From 41bb1e89e70c70d34d5471361de2da3bea23ae62 Mon Sep 17 00:00:00 2001 From: Durable Workflow Date: Fri, 23 Oct 2026 19:17:21 +0000 Subject: [PATCH 01/10] Serve message streams like Server 2.11.1 Appends, diagnostics, delivery as the runtime-reserved signal, worker cursor acknowledgements and waits on workflow task completions, and the hand-over of the cursor and unconsumed messages to a continued run, with the message-streams parity fixture recorded from published PHP 2.11.1. A signal consumed by a condition wait now records no SignalApplied unless it is declared, the instance cursor event carries its task snapshot, and a completion reports the next message task, as PHP does. Retention holds runs whose instance has inbound streams until stream cleanup is ported. --- .github/workflows/server-parity.yml | 4 +- .../2026-10-23-message-streams.md | 64 + docs/rust-server-port.md | 14 +- rust/src/runtime/continuations.rs | 10 +- rust/src/runtime/execution.rs | 89 +- rust/src/runtime/http.rs | 54 +- rust/src/runtime/message_streams.rs | 719 +++++++++ rust/src/runtime/mod.rs | 1 + rust/src/runtime/retention.rs | 7 +- rust/src/runtime/store.rs | 223 ++- rust/tests/execution.rs | 2 + rust/tests/execution/message_streams.rs | 178 +++ .../conformance/server-parity/contract.mjs | 3 + .../message-streams-contract.mjs | 191 +++ .../server-parity/message-streams-probe.php | 163 ++ scripts/conformance/server-parity/probe.php | 6 + .../ServerParity/message-streams.json | 182 +++ .../message-streams-observation.json | 1323 +++++++++++++++++ ...ServerParityMessageStreamsContractTest.mjs | 81 + 19 files changed, 3229 insertions(+), 85 deletions(-) create mode 100644 docs/rust-server-port-checkpoints/2026-10-23-message-streams.md create mode 100644 rust/src/runtime/message_streams.rs create mode 100644 rust/tests/execution/message_streams.rs create mode 100644 scripts/conformance/server-parity/message-streams-contract.mjs create mode 100644 scripts/conformance/server-parity/message-streams-probe.php create mode 100644 tests/Fixtures/ServerParity/message-streams.json create mode 100644 tests/Fixtures/ServerParityExpected/message-streams-observation.json create mode 100644 tests/Unit/ServerParityMessageStreamsContractTest.mjs diff --git a/.github/workflows/server-parity.yml b/.github/workflows/server-parity.yml index 679c1d450..c37129d0e 100644 --- a/.github/workflows/server-parity.yml +++ b/.github/workflows/server-parity.yml @@ -41,6 +41,7 @@ on: - tests/Unit/ServerParityPayloadCleanupContractTest.mjs - tests/Unit/ServerParityRepairPassContractTest.mjs - tests/Unit/ServerParityNamespaceDeleteContractTest.mjs + - tests/Unit/ServerParityMessageStreamsContractTest.mjs - tests/Unit/ServerParityWaitingHistoryContractTest.mjs - tests/Unit/ServerParityTaskRetryContractTest.mjs - tests/Unit/ServerParityPublishedPatchContractTest.mjs @@ -95,6 +96,7 @@ on: - tests/Unit/ServerParityPayloadCleanupContractTest.mjs - tests/Unit/ServerParityRepairPassContractTest.mjs - tests/Unit/ServerParityNamespaceDeleteContractTest.mjs + - tests/Unit/ServerParityMessageStreamsContractTest.mjs - tests/Unit/ServerParityWaitingHistoryContractTest.mjs - tests/Unit/ServerParityTaskRetryContractTest.mjs - tests/Unit/ServerParityPublishedPatchContractTest.mjs @@ -1518,7 +1520,7 @@ jobs: -e COMPOSER_HOME=/tmp/composer -v "$PWD:/source" -w /source "$image" \ install --working-dir=scripts/conformance/server-parity --no-interaction --no-progress docker run --rm --user=1000:1000 --entrypoint node -v "$PWD:/source:ro" -w /source \ - "$image" --test tests/Unit/ServerParityRunnerTest.mjs tests/Unit/ServerParityCooperativeContractTest.mjs tests/Unit/ServerParityChildCancellationContractTest.mjs tests/Unit/ServerParityScheduleContractTest.mjs tests/Unit/ServerParityVisibilityContractTest.mjs tests/Unit/ServerParityAdmissionContractTest.mjs tests/Unit/ServerParityRoleTokenContractTest.mjs tests/Unit/ServerParityWorkerDeregistrationContractTest.mjs tests/Unit/ServerParitySearchAttributeContractTest.mjs tests/Unit/ServerParityWorkerListingContractTest.mjs tests/Unit/ServerParityWorkerReplacementContractTest.mjs tests/Unit/ServerParityPatchDeploymentContractTest.mjs tests/Unit/ServerParityWaitingHistoryContractTest.mjs tests/Unit/ServerParityTaskRetryContractTest.mjs tests/Unit/ServerParityPublishedPatchContractTest.mjs tests/Unit/ServerParityPublishedPatchTransportTest.mjs tests/Unit/ServerParityRecordFileTest.mjs tests/Unit/ServerParityLateWorkflowCompletionContractTest.mjs tests/Unit/ServerParityStorageTestContractTest.mjs tests/Unit/ServerParitySchedulePoliciesContractTest.mjs tests/Unit/ServerParitySignalsBeforeWorkerContractTest.mjs tests/Unit/ServerParityScheduleCatalogContractTest.mjs tests/Unit/ServerParityOperatorMetricsContractTest.mjs tests/Unit/ServerParityActivityLeaseRenewalContractTest.mjs tests/Unit/ServerParityServiceCatalogContractTest.mjs tests/Unit/ServerParitySystemMetricsContractTest.mjs tests/Unit/ServerParityPrometheusMetricsContractTest.mjs tests/Unit/ServerParityPayloadCleanupContractTest.mjs tests/Unit/ServerParityRepairPassContractTest.mjs tests/Unit/ServerParityNamespaceDeleteContractTest.mjs > parity-evidence/comparator-tests.txt + "$image" --test tests/Unit/ServerParityRunnerTest.mjs tests/Unit/ServerParityCooperativeContractTest.mjs tests/Unit/ServerParityChildCancellationContractTest.mjs tests/Unit/ServerParityScheduleContractTest.mjs tests/Unit/ServerParityVisibilityContractTest.mjs tests/Unit/ServerParityAdmissionContractTest.mjs tests/Unit/ServerParityRoleTokenContractTest.mjs tests/Unit/ServerParityWorkerDeregistrationContractTest.mjs tests/Unit/ServerParitySearchAttributeContractTest.mjs tests/Unit/ServerParityWorkerListingContractTest.mjs tests/Unit/ServerParityWorkerReplacementContractTest.mjs tests/Unit/ServerParityPatchDeploymentContractTest.mjs tests/Unit/ServerParityWaitingHistoryContractTest.mjs tests/Unit/ServerParityTaskRetryContractTest.mjs tests/Unit/ServerParityPublishedPatchContractTest.mjs tests/Unit/ServerParityPublishedPatchTransportTest.mjs tests/Unit/ServerParityRecordFileTest.mjs tests/Unit/ServerParityLateWorkflowCompletionContractTest.mjs tests/Unit/ServerParityStorageTestContractTest.mjs tests/Unit/ServerParitySchedulePoliciesContractTest.mjs tests/Unit/ServerParitySignalsBeforeWorkerContractTest.mjs tests/Unit/ServerParityScheduleCatalogContractTest.mjs tests/Unit/ServerParityOperatorMetricsContractTest.mjs tests/Unit/ServerParityActivityLeaseRenewalContractTest.mjs tests/Unit/ServerParityServiceCatalogContractTest.mjs tests/Unit/ServerParitySystemMetricsContractTest.mjs tests/Unit/ServerParityPrometheusMetricsContractTest.mjs tests/Unit/ServerParityPayloadCleanupContractTest.mjs tests/Unit/ServerParityRepairPassContractTest.mjs tests/Unit/ServerParityNamespaceDeleteContractTest.mjs tests/Unit/ServerParityMessageStreamsContractTest.mjs > parity-evidence/comparator-tests.txt docker run --rm --user=1000:1000 --cpus=1 --memory=256m --memory-swap=256m \ --network=none --entrypoint node -v "$PWD:/source:ro" -w /source \ "$image" --test tests/Unit/ServerParityCodecDeadlineLoadTest.mjs > parity-evidence/codec-load-tests.txt diff --git a/docs/rust-server-port-checkpoints/2026-10-23-message-streams.md b/docs/rust-server-port-checkpoints/2026-10-23-message-streams.md new file mode 100644 index 000000000..c369f493f --- /dev/null +++ b/docs/rust-server-port-checkpoints/2026-10-23-message-streams.md @@ -0,0 +1,64 @@ +# October 23, message streams + +- **The gap.** All three published SDKs declare `message-streams` from + worker protocol 1.15. Native refused a worker that asked for them, did not + serve the append and diagnostics routes, and ignored + `message_stream_cursors` and `message_stream_waits` on a completion. +- **How PHP 2.11.1 does it.** Learned from its source, then observed live + against the published image: + - Each accepted message is delivered to the current run as the + runtime-reserved signal `__durable_workflow_message_stream`, in the + append's transaction. The signal skips the declared-signal and argument + checks but not the pending-signal and payload ceilings. + - SDK receives open a condition wait keyed by the stream cursor. A stream + message wakes that wait. + - When a signal wakes a condition wait, PHP consumes it without + `SignalApplied`, unless the run's durable contract declares that signal. + This holds for any undeclared signal, not only stream messages. + - The stream cursor's `MessageCursorAdvanced` is recorded after the + completion, with the completed task's snapshot. + - On continue-as-new, the next run receives a payload-free cursor + checkpoint for each acknowledged stream, then every unconsumed message + again, by append order. The old run rejects the signals it had not + consumed. +- **What native does now.** + - `POST /api/workflows/{id}/message-streams/{stream}/messages`, with PHP's + validation order, refusals and diagnostics of malformed input. + - `GET /api/workflows/{id}/message-streams` and + `GET /api/workflows/{id}/message-streams/{stream}`. + - Completion cursors and waits: validated before anything changes, + recorded in the completion's transaction, handed over on continue-as-new. + - The worker capability is PHP's contract with `supported: true`, and + cluster info publishes `message_streams_contract`. + - Continue-as-new no longer refuses a run with message streams. +- **Native changes found on the way.** + - A signal consumed by a condition wait no longer records `SignalApplied` + unless it is declared, and keeps the wait's sequence. + - The instance-level `MessageCursorAdvanced` carries the task snapshot, as + PHP's does. + - A completion reports the run's next message task in + `created_task_ids`, as PHP does. +- **How it was checked.** + - The `message-streams` fixture: 15 appends, 8 refused completions, the + deliveries each worker read, both runs' full event sequences, and the + diagnostics at each step. + - Published PHP 2.11.1 and native pass it on SQLite. The probe's output + is the same size on both, and both runs' event sequences are identical. + - The contract test refuses 50 changed observations, among them a + `SignalApplied` for a consumed message, a missing next message task, a + redelivered consumed message and a missing cursor checkpoint. +- **Not done yet** (Server #325 follow-ups): + - **External payload references.** Native refuses a reference as stream + input with 501 `message_stream_external_payload_unavailable`. PHP keeps + the reference and resolves it when history reaches a worker. + - **Cleanup and upgrade.** Retention cleanup of streams + (`MessageStreamRetentionCleanup`) is not ported. Takeover of PHP data + with streams is still refused. + - **Cluster `capabilities` map.** Native has none yet, so + `capabilities.message_streams` is absent; that map is the cluster info + work. + - **History presentation, outside streams.** Task snapshots lack PHP's + `last_dispatch_attempt_at` and `last_dispatched_at`. A consumed signal's + `SignalReceived` lacks `workflow_sequence` in poll history. + - **Condition fingerprint.** Native requires `condition_definition_fingerprint` + on `open_condition_wait`; PHP accepts a wait without one. diff --git a/docs/rust-server-port.md b/docs/rust-server-port.md index 96247ce9f..db8edcd57 100644 --- a/docs/rust-server-port.md +++ b/docs/rust-server-port.md @@ -148,7 +148,10 @@ any other unknown path or method answers PHP's own 404 or 405 - **Run operations.** - Imports from embedded v2 and Waterline v1. - **Other families.** - - Message streams. + - Message streams are served + ([checkpoint](rust-server-port-checkpoints/2026-10-23-message-streams.md)), + except external payload references as input, stream retention cleanup + and takeover of PHP data with streams. - Durable workflow stream items and close. - Service endpoints. (The service catalog reads from Server 2.11 are served: native never holds catalog rows, so every list is PHP's empty page and every @@ -265,9 +268,12 @@ contract fixture before its correction. Add a fixture for each before changing them. - **Retention of runs with external payloads or message streams.** Published PHP deletes a pruned run's external payload objects and releases - its message stream items. Native does not clean either up yet. It skips - such a run with `external_payload_cleanup_unavailable` or - `message_stream_cleanup_unavailable` and keeps its details. + its message stream items. It also compacts its instance's consumed inbound + stream items, and removes the instance's streams with its last run. Native + does not clean any of these up yet. It skips such a run, including any run + whose instance has inbound streams, with + `external_payload_cleanup_unavailable` or + `message_stream_cleanup_unavailable`, and keeps its details. - **Deeply nested JSON bodies.** Published PHP decodes JSON up to 512 levels deep. Native checks every body as PHP does before any route, but its handlers parse with serde_json's 128-level limit. A body nested 129 to 512 diff --git a/rust/src/runtime/continuations.rs b/rust/src/runtime/continuations.rs index 7a5d4d62c..e8590924a 100644 --- a/rust/src/runtime/continuations.rs +++ b/rust/src/runtime/continuations.rs @@ -105,17 +105,15 @@ where } // Published PHP continues a run whatever it has open. Its activities, // timers, children and waits stay with the closed predecessor, and no - // parent-close policy applies. Streams, service calls and live - // queries are not transferred by native yet. + // parent-close policy applies. Message streams pass to the next run + // when the completion records its stream acknowledgements. Durable + // output streams, service calls and live queries are not transferred + // by native yet. for (sql, selected) in [ ( "SELECT COUNT(*) FROM workflow_durable_streams WHERE workflow_instance_id=$1", workflow.as_str(), ), - ( - "SELECT COUNT(*) FROM workflow_inbound_streams WHERE workflow_instance_id=$1", - workflow.as_str(), - ), ( "SELECT COUNT(*) FROM workflow_service_calls WHERE caller_workflow_run_id=$1", old.as_str(), diff --git a/rust/src/runtime/execution.rs b/rust/src/runtime/execution.rs index fa6a57a4f..5c2772790 100644 --- a/rust/src/runtime/execution.rs +++ b/rust/src/runtime/execution.rs @@ -585,6 +585,89 @@ impl Runtime { Ok(view) } + delegate!(message_stream_workflow_bound(workflow_id: &str) -> Result); + delegate!(record_malformed_message(workflow_id: &str, stream: &str, message_id: &Value) -> Result<()>); + delegate!(message_stream_diagnostics(workflow_id: &str, stream: Option<&str>) -> Result>); + + /// `MessageStreamController::append`: the stream name, the workflow in + /// this namespace, the request rules and the payload envelope, in PHP's + /// order. A refused request or payload counts as malformed input. + pub(crate) async fn append_message(&self, workflow_id: &str, stream: &str, body: Value, context: Value) -> Result<(StatusCode, Value)> { + super::message_streams::check_path_name(stream)?; + if !self.message_stream_workflow_bound(workflow_id).await? { + return Err(super::RuntimeError::Protocol { + status: StatusCode::NOT_FOUND, + response: json!({"accepted":false,"workflow_id":workflow_id,"stream_name":stream,"outcome":"rejected","reason":"instance_not_found"}), + }); + } + let body = super::laravel_rules::normalize(&body); + let blob = match self.message_payload(&body).await { + Ok(blob) => blob, + Err(error) => { + if matches!(&error, super::RuntimeError::Protocol { status, .. } | super::RuntimeError::Refused { status, .. } if *status == StatusCode::UNPROCESSABLE_ENTITY) { + self.record_malformed_message(workflow_id, stream, &body["message_id"]).await?; + } + return Err(error); + } + }; + let message_id = body["message_id"].as_str().unwrap_or_default().to_owned(); + let (codec, structural) = (self.signal_codec.clone(), self.structural); + match &*self.storage { + Storage::Sqlite(store) => store.append_message(workflow_id, stream, &message_id, "avro", &blob, &context, codec, structural).await, + Storage::Postgres(store) => store.append_message(workflow_id, stream, &message_id, "avro", &blob, &context, codec, structural).await, + Storage::MySql(store) => store.append_message(workflow_id, stream, &message_id, "avro", &blob, &context, codec, structural).await, + } + } + + /// The request rules, then `AvroPayloadEnvelopeResolver::resolve`: no + /// input is an empty argument list, plain values are encoded, and an + /// inline envelope keeps its exact bytes. + async fn message_payload(&self, body: &Value) -> Result { + super::message_streams::check_append(body)?; + if body.get("input").is_none_or(Value::is_null) { + return super::queries::empty_arguments(self.signal_codec.clone()).await; + } + super::input_envelope::transport(body)?; + if let Some(blob) = self.plain_arguments(body).await? { + return Ok(blob); + } + if self.prepare_external(&body["input"]).await?.is_some() { + // PHP keeps the reference and resolves it for workers; native + // does not resolve stream payload references yet. + return Err(super::RuntimeError::Protocol { + status: StatusCode::NOT_IMPLEMENTED, + response: json!({"accepted":false,"workflow_id":null,"message_id":body["message_id"],"outcome":"rejected", + "reason":"message_stream_external_payload_unavailable"}), + }); + } + self.decodable(&body["input"]["blob"]).await?; + super::envelope(body, "input") + } + + /// `MessageStreamController::index` and `show`. + pub(crate) async fn message_streams(&self, workflow_id: &str, stream: Option<&str>) -> Result { + if let Some(stream) = stream { + super::message_streams::check_path_name(stream)?; + } + if !self.message_stream_workflow_bound(workflow_id).await? { + return Err(super::RuntimeError::Protocol { + status: StatusCode::NOT_FOUND, + response: json!({"message":"Workflow not found.","reason":"instance_not_found"}), + }); + } + let streams = self.message_stream_diagnostics(workflow_id, stream).await?; + match stream { + None => Ok(json!({"workflow_id":workflow_id,"count":streams.len(),"streams":streams})), + Some(stream) => match streams.into_iter().next() { + Some(found) => Ok(json!({"workflow_id":workflow_id,"stream":found})), + None => Err(super::RuntimeError::Protocol { + status: StatusCode::NOT_FOUND, + response: json!({"workflow_id":workflow_id,"stream_name":stream,"reason":"message_stream_not_found"}), + }), + }, + } + } + /// `ActivityController::index`. pub(crate) async fn standalone_activities(&self, query: Value) -> Result { let request = super::standalone_activities::list_request(&query)?; @@ -1256,17 +1339,17 @@ impl Runtime { match &*self.storage { Storage::Sqlite(store) => { store - .complete_workflow(task_id, body, self.transition_codec.clone(), external, worker, protocol) + .complete_workflow(task_id, body, self.transition_codec.clone(), external, worker, protocol, self.structural) .await } Storage::Postgres(store) => { store - .complete_workflow(task_id, body, self.transition_codec.clone(), external, worker, protocol) + .complete_workflow(task_id, body, self.transition_codec.clone(), external, worker, protocol, self.structural) .await } Storage::MySql(store) => { store - .complete_workflow(task_id, body, self.transition_codec.clone(), external, worker, protocol) + .complete_workflow(task_id, body, self.transition_codec.clone(), external, worker, protocol, self.structural) .await } } diff --git a/rust/src/runtime/http.rs b/rust/src/runtime/http.rs index 98febdb4c..3bdc13127 100644 --- a/rust/src/runtime/http.rs +++ b/rust/src/runtime/http.rs @@ -120,6 +120,12 @@ pub fn router(runtime: Runtime) -> Router { post(repair_run), ) .route("/api/workflows/{workflow_id}/debug", get(debug_current)) + .route("/api/workflows/{workflow_id}/message-streams", get(list_message_streams)) + .route("/api/workflows/{workflow_id}/message-streams/{stream_name}", get(show_message_stream)) + .route( + "/api/workflows/{workflow_id}/message-streams/{stream_name}/messages", + post(append_message), + ) .route( "/api/workflows/{workflow_id}/runs/{run_id}/debug", get(debug_run), @@ -919,6 +925,7 @@ async fn cluster() -> Json { "auth_composition_contract": auth_composition, "principal_attribution_contract": principal_attribution, "service_catalog_read_contract": service_catalog, + "message_streams_contract": super::message_streams::manifest(), "payload_codec": "avro", "qualified_for_php_takeover": false}), ) } @@ -960,6 +967,39 @@ fn present(body: &Value, field: &str) -> Option { body.get(field).filter(|value| !value.is_null()).cloned() } +/// `POST /api/workflows/{workflow_id}/message-streams/{stream_name}/messages`. +async fn append_message( + Extension(runtime): Extension, + request: CommandRequest, + Path((workflow_id, stream)): Path<(String, String)>, + Json(body): Json, +) -> Result<(StatusCode, Json)> { + let normalized = super::laravel_rules::normalize(&body); + let context = request.context(&body, &workflow_id, "message_stream_append", metadata(&[ + ("stream_name", Some(json!(stream))), + ("message_id", present(&normalized, "message_id")), + ("request_id", present(&normalized, "request_id")), + ])); + let (status, response) = runtime.append_message(&workflow_id, &stream, body, context).await?; + Ok((status, Json(response))) +} + +/// `GET /api/workflows/{workflow_id}/message-streams`. +async fn list_message_streams( + Extension(runtime): Extension, + Path(workflow_id): Path, +) -> Result> { + Ok(Json(runtime.message_streams(&workflow_id, None).await?)) +} + +/// `GET /api/workflows/{workflow_id}/message-streams/{stream_name}`. +async fn show_message_stream( + Extension(runtime): Extension, + Path((workflow_id, stream)): Path<(String, String)>, +) -> Result> { + Ok(Json(runtime.message_streams(&workflow_id, Some(&stream)).await?)) +} + fn signal_context(request: &CommandRequest, body: &Value, workflow_id: &str, name: &str) -> Value { request.context(body, workflow_id, "signal", metadata(&[ ("request_id", present(body, "request_id")), @@ -1853,7 +1893,9 @@ async fn complete_workflow( headers: HeaderMap, Json(body): Json, ) -> Result> { - // PHP's transport rules come before every guard, ownership included. + // PHP validates the message-stream fields first, then its transport + // rules, before every guard, ownership included. + super::message_streams::check_completion_fields(&body).map_err(worker_refusal)?; let body = super::completion_rules::checked(body).map_err(worker_refusal)?; if body["commands"].as_array().is_some_and(|commands| { commands @@ -1878,6 +1920,16 @@ async fn complete_workflow( "remediation":format!("Complete typed search-attribute updates only through server nodes advertising worker protocol {minimum} or newer.")}), }); } + if !super::message_streams::Completion::of(&body).is_empty() && !protocol_at_least(&headers, 15)? { + let minimum = super::message_streams::MINIMUM_PROTOCOL_VERSION; + return Err(super::RuntimeError::Protocol { + status: StatusCode::CONFLICT, + response: json!({"task_id":task_id,"workflow_task_attempt":body["workflow_task_attempt"], + "outcome":"rejected","reason":"message_streams_unavailable", + "requested_version":request_protocol(&headers)?,"minimum_protocol_version":minimum, + "remediation":format!("Send the X-Durable-Workflow-Protocol-Version header with worker protocol {minimum} or newer.")}), + }); + } let attempt = body["workflow_task_attempt"].clone(); runtime .complete_workflow(&task_id, body, Some(request.principal().clone()), headers.get("x-durable-workflow-protocol-version").and_then(|value| value.to_str().ok())) diff --git a/rust/src/runtime/message_streams.rs b/rust/src/runtime/message_streams.rs new file mode 100644 index 000000000..42e87ab1a --- /dev/null +++ b/rust/src/runtime/message_streams.rs @@ -0,0 +1,719 @@ +//! Message streams, as published Server 2.11.1 serves them +//! (`MessageStreamController`, `MessageStreamService`, +//! `MessageStreamsContract`). +//! +//! A client appends ordered messages to a named stream of a workflow +//! instance. The server assigns each its position and delivers it to the +//! current run as the runtime-reserved `__durable_workflow_message_stream` +//! signal, in the append's transaction. Workers acknowledge what they consumed +//! with `message_stream_cursors` on a workflow task completion, which records +//! `MessageCursorAdvanced`, and report open receives with +//! `message_stream_waits`. When a run continues as new, the next run receives +//! a payload-free cursor checkpoint and every unconsumed message again. +//! +//! An external payload reference as message input is refused for now: +//! published PHP keeps it by reference and resolves it when history crosses +//! to a worker, which native does not do yet. +use super::laravel_rules::{self, Rule}; +use super::store::{PreparedControlArguments, SignalWrite, Store, id, now}; +use super::transaction::Transaction; +use super::{Result, RuntimeError, backend::Backend}; +use axum::http::StatusCode; +use serde_json::{Value, json}; +use sqlx::{ColumnIndex, Decode, Encode, Executor, IntoArguments, Type}; +use std::collections::BTreeMap; +use std::sync::Arc; +use tokio::sync::Semaphore; + +pub(super) const INTERNAL_SIGNAL: &str = "__durable_workflow_message_stream"; +const MESSAGE_SCHEMA: &str = "durable-workflow.v2.message-stream.message"; +const CURSOR_SCHEMA: &str = "durable-workflow.v2.message-stream.cursor"; +pub(super) const MINIMUM_PROTOCOL_VERSION: &str = "1.15"; +const MAX_BATCH: i64 = 100; +const NAME_MESSAGE: &str = "must contain 1-128 letters, numbers, periods, underscores, colons, or hyphens."; + +/// `MessageStreamsContract::manifest`. +pub(super) fn manifest() -> Value { + json!({ + "schema": "durable-workflow.v2.message-streams.contract", + "version": 1, + "capability_flag": "message_streams", + "minimum_worker_protocol_version": MINIMUM_PROTOCOL_VERSION, + "scope": "workflow_instance", + "transport": { + "append_path": "/api/workflows/{workflow_id}/message-streams/{stream_name}/messages", + "diagnostics_path": "/api/workflows/{workflow_id}/message-streams", + "internal_signal": INTERNAL_SIGNAL, + "message_schema": MESSAGE_SCHEMA, + "cursor_schema": CURSOR_SCHEMA, + "payload_envelope": "inline_avro_at_the_server_to_worker_delivery_boundary", + "durable_payload": "exact_inline_avro_bytes_or_the_verified_external_storage_reference", + "payload_visibility": "payload_is_present_only_in_durable_signal_history_and_is_omitted_from_stream_diagnostics", + }, + "identity": { + "producer_key": "message_id", + "order_key": "position", + "first_position": 1, + "duplicate_same_payload": "return_existing_position_without_redelivery", + "duplicate_different_payload": "reject_message_identity_conflict", + }, + "cursor": { + "ownership": "server", + "worker_completion_field": "message_stream_cursors", + "monotonic": true, + "advancement": "highest_contiguous_consumed_position", + "history_event": "MessageCursorAdvanced", + "continue_as_new": "a_payload_free_cursor_checkpoint_and_unconsumed_messages_are_delivered_to_the_current_run", + }, + "waiting": { + "worker_completion_field": "message_stream_waits", + "durable_wakeup": "runtime_reserved_signal_task_resume", + "pending_wait_is_diagnostic": true, + }, + "batch": { + "minimum": 1, + "maximum": MAX_BATCH, + "waits_for": "first_message", + "returns": "up_to_maximum_currently_available_messages", + }, + "replay": { + "ordering_source": "server_assigned_position", + "deduplication_source": "message_id_and_instance_cursor", + "worker_replacement": "history_replay_reconstructs_pending_messages_and_cursor_acknowledgement_is_idempotent", + "server_restart": "stream_items_cursor_and_wait_state_are_database_durable", + "external_payloads": "references_remain_reference_backed_at_rest_and_are_resolved_for_worker_delivery_then_removed_with_namespace_retention", + }, + "sdk_handoffs": { + "php": "WorkflowContext::messageStream(string)->receive(int)", + "python": "WorkflowContext.message_stream(str).receive(max_items)", + "rust": "WorkflowContext::message_stream(name).receive(max_items).await", + }, + "conformance": { + "runtimes": ["php", "python", "rust"], + "targets": ["server", "cloud"], + "required_scenarios": [ + "repeated_messages_are_observed_in_position_order_exactly_once", + "duplicate_delivery_preserves_one_logical_message", + "worker_is_replaced_mid_stream", + "continue_as_new_retains_cursor_and_pending_messages", + "server_restart_retains_cursor_wait_and_order", + ], + "artifacts": "published", + }, + }) +} + +/// The worker capability: the manifest, supported. +pub(super) fn worker_capability() -> Value { + let mut capability = manifest(); + capability["supported"] = json!(true); + capability +} + +fn identifier(value: &str) -> bool { + !value.is_empty() && value.bytes().all(|byte| byte.is_ascii_alphanumeric() || matches!(byte, b'.' | b'_' | b':' | b'-')) +} + +/// `/^[A-Za-z0-9._:-]{1,128}$/`. +pub(super) fn valid_name(name: &str) -> bool { + name.len() <= 128 && identifier(name) +} + +fn invalid(field: &str, message: String) -> RuntimeError { + laravel_rules::failure(vec![(field.to_owned(), vec![message])]).expect("one message") +} + +/// `validateStreamName`: a path stream name outside the pattern. +pub(super) fn check_path_name(name: &str) -> Result<()> { + if valid_name(name) { + Ok(()) + } else { + Err(invalid("stream_name", format!("The stream name {NAME_MESSAGE}"))) + } +} + +/// The append body's rules, after Laravel's normalization. +const APPEND_RULES: &laravel_rules::Rules = &[ + ("message_id", &[Rule::Required, Rule::String, Rule::Max(191), Rule::Pattern(identifier)]), + ("input", &[Rule::Nullable, Rule::Array]), + ("request_id", &[Rule::Nullable, Rule::String, Rule::Max(255)]), +]; + +pub(super) fn check_append(body: &Value) -> Result<()> { + laravel_rules::validate(body, APPEND_RULES) +} + +/// `diagnosticMessageId`: a refused message id as diagnostics keep it. +pub(super) fn diagnostic_message_id(value: &Value) -> String { + match value.as_str() { + None => "[omitted]".into(), + Some(id) if id.len() <= 191 && identifier(id) => id.into(), + Some(id) => { + use sha2::Digest; + format!("sha256:{}", hex(sha2::Sha256::digest(id.as_bytes()))) + } + } +} + +/// The payload hash PHP stores: sha256 of `codec \0 blob`. +pub(super) fn payload_hash(codec: &str, blob: &str) -> String { + use sha2::Digest; + let mut digest = sha2::Sha256::new(); + digest.update(codec.as_bytes()); + digest.update([0]); + digest.update(blob.as_bytes()); + hex(digest.finalize()) +} + +fn hex(digest: impl AsRef<[u8]>) -> String { + digest.as_ref().iter().map(|byte| format!("{byte:02x}")).collect() +} + +/// The worker protocol's message-stream fields on a completion, checked as +/// PHP's first `validate` call does, before the completion's own rules. +const COMPLETION_RULES: &laravel_rules::Rules = &[ + ("message_stream_cursors", &[Rule::Nullable, Rule::Array, Rule::Max(100)]), + ("message_stream_cursors.*.stream_name", &[Rule::Required, Rule::String, Rule::Max(128)]), + ("message_stream_cursors.*.through_position", &[Rule::Required, Rule::Integer, Rule::Min(0)]), + ("message_stream_waits", &[Rule::Nullable, Rule::Array, Rule::Max(100)]), + ("message_stream_waits.*.stream_name", &[Rule::Required, Rule::String, Rule::Max(128)]), + ("message_stream_waits.*.after_position", &[Rule::Required, Rule::Integer, Rule::Min(0)]), +]; + +pub(super) fn check_completion_fields(body: &Value) -> Result<()> { + laravel_rules::validate(&laravel_rules::normalize(body), COMPLETION_RULES) +} + +/// A completion's cursor advances and pending waits, in request order. +pub(super) struct Completion { + pub(super) cursors: Vec, + pub(super) waits: Vec, +} + +impl Completion { + pub(super) fn of(body: &Value) -> Self { + let list = |field: &str| match &body[field] { + Value::Array(items) => items.clone(), + Value::Object(items) => items.values().cloned().collect(), + _ => Vec::new(), + }; + Self { cursors: list("message_stream_cursors"), waits: list("message_stream_waits") } + } + + pub(super) fn is_empty(&self) -> bool { + self.cursors.is_empty() && self.waits.is_empty() + } +} + +/// `metadataStreamName`. +fn metadata_name(item: &Value, field: &str) -> Result { + match item["stream_name"].as_str() { + Some(name) if valid_name(name) => Ok(name.to_owned()), + _ => Err(invalid(field, format!("The stream_name {NAME_MESSAGE}"))), + } +} + +fn non_negative(value: &Value) -> Option { + value.as_i64().filter(|value| *value >= 0) +} + +/// The rejection PHP's service answers for an append it cannot accept. +fn rejection(status: StatusCode, reason: &str, workflow_id: &str, stream: &str, message_id: &str, extra: Value) -> RuntimeError { + let mut response = json!({"accepted":false,"duplicate":false,"workflow_id":workflow_id,"stream_name":stream, + "message_id":message_id,"position":null,"outcome":"rejected","reason":reason}); + if let Value::Object(extra) = extra { + response.as_object_mut().expect("object").extend(extra); + } + RuntimeError::Protocol { status, response } +} + +/// The value a stream delivery signal carries: one map in a list. +fn map(entries: Vec<(&str, crate::codec::Value)>) -> crate::codec::Value { + crate::codec::Value::Map(entries.into_iter().map(|(key, value)| (key.to_owned(), value)).collect::>()) +} + +/// A runtime signal's encoded arguments and command payload. +async fn runtime_arguments(codec: Arc, delivery: crate::codec::Value) -> Result { + let values = vec![delivery]; + let arguments = super::queries::encode_arguments(codec.clone(), crate::codec::Value::Array(values.clone())).await?; + let command = super::queries::encode_arguments( + codec.clone(), + map(vec![ + ("name", crate::codec::Value::String(INTERNAL_SIGNAL.into())), + ("arguments", crate::codec::Value::Array(values.clone())), + ("validation_errors", crate::codec::Value::Array(vec![])), + ]), + ) + .await?; + let value = super::queries::encode_arguments(codec, values[0].clone()).await?; + Ok(PreparedControlArguments { arguments, values, value, command }) +} + +/// `CommandContext::controlPlane`: the context of a delivery with no request. +fn control_plane_context() -> Value { + json!({"caller":{"type":"control_plane","label":"Control Plane"},"auth":{"status":"not_applicable","method":"none"}}) +} + +impl Store +where + for<'c> &'c mut DB::Connection: Executor<'c, Database = DB>, + DB::Arguments: IntoArguments, + for<'q> &'q str: Encode<'q, DB> + Type, + for<'q> Option<&'q str>: Encode<'q, DB>, + for<'q> Option: Encode<'q, DB>, + for<'q> String: Encode<'q, DB> + Type, + for<'q> i64: Encode<'q, DB> + Type, + for<'r> i64: Decode<'r, DB>, + usize: ColumnIndex, +{ + /// `NamespaceWorkflowScope::workflowBound`. + pub(crate) async fn message_stream_workflow_bound(&self, workflow_id: &str) -> Result { + Ok(Self::query("SELECT id FROM workflow_instances WHERE id=$1 AND namespace=$2") + .bind(workflow_id) + .bind(self.namespace.as_ref()) + .fetch_optional(&self.pool) + .await? + .is_some()) + } + + /// The stream row, created at position zero when absent. + async fn locked_stream(&self, tx: &mut Transaction<'_, DB>, workflow_id: &str, stream: &str) -> Result { + const SELECT: &str = "SELECT * FROM workflow_inbound_streams WHERE namespace=$1 AND workflow_instance_id=$2 AND stream_name=$3"; + if let Some(row) = Self::query(SELECT) + .bind(self.namespace.as_ref()) + .bind(workflow_id) + .bind(stream) + .fetch_optional(&mut **tx) + .await? + { + return Ok(row); + } + let at = now(); + Self::query("INSERT INTO workflow_inbound_streams(namespace,workflow_instance_id,stream_name,last_position,cursor_position,duplicate_count,malformed_count,created_at,updated_at) VALUES ($1,$2,$3,0,0,0,0,$4,$5)") + .bind(self.namespace.as_ref()).bind(workflow_id).bind(stream) + .bind(DB::bind_time(at)).bind(DB::bind_time(at)).execute(&mut **tx).await?; + Ok(Self::query(SELECT).bind(self.namespace.as_ref()).bind(workflow_id).bind(stream).fetch_one(&mut **tx).await?) + } + + /// The instance's current run, if it is still open. + async fn active_current_run(tx: &mut Transaction<'_, DB>, workflow_id: &str) -> Result> { + let Some(run) = Self::query("SELECT r.* FROM workflow_instances i JOIN workflow_runs r ON r.id=i.current_run_id WHERE i.id=$1") + .bind(workflow_id) + .fetch_optional(&mut **tx) + .await? + else { + return Ok(None); + }; + let open = matches!(DB::string(&run, "status")?.as_str(), "pending" | "running" | "waiting"); + Ok(open.then_some(run)) + } + + /// `MessageStreamService::append`, in one transaction. + #[allow(clippy::too_many_arguments)] + pub(crate) async fn append_message( + &self, + workflow_id: &str, + stream: &str, + message_id: &str, + payload_codec: &str, + payload_blob: &str, + context: &Value, + codec: Arc, + limits: super::memo_limits::StructuralLimits, + ) -> Result<(StatusCode, Value)> { + let mut tx = self.begin().await?; + let bound = Self::query("SELECT id FROM workflow_instances WHERE id=$1 AND namespace=$2") + .bind(workflow_id).bind(self.namespace.as_ref()).fetch_optional(&mut *tx).await?; + if bound.is_none() { + return Err(rejection(StatusCode::NOT_FOUND, "instance_not_found", workflow_id, stream, message_id, json!({}))); + } + let Some(run) = Self::active_current_run(&mut tx, workflow_id).await? else { + return Err(rejection(StatusCode::CONFLICT, "run_not_active", workflow_id, stream, message_id, json!({}))); + }; + let row = self.locked_stream(&mut tx, workflow_id, stream).await?; + let stream_id = DB::number(&row, "id")?; + let cursor = DB::number(&row, "cursor_position")?; + let hash = payload_hash(payload_codec, payload_blob); + let at = now(); + if let Some(existing) = Self::query("SELECT position,payload_hash FROM workflow_inbound_stream_items WHERE stream_id=$1 AND message_id=$2") + .bind(stream_id).bind(message_id).fetch_optional(&mut *tx).await? + { + let same = DB::string(&existing, "payload_hash")? == hash; + let outcome = if same { "duplicate" } else { "message_identity_conflict" }; + Self::query("UPDATE workflow_inbound_streams SET duplicate_count=duplicate_count+1,last_input_outcome=$1,last_input_message_id=$2,last_input_at=$3,updated_at=$4 WHERE id=$5") + .bind(outcome).bind(message_id).bind(DB::bind_time(at)).bind(DB::bind_time(at)).bind(stream_id) + .execute(&mut *tx).await?; + tx.commit().await?; + let response = json!({"accepted":same,"duplicate":same,"workflow_id":workflow_id,"stream_name":stream, + "message_id":message_id,"position":DB::number(&existing,"position")?,"cursor_position":cursor, + "outcome":if same { "duplicate" } else { "rejected" },"reason":if same { Value::Null } else { json!("message_identity_conflict") }}); + return Ok((if same { StatusCode::OK } else { StatusCode::CONFLICT }, response)); + } + let position = DB::number(&row, "last_position")? + 1; + Self::query("INSERT INTO workflow_inbound_stream_items(stream_id,namespace,workflow_instance_id,stream_name,message_id,position,payload_codec,payload_blob,payload_hash,created_at,updated_at) VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11)") + .bind(stream_id).bind(self.namespace.as_ref()).bind(workflow_id).bind(stream).bind(message_id).bind(position) + .bind(payload_codec).bind(payload_blob).bind(&hash).bind(DB::bind_time(at)).bind(DB::bind_time(at)) + .execute(&mut *tx).await?; + Self::query("UPDATE workflow_inbound_streams SET last_position=$1,waiting_run_id=NULL,waiting_after_position=NULL,waiting_since=NULL,last_input_outcome='accepted',last_input_message_id=$2,last_input_at=$3,updated_at=$4 WHERE id=$5") + .bind(position).bind(message_id).bind(DB::bind_time(at)).bind(DB::bind_time(at)).bind(stream_id) + .execute(&mut *tx).await?; + let delivery = Self::message_delivery(stream, message_id, position, payload_codec, payload_blob); + let run_id = DB::string(&run, "id")?; + match Self::deliver_runtime_signal(&mut tx, workflow_id, delivery, context, codec, limits).await? { + Ok(command_id) => { + Self::mark_delivered(&mut tx, stream_id, message_id, &run_id, &command_id).await?; + } + Err(reason) => { + Self::query("DELETE FROM workflow_inbound_stream_items WHERE stream_id=$1 AND message_id=$2") + .bind(stream_id).bind(message_id).execute(&mut *tx).await?; + Self::query("UPDATE workflow_inbound_streams SET last_position=$1,last_input_outcome='message_stream_not_supported' WHERE id=$2") + .bind(position - 1).bind(stream_id).execute(&mut *tx).await?; + tx.commit().await?; + self.wake.notify_waiters(); + return Err(rejection(StatusCode::CONFLICT, "message_stream_not_supported", workflow_id, stream, message_id, + json!({"detail":reason,"remediation":"Use a PHP, Python, or Rust worker that advertises the message_streams capability."}))); + } + } + tx.summary_changed(&run_id); + tx.commit().await?; + self.wake.notify_waiters(); + Ok((StatusCode::ACCEPTED, json!({"accepted":true,"duplicate":false,"workflow_id":workflow_id,"run_id":run_id, + "stream_name":stream,"message_id":message_id,"position":position,"cursor_position":cursor, + "outcome":"accepted","reason":null}))) + } + + /// `MessageStreamService::recordMalformed`: counts a refused append + /// against the stream of an open run, keeping no payload. + pub(crate) async fn record_malformed_message(&self, workflow_id: &str, stream: &str, message_id: &Value) -> Result<()> { + let mut tx = self.begin().await?; + let bound = Self::query("SELECT id FROM workflow_instances WHERE id=$1 AND namespace=$2") + .bind(workflow_id).bind(self.namespace.as_ref()).fetch_optional(&mut *tx).await?; + if bound.is_none() || Self::active_current_run(&mut tx, workflow_id).await?.is_none() { + return Ok(()); + } + let row = self.locked_stream(&mut tx, workflow_id, stream).await?; + let at = now(); + Self::query("UPDATE workflow_inbound_streams SET malformed_count=malformed_count+1,last_input_outcome='malformed',last_input_message_id=$1,last_input_at=$2,updated_at=$3 WHERE id=$4") + .bind(diagnostic_message_id(message_id)).bind(DB::bind_time(at)).bind(DB::bind_time(at)).bind(DB::number(&row, "id")?) + .execute(&mut *tx).await?; + tx.commit().await?; + Ok(()) + } + + /// `MessageStreamService::diagnostics`: every stream of the instance, or + /// the named one, by name, without payloads. + pub(crate) async fn message_stream_diagnostics(&self, workflow_id: &str, stream: Option<&str>) -> Result> { + let rows = match stream { + Some(stream) => Self::query("SELECT * FROM workflow_inbound_streams WHERE namespace=$1 AND workflow_instance_id=$2 AND stream_name=$3 ORDER BY stream_name") + .bind(self.namespace.as_ref()).bind(workflow_id).bind(stream).fetch_all(&self.pool).await?, + None => Self::query("SELECT * FROM workflow_inbound_streams WHERE namespace=$1 AND workflow_instance_id=$2 ORDER BY stream_name") + .bind(self.namespace.as_ref()).bind(workflow_id).fetch_all(&self.pool).await?, + }; + let time = |row: &DB::Row, field: &str| -> Result { + Ok(super::task_queues::json_time(DB::optional_instant(row, field)?)) + }; + rows.iter() + .map(|row| { + let last = DB::number(row, "last_position")?; + let cursor = DB::number(row, "cursor_position")?; + let wait = match DB::optional_string(row, "waiting_run_id")? { + Some(run_id) => json!({"run_id":run_id,"after_position":DB::optional_number(row,"waiting_after_position")?, + "since":time(row,"waiting_since")?}), + None => Value::Null, + }; + Ok(json!({"stream_name":DB::string(row,"stream_name")?,"last_position":last,"cursor_position":cursor, + "pending_count":(last - cursor).max(0),"pending_wait":wait, + "duplicate_count":DB::number(row,"duplicate_count")?,"malformed_count":DB::number(row,"malformed_count")?, + "last_input_outcome":DB::optional_string(row,"last_input_outcome")?, + "last_input_message_id":DB::optional_string(row,"last_input_message_id")?, + "last_input_at":time(row,"last_input_at")?})) + }) + .collect() + } + + /// `MessageStreamService::validateCompletion`: the worker's cursor + /// advances and waits, checked before the completion changes anything. + pub(super) async fn validate_stream_completion( + &self, + tx: &mut Transaction<'_, DB>, + instance_id: &str, + run_id: &str, + completion: &Completion, + ) -> Result<()> { + let mut proposed = BTreeMap::new(); + for (index, cursor) in completion.cursors.iter().enumerate() { + let field = |name: &str| format!("message_stream_cursors.{index}.{name}"); + let stream = metadata_name(cursor, &field("stream_name"))?; + if proposed.contains_key(&stream) { + return Err(invalid(&field("stream_name"), "A stream may be acknowledged only once per workflow task completion.".into())); + } + let Some(through) = non_negative(&cursor["through_position"]) else { + return Err(invalid(&field("through_position"), "The through_position must be a non-negative integer.".into())); + }; + let row = Self::query("SELECT id,last_position,cursor_position FROM workflow_inbound_streams WHERE namespace=$1 AND workflow_instance_id=$2 AND stream_name=$3") + .bind(self.namespace.as_ref()).bind(instance_id).bind(&stream).fetch_optional(&mut **tx).await?; + let Some(row) = row.filter(|row| DB::number(row, "last_position").is_ok_and(|last| through <= last)) else { + return Err(invalid(&field("through_position"), "The cursor cannot advance beyond the last accepted stream position.".into())); + }; + let current = DB::number(&row, "cursor_position")?; + if through > current { + let delivered = Self::scalar("SELECT COUNT(*) FROM workflow_inbound_stream_items WHERE stream_id=$1 AND position>$2 AND position<=$3 AND delivered_run_id=$4") + .bind(DB::number(&row, "id")?).bind(current).bind(through).bind(run_id).fetch_one(&mut **tx).await?; + if delivered != through - current { + return Err(invalid(&field("through_position"), "The cursor must advance contiguously through messages delivered to this workflow run.".into())); + } + } + proposed.insert(stream, current.max(through)); + } + let mut waiting = std::collections::BTreeSet::new(); + for (index, wait) in completion.waits.iter().enumerate() { + let field = |name: &str| format!("message_stream_waits.{index}.{name}"); + let stream = metadata_name(wait, &field("stream_name"))?; + if !waiting.insert(stream.clone()) { + return Err(invalid(&field("stream_name"), "A stream may have only one pending wait per workflow task completion.".into())); + } + let Some(after) = non_negative(&wait["after_position"]) else { + return Err(invalid(&field("after_position"), "The after_position must be a non-negative integer.".into())); + }; + let expected = match proposed.get(&stream) { + Some(cursor) => *cursor, + None => Self::query("SELECT cursor_position FROM workflow_inbound_streams WHERE namespace=$1 AND workflow_instance_id=$2 AND stream_name=$3") + .bind(self.namespace.as_ref()).bind(instance_id).bind(&stream).fetch_optional(&mut **tx).await? + .map(|row| DB::number(&row, "cursor_position")).transpose()?.unwrap_or(0), + }; + if after != expected { + return Err(invalid(&field("after_position"), "The pending wait must start after the durable stream cursor.".into())); + } + } + Ok(()) + } + + /// `MessageStreamService::recordCompletion`, after the completion has + /// been applied in the same transaction: consumed messages, each moved + /// cursor with `MessageCursorAdvanced`, pending waits, then the hand-over + /// to a run that continued this one. + pub(super) async fn record_stream_completion( + &self, + tx: &mut Transaction<'_, DB>, + instance_id: &str, + run_id: &str, + task_id: &str, + completion: &Completion, + codec: Arc, + limits: super::memo_limits::StructuralLimits, + ) -> Result<()> { + for cursor in &completion.cursors { + let stream = cursor["stream_name"].as_str().unwrap_or_default(); + let through = cursor["through_position"].as_i64().unwrap_or(0); + let row = self.locked_stream(tx, instance_id, stream).await?; + let previous = DB::number(&row, "cursor_position")?; + if through <= previous { + continue; + } + let at = now(); + Self::query("UPDATE workflow_inbound_stream_items SET consumed_run_id=$1,consumed_task_id=$2,consumed_at=$3,updated_at=$4 WHERE stream_id=$5 AND position>$6 AND position<=$7") + .bind(run_id).bind(task_id).bind(DB::bind_time(at)).bind(DB::bind_time(at)).bind(DB::number(&row, "id")?) + .bind(previous).bind(through).execute(&mut **tx).await?; + Self::query("UPDATE workflow_inbound_streams SET cursor_position=$1,waiting_run_id=NULL,waiting_after_position=NULL,waiting_since=NULL,updated_at=$2 WHERE id=$3") + .bind(through).bind(DB::bind_time(at)).bind(DB::number(&row, "id")?).execute(&mut **tx).await?; + let task = Self::query("SELECT * FROM workflow_tasks WHERE id=$1").bind(task_id).fetch_one(&mut **tx).await?; + Self::append(tx, run_id, "MessageCursorAdvanced", json!({"stream_key":stream,"previous_position":previous, + "new_position":through,"task":Self::task_snapshot(&task)?}), Some(task_id), None).await?; + } + for wait in &completion.waits { + let stream = wait["stream_name"].as_str().unwrap_or_default(); + let after = wait["after_position"].as_i64().unwrap_or(0); + let row = self.locked_stream(tx, instance_id, stream).await?; + if DB::number(&row, "last_position")? > after { + continue; + } + let at = now(); + Self::query("UPDATE workflow_inbound_streams SET waiting_run_id=$1,waiting_after_position=$2,waiting_since=$3,updated_at=$4 WHERE id=$5") + .bind(run_id).bind(after).bind(DB::bind_time(at)).bind(DB::bind_time(at)).bind(DB::number(&row, "id")?) + .execute(&mut **tx).await?; + } + let current = Self::query("SELECT current_run_id FROM workflow_instances WHERE id=$1") + .bind(instance_id).fetch_one(&mut **tx).await?; + let current = DB::optional_string(¤t, "current_run_id")?; + if let Some(current) = current.filter(|current| current != run_id) { + self.carry_streams_forward(tx, instance_id, ¤t, codec, limits).await?; + } + Ok(()) + } + + /// `carryForward`: a cursor checkpoint for each acknowledged stream the + /// current run has not received, then every unconsumed message the + /// current run has not received, by append order. + async fn carry_streams_forward( + &self, + tx: &mut Transaction<'_, DB>, + instance_id: &str, + current: &str, + codec: Arc, + limits: super::memo_limits::StructuralLimits, + ) -> Result<()> { + let context = control_plane_context(); + let streams = Self::query("SELECT id,stream_name,cursor_position FROM workflow_inbound_streams WHERE namespace=$1 AND workflow_instance_id=$2 AND cursor_position>0 AND (cursor_checkpoint_run_id IS NULL OR cursor_checkpoint_run_id<>$3) ORDER BY stream_name") + .bind(self.namespace.as_ref()).bind(instance_id).bind(current).fetch_all(&mut **tx).await?; + for stream in streams { + let checkpoint = map(vec![ + ("schema", crate::codec::Value::String(CURSOR_SCHEMA.into())), + ("stream_name", crate::codec::Value::String(DB::string(&stream, "stream_name")?)), + ("through_position", crate::codec::Value::Long(DB::number(&stream, "cursor_position")?)), + ]); + if Self::deliver_runtime_signal(tx, instance_id, checkpoint, &context, codec.clone(), limits).await?.is_err() { + return Err(RuntimeError::Refused { status: StatusCode::INTERNAL_SERVER_ERROR, reason: "message_stream_cursor_carry_failed" }); + } + Self::query("UPDATE workflow_inbound_streams SET cursor_checkpoint_run_id=$1 WHERE id=$2") + .bind(current).bind(DB::number(&stream, "id")?).execute(&mut **tx).await?; + } + loop { + let items = Self::query("SELECT stream_id,stream_name,message_id,position,payload_codec,payload_blob FROM workflow_inbound_stream_items WHERE namespace=$1 AND workflow_instance_id=$2 AND consumed_at IS NULL AND (delivered_run_id IS NULL OR delivered_run_id<>$3) ORDER BY id LIMIT 100") + .bind(self.namespace.as_ref()).bind(instance_id).bind(current).fetch_all(&mut **tx).await?; + if items.is_empty() { + return Ok(()); + } + for item in items { + let (stream, message_id) = (DB::string(&item, "stream_name")?, DB::string(&item, "message_id")?); + let delivery = Self::message_delivery(&stream, &message_id, DB::number(&item, "position")?, + &DB::string(&item, "payload_codec")?, &DB::string(&item, "payload_blob")?); + match Self::deliver_runtime_signal(tx, instance_id, delivery, &context, codec.clone(), limits).await? { + Ok(command_id) => { + Self::mark_delivered(tx, DB::number(&item, "stream_id")?, &message_id, current, &command_id).await?; + } + Err(_) => { + return Err(RuntimeError::Refused { status: StatusCode::INTERNAL_SERVER_ERROR, reason: "message_stream_carry_failed" }); + } + } + } + } + } + + /// The message a stream delivers: its schema, identity, position and the + /// inline payload envelope. + fn message_delivery(stream: &str, message_id: &str, position: i64, payload_codec: &str, payload_blob: &str) -> crate::codec::Value { + use crate::codec::Value as Payload; + map(vec![ + ("schema", Payload::String(MESSAGE_SCHEMA.into())), + ("stream_name", Payload::String(stream.into())), + ("message_id", Payload::String(message_id.into())), + ("position", Payload::Long(position)), + ("payload_envelope", map(vec![ + ("codec", Payload::String(payload_codec.into())), + ("blob", Payload::String(payload_blob.into())), + ])), + ]) + } + + async fn mark_delivered(tx: &mut Transaction<'_, DB>, stream_id: i64, message_id: &str, run_id: &str, command_id: &str) -> Result<()> { + let at = now(); + Self::query("UPDATE workflow_inbound_stream_items SET delivered_run_id=$1,workflow_command_id=$2,delivered_at=$3,updated_at=$4 WHERE stream_id=$5 AND message_id=$6") + .bind(run_id).bind(command_id).bind(DB::bind_time(at)).bind(DB::bind_time(at)).bind(stream_id).bind(message_id) + .execute(&mut **tx).await?; + Ok(()) + } + + /// `WorkflowStub::attemptRuntimeSignalWithArguments` for the instance's + /// current run: no declared-signal admission or argument contract, but + /// the pending-signal and payload ceilings still apply. The accepted + /// command's id, or the refusal's reason. + async fn deliver_runtime_signal( + tx: &mut Transaction<'_, DB>, + workflow_id: &str, + delivery: crate::codec::Value, + context: &Value, + codec: Arc, + limits: super::memo_limits::StructuralLimits, + ) -> Result> { + let instance = Self::query("SELECT current_run_id,last_message_sequence FROM workflow_instances WHERE id=$1") + .bind(workflow_id).fetch_one(&mut **tx).await?; + let run_id = DB::string(&instance, "current_run_id")?; + let run = Self::query("SELECT * FROM workflow_runs WHERE id=$1").bind(&run_id).fetch_one(&mut **tx).await?; + if !matches!(DB::string(&run, "status")?.as_str(), "pending" | "running" | "waiting") { + return Ok(Err("run_not_active")); + } + let prepared = runtime_arguments(codec.clone(), delivery).await?; + let command_id = id(); + let signal_id = id(); + let command_sequence = DB::number(&run, "last_command_sequence")? + 1; + let message_sequence = DB::number(&instance, "last_message_sequence")? + 1; + let workflow_type = DB::string(&run, "workflow_type")?; + let write = SignalWrite { workflow_id, run_id: &run_id, workflow_type: &workflow_type, name: INTERNAL_SIGNAL, + command_id: &command_id, signal_id: &signal_id, command_sequence, message_sequence, context }; + let pending = Self::scalar("SELECT COUNT(*) FROM workflow_signal_records WHERE workflow_run_id=$1 AND status='received'") + .bind(&run_id).fetch_one(&mut **tx).await?; + let exceeded = super::memo_limits::Exceeded::pending_signals(limits.pending_signals, pending) + .or_else(|| super::memo_limits::Exceeded::payload(limits.payload_bytes, &prepared.arguments)); + if let Some(exceeded) = exceeded { + let oversized = exceeded.kind == "payload_size"; + let (values, errors, diagnostics) = if oversized { + (vec![], json!({"arguments":[exceeded.message]}), exceeded.fields()) + } else { + (prepared.values.clone(), json!([]), json!({})) + }; + let command = super::store::prepare_rejected_command(codec.clone(), INTERNAL_SIGNAL.into(), values, + errors.clone(), diagnostics, "signal_codec_unavailable").await?; + let arguments = if oversized { super::queries::empty_arguments(codec).await? } else { prepared.arguments.clone() }; + let row_errors = if oversized { errors } else { json!([]) }; + Self::record_rejected_signal(tx, &write, None, "structural_limit_exceeded", &command, &arguments, &row_errors).await?; + return Ok(Err("structural_limit_exceeded")); + } + Self::record_accepted_signal(tx, &write, &prepared.command, &prepared.arguments, &DB::string(&run, "queue")?).await?; + tx.summary_changed(&run_id); + Ok(Ok(command_id)) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn names_follow_phps_pattern() { + assert!(valid_name("orders")); + assert!(valid_name("a.b_c:d-e")); + assert!(valid_name(&"x".repeat(128))); + assert!(!valid_name(&"x".repeat(129))); + assert!(!valid_name("")); + assert!(!valid_name("bad name")); + assert!(!valid_name("bad/name")); + } + + #[test] + fn refused_message_ids_are_bounded_for_diagnostics() { + assert_eq!(diagnostic_message_id(&Value::Null), "[omitted]"); + assert_eq!(diagnostic_message_id(&json!(7)), "[omitted]"); + assert_eq!(diagnostic_message_id(&json!("m3")), "m3"); + let long = "a".repeat(192); + assert_eq!(diagnostic_message_id(&json!(long)).len(), "sha256:".len() + 64); + assert!(diagnostic_message_id(&json!("bad id!")).starts_with("sha256:")); + } + + #[test] + fn payload_hash_covers_codec_and_blob() { + assert_ne!(payload_hash("avro", "a"), payload_hash("avro", "b")); + assert_ne!(payload_hash("avro", "ab"), payload_hash("avroa", "b")); + assert_eq!(payload_hash("avro", "x").len(), 64); + } + + #[test] + fn the_worker_capability_is_the_supported_manifest() { + let capability = worker_capability(); + assert_eq!(capability["supported"], true); + assert_eq!(capability["minimum_worker_protocol_version"], "1.15"); + assert_eq!(capability["transport"]["internal_signal"], INTERNAL_SIGNAL); + assert_eq!(manifest()["batch"]["maximum"], 100); + } + + #[test] + fn completion_fields_follow_laravels_rules() { + assert!(check_completion_fields(&json!({})).is_ok()); + assert!(check_completion_fields(&json!({"message_stream_cursors":[{"stream_name":"s","through_position":0}]})).is_ok()); + let Err(RuntimeError::Protocol { response, .. }) = + check_completion_fields(&json!({"message_stream_cursors":[{"stream_name":"s","through_position":-1}]})) + else { + panic!("expected a validation failure"); + }; + assert_eq!(response["message"], "The message_stream_cursors.0.through_position field must be at least 0."); + } +} diff --git a/rust/src/runtime/mod.rs b/rust/src/runtime/mod.rs index d60cb0473..ff9968292 100644 --- a/rust/src/runtime/mod.rs +++ b/rust/src/runtime/mod.rs @@ -117,6 +117,7 @@ mod service_catalog; mod sqlite; mod start_definition; mod start_request; +mod message_streams; mod storage_test; mod store; mod system_metrics; diff --git a/rust/src/runtime/retention.rs b/rust/src/runtime/retention.rs index 9e888fde5..b1a8e1047 100644 --- a/rust/src/runtime/retention.rs +++ b/rust/src/runtime/retention.rs @@ -78,9 +78,12 @@ pub(super) const STREAM_COUNTS: [&str; 3] = ["message_streams_deleted", "message_stream_items_deleted", "message_stream_items_compacted"]; /// Counts that hold a run's details, the run-id binds of each, and the -/// skip reason. An abandoned activity's hold is read from history. +/// skip reason. An abandoned activity's hold is read from history. A run +/// whose instance has inbound message streams is held too: native does not +/// compact or remove stream state as PHP's MessageStreamRetentionCleanup +/// does yet, so it keeps the run rather than orphan the streams. pub(super) const HOLDS: &[(&str, usize, &str)] = &[ - ("SELECT (SELECT COUNT(*) FROM workflow_durable_streams WHERE workflow_run_id=$1)+(SELECT COUNT(*) FROM workflow_messages WHERE workflow_run_id=$2)", 2, "message_stream_cleanup_unavailable"), + ("SELECT (SELECT COUNT(*) FROM workflow_durable_streams WHERE workflow_run_id=$1)+(SELECT COUNT(*) FROM workflow_messages WHERE workflow_run_id=$2)+(SELECT COUNT(*) FROM workflow_inbound_streams WHERE workflow_instance_id IN (SELECT workflow_instance_id FROM workflow_runs WHERE id=$3))", 3, "message_stream_cleanup_unavailable"), (REFERENCES_SQL, 5, "external_payload_cleanup_unavailable"), ]; diff --git a/rust/src/runtime/store.rs b/rust/src/runtime/store.rs index 93c299d12..a0c7a0fe5 100644 --- a/rust/src/runtime/store.rs +++ b/rust/src/runtime/store.rs @@ -15,8 +15,21 @@ pub(super) struct PreparedControlArguments { pub(super) value: String, pub(super) command: String, } +/// One signal command's identity and sequences in its run, as both the +/// public signal route and the runtime's message-stream deliveries write it. +pub(super) struct SignalWrite<'a> { + pub(super) workflow_id: &'a str, + pub(super) run_id: &'a str, + pub(super) workflow_type: &'a str, + pub(super) name: &'a str, + pub(super) command_id: &'a str, + pub(super) signal_id: &'a str, + pub(super) command_sequence: i64, + pub(super) message_sequence: i64, + pub(super) context: &'a Value, +} /// The runtime-owned message-stream delivery signal (`WorkflowStub::MESSAGE_STREAM_RUNTIME_SIGNAL`). -const MESSAGE_STREAM_SIGNAL: &str = "__durable_workflow_message_stream"; +const MESSAGE_STREAM_SIGNAL: &str = super::message_streams::INTERNAL_SIGNAL; pub(super) const TASK_CANDIDATES_SQL: &str = "SELECT t.*,r.workflow_type FROM workflow_tasks t JOIN workflow_runs r ON r.id=t.workflow_run_id WHERE t.namespace=$1 AND t.queue=$2 AND t.task_type=$3 AND r.status IN ('pending','running','waiting') AND (t.status='ready' OR (t.status='leased' AND t.task_type='workflow' AND t.lease_expires_at<=$4)) AND (t.available_at IS NULL OR t.available_at<=$5) AND (COALESCE(NULLIF(t.compatibility,''),NULLIF(r.compatibility,'')) IS NULL OR COALESCE(NULLIF(t.compatibility,''),NULLIF(r.compatibility,''))=$6) ORDER BY t.priority,t.available_at,t.id LIMIT 100"; /// `nextEligibleSignal`: a run's received signals in PHP's order: command /// sequence (unsequenced last), receipt time, then id. @@ -779,25 +792,9 @@ where "signal_codec_unavailable", ) .await?; - let rejected_at = now(); - Self::query("INSERT INTO workflow_commands(id,workflow_instance_id,workflow_run_id,resolved_workflow_run_id,command_type,source,status,outcome,workflow_type,payload_codec,payload,rejection_reason,command_sequence,message_sequence,rejected_at,context) VALUES ($1,$2,$3,$4,'signal','control_plane','rejected',$5,$6,'avro',$7,$8,$9,$10,$11,$12)") - .bind(&command_id).bind(workflow_id).bind(¤t_run).bind(¤t_run).bind(outcome).bind(&workflow_type) - .bind(&prepared.command).bind(reason).bind(command_sequence).bind(message_sequence).bind(DB::bind_time(rejected_at)) - .bind(DB::document(&context)).execute(&mut *tx).await?; - Self::query("INSERT INTO workflow_signal_records(id,workflow_command_id,workflow_instance_id,workflow_run_id,resolved_workflow_run_id,signal_name,status,outcome,command_sequence,payload_codec,arguments,validation_errors,rejection_reason,rejected_at,closed_at) VALUES ($1,$2,$3,$4,$5,$6,'rejected',$7,$8,'avro',$9,$10,$11,$12,$13)") - .bind(&signal_id).bind(&command_id).bind(workflow_id).bind(¤t_run).bind(¤t_run).bind(name) - .bind(outcome).bind(command_sequence).bind(&row_arguments).bind(DB::document(&row_errors)).bind(reason) - .bind(DB::bind_time(rejected_at)).bind(DB::bind_time(rejected_at)).execute(&mut *tx).await?; - Self::query("UPDATE workflow_runs SET last_command_sequence=$1 WHERE id=$2") - .bind(command_sequence) - .bind(¤t_run) - .execute(&mut *tx) - .await?; - Self::query("UPDATE workflow_instances SET last_message_sequence=$1 WHERE id=$2") - .bind(message_sequence) - .bind(workflow_id) - .execute(&mut *tx) - .await?; + let write = SignalWrite { workflow_id, run_id: ¤t_run, workflow_type: &workflow_type, name, + command_id: &command_id, signal_id: &signal_id, command_sequence, message_sequence, context: &context }; + Self::record_rejected_signal(&mut tx, &write, outcome, reason, &prepared.command, &row_arguments, &row_errors).await?; tx.commit().await?; let mut response = json!({"workflow_id":workflow_id,"run_id":current_run,"resolved_run_id":current_run, "requested_run_id":null,"signal_name":name,"command_id":command_id,"command_sequence":command_sequence, @@ -812,40 +809,88 @@ where // signal wait reservation or notification is added here. return Err(super::RuntimeError::Protocol { status, response }); } - let wait = Self::open_wait(&mut tx, ¤t_run).await?; - let mut wait_id = format!("signal-command:{command_id}"); - if let Some(wait) = wait.as_ref().filter(|wait| wait["signal_name"] == name) { + let workflow_type = DB::string(&run, "workflow_type")?; + let write = SignalWrite { workflow_id, run_id: ¤t_run, workflow_type: &workflow_type, name, + command_id: &command_id, signal_id: &signal_id, command_sequence, message_sequence, context: &context }; + Self::record_accepted_signal(&mut tx, &write, &prepared.command, &prepared.arguments, &DB::string(&run, "queue")?).await?; + tx.commit().await?; + self.wake.notify_waiters(); + Ok( + json!({"workflow_id":workflow_id,"run_id":current_run,"resolved_run_id":current_run,"requested_run_id":null,"signal_name":name,"command_id":command_id,"command_sequence":command_sequence,"workflow_type":DB::string(&run, "workflow_type")?,"command_status":"accepted","command_source":"control_plane","target_scope":"instance","outcome":"signal_received","reason":null,"rejection_reason":null,"validation_errors":[]}), + ) + } + + /// A refused signal's durable admission record: its rejected command, + /// its rejected signal row and the advanced sequences. No history, task + /// or wait reservation is added. + pub(super) async fn record_rejected_signal( + tx: &mut Transaction<'_, DB>, + write: &SignalWrite<'_>, + outcome: Option<&str>, + reason: &str, + command: &str, + arguments: &str, + errors: &Value, + ) -> Result<()> { + let rejected_at = now(); + Self::query("INSERT INTO workflow_commands(id,workflow_instance_id,workflow_run_id,resolved_workflow_run_id,command_type,source,status,outcome,workflow_type,payload_codec,payload,rejection_reason,command_sequence,message_sequence,rejected_at,context) VALUES ($1,$2,$3,$4,'signal','control_plane','rejected',$5,$6,'avro',$7,$8,$9,$10,$11,$12)") + .bind(write.command_id).bind(write.workflow_id).bind(write.run_id).bind(write.run_id).bind(outcome).bind(write.workflow_type) + .bind(command).bind(reason).bind(write.command_sequence).bind(write.message_sequence).bind(DB::bind_time(rejected_at)) + .bind(DB::document(write.context)).execute(&mut **tx).await?; + Self::query("INSERT INTO workflow_signal_records(id,workflow_command_id,workflow_instance_id,workflow_run_id,resolved_workflow_run_id,signal_name,status,outcome,command_sequence,payload_codec,arguments,validation_errors,rejection_reason,rejected_at,closed_at) VALUES ($1,$2,$3,$4,$5,$6,'rejected',$7,$8,'avro',$9,$10,$11,$12,$13)") + .bind(write.signal_id).bind(write.command_id).bind(write.workflow_id).bind(write.run_id).bind(write.run_id).bind(write.name) + .bind(outcome).bind(write.command_sequence).bind(arguments).bind(DB::document(errors)).bind(reason) + .bind(DB::bind_time(rejected_at)).bind(DB::bind_time(rejected_at)).execute(&mut **tx).await?; + Self::advance_signal_sequences(tx, write).await + } + + /// `recordAcceptedSignalCommand`, `recordAcceptedSignal` and + /// `recordSignalReceivedHistory`: the accepted command, the received + /// signal reserving an open wait for its name, the advanced sequences, + /// `SignalReceived`, then the run's next message task if none is open. + pub(super) async fn record_accepted_signal( + tx: &mut Transaction<'_, DB>, + write: &SignalWrite<'_>, + command: &str, + arguments: &str, + queue: &str, + ) -> Result<()> { + let wait = Self::open_wait(tx, write.run_id).await?; + let mut wait_id = format!("signal-command:{}", write.command_id); + if let Some(wait) = wait.as_ref().filter(|wait| wait["signal_name"] == write.name) { let reserved = Self::scalar("SELECT COUNT(*) FROM workflow_signal_records WHERE workflow_run_id=$1 AND signal_wait_id=$2 AND status='received'") - .bind(¤t_run).bind(text(wait, "signal_wait_id")?).fetch_one(&mut *tx).await?; + .bind(write.run_id).bind(text(wait, "signal_wait_id")?).fetch_one(&mut **tx).await?; if reserved == 0 { wait_id = text(wait, "signal_wait_id")?.to_owned(); } } let accepted_at = now(); Self::query("INSERT INTO workflow_commands(id,workflow_instance_id,workflow_run_id,resolved_workflow_run_id,command_type,source,status,outcome,workflow_type,payload_codec,payload,command_sequence,message_sequence,accepted_at,context) VALUES ($1,$2,$3,$4,'signal','control_plane','accepted','signal_received',$5,'avro',$6,$7,$8,$9,$10)") - .bind(&command_id).bind(workflow_id).bind(¤t_run).bind(¤t_run).bind(DB::string(&run, "workflow_type")?) - .bind(&prepared.command).bind(command_sequence).bind(message_sequence).bind(DB::bind_time(accepted_at)) - .bind(DB::document(&context)).execute(&mut *tx).await?; + .bind(write.command_id).bind(write.workflow_id).bind(write.run_id).bind(write.run_id).bind(write.workflow_type) + .bind(command).bind(write.command_sequence).bind(write.message_sequence).bind(DB::bind_time(accepted_at)) + .bind(DB::document(write.context)).execute(&mut **tx).await?; Self::query("INSERT INTO workflow_signal_records(id,workflow_command_id,workflow_instance_id,workflow_run_id,resolved_workflow_run_id,signal_name,signal_wait_id,status,outcome,command_sequence,payload_codec,arguments,received_at) VALUES ($1,$2,$3,$4,$5,$6,$7,'received','signal_received',$8,'avro',$9,$10)") - .bind(&signal_id).bind(&command_id).bind(workflow_id).bind(¤t_run).bind(¤t_run).bind(name).bind(&wait_id) - .bind(command_sequence).bind(&prepared.arguments).bind(DB::bind_time(accepted_at)).execute(&mut *tx).await?; + .bind(write.signal_id).bind(write.command_id).bind(write.workflow_id).bind(write.run_id).bind(write.run_id).bind(write.name).bind(&wait_id) + .bind(write.command_sequence).bind(arguments).bind(DB::bind_time(accepted_at)).execute(&mut **tx).await?; + Self::advance_signal_sequences(tx, write).await?; + Self::append(tx, write.run_id, "SignalReceived", json!({"workflow_command_id":write.command_id,"signal_id":write.signal_id, + "workflow_instance_id":write.workflow_id,"workflow_run_id":write.run_id,"signal_name":write.name,"signal_wait_id":wait_id, + "arguments":wire(arguments),"payload_codec":"avro"}), None, Some(write.command_id)).await?; + Self::enqueue_next_message(tx, write.run_id, queue).await + } + + async fn advance_signal_sequences(tx: &mut Transaction<'_, DB>, write: &SignalWrite<'_>) -> Result<()> { Self::query("UPDATE workflow_runs SET last_command_sequence=$1 WHERE id=$2") - .bind(command_sequence) - .bind(¤t_run) - .execute(&mut *tx) + .bind(write.command_sequence) + .bind(write.run_id) + .execute(&mut **tx) .await?; Self::query("UPDATE workflow_instances SET last_message_sequence=$1 WHERE id=$2") - .bind(message_sequence) - .bind(workflow_id) - .execute(&mut *tx) + .bind(write.message_sequence) + .bind(write.workflow_id) + .execute(&mut **tx) .await?; - Self::append(&mut tx, ¤t_run, "SignalReceived", json!({"workflow_command_id":command_id,"signal_id":signal_id,"workflow_instance_id":workflow_id,"workflow_run_id":current_run,"signal_name":name,"signal_wait_id":wait_id,"arguments":wire(&prepared.arguments),"payload_codec":"avro"}), None, Some(&command_id)).await?; - Self::enqueue_next_message(&mut tx, ¤t_run, &DB::string(&run, "queue")?).await?; - tx.commit().await?; - self.wake.notify_waiters(); - Ok( - json!({"workflow_id":workflow_id,"run_id":current_run,"resolved_run_id":current_run,"requested_run_id":null,"signal_name":name,"command_id":command_id,"command_sequence":command_sequence,"workflow_type":DB::string(&run, "workflow_type")?,"command_status":"accepted","command_source":"control_plane","target_scope":"instance","outcome":"signal_received","reason":null,"rejection_reason":null,"validation_errors":[]}), - ) + Ok(()) } pub(super) async fn authored_sequence( @@ -1043,6 +1088,17 @@ where run_id: &str, queue: &str, ) -> Result<()> { + Self::next_message_task(tx, run_id, queue).await.map(drop) + } + + /// `PendingMessageTask::createForRun`: the run's next update or signal + /// task, when it is open, has no open workflow task and holds a message + /// that can advance it. The created task's id. + pub(super) async fn next_message_task( + tx: &mut Transaction<'_, DB>, + run_id: &str, + queue: &str, + ) -> Result> { let status = Self::query("SELECT status FROM workflow_runs WHERE id=$1") .bind(run_id) .fetch_one(&mut **tx) @@ -1051,12 +1107,12 @@ where DB::string(&status, "status")?.as_str(), "pending" | "running" | "waiting" ) { - return Ok(()); + return Ok(None); } let open = Self::scalar("SELECT COUNT(*) FROM workflow_tasks WHERE workflow_run_id=$1 AND task_type='workflow' AND status IN ('ready','leased')") .bind(run_id).fetch_one(&mut **tx).await?; if open != 0 { - return Ok(()); + return Ok(None); } // `nextReadyUpdate`: the earliest accepted update, unless an earlier // signal is still unapplied. That signal then goes first. @@ -1074,7 +1130,7 @@ where .await? .is_none() { - Self::create_update_task( + let task_id = Self::create_update_task( tx, run_id, queue, @@ -1082,7 +1138,7 @@ where &DB::string(&update, "workflow_command_id")?, ) .await?; - return Ok(()); + return Ok(Some(task_id)); } let (wait, condition_open, declared) = Self::signal_delivery(tx, run_id).await?; let signals = Self::query(RECEIVED_SIGNALS).bind(run_id).fetch_all(&mut **tx).await?; @@ -1092,10 +1148,10 @@ where if !Self::signal_can_advance(wait.as_ref(), condition_open, declared.as_deref(), &name, &signal_wait_id) { continue; } - Self::create_task(tx, run_id, queue, "workflow", json!({"resume_source_kind":"workflow_signal","workflow_signal_id":DB::string(&signal,"id")?,"signal_name":name,"signal_wait_id":signal_wait_id})).await?; - break; + let task_id = Self::create_task(tx, run_id, queue, "workflow", json!({"resume_source_kind":"workflow_signal","workflow_signal_id":DB::string(&signal,"id")?,"signal_name":name,"signal_wait_id":signal_wait_id})).await?; + return Ok(Some(task_id)); } - Ok(()) + Ok(None) } async fn apply_task_signal( @@ -1124,15 +1180,20 @@ where let wait = open.clone().filter(|wait| { wait.get("signal_name").is_none() || (wait["signal_name"] == name && wait["signal_wait_id"] == wait_id) }); - if wait.is_none() - && !Self::declared_signals(tx, run_id) - .await? - .is_some_and(|names| names.contains(&name)) - { + let declared = Self::declared_signals(tx, run_id) + .await? + .is_some_and(|names| names.contains(&name)); + if wait.is_none() && !declared { let reason = if open.is_some() { "signal_wait_mismatch" } else { "signal_wait_not_open" }; return Err(refuse(StatusCode::CONFLICT, reason)); } let direct = wait.as_ref().is_some_and(|wait| wait.get("signal_name").is_some()); + // `recordAppliedSignalForSignalResume` applies a signal its open + // signal wait matches, or one the durable contract declares, with + // `SignalApplied`. Otherwise `markConditionWaitSignalConsumed`: the + // open condition wait consumes it at the wait's sequence, without + // that event (an undeclared or runtime-reserved signal). + let applies = direct || declared; let condition_timer = match &wait { Some(wait) if !direct => Self::condition_timer(tx, run_id, wait).await?, _ => None, @@ -1156,8 +1217,9 @@ where .bind(&command_id) .execute(&mut **tx) .await?; + let recorded_sequence = if direct || !declared { sequence } else { None }; Self::query("UPDATE workflow_signal_records SET status='applied',workflow_sequence=$1,applied_at=$2,closed_at=$3 WHERE id=$4") - .bind(DB::optional_integer(sequence)).bind(DB::bind_time(applied_at)).bind(DB::bind_time(applied_at)).bind(signal_id).execute(&mut **tx).await?; + .bind(DB::optional_integer(recorded_sequence)).bind(DB::bind_time(applied_at)).bind(DB::bind_time(applied_at)).bind(signal_id).execute(&mut **tx).await?; let run = Self::query( "SELECT workflow_instance_id,message_cursor_position FROM workflow_runs WHERE id=$1", ) @@ -1177,7 +1239,7 @@ where .bind(run_id) .execute(&mut **tx) .await?; - Self::append(tx, run_id, "MessageCursorAdvanced", json!({"stream_key":format!("instance:{}",DB::string(&run,"workflow_instance_id")?),"previous_position":previous,"new_position":position}), Some(task_id), None).await?; + Self::append(tx, run_id, "MessageCursorAdvanced", json!({"stream_key":format!("instance:{}",DB::string(&run,"workflow_instance_id")?),"previous_position":previous,"new_position":position,"task":Self::task_snapshot(task)?}), Some(task_id), None).await?; if let Some(wait) = wait.as_ref().filter(|_| direct) { Self::cancel_wait_timeout(tx, run_id, wait, task_id, applied_at).await?; } @@ -1185,15 +1247,17 @@ where if direct { applied["sequence"] = json!(sequence); } - Self::append( - tx, - run_id, - "SignalApplied", - applied, - Some(task_id), - Some(&command_id), - ) - .await?; + if applies { + Self::append( + tx, + run_id, + "SignalApplied", + applied, + Some(task_id), + Some(&command_id), + ) + .await?; + } if !direct && let Some(mut satisfied) = wait && !condition_timer.as_ref().is_some_and(|(_, fired)| *fired) @@ -2106,7 +2170,9 @@ where external: super::external_resolution::PreparedCommands, worker: Option, protocol: Option<&str>, + limits: super::memo_limits::StructuralLimits, ) -> Result { + let transition_codec_for_streams = transition_codec.clone(); let commands = body["commands"] .as_array() .filter(|c| !c.is_empty() && c.len() <= 100) @@ -2262,6 +2328,17 @@ where ); } Self::workflow_task_guard(&mut tx, &task, &body).await?; + // `MessageStreamService::validateCompletion`, before anything changes. + let streams = super::message_streams::Completion::of(&body); + let stream_instance = Self::query("SELECT workflow_instance_id FROM workflow_runs WHERE id=$1") + .bind(DB::string(&task, "workflow_run_id")?) + .fetch_one(&mut *tx) + .await?; + let stream_instance = DB::string(&stream_instance, "workflow_instance_id")?; + if !streams.is_empty() { + self.validate_stream_completion(&mut tx, &stream_instance, &DB::string(&task, "workflow_run_id")?, &streams) + .await?; + } // `childCancellationPolicyGuard`: a child that requests cancellation // on parent close, or propagates the parent's cancellation, needs a // cooperative worker's claim and a protocol 1.20 request. @@ -2654,10 +2731,14 @@ where .bind(task_id) .execute(&mut *tx) .await?; - Self::record_completion(&mut tx, task_id, &json!({COMPLETION_RECEIPT: {"request": body, "created_task_ids": created}})).await?; Self::resume_undelivered_cancellation(&mut tx, &run_id, &DB::string(&run, "queue")?) .await?; - Self::enqueue_next_message(&mut tx, &run_id, &DB::string(&run, "queue")?).await?; + // PHP reports the run's next message task with the tasks the + // commands created. + if let Some(next) = Self::next_message_task(&mut tx, &run_id, &DB::string(&run, "queue")?).await? { + created.push(next); + } + Self::record_completion(&mut tx, task_id, &json!({COMPLETION_RECEIPT: {"request": body, "created_task_ids": created}})).await?; // PHP's executor leaves every run a completed task did not close // waiting, even when the task recorded only markers or side effects. Self::query("UPDATE workflow_runs SET status='waiting' WHERE id=$1 AND status IN ('pending','running')") @@ -2672,6 +2753,12 @@ where if let Some(worker) = &worker { Self::attribute_worker_terminal(&mut tx, &run_id, task_id, worker).await?; } + // `MessageStreamService::recordCompletion`: only a continued run + // changes the instance's current run, which then receives the streams. + if !streams.is_empty() || commands.iter().any(|command| command["type"] == "continue_as_new") { + self.record_stream_completion(&mut tx, &stream_instance, &run_id, task_id, &streams, transition_codec_for_streams, limits) + .await?; + } tx.summary_changed(&run_id); tx.commit().await?; self.wake.notify_waiters(); @@ -3960,7 +4047,7 @@ pub(crate) fn capabilities() -> Value { let mut value = json!({"supported_workflow_task_commands": ["schedule_activity", "start_timer", "start_child_workflow", "open_condition_wait", "open_signal_wait", "record_version_marker", "record_side_effect", "complete_workflow", "fail_workflow", "continue_as_new", "complete_update", "fail_update"], "worker_deregistration_fencing":{"schema":"durable-workflow.v2.worker-deregistration.v1","supported":true,"receipt_retention_seconds":600,"endpoint":"/worker/registrations/{workerId}/deregister"}, "workflow_memo_updates": false, "cooperative_cancellation": true, "prepared_local_activities": false, - "worker_sessions": false, "sticky_execution": false, "local_activities": false, "message_streams": false, + "worker_sessions": false, "sticky_execution": false, "local_activities": false, "message_streams": super::message_streams::worker_capability(), "workflow_updates": true, "query_tasks": true, "query_task_authority":true, "query_task_poll_request_idempotency": true, "activity_lease_renewal": true}); value["supported_workflow_task_commands"] diff --git a/rust/tests/execution.rs b/rust/tests/execution.rs index c3ce551e4..551fa00cf 100644 --- a/rust/tests/execution.rs +++ b/rust/tests/execution.rs @@ -109,6 +109,8 @@ mod parent_close_policies; mod history_pages; #[path = "execution/declared_signals.rs"] mod declared_signals; +#[path = "execution/message_streams.rs"] +mod message_streams; #[path = "execution/poll_waits.rs"] mod poll_waits; #[path = "execution/child_cascade.rs"] diff --git a/rust/tests/execution/message_streams.rs b/rust/tests/execution/message_streams.rs new file mode 100644 index 000000000..254cfb6f8 --- /dev/null +++ b/rust/tests/execution/message_streams.rs @@ -0,0 +1,178 @@ +use super::*; + +// Server 2.11.1 message streams: ordered, deduplicated appends delivered to +// the current run as the runtime-reserved signal; worker acknowledgements and +// waits on workflow task completions; the hand-over to a continued run. + +const STREAM_SIGNAL: &str = "__durable_workflow_message_stream"; + +fn stream_worker() -> Value { + json!({"worker_id":"streams","task_queue":"test","runtime":"php","supported_workflow_types":["echo"], + "supported_activity_types":[],"capabilities":["message_streams"]}) +} + +/// An SDK receive: a condition wait keyed by the stream cursor. +fn receive(cursor: i64) -> Value { + json!([{"type":"open_condition_wait","condition_key":format!("message-stream:orders:{cursor}"), + "condition_definition_fingerprint":format!("sha256:{}", "a".repeat(64))}]) +} + +async fn append(app: &Router, workflow: &str, stream: &str, body: Value) -> (StatusCode, Value) { + request(app, "POST", &format!("/api/workflows/{workflow}/message-streams/{stream}/messages"), body).await +} + +async fn complete(app: &Router, task: &Value, commands: Value, extra: Value, protocol: &str) -> (StatusCode, Value) { + let mut body = completion(task, commands); + body.as_object_mut().unwrap().extend(extra.as_object().unwrap().clone()); + request_protocol(app, "POST", &format!("/api/worker/workflow-tasks/{}/complete", task["task_id"].as_str().unwrap()), body, protocol).await +} + +async fn streams(app: &Router, workflow: &str) -> Value { + let (status, body) = request(app, "GET", &format!("/api/workflows/{workflow}/message-streams"), Value::Null).await; + assert_eq!(status, StatusCode::OK, "{body}"); + body +} + +/// The stream deliveries a history holds: (schema tail, stream, message id or cursor). +fn deliveries(history: &Value) -> Vec<(String, String, String)> { + history.as_array().unwrap().iter() + .filter(|event| event["event_type"] == "SignalReceived" && event["payload"]["signal_name"] == STREAM_SIGNAL) + .map(|event| { + let blob = event["payload"]["arguments"]["blob"].as_str().unwrap(); + let Payload::Array(values) = durable_workflow_server::codec::ValueCodec::new().unwrap().decode(blob).unwrap() else { + panic!("one argument list"); + }; + let Payload::Map(delivery) = &values[0] else { panic!("one delivery map") }; + let text = |key: &str| match delivery.get(key) { + Some(Payload::String(value)) => value.clone(), + Some(Payload::Long(value)) => value.to_string(), + other => panic!("{key}: {other:?}"), + }; + let schema = text("schema"); + let marker = if schema.ends_with(".cursor") { text("through_position") } else { text("message_id") }; + (schema.rsplit('.').next().unwrap().to_owned(), text("stream_name"), marker) + }) + .collect() +} + +fn types(history: &Value) -> Vec { + history.as_array().unwrap().iter().map(|event| event["event_type"].as_str().unwrap().to_owned()).collect() +} + +#[tokio::test] +async fn appended_messages_reach_the_run_and_follow_it_when_it_continues() { + let database = TestDatabase::new().await; + let app = router(database.open().await.unwrap()); + assert_eq!(request(&app, "POST", "/api/worker/register", stream_worker()).await.0, StatusCode::CREATED); + let started = start(&app, "streams-flow").await; + let task = poll(&app, "streams", "workflow").await; + let waits = json!({"message_stream_waits":[{"stream_name":"orders","after_position":0}]}); + let (status, body) = complete(&app, &task, receive(0), waits, "1.20").await; + assert_eq!(status, StatusCode::OK, "{body}"); + let listed = streams(&app, "streams-flow").await; + assert_eq!(listed["streams"][0]["pending_wait"]["after_position"], 0, "{listed}"); + assert_eq!(listed["streams"][0]["pending_wait"]["run_id"], started["run_id"]); + + let (status, first) = append(&app, "streams-flow", "orders", json!({"message_id":"m1","input":[{"n":1}]})).await; + assert_eq!((status, &first["position"], &first["run_id"]), (StatusCode::ACCEPTED, &json!(1), &started["run_id"]), "{first}"); + let (status, again) = append(&app, "streams-flow", "orders", json!({"message_id":"m1","input":[{"n":1}]})).await; + assert_eq!((status, &again["duplicate"], &again["position"]), (StatusCode::OK, &json!(true), &json!(1)), "{again}"); + assert!(again.get("run_id").is_none(), "a duplicate names no run: {again}"); + let (status, conflict) = append(&app, "streams-flow", "orders", json!({"message_id":"m1","input":[{"n":9}]})).await; + assert_eq!((status, &conflict["reason"]), (StatusCode::CONFLICT, &json!("message_identity_conflict")), "{conflict}"); + assert_eq!(append(&app, "streams-flow", "orders", json!({"message_id":"m2","input":[{"n":2}]})).await.1["position"], 2); + let listed = streams(&app, "streams-flow").await; + let orders = &listed["streams"][0]; + assert_eq!((&orders["last_position"], &orders["pending_count"], &orders["duplicate_count"]), (&json!(2), &json!(2), &json!(2)), "{listed}"); + assert!(orders["pending_wait"].is_null(), "an append clears the wait: {listed}"); + + // The condition wait wakes for the first message; the worker reads both. + let task = poll(&app, "streams", "workflow").await; + assert_eq!(deliveries(&task["history_events"]), [("message".into(), "orders".into(), "m1".into()), ("message".into(), "orders".into(), "m2".into())]); + let cursor = json!({"message_stream_cursors":[{"stream_name":"orders","through_position":1}]}); + let (status, body) = complete(&app, &task, receive(1), cursor, "1.20").await; + assert_eq!(status, StatusCode::OK, "{body}"); + assert_eq!(body["created_task_ids"].as_array().unwrap().len(), 1, "the next message task is reported: {body}"); + let history = run_history(&app, &started["workflow_id"], &started["run_id"]).await; + assert!(!types(&history).contains(&"SignalApplied".to_owned()), "the condition wait consumes the message: {history}"); + let cursors: Vec<_> = history.as_array().unwrap().iter().filter(|event| event["event_type"] == "MessageCursorAdvanced").collect(); + assert_eq!(cursors.len(), 2, "{history}"); + assert_eq!(cursors[0]["payload"]["stream_key"], "instance:streams-flow"); + assert_eq!(cursors[0]["payload"]["task"]["status"], "leased"); + assert_eq!((&cursors[1]["payload"]["stream_key"], &cursors[1]["payload"]["new_position"]), (&json!("orders"), &json!(1))); + assert_eq!(cursors[1]["payload"]["task"]["status"], "completed"); + assert_eq!(streams(&app, "streams-flow").await["streams"][0]["cursor_position"], 1); + + // Continue as new acknowledging the second message: the next run gets the + // cursor and the third message, which was still unconsumed. + assert_eq!(append(&app, "streams-flow", "orders", json!({"message_id":"m3","input":[{"n":3}]})).await.1["position"], 3); + let task = poll(&app, "streams", "workflow").await; + let cursor = json!({"message_stream_cursors":[{"stream_name":"orders","through_position":2}]}); + let (status, body) = complete(&app, &task, json!([{"type":"continue_as_new","arguments":envelope(Payload::Array(vec![]))}]), cursor, "1.20").await; + assert_eq!(status, StatusCode::OK, "{body}"); + let task = poll(&app, "streams", "workflow").await; + assert_ne!(task["run_id"], started["run_id"], "{task}"); + assert_eq!(deliveries(&task["history_events"]), [("cursor".into(), "orders".into(), "2".into()), ("message".into(), "orders".into(), "m3".into())]); + let cursor = json!({"message_stream_cursors":[{"stream_name":"orders","through_position":3}]}); + let (status, body) = complete(&app, &task, json!([{"type":"complete_workflow","result":envelope(Payload::String("done".into()))}]), cursor, "1.20").await; + assert_eq!(status, StatusCode::OK, "{body}"); + let orders = &streams(&app, "streams-flow").await["streams"][0]; + assert_eq!((&orders["cursor_position"], &orders["pending_count"]), (&json!(3), &json!(0)), "{orders}"); + let (status, closed) = append(&app, "streams-flow", "orders", json!({"message_id":"m1","input":[{"n":1}]})).await; + assert_eq!((status, &closed["reason"]), (StatusCode::CONFLICT, &json!("run_not_active")), "{closed}"); +} + +#[tokio::test] +async fn refused_appends_and_acknowledgements_change_nothing_but_the_counts() { + let database = TestDatabase::new().await; + let app = router(database.open().await.unwrap()); + assert_eq!(request(&app, "POST", "/api/worker/register", stream_worker()).await.0, StatusCode::CREATED); + start(&app, "streams-refused").await; + let task = poll(&app, "streams", "workflow").await; + for (extra, field) in [ + (json!({"message_stream_waits":[{"stream_name":"orders","after_position":1}]}), "message_stream_waits.0.after_position"), + (json!({"message_stream_cursors":[{"stream_name":"orders","through_position":1}]}), "message_stream_cursors.0.through_position"), + (json!({"message_stream_waits":[{"stream_name":"bad name","after_position":0}]}), "message_stream_waits.0.stream_name"), + (json!({"message_stream_cursors":[{"stream_name":"orders","through_position":-1}]}), "message_stream_cursors.0.through_position"), + ] { + let (status, body) = complete(&app, &task, receive(0), extra, "1.20").await; + assert_eq!(status, StatusCode::UNPROCESSABLE_ENTITY, "{body}"); + assert!(body["errors"].get(field).is_some(), "{field}: {body}"); + } + let (status, body) = complete(&app, &task, receive(0), json!({"message_stream_waits":[{"stream_name":"orders","after_position":0}]}), "1.14").await; + assert_eq!((status, &body["reason"]), (StatusCode::CONFLICT, &json!("message_streams_unavailable")), "{body}"); + assert_eq!(streams(&app, "streams-refused").await["count"], 0, "a refused completion records no stream"); + + for (body, field) in [(json!({"input":[1]}), "message_id"), (json!({"message_id":"bad id!"}), "message_id"), + (json!({"message_id":"x".repeat(192)}), "message_id"), (json!({"message_id":"m1","input":"text"}), "input")] { + let (status, refused) = append(&app, "streams-refused", "orders", body).await; + assert_eq!(status, StatusCode::UNPROCESSABLE_ENTITY, "{refused}"); + assert!(refused["errors"].get(field).is_some(), "{refused}"); + } + let (status, refused) = append(&app, "streams-refused", "bad%20name", json!({"message_id":"m1"})).await; + assert_eq!(status, StatusCode::UNPROCESSABLE_ENTITY, "{refused}"); + let (status, refused) = append(&app, "streams-missing", "orders", json!({"message_id":"m1"})).await; + assert_eq!((status, &refused["reason"]), (StatusCode::NOT_FOUND, &json!("instance_not_found")), "{refused}"); + let orders = &streams(&app, "streams-refused").await["streams"][0]; + assert_eq!((&orders["malformed_count"], &orders["last_position"]), (&json!(4), &json!(0)), "{orders}"); + // The last refused append kept a valid id; the refused stream name and + // the unknown workflow count nowhere. + assert_eq!((&orders["last_input_outcome"], &orders["last_input_message_id"]), (&json!("malformed"), &json!("m1")), "{orders}"); + let (status, refused) = request(&app, "POST", &format!("/api/workflows/streams-refused/signal/{STREAM_SIGNAL}"), json!([1])).await; + assert_eq!((status, &refused["reason"]), (StatusCode::UNPROCESSABLE_ENTITY, &json!("runtime_reserved_signal")), "{refused}"); + let (status, missing) = request(&app, "GET", "/api/workflows/streams-refused/message-streams/other", Value::Null).await; + assert_eq!((status, &missing["reason"]), (StatusCode::NOT_FOUND, &json!("message_stream_not_found")), "{missing}"); +} + +#[tokio::test] +async fn cluster_info_and_workers_see_the_contract() { + let database = TestDatabase::new().await; + let app = router(database.open().await.unwrap()); + let (status, info) = request(&app, "GET", "/api/cluster/info", Value::Null).await; + assert_eq!(status, StatusCode::OK, "{info}"); + assert_eq!(info["message_streams_contract"]["schema"], "durable-workflow.v2.message-streams.contract"); + assert_eq!(info["message_streams_contract"]["transport"]["internal_signal"], STREAM_SIGNAL); + let capability = &info["worker_protocol"]["server_capabilities"]["message_streams"]; + assert_eq!(capability["supported"], true, "{capability}"); + assert_eq!(capability["minimum_worker_protocol_version"], "1.15"); +} diff --git a/scripts/conformance/server-parity/contract.mjs b/scripts/conformance/server-parity/contract.mjs index 85f39f1ac..a04c2efdc 100644 --- a/scripts/conformance/server-parity/contract.mjs +++ b/scripts/conformance/server-parity/contract.mjs @@ -19,6 +19,7 @@ import {checkPrometheusMetrics} from './prometheus-metrics-contract.mjs'; import {checkPayloadCleanup} from './payload-cleanup-contract.mjs'; import {checkRepairPass} from './repair-pass-contract.mjs'; import {checkNamespaceDelete} from './namespace-delete-contract.mjs'; +import {checkMessageStreams} from './message-streams-contract.mjs'; import {checkSchedulePolicies} from './schedule-policies-contract.mjs'; import {checkScheduleCatalog} from './schedule-catalog-contract.mjs'; import {checkSignalsBeforeWorker} from './signals-before-worker-contract.mjs'; @@ -155,6 +156,7 @@ export function checkObservation(fixture, observation, workflowId, artifacts) { const payloadCleanup = fixture.payload_cleanup ? checkPayloadCleanup(fixture, observation, workflowId) : null; const repairPass = fixture.repair_pass ? checkRepairPass(fixture, observation, workflowId) : null; const namespaceDelete = fixture.namespace_delete ? checkNamespaceDelete(fixture, observation, workflowId) : null; + const messageStreams = fixture.message_streams ? checkMessageStreams(fixture, observation, workflowId) : null; const schedulePolicies = fixture.schedule_policies ? checkSchedulePolicies(fixture, observation, workflowId) : null; const scheduleCatalog = fixture.schedule_catalog ? checkScheduleCatalog(fixture, observation, workflowId) : null; const signalsBeforeWorker = fixture.signals_before_worker ? checkSignalsBeforeWorker(fixture, observation, workflowId) : null; @@ -708,6 +710,7 @@ export function checkObservation(fixture, observation, workflowId, artifacts) { ...(payloadCleanup ? {payload_cleanup: payloadCleanup} : {}), ...(repairPass ? {repair_pass: repairPass} : {}), ...(namespaceDelete ? {namespace_delete: namespaceDelete} : {}), + ...(messageStreams ? {message_streams: messageStreams} : {}), ...(schedulePolicies ? {schedule_policies: schedulePolicies} : {}), ...(scheduleCatalog ? {schedule_catalog: scheduleCatalog} : {}), ...(signalsBeforeWorker ? {signals_before_worker: signalsBeforeWorker} : {}), diff --git a/scripts/conformance/server-parity/message-streams-contract.mjs b/scripts/conformance/server-parity/message-streams-contract.mjs new file mode 100644 index 000000000..829508c2a --- /dev/null +++ b/scripts/conformance/server-parity/message-streams-contract.mjs @@ -0,0 +1,191 @@ +import assert from 'node:assert/strict'; + +const TIME = /^\d{4}-\d\d-\d\dT\d\d:\d\d:\d\d\.\d{6}Z$/; +const MESSAGE = 'durable-workflow.v2.message-stream.message'; +const CURSOR = 'durable-workflow.v2.message-stream.cursor'; + +// Laravel's 422 with one attribute and its one message. +function refused(read, field, message, label) { + assert.equal(read.status, 422, `${label} status`); + assert.deepStrictEqual(read.response, {message, reason: 'validation_failed', errors: {[field]: [message]}, + validation_errors: {[field]: [message]}}, `${label} body`); +} + +// One stream's diagnostics: exact counts, the pending wait by run and a +// microsecond UTC time, and the last input. +function checkStream(entry, expected, runId, label) { + const {pending_wait: wait, last_input_at: inputAt, ...rest} = entry; + const {wait: expectedWait = null, input = null, ...counts} = expected; + assert.deepStrictEqual(rest, {...counts, pending_count: counts.last_position - counts.cursor_position, + duplicate_count: counts.duplicate_count ?? 0, malformed_count: counts.malformed_count ?? 0, + last_input_outcome: input?.outcome ?? null, last_input_message_id: input?.message_id ?? null}, `${label} counts`); + if (expectedWait === null) { + assert.equal(wait, null, `${label} has no pending wait`); + } else { + assert.equal(wait?.run_id, runId, `${label} pending wait run`); + assert.equal(wait?.after_position, expectedWait.after_position, `${label} pending wait position`); + assert.match(wait?.since ?? '', TIME, `${label} pending wait time`); + } + if (input === null) { + assert.equal(inputAt, null, `${label} has no input yet`); + } else { + assert.match(inputAt ?? '', TIME, `${label} last input time`); + } +} + +function checkList(read, workflowId, streams, runId, label) { + assert.equal(read.status, 200, `${label} status`); + assert.equal(read.response.workflow_id, workflowId, `${label} workflow`); + assert.equal(read.response.count, Object.keys(streams).length, `${label} count`); + assert.deepStrictEqual(read.response.streams.map(entry => entry.stream_name), Object.keys(streams), `${label} streams by name`); + for (const entry of read.response.streams) { + checkStream(entry, {stream_name: entry.stream_name, ...streams[entry.stream_name]}, runId, `${label} ${entry.stream_name}`); + } +} + +function accepted(read, workflowId, runId, stream, messageId, position, cursor, label) { + assert.equal(read.status, 202, `${label} status`); + assert.deepStrictEqual(read.response, {accepted: true, duplicate: false, workflow_id: workflowId, run_id: runId, + stream_name: stream, message_id: messageId, position, cursor_position: cursor, outcome: 'accepted', reason: null}, `${label} body`); +} + +function completed(read, runId, status, created, label) { + assert.equal(read.status, 200, `${label} status`); + const {task_id: taskId, created_task_ids: createdIds, ...rest} = read.response; + assert.equal(typeof taskId, 'string', `${label} task`); + assert.deepStrictEqual(rest, {workflow_task_attempt: 1, outcome: 'completed', recorded: true, run_id: runId, + run_status: status, reason: null}, `${label} outcome`); + assert.equal(createdIds.length, created, `${label} created tasks, the next message task included`); +} + +// A message delivery or cursor checkpoint as the worker reads it. +function delivery(stream, messageId, position, payload) { + return [{schema: MESSAGE, stream_name: stream, message_id: messageId, position, payload_envelope: ['blob', 'codec'], payload}]; +} + +// Reviewed message streams (published Server 2.11.1). +export function checkMessageStreams(fixture, observation, workflowId) { + const definition = fixture.message_streams; + const state = observation.message_streams; + // The same for every target: embedded applications have no HTTP message streams. + const projection = {scope: 'message_streams', first_run_events: definition.first_run_events.length, + continued_run_events: definition.continued_run_events.length}; + assert.ok(state && typeof state === 'object', 'actual message stream observation'); + if (observation.mode === 'embedded') { + assert.deepStrictEqual(state, {applicable: false, reason: 'embedded_has_no_http_message_streams'}, + 'embedded message streams explicitly inapplicable'); + return projection; + } + const id = `${workflowId}-ms`; + const {stream, other_stream: other, refusals: text, messages} = definition; + const run = state.run_id; + assert.equal(state.workflow_id, id, 'stream workflow'); + assert.deepStrictEqual(state.cluster_contract, definition.manifest, 'cluster info publishes the contract'); + assert.deepStrictEqual(state.worker_capability, {...definition.manifest, supported: true}, 'workers see the supported contract'); + + assert.deepStrictEqual(state.list_before, {status: 200, response: {workflow_id: id, count: 0, streams: []}}, 'no streams yet'); + assert.deepStrictEqual(state.show_before, {status: 404, response: {workflow_id: id, stream_name: stream, reason: 'message_stream_not_found'}}, + 'an unknown stream'); + refused(state.show_bad_name, 'stream_name', text.stream_name, 'a stream name outside the pattern'); + assert.deepStrictEqual(state.list_unknown, {status: 404, response: {message: 'Workflow not found.', reason: 'instance_not_found'}}, + 'an unknown workflow'); + assert.deepStrictEqual(state.poll1.history, [], 'nothing delivered before the first append'); + + const first = state.complete1_refusals; + refused(first.wait_ahead, 'message_stream_waits.0.after_position', text.wait_after_cursor, 'a wait ahead of the cursor'); + refused(first.cursor_ahead, 'message_stream_cursors.0.through_position', text.cursor_beyond, 'a cursor past the last message'); + refused(first.bad_name, 'message_stream_waits.0.stream_name', text.metadata_stream_name, 'a wait stream name outside the pattern'); + refused(first.repeated_wait, 'message_stream_waits.1.stream_name', text.duplicate_wait, 'two waits on one stream'); + assert.equal(first.old_protocol.status, 409, 'stream metadata before protocol 1.15'); + const {task_id: oldTask, ...oldProtocol} = first.old_protocol.response; + assert.equal(typeof oldTask, 'string', 'the refused task'); + assert.deepStrictEqual(oldProtocol, {workflow_task_attempt: 1, outcome: 'rejected', reason: 'message_streams_unavailable', + requested_version: '1.14', minimum_protocol_version: '1.15', remediation: text.old_protocol_remediation}, 'the protocol refusal'); + completed(state.complete1, run, 'waiting', 0, 'the first completion with a wait'); + checkList(state.list_waiting, id, {[stream]: {last_position: 0, cursor_position: 0, wait: {after_position: 0}}}, run, 'the waiting stream'); + + const appends = state.appends; + accepted(appends.m1, id, run, stream, 'm1', 1, 0, 'the first message'); + assert.deepStrictEqual(appends.m1_duplicate, {status: 200, response: {accepted: true, duplicate: true, workflow_id: id, stream_name: stream, + message_id: 'm1', position: 1, cursor_position: 0, outcome: 'duplicate', reason: null}}, 'the same message again'); + assert.deepStrictEqual(appends.m1_conflict, {status: 409, response: {accepted: false, duplicate: false, workflow_id: id, stream_name: stream, + message_id: 'm1', position: 1, cursor_position: 0, outcome: 'rejected', reason: 'message_identity_conflict'}}, 'the id with another payload'); + assert.deepStrictEqual(appends.m2_sdk.response, {accepted: true, duplicate: false, workflow_id: id, run_id: run, stream_name: stream, + message_id: 'm2', position: 2, cursor_position: 0, outcome: 'accepted', reason: null}, 'the published SDK append'); + refused(appends.no_id, 'message_id', text.message_id_required, 'no message id'); + refused(appends.bad_id, 'message_id', text.message_id_format, 'a message id outside the pattern'); + refused(appends.long_id, 'message_id', text.message_id_max, 'a message id too long'); + refused(appends.bad_input, 'input', text.input_array, 'input that is not a list'); + refused(appends.bad_stream, 'stream_name', text.stream_name, 'an append to a stream name outside the pattern'); + assert.deepStrictEqual(appends.unknown_workflow, {status: 404, response: {accepted: false, workflow_id: `${id}-missing`, + stream_name: stream, outcome: 'rejected', reason: 'instance_not_found'}}, 'an append to an unknown workflow'); + accepted(appends.other_stream, id, run, other, 'a1', 1, 0, 'a message without input on another stream'); + assert.equal(appends.reserved_signal.status, 422, 'the runtime signal is not a public signal'); + assert.equal(appends.reserved_signal.response.reason, 'runtime_reserved_signal', 'the reserved signal refusal'); + // Four refused appends to the stream are malformed; the last kept the + // valid id m3. The refused stream name counts nowhere. + const appended = { + [other]: {last_position: 1, cursor_position: 0, input: {outcome: 'accepted', message_id: 'a1'}}, + [stream]: {last_position: 2, cursor_position: 0, duplicate_count: 2, malformed_count: 4, input: {outcome: 'malformed', message_id: 'm3'}}, + }; + checkList(state.list_after_appends, id, appended, run, 'after the appends'); + assert.equal(state.show_after_appends.status, 200, 'one stream'); + assert.deepStrictEqual(state.show_after_appends.response, {workflow_id: id, + stream: state.list_after_appends.response.streams.find(entry => entry.stream_name === stream)}, 'one stream as listed'); + + assert.equal(state.poll2.run_id, run, 'the woken run'); + assert.deepStrictEqual(state.poll2.history, [ + {event_type: 'SignalReceived', deliveries: delivery(stream, 'm1', 1, messages.m1)}, + {event_type: 'SignalReceived', deliveries: delivery(stream, 'm2', 2, messages.m2)}, + {event_type: 'SignalReceived', deliveries: delivery(other, 'a1', 1, [])}, + ], 'each message is one runtime signal, in append order'); + const second = state.complete2_refusals; + refused(second.beyond, 'message_stream_cursors.0.through_position', text.cursor_beyond, 'a cursor past the last message'); + refused(second.negative, 'message_stream_cursors.0.through_position', text.negative_cursor, 'a negative cursor'); + refused(second.wait_behind, 'message_stream_waits.0.after_position', text.wait_after_cursor, 'a wait behind the new cursor'); + // Two messages are still unconsumed, so the next message task follows. + completed(state.complete2, run, 'waiting', 1, 'the first acknowledgement'); + assert.deepStrictEqual(state.history_after2.filter(event => event.event_type === 'MessageCursorAdvanced'), [ + {event_type: 'MessageCursorAdvanced', stream_key: 'instance', previous_position: 0, new_position: 1, task_status: 'leased'}, + {event_type: 'MessageCursorAdvanced', stream_key: stream, previous_position: 0, new_position: 1, task_status: 'completed'}, + ], 'the instance cursor while the task runs, then the stream cursor once it completed'); + checkList(state.list_after2, id, {...appended, [stream]: {...appended[stream], cursor_position: 1}}, run, 'after the acknowledgement'); + + accepted(appends.m3, id, run, stream, 'm3', 3, 1, 'the third message'); + assert.equal(state.poll3.run_id, run, 'the same run again'); + assert.deepStrictEqual(state.poll3.history.filter(event => event.event_type === 'SignalReceived').at(-1), + {event_type: 'SignalReceived', deliveries: delivery(stream, 'm3', 3, messages.m3)}, 'the third message delivered'); + completed(state.complete3, run, 'completed', 1, 'continue as new'); + checkList(state.list_after_continue, id, {...appended, [stream]: {...appended[stream], last_position: 3, cursor_position: 2, + input: {outcome: 'accepted', message_id: 'm3'}}}, run, 'after continue as new'); + + const continued = state.continued_run_id; + assert.equal(state.poll4.run_id, continued, 'the continued run'); + assert.notEqual(continued, run, 'a new run'); + assert.deepStrictEqual(state.poll4.history, [ + {event_type: 'SignalReceived', deliveries: [{schema: CURSOR, stream_name: stream, through_position: 2}]}, + {event_type: 'SignalReceived', deliveries: delivery(other, 'a1', 1, [])}, + {event_type: 'SignalReceived', deliveries: delivery(stream, 'm3', 3, messages.m3)}, + ], 'the continued run receives the cursor, then every unconsumed message by append order'); + completed(state.complete4, continued, 'completed', 0, 'the continued run completes'); + checkList(state.list_final, id, {...appended, [stream]: {...appended[stream], last_position: 3, cursor_position: 3, + input: {outcome: 'accepted', message_id: 'm3'}}}, run, 'the final streams'); + for (const [name, messageId] of [['closed', 'm4'], ['closed_duplicate', 'm1']]) { + assert.deepStrictEqual(appends[name], {status: 409, response: {accepted: false, duplicate: false, workflow_id: id, stream_name: stream, + message_id: messageId, position: null, outcome: 'rejected', reason: 'run_not_active'}}, `${name}: the instance has no open run`); + } + + assert.deepStrictEqual(state.first_run_events, definition.first_run_events, + 'the first run: deliveries consumed by the condition waits without SignalApplied, the rest rejected at the close'); + assert.deepStrictEqual(state.continued_run_events, definition.continued_run_events, 'the continued run'); + assert.deepStrictEqual(state.first_run_streams.filter(event => event.event_type === 'MessageCursorAdvanced').map(event => event.stream_key), + ['instance', stream, 'instance', stream], 'first run cursors'); + assert.ok(!JSON.stringify(state.list_final).includes('"n":'), 'diagnostics never carry payloads'); + for (const event of [...state.first_run_streams, ...state.continued_run_streams]) { + if (event.event_type === 'SignalReceived') { + assert.equal(event.deliveries.length, 1, 'one delivery per runtime signal'); + } + } + + return projection; +} diff --git a/scripts/conformance/server-parity/message-streams-probe.php b/scripts/conformance/server-parity/message-streams-probe.php new file mode 100644 index 000000000..bd1869b16 --- /dev/null +++ b/scripts/conformance/server-parity/message-streams-probe.php @@ -0,0 +1,163 @@ +payloadCodec(); + $token = getenv('DW_PARITY_TOKEN') ?: null; + // The SQLite reference shares its database with a one-second maintenance + // loop; retry its retryable lock-pressure refusals, as the other probes do. + $stack = HandlerStack::create(); + $stack->push(LockPressurePollRetries::middleware(), 'lock_pressure_poll_retry'); + $http = new MessageStreamsHttp(['base_uri' => rtrim($url, '/').'/api/', 'http_errors' => false, + 'connect_timeout' => 1, 'timeout' => 30, 'handler' => $stack]); + $raw = static function (string $method, string $path, ?array $body = null, ?string $worker = null) use ($http, $token): array { + $response = $http->request($method, $path, ['headers' => array_filter([ + 'Authorization' => $token === null ? null : 'Bearer '.$token, + 'Accept' => 'application/json', 'X-Namespace' => 'default', + $worker !== null ? 'X-Durable-Workflow-Protocol-Version' : 'X-Durable-Workflow-Control-Plane-Version' => $worker ?? '2', + ]), ...($body === null ? [] : ['json' => $body])]); + $decoded = json_decode((string) $response->getBody(), true, flags: JSON_THROW_ON_ERROR); + if (is_array($decoded)) { + unset($decoded['protocol_version'], $decoded['server_capabilities']); + } + + return ['status' => $response->getStatusCode(), 'response' => $decoded]; + }; + $append = static fn (string $streamName, array $body): array => $raw('POST', + "workflows/{$id}/message-streams/".rawurlencode($streamName).'/messages', $body); + $complete = static fn (array $task, array $body, string $protocol = '1.20'): array => $raw('POST', + "worker/workflow-tasks/{$task['task_id']}/complete", + ['lease_owner' => $task['lease_owner'] ?? $worker, 'workflow_task_attempt' => $task['workflow_task_attempt']] + $body, $protocol); + $wait = static fn (int $cursor): array => ['type' => 'open_condition_wait', + 'condition_key' => "message-stream:{$stream}:{$cursor}", + 'condition_definition_fingerprint' => 'sha256:'.hash('sha256', $definition['condition_fingerprint_source'])]; + // A stream's deliveries and cursor moves in history, decoded. + $streamEvents = static function (array $events) use ($codec, $id): array { + $rows = []; + foreach ($events as $event) { + $payload = $event['payload'] ?? []; + if (($event['event_type'] ?? null) === 'SignalReceived' + && ($payload['signal_name'] ?? null) === '__durable_workflow_message_stream') { + $deliveries = $codec->decodeEnvelope($payload['arguments']); + foreach ($deliveries as &$delivery) { + if (is_array($delivery['payload_envelope'] ?? null)) { + $envelope = $delivery['payload_envelope']; + $delivery['payload_envelope'] = array_keys($envelope); + sort($delivery['payload_envelope']); + $delivery['payload'] = $codec->decodeEnvelope($envelope); + } + } + unset($delivery); + $rows[] = ['event_type' => 'SignalReceived', 'deliveries' => $deliveries]; + } elseif (($event['event_type'] ?? null) === 'MessageCursorAdvanced') { + $key = $payload['stream_key'] ?? null; + $rows[] = ['event_type' => 'MessageCursorAdvanced', + 'stream_key' => $key === "instance:{$id}" ? 'instance' : $key, + 'previous_position' => $payload['previous_position'] ?? null, + 'new_position' => $payload['new_position'] ?? null, + 'task_status' => $payload['task']['status'] ?? null]; + } + } + + return $rows; + }; + $history = static function (string $runId) use ($raw, $id): array { + $read = $raw('GET', "workflows/{$id}/runs/{$runId}/history"); + + return $read['response']['events'] ?? []; + }; + $poll = static function () use ($client, $worker, $queue, $streamEvents): array { + $task = $client->pollWorkflowTask($worker, $queue, 0); + if (! is_array($task)) { + throw new RuntimeException('No workflow task was ready for the message stream run.'); + } + + return [$task, ['run_id' => $task['run_id'] ?? null, 'history' => $streamEvents($task['history_events'] ?? [])]]; + }; + $state = ['applicable' => true, 'workflow_id' => $id]; + $info = $raw('GET', 'cluster/info')['response']; + $state['cluster_contract'] = $info['message_streams_contract'] ?? null; + $state['worker_capability'] = $info['worker_protocol']['server_capabilities']['message_streams'] ?? null; + $state['run_id'] = $client->startWorkflow($definition['workflow_type'], $id, $queue, [$fixture['input']])->selectedRunId; + $state['list_before'] = $raw('GET', "workflows/{$id}/message-streams"); + $state['show_before'] = $raw('GET', "workflows/{$id}/message-streams/{$stream}"); + $state['show_bad_name'] = $raw('GET', "workflows/{$id}/message-streams/".rawurlencode($definition['bad_stream'])); + $state['list_unknown'] = $raw('GET', "workflows/{$id}-missing/message-streams"); + $client->registerWorker($worker, $queue, [$definition['workflow_type']], []); + [$task, $state['poll1']] = $poll(); + $state['complete1_refusals'] = [ + 'wait_ahead' => $complete($task, ['commands' => [$wait(0)], 'message_stream_waits' => [['stream_name' => $stream, 'after_position' => 1]]]), + 'cursor_ahead' => $complete($task, ['commands' => [$wait(0)], 'message_stream_cursors' => [['stream_name' => $stream, 'through_position' => 1]]]), + 'bad_name' => $complete($task, ['commands' => [$wait(0)], 'message_stream_waits' => [['stream_name' => 'bad name', 'after_position' => 0]]]), + 'repeated_wait' => $complete($task, ['commands' => [$wait(0)], 'message_stream_waits' => [ + ['stream_name' => $stream, 'after_position' => 0], ['stream_name' => $stream, 'after_position' => 0]]]), + 'old_protocol' => $complete($task, ['commands' => [$wait(0)], 'message_stream_waits' => [['stream_name' => $stream, 'after_position' => 0]]], '1.14'), + ]; + $state['complete1'] = $complete($task, ['commands' => [$wait(0)], 'message_stream_waits' => [['stream_name' => $stream, 'after_position' => 0]]]); + $state['list_waiting'] = $raw('GET', "workflows/{$id}/message-streams"); + $messages = $definition['messages']; + $state['appends'] = [ + 'm1' => $append($stream, ['message_id' => 'm1', 'input' => $messages['m1']]), + 'm1_duplicate' => $append($stream, ['message_id' => 'm1', 'input' => $messages['m1']]), + 'm1_conflict' => $append($stream, ['message_id' => 'm1', 'input' => $messages['conflict']]), + // The published SDK client's own append. + 'm2_sdk' => ['status' => null, 'response' => $client->appendMessageStream($id, $stream, 'm2', $messages['m2'])], + 'no_id' => $append($stream, ['input' => $messages['m3']]), + 'bad_id' => $append($stream, ['message_id' => 'bad id!', 'input' => $messages['m3']]), + 'long_id' => $append($stream, ['message_id' => str_repeat('a', $definition['long_message_id_length']), 'input' => $messages['m3']]), + 'bad_input' => $append($stream, ['message_id' => 'm3', 'input' => 'text']), + 'bad_stream' => $append($definition['bad_stream'], ['message_id' => 'm3', 'input' => $messages['m3']]), + 'unknown_workflow' => $raw('POST', "workflows/{$id}-missing/message-streams/{$stream}/messages", ['message_id' => 'm1', 'input' => [1]]), + 'other_stream' => $append($definition['other_stream'], ['message_id' => 'a1']), + 'reserved_signal' => $raw('POST', "workflows/{$id}/signal/__durable_workflow_message_stream", ['input' => [1]]), + ]; + $state['list_after_appends'] = $raw('GET', "workflows/{$id}/message-streams"); + $state['show_after_appends'] = $raw('GET', "workflows/{$id}/message-streams/{$stream}"); + [$task, $state['poll2']] = $poll(); + $state['complete2_refusals'] = [ + 'beyond' => $complete($task, ['commands' => [$wait(1)], 'message_stream_cursors' => [['stream_name' => $stream, 'through_position' => 3]]]), + 'negative' => $complete($task, ['commands' => [$wait(1)], 'message_stream_cursors' => [['stream_name' => $stream, 'through_position' => -1]]]), + 'wait_behind' => $complete($task, ['commands' => [$wait(1)], 'message_stream_cursors' => [['stream_name' => $stream, 'through_position' => 1]], + 'message_stream_waits' => [['stream_name' => $stream, 'after_position' => 0]]]), + ]; + $state['complete2'] = $complete($task, ['commands' => [$wait(1)], 'message_stream_cursors' => [['stream_name' => $stream, 'through_position' => 1]]]); + $state['history_after2'] = $streamEvents($history($state['run_id'])); + $state['list_after2'] = $raw('GET', "workflows/{$id}/message-streams"); + $state['appends']['m3'] = $append($stream, ['message_id' => 'm3', 'input' => $messages['m3']]); + [$task, $state['poll3']] = $poll(); + $state['complete3'] = $complete($task, ['commands' => [['type' => 'continue_as_new', 'arguments' => $codec->envelope([$fixture['input']])]], + 'message_stream_cursors' => [['stream_name' => $stream, 'through_position' => 2]]]); + $state['list_after_continue'] = $raw('GET', "workflows/{$id}/message-streams"); + [$task, $state['poll4']] = $poll(); + $state['complete4'] = $complete($task, ['commands' => [['type' => 'complete_workflow', 'result' => $codec->envelope('done')]], + 'message_stream_cursors' => [['stream_name' => $stream, 'through_position' => 3]]]); + $state['continued_run_id'] = $state['poll4']['run_id']; + $state['list_final'] = $raw('GET', "workflows/{$id}/message-streams"); + $state['appends']['closed'] = $append($stream, ['message_id' => 'm4', 'input' => [1]]); + $state['appends']['closed_duplicate'] = $append($stream, ['message_id' => 'm1', 'input' => $messages['m1']]); + $eventTypes = static fn (array $events): array => array_map(static fn (array $event): string => $event['event_type'], $events); + $firstRun = $history($state['run_id']); + $continuedRun = $history($state['continued_run_id']); + $state['first_run_events'] = $eventTypes($firstRun); + $state['first_run_streams'] = $streamEvents($firstRun); + $state['continued_run_events'] = $eventTypes($continuedRun); + $state['continued_run_streams'] = $streamEvents($continuedRun); + + return $state; +} diff --git a/scripts/conformance/server-parity/probe.php b/scripts/conformance/server-parity/probe.php index b6339da20..2f4ea2339 100644 --- a/scripts/conformance/server-parity/probe.php +++ b/scripts/conformance/server-parity/probe.php @@ -437,6 +437,7 @@ function httpObservation(array $fixture, string $workflowId, string $namespace, $payloadCleanup = isset($fixture['payload_cleanup']) ? finishHttpPayloadCleanup($client, $fixture, $workflowId, $url) : null; $repairPass = isset($fixture['repair_pass']) ? finishHttpRepairPass($client, $fixture, $workflowId, $url) : null; $namespaceDelete = isset($fixture['namespace_delete']) ? finishHttpNamespaceDelete($client, $fixture, $workflowId, $queue, $url) : null; + $messageStreams = isset($fixture['message_streams']) ? finishHttpMessageStreams($client, $fixture, $workflowId, $url) : null; $schedulePolicies = isset($fixture['schedule_policies']) ? finishHttpSchedulePolicies($client, $fixture, $workflowId, $url) : null; $scheduleCatalog = isset($fixture['schedule_catalog']) ? finishHttpSchedulePolicies($client, $fixture, $workflowId, $url, 'schedule_catalog') @@ -543,6 +544,7 @@ function httpObservation(array $fixture, string $workflowId, string $namespace, 'payload_cleanup' => $payloadCleanup, 'repair_pass' => $repairPass, 'namespace_delete' => $namespaceDelete, + 'message_streams' => $messageStreams, 'schedule_policies' => $schedulePolicies, 'schedule_catalog' => $scheduleCatalog, 'signals_before_worker' => $signalsBeforeWorker, @@ -853,6 +855,7 @@ function embeddedObservation(array $fixture, string $workflowId, string $namespa require __DIR__.'/payload-cleanup-probe.php'; require __DIR__.'/repair-pass-probe.php'; require __DIR__.'/namespace-delete-probe.php'; + require __DIR__.'/message-streams-probe.php'; require __DIR__.'/schedule-policies-probe.php'; require __DIR__.'/signals-before-worker-probe.php'; require __DIR__.'/worker-deregistration-probe.php'; @@ -1032,6 +1035,9 @@ function embeddedObservation(array $fixture, string $workflowId, string $namespa if ($mode === 'embedded' && isset($fixture['namespace_delete'])) { $observation['namespace_delete'] = ['applicable' => false, 'reason' => 'embedded_has_no_http_namespaces']; } + if ($mode === 'embedded' && isset($fixture['message_streams'])) { + $observation['message_streams'] = ['applicable' => false, 'reason' => 'embedded_has_no_http_message_streams']; + } if ($mode === 'embedded' && isset($fixture['schedule_policies'])) { $observation['schedule_policies'] = ['applicable' => false, 'reason' => 'embedded_has_no_http_schedules']; } diff --git a/tests/Fixtures/ServerParity/message-streams.json b/tests/Fixtures/ServerParity/message-streams.json new file mode 100644 index 000000000..49b0ea7ef --- /dev/null +++ b/tests/Fixtures/ServerParity/message-streams.json @@ -0,0 +1,182 @@ +{ + "schema": "durable-workflow.server-parity-fixture/v1", + "id": "message-streams", + "review_state": "reviewed", + "workflow_type": "parity.v1.echo", + "input": { + "message": "message streams λ", + "count": 9007199254740993, + "optional": null + }, + "typed_value": { + "type": "map", + "value": { + "message": { + "type": "string", + "value": "message streams λ" + }, + "count": { + "type": "int64", + "value": "9007199254740993" + }, + "optional": { + "type": "null", + "value": null + } + } + }, + "activity": false, + "expected_events": [ + "StartAccepted", + "WorkflowStarted", + "WorkflowCompleted" + ], + "message_streams": { + "workflow_type": "parity.v1.message_streams", + "stream": "orders", + "other_stream": "audit", + "bad_stream": "bad name!", + "condition_fingerprint_source": "message-stream:orders", + "manifest": { + "schema": "durable-workflow.v2.message-streams.contract", + "version": 1, + "capability_flag": "message_streams", + "minimum_worker_protocol_version": "1.15", + "scope": "workflow_instance", + "transport": { + "append_path": "/api/workflows/{workflow_id}/message-streams/{stream_name}/messages", + "diagnostics_path": "/api/workflows/{workflow_id}/message-streams", + "internal_signal": "__durable_workflow_message_stream", + "message_schema": "durable-workflow.v2.message-stream.message", + "cursor_schema": "durable-workflow.v2.message-stream.cursor", + "payload_envelope": "inline_avro_at_the_server_to_worker_delivery_boundary", + "durable_payload": "exact_inline_avro_bytes_or_the_verified_external_storage_reference", + "payload_visibility": "payload_is_present_only_in_durable_signal_history_and_is_omitted_from_stream_diagnostics" + }, + "identity": { + "producer_key": "message_id", + "order_key": "position", + "first_position": 1, + "duplicate_same_payload": "return_existing_position_without_redelivery", + "duplicate_different_payload": "reject_message_identity_conflict" + }, + "cursor": { + "ownership": "server", + "worker_completion_field": "message_stream_cursors", + "monotonic": true, + "advancement": "highest_contiguous_consumed_position", + "history_event": "MessageCursorAdvanced", + "continue_as_new": "a_payload_free_cursor_checkpoint_and_unconsumed_messages_are_delivered_to_the_current_run" + }, + "waiting": { + "worker_completion_field": "message_stream_waits", + "durable_wakeup": "runtime_reserved_signal_task_resume", + "pending_wait_is_diagnostic": true + }, + "batch": { + "minimum": 1, + "maximum": 100, + "waits_for": "first_message", + "returns": "up_to_maximum_currently_available_messages" + }, + "replay": { + "ordering_source": "server_assigned_position", + "deduplication_source": "message_id_and_instance_cursor", + "worker_replacement": "history_replay_reconstructs_pending_messages_and_cursor_acknowledgement_is_idempotent", + "server_restart": "stream_items_cursor_and_wait_state_are_database_durable", + "external_payloads": "references_remain_reference_backed_at_rest_and_are_resolved_for_worker_delivery_then_removed_with_namespace_retention" + }, + "sdk_handoffs": { + "php": "WorkflowContext::messageStream(string)->receive(int)", + "python": "WorkflowContext.message_stream(str).receive(max_items)", + "rust": "WorkflowContext::message_stream(name).receive(max_items).await" + }, + "conformance": { + "runtimes": [ + "php", + "python", + "rust" + ], + "targets": [ + "server", + "cloud" + ], + "required_scenarios": [ + "repeated_messages_are_observed_in_position_order_exactly_once", + "duplicate_delivery_preserves_one_logical_message", + "worker_is_replaced_mid_stream", + "continue_as_new_retains_cursor_and_pending_messages", + "server_restart_retains_cursor_wait_and_order" + ], + "artifacts": "published" + } + }, + "messages": { + "m1": [ + { + "n": 1 + } + ], + "m2": [ + { + "n": 2 + } + ], + "m3": [ + { + "n": 3 + } + ], + "conflict": [ + { + "n": 9 + } + ] + }, + "long_message_id_length": 192, + "refusals": { + "stream_name": "The stream name must contain 1-128 letters, numbers, periods, underscores, colons, or hyphens.", + "metadata_stream_name": "The stream_name must contain 1-128 letters, numbers, periods, underscores, colons, or hyphens.", + "message_id_required": "The message id field is required.", + "message_id_format": "The message id field format is invalid.", + "message_id_max": "The message id field must not be greater than 191 characters.", + "input_array": "The input field must be an array.", + "wait_after_cursor": "The pending wait must start after the durable stream cursor.", + "cursor_beyond": "The cursor cannot advance beyond the last accepted stream position.", + "duplicate_wait": "A stream may have only one pending wait per workflow task completion.", + "negative_cursor": "The message_stream_cursors.0.through_position field must be at least 0.", + "old_protocol_remediation": "Send the X-Durable-Workflow-Protocol-Version header with worker protocol 1.15 or newer." + }, + "first_run_events": [ + "StartAccepted", + "WorkflowStarted", + "ConditionWaitOpened", + "SignalReceived", + "SignalReceived", + "SignalReceived", + "MessageCursorAdvanced", + "ConditionWaitSatisfied", + "ConditionWaitOpened", + "MessageCursorAdvanced", + "SignalReceived", + "MessageCursorAdvanced", + "ConditionWaitSatisfied", + "WorkflowContinuedAsNew", + "SignalRejected", + "SignalRejected", + "MessageCursorAdvanced" + ], + "continued_run_events": [ + "WorkflowStarted", + "SignalReceived", + "SignalReceived", + "SignalReceived", + "WorkflowCompleted", + "SignalRejected", + "SignalRejected", + "SignalRejected", + "MessageCursorAdvanced" + ] + }, + "contract": "Message streams (published Server 2.11.1, MessageStreamController and MessageStreamService over the Workflow package's runtime-reserved signal). Cluster info publishes the message streams contract and the worker capability the same contract with supported true. A client appends ordered messages to a named stream of a workflow instance: the first message is position 1, a repeated message id with the same payload is a duplicate at its original position, and with another payload a 409 message_identity_conflict. Each accepted message reaches the current run as a __durable_workflow_message_stream SignalReceived whose single argument holds the message schema, stream name, message id, position and the inline payload envelope. Refused requests are counted as malformed without keeping their payload, by a diagnostic message id. Diagnostics list every stream of the instance by name with positions, pending counts, the pending wait, duplicate and malformed counts and the last input, without payloads. A worker on protocol 1.15 or newer reports open receives with message_stream_waits and consumed messages with message_stream_cursors on a workflow task completion; the server checks both before the completion changes anything, records MessageCursorAdvanced for each moved cursor and clears the wait. An open condition wait is woken by a stream message, which it consumes without SignalApplied. When the run continues as new, the next run receives a payload-free cursor checkpoint for each acknowledged stream and every unconsumed message again, in append order; signals the old run had not consumed are rejected. Appends to a closed run are refused with run_not_active, duplicates included. Embedded applications have no HTTP message streams." +} diff --git a/tests/Fixtures/ServerParityExpected/message-streams-observation.json b/tests/Fixtures/ServerParityExpected/message-streams-observation.json new file mode 100644 index 000000000..215042c4c --- /dev/null +++ b/tests/Fixtures/ServerParityExpected/message-streams-observation.json @@ -0,0 +1,1323 @@ +{ + "recorded_from": "published Server 2.11.1 (ghcr.io/durable-workflow/server@sha256:bf8bed13f5f809ee3425233180a581353eac4683adbee5c4ecbf63bb18bbd102) on SQLite, workflow id probe-5", + "message_streams": { + "applicable": true, + "workflow_id": "probe-5-ms", + "cluster_contract": { + "schema": "durable-workflow.v2.message-streams.contract", + "version": 1, + "capability_flag": "message_streams", + "minimum_worker_protocol_version": "1.15", + "scope": "workflow_instance", + "transport": { + "append_path": "/api/workflows/{workflow_id}/message-streams/{stream_name}/messages", + "diagnostics_path": "/api/workflows/{workflow_id}/message-streams", + "internal_signal": "__durable_workflow_message_stream", + "message_schema": "durable-workflow.v2.message-stream.message", + "cursor_schema": "durable-workflow.v2.message-stream.cursor", + "payload_envelope": "inline_avro_at_the_server_to_worker_delivery_boundary", + "durable_payload": "exact_inline_avro_bytes_or_the_verified_external_storage_reference", + "payload_visibility": "payload_is_present_only_in_durable_signal_history_and_is_omitted_from_stream_diagnostics" + }, + "identity": { + "producer_key": "message_id", + "order_key": "position", + "first_position": 1, + "duplicate_same_payload": "return_existing_position_without_redelivery", + "duplicate_different_payload": "reject_message_identity_conflict" + }, + "cursor": { + "ownership": "server", + "worker_completion_field": "message_stream_cursors", + "monotonic": true, + "advancement": "highest_contiguous_consumed_position", + "history_event": "MessageCursorAdvanced", + "continue_as_new": "a_payload_free_cursor_checkpoint_and_unconsumed_messages_are_delivered_to_the_current_run" + }, + "waiting": { + "worker_completion_field": "message_stream_waits", + "durable_wakeup": "runtime_reserved_signal_task_resume", + "pending_wait_is_diagnostic": true + }, + "batch": { + "minimum": 1, + "maximum": 100, + "waits_for": "first_message", + "returns": "up_to_maximum_currently_available_messages" + }, + "replay": { + "ordering_source": "server_assigned_position", + "deduplication_source": "message_id_and_instance_cursor", + "worker_replacement": "history_replay_reconstructs_pending_messages_and_cursor_acknowledgement_is_idempotent", + "server_restart": "stream_items_cursor_and_wait_state_are_database_durable", + "external_payloads": "references_remain_reference_backed_at_rest_and_are_resolved_for_worker_delivery_then_removed_with_namespace_retention" + }, + "sdk_handoffs": { + "php": "WorkflowContext::messageStream(string)->receive(int)", + "python": "WorkflowContext.message_stream(str).receive(max_items)", + "rust": "WorkflowContext::message_stream(name).receive(max_items).await" + }, + "conformance": { + "runtimes": [ + "php", + "python", + "rust" + ], + "targets": [ + "server", + "cloud" + ], + "required_scenarios": [ + "repeated_messages_are_observed_in_position_order_exactly_once", + "duplicate_delivery_preserves_one_logical_message", + "worker_is_replaced_mid_stream", + "continue_as_new_retains_cursor_and_pending_messages", + "server_restart_retains_cursor_wait_and_order" + ], + "artifacts": "published" + } + }, + "worker_capability": { + "schema": "durable-workflow.v2.message-streams.contract", + "version": 1, + "capability_flag": "message_streams", + "minimum_worker_protocol_version": "1.15", + "scope": "workflow_instance", + "transport": { + "append_path": "/api/workflows/{workflow_id}/message-streams/{stream_name}/messages", + "diagnostics_path": "/api/workflows/{workflow_id}/message-streams", + "internal_signal": "__durable_workflow_message_stream", + "message_schema": "durable-workflow.v2.message-stream.message", + "cursor_schema": "durable-workflow.v2.message-stream.cursor", + "payload_envelope": "inline_avro_at_the_server_to_worker_delivery_boundary", + "durable_payload": "exact_inline_avro_bytes_or_the_verified_external_storage_reference", + "payload_visibility": "payload_is_present_only_in_durable_signal_history_and_is_omitted_from_stream_diagnostics" + }, + "identity": { + "producer_key": "message_id", + "order_key": "position", + "first_position": 1, + "duplicate_same_payload": "return_existing_position_without_redelivery", + "duplicate_different_payload": "reject_message_identity_conflict" + }, + "cursor": { + "ownership": "server", + "worker_completion_field": "message_stream_cursors", + "monotonic": true, + "advancement": "highest_contiguous_consumed_position", + "history_event": "MessageCursorAdvanced", + "continue_as_new": "a_payload_free_cursor_checkpoint_and_unconsumed_messages_are_delivered_to_the_current_run" + }, + "waiting": { + "worker_completion_field": "message_stream_waits", + "durable_wakeup": "runtime_reserved_signal_task_resume", + "pending_wait_is_diagnostic": true + }, + "batch": { + "minimum": 1, + "maximum": 100, + "waits_for": "first_message", + "returns": "up_to_maximum_currently_available_messages" + }, + "replay": { + "ordering_source": "server_assigned_position", + "deduplication_source": "message_id_and_instance_cursor", + "worker_replacement": "history_replay_reconstructs_pending_messages_and_cursor_acknowledgement_is_idempotent", + "server_restart": "stream_items_cursor_and_wait_state_are_database_durable", + "external_payloads": "references_remain_reference_backed_at_rest_and_are_resolved_for_worker_delivery_then_removed_with_namespace_retention" + }, + "sdk_handoffs": { + "php": "WorkflowContext::messageStream(string)->receive(int)", + "python": "WorkflowContext.message_stream(str).receive(max_items)", + "rust": "WorkflowContext::message_stream(name).receive(max_items).await" + }, + "conformance": { + "runtimes": [ + "php", + "python", + "rust" + ], + "targets": [ + "server", + "cloud" + ], + "required_scenarios": [ + "repeated_messages_are_observed_in_position_order_exactly_once", + "duplicate_delivery_preserves_one_logical_message", + "worker_is_replaced_mid_stream", + "continue_as_new_retains_cursor_and_pending_messages", + "server_restart_retains_cursor_wait_and_order" + ], + "artifacts": "published" + }, + "supported": true + }, + "run_id": "01m5n2d2gpbkqp605f4w699nfy", + "list_before": { + "status": 200, + "response": { + "workflow_id": "probe-5-ms", + "count": 0, + "streams": [] + } + }, + "show_before": { + "status": 404, + "response": { + "workflow_id": "probe-5-ms", + "stream_name": "orders", + "reason": "message_stream_not_found" + } + }, + "show_bad_name": { + "status": 422, + "response": { + "message": "The stream name must contain 1-128 letters, numbers, periods, underscores, colons, or hyphens.", + "reason": "validation_failed", + "errors": { + "stream_name": [ + "The stream name must contain 1-128 letters, numbers, periods, underscores, colons, or hyphens." + ] + }, + "validation_errors": { + "stream_name": [ + "The stream name must contain 1-128 letters, numbers, periods, underscores, colons, or hyphens." + ] + } + } + }, + "list_unknown": { + "status": 404, + "response": { + "message": "Workflow not found.", + "reason": "instance_not_found" + } + }, + "poll1": { + "run_id": "01m5n2d2gpbkqp605f4w699nfy", + "history": [] + }, + "complete1_refusals": { + "wait_ahead": { + "status": 422, + "response": { + "message": "The pending wait must start after the durable stream cursor.", + "reason": "validation_failed", + "errors": { + "message_stream_waits.0.after_position": [ + "The pending wait must start after the durable stream cursor." + ] + }, + "validation_errors": { + "message_stream_waits.0.after_position": [ + "The pending wait must start after the durable stream cursor." + ] + } + } + }, + "cursor_ahead": { + "status": 422, + "response": { + "message": "The cursor cannot advance beyond the last accepted stream position.", + "reason": "validation_failed", + "errors": { + "message_stream_cursors.0.through_position": [ + "The cursor cannot advance beyond the last accepted stream position." + ] + }, + "validation_errors": { + "message_stream_cursors.0.through_position": [ + "The cursor cannot advance beyond the last accepted stream position." + ] + } + } + }, + "bad_name": { + "status": 422, + "response": { + "message": "The stream_name must contain 1-128 letters, numbers, periods, underscores, colons, or hyphens.", + "reason": "validation_failed", + "errors": { + "message_stream_waits.0.stream_name": [ + "The stream_name must contain 1-128 letters, numbers, periods, underscores, colons, or hyphens." + ] + }, + "validation_errors": { + "message_stream_waits.0.stream_name": [ + "The stream_name must contain 1-128 letters, numbers, periods, underscores, colons, or hyphens." + ] + } + } + }, + "repeated_wait": { + "status": 422, + "response": { + "message": "A stream may have only one pending wait per workflow task completion.", + "reason": "validation_failed", + "errors": { + "message_stream_waits.1.stream_name": [ + "A stream may have only one pending wait per workflow task completion." + ] + }, + "validation_errors": { + "message_stream_waits.1.stream_name": [ + "A stream may have only one pending wait per workflow task completion." + ] + } + } + }, + "old_protocol": { + "status": 409, + "response": { + "task_id": "01m5n2d2h1y9p7kk8qea67rxxj", + "workflow_task_attempt": 1, + "outcome": "rejected", + "reason": "message_streams_unavailable", + "requested_version": "1.14", + "minimum_protocol_version": "1.15", + "remediation": "Send the X-Durable-Workflow-Protocol-Version header with worker protocol 1.15 or newer." + } + } + }, + "complete1": { + "status": 200, + "response": { + "task_id": "01m5n2d2h1y9p7kk8qea67rxxj", + "workflow_task_attempt": 1, + "outcome": "completed", + "recorded": true, + "run_id": "01m5n2d2gpbkqp605f4w699nfy", + "run_status": "waiting", + "created_task_ids": [], + "reason": null + } + }, + "list_waiting": { + "status": 200, + "response": { + "workflow_id": "probe-5-ms", + "count": 1, + "streams": [ + { + "stream_name": "orders", + "last_position": 0, + "cursor_position": 0, + "pending_count": 0, + "pending_wait": { + "run_id": "01m5n2d2gpbkqp605f4w699nfy", + "after_position": 0, + "since": "2026-10-23T19:04:33.000000Z" + }, + "duplicate_count": 0, + "malformed_count": 0, + "last_input_outcome": null, + "last_input_message_id": null, + "last_input_at": null + } + ] + } + }, + "appends": { + "m1": { + "status": 202, + "response": { + "accepted": true, + "duplicate": false, + "workflow_id": "probe-5-ms", + "run_id": "01m5n2d2gpbkqp605f4w699nfy", + "stream_name": "orders", + "message_id": "m1", + "position": 1, + "cursor_position": 0, + "outcome": "accepted", + "reason": null + } + }, + "m1_duplicate": { + "status": 200, + "response": { + "accepted": true, + "duplicate": true, + "workflow_id": "probe-5-ms", + "stream_name": "orders", + "message_id": "m1", + "position": 1, + "cursor_position": 0, + "outcome": "duplicate", + "reason": null + } + }, + "m1_conflict": { + "status": 409, + "response": { + "accepted": false, + "duplicate": false, + "workflow_id": "probe-5-ms", + "stream_name": "orders", + "message_id": "m1", + "position": 1, + "cursor_position": 0, + "outcome": "rejected", + "reason": "message_identity_conflict" + } + }, + "m2_sdk": { + "status": null, + "response": { + "accepted": true, + "duplicate": false, + "workflow_id": "probe-5-ms", + "run_id": "01m5n2d2gpbkqp605f4w699nfy", + "stream_name": "orders", + "message_id": "m2", + "position": 2, + "cursor_position": 0, + "outcome": "accepted", + "reason": null + } + }, + "no_id": { + "status": 422, + "response": { + "message": "The message id field is required.", + "reason": "validation_failed", + "errors": { + "message_id": [ + "The message id field is required." + ] + }, + "validation_errors": { + "message_id": [ + "The message id field is required." + ] + } + } + }, + "bad_id": { + "status": 422, + "response": { + "message": "The message id field format is invalid.", + "reason": "validation_failed", + "errors": { + "message_id": [ + "The message id field format is invalid." + ] + }, + "validation_errors": { + "message_id": [ + "The message id field format is invalid." + ] + } + } + }, + "long_id": { + "status": 422, + "response": { + "message": "The message id field must not be greater than 191 characters.", + "reason": "validation_failed", + "errors": { + "message_id": [ + "The message id field must not be greater than 191 characters." + ] + }, + "validation_errors": { + "message_id": [ + "The message id field must not be greater than 191 characters." + ] + } + } + }, + "bad_input": { + "status": 422, + "response": { + "message": "The input field must be an array.", + "reason": "validation_failed", + "errors": { + "input": [ + "The input field must be an array." + ] + }, + "validation_errors": { + "input": [ + "The input field must be an array." + ] + } + } + }, + "bad_stream": { + "status": 422, + "response": { + "message": "The stream name must contain 1-128 letters, numbers, periods, underscores, colons, or hyphens.", + "reason": "validation_failed", + "errors": { + "stream_name": [ + "The stream name must contain 1-128 letters, numbers, periods, underscores, colons, or hyphens." + ] + }, + "validation_errors": { + "stream_name": [ + "The stream name must contain 1-128 letters, numbers, periods, underscores, colons, or hyphens." + ] + } + } + }, + "unknown_workflow": { + "status": 404, + "response": { + "accepted": false, + "workflow_id": "probe-5-ms-missing", + "stream_name": "orders", + "outcome": "rejected", + "reason": "instance_not_found" + } + }, + "other_stream": { + "status": 202, + "response": { + "accepted": true, + "duplicate": false, + "workflow_id": "probe-5-ms", + "run_id": "01m5n2d2gpbkqp605f4w699nfy", + "stream_name": "audit", + "message_id": "a1", + "position": 1, + "cursor_position": 0, + "outcome": "accepted", + "reason": null + } + }, + "reserved_signal": { + "status": 422, + "response": { + "workflow_id": "probe-5-ms", + "signal_name": "__durable_workflow_message_stream", + "outcome": "rejected", + "reason": "runtime_reserved_signal", + "message": "This signal name is reserved for the durable runtime transport.", + "accepted": false, + "control_plane": { + "schema": "durable-workflow.v2.control-plane-response", + "version": 1, + "operation": "signal", + "workflow_id": "probe-5-ms", + "contract": { + "schema": "durable-workflow.v2.control-plane-response.contract", + "version": 1, + "legacy_field_policy": "reject_non_canonical", + "legacy_fields": { + "query": "query_name", + "signal": "signal_name", + "update": "update_name", + "wait_policy": "wait_for" + }, + "required_fields": [ + "workflow_id", + "operation_name", + "operation_name_field" + ], + "success_fields": [ + "outcome" + ], + "rejection_fields": [ + "workflow_id", + "run_id", + "signal_name", + "command_status", + "command_source", + "target_scope", + "outcome", + "reason", + "rejection_reason", + "rejection_category", + "message", + "command_contract_source", + "command_contract_backfill_needed", + "command_contract_backfill_available", + "declared_signals", + "signal_admission", + "retryable", + "error_id" + ], + "rejection_reasons": [ + "instance_not_found", + "historical_run_command_rejected", + "unknown_signal", + "invalid_signal_arguments", + "run_not_active", + "instance_not_started", + "selected_run_not_current", + "configured_workflow_type_invalid", + "v1_projection_read_only", + "backend_lock_pressure", + "control_plane_internal_error" + ] + }, + "operation_name": "__durable_workflow_message_stream", + "operation_name_field": "signal_name", + "outcome": "rejected", + "reason": "runtime_reserved_signal", + "message": "This signal name is reserved for the durable runtime transport." + } + } + }, + "m3": { + "status": 202, + "response": { + "accepted": true, + "duplicate": false, + "workflow_id": "probe-5-ms", + "run_id": "01m5n2d2gpbkqp605f4w699nfy", + "stream_name": "orders", + "message_id": "m3", + "position": 3, + "cursor_position": 1, + "outcome": "accepted", + "reason": null + } + }, + "closed": { + "status": 409, + "response": { + "accepted": false, + "duplicate": false, + "workflow_id": "probe-5-ms", + "stream_name": "orders", + "message_id": "m4", + "position": null, + "outcome": "rejected", + "reason": "run_not_active" + } + }, + "closed_duplicate": { + "status": 409, + "response": { + "accepted": false, + "duplicate": false, + "workflow_id": "probe-5-ms", + "stream_name": "orders", + "message_id": "m1", + "position": null, + "outcome": "rejected", + "reason": "run_not_active" + } + } + }, + "list_after_appends": { + "status": 200, + "response": { + "workflow_id": "probe-5-ms", + "count": 2, + "streams": [ + { + "stream_name": "audit", + "last_position": 1, + "cursor_position": 0, + "pending_count": 1, + "pending_wait": null, + "duplicate_count": 0, + "malformed_count": 0, + "last_input_outcome": "accepted", + "last_input_message_id": "a1", + "last_input_at": "2026-10-23T19:04:34.000000Z" + }, + { + "stream_name": "orders", + "last_position": 2, + "cursor_position": 0, + "pending_count": 2, + "pending_wait": null, + "duplicate_count": 2, + "malformed_count": 4, + "last_input_outcome": "malformed", + "last_input_message_id": "m3", + "last_input_at": "2026-10-23T19:04:34.000000Z" + } + ] + } + }, + "show_after_appends": { + "status": 200, + "response": { + "workflow_id": "probe-5-ms", + "stream": { + "stream_name": "orders", + "last_position": 2, + "cursor_position": 0, + "pending_count": 2, + "pending_wait": null, + "duplicate_count": 2, + "malformed_count": 4, + "last_input_outcome": "malformed", + "last_input_message_id": "m3", + "last_input_at": "2026-10-23T19:04:34.000000Z" + } + } + }, + "poll2": { + "run_id": "01m5n2d2gpbkqp605f4w699nfy", + "history": [ + { + "event_type": "SignalReceived", + "deliveries": [ + { + "schema": "durable-workflow.v2.message-stream.message", + "stream_name": "orders", + "message_id": "m1", + "position": 1, + "payload_envelope": [ + "blob", + "codec" + ], + "payload": [ + { + "n": 1 + } + ] + } + ] + }, + { + "event_type": "SignalReceived", + "deliveries": [ + { + "schema": "durable-workflow.v2.message-stream.message", + "stream_name": "orders", + "message_id": "m2", + "position": 2, + "payload_envelope": [ + "blob", + "codec" + ], + "payload": [ + { + "n": 2 + } + ] + } + ] + }, + { + "event_type": "SignalReceived", + "deliveries": [ + { + "schema": "durable-workflow.v2.message-stream.message", + "stream_name": "audit", + "message_id": "a1", + "position": 1, + "payload_envelope": [ + "blob", + "codec" + ], + "payload": [] + } + ] + } + ] + }, + "complete2_refusals": { + "beyond": { + "status": 422, + "response": { + "message": "The cursor cannot advance beyond the last accepted stream position.", + "reason": "validation_failed", + "errors": { + "message_stream_cursors.0.through_position": [ + "The cursor cannot advance beyond the last accepted stream position." + ] + }, + "validation_errors": { + "message_stream_cursors.0.through_position": [ + "The cursor cannot advance beyond the last accepted stream position." + ] + } + } + }, + "negative": { + "status": 422, + "response": { + "message": "The message_stream_cursors.0.through_position field must be at least 0.", + "reason": "validation_failed", + "errors": { + "message_stream_cursors.0.through_position": [ + "The message_stream_cursors.0.through_position field must be at least 0." + ] + }, + "validation_errors": { + "message_stream_cursors.0.through_position": [ + "The message_stream_cursors.0.through_position field must be at least 0." + ] + } + } + }, + "wait_behind": { + "status": 422, + "response": { + "message": "The pending wait must start after the durable stream cursor.", + "reason": "validation_failed", + "errors": { + "message_stream_waits.0.after_position": [ + "The pending wait must start after the durable stream cursor." + ] + }, + "validation_errors": { + "message_stream_waits.0.after_position": [ + "The pending wait must start after the durable stream cursor." + ] + } + } + } + }, + "complete2": { + "status": 200, + "response": { + "task_id": "01m5n2d3eb25p3d5t00y35nhhk", + "workflow_task_attempt": 1, + "outcome": "completed", + "recorded": true, + "run_id": "01m5n2d2gpbkqp605f4w699nfy", + "run_status": "waiting", + "created_task_ids": [ + "01m5n2d516v62fqj8j4dzkb82t" + ], + "reason": null + } + }, + "history_after2": [ + { + "event_type": "SignalReceived", + "deliveries": [ + { + "schema": "durable-workflow.v2.message-stream.message", + "stream_name": "orders", + "message_id": "m1", + "position": 1, + "payload_envelope": [ + "blob", + "codec" + ], + "payload": [ + { + "n": 1 + } + ] + } + ] + }, + { + "event_type": "SignalReceived", + "deliveries": [ + { + "schema": "durable-workflow.v2.message-stream.message", + "stream_name": "orders", + "message_id": "m2", + "position": 2, + "payload_envelope": [ + "blob", + "codec" + ], + "payload": [ + { + "n": 2 + } + ] + } + ] + }, + { + "event_type": "SignalReceived", + "deliveries": [ + { + "schema": "durable-workflow.v2.message-stream.message", + "stream_name": "audit", + "message_id": "a1", + "position": 1, + "payload_envelope": [ + "blob", + "codec" + ], + "payload": [] + } + ] + }, + { + "event_type": "MessageCursorAdvanced", + "stream_key": "instance", + "previous_position": 0, + "new_position": 1, + "task_status": "leased" + }, + { + "event_type": "MessageCursorAdvanced", + "stream_key": "orders", + "previous_position": 0, + "new_position": 1, + "task_status": "completed" + } + ], + "list_after2": { + "status": 200, + "response": { + "workflow_id": "probe-5-ms", + "count": 2, + "streams": [ + { + "stream_name": "audit", + "last_position": 1, + "cursor_position": 0, + "pending_count": 1, + "pending_wait": null, + "duplicate_count": 0, + "malformed_count": 0, + "last_input_outcome": "accepted", + "last_input_message_id": "a1", + "last_input_at": "2026-10-23T19:04:34.000000Z" + }, + { + "stream_name": "orders", + "last_position": 2, + "cursor_position": 1, + "pending_count": 1, + "pending_wait": null, + "duplicate_count": 2, + "malformed_count": 4, + "last_input_outcome": "malformed", + "last_input_message_id": "m3", + "last_input_at": "2026-10-23T19:04:34.000000Z" + } + ] + } + }, + "poll3": { + "run_id": "01m5n2d2gpbkqp605f4w699nfy", + "history": [ + { + "event_type": "SignalReceived", + "deliveries": [ + { + "schema": "durable-workflow.v2.message-stream.message", + "stream_name": "orders", + "message_id": "m1", + "position": 1, + "payload_envelope": [ + "blob", + "codec" + ], + "payload": [ + { + "n": 1 + } + ] + } + ] + }, + { + "event_type": "SignalReceived", + "deliveries": [ + { + "schema": "durable-workflow.v2.message-stream.message", + "stream_name": "orders", + "message_id": "m2", + "position": 2, + "payload_envelope": [ + "blob", + "codec" + ], + "payload": [ + { + "n": 2 + } + ] + } + ] + }, + { + "event_type": "SignalReceived", + "deliveries": [ + { + "schema": "durable-workflow.v2.message-stream.message", + "stream_name": "audit", + "message_id": "a1", + "position": 1, + "payload_envelope": [ + "blob", + "codec" + ], + "payload": [] + } + ] + }, + { + "event_type": "MessageCursorAdvanced", + "stream_key": "instance", + "previous_position": 0, + "new_position": 1, + "task_status": "leased" + }, + { + "event_type": "MessageCursorAdvanced", + "stream_key": "orders", + "previous_position": 0, + "new_position": 1, + "task_status": "completed" + }, + { + "event_type": "SignalReceived", + "deliveries": [ + { + "schema": "durable-workflow.v2.message-stream.message", + "stream_name": "orders", + "message_id": "m3", + "position": 3, + "payload_envelope": [ + "blob", + "codec" + ], + "payload": [ + { + "n": 3 + } + ] + } + ] + } + ] + }, + "complete3": { + "status": 200, + "response": { + "task_id": "01m5n2d516v62fqj8j4dzkb82t", + "workflow_task_attempt": 1, + "outcome": "completed", + "recorded": true, + "run_id": "01m5n2d2gpbkqp605f4w699nfy", + "run_status": "completed", + "created_task_ids": [ + "01m5n2d5gnm3yatnb4jaqtvfnk" + ], + "reason": null + } + }, + "list_after_continue": { + "status": 200, + "response": { + "workflow_id": "probe-5-ms", + "count": 2, + "streams": [ + { + "stream_name": "audit", + "last_position": 1, + "cursor_position": 0, + "pending_count": 1, + "pending_wait": null, + "duplicate_count": 0, + "malformed_count": 0, + "last_input_outcome": "accepted", + "last_input_message_id": "a1", + "last_input_at": "2026-10-23T19:04:34.000000Z" + }, + { + "stream_name": "orders", + "last_position": 3, + "cursor_position": 2, + "pending_count": 1, + "pending_wait": null, + "duplicate_count": 2, + "malformed_count": 4, + "last_input_outcome": "accepted", + "last_input_message_id": "m3", + "last_input_at": "2026-10-23T19:04:35.000000Z" + } + ] + } + }, + "poll4": { + "run_id": "01m5n2d5g530fnzmcjg1mtmffs", + "history": [ + { + "event_type": "SignalReceived", + "deliveries": [ + { + "schema": "durable-workflow.v2.message-stream.cursor", + "stream_name": "orders", + "through_position": 2 + } + ] + }, + { + "event_type": "SignalReceived", + "deliveries": [ + { + "schema": "durable-workflow.v2.message-stream.message", + "stream_name": "audit", + "message_id": "a1", + "position": 1, + "payload_envelope": [ + "blob", + "codec" + ], + "payload": [] + } + ] + }, + { + "event_type": "SignalReceived", + "deliveries": [ + { + "schema": "durable-workflow.v2.message-stream.message", + "stream_name": "orders", + "message_id": "m3", + "position": 3, + "payload_envelope": [ + "blob", + "codec" + ], + "payload": [ + { + "n": 3 + } + ] + } + ] + } + ] + }, + "complete4": { + "status": 200, + "response": { + "task_id": "01m5n2d5gnm3yatnb4jaqtvfnk", + "workflow_task_attempt": 1, + "outcome": "completed", + "recorded": true, + "run_id": "01m5n2d5g530fnzmcjg1mtmffs", + "run_status": "completed", + "created_task_ids": [], + "reason": null + } + }, + "continued_run_id": "01m5n2d5g530fnzmcjg1mtmffs", + "list_final": { + "status": 200, + "response": { + "workflow_id": "probe-5-ms", + "count": 2, + "streams": [ + { + "stream_name": "audit", + "last_position": 1, + "cursor_position": 0, + "pending_count": 1, + "pending_wait": null, + "duplicate_count": 0, + "malformed_count": 0, + "last_input_outcome": "accepted", + "last_input_message_id": "a1", + "last_input_at": "2026-10-23T19:04:34.000000Z" + }, + { + "stream_name": "orders", + "last_position": 3, + "cursor_position": 3, + "pending_count": 0, + "pending_wait": null, + "duplicate_count": 2, + "malformed_count": 4, + "last_input_outcome": "accepted", + "last_input_message_id": "m3", + "last_input_at": "2026-10-23T19:04:35.000000Z" + } + ] + } + }, + "first_run_events": [ + "StartAccepted", + "WorkflowStarted", + "ConditionWaitOpened", + "SignalReceived", + "SignalReceived", + "SignalReceived", + "MessageCursorAdvanced", + "ConditionWaitSatisfied", + "ConditionWaitOpened", + "MessageCursorAdvanced", + "SignalReceived", + "MessageCursorAdvanced", + "ConditionWaitSatisfied", + "WorkflowContinuedAsNew", + "SignalRejected", + "SignalRejected", + "MessageCursorAdvanced" + ], + "first_run_streams": [ + { + "event_type": "SignalReceived", + "deliveries": [ + { + "schema": "durable-workflow.v2.message-stream.message", + "stream_name": "orders", + "message_id": "m1", + "position": 1, + "payload_envelope": [ + "blob", + "codec" + ], + "payload": [ + { + "n": 1 + } + ] + } + ] + }, + { + "event_type": "SignalReceived", + "deliveries": [ + { + "schema": "durable-workflow.v2.message-stream.message", + "stream_name": "orders", + "message_id": "m2", + "position": 2, + "payload_envelope": [ + "blob", + "codec" + ], + "payload": [ + { + "n": 2 + } + ] + } + ] + }, + { + "event_type": "SignalReceived", + "deliveries": [ + { + "schema": "durable-workflow.v2.message-stream.message", + "stream_name": "audit", + "message_id": "a1", + "position": 1, + "payload_envelope": [ + "blob", + "codec" + ], + "payload": [] + } + ] + }, + { + "event_type": "MessageCursorAdvanced", + "stream_key": "instance", + "previous_position": 0, + "new_position": 1, + "task_status": "leased" + }, + { + "event_type": "MessageCursorAdvanced", + "stream_key": "orders", + "previous_position": 0, + "new_position": 1, + "task_status": "completed" + }, + { + "event_type": "SignalReceived", + "deliveries": [ + { + "schema": "durable-workflow.v2.message-stream.message", + "stream_name": "orders", + "message_id": "m3", + "position": 3, + "payload_envelope": [ + "blob", + "codec" + ], + "payload": [ + { + "n": 3 + } + ] + } + ] + }, + { + "event_type": "MessageCursorAdvanced", + "stream_key": "instance", + "previous_position": 1, + "new_position": 2, + "task_status": "leased" + }, + { + "event_type": "MessageCursorAdvanced", + "stream_key": "orders", + "previous_position": 1, + "new_position": 2, + "task_status": "completed" + } + ], + "continued_run_events": [ + "WorkflowStarted", + "SignalReceived", + "SignalReceived", + "SignalReceived", + "WorkflowCompleted", + "SignalRejected", + "SignalRejected", + "SignalRejected", + "MessageCursorAdvanced" + ], + "continued_run_streams": [ + { + "event_type": "SignalReceived", + "deliveries": [ + { + "schema": "durable-workflow.v2.message-stream.cursor", + "stream_name": "orders", + "through_position": 2 + } + ] + }, + { + "event_type": "SignalReceived", + "deliveries": [ + { + "schema": "durable-workflow.v2.message-stream.message", + "stream_name": "audit", + "message_id": "a1", + "position": 1, + "payload_envelope": [ + "blob", + "codec" + ], + "payload": [] + } + ] + }, + { + "event_type": "SignalReceived", + "deliveries": [ + { + "schema": "durable-workflow.v2.message-stream.message", + "stream_name": "orders", + "message_id": "m3", + "position": 3, + "payload_envelope": [ + "blob", + "codec" + ], + "payload": [ + { + "n": 3 + } + ] + } + ] + }, + { + "event_type": "MessageCursorAdvanced", + "stream_key": "orders", + "previous_position": 2, + "new_position": 3, + "task_status": "completed" + } + ] + } +} diff --git a/tests/Unit/ServerParityMessageStreamsContractTest.mjs b/tests/Unit/ServerParityMessageStreamsContractTest.mjs new file mode 100644 index 000000000..f98be161b --- /dev/null +++ b/tests/Unit/ServerParityMessageStreamsContractTest.mjs @@ -0,0 +1,81 @@ +import test from 'node:test'; +import assert from 'node:assert/strict'; +import {readFileSync} from 'node:fs'; +import {checkMessageStreams} from '../../scripts/conformance/server-parity/message-streams-contract.mjs'; + +const fixture = JSON.parse(readFileSync(new URL('../Fixtures/ServerParity/message-streams.json', import.meta.url))); +// What published Server 2.11.1 answered for this fixture's probe. +const recorded = JSON.parse(readFileSync(new URL('../Fixtures/ServerParityExpected/message-streams-observation.json', import.meta.url))); +const model = () => structuredClone({mode: 'http', message_streams: recorded.message_streams}); +const projection = {scope: 'message_streams', first_run_events: 17, continued_run_events: 9}; +const check = observation => checkMessageStreams(fixture, observation, 'probe-5'); +const stream = (state, step, name) => state[step].response.streams.find(entry => entry.stream_name === name); + +test('published PHP message streams pass', () => assert.deepStrictEqual(check(model()), projection)); +test('embedded explicitly declares HTTP message streams inapplicable', () => assert.deepStrictEqual(check( + {mode: 'embedded', message_streams: {applicable: false, reason: 'embedded_has_no_http_message_streams'}}), projection)); +test('the recorded observation keeps the published reference', () => { + assert.match(recorded.recorded_from, /^published Server 2\.11\.1 /); + assert.equal(recorded.message_streams.workflow_id, 'probe-5-ms'); +}); + +for (const [label, mutate] of [ + ['a missing observation', s => { for (const key of Object.keys(s)) delete s[key]; }], + ['no contract in cluster info', s => s.cluster_contract = null], + ['another contract version', s => s.cluster_contract.version = 2], + ['the capability as a plain flag', s => s.worker_capability = false], + ['an unsupported capability', s => s.worker_capability.supported = false], + ['a stream before any append', s => s.list_before.response.count = 1], + ['an unknown stream answered', s => s.show_before = {status: 200, response: {workflow_id: s.workflow_id, stream: null}}], + ['a bad stream name served', s => s.show_bad_name = {status: 404, response: {reason: 'message_stream_not_found'}}], + ['an unknown workflow listed', s => s.list_unknown = {status: 200, response: {workflow_id: 'x', count: 0, streams: []}}], + ['a wait ahead of the cursor accepted', s => s.complete1_refusals.wait_ahead = s.complete1], + ['a cursor past the stream accepted', s => s.complete1_refusals.cursor_ahead.status = 200], + ['two waits on one stream accepted', s => s.complete1_refusals.repeated_wait.response.errors = {}], + ['stream metadata before protocol 1.15 accepted', s => s.complete1_refusals.old_protocol.status = 200], + ['the wait not recorded', s => stream(s, 'list_waiting', 'orders').pending_wait = null], + ['a wait time without microseconds', s => stream(s, 'list_waiting', 'orders').pending_wait.since = '2026-10-23T18:00:00Z'], + ['the first message at position 0', s => s.appends.m1.response.position = 0], + ['an accepted append without its run', s => delete s.appends.m1.response.run_id], + ['a duplicate appended again', s => s.appends.m1_duplicate = {status: 202, response: {...s.appends.m2_sdk.response, message_id: 'm1', position: 3}}], + ['a conflicting payload accepted', s => Object.assign(s.appends.m1_conflict, {status: 200})], + ['a missing message id accepted', s => s.appends.no_id.status = 202], + ['a long message id accepted', s => s.appends.long_id.status = 202], + ['input that is not a list accepted', s => s.appends.bad_input.status = 202], + ['an append to an unknown workflow with the service shape', s => s.appends.unknown_workflow.response.duplicate = false], + ['the runtime signal accepted from a client', s => s.appends.reserved_signal = {status: 202, response: {outcome: 'signal_received'}}], + ['a refused stream name counted', s => stream(s, 'list_after_appends', 'orders').malformed_count = 5], + ['a malformed append not counted', s => stream(s, 'list_after_appends', 'orders').malformed_count = 3], + ['a raw refused message id kept', s => stream(s, 'list_after_appends', 'orders').last_input_message_id = 'bad id!'], + ['a wait kept after an append', s => { stream(s, 'list_after_appends', 'orders').pending_wait = stream(s, 'list_waiting', 'orders').pending_wait; }], + ['streams listed out of name order', s => s.list_after_appends.response.streams.reverse()], + ['a show unlike the list', s => s.show_after_appends.response.stream.duplicate_count = 0], + ['messages delivered out of order', s => s.poll2.history.reverse()], + ['two messages in one signal', s => s.poll2.history[0].deliveries.push(s.poll2.history[1].deliveries[0])], + ['a payload delivered by reference', s => s.poll2.history[0].deliveries[0].payload_envelope = ['codec', 'external_storage']], + ['a negative cursor accepted', s => s.complete2_refusals.negative.status = 200], + ['a wait behind the new cursor accepted', s => s.complete2_refusals.wait_behind.status = 200], + ['the next message task not reported', s => s.complete2.response.created_task_ids = []], + ['no stream cursor event', s => s.history_after2 = s.history_after2.filter(event => event.stream_key !== 'orders')], + ['the stream cursor before the task completed', s => { + s.history_after2.find(event => event.stream_key === 'orders').task_status = 'leased'; + }], + ['the cursor not moved', s => stream(s, 'list_after2', 'orders').cursor_position = 0], + ['the third message without the cursor', s => s.appends.m3.response.cursor_position = 0], + ['the cursor not carried to the continued run', s => s.poll4.history.shift()], + ['a consumed message delivered again', s => s.poll4.history.splice(1, 0, structuredClone(s.poll2.history[0]))], + ['unconsumed messages out of append order', s => s.poll4.history.push(s.poll4.history.splice(1, 1)[0])], + ['the continuation as the same run', s => s.continued_run_id = s.run_id], + ['an append to a closed instance accepted', s => s.appends.closed.status = 202], + ['a closed duplicate answered as a duplicate', s => Object.assign(s.appends.closed_duplicate.response, {reason: null, outcome: 'duplicate'})], + ['SignalApplied for a consumed stream message', s => s.first_run_events.splice(7, 0, 'SignalApplied')], + ['unconsumed signals not rejected', s => s.first_run_events = s.first_run_events.filter(type => type !== 'SignalRejected')], + ['a continued run without its final cursor', s => s.continued_run_events.pop()], + ['diagnostics with payloads', s => { stream(s, 'list_final', 'orders').payload = [{n: 3}]; }], +]) { + test(`refuses ${label}`, () => { + const observation = model(); + mutate(observation.message_streams); + assert.throws(() => check(observation)); + }); +} From 1414996fab53155cf324fa6300f1875c69e53c20 Mon Sep 17 00:00:00 2001 From: Durable Workflow Date: Fri, 23 Oct 2026 20:40:20 +0000 Subject: [PATCH 02/10] Accept condition waits without a key or fingerprint like PHP Published PHP treats an open_condition_wait's condition_key and condition_definition_fingerprint as optional strings of any form, trimmed and left out when empty. Native required both and a sha256 fingerprint. The wait projection and run summary now read a keyless wait as PHP does. --- .../2026-10-23-condition-wait-fields.md | 21 +++++++ rust/src/runtime/external_wait_summary.rs | 37 ++++++++--- rust/src/runtime/store.rs | 24 ++++--- rust/src/runtime/wait_projection.rs | 3 +- rust/tests/execution.rs | 2 + .../execution/keyless_condition_waits.rs | 63 +++++++++++++++++++ 6 files changed, 130 insertions(+), 20 deletions(-) create mode 100644 docs/rust-server-port-checkpoints/2026-10-23-condition-wait-fields.md create mode 100644 rust/tests/execution/keyless_condition_waits.rs diff --git a/docs/rust-server-port-checkpoints/2026-10-23-condition-wait-fields.md b/docs/rust-server-port-checkpoints/2026-10-23-condition-wait-fields.md new file mode 100644 index 000000000..b96d9b562 --- /dev/null +++ b/docs/rust-server-port-checkpoints/2026-10-23-condition-wait-fields.md @@ -0,0 +1,21 @@ +# October 23, optional condition wait fields + +- **The gap.** Native refused an `open_condition_wait` command without a + `condition_key`, or without a `condition_definition_fingerprint` of the form + `sha256:` followed by 64 hex digits. Published PHP accepts both. +- **How PHP 2.11.1 does it.** `normalizeOpenConditionWaitCommand` treats the key + and the fingerprint as optional strings of any form. It trims them and leaves + them out when empty. `applyOpenConditionWait` records only the fields that + remain, on `ConditionWaitOpened` and on a condition timeout's + `TimerScheduled`. Run summaries name a keyless wait "Waiting for condition", + without a key. Observed live against the published image: a bare + `open_condition_wait`, a key of `" ready "` with the fingerprint + `"any form"`, and blank values all behave this way. +- **What native does now.** The same. The open wait projection stores no + target name for a keyless wait, and the run summary reads such a wait as + PHP does. A key or fingerprint that is not a string is still refused by the + completion transport rules. +- **How it was checked.** A new integration test covers a bare wait, a padded + key with an arbitrary fingerprint, blank values with a timeout, and a key + that is not a string. The 63 existing condition, timed-condition and + observation tests still pass. diff --git a/rust/src/runtime/external_wait_summary.rs b/rust/src/runtime/external_wait_summary.rs index 0f9a959a1..06abf03fa 100644 --- a/rust/src/runtime/external_wait_summary.rs +++ b/rust/src/runtime/external_wait_summary.rs @@ -19,6 +19,8 @@ pub(super) struct ExternalWaitHistory { struct ExpectedWait { kind: &'static str, + /// The signal name, or the condition key; PHP allows a condition + /// without one, which reads as empty. name: String, sequence: i64, timeout: Option, @@ -27,6 +29,26 @@ struct ExpectedWait { identity: Value, } +/// A wait's name: a signal always has one; a condition's key is optional. +fn wait_name<'a>(kind: &str, value: &'a Value) -> Option<&'a str> { + match value.as_str().filter(|name| !name.is_empty()) { + Some(name) => Some(name), + None if kind == "condition" && value.is_null() => Some(""), + None => None, + } +} + +impl ExpectedWait { + /// `conditionLabel`: the kind, then the name when there is one. + fn label(&self) -> String { + if self.name.is_empty() { + self.kind.to_owned() + } else { + format!("{} {}", self.kind, self.name) + } + } +} + impl ExternalWaitHistory { pub(super) fn apply(&mut self, kind: &str, payload: &Value) -> bool { if matches!(kind, "TimerScheduled" | "TimerFired" | "TimerCancelled") { @@ -70,7 +92,7 @@ impl ExternalWaitHistory { } let (Some(id), Some(name), Some(sequence)) = ( payload[id_key].as_str().filter(|v| !v.is_empty()), - payload[name_key].as_str().filter(|v| !v.is_empty()), + wait_name(wait_kind, &payload[name_key]), payload["sequence"].as_i64().filter(|v| *v > 0), ) else { return false; @@ -102,7 +124,7 @@ impl ExternalWaitHistory { let (Some(timer_id), Some(wait_id), Some(name), Some(sequence), Some(delay)) = ( payload["timer_id"].as_str().filter(|v| !v.is_empty()), payload[id_key].as_str().filter(|v| !v.is_empty()), - payload[name_key].as_str().filter(|v| !v.is_empty()), + wait_name(wait_kind, &payload[name_key]), payload["sequence"].as_i64(), payload["delay_seconds"].as_i64(), ) else { @@ -230,14 +252,11 @@ where kind: expected.kind, id: DB::string(row, "wait_id")?, reason: if expected.fired { - format!( - "Waiting to apply {} {} timeout", - expected.kind, expected.name - ) + format!("Waiting to apply {} timeout", expected.label()) } else if expected.timeout.is_some() { - format!("Waiting for {} {} or timeout", expected.kind, expected.name) + format!("Waiting for {} or timeout", expected.label()) } else { - format!("Waiting for {} {}", expected.kind, expected.name) + format!("Waiting for {}", expected.label()) }, started_at: DB::optional_instant(row, "opened_at")?, deadline_at: expected.timer.as_ref().map(|(_, at)| *at), @@ -272,7 +291,7 @@ where } fn external_wait_matches(row: &DB::Row, expected: &ExpectedWait) -> Result { if DB::string(row, "kind")? != expected.kind - || DB::optional_string(row, "target_name")?.as_deref() != Some(expected.name.as_str()) + || DB::optional_string(row, "target_name")?.unwrap_or_default() != expected.name || DB::optional_number(row, "sequence")? != Some(expected.sequence) || DB::optional_string(row, "history_authority")?.as_deref() != Some("typed_history") || DB::optional_string(row, "source_status")?.as_deref() diff --git a/rust/src/runtime/store.rs b/rust/src/runtime/store.rs index 93c299d12..e86601249 100644 --- a/rust/src/runtime/store.rs +++ b/rust/src/runtime/store.rs @@ -2199,17 +2199,9 @@ where { return Err(refuse(StatusCode::UNPROCESSABLE_ENTITY, "invalid_request")); } - text(command, "condition_key")?; + // `normalizeOpenConditionWaitCommand`: the key and the + // definition fingerprint are optional strings, of any form. super::wait_timeouts::timeout_for(command, "invalid_condition_timeout")?; - let fingerprint = text(command, "condition_definition_fingerprint")?; - if !fingerprint.strip_prefix("sha256:").is_some_and(|hash| { - hash.len() == 64 && hash.bytes().all(|byte| byte.is_ascii_hexdigit()) - }) { - return Err(refuse( - StatusCode::UNPROCESSABLE_ENTITY, - "invalid_condition_fingerprint", - )); - } } "open_signal_wait" => { reject_fields(command, &["type", "signal_name", "timeout_seconds"])?; @@ -2555,6 +2547,18 @@ where .unwrap() .remove("condition_wait_occurrence_id"); } + // `normalizeOptionalString`: PHP trims the condition's key and + // definition fingerprint and drops them when empty. + if !direct { + for field in ["condition_key", "condition_definition_fingerprint"] { + match payload[field].as_str().map(str::trim).filter(|value| !value.is_empty()) { + Some(value) => payload[field] = json!(value), + None => { + payload.as_object_mut().unwrap().remove(field); + } + } + } + } let pending_wait_id = if direct { let pending = Self::query("SELECT signal_wait_id FROM workflow_signal_records WHERE workflow_run_id=$1 AND signal_name=$2 AND status='received' ORDER BY command_sequence,id LIMIT 1") .bind(&run_id).bind(text(command, "signal_name")?).fetch_optional(&mut *tx).await?; diff --git a/rust/src/runtime/wait_projection.rs b/rust/src/runtime/wait_projection.rs index 6467aa436..6dfdfc2ba 100644 --- a/rust/src/runtime/wait_projection.rs +++ b/rust/src/runtime/wait_projection.rs @@ -267,7 +267,8 @@ where ) -> Result<()> { if event_type == "ConditionWaitOpened" { let wait_id = super::text(payload, "condition_wait_id")?; - let key = super::text(payload, "condition_key")?; + // PHP records a condition wait without a key when none was given. + let key = payload["condition_key"].as_str(); let sequence = wait_sequence(payload)?; let projection_id = projection_id(run_id, wait_id); let summary = payload["timeout_seconds"].as_i64().map_or_else( diff --git a/rust/tests/execution.rs b/rust/tests/execution.rs index c3ce551e4..55a6b36ff 100644 --- a/rust/tests/execution.rs +++ b/rust/tests/execution.rs @@ -87,6 +87,8 @@ mod rejected_updates; mod continue_messages; #[path = "execution/update_wait.rs"] mod update_wait; +#[path = "execution/keyless_condition_waits.rs"] +mod keyless_condition_waits; #[path = "execution/payload_limits.rs"] mod payload_limits; #[path = "execution/start_definition.rs"] diff --git a/rust/tests/execution/keyless_condition_waits.rs b/rust/tests/execution/keyless_condition_waits.rs new file mode 100644 index 000000000..2d66bfa6a --- /dev/null +++ b/rust/tests/execution/keyless_condition_waits.rs @@ -0,0 +1,63 @@ +use super::*; + +// Workflow 2.9's `normalizeOpenConditionWaitCommand`: a condition wait's key +// and definition fingerprint are optional strings of any form. PHP trims them +// and leaves them out when empty. + +fn opened(history: &Value) -> Value { + history.as_array().unwrap().iter().rev().find(|event| event["event_type"] == "ConditionWaitOpened") + .map(|event| event["payload"].clone()).unwrap() +} + +#[tokio::test] +async fn condition_waits_need_neither_a_key_nor_a_fingerprint() { + let database = TestDatabase::new().await; + let app = router(database.open().await.unwrap()); + let started = start(&app, "keyless-condition").await; + register(&app, "keyless", json!(["echo"]), json!([])).await; + let task = poll(&app, "keyless", "workflow").await; + let (status, body) = finish_task(&app, &task, json!([{"type":"open_condition_wait"}])).await; + assert_eq!(status, StatusCode::OK, "{body}"); + let history = run_history(&app, &started["workflow_id"], &started["run_id"]).await; + let payload = opened(&history); + assert!(payload.get("condition_key").is_none() && payload.get("condition_definition_fingerprint").is_none(), "{payload}"); + assert_eq!(task_retry::rows(&database, + "SELECT COUNT(*) FROM workflow_run_summaries WHERE wait_kind='condition' AND wait_reason='Waiting for condition'").await, 1); + assert_eq!(task_retry::rows(&database, + "SELECT COUNT(*) FROM workflow_run_waits WHERE kind='condition' AND status='open' AND target_name IS NULL").await, 1); + + // A signal wakes the wait; the next one keeps a trimmed key and any + // fingerprint, and drops a blank one. + let (status, body) = request(&app, "POST", "/api/workflows/keyless-condition/signal/go", json!([])).await; + assert_eq!(status, StatusCode::ACCEPTED, "{body}"); + let task = poll(&app, "keyless", "workflow").await; + let (status, body) = finish_task(&app, &task, json!([{"type":"open_condition_wait","condition_key":" ready ", + "condition_definition_fingerprint":"any form"}])).await; + assert_eq!(status, StatusCode::OK, "{body}"); + let payload = opened(&run_history(&app, &started["workflow_id"], &started["run_id"]).await); + assert_eq!((payload["condition_key"].clone(), payload["condition_definition_fingerprint"].clone()), + (json!("ready"), json!("any form")), "{payload}"); + assert_eq!(task_retry::rows(&database, + "SELECT COUNT(*) FROM workflow_run_summaries WHERE wait_reason='Waiting for condition ready'").await, 1); + + let (status, body) = request(&app, "POST", "/api/workflows/keyless-condition/signal/again", json!([])).await; + assert_eq!(status, StatusCode::ACCEPTED, "{body}"); + let task = poll(&app, "keyless", "workflow").await; + let (status, body) = finish_task(&app, &task, json!([{"type":"open_condition_wait","condition_key":" ", + "condition_definition_fingerprint":"","timeout_seconds":60}])).await; + assert_eq!(status, StatusCode::OK, "{body}"); + let history = run_history(&app, &started["workflow_id"], &started["run_id"]).await; + let payload = opened(&history); + assert!(payload.get("condition_key").is_none() && payload.get("condition_definition_fingerprint").is_none(), "{payload}"); + let scheduled = history.as_array().unwrap().iter().rev().find(|event| event["event_type"] == "TimerScheduled").unwrap(); + assert!(scheduled["payload"].get("condition_key").is_none(), "{scheduled}"); + assert_eq!(task_retry::rows(&database, + "SELECT COUNT(*) FROM workflow_run_summaries WHERE wait_reason='Waiting for condition or timeout'").await, 1); + + // A key that is not a string is still refused. + let (status, body) = request(&app, "POST", "/api/workflows/keyless-condition/signal/third", json!([])).await; + assert_eq!(status, StatusCode::ACCEPTED, "{body}"); + let task = poll(&app, "keyless", "workflow").await; + let (status, body) = finish_task(&app, &task, json!([{"type":"open_condition_wait","condition_key":7}])).await; + assert_eq!(status, StatusCode::UNPROCESSABLE_ENTITY, "{body}"); +} From 0ec63aae91f51015ef143b4630f9a08fb37e31ed Mon Sep 17 00:00:00 2001 From: Durable Workflow Date: Fri, 23 Oct 2026 20:49:49 +0000 Subject: [PATCH 03/10] Snapshot the task on every event PHP records with it Published PHP's WorkflowHistoryEvent::record adds the task's snapshot to each event it records with the task model. Native now adds it centrally for those event types, uses the full snapshot on TimerScheduled and the activity events, and leases a timer task to itself while it fires, as RunTimerTask does, so TimerFired carries the leased timer task. --- .../2026-10-23-task-snapshots.md | 36 ++++++++++ rust/src/runtime/activity_failures.rs | 4 +- rust/src/runtime/store.rs | 66 +++++++++++++++++-- .../execution/worker_history_presentation.rs | 59 +++++++++++++++++ 4 files changed, 157 insertions(+), 8 deletions(-) create mode 100644 docs/rust-server-port-checkpoints/2026-10-23-task-snapshots.md diff --git a/docs/rust-server-port-checkpoints/2026-10-23-task-snapshots.md b/docs/rust-server-port-checkpoints/2026-10-23-task-snapshots.md new file mode 100644 index 000000000..10c5a1ce1 --- /dev/null +++ b/docs/rust-server-port-checkpoints/2026-10-23-task-snapshots.md @@ -0,0 +1,36 @@ +# October 23, task snapshots on history events + +- **The gap.** Published PHP's `WorkflowHistoryEvent::record` adds a `task` + snapshot to every event it records with the task model in hand. Native + added snapshots event by event and missed several. Driving one workflow + through a side effect, an activity, a timer, a child, a signal wait and a + condition wait against both servers showed that native had no snapshot on: + - `TimerFired`, `ChildWorkflowScheduled`, `ChildRunStarted`; + - `SignalWaitOpened`, `ConditionWaitOpened`, `ConditionWaitSatisfied`; + - `SignalApplied` and the instance `MessageCursorAdvanced`. + + Native's own snapshots on `TimerScheduled`, `ActivityStarted`, + `ActivityCompleted` and `ActivityFailed` also left out fields that PHP + keeps, such as `queue`, `connection`, `available_at` and `repair_count`. +- **What native does now.** + - When an event of a type PHP records with the task model is appended with + a task and no snapshot of its own, native adds the task's snapshot. The + list of those event types comes from the Workflow package's source. + - `TimerScheduled` uses the full snapshot. The activity events' snapshots + keep PHP's fields. + - As `RunTimerTask` does, a timer task is leased to itself while its timer + fires: status leased, `leased_at`, `lease_owner` set to the task id, a + workflow-task lease, and one more attempt. Its lease is cleared once the + timer has fired. `TimerFired` therefore carries the leased timer task. +- **How it was checked.** The same probe now finds the same events with + snapshots on both servers, with the same fields. A new integration test + checks the timer, child, signal-wait, signal and condition-wait events. +- **Left as they are.** + - PHP's command snapshots carry `request_route_name`, a name Laravel + generates at random for the deployment's cached routes. Native has no + such name. + - Native stamps a dispatch on every new task, so the snapshot of a task + created inside a completion keeps `last_dispatch_attempt_at` and + `last_dispatched_at`, which PHP's does not (Server #325 follow-up). + - A message stream's `SignalApplied` and the condition wait it satisfies + follow the message streams change (#709). diff --git a/rust/src/runtime/activity_failures.rs b/rust/src/runtime/activity_failures.rs index 062f7720a..e24c73bb7 100644 --- a/rust/src/runtime/activity_failures.rs +++ b/rust/src/runtime/activity_failures.rs @@ -540,7 +540,9 @@ where } payload["task"] = json!({"id":task_id,"type":"activity","status":"leased", "available_at":DB::optional_instant(&task,"available_at")?,"leased_at":DB::optional_instant(&task,"leased_at")?, - "lease_owner":DB::optional_string(&task,"lease_owner")?,"attempt_count":count}); + "lease_owner":DB::optional_string(&task,"lease_owner")?,"lease_expires_at":DB::optional_instant(&task,"lease_expires_at")?, + "attempt_count":count,"repair_count":DB::number(&task,"repair_count")?,"queue":DB::string(&task,"queue")?, + "connection":DB::PHP_CONNECTION}); let failed_event = payload.clone(); Self::append( &mut tx, diff --git a/rust/src/runtime/store.rs b/rust/src/runtime/store.rs index 03dfa699a..2995d2b9a 100644 --- a/rust/src/runtime/store.rs +++ b/rust/src/runtime/store.rs @@ -15,6 +15,37 @@ pub(super) struct PreparedControlArguments { pub(super) value: String, pub(super) command: String, } +/// The events PHP's Workflow package records with the task model in hand, +/// which therefore carry the task's snapshot. +const TASK_SNAPSHOT_EVENTS: &[&str] = &[ + "ActivityCompleted", + "ActivityFailed", + "ActivityHeartbeatRecorded", + "ActivityRetryScheduled", + "ActivityScheduled", + "ActivityStarted", + "ActivityTimedOut", + "ChildRunStarted", + "ChildWorkflowScheduled", + "ConditionWaitOpened", + "ConditionWaitSatisfied", + "ConditionWaitTimedOut", + "FailureHandled", + "MemoUpserted", + "MessageCursorAdvanced", + "SearchAttributesUpserted", + "SelectionOperationCancelled", + "SideEffectRecorded", + "SignalApplied", + "SignalWaitOpened", + "TimerFired", + "TimerScheduled", + "VersionMarkerRecorded", + "WorkflowCompleted", + "WorkflowContinuedAsNew", + "WorkflowFailed", + "WorkflowTimedOut", +]; /// The runtime-owned message-stream delivery signal (`WorkflowStub::MESSAGE_STREAM_RUNTIME_SIGNAL`). const MESSAGE_STREAM_SIGNAL: &str = "__durable_workflow_message_stream"; pub(super) const TASK_CANDIDATES_SQL: &str = "SELECT t.*,r.workflow_type FROM workflow_tasks t JOIN workflow_runs r ON r.id=t.workflow_run_id WHERE t.namespace=$1 AND t.queue=$2 AND t.task_type=$3 AND r.status IN ('pending','running','waiting') AND (t.status='ready' OR (t.status='leased' AND t.task_type='workflow' AND t.lease_expires_at<=$4)) AND (t.available_at IS NULL OR t.available_at<=$5) AND (COALESCE(NULLIF(t.compatibility,''),NULLIF(r.compatibility,'')) IS NULL OR COALESCE(NULLIF(t.compatibility,''),NULLIF(r.compatibility,''))=$6) ORDER BY t.priority,t.available_at,t.id LIMIT 100"; @@ -1709,7 +1740,8 @@ where payload["task"] = json!({"id":task_id,"type":"activity","status":"leased", "repair_count":DB::number(&task,"repair_count")?, "available_at":DB::optional_instant(&task,"available_at")?,"leased_at":leased_at, - "lease_owner":worker_id,"lease_expires_at":expires,"attempt_count":attempt,"queue":queue}); + "lease_owner":worker_id,"lease_expires_at":expires,"attempt_count":attempt,"queue":queue, + "connection":DB::PHP_CONNECTION}); Self::append_at( &mut tx, &run_id, @@ -2490,8 +2522,7 @@ where Self::query("INSERT INTO workflow_run_timers(id,workflow_run_id,sequence,status,delay_seconds,fire_at) VALUES ($1,$2,$3,'pending',$4,$5)") .bind(&timer_id).bind(&run_id).bind(sequence).bind(delay).bind(DB::bind_time(fire_at)).execute(&mut *tx).await?; let mut timer_payload = json!({"timer_id":timer_id,"sequence":sequence,"delay_seconds":delay,"fire_at":fire_at, - "task":{"id":task_id,"type":"workflow","status":"leased","lease_owner":DB::optional_string(&task,"lease_owner")?, - "lease_expires_at":DB::optional_instant(&task,"lease_expires_at")?,"attempt_count":DB::number(&task,"attempt_count")?}}); + "task":Self::task_snapshot(&task)?}); super::selections::merge(&mut timer_payload, &metadata); Self::append_at( &mut tx, @@ -2889,7 +2920,8 @@ where payload["task"] = json!({"id":task_id,"type":"activity","status":"leased", "repair_count":DB::number(&task,"repair_count")?, "available_at":DB::optional_instant(&task,"available_at")?,"leased_at":DB::optional_instant(&task,"leased_at")?, - "lease_owner":DB::optional_string(&task,"lease_owner")?,"attempt_count":DB::number(&task,"attempt_count")?}); + "lease_owner":DB::optional_string(&task,"lease_owner")?,"lease_expires_at":DB::optional_instant(&task,"lease_expires_at")?, + "attempt_count":DB::number(&task,"attempt_count")?,"queue":DB::string(&task,"queue")?,"connection":DB::PHP_CONNECTION}); Self::append( &mut tx, &run_id, @@ -3015,6 +3047,15 @@ where json!({"timer_id":timer_id}) }; super::selections::merge(&mut payload, &metadata); + // `RunTimerTask`: the timer task is leased to itself while its + // timer fires, so `TimerFired` carries the leased task. + Self::query("UPDATE workflow_tasks SET status='leased',leased_at=$1,lease_owner=$2,lease_expires_at=$3,attempt_count=attempt_count+1 WHERE id=$4") + .bind(DB::bind_time(fired_at)) + .bind(&task_id) + .bind(DB::bind_time(fired_at + chrono::Duration::seconds(self.workflow_task_lease_seconds))) + .bind(&task_id) + .execute(&mut *tx) + .await?; Self::append( &mut tx, &run_id, @@ -3038,7 +3079,7 @@ where .bind(&timer_id) .execute(&mut *tx) .await?; - Self::query("UPDATE workflow_tasks SET status='completed' WHERE id=$1") + Self::query("UPDATE workflow_tasks SET status='completed',lease_expires_at=NULL WHERE id=$1") .bind(&task_id) .execute(&mut *tx) .await?; @@ -3367,9 +3408,20 @@ where .execute(&mut **tx) .await?; let recorded_at = recorded_at.unwrap_or_else(now); - // `WorkflowHistoryEvent::record`: an event recorded with a command - // carries that command's snapshot. let mut payload = payload; + // `WorkflowHistoryEvent::record`: an event PHP records with its task + // model carries that task's snapshot. + if let Some(task_id) = task_id + && TASK_SNAPSHOT_EVENTS.contains(&event_type) + && payload.as_object().is_some_and(|fields| !fields.contains_key("task")) + && let Some(task) = Self::query("SELECT * FROM workflow_tasks WHERE id=$1") + .bind(task_id) + .fetch_optional(&mut **tx) + .await? + { + payload["task"] = Self::task_snapshot(&task)?; + } + // An event recorded with a command carries that command's snapshot. if let Some(command_id) = command_id.filter(|_| command_snapshot) && payload.as_object().is_some_and(|fields| !fields.contains_key("command")) && let Some(snapshot) = Self::command_snapshot(tx, command_id).await? diff --git a/rust/tests/execution/worker_history_presentation.rs b/rust/tests/execution/worker_history_presentation.rs index db76a3d75..401ac593f 100644 --- a/rust/tests/execution/worker_history_presentation.rs +++ b/rust/tests/execution/worker_history_presentation.rs @@ -62,3 +62,62 @@ async fn a_worker_reads_the_sequence_that_consumed_a_signal() { } assert!(completed["payload"]["task"].get("last_claim_error").is_none(), "unset fields stay out: {completed}"); } + +// `WorkflowHistoryEvent::record` with the task model: each event PHP records +// while handling a task carries that task's snapshot, and a fired timer +// carries its timer task, leased to itself. +#[tokio::test] +async fn events_recorded_with_a_task_carry_its_snapshot() { + let database = TestDatabase::new().await; + let app = router(database.open().await.unwrap()); + let started = start(&app, "snapshot-events").await; + register(&app, "snapshotter", json!(["echo", "child"]), json!([])).await; + let events = || run_history(&app, &started["workflow_id"], &started["run_id"]); + let snapshot_of = |history: &Value, kind: &str| -> Value { + history.as_array().unwrap().iter().rev().find(|event| event["event_type"] == kind) + .map(|event| event["payload"]["task"].clone()).unwrap_or(Value::Null) + }; + + let task = poll(&app, "snapshotter", "workflow").await; + assert_eq!(finish_task(&app, &task, json!([{"type":"start_timer","delay_seconds":1}])).await.0, StatusCode::OK); + let scheduled = snapshot_of(&events().await, "TimerScheduled"); + assert_eq!((scheduled["id"].clone(), scheduled["type"].clone()), (task["task_id"].clone(), json!("workflow")), "{scheduled}"); + for field in ["queue", "connection", "available_at", "leased_at", "repair_count"] { + assert!(scheduled.get(field).is_some(), "{field}: {scheduled}"); + } + let deadline = tokio::time::Instant::now() + std::time::Duration::from_secs(15); + let fired = loop { + let fired = snapshot_of(&events().await, "TimerFired"); + if !fired.is_null() { + break fired; + } + assert!(tokio::time::Instant::now() < deadline, "the timer never fired"); + tokio::time::sleep(std::time::Duration::from_millis(100)).await; + }; + assert_eq!((fired["type"].clone(), fired["status"].clone(), fired["attempt_count"].clone()), + (json!("timer"), json!("leased"), json!(1)), "{fired}"); + assert_eq!(fired["lease_owner"], fired["id"], "{fired}"); + assert!(fired["lease_expires_at"].is_string() && fired["leased_at"].is_string(), "{fired}"); + + let task = poll(&app, "snapshotter", "workflow").await; + assert_eq!(finish_task(&app, &task, json!([ + {"type":"start_child_workflow","workflow_type":"child","arguments":envelope(Payload::Array(vec![]))}, + {"type":"open_signal_wait","signal_name":"go"}])).await.0, StatusCode::OK); + let history = events().await; + for kind in ["ChildWorkflowScheduled", "ChildRunStarted", "SignalWaitOpened"] { + assert_eq!(snapshot_of(&history, kind)["id"], task["task_id"], "{kind}: {history}"); + } + let (status, body) = request(&app, "POST", "/api/workflows/snapshot-events/signal/go", json!([])).await; + assert_eq!(status, StatusCode::ACCEPTED, "{body}"); + let mut task = poll(&app, "snapshotter", "workflow").await; + // The child's first task may come first on the shared queue. + if task["workflow_type"] == "child" { + assert_eq!(finish_task(&app, &task, json!([{"type":"complete_workflow","result":envelope(Payload::Null)}])).await.0, StatusCode::OK); + task = poll(&app, "snapshotter", "workflow").await; + } + assert_eq!(finish_task(&app, &task, json!([{"type":"open_condition_wait","condition_key":"k", + "condition_definition_fingerprint":format!("sha256:{}", "c".repeat(64))}])).await.0, StatusCode::OK); + let history = events().await; + assert_eq!(snapshot_of(&history, "SignalApplied")["id"], task["task_id"], "{history}"); + assert_eq!(snapshot_of(&history, "ConditionWaitOpened")["id"], task["task_id"], "{history}"); +} From 05a6110cb4122c62cf18e8b52c9437c7bf08116a Mon Sep 17 00:00:00 2001 From: Durable Workflow Date: Fri, 23 Oct 2026 21:04:16 +0000 Subject: [PATCH 04/10] Describe each command's workflow class and preview flag like PHP PHP's run description names the run's workflow class on every command and says whether the command's payload preview is omitted. Native left both out. Only a stored external reference is read to decide the flag. --- .../2026-10-23-task-snapshots.md | 10 +++++++++- rust/src/runtime/run_describe.rs | 2 +- rust/src/runtime/store.rs | 11 +++++++++-- 3 files changed, 19 insertions(+), 4 deletions(-) diff --git a/docs/rust-server-port-checkpoints/2026-10-23-task-snapshots.md b/docs/rust-server-port-checkpoints/2026-10-23-task-snapshots.md index 10c5a1ce1..87931cf15 100644 --- a/docs/rust-server-port-checkpoints/2026-10-23-task-snapshots.md +++ b/docs/rust-server-port-checkpoints/2026-10-23-task-snapshots.md @@ -1,4 +1,4 @@ -# October 23, task snapshots on history events +# October 23, task snapshots on history events and described commands - **The gap.** Published PHP's `WorkflowHistoryEvent::record` adds a `task` snapshot to every event it records with the task model in hand. Native @@ -25,6 +25,14 @@ - **How it was checked.** The same probe now finds the same events with snapshots on both servers, with the same fields. A new integration test checks the timer, child, signal-wait, signal and condition-wait events. +- **Described commands.** A field-by-field diff of the same workflow's + description on both servers found two more gaps. Each command PHP + describes names the run's `workflow_class` (`WorkflowCommand::record` keeps + it), and says whether its payload preview is omitted + (`payload_preview_omitted`, true only for a stored external payload over the + size limit). Native now sends both. With fresh workers on both sides, the + descriptions then differ only in `request_route_name` and the + `request_fingerprint` hashed from it. - **Left as they are.** - PHP's command snapshots carry `request_route_name`, a name Laravel generates at random for the deployment's cached routes. Native has no diff --git a/rust/src/runtime/run_describe.rs b/rust/src/runtime/run_describe.rs index 070e4e364..ae9e6ae3b 100644 --- a/rust/src/runtime/run_describe.rs +++ b/rust/src/runtime/run_describe.rs @@ -31,7 +31,7 @@ fn max_payload_bytes() -> i64 { } /// `oversizedPayloadPreview`: an external payload larger than the limit. -fn oversized(envelope: &Value) -> bool { +pub(super) fn oversized(envelope: &Value) -> bool { envelope["external_storage"]["size_bytes"].as_i64().is_some_and(|size| size > max_payload_bytes()) } diff --git a/rust/src/runtime/store.rs b/rust/src/runtime/store.rs index 2995d2b9a..268e891b8 100644 --- a/rust/src/runtime/store.rs +++ b/rust/src/runtime/store.rs @@ -553,7 +553,9 @@ where // refused signals which intentionally have no workflow history event. // Select metadata only: large customer payloads are never decoded or // copied merely to display the command's target and admission result. - let command_rows = Self::query("SELECT c.id,c.command_sequence,c.command_type,c.target_scope,c.requested_workflow_run_id,c.resolved_workflow_run_id,c.source,c.status,c.outcome,c.rejection_reason,c.workflow_type,c.workflow_class,c.accepted_at,c.applied_at,c.rejected_at,c.context,s.signal_name FROM workflow_commands c LEFT JOIN workflow_signal_records s ON s.workflow_command_id=c.id WHERE c.workflow_run_id=$1 AND c.workflow_instance_id=$2 ORDER BY c.command_sequence,c.created_at,c.id") + // Only a stored external reference is read, to tell whether its preview + // is omitted; inline payloads stay unread. + let command_rows = Self::query("SELECT c.id,c.command_sequence,c.command_type,c.target_scope,c.requested_workflow_run_id,c.resolved_workflow_run_id,c.source,c.status,c.outcome,c.rejection_reason,c.workflow_type,c.workflow_class,c.accepted_at,c.applied_at,c.rejected_at,c.context,s.signal_name,CASE WHEN c.payload LIKE 'dw-%' THEN c.payload END AS stored_payload FROM workflow_commands c LEFT JOIN workflow_signal_records s ON s.workflow_command_id=c.id WHERE c.workflow_run_id=$1 AND c.workflow_instance_id=$2 ORDER BY c.command_sequence,c.created_at,c.id") .bind(DB::string(&row, "id")?).bind(workflow_id).fetch_all(&self.pool).await?; let mut commands = Vec::with_capacity(command_rows.len()); for command in command_rows { @@ -564,7 +566,12 @@ where "target_name":DB::optional_string(&command,"signal_name")?,"source":DB::string(&command,"source")?, "status":DB::string(&command,"status")?,"outcome":DB::optional_string(&command,"outcome")?, "reason":DB::optional_string(&command,"rejection_reason")?,"rejection_reason":DB::optional_string(&command,"rejection_reason")?, - "workflow_type":DB::optional_string(&command,"workflow_type")?,"workflow_class":DB::optional_string(&command,"workflow_class")?, + // `WorkflowCommand::record` keeps the run's class on each command. + "workflow_type":DB::optional_string(&command,"workflow_type")?, + "workflow_class":DB::optional_string(&command,"workflow_class")?.or(DB::optional_string(&row,"workflow_class")?), + "payload_preview_omitted":DB::optional_string(&command,"stored_payload")?.as_deref() + .and_then(super::external_resolution::stored_envelope) + .is_some_and(|envelope| super::run_describe::oversized(&envelope)), "accepted_at":DB::optional_instant(&command,"accepted_at")?,"applied_at":DB::optional_instant(&command,"applied_at")?, "rejected_at":DB::optional_instant(&command,"rejected_at")?}); let fields = public.as_object_mut().expect("command object"); From 65772abac8ecb65f7bde7f85c56b80bb77054777 Mon Sep 17 00:00:00 2001 From: Durable Workflow Date: Sat, 24 Oct 2026 00:50:27 +0000 Subject: [PATCH 05/10] Rust #325: preview omission reads native stored references; exact prefix; run-bound task snapshots Security review of #718: - oversized() reads native's external_payload envelope as well as PHP's external_storage, so a stored reference over the payload limit omits the preview (and the target name, as PHP does). Run input and output previews use the same check. - The described-command query reads a payload only when its first 30 characters equal native's stored prefix, compared case-sensitively on each backend (COLLATE BINARY, binary cast, PostgreSQL text). - record() reads a task snapshot only from a task of the event's own run. Tests: a real stored reference at the 2 MiB limit and one byte above, upper-case/v2/malformed/large-inline counterfactuals, the query itself on SQLite, and a mismatched-run event with no snapshot. --- .../2026-10-23-task-snapshots.md | 27 ++++ rust/src/runtime/backend.rs | 8 ++ rust/src/runtime/run_describe.rs | 8 +- rust/src/runtime/store.rs | 37 ++++- rust/src/runtime/store_record_tests.rs | 128 ++++++++++++++++++ .../execution/external_payloads/commands.rs | 57 ++++++++ 6 files changed, 258 insertions(+), 7 deletions(-) create mode 100644 rust/src/runtime/store_record_tests.rs diff --git a/docs/rust-server-port-checkpoints/2026-10-23-task-snapshots.md b/docs/rust-server-port-checkpoints/2026-10-23-task-snapshots.md index 87931cf15..5ea9a10aa 100644 --- a/docs/rust-server-port-checkpoints/2026-10-23-task-snapshots.md +++ b/docs/rust-server-port-checkpoints/2026-10-23-task-snapshots.md @@ -33,6 +33,33 @@ size limit). Native now sends both. With fresh workers on both sides, the descriptions then differ only in `request_route_name` and the `request_fingerprint` hashed from it. +- **Security review corrections.** + - The size check read only PHP's `external_storage` envelope. Native's + stored envelope names its reference `external_payload`, so the flag was + always false. It now reads both, which also corrects the run's input and + output previews. As PHP does, an omitted preview also omits the + command's `target_name`. + - The query read any payload starting with `dw-`, and SQLite's `LIKE` and + MySQL's case-insensitive collation ignore case. It now reads a payload + only when its first 30 characters equal native's stored prefix exactly: + `COLLATE BINARY` on SQLite, a binary cast on MySQL and MariaDB, and + PostgreSQL's case-sensitive text comparison. On MySQL 8.0 and MariaDB + 10.11 the binary cast refuses the upper-case prefix that the column's + collation would match. + - A task snapshot is read only from a task of the event's own run. + - Tests: + - a real uploaded reference on a signal, then the same reference stored + at the 2 MiB limit (not omitted) and one byte over (omitted, with no + target name); the described command never includes the reference; + - counterfactuals that are not native references: an upper-case or v2 + prefix, the prefix with no valid reference after it, and a large inline + payload starting with `dw-`; + - on SQLite, the query itself returns the payload only for the exact + prefix; + - an event recorded on one run with another run's task carries no + snapshot. + + Each test fails without its correction. - **Left as they are.** - PHP's command snapshots carry `request_route_name`, a name Laravel generates at random for the deployment's cached routes. Native has no diff --git a/rust/src/runtime/backend.rs b/rust/src/runtime/backend.rs index 722cd8f75..22eb45864 100644 --- a/rust/src/runtime/backend.rs +++ b/rust/src/runtime/backend.rs @@ -35,6 +35,9 @@ pub(super) trait Backend: Database { const TIMER_SCHEDULE_SQL: &str = "SELECT payload FROM workflow_history_events WHERE workflow_run_id=$1 AND event_type='TimerScheduled' AND payload->>'timer_id'=$2 ORDER BY sequence DESC LIMIT 1"; /// The history sequence of a signal's `SignalReceived` event. const SIGNAL_RECEIVED_SEQUENCE_SQL: &str = "SELECT sequence FROM workflow_history_events WHERE workflow_run_id=$1 AND event_type='SignalReceived' AND payload->>'signal_id'=$2 ORDER BY sequence LIMIT 1"; + // PostgreSQL compares text case-sensitively. + const RUN_COMMANDS_SQL: &str = + super::store::run_commands_sql!("substr(c.payload,1,30)='dw-native-external-payload-v1:'"); const SELECTION_RESOLUTION_SQL: &str = "SELECT id,payload FROM workflow_history_events WHERE workflow_run_id=$1 AND event_type='SelectionResolved' AND payload->>'selection_group_id'=$2 ORDER BY sequence LIMIT 1"; // Frozen run command/task repair counters use each backend's actual width. const REPAIR_COUNTER_MAX: i64 = i32::MAX as i64; @@ -185,6 +188,8 @@ ORDER BY t.available_order,t.id LIMIT 100 const TIMER_TASK_SQL: &str = "SELECT id,payload FROM workflow_tasks WHERE workflow_run_id=$1 AND task_type='timer' AND status='ready' AND json_extract(payload,'$.timer_id')=$2 AND namespace=$3"; const TIMER_SCHEDULE_SQL: &str = "SELECT payload FROM workflow_history_events WHERE workflow_run_id=$1 AND event_type='TimerScheduled' AND json_extract(payload,'$.timer_id')=$2 ORDER BY sequence DESC LIMIT 1"; const SIGNAL_RECEIVED_SEQUENCE_SQL: &str = "SELECT sequence FROM workflow_history_events WHERE workflow_run_id=$1 AND event_type='SignalReceived' AND json_extract(payload,'$.signal_id')=$2 ORDER BY sequence LIMIT 1"; + const RUN_COMMANDS_SQL: &str = + super::store::run_commands_sql!("substr(c.payload,1,30) COLLATE BINARY='dw-native-external-payload-v1:'"); const HISTORY_PAGE: &str = "SELECT * FROM (SELECT *,SUM(length(CAST(payload AS BLOB))) OVER (ORDER BY sequence) AS page_bytes FROM workflow_history_events WHERE workflow_run_id=$1 AND sequence>$2 ORDER BY sequence LIMIT $3) AS page WHERE page_bytes<=8388608 ORDER BY sequence"; // Normalize only the two supported UTC storage shapes. Removing trailing // fractional zeros preserves exact decimal ordering, including earlier @@ -457,6 +462,9 @@ impl Backend for MySql { const TIMER_TASK_SQL: &str = "SELECT id,payload FROM workflow_tasks WHERE workflow_run_id=? AND task_type='timer' AND status='ready' AND JSON_UNQUOTE(JSON_EXTRACT(payload,'$.timer_id'))=? AND BINARY namespace=BINARY ?"; const TIMER_SCHEDULE_SQL: &str = "SELECT payload FROM workflow_history_events WHERE workflow_run_id=? AND event_type='TimerScheduled' AND JSON_UNQUOTE(JSON_EXTRACT(payload,'$.timer_id'))=? ORDER BY sequence DESC LIMIT 1"; const SIGNAL_RECEIVED_SEQUENCE_SQL: &str = "SELECT sequence FROM workflow_history_events WHERE workflow_run_id=? AND event_type='SignalReceived' AND JSON_UNQUOTE(JSON_EXTRACT(payload,'$.signal_id'))=? ORDER BY sequence LIMIT 1"; + // The payload column's collation ignores case; compare bytes. + const RUN_COMMANDS_SQL: &str = + super::store::run_commands_sql!("CAST(SUBSTRING(c.payload,1,30) AS BINARY)='dw-native-external-payload-v1:'"); // First use the preserved run/sequence index to materialize only IDs and // byte counts for this bounded page. Sorting JSON payloads in the window // exhausts MySQL's default sort buffer before the size rejection can run. diff --git a/rust/src/runtime/run_describe.rs b/rust/src/runtime/run_describe.rs index ae9e6ae3b..c18bf74d1 100644 --- a/rust/src/runtime/run_describe.rs +++ b/rust/src/runtime/run_describe.rs @@ -31,8 +31,12 @@ fn max_payload_bytes() -> i64 { } /// `oversizedPayloadPreview`: an external payload larger than the limit. +/// PHP's stored envelope names its reference `external_storage`; native's +/// names it `external_payload`. pub(super) fn oversized(envelope: &Value) -> bool { - envelope["external_storage"]["size_bytes"].as_i64().is_some_and(|size| size > max_payload_bytes()) + ["external_storage", "external_payload"] + .iter() + .any(|key| envelope[key]["size_bytes"].as_i64().is_some_and(|size| size > max_payload_bytes())) } /// Carbon's `toIso8601String()`, which PHP's run description uses. @@ -315,6 +319,8 @@ mod tests { assert!(oversized(&json!({"external_storage":{"size_bytes":2 * 1024 * 1024 + 1}}))); assert!(!oversized(&json!({"external_storage":{"size_bytes":2 * 1024 * 1024}}))); assert!(!oversized(&json!({"codec":"avro","blob":"AA=="}))); + assert!(oversized(&json!({"codec":"avro","external_payload":{"size_bytes":2 * 1024 * 1024 + 1}}))); + assert!(!oversized(&json!({"codec":"avro","external_payload":{"size_bytes":2 * 1024 * 1024}}))); assert_eq!(without_empty(json!({"a":null,"b":[],"c":{},"d":0})), json!({"c":{},"d":0})); let time = DateTime::parse_from_rfc3339("2026-10-21T07:38:51.123Z").unwrap().with_timezone(&Utc); assert_eq!(iso8601(Some(time)), json!("2026-10-21T07:38:51+00:00")); diff --git a/rust/src/runtime/store.rs b/rust/src/runtime/store.rs index e09c281fe..8b970ff41 100644 --- a/rust/src/runtime/store.rs +++ b/rust/src/runtime/store.rs @@ -46,6 +46,20 @@ const TASK_SNAPSHOT_EVENTS: &[&str] = &[ "WorkflowFailed", "WorkflowTimedOut", ]; +/// Describe's command journal. `$stored` is each backend's exact, +/// case-sensitive test for native's stored-reference prefix +/// (`external_resolution::STORED_PREFIX`), so a command's payload is read only +/// when it is a stored reference. +macro_rules! run_commands_sql { + ($stored:literal) => { + concat!( + "SELECT c.id,c.command_sequence,c.command_type,c.target_scope,c.requested_workflow_run_id,c.resolved_workflow_run_id,c.source,c.status,c.outcome,c.rejection_reason,c.workflow_type,c.workflow_class,c.accepted_at,c.applied_at,c.rejected_at,c.context,s.signal_name,CASE WHEN ", + $stored, + " THEN c.payload END AS stored_payload FROM workflow_commands c LEFT JOIN workflow_signal_records s ON s.workflow_command_id=c.id WHERE c.workflow_run_id=$1 AND c.workflow_instance_id=$2 ORDER BY c.command_sequence,c.created_at,c.id" + ) + }; +} +pub(super) use run_commands_sql; /// The runtime-owned message-stream delivery signal (`WorkflowStub::MESSAGE_STREAM_RUNTIME_SIGNAL`). const MESSAGE_STREAM_SIGNAL: &str = "__durable_workflow_message_stream"; pub(super) const TASK_CANDIDATES_SQL: &str = "SELECT t.*,r.workflow_type FROM workflow_tasks t JOIN workflow_runs r ON r.id=t.workflow_run_id WHERE t.namespace=$1 AND t.queue=$2 AND t.task_type=$3 AND r.status IN ('pending','running','waiting') AND (t.status='ready' OR (t.status='leased' AND t.task_type='workflow' AND t.lease_expires_at<=$4)) AND (t.available_at IS NULL OR t.available_at<=$5) AND (COALESCE(NULLIF(t.compatibility,''),NULLIF(r.compatibility,'')) IS NULL OR COALESCE(NULLIF(t.compatibility,''),NULLIF(r.compatibility,''))=$6) ORDER BY t.priority,t.available_at,t.id LIMIT 100"; @@ -555,23 +569,28 @@ where // copied merely to display the command's target and admission result. // Only a stored external reference is read, to tell whether its preview // is omitted; inline payloads stay unread. - let command_rows = Self::query("SELECT c.id,c.command_sequence,c.command_type,c.target_scope,c.requested_workflow_run_id,c.resolved_workflow_run_id,c.source,c.status,c.outcome,c.rejection_reason,c.workflow_type,c.workflow_class,c.accepted_at,c.applied_at,c.rejected_at,c.context,s.signal_name,CASE WHEN c.payload LIKE 'dw-%' THEN c.payload END AS stored_payload FROM workflow_commands c LEFT JOIN workflow_signal_records s ON s.workflow_command_id=c.id WHERE c.workflow_run_id=$1 AND c.workflow_instance_id=$2 ORDER BY c.command_sequence,c.created_at,c.id") + let command_rows = Self::query(DB::RUN_COMMANDS_SQL) .bind(DB::string(&row, "id")?).bind(workflow_id).fetch_all(&self.pool).await?; let mut commands = Vec::with_capacity(command_rows.len()); for command in command_rows { + // `oversizedPayloadPreview` over the stored reference's envelope; + // an omitted preview also omits the target name, as PHP does. + let omitted = DB::optional_string(&command, "stored_payload")? + .as_deref() + .and_then(super::external_resolution::stored_envelope) + .is_some_and(|envelope| super::run_describe::oversized(&envelope)); + let target_name = DB::optional_string(&command, "signal_name")?.filter(|_| !omitted); let mut public = json!({"id":DB::string(&command,"id")?,"sequence":DB::optional_number(&command,"command_sequence")?, "type":DB::string(&command,"command_type")?,"target_scope":DB::string(&command,"target_scope")?, "requested_run_id":DB::optional_string(&command,"requested_workflow_run_id")?, "resolved_run_id":DB::optional_string(&command,"resolved_workflow_run_id")?, - "target_name":DB::optional_string(&command,"signal_name")?,"source":DB::string(&command,"source")?, + "target_name":target_name,"source":DB::string(&command,"source")?, "status":DB::string(&command,"status")?,"outcome":DB::optional_string(&command,"outcome")?, "reason":DB::optional_string(&command,"rejection_reason")?,"rejection_reason":DB::optional_string(&command,"rejection_reason")?, // `WorkflowCommand::record` keeps the run's class on each command. "workflow_type":DB::optional_string(&command,"workflow_type")?, "workflow_class":DB::optional_string(&command,"workflow_class")?.or(DB::optional_string(&row,"workflow_class")?), - "payload_preview_omitted":DB::optional_string(&command,"stored_payload")?.as_deref() - .and_then(super::external_resolution::stored_envelope) - .is_some_and(|envelope| super::run_describe::oversized(&envelope)), + "payload_preview_omitted":omitted, "accepted_at":DB::optional_instant(&command,"accepted_at")?,"applied_at":DB::optional_instant(&command,"applied_at")?, "rejected_at":DB::optional_instant(&command,"rejected_at")?}); let fields = public.as_object_mut().expect("command object"); @@ -3425,11 +3444,13 @@ where if let Some(task_id) = task_id && TASK_SNAPSHOT_EVENTS.contains(&event_type) && payload.as_object().is_some_and(|fields| !fields.contains_key("task")) - && let Some(task) = Self::query("SELECT * FROM workflow_tasks WHERE id=$1") + && let Some(task) = Self::query("SELECT * FROM workflow_tasks WHERE id=$1 AND workflow_run_id=$2") .bind(task_id) + .bind(run_id) .fetch_optional(&mut **tx) .await? { + // Only the run's own task: another run's is never copied. payload["task"] = Self::task_snapshot(&task)?; } // An event recorded with a command carries that command's snapshot. @@ -4082,3 +4103,7 @@ pub(crate) fn capabilities() -> Value { } value } + +#[cfg(test)] +#[path = "store_record_tests.rs"] +mod record_tests; diff --git a/rust/src/runtime/store_record_tests.rs b/rust/src/runtime/store_record_tests.rs new file mode 100644 index 000000000..1f312ffeb --- /dev/null +++ b/rust/src/runtime/store_record_tests.rs @@ -0,0 +1,128 @@ +//! History recording and described-command boundaries that a public route +//! cannot reach with mismatched identifiers. +use super::Store; +use crate::runtime::backend::Backend; +use crate::runtime::external_resolution::STORED_PREFIX; +use serde_json::{Value, json}; +use std::collections::BTreeMap; + +#[test] +fn described_commands_read_only_the_exact_native_stored_prefix() { + // Each backend compares the first 30 characters, byte for byte, with + // the whole prefix. + assert_eq!(STORED_PREFIX.len(), 30); + let quoted = format!("'{STORED_PREFIX}'"); + for (backend, sql, comparison) in [ + ("sqlite", ::RUN_COMMANDS_SQL, "substr(c.payload,1,30) COLLATE BINARY="), + ("postgres", ::RUN_COMMANDS_SQL, "substr(c.payload,1,30)="), + ("mysql", ::RUN_COMMANDS_SQL, "CAST(SUBSTRING(c.payload,1,30) AS BINARY)="), + ] { + assert!(sql.contains(&format!("CASE WHEN {comparison}{quoted} THEN c.payload END")), "{backend}: {sql}"); + assert!(!sql.contains("LIKE"), "{backend}: {sql}"); + } +} + +async fn recorded_task(store: &Store, run_id: &str) -> Value { + let row = Store::::query( + "SELECT payload FROM workflow_history_events WHERE workflow_run_id=$1 AND event_type='SideEffectRecorded'", + ) + .bind(run_id) + .fetch_one(&store.pool) + .await + .unwrap(); + ::document_row(&row, "payload").unwrap()["task"].clone() +} + +#[tokio::test] +async fn a_task_snapshot_comes_only_from_the_events_own_run() { + let directory = tempfile::tempdir().unwrap(); + let store = Store::::new( + crate::runtime::sqlite::open(directory.path().join("record.sqlite").to_str().unwrap()) + .await + .unwrap(), + ); + store + .register( + json!({"worker_id":"record-worker","task_queue":"test","runtime":"php", + "supported_workflow_types":["echo"],"supported_activity_types":[]}), + &crate::runtime::QueryTaskConfig::default(), + None, + ) + .await + .unwrap(); + let mut runs = Vec::new(); + for workflow_id in ["record-own", "record-other"] { + let started = store + .start( + json!({"workflow_id":workflow_id,"workflow_type":"echo","task_queue":"test", + "input":{"codec":"avro","blob":"AA=="}}), + BTreeMap::new(), + None, + None, + ) + .await + .unwrap(); + runs.push(started["run_id"].as_str().unwrap().to_owned()); + } + let task = Store::::query("SELECT id FROM workflow_tasks WHERE workflow_run_id=$1") + .bind(&runs[0]) + .fetch_one(&store.pool) + .await + .unwrap(); + let task_id = ::string(&task, "id").unwrap(); + + let mut tx = store.begin().await.unwrap(); + for run_id in &runs { + Store::::record(&mut tx, run_id, "SideEffectRecorded", json!({"sequence":1}), + Some(&task_id), None, None, false) + .await + .unwrap(); + } + tx.commit().await.unwrap(); + + assert_eq!(recorded_task(&store, &runs[0]).await["id"], json!(task_id), "the run's own task"); + // Recorded on the other run with the first run's task, the event carries + // no snapshot at all. + assert!(recorded_task(&store, &runs[1]).await.is_null()); +} + +// The query itself, run on rows whose payloads begin like a stored +// reference: only native's exact prefix is selected. +#[tokio::test] +async fn described_commands_select_no_inline_payload() { + let directory = tempfile::tempdir().unwrap(); + let store = Store::::new( + crate::runtime::sqlite::open(directory.path().join("commands.sqlite").to_str().unwrap()) + .await + .unwrap(), + ); + let exact = format!("{STORED_PREFIX}{{}}"); + let rows = [ + ("c1", exact.clone(), true), + ("c2", exact.to_uppercase(), false), + ("c3", format!("dw-native-external-payload-v1{}", "A".repeat(4096)), false), + ("c4", format!("dw-{}", "A".repeat(4096)), false), + ("c5", "dw-native".to_owned(), false), + ]; + for (sequence, (id, payload, _)) in rows.iter().enumerate() { + Store::::query("INSERT INTO workflow_commands(id,workflow_instance_id,workflow_run_id,command_type,status,payload,command_sequence) VALUES ($1,'instance','run','signal','accepted',$2,$3)") + .bind(*id) + .bind(payload.as_str()) + .bind(sequence as i64 + 1) + .execute(&store.pool) + .await + .unwrap(); + } + let selected = Store::::query(::RUN_COMMANDS_SQL) + .bind("run") + .bind("instance") + .fetch_all(&store.pool) + .await + .unwrap(); + assert_eq!(selected.len(), rows.len()); + for (row, (id, payload, read)) in selected.iter().zip(&rows) { + assert_eq!(::string(row, "id").unwrap(), *id); + let stored = ::optional_string(row, "stored_payload").unwrap(); + assert_eq!(stored.as_deref(), read.then_some(payload.as_str()), "{id}"); + } +} diff --git a/rust/tests/execution/external_payloads/commands.rs b/rust/tests/execution/external_payloads/commands.rs index 82237259c..bb962993b 100644 --- a/rust/tests/execution/external_payloads/commands.rs +++ b/rust/tests/execution/external_payloads/commands.rs @@ -151,3 +151,60 @@ async fn signal_update_and_query_input_resolve_runtime_references_like_php() { runtime.close().await; database.remove().await; } + +/// The described signal command, which must show only metadata. +async fn described_signal(app: &Router, workflow_id: &str) -> Value { + let (status, body) = json_request(app, "GET", &format!("/api/workflows/{workflow_id}"), "operator-token", Value::Null).await; + assert_eq!(status, StatusCode::OK, "{body}"); + let command = body["commands"].as_array().unwrap().iter().find(|command| command["type"] == "signal").unwrap().clone(); + for field in ["payload", "external_payload", "external_storage", "reference_id", "sha256", "size_bytes", "arguments"] { + assert!(command.get(field).is_none(), "{field}: {command}"); + } + assert!(!command.to_string().contains("ep_"), "no reference identity: {command}"); + command +} + +// `WorkflowController::oversizedPayloadPreview` over each command's stored +// envelope: only a stored external reference larger than the payload limit +// (2 MiB by default) omits the preview, and then the target name too. +#[tokio::test] +async fn described_commands_omit_only_an_oversized_stored_reference() { + let temp = tempfile::tempdir().unwrap(); + let (database, runtime, app) = setup(ExternalPayloadConfig::default(), &temp.path().join("objects")).await; + waiting_run(&app, "described-reference", json!({"worker_id":"describe-worker","task_queue":"described-reference", + "runtime":"php","supported_workflow_types":["echo"],"supported_activity_types":[]})).await; + let reference = upload(&app, &argument("signal")).await; + let (status, body) = json_request(&app, "POST", "/api/workflows/described-reference/signal/s", "operator-token", + json!({"input":{"codec":"avro","external_payload":reference}})).await; + assert_eq!(status, StatusCode::ACCEPTED, "{body}"); + let command = described_signal(&app, "described-reference").await; + assert_eq!((command["payload_preview_omitted"].clone(), command["target_name"].clone()), (json!(false), json!("s")), "{command}"); + let command_id = command["id"].as_str().unwrap().to_owned(); + + // The same stored reference, as native writes it, at sizes around the + // limit, and counterfactuals that are not native stored references. + let limit = 2 * 1024 * 1024_i64; + let stored = |size: i64| { + let mut reference = reference.clone(); + reference["size_bytes"] = json!(size); + format!("dw-native-external-payload-v1:{reference}") + }; + for (payload, omitted) in [ + (stored(limit), false), + (stored(limit + 1), true), + // Not native's exact prefix, so never read. + (stored(limit + 1).replace("dw-native-external-payload-v1:", "DW-NATIVE-EXTERNAL-PAYLOAD-V1:"), false), + (stored(limit + 1).replace("dw-native-external-payload-v1:", "dw-native-external-payload-v2:"), false), + // Native's prefix with no valid reference after it. + (format!("dw-native-external-payload-v1:{{\"size_bytes\":{}}}", limit + 1), false), + // A large inline payload. + (format!("dw-{}", "A".repeat(limit as usize + 1)), false), + ] { + database.execute(sqlx::AssertSqlSafe(format!("UPDATE workflow_commands SET payload='{payload}' WHERE id='{command_id}'"))).await; + let command = described_signal(&app, "described-reference").await; + assert_eq!(command["payload_preview_omitted"], json!(omitted), "{}: {command}", &payload[..payload.len().min(48)]); + assert_eq!(command.get("target_name").is_none(), omitted, "{command}"); + } + runtime.close().await; + database.remove().await; +} From dc1b4e0ae26992d2ac45f450c2e4fd01e82e652b Mon Sep 17 00:00:00 2001 From: Durable Workflow Date: Sat, 24 Oct 2026 00:58:48 +0000 Subject: [PATCH 06/10] Server parity: run the SQLite published-SDK patch record without PHP's maintenance loop (#721) On SQLite the published PHP API server and its maintenance loop share one database file. A contended poll gets PHP's typed backend_lock_pressure refusal, and the published Python run_until helper ends its Worker on it (sdk-python #218). The cohort tests patch replacement, not writer contention: stop the loop for the PHP record, restart it right after, and record and assert its state. The observer still never retries a poll refusal; MySQL and PostgreSQL jobs are unchanged. --- .../published-sdk-patch-profile.sh | 24 +++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/scripts/conformance/server-parity/published-sdk-patch-profile.sh b/scripts/conformance/server-parity/published-sdk-patch-profile.sh index 5e012bd34..221adce99 100644 --- a/scripts/conformance/server-parity/published-sdk-patch-profile.sh +++ b/scripts/conformance/server-parity/published-sdk-patch-profile.sh @@ -25,13 +25,37 @@ shared=(-v "$PWD:/source:ro" -v "$PWD/published-sdk-adapters:/sdk-adapters:ro" -e DW_PARITY_RUST_PATCH_WORKER=/sdk-adapters/rust-worker -e PYTHONDONTWRITEBYTECODE=1 -e DW_PARITY_TOKEN=parity-test-token -e DW_PARITY_RUNNER_REVISION) export DW_PARITY_RUNNER_REVISION=$(git rev-parse HEAD) +# Server #721. On SQLite the published PHP API server and its maintenance loop +# share one database file. SQLite refuses a deferred transaction's upgrade to a +# writer at once, without a busy wait, so PHP answers a contended poll with its +# typed backend_lock_pressure refusal. The published Python run_until helper +# then ends its Worker (sdk-python #218). This cohort tests patch replacement, +# not writer contention, so the PHP record runs with the maintenance loop +# stopped, and the loop restarts right after it. The fixtures use no timers or +# repair. The observer still never retries a poll refusal. +maintenance="$RESOURCE_SCOPE-php-maintenance" +schedule=parity-evidence/published-sdk-patch/maintenance-schedule.txt for target in php rust; do if test "$target" = php; then url="$php_url"; else url="$native_url"; fi + if test "$target" = php && test "$backend" = sqlite; then + echo "before: $(docker inspect -f '{{.State.Status}}' "$maintenance")" > "$schedule" + grep -qx 'before: running' "$schedule" + docker stop --time 2 "$maintenance" > /dev/null + trap 'docker start "$maintenance" > /dev/null' EXIT + echo "during php record: $(docker inspect -f '{{.State.Status}}' "$maintenance")" >> "$schedule" + grep -qx 'during php record: exited' "$schedule" + fi docker run --rm --user=1000:1000 --cpus=1 --memory=512m --memory-swap=512m \ --network "$RESOURCE_SCOPE" --entrypoint node "${shared[@]}" "$image" \ scripts/conformance/server-parity.mjs record --mode http --url "$url" --target "$target" \ --prefix published-sdk-v1 --artifacts "$profile/artifacts.json" "${fixtures[@]}" \ --output "/evidence/published-sdk-patch/$target/record.json" + if test "$target" = php && test "$backend" = sqlite; then + docker start "$maintenance" > /dev/null + trap - EXIT + echo "after php record: $(docker inspect -f '{{.State.Status}}' "$maintenance")" >> "$schedule" + grep -qx 'after php record: running' "$schedule" + fi done case "$backend" in mysql) embedded=(-e DB_CONNECTION=mysql -e DB_HOST=parity-mysql -e DB_DATABASE=embedded_ref From 17f63ca8d63bdc2a541c13cd4475bd89dccb7a65 Mon Sep 17 00:00:00 2001 From: Durable Workflow Date: Sat, 24 Oct 2026 01:39:56 +0000 Subject: [PATCH 07/10] Server parity: arm the maintenance restore before the stop and keep it until a sustained restart (#721) Security review of #722: the restore trap was installed only after a successful stop and disarmed before the restart was checked. Move the window into published-sdk-patch/maintenance.sh: arm restoration before the stop, restart with up to three attempts that each require the loop to stay running across three checks, and disarm only after that. A failed restart records the observed state and fails with restoration still armed. tests/Unit/PatchCohortMaintenanceTest.mjs injects a stop that exits nonzero after taking effect, a failed PHP record, and a start that exits at once, with a fake docker. The previous ordering fails the first and the last. --- .github/workflows/phpunit-feature.yml | 3 + .../published-sdk-patch-profile.sh | 33 +++---- .../published-sdk-patch/maintenance.sh | 58 ++++++++++++ tests/Unit/PatchCohortMaintenanceTest.mjs | 90 +++++++++++++++++++ 4 files changed, 163 insertions(+), 21 deletions(-) create mode 100644 scripts/conformance/server-parity/published-sdk-patch/maintenance.sh create mode 100644 tests/Unit/PatchCohortMaintenanceTest.mjs diff --git a/.github/workflows/phpunit-feature.yml b/.github/workflows/phpunit-feature.yml index 9cecaf1bf..c4e233777 100644 --- a/.github/workflows/phpunit-feature.yml +++ b/.github/workflows/phpunit-feature.yml @@ -71,6 +71,9 @@ jobs: - name: Require safe PHP image build context run: node --test tests/Unit/PhpImageContextTest.mjs + - name: Restore the patch cohort's maintenance loop on every path + run: node --test tests/Unit/PatchCohortMaintenanceTest.mjs + - name: Validate embedded Workflow package authority run: php scripts/ci/test-workflow-package-authority.php diff --git a/scripts/conformance/server-parity/published-sdk-patch-profile.sh b/scripts/conformance/server-parity/published-sdk-patch-profile.sh index 221adce99..c24c58762 100644 --- a/scripts/conformance/server-parity/published-sdk-patch-profile.sh +++ b/scripts/conformance/server-parity/published-sdk-patch-profile.sh @@ -33,30 +33,21 @@ export DW_PARITY_RUNNER_REVISION=$(git rev-parse HEAD) # not writer contention, so the PHP record runs with the maintenance loop # stopped, and the loop restarts right after it. The fixtures use no timers or # repair. The observer still never retries a poll refusal. -maintenance="$RESOURCE_SCOPE-php-maintenance" -schedule=parity-evidence/published-sdk-patch/maintenance-schedule.txt -for target in php rust; do - if test "$target" = php; then url="$php_url"; else url="$native_url"; fi - if test "$target" = php && test "$backend" = sqlite; then - echo "before: $(docker inspect -f '{{.State.Status}}' "$maintenance")" > "$schedule" - grep -qx 'before: running' "$schedule" - docker stop --time 2 "$maintenance" > /dev/null - trap 'docker start "$maintenance" > /dev/null' EXIT - echo "during php record: $(docker inspect -f '{{.State.Status}}' "$maintenance")" >> "$schedule" - grep -qx 'during php record: exited' "$schedule" - fi +. scripts/conformance/server-parity/published-sdk-patch/maintenance.sh +record() { docker run --rm --user=1000:1000 --cpus=1 --memory=512m --memory-swap=512m \ --network "$RESOURCE_SCOPE" --entrypoint node "${shared[@]}" "$image" \ - scripts/conformance/server-parity.mjs record --mode http --url "$url" --target "$target" \ + scripts/conformance/server-parity.mjs record --mode http --url "$1" --target "$2" \ --prefix published-sdk-v1 --artifacts "$profile/artifacts.json" "${fixtures[@]}" \ - --output "/evidence/published-sdk-patch/$target/record.json" - if test "$target" = php && test "$backend" = sqlite; then - docker start "$maintenance" > /dev/null - trap - EXIT - echo "after php record: $(docker inspect -f '{{.State.Status}}' "$maintenance")" >> "$schedule" - grep -qx 'after php record: running' "$schedule" - fi -done + --output "/evidence/published-sdk-patch/$2/record.json" +} +if test "$backend" = sqlite; then + run_without_maintenance "$RESOURCE_SCOPE-php-maintenance" \ + parity-evidence/published-sdk-patch/maintenance-schedule.txt -- record "$php_url" php +else + record "$php_url" php +fi +record "$native_url" rust case "$backend" in mysql) embedded=(-e DB_CONNECTION=mysql -e DB_HOST=parity-mysql -e DB_DATABASE=embedded_ref -e DB_USERNAME=root -e DB_PASSWORD=parity-disposable-password) ;; diff --git a/scripts/conformance/server-parity/published-sdk-patch/maintenance.sh b/scripts/conformance/server-parity/published-sdk-patch/maintenance.sh new file mode 100644 index 000000000..71cdebacb --- /dev/null +++ b/scripts/conformance/server-parity/published-sdk-patch/maintenance.sh @@ -0,0 +1,58 @@ +# Sourced by published-sdk-patch-profile.sh (Server #721). +# +# run_without_maintenance CONTAINER SCHEDULE -- COMMAND... +# +# Runs COMMAND while the maintenance container is stopped, then restarts it. +# Restoration is armed before the stop is attempted and stays armed until the +# container has stayed running for a sustained check, so a failed stop, a +# failed COMMAND or a restart that exits at once never leaves it stopped. +# SCHEDULE records the container's state before, during and after. + +maintenance_poll_seconds="${DW_PARITY_MAINTENANCE_POLL_SECONDS:-1}" + +maintenance_state() { + docker inspect -f '{{.State.Status}}' "$1" 2>/dev/null || echo unknown +} + +# Running at each of three checks, one poll interval apart. +maintenance_stays_running() { + local check + for check in 1 2 3; do + test "$(maintenance_state "$1")" = running || return 1 + sleep "$maintenance_poll_seconds" + done + test "$(maintenance_state "$1")" = running +} + +# Up to three starts, each followed by the sustained check. +maintenance_restore() { + local attempt + for attempt in 1 2 3; do + docker start "$1" > /dev/null 2>&1 || true + if maintenance_stays_running "$1"; then + return 0 + fi + done + echo "maintenance container $1 did not stay running: $(maintenance_state "$1")" >&2 + return 1 +} + +run_without_maintenance() { + local container="$1" schedule="$2" + shift 2 + test "$1" = -- && shift + echo "before: $(maintenance_state "$container")" > "$schedule" + grep -qx 'before: running' "$schedule" + # shellcheck disable=SC2064 + trap "maintenance_restore '$container' || true" EXIT + docker stop --time 2 "$container" > /dev/null + echo "during php record: $(maintenance_state "$container")" >> "$schedule" + grep -qx 'during php record: exited' "$schedule" + "$@" + if ! maintenance_restore "$container"; then + echo "after php record: $(maintenance_state "$container")" >> "$schedule" + return 1 + fi + trap - EXIT + echo "after php record: running for ${maintenance_poll_seconds}s x3" >> "$schedule" +} diff --git a/tests/Unit/PatchCohortMaintenanceTest.mjs b/tests/Unit/PatchCohortMaintenanceTest.mjs new file mode 100644 index 000000000..9283d780e --- /dev/null +++ b/tests/Unit/PatchCohortMaintenanceTest.mjs @@ -0,0 +1,90 @@ +// Server #721: the SQLite published-SDK patch cohort stops PHP's maintenance +// loop for its PHP record. Every failure path must leave the loop restored or +// fail loudly, and never record a restart that did not stay running. +import test from 'node:test'; +import assert from 'node:assert/strict'; +import {chmodSync, existsSync, mkdtempSync, readFileSync, writeFileSync} from 'node:fs'; +import {tmpdir} from 'node:os'; +import {join, resolve} from 'node:path'; +import {spawnSync} from 'node:child_process'; + +const helper = resolve('scripts/conformance/server-parity/published-sdk-patch/maintenance.sh'); + +// One container's state in a file; every call is logged. +// SCENARIO=stop-nonzero: the stop takes effect but exits 1. +// SCENARIO=immediate-exit: start exits 0, the container runs for one +// inspection and then exits. +const fakeDocker = `#!/usr/bin/env bash +set -u +dir=$(dirname "$0") +echo "$*" >> "$dir/calls" +case "$1" in + inspect) + state=$(cat "$dir/state") + if test "$state" = running-once; then echo exited > "$dir/state"; state=running; fi + echo "$state" ;; + stop) echo exited > "$dir/state"; test "$SCENARIO" != stop-nonzero ;; + start) + if test "$SCENARIO" = immediate-exit; then echo running-once > "$dir/state"; else echo running > "$dir/state"; fi ;; + *) exit 2 ;; +esac +`; + +function run(scenario, command) { + const dir = mkdtempSync(join(tmpdir(), 'patch-maintenance-')); + writeFileSync(join(dir, 'docker'), fakeDocker); + chmodSync(join(dir, 'docker'), 0o755); + writeFileSync(join(dir, 'state'), 'running\n'); + writeFileSync(join(dir, 'calls'), ''); + const result = spawnSync('bash', ['-c', + `set -euo pipefail; . "$HELPER"; run_without_maintenance maint "$SCHEDULE" -- ${command}`], { + encoding: 'utf8', + env: {...process.env, PATH: `${dir}:${process.env.PATH}`, HELPER: helper, + SCHEDULE: join(dir, 'schedule.txt'), SCENARIO: scenario, DW_PARITY_MAINTENANCE_POLL_SECONDS: '0.05'}, + }); + const read = name => (existsSync(join(dir, name)) ? readFileSync(join(dir, name), 'utf8') : ''); + const calls = read('calls').trim().split('\n'); + return { + status: result.status, + stderr: result.stderr, + schedule: read('schedule.txt').trim().split('\n'), + starts: calls.filter(call => call.startsWith('start ')).length, + stopIndex: calls.findIndex(call => call.startsWith('stop ')), + lastStart: calls.findLastIndex(call => call.startsWith('start ')), + state: read('state').trim(), + }; +} + +test('a normal window stops, runs, and restores a loop that stays running', () => { + const result = run('normal', 'true'); + assert.equal(result.status, 0, result.stderr); + assert.deepEqual(result.schedule, ['before: running', 'during php record: exited', 'after php record: running for 0.05s x3']); + assert.equal(result.starts, 1, 'restoration is disarmed after the sustained check'); + assert.equal(result.state, 'running'); +}); + +test('a stop that takes effect but exits nonzero still restores', () => { + const result = run('stop-nonzero', 'true'); + assert.notEqual(result.status, 0); + assert.ok(result.lastStart > result.stopIndex, 'a start follows the stop'); + assert.equal(result.state, 'running'); + assert.deepEqual(result.schedule, ['before: running']); +}); + +test('a failed PHP record restores the loop', () => { + const result = run('normal', 'false'); + assert.notEqual(result.status, 0); + assert.equal(result.state, 'running'); + assert.ok(result.starts >= 1); + assert.ok(!result.schedule.some(line => line.startsWith('after php record'))); +}); + +test('a restart that exits at once stays armed and fails without recording running', () => { + const result = run('immediate-exit', 'true'); + assert.notEqual(result.status, 0); + assert.ok(result.schedule.includes('after php record: exited'), result.schedule.join('\n')); + assert.ok(!result.schedule.some(line => line.startsWith('after php record: running'))); + // Three attempts after the record, then three more from the still-armed trap. + assert.equal(result.starts, 6); + assert.match(result.stderr, /did not stay running/); +}); From 38717d3372767b72cf17851b0bd16ca935515e81 Mon Sep 17 00:00:00 2001 From: Durable Workflow Date: Sat, 24 Oct 2026 01:43:15 +0000 Subject: [PATCH 08/10] Rust #325: read a signal's stored reference from its signal record Security review of #718 at 65772abac: native keeps a signal's stored input reference in workflow_signal_records.arguments; workflow_commands.payload holds the encoded command. Read the exact native sentinel from s.arguments for signals and from c.payload for starts only, case-sensitively per backend. The describe test now proves the real signal row is the source, mutates that row around the limit with the same counterfactuals, shows an oversized sentinel in a signal's command row does not count, and covers the start command's row. --- .../2026-10-23-task-snapshots.md | 30 +++++---- rust/src/runtime/backend.rs | 15 ++++- rust/src/runtime/store.rs | 18 +++--- rust/src/runtime/store_record_tests.rs | 62 +++++++++++++------ rust/tests/execution/external_payloads.rs | 2 +- .../execution/external_payloads/commands.rs | 50 +++++++++++---- 6 files changed, 124 insertions(+), 53 deletions(-) diff --git a/docs/rust-server-port-checkpoints/2026-10-23-task-snapshots.md b/docs/rust-server-port-checkpoints/2026-10-23-task-snapshots.md index 5ea9a10aa..f139ee749 100644 --- a/docs/rust-server-port-checkpoints/2026-10-23-task-snapshots.md +++ b/docs/rust-server-port-checkpoints/2026-10-23-task-snapshots.md @@ -39,23 +39,31 @@ always false. It now reads both, which also corrects the run's input and output previews. As PHP does, an omitted preview also omits the command's `target_name`. - - The query read any payload starting with `dw-`, and SQLite's `LIKE` and - MySQL's case-insensitive collation ignore case. It now reads a payload - only when its first 30 characters equal native's stored prefix exactly: - `COLLATE BINARY` on SQLite, a binary cast on MySQL and MariaDB, and - PostgreSQL's case-sensitive text comparison. On MySQL 8.0 and MariaDB + - The query read any command payload starting with `dw-`, and SQLite's + `LIKE` and MySQL's case-insensitive collation ignore case. It also read the + wrong column for signals. Native keeps a signal's stored input reference + in its signal record (`workflow_signal_records.arguments`) and a start's + in the command row (`workflow_commands.payload`). Other commands keep + none. The query now reads only that column for each of the two command + types, and only when its first 30 characters equal native's stored prefix + exactly: `COLLATE BINARY` on SQLite, a binary cast on MySQL and MariaDB, + and PostgreSQL's case-sensitive text comparison. On MySQL 8.0 and MariaDB 10.11 the binary cast refuses the upper-case prefix that the column's collation would match. - A task snapshot is read only from a task of the event's own run. - Tests: - - a real uploaded reference on a signal, then the same reference stored - at the 2 MiB limit (not omitted) and one byte over (omitted, with no - target name); the described command never includes the reference; + - a real uploaded reference on a signal, found in the signal record and + not in the command row; + - that signal record rewritten with the same reference at the 2 MiB limit + (not omitted) and one byte over (omitted, with no target name). The + described command never includes the reference; - counterfactuals that are not native references: an upper-case or v2 prefix, the prefix with no valid reference after it, and a large inline - payload starting with `dw-`; - - on SQLite, the query itself returns the payload only for the exact - prefix; + payload starting with `dw-`. An oversized reference in the signal's + command row is not counted either; + - the same sizes and counterfactuals in a start command's row; + - on SQLite, the query itself returns only the exact prefix in the + command type's own column; - an event recorded on one run with another run's task carries no snapshot. diff --git a/rust/src/runtime/backend.rs b/rust/src/runtime/backend.rs index 22eb45864..67a823065 100644 --- a/rust/src/runtime/backend.rs +++ b/rust/src/runtime/backend.rs @@ -37,7 +37,10 @@ pub(super) trait Backend: Database { const SIGNAL_RECEIVED_SEQUENCE_SQL: &str = "SELECT sequence FROM workflow_history_events WHERE workflow_run_id=$1 AND event_type='SignalReceived' AND payload->>'signal_id'=$2 ORDER BY sequence LIMIT 1"; // PostgreSQL compares text case-sensitively. const RUN_COMMANDS_SQL: &str = - super::store::run_commands_sql!("substr(c.payload,1,30)='dw-native-external-payload-v1:'"); + super::store::run_commands_sql!( + "substr(s.arguments,1,30)='dw-native-external-payload-v1:'", + "substr(c.payload,1,30)='dw-native-external-payload-v1:'" + ); const SELECTION_RESOLUTION_SQL: &str = "SELECT id,payload FROM workflow_history_events WHERE workflow_run_id=$1 AND event_type='SelectionResolved' AND payload->>'selection_group_id'=$2 ORDER BY sequence LIMIT 1"; // Frozen run command/task repair counters use each backend's actual width. const REPAIR_COUNTER_MAX: i64 = i32::MAX as i64; @@ -189,7 +192,10 @@ ORDER BY t.available_order,t.id LIMIT 100 const TIMER_SCHEDULE_SQL: &str = "SELECT payload FROM workflow_history_events WHERE workflow_run_id=$1 AND event_type='TimerScheduled' AND json_extract(payload,'$.timer_id')=$2 ORDER BY sequence DESC LIMIT 1"; const SIGNAL_RECEIVED_SEQUENCE_SQL: &str = "SELECT sequence FROM workflow_history_events WHERE workflow_run_id=$1 AND event_type='SignalReceived' AND json_extract(payload,'$.signal_id')=$2 ORDER BY sequence LIMIT 1"; const RUN_COMMANDS_SQL: &str = - super::store::run_commands_sql!("substr(c.payload,1,30) COLLATE BINARY='dw-native-external-payload-v1:'"); + super::store::run_commands_sql!( + "substr(s.arguments,1,30) COLLATE BINARY='dw-native-external-payload-v1:'", + "substr(c.payload,1,30) COLLATE BINARY='dw-native-external-payload-v1:'" + ); const HISTORY_PAGE: &str = "SELECT * FROM (SELECT *,SUM(length(CAST(payload AS BLOB))) OVER (ORDER BY sequence) AS page_bytes FROM workflow_history_events WHERE workflow_run_id=$1 AND sequence>$2 ORDER BY sequence LIMIT $3) AS page WHERE page_bytes<=8388608 ORDER BY sequence"; // Normalize only the two supported UTC storage shapes. Removing trailing // fractional zeros preserves exact decimal ordering, including earlier @@ -464,7 +470,10 @@ impl Backend for MySql { const SIGNAL_RECEIVED_SEQUENCE_SQL: &str = "SELECT sequence FROM workflow_history_events WHERE workflow_run_id=? AND event_type='SignalReceived' AND JSON_UNQUOTE(JSON_EXTRACT(payload,'$.signal_id'))=? ORDER BY sequence LIMIT 1"; // The payload column's collation ignores case; compare bytes. const RUN_COMMANDS_SQL: &str = - super::store::run_commands_sql!("CAST(SUBSTRING(c.payload,1,30) AS BINARY)='dw-native-external-payload-v1:'"); + super::store::run_commands_sql!( + "CAST(SUBSTRING(s.arguments,1,30) AS BINARY)='dw-native-external-payload-v1:'", + "CAST(SUBSTRING(c.payload,1,30) AS BINARY)='dw-native-external-payload-v1:'" + ); // First use the preserved run/sequence index to materialize only IDs and // byte counts for this bounded page. Sorting JSON payloads in the window // exhausts MySQL's default sort buffer before the size rejection can run. diff --git a/rust/src/runtime/store.rs b/rust/src/runtime/store.rs index 8b970ff41..8c0133c61 100644 --- a/rust/src/runtime/store.rs +++ b/rust/src/runtime/store.rs @@ -46,15 +46,19 @@ const TASK_SNAPSHOT_EVENTS: &[&str] = &[ "WorkflowFailed", "WorkflowTimedOut", ]; -/// Describe's command journal. `$stored` is each backend's exact, -/// case-sensitive test for native's stored-reference prefix -/// (`external_resolution::STORED_PREFIX`), so a command's payload is read only -/// when it is a stored reference. +/// Describe's command journal. Each backend passes its exact, case-sensitive +/// test for native's stored-reference prefix +/// (`external_resolution::STORED_PREFIX`) over the column that holds a +/// command's input: a signal's in its signal record (`s.arguments`), a start's +/// in the command row (`c.payload`). Other commands keep no stored reference. +/// Only a stored reference is read; inline payloads are not. macro_rules! run_commands_sql { - ($stored:literal) => { + ($signal:literal, $start:literal) => { concat!( - "SELECT c.id,c.command_sequence,c.command_type,c.target_scope,c.requested_workflow_run_id,c.resolved_workflow_run_id,c.source,c.status,c.outcome,c.rejection_reason,c.workflow_type,c.workflow_class,c.accepted_at,c.applied_at,c.rejected_at,c.context,s.signal_name,CASE WHEN ", - $stored, + "SELECT c.id,c.command_sequence,c.command_type,c.target_scope,c.requested_workflow_run_id,c.resolved_workflow_run_id,c.source,c.status,c.outcome,c.rejection_reason,c.workflow_type,c.workflow_class,c.accepted_at,c.applied_at,c.rejected_at,c.context,s.signal_name,CASE WHEN c.command_type='signal' AND ", + $signal, + " THEN s.arguments WHEN c.command_type='start' AND ", + $start, " THEN c.payload END AS stored_payload FROM workflow_commands c LEFT JOIN workflow_signal_records s ON s.workflow_command_id=c.id WHERE c.workflow_run_id=$1 AND c.workflow_instance_id=$2 ORDER BY c.command_sequence,c.created_at,c.id" ) }; diff --git a/rust/src/runtime/store_record_tests.rs b/rust/src/runtime/store_record_tests.rs index 1f312ffeb..9529c9473 100644 --- a/rust/src/runtime/store_record_tests.rs +++ b/rust/src/runtime/store_record_tests.rs @@ -8,16 +8,21 @@ use std::collections::BTreeMap; #[test] fn described_commands_read_only_the_exact_native_stored_prefix() { - // Each backend compares the first 30 characters, byte for byte, with - // the whole prefix. + // Each backend compares the first 30 characters, byte for byte, with the + // whole prefix: a signal's in its signal record, a start's in its command. assert_eq!(STORED_PREFIX.len(), 30); let quoted = format!("'{STORED_PREFIX}'"); - for (backend, sql, comparison) in [ - ("sqlite", ::RUN_COMMANDS_SQL, "substr(c.payload,1,30) COLLATE BINARY="), - ("postgres", ::RUN_COMMANDS_SQL, "substr(c.payload,1,30)="), - ("mysql", ::RUN_COMMANDS_SQL, "CAST(SUBSTRING(c.payload,1,30) AS BINARY)="), + for (backend, sql, signal, start) in [ + ("sqlite", ::RUN_COMMANDS_SQL, + "substr(s.arguments,1,30) COLLATE BINARY=", "substr(c.payload,1,30) COLLATE BINARY="), + ("postgres", ::RUN_COMMANDS_SQL, + "substr(s.arguments,1,30)=", "substr(c.payload,1,30)="), + ("mysql", ::RUN_COMMANDS_SQL, + "CAST(SUBSTRING(s.arguments,1,30) AS BINARY)=", "CAST(SUBSTRING(c.payload,1,30) AS BINARY)="), ] { - assert!(sql.contains(&format!("CASE WHEN {comparison}{quoted} THEN c.payload END")), "{backend}: {sql}"); + let expected = format!("CASE WHEN c.command_type='signal' AND {signal}{quoted} THEN s.arguments \ + WHEN c.command_type='start' AND {start}{quoted} THEN c.payload END"); + assert!(sql.contains(&expected), "{backend}: {sql}"); assert!(!sql.contains("LIKE"), "{backend}: {sql}"); } } @@ -86,8 +91,8 @@ async fn a_task_snapshot_comes_only_from_the_events_own_run() { assert!(recorded_task(&store, &runs[1]).await.is_null()); } -// The query itself, run on rows whose payloads begin like a stored -// reference: only native's exact prefix is selected. +// The query itself, run on rows whose inputs begin like a stored reference: +// only native's exact prefix in the command type's own input column is read. #[tokio::test] async fn described_commands_select_no_inline_payload() { let directory = tempfile::tempdir().unwrap(); @@ -97,21 +102,40 @@ async fn described_commands_select_no_inline_payload() { .unwrap(), ); let exact = format!("{STORED_PREFIX}{{}}"); + let large = format!("dw-{}", "A".repeat(4096)); + let near = format!("dw-native-external-payload-v1{}", "A".repeat(4096)); + let upper = exact.to_uppercase(); + // (id, type, command payload, signal arguments, read) let rows = [ - ("c1", exact.clone(), true), - ("c2", exact.to_uppercase(), false), - ("c3", format!("dw-native-external-payload-v1{}", "A".repeat(4096)), false), - ("c4", format!("dw-{}", "A".repeat(4096)), false), - ("c5", "dw-native".to_owned(), false), + ("c1", "signal", "AA==", Some(exact.as_str()), Some(exact.as_str())), + ("c2", "signal", "AA==", Some(upper.as_str()), None), + ("c3", "signal", "AA==", Some(near.as_str()), None), + ("c4", "signal", "AA==", Some(large.as_str()), None), + // A signal's command row is not its input. + ("c5", "signal", exact.as_str(), Some("AA=="), None), + ("c6", "start", exact.as_str(), None, Some(exact.as_str())), + ("c7", "start", large.as_str(), None, None), + // Other commands keep no stored reference. + ("c8", "update", exact.as_str(), None, None), ]; - for (sequence, (id, payload, _)) in rows.iter().enumerate() { - Store::::query("INSERT INTO workflow_commands(id,workflow_instance_id,workflow_run_id,command_type,status,payload,command_sequence) VALUES ($1,'instance','run','signal','accepted',$2,$3)") + for (sequence, (id, kind, payload, arguments, _)) in rows.iter().enumerate() { + Store::::query("INSERT INTO workflow_commands(id,workflow_instance_id,workflow_run_id,command_type,status,payload,command_sequence) VALUES ($1,'instance','run',$2,'accepted',$3,$4)") .bind(*id) - .bind(payload.as_str()) + .bind(*kind) + .bind(*payload) .bind(sequence as i64 + 1) .execute(&store.pool) .await .unwrap(); + if let Some(arguments) = arguments { + Store::::query("INSERT INTO workflow_signal_records(id,workflow_command_id,signal_name,status,arguments) VALUES ($1,$2,'s','received',$3)") + .bind(format!("s-{id}")) + .bind(*id) + .bind(*arguments) + .execute(&store.pool) + .await + .unwrap(); + } } let selected = Store::::query(::RUN_COMMANDS_SQL) .bind("run") @@ -120,9 +144,9 @@ async fn described_commands_select_no_inline_payload() { .await .unwrap(); assert_eq!(selected.len(), rows.len()); - for (row, (id, payload, read)) in selected.iter().zip(&rows) { + for (row, (id, _, _, _, read)) in selected.iter().zip(&rows) { assert_eq!(::string(row, "id").unwrap(), *id); let stored = ::optional_string(row, "stored_payload").unwrap(); - assert_eq!(stored.as_deref(), read.then_some(payload.as_str()), "{id}"); + assert_eq!(stored.as_deref(), *read, "{id}"); } } diff --git a/rust/tests/execution/external_payloads.rs b/rust/tests/execution/external_payloads.rs index c3771c42e..5acb9e25a 100644 --- a/rust/tests/execution/external_payloads.rs +++ b/rust/tests/execution/external_payloads.rs @@ -133,7 +133,7 @@ async fn unclaimed(database: &TestDatabase) -> i64 { } /// One integer the database answers. -async fn count(database: &TestDatabase, query: &'static str) -> i64 { +async fn count(database: &TestDatabase, query: impl sqlx::SqlSafeStr) -> i64 { match database { TestDatabase::Sqlite(dir) => { let pool = sqlx::SqlitePool::connect(&format!( diff --git a/rust/tests/execution/external_payloads/commands.rs b/rust/tests/execution/external_payloads/commands.rs index bb962993b..e93b63183 100644 --- a/rust/tests/execution/external_payloads/commands.rs +++ b/rust/tests/execution/external_payloads/commands.rs @@ -152,11 +152,11 @@ async fn signal_update_and_query_input_resolve_runtime_references_like_php() { database.remove().await; } -/// The described signal command, which must show only metadata. -async fn described_signal(app: &Router, workflow_id: &str) -> Value { +/// One described command of `kind`, which must show only metadata. +async fn described(app: &Router, workflow_id: &str, kind: &str) -> Value { let (status, body) = json_request(app, "GET", &format!("/api/workflows/{workflow_id}"), "operator-token", Value::Null).await; assert_eq!(status, StatusCode::OK, "{body}"); - let command = body["commands"].as_array().unwrap().iter().find(|command| command["type"] == "signal").unwrap().clone(); + let command = body["commands"].as_array().unwrap().iter().find(|command| command["type"] == kind).unwrap().clone(); for field in ["payload", "external_payload", "external_storage", "reference_id", "sha256", "size_bytes", "arguments"] { assert!(command.get(field).is_none(), "{field}: {command}"); } @@ -166,7 +166,9 @@ async fn described_signal(app: &Router, workflow_id: &str) -> Value { // `WorkflowController::oversizedPayloadPreview` over each command's stored // envelope: only a stored external reference larger than the payload limit -// (2 MiB by default) omits the preview, and then the target name too. +// (2 MiB by default) omits the preview, and then the target name too. Native +// keeps a signal's stored reference in its signal record and a start's in the +// command row. #[tokio::test] async fn described_commands_omit_only_an_oversized_stored_reference() { let temp = tempfile::tempdir().unwrap(); @@ -177,9 +179,15 @@ async fn described_commands_omit_only_an_oversized_stored_reference() { let (status, body) = json_request(&app, "POST", "/api/workflows/described-reference/signal/s", "operator-token", json!({"input":{"codec":"avro","external_payload":reference}})).await; assert_eq!(status, StatusCode::ACCEPTED, "{body}"); - let command = described_signal(&app, "described-reference").await; + let command = described(&app, "described-reference", "signal").await; assert_eq!((command["payload_preview_omitted"].clone(), command["target_name"].clone()), (json!(false), json!("s")), "{command}"); - let command_id = command["id"].as_str().unwrap().to_owned(); + let signal = command["id"].as_str().unwrap().to_owned(); + // The real stored reference is in the signal record, not the command row. + let identity = reference["reference_id"].as_str().unwrap(); + assert_eq!(count(&database, sqlx::AssertSqlSafe(format!( + "SELECT COUNT(*) FROM workflow_signal_records WHERE workflow_command_id='{signal}' AND arguments LIKE 'dw-native-external-payload-v1:%{identity}%'"))).await, 1); + assert_eq!(count(&database, sqlx::AssertSqlSafe(format!( + "SELECT COUNT(*) FROM workflow_commands WHERE id='{signal}' AND payload LIKE 'dw-native%'"))).await, 0); // The same stored reference, as native writes it, at sizes around the // limit, and counterfactuals that are not native stored references. @@ -189,7 +197,7 @@ async fn described_commands_omit_only_an_oversized_stored_reference() { reference["size_bytes"] = json!(size); format!("dw-native-external-payload-v1:{reference}") }; - for (payload, omitted) in [ + let cases = [ (stored(limit), false), (stored(limit + 1), true), // Not native's exact prefix, so never read. @@ -199,11 +207,29 @@ async fn described_commands_omit_only_an_oversized_stored_reference() { (format!("dw-native-external-payload-v1:{{\"size_bytes\":{}}}", limit + 1), false), // A large inline payload. (format!("dw-{}", "A".repeat(limit as usize + 1)), false), - ] { - database.execute(sqlx::AssertSqlSafe(format!("UPDATE workflow_commands SET payload='{payload}' WHERE id='{command_id}'"))).await; - let command = described_signal(&app, "described-reference").await; - assert_eq!(command["payload_preview_omitted"], json!(omitted), "{}: {command}", &payload[..payload.len().min(48)]); - assert_eq!(command.get("target_name").is_none(), omitted, "{command}"); + ]; + for (arguments, omitted) in &cases { + database.execute(sqlx::AssertSqlSafe(format!( + "UPDATE workflow_signal_records SET arguments='{arguments}' WHERE workflow_command_id='{signal}'"))).await; + let command = described(&app, "described-reference", "signal").await; + assert_eq!(command["payload_preview_omitted"], json!(omitted), "{}: {command}", &arguments[..arguments.len().min(48)]); + assert_eq!(command.get("target_name").is_none(), *omitted, "{command}"); + } + // An oversized reference in the signal's command row is not its input. + database.execute(sqlx::AssertSqlSafe(format!( + "UPDATE workflow_signal_records SET arguments='{}' WHERE workflow_command_id='{signal}'", stored(limit)))).await; + database.execute(sqlx::AssertSqlSafe(format!( + "UPDATE workflow_commands SET payload='{}' WHERE id='{signal}'", stored(limit + 1)))).await; + let command = described(&app, "described-reference", "signal").await; + assert_eq!((command["payload_preview_omitted"].clone(), command["target_name"].clone()), (json!(false), json!("s")), "{command}"); + + // A start keeps its stored input reference in the command row. + let start = described(&app, "described-reference", "start").await["id"].as_str().unwrap().to_owned(); + for (payload, omitted) in &cases { + database.execute(sqlx::AssertSqlSafe(format!( + "UPDATE workflow_commands SET payload='{payload}' WHERE id='{start}'"))).await; + let command = described(&app, "described-reference", "start").await; + assert_eq!(command["payload_preview_omitted"], json!(omitted), "start {}: {command}", &payload[..payload.len().min(48)]); } runtime.close().await; database.remove().await; From 17029019b8b38850134c7d1a4503a7a730fb3f71 Mon Sep 17 00:00:00 2001 From: Durable Workflow Date: Sat, 24 Oct 2026 02:19:19 +0000 Subject: [PATCH 09/10] Server parity: name the maintenance container from the job identity (#721) The Docker workflow isolation policy requires a stopped container's name to resolve to github.run_id, run_attempt and job in the same source. Name the SQLite job's maintenance container from GITHUB_RUN_ID, GITHUB_RUN_ATTEMPT and GITHUB_JOB, refuse to act unless it equals the job's $RESOURCE_SCOPE-php-maintenance, and add a control proving a container outside the job scope is never touched. --- .../published-sdk-patch-profile.sh | 4 +- .../published-sdk-patch/maintenance.sh | 46 ++++++++++--------- tests/Unit/PatchCohortMaintenanceTest.mjs | 16 +++++-- 3 files changed, 40 insertions(+), 26 deletions(-) diff --git a/scripts/conformance/server-parity/published-sdk-patch-profile.sh b/scripts/conformance/server-parity/published-sdk-patch-profile.sh index c24c58762..738c21d1d 100644 --- a/scripts/conformance/server-parity/published-sdk-patch-profile.sh +++ b/scripts/conformance/server-parity/published-sdk-patch-profile.sh @@ -42,8 +42,8 @@ record() { --output "/evidence/published-sdk-patch/$2/record.json" } if test "$backend" = sqlite; then - run_without_maintenance "$RESOURCE_SCOPE-php-maintenance" \ - parity-evidence/published-sdk-patch/maintenance-schedule.txt -- record "$php_url" php + run_without_maintenance parity-evidence/published-sdk-patch/maintenance-schedule.txt -- \ + record "$php_url" php else record "$php_url" php fi diff --git a/scripts/conformance/server-parity/published-sdk-patch/maintenance.sh b/scripts/conformance/server-parity/published-sdk-patch/maintenance.sh index 71cdebacb..33528001f 100644 --- a/scripts/conformance/server-parity/published-sdk-patch/maintenance.sh +++ b/scripts/conformance/server-parity/published-sdk-patch/maintenance.sh @@ -1,56 +1,60 @@ # Sourced by published-sdk-patch-profile.sh (Server #721). # -# run_without_maintenance CONTAINER SCHEDULE -- COMMAND... +# run_without_maintenance SCHEDULE -- COMMAND... # -# Runs COMMAND while the maintenance container is stopped, then restarts it. -# Restoration is armed before the stop is attempted and stays armed until the -# container has stayed running for a sustained check, so a failed stop, a -# failed COMMAND or a restart that exits at once never leaves it stopped. -# SCHEDULE records the container's state before, during and after. +# Runs COMMAND while the SQLite job's PHP maintenance container is stopped, +# then restarts it. Restoration is armed before the stop is attempted and +# stays armed until the container has stayed running for a sustained check, +# so a failed stop, a failed COMMAND or a restart that exits at once never +# leaves it stopped. SCHEDULE records the container's state before, during +# and after. maintenance_poll_seconds="${DW_PARITY_MAINTENANCE_POLL_SECONDS:-1}" +# The SQLite job's container, named from this job's identity. +maintenance_container="dw-parity-${GITHUB_RUN_ID:-local}-${GITHUB_RUN_ATTEMPT:-1}-${GITHUB_JOB:-parity}-php-maintenance" maintenance_state() { - docker inspect -f '{{.State.Status}}' "$1" 2>/dev/null || echo unknown + docker inspect -f '{{.State.Status}}' "$maintenance_container" 2>/dev/null || echo unknown } # Running at each of three checks, one poll interval apart. maintenance_stays_running() { local check for check in 1 2 3; do - test "$(maintenance_state "$1")" = running || return 1 + test "$(maintenance_state)" = running || return 1 sleep "$maintenance_poll_seconds" done - test "$(maintenance_state "$1")" = running + test "$(maintenance_state)" = running } # Up to three starts, each followed by the sustained check. maintenance_restore() { local attempt for attempt in 1 2 3; do - docker start "$1" > /dev/null 2>&1 || true - if maintenance_stays_running "$1"; then + docker start "$maintenance_container" >/dev/null 2>&1 || true + if maintenance_stays_running; then return 0 fi done - echo "maintenance container $1 did not stay running: $(maintenance_state "$1")" >&2 + echo "maintenance container $maintenance_container did not stay running: $(maintenance_state)" >&2 return 1 } run_without_maintenance() { - local container="$1" schedule="$2" - shift 2 + local schedule="$1" + shift test "$1" = -- && shift - echo "before: $(maintenance_state "$container")" > "$schedule" + # The job started this container as "$RESOURCE_SCOPE-php-maintenance". + test "$maintenance_container" = "$RESOURCE_SCOPE-php-maintenance" + echo "before: $(maintenance_state)" > "$schedule" grep -qx 'before: running' "$schedule" - # shellcheck disable=SC2064 - trap "maintenance_restore '$container' || true" EXIT - docker stop --time 2 "$container" > /dev/null - echo "during php record: $(maintenance_state "$container")" >> "$schedule" + trap 'maintenance_restore || true' EXIT + docker stop --time 2 "$maintenance_container" >/dev/null + echo "during php record: $(maintenance_state)" >> "$schedule" grep -qx 'during php record: exited' "$schedule" "$@" - if ! maintenance_restore "$container"; then - echo "after php record: $(maintenance_state "$container")" >> "$schedule" + if ! maintenance_restore; then + echo "after php record: $(maintenance_state)" >> "$schedule" return 1 fi trap - EXIT diff --git a/tests/Unit/PatchCohortMaintenanceTest.mjs b/tests/Unit/PatchCohortMaintenanceTest.mjs index 9283d780e..24398e2cc 100644 --- a/tests/Unit/PatchCohortMaintenanceTest.mjs +++ b/tests/Unit/PatchCohortMaintenanceTest.mjs @@ -18,6 +18,7 @@ const fakeDocker = `#!/usr/bin/env bash set -u dir=$(dirname "$0") echo "$*" >> "$dir/calls" +test "\${!#}" = dw-parity-7-2-sqlite-php-maintenance || exit 3 case "$1" in inspect) state=$(cat "$dir/state") @@ -30,17 +31,18 @@ case "$1" in esac `; -function run(scenario, command) { +function run(scenario, command, scope = 'dw-parity-7-2-sqlite') { const dir = mkdtempSync(join(tmpdir(), 'patch-maintenance-')); writeFileSync(join(dir, 'docker'), fakeDocker); chmodSync(join(dir, 'docker'), 0o755); writeFileSync(join(dir, 'state'), 'running\n'); writeFileSync(join(dir, 'calls'), ''); const result = spawnSync('bash', ['-c', - `set -euo pipefail; . "$HELPER"; run_without_maintenance maint "$SCHEDULE" -- ${command}`], { + `set -euo pipefail; . "$HELPER"; run_without_maintenance "$SCHEDULE" -- ${command}`], { encoding: 'utf8', env: {...process.env, PATH: `${dir}:${process.env.PATH}`, HELPER: helper, - SCHEDULE: join(dir, 'schedule.txt'), SCENARIO: scenario, DW_PARITY_MAINTENANCE_POLL_SECONDS: '0.05'}, + SCHEDULE: join(dir, 'schedule.txt'), SCENARIO: scenario, DW_PARITY_MAINTENANCE_POLL_SECONDS: '0.05', + GITHUB_RUN_ID: '7', GITHUB_RUN_ATTEMPT: '2', GITHUB_JOB: 'sqlite', RESOURCE_SCOPE: scope}, }); const read = name => (existsSync(join(dir, name)) ? readFileSync(join(dir, name), 'utf8') : ''); const calls = read('calls').trim().split('\n'); @@ -88,3 +90,11 @@ test('a restart that exits at once stays armed and fails without recording runni assert.equal(result.starts, 6); assert.match(result.stderr, /did not stay running/); }); + +test('a container outside the job scope is never touched', () => { + const result = run('normal', 'true', 'dw-parity-other-job'); + assert.notEqual(result.status, 0); + assert.equal(result.starts, 0); + assert.equal(result.stopIndex, -1); + assert.equal(result.state, 'running'); +}); From 6bde9459cf0a2d03d5e4c3be75dd97ca1d287deb Mon Sep 17 00:00:00 2001 From: Richard McDaniel <1130888+rmcdaniel@users.noreply.github.com> Date: Sat, 24 Oct 2026 08:51:33 +0000 Subject: [PATCH 10/10] Message streams are served: drop them from the unserved route list #709 serves the three message-stream routes, which the route coverage test reported as now served. Native serves 129 of 175 route methods at this tree; 46 remain. --- rust/tests/fixtures/unserved-php-routes.v2.11.1.txt | 3 --- 1 file changed, 3 deletions(-) diff --git a/rust/tests/fixtures/unserved-php-routes.v2.11.1.txt b/rust/tests/fixtures/unserved-php-routes.v2.11.1.txt index 886219a91..5bd1bdd32 100644 --- a/rust/tests/fixtures/unserved-php-routes.v2.11.1.txt +++ b/rust/tests/fixtures/unserved-php-routes.v2.11.1.txt @@ -43,9 +43,6 @@ POST /api/worker/workflow-tasks/{taskId}/local-activities/{attemptId}/heartbeat POST /api/worker/workflow-tasks/{taskId}/local-activities/{attemptId}/outcome POST /api/workflows/import/embedded-v2 POST /api/workflows/import/waterline-v1 -GET /api/workflows/{workflowId}/message-streams -GET /api/workflows/{workflowId}/message-streams/{streamName} -POST /api/workflows/{workflowId}/message-streams/{streamName}/messages GET /api/workflows/{workflowId}/nexus-operations GET /api/workflows/{workflowId}/runs/{runId}/nexus-operations POST /api/workflows/{workflowId}/runs/{runId}/streams/{streamName}/close