Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 3 additions & 1 deletion .github/workflows/server-parity.yml
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,7 @@ on:
- tests/Unit/ServerParityActivityLeaseRenewalContractTest.mjs
- tests/Unit/ServerParityServiceCatalogContractTest.mjs
- tests/Unit/ServerParitySystemMetricsContractTest.mjs
- tests/Unit/ServerParityPrometheusMetricsContractTest.mjs
- tests/Unit/ServerParityNamespaceDeleteContractTest.mjs
- tests/Unit/ServerParityWaitingHistoryContractTest.mjs
- tests/Unit/ServerParityTaskRetryContractTest.mjs
Expand Down Expand Up @@ -88,6 +89,7 @@ on:
- tests/Unit/ServerParityActivityLeaseRenewalContractTest.mjs
- tests/Unit/ServerParityServiceCatalogContractTest.mjs
- tests/Unit/ServerParitySystemMetricsContractTest.mjs
- tests/Unit/ServerParityPrometheusMetricsContractTest.mjs
- tests/Unit/ServerParityNamespaceDeleteContractTest.mjs
- tests/Unit/ServerParityWaitingHistoryContractTest.mjs
- tests/Unit/ServerParityTaskRetryContractTest.mjs
Expand Down Expand Up @@ -1506,7 +1508,7 @@ jobs:
-e COMPOSER_HOME=/tmp/composer -v "$PWD:/source" -w /source "$image" \
install --working-dir=scripts/conformance/server-parity --no-interaction --no-progress
docker run --rm --user=1000:1000 --entrypoint node -v "$PWD:/source:ro" -w /source \
"$image" --test tests/Unit/ServerParityRunnerTest.mjs tests/Unit/ServerParityCooperativeContractTest.mjs tests/Unit/ServerParityChildCancellationContractTest.mjs tests/Unit/ServerParityScheduleContractTest.mjs tests/Unit/ServerParityVisibilityContractTest.mjs tests/Unit/ServerParityAdmissionContractTest.mjs tests/Unit/ServerParityRoleTokenContractTest.mjs tests/Unit/ServerParityWorkerDeregistrationContractTest.mjs tests/Unit/ServerParitySearchAttributeContractTest.mjs tests/Unit/ServerParityWorkerListingContractTest.mjs tests/Unit/ServerParityWorkerReplacementContractTest.mjs tests/Unit/ServerParityPatchDeploymentContractTest.mjs tests/Unit/ServerParityWaitingHistoryContractTest.mjs tests/Unit/ServerParityTaskRetryContractTest.mjs tests/Unit/ServerParityPublishedPatchContractTest.mjs tests/Unit/ServerParityPublishedPatchTransportTest.mjs tests/Unit/ServerParityRecordFileTest.mjs tests/Unit/ServerParityLateWorkflowCompletionContractTest.mjs tests/Unit/ServerParityStorageTestContractTest.mjs tests/Unit/ServerParitySchedulePoliciesContractTest.mjs tests/Unit/ServerParitySignalsBeforeWorkerContractTest.mjs tests/Unit/ServerParityScheduleCatalogContractTest.mjs tests/Unit/ServerParityOperatorMetricsContractTest.mjs tests/Unit/ServerParityActivityLeaseRenewalContractTest.mjs tests/Unit/ServerParityServiceCatalogContractTest.mjs tests/Unit/ServerParitySystemMetricsContractTest.mjs tests/Unit/ServerParityNamespaceDeleteContractTest.mjs > parity-evidence/comparator-tests.txt
"$image" --test tests/Unit/ServerParityRunnerTest.mjs tests/Unit/ServerParityCooperativeContractTest.mjs tests/Unit/ServerParityChildCancellationContractTest.mjs tests/Unit/ServerParityScheduleContractTest.mjs tests/Unit/ServerParityVisibilityContractTest.mjs tests/Unit/ServerParityAdmissionContractTest.mjs tests/Unit/ServerParityRoleTokenContractTest.mjs tests/Unit/ServerParityWorkerDeregistrationContractTest.mjs tests/Unit/ServerParitySearchAttributeContractTest.mjs tests/Unit/ServerParityWorkerListingContractTest.mjs tests/Unit/ServerParityWorkerReplacementContractTest.mjs tests/Unit/ServerParityPatchDeploymentContractTest.mjs tests/Unit/ServerParityWaitingHistoryContractTest.mjs tests/Unit/ServerParityTaskRetryContractTest.mjs tests/Unit/ServerParityPublishedPatchContractTest.mjs tests/Unit/ServerParityPublishedPatchTransportTest.mjs tests/Unit/ServerParityRecordFileTest.mjs tests/Unit/ServerParityLateWorkflowCompletionContractTest.mjs tests/Unit/ServerParityStorageTestContractTest.mjs tests/Unit/ServerParitySchedulePoliciesContractTest.mjs tests/Unit/ServerParitySignalsBeforeWorkerContractTest.mjs tests/Unit/ServerParityScheduleCatalogContractTest.mjs tests/Unit/ServerParityOperatorMetricsContractTest.mjs tests/Unit/ServerParityActivityLeaseRenewalContractTest.mjs tests/Unit/ServerParityServiceCatalogContractTest.mjs tests/Unit/ServerParitySystemMetricsContractTest.mjs tests/Unit/ServerParityPrometheusMetricsContractTest.mjs tests/Unit/ServerParityNamespaceDeleteContractTest.mjs > parity-evidence/comparator-tests.txt
docker run --rm --user=1000:1000 --cpus=1 --memory=256m --memory-swap=256m \
--network=none --entrypoint node -v "$PWD:/source:ro" -w /source \
"$image" --test tests/Unit/ServerParityCodecDeadlineLoadTest.mjs > parity-evidence/codec-load-tests.txt
Expand Down
58 changes: 58 additions & 0 deletions docs/rust-server-port-checkpoints/2026-10-22-prometheus-metrics.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,58 @@
# October 22, Prometheus metrics summary

- **Source.** Server 2.11.1 `SystemController::prometheusMetrics` and
`PrometheusMetricsSummary::snapshot`, with `BoundedMetricPolicy` and the
declared dimensions in `config/dw-bounded-growth.php`.
- **Served.** Native answers admin-only `GET /api/system/prometheus-metrics`
in the default or a named namespace:
- workflow series by task queue and workflow type: started, completed,
failed, cancelled and terminated totals, running runs and a latency
histogram from run summaries;
- activity series by task queue, workflow type and activity type, with the
same counters and a latency histogram from activity executions;
- task-queue runtime state: ready, due, delayed, leased and expired tasks,
tasks added and dispatched in the last minute, and fresh worker pollers,
slot capacity and available slots.

Each family keeps the first 100 series in PHP's order. Native reads at
most 101 candidate label sets, then aggregates only the series up to the
last candidate. The disclosure reports observed, reported and suppressed
series and the suppressed starts, with `lower_bound` precision once the
limit is reached. Native refuses the series-limit settings, so the limit is
always PHP's default of 100.
- **Times.** Task and worker comparisons bind whole seconds, as Laravel does
and as native's operator metrics already do. A task made available within
the current second still counts as delayed. Workers are fresh within
`DW_WORKER_STALE_AFTER_SECONDS`, by default `max(3 × heartbeat, 30)`.
- **Decision: cumulative latency buckets.** Published PHP binds each bucket
bound as text. SQLite orders every number before every text value, so PHP
on SQLite counts every duration in every bucket
([server#683](https://github.com/durable-workflow/server/issues/683)).
Native compares durations with numeric bounds and reports the cumulative
histogram on every backend. The fixture accepts PHP's SQLite shape as a
recorded reference defect; native's tests pin the cumulative buckets with
fixed durations on SQLite, PostgreSQL and MySQL.
- **Correction: undispatched activity tasks.** PHP's worker completion
publishes only the timer tasks it creates. An activity task scheduled by a
workflow task therefore keeps null `last_dispatched_at` and
`last_dispatch_attempt_at` until a poller claims it, and its queue's
dispatched-in-the-last-minute counter stays 0. Native stamped it
dispatched. It now leaves those columns null for that path, as PHP does.
Other task paths keep their stamps.
- **Fixture.** `prometheus-metrics` reads a fresh namespace, then:
- registers a worker with four workflow and three activity slots on the
first queue;
- completes a run there after one activity;
- terminates a second run there;
- starts a run on a second queue that has no worker.

Counters, task-queue state, worker capacity and the disclosure are exact.
Each counted run and activity lasts at least 1.1 seconds, so latency sums
and the empty low buckets are checked against that bound.
- **Evidence.** Native tests pass on SQLite, PostgreSQL and MySQL, and the
native suite with `--all-targets` passes on SQLite (649 tests).
A local copy of the SQLite parity step passed all 53 fixtures on embedded
PHP and native, including `prometheus-metrics`. Published PHP passed
`prometheus-metrics` too; its only difference was `storage-test`, an
intermittent 503 from PHP's external payload registration on that loaded
host, which passed in the previous local run.
19 changes: 11 additions & 8 deletions docs/rust-server-port.md
Original file line number Diff line number Diff line change
Expand Up @@ -102,14 +102,17 @@ These published PHP routes on this tree have no native handler yet, by family.

- **Used by the CLI or Waterline.**
- Standalone activities: list, describe and start.
- The Prometheus metrics summary, the repair pass, and external payload
cleanup and its pass. (Namespace delete is served with the
`namespace-delete` fixture. The JSON `GET|POST /api/system/metrics` is
served with the `system-metrics` fixture: namespace failure, durable-state
and payload counts are exact; native enforces no admission, durable-state
or payload quota, so limits are null and rejections zero, as on an
unconfigured PHP server. Projection drift is server-wide, and native's
only drift source is a run without its summary.)
- The repair pass, and external payload cleanup and its pass. (Namespace
delete is served with the `namespace-delete` fixture. The JSON
`GET|POST /api/system/metrics` is served with the `system-metrics`
fixture: namespace failure, durable-state and payload counts are exact;
native enforces no admission, durable-state or payload quota, so limits
are null and rejections zero, as on an unconfigured PHP server.
Projection drift is server-wide, and native's only drift source is a run
without its summary. The Prometheus summary
`GET /api/system/prometheus-metrics` is served with the
`prometheus-metrics` fixture, with PHP's default limit of 100 series per
family.)
- Task queue priority and fairness.
- **Worker protocol features.**
- Worker sessions.
Expand Down
1 change: 1 addition & 0 deletions rust/src/runtime/execution.rs
Original file line number Diff line number Diff line change
Expand Up @@ -340,6 +340,7 @@ impl Runtime {
delegate!(operator_dashboard(config: &super::operator_metrics::MetricsConfig, bounded: bool) -> Result<Value>);
delegate!(repair_status() -> Result<Value>);
delegate!(system_metrics() -> Result<Value>);
delegate!(prometheus_metrics() -> Result<Value>);
delegate!(retention_status(limit: i64) -> Result<Value>);
delegate!(retention_pass(limit: i64, run_ids: Vec<String>) -> Result<(Value, bool)>);
delegate!(activity_timeout_status(limit: i64) -> Result<Value>);
Expand Down
8 changes: 8 additions & 0 deletions rust/src/runtime/http.rs
Original file line number Diff line number Diff line change
Expand Up @@ -204,6 +204,7 @@ pub fn router(runtime: Runtime) -> Router {
.route("/api/system/operator-metrics", get(operator_metrics))
.route("/api/system/health", get(system_health))
.route("/api/system/metrics", get(system_metrics).post(system_metrics))
.route("/api/system/prometheus-metrics", get(prometheus_metrics))
.route("/api/system/operator-dashboard", get(operator_dashboard))
.route("/api/system/operator-dashboard/bounded", get(bounded_operator_dashboard))
.route(
Expand Down Expand Up @@ -650,6 +651,9 @@ fn named_namespace_operation(method: &str, path: &str) -> bool {
if path == "/api/system/metrics" {
return matches!(method, "GET" | "POST");
}
if path == "/api/system/prometheus-metrics" {
return method == "GET";
}
if let Some(path) = path.strip_prefix("/api/workflows/") {
let parts: Vec<_> = path.split('/').collect();
if matches!(
Expand Down Expand Up @@ -1482,6 +1486,10 @@ async fn system_metrics(Extension(runtime): Extension<Runtime>) -> Result<Json<V
Ok(Json(runtime.system_metrics().await?))
}

async fn prometheus_metrics(Extension(runtime): Extension<Runtime>) -> Result<Json<Value>> {
Ok(Json(runtime.prometheus_metrics().await?))
}

/// PHP answers 503 while a check reports an error.
async fn system_health(Extension(runtime): Extension<Runtime>) -> Result<Response> {
let (unavailable, body) = runtime.system_health(&metrics_config(&runtime)).await?;
Expand Down
1 change: 1 addition & 0 deletions rust/src/runtime/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -73,6 +73,7 @@ mod payload_limits;
mod poll_readiness;
pub mod postgres;
mod previews;
mod prometheus_metrics;
mod queries;
mod query_authority;
mod repair_candidates;
Expand Down
2 changes: 1 addition & 1 deletion rust/src/runtime/operator_metrics.rs
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,7 @@ use sqlx::{ColumnIndex, Decode, Encode, Executor, IntoArguments, Type};
pub(super) const REDISPATCH_AFTER_SECONDS: i64 = 3;

/// Laravel binds a bare timestamp as `Y-m-d H:i:s`.
fn whole_second(at: DateTime<Utc>) -> DateTime<Utc> {
pub(super) fn whole_second(at: DateTime<Utc>) -> DateTime<Utc> {
at.with_nanosecond(0).unwrap_or(at)
}

Expand Down
Loading
Loading