From 32fe70df8a4f52bf07189a51c481411551d3bde3 Mon Sep 17 00:00:00 2001 From: Matt Aitken Date: Sat, 29 Aug 2026 13:27:01 +0100 Subject: [PATCH 01/32] feat(webapp,run-engine,core,clickhouse): surface total concurrency in metrics and dashboard Queues with a totalConcurrencyLimit now report how they use it. The gauge pipeline emits total running and the stored cap, ClickHouse aggregates them into the queue metrics tiers, the queues list gets a Total column, the queue detail page charts total running against the cap, and the per-key table shows each key's effective limit including per-key overrides. Queue retrieve and list API responses include the same totals. --- .changeset/queue-total-concurrency-stats.md | 5 + .../v3/QueueListPresenter.server.ts | 25 +++- .../v3/QueueRetrievePresenter.server.ts | 20 ++++ .../route.tsx | 32 ++++- .../route.tsx | 69 ++++++++++- ...ueueParam.concurrency.combined.override.ts | 3 + ....$queueParam.concurrency.combined.reset.ts | 3 + ...queues.$queueParam.concurrency.override.ts | 3 + ...v1.queues.$queueParam.concurrency.reset.ts | 3 + .../resources.queues.concurrency-keys.ts | 10 +- apps/webapp/app/v3/querySchemas.ts | 24 ++++ apps/webapp/app/v3/queueMetricsMapping.ts | 2 + ...42_add_queue_metrics_total_concurrency.sql | 109 ++++++++++++++++++ .../clickhouse/src/queueMetrics.ts | 2 + internal-packages/metrics-pipeline/src/lua.ts | 18 ++- .../run-engine/src/engine/index.ts | 14 +++ .../run-engine/src/run-queue/index.ts | 49 +++++++- packages/core/src/v3/schemas/queues.ts | 15 +++ 18 files changed, 394 insertions(+), 12 deletions(-) create mode 100644 .changeset/queue-total-concurrency-stats.md create mode 100644 internal-packages/clickhouse/schema/042_add_queue_metrics_total_concurrency.sql diff --git a/.changeset/queue-total-concurrency-stats.md b/.changeset/queue-total-concurrency-stats.md new file mode 100644 index 00000000000..a70da24d1fb --- /dev/null +++ b/.changeset/queue-total-concurrency-stats.md @@ -0,0 +1,5 @@ +--- +"@trigger.dev/core": patch +--- + +Queue retrieve and list API responses now report total concurrency usage. When a queue has a `totalConcurrencyLimit`, `concurrency.total` includes the effective cap, the declared base, any active override, and how many runs are in flight across all concurrency keys. diff --git a/apps/webapp/app/presenters/v3/QueueListPresenter.server.ts b/apps/webapp/app/presenters/v3/QueueListPresenter.server.ts index 0dc3daa9856..d78e98ade4c 100644 --- a/apps/webapp/app/presenters/v3/QueueListPresenter.server.ts +++ b/apps/webapp/app/presenters/v3/QueueListPresenter.server.ts @@ -9,7 +9,10 @@ import { engine } from "~/v3/runEngine.server"; import { BasePresenter } from "./basePresenter.server"; import { toQueueItem } from "./QueueRetrievePresenter.server"; -type QueueListEngine = Pick; +type QueueListEngine = Pick< + RunEngine, + "lengthOfQueues" | "currentConcurrencyOfQueues" | "totalConcurrencyOfQueues" +>; export const QUEUE_LIST_DEFAULT_ITEMS_PER_PAGE = 25; const MAX_ITEMS_PER_PAGE = 100; @@ -34,6 +37,9 @@ const queueListSelect = { concurrencyLimitOverriddenAt: true, concurrencyLimitOverriddenBy: true, concurrencyLimitOverridePercent: true, + totalConcurrencyLimit: true, + totalConcurrencyLimitBase: true, + totalConcurrencyLimitOverriddenAt: true, type: true, paused: true, } satisfies Prisma.TaskQueueSelect; @@ -333,11 +339,15 @@ export class QueueListPresenter extends BasePresenter { concurrencyLimitOverriddenAt: Date | null; concurrencyLimitOverriddenBy: string | null; concurrencyLimitOverridePercent: Prisma.Decimal | null; + totalConcurrencyLimit: number | null; + totalConcurrencyLimitBase: number | null; + totalConcurrencyLimitOverriddenAt: Date | null; type: TaskQueueType; paused: boolean; }[] ): Promise { - const [queuedByQueue, runningByQueue] = await Promise.all([ + const queuesWithTotalCap = queues.filter((q) => q.totalConcurrencyLimit !== null); + const [queuedByQueue, runningByQueue, totalRunningByQueue] = await Promise.all([ this.engineClient.lengthOfQueues( environment, queues.map((q) => q.name) @@ -346,6 +356,12 @@ export class QueueListPresenter extends BasePresenter { environment, queues.map((q) => q.name) ), + queuesWithTotalCap.length > 0 + ? this.engineClient.totalConcurrencyOfQueues( + environment, + queuesWithTotalCap.map((q) => q.name) + ) + : Promise.resolve({} as Record), ]); // Manually "join" the overridden users because there is no way to implement the relationship @@ -373,6 +389,11 @@ export class QueueListPresenter extends BasePresenter { ? (overriddenByMap.get(queue.concurrencyLimitOverriddenBy) ?? null) : null, paused: queue.paused, + totalConcurrencyLimit: queue.totalConcurrencyLimit, + totalConcurrencyLimitBase: queue.totalConcurrencyLimitBase, + totalConcurrencyLimitOverriddenAt: queue.totalConcurrencyLimitOverriddenAt, + totalRunning: + queue.totalConcurrencyLimit !== null ? (totalRunningByQueue[queue.name] ?? 0) : null, }), // Prisma returns Decimal; the client only needs a plain number (null for absolute overrides). concurrencyLimitOverridePercent: diff --git a/apps/webapp/app/presenters/v3/QueueRetrievePresenter.server.ts b/apps/webapp/app/presenters/v3/QueueRetrievePresenter.server.ts index f6918394e5c..e4777ceb13e 100644 --- a/apps/webapp/app/presenters/v3/QueueRetrievePresenter.server.ts +++ b/apps/webapp/app/presenters/v3/QueueRetrievePresenter.server.ts @@ -90,6 +90,7 @@ export class QueueRetrievePresenter extends BasePresenter { const results = await Promise.all([ engine.lengthOfQueues(environment, [queue.name]), engine.currentConcurrencyOfQueues(environment, [queue.name]), + engine.totalConcurrencyOfQueues(environment, [queue.name]), ]); // Transform queues to include running and queued counts @@ -107,6 +108,11 @@ export class QueueRetrievePresenter extends BasePresenter { concurrencyLimitOverriddenAt: queue.concurrencyLimitOverriddenAt ?? null, concurrencyLimitOverriddenBy: queue.concurrencyLimitOverriddenBy ?? null, paused: queue.paused, + totalConcurrencyLimit: queue.totalConcurrencyLimit ?? null, + totalConcurrencyLimitBase: queue.totalConcurrencyLimitBase ?? null, + totalConcurrencyLimitOverriddenAt: queue.totalConcurrencyLimitOverriddenAt ?? null, + totalRunning: + queue.totalConcurrencyLimit != null ? (results[2]?.[queue.name] ?? 0) : null, }), // The percent source-of-truth for percent-based overrides isn't part of the shared // `QueueItem` schema (that's a public contract), so we surface it as an extra field on @@ -148,6 +154,10 @@ export function toQueueItem(data: { concurrencyLimitOverriddenAt: Date | null; concurrencyLimitOverriddenBy: User | null; paused: boolean; + totalConcurrencyLimit?: number | null; + totalConcurrencyLimitBase?: number | null; + totalConcurrencyLimitOverriddenAt?: Date | null; + totalRunning?: number | null; }): QueueItem & { releaseConcurrencyOnWaitpoint: boolean } { return { id: data.friendlyId, @@ -164,6 +174,16 @@ export function toQueueItem(data: { override: data.concurrencyLimitOverriddenAt ? data.concurrencyLimit : null, overriddenBy: toQueueConcurrencyOverriddenBy(data.concurrencyLimitOverriddenBy), overriddenAt: data.concurrencyLimitOverriddenAt, + total: + data.totalConcurrencyLimit !== undefined + ? { + current: data.totalConcurrencyLimit, + base: data.totalConcurrencyLimitBase ?? null, + override: data.totalConcurrencyLimitOverriddenAt ? data.totalConcurrencyLimit : null, + overriddenAt: data.totalConcurrencyLimitOverriddenAt ?? null, + running: data.totalRunning ?? null, + } + : undefined, }, // TODO: This needs to be removed but keeping this here for now to avoid breaking existing clients releaseConcurrencyOnWaitpoint: true, diff --git a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.queues/route.tsx b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.queues/route.tsx index 475f25362dc..5ec0ada0371 100644 --- a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.queues/route.tsx +++ b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.queues/route.tsx @@ -705,6 +705,13 @@ function QueuesWithMetricsView() { Queued Running Limit + + Total + + = + Math.min( + queue.concurrency.total.current, + environment.concurrencyLimit + ) && + "text-warning" + )} + > + {queue.concurrency?.total?.current != null + ? `${queue.concurrency.total.running ?? 0}/${Math.min( + queue.concurrency.total.current, + environment.concurrencyLimit + )}` + : "–"} + - +
{hasFilters ? "No queues found matching your filters" : "No queues found"} diff --git a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.queues_.$queueParam/route.tsx b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.queues_.$queueParam/route.tsx index 7ddcdb1d607..ff4fc786bc5 100644 --- a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.queues_.$queueParam/route.tsx +++ b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.queues_.$queueParam/route.tsx @@ -392,7 +392,12 @@ export default function Page() { ) ) : ( - + )} @@ -402,7 +407,13 @@ export default function Page() { {view === "keys" && hasKeys ? ( <> - + {selectedKey ? ( @@ -436,10 +447,12 @@ function OverviewCharts({ ids, timeRange, queueName, + hasTotalLimit, }: { ids: Ids; timeRange: TimeRangeParams; queueName: string; + hasTotalLimit: boolean; }) { const zoomToTimeFilter = useZoomToTimeFilter(); return ( @@ -479,6 +492,37 @@ function OverviewCharts({ // leading zeros so the reference line doesn't start with a false 0→limit step. carryBackfill={["limit"]} /> + {hasTotalLimit ? ( + + Runs in flight across ALL concurrency keys ( + ) versus the queue's total limit ( + + ). + + } + showLegend + className="aspect-[2/1]" + query={`SELECT timeBucket() AS t, max(max_total_running) AS running, least(max(max_total_limit), max(max_env_limit)) AS cap\nFROM queue_metrics\nGROUP BY t\nORDER BY t`} + fillGaps + minBucketSeconds={SYNCED_CHART_MIN_BUCKET_SECONDS} + ids={ids} + timeRange={timeRange} + queueName={queueName} + series={[ + { key: "cap", label: "Total limit", color: COLORS.limit }, + { key: "running", label: "Running", color: COLORS.running }, + ]} + thresholdStroke={{ + series: "running", + valueFromSeries: "cap", + aboveColor: "var(--color-warning)", + }} + carryBackfill={["cap"]} + /> + ) : null} Key Queued now Running now + + Limit + Oldest wait Started Peak backlog @@ -976,11 +1031,11 @@ function KeyStatsTable({ {showLoading ? ( - + Loading… ) : rows.length === 0 ? ( - + {search ? `No keys match “${search}”` : "No concurrency keys"} ) : ( @@ -994,6 +1049,12 @@ function KeyStatsTable({ {row.key} {row.queued.toLocaleString()} {row.running.toLocaleString()} + + {Math.min(row.limitOverride ?? defaultKeyLimit, envLimit).toLocaleString()} + {row.oldestWaitMs === null ? "–" : formatWaitMs(row.oldestWaitMs)} diff --git a/apps/webapp/app/routes/api.v1.queues.$queueParam.concurrency.combined.override.ts b/apps/webapp/app/routes/api.v1.queues.$queueParam.concurrency.combined.override.ts index c643b77965a..77688a9fcc5 100644 --- a/apps/webapp/app/routes/api.v1.queues.$queueParam.concurrency.combined.override.ts +++ b/apps/webapp/app/routes/api.v1.queues.$queueParam.concurrency.combined.override.ts @@ -46,6 +46,9 @@ const route = createActionApiRoute( concurrencyLimitOverriddenAt: queue.concurrencyLimitOverriddenAt, concurrencyLimitOverriddenBy: null, paused: queue.paused, + totalConcurrencyLimit: queue.totalConcurrencyLimit, + totalConcurrencyLimitBase: queue.totalConcurrencyLimitBase, + totalConcurrencyLimitOverriddenAt: queue.totalConcurrencyLimitOverriddenAt, }), { status: 200 } ); diff --git a/apps/webapp/app/routes/api.v1.queues.$queueParam.concurrency.combined.reset.ts b/apps/webapp/app/routes/api.v1.queues.$queueParam.concurrency.combined.reset.ts index b2841f1efe6..0e588716658 100644 --- a/apps/webapp/app/routes/api.v1.queues.$queueParam.concurrency.combined.reset.ts +++ b/apps/webapp/app/routes/api.v1.queues.$queueParam.concurrency.combined.reset.ts @@ -45,6 +45,9 @@ const route = createActionApiRoute( concurrencyLimitOverriddenAt: queue.concurrencyLimitOverriddenAt, concurrencyLimitOverriddenBy: null, paused: queue.paused, + totalConcurrencyLimit: queue.totalConcurrencyLimit, + totalConcurrencyLimitBase: queue.totalConcurrencyLimitBase, + totalConcurrencyLimitOverriddenAt: queue.totalConcurrencyLimitOverriddenAt, }), { status: 200 } ); diff --git a/apps/webapp/app/routes/api.v1.queues.$queueParam.concurrency.override.ts b/apps/webapp/app/routes/api.v1.queues.$queueParam.concurrency.override.ts index 90f5772c5d3..42bb2008682 100644 --- a/apps/webapp/app/routes/api.v1.queues.$queueParam.concurrency.override.ts +++ b/apps/webapp/app/routes/api.v1.queues.$queueParam.concurrency.override.ts @@ -61,6 +61,9 @@ const route = createActionApiRoute( concurrencyLimitOverriddenAt: queue.concurrencyLimitOverriddenAt, concurrencyLimitOverriddenBy: null, paused: queue.paused, + totalConcurrencyLimit: queue.totalConcurrencyLimit, + totalConcurrencyLimitBase: queue.totalConcurrencyLimitBase, + totalConcurrencyLimitOverriddenAt: queue.totalConcurrencyLimitOverriddenAt, }), { status: 200 } ); diff --git a/apps/webapp/app/routes/api.v1.queues.$queueParam.concurrency.reset.ts b/apps/webapp/app/routes/api.v1.queues.$queueParam.concurrency.reset.ts index 503d875e471..3f36e629f09 100644 --- a/apps/webapp/app/routes/api.v1.queues.$queueParam.concurrency.reset.ts +++ b/apps/webapp/app/routes/api.v1.queues.$queueParam.concurrency.reset.ts @@ -43,6 +43,9 @@ const route = createActionApiRoute( concurrencyLimitOverriddenAt: queue.concurrencyLimitOverriddenAt, concurrencyLimitOverriddenBy: null, paused: queue.paused, + totalConcurrencyLimit: queue.totalConcurrencyLimit, + totalConcurrencyLimitBase: queue.totalConcurrencyLimitBase, + totalConcurrencyLimitOverriddenAt: queue.totalConcurrencyLimitOverriddenAt, }), { status: 200 } ); diff --git a/apps/webapp/app/routes/resources.queues.concurrency-keys.ts b/apps/webapp/app/routes/resources.queues.concurrency-keys.ts index 67c2b9f500a..662013694ef 100644 --- a/apps/webapp/app/routes/resources.queues.concurrency-keys.ts +++ b/apps/webapp/app/routes/resources.queues.concurrency-keys.ts @@ -43,6 +43,8 @@ export type ConcurrencyKeyRow = { peakBacklog: number; peakRunning: number; meanWaitMs: number; + /** Per-key concurrency limit override, when one is set for this key (null = inherits the queue limit). */ + limitOverride: number | null; }; export type ConcurrencyKeysResponse = @@ -151,8 +153,11 @@ export const action = async ({ request }: ActionFunctionArgs) => { const total = rankingRows?.[0]?.ranked_total ?? 0; const keys = (rankingRows ?? []).map((r) => r.concurrency_key); - // Enrich just this page's keys with live "now" counts from Redis. - const live = await engine.concurrencyKeyLiveStats(environment, queueName, keys); + // Enrich just this page's keys with live "now" counts and any per-key limit overrides from Redis. + const [live, keyLimitOverrides] = await Promise.all([ + engine.concurrencyKeyLiveStats(environment, queueName, keys), + engine.runQueue.getQueueConcurrencyKeyLimits(environment, queueName), + ]); const loadedAt = Date.now(); const rows: ConcurrencyKeyRow[] = (rankingRows ?? []).map((r) => { @@ -168,6 +173,7 @@ export const action = async ({ request }: ActionFunctionArgs) => { peakBacklog: r.peak_backlog, peakRunning: r.peak_running, meanWaitMs: r.mean_wait_ms, + limitOverride: keyLimitOverrides[r.concurrency_key] ?? null, }; }); diff --git a/apps/webapp/app/v3/querySchemas.ts b/apps/webapp/app/v3/querySchemas.ts index 690bbaf5396..8267a0f8020 100644 --- a/apps/webapp/app/v3/querySchemas.ts +++ b/apps/webapp/app/v3/querySchemas.ts @@ -770,6 +770,22 @@ const queueMetricsSchema: TableSchema = { fillMode: "carry", }), }, + max_total_running: { + name: "max_total_running", + ...column("UInt32", { + description: + "Peak in-flight runs across ALL concurrency keys of the queue in the bucket (only emitted for keyed queues). Aggregate with max().", + fillMode: "carry", + }), + }, + max_total_limit: { + name: "max_total_limit", + ...column("UInt32", { + description: + "The queue's total concurrency limit across all keys, as stored (0 = no cap; clamp against max_env_limit). Aggregate with max().", + fillMode: "carry", + }), + }, max_ck_backlogged: { name: "max_ck_backlogged", ...column("UInt32", { @@ -1406,6 +1422,14 @@ const queueMetricsByKeySchema: TableSchema = { fillMode: "carry", }), }, + max_limit: { + name: "max_limit", + ...column("UInt32", { + description: + "The effective concurrency limit for this key (the queue limit, or its per-key override). Aggregate with max().", + fillMode: "carry", + }), + }, wait_ms_sum: { name: "wait_ms_sum", ...column("UInt64", { diff --git a/apps/webapp/app/v3/queueMetricsMapping.ts b/apps/webapp/app/v3/queueMetricsMapping.ts index 9433b361a88..d341f4a63cd 100644 --- a/apps/webapp/app/v3/queueMetricsMapping.ts +++ b/apps/webapp/app/v3/queueMetricsMapping.ts @@ -131,6 +131,8 @@ export function mapEntryToRows( throttled: num(f.thr), ck_backlogged: num(f.ckq), ck_max_wait_ms: num(f.ckw), + total_running: num(f.tcc), + total_limit: num(f.tlim), }, ]; } diff --git a/internal-packages/clickhouse/schema/042_add_queue_metrics_total_concurrency.sql b/internal-packages/clickhouse/schema/042_add_queue_metrics_total_concurrency.sql new file mode 100644 index 00000000000..7711effa6e3 --- /dev/null +++ b/internal-packages/clickhouse/schema/042_add_queue_metrics_total_concurrency.sql @@ -0,0 +1,109 @@ +-- +goose Up + +-- Total-concurrency gauges: total_running is the in-flight count across ALL +-- concurrency-key variants of a queue (the groupConcurrency set), total_limit the +-- RAW stored total cap (0 = none; readers clamp against max_env_limit). Emitted on +-- base-queue gauge rows only. Per-key gauge rows now carry the EFFECTIVE per-key +-- limit in queue_limit (override-aware), surfaced in the ck tier as max_limit. + +ALTER TABLE trigger_dev.queue_metrics_raw_v1 + ADD COLUMN IF NOT EXISTS total_running UInt32 DEFAULT 0, + ADD COLUMN IF NOT EXISTS total_limit UInt32 DEFAULT 0; + +ALTER TABLE trigger_dev.queue_metrics_v1 + ADD COLUMN IF NOT EXISTS max_total_running SimpleAggregateFunction(max, UInt32), + ADD COLUMN IF NOT EXISTS max_total_limit SimpleAggregateFunction(max, UInt32); + +ALTER TABLE trigger_dev.queue_metrics_5m_v1 + ADD COLUMN IF NOT EXISTS max_total_running SimpleAggregateFunction(max, UInt32), + ADD COLUMN IF NOT EXISTS max_total_limit SimpleAggregateFunction(max, UInt32); + +ALTER TABLE trigger_dev.queue_metrics_ck_v1 + ADD COLUMN IF NOT EXISTS max_limit SimpleAggregateFunction(max, UInt32); + +-- Materialized views cannot be altered: recreate them with the new columns. The 5m +-- MV MUST keep reading raw, never cascade off queue_metrics_v1 (out-of-time-order +-- deltaSumTimestamp merges double-count bridging spans). + +DROP VIEW IF EXISTS trigger_dev.queue_metrics_mv_v1; +CREATE MATERIALIZED VIEW IF NOT EXISTS trigger_dev.queue_metrics_mv_v1 +TO trigger_dev.queue_metrics_v1 AS +SELECT + organization_id, project_id, environment_id, queue_name, + toStartOfInterval(event_time, INTERVAL 10 SECOND) AS bucket_start, + deltaSumTimestampStateIf(cumulative, order_key, op = 'enqueue' AND concurrency_key = '') AS enqueue_delta, + deltaSumTimestampStateIf(cumulative, order_key, op = 'started' AND concurrency_key = '') AS started_delta, + deltaSumTimestampStateIf(cumulative, order_key, op = 'ack' AND concurrency_key = '') AS ack_delta, + deltaSumTimestampStateIf(cumulative, order_key, op = 'nack' AND concurrency_key = '') AS nack_delta, + deltaSumTimestampStateIf(cumulative, order_key, op = 'dlq' AND concurrency_key = '') AS dlq_delta, + sum(throttled) AS throttled_count, + max(queued) AS max_queued, + max(running) AS max_running, + max(queue_limit) AS max_limit, + max(env_queued) AS max_env_queued, + max(env_running) AS max_env_running, + max(env_limit) AS max_env_limit, + max(ck_backlogged) AS max_ck_backlogged, + max(ck_max_wait_ms) AS max_ck_wait_ms, + max(total_running) AS max_total_running, + max(total_limit) AS max_total_limit, + sumIf(wait_ms, op = 'started' AND concurrency_key = '') AS wait_ms_sum, + countIf(op = 'started' AND wait_ms > 0 AND concurrency_key = '') AS wait_ms_count, + quantilesStateIf(0.5, 0.9, 0.95, 0.99)(wait_ms, op = 'started' AND wait_ms > 0 AND concurrency_key = '') AS wait_quantiles +FROM trigger_dev.queue_metrics_raw_v1 +GROUP BY organization_id, project_id, environment_id, queue_name, bucket_start; + +DROP VIEW IF EXISTS trigger_dev.queue_metrics_5m_mv_v1; +CREATE MATERIALIZED VIEW IF NOT EXISTS trigger_dev.queue_metrics_5m_mv_v1 +TO trigger_dev.queue_metrics_5m_v1 AS +SELECT + organization_id, project_id, environment_id, queue_name, + toStartOfInterval(event_time, INTERVAL 5 MINUTE) AS bucket_start, + deltaSumTimestampStateIf(cumulative, order_key, op = 'enqueue' AND concurrency_key = '') AS enqueue_delta, + deltaSumTimestampStateIf(cumulative, order_key, op = 'started' AND concurrency_key = '') AS started_delta, + deltaSumTimestampStateIf(cumulative, order_key, op = 'ack' AND concurrency_key = '') AS ack_delta, + deltaSumTimestampStateIf(cumulative, order_key, op = 'nack' AND concurrency_key = '') AS nack_delta, + deltaSumTimestampStateIf(cumulative, order_key, op = 'dlq' AND concurrency_key = '') AS dlq_delta, + sum(throttled) AS throttled_count, + max(queued) AS max_queued, + max(running) AS max_running, + max(queue_limit) AS max_limit, + max(env_queued) AS max_env_queued, + max(env_running) AS max_env_running, + max(env_limit) AS max_env_limit, + max(ck_backlogged) AS max_ck_backlogged, + max(ck_max_wait_ms) AS max_ck_wait_ms, + max(total_running) AS max_total_running, + max(total_limit) AS max_total_limit, + sumIf(wait_ms, op = 'started' AND concurrency_key = '') AS wait_ms_sum, + countIf(op = 'started' AND wait_ms > 0 AND concurrency_key = '') AS wait_ms_count, + quantilesStateIf(0.5, 0.9, 0.95, 0.99)(wait_ms, op = 'started' AND wait_ms > 0 AND concurrency_key = '') AS wait_quantiles +FROM trigger_dev.queue_metrics_raw_v1 +GROUP BY organization_id, project_id, environment_id, queue_name, bucket_start; + +DROP VIEW IF EXISTS trigger_dev.queue_metrics_ck_mv_v1; +CREATE MATERIALIZED VIEW IF NOT EXISTS trigger_dev.queue_metrics_ck_mv_v1 +TO trigger_dev.queue_metrics_ck_v1 AS +SELECT + organization_id, project_id, environment_id, queue_name, concurrency_key, + toStartOfInterval(event_time, INTERVAL 10 SECOND) AS bucket_start, + deltaSumTimestampStateIf(cumulative, order_key, op = 'enqueue') AS enqueue_delta, + deltaSumTimestampStateIf(cumulative, order_key, op = 'started') AS started_delta, + deltaSumTimestampStateIf(cumulative, order_key, op = 'ack') AS ack_delta, + maxIf(queued, op = 'gauge') AS max_queued, + maxIf(running, op = 'gauge') AS max_running, + maxIf(queue_limit, op = 'gauge') AS max_limit, + sumIf(wait_ms, op = 'started') AS wait_ms_sum, + countIf(op = 'started' AND wait_ms > 0) AS wait_ms_count +FROM trigger_dev.queue_metrics_raw_v1 +WHERE concurrency_key != '' +GROUP BY organization_id, project_id, environment_id, queue_name, concurrency_key, bucket_start; + +-- +goose Down +DROP VIEW IF EXISTS trigger_dev.queue_metrics_ck_mv_v1; +DROP VIEW IF EXISTS trigger_dev.queue_metrics_5m_mv_v1; +DROP VIEW IF EXISTS trigger_dev.queue_metrics_mv_v1; +ALTER TABLE trigger_dev.queue_metrics_ck_v1 DROP COLUMN IF EXISTS max_limit; +ALTER TABLE trigger_dev.queue_metrics_5m_v1 DROP COLUMN IF EXISTS max_total_running, DROP COLUMN IF EXISTS max_total_limit; +ALTER TABLE trigger_dev.queue_metrics_v1 DROP COLUMN IF EXISTS max_total_running, DROP COLUMN IF EXISTS max_total_limit; +ALTER TABLE trigger_dev.queue_metrics_raw_v1 DROP COLUMN IF EXISTS total_running, DROP COLUMN IF EXISTS total_limit; diff --git a/internal-packages/clickhouse/src/queueMetrics.ts b/internal-packages/clickhouse/src/queueMetrics.ts index 39576b4a0a3..aa3cf5296d2 100644 --- a/internal-packages/clickhouse/src/queueMetrics.ts +++ b/internal-packages/clickhouse/src/queueMetrics.ts @@ -21,6 +21,8 @@ export const QueueMetricsRawV1Input = z.object({ throttled: z.number().optional(), ck_backlogged: z.number().optional(), ck_max_wait_ms: z.number().optional(), + total_running: z.number().optional(), + total_limit: z.number().optional(), wait_ms: z.number().optional(), cumulative: z.number().optional(), }); diff --git a/internal-packages/metrics-pipeline/src/lua.ts b/internal-packages/metrics-pipeline/src/lua.ts index 64f3b896c0d..701f608308a 100644 --- a/internal-packages/metrics-pipeline/src/lua.ts +++ b/internal-packages/metrics-pipeline/src/lua.ts @@ -17,6 +17,10 @@ export type GaugeComputeLuaParams = { // CK-health extras (both or neither): appended as an optional gauge tail, gauge[8]/gauge[9]. ckBacklogged?: string; ckMaxWaitMs?: string; + // Total-concurrency extras (both or neither, and only with the CK extras): appended as + // gauge[10]/gauge[11]. totalLimit is the RAW stored limit (0 = none); readers clamp. + totalRunning?: string; + totalLimit?: string; }; // Computes an op=gauge snapshot into the enclosing script's `__qm_g` local (a flat @@ -26,11 +30,21 @@ export type GaugeComputeLuaParams = { export function createMetricsGaugeComputeLua(params: GaugeComputeLuaParams): string { const throttled = params.throttledExpr ?? "__cc >= __lim and __ql > 0"; const hasCk = params.ckBacklogged != null && params.ckMaxWaitMs != null; - const gauge = hasCk + const hasTotal = params.totalRunning != null && params.totalLimit != null; + if (hasTotal && !hasCk) { + throw new Error("gauge totalRunning/totalLimit extras require the CK extras"); + } + const gauge = hasTotal ? ` local __ckq = tonumber(${params.ckBacklogged}) or 0 local __ckw = tonumber(${params.ckMaxWaitMs}) or 0 + local __tcc = tonumber(${params.totalRunning}) or 0 + local __tlim = tonumber(${params.totalLimit}) or 0 + __qm_g = {__ql, __cc, __lim, __eql, __ec, __elim, __thr, __ckq, __ckw, __tcc, __tlim}` + : hasCk + ? ` local __ckq = tonumber(${params.ckBacklogged}) or 0 + local __ckw = tonumber(${params.ckMaxWaitMs}) or 0 __qm_g = {__ql, __cc, __lim, __eql, __ec, __elim, __thr, __ckq, __ckw}` - : ` __qm_g = {__ql, __cc, __lim, __eql, __ec, __elim, __thr}`; + : ` __qm_g = {__ql, __cc, __lim, __eql, __ec, __elim, __thr}`; return ` if ${params.enabledArg} then diff --git a/internal-packages/run-engine/src/engine/index.ts b/internal-packages/run-engine/src/engine/index.ts index 40e3d7bc336..1909df7e9c6 100644 --- a/internal-packages/run-engine/src/engine/index.ts +++ b/internal-packages/run-engine/src/engine/index.ts @@ -1748,6 +1748,20 @@ export class RunEngine { return this.runQueue.currentConcurrencyOfQueues(environment, queues); } + async totalConcurrencyOfQueues( + environment: MinimalAuthenticatedEnvironment, + queues: string[] + ): Promise> { + return this.runQueue.totalConcurrencyOfQueues(environment, queues); + } + + async totalConcurrencyLimitsOfQueues( + environment: MinimalAuthenticatedEnvironment, + queues: string[] + ): Promise> { + return this.runQueue.totalConcurrencyLimitsOfQueues(environment, queues); + } + async concurrencyKeyBreakdown( environment: MinimalAuthenticatedEnvironment, queue: string, diff --git a/internal-packages/run-engine/src/run-queue/index.ts b/internal-packages/run-engine/src/run-queue/index.ts index 4b6d61059f4..35448941232 100644 --- a/internal-packages/run-engine/src/run-queue/index.ts +++ b/internal-packages/run-engine/src/run-queue/index.ts @@ -219,7 +219,8 @@ const QUEUE_METRICS_CK_ENQUEUE_GAUGE_LUA = createMetricsGaugeComputeLua({ enabledArg: "ARGV[#ARGV] == '1'", queued: "redis.call('ZCARD', queueKey)", running: "redis.call('SCARD', queueCurrentConcurrencyKey)", - queueLimit: "redis.call('GET', queueConcurrencyLimitKey) or '1000000'", + queueLimit: + "redis.call('HGET', ckLimitsKey, queueName) or redis.call('GET', queueConcurrencyLimitKey) or '1000000'", envQueued: "redis.call('ZCARD', envQueueKey)", envRunning: "redis.call('SCARD', envCurrentConcurrencyKey)", envLimit: "redis.call('GET', envConcurrencyLimitKey) or defaultEnvConcurrencyLimit", @@ -250,6 +251,8 @@ const QUEUE_METRICS_CK_DEQUEUE_GAUGE_LUA = createMetricsGaugeComputeLua({ envLimit: "redis.call('GET', envConcurrencyLimitKey) or defaultEnvConcurrencyLimit", throttledExpr: "false", ...QUEUE_METRICS_CK_GAUGE_EXTRAS, + totalRunning: "redis.call('SCARD', groupConcurrencyKey)", + totalLimit: "redis.call('GET', totalConcurrencyLimitKey) or '0'", }); /** Injected queue-metrics stream emitter; all calls are no-ops when metrics are disabled. */ @@ -710,6 +713,46 @@ export class RunQueue { return limits; } + /** Batch variant of totalConcurrencyOfQueue: one pipeline of group SCARDs. */ + public async totalConcurrencyOfQueues( + env: MinimalAuthenticatedEnvironment, + queues: string[] + ): Promise> { + const pipeline = this.redis.pipeline(); + queues.forEach((queue) => { + pipeline.scard(this.keys.queueGroupConcurrencyKey(env, queue)); + }); + + const results = await pipeline.exec(); + + return queues.reduce( + (acc, queue, index) => { + const value = results?.[index]?.[1]; + acc[queue] = typeof value === "number" ? value : 0; + return acc; + }, + {} as Record + ); + } + + /** Batch read of the RAW stored total concurrency limits (undefined = no cap). */ + public async totalConcurrencyLimitsOfQueues( + env: MinimalAuthenticatedEnvironment, + queues: string[] + ): Promise> { + const keys = queues.map((queue) => this.keys.queueTotalConcurrencyLimitKey(env, queue)); + const values = keys.length > 0 ? await this.redis.mget(...keys) : []; + + return queues.reduce( + (acc, queue, index) => { + const value = values[index]; + acc[queue] = value != null ? Number(value) : undefined; + return acc; + }, + {} as Record + ); + } + public async updateEnvConcurrencyLimits(env: MinimalAuthenticatedEnvironment) { await this.#callUpdateEnvironmentConcurrencyLimits({ envConcurrencyLimitKey: this.keys.envConcurrencyLimitKey(env), @@ -2357,6 +2400,10 @@ export class RunQueue { fields.ckq = ckq; fields.ckw = ckw; } + if (gauge.length >= 11) { + fields.tcc = gauge[9]; + fields.tlim = gauge[10]; + } this.options.queueMetrics?.emitGauge(queue, fields); } diff --git a/packages/core/src/v3/schemas/queues.ts b/packages/core/src/v3/schemas/queues.ts index 34a47b34e3e..f3039f8075a 100644 --- a/packages/core/src/v3/schemas/queues.ts +++ b/packages/core/src/v3/schemas/queues.ts @@ -45,6 +45,21 @@ export const QueueItem = z.object({ overriddenAt: z.coerce.date().nullable(), /** Who overrode the concurrency limit (will be null if overridden via the API) */ overriddenBy: z.string().nullable(), + /** The total concurrency cap across all concurrencyKey values of the queue */ + total: z + .object({ + /** The effective/current total concurrency limit (null = no cap) */ + current: z.number().nullable(), + /** The declared total limit an override reverts to on reset */ + base: z.number().nullable(), + /** The overridden total limit, when an override is active */ + override: z.number().nullable(), + /** When the total override was applied */ + overriddenAt: z.coerce.date().nullable(), + /** Runs currently in flight across all concurrencyKey values */ + running: z.number().nullable(), + }) + .optional(), }) .optional(), }); From 2f98349303157d61e1d337d3ea47a2364f23b25d Mon Sep 17 00:00:00 2001 From: Matt Aitken Date: Sat, 29 Aug 2026 13:38:22 +0100 Subject: [PATCH 02/32] fix(run-engine,clickhouse,webapp): total gauges on enqueue paths; restore views on rollback The CK enqueue gauges (fast path and queued path) now sample total running and the stored cap, so metric buckets fed only by enqueues no longer record zero totals. The migration's down section recreates the pre-existing materialized view definitions so ingestion keeps flowing after a rollback. The per-key table reads only the page's overrides with one HMGET instead of loading the queue's whole override hash. --- .../resources.queues.concurrency-keys.ts | 2 +- ...42_add_queue_metrics_total_concurrency.sql | 68 +++++++++++++++++++ .../run-engine/src/run-queue/index.ts | 36 +++++++++- 3 files changed, 103 insertions(+), 3 deletions(-) diff --git a/apps/webapp/app/routes/resources.queues.concurrency-keys.ts b/apps/webapp/app/routes/resources.queues.concurrency-keys.ts index 662013694ef..8c590554e51 100644 --- a/apps/webapp/app/routes/resources.queues.concurrency-keys.ts +++ b/apps/webapp/app/routes/resources.queues.concurrency-keys.ts @@ -156,7 +156,7 @@ export const action = async ({ request }: ActionFunctionArgs) => { // Enrich just this page's keys with live "now" counts and any per-key limit overrides from Redis. const [live, keyLimitOverrides] = await Promise.all([ engine.concurrencyKeyLiveStats(environment, queueName, keys), - engine.runQueue.getQueueConcurrencyKeyLimits(environment, queueName), + engine.runQueue.getQueueConcurrencyKeyLimitsForKeys(environment, queueName, keys), ]); const loadedAt = Date.now(); diff --git a/internal-packages/clickhouse/schema/042_add_queue_metrics_total_concurrency.sql b/internal-packages/clickhouse/schema/042_add_queue_metrics_total_concurrency.sql index 7711effa6e3..8bb19faeef1 100644 --- a/internal-packages/clickhouse/schema/042_add_queue_metrics_total_concurrency.sql +++ b/internal-packages/clickhouse/schema/042_add_queue_metrics_total_concurrency.sql @@ -107,3 +107,71 @@ ALTER TABLE trigger_dev.queue_metrics_ck_v1 DROP COLUMN IF EXISTS max_limit; ALTER TABLE trigger_dev.queue_metrics_5m_v1 DROP COLUMN IF EXISTS max_total_running, DROP COLUMN IF EXISTS max_total_limit; ALTER TABLE trigger_dev.queue_metrics_v1 DROP COLUMN IF EXISTS max_total_running, DROP COLUMN IF EXISTS max_total_limit; ALTER TABLE trigger_dev.queue_metrics_raw_v1 DROP COLUMN IF EXISTS total_running, DROP COLUMN IF EXISTS total_limit; + +-- Recreate the pre-042 materialized views (the definitions from 036) so ingestion keeps +-- feeding every aggregate table after a rollback. +CREATE MATERIALIZED VIEW IF NOT EXISTS trigger_dev.queue_metrics_mv_v1 +TO trigger_dev.queue_metrics_v1 AS +SELECT + organization_id, project_id, environment_id, queue_name, + toStartOfInterval(event_time, INTERVAL 10 SECOND) AS bucket_start, + deltaSumTimestampStateIf(cumulative, order_key, op = 'enqueue' AND concurrency_key = '') AS enqueue_delta, + deltaSumTimestampStateIf(cumulative, order_key, op = 'started' AND concurrency_key = '') AS started_delta, + deltaSumTimestampStateIf(cumulative, order_key, op = 'ack' AND concurrency_key = '') AS ack_delta, + deltaSumTimestampStateIf(cumulative, order_key, op = 'nack' AND concurrency_key = '') AS nack_delta, + deltaSumTimestampStateIf(cumulative, order_key, op = 'dlq' AND concurrency_key = '') AS dlq_delta, + sum(throttled) AS throttled_count, + max(queued) AS max_queued, + max(running) AS max_running, + max(queue_limit) AS max_limit, + max(env_queued) AS max_env_queued, + max(env_running) AS max_env_running, + max(env_limit) AS max_env_limit, + max(ck_backlogged) AS max_ck_backlogged, + max(ck_max_wait_ms) AS max_ck_wait_ms, + sumIf(wait_ms, op = 'started' AND concurrency_key = '') AS wait_ms_sum, + countIf(op = 'started' AND wait_ms > 0 AND concurrency_key = '') AS wait_ms_count, + quantilesStateIf(0.5, 0.9, 0.95, 0.99)(wait_ms, op = 'started' AND wait_ms > 0 AND concurrency_key = '') AS wait_quantiles +FROM trigger_dev.queue_metrics_raw_v1 +GROUP BY organization_id, project_id, environment_id, queue_name, bucket_start; + +CREATE MATERIALIZED VIEW IF NOT EXISTS trigger_dev.queue_metrics_5m_mv_v1 +TO trigger_dev.queue_metrics_5m_v1 AS +SELECT + organization_id, project_id, environment_id, queue_name, + toStartOfInterval(event_time, INTERVAL 5 MINUTE) AS bucket_start, + deltaSumTimestampStateIf(cumulative, order_key, op = 'enqueue' AND concurrency_key = '') AS enqueue_delta, + deltaSumTimestampStateIf(cumulative, order_key, op = 'started' AND concurrency_key = '') AS started_delta, + deltaSumTimestampStateIf(cumulative, order_key, op = 'ack' AND concurrency_key = '') AS ack_delta, + deltaSumTimestampStateIf(cumulative, order_key, op = 'nack' AND concurrency_key = '') AS nack_delta, + deltaSumTimestampStateIf(cumulative, order_key, op = 'dlq' AND concurrency_key = '') AS dlq_delta, + sum(throttled) AS throttled_count, + max(queued) AS max_queued, + max(running) AS max_running, + max(queue_limit) AS max_limit, + max(env_queued) AS max_env_queued, + max(env_running) AS max_env_running, + max(env_limit) AS max_env_limit, + max(ck_backlogged) AS max_ck_backlogged, + max(ck_max_wait_ms) AS max_ck_wait_ms, + sumIf(wait_ms, op = 'started' AND concurrency_key = '') AS wait_ms_sum, + countIf(op = 'started' AND wait_ms > 0 AND concurrency_key = '') AS wait_ms_count, + quantilesStateIf(0.5, 0.9, 0.95, 0.99)(wait_ms, op = 'started' AND wait_ms > 0 AND concurrency_key = '') AS wait_quantiles +FROM trigger_dev.queue_metrics_raw_v1 +GROUP BY organization_id, project_id, environment_id, queue_name, bucket_start; + +CREATE MATERIALIZED VIEW IF NOT EXISTS trigger_dev.queue_metrics_ck_mv_v1 +TO trigger_dev.queue_metrics_ck_v1 AS +SELECT + organization_id, project_id, environment_id, queue_name, concurrency_key, + toStartOfInterval(event_time, INTERVAL 10 SECOND) AS bucket_start, + deltaSumTimestampStateIf(cumulative, order_key, op = 'enqueue') AS enqueue_delta, + deltaSumTimestampStateIf(cumulative, order_key, op = 'started') AS started_delta, + deltaSumTimestampStateIf(cumulative, order_key, op = 'ack') AS ack_delta, + maxIf(queued, op = 'gauge') AS max_queued, + maxIf(running, op = 'gauge') AS max_running, + sumIf(wait_ms, op = 'started') AS wait_ms_sum, + countIf(op = 'started' AND wait_ms > 0) AS wait_ms_count +FROM trigger_dev.queue_metrics_raw_v1 +WHERE concurrency_key != '' +GROUP BY organization_id, project_id, environment_id, queue_name, concurrency_key, bucket_start; diff --git a/internal-packages/run-engine/src/run-queue/index.ts b/internal-packages/run-engine/src/run-queue/index.ts index 35448941232..311e7603738 100644 --- a/internal-packages/run-engine/src/run-queue/index.ts +++ b/internal-packages/run-engine/src/run-queue/index.ts @@ -214,6 +214,14 @@ const QUEUE_METRICS_CK_GAUGE_EXTRAS = { ckMaxWaitMs: "__ckwait", }; +// Total-concurrency tail (gauge[10]/gauge[11]): live group cardinality + raw stored cap. +// Requires groupConcurrencyKey/totalConcurrencyLimitKey locals; the CK scripts that actually +// run (the Tracked variants and the CK dequeue) all declare them for the total-cap gate. +const QUEUE_METRICS_TOTAL_GAUGE_EXTRAS = { + totalRunning: "redis.call('SCARD', groupConcurrencyKey)", + totalLimit: "redis.call('GET', totalConcurrencyLimitKey) or '0'", +}; + // CK enqueue variants of the two gauges above, extended with the CK-health tail. const QUEUE_METRICS_CK_ENQUEUE_GAUGE_LUA = createMetricsGaugeComputeLua({ enabledArg: "ARGV[#ARGV] == '1'", @@ -225,6 +233,7 @@ const QUEUE_METRICS_CK_ENQUEUE_GAUGE_LUA = createMetricsGaugeComputeLua({ envRunning: "redis.call('SCARD', envCurrentConcurrencyKey)", envLimit: "redis.call('GET', envConcurrencyLimitKey) or defaultEnvConcurrencyLimit", ...QUEUE_METRICS_CK_GAUGE_EXTRAS, + ...QUEUE_METRICS_TOTAL_GAUGE_EXTRAS, }); const QUEUE_METRICS_CK_ENQUEUE_FASTPATH_GAUGE_LUA = createMetricsGaugeComputeLua({ @@ -236,6 +245,7 @@ const QUEUE_METRICS_CK_ENQUEUE_FASTPATH_GAUGE_LUA = createMetricsGaugeComputeLua envRunning: "envCurrent", envLimit: "envLimit", ...QUEUE_METRICS_CK_GAUGE_EXTRAS, + ...QUEUE_METRICS_TOTAL_GAUGE_EXTRAS, }); // CK dequeue: depth/running from the per-base-queue aggregate counters the run-queue already @@ -251,8 +261,7 @@ const QUEUE_METRICS_CK_DEQUEUE_GAUGE_LUA = createMetricsGaugeComputeLua({ envLimit: "redis.call('GET', envConcurrencyLimitKey) or defaultEnvConcurrencyLimit", throttledExpr: "false", ...QUEUE_METRICS_CK_GAUGE_EXTRAS, - totalRunning: "redis.call('SCARD', groupConcurrencyKey)", - totalLimit: "redis.call('GET', totalConcurrencyLimitKey) or '0'", + ...QUEUE_METRICS_TOTAL_GAUGE_EXTRAS, }); /** Injected queue-metrics stream emitter; all calls are no-ops when metrics are disabled. */ @@ -713,6 +722,29 @@ export class RunQueue { return limits; } + /** Per-key limit overrides for just the given keys: one HMGET, O(keys) not O(overrides). */ + public async getQueueConcurrencyKeyLimitsForKeys( + env: MinimalAuthenticatedEnvironment, + queue: string, + concurrencyKeys: string[] + ): Promise> { + if (concurrencyKeys.length === 0) { + return {}; + } + + const fields = concurrencyKeys.map((key) => this.keys.queueKey(env, queue, key)); + const values = await this.redis.hmget(this.keys.queueCkLimitsKey(env, queue), ...fields); + + const limits: Record = {}; + concurrencyKeys.forEach((key, index) => { + const value = values[index]; + if (value != null) { + limits[key] = Number(value); + } + }); + return limits; + } + /** Batch variant of totalConcurrencyOfQueue: one pipeline of group SCARDs. */ public async totalConcurrencyOfQueues( env: MinimalAuthenticatedEnvironment, From 5ad130bf04557d01c0256c74c2e32ad8c38e6f4d Mon Sep 17 00:00:00 2001 From: Matt Aitken Date: Sat, 29 Aug 2026 16:48:06 +0100 Subject: [PATCH 03/32] fix(clickhouse): keep migration comments semicolon-free The test harness splits a migration's up section on semicolons, so a semicolon inside a comment yields a comment-only statement that ClickHouse rejects as an empty query and every container-backed suite fails at setup. --- .../schema/042_add_queue_metrics_total_concurrency.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal-packages/clickhouse/schema/042_add_queue_metrics_total_concurrency.sql b/internal-packages/clickhouse/schema/042_add_queue_metrics_total_concurrency.sql index 8bb19faeef1..f45e6421ae3 100644 --- a/internal-packages/clickhouse/schema/042_add_queue_metrics_total_concurrency.sql +++ b/internal-packages/clickhouse/schema/042_add_queue_metrics_total_concurrency.sql @@ -2,7 +2,7 @@ -- Total-concurrency gauges: total_running is the in-flight count across ALL -- concurrency-key variants of a queue (the groupConcurrency set), total_limit the --- RAW stored total cap (0 = none; readers clamp against max_env_limit). Emitted on +-- RAW stored total cap (0 = none, readers clamp against max_env_limit). Emitted on -- base-queue gauge rows only. Per-key gauge rows now carry the EFFECTIVE per-key -- limit in queue_limit (override-aware), surfaced in the ck tier as max_limit. From 24b2fd2ed9134f4dc9cc2e2e977e01175dd1eeb2 Mon Sep 17 00:00:00 2001 From: Matt Aitken Date: Sat, 29 Aug 2026 17:01:59 +0100 Subject: [PATCH 04/32] fix(webapp): skip the total concurrency read when the queue has no cap Queue retrieve only asks the engine for total running when a total limit is set, matching the list presenter and avoiding a pointless read for the common uncapped case. --- .../webapp/app/presenters/v3/QueueRetrievePresenter.server.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/apps/webapp/app/presenters/v3/QueueRetrievePresenter.server.ts b/apps/webapp/app/presenters/v3/QueueRetrievePresenter.server.ts index e4777ceb13e..26811a8800e 100644 --- a/apps/webapp/app/presenters/v3/QueueRetrievePresenter.server.ts +++ b/apps/webapp/app/presenters/v3/QueueRetrievePresenter.server.ts @@ -90,7 +90,9 @@ export class QueueRetrievePresenter extends BasePresenter { const results = await Promise.all([ engine.lengthOfQueues(environment, [queue.name]), engine.currentConcurrencyOfQueues(environment, [queue.name]), - engine.totalConcurrencyOfQueues(environment, [queue.name]), + queue.totalConcurrencyLimit != null + ? engine.totalConcurrencyOfQueues(environment, [queue.name]) + : undefined, ]); // Transform queues to include running and queued counts From 45705a36f2d89e03a6896f1af9a92ac44d57847c Mon Sep 17 00:00:00 2001 From: Matt Aitken Date: Sat, 29 Aug 2026 18:21:57 +0100 Subject: [PATCH 05/32] feat(webapp): show the Total column in the non-metrics queues table too The total concurrency numbers come from live Redis, not the metrics pipeline, so the column belongs in both tables rather than only behind the queue metrics UI gate. --- .../route.tsx | 29 ++++++++++++++++++- 1 file changed, 28 insertions(+), 1 deletion(-) diff --git a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.queues/route.tsx b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.queues/route.tsx index 5ec0ada0371..5e830fd68a0 100644 --- a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.queues/route.tsx +++ b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.queues/route.tsx @@ -1813,6 +1813,12 @@ function ClassicQueuesView() { Queued Running Limit + + Total + {limit} + = + Math.min( + queue.concurrency.total.current, + environment.concurrencyLimit + ) && + "text-warning" + )} + > + {queue.concurrency?.total?.current != null + ? `${queue.concurrency.total.running ?? 0}/${Math.min( + queue.concurrency.total.current, + environment.concurrencyLimit + )}` + : "–"} + - +
{hasFilters ? "No queues found matching your filters" : "No queues found"} From 19b9e84b627dc468bec1a53bb6be981047d07506 Mon Sep 17 00:00:00 2001 From: Matt Aitken Date: Sat, 29 Aug 2026 18:33:02 +0100 Subject: [PATCH 06/32] feat(webapp): fold the total cap into the Limit column A separate Total column implied every queue should have one, and its dash read as a missing limit on queues that never use concurrency keys. Only queues that declare a totalConcurrencyLimit now change: their Limit cell reads as per-key plus total (e.g. 1 /key, 3 total) and Running turns warning-colored when the total cap is saturated. Plain queues are unchanged. --- .../route.tsx | 88 ++++++++----------- 1 file changed, 38 insertions(+), 50 deletions(-) diff --git a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.queues/route.tsx b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.queues/route.tsx index 5e830fd68a0..2caf4f95802 100644 --- a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.queues/route.tsx +++ b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.queues/route.tsx @@ -704,13 +704,12 @@ function QueuesWithMetricsView() { Name Queued Running - Limit - Total + Limit 0 && "text-text-bright" + queue.concurrency?.total?.current != null && + queue.running >= + Math.min( + queue.concurrency.total.current, + environment.concurrencyLimit + ) + ? "text-warning" + : queue.running > 0 && "text-text-bright" )} > {queue.running} @@ -879,29 +885,16 @@ function QueuesWithMetricsView() { ) : ( limit )} - - = - Math.min( - queue.concurrency.total.current, - environment.concurrencyLimit - ) && - "text-warning" - )} - > - {queue.concurrency?.total?.current != null - ? `${queue.concurrency.total.running ?? 0}/${Math.min( + {queue.concurrency?.total?.current != null ? ( + + /key ·{" "} + {Math.min( queue.concurrency.total.current, environment.concurrencyLimit - )}` - : "–"} + )}{" "} + total + + ) : null} - +
{hasFilters ? "No queues found matching your filters" : "No queues found"} @@ -1812,12 +1805,11 @@ function ClassicQueuesView() { Name Queued Running - Limit - Total + Limit 0 && "text-text-bright", + queue.concurrency?.total?.current != null && + queue.running >= + Math.min( + queue.concurrency.total.current, + environment.concurrencyLimit + ) + ? "text-warning" + : queue.running > 0 && "text-text-bright", isAtConcurrencyLimit && "text-warning" )} > @@ -1940,27 +1939,16 @@ function ClassicQueuesView() { )} > {limit} - - = - Math.min( - queue.concurrency.total.current, - environment.concurrencyLimit - ) && - "text-warning" - )} - > - {queue.concurrency?.total?.current != null - ? `${queue.concurrency.total.running ?? 0}/${Math.min( + {queue.concurrency?.total?.current != null ? ( + + /key ·{" "} + {Math.min( queue.concurrency.total.current, environment.concurrencyLimit - )}` - : "–"} + )}{" "} + total + + ) : null} - +
{hasFilters ? "No queues found matching your filters" : "No queues found"} From f5855b34deab8c75d4df59078c560a096015cebd Mon Sep 17 00:00:00 2001 From: Matt Aitken Date: Sat, 29 Aug 2026 18:39:14 +0100 Subject: [PATCH 07/32] fix(webapp): saturate the total-cap warning on keyed runs only The total cap gates keyed admissions, so the Running cell now warns off the group count rather than the aggregate that also includes unkeyed runs. --- .../route.tsx | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.queues/route.tsx b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.queues/route.tsx index 2caf4f95802..0364e9ca1b7 100644 --- a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.queues/route.tsx +++ b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.queues/route.tsx @@ -854,7 +854,7 @@ function QueuesWithMetricsView() { "w-[1%]", queue.paused ? "opacity-50" : undefined, queue.concurrency?.total?.current != null && - queue.running >= + (queue.concurrency.total.running ?? 0) >= Math.min( queue.concurrency.total.current, environment.concurrencyLimit @@ -1918,7 +1918,7 @@ function ClassicQueuesView() { "w-[1%] pl-16 tabular-nums", queue.paused ? "opacity-50" : undefined, queue.concurrency?.total?.current != null && - queue.running >= + (queue.concurrency.total.running ?? 0) >= Math.min( queue.concurrency.total.current, environment.concurrencyLimit From 24eb8395a6c73bc1418627ed213eba4b3be744c8 Mon Sep 17 00:00:00 2001 From: Matt Aitken Date: Sat, 29 Aug 2026 19:11:06 +0100 Subject: [PATCH 08/32] refactor(webapp,core,clickhouse): combined concurrency in responses, dashboard and metrics Queue API responses expose concurrency.combined, the dashboard says combined, and the new metrics columns are named combined_running and combined_limit. --- .../v3/QueueRetrievePresenter.server.ts | 2 +- .../route.tsx | 28 ++++++++--------- .../route.tsx | 10 +++---- apps/webapp/app/v3/querySchemas.ts | 10 +++---- apps/webapp/app/v3/queueMetricsMapping.ts | 4 +-- ...dd_queue_metrics_combined_concurrency.sql} | 30 +++++++++---------- .../clickhouse/src/queueMetrics.ts | 4 +-- packages/core/src/v3/schemas/queues.ts | 12 ++++---- 8 files changed, 50 insertions(+), 50 deletions(-) rename internal-packages/clickhouse/schema/{042_add_queue_metrics_total_concurrency.sql => 042_add_queue_metrics_combined_concurrency.sql} (91%) diff --git a/apps/webapp/app/presenters/v3/QueueRetrievePresenter.server.ts b/apps/webapp/app/presenters/v3/QueueRetrievePresenter.server.ts index 26811a8800e..1ce08e4c628 100644 --- a/apps/webapp/app/presenters/v3/QueueRetrievePresenter.server.ts +++ b/apps/webapp/app/presenters/v3/QueueRetrievePresenter.server.ts @@ -176,7 +176,7 @@ export function toQueueItem(data: { override: data.concurrencyLimitOverriddenAt ? data.concurrencyLimit : null, overriddenBy: toQueueConcurrencyOverriddenBy(data.concurrencyLimitOverriddenBy), overriddenAt: data.concurrencyLimitOverriddenAt, - total: + combined: data.totalConcurrencyLimit !== undefined ? { current: data.totalConcurrencyLimit, diff --git a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.queues/route.tsx b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.queues/route.tsx index 0364e9ca1b7..a8ba01515a0 100644 --- a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.queues/route.tsx +++ b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.queues/route.tsx @@ -707,7 +707,7 @@ function QueuesWithMetricsView() { Limit @@ -853,10 +853,10 @@ function QueuesWithMetricsView() { className={cn( "w-[1%]", queue.paused ? "opacity-50" : undefined, - queue.concurrency?.total?.current != null && - (queue.concurrency.total.running ?? 0) >= + queue.concurrency?.combined?.current != null && + (queue.concurrency.combined.running ?? 0) >= Math.min( - queue.concurrency.total.current, + queue.concurrency.combined.current, environment.concurrencyLimit ) ? "text-warning" @@ -885,14 +885,14 @@ function QueuesWithMetricsView() { ) : ( limit )} - {queue.concurrency?.total?.current != null ? ( + {queue.concurrency?.combined?.current != null ? ( /key ·{" "} {Math.min( - queue.concurrency.total.current, + queue.concurrency.combined.current, environment.concurrencyLimit )}{" "} - total + combined ) : null} @@ -1807,7 +1807,7 @@ function ClassicQueuesView() { Running Limit @@ -1917,10 +1917,10 @@ function ClassicQueuesView() { className={cn( "w-[1%] pl-16 tabular-nums", queue.paused ? "opacity-50" : undefined, - queue.concurrency?.total?.current != null && - (queue.concurrency.total.running ?? 0) >= + queue.concurrency?.combined?.current != null && + (queue.concurrency.combined.running ?? 0) >= Math.min( - queue.concurrency.total.current, + queue.concurrency.combined.current, environment.concurrencyLimit ) ? "text-warning" @@ -1939,14 +1939,14 @@ function ClassicQueuesView() { )} > {limit} - {queue.concurrency?.total?.current != null ? ( + {queue.concurrency?.combined?.current != null ? ( /key ·{" "} {Math.min( - queue.concurrency.total.current, + queue.concurrency.combined.current, environment.concurrencyLimit )}{" "} - total + combined ) : null} diff --git a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.queues_.$queueParam/route.tsx b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.queues_.$queueParam/route.tsx index ff4fc786bc5..8e445fa6111 100644 --- a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.queues_.$queueParam/route.tsx +++ b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.queues_.$queueParam/route.tsx @@ -396,7 +396,7 @@ export default function Page() { ids={ids} timeRange={timeRange} queueName={fullName} - hasTotalLimit={queue.concurrency?.total?.current != null} + hasTotalLimit={queue.concurrency?.combined?.current != null} /> )} @@ -494,25 +494,25 @@ function OverviewCharts({ /> {hasTotalLimit ? ( Runs in flight across ALL concurrency keys ( - ) versus the queue's total limit ( + ) versus the queue's combined limit ( ). } showLegend className="aspect-[2/1]" - query={`SELECT timeBucket() AS t, max(max_total_running) AS running, least(max(max_total_limit), max(max_env_limit)) AS cap\nFROM queue_metrics\nGROUP BY t\nORDER BY t`} + query={`SELECT timeBucket() AS t, max(max_combined_running) AS running, least(max(max_combined_limit), max(max_env_limit)) AS cap\nFROM queue_metrics\nGROUP BY t\nORDER BY t`} fillGaps minBucketSeconds={SYNCED_CHART_MIN_BUCKET_SECONDS} ids={ids} timeRange={timeRange} queueName={queueName} series={[ - { key: "cap", label: "Total limit", color: COLORS.limit }, + { key: "cap", label: "Combined limit", color: COLORS.limit }, { key: "running", label: "Running", color: COLORS.running }, ]} thresholdStroke={{ diff --git a/apps/webapp/app/v3/querySchemas.ts b/apps/webapp/app/v3/querySchemas.ts index 8267a0f8020..05cd7f0b394 100644 --- a/apps/webapp/app/v3/querySchemas.ts +++ b/apps/webapp/app/v3/querySchemas.ts @@ -770,19 +770,19 @@ const queueMetricsSchema: TableSchema = { fillMode: "carry", }), }, - max_total_running: { - name: "max_total_running", + max_combined_running: { + name: "max_combined_running", ...column("UInt32", { description: "Peak in-flight runs across ALL concurrency keys of the queue in the bucket (only emitted for keyed queues). Aggregate with max().", fillMode: "carry", }), }, - max_total_limit: { - name: "max_total_limit", + max_combined_limit: { + name: "max_combined_limit", ...column("UInt32", { description: - "The queue's total concurrency limit across all keys, as stored (0 = no cap; clamp against max_env_limit). Aggregate with max().", + "The queue's combined concurrency limit across all keys, as stored (0 = no cap; clamp against max_env_limit). Aggregate with max().", fillMode: "carry", }), }, diff --git a/apps/webapp/app/v3/queueMetricsMapping.ts b/apps/webapp/app/v3/queueMetricsMapping.ts index d341f4a63cd..f093dc3f027 100644 --- a/apps/webapp/app/v3/queueMetricsMapping.ts +++ b/apps/webapp/app/v3/queueMetricsMapping.ts @@ -131,8 +131,8 @@ export function mapEntryToRows( throttled: num(f.thr), ck_backlogged: num(f.ckq), ck_max_wait_ms: num(f.ckw), - total_running: num(f.tcc), - total_limit: num(f.tlim), + combined_running: num(f.tcc), + combined_limit: num(f.tlim), }, ]; } diff --git a/internal-packages/clickhouse/schema/042_add_queue_metrics_total_concurrency.sql b/internal-packages/clickhouse/schema/042_add_queue_metrics_combined_concurrency.sql similarity index 91% rename from internal-packages/clickhouse/schema/042_add_queue_metrics_total_concurrency.sql rename to internal-packages/clickhouse/schema/042_add_queue_metrics_combined_concurrency.sql index f45e6421ae3..03cb133799a 100644 --- a/internal-packages/clickhouse/schema/042_add_queue_metrics_total_concurrency.sql +++ b/internal-packages/clickhouse/schema/042_add_queue_metrics_combined_concurrency.sql @@ -1,22 +1,22 @@ -- +goose Up --- Total-concurrency gauges: total_running is the in-flight count across ALL --- concurrency-key variants of a queue (the groupConcurrency set), total_limit the +-- Total-concurrency gauges: combined_running is the in-flight count across ALL +-- concurrency-key variants of a queue (the groupConcurrency set), combined_limit the -- RAW stored total cap (0 = none, readers clamp against max_env_limit). Emitted on -- base-queue gauge rows only. Per-key gauge rows now carry the EFFECTIVE per-key -- limit in queue_limit (override-aware), surfaced in the ck tier as max_limit. ALTER TABLE trigger_dev.queue_metrics_raw_v1 - ADD COLUMN IF NOT EXISTS total_running UInt32 DEFAULT 0, - ADD COLUMN IF NOT EXISTS total_limit UInt32 DEFAULT 0; + ADD COLUMN IF NOT EXISTS combined_running UInt32 DEFAULT 0, + ADD COLUMN IF NOT EXISTS combined_limit UInt32 DEFAULT 0; ALTER TABLE trigger_dev.queue_metrics_v1 - ADD COLUMN IF NOT EXISTS max_total_running SimpleAggregateFunction(max, UInt32), - ADD COLUMN IF NOT EXISTS max_total_limit SimpleAggregateFunction(max, UInt32); + ADD COLUMN IF NOT EXISTS max_combined_running SimpleAggregateFunction(max, UInt32), + ADD COLUMN IF NOT EXISTS max_combined_limit SimpleAggregateFunction(max, UInt32); ALTER TABLE trigger_dev.queue_metrics_5m_v1 - ADD COLUMN IF NOT EXISTS max_total_running SimpleAggregateFunction(max, UInt32), - ADD COLUMN IF NOT EXISTS max_total_limit SimpleAggregateFunction(max, UInt32); + ADD COLUMN IF NOT EXISTS max_combined_running SimpleAggregateFunction(max, UInt32), + ADD COLUMN IF NOT EXISTS max_combined_limit SimpleAggregateFunction(max, UInt32); ALTER TABLE trigger_dev.queue_metrics_ck_v1 ADD COLUMN IF NOT EXISTS max_limit SimpleAggregateFunction(max, UInt32); @@ -45,8 +45,8 @@ SELECT max(env_limit) AS max_env_limit, max(ck_backlogged) AS max_ck_backlogged, max(ck_max_wait_ms) AS max_ck_wait_ms, - max(total_running) AS max_total_running, - max(total_limit) AS max_total_limit, + max(combined_running) AS max_combined_running, + max(combined_limit) AS max_combined_limit, sumIf(wait_ms, op = 'started' AND concurrency_key = '') AS wait_ms_sum, countIf(op = 'started' AND wait_ms > 0 AND concurrency_key = '') AS wait_ms_count, quantilesStateIf(0.5, 0.9, 0.95, 0.99)(wait_ms, op = 'started' AND wait_ms > 0 AND concurrency_key = '') AS wait_quantiles @@ -73,8 +73,8 @@ SELECT max(env_limit) AS max_env_limit, max(ck_backlogged) AS max_ck_backlogged, max(ck_max_wait_ms) AS max_ck_wait_ms, - max(total_running) AS max_total_running, - max(total_limit) AS max_total_limit, + max(combined_running) AS max_combined_running, + max(combined_limit) AS max_combined_limit, sumIf(wait_ms, op = 'started' AND concurrency_key = '') AS wait_ms_sum, countIf(op = 'started' AND wait_ms > 0 AND concurrency_key = '') AS wait_ms_count, quantilesStateIf(0.5, 0.9, 0.95, 0.99)(wait_ms, op = 'started' AND wait_ms > 0 AND concurrency_key = '') AS wait_quantiles @@ -104,9 +104,9 @@ DROP VIEW IF EXISTS trigger_dev.queue_metrics_ck_mv_v1; DROP VIEW IF EXISTS trigger_dev.queue_metrics_5m_mv_v1; DROP VIEW IF EXISTS trigger_dev.queue_metrics_mv_v1; ALTER TABLE trigger_dev.queue_metrics_ck_v1 DROP COLUMN IF EXISTS max_limit; -ALTER TABLE trigger_dev.queue_metrics_5m_v1 DROP COLUMN IF EXISTS max_total_running, DROP COLUMN IF EXISTS max_total_limit; -ALTER TABLE trigger_dev.queue_metrics_v1 DROP COLUMN IF EXISTS max_total_running, DROP COLUMN IF EXISTS max_total_limit; -ALTER TABLE trigger_dev.queue_metrics_raw_v1 DROP COLUMN IF EXISTS total_running, DROP COLUMN IF EXISTS total_limit; +ALTER TABLE trigger_dev.queue_metrics_5m_v1 DROP COLUMN IF EXISTS max_combined_running, DROP COLUMN IF EXISTS max_combined_limit; +ALTER TABLE trigger_dev.queue_metrics_v1 DROP COLUMN IF EXISTS max_combined_running, DROP COLUMN IF EXISTS max_combined_limit; +ALTER TABLE trigger_dev.queue_metrics_raw_v1 DROP COLUMN IF EXISTS combined_running, DROP COLUMN IF EXISTS combined_limit; -- Recreate the pre-042 materialized views (the definitions from 036) so ingestion keeps -- feeding every aggregate table after a rollback. diff --git a/internal-packages/clickhouse/src/queueMetrics.ts b/internal-packages/clickhouse/src/queueMetrics.ts index aa3cf5296d2..f3a6be695e4 100644 --- a/internal-packages/clickhouse/src/queueMetrics.ts +++ b/internal-packages/clickhouse/src/queueMetrics.ts @@ -21,8 +21,8 @@ export const QueueMetricsRawV1Input = z.object({ throttled: z.number().optional(), ck_backlogged: z.number().optional(), ck_max_wait_ms: z.number().optional(), - total_running: z.number().optional(), - total_limit: z.number().optional(), + combined_running: z.number().optional(), + combined_limit: z.number().optional(), wait_ms: z.number().optional(), cumulative: z.number().optional(), }); diff --git a/packages/core/src/v3/schemas/queues.ts b/packages/core/src/v3/schemas/queues.ts index f3039f8075a..9ca282fb33d 100644 --- a/packages/core/src/v3/schemas/queues.ts +++ b/packages/core/src/v3/schemas/queues.ts @@ -45,16 +45,16 @@ export const QueueItem = z.object({ overriddenAt: z.coerce.date().nullable(), /** Who overrode the concurrency limit (will be null if overridden via the API) */ overriddenBy: z.string().nullable(), - /** The total concurrency cap across all concurrencyKey values of the queue */ - total: z + /** The combined concurrency cap across all concurrencyKey values of the queue */ + combined: z .object({ - /** The effective/current total concurrency limit (null = no cap) */ + /** The effective/current combined concurrency limit (null = no cap) */ current: z.number().nullable(), - /** The declared total limit an override reverts to on reset */ + /** The declared combined limit an override reverts to on reset */ base: z.number().nullable(), - /** The overridden total limit, when an override is active */ + /** The overridden combined limit, when an override is active */ override: z.number().nullable(), - /** When the total override was applied */ + /** When the combined override was applied */ overriddenAt: z.coerce.date().nullable(), /** Runs currently in flight across all concurrencyKey values */ running: z.number().nullable(), From 836394a96409f79a7f559af18f00721c69d7ea24 Mon Sep 17 00:00:00 2001 From: Matt Aitken Date: Sat, 29 Aug 2026 19:35:52 +0100 Subject: [PATCH 09/32] feat(webapp): bracketed combined limit in the Limit column Queues that set a combinedConcurrencyLimit show it bracketed next to the per-key limit with a fine dashed underline and an explanatory tooltip; the Limit header tooltip is width-capped. Queues without one are unchanged. --- .../route.tsx | 74 ++++++++++++++----- 1 file changed, 56 insertions(+), 18 deletions(-) diff --git a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.queues/route.tsx b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.queues/route.tsx index a8ba01515a0..bc2304b9629 100644 --- a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.queues/route.tsx +++ b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.queues/route.tsx @@ -707,7 +707,8 @@ function QueuesWithMetricsView() { Limit @@ -886,14 +887,32 @@ function QueuesWithMetricsView() { limit )} {queue.concurrency?.combined?.current != null ? ( - - /key ·{" "} - {Math.min( - queue.concurrency.combined.current, - environment.concurrencyLimit - )}{" "} - combined - + + ( + {Math.min( + queue.concurrency.combined.current, + environment.concurrencyLimit + )} + ) + + } + content={ + <> + Combined limit: at most{" "} + {Math.min( + queue.concurrency.combined.current, + environment.concurrencyLimit + )}{" "} + runs across all concurrency keys of this queue. The main limit + applies to each key separately. + + } + className="max-w-[260px]" + /> ) : null} Running Limit @@ -1940,14 +1960,32 @@ function ClassicQueuesView() { > {limit} {queue.concurrency?.combined?.current != null ? ( - - /key ·{" "} - {Math.min( - queue.concurrency.combined.current, - environment.concurrencyLimit - )}{" "} - combined - + + ( + {Math.min( + queue.concurrency.combined.current, + environment.concurrencyLimit + )} + ) + + } + content={ + <> + Combined limit: at most{" "} + {Math.min( + queue.concurrency.combined.current, + environment.concurrencyLimit + )}{" "} + runs across all concurrency keys of this queue. The main limit + applies to each key separately. + + } + className="max-w-[260px]" + /> ) : null} Date: Sat, 29 Aug 2026 19:40:12 +0100 Subject: [PATCH 10/32] fix(webapp): combined-limit tooltip renders beside the cell link The tooltip trigger is a button, so nesting it in the Limit cell's link made clicking it navigate; it now renders as the cell's trailing adornment. --- .../route.tsx | 61 ++++++++++--------- 1 file changed, 33 insertions(+), 28 deletions(-) diff --git a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.queues/route.tsx b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.queues/route.tsx index bc2304b9629..ee2354ea1af 100644 --- a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.queues/route.tsx +++ b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.queues/route.tsx @@ -875,6 +875,39 @@ function QueuesWithMetricsView() { queue.paused ? "opacity-50" : undefined, queue.concurrency?.overriddenAt && "font-medium text-text-bright" )} + // The combined-limit hint is a tooltip button, so it renders beside the + // link (trailing) rather than nested inside the ; the number stays the + // link. + trailingContent={ + queue.concurrency?.combined?.current != null ? ( + + ( + {Math.min( + queue.concurrency.combined.current, + environment.concurrencyLimit + )} + ) + + } + content={ + <> + Combined limit: at most{" "} + {Math.min( + queue.concurrency.combined.current, + environment.concurrencyLimit + )}{" "} + runs across all concurrency keys of this queue. The main limit + applies to each key separately. + + } + className="max-w-[260px]" + /> + ) : undefined + } > {queue.concurrencyLimitOverridePercent !== null ? ( <> @@ -886,34 +919,6 @@ function QueuesWithMetricsView() { ) : ( limit )} - {queue.concurrency?.combined?.current != null ? ( - - ( - {Math.min( - queue.concurrency.combined.current, - environment.concurrencyLimit - )} - ) - - } - content={ - <> - Combined limit: at most{" "} - {Math.min( - queue.concurrency.combined.current, - environment.concurrencyLimit - )}{" "} - runs across all concurrency keys of this queue. The main limit - applies to each key separately. - - } - className="max-w-[260px]" - /> - ) : null} Date: Mon, 31 Aug 2026 10:30:33 +0100 Subject: [PATCH 11/32] Better tooltip message --- .../route.tsx | 550 +++++++++++++----- 1 file changed, 402 insertions(+), 148 deletions(-) diff --git a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.queues/route.tsx b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.queues/route.tsx index ee2354ea1af..748be64dee1 100644 --- a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.queues/route.tsx +++ b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.queues/route.tsx @@ -8,7 +8,10 @@ import { } from "@heroicons/react/20/solid"; import { DialogClose } from "@radix-ui/react-dialog"; import { Form, useNavigation } from "@remix-run/react"; -import { type ActionFunctionArgs, type LoaderFunctionArgs } from "@remix-run/server-runtime"; +import { + type ActionFunctionArgs, + type LoaderFunctionArgs, +} from "@remix-run/server-runtime"; import type { RuntimeEnvironmentType } from "@trigger.dev/database"; import { useEffect, useMemo, useState, type ReactNode } from "react"; import { QueuesIcon } from "~/assets/icons/QueuesIcon"; @@ -22,10 +25,19 @@ import { PageBody, PageContainer } from "~/components/layout/AppLayout"; import { MetricsLayout } from "~/components/layout/MetricsLayout"; import { Badge } from "~/components/primitives/Badge"; import { Button, LinkButton } from "~/components/primitives/Buttons"; -import { Dialog, DialogContent, DialogHeader, DialogTrigger } from "~/components/primitives/Dialog"; +import { + Dialog, + DialogContent, + DialogHeader, + DialogTrigger, +} from "~/components/primitives/Dialog"; import { FormButtons } from "~/components/primitives/FormButtons"; import { Header3 } from "~/components/primitives/Headers"; -import { NavBar, PageAccessories, PageTitle } from "~/components/primitives/PageHeader"; +import { + NavBar, + PageAccessories, + PageTitle, +} from "~/components/primitives/PageHeader"; import { PaginationControls } from "~/components/primitives/Pagination"; import { Paragraph } from "~/components/primitives/Paragraph"; import { PopoverMenuItem } from "~/components/primitives/Popover"; @@ -55,7 +67,10 @@ import { useAutoRevalidate } from "~/hooks/useAutoRevalidate"; import { useEnvironment } from "~/hooks/useEnvironment"; import { useOrganization } from "~/hooks/useOrganizations"; import { useProject } from "~/hooks/useProject"; -import { redirectWithErrorMessage, redirectWithSuccessMessage } from "~/models/message.server"; +import { + redirectWithErrorMessage, + redirectWithSuccessMessage, +} from "~/models/message.server"; import { findProjectBySlug } from "~/models/project.server"; import { findEnvironmentBySlug } from "~/models/runtimeEnvironment.server"; import { EnvironmentQueuePresenter } from "~/presenters/v3/EnvironmentQueuePresenter.server"; @@ -64,12 +79,18 @@ import { QueueMetricsPresenter, type QueueListMetric, } from "~/presenters/v3/QueueMetricsPresenter.server"; -import { TimeFilter, timeFilterFromTo } from "~/components/runs/v3/SharedFilters"; +import { + TimeFilter, + timeFilterFromTo, +} from "~/components/runs/v3/SharedFilters"; import { useSearchParams } from "~/hooks/useSearchParam"; import { parseFiniteInt } from "~/utils/searchParams"; import { MiniLineChart } from "~/components/metrics/MiniLineChart"; import { buildActivityTimeAxis } from "~/components/primitives/charts/activityTimeAxis"; -import { Chart, type ChartConfig } from "~/components/primitives/charts/ChartCompound"; +import { + Chart, + type ChartConfig, +} from "~/components/primitives/charts/ChartCompound"; import { ChartCard } from "~/components/primitives/charts/ChartCard"; import { ChartSyncProvider } from "~/components/primitives/charts/ChartSyncContext"; import { useZoomToTimeFilter } from "~/hooks/useZoomToTimeFilter"; @@ -115,6 +136,7 @@ import { import { queueMetricsMaxPeriodDays } from "~/components/queues/queueMetricsPeriod.server"; import { isQueueAtCapacity } from "~/components/queues/queue-thresholds"; import { pageMeta } from "~/utils/pageTitle"; +import { InlineCode } from "~/components/code/InlineCode"; const SearchParamsSchema = z.object({ query: z.string().optional(), @@ -145,14 +167,19 @@ export const meta = pageMeta("Queues"); export const loader = async ({ request, params }: LoaderFunctionArgs) => { const userId = await requireUserId(request); - const { organizationSlug, projectParam, envParam } = EnvironmentParamSchema.parse(params); + const { organizationSlug, projectParam, envParam } = + EnvironmentParamSchema.parse(params); const url = new URL(request.url); const { page, query, period, from, to, sort } = SearchParamsSchema.parse( - Object.fromEntries(url.searchParams) + Object.fromEntries(url.searchParams), ); - const project = await findProjectBySlug(organizationSlug, projectParam, userId); + const project = await findProjectBySlug( + organizationSlug, + projectParam, + userId, + ); if (!project) { throw new Response(undefined, { status: 404, @@ -181,7 +208,7 @@ export const loader = async ({ request, params }: LoaderFunctionArgs) => { : QUEUE_METRICS_RETENTION_DAYS; const defaultPeriod = clampQueueMetricsPeriod( queueMetricsPeriodFromRequest(request), - maxPeriodDays + maxPeriodDays, ); try { @@ -213,18 +240,23 @@ export const loader = async ({ request, params }: LoaderFunctionArgs) => { try { const presenter = new QueueMetricsPresenter(); const queueNames = queues.queues.map((q) => - q.type === "task" ? `task/${q.name}` : q.name + q.type === "task" ? `task/${q.name}` : q.name, ); const timeRange = clipQueueMetricsWindow( timeFilterFromTo({ period: - resolveQueueMetricsPeriod({ period, from, to, defaultPeriod, maxPeriodDays }) ?? - undefined, + resolveQueueMetricsPeriod({ + period, + from, + to, + defaultPeriod, + maxPeriodDays, + }) ?? undefined, from: parseFiniteInt(from), to: parseFiniteInt(to), defaultPeriod, }), - maxPeriodDays + maxPeriodDays, ); const queueMetrics = queueNames.length > 0 @@ -243,18 +275,25 @@ export const loader = async ({ request, params }: LoaderFunctionArgs) => { }; } } catch (error) { - logger.warn("Queue list metrics unavailable, rendering without them", { error }); + logger.warn("Queue list metrics unavailable, rendering without them", { + error, + }); } } // Allocation summary (Environment limit + Allocated tiles) is additive; a presenter // failure must not 400 the page, so fail open to null like the metrics block above. - let allocation: Awaited> | null = null; + let allocation: Awaited< + ReturnType + > | null = null; if (queueMetricsUiEnabled) { try { allocation = await new QueueAllocationPresenter().call({ environment }); } catch (error) { - logger.warn("Queue allocation summary unavailable, rendering without it", { error }); + logger.warn( + "Queue allocation summary unavailable, rendering without it", + { error }, + ); } } @@ -272,7 +311,8 @@ export const loader = async ({ request, params }: LoaderFunctionArgs) => { console.error(error); throw new Response(undefined, { status: 400, - statusText: "Something went wrong, if this problem persists please contact support.", + statusText: + "Something went wrong, if this problem persists please contact support.", }); } }; @@ -283,13 +323,18 @@ export const action = async ({ request, params }: ActionFunctionArgs) => { return redirectWithErrorMessage( `/orgs/${params.organizationSlug}/projects/${params.projectParam}/env/${params.envParam}/queues`, request, - "Wrong method" + "Wrong method", ); } - const { organizationSlug, projectParam, envParam } = EnvironmentParamSchema.parse(params); + const { organizationSlug, projectParam, envParam } = + EnvironmentParamSchema.parse(params); - const project = await findProjectBySlug(organizationSlug, projectParam, userId); + const project = await findProjectBySlug( + organizationSlug, + projectParam, + userId, + ); if (!project) { throw new Response(undefined, { status: 404, @@ -312,7 +357,11 @@ export const action = async ({ request, params }: ActionFunctionArgs) => { const redirectPath = `/orgs/${organizationSlug}/projects/${projectParam}/env/${envParam}/queues${url.search}`; if (environment.archivedAt) { - return redirectWithErrorMessage(redirectPath, request, "This branch is archived"); + return redirectWithErrorMessage( + redirectPath, + request, + "This branch is archived", + ); } // Per-queue actions (pause/resume/override/remove-override) are shared with the queue detail @@ -335,7 +384,11 @@ export const action = async ({ request, params }: ActionFunctionArgs) => { if (!result.success) { return redirectWithErrorMessage(redirectPath, request, result.error); } - return redirectWithSuccessMessage(redirectPath, request, "Environment paused"); + return redirectWithSuccessMessage( + redirectPath, + request, + "Environment paused", + ); } case "environment-resume": { const resumeService = new PauseEnvironmentService(); @@ -343,10 +396,18 @@ export const action = async ({ request, params }: ActionFunctionArgs) => { if (!result.success) { return redirectWithErrorMessage(redirectPath, request, result.error); } - return redirectWithSuccessMessage(redirectPath, request, "Environment resumed"); + return redirectWithSuccessMessage( + redirectPath, + request, + "Environment resumed", + ); } default: - return redirectWithErrorMessage(redirectPath, request, "Something went wrong"); + return redirectWithErrorMessage( + redirectPath, + request, + "Something went wrong", + ); } }; @@ -359,14 +420,19 @@ function getEnvConcurrencyLimitStatus(environment: { burstFactor: number; }) { const limitStatus = - environment.running === environment.concurrencyLimit * environment.burstFactor + environment.running === + environment.concurrencyLimit * environment.burstFactor ? "limit" : environment.running > environment.concurrencyLimit ? "burst" : "within"; const limitClassName = - limitStatus === "burst" ? "text-warning" : limitStatus === "limit" ? "text-error" : undefined; + limitStatus === "burst" + ? "text-warning" + : limitStatus === "limit" + ? "text-error" + : undefined; return { limitStatus, limitClassName }; } @@ -375,7 +441,11 @@ export default function Page() { // Per-org flag decides which whole page renders. Off => the classic Queues page, // byte-for-byte the pre-metrics UI. Each branch is its own component (own hooks). const { queueMetricsUiEnabled } = useTypedLoaderData(); - return queueMetricsUiEnabled ? : ; + return queueMetricsUiEnabled ? ( + + ) : ( + + ); } function QueuesWithMetricsView() { @@ -435,20 +505,23 @@ function QueuesWithMetricsView() { defaultPeriod: QUEUE_LIVE_BLOCKS_PERIOD, fillGaps: false, refreshIntervalMs: 15_000, - } + }, ); const lastLiveBlockRow = liveBlockRows.length > 0 ? liveBlockRows[liveBlockRows.length - 1] : null; // Only trust the gauge while its newest bucket is fresh. A row painted from the hook's cache on // client-side nav-back (responseCache), or a quiet env whose latest bucket is minutes old, must // not override the loader's Redis-exact live values with a stale count. - const lastLiveBucketMs = lastLiveBlockRow ? tileTimeToMs(lastLiveBlockRow.t) : NaN; + const lastLiveBucketMs = lastLiveBlockRow + ? tileTimeToMs(lastLiveBlockRow.t) + : NaN; const liveBlockIsFresh = useIsMetricResponseFresh( responseReceivedAt, lastLiveBucketMs, - LIVE_GAUGE_FRESH_MS + LIVE_GAUGE_FRESH_MS, ); - const freshLiveBlockRow = lastLiveBlockRow && liveBlockIsFresh ? lastLiveBlockRow : null; + const freshLiveBlockRow = + lastLiveBlockRow && liveBlockIsFresh ? lastLiveBlockRow : null; const envQueuedLive = freshLiveBlockRow ? tileNumber(freshLiveBlockRow.env_queued) : environment.queued; @@ -461,7 +534,8 @@ function QueuesWithMetricsView() { const envLimit = environment.concurrencyLimit; const burstLimit = Math.round(envLimit * environment.burstFactor); const allocated = allocation?.allocated ?? 0; - const allocationPct = envLimit > 0 ? Math.round((allocated / envLimit) * 100) : 0; + const allocationPct = + envLimit > 0 ? Math.round((allocated / envLimit) * 100) : 0; // Running-block tinting (burst/limit) tracks the live running value, not the loader snapshot. const { limitStatus, limitClassName } = getEnvConcurrencyLimitStatus({ @@ -522,7 +596,11 @@ function QueuesWithMetricsView() { paused : undefined} + suffix={ + env.paused ? ( + paused + ) : undefined + } animate accessory={ @@ -540,7 +618,9 @@ function QueuesWithMetricsView() { /> } - valueClassName={env.paused ? "text-warning tabular-nums" : "tabular-nums"} + valueClassName={ + env.paused ? "text-warning tabular-nums" : "tabular-nums" + } compactThreshold={1000000} /> - Including {envRunningLive - environment.concurrencyLimit} burst runs{" "} - + Including {envRunningLive - environment.concurrencyLimit}{" "} + burst runs ) : limitStatus === "limit" ? ( "At concurrency limit" @@ -592,13 +675,21 @@ function QueuesWithMetricsView() { } value={allocation ? allocated : undefined} formattedValue={allocation ? undefined : "–"} - suffix={allocation ? `${allocationPct}% of the environment limit` : undefined} + suffix={ + allocation + ? `${allocationPct}% of the environment limit` + : undefined + } suffixClassName="text-text-dimmed" /> 1 ? `bursts up to ${burstLimit}` : undefined} + suffix={ + environment.burstFactor > 1 + ? `bursts up to ${burstLimit}` + : undefined + } suffixClassName="text-text-dimmed" accessory={ plan ? ( @@ -613,7 +704,10 @@ function QueuesWithMetricsView() { ) : ( Limit @@ -719,16 +814,17 @@ function QueuesWithMetricsView() { tooltip={

- Environment: uses the environment - limit of {environment.concurrencyLimit}. + Environment: + uses the environment limit of{" "} + {environment.concurrencyLimit}.

- User: a limit you set in your - code. + User: a limit + you set in your code.

- Override: a limit you set here or - via the API. + Override: a + limit you set here or via the API.

} @@ -748,8 +844,8 @@ function QueuesWithMetricsView() { disableTooltipHoverableContent tooltip={ <> - How many runs were waiting, over the selected time. marks - where the queue was throttled. + How many runs were waiting, over the selected time.{" "} + marks where the queue was throttled. } > @@ -763,16 +859,22 @@ function QueuesWithMetricsView() { {queueRows.length > 0 ? ( queueRows.map((queue) => { - const limit = queue.concurrencyLimit ?? environment.concurrencyLimit; + const limit = + queue.concurrencyLimit ?? environment.concurrencyLimit; const isAtConcurrencyLimit = queue.running >= limit; const isAtQueueLimit = environment.queueSizeLimit !== null && queue.queued >= environment.queueSizeLimit; const queueFilterableName = queueMetricsKey(queue); const queueMetric = metricsByQueue[queueFilterableName]; - const queueDetailPath = v3QueuePath(organization, project, env, { - friendlyId: queue.id, - }); + const queueDetailPath = v3QueuePath( + organization, + project, + env, + { + friendlyId: queue.id, + }, + ); return ( ) : ( ) @@ -811,7 +913,9 @@ function QueuesWithMetricsView() { trailingContent={ isAtConcurrencyLimit ? ( } + button={ + + } content="At concurrency limit: this queue is running as many runs as its limit allows; new runs wait in the backlog." className="max-w-[230px]" disableHoverableContent @@ -820,11 +924,16 @@ function QueuesWithMetricsView() { } > - + {queue.name} {queue.paused ? ( - + Paused ) : null} @@ -842,7 +951,7 @@ function QueuesWithMetricsView() { className={cn( "w-[1%]", queue.paused ? "opacity-50" : undefined, - isAtQueueLimit && "text-error" + isAtQueueLimit && "text-error", )} > {queue.queued} @@ -858,10 +967,10 @@ function QueuesWithMetricsView() { (queue.concurrency.combined.running ?? 0) >= Math.min( queue.concurrency.combined.current, - environment.concurrencyLimit + environment.concurrencyLimit, ) ? "text-warning" - : queue.running > 0 && "text-text-bright" + : queue.running > 0 && "text-text-bright", )} > {queue.running} @@ -873,7 +982,8 @@ function QueuesWithMetricsView() { className={cn( "w-[1%]", queue.paused ? "opacity-50" : undefined, - queue.concurrency?.overriddenAt && "font-medium text-text-bright" + queue.concurrency?.overriddenAt && + "font-medium text-text-bright", )} // The combined-limit hint is a tooltip button, so it renders beside the // link (trailing) rather than nested inside the ; the number stays the @@ -888,7 +998,7 @@ function QueuesWithMetricsView() { ( {Math.min( queue.concurrency.combined.current, - environment.concurrencyLimit + environment.concurrencyLimit, )} ) @@ -898,10 +1008,11 @@ function QueuesWithMetricsView() { Combined limit: at most{" "} {Math.min( queue.concurrency.combined.current, - environment.concurrencyLimit + environment.concurrencyLimit, )}{" "} - runs across all concurrency keys of this queue. The main limit - applies to each key separately. + runs across all concurrency keys of this + queue. The main limit applies to each key + separately. } className="max-w-[260px]" @@ -913,7 +1024,11 @@ function QueuesWithMetricsView() { <> {limit} - ({formatOverridePercent(queue.concurrencyLimitOverridePercent)}%) + ( + {formatOverridePercent( + queue.concurrencyLimitOverridePercent, + )} + %) ) : ( @@ -924,7 +1039,10 @@ function QueuesWithMetricsView() { to={queueDetailPath} alignment="right" actionClassName="pl-16" - className={cn("w-[1%]", queue.paused ? "opacity-50" : undefined)} + className={cn( + "w-[1%]", + queue.paused ? "opacity-50" : undefined, + )} // Keep the whole row navigable: the override explainer is a tooltip // button, so it renders beside the link (trailing) rather than nested // inside the , and the label itself stays the link. @@ -934,7 +1052,7 @@ function QueuesWithMetricsView() { content={ queue.concurrencyLimitOverridePercent !== null ? `Overridden at ${formatOverridePercent( - queue.concurrencyLimitOverridePercent + queue.concurrencyLimitOverridePercent, )}% of the environment limit.` : `This queue's concurrency limit has been manually overridden to ${limit}.` } @@ -994,7 +1112,9 @@ function QueuesWithMetricsView() { peakTooltip={ queueMetric && queueMetric.throttledTotal > 0 ? `Peak queued; this queue was throttled ${queueMetric.throttledTotal.toLocaleString()} ${ - queueMetric.throttledTotal === 1 ? "time" : "times" + queueMetric.throttledTotal === 1 + ? "time" + : "times" } in this period` : "Peak queued in this period" } @@ -1002,8 +1122,16 @@ function QueuesWithMetricsView() { } - hiddenButtons={!queue.paused && } + visibleButtons={ + queue.paused && ( + + ) + } + hiddenButtons={ + !queue.paused && ( + + ) + } popoverContent={ <> {queue.paused ? ( @@ -1056,7 +1184,9 @@ function QueuesWithMetricsView() { /> } @@ -1069,7 +1199,9 @@ function QueuesWithMetricsView() {
- {hasFilters ? "No queues found matching your filters" : "No queues found"} + {hasFilters + ? "No queues found matching your filters" + : "No queues found"}
@@ -1099,7 +1231,8 @@ function EnvironmentPauseResumeButton({ }, [navigation.state]); const isLoading = Boolean( - navigation.formData?.get("action") === (env.paused ? "environment-resume" : "environment-pause") + navigation.formData?.get("action") === + (env.paused ? "environment-resume" : "environment-pause"), ); return ( @@ -1114,7 +1247,9 @@ function EnvironmentPauseResumeButton({ type="button" variant="secondary/small" LeadingIcon={env.paused ? PlayIcon : PauseIcon} - leadingIconClassName={env.paused ? "text-success" : "text-warning"} + leadingIconClassName={ + env.paused ? "text-success" : "text-warning" + } className={ env.paused ? "border-success/60 text-success [&_span]:text-success hover:border-success" @@ -1142,13 +1277,15 @@ function EnvironmentPauseResumeButton({
- {env.paused ? "Resume environment?" : "Pause environment?"} + + {env.paused ? "Resume environment?" : "Pause environment?"} +
{env.paused ? `This will allow runs to be dequeued in ${environmentFullTitle(env)} again.` : `This will pause all runs from being dequeued in ${environmentFullTitle( - env + env, )}. Any executing runs will continue to run.`}
setIsOpen(false)}> @@ -1164,7 +1301,13 @@ function EnvironmentPauseResumeButton({ disabled={isLoading} variant={env.paused ? "primary/medium" : "danger/medium"} LeadingIcon={ - isLoading ? : env.paused ? PlayIcon : PauseIcon + isLoading ? ( + + ) : env.paused ? ( + PlayIcon + ) : ( + PauseIcon + ) } shortcut={{ modifiers: ["mod"], key: "enter" }} > @@ -1188,7 +1331,7 @@ function EnvironmentPauseResumeButton({ export function isEnvironmentPauseResumeFormSubmission( formMethod: string | undefined, - formData: FormData | undefined + formData: FormData | undefined, ) { if (!formMethod || !formData) { return false; @@ -1202,7 +1345,13 @@ export function isEnvironmentPauseResumeFormSubmission( } export function QueueFilters() { - return ; + return ( + + ); } type MetricTileRow = Record; @@ -1277,7 +1426,10 @@ function tileTimeToMs(value: number | string | null): number { /** Peak of a series, ignoring the buckets it has nothing to say about. */ function peakOf(points: TilePoint[]): number { - return points.reduce((max, p) => (p.value === null ? max : Math.max(max, p.value)), 0); + return points.reduce( + (max, p) => (p.value === null ? max : Math.max(max, p.value)), + 0, + ); } const SCHEDULING_DELAY_QUERY = `SELECT timeBucket() AS t,\n round(quantilesTDigestMerge(0.5, 0.9, 0.95, 0.99)(wait_quantiles)[3]) AS p95,\n sum(wait_ms_count) AS samples\nFROM env_metrics\nGROUP BY t\nORDER BY t`; @@ -1290,8 +1442,8 @@ const QUEUE_HEADER_TILES: QueueHeaderTile[] = [ label: "Env saturation", description: ( <> - How much of the environment's concurrency is in use. Turns above 100%, - when it's into burst capacity. + How much of the environment's concurrency is in use. Turns{" "} + above 100%, when it's into burst capacity. ), color: "var(--color-queues-chart)", @@ -1300,17 +1452,23 @@ const QUEUE_HEADER_TILES: QueueHeaderTile[] = [ { color: "var(--color-warning)", label: "Over limit" }, ], query: `SELECT timeBucket() AS t,\n max(max_env_running) AS running,\n max(max_env_limit) AS env_limit\nFROM env_metrics\nGROUP BY t\nORDER BY t`, - formatValue: (v) => (v > 100 ? `${v}% — over the environment limit` : `${v}%`), + formatValue: (v) => + v > 100 ? `${v}% — over the environment limit` : `${v}%`, formatAxis: (v) => `${v}%`, derive: (rows) => { const points = rows.map((r) => { const limit = tileNumber(r.env_limit); return { bucket: tileTimeToMs(r.t), - value: limit > 0 ? Math.round((tileNumber(r.running) / limit) * 100) : 0, + value: + limit > 0 ? Math.round((tileNumber(r.running) / limit) * 100) : 0, }; }); - return { points, total: peakOf(points), formatTotal: (v) => `${v}% peak` }; + return { + points, + total: peakOf(points), + formatTotal: (v) => `${v}% peak`, + }; }, }, { @@ -1324,7 +1482,11 @@ const QUEUE_HEADER_TILES: QueueHeaderTile[] = [ bucket: tileTimeToMs(r.t), value: tileNumber(r.queued), })); - return { points, total: peakOf(points), formatTotal: (v) => `${v.toLocaleString()} peak` }; + return { + points, + total: peakOf(points), + formatTotal: (v) => `${v.toLocaleString()} peak`, + }; }, }, { @@ -1332,8 +1494,8 @@ const QUEUE_HEADER_TILES: QueueHeaderTile[] = [ label: "Scheduling delay p95", description: ( <> - How long runs wait before they start (95% start faster than this). Turns {" "} - above 1 minute. + How long runs wait before they start (95% start faster than this). Turns{" "} + above 1 minute. ), totalTooltip: "The worst p95 in the selected window.", @@ -1362,8 +1524,9 @@ const QUEUE_HEADER_TILES: QueueHeaderTile[] = [ */ derive: (rows) => { const worst = rows.reduce( - (max, r) => (tileNumber(r.samples) > 0 ? Math.max(max, tileNumber(r.p95)) : max), - 0 + (max, r) => + tileNumber(r.samples) > 0 ? Math.max(max, tileNumber(r.p95)) : max, + 0, ); return { total: worst, @@ -1377,7 +1540,8 @@ const QUEUE_HEADER_TILES: QueueHeaderTile[] = [ id: "throttled", label: "Throttled", description: "How often runs were held back by a limit.", - totalTooltip: "The share of the selected window with at least one blocked dequeue.", + totalTooltip: + "The share of the selected window with at least one blocked dequeue.", color: "var(--color-queues-chart)", legend: [{ color: "var(--color-warning)", label: "Throttled" }], query: THROTTLED_QUERY, @@ -1398,7 +1562,8 @@ const QUEUE_HEADER_TILES: QueueHeaderTile[] = [ */ derive: (rows) => { const nonzero = rows.filter((r) => tileNumber(r.throttled) > 0).length; - const pct = rows.length > 0 ? Math.round((nonzero / rows.length) * 100) : 0; + const pct = + rows.length > 0 ? Math.round((nonzero / rows.length) * 100) : 0; return { total: pct, formatTotal: (v) => `${v}% of current period`, @@ -1464,10 +1629,13 @@ function QueueEnvMetricChart({ const derived = tile.derive(rows); const points = derived.points; - const plottedBucketMs = points.length > 1 ? points[1]!.bucket - points[0]!.bucket : 0; + const plottedBucketMs = + points.length > 1 ? points[1]!.bucket - points[0]!.bucket : 0; const floorWidenedBuckets = - plottedBucketMs > 0 && plottedBucketMs <= HERO_CHART_MIN_BUCKET_SECONDS * 1000; - const readoutQuery = tile.readout && floorWidenedBuckets ? tile.readout.query : ""; + plottedBucketMs > 0 && + plottedBucketMs <= HERO_CHART_MIN_BUCKET_SECONDS * 1000; + const readoutQuery = + tile.readout && floorWidenedBuckets ? tile.readout.query : ""; const readoutResult = useMetricResourceQuery(readoutQuery, sharedOptions); const { total, formatTotal, totalClassName } = tile.readout @@ -1486,11 +1654,12 @@ function QueueEnvMetricChart({ const chartConfig = useMemo( () => ({ [tile.id]: { label: tile.label, color: lineColor } }), - [tile.id, tile.label, lineColor] + [tile.id, tile.label, lineColor], ); const { tickFormatter, tooltipLabelFormatter } = buildActivityTimeAxis(data); - const hasData = data.length > 0 && data.some((p) => Number(p[tile.id] ?? 0) > 0); + const hasData = + data.length > 0 && data.some((p) => Number(p[tile.id] ?? 0) > 0); // Peak readout lives in the card title (ChartCard has no dedicated value slot). A zero/empty // total renders no readout at all (skipping "0% peak", "0 peak", "0" and the p95 "–" placeholder) @@ -1525,7 +1694,7 @@ function QueueEnvMetricChart({ {peak} @@ -1539,7 +1708,7 @@ function QueueEnvMetricChart({ {peak} @@ -1586,7 +1755,9 @@ function QueueEnvMetricChart({ thresholdStroke={thresholdStroke} warningOverlay={warningOverlay} xAxisProps={{ tickFormatter }} - yAxisProps={tile.formatAxis ? { tickFormatter: tile.formatAxis } : undefined} + yAxisProps={ + tile.formatAxis ? { tickFormatter: tile.formatAxis } : undefined + } tooltipLabelFormatter={tooltipLabelFormatter} tooltipValueFormatter={tile.formatValue} /> @@ -1618,11 +1789,21 @@ type QueueHealth = { limit: number; }; -type QueueHealthLabel = "Paused" | "At capacity" | "Backlogged" | "Active" | "Idle"; +type QueueHealthLabel = + | "Paused" + | "At capacity" + | "Backlogged" + | "Active" + | "Idle"; // Single source of truth for the queue health decision, shared by the badge and the table's // health-column sort so the sorted order always matches the labels shown. -function queueHealthLabel({ paused, running, queued, limit }: QueueHealth): QueueHealthLabel { +function queueHealthLabel({ + paused, + running, + queued, + limit, +}: QueueHealth): QueueHealthLabel { if (paused) return "Paused"; if (isQueueAtCapacity({ running, queued, limit })) return "At capacity"; if (queued > 0) return "Backlogged"; @@ -1633,8 +1814,10 @@ function queueHealthLabel({ paused, running, queued, limit }: QueueHealth): Queu // Tint + colored text, sized like the error status chips (see ErrorStatusBadge). const QUEUE_HEALTH_STYLES: Record = { Paused: "bg-warning/10 text-warning system:bg-warning system:text-white", - "At capacity": "bg-warning/10 text-warning system:bg-warning system:text-white", - Backlogged: "bg-blue-500/10 text-blue-500 system:bg-blue-500 system:text-white", + "At capacity": + "bg-warning/10 text-warning system:bg-warning system:text-white", + Backlogged: + "bg-blue-500/10 text-blue-500 system:bg-blue-500 system:text-white", Active: "bg-success/10 text-success system:bg-success system:text-white", Idle: "bg-charcoal-500/10 text-text-dimmed system:bg-charcoal-500 system:text-white", }; @@ -1645,7 +1828,7 @@ function QueueHealthBadge(health: QueueHealth) { {label} @@ -1667,14 +1850,21 @@ function formatWaitMs(ms: number): string { // Drop a trailing ".00" from whole percentages so "50.00" reads as "50" but "12.50" is preserved. function formatOverridePercent(percent: number): string { - return Number.isInteger(percent) ? percent.toString() : percent.toFixed(2).replace(/\.?0+$/, ""); + return Number.isInteger(percent) + ? percent.toString() + : percent.toFixed(2).replace(/\.?0+$/, ""); } // Classic Queues page, restored verbatim from before the Queue Metrics feature. Rendered // when queueMetricsUiEnabled is off so a gated org sees exactly the pre-metrics UI. function ClassicQueuesView() { - const { environment, queues, pagination, hasFilters, autoReloadPollIntervalMs } = - useTypedLoaderData(); + const { + environment, + queues, + pagination, + hasFilters, + autoReloadPollIntervalMs, + } = useTypedLoaderData(); const organization = useOrganization(); const project = useProject(); @@ -1683,7 +1873,8 @@ function ClassicQueuesView() { useAutoRevalidate({ interval: autoReloadPollIntervalMs, onFocus: true }); - const { limitStatus, limitClassName } = getEnvConcurrencyLimitStatus(environment); + const { limitStatus, limitClassName } = + getEnvConcurrencyLimitStatus(environment); return ( @@ -1708,7 +1899,11 @@ function ClassicQueuesView() { paused : undefined} + suffix={ + env.paused ? ( + paused + ) : undefined + } animate accessory={
@@ -1730,7 +1925,9 @@ function ClassicQueuesView() { />
} - valueClassName={env.paused ? "text-warning tabular-nums" : "tabular-nums"} + valueClassName={ + env.paused ? "text-warning tabular-nums" : "tabular-nums" + } compactThreshold={1000000} /> - Including {environment.running - environment.concurrencyLimit} burst runs{" "} - + Including{" "} + {environment.running - environment.concurrencyLimit} burst + runs ) : limitStatus === "limit" ? ( "At concurrency limit" @@ -1779,17 +1977,19 @@ function ClassicQueuesView() { - Burst limit {environment.burstFactor * environment.concurrencyLimit}{" "} + Burst limit{" "} + {environment.burstFactor * environment.concurrencyLimit}{" "} ) : undefined } accessory={ plan ? ( - plan?.v3Subscription?.plan?.limits.concurrentRuns.canExceed ? ( + plan?.v3Subscription?.plan?.limits.concurrentRuns + .canExceed ? ( ) : (
@@ -1831,7 +2040,7 @@ function ClassicQueuesView() { Running Limit @@ -1847,8 +2056,8 @@ function ClassicQueuesView() { className="text-wrap! text-text-dimmed" spacing > - This queue is limited by your environment's concurrency limit of{" "} - {environment.concurrencyLimit}. + This queue is limited by your environment's + concurrency limit of {environment.concurrencyLimit}.
@@ -1858,7 +2067,8 @@ function ClassicQueuesView() { className="text-wrap! text-text-dimmed" spacing > - This queue is limited by a concurrency limit set in your code. + This queue is limited by a concurrency limit set in + your code.
@@ -1868,8 +2078,8 @@ function ClassicQueuesView() { className="text-wrap! text-text-dimmed" spacing > - This queue's concurrency limit has been manually overridden from the - dashboard or API. + This queue's concurrency limit has been manually + overridden from the dashboard or API.
@@ -1885,7 +2095,8 @@ function ClassicQueuesView() { {queues.length > 0 ? ( queues.map((queue) => { - const limit = queue.concurrencyLimit ?? environment.concurrencyLimit; + const limit = + queue.concurrencyLimit ?? environment.concurrencyLimit; const isAtConcurrencyLimit = queue.running >= limit; const isAtQueueLimit = environment.queueSizeLimit !== null && @@ -1901,7 +2112,10 @@ function ClassicQueuesView() { {queue.concurrency?.overriddenAt ? ( + Concurrency limit overridden } @@ -1911,17 +2125,26 @@ function ClassicQueuesView() { /> ) : null} {queue.paused ? ( - + Paused ) : null} {isAtQueueLimit ? ( - + At queue limit ) : null} {isAtConcurrencyLimit ? ( - + At concurrency limit ) : null} @@ -1932,7 +2155,7 @@ function ClassicQueuesView() { className={cn( "w-[1%] pl-16 tabular-nums", queue.paused ? "opacity-50" : undefined, - isAtQueueLimit && "text-error" + isAtQueueLimit && "text-error", )} > {queue.queued} @@ -1946,11 +2169,11 @@ function ClassicQueuesView() { (queue.concurrency.combined.running ?? 0) >= Math.min( queue.concurrency.combined.current, - environment.concurrencyLimit + environment.concurrencyLimit, ) ? "text-warning" : queue.running > 0 && "text-text-bright", - isAtConcurrencyLimit && "text-warning" + isAtConcurrencyLimit && "text-warning", )} > {queue.running} @@ -1960,7 +2183,8 @@ function ClassicQueuesView() { className={cn( "w-[1%] pl-16 tabular-nums", queue.paused ? "opacity-50" : undefined, - queue.concurrency?.overriddenAt && "font-medium text-text-bright" + queue.concurrency?.overriddenAt && + "font-medium text-text-bright", )} > {limit} @@ -1973,7 +2197,7 @@ function ClassicQueuesView() { ( {Math.min( queue.concurrency.combined.current, - environment.concurrencyLimit + environment.concurrencyLimit, )} ) @@ -1983,10 +2207,11 @@ function ClassicQueuesView() { Combined limit: at most{" "} {Math.min( queue.concurrency.combined.current, - environment.concurrencyLimit + environment.concurrencyLimit, )}{" "} - runs across all concurrency keys of this queue. The main limit - applies to each key separately. + runs across all concurrency keys of this + queue. The main limit applies to each key + separately. } className="max-w-[260px]" @@ -1999,7 +2224,8 @@ function ClassicQueuesView() { "w-[1%] pl-16", queue.paused ? "opacity-50" : undefined, isAtConcurrencyLimit && "text-warning", - queue.concurrency?.overriddenAt && "font-medium text-text-bright" + queue.concurrency?.overriddenAt && + "font-medium text-text-bright", )} > {queue.concurrency?.overriddenAt ? ( @@ -2012,8 +2238,16 @@ function ClassicQueuesView() { } - hiddenButtons={!queue.paused && } + visibleButtons={ + queue.paused && ( + + ) + } + hiddenButtons={ + !queue.paused && ( + + ) + } popoverContent={ <> {queue.paused ? ( @@ -2066,7 +2300,9 @@ function ClassicQueuesView() { /> } @@ -2079,7 +2315,9 @@ function ClassicQueuesView() {
- {hasFilters ? "No queues found matching your filters" : "No queues found"} + {hasFilters + ? "No queues found matching your filters" + : "No queues found"}
@@ -2110,3 +2348,19 @@ function BurstFactorTooltip({ /> ); } + +const limitTooltip = ( + <> + + How many runs can execute at once.{" "} + + + 1 (20) means 1 run + per concurrency key, but at most 20 runs across all keys. Set using{" "} + + combinedConcurrencyLimit + {" "} + in your code. + + +); From f18de74a3bf4b43b0c15eab7b23ac41d5a3fb6b0 Mon Sep 17 00:00:00 2001 From: Matt Aitken Date: Mon, 31 Aug 2026 10:37:07 +0100 Subject: [PATCH 12/32] docs(core): combined.current is the declared cap, clamped at admit time Also reflows an import to the formatter's current output. --- .../route.tsx | 512 +++++------------- 1 file changed, 144 insertions(+), 368 deletions(-) diff --git a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.queues/route.tsx b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.queues/route.tsx index 748be64dee1..12b29480751 100644 --- a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.queues/route.tsx +++ b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.queues/route.tsx @@ -8,10 +8,7 @@ import { } from "@heroicons/react/20/solid"; import { DialogClose } from "@radix-ui/react-dialog"; import { Form, useNavigation } from "@remix-run/react"; -import { - type ActionFunctionArgs, - type LoaderFunctionArgs, -} from "@remix-run/server-runtime"; +import { type ActionFunctionArgs, type LoaderFunctionArgs } from "@remix-run/server-runtime"; import type { RuntimeEnvironmentType } from "@trigger.dev/database"; import { useEffect, useMemo, useState, type ReactNode } from "react"; import { QueuesIcon } from "~/assets/icons/QueuesIcon"; @@ -25,19 +22,10 @@ import { PageBody, PageContainer } from "~/components/layout/AppLayout"; import { MetricsLayout } from "~/components/layout/MetricsLayout"; import { Badge } from "~/components/primitives/Badge"; import { Button, LinkButton } from "~/components/primitives/Buttons"; -import { - Dialog, - DialogContent, - DialogHeader, - DialogTrigger, -} from "~/components/primitives/Dialog"; +import { Dialog, DialogContent, DialogHeader, DialogTrigger } from "~/components/primitives/Dialog"; import { FormButtons } from "~/components/primitives/FormButtons"; import { Header3 } from "~/components/primitives/Headers"; -import { - NavBar, - PageAccessories, - PageTitle, -} from "~/components/primitives/PageHeader"; +import { NavBar, PageAccessories, PageTitle } from "~/components/primitives/PageHeader"; import { PaginationControls } from "~/components/primitives/Pagination"; import { Paragraph } from "~/components/primitives/Paragraph"; import { PopoverMenuItem } from "~/components/primitives/Popover"; @@ -67,10 +55,7 @@ import { useAutoRevalidate } from "~/hooks/useAutoRevalidate"; import { useEnvironment } from "~/hooks/useEnvironment"; import { useOrganization } from "~/hooks/useOrganizations"; import { useProject } from "~/hooks/useProject"; -import { - redirectWithErrorMessage, - redirectWithSuccessMessage, -} from "~/models/message.server"; +import { redirectWithErrorMessage, redirectWithSuccessMessage } from "~/models/message.server"; import { findProjectBySlug } from "~/models/project.server"; import { findEnvironmentBySlug } from "~/models/runtimeEnvironment.server"; import { EnvironmentQueuePresenter } from "~/presenters/v3/EnvironmentQueuePresenter.server"; @@ -79,18 +64,12 @@ import { QueueMetricsPresenter, type QueueListMetric, } from "~/presenters/v3/QueueMetricsPresenter.server"; -import { - TimeFilter, - timeFilterFromTo, -} from "~/components/runs/v3/SharedFilters"; +import { TimeFilter, timeFilterFromTo } from "~/components/runs/v3/SharedFilters"; import { useSearchParams } from "~/hooks/useSearchParam"; import { parseFiniteInt } from "~/utils/searchParams"; import { MiniLineChart } from "~/components/metrics/MiniLineChart"; import { buildActivityTimeAxis } from "~/components/primitives/charts/activityTimeAxis"; -import { - Chart, - type ChartConfig, -} from "~/components/primitives/charts/ChartCompound"; +import { Chart, type ChartConfig } from "~/components/primitives/charts/ChartCompound"; import { ChartCard } from "~/components/primitives/charts/ChartCard"; import { ChartSyncProvider } from "~/components/primitives/charts/ChartSyncContext"; import { useZoomToTimeFilter } from "~/hooks/useZoomToTimeFilter"; @@ -167,19 +146,14 @@ export const meta = pageMeta("Queues"); export const loader = async ({ request, params }: LoaderFunctionArgs) => { const userId = await requireUserId(request); - const { organizationSlug, projectParam, envParam } = - EnvironmentParamSchema.parse(params); + const { organizationSlug, projectParam, envParam } = EnvironmentParamSchema.parse(params); const url = new URL(request.url); const { page, query, period, from, to, sort } = SearchParamsSchema.parse( - Object.fromEntries(url.searchParams), + Object.fromEntries(url.searchParams) ); - const project = await findProjectBySlug( - organizationSlug, - projectParam, - userId, - ); + const project = await findProjectBySlug(organizationSlug, projectParam, userId); if (!project) { throw new Response(undefined, { status: 404, @@ -208,7 +182,7 @@ export const loader = async ({ request, params }: LoaderFunctionArgs) => { : QUEUE_METRICS_RETENTION_DAYS; const defaultPeriod = clampQueueMetricsPeriod( queueMetricsPeriodFromRequest(request), - maxPeriodDays, + maxPeriodDays ); try { @@ -240,7 +214,7 @@ export const loader = async ({ request, params }: LoaderFunctionArgs) => { try { const presenter = new QueueMetricsPresenter(); const queueNames = queues.queues.map((q) => - q.type === "task" ? `task/${q.name}` : q.name, + q.type === "task" ? `task/${q.name}` : q.name ); const timeRange = clipQueueMetricsWindow( timeFilterFromTo({ @@ -256,7 +230,7 @@ export const loader = async ({ request, params }: LoaderFunctionArgs) => { to: parseFiniteInt(to), defaultPeriod, }), - maxPeriodDays, + maxPeriodDays ); const queueMetrics = queueNames.length > 0 @@ -283,17 +257,12 @@ export const loader = async ({ request, params }: LoaderFunctionArgs) => { // Allocation summary (Environment limit + Allocated tiles) is additive; a presenter // failure must not 400 the page, so fail open to null like the metrics block above. - let allocation: Awaited< - ReturnType - > | null = null; + let allocation: Awaited> | null = null; if (queueMetricsUiEnabled) { try { allocation = await new QueueAllocationPresenter().call({ environment }); } catch (error) { - logger.warn( - "Queue allocation summary unavailable, rendering without it", - { error }, - ); + logger.warn("Queue allocation summary unavailable, rendering without it", { error }); } } @@ -311,8 +280,7 @@ export const loader = async ({ request, params }: LoaderFunctionArgs) => { console.error(error); throw new Response(undefined, { status: 400, - statusText: - "Something went wrong, if this problem persists please contact support.", + statusText: "Something went wrong, if this problem persists please contact support.", }); } }; @@ -323,18 +291,13 @@ export const action = async ({ request, params }: ActionFunctionArgs) => { return redirectWithErrorMessage( `/orgs/${params.organizationSlug}/projects/${params.projectParam}/env/${params.envParam}/queues`, request, - "Wrong method", + "Wrong method" ); } - const { organizationSlug, projectParam, envParam } = - EnvironmentParamSchema.parse(params); + const { organizationSlug, projectParam, envParam } = EnvironmentParamSchema.parse(params); - const project = await findProjectBySlug( - organizationSlug, - projectParam, - userId, - ); + const project = await findProjectBySlug(organizationSlug, projectParam, userId); if (!project) { throw new Response(undefined, { status: 404, @@ -357,11 +320,7 @@ export const action = async ({ request, params }: ActionFunctionArgs) => { const redirectPath = `/orgs/${organizationSlug}/projects/${projectParam}/env/${envParam}/queues${url.search}`; if (environment.archivedAt) { - return redirectWithErrorMessage( - redirectPath, - request, - "This branch is archived", - ); + return redirectWithErrorMessage(redirectPath, request, "This branch is archived"); } // Per-queue actions (pause/resume/override/remove-override) are shared with the queue detail @@ -384,11 +343,7 @@ export const action = async ({ request, params }: ActionFunctionArgs) => { if (!result.success) { return redirectWithErrorMessage(redirectPath, request, result.error); } - return redirectWithSuccessMessage( - redirectPath, - request, - "Environment paused", - ); + return redirectWithSuccessMessage(redirectPath, request, "Environment paused"); } case "environment-resume": { const resumeService = new PauseEnvironmentService(); @@ -396,18 +351,10 @@ export const action = async ({ request, params }: ActionFunctionArgs) => { if (!result.success) { return redirectWithErrorMessage(redirectPath, request, result.error); } - return redirectWithSuccessMessage( - redirectPath, - request, - "Environment resumed", - ); + return redirectWithSuccessMessage(redirectPath, request, "Environment resumed"); } default: - return redirectWithErrorMessage( - redirectPath, - request, - "Something went wrong", - ); + return redirectWithErrorMessage(redirectPath, request, "Something went wrong"); } }; @@ -420,19 +367,14 @@ function getEnvConcurrencyLimitStatus(environment: { burstFactor: number; }) { const limitStatus = - environment.running === - environment.concurrencyLimit * environment.burstFactor + environment.running === environment.concurrencyLimit * environment.burstFactor ? "limit" : environment.running > environment.concurrencyLimit ? "burst" : "within"; const limitClassName = - limitStatus === "burst" - ? "text-warning" - : limitStatus === "limit" - ? "text-error" - : undefined; + limitStatus === "burst" ? "text-warning" : limitStatus === "limit" ? "text-error" : undefined; return { limitStatus, limitClassName }; } @@ -441,11 +383,7 @@ export default function Page() { // Per-org flag decides which whole page renders. Off => the classic Queues page, // byte-for-byte the pre-metrics UI. Each branch is its own component (own hooks). const { queueMetricsUiEnabled } = useTypedLoaderData(); - return queueMetricsUiEnabled ? ( - - ) : ( - - ); + return queueMetricsUiEnabled ? : ; } function QueuesWithMetricsView() { @@ -505,23 +443,20 @@ function QueuesWithMetricsView() { defaultPeriod: QUEUE_LIVE_BLOCKS_PERIOD, fillGaps: false, refreshIntervalMs: 15_000, - }, + } ); const lastLiveBlockRow = liveBlockRows.length > 0 ? liveBlockRows[liveBlockRows.length - 1] : null; // Only trust the gauge while its newest bucket is fresh. A row painted from the hook's cache on // client-side nav-back (responseCache), or a quiet env whose latest bucket is minutes old, must // not override the loader's Redis-exact live values with a stale count. - const lastLiveBucketMs = lastLiveBlockRow - ? tileTimeToMs(lastLiveBlockRow.t) - : NaN; + const lastLiveBucketMs = lastLiveBlockRow ? tileTimeToMs(lastLiveBlockRow.t) : NaN; const liveBlockIsFresh = useIsMetricResponseFresh( responseReceivedAt, lastLiveBucketMs, - LIVE_GAUGE_FRESH_MS, + LIVE_GAUGE_FRESH_MS ); - const freshLiveBlockRow = - lastLiveBlockRow && liveBlockIsFresh ? lastLiveBlockRow : null; + const freshLiveBlockRow = lastLiveBlockRow && liveBlockIsFresh ? lastLiveBlockRow : null; const envQueuedLive = freshLiveBlockRow ? tileNumber(freshLiveBlockRow.env_queued) : environment.queued; @@ -534,8 +469,7 @@ function QueuesWithMetricsView() { const envLimit = environment.concurrencyLimit; const burstLimit = Math.round(envLimit * environment.burstFactor); const allocated = allocation?.allocated ?? 0; - const allocationPct = - envLimit > 0 ? Math.round((allocated / envLimit) * 100) : 0; + const allocationPct = envLimit > 0 ? Math.round((allocated / envLimit) * 100) : 0; // Running-block tinting (burst/limit) tracks the live running value, not the loader snapshot. const { limitStatus, limitClassName } = getEnvConcurrencyLimitStatus({ @@ -596,11 +530,7 @@ function QueuesWithMetricsView() { paused - ) : undefined - } + suffix={env.paused ? paused : undefined} animate accessory={ @@ -618,9 +548,7 @@ function QueuesWithMetricsView() { /> } - valueClassName={ - env.paused ? "text-warning tabular-nums" : "tabular-nums" - } + valueClassName={env.paused ? "text-warning tabular-nums" : "tabular-nums"} compactThreshold={1000000} /> - Including {envRunningLive - environment.concurrencyLimit}{" "} - burst runs + Including {envRunningLive - environment.concurrencyLimit} burst runs{" "} + ) : limitStatus === "limit" ? ( "At concurrency limit" @@ -675,21 +600,13 @@ function QueuesWithMetricsView() { } value={allocation ? allocated : undefined} formattedValue={allocation ? undefined : "–"} - suffix={ - allocation - ? `${allocationPct}% of the environment limit` - : undefined - } + suffix={allocation ? `${allocationPct}% of the environment limit` : undefined} suffixClassName="text-text-dimmed" /> 1 - ? `bursts up to ${burstLimit}` - : undefined - } + suffix={environment.burstFactor > 1 ? `bursts up to ${burstLimit}` : undefined} suffixClassName="text-text-dimmed" accessory={ plan ? ( @@ -704,10 +621,7 @@ function QueuesWithMetricsView() { ) : (

- Environment: - uses the environment limit of{" "} - {environment.concurrencyLimit}. + Environment: uses the environment + limit of {environment.concurrencyLimit}.

- User: a limit - you set in your code. + User: a limit you set in your + code.

- Override: a - limit you set here or via the API. + Override: a limit you set here or + via the API.

} @@ -844,8 +756,8 @@ function QueuesWithMetricsView() { disableTooltipHoverableContent tooltip={ <> - How many runs were waiting, over the selected time.{" "} - marks where the queue was throttled. + How many runs were waiting, over the selected time. marks + where the queue was throttled. } > @@ -859,22 +771,16 @@ function QueuesWithMetricsView() { {queueRows.length > 0 ? ( queueRows.map((queue) => { - const limit = - queue.concurrencyLimit ?? environment.concurrencyLimit; + const limit = queue.concurrencyLimit ?? environment.concurrencyLimit; const isAtConcurrencyLimit = queue.running >= limit; const isAtQueueLimit = environment.queueSizeLimit !== null && queue.queued >= environment.queueSizeLimit; const queueFilterableName = queueMetricsKey(queue); const queueMetric = metricsByQueue[queueFilterableName]; - const queueDetailPath = v3QueuePath( - organization, - project, - env, - { - friendlyId: queue.id, - }, - ); + const queueDetailPath = v3QueuePath(organization, project, env, { + friendlyId: queue.id, + }); return ( ) : ( ) @@ -913,9 +819,7 @@ function QueuesWithMetricsView() { trailingContent={ isAtConcurrencyLimit ? ( - } + button={} content="At concurrency limit: this queue is running as many runs as its limit allows; new runs wait in the backlog." className="max-w-[230px]" disableHoverableContent @@ -924,16 +828,11 @@ function QueuesWithMetricsView() { } > - + {queue.name} {queue.paused ? ( - + Paused ) : null} @@ -951,7 +850,7 @@ function QueuesWithMetricsView() { className={cn( "w-[1%]", queue.paused ? "opacity-50" : undefined, - isAtQueueLimit && "text-error", + isAtQueueLimit && "text-error" )} > {queue.queued} @@ -967,10 +866,10 @@ function QueuesWithMetricsView() { (queue.concurrency.combined.running ?? 0) >= Math.min( queue.concurrency.combined.current, - environment.concurrencyLimit, + environment.concurrencyLimit ) ? "text-warning" - : queue.running > 0 && "text-text-bright", + : queue.running > 0 && "text-text-bright" )} > {queue.running} @@ -982,8 +881,7 @@ function QueuesWithMetricsView() { className={cn( "w-[1%]", queue.paused ? "opacity-50" : undefined, - queue.concurrency?.overriddenAt && - "font-medium text-text-bright", + queue.concurrency?.overriddenAt && "font-medium text-text-bright" )} // The combined-limit hint is a tooltip button, so it renders beside the // link (trailing) rather than nested inside the ; the number stays the @@ -998,7 +896,7 @@ function QueuesWithMetricsView() { ( {Math.min( queue.concurrency.combined.current, - environment.concurrencyLimit, + environment.concurrencyLimit )} ) @@ -1008,11 +906,10 @@ function QueuesWithMetricsView() { Combined limit: at most{" "} {Math.min( queue.concurrency.combined.current, - environment.concurrencyLimit, + environment.concurrencyLimit )}{" "} - runs across all concurrency keys of this - queue. The main limit applies to each key - separately. + runs across all concurrency keys of this queue. The main limit + applies to each key separately. } className="max-w-[260px]" @@ -1024,10 +921,7 @@ function QueuesWithMetricsView() { <> {limit} - ( - {formatOverridePercent( - queue.concurrencyLimitOverridePercent, - )} + ({formatOverridePercent(queue.concurrencyLimitOverridePercent)} %) @@ -1039,10 +933,7 @@ function QueuesWithMetricsView() { to={queueDetailPath} alignment="right" actionClassName="pl-16" - className={cn( - "w-[1%]", - queue.paused ? "opacity-50" : undefined, - )} + className={cn("w-[1%]", queue.paused ? "opacity-50" : undefined)} // Keep the whole row navigable: the override explainer is a tooltip // button, so it renders beside the link (trailing) rather than nested // inside the , and the label itself stays the link. @@ -1052,7 +943,7 @@ function QueuesWithMetricsView() { content={ queue.concurrencyLimitOverridePercent !== null ? `Overridden at ${formatOverridePercent( - queue.concurrencyLimitOverridePercent, + queue.concurrencyLimitOverridePercent )}% of the environment limit.` : `This queue's concurrency limit has been manually overridden to ${limit}.` } @@ -1112,9 +1003,7 @@ function QueuesWithMetricsView() { peakTooltip={ queueMetric && queueMetric.throttledTotal > 0 ? `Peak queued; this queue was throttled ${queueMetric.throttledTotal.toLocaleString()} ${ - queueMetric.throttledTotal === 1 - ? "time" - : "times" + queueMetric.throttledTotal === 1 ? "time" : "times" } in this period` : "Peak queued in this period" } @@ -1122,16 +1011,8 @@ function QueuesWithMetricsView() { - ) - } - hiddenButtons={ - !queue.paused && ( - - ) - } + visibleButtons={queue.paused && } + hiddenButtons={!queue.paused && } popoverContent={ <> {queue.paused ? ( @@ -1184,9 +1065,7 @@ function QueuesWithMetricsView() { /> } @@ -1199,9 +1078,7 @@ function QueuesWithMetricsView() {
- {hasFilters - ? "No queues found matching your filters" - : "No queues found"} + {hasFilters ? "No queues found matching your filters" : "No queues found"}
@@ -1231,8 +1108,7 @@ function EnvironmentPauseResumeButton({ }, [navigation.state]); const isLoading = Boolean( - navigation.formData?.get("action") === - (env.paused ? "environment-resume" : "environment-pause"), + navigation.formData?.get("action") === (env.paused ? "environment-resume" : "environment-pause") ); return ( @@ -1247,9 +1123,7 @@ function EnvironmentPauseResumeButton({ type="button" variant="secondary/small" LeadingIcon={env.paused ? PlayIcon : PauseIcon} - leadingIconClassName={ - env.paused ? "text-success" : "text-warning" - } + leadingIconClassName={env.paused ? "text-success" : "text-warning"} className={ env.paused ? "border-success/60 text-success [&_span]:text-success hover:border-success" @@ -1277,15 +1151,13 @@ function EnvironmentPauseResumeButton({ - - {env.paused ? "Resume environment?" : "Pause environment?"} - + {env.paused ? "Resume environment?" : "Pause environment?"}
{env.paused ? `This will allow runs to be dequeued in ${environmentFullTitle(env)} again.` : `This will pause all runs from being dequeued in ${environmentFullTitle( - env, + env )}. Any executing runs will continue to run.`} setIsOpen(false)}> @@ -1301,13 +1173,7 @@ function EnvironmentPauseResumeButton({ disabled={isLoading} variant={env.paused ? "primary/medium" : "danger/medium"} LeadingIcon={ - isLoading ? ( - - ) : env.paused ? ( - PlayIcon - ) : ( - PauseIcon - ) + isLoading ? : env.paused ? PlayIcon : PauseIcon } shortcut={{ modifiers: ["mod"], key: "enter" }} > @@ -1331,7 +1197,7 @@ function EnvironmentPauseResumeButton({ export function isEnvironmentPauseResumeFormSubmission( formMethod: string | undefined, - formData: FormData | undefined, + formData: FormData | undefined ) { if (!formMethod || !formData) { return false; @@ -1345,13 +1211,7 @@ export function isEnvironmentPauseResumeFormSubmission( } export function QueueFilters() { - return ( - - ); + return ; } type MetricTileRow = Record; @@ -1426,10 +1286,7 @@ function tileTimeToMs(value: number | string | null): number { /** Peak of a series, ignoring the buckets it has nothing to say about. */ function peakOf(points: TilePoint[]): number { - return points.reduce( - (max, p) => (p.value === null ? max : Math.max(max, p.value)), - 0, - ); + return points.reduce((max, p) => (p.value === null ? max : Math.max(max, p.value)), 0); } const SCHEDULING_DELAY_QUERY = `SELECT timeBucket() AS t,\n round(quantilesTDigestMerge(0.5, 0.9, 0.95, 0.99)(wait_quantiles)[3]) AS p95,\n sum(wait_ms_count) AS samples\nFROM env_metrics\nGROUP BY t\nORDER BY t`; @@ -1442,8 +1299,8 @@ const QUEUE_HEADER_TILES: QueueHeaderTile[] = [ label: "Env saturation", description: ( <> - How much of the environment's concurrency is in use. Turns{" "} - above 100%, when it's into burst capacity. + How much of the environment's concurrency is in use. Turns above 100%, + when it's into burst capacity. ), color: "var(--color-queues-chart)", @@ -1452,16 +1309,14 @@ const QUEUE_HEADER_TILES: QueueHeaderTile[] = [ { color: "var(--color-warning)", label: "Over limit" }, ], query: `SELECT timeBucket() AS t,\n max(max_env_running) AS running,\n max(max_env_limit) AS env_limit\nFROM env_metrics\nGROUP BY t\nORDER BY t`, - formatValue: (v) => - v > 100 ? `${v}% — over the environment limit` : `${v}%`, + formatValue: (v) => (v > 100 ? `${v}% — over the environment limit` : `${v}%`), formatAxis: (v) => `${v}%`, derive: (rows) => { const points = rows.map((r) => { const limit = tileNumber(r.env_limit); return { bucket: tileTimeToMs(r.t), - value: - limit > 0 ? Math.round((tileNumber(r.running) / limit) * 100) : 0, + value: limit > 0 ? Math.round((tileNumber(r.running) / limit) * 100) : 0, }; }); return { @@ -1494,8 +1349,8 @@ const QUEUE_HEADER_TILES: QueueHeaderTile[] = [ label: "Scheduling delay p95", description: ( <> - How long runs wait before they start (95% start faster than this). Turns{" "} - above 1 minute. + How long runs wait before they start (95% start faster than this). Turns {" "} + above 1 minute. ), totalTooltip: "The worst p95 in the selected window.", @@ -1524,9 +1379,8 @@ const QUEUE_HEADER_TILES: QueueHeaderTile[] = [ */ derive: (rows) => { const worst = rows.reduce( - (max, r) => - tileNumber(r.samples) > 0 ? Math.max(max, tileNumber(r.p95)) : max, - 0, + (max, r) => (tileNumber(r.samples) > 0 ? Math.max(max, tileNumber(r.p95)) : max), + 0 ); return { total: worst, @@ -1540,8 +1394,7 @@ const QUEUE_HEADER_TILES: QueueHeaderTile[] = [ id: "throttled", label: "Throttled", description: "How often runs were held back by a limit.", - totalTooltip: - "The share of the selected window with at least one blocked dequeue.", + totalTooltip: "The share of the selected window with at least one blocked dequeue.", color: "var(--color-queues-chart)", legend: [{ color: "var(--color-warning)", label: "Throttled" }], query: THROTTLED_QUERY, @@ -1562,8 +1415,7 @@ const QUEUE_HEADER_TILES: QueueHeaderTile[] = [ */ derive: (rows) => { const nonzero = rows.filter((r) => tileNumber(r.throttled) > 0).length; - const pct = - rows.length > 0 ? Math.round((nonzero / rows.length) * 100) : 0; + const pct = rows.length > 0 ? Math.round((nonzero / rows.length) * 100) : 0; return { total: pct, formatTotal: (v) => `${v}% of current period`, @@ -1629,13 +1481,10 @@ function QueueEnvMetricChart({ const derived = tile.derive(rows); const points = derived.points; - const plottedBucketMs = - points.length > 1 ? points[1]!.bucket - points[0]!.bucket : 0; + const plottedBucketMs = points.length > 1 ? points[1]!.bucket - points[0]!.bucket : 0; const floorWidenedBuckets = - plottedBucketMs > 0 && - plottedBucketMs <= HERO_CHART_MIN_BUCKET_SECONDS * 1000; - const readoutQuery = - tile.readout && floorWidenedBuckets ? tile.readout.query : ""; + plottedBucketMs > 0 && plottedBucketMs <= HERO_CHART_MIN_BUCKET_SECONDS * 1000; + const readoutQuery = tile.readout && floorWidenedBuckets ? tile.readout.query : ""; const readoutResult = useMetricResourceQuery(readoutQuery, sharedOptions); const { total, formatTotal, totalClassName } = tile.readout @@ -1654,12 +1503,11 @@ function QueueEnvMetricChart({ const chartConfig = useMemo( () => ({ [tile.id]: { label: tile.label, color: lineColor } }), - [tile.id, tile.label, lineColor], + [tile.id, tile.label, lineColor] ); const { tickFormatter, tooltipLabelFormatter } = buildActivityTimeAxis(data); - const hasData = - data.length > 0 && data.some((p) => Number(p[tile.id] ?? 0) > 0); + const hasData = data.length > 0 && data.some((p) => Number(p[tile.id] ?? 0) > 0); // Peak readout lives in the card title (ChartCard has no dedicated value slot). A zero/empty // total renders no readout at all (skipping "0% peak", "0 peak", "0" and the p95 "–" placeholder) @@ -1694,7 +1542,7 @@ function QueueEnvMetricChart({ {peak} @@ -1708,7 +1556,7 @@ function QueueEnvMetricChart({ {peak} @@ -1755,9 +1603,7 @@ function QueueEnvMetricChart({ thresholdStroke={thresholdStroke} warningOverlay={warningOverlay} xAxisProps={{ tickFormatter }} - yAxisProps={ - tile.formatAxis ? { tickFormatter: tile.formatAxis } : undefined - } + yAxisProps={tile.formatAxis ? { tickFormatter: tile.formatAxis } : undefined} tooltipLabelFormatter={tooltipLabelFormatter} tooltipValueFormatter={tile.formatValue} /> @@ -1789,21 +1635,11 @@ type QueueHealth = { limit: number; }; -type QueueHealthLabel = - | "Paused" - | "At capacity" - | "Backlogged" - | "Active" - | "Idle"; +type QueueHealthLabel = "Paused" | "At capacity" | "Backlogged" | "Active" | "Idle"; // Single source of truth for the queue health decision, shared by the badge and the table's // health-column sort so the sorted order always matches the labels shown. -function queueHealthLabel({ - paused, - running, - queued, - limit, -}: QueueHealth): QueueHealthLabel { +function queueHealthLabel({ paused, running, queued, limit }: QueueHealth): QueueHealthLabel { if (paused) return "Paused"; if (isQueueAtCapacity({ running, queued, limit })) return "At capacity"; if (queued > 0) return "Backlogged"; @@ -1814,10 +1650,8 @@ function queueHealthLabel({ // Tint + colored text, sized like the error status chips (see ErrorStatusBadge). const QUEUE_HEALTH_STYLES: Record = { Paused: "bg-warning/10 text-warning system:bg-warning system:text-white", - "At capacity": - "bg-warning/10 text-warning system:bg-warning system:text-white", - Backlogged: - "bg-blue-500/10 text-blue-500 system:bg-blue-500 system:text-white", + "At capacity": "bg-warning/10 text-warning system:bg-warning system:text-white", + Backlogged: "bg-blue-500/10 text-blue-500 system:bg-blue-500 system:text-white", Active: "bg-success/10 text-success system:bg-success system:text-white", Idle: "bg-charcoal-500/10 text-text-dimmed system:bg-charcoal-500 system:text-white", }; @@ -1828,7 +1662,7 @@ function QueueHealthBadge(health: QueueHealth) { {label} @@ -1850,21 +1684,14 @@ function formatWaitMs(ms: number): string { // Drop a trailing ".00" from whole percentages so "50.00" reads as "50" but "12.50" is preserved. function formatOverridePercent(percent: number): string { - return Number.isInteger(percent) - ? percent.toString() - : percent.toFixed(2).replace(/\.?0+$/, ""); + return Number.isInteger(percent) ? percent.toString() : percent.toFixed(2).replace(/\.?0+$/, ""); } // Classic Queues page, restored verbatim from before the Queue Metrics feature. Rendered // when queueMetricsUiEnabled is off so a gated org sees exactly the pre-metrics UI. function ClassicQueuesView() { - const { - environment, - queues, - pagination, - hasFilters, - autoReloadPollIntervalMs, - } = useTypedLoaderData(); + const { environment, queues, pagination, hasFilters, autoReloadPollIntervalMs } = + useTypedLoaderData(); const organization = useOrganization(); const project = useProject(); @@ -1873,8 +1700,7 @@ function ClassicQueuesView() { useAutoRevalidate({ interval: autoReloadPollIntervalMs, onFocus: true }); - const { limitStatus, limitClassName } = - getEnvConcurrencyLimitStatus(environment); + const { limitStatus, limitClassName } = getEnvConcurrencyLimitStatus(environment); return ( @@ -1899,11 +1725,7 @@ function ClassicQueuesView() { paused - ) : undefined - } + suffix={env.paused ? paused : undefined} animate accessory={
@@ -1925,9 +1747,7 @@ function ClassicQueuesView() { />
} - valueClassName={ - env.paused ? "text-warning tabular-nums" : "tabular-nums" - } + valueClassName={env.paused ? "text-warning tabular-nums" : "tabular-nums"} compactThreshold={1000000} /> - Including{" "} - {environment.running - environment.concurrencyLimit} burst - runs + Including {environment.running - environment.concurrencyLimit} burst runs{" "} + ) : limitStatus === "limit" ? ( "At concurrency limit" @@ -1977,19 +1796,17 @@ function ClassicQueuesView() { - Burst limit{" "} - {environment.burstFactor * environment.concurrencyLimit}{" "} + Burst limit {environment.burstFactor * environment.concurrencyLimit}{" "} ) : undefined } accessory={ plan ? ( - plan?.v3Subscription?.plan?.limits.concurrentRuns - .canExceed ? ( + plan?.v3Subscription?.plan?.limits.concurrentRuns.canExceed ? ( ) : (
@@ -2056,8 +1864,8 @@ function ClassicQueuesView() { className="text-wrap! text-text-dimmed" spacing > - This queue is limited by your environment's - concurrency limit of {environment.concurrencyLimit}. + This queue is limited by your environment's concurrency limit of{" "} + {environment.concurrencyLimit}.
@@ -2067,8 +1875,7 @@ function ClassicQueuesView() { className="text-wrap! text-text-dimmed" spacing > - This queue is limited by a concurrency limit set in - your code. + This queue is limited by a concurrency limit set in your code.
@@ -2078,8 +1885,8 @@ function ClassicQueuesView() { className="text-wrap! text-text-dimmed" spacing > - This queue's concurrency limit has been manually - overridden from the dashboard or API. + This queue's concurrency limit has been manually overridden from the + dashboard or API.
@@ -2095,8 +1902,7 @@ function ClassicQueuesView() { {queues.length > 0 ? ( queues.map((queue) => { - const limit = - queue.concurrencyLimit ?? environment.concurrencyLimit; + const limit = queue.concurrencyLimit ?? environment.concurrencyLimit; const isAtConcurrencyLimit = queue.running >= limit; const isAtQueueLimit = environment.queueSizeLimit !== null && @@ -2112,10 +1918,7 @@ function ClassicQueuesView() { {queue.concurrency?.overriddenAt ? ( + Concurrency limit overridden } @@ -2125,26 +1928,17 @@ function ClassicQueuesView() { /> ) : null} {queue.paused ? ( - + Paused ) : null} {isAtQueueLimit ? ( - + At queue limit ) : null} {isAtConcurrencyLimit ? ( - + At concurrency limit ) : null} @@ -2155,7 +1949,7 @@ function ClassicQueuesView() { className={cn( "w-[1%] pl-16 tabular-nums", queue.paused ? "opacity-50" : undefined, - isAtQueueLimit && "text-error", + isAtQueueLimit && "text-error" )} > {queue.queued} @@ -2169,11 +1963,11 @@ function ClassicQueuesView() { (queue.concurrency.combined.running ?? 0) >= Math.min( queue.concurrency.combined.current, - environment.concurrencyLimit, + environment.concurrencyLimit ) ? "text-warning" : queue.running > 0 && "text-text-bright", - isAtConcurrencyLimit && "text-warning", + isAtConcurrencyLimit && "text-warning" )} > {queue.running} @@ -2183,8 +1977,7 @@ function ClassicQueuesView() { className={cn( "w-[1%] pl-16 tabular-nums", queue.paused ? "opacity-50" : undefined, - queue.concurrency?.overriddenAt && - "font-medium text-text-bright", + queue.concurrency?.overriddenAt && "font-medium text-text-bright" )} > {limit} @@ -2197,7 +1990,7 @@ function ClassicQueuesView() { ( {Math.min( queue.concurrency.combined.current, - environment.concurrencyLimit, + environment.concurrencyLimit )} ) @@ -2207,11 +2000,10 @@ function ClassicQueuesView() { Combined limit: at most{" "} {Math.min( queue.concurrency.combined.current, - environment.concurrencyLimit, + environment.concurrencyLimit )}{" "} - runs across all concurrency keys of this - queue. The main limit applies to each key - separately. + runs across all concurrency keys of this queue. The main limit + applies to each key separately. } className="max-w-[260px]" @@ -2224,8 +2016,7 @@ function ClassicQueuesView() { "w-[1%] pl-16", queue.paused ? "opacity-50" : undefined, isAtConcurrencyLimit && "text-warning", - queue.concurrency?.overriddenAt && - "font-medium text-text-bright", + queue.concurrency?.overriddenAt && "font-medium text-text-bright" )} > {queue.concurrency?.overriddenAt ? ( @@ -2238,16 +2029,8 @@ function ClassicQueuesView() { - ) - } - hiddenButtons={ - !queue.paused && ( - - ) - } + visibleButtons={queue.paused && } + hiddenButtons={!queue.paused && } popoverContent={ <> {queue.paused ? ( @@ -2300,9 +2083,7 @@ function ClassicQueuesView() { /> } @@ -2315,9 +2096,7 @@ function ClassicQueuesView() {
- {hasFilters - ? "No queues found matching your filters" - : "No queues found"} + {hasFilters ? "No queues found matching your filters" : "No queues found"}
@@ -2355,12 +2134,9 @@ const limitTooltip = ( How many runs can execute at once.{" "} - 1 (20) means 1 run - per concurrency key, but at most 20 runs across all keys. Set using{" "} - - combinedConcurrencyLimit - {" "} - in your code. + 1 (20) means 1 run per concurrency key, + but at most 20 runs across all keys. Set using{" "} + combinedConcurrencyLimit in your code. ); From b46032d666a631661f93b9df85c94e3e9c2b14a1 Mon Sep 17 00:00:00 2001 From: Matt Aitken Date: Mon, 31 Aug 2026 10:47:42 +0100 Subject: [PATCH 13/32] fix(run-engine): sample the combined gauge after batch admission The dequeue gauge ran before the loop, so a queue's first batch from idle and its final drain were never sampled with their runs in flight and the combined chart under-reported. The successful path now re-samples after admissions; early returns keep the entry sample. Also documents that combined.current is the declared cap, clamped at admit time. --- internal-packages/run-engine/src/run-queue/index.ts | 4 ++++ packages/core/src/v3/schemas/queues.ts | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/internal-packages/run-engine/src/run-queue/index.ts b/internal-packages/run-engine/src/run-queue/index.ts index 311e7603738..3865d068d35 100644 --- a/internal-packages/run-engine/src/run-queue/index.ts +++ b/internal-packages/run-engine/src/run-queue/index.ts @@ -5277,6 +5277,10 @@ else redis.call('ZADD', masterQueueKey, earliestIdx[2], ckWildcardName) end +-- Re-sample the gauge so the emitted snapshot includes this batch's admissions; +-- the top-of-script sample only covers the early returns where nothing was admitted. +${QUEUE_METRICS_CK_DEQUEUE_GAUGE_LUA} + return __qmret(results) `, }); diff --git a/packages/core/src/v3/schemas/queues.ts b/packages/core/src/v3/schemas/queues.ts index 9ca282fb33d..cf604c3c298 100644 --- a/packages/core/src/v3/schemas/queues.ts +++ b/packages/core/src/v3/schemas/queues.ts @@ -48,7 +48,7 @@ export const QueueItem = z.object({ /** The combined concurrency cap across all concurrencyKey values of the queue */ combined: z .object({ - /** The effective/current combined concurrency limit (null = no cap) */ + /** The current combined concurrency limit as declared or overridden (null = no cap). Enforcement clamps it to the environment concurrency limit at admit time. */ current: z.number().nullable(), /** The declared combined limit an override reverts to on reset */ base: z.number().nullable(), From a8fa96cee1399f6686c42479d7b5af3c5c376d06 Mon Sep 17 00:00:00 2001 From: Matt Aitken Date: Mon, 31 Aug 2026 12:59:17 +0100 Subject: [PATCH 14/32] refactor(run-engine,webapp): drop per-key override admit reads and limit column Removes the override-aware admit and gauge reads from the CK Lua scripts and the per-key limit column, following the removal of runtime per-key overrides from this stack. --- .../route.tsx | 29 +-- apps/webapp/app/v3/querySchemas.ts | 2 +- ...add_queue_metrics_combined_concurrency.sql | 2 +- .../run-engine/src/run-queue/index.ts | 170 ++---------------- 4 files changed, 16 insertions(+), 187 deletions(-) diff --git a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.queues_.$queueParam/route.tsx b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.queues_.$queueParam/route.tsx index 8e445fa6111..933baafc43a 100644 --- a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.queues_.$queueParam/route.tsx +++ b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.queues_.$queueParam/route.tsx @@ -407,13 +407,7 @@ export default function Page() { {view === "keys" && hasKeys ? ( <> - + {selectedKey ? ( @@ -966,15 +960,10 @@ function KeyStatsTable({ ids, timeRange, queueName, - defaultKeyLimit, - envLimit, }: { ids: Ids; timeRange: TimeRangeParams; queueName: string; - /** The limit a key inherits when it has no override (the queue's limit, else the env limit). */ - defaultKeyLimit: number; - envLimit: number; }) { const { value, replace, del } = useSearchParams(); const selectedKey = value("key"); @@ -1017,12 +1006,6 @@ function KeyStatsTable({ Key Queued now Running now - - Limit - Oldest wait Started Peak backlog @@ -1031,11 +1014,11 @@ function KeyStatsTable({ {showLoading ? ( - + Loading… ) : rows.length === 0 ? ( - + {search ? `No keys match “${search}”` : "No concurrency keys"} ) : ( @@ -1049,12 +1032,6 @@ function KeyStatsTable({ {row.key} {row.queued.toLocaleString()} {row.running.toLocaleString()} - - {Math.min(row.limitOverride ?? defaultKeyLimit, envLimit).toLocaleString()} - {row.oldestWaitMs === null ? "–" : formatWaitMs(row.oldestWaitMs)} diff --git a/apps/webapp/app/v3/querySchemas.ts b/apps/webapp/app/v3/querySchemas.ts index 05cd7f0b394..3ec1523c83f 100644 --- a/apps/webapp/app/v3/querySchemas.ts +++ b/apps/webapp/app/v3/querySchemas.ts @@ -1426,7 +1426,7 @@ const queueMetricsByKeySchema: TableSchema = { name: "max_limit", ...column("UInt32", { description: - "The effective concurrency limit for this key (the queue limit, or its per-key override). Aggregate with max().", + "The queue concurrency limit that applied to this key in the bucket. Aggregate with max().", fillMode: "carry", }), }, diff --git a/internal-packages/clickhouse/schema/042_add_queue_metrics_combined_concurrency.sql b/internal-packages/clickhouse/schema/042_add_queue_metrics_combined_concurrency.sql index 03cb133799a..57c6b290efc 100644 --- a/internal-packages/clickhouse/schema/042_add_queue_metrics_combined_concurrency.sql +++ b/internal-packages/clickhouse/schema/042_add_queue_metrics_combined_concurrency.sql @@ -4,7 +4,7 @@ -- concurrency-key variants of a queue (the groupConcurrency set), combined_limit the -- RAW stored total cap (0 = none, readers clamp against max_env_limit). Emitted on -- base-queue gauge rows only. Per-key gauge rows now carry the EFFECTIVE per-key --- limit in queue_limit (override-aware), surfaced in the ck tier as max_limit. +-- limit in queue_limit, surfaced in the ck tier as max_limit. ALTER TABLE trigger_dev.queue_metrics_raw_v1 ADD COLUMN IF NOT EXISTS combined_running UInt32 DEFAULT 0, diff --git a/internal-packages/run-engine/src/run-queue/index.ts b/internal-packages/run-engine/src/run-queue/index.ts index 3865d068d35..8ba389eac3f 100644 --- a/internal-packages/run-engine/src/run-queue/index.ts +++ b/internal-packages/run-engine/src/run-queue/index.ts @@ -114,18 +114,12 @@ local function __gateReconcile(setKey, msgKeyPrefix, reconcileKeyPrefix) end end -local function __gatesHaveCapacity(gatesKeyPrefix, msg, messageId, envLimit, msgKeyPrefix, ckOverridesEnabled) +local function __gatesHaveCapacity(gatesKeyPrefix, msg, messageId, envLimit, msgKeyPrefix) if not msg.gates then return true end for _, gate in ipairs(msg.gates) do local base, variant, gateKey = __gateKeys(gatesKeyPrefix, msg, gate) local occupancy = tonumber(redis.call('SCARD', variant .. ':currentConcurrency') or '0') local perKeyLimit = math.min(tonumber(redis.call('GET', base .. ':concurrency') or '1000000'), envLimit) - if ckOverridesEnabled and gateKey and gateKey ~= '' then - local gateOverride = redis.call('HGET', base .. ':ckLimits', string.sub(variant, #gatesKeyPrefix + 1)) - if gateOverride then - perKeyLimit = math.min(tonumber(gateOverride), envLimit) - end - end if occupancy >= perKeyLimit and redis.call('SISMEMBER', variant .. ':currentConcurrency', messageId) == 0 then __gateReconcile(variant .. ':currentConcurrency', msgKeyPrefix, gatesKeyPrefix) return false @@ -227,8 +221,7 @@ const QUEUE_METRICS_CK_ENQUEUE_GAUGE_LUA = createMetricsGaugeComputeLua({ enabledArg: "ARGV[#ARGV] == '1'", queued: "redis.call('ZCARD', queueKey)", running: "redis.call('SCARD', queueCurrentConcurrencyKey)", - queueLimit: - "redis.call('HGET', ckLimitsKey, queueName) or redis.call('GET', queueConcurrencyLimitKey) or '1000000'", + queueLimit: "redis.call('GET', queueConcurrencyLimitKey) or '1000000'", envQueued: "redis.call('ZCARD', envQueueKey)", envRunning: "redis.call('SCARD', envCurrentConcurrencyKey)", envLimit: "redis.call('GET', envConcurrencyLimitKey) or defaultEnvConcurrencyLimit", @@ -275,13 +268,6 @@ export interface RunQueueMetricsEmitter { emitGauge(shardKey: string, fields: Record): void; } -export class RunQueueConcurrencyKeyLimitExceededError extends Error { - constructor(message: string) { - super(message); - this.name = "RunQueueConcurrencyKeyLimitExceededError"; - } -} - export type RunQueueOptions = { name: string; tracer: Tracer; @@ -346,7 +332,6 @@ export type RunQueueOptions = { */ gatesEnabled?: boolean; /** Cap on per-concurrency-key limit overrides stored per queue. Default 1000. */ - maxConcurrencyKeyOverridesPerQueue?: number; workerOptions?: { pollIntervalMs?: number; immediatePollIntervalMs?: number; @@ -460,7 +445,6 @@ export class RunQueue { private queueSelectionStrategy: RunQueueSelectionStrategy; private shardCount: number; private counterTtlSeconds: number; - private maxConcurrencyKeyOverridesPerQueue: number; private abortController: AbortController; private worker: Worker; private workerQueueResolver: WorkerQueueResolver; @@ -471,7 +455,6 @@ export class RunQueue { constructor(public readonly options: RunQueueOptions) { this.shardCount = options.shardCount ?? 2; this.counterTtlSeconds = options.counterTtlSeconds ?? 86400; - this.maxConcurrencyKeyOverridesPerQueue = options.maxConcurrencyKeyOverridesPerQueue ?? 1000; this.retryOptions = options.retryOptions ?? defaultRetrySettings; this.redis = createRedisClient(options.redis, { onError: (error) => { @@ -666,85 +649,6 @@ export class RunQueue { return this.redis.scard(this.keys.queueGroupConcurrencyKey(env, queue)); } - /** - * Sets a per-concurrency-key limit override for a queue. The stored value is the - * raw requested limit; admit paths clamp to the environment limit at read time. - * Throws RunQueueConcurrencyKeyLimitExceededError when a NEW key would push the - * queue past maxConcurrencyKeyOverridesPerQueue (updates to existing keys always - * succeed). - */ - public async updateQueueConcurrencyKeyLimit( - env: MinimalAuthenticatedEnvironment, - queue: string, - concurrencyKey: string, - limit: number - ) { - const result = await this.redis.setQueueConcurrencyKeyLimit( - this.keys.queueCkLimitsKey(env, queue), - this.keys.queueKey(env, queue, concurrencyKey), - String(limit), - String(this.maxConcurrencyKeyOverridesPerQueue) - ); - - if (result === 0) { - throw new RunQueueConcurrencyKeyLimitExceededError( - `Cannot add a concurrency key override to queue ${queue}: the queue already has ${this.maxConcurrencyKeyOverridesPerQueue} overrides` - ); - } - } - - public async removeQueueConcurrencyKeyLimit( - env: MinimalAuthenticatedEnvironment, - queue: string, - concurrencyKey: string - ) { - return this.redis.hdel( - this.keys.queueCkLimitsKey(env, queue), - this.keys.queueKey(env, queue, concurrencyKey) - ); - } - - /** Returns the raw per-concurrency-key limit overrides for a queue, keyed by concurrency key value. */ - public async getQueueConcurrencyKeyLimits( - env: MinimalAuthenticatedEnvironment, - queue: string - ): Promise> { - const raw = await this.redis.hgetall(this.keys.queueCkLimitsKey(env, queue)); - - const limits: Record = {}; - for (const [variantName, value] of Object.entries(raw)) { - const ckIndex = variantName.indexOf(":ck:"); - if (ckIndex === -1) { - continue; - } - limits[variantName.slice(ckIndex + 4)] = Number(value); - } - return limits; - } - - /** Per-key limit overrides for just the given keys: one HMGET, O(keys) not O(overrides). */ - public async getQueueConcurrencyKeyLimitsForKeys( - env: MinimalAuthenticatedEnvironment, - queue: string, - concurrencyKeys: string[] - ): Promise> { - if (concurrencyKeys.length === 0) { - return {}; - } - - const fields = concurrencyKeys.map((key) => this.keys.queueKey(env, queue, key)); - const values = await this.redis.hmget(this.keys.queueCkLimitsKey(env, queue), ...fields); - - const limits: Record = {}; - concurrencyKeys.forEach((key, index) => { - const value = values[index]; - if (value != null) { - limits[key] = Number(value); - } - }); - return limits; - } - /** Batch variant of totalConcurrencyOfQueue: one pipeline of group SCARDs. */ public async totalConcurrencyOfQueues( env: MinimalAuthenticatedEnvironment, @@ -2538,7 +2442,6 @@ export class RunQueue { const totalConcurrencyLimitKey = this.keys.queueTotalConcurrencyLimitKeyFromQueue( message.queue ); - const ckLimitsKey = this.keys.queueCkLimitsKeyFromQueue(message.queue); const totalConcurrencyEnabledArg = this.options.totalConcurrencyEnabled ? "1" : "0"; if (ttlInfo) { @@ -2562,7 +2465,6 @@ export class RunQueue { baseQueueKey, groupConcurrencyKey, totalConcurrencyLimitKey, - ckLimitsKey, // args queueName, messageId, @@ -2602,7 +2504,6 @@ export class RunQueue { baseQueueKey, groupConcurrencyKey, totalConcurrencyLimitKey, - ckLimitsKey, // args queueName, messageId, @@ -2893,7 +2794,6 @@ export class RunQueue { runningCounterKey, this.keys.queueGroupConcurrencyKeyFromQueue(ckWildcardQueue), this.keys.queueTotalConcurrencyLimitKeyFromQueue(ckWildcardQueue), - this.keys.queueCkLimitsKeyFromQueue(ckWildcardQueue), //args ckWildcardQueue, String(Date.now()), @@ -3790,7 +3690,7 @@ if enableFastPath == '1' then local okDecode, decoded = pcall(cjson.decode, messageData) if okDecode and type(decoded) == 'table' and decoded.gates then gateMsg = decoded - gatesAllowFastPath = __gatesHaveCapacity(keyPrefix, decoded, messageId, envLimit, nil, totalConcurrencyEnabled) + gatesAllowFastPath = __gatesHaveCapacity(keyPrefix, decoded, messageId, envLimit, nil) end end @@ -3905,7 +3805,7 @@ if enableFastPath == '1' then local okDecode, decoded = pcall(cjson.decode, messageData) if okDecode and type(decoded) == 'table' and decoded.gates then gateMsg = decoded - gatesAllowFastPath = __gatesHaveCapacity(keyPrefix, decoded, messageId, envLimit, nil, totalConcurrencyEnabled) + gatesAllowFastPath = __gatesHaveCapacity(keyPrefix, decoded, messageId, envLimit, nil) end end @@ -4184,7 +4084,7 @@ return __qmret(0) // *Tracked variants of dequeueMessageFromKey and the ack/nack/dlq/release/clear // scripts. this.redis.defineCommand("enqueueMessageCkTracked", { - numberOfKeys: 18, + numberOfKeys: 17, lua: ` local masterQueueKey = KEYS[1] local queueKey = KEYS[2] @@ -4206,7 +4106,6 @@ local baseQueueKey = KEYS[15] -- Total-cap keys (KEYS 16-17) local groupConcurrencyKey = KEYS[16] local totalConcurrencyLimitKey = KEYS[17] -local ckLimitsKey = KEYS[18] local queueName = ARGV[1] local messageId = ARGV[2] @@ -4245,10 +4144,6 @@ if enableFastPath == '1' then envLimit ) if totalConcurrencyEnabled then - local perKeyOverride = redis.call('HGET', ckLimitsKey, queueName) - if perKeyOverride then - queueLimit = math.min(tonumber(perKeyOverride), envLimit) - end end if queueCurrent < queueLimit then @@ -4272,7 +4167,7 @@ if enableFastPath == '1' then local okDecode, decoded = pcall(cjson.decode, messageData) if okDecode and type(decoded) == 'table' and decoded.gates then gateMsg = decoded - gatesAllowFastPath = __gatesHaveCapacity(keyPrefix, decoded, messageId, envLimit, nil, totalConcurrencyEnabled) + gatesAllowFastPath = __gatesHaveCapacity(keyPrefix, decoded, messageId, envLimit, nil) end end @@ -4361,7 +4256,7 @@ return __qmret(0) }); this.redis.defineCommand("enqueueMessageWithTtlCkTracked", { - numberOfKeys: 19, + numberOfKeys: 18, lua: ` local masterQueueKey = KEYS[1] local queueKey = KEYS[2] @@ -4384,7 +4279,6 @@ local baseQueueKey = KEYS[16] -- Total-cap keys (KEYS 17-18) local groupConcurrencyKey = KEYS[17] local totalConcurrencyLimitKey = KEYS[18] -local ckLimitsKey = KEYS[19] local queueName = ARGV[1] local messageId = ARGV[2] @@ -4425,10 +4319,6 @@ if enableFastPath == '1' then envLimit ) if totalConcurrencyEnabled then - local perKeyOverride = redis.call('HGET', ckLimitsKey, queueName) - if perKeyOverride then - queueLimit = math.min(tonumber(perKeyOverride), envLimit) - end end if queueCurrent < queueLimit then @@ -4450,7 +4340,7 @@ if enableFastPath == '1' then local okDecode, decoded = pcall(cjson.decode, messageData) if okDecode and type(decoded) == 'table' and decoded.gates then gateMsg = decoded - gatesAllowFastPath = __gatesHaveCapacity(keyPrefix, decoded, messageId, envLimit, nil, totalConcurrencyEnabled) + gatesAllowFastPath = __gatesHaveCapacity(keyPrefix, decoded, messageId, envLimit, nil) end end @@ -4853,7 +4743,7 @@ for i = 1, #messages, 2 do else local gatesAllow = true if gatesEnabled then - gatesAllow = __gatesHaveCapacity(keyPrefix, messageData, messageId, envConcurrencyLimit, messageKeyPrefix, totalConcurrencyEnabled) + gatesAllow = __gatesHaveCapacity(keyPrefix, messageData, messageId, envConcurrencyLimit, messageKeyPrefix) end if gatesAllow then @@ -5055,7 +4945,7 @@ return results // (normal dequeue, TTL-expired, or stale-orphan path — all of which were // counted at enqueue time). this.redis.defineCommand("dequeueMessagesFromCkQueueTracked", { - numberOfKeys: 14, + numberOfKeys: 13, lua: ` local ckIndexKey = KEYS[1] local queueConcurrencyLimitKey = KEYS[2] @@ -5070,7 +4960,6 @@ local lengthCounterKey = KEYS[10] local runningCounterKey = KEYS[11] local groupConcurrencyKey = KEYS[12] local totalConcurrencyLimitKey = KEYS[13] -local ckLimitsKey = KEYS[14] local ckWildcardName = ARGV[1] local currentTime = tonumber(ARGV[2]) @@ -5162,12 +5051,6 @@ for _, ckQueueName in ipairs(ckQueues) do local ckCurrentConcurrency = tonumber(redis.call('SCARD', ckConcurrencyKey) or '0') local perKeyLimit = queueConcurrencyLimit - if totalConcurrencyEnabled then - local perKeyOverride = redis.call('HGET', ckLimitsKey, ckQueueName) - if perKeyOverride then - perKeyLimit = math.min(tonumber(perKeyOverride), envConcurrencyLimit) - end - end if ckCurrentConcurrency >= perKeyLimit then -- Back a blocked variant off so it cannot pin the bounded candidate window @@ -5202,7 +5085,7 @@ for _, ckQueueName in ipairs(ckQueues) do else local gatesAllow = true if gatesEnabled then - gatesAllow = __gatesHaveCapacity(keyPrefix, messageData, messageId, envConcurrencyLimit, messageKeyPrefix, totalConcurrencyEnabled) + gatesAllow = __gatesHaveCapacity(keyPrefix, messageData, messageId, envConcurrencyLimit, messageKeyPrefix) end if not gatesAllow then blockedByGates = true @@ -6104,26 +5987,6 @@ __gatesRelease(keyPrefix, redis.call('GET', messageKey), messageId) `, }); - this.redis.defineCommand("setQueueConcurrencyKeyLimit", { - numberOfKeys: 1, - lua: ` -local ckLimitsKey = KEYS[1] - -local fieldName = ARGV[1] -local limit = ARGV[2] -local maxFields = tonumber(ARGV[3]) - -if redis.call('HEXISTS', ckLimitsKey, fieldName) == 0 then - if redis.call('HLEN', ckLimitsKey) >= maxFields then - return 0 - end -end - -redis.call('HSET', ckLimitsKey, fieldName, limit) -return 1 -`, - }); - this.redis.defineCommand("updateEnvironmentConcurrencyLimits", { numberOfKeys: 2, lua: ` @@ -6488,14 +6351,6 @@ declare module "@internal/redis" { callback?: Callback ): Result; - setQueueConcurrencyKeyLimit( - ckLimitsKey: string, - fieldName: string, - limit: string, - maxFields: string, - callback?: Callback - ): Result; - updateEnvironmentConcurrencyLimits( // keys envConcurrencyLimitKey: string, @@ -6682,7 +6537,6 @@ declare module "@internal/redis" { baseQueueKey: string, groupConcurrencyKey: string, totalConcurrencyLimitKey: string, - ckLimitsKey: string, queueName: string, messageId: string, messageData: string, @@ -6720,7 +6574,6 @@ declare module "@internal/redis" { baseQueueKey: string, groupConcurrencyKey: string, totalConcurrencyLimitKey: string, - ckLimitsKey: string, queueName: string, messageId: string, messageData: string, @@ -6755,7 +6608,6 @@ declare module "@internal/redis" { runningCounterKey: string, groupConcurrencyKey: string, totalConcurrencyLimitKey: string, - ckLimitsKey: string, ckWildcardName: string, currentTime: string, defaultEnvConcurrencyLimit: string, From 5364092524c4550d554334d3a5704f6253402874 Mon Sep 17 00:00:00 2001 From: Matt Aitken Date: Mon, 31 Aug 2026 12:59:38 +0100 Subject: [PATCH 15/32] refactor(webapp): concurrency keys resource stops reading per-key overrides --- .../app/routes/resources.queues.concurrency-keys.ts | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/apps/webapp/app/routes/resources.queues.concurrency-keys.ts b/apps/webapp/app/routes/resources.queues.concurrency-keys.ts index 8c590554e51..67c2b9f500a 100644 --- a/apps/webapp/app/routes/resources.queues.concurrency-keys.ts +++ b/apps/webapp/app/routes/resources.queues.concurrency-keys.ts @@ -43,8 +43,6 @@ export type ConcurrencyKeyRow = { peakBacklog: number; peakRunning: number; meanWaitMs: number; - /** Per-key concurrency limit override, when one is set for this key (null = inherits the queue limit). */ - limitOverride: number | null; }; export type ConcurrencyKeysResponse = @@ -153,11 +151,8 @@ export const action = async ({ request }: ActionFunctionArgs) => { const total = rankingRows?.[0]?.ranked_total ?? 0; const keys = (rankingRows ?? []).map((r) => r.concurrency_key); - // Enrich just this page's keys with live "now" counts and any per-key limit overrides from Redis. - const [live, keyLimitOverrides] = await Promise.all([ - engine.concurrencyKeyLiveStats(environment, queueName, keys), - engine.runQueue.getQueueConcurrencyKeyLimitsForKeys(environment, queueName, keys), - ]); + // Enrich just this page's keys with live "now" counts from Redis. + const live = await engine.concurrencyKeyLiveStats(environment, queueName, keys); const loadedAt = Date.now(); const rows: ConcurrencyKeyRow[] = (rankingRows ?? []).map((r) => { @@ -173,7 +168,6 @@ export const action = async ({ request }: ActionFunctionArgs) => { peakBacklog: r.peak_backlog, peakRunning: r.peak_running, meanWaitMs: r.mean_wait_ms, - limitOverride: keyLimitOverrides[r.concurrency_key] ?? null, }; }); From 5ea5ed56d6d1017a295a3a505d293d52f5727cb9 Mon Sep 17 00:00:00 2001 From: Matt Aitken Date: Mon, 31 Aug 2026 13:11:29 +0100 Subject: [PATCH 16/32] refactor(run-engine): drop the now-unreferenced ck-limits key builders --- internal-packages/run-engine/src/run-queue/keyProducer.ts | 8 -------- internal-packages/run-engine/src/run-queue/types.ts | 3 --- 2 files changed, 11 deletions(-) diff --git a/internal-packages/run-engine/src/run-queue/keyProducer.ts b/internal-packages/run-engine/src/run-queue/keyProducer.ts index 120e04f8c38..98028f5af7b 100644 --- a/internal-packages/run-engine/src/run-queue/keyProducer.ts +++ b/internal-packages/run-engine/src/run-queue/keyProducer.ts @@ -366,14 +366,6 @@ export class RunQueueFullKeyProducer implements RunQueueKeyProducer { return `${this.baseQueueKeyFromQueue(queue)}:${constants.TOTAL_CONCURRENCY_LIMIT_PART}`; } - queueCkLimitsKey(env: RunQueueKeyProducerEnvironment, queue: string): string { - return `${this.queueKey(env, queue)}:ckLimits`; - } - - queueCkLimitsKeyFromQueue(queue: string): string { - return `${this.baseQueueKeyFromQueue(queue)}:ckLimits`; - } - isCkWildcard(queue: string): boolean { return queue.endsWith(":ck:*"); } diff --git a/internal-packages/run-engine/src/run-queue/types.ts b/internal-packages/run-engine/src/run-queue/types.ts index 2961b642314..2cbfe40c775 100644 --- a/internal-packages/run-engine/src/run-queue/types.ts +++ b/internal-packages/run-engine/src/run-queue/types.ts @@ -111,9 +111,6 @@ export interface RunQueueKeyProducer { queueTotalConcurrencyLimitKey(env: RunQueueKeyProducerEnvironment, queue: string): string; queueTotalConcurrencyLimitKeyFromQueue(queue: string): string; - queueCkLimitsKey(env: RunQueueKeyProducerEnvironment, queue: string): string; - queueCkLimitsKeyFromQueue(queue: string): string; - //env oncurrency envCurrentConcurrencyKey(env: EnvDescriptor): string; envCurrentConcurrencyKey(env: RunQueueKeyProducerEnvironment): string; From 8bf694a79b08a23321870588dc9089ad049727bb Mon Sep 17 00:00:00 2001 From: Matt Aitken Date: Mon, 31 Aug 2026 13:44:36 +0100 Subject: [PATCH 17/32] chore: lift the run-queue knip ignore The class the ignore covered is deleted at this level, so the merged result carries no dead-code exemption. --- knip.json | 3 --- 1 file changed, 3 deletions(-) diff --git a/knip.json b/knip.json index c8e7f4027dd..84456756ca1 100644 --- a/knip.json +++ b/knip.json @@ -27,9 +27,6 @@ ], "ignoreDependencies": ["@sentry/cli", "assert", "util"] }, - "internal-packages/run-engine": { - "ignore": ["src/run-queue/index.ts"] - }, "internal-packages/dashboard-agent": { "entry": ["trigger.config.ts", "src/investigation-sweep.ts", "src/maintenance.ts"], "ignoreBinaries": ["rg"] From 5428855936c55f179592c07c554c33bc35774b67 Mon Sep 17 00:00:00 2001 From: Matt Aitken Date: Mon, 31 Aug 2026 14:03:19 +0100 Subject: [PATCH 18/32] fix(run-engine,webapp,clickhouse): review fixes for the metrics tier The plain dequeue gauge re-samples after admissions like the keyed one, the repair path clears a keyed run's variant and group slots by concurrency key, pause responses include the combined limit, stale wording and a leftover changeset from before the rename are cleaned up, and two empty flag blocks are removed from the fast-path scripts. --- apps/webapp/app/v3/querySchemas.ts | 2 +- .../app/v3/services/pauseQueue.server.ts | 3 ++ ...add_queue_metrics_combined_concurrency.sql | 5 +-- .../run-engine/src/engine/index.ts | 2 ++ .../run-engine/src/run-queue/index.ts | 34 +++++++++++++------ 5 files changed, 32 insertions(+), 14 deletions(-) diff --git a/apps/webapp/app/v3/querySchemas.ts b/apps/webapp/app/v3/querySchemas.ts index 3ec1523c83f..b33b205cc3c 100644 --- a/apps/webapp/app/v3/querySchemas.ts +++ b/apps/webapp/app/v3/querySchemas.ts @@ -1426,7 +1426,7 @@ const queueMetricsByKeySchema: TableSchema = { name: "max_limit", ...column("UInt32", { description: - "The queue concurrency limit that applied to this key in the bucket. Aggregate with max().", + "The queue concurrency limit that applied to this key in the bucket (1000000 = no explicit limit). Aggregate with max().", fillMode: "carry", }), }, diff --git a/apps/webapp/app/v3/services/pauseQueue.server.ts b/apps/webapp/app/v3/services/pauseQueue.server.ts index aa3e21f9727..97cc598daf0 100644 --- a/apps/webapp/app/v3/services/pauseQueue.server.ts +++ b/apps/webapp/app/v3/services/pauseQueue.server.ts @@ -92,6 +92,9 @@ export class PauseQueueService extends BaseService { concurrencyLimitOverriddenAt: updatedQueue.concurrencyLimitOverriddenAt ?? null, concurrencyLimitOverriddenBy: queue.concurrencyLimitOverriddenBy ?? null, paused: updatedQueue.paused, + totalConcurrencyLimit: updatedQueue.totalConcurrencyLimit ?? null, + totalConcurrencyLimitBase: updatedQueue.totalConcurrencyLimitBase ?? null, + totalConcurrencyLimitOverriddenAt: updatedQueue.totalConcurrencyLimitOverriddenAt ?? null, }), }; } catch (error) { diff --git a/internal-packages/clickhouse/schema/042_add_queue_metrics_combined_concurrency.sql b/internal-packages/clickhouse/schema/042_add_queue_metrics_combined_concurrency.sql index 57c6b290efc..dd0c4c53b34 100644 --- a/internal-packages/clickhouse/schema/042_add_queue_metrics_combined_concurrency.sql +++ b/internal-packages/clickhouse/schema/042_add_queue_metrics_combined_concurrency.sql @@ -3,8 +3,9 @@ -- Total-concurrency gauges: combined_running is the in-flight count across ALL -- concurrency-key variants of a queue (the groupConcurrency set), combined_limit the -- RAW stored total cap (0 = none, readers clamp against max_env_limit). Emitted on --- base-queue gauge rows only. Per-key gauge rows now carry the EFFECTIVE per-key --- limit in queue_limit, surfaced in the ck tier as max_limit. +-- base-queue gauge rows only. Per-key gauge rows carry the queue concurrency +-- limit that applied in queue_limit, surfaced in the ck tier as max_limit +-- (1000000 = no explicit limit). ALTER TABLE trigger_dev.queue_metrics_raw_v1 ADD COLUMN IF NOT EXISTS combined_running UInt32 DEFAULT 0, diff --git a/internal-packages/run-engine/src/engine/index.ts b/internal-packages/run-engine/src/engine/index.ts index 1909df7e9c6..6347a2d1e60 100644 --- a/internal-packages/run-engine/src/engine/index.ts +++ b/internal-packages/run-engine/src/engine/index.ts @@ -3001,6 +3001,7 @@ export class RunEngine { { select: { queue: true, + concurrencyKey: true, }, }, this.prisma @@ -3022,6 +3023,7 @@ export class RunEngine { runId, orgId: latestSnapshot.organizationId, queue: taskRun.queue, + concurrencyKey: taskRun.concurrencyKey ?? undefined, env: { id: latestSnapshot.environmentId, type: latestSnapshot.environmentType, diff --git a/internal-packages/run-engine/src/run-queue/index.ts b/internal-packages/run-engine/src/run-queue/index.ts index 8ba389eac3f..a28ab2542c4 100644 --- a/internal-packages/run-engine/src/run-queue/index.ts +++ b/internal-packages/run-engine/src/run-queue/index.ts @@ -331,7 +331,6 @@ export type RunQueueOptions = { * the total cap covering releases from builds without the mirror. */ gatesEnabled?: boolean; - /** Cap on per-concurrency-key limit overrides stored per queue. Default 1000. */ workerOptions?: { pollIntervalMs?: number; immediatePollIntervalMs?: number; @@ -1549,6 +1548,7 @@ export class RunQueue { runId: string; orgId: string; queue: string; + concurrencyKey?: string; env: RunQueueKeyProducerEnvironment; }) { return this.#callClearMessageFromConcurrencySets(params); @@ -3084,18 +3084,30 @@ export class RunQueue { runId, orgId, queue, + concurrencyKey, env, }: { runId: string; orgId: string; queue: string; + concurrencyKey?: string; env: RunQueueKeyProducerEnvironment; }) { const messageId = runId; const messageKey = this.keys.messageKey(orgId, messageId); - const queueCurrentConcurrencyKey = this.keys.queueCurrentConcurrencyKey(env, queue); + /** + * Callers pass the bare TaskRun queue name plus its concurrencyKey; the run's + * slots live on the ck variant, and the tracked clear additionally mirrors the + * group set and counters that only keyed queues maintain. + */ + const fullQueue = concurrencyKey ? this.keys.queueKey(env, queue, concurrencyKey) : queue; + const queueCurrentConcurrencyKey = this.keys.queueCurrentConcurrencyKey( + env, + queue, + concurrencyKey + ); const envCurrentConcurrencyKey = this.keys.envCurrentConcurrencyKey(env); - const queueCurrentDequeuedKey = this.keys.queueCurrentDequeuedKey(env, queue); + const queueCurrentDequeuedKey = this.keys.queueCurrentDequeuedKey(env, queue, concurrencyKey); const envCurrentDequeuedKey = this.keys.envCurrentDequeuedKey(env); this.logger.debug("Calling clearMessageFromConcurrencySets", { @@ -3110,15 +3122,15 @@ export class RunQueue { service: this.name, }); - if (queue.includes(":ck:")) { + if (fullQueue.includes(":ck:")) { return this.redis.clearMessageFromConcurrencySetsTracked( queueCurrentConcurrencyKey, envCurrentConcurrencyKey, queueCurrentDequeuedKey, envCurrentDequeuedKey, - this.keys.queueRunningCounterKeyFromQueue(queue), - this.keys.ckIndexKeyFromQueue(queue), - this.keys.queueGroupConcurrencyKeyFromQueue(queue), + this.keys.queueRunningCounterKeyFromQueue(fullQueue), + this.keys.ckIndexKeyFromQueue(fullQueue), + this.keys.queueGroupConcurrencyKeyFromQueue(fullQueue), messageKey, messageId, this.options.redis.keyPrefix ?? "", @@ -4143,8 +4155,6 @@ if enableFastPath == '1' then tonumber(redis.call('GET', queueConcurrencyLimitKey) or '1000000'), envLimit ) - if totalConcurrencyEnabled then - end if queueCurrent < queueLimit then -- Total-cap gate: a fast-path admit consumes a group slot, so it must @@ -4318,8 +4328,6 @@ if enableFastPath == '1' then tonumber(redis.call('GET', queueConcurrencyLimitKey) or '1000000'), envLimit ) - if totalConcurrencyEnabled then - end if queueCurrent < queueLimit then -- Total-cap gate: see enqueueMessageCkTracked. @@ -4785,6 +4793,10 @@ else redis.call('ZADD', masterQueueKey, earliestMessage[2], queueName) end +-- Re-sample the gauge so the emitted snapshot includes this batch's admissions; +-- the top-of-script sample only covers the early returns where nothing was admitted. +${QUEUE_METRICS_GAUGE_LUA} + -- Return results as a flat array: [messageId1, messageScore1, messagePayload1, messageId2, messageScore2, messagePayload2, ...] return __qmret(results) `, From 7df40bcd4a075b7beb192ce601ab916e8047dbc6 Mon Sep 17 00:00:00 2001 From: Matt Aitken Date: Mon, 31 Aug 2026 14:03:43 +0100 Subject: [PATCH 19/32] chore: drop the pre-rename changeset superseded by the combined one --- .changeset/queue-total-concurrency-stats.md | 5 ----- 1 file changed, 5 deletions(-) delete mode 100644 .changeset/queue-total-concurrency-stats.md diff --git a/.changeset/queue-total-concurrency-stats.md b/.changeset/queue-total-concurrency-stats.md deleted file mode 100644 index a70da24d1fb..00000000000 --- a/.changeset/queue-total-concurrency-stats.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -"@trigger.dev/core": patch ---- - -Queue retrieve and list API responses now report total concurrency usage. When a queue has a `totalConcurrencyLimit`, `concurrency.total` includes the effective cap, the declared base, any active override, and how many runs are in flight across all concurrency keys. From 4585e2a89e7d8dc4bf2984f16e55e11883f26a17 Mon Sep 17 00:00:00 2001 From: Matt Aitken Date: Mon, 31 Aug 2026 17:25:54 +0100 Subject: [PATCH 20/32] perf(run-engine): share the combined-limit read between the admit gate and gauges The CK enqueue and dequeue scripts read the combined concurrency limit key once for admission and again for the metrics gauge tail. A per-call memo makes whichever runs first do the single GET; limits cannot change mid-script, so the value stays exact. Group cardinality remains a fresh read because gauges must reflect post-admission state. --- .../run-engine/src/run-queue/index.ts | 34 +++++++++++++++---- 1 file changed, 28 insertions(+), 6 deletions(-) diff --git a/internal-packages/run-engine/src/run-queue/index.ts b/internal-packages/run-engine/src/run-queue/index.ts index a28ab2542c4..f7ea1acdfd7 100644 --- a/internal-packages/run-engine/src/run-queue/index.ts +++ b/internal-packages/run-engine/src/run-queue/index.ts @@ -209,11 +209,12 @@ const QUEUE_METRICS_CK_GAUGE_EXTRAS = { }; // Total-concurrency tail (gauge[10]/gauge[11]): live group cardinality + raw stored cap. -// Requires groupConcurrencyKey/totalConcurrencyLimitKey locals; the CK scripts that actually -// run (the Tracked variants and the CK dequeue) all declare them for the total-cap gate. +// Requires the groupConcurrencyKey local and the __totalLimitRaw memo (one GET shared with +// the total-cap gate); the CK scripts that run this (the Tracked variants and the CK +// dequeue) declare both. The group SCARD stays a fresh read: it must be post-admission. const QUEUE_METRICS_TOTAL_GAUGE_EXTRAS = { totalRunning: "redis.call('SCARD', groupConcurrencyKey)", - totalLimit: "redis.call('GET', totalConcurrencyLimitKey) or '0'", + totalLimit: "__totalLimitRaw() or '0'", }; // CK enqueue variants of the two gauges above, extended with the CK-health tail. @@ -4118,6 +4119,13 @@ local baseQueueKey = KEYS[15] -- Total-cap keys (KEYS 16-17) local groupConcurrencyKey = KEYS[16] local totalConcurrencyLimitKey = KEYS[17] +local __rawTotalLimit = nil +local function __totalLimitRaw() + if __rawTotalLimit == nil then + __rawTotalLimit = redis.call('GET', totalConcurrencyLimitKey) or false + end + return __rawTotalLimit +end local queueName = ARGV[1] local messageId = ARGV[2] @@ -4162,7 +4170,7 @@ if enableFastPath == '1' then -- slow path (the message queues; the dequeue gate holds it). local totalAllowsFastPath = true if totalConcurrencyEnabled then - local rawTotalLimit = redis.call('GET', totalConcurrencyLimitKey) + local rawTotalLimit = __totalLimitRaw() if rawTotalLimit then local totalLimit = math.min(tonumber(rawTotalLimit), envLimit) if tonumber(redis.call('SCARD', groupConcurrencyKey) or '0') >= totalLimit then @@ -4289,6 +4297,13 @@ local baseQueueKey = KEYS[16] -- Total-cap keys (KEYS 17-18) local groupConcurrencyKey = KEYS[17] local totalConcurrencyLimitKey = KEYS[18] +local __rawTotalLimit = nil +local function __totalLimitRaw() + if __rawTotalLimit == nil then + __rawTotalLimit = redis.call('GET', totalConcurrencyLimitKey) or false + end + return __rawTotalLimit +end local queueName = ARGV[1] local messageId = ARGV[2] @@ -4333,7 +4348,7 @@ if enableFastPath == '1' then -- Total-cap gate: see enqueueMessageCkTracked. local totalAllowsFastPath = true if totalConcurrencyEnabled then - local rawTotalLimit = redis.call('GET', totalConcurrencyLimitKey) + local rawTotalLimit = __totalLimitRaw() if rawTotalLimit then local totalLimit = math.min(tonumber(rawTotalLimit), envLimit) if tonumber(redis.call('SCARD', groupConcurrencyKey) or '0') >= totalLimit then @@ -4972,6 +4987,13 @@ local lengthCounterKey = KEYS[10] local runningCounterKey = KEYS[11] local groupConcurrencyKey = KEYS[12] local totalConcurrencyLimitKey = KEYS[13] +local __rawTotalLimit = nil +local function __totalLimitRaw() + if __rawTotalLimit == nil then + __rawTotalLimit = redis.call('GET', totalConcurrencyLimitKey) or false + end + return __rawTotalLimit +end local ckWildcardName = ARGV[1] local currentTime = tonumber(ARGV[2]) @@ -5014,7 +5036,7 @@ local actualMaxCount = math.min(maxCount, envAvailableCapacity) -- behind, and blocking on it would deadlock the run against itself). local totalHeadroom = nil if totalConcurrencyEnabled then - local rawTotalLimit = redis.call('GET', totalConcurrencyLimitKey) + local rawTotalLimit = __totalLimitRaw() if rawTotalLimit then local totalConcurrencyLimit = math.min(tonumber(rawTotalLimit), envConcurrencyLimit) local groupCurrentConcurrency = tonumber(redis.call('SCARD', groupConcurrencyKey) or '0') From f1ca125f4f34d8e47d3a061b66f501df8f1557bc Mon Sep 17 00:00:00 2001 From: Matt Aitken Date: Mon, 31 Aug 2026 17:52:44 +0100 Subject: [PATCH 21/32] perf(run-engine): dequeue gauges sample once, at return The gauge slot is single-valued and the last write wins, so on the success path the post-admission resample made the entry sample pure waste. A return wrapper computes the gauge exactly once per call at exit, keeping every emitted value identical while dropping the discarded reads (about six per plain dequeue, ten per keyed dequeue at full sampling). --- .../run-engine/src/run-queue/index.ts | 31 +++++++++++++------ 1 file changed, 21 insertions(+), 10 deletions(-) diff --git a/internal-packages/run-engine/src/run-queue/index.ts b/internal-packages/run-engine/src/run-queue/index.ts index f7ea1acdfd7..f35a3a51dae 100644 --- a/internal-packages/run-engine/src/run-queue/index.ts +++ b/internal-packages/run-engine/src/run-queue/index.ts @@ -173,8 +173,9 @@ local __qm_g = false local function __qmret(r) if r == nil then r = false end return {r, __qm_g} end`; // Fresh-read gauge for splice points with no reusable locals: enqueue slow-path (before -// return 0) and the base dequeue top. Gated on the last ARGV so it is inert unless the -// caller opts in. CK queues emit per-subqueue depth (queue_name aggregates via the MV). +// return 0) and the base dequeue's sample-at-return wrapper. Gated on the last ARGV so it +// is inert unless the caller opts in. CK queues emit per-subqueue depth (queue_name +// aggregates via the MV). const QUEUE_METRICS_GAUGE_LUA = createMetricsGaugeComputeLua({ enabledArg: "ARGV[#ARGV] == '1'", queued: "redis.call('ZCARD', queueKey)", @@ -4696,7 +4697,16 @@ local gatesEnabled = ARGV[7] == '1' local totalConcurrencyEnabled = ARGV[8] == '1' ${QUEUE_METRICS_GAUGE_PRELUDE} ${QUEUE_GATES_LUA_HELPERS} +-- Sample-at-return: the gauge is computed once, by the return wrapper, so every +-- exit emits the state as of that exit (post-admission on the success path) and +-- no path pays for a sample that a later one would overwrite. +local function __qmsample() ${QUEUE_METRICS_GAUGE_LUA} +end +do + local __qmret_inner = __qmret + __qmret = function(r) __qmsample() return __qmret_inner(r) end +end -- Check current env concurrency against the limit local envCurrentConcurrency = tonumber(redis.call('SCARD', envCurrentConcurrencyKey) or '0') @@ -4808,10 +4818,6 @@ else redis.call('ZADD', masterQueueKey, earliestMessage[2], queueName) end --- Re-sample the gauge so the emitted snapshot includes this batch's admissions; --- the top-of-script sample only covers the early returns where nothing was admitted. -${QUEUE_METRICS_GAUGE_LUA} - -- Return results as a flat array: [messageId1, messageScore1, messagePayload1, messageId2, messageScore2, messagePayload2, ...] return __qmret(results) `, @@ -5005,7 +5011,16 @@ local totalConcurrencyEnabled = ARGV[7] == '1' local gatesEnabled = ARGV[8] == '1' ${QUEUE_METRICS_GAUGE_PRELUDE} ${QUEUE_GATES_LUA_HELPERS} +-- Sample-at-return: the gauge is computed once, by the return wrapper, so every +-- exit emits the state as of that exit (post-admission on the success path) and +-- no path pays for a sample that a later one would overwrite. +local function __qmsample() ${QUEUE_METRICS_CK_DEQUEUE_GAUGE_LUA} +end +do + local __qmret_inner = __qmret + __qmret = function(r) __qmsample() return __qmret_inner(r) end +end local function decrLengthCounter() if tonumber(redis.call('GET', lengthCounterKey) or '0') > 0 then @@ -5194,10 +5209,6 @@ else redis.call('ZADD', masterQueueKey, earliestIdx[2], ckWildcardName) end --- Re-sample the gauge so the emitted snapshot includes this batch's admissions; --- the top-of-script sample only covers the early returns where nothing was admitted. -${QUEUE_METRICS_CK_DEQUEUE_GAUGE_LUA} - return __qmret(results) `, }); From ef18be5e9f86e36744bb44e61101293c22b21f15 Mon Sep 17 00:00:00 2001 From: Matt Aitken Date: Mon, 31 Aug 2026 18:07:55 +0100 Subject: [PATCH 22/32] test(run-engine): pin dequeue-emitted gauges so a sampling regression fails the suite The gauge assertions were all satisfiable by enqueue-emitted gauges, so breaking the dequeue scripts' sample-at-return wrapper left the suite green. The base test now requires the post-admission reading (running 1, queued 0) and the CK test requires the wildcard aggregate only the CK dequeue emits. Verified by mutation: disabling the wrapper fails both. --- .../run-engine/src/run-queue/metrics.test.ts | 22 ++++++++++++++----- 1 file changed, 16 insertions(+), 6 deletions(-) diff --git a/internal-packages/run-engine/src/run-queue/metrics.test.ts b/internal-packages/run-engine/src/run-queue/metrics.test.ts index ebfc295470e..d16b8048d95 100644 --- a/internal-packages/run-engine/src/run-queue/metrics.test.ts +++ b/internal-packages/run-engine/src/run-queue/metrics.test.ts @@ -123,7 +123,10 @@ describe("RunQueue queue-metrics emission", () => { const entries = await waitForEntries(redis, definition, (es) => { const seen = es.map((e) => e.fields.op); - return ["enqueue", "gauge", "started", "ack"].every((o) => seen.includes(o)); + if (!["enqueue", "gauge", "started", "ack"].every((o) => seen.includes(o))) return false; + return es.some( + (e) => e.fields.op === "gauge" && e.fields.cc === "1" && e.fields.ql === "0" + ); }); const ops = entries.map((e) => e.fields.op); expect(ops).toContain("enqueue"); @@ -141,6 +144,14 @@ describe("RunQueue queue-metrics emission", () => { expect(gauge!.fields.ckq).toBeUndefined(); expect(gauge!.fields.ckw).toBeUndefined(); + // Pins the dequeue script's sample-at-return wrapper: only the dequeue emits the + // post-admission reading (running 1, queued 0); the enqueue gauge sees the inverse. + const dequeueGauge = entries.find( + (e) => e.fields.op === "gauge" && e.fields.cc === "1" && e.fields.ql === "0" + ); + assertGauge(dequeueGauge); + expect(dequeueGauge!.fields.q).toContain("task/my-task"); + // The first counter emission also seeds a cum=0 baseline (no wait); the real reading // carries wait. Pick the reading (cum > 0). const started = entries.find((e) => e.fields.op === "started" && Number(e.fields.cum) > 0); @@ -283,14 +294,13 @@ describe("RunQueue queue-metrics emission", () => { expect(dequeued?.messageId).toBe(message.runId); const entries = await waitForEntries(redis, definition, (es) => - es.some( - (e) => e.fields.op === "gauge" && e.fields.q.includes(":ck:") && e.fields.thr === "0" - ) + es.some((e) => e.fields.op === "gauge" && e.fields.q.includes(":ck:*")) ); const gauges = entries.filter((e) => e.fields.op === "gauge"); expect(gauges.length).toBeGreaterThan(0); - // The aggregate CK dequeue gauge targets the CK wildcard and never sets thr. - const aggregate = gauges.find((e) => e.fields.q.includes(":ck:") && e.fields.thr === "0"); + // The aggregate gauge targets the CK wildcard and only the CK dequeue script emits + // it, so this pins that script's sample-at-return wrapper. + const aggregate = gauges.find((e) => e.fields.q.includes(":ck:*")); assertGauge(aggregate); expect(Number(aggregate!.fields.ql)).toBeGreaterThanOrEqual(0); expect(Number(aggregate!.fields.cc)).toBeGreaterThanOrEqual(0); From 48d9bae195baa336280150f3c4cc93f97f8a30d8 Mon Sep 17 00:00:00 2001 From: Matt Aitken Date: Mon, 31 Aug 2026 18:33:33 +0100 Subject: [PATCH 23/32] test(run-engine): wait for the metrics emitter connection before exercising it The emitter drops emissions until its Redis client is ready, and the tests enqueued immediately after constructing it, so the first counter entry was occasionally lost and the suite flaked roughly two runs in eighty. Awaiting readiness makes every emission land. --- internal-packages/run-engine/src/run-queue/metrics.test.ts | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/internal-packages/run-engine/src/run-queue/metrics.test.ts b/internal-packages/run-engine/src/run-queue/metrics.test.ts index d16b8048d95..efd00cd6aa1 100644 --- a/internal-packages/run-engine/src/run-queue/metrics.test.ts +++ b/internal-packages/run-engine/src/run-queue/metrics.test.ts @@ -81,6 +81,7 @@ describe("RunQueue queue-metrics emission", () => { definition, flag: { enabled: () => true }, }); + await emitter.waitUntilReady(); const queue = new RunQueue({ name: "rq", @@ -183,6 +184,7 @@ describe("RunQueue queue-metrics emission", () => { definition, flag: { enabled: () => true }, }); + await emitter.waitUntilReady(); const queue = new RunQueue({ name: "rq", tracer: trace.getTracer("rq"), @@ -255,6 +257,7 @@ describe("RunQueue queue-metrics emission", () => { maxLen: 1000, }; const emitter = new MetricsStreamEmitter({ redis, definition, flag: { enabled: () => true } }); + await emitter.waitUntilReady(); const queue = new RunQueue({ name: "rq", tracer: trace.getTracer("rq"), @@ -354,6 +357,7 @@ describe("RunQueue queue-metrics emission", () => { flag: { enabled: () => true }, gaugeSampleRate: 0, }); + await emitter.waitUntilReady(); const queue = new RunQueue({ name: "rq", tracer: trace.getTracer("rq"), From 481ee26b2fca6d5ef068e124969bc397a1ba1278 Mon Sep 17 00:00:00 2001 From: Matt Aitken Date: Mon, 31 Aug 2026 18:49:08 +0100 Subject: [PATCH 24/32] test(run-engine,metrics-pipeline): bound emitter-readiness waits and cover the consumer round-trip An unreachable Redis leaves waitUntilReady pending forever, so the bounded wait fails fast with a descriptive error instead of burning the test timeout. The consumer round-trip test gains the same readiness wait its gauge sibling already had, closing the remaining first-emission drop flake. --- .../metrics-pipeline/src/consumer.test.ts | 1 + .../run-engine/src/run-queue/metrics.test.ts | 19 +++++++++++++++---- 2 files changed, 16 insertions(+), 4 deletions(-) diff --git a/internal-packages/metrics-pipeline/src/consumer.test.ts b/internal-packages/metrics-pipeline/src/consumer.test.ts index 672fa426999..cb111ee1534 100644 --- a/internal-packages/metrics-pipeline/src/consumer.test.ts +++ b/internal-packages/metrics-pipeline/src/consumer.test.ts @@ -43,6 +43,7 @@ redisTest( }); await consumer.start(); + await emitter.waitUntilReady(); emitter.emit("queueA", { op: "enqueue", q: "queueA" }); emitter.emit("queueB", { op: "started", q: "queueB", wait: 42 }); diff --git a/internal-packages/run-engine/src/run-queue/metrics.test.ts b/internal-packages/run-engine/src/run-queue/metrics.test.ts index efd00cd6aa1..8b3872ddc2f 100644 --- a/internal-packages/run-engine/src/run-queue/metrics.test.ts +++ b/internal-packages/run-engine/src/run-queue/metrics.test.ts @@ -24,6 +24,17 @@ const authenticatedEnvDev = { organization: { id: "o1234" }, }; +// A dead Redis leaves waitUntilReady() pending forever (the client retries +// indefinitely), which would burn the whole test timeout with no diagnostic. +async function emitterReady(emitter: MetricsStreamEmitter) { + await Promise.race([ + emitter.waitUntilReady(), + setTimeout(15_000).then(() => { + throw new Error("metrics emitter Redis connection never became ready"); + }), + ]); +} + async function readAllEntries( redisOptions: { host: string; @@ -81,7 +92,7 @@ describe("RunQueue queue-metrics emission", () => { definition, flag: { enabled: () => true }, }); - await emitter.waitUntilReady(); + await emitterReady(emitter); const queue = new RunQueue({ name: "rq", @@ -184,7 +195,7 @@ describe("RunQueue queue-metrics emission", () => { definition, flag: { enabled: () => true }, }); - await emitter.waitUntilReady(); + await emitterReady(emitter); const queue = new RunQueue({ name: "rq", tracer: trace.getTracer("rq"), @@ -257,7 +268,7 @@ describe("RunQueue queue-metrics emission", () => { maxLen: 1000, }; const emitter = new MetricsStreamEmitter({ redis, definition, flag: { enabled: () => true } }); - await emitter.waitUntilReady(); + await emitterReady(emitter); const queue = new RunQueue({ name: "rq", tracer: trace.getTracer("rq"), @@ -357,7 +368,7 @@ describe("RunQueue queue-metrics emission", () => { flag: { enabled: () => true }, gaugeSampleRate: 0, }); - await emitter.waitUntilReady(); + await emitterReady(emitter); const queue = new RunQueue({ name: "rq", tracer: trace.getTracer("rq"), From 201a648e861b2b0e411f5fcb0b0b3dd6e7abab51 Mon Sep 17 00:00:00 2001 From: Matt Aitken Date: Mon, 31 Aug 2026 18:51:27 +0100 Subject: [PATCH 25/32] test(run-engine): abort the readiness race timer so its losing branch cannot reject unhandled --- .../run-engine/src/run-queue/metrics.test.ts | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/internal-packages/run-engine/src/run-queue/metrics.test.ts b/internal-packages/run-engine/src/run-queue/metrics.test.ts index 8b3872ddc2f..4c3f17125cf 100644 --- a/internal-packages/run-engine/src/run-queue/metrics.test.ts +++ b/internal-packages/run-engine/src/run-queue/metrics.test.ts @@ -26,13 +26,16 @@ const authenticatedEnvDev = { // A dead Redis leaves waitUntilReady() pending forever (the client retries // indefinitely), which would burn the whole test timeout with no diagnostic. +// Races values, not throws: a rejection in the losing branch of a settled race +// is an unhandled rejection, so the timer is aborted and swallowed instead. async function emitterReady(emitter: MetricsStreamEmitter) { - await Promise.race([ - emitter.waitUntilReady(), - setTimeout(15_000).then(() => { - throw new Error("metrics emitter Redis connection never became ready"); - }), - ]); + const abort = new AbortController(); + const timedOut = setTimeout(15_000, "timeout", { signal: abort.signal }).catch(() => "aborted"); + const winner = await Promise.race([emitter.waitUntilReady().then(() => "ready"), timedOut]); + abort.abort(); + if (winner === "timeout") { + throw new Error("metrics emitter Redis connection never became ready"); + } } async function readAllEntries( From 376e36cb2e923380f043256520d515c8a25a2f40 Mon Sep 17 00:00:00 2001 From: Matt Aitken Date: Mon, 31 Aug 2026 18:56:07 +0100 Subject: [PATCH 26/32] test(run-engine): close the emitter when the readiness wait times out --- internal-packages/run-engine/src/run-queue/metrics.test.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/internal-packages/run-engine/src/run-queue/metrics.test.ts b/internal-packages/run-engine/src/run-queue/metrics.test.ts index 4c3f17125cf..d82ddb35305 100644 --- a/internal-packages/run-engine/src/run-queue/metrics.test.ts +++ b/internal-packages/run-engine/src/run-queue/metrics.test.ts @@ -34,6 +34,7 @@ async function emitterReady(emitter: MetricsStreamEmitter) { const winner = await Promise.race([emitter.waitUntilReady().then(() => "ready"), timedOut]); abort.abort(); if (winner === "timeout") { + await emitter.close().catch(() => {}); throw new Error("metrics emitter Redis connection never became ready"); } } From ca263c94aee2d3b951a923017790a20f8c2de25a Mon Sep 17 00:00:00 2001 From: Matt Aitken Date: Mon, 31 Aug 2026 19:01:29 +0100 Subject: [PATCH 27/32] test(metrics-pipeline,run-engine): readiness wait for the per-stream test, honest timer comment The per-stream batches test carried the same first-emission drop race as its siblings; it now waits for the emitter connection too. The readiness helper's comment claimed a losing race branch rejects unhandled, which is not how Promise.race behaves (it handles every input); the abort's real benefit is releasing the timer promptly. --- internal-packages/metrics-pipeline/src/consumer.test.ts | 1 + internal-packages/run-engine/src/run-queue/metrics.test.ts | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/internal-packages/metrics-pipeline/src/consumer.test.ts b/internal-packages/metrics-pipeline/src/consumer.test.ts index cb111ee1534..f9f59335249 100644 --- a/internal-packages/metrics-pipeline/src/consumer.test.ts +++ b/internal-packages/metrics-pipeline/src/consumer.test.ts @@ -157,6 +157,7 @@ redisTest( }); await consumer.start(); + await emitter.waitUntilReady(); emitter.emit(a, { op: "enqueue", q: a }); emitter.emit(b, { op: "enqueue", q: b }); await waitFor(() => inserted.flatMap((i) => i.rows).length >= 2); diff --git a/internal-packages/run-engine/src/run-queue/metrics.test.ts b/internal-packages/run-engine/src/run-queue/metrics.test.ts index d82ddb35305..1b47238db89 100644 --- a/internal-packages/run-engine/src/run-queue/metrics.test.ts +++ b/internal-packages/run-engine/src/run-queue/metrics.test.ts @@ -26,8 +26,8 @@ const authenticatedEnvDev = { // A dead Redis leaves waitUntilReady() pending forever (the client retries // indefinitely), which would burn the whole test timeout with no diagnostic. -// Races values, not throws: a rejection in the losing branch of a settled race -// is an unhandled rejection, so the timer is aborted and swallowed instead. +// The abort releases the losing timer promptly so it cannot hold an event +// loop open for the remaining 15s after a fast ready. async function emitterReady(emitter: MetricsStreamEmitter) { const abort = new AbortController(); const timedOut = setTimeout(15_000, "timeout", { signal: abort.signal }).catch(() => "aborted"); From 9579de07c6ec6e4ff0f972d33cf29b578472fe3a Mon Sep 17 00:00:00 2001 From: Matt Aitken Date: Mon, 31 Aug 2026 19:10:57 +0100 Subject: [PATCH 28/32] test(run-engine): fire-and-forget the emitter close on readiness timeout A quit written to a socket that accepted but never completes the handshake never settles, which made the diagnostic throw unreachable. Closing without awaiting keeps the fast, descriptive failure. --- internal-packages/run-engine/src/run-queue/metrics.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal-packages/run-engine/src/run-queue/metrics.test.ts b/internal-packages/run-engine/src/run-queue/metrics.test.ts index 1b47238db89..edae6f8cc30 100644 --- a/internal-packages/run-engine/src/run-queue/metrics.test.ts +++ b/internal-packages/run-engine/src/run-queue/metrics.test.ts @@ -34,7 +34,7 @@ async function emitterReady(emitter: MetricsStreamEmitter) { const winner = await Promise.race([emitter.waitUntilReady().then(() => "ready"), timedOut]); abort.abort(); if (winner === "timeout") { - await emitter.close().catch(() => {}); + void emitter.close().catch(() => {}); throw new Error("metrics emitter Redis connection never became ready"); } } From 2de162a2b47adbbe72d75d3090166f498d3824cb Mon Sep 17 00:00:00 2001 From: Matt Aitken Date: Sun, 6 Sep 2026 13:19:40 +0100 Subject: [PATCH 29/32] rename(webapp,clickhouse): concurrency vocabulary for the unlaunched metrics surfaces The Query tables become concurrency_metrics and concurrency_metrics_by_key, and the combined_* columns become total_* across the query schemas, the insert mapping, the ClickHouse migration, and every TRQL chart, matching the perKey/total shape the SDK will expose. Internal names (stream keys, env vars, ClickHouse table names) are unchanged. None of these surfaces have launched, so this is the last free moment for the rename. --- .../components/queues/QueueMetricCards.tsx | 2 +- .../presenters/v3/BuiltInDashboards.server.ts | 10 ++--- .../v3/reports/health/health-data.ts | 4 +- .../presenters/v3/reports/report-registry.ts | 4 +- .../route.tsx | 4 +- .../route.tsx | 43 ++++++++++--------- .../route.tsx | 2 +- .../api.v1.queues.$queueParam.metrics.ts | 2 +- .../route.tsx | 4 +- apps/webapp/app/v3/querySchemas.ts | 16 +++---- apps/webapp/app/v3/queueMetricsMapping.ts | 4 +- apps/webapp/test/reportHealthData.test.ts | 4 +- ...2_add_queue_metrics_total_concurrency.sql} | 30 ++++++------- .../clickhouse/src/queueMetrics.ts | 4 +- 14 files changed, 68 insertions(+), 65 deletions(-) rename internal-packages/clickhouse/schema/{042_add_queue_metrics_combined_concurrency.sql => 042_add_queue_metrics_total_concurrency.sql} (91%) diff --git a/apps/webapp/app/components/queues/QueueMetricCards.tsx b/apps/webapp/app/components/queues/QueueMetricCards.tsx index 7c87a1e2d02..6b6692bcd9e 100644 --- a/apps/webapp/app/components/queues/QueueMetricCards.tsx +++ b/apps/webapp/app/components/queues/QueueMetricCards.tsx @@ -340,7 +340,7 @@ export function QueueSidebarStats({ }; const { rows, showLoading } = useQueueMetric( - `SELECT max(max_queued) AS peak_queued,\n round(quantilesMerge(0.5, 0.9, 0.95, 0.99)(wait_quantiles)[3]) AS worst_p95\nFROM queue_metrics`, + `SELECT max(max_queued) AS peak_queued,\n round(quantilesMerge(0.5, 0.9, 0.95, 0.99)(wait_quantiles)[3]) AS worst_p95\nFROM concurrency_metrics`, { ids, timeRange, queueName, defaultPeriod } ); const row = rows[0]; diff --git a/apps/webapp/app/presenters/v3/BuiltInDashboards.server.ts b/apps/webapp/app/presenters/v3/BuiltInDashboards.server.ts index d831568248d..7c609322ab2 100644 --- a/apps/webapp/app/presenters/v3/BuiltInDashboards.server.ts +++ b/apps/webapp/app/presenters/v3/BuiltInDashboards.server.ts @@ -634,7 +634,7 @@ const queuesDashboard: BuiltInDashboard = { "t-pressure": { title: "Queue pressure", query: "", display: { type: "title" } }, pressure: { title: "Queue pressure", - query: `SELECT queue,\n argMax(max_running, bucket_start) AS running,\n argMax(max_queued, bucket_start) AS queued,\n argMax(max_limit, bucket_start) AS limit,\n running + queued AS demand,\n max(max_queued) AS peak_queued,\n sum(throttled_count) AS throttled,\n multiIf(running >= limit AND queued > 0, 'queue-limited', queued > 0, 'backlogged', 'healthy') AS status\nFROM queue_metrics\nGROUP BY queue\nORDER BY peak_queued DESC`, + query: `SELECT queue,\n argMax(max_running, bucket_start) AS running,\n argMax(max_queued, bucket_start) AS queued,\n argMax(max_limit, bucket_start) AS limit,\n running + queued AS demand,\n max(max_queued) AS peak_queued,\n sum(throttled_count) AS throttled,\n multiIf(running >= limit AND queued > 0, 'queue-limited', queued > 0, 'backlogged', 'healthy') AS status\nFROM concurrency_metrics\nGROUP BY queue\nORDER BY peak_queued DESC`, display: { type: "table", prettyFormatting: true, @@ -644,7 +644,7 @@ const queuesDashboard: BuiltInDashboard = { "t-trends": { title: "Per-queue trends", query: "", display: { type: "title" } }, "running-q": { title: "Running by queue", - query: `SELECT timeBucket() AS t, queue, max(max_running) AS running\nFROM queue_metrics\nGROUP BY t, queue\nORDER BY t`, + query: `SELECT timeBucket() AS t, queue, max(max_running) AS running\nFROM concurrency_metrics\nGROUP BY t, queue\nORDER BY t`, // Grouped gauge: carry each queue's running across idle buckets (per-group LOCF). fillGaps: true, display: { @@ -661,7 +661,7 @@ const queuesDashboard: BuiltInDashboard = { }, "queued-q": { title: "Queue depth (backlog) by queue", - query: `SELECT timeBucket() AS t, queue, max(max_queued) AS queued\nFROM queue_metrics\nGROUP BY t, queue\nORDER BY t`, + query: `SELECT timeBucket() AS t, queue, max(max_queued) AS queued\nFROM concurrency_metrics\nGROUP BY t, queue\nORDER BY t`, // Grouped gauge: carry each queue's backlog across idle buckets (per-group LOCF). fillGaps: true, display: { @@ -678,7 +678,7 @@ const queuesDashboard: BuiltInDashboard = { }, "throttled-q": { title: "Throttled buckets by queue", - query: `SELECT timeBucket() AS t, queue, sum(throttled_count) AS throttled\nFROM queue_metrics\nGROUP BY t, queue\nORDER BY t`, + query: `SELECT timeBucket() AS t, queue, sum(throttled_count) AS throttled\nFROM concurrency_metrics\nGROUP BY t, queue\nORDER BY t`, // Grouped counter: per-group zero-fill so idle buckets read 0, not a gap. fillGaps: true, display: { @@ -697,7 +697,7 @@ const queuesDashboard: BuiltInDashboard = { title: "Enqueued vs started", // Counter states merge per queue, then sum outside: a single merge across queues // mixes unrelated odometers and returns wrong totals. - query: `SELECT t, sum(enq) AS enqueued, sum(st) AS started\nFROM (\n SELECT timeBucket() AS t, queue,\n deltaSumTimestampMerge(enqueue_delta) AS enq,\n deltaSumTimestampMerge(started_delta) AS st\n FROM queue_metrics\n GROUP BY t, queue\n)\nGROUP BY t\nORDER BY t`, + query: `SELECT t, sum(enq) AS enqueued, sum(st) AS started\nFROM (\n SELECT timeBucket() AS t, queue,\n deltaSumTimestampMerge(enqueue_delta) AS enq,\n deltaSumTimestampMerge(started_delta) AS st\n FROM concurrency_metrics\n GROUP BY t, queue\n)\nGROUP BY t\nORDER BY t`, display: { type: "chart", chartType: "line", diff --git a/apps/webapp/app/presenters/v3/reports/health/health-data.ts b/apps/webapp/app/presenters/v3/reports/health/health-data.ts index 3c14b498e57..de2cc01b4cd 100644 --- a/apps/webapp/app/presenters/v3/reports/health/health-data.ts +++ b/apps/webapp/app/presenters/v3/reports/health/health-data.ts @@ -212,7 +212,7 @@ function queueWorstQuery(): string { return `SELECT queue AS name, argMax(max_queued, bucket_start) AS latest_queued -FROM queue_metrics +FROM concurrency_metrics GROUP BY queue ORDER BY latest_queued DESC LIMIT 20`; @@ -228,7 +228,7 @@ FROM ( SELECT deltaSumTimestampMerge(dlq_delta) AS dlq, argMax(max_queued, bucket_start) AS latest_queued - FROM queue_metrics + FROM concurrency_metrics GROUP BY queue )`; } diff --git a/apps/webapp/app/presenters/v3/reports/report-registry.ts b/apps/webapp/app/presenters/v3/reports/report-registry.ts index 23cc178befb..d8b3bc4e798 100644 --- a/apps/webapp/app/presenters/v3/reports/report-registry.ts +++ b/apps/webapp/app/presenters/v3/reports/report-registry.ts @@ -4,7 +4,7 @@ import { loadHealthInput } from "./health/health-data"; import { type ReportViewModel } from "./report-view-model"; /** A query table a report may read. Same table names the query API authorizes against. */ -export type ReportQueryTable = "runs" | "env_metrics" | "queue_metrics"; +export type ReportQueryTable = "runs" | "env_metrics" | "concurrency_metrics"; export type ReportLoader = { /** Authorization metadata: the route derives its per-table JWT scope check from this. */ @@ -19,7 +19,7 @@ function defineReport(loader: ReportLoader): ReportLoader> = { health: defineReport({ - tables: ["runs", "env_metrics", "queue_metrics"], + tables: ["runs", "env_metrics", "concurrency_metrics"], load: (env, period) => loadHealthInput(env, period), interpret: interpretHealth, }), diff --git a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.queues/route.tsx b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.queues/route.tsx index 12b29480751..b3eab7a7249 100644 --- a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.queues/route.tsx +++ b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.queues/route.tsx @@ -128,7 +128,7 @@ const SearchParamsSchema = z.object({ // The live "Queued" / "Running" header blocks poll ClickHouse on a short cadence so they stay // current after first paint. They read the env-wide gauges from env_metrics (the env-level rollup -// of queue_metrics, cheapest for a dimension-free query), always over a fixed 15m window regardless +// of concurrency_metrics, cheapest for a dimension-free query), always over a fixed 15m window regardless // of the chart/table period, and are NOT scoped to the visible queue set (the blocks are env-wide). const QUEUE_LIVE_BLOCKS_PERIOD = "15m"; const QUEUE_LIVE_BLOCKS_QUERY = @@ -1670,7 +1670,7 @@ function QueueHealthBadge(health: QueueHealth) { ); } -// The `queue_metrics`-prefixed key a queue is stored under (task queues are prefixed `task/`). +// The metrics-row key a queue is stored under (task queues are prefixed `task/`). function queueMetricsKey(queue: { type: string; name: string }): string { return `${queue.type === "task" ? "task/" : ""}${queue.name}`; } diff --git a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.queues_.$queueParam/route.tsx b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.queues_.$queueParam/route.tsx index 933baafc43a..ab2f7f5eb46 100644 --- a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.queues_.$queueParam/route.tsx +++ b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.queues_.$queueParam/route.tsx @@ -266,7 +266,7 @@ export default function Page() { // The Concurrency keys tab exists only for queues with key activity: live keys in the // ckIndex, or nonzero CK history in the selected range (one cached scalar query decides). const { rows: gateRows, showLoading: gateLoading } = useQueueMetric( - `SELECT max(max_ck_backlogged) AS peak_keys, max(max_ck_wait_ms) AS peak_wait\nFROM queue_metrics`, + `SELECT max(max_ck_backlogged) AS peak_keys, max(max_ck_wait_ms) AS peak_wait\nFROM concurrency_metrics`, { ids, timeRange, queueName: fullName } ); const gateRow = gateRows[0]; @@ -463,7 +463,7 @@ function OverviewCharts({ } showLegend className="aspect-[2/1]" - query={`SELECT timeBucket() AS t, max(max_running) AS running, max(max_limit) AS limit\nFROM queue_metrics\nGROUP BY t\nORDER BY t`} + query={`SELECT timeBucket() AS t, max(max_running) AS running, max(max_limit) AS limit\nFROM concurrency_metrics\nGROUP BY t\nORDER BY t`} fillGaps minBucketSeconds={SYNCED_CHART_MIN_BUCKET_SECONDS} ids={ids} @@ -499,7 +499,7 @@ function OverviewCharts({ } showLegend className="aspect-[2/1]" - query={`SELECT timeBucket() AS t, max(max_combined_running) AS running, least(max(max_combined_limit), max(max_env_limit)) AS cap\nFROM queue_metrics\nGROUP BY t\nORDER BY t`} + query={`SELECT timeBucket() AS t, max(max_total_running) AS running, least(max(max_total_limit), max(max_env_limit)) AS cap\nFROM concurrency_metrics\nGROUP BY t\nORDER BY t`} fillGaps minBucketSeconds={SYNCED_CHART_MIN_BUCKET_SECONDS} ids={ids} @@ -521,7 +521,7 @@ function OverviewCharts({ title="Queue depth" info="How many runs are waiting in this queue over time." className="aspect-[2/1]" - query={`SELECT timeBucket() AS t, max(max_queued) AS queued\nFROM queue_metrics\nGROUP BY t\nORDER BY t`} + query={`SELECT timeBucket() AS t, max(max_queued) AS queued\nFROM concurrency_metrics\nGROUP BY t\nORDER BY t`} fillGaps minBucketSeconds={SYNCED_CHART_MIN_BUCKET_SECONDS} ids={ids} @@ -541,7 +541,7 @@ function OverviewCharts({ showLegend extraLegend={[{ color: "var(--color-warning)", label: "Falling behind" }]} className="aspect-[2/1]" - query={`SELECT timeBucket() AS t,\n deltaSumTimestampMerge(enqueue_delta) AS enqueued,\n deltaSumTimestampMerge(started_delta) AS started\nFROM queue_metrics\nGROUP BY t\nORDER BY t`} + query={`SELECT timeBucket() AS t,\n deltaSumTimestampMerge(enqueue_delta) AS enqueued,\n deltaSumTimestampMerge(started_delta) AS started\nFROM concurrency_metrics\nGROUP BY t\nORDER BY t`} fillGaps minBucketSeconds={SYNCED_CHART_MIN_BUCKET_SECONDS} ids={ids} @@ -560,7 +560,7 @@ function OverviewCharts({ info="How long runs wait before they start." showLegend className="aspect-[2/1]" - query={`SELECT timeBucket() AS t,\n round(quantilesMerge(0.5, 0.9, 0.95, 0.99)(wait_quantiles)[1]) AS p50,\n round(quantilesMerge(0.5, 0.9, 0.95, 0.99)(wait_quantiles)[3]) AS p95,\n round(quantilesMerge(0.5, 0.9, 0.95, 0.99)(wait_quantiles)[4]) AS p99,\n sum(wait_ms_count) AS samples\nFROM queue_metrics\nGROUP BY t\nORDER BY t`} + query={`SELECT timeBucket() AS t,\n round(quantilesMerge(0.5, 0.9, 0.95, 0.99)(wait_quantiles)[1]) AS p50,\n round(quantilesMerge(0.5, 0.9, 0.95, 0.99)(wait_quantiles)[3]) AS p95,\n round(quantilesMerge(0.5, 0.9, 0.95, 0.99)(wait_quantiles)[4]) AS p99,\n sum(wait_ms_count) AS samples\nFROM concurrency_metrics\nGROUP BY t\nORDER BY t`} fillGaps minBucketSeconds={SYNCED_CHART_MIN_BUCKET_SECONDS} sampleCountColumn="samples" @@ -583,7 +583,7 @@ function OverviewCharts({ } className="aspect-[2/1] sm:col-span-2 sm:aspect-[4/1]" - query={`SELECT timeBucket() AS t, sum(throttled_count) AS throttled\nFROM queue_metrics\nGROUP BY t\nORDER BY t`} + query={`SELECT timeBucket() AS t, sum(throttled_count) AS throttled\nFROM concurrency_metrics\nGROUP BY t\nORDER BY t`} fillGaps minBucketSeconds={SYNCED_CHART_MIN_BUCKET_SECONDS} ids={ids} @@ -698,7 +698,7 @@ function ConcurrencyKeyCharts({ title="Keys with backlog" info="Keys with runs waiting at once." className="aspect-[2/1]" - query={`SELECT timeBucket() AS t, max(max_ck_backlogged) AS keys\nFROM queue_metrics\nGROUP BY t\nORDER BY t`} + query={`SELECT timeBucket() AS t, max(max_ck_backlogged) AS keys\nFROM concurrency_metrics\nGROUP BY t\nORDER BY t`} fillGaps ids={ids} timeRange={timeRange} @@ -720,7 +720,7 @@ function ConcurrencyKeyCharts({ ) : null } className="aspect-[2/1]" - query={`SELECT timeBucket() AS t, max(max_ck_wait_ms) AS wait\nFROM queue_metrics\nGROUP BY t\nORDER BY t`} + query={`SELECT timeBucket() AS t, max(max_ck_wait_ms) AS wait\nFROM concurrency_metrics\nGROUP BY t\nORDER BY t`} fillGaps ids={ids} timeRange={timeRange} @@ -782,7 +782,7 @@ type GroupedKeyChartProps = { // search can match keys outside the top 8; then filter by the search and keep the top 8 of those. function GroupedKeyChartCard(props: GroupedKeyChartProps) { const { rows, showLoading, failed } = useQueueMetric( - `SELECT concurrency_key, ${props.rankExpr} AS peak\nFROM queue_metrics_by_key\nGROUP BY concurrency_key\nORDER BY peak DESC\nLIMIT 50`, + `SELECT concurrency_key, ${props.rankExpr} AS peak\nFROM concurrency_metrics_by_key\nGROUP BY concurrency_key\nORDER BY peak DESC\nLIMIT 50`, { ids: props.ids, timeRange: props.timeRange, queueName: props.queueName } ); const keyFilter = props.keyFilter; @@ -810,7 +810,7 @@ function GroupedKeySeries({ }: GroupedKeyChartProps & { keys: string[] }) { const inList = keys.map((k) => `'${trqlString(k)}'`).join(", "); const { rows, showLoading, failed } = useQueueMetric( - `SELECT timeBucket() AS t, concurrency_key, ${seriesExpr} AS v\nFROM queue_metrics_by_key\nWHERE concurrency_key IN (${inList})\nGROUP BY t, concurrency_key\nORDER BY t`, + `SELECT timeBucket() AS t, concurrency_key, ${seriesExpr} AS v\nFROM concurrency_metrics_by_key\nWHERE concurrency_key IN (${inList})\nGROUP BY t, concurrency_key\nORDER BY t`, { ids, timeRange, queueName, fillGaps } ); @@ -1074,7 +1074,7 @@ function KeyDrilldown({ } className="aspect-[2/1]" - query={`SELECT timeBucket() AS t, max(max_queued) AS queued, max(max_running) AS running\nFROM queue_metrics_by_key\nWHERE ${pin}\nGROUP BY t\nORDER BY t`} + query={`SELECT timeBucket() AS t, max(max_queued) AS queued, max(max_running) AS running\nFROM concurrency_metrics_by_key\nWHERE ${pin}\nGROUP BY t\nORDER BY t`} fillGaps minBucketSeconds={SYNCED_CHART_MIN_BUCKET_SECONDS} ids={ids} @@ -1089,7 +1089,7 @@ function KeyDrilldown({ 0, round(sum(wait_ms_sum) / sum(wait_ms_count)), 0) AS wait, sum(wait_ms_count) AS samples\nFROM queue_metrics_by_key\nWHERE ${pin}\nGROUP BY t\nORDER BY t`} + query={`SELECT timeBucket() AS t, if(sum(wait_ms_count) > 0, round(sum(wait_ms_sum) / sum(wait_ms_count)), 0) AS wait, sum(wait_ms_count) AS samples\nFROM concurrency_metrics_by_key\nWHERE ${pin}\nGROUP BY t\nORDER BY t`} fillGaps minBucketSeconds={SYNCED_CHART_MIN_BUCKET_SECONDS} sampleCountColumn="samples" @@ -1145,18 +1145,21 @@ function QueueStats({ timeRange: TimeRangeParams; queueName: string; }) { - const { rows } = useQueueMetric(`SELECT max(max_queued) AS peak_queued\nFROM queue_metrics`, { - ids, - timeRange, - queueName, - }); + const { rows } = useQueueMetric( + `SELECT max(max_queued) AS peak_queued\nFROM concurrency_metrics`, + { + ids, + timeRange, + queueName, + } + ); const peakQueued = rows[0] ? toNumber(rows[0].peak_queued) : 0; // Latest gauges from ClickHouse, polled every 15s so the live blocks keep ticking after first // paint. Read the newest bucket (largest t); until the first poll lands liveRows is empty and the // *Live values stay null, so the blocks show the loader values instead of flashing 0. const { rows: liveRows, responseReceivedAt } = useQueueMetric( - `SELECT timeBucket() AS t, max(max_running) AS running, max(max_queued) AS queued, max(max_limit) AS q_limit, max(max_ck_wait_ms) AS ck_wait FROM queue_metrics GROUP BY t ORDER BY t`, + `SELECT timeBucket() AS t, max(max_running) AS running, max(max_queued) AS queued, max(max_limit) AS q_limit, max(max_ck_wait_ms) AS ck_wait FROM concurrency_metrics GROUP BY t ORDER BY t`, { ids, timeRange: { period: "15m", from: null, to: null }, diff --git a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.tasks.standard.$taskParam/route.tsx b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.tasks.standard.$taskParam/route.tsx index 56f553547b1..ab8e5be48d6 100644 --- a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.tasks.standard.$taskParam/route.tsx +++ b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.tasks.standard.$taskParam/route.tsx @@ -527,7 +527,7 @@ function TaskActivityCard({ > {view === "queue" ? ( 1, // dummy — the queue name isn't resolved against Postgres authorization: { action: "read", - resource: () => ({ type: "query", id: "queue_metrics" }), + resource: () => ({ type: "query", id: "concurrency_metrics" }), }, }, async ({ params, searchParams, authentication }) => { diff --git a/apps/webapp/app/routes/resources.orgs.$organizationSlug.projects.$projectParam.env.$envParam.runs.$runParam.spans.$spanParam/route.tsx b/apps/webapp/app/routes/resources.orgs.$organizationSlug.projects.$projectParam.env.$envParam.runs.$runParam.spans.$spanParam/route.tsx index 02571272fec..d81a5a4b664 100644 --- a/apps/webapp/app/routes/resources.orgs.$organizationSlug.projects.$projectParam.env.$envParam.runs.$runParam.spans.$spanParam/route.tsx +++ b/apps/webapp/app/routes/resources.orgs.$organizationSlug.projects.$projectParam.env.$envParam.runs.$runParam.spans.$spanParam/route.tsx @@ -1342,7 +1342,7 @@ function WaitingInQueueBlock({ responseReceivedAt, lastSuccessfulResponseAt, } = useQueueMetric( - `SELECT timeBucket() AS t, max(max_running) AS running, max(max_queued) AS queued, max(max_limit) AS q_limit\nFROM queue_metrics\nGROUP BY t\nORDER BY t`, + `SELECT timeBucket() AS t, max(max_running) AS running, max(max_queued) AS queued, max(max_limit) AS q_limit\nFROM concurrency_metrics\nGROUP BY t\nORDER BY t`, { ids: waiting.ids, timeRange: { period: "15m", from: null, to: null }, @@ -1441,7 +1441,7 @@ function WaitingInQueueBlock({
{ - it("measured path: queue_metrics source, real pending, parsed dlq, window from timeRange", async () => { + it("measured path: concurrency_metrics source, real pending, parsed dlq, window from timeRange", async () => { const input = await loadHealthInput( fakeEnv, "1h", diff --git a/internal-packages/clickhouse/schema/042_add_queue_metrics_combined_concurrency.sql b/internal-packages/clickhouse/schema/042_add_queue_metrics_total_concurrency.sql similarity index 91% rename from internal-packages/clickhouse/schema/042_add_queue_metrics_combined_concurrency.sql rename to internal-packages/clickhouse/schema/042_add_queue_metrics_total_concurrency.sql index dd0c4c53b34..f0cef36b4c8 100644 --- a/internal-packages/clickhouse/schema/042_add_queue_metrics_combined_concurrency.sql +++ b/internal-packages/clickhouse/schema/042_add_queue_metrics_total_concurrency.sql @@ -1,23 +1,23 @@ -- +goose Up --- Total-concurrency gauges: combined_running is the in-flight count across ALL --- concurrency-key variants of a queue (the groupConcurrency set), combined_limit the +-- Total-concurrency gauges: total_running is the in-flight count across ALL +-- concurrency-key variants of a queue (the groupConcurrency set), total_limit the -- RAW stored total cap (0 = none, readers clamp against max_env_limit). Emitted on -- base-queue gauge rows only. Per-key gauge rows carry the queue concurrency -- limit that applied in queue_limit, surfaced in the ck tier as max_limit -- (1000000 = no explicit limit). ALTER TABLE trigger_dev.queue_metrics_raw_v1 - ADD COLUMN IF NOT EXISTS combined_running UInt32 DEFAULT 0, - ADD COLUMN IF NOT EXISTS combined_limit UInt32 DEFAULT 0; + ADD COLUMN IF NOT EXISTS total_running UInt32 DEFAULT 0, + ADD COLUMN IF NOT EXISTS total_limit UInt32 DEFAULT 0; ALTER TABLE trigger_dev.queue_metrics_v1 - ADD COLUMN IF NOT EXISTS max_combined_running SimpleAggregateFunction(max, UInt32), - ADD COLUMN IF NOT EXISTS max_combined_limit SimpleAggregateFunction(max, UInt32); + ADD COLUMN IF NOT EXISTS max_total_running SimpleAggregateFunction(max, UInt32), + ADD COLUMN IF NOT EXISTS max_total_limit SimpleAggregateFunction(max, UInt32); ALTER TABLE trigger_dev.queue_metrics_5m_v1 - ADD COLUMN IF NOT EXISTS max_combined_running SimpleAggregateFunction(max, UInt32), - ADD COLUMN IF NOT EXISTS max_combined_limit SimpleAggregateFunction(max, UInt32); + ADD COLUMN IF NOT EXISTS max_total_running SimpleAggregateFunction(max, UInt32), + ADD COLUMN IF NOT EXISTS max_total_limit SimpleAggregateFunction(max, UInt32); ALTER TABLE trigger_dev.queue_metrics_ck_v1 ADD COLUMN IF NOT EXISTS max_limit SimpleAggregateFunction(max, UInt32); @@ -46,8 +46,8 @@ SELECT max(env_limit) AS max_env_limit, max(ck_backlogged) AS max_ck_backlogged, max(ck_max_wait_ms) AS max_ck_wait_ms, - max(combined_running) AS max_combined_running, - max(combined_limit) AS max_combined_limit, + max(total_running) AS max_total_running, + max(total_limit) AS max_total_limit, sumIf(wait_ms, op = 'started' AND concurrency_key = '') AS wait_ms_sum, countIf(op = 'started' AND wait_ms > 0 AND concurrency_key = '') AS wait_ms_count, quantilesStateIf(0.5, 0.9, 0.95, 0.99)(wait_ms, op = 'started' AND wait_ms > 0 AND concurrency_key = '') AS wait_quantiles @@ -74,8 +74,8 @@ SELECT max(env_limit) AS max_env_limit, max(ck_backlogged) AS max_ck_backlogged, max(ck_max_wait_ms) AS max_ck_wait_ms, - max(combined_running) AS max_combined_running, - max(combined_limit) AS max_combined_limit, + max(total_running) AS max_total_running, + max(total_limit) AS max_total_limit, sumIf(wait_ms, op = 'started' AND concurrency_key = '') AS wait_ms_sum, countIf(op = 'started' AND wait_ms > 0 AND concurrency_key = '') AS wait_ms_count, quantilesStateIf(0.5, 0.9, 0.95, 0.99)(wait_ms, op = 'started' AND wait_ms > 0 AND concurrency_key = '') AS wait_quantiles @@ -105,9 +105,9 @@ DROP VIEW IF EXISTS trigger_dev.queue_metrics_ck_mv_v1; DROP VIEW IF EXISTS trigger_dev.queue_metrics_5m_mv_v1; DROP VIEW IF EXISTS trigger_dev.queue_metrics_mv_v1; ALTER TABLE trigger_dev.queue_metrics_ck_v1 DROP COLUMN IF EXISTS max_limit; -ALTER TABLE trigger_dev.queue_metrics_5m_v1 DROP COLUMN IF EXISTS max_combined_running, DROP COLUMN IF EXISTS max_combined_limit; -ALTER TABLE trigger_dev.queue_metrics_v1 DROP COLUMN IF EXISTS max_combined_running, DROP COLUMN IF EXISTS max_combined_limit; -ALTER TABLE trigger_dev.queue_metrics_raw_v1 DROP COLUMN IF EXISTS combined_running, DROP COLUMN IF EXISTS combined_limit; +ALTER TABLE trigger_dev.queue_metrics_5m_v1 DROP COLUMN IF EXISTS max_total_running, DROP COLUMN IF EXISTS max_total_limit; +ALTER TABLE trigger_dev.queue_metrics_v1 DROP COLUMN IF EXISTS max_total_running, DROP COLUMN IF EXISTS max_total_limit; +ALTER TABLE trigger_dev.queue_metrics_raw_v1 DROP COLUMN IF EXISTS total_running, DROP COLUMN IF EXISTS total_limit; -- Recreate the pre-042 materialized views (the definitions from 036) so ingestion keeps -- feeding every aggregate table after a rollback. diff --git a/internal-packages/clickhouse/src/queueMetrics.ts b/internal-packages/clickhouse/src/queueMetrics.ts index f3a6be695e4..aa3cf5296d2 100644 --- a/internal-packages/clickhouse/src/queueMetrics.ts +++ b/internal-packages/clickhouse/src/queueMetrics.ts @@ -21,8 +21,8 @@ export const QueueMetricsRawV1Input = z.object({ throttled: z.number().optional(), ck_backlogged: z.number().optional(), ck_max_wait_ms: z.number().optional(), - combined_running: z.number().optional(), - combined_limit: z.number().optional(), + total_running: z.number().optional(), + total_limit: z.number().optional(), wait_ms: z.number().optional(), cumulative: z.number().optional(), }); From 5bab9e15e584f4d7d3a9e100a08d9decb0bbb97b Mon Sep 17 00:00:00 2001 From: Matt Aitken Date: Sun, 6 Sep 2026 13:33:29 +0100 Subject: [PATCH 30/32] fix(webapp,clickhouse,run-engine): rename follow-ups from review Renumbers the metrics migration to 044 (main took 042 and 043), updates the two tests that pin the query-table and authorization ids, keeps the combined-cap chart from drawing a zero cap for history before a cap existed, and drops the flag docblock's stale per-key override paragraph. --- .../route.tsx | 2 +- apps/webapp/test/dashboardAgentToolScopes.test.ts | 2 +- apps/webapp/test/reportsApiRoute.test.ts | 10 +++++++--- ...sql => 044_add_queue_metrics_total_concurrency.sql} | 0 internal-packages/run-engine/src/run-queue/index.ts | 4 ---- 5 files changed, 9 insertions(+), 9 deletions(-) rename internal-packages/clickhouse/schema/{042_add_queue_metrics_total_concurrency.sql => 044_add_queue_metrics_total_concurrency.sql} (100%) diff --git a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.queues_.$queueParam/route.tsx b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.queues_.$queueParam/route.tsx index ab2f7f5eb46..3e830514338 100644 --- a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.queues_.$queueParam/route.tsx +++ b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.queues_.$queueParam/route.tsx @@ -499,7 +499,7 @@ function OverviewCharts({ } showLegend className="aspect-[2/1]" - query={`SELECT timeBucket() AS t, max(max_total_running) AS running, least(max(max_total_limit), max(max_env_limit)) AS cap\nFROM concurrency_metrics\nGROUP BY t\nORDER BY t`} + query={`SELECT timeBucket() AS t, max(max_total_running) AS running, least(nullIf(max(max_total_limit), 0), max(max_env_limit)) AS cap\nFROM concurrency_metrics\nGROUP BY t\nORDER BY t`} fillGaps minBucketSeconds={SYNCED_CHART_MIN_BUCKET_SECONDS} ids={ids} diff --git a/apps/webapp/test/dashboardAgentToolScopes.test.ts b/apps/webapp/test/dashboardAgentToolScopes.test.ts index 6eabb2d8eb7..7f850c3d3a4 100644 --- a/apps/webapp/test/dashboardAgentToolScopes.test.ts +++ b/apps/webapp/test/dashboardAgentToolScopes.test.ts @@ -34,7 +34,7 @@ const VIA_ENV_JWT: Read[] = [ { tool: "get_queue (metrics)", path: "/api/v1/queues/:name/metrics", - resource: { type: "query", id: "queue_metrics" }, + resource: { type: "query", id: "concurrency_metrics" }, }, { tool: "get_queue (live row)", path: "/api/v1/queues/:name", resource: { type: "queues" } }, { diff --git a/apps/webapp/test/reportsApiRoute.test.ts b/apps/webapp/test/reportsApiRoute.test.ts index f027c0d9eb3..37043b6513b 100644 --- a/apps/webapp/test/reportsApiRoute.test.ts +++ b/apps/webapp/test/reportsApiRoute.test.ts @@ -105,7 +105,7 @@ describe("api.v1.reports.$key — authorization", () => { expect(requiredResources("health")).toEqual([ { type: "query", id: "runs" }, { type: "query", id: "env_metrics" }, - { type: "query", id: "queue_metrics" }, + { type: "query", id: "concurrency_metrics" }, ]); }); @@ -118,7 +118,7 @@ describe("api.v1.reports.$key — authorization", () => { describe("reportQueryTables — scope derivation from the registry", () => { const registry: Record = { - health: { tables: ["runs", "env_metrics", "queue_metrics"] }, + health: { tables: ["runs", "env_metrics", "concurrency_metrics"] }, narrow: { tables: ["runs"] }, }; @@ -127,7 +127,11 @@ describe("reportQueryTables — scope derivation from the registry", () => { }); it("still gives the wider report all of its tables", () => { - expect(reportQueryTables("health", registry)).toEqual(["runs", "env_metrics", "queue_metrics"]); + expect(reportQueryTables("health", registry)).toEqual([ + "runs", + "env_metrics", + "concurrency_metrics", + ]); }); it("returns no tables for an unknown key", () => { diff --git a/internal-packages/clickhouse/schema/042_add_queue_metrics_total_concurrency.sql b/internal-packages/clickhouse/schema/044_add_queue_metrics_total_concurrency.sql similarity index 100% rename from internal-packages/clickhouse/schema/042_add_queue_metrics_total_concurrency.sql rename to internal-packages/clickhouse/schema/044_add_queue_metrics_total_concurrency.sql diff --git a/internal-packages/run-engine/src/run-queue/index.ts b/internal-packages/run-engine/src/run-queue/index.ts index f35a3a51dae..b5b39efdac7 100644 --- a/internal-packages/run-engine/src/run-queue/index.ts +++ b/internal-packages/run-engine/src/run-queue/index.ts @@ -318,10 +318,6 @@ export type RunQueueOptions = { * that dead-lettered or suspended through a mirror-less path. Enabling only after * every instance runs this build avoids the noise but is no longer load-bearing * for correctness. - * - * Per-concurrency-key limit overrides are part of the same concurrency-limits - * feature and are deliberately enforced behind this flag too: writes are always - * accepted and durable, and enforcement of both arrives together. */ totalConcurrencyEnabled?: boolean; /** From 6ed3a384aa80b16264397475aeb004abf93e8c28 Mon Sep 17 00:00:00 2001 From: Matt Aitken Date: Sun, 6 Sep 2026 13:52:21 +0100 Subject: [PATCH 31/32] fix(webapp,clickhouse): pre-cap history keeps its truthful gap in the combined chart The config-gauge back-fill exists to cover leading buckets with no samples, but it also overwrote sampled history from before a combined cap existed, showing a cap that was never in force. The cap series now carries a sampled guard column and the back-fill skips sampled buckets. Also corrects the renumbered migration's pre-044 comment. --- .../app/components/queues/QueueMetricCards.tsx | 14 ++++++++++++-- .../route.tsx | 3 ++- .../044_add_queue_metrics_total_concurrency.sql | 2 +- 3 files changed, 15 insertions(+), 4 deletions(-) diff --git a/apps/webapp/app/components/queues/QueueMetricCards.tsx b/apps/webapp/app/components/queues/QueueMetricCards.tsx index 6b6692bcd9e..e878d5536e2 100644 --- a/apps/webapp/app/components/queues/QueueMetricCards.tsx +++ b/apps/webapp/app/components/queues/QueueMetricCards.tsx @@ -102,6 +102,12 @@ type QueueMetricChartProps = { * are config values that existed all along, so carry the first value backward instead. */ carryBackfill?: string[]; + /** + * Column that marks a bucket as genuinely sampled. When set, carryBackfill only + * overwrites buckets where this column is absent or zero, so history from before + * a config value existed keeps its truthful gap instead of inheriting the value. + */ + carryBackfillGuard?: string; /** Show the series legend below the chart (use for multi-series charts). */ showLegend?: boolean; /** @@ -141,6 +147,7 @@ export function QueueMetricChart({ defaultPeriod, warningOverlay, carryBackfill, + carryBackfillGuard, thresholdStroke, onHasDataChange, minBucketSeconds, @@ -174,12 +181,15 @@ export function QueueMetricChart({ const first = points.findIndex((p) => toNumber(p[key]) > 0); if (first > 0) { const value = points[first]![key]!; - for (let i = 0; i < first; i++) points[i]![key] = value; + for (let i = 0; i < first; i++) { + if (carryBackfillGuard && toNumber(points[i]![carryBackfillGuard]) > 0) continue; + points[i]![key] = value; + } } } } return points; - }, [rows, series, carryBackfill, sampleCountColumn]); + }, [rows, series, carryBackfill, carryBackfillGuard, sampleCountColumn]); const chartConfig = useMemo(() => { const cfg: ChartConfig = {}; diff --git a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.queues_.$queueParam/route.tsx b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.queues_.$queueParam/route.tsx index 3e830514338..ffffc195b3a 100644 --- a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.queues_.$queueParam/route.tsx +++ b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.queues_.$queueParam/route.tsx @@ -499,7 +499,7 @@ function OverviewCharts({ } showLegend className="aspect-[2/1]" - query={`SELECT timeBucket() AS t, max(max_total_running) AS running, least(nullIf(max(max_total_limit), 0), max(max_env_limit)) AS cap\nFROM concurrency_metrics\nGROUP BY t\nORDER BY t`} + query={`SELECT timeBucket() AS t, max(max_total_running) AS running, least(nullIf(max(max_total_limit), 0), max(max_env_limit)) AS cap, max(max_env_limit) AS sampled\nFROM concurrency_metrics\nGROUP BY t\nORDER BY t`} fillGaps minBucketSeconds={SYNCED_CHART_MIN_BUCKET_SECONDS} ids={ids} @@ -515,6 +515,7 @@ function OverviewCharts({ aboveColor: "var(--color-warning)", }} carryBackfill={["cap"]} + carryBackfillGuard="sampled" /> ) : null} Date: Sun, 6 Sep 2026 14:00:37 +0100 Subject: [PATCH 32/32] fix(webapp): carry-guard column reaches the chart points, Total naming in the UI The sampled guard was queried but dropped when rows became chart points, so the back-fill guard always read zero and pre-cap history was still overwritten; the guard column is now copied onto each point. User-visible strings move from Combined to Total, matching the perKey/total vocabulary. --- apps/webapp/app/components/queues/QueueMetricCards.tsx | 1 + .../route.tsx | 4 ++-- .../route.tsx | 4 ++-- 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/apps/webapp/app/components/queues/QueueMetricCards.tsx b/apps/webapp/app/components/queues/QueueMetricCards.tsx index e878d5536e2..8a9fa8f5549 100644 --- a/apps/webapp/app/components/queues/QueueMetricCards.tsx +++ b/apps/webapp/app/components/queues/QueueMetricCards.tsx @@ -170,6 +170,7 @@ export function QueueMetricChart({ }; const hasSamples = sampleCountColumn ? toNumber(r[sampleCountColumn]) > 0 : true; for (const s of series) point[s.key] = hasSamples ? toNumber(r[s.key]) : null; + if (carryBackfillGuard) point[carryBackfillGuard] = toNumber(r[carryBackfillGuard]); return point; }) .filter((p) => Number.isFinite(p.bucket)); diff --git a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.queues/route.tsx b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.queues/route.tsx index b3eab7a7249..1919196bdd0 100644 --- a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.queues/route.tsx +++ b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.queues/route.tsx @@ -903,7 +903,7 @@ function QueuesWithMetricsView() { } content={ <> - Combined limit: at most{" "} + Total limit: at most{" "} {Math.min( queue.concurrency.combined.current, environment.concurrencyLimit @@ -1997,7 +1997,7 @@ function ClassicQueuesView() { } content={ <> - Combined limit: at most{" "} + Total limit: at most{" "} {Math.min( queue.concurrency.combined.current, environment.concurrencyLimit diff --git a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.queues_.$queueParam/route.tsx b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.queues_.$queueParam/route.tsx index ffffc195b3a..375ba2ca0f2 100644 --- a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.queues_.$queueParam/route.tsx +++ b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.queues_.$queueParam/route.tsx @@ -488,7 +488,7 @@ function OverviewCharts({ /> {hasTotalLimit ? ( Runs in flight across ALL concurrency keys ( @@ -506,7 +506,7 @@ function OverviewCharts({ timeRange={timeRange} queueName={queueName} series={[ - { key: "cap", label: "Combined limit", color: COLORS.limit }, + { key: "cap", label: "Total limit", color: COLORS.limit }, { key: "running", label: "Running", color: COLORS.running }, ]} thresholdStroke={{