feat(status-pages): page service, publisher, break-glass import, auto incidents

apps/alerting/src/scheduled.test.ts

@@ -20,7 +20,10 @@ const cronCases = [
 	],
 	["*/15 * * * *", ["digest", "supabaseInsights"]],
 	["0 * * * *", ["serviceMapRollup", "supabaseDiscovery", "awsConnections"]],
-	["* * * * *", ["alert", "alertLongQuery", "error", "escalation", "fixVerification", "supabaseLogs"]],
+	[
+		"* * * * *",
+		["alert", "alertLongQuery", "error", "escalation", "fixVerification", "statusPages", "supabaseLogs"],
+	],
 ] as const
 
 /** Every tick recording its name when it runs, with `overrides` swapped in. */
@@ -45,6 +48,7 @@ const recordingTicks = (
 		prReviewPostMerge: tick("prReviewPostMerge"),
 		railwayMetrics: tick("railwayMetrics"),
 		serviceMapRollup: tick("serviceMapRollup"),
+		statusPages: tick("statusPages"),
 		supabaseDiscovery: tick("supabaseDiscovery"),
 		supabaseLogs: tick("supabaseLogs"),
 		supabaseInsights: tick("supabaseInsights"),

apps/alerting/src/scheduled.ts

@@ -28,17 +28,33 @@ import { SupabaseConnectionService } from "@maple/backend/services/integrations/
 import { SupabaseInsightsService } from "@maple/backend/services/integrations/SupabaseInsightsService"
 import { SupabaseLogsService } from "@maple/backend/services/integrations/SupabaseLogsService"
 import { ServiceMapRollupService } from "@maple/backend/services/dashboards/ServiceMapRollupService"
+import { StatusPagePublisher } from "@maple/backend/services/status-pages/StatusPagePublisher"
+import { StatusPageReconciler } from "@maple/backend/services/status-pages/StatusPageReconciler"
+import {
+	type StatusPageBindings,
+	StatusPageStore,
+} from "@maple/backend/services/status-pages/StatusPageStore"
 import { layerMapleDbConnection } from "@maple/backend/platform/pg-connection-source"
 import { summarizeCause } from "@maple/backend/platform/describe-cause"
 import { withPgConnectionScope } from "@maple/backend/platform/pg-connection-scope"
 import { EmailSender, type EmailSenderClient } from "@maple/backend/platform/bindings"
 import { type ChatSessionNamespace, chatSessionsLayerIfBound } from "@maple/backend/platform/chat-sessions"
 import { envPorts } from "@maple/backend/platform/env-ports"
-import { Cause, Effect, Layer, Match, Option } from "effect"
+import { Cause, Clock, Effect, Layer, Match, Option } from "effect"
 import { FetchHttpClient } from "effect/http"
 import { isLaneOf, scheduleFor, type TickLane } from "./schedule.ts"
 import type { AlertingWorkerEnv } from "./worker.ts"
 
+/** The status page publisher, only when its KV and R2 bindings exist on this Worker. */
+const statusPagesLayerIfBound = (bindings: Option.Option<StatusPageBindings>) =>
+	Option.match(bindings, {
+		onNone: () => Layer.empty,
+		onSome: ({ kv, bucket }) =>
+			Layer.mergeAll(StatusPagePublisher.layer, StatusPageReconciler.layer).pipe(
+				Layer.provide(StatusPageStore.fromBindings(kv, bucket)),
+			),
+	})
+
 /**
  * The tick graph for one fire. Deliberately without a tracer or logger of its
  * own: those come from the SDK the Worker bridge builds into each fire's
@@ -50,6 +66,7 @@ export const layerTick = (
 	env: AlertingWorkerEnv,
 	email: Option.Option<EmailSenderClient> = Option.none(),
 	chatSessions?: ChatSessionNamespace,
+	statusPages: Option.Option<StatusPageBindings> = Option.none(),
 ) =>
 	Layer.mergeAll(
 		AlertsService.layer,
@@ -73,6 +90,7 @@ export const layerTick = (
 		// Read by `maybeEnqueueTriage` when present; its absence means every
 		// incident opened here is investigated unclassified.
 		IncidentClassifier.layer,
+		statusPagesLayerIfBound(statusPages),
 	).pipe(
 		Layer.provide(PullRequestLookupLive),
 		Layer.provide(Layer.mergeAll(Env.layer, DatabasePgLive, EdgeCacheServiceLive, FetchHttpClient.layer)),
@@ -400,6 +418,43 @@ const prReviewPostMergeTick = makeTick(
 			: undefined,
 )
 
+/**
+ * Imports break-glass updates, repairs auto incidents, then sweeps: edge cleanups, lagging pages
+ * and the heartbeat (written by the sweep only once it reached the database). Each step isolated.
+ */
+const statusPagesTick = Effect.gen(function* () {
+	const publisher = yield* Effect.serviceOption(StatusPagePublisher)
+	const reconciler = yield* Effect.serviceOption(StatusPageReconciler)
+	if (Option.isNone(publisher) || Option.isNone(reconciler)) {
+		return yield* Effect.logInfo("Status page storage is not bound; skipping status page publishing")
+	}
+	yield* makeTick(reconciler.value.reconcileAll(), "status_page_reconcile", (result) =>
+		result.imported + result.cleared + result.failed > 0
+			? {
+					statusPageImported: result.imported,
+					statusPageCleared: result.cleared,
+					statusPageFailures: result.failed,
+				}
+			: undefined,
+	)
+	yield* makeTick(reconciler.value.repairAutoIncidents(), "status_page_repair", (repaired) =>
+		repaired > 0 ? { statusPageAutoIncidentsRepaired: repaired } : undefined,
+	)
+	const nowMs = yield* Clock.currentTimeMillis
+	yield* makeTick(publisher.value.sweep(nowMs), "status_page_publish", (result) =>
+		result.due + result.cleaned + result.cleanupsFailed > 0
+			? {
+					statusPagesDue: result.due,
+					statusPagesWritten: result.written,
+					statusPagesUnchanged: result.unchanged,
+					statusPageFailures: result.failed,
+					statusPagesCleaned: result.cleaned,
+					statusPageCleanupFailures: result.cleanupsFailed,
+				}
+			: undefined,
+	)
+})
+
 export interface ScheduledTickPrograms<R = never> {
 	readonly alert: Effect.Effect<void, never, R>
 	readonly alertLongQuery: Effect.Effect<void, never, R>
@@ -416,6 +471,7 @@ export interface ScheduledTickPrograms<R = never> {
 	readonly prReviewPostMerge: Effect.Effect<void, never, R>
 	readonly railwayMetrics: Effect.Effect<void, never, R>
 	readonly serviceMapRollup: Effect.Effect<void, never, R>
+	readonly statusPages: Effect.Effect<void, never, R>
 	readonly supabaseDiscovery: Effect.Effect<void, never, R>
 	readonly supabaseLogs: Effect.Effect<void, never, R>
 	readonly supabaseInsights: Effect.Effect<void, never, R>
@@ -430,7 +486,8 @@ const laneProgram = <R>(lane: TickLane, ticks: ScheduledTickPrograms<R>): Effect
 		// a window this minute's error tick just refuted must already be settled
 		// when the verification tick looks at it.
 		Match.when("error", () => Effect.andThen(ticks.error, ticks.fixVerification)),
-		Match.when("escalation", () => ticks.escalation),
+		// Status pages ride the escalation lane: the minute cron is one connection under its cap.
+		Match.when("escalation", () => Effect.andThen(ticks.escalation, ticks.statusPages)),
 		Match.when("anomaly", () => ticks.anomaly),
 		Match.when("cloudflare_analytics", () => ticks.cloudflareAnalytics),
 		Match.when("gcp_metrics", () => ticks.gcpMetrics),
@@ -516,6 +573,7 @@ export const scheduledTicks: ScheduledTickPrograms<ScheduledServices> = {
 	prReviewPostMerge: prReviewPostMergeTick,
 	railwayMetrics: railwayMetricsTick,
 	serviceMapRollup: serviceMapRollupTick,
+	statusPages: statusPagesTick,
 	supabaseDiscovery: supabaseDiscoveryTick,
 	supabaseLogs: supabaseLogsTick,
 	supabaseInsights: supabaseInsightsTick,
@@ -534,10 +592,11 @@ export const runScheduled = (
 	env: AlertingWorkerEnv,
 	email: Option.Option<EmailSenderClient>,
 	chatSessions: ChatSessionNamespace,
+	statusPages: Option.Option<StatusPageBindings>,
 ): Effect.Effect<void, unknown> =>
 	withPgConnectionScope(selectLaneProgram(cron, lane, scheduledTicks)).pipe(
 		// One lane is one application run: the layer is built here and released
 		// with it, as the async entry's ManagedRuntime was.
 		// oxlint-disable-next-line effecttsgo/strict-effect-provide
-		Effect.provide(layerTick(env, email, chatSessions)),
+		Effect.provide(layerTick(env, email, chatSessions, statusPages)),
 	)

apps/alerting/src/worker.ts

@@ -13,6 +13,8 @@ import {
 	MapleStack,
 	type MapleStage,
 	mapleWorkerProps,
+	StatusSnapshotsBucket,
+	StatusSnapshotsKv,
 } from "@maple/infra/cloudflare"
 import {
 	apnsEnv,
@@ -36,6 +38,7 @@ import { bindChatSessions } from "@maple/backend/platform/chat-sessions"
 import { bindEmailSender } from "@maple/backend/platform/email-sender"
 import { chatConnectorOutboundConfigKeys } from "@maple/chat-platform"
 import * as Cloudflare from "@alchemy.run/cloudflare"
+import { RuntimeContext } from "alchemy/RuntimeContext"
 import { Cause, Config, Effect, Layer, Option, Ref } from "effect"
 import { FetchHttpClient, HttpClient, HttpServerResponse } from "effect/http"
 import { ALERTING_CRONS, scheduleFor } from "./schedule.ts"
@@ -49,6 +52,11 @@ import { CHAT_ANTICIPATED_ERROR_IDENTIFIERS } from "@maple/chat-platform/anticip
  */
 export type AlertingWorkerEnv = Record<string, unknown>
 
+/** Discharge alchemy's phantom color, the way alchemy's own runtime helpers do. */
+const runtime = <A, E>(effect: Effect.Effect<A, E, RuntimeContext>): Effect.Effect<A, E> =>
+	// oxlint-disable-next-line effecttsgo/strict-effect-provide
+	Effect.provide(effect, RuntimeContext.phantom)
+
 /** Config-sourced env; largely shared with api via `@maple/infra/env`. */
 const configuredEnv = (stage: MapleStage, region: MapleRegion, domains: MapleDomains) =>
 	merge(
@@ -151,6 +159,9 @@ export default class Alerting extends Cloudflare.Worker<Alerting>()(
 		const email = yield* bindEmailSender
 		// maple-ai's chat Durable Object, where ticks start investigations; handed to each fire's graph.
 		const chatSessions = yield* bindChatSessions
+		// Where published status pages live; ticks publish snapshots and import break-glass posts.
+		const statusKv = yield* Cloudflare.KV.ReadWriteNamespace(StatusSnapshotsKv)
+		const statusBucket = yield* Cloudflare.R2.ReadWriteBucket(StatusSnapshotsBucket)
 		// Once per isolate, not once per fire.
 		const loggedNonProdSkip = yield* Ref.make(false)
 		// Once per isolate too: an isolate never changes data center. A failed lookup stays
@@ -167,7 +178,8 @@ export default class Alerting extends Cloudflare.Worker<Alerting>()(
 		const runTick = (cron: string, lane: string, env: Record<string, unknown>) =>
 			Effect.gen(function* () {
 				const { runScheduled } = yield* scheduled
-				yield* runScheduled(cron, lane, env, email, chatSessions)
+				const [kv, bucket] = yield* runtime(Effect.all([statusKv.raw, statusBucket.raw]))
+				yield* runScheduled(cron, lane, env, email, chatSessions, Option.some({ kv, bucket }))
 			}).pipe(
 				// Interrupts are isolate teardown, not a failed run.
 				Effect.catchCause((cause) =>
@@ -284,6 +296,8 @@ export default class Alerting extends Cloudflare.Worker<Alerting>()(
 			Layer.mergeAll(
 				Cloudflare.Hyperdrive.ConnectBinding,
 				Cloudflare.Email.SendBinding,
+				Cloudflare.KV.ReadWriteNamespaceBinding,
+				Cloudflare.R2.ReadWriteBucketBinding,
 				Cloudflare.Workers.CronEventSourceLive,
 				FetchHttpClient.layer,
 				WorkerTelemetry({

apps/api/src/runtime/service-graph.ts

@@ -56,6 +56,7 @@ import { SupabaseOAuthService } from "@maple/backend/services/auth/SupabaseOAuth
 import { OrgOpenRouterKeyService } from "@maple/backend/services/org/OrgOpenRouterKeyService"
 import { ScrapeTargetsService } from "@maple/backend/services/integrations/ScrapeTargetsService"
 import { UptimeMonitorsService } from "@maple/backend/services/uptime/UptimeMonitorsService"
+import { StatusPagesService } from "@maple/backend/services/status-pages/StatusPagesService"
 import { ChatWorkspaceService } from "@maple/backend/services/integrations/ChatWorkspaceService"
 import { TinybirdOrgTokenService } from "@maple/backend/services/integrations/TinybirdOrgTokenService"
 import { PlanetScaleWebhookQueue } from "@maple/backend/services/integrations/planetscale/PlanetScaleWebhookQueue"
@@ -129,6 +130,7 @@ export const HttpServicesLive = Layer.mergeAll(
 	CancellationReviewService.layer,
 	ScrapeTargetsService.layer,
 	UptimeMonitorsService.layer,
+	StatusPagesService.layer,
 	PlanetScaleConnectionService.layer,
 	PlanetScaleService.layer,
 	PlanetScaleWebhookService.layer,

apps/status/src/worker.ts

@@ -4,15 +4,19 @@
  * No service bindings or Hyperdrive: serving never calls Maple. Traces export to Maple's ingest after
  * the response, best effort, so an outage loses them without touching the page.
  */
-import { MapleStack, mapleWorkerProps } from "@maple/infra/cloudflare"
+import {
+	MapleStack,
+	mapleWorkerProps,
+	StatusSnapshotsBucket,
+	StatusSnapshotsKv,
+} from "@maple/infra/cloudflare"
 import { selfObservabilityEnv } from "@maple/infra/env"
 import { WorkerTelemetry } from "@maple/infra/worker-telemetry"
 import * as Cloudflare from "@alchemy.run/cloudflare"
 import { RuntimeContext } from "alchemy/RuntimeContext"
 import { Clock, Effect, Layer } from "effect"
 import { HttpServerRequest, HttpServerResponse } from "effect/http"
 import { handleRequest } from "./handler"
-import { StatusSnapshotsBucket, StatusSnapshotsKv } from "./resources"
 import { nativeStorage } from "./storage"
 
 /** `__ALCHEMY_RUNTIME__` folds to `true` in the bundle, so the stack-side branch is tree-shaken. */

packages/backend/src/services/alerts/AlertsService.ts

@@ -161,6 +161,10 @@ import { makeAlertDestinationDelivery, parseAlertDestinationEncryptionKey } from
 import { AlertReadModelsService, type AlertReadModelsServiceApi } from "./AlertReadModelsService"
 import { AlertRulesService, makeAlertRulePersistence, type AlertRulesServiceApi } from "./AlertRulesService"
 import { UptimeAlertSource } from "@maple/backend/services/uptime/UptimeAlertSource"
+import {
+	openForAlertIncident,
+	resolveForAlertIncident,
+} from "@maple/backend/services/status-pages/auto-incidents"
 import {
 	decodeStoredAlertRuleMetadata,
 	storedRuleCondition,
@@ -597,6 +601,9 @@ export interface AlertsServiceApi
 	>
 }
 
+/** Longest an uptime incident's status-page write may delay the alert tick. */
+const STATUS_PAGE_SYNC_TIMEOUT = "3 seconds"
+
 export class AlertsService extends Context.Service<AlertsService, AlertsServiceApi>()(
 	"@maple/backend/services/AlertsService",
 	{
@@ -653,6 +660,33 @@ export class AlertsService extends Context.Service<AlertsService, AlertsServiceA
 
 			const dbExecute = makeDbExecute(database, "AlertsService", makePersistenceError)
 
+			// Status pages showing an uptime rule's monitor follow its incidents. Runs after the
+			// alert write commits, bounded so a slow page write cannot hold the notification behind
+			// it; a failure or timeout is logged (the repair sweep catches up) and never blocks alerting.
+			const syncStatusPages = <E>(
+				signalType: string,
+				context: { readonly orgId: OrgId; readonly alertIncidentId: string },
+				write: (db: MapleDb) => Effect.Effect<unknown, E>,
+			) =>
+				signalType !== "uptime"
+					? Effect.void
+					: database
+							.execute((db) => db.transaction(write(db)))
+							.pipe(
+								Effect.timeout(STATUS_PAGE_SYNC_TIMEOUT),
+								Effect.asVoid,
+								// A defect too: a status page sync must never abort the alert tick.
+								Effect.catchCause((cause) =>
+									Effect.logWarning("Status page auto incident sync failed").pipe(
+										Effect.annotateLogs({
+											orgId: context.orgId,
+											"maple.alert.incident_id": context.alertIncidentId,
+											"error.message": summarizeCause(cause),
+										}),
+									),
+								),
+							)
+
 			/**
 			 * Run `write` only while `claim` still owns the rule. The claim row is
 			 * locked FOR UPDATE and re-read in the same transaction as the write, so a
@@ -3126,6 +3160,17 @@ export class AlertsService extends Context.Service<AlertsService, AlertsServiceA
 								severity: lifecycle.severity,
 							}
 						}
+						yield* syncStatusPages(
+							normalized.signalType,
+							{ orgId: row.orgId, alertIncidentId: incidentId },
+							(db) =>
+								openForAlertIncident(db, {
+									orgId: row.orgId,
+									ruleId: row.id,
+									alertIncidentId: incidentId,
+									at: timestamp,
+								}),
+						)
 						if (lifecycle.notificationSuppression === "flapping") {
 							yield* Effect.logInfo("Skipping trigger notification for flapping incident").pipe(
 								Effect.annotateLogs({
@@ -3291,6 +3336,16 @@ export class AlertsService extends Context.Service<AlertsService, AlertsServiceA
 								severity: lifecycle.severity,
 							}
 						}
+						yield* syncStatusPages(
+							normalized.signalType,
+							{ orgId: row.orgId, alertIncidentId: openIncident.id },
+							(db) =>
+								resolveForAlertIncident(db, {
+									orgId: row.orgId,
+									alertIncidentId: openIncident.id,
+									at: timestamp,
+								}),
+						)
 						if (openIncident.holdReason !== null) {
 							yield* Metric.update(AlertingMetrics.incidentsResolvedAfterHoldTotal, 1)
 						}
@@ -3577,6 +3632,16 @@ export class AlertsService extends Context.Service<AlertsService, AlertsServiceA
 							),
 						)
 						if (resolved.length === 0) return false
+						yield* syncStatusPages(
+							normalized.signalType,
+							{ orgId, alertIncidentId: incident.id },
+							(db) =>
+								resolveForAlertIncident(db, {
+									orgId,
+									alertIncidentId: incident.id,
+									at: timestamp,
+								}),
+						)
 						if (incident.holdReason !== null) {
 							yield* Metric.update(AlertingMetrics.incidentsResolvedAfterHoldTotal, 1)
 						}
@@ -3750,6 +3815,16 @@ export class AlertsService extends Context.Service<AlertsService, AlertsServiceA
 								),
 							)
 							if (resolved.length === 0) return false
+							yield* syncStatusPages(
+								normalized.signalType,
+								{ orgId, alertIncidentId: incident.id },
+								(db) =>
+									resolveForAlertIncident(db, {
+										orgId,
+										alertIncidentId: incident.id,
+										at: timestamp,
+									}),
+							)
 							if (incident.holdReason !== null) {
 								yield* Metric.update(AlertingMetrics.incidentsResolvedAfterHoldTotal, 1)
 							}