fix(webapp): close a latch bypass, and bound the new configuration

The latch check took its argument optionally so as not to disturb
existing callers. One admin route omitted it and therefore skipped the
check entirely, which let a per-organisation override be enabled with
the latch unset: those runs would be resident with their transitions
skipped, and their heads would freeze. An optional safety argument
disables the safety at every caller that forgets it, so it is now
required and the compiler enumerates the call sites. There was exactly
one.

The latch is deployment-wide, like the dial and the hard stop, so an
organisation save now strips it rather than reporting success for a
setting nothing reads from an organisation row.

Bounds on the new numeric configuration. A zero cache lifetime or size
is a constructor error, a zero sweep budget truncates every pass so
rule 2 can never converge, and a zero command timeout fails every
command. Jitter may legitimately be zero.

The cluster service pins its image by digest, as the repository
requires.

Author: d-cs

apps/webapp/app/env.server.ts

@@ -1325,11 +1325,15 @@ const EnvironmentSchema = z
       .int()
       .default(2 * 60 * 60 * 1000),
     RUN_ENGINE_SNAPSHOT_STORE_GC_SWEEP_SCHEDULE: z.string().default("0 */6 * * *"),
-    RUN_ENGINE_SNAPSHOT_STORE_GC_SWEEP_JITTER_IN_MS: z.coerce.number().int().default(60_000),
+    RUN_ENGINE_SNAPSHOT_STORE_GC_SWEEP_JITTER_IN_MS: z.coerce.number().int().min(0).default(60_000),
     // An existing run costs ~4 serial round trips and the orphan-marker clear cannot be batched
     // (cross-slot pipelines are rejected), so a full pass is hours, not minutes. A budget that
     // truncates every pass stops rule 2 converging, because it needs consecutive sightings.
-    RUN_ENGINE_SNAPSHOT_STORE_GC_SWEEP_BUDGET_MS: z.coerce.number().int().default(10_800_000),
+    RUN_ENGINE_SNAPSHOT_STORE_GC_SWEEP_BUDGET_MS: z.coerce
+      .number()
+      .int()
+      .positive()
+      .default(10_800_000),
     /**
      * RETIRED. The hard stop is the snapshotStoreHalt feature flag and nothing else: an environment
      * variable converged over a rolling deploy rather than a flag interval, and during that window a
@@ -1341,8 +1345,16 @@ const EnvironmentSchema = z
      * ignored.
      */
     RUN_ENGINE_SNAPSHOT_STORE_HALT: z.string().optional(),
-    RUN_ENGINE_SNAPSHOT_STORE_ORG_MODE_CACHE_TTL_MS: z.coerce.number().int().default(30_000),
-    RUN_ENGINE_SNAPSHOT_STORE_ORG_MODE_CACHE_MAX: z.coerce.number().int().default(10_000),
+    RUN_ENGINE_SNAPSHOT_STORE_ORG_MODE_CACHE_TTL_MS: z.coerce
+      .number()
+      .int()
+      .positive()
+      .default(30_000),
+    RUN_ENGINE_SNAPSHOT_STORE_ORG_MODE_CACHE_MAX: z.coerce
+      .number()
+      .int()
+      .positive()
+      .default(10_000),
     // No fallback to REDIS_*: this is a distinct durable endpoint and must be set explicitly, or
     // execution state silently lands on the general-purpose cache.
     RUN_ENGINE_SNAPSHOT_STORE_REDIS_HOST: z.string().optional(),
@@ -1354,7 +1366,11 @@ const EnvironmentSchema = z
     // Fails an append fast rather than letting it wait on an unreachable endpoint. Postgres is
     // authoritative below the final dial position, so a refused append costs a mirrored write; a
     // blocked one costs the request.
-    RUN_ENGINE_SNAPSHOT_STORE_REDIS_COMMAND_TIMEOUT_MS: z.coerce.number().int().default(500),
+    RUN_ENGINE_SNAPSHOT_STORE_REDIS_COMMAND_TIMEOUT_MS: z.coerce
+      .number()
+      .int()
+      .positive()
+      .default(500),
 
     RUN_ENGINE_DEV_PRESENCE_REDIS_HOST: z
       .string()

apps/webapp/app/routes/admin.api.v2.orgs.$organizationId.feature-flags.ts

@@ -6,6 +6,7 @@ import { env } from "~/env.server";
 import { prisma } from "~/db.server";
 import { requireUser } from "~/services/session.server";
 import { controlPlaneResolver } from "~/v3/runOpsMigration/controlPlaneResolver.server";
+import { globalFlagsRegistry } from "~/v3/globalFlagsRegistry.server";
 import { snapshotStoreFlagSaveError } from "~/v3/snapshotStoreFlagGuard.server";
 import { invalidateSnapshotStoreOrgMode } from "~/v3/snapshotStoreMode.server";
 import { selectMintBaselineSource, stampMintKindFlip } from "~/v3/runOpsMigration/mintFlipGrace";
@@ -149,6 +150,9 @@ export async function action({ request, params }: ActionFunctionArgs) {
 
   const snapshotStoreError = snapshotStoreFlagSaveError(requestedFlags, {
     redisHostConfigured: !!env.RUN_ENGINE_SNAPSHOT_STORE_REDIS_HOST,
+    // Read from the live registry, not the payload: the latch must ALREADY be true before anything
+    // can be enabled, or a run born in the gap is resident with its transitions skipped.
+    everEnabled: globalFlagsRegistry.current()?.[FEATURE_FLAG.snapshotStoreEverEnabled] === true,
   });
   if (snapshotStoreError) {
     return json({ error: snapshotStoreError }, { status: 400 });

apps/webapp/app/v3/featureFlags.ts

@@ -245,7 +245,13 @@ export const ORG_LOCKED_FLAGS: FeatureFlagKey[] = [
  * consults it, so the line is held here — the same way the mint grace stamps are stripped.
  */
 export function withoutOrgForbiddenSnapshotKeys<T extends Record<string, unknown>>(values: T): T {
-  const forbidden = [FEATURE_FLAG.snapshotStoreMode, FEATURE_FLAG.snapshotStoreHalt] as const;
+  const forbidden = [
+    FEATURE_FLAG.snapshotStoreMode,
+    FEATURE_FLAG.snapshotStoreHalt,
+    // Deployment-wide, like the other two. Nothing reads it from an organisation row, so accepting
+    // it on an organisation save reports success for a setting that does nothing.
+    FEATURE_FLAG.snapshotStoreEverEnabled,
+  ] as const;
   if (!forbidden.some((key) => key in values)) return values;
 
   const rest = { ...values };

apps/webapp/app/v3/snapshotStoreFlagGuard.server.ts

@@ -7,7 +7,10 @@ import { FEATURE_FLAG } from "~/v3/featureFlags";
  */
 export function snapshotStoreFlagSaveError(
   requested: Record<string, unknown>,
-  opts: { redisHostConfigured: boolean; everEnabled?: boolean }
+  // Both REQUIRED. `everEnabled` was optional so as not to disturb existing callers, and a route
+  // that omitted it silently skipped the latch check: an optional safety argument disables the
+  // safety at every caller that forgets it. Required means the compiler enumerates them.
+  opts: { redisHostConfigured: boolean; everEnabled: boolean }
 ): string | undefined {
   // Both keys, because either one past `off` is equally silent without a connection, and either one
   // equally makes a run resident once there is one.
@@ -26,7 +29,7 @@ export function snapshotStoreFlagSaveError(
   // entirely while it is unset, so a run born after the dial moved but before the latch landed would
   // be resident with its transitions skipped, and its head would freeze while Postgres moved on.
   // Refusing here makes that ordering impossible to get wrong rather than merely documented.
-  if (opts.everEnabled === false) {
+  if (!opts.everEnabled) {
     for (const { key, value } of enabling) {
       return `Cannot set ${key} to "${String(value)}" before ${FEATURE_FLAG.snapshotStoreEverEnabled} is true. Set that flag first: until it is, transitions skip the store entirely, so a run born now would be resident with its transitions skipped and its head would freeze.`;
     }

apps/webapp/test/snapshotStoreFlagGuard.test.ts

@@ -15,7 +15,7 @@ describe("snapshotStoreFlagSaveError", () => {
     expect(
       snapshotStoreFlagSaveError(
         { snapshotStoreMode: "dual-write" },
-        { redisHostConfigured: false }
+        { redisHostConfigured: false, everEnabled: true }
       )
     ).toMatch(/RUN_ENGINE_SNAPSHOT_STORE_REDIS_HOST/);
   });
@@ -24,32 +24,44 @@ describe("snapshotStoreFlagSaveError", () => {
     expect(
       snapshotStoreFlagSaveError(
         { snapshotStoreMode: "redis-read" },
-        { redisHostConfigured: false }
+        { redisHostConfigured: false, everEnabled: true }
       )
     ).toMatch(/redis-read/);
   });
 
   it("allows a flip past off once the host is configured", () => {
     expect(
-      snapshotStoreFlagSaveError({ snapshotStoreMode: "dual-write" }, { redisHostConfigured: true })
+      snapshotStoreFlagSaveError(
+        { snapshotStoreMode: "dual-write" },
+        { redisHostConfigured: true, everEnabled: true }
+      )
     ).toBeUndefined();
   });
 
   it("allows off with no host, because that is the default state", () => {
     expect(
-      snapshotStoreFlagSaveError({ snapshotStoreMode: "off" }, { redisHostConfigured: false })
+      snapshotStoreFlagSaveError(
+        { snapshotStoreMode: "off" },
+        { redisHostConfigured: false, everEnabled: true }
+      )
     ).toBeUndefined();
   });
 
   it("ignores a payload that does not mention the dial", () => {
     expect(
-      snapshotStoreFlagSaveError({ runOpsMintKind: "cuid" }, { redisHostConfigured: false })
+      snapshotStoreFlagSaveError(
+        { runOpsMintKind: "cuid" },
+        { redisHostConfigured: false, everEnabled: true }
+      )
     ).toBeUndefined();
   });
 
   it("ignores a non-string dial value and leaves it to schema validation", () => {
     expect(
-      snapshotStoreFlagSaveError({ snapshotStoreMode: 3 }, { redisHostConfigured: false })
+      snapshotStoreFlagSaveError(
+        { snapshotStoreMode: 3 },
+        { redisHostConfigured: false, everEnabled: true }
+      )
     ).toBeUndefined();
   });
 
@@ -58,14 +70,17 @@ describe("snapshotStoreFlagSaveError", () => {
     expect(
       snapshotStoreFlagSaveError(
         { snapshotStoreOrgMode: "dual-write" },
-        { redisHostConfigured: false }
+        { redisHostConfigured: false, everEnabled: true }
       )
     ).toMatch(/RUN_ENGINE_SNAPSHOT_STORE_REDIS_HOST/);
   });
 
   it("allows a per-organisation off with no host", () => {
     expect(
-      snapshotStoreFlagSaveError({ snapshotStoreOrgMode: "off" }, { redisHostConfigured: false })
+      snapshotStoreFlagSaveError(
+        { snapshotStoreOrgMode: "off" },
+        { redisHostConfigured: false, everEnabled: true }
+      )
     ).toBeUndefined();
   });
 });

docker/docker-compose.extras.yml

@@ -136,7 +136,7 @@ services:
   # ioredis discovers the other five nodes from that one seed.
   redis-cluster:
     container_name: ${CONTAINER_PREFIX:-}redis-cluster
-    image: redis:7.2
+    image: redis:7.2@sha256:74566c6910d13ae61e7ce73ebd3127438a1fe805b309b097c323142719ec8a5b
     restart: always
     volumes:
       - ./config/redis-cluster-init.sh:/init.sh:ro