fix(webapp): retry the distinct-database probe before failing closed

The probe fails closed, so one store being briefly unreachable collapsed the
deployment to single-DB and the boot interlock then refused the boot. With more
than two stores configured that turns a transient blip on any one of them into
a fleet-wide startup failure.

Each target now gets a bounded number of attempts with a short backoff before
the probe gives up. Failing closed is unchanged once the budget is exhausted:
"distinct" stays a positive claim a failed probe cannot support. A genuine
duplicate is a final answer and is never retried, so a misconfigured deployment
still fails on the first pass.

Retries are per target, so one slow store does not re-probe the stores that
already answered.

Author: d-cs

apps/webapp/app/v3/runOpsMigration/distinctDbSentinel.server.ts

@@ -64,6 +64,48 @@ export async function probeControlPlaneCoresidency(
 
 export type DistinctTarget = { id: string; url: string };
 
+/** Injection seam for the retry tests: no containers, no real waiting. */
+export type DistinctProbeOptions = {
+  logger?: { warn: (msg: string, meta?: Record<string, unknown>) => void };
+  readFingerprint?: (url: string) => Promise<DatabaseFingerprint>;
+  /** Total attempts per target, including the first. Bounded so boot latency stays bounded. */
+  attempts?: number;
+  sleep?: (ms: number) => Promise<void>;
+};
+
+const DEFAULT_PROBE_ATTEMPTS = 3;
+const RETRY_BASE_DELAY_MS = 250;
+
+const defaultSleep = (ms: number) => new Promise<void>((resolve) => setTimeout(resolve, ms));
+
+/**
+ * Read one fingerprint, retrying a bounded number of times.
+ *
+ * The probe fails CLOSED, and that must not change: "distinct" is a positive claim a failed probe
+ * cannot support. But failing closed on the first blip means one shard being briefly unreachable
+ * collapses the deployment to single-DB, and the boot interlock then refuses the boot for the whole
+ * fleet. A transient error deserves a retry; a persistent one still fails closed, just later.
+ */
+async function readFingerprintWithRetry(
+  url: string,
+  read: (url: string) => Promise<DatabaseFingerprint>,
+  attempts: number,
+  sleep: (ms: number) => Promise<void>
+): Promise<DatabaseFingerprint> {
+  let lastError: unknown;
+  for (let attempt = 1; attempt <= attempts; attempt++) {
+    try {
+      return await read(url);
+    } catch (error) {
+      lastError = error;
+      if (attempt < attempts) {
+        await sleep(RETRY_BASE_DELAY_MS * attempt);
+      }
+    }
+  }
+  throw lastError;
+}
+
 /**
  * Set uniqueness over every store that owns its own database. Fail-closed: a probe that cannot
  * answer returns NOT distinct, because "distinct" is a positive claim a failed probe cannot support.
@@ -77,14 +119,22 @@ export type DistinctTarget = { id: string; url: string };
  */
 export async function probeDistinctStores(
   targets: DistinctTarget[],
-  opts?: { logger?: { warn: (msg: string, meta?: Record<string, unknown>) => void } }
+  opts?: DistinctProbeOptions
 ): Promise<{ distinct: true } | { distinct: false; reason: string }> {
   if (targets.length < 2) {
     return { distinct: true };
   }
 
+  const read = opts?.readFingerprint ?? readDatabaseFingerprint;
+  const attempts = opts?.attempts ?? DEFAULT_PROBE_ATTEMPTS;
+  const sleep = opts?.sleep ?? defaultSleep;
+
   try {
-    const fingerprints = await Promise.all(targets.map((t) => readDatabaseFingerprint(t.url)));
+    // Retry per TARGET, not around the whole set: one slow shard must not re-probe the stores that
+    // already answered. A duplicate verdict below is final and is never retried.
+    const fingerprints = await Promise.all(
+      targets.map((t) => readFingerprintWithRetry(t.url, read, attempts, sleep))
+    );
 
     const seen = new Map<string, string>();
     for (const [index, target] of targets.entries()) {
@@ -104,7 +154,9 @@ export async function probeDistinctStores(
 
     return { distinct: true };
   } catch (error) {
-    const reason = `distinct-db sentinel probe failed; failing closed (single-DB). ${String(error)}`;
+    const reason =
+      `distinct-db sentinel probe failed after ${opts?.attempts ?? DEFAULT_PROBE_ATTEMPTS} ` +
+      `attempt(s); failing closed (single-DB). ${String(error)}`;
     opts?.logger?.warn(reason, { error });
     return { distinct: false, reason };
   }
@@ -115,7 +167,7 @@ export async function probeDistinctStores(
 export async function probeDistinctDatabases(
   legacyUrl: string,
   newUrl: string,
-  opts?: { logger?: { warn: (msg: string, meta?: Record<string, unknown>) => void } }
+  opts?: DistinctProbeOptions
 ): Promise<{ distinct: true } | { distinct: false; reason: string }> {
   return probeDistinctStores(
     [

apps/webapp/test/v3/runOpsMigration/distinctDbSentinel.server.test.ts

@@ -1,6 +1,6 @@
 import { heteroPostgresTest } from "@internal/testcontainers";
 import { PrismaClient } from "@trigger.dev/database";
-import { describe, expect, vi } from "vitest";
+import { describe, expect, it, vi } from "vitest";
 import {
   probeDistinctDatabases,
   probeDistinctStores,
@@ -66,6 +66,78 @@ describe("probeDistinctDatabases", () => {
   );
 });
 
+// A transient failure on ONE store must not refuse the boot fleet-wide. The probe fails closed,
+// so an unretried blip on any shard collapses the whole deployment to single-DB and the boot
+// interlock then throws. Retry a bounded number of times, then fail closed exactly as before.
+describe("probeDistinctStores bounded retry", () => {
+  const fp = (sysId: string, db: string) => ({ systemIdentifier: sysId, databaseName: db });
+
+  it("recovers when a transient failure clears within the retry budget", async () => {
+    let calls = 0;
+    const readFingerprint = vi.fn(async (url: string) => {
+      calls++;
+      if (calls === 2) throw new Error("ECONNREFUSED");
+      return fp("sys", url);
+    });
+    const result = await probeDistinctStores(
+      [
+        { id: "new", url: "a" },
+        { id: "shard-a", url: "b" },
+      ],
+      { readFingerprint, attempts: 3, sleep: async () => {} }
+    );
+    expect(result).toEqual({ distinct: true });
+  });
+
+  it("fails closed once the retry budget is exhausted", async () => {
+    const readFingerprint = vi.fn(async () => {
+      throw new Error("ECONNREFUSED");
+    });
+    const result = await probeDistinctStores(
+      [
+        { id: "new", url: "a" },
+        { id: "shard-a", url: "b" },
+      ],
+      { readFingerprint, attempts: 3, sleep: async () => {} }
+    );
+    expect(result).toMatchObject({ distinct: false });
+  });
+
+  it("bounds the attempts it makes", async () => {
+    const readFingerprint = vi.fn(async () => {
+      throw new Error("ECONNREFUSED");
+    });
+    await probeDistinctStores(
+      [
+        { id: "a", url: "a" },
+        { id: "b", url: "b" },
+      ],
+      {
+        readFingerprint,
+        attempts: 3,
+        sleep: async () => {},
+      }
+    );
+    // 2 targets x 3 attempts each, and no more.
+    expect(readFingerprint).toHaveBeenCalledTimes(6);
+  });
+
+  // A duplicate is a correct, final answer. Retrying it would delay every boot of a genuinely
+  // misconfigured deployment for no benefit.
+  it("does not retry a genuine duplicate", async () => {
+    const readFingerprint = vi.fn(async () => fp("sys", "same"));
+    const result = await probeDistinctStores(
+      [
+        { id: "new", url: "a" },
+        { id: "shard-a", url: "b" },
+      ],
+      { readFingerprint, attempts: 3, sleep: async () => {} }
+    );
+    expect(result).toMatchObject({ distinct: false });
+    expect(readFingerprint).toHaveBeenCalledTimes(2);
+  });
+});
+
 describe("probeDistinctStores (set uniqueness at N)", () => {
   heteroPostgresTest(
     "reports distinct for two separate physical clusters",