fix(daemon): re-issue the interrupted request against a replacement daemon

Makefile.cbm

@@ -468,6 +468,7 @@ DAEMON_SRCS = \
     src/daemon/runtime.c \
     src/daemon/application.c \
     src/daemon/frontend.c \
+    src/daemon/frontend_liveness.c \
     src/daemon/host.c \
     src/daemon/bootstrap.c \
     src/daemon/ipc.c

src/daemon/bootstrap.c

@@ -9,6 +9,7 @@
 #include "foundation/compat_fs.h"
 #include "foundation/log.h"
 #include "foundation/platform.h"
+#include "foundation/subprocess.h"
 
 #include <errno.h>
 #include <limits.h>
@@ -22,7 +23,6 @@
 #ifndef WIN32_LEAN_AND_MEAN
 #define WIN32_LEAN_AND_MEAN
 #endif
-#include "foundation/subprocess.h"
 #include "foundation/win_utf8.h"
 #include <windows.h>
 #else
@@ -1301,17 +1301,8 @@ static bool bootstrap_production_spawn(void *context,
     return true;
 }
 #else
-static void bootstrap_child_close_fds(void) {
-    long open_max = sysconf(_SC_OPEN_MAX);
-    if (open_max < 0 || open_max > 1048576L) {
-        open_max = 65536L;
-    }
-    for (int fd = 3; fd < open_max; fd++) {
-        (void)close(fd);
-    }
-}
-
-static void bootstrap_daemon_grandchild(const cbm_daemon_bootstrap_launch_spec_t *spec) {
+static void bootstrap_daemon_grandchild(const cbm_daemon_bootstrap_launch_spec_t *spec,
+                                        long max_fd) {
     (void)umask(077);
     sigset_t empty;
     (void)sigemptyset(&empty);
@@ -1324,7 +1315,7 @@ static void bootstrap_daemon_grandchild(const cbm_daemon_bootstrap_launch_spec_t
     if (null_fd > STDERR_FILENO) {
         (void)close(null_fd);
     }
-    bootstrap_child_close_fds();
+    (void)cbm_posix_close_fds_from(STDERR_FILENO + 1, max_fd);
     execv(spec->executable_path, (char *const *)spec->argv);
     _exit(127);
 }
@@ -1335,6 +1326,10 @@ static bool bootstrap_production_spawn(void *context,
     if (!spec || !spec->detached || spec->inherit_standard_handles || spec->use_shell) {
         return false;
     }
+    /* Read the cap before fork: sysconf() may take libc locks, and the fork
+     * child may not (the frontend reconnects from a worker thread). The
+     * derivation is shared with the subprocess spawn path. */
+    long max_fd = cbm_posix_fd_close_cap();
     pid_t first = fork();
     if (first < 0) {
         return false;
@@ -1350,7 +1345,7 @@ static bool bootstrap_production_spawn(void *context,
         if (daemon > 0) {
             _exit(0);
         }
-        bootstrap_daemon_grandchild(spec);
+        bootstrap_daemon_grandchild(spec, max_fd);
     }
 
     int status = 0;

src/daemon/frontend.h

@@ -17,6 +17,25 @@ typedef struct cbm_daemon_maintenance_monitor cbm_daemon_maintenance_monitor_t;
  * outlive the bounded grace period. */
 typedef bool (*cbm_daemon_maintenance_cancel_fn)(void *context);
 
+/* #2796: invoked on the frontend worker thread when a request exchange fails
+ * with a transport error while the MCP client's session is still live.
+ * dead_client remains owned by the frontend, which closes it after the swap
+ * and frees it only once every frontend thread has stopped; the callback must
+ * not use or close it. The callback establishes a replacement (connect-or-spawn
+ * plus session context) and returns it — NULL leaves the transport failure
+ * fatal. The returned client must be fresh, open, and distinct from every
+ * client returned earlier in this run (dead_client included); the frontend
+ * takes ownership: it closes the client if the swap is refused, and
+ * closes/frees it at the end of a successful run. The callback runs at most
+ * three times per run. It may take as long as a cold daemon spawn; the EOF
+ * drain and the join watchdog grant at most a hard 120 s grace per attempt.
+ * A maintenance activation still fail-stops the process on the monitor's own
+ * shorter grace, because a binary update must not wait for a reconnect. It
+ * must not read or write the frontend's stdin/stdout; stderr diagnostics are
+ * allowed, but the callback must not assume that stream is being drained. */
+typedef cbm_daemon_runtime_client_t *(*cbm_daemon_frontend_daemon_lost_fn)(
+    void *context, cbm_daemon_runtime_client_t *dead_client);
+
 /* Parse one JSON-RPC message and recognize only the exact cancellation
  * notification method. String contents, nested fields, method prefixes, and
  * requests carrying an id are not cancellation notifications. */
@@ -66,9 +85,17 @@ uint64_t cbm_daemon_frontend_test_worker_idle_cycles(void);
  * either thread is blocked in stdio, requests cooperative cancellation for the
  * exact active request, and then bounds process exit. Kernel IPC close cancels
  * only this session's daemon work. EOF/parse failure closes the authenticated
- * session. An unexpected daemon transport failure likewise terminates the
- * process so an agent waiting with stdin still open observes server EOF. */
+ * session. An unexpected daemon transport failure is retried against a
+ * replacement daemon through daemon_lost (bounded, #2796); daemon answers such
+ * as busy, rejected, handler error or cancellation are not retried. A replayed
+ * request may already have executed in the dead daemon: a transport failure
+ * cannot distinguish "never sent" from "reply lost", so the interrupted call
+ * has at-least-once semantics. Without the callback — or once the retry budget
+ * is spent — the worker terminates the process immediately so an agent waiting
+ * with stdin still open observes server EOF. */
 int cbm_daemon_frontend_mcp_run(cbm_daemon_runtime_client_t *client,
-                                cbm_version_cohort_manager_t *cohort_manager, FILE *in, FILE *out);
+                                cbm_version_cohort_manager_t *cohort_manager, FILE *in, FILE *out,
+                                cbm_daemon_frontend_daemon_lost_fn daemon_lost,
+                                void *daemon_lost_context);
 
 #endif /* CBM_DAEMON_FRONTEND_H */

src/daemon/frontend_liveness.c

@@ -0,0 +1,74 @@
+/*
+ * frontend_liveness.c — deterministic deadline policy for the frontend
+ * waiters. No clocks, no atomics, no frontend state: callers pass a sampled
+ * `now`, the reconnect counter and attempt start, and whether a replacement is
+ * running.
+ */
+#include "daemon/frontend_liveness.h"
+
+static uint64_t frontend_liveness_deadline(uint64_t start, uint64_t window_ms) {
+    return start > UINT64_MAX - window_ms ? UINT64_MAX : start + window_ms;
+}
+
+static uint64_t frontend_liveness_anchor(uint64_t now, uint64_t attempt_started_ms) {
+    /* A start after `now` would mean the sampler saw the attempt before its
+     * timestamp; prefer the local observation so the deadline cannot move
+     * backwards. */
+    return attempt_started_ms != 0 && attempt_started_ms <= now ? attempt_started_ms : now;
+}
+
+void cbm_daemon_frontend_liveness_init(cbm_daemon_frontend_liveness_t *liveness, uint64_t now,
+                                       unsigned int attempts) {
+    liveness->last_progress_ms = now;
+    liveness->reconnect_since_ms = 0;
+    liveness->seen_attempts = attempts;
+    liveness->was_reconnecting = false;
+}
+
+bool cbm_daemon_frontend_liveness_wait(cbm_daemon_frontend_liveness_t *liveness, uint64_t now,
+                                       unsigned int attempts, bool reconnecting,
+                                       uint64_t attempt_started_ms, uint64_t base_window_ms) {
+    if (attempts != liveness->seen_attempts) {
+        liveness->seen_attempts = attempts;
+        liveness->last_progress_ms = now;
+        liveness->reconnect_since_ms = 0;
+        liveness->was_reconnecting = false;
+    }
+    if (reconnecting) {
+        if (!liveness->was_reconnecting) {
+            liveness->reconnect_since_ms = frontend_liveness_anchor(now, attempt_started_ms);
+        }
+    } else if (liveness->was_reconnecting) {
+        /* The attempt ended; the retried exchange gets its own stuck window
+         * instead of inheriting the spawn's remainder. */
+        liveness->last_progress_ms = now;
+        liveness->reconnect_since_ms = 0;
+    }
+    liveness->was_reconnecting = reconnecting;
+
+    uint64_t deadline = frontend_liveness_deadline(liveness->last_progress_ms, base_window_ms);
+    if (reconnecting) {
+        uint64_t reconnect_deadline = frontend_liveness_deadline(
+            liveness->reconnect_since_ms, CBM_DAEMON_FRONTEND_RECONNECT_GRACE_MS);
+        if (reconnect_deadline > deadline) {
+            deadline = reconnect_deadline;
+        }
+    }
+    return now < deadline;
+}
+
+void cbm_daemon_frontend_liveness_progress(cbm_daemon_frontend_liveness_t *liveness, uint64_t now) {
+    liveness->last_progress_ms = now;
+}
+
+bool cbm_daemon_frontend_liveness_reconnect_late(cbm_daemon_frontend_liveness_t *liveness,
+                                                 uint64_t now, uint64_t attempt_started_ms) {
+    if (liveness->was_reconnecting) {
+        /* This attempt already has its grace; re-anchoring here would let a
+         * repeated re-sample extend a hung attempt without bound. */
+        return false;
+    }
+    liveness->reconnect_since_ms = frontend_liveness_anchor(now, attempt_started_ms);
+    liveness->was_reconnecting = true;
+    return true;
+}

src/daemon/frontend_liveness.h

@@ -0,0 +1,61 @@
+/*
+ * frontend_liveness.h — no-progress policy shared by the frontend's waiters.
+ *
+ * Two threads wait for the worker with different ordinary windows: the stdin
+ * reader's EOF drain (15 s) and the join watchdog (5 s). Both must treat a
+ * replacement-daemon attempt as forward progress, because a cold daemon start
+ * can outlast either window. Keeping that arithmetic in each loop is how the
+ * two policies drifted once (#2796): the drain granted the spawn its grace
+ * while the watchdog killed the same worker five seconds in. Each waiter
+ * supplies its own ordinary window; the reconnect grace is shared here and
+ * anchored to the attempt's real start, so two waiters observing the same
+ * attempt spend one 120 s budget rather than two.
+ */
+#ifndef CBM_DAEMON_FRONTEND_LIVENESS_H
+#define CBM_DAEMON_FRONTEND_LIVENESS_H
+
+#include <stdbool.h>
+#include <stdint.h>
+
+/* One replacement spawn may take as long as a cold daemon start (connect,
+ * bootstrap, and possibly an upgrade drain). This bounds a single attempt, not
+ * the run: the reconnect budget bounds the run. */
+#define CBM_DAEMON_FRONTEND_RECONNECT_GRACE_MS 120000U
+
+typedef struct {
+    uint64_t last_progress_ms;
+    uint64_t reconnect_since_ms;
+    unsigned int seen_attempts;
+    bool was_reconnecting;
+} cbm_daemon_frontend_liveness_t;
+
+/* Start a window at `now`; `attempts` is the reconnect counter's current
+ * value (a nonzero value when a waiter starts mid-attempt). */
+void cbm_daemon_frontend_liveness_init(cbm_daemon_frontend_liveness_t *liveness, uint64_t now,
+                                       unsigned int attempts);
+
+/* Observe one poll and report whether the caller may keep waiting.
+ * `base_window_ms` is the ordinary no-progress window. A reconnect in progress
+ * extends the deadline to that attempt's grace. `attempt_started_ms` is the
+ * attempt's real start (0 when the caller has no timestamp): anchoring to it
+ * lets the drain and the watchdog share one budget for the same attempt.
+ * A changed attempt counter (an attempt that started and ended between polls)
+ * restarts the window, so a fast failure is not charged to the previous
+ * attempt. */
+bool cbm_daemon_frontend_liveness_wait(cbm_daemon_frontend_liveness_t *liveness, uint64_t now,
+                                       unsigned int attempts, bool reconnecting,
+                                       uint64_t attempt_started_ms, uint64_t base_window_ms);
+
+/* A unit of work completed: restart the ordinary window. The per-attempt
+ * reconnect grace is not extended. */
+void cbm_daemon_frontend_liveness_progress(cbm_daemon_frontend_liveness_t *liveness, uint64_t now);
+
+/* The deadline expired, but a re-sample showed a reconnect that started after
+ * the poll: grant that attempt its grace instead of stopping underneath it.
+ * Returns true when a grace was granted; repeated calls for the same attempt
+ * are no-ops and return false, so the caller cannot extend a hung attempt by
+ * re-sampling. */
+bool cbm_daemon_frontend_liveness_reconnect_late(cbm_daemon_frontend_liveness_t *liveness,
+                                                 uint64_t now, uint64_t attempt_started_ms);
+
+#endif /* CBM_DAEMON_FRONTEND_LIVENESS_H */

src/foundation/subprocess.c

@@ -27,6 +27,7 @@
 #else
 #include <errno.h>
 #include <fcntl.h>
+#include <limits.h>
 #include <signal.h>
 #ifdef __APPLE__
 #include <paths.h>
@@ -1434,7 +1435,15 @@ static void cbm_posix_reset_child_signals(void) {
 static bool g_force_close_range_enosys = false;
 #endif
 
-static cbm_fd_close_strategy_t cbm_posix_close_fds_from(int lowfd, long max_fd) {
+long cbm_posix_fd_close_cap(void) {
+    long max_fd = sysconf(_SC_OPEN_MAX);
+    if (max_fd < 0 || max_fd > 1048576L) {
+        max_fd = 65536L;
+    }
+    return max_fd;
+}
+
+cbm_fd_close_strategy_t cbm_posix_close_fds_from(int lowfd, long max_fd) {
 #if defined(__linux__) && defined(SYS_close_range)
     bool range_unavailable = false;
 #ifdef CBM_ENABLE_TEST_SEAMS
@@ -1448,6 +1457,13 @@ static cbm_fd_close_strategy_t cbm_posix_close_fds_from(int lowfd, long max_fd)
     closefrom(lowfd);
     return CBM_FD_CLOSEFROM;
 #endif
+    /* close() takes an int, so descriptors above INT_MAX cannot exist. Clamp
+     * inside the implementation: the exported function must not depend on a
+     * caller respecting the cap contract, or an int loop variable would wrap
+     * and never terminate in the fork child. */
+    if (max_fd > INT_MAX) {
+        max_fd = INT_MAX;
+    }
     for (int fd = lowfd; fd < max_fd; fd++) {
         (void)close(fd);
     }
@@ -1701,10 +1717,7 @@ static int cbm_subprocess_spawn_posix(cbm_subprocess_t *process) {
     }
     int child_output = stdout_capture >= 0 ? stdout_capture : output;
 
-    long max_fd = sysconf(_SC_OPEN_MAX);
-    if (max_fd < 0 || max_fd > 1048576L) {
-        max_fd = 65536L;
-    }
+    long max_fd = cbm_posix_fd_close_cap();
 
     pid_t pid = -1;
 #ifdef __APPLE__

src/foundation/subprocess.h

@@ -232,6 +232,20 @@ typedef enum {
     CBM_FD_CLOSE_LOOP       /* close() per descriptor up to _SC_OPEN_MAX: O(RLIMIT_NOFILE) */
 } cbm_fd_close_strategy_t;
 
+#ifndef _WIN32
+/* Fallback loop bound for the descriptor sweep below: sysconf(_SC_OPEN_MAX),
+ * or 65536 when that is unavailable or above 1048576. Callers invoke this
+ * BEFORE fork(); sysconf may take libc locks. */
+long cbm_posix_fd_close_cap(void);
+
+/* Close every descriptor >= lowfd in a fork+exec child (#1484, #2796).
+ * async-signal-safe: a raw close_range syscall, closefrom(), or close() only —
+ * no allocation and no stdio, because this runs between fork and exec in a
+ * child of a possibly multi-threaded process. `max_fd` is the cap returned by
+ * cbm_posix_fd_close_cap() in the parent. */
+cbm_fd_close_strategy_t cbm_posix_close_fds_from(int lowfd, long max_fd);
+#endif
+
 #if defined(CBM_ENABLE_TEST_SEAMS) && !defined(_WIN32)
 /* Run the child's close-inherited-descriptors step in the CALLING process for
  * descriptors >= lowfd (callers pick a lowfd above everything they need) and