diff --git a/Makefile.cbm b/Makefile.cbm index 3b93146f1f..f6cf8b8331 100644 --- a/Makefile.cbm +++ b/Makefile.cbm @@ -468,6 +468,7 @@ DAEMON_SRCS = \ src/daemon/runtime.c \ src/daemon/application.c \ src/daemon/frontend.c \ + src/daemon/frontend_liveness.c \ src/daemon/host.c \ src/daemon/bootstrap.c \ src/daemon/ipc.c diff --git a/src/daemon/bootstrap.c b/src/daemon/bootstrap.c index cb384cd20c..e3221841e4 100644 --- a/src/daemon/bootstrap.c +++ b/src/daemon/bootstrap.c @@ -9,6 +9,7 @@ #include "foundation/compat_fs.h" #include "foundation/log.h" #include "foundation/platform.h" +#include "foundation/subprocess.h" #include #include @@ -22,7 +23,6 @@ #ifndef WIN32_LEAN_AND_MEAN #define WIN32_LEAN_AND_MEAN #endif -#include "foundation/subprocess.h" #include "foundation/win_utf8.h" #include #else @@ -1301,17 +1301,8 @@ static bool bootstrap_production_spawn(void *context, return true; } #else -static void bootstrap_child_close_fds(void) { - long open_max = sysconf(_SC_OPEN_MAX); - if (open_max < 0 || open_max > 1048576L) { - open_max = 65536L; - } - for (int fd = 3; fd < open_max; fd++) { - (void)close(fd); - } -} - -static void bootstrap_daemon_grandchild(const cbm_daemon_bootstrap_launch_spec_t *spec) { +static void bootstrap_daemon_grandchild(const cbm_daemon_bootstrap_launch_spec_t *spec, + long max_fd) { (void)umask(077); sigset_t empty; (void)sigemptyset(&empty); @@ -1324,7 +1315,7 @@ static void bootstrap_daemon_grandchild(const cbm_daemon_bootstrap_launch_spec_t if (null_fd > STDERR_FILENO) { (void)close(null_fd); } - bootstrap_child_close_fds(); + (void)cbm_posix_close_fds_from(STDERR_FILENO + 1, max_fd); execv(spec->executable_path, (char *const *)spec->argv); _exit(127); } @@ -1335,6 +1326,10 @@ static bool bootstrap_production_spawn(void *context, if (!spec || !spec->detached || spec->inherit_standard_handles || spec->use_shell) { return false; } + /* Read the cap before fork: sysconf() may take libc locks, and the fork + * child may not (the frontend reconnects from a worker thread). The + * derivation is shared with the subprocess spawn path. */ + long max_fd = cbm_posix_fd_close_cap(); pid_t first = fork(); if (first < 0) { return false; @@ -1350,7 +1345,7 @@ static bool bootstrap_production_spawn(void *context, if (daemon > 0) { _exit(0); } - bootstrap_daemon_grandchild(spec); + bootstrap_daemon_grandchild(spec, max_fd); } int status = 0; diff --git a/src/daemon/frontend.c b/src/daemon/frontend.c index cbe4b282e3..f53d7b3d79 100644 --- a/src/daemon/frontend.c +++ b/src/daemon/frontend.c @@ -4,6 +4,7 @@ #include "daemon/frontend.h" #include "daemon/application.h" +#include "daemon/frontend_liveness.h" #include "foundation/compat.h" #include "foundation/compat_thread.h" #include "foundation/platform.h" @@ -39,6 +40,12 @@ enum { * receiving every response while a genuinely stuck request still gets * cancelled and the frontend exits cleanly. */ FRONTEND_EOF_DRAIN_MS = 15000, + /* #2796: replacement-daemon attempts per frontend run once the transport + * fails mid-session. Bounded so a daemon that cannot come back still + * terminates the process exactly like the pre-reconnect behavior. A + * replacement spawn may legitimately outlast one drain window; the shared + * per-attempt grace lives in frontend_liveness.h. */ + FRONTEND_DAEMON_RECONNECT_MAX = 3, FRONTEND_MAINTENANCE_POLL_MS = 10, /* Cadence for the quiet no-maintenance steady state. Every presence probe * try-acquires the maintenance marker lock, and on Windows that @@ -61,6 +68,13 @@ enum { FRONTEND_PARTICIPANT_NAME_CAP = 64, }; +/* A grace shorter than either ordinary window would cut a legitimate spawn + * off at the old deadline, silently restoring the pre-#2796 fatal path. */ +_Static_assert(CBM_DAEMON_FRONTEND_RECONNECT_GRACE_MS >= FRONTEND_EOF_DRAIN_MS, + "reconnect grace must cover the EOF drain window"); +_Static_assert(CBM_DAEMON_FRONTEND_RECONNECT_GRACE_MS >= FRONTEND_JOIN_WATCHDOG_MS, + "reconnect grace must cover the join watchdog window"); + typedef struct { char *message; size_t length; @@ -90,6 +104,30 @@ typedef struct { const char *active_id_str; cbm_daemon_runtime_application_token_t active_request_token; bool failed; + /* #2796: replacement supplier for a dead daemon transport, plus the retry + * budget already spent in this run. daemon_lost==NULL keeps the legacy + * fatal behavior. */ + cbm_daemon_frontend_daemon_lost_fn daemon_lost; + void *daemon_lost_context; + /* Read by the worker and the EOF drain (grace reset per attempt). */ + atomic_uint daemon_reconnects; + /* A cancellation for the active request arrived while its token was + * invalid during a reconnect; deliver it with the re-minted token. */ + bool cancel_pending; + /* Clients replaced by a reconnect. close_begin has run, but the allocation + * stays alive until the run tail: the stdin reader and the maintenance + * monitor may still be using a pointer they read before the swap, and + * close_finish frees. Bounded by FRONTEND_DAEMON_RECONNECT_MAX. */ + cbm_daemon_runtime_client_t *retired[FRONTEND_DAEMON_RECONNECT_MAX]; + size_t retired_count; + /* True while the daemon_lost callback runs. A cold replacement spawn can + * outlast FRONTEND_EOF_DRAIN_MS, and the drain must treat that time as + * forward progress rather than a stuck request. */ + atomic_bool reconnect_active; + /* cbm_now_ms() when the in-flight attempt started (0 when none has run). + * Both waiters anchor the per-attempt grace here, so the drain and the + * join watchdog spend one 120 s budget for the same attempt. */ + atomic_uint_fast64_t reconnect_started_ms; /* Monotonic count of fully processed queue items (responses written or * cancellations acknowledged). The EOF drain below watches it to tell a * batch that is still making progress from a genuinely stuck request. */ @@ -97,6 +135,7 @@ typedef struct { } frontend_state_t; typedef struct { + frontend_state_t *state; atomic_bool complete; } frontend_join_watchdog_t; @@ -306,20 +345,39 @@ static void frontend_input_closed(frontend_state_t *state) { static void *frontend_join_watchdog(void *opaque) { frontend_join_watchdog_t *watchdog = opaque; - uint64_t now = cbm_now_ms(); - uint64_t deadline = - now > UINT64_MAX - FRONTEND_JOIN_WATCHDOG_MS ? UINT64_MAX : now + FRONTEND_JOIN_WATCHDOG_MS; - while (!atomic_load_explicit(&watchdog->complete, memory_order_acquire) && - cbm_now_ms() < deadline) { + frontend_state_t *state = watchdog->state; + cbm_daemon_frontend_liveness_t liveness; + cbm_daemon_frontend_liveness_init( + &liveness, cbm_now_ms(), + atomic_load_explicit(&state->daemon_reconnects, memory_order_acquire)); + while (!atomic_load_explicit(&watchdog->complete, memory_order_acquire)) { + uint64_t now = cbm_now_ms(); + unsigned int attempts = + atomic_load_explicit(&state->daemon_reconnects, memory_order_acquire); + bool reconnecting = atomic_load_explicit(&state->reconnect_active, memory_order_acquire); + uint64_t attempt_started = + atomic_load_explicit(&state->reconnect_started_ms, memory_order_acquire); + if (!cbm_daemon_frontend_liveness_wait(&liveness, now, attempts, reconnecting, + attempt_started, FRONTEND_JOIN_WATCHDOG_MS)) { + /* The worker may have completed between the poll and the + * deadline; do not fail-stop a join that is already finishing. + * `complete` is published only after the main thread's join + * returns, so also query the worker state to cover the + * done-but-not-yet-joined window. */ + if (atomic_load_explicit(&watchdog->complete, memory_order_acquire) || + frontend_worker_is_done(state)) { + break; + } + /* A thin frontend owns no durable state. If stdout is + * backpressured, fclose/IPC cancellation cannot portably wake its + * worker on every platform. Fail-stop releases the authenticated + * connection and lets the daemon cancel this exact session instead + * of hanging forever. A replacement spawn in progress gets the + * same bounded grace as in the EOF drain. */ + _Exit(EXIT_FAILURE); + } cbm_usleep(FRONTEND_WAIT_US); } - if (!atomic_load_explicit(&watchdog->complete, memory_order_acquire)) { - /* A thin frontend owns no durable state. If stdout is backpressured, - * fclose/IPC cancellation cannot portably wake its worker on every - * platform. Fail-stop releases the authenticated connection and lets - * the daemon cancel this exact session instead of hanging forever. */ - _Exit(EXIT_FAILURE); - } return NULL; } @@ -347,6 +405,7 @@ static bool frontend_pop_begin(frontend_state_t *state, frontend_item_t *item) { state->count--; state->queued_bytes -= item->length; state->in_request = true; + state->cancel_pending = false; state->active_has_id = item->has_id; state->active_id = item->id; state->active_id_str = item->id_str; @@ -365,6 +424,7 @@ static void frontend_end_request(frontend_state_t *state, bool failed) { state->active_id = 0; state->active_id_str = NULL; state->active_request_token = CBM_DAEMON_RUNTIME_APPLICATION_TOKEN_INVALID; + state->cancel_pending = false; state->failed = state->failed || failed; cbm_mutex_unlock(&state->mutex); } @@ -458,6 +518,9 @@ typedef enum { FRONTEND_CANCELLATION_STALE = 1, FRONTEND_CANCELLATION_QUEUED = 2, FRONTEND_CANCELLATION_ACTIVE = 3, + /* Matched the active request while its token was invalid during a + * reconnect; delivered when the retry publishes the new token. */ + FRONTEND_CANCELLATION_DEFERRED = 4, } frontend_cancellation_route_t; /* Cancellation is an out-of-band frontend control message. A matching active @@ -466,10 +529,14 @@ typedef enum { * without ever reaching the daemon. Stale/invalid targets are ignored. */ static frontend_cancellation_route_t frontend_route_cancellation( frontend_state_t *state, const char *message, - cbm_daemon_runtime_application_token_t *request_token_out) { + cbm_daemon_runtime_application_token_t *request_token_out, + cbm_daemon_runtime_client_t **request_client_out) { if (request_token_out) { *request_token_out = CBM_DAEMON_RUNTIME_APPLICATION_TOKEN_INVALID; } + if (request_client_out) { + *request_client_out = NULL; + } cbm_jsonrpc_request_t request = {0}; if (!frontend_parse_cancellation(message, &request)) { return FRONTEND_CANCELLATION_NONE; @@ -478,13 +545,25 @@ static frontend_cancellation_route_t frontend_route_cancellation( frontend_cancellation_route_t route = FRONTEND_CANCELLATION_STALE; cbm_mutex_lock(&state->mutex); if (state->in_request && state->active_has_id && - state->active_request_token != CBM_DAEMON_RUNTIME_APPLICATION_TOKEN_INVALID && cbm_mcp_cancel_request_matches(request.params_raw, state->active_id, state->active_id_str)) { - if (request_token_out) { - *request_token_out = state->active_request_token; + /* Remember the cancel even while the token is invalid during a + * reconnect: the retry delivers it with the freshly minted token. */ + state->cancel_pending = true; + if (state->active_request_token != CBM_DAEMON_RUNTIME_APPLICATION_TOKEN_INVALID) { + if (request_token_out) { + *request_token_out = state->active_request_token; + } + /* Snapshot the owner under the same lock as the token: the worker + * may already have swapped clients, and the token is only + * meaningful on the client that minted it. */ + if (request_client_out) { + *request_client_out = state->client; + } + route = FRONTEND_CANCELLATION_ACTIVE; + } else { + route = FRONTEND_CANCELLATION_DEFERRED; } - route = FRONTEND_CANCELLATION_ACTIVE; } else { for (size_t offset = 0; offset < state->count; offset++) { size_t index = (state->head + offset) % FRONTEND_QUEUE_CAPACITY; @@ -502,6 +581,126 @@ static frontend_cancellation_route_t frontend_route_cancellation( return route; } +/* #2796: swap in a replacement daemon client after a transport failure. The + * request token minted for the interrupted exchange was scoped to the dead + * client, so the caller must re-reserve one before re-issuing the request. + * The callback runs unmutexed (a cold daemon spawn takes seconds); the swap + * itself refuses once teardown has begun so no replacement outlives the run. + * The dead client stays allocated until the run tail because the stdin reader + * and the maintenance monitor can still hold a pointer read before the swap; + * close_begin marks it closed without freeing. */ +static bool frontend_daemon_reconnect(frontend_state_t *state) { + unsigned int attempts = atomic_load_explicit(&state->daemon_reconnects, memory_order_relaxed); + if (!state->daemon_lost || attempts >= FRONTEND_DAEMON_RECONNECT_MAX) { + return false; + } + cbm_mutex_lock(&state->mutex); + bool stopping = state->stopping; + cbm_daemon_runtime_client_t *dead = state->client; + if (!stopping) { + /* The published token belongs to the dead client. Invalidate it for + * the duration of the callback (a cold spawn takes seconds); a + * cancellation that arrives meanwhile is remembered in cancel_pending + * and delivered with the re-minted token. */ + state->active_request_token = CBM_DAEMON_RUNTIME_APPLICATION_TOKEN_INVALID; + } + cbm_mutex_unlock(&state->mutex); + if (stopping) { + return false; + } + /* Publish the attempt's timestamp and active flag BEFORE incrementing the + * counter. A waiter that observes the new counter (acquire) must then see + * this attempt's timestamp too; incrementing first would let it pair a new + * counter with the previous attempt's timestamp and compress the new + * attempt's grace. */ + atomic_store_explicit(&state->reconnect_started_ms, cbm_now_ms(), memory_order_release); + atomic_store_explicit(&state->reconnect_active, true, memory_order_release); + atomic_fetch_add_explicit(&state->daemon_reconnects, 1, memory_order_release); + cbm_daemon_runtime_client_t *replacement = state->daemon_lost(state->daemon_lost_context, dead); + atomic_store_explicit(&state->reconnect_active, false, memory_order_release); + if (!replacement || replacement == dead) { + return false; + } + /* A supplier that hands back a client retired earlier in this run would be + * appended twice and freed twice at the run tail. */ + for (size_t i = 0; i < state->retired_count; i++) { + if (state->retired[i] == replacement) { + return false; + } + } + cbm_mutex_lock(&state->mutex); + if (state->stopping || state->client != dead) { + cbm_mutex_unlock(&state->mutex); + (void)cbm_daemon_runtime_client_close(replacement, FRONTEND_CLOSE_TIMEOUT_MS); + return false; + } + state->client = replacement; + cbm_mutex_unlock(&state->mutex); + (void)cbm_daemon_runtime_client_close_begin(dead); + /* daemon_reconnects bounds this to the array size; guard locally so a + * future caller cannot corrupt the run. */ + if (state->retired_count < FRONTEND_DAEMON_RECONNECT_MAX) { + state->retired[state->retired_count++] = dead; + } + return true; +} + +/* #2796: one request exchange with replacement-daemon retries. Returns the + * final status; response and response_length are set only on OK. */ +static cbm_daemon_runtime_application_status_t frontend_exchange_with_reconnect( + frontend_state_t *state, frontend_item_t *item, uint8_t **response, uint32_t *response_length) { + cbm_daemon_runtime_application_status_t status; + for (;;) { + status = cbm_daemon_application_client_mcp_tagged(state->client, item->request_token, + item->message, response, response_length, + FRONTEND_REQUEST_TIMEOUT_MS); + if (status == CBM_DAEMON_RUNTIME_APPLICATION_OK) { + return status; + } + /* Only a transport failure triggers recovery. BUSY, UNAVAILABLE, + * REJECTED, HANDLER_ERROR and CANCELLED are daemon answers and must + * keep their existing meaning; CANCELLED in particular must never be + * re-issued as a fresh request. A dead transport is not this session's + * death: swap in a replacement client, re-mint the request token + * (tokens are scoped to the client that reserved them), and re-issue + * the interrupted request. + * + * TRANSPORT_ERROR also covers local allocation failures in the + * application layer while the connection is still healthy; replaying + * such a request is harmless (it never left the client) but costs a + * reconnect. It also covers a reply the client could not copy after + * the daemon executed the request (runtime response-copy failure). + * Such a request is replayed even though it may have run; the + * alternative is losing both the response and the session, and the + * transport failure carries no evidence either way. */ + if (status != CBM_DAEMON_RUNTIME_APPLICATION_TRANSPORT_ERROR || + !frontend_daemon_reconnect(state)) { + return status; + } + /* Reserve and publish under one lock, like frontend_pop_begin. A + * cancellation that arrived while the reconnect held the token + * invalid is remembered in cancel_pending and delivered with this + * fresh token. */ + cbm_mutex_lock(&state->mutex); + bool minted = cbm_daemon_runtime_client_application_token_reserve(state->client, + &item->request_token); + bool deliver_cancel = minted && state->cancel_pending; + if (minted) { + state->active_request_token = item->request_token; + state->cancel_pending = false; + } + cbm_mutex_unlock(&state->mutex); + if (!minted) { + /* The replacement is installed but cannot mint a token (it is + * already closing or unusable); the run teardown closes it. */ + return status; + } + if (deliver_cancel) { + (void)cbm_daemon_runtime_client_application_cancel(state->client, item->request_token); + } + } +} + static void *frontend_worker(void *opaque) { frontend_state_t *state = opaque; for (;;) { @@ -523,9 +722,7 @@ static void *frontend_worker(void *opaque) { failed = !frontend_write_cancelled_response(state->out, &item); } else { cbm_daemon_runtime_application_status_t status = - cbm_daemon_application_client_mcp_tagged(state->client, item.request_token, - item.message, &response, &response_length, - FRONTEND_REQUEST_TIMEOUT_MS); + frontend_exchange_with_reconnect(state, &item, &response, &response_length); if (status == CBM_DAEMON_RUNTIME_APPLICATION_CANCELLED) { failed = !frontend_write_cancelled_response(state->out, &item); } else { @@ -544,12 +741,14 @@ static void *frontend_worker(void *opaque) { atomic_fetch_add_explicit(&state->completed_items, 1, memory_order_release); if (failed) { if (!expected_stop) { - /* A failed daemon transport cannot wake a thread blocked in - * stdio portably. This frontend owns no state: immediate - * process exit closes the kernel IPC handle, which cancels - * daemon session ownership without a detached reader or an - * unsafe cross-thread fclose. Logging or flushing here could - * itself block on agent-owned stdout/stderr. */ + /* #2796: reached only without a daemon_lost callback or after + * its retry budget is spent. A failed daemon transport cannot + * wake a thread blocked in stdio portably. This frontend owns + * no state: immediate process exit closes the kernel IPC + * handle, which cancels daemon session ownership without a + * detached reader or an unsafe cross-thread fclose. Logging or + * flushing here could itself block on agent-owned + * stdout/stderr. */ _Exit(EXIT_FAILURE); } break; @@ -632,32 +831,47 @@ static bool frontend_enqueue(frontend_state_t *state, char *message, bool conten static bool frontend_stop_begin(frontend_state_t *state) { cbm_mutex_lock(&state->mutex); state->stopping = true; + cbm_daemon_runtime_client_t *client = state->client; cbm_mutex_unlock(&state->mutex); /* Retain the client allocation until the worker is joined. This covers the * boundary where the worker has claimed an item but has not yet entered the - * runtime exchange: a late call observes closing instead of freed memory. */ - return cbm_daemon_runtime_client_close_begin(state->client); + * runtime exchange: a late call observes closing instead of freed memory. + * The worker cannot swap after stopping is published, so this snapshot is + * the client this run owns. */ + return cbm_daemon_runtime_client_close_begin(client); } static bool frontend_cancel_for_maintenance(void *opaque) { frontend_state_t *state = opaque; cbm_daemon_runtime_application_token_t request_token = CBM_DAEMON_RUNTIME_APPLICATION_TOKEN_INVALID; + cbm_daemon_runtime_client_t *client = NULL; + bool in_request; cbm_mutex_lock(&state->mutex); state->stopping = true; - if (state->in_request) { + in_request = state->in_request; + if (in_request) { request_token = state->active_request_token; + client = state->client; } cbm_mutex_unlock(&state->mutex); - if (request_token == CBM_DAEMON_RUNTIME_APPLICATION_TOKEN_INVALID) { + if (!in_request) { return false; } - return cbm_daemon_runtime_client_application_cancel(state->client, request_token) == + if (request_token == CBM_DAEMON_RUNTIME_APPLICATION_TOKEN_INVALID || !client) { + /* A reconnect holds the token invalid while a request is still in + * flight: report the cancel as requested so the monitor takes its + * bounded grace instead of exiting immediately. */ + return true; + } + return cbm_daemon_runtime_client_application_cancel(client, request_token) == CBM_DAEMON_RUNTIME_CANCEL_ACCEPTED; } int cbm_daemon_frontend_mcp_run(cbm_daemon_runtime_client_t *client, - cbm_version_cohort_manager_t *cohort_manager, FILE *in, FILE *out) { + cbm_version_cohort_manager_t *cohort_manager, FILE *in, FILE *out, + cbm_daemon_frontend_daemon_lost_fn daemon_lost, + void *daemon_lost_context) { if (!client || !cohort_manager || !in || !out) { return -1; } @@ -665,9 +879,14 @@ int cbm_daemon_frontend_mcp_run(cbm_daemon_runtime_client_t *client, .client = client, .cohort_manager = cohort_manager, .out = out, + .daemon_lost = daemon_lost, + .daemon_lost_context = daemon_lost_context, }; cbm_mutex_init(&state.mutex); atomic_init(&state.completed_items, 0); + atomic_init(&state.daemon_reconnects, 0); + atomic_init(&state.reconnect_active, false); + atomic_init(&state.reconnect_started_ms, 0); cbm_daemon_maintenance_monitor_t *maintenance_monitor = frontend_maintenance_monitor_start(cohort_manager, frontend_cancel_for_maintenance, &state, EXIT_SUCCESS, "MCP frontend", true); @@ -701,18 +920,17 @@ int cbm_daemon_frontend_mcp_run(cbm_daemon_runtime_client_t *client, } cbm_daemon_runtime_application_token_t cancel_token = CBM_DAEMON_RUNTIME_APPLICATION_TOKEN_INVALID; + cbm_daemon_runtime_client_t *cancel_client = NULL; frontend_cancellation_route_t cancellation = - frontend_route_cancellation(&state, message, &cancel_token); + frontend_route_cancellation(&state, message, &cancel_token, &cancel_client); if (cancellation != FRONTEND_CANCELLATION_NONE) { free(message); if (cancellation == FRONTEND_CANCELLATION_ACTIVE) { - cbm_daemon_runtime_cancel_result_t cancelled = - cbm_daemon_runtime_client_application_cancel(state.client, cancel_token); - if (cancelled == CBM_DAEMON_RUNTIME_CANCEL_ERROR) { - result = -1; - close_begun = frontend_stop_begin(&state); - break; - } + /* Cancellation is best-effort: ERROR means the client is + * already unusable or closing. The in-flight exchange observes + * the same failure (or stop), and the worker owns the recovery + * decision instead of the reader tearing the session down. */ + (void)cbm_daemon_runtime_client_application_cancel(cancel_client, cancel_token); } /* A queued cancellation is represented on that queue item; stale * and malformed targets are intentionally side-effect free. */ @@ -740,14 +958,44 @@ int cbm_daemon_frontend_mcp_run(cbm_daemon_runtime_client_t *client, * writing — file redirects, piped harnesses, hook one-shots — * intermittently lost their tail responses. */ uint64_t drained = atomic_load_explicit(&state.completed_items, memory_order_acquire); - uint64_t drain_deadline = cbm_now_ms() + FRONTEND_EOF_DRAIN_MS; - while (!frontend_worker_is_done(&state) && cbm_now_ms() < drain_deadline) { + cbm_daemon_frontend_liveness_t liveness; + cbm_daemon_frontend_liveness_init( + &liveness, cbm_now_ms(), + atomic_load_explicit(&state.daemon_reconnects, memory_order_acquire)); + for (;;) { + if (frontend_worker_is_done(&state)) { + break; + } + uint64_t now = cbm_now_ms(); + unsigned int attempts = + atomic_load_explicit(&state.daemon_reconnects, memory_order_acquire); + bool reconnecting = atomic_load_explicit(&state.reconnect_active, memory_order_acquire); + uint64_t attempt_started = + atomic_load_explicit(&state.reconnect_started_ms, memory_order_acquire); + if (!cbm_daemon_frontend_liveness_wait(&liveness, now, attempts, reconnecting, + attempt_started, FRONTEND_EOF_DRAIN_MS)) { + if (frontend_worker_is_done(&state)) { + break; + } + /* Re-sample once: an attempt that became visible after the + * poll above is granted its grace instead of being cut off by + * stop_begin. */ + bool late_reconnect = + atomic_load_explicit(&state.reconnect_active, memory_order_acquire); + uint64_t late_started = + atomic_load_explicit(&state.reconnect_started_ms, memory_order_acquire); + if (late_reconnect && + cbm_daemon_frontend_liveness_reconnect_late(&liveness, now, late_started)) { + continue; + } + break; + } cbm_usleep(FRONTEND_WAIT_US); uint64_t now_drained = atomic_load_explicit(&state.completed_items, memory_order_acquire); if (now_drained != drained) { drained = now_drained; - drain_deadline = cbm_now_ms() + FRONTEND_EOF_DRAIN_MS; + cbm_daemon_frontend_liveness_progress(&liveness, cbm_now_ms()); } } } @@ -758,6 +1006,7 @@ int cbm_daemon_frontend_mcp_run(cbm_daemon_runtime_client_t *client, cbm_thread_t watchdog_thread; bool watchdog_started = false; if (!frontend_worker_is_done(&state)) { + watchdog.state = &state; atomic_init(&watchdog.complete, false); if (cbm_thread_create(&watchdog_thread, 0, frontend_join_watchdog, &watchdog) != 0) { _Exit(EXIT_FAILURE); @@ -782,6 +1031,11 @@ int cbm_daemon_frontend_mcp_run(cbm_daemon_runtime_client_t *client, } else { result = -1; } + /* The worker is joined and the maintenance monitor is stopped, so no other + * thread can still reference a client retired by a reconnect. */ + for (size_t i = 0; i < state.retired_count; i++) { + (void)cbm_daemon_runtime_client_close_finish(state.retired[i], FRONTEND_CLOSE_TIMEOUT_MS); + } for (size_t i = 0; i < FRONTEND_QUEUE_CAPACITY; i++) { frontend_item_free(&state.queue[i]); } diff --git a/src/daemon/frontend.h b/src/daemon/frontend.h index 9d172606e0..fb67a4e7ed 100644 --- a/src/daemon/frontend.h +++ b/src/daemon/frontend.h @@ -17,6 +17,25 @@ typedef struct cbm_daemon_maintenance_monitor cbm_daemon_maintenance_monitor_t; * outlive the bounded grace period. */ typedef bool (*cbm_daemon_maintenance_cancel_fn)(void *context); +/* #2796: invoked on the frontend worker thread when a request exchange fails + * with a transport error while the MCP client's session is still live. + * dead_client remains owned by the frontend, which closes it after the swap + * and frees it only once every frontend thread has stopped; the callback must + * not use or close it. The callback establishes a replacement (connect-or-spawn + * plus session context) and returns it — NULL leaves the transport failure + * fatal. The returned client must be fresh, open, and distinct from every + * client returned earlier in this run (dead_client included); the frontend + * takes ownership: it closes the client if the swap is refused, and + * closes/frees it at the end of a successful run. The callback runs at most + * three times per run. It may take as long as a cold daemon spawn; the EOF + * drain and the join watchdog grant at most a hard 120 s grace per attempt. + * A maintenance activation still fail-stops the process on the monitor's own + * shorter grace, because a binary update must not wait for a reconnect. It + * must not read or write the frontend's stdin/stdout; stderr diagnostics are + * allowed, but the callback must not assume that stream is being drained. */ +typedef cbm_daemon_runtime_client_t *(*cbm_daemon_frontend_daemon_lost_fn)( + void *context, cbm_daemon_runtime_client_t *dead_client); + /* Parse one JSON-RPC message and recognize only the exact cancellation * notification method. String contents, nested fields, method prefixes, and * requests carrying an id are not cancellation notifications. */ @@ -66,9 +85,17 @@ uint64_t cbm_daemon_frontend_test_worker_idle_cycles(void); * either thread is blocked in stdio, requests cooperative cancellation for the * exact active request, and then bounds process exit. Kernel IPC close cancels * only this session's daemon work. EOF/parse failure closes the authenticated - * session. An unexpected daemon transport failure likewise terminates the - * process so an agent waiting with stdin still open observes server EOF. */ + * session. An unexpected daemon transport failure is retried against a + * replacement daemon through daemon_lost (bounded, #2796); daemon answers such + * as busy, rejected, handler error or cancellation are not retried. A replayed + * request may already have executed in the dead daemon: a transport failure + * cannot distinguish "never sent" from "reply lost", so the interrupted call + * has at-least-once semantics. Without the callback — or once the retry budget + * is spent — the worker terminates the process immediately so an agent waiting + * with stdin still open observes server EOF. */ int cbm_daemon_frontend_mcp_run(cbm_daemon_runtime_client_t *client, - cbm_version_cohort_manager_t *cohort_manager, FILE *in, FILE *out); + cbm_version_cohort_manager_t *cohort_manager, FILE *in, FILE *out, + cbm_daemon_frontend_daemon_lost_fn daemon_lost, + void *daemon_lost_context); #endif /* CBM_DAEMON_FRONTEND_H */ diff --git a/src/daemon/frontend_liveness.c b/src/daemon/frontend_liveness.c new file mode 100644 index 0000000000..289145f8fc --- /dev/null +++ b/src/daemon/frontend_liveness.c @@ -0,0 +1,74 @@ +/* + * frontend_liveness.c — deterministic deadline policy for the frontend + * waiters. No clocks, no atomics, no frontend state: callers pass a sampled + * `now`, the reconnect counter and attempt start, and whether a replacement is + * running. + */ +#include "daemon/frontend_liveness.h" + +static uint64_t frontend_liveness_deadline(uint64_t start, uint64_t window_ms) { + return start > UINT64_MAX - window_ms ? UINT64_MAX : start + window_ms; +} + +static uint64_t frontend_liveness_anchor(uint64_t now, uint64_t attempt_started_ms) { + /* A start after `now` would mean the sampler saw the attempt before its + * timestamp; prefer the local observation so the deadline cannot move + * backwards. */ + return attempt_started_ms != 0 && attempt_started_ms <= now ? attempt_started_ms : now; +} + +void cbm_daemon_frontend_liveness_init(cbm_daemon_frontend_liveness_t *liveness, uint64_t now, + unsigned int attempts) { + liveness->last_progress_ms = now; + liveness->reconnect_since_ms = 0; + liveness->seen_attempts = attempts; + liveness->was_reconnecting = false; +} + +bool cbm_daemon_frontend_liveness_wait(cbm_daemon_frontend_liveness_t *liveness, uint64_t now, + unsigned int attempts, bool reconnecting, + uint64_t attempt_started_ms, uint64_t base_window_ms) { + if (attempts != liveness->seen_attempts) { + liveness->seen_attempts = attempts; + liveness->last_progress_ms = now; + liveness->reconnect_since_ms = 0; + liveness->was_reconnecting = false; + } + if (reconnecting) { + if (!liveness->was_reconnecting) { + liveness->reconnect_since_ms = frontend_liveness_anchor(now, attempt_started_ms); + } + } else if (liveness->was_reconnecting) { + /* The attempt ended; the retried exchange gets its own stuck window + * instead of inheriting the spawn's remainder. */ + liveness->last_progress_ms = now; + liveness->reconnect_since_ms = 0; + } + liveness->was_reconnecting = reconnecting; + + uint64_t deadline = frontend_liveness_deadline(liveness->last_progress_ms, base_window_ms); + if (reconnecting) { + uint64_t reconnect_deadline = frontend_liveness_deadline( + liveness->reconnect_since_ms, CBM_DAEMON_FRONTEND_RECONNECT_GRACE_MS); + if (reconnect_deadline > deadline) { + deadline = reconnect_deadline; + } + } + return now < deadline; +} + +void cbm_daemon_frontend_liveness_progress(cbm_daemon_frontend_liveness_t *liveness, uint64_t now) { + liveness->last_progress_ms = now; +} + +bool cbm_daemon_frontend_liveness_reconnect_late(cbm_daemon_frontend_liveness_t *liveness, + uint64_t now, uint64_t attempt_started_ms) { + if (liveness->was_reconnecting) { + /* This attempt already has its grace; re-anchoring here would let a + * repeated re-sample extend a hung attempt without bound. */ + return false; + } + liveness->reconnect_since_ms = frontend_liveness_anchor(now, attempt_started_ms); + liveness->was_reconnecting = true; + return true; +} diff --git a/src/daemon/frontend_liveness.h b/src/daemon/frontend_liveness.h new file mode 100644 index 0000000000..8f2ef2fb70 --- /dev/null +++ b/src/daemon/frontend_liveness.h @@ -0,0 +1,61 @@ +/* + * frontend_liveness.h — no-progress policy shared by the frontend's waiters. + * + * Two threads wait for the worker with different ordinary windows: the stdin + * reader's EOF drain (15 s) and the join watchdog (5 s). Both must treat a + * replacement-daemon attempt as forward progress, because a cold daemon start + * can outlast either window. Keeping that arithmetic in each loop is how the + * two policies drifted once (#2796): the drain granted the spawn its grace + * while the watchdog killed the same worker five seconds in. Each waiter + * supplies its own ordinary window; the reconnect grace is shared here and + * anchored to the attempt's real start, so two waiters observing the same + * attempt spend one 120 s budget rather than two. + */ +#ifndef CBM_DAEMON_FRONTEND_LIVENESS_H +#define CBM_DAEMON_FRONTEND_LIVENESS_H + +#include +#include + +/* One replacement spawn may take as long as a cold daemon start (connect, + * bootstrap, and possibly an upgrade drain). This bounds a single attempt, not + * the run: the reconnect budget bounds the run. */ +#define CBM_DAEMON_FRONTEND_RECONNECT_GRACE_MS 120000U + +typedef struct { + uint64_t last_progress_ms; + uint64_t reconnect_since_ms; + unsigned int seen_attempts; + bool was_reconnecting; +} cbm_daemon_frontend_liveness_t; + +/* Start a window at `now`; `attempts` is the reconnect counter's current + * value (a nonzero value when a waiter starts mid-attempt). */ +void cbm_daemon_frontend_liveness_init(cbm_daemon_frontend_liveness_t *liveness, uint64_t now, + unsigned int attempts); + +/* Observe one poll and report whether the caller may keep waiting. + * `base_window_ms` is the ordinary no-progress window. A reconnect in progress + * extends the deadline to that attempt's grace. `attempt_started_ms` is the + * attempt's real start (0 when the caller has no timestamp): anchoring to it + * lets the drain and the watchdog share one budget for the same attempt. + * A changed attempt counter (an attempt that started and ended between polls) + * restarts the window, so a fast failure is not charged to the previous + * attempt. */ +bool cbm_daemon_frontend_liveness_wait(cbm_daemon_frontend_liveness_t *liveness, uint64_t now, + unsigned int attempts, bool reconnecting, + uint64_t attempt_started_ms, uint64_t base_window_ms); + +/* A unit of work completed: restart the ordinary window. The per-attempt + * reconnect grace is not extended. */ +void cbm_daemon_frontend_liveness_progress(cbm_daemon_frontend_liveness_t *liveness, uint64_t now); + +/* The deadline expired, but a re-sample showed a reconnect that started after + * the poll: grant that attempt its grace instead of stopping underneath it. + * Returns true when a grace was granted; repeated calls for the same attempt + * are no-ops and return false, so the caller cannot extend a hung attempt by + * re-sampling. */ +bool cbm_daemon_frontend_liveness_reconnect_late(cbm_daemon_frontend_liveness_t *liveness, + uint64_t now, uint64_t attempt_started_ms); + +#endif /* CBM_DAEMON_FRONTEND_LIVENESS_H */ diff --git a/src/foundation/subprocess.c b/src/foundation/subprocess.c index 5e3a549ed2..4e703eb233 100644 --- a/src/foundation/subprocess.c +++ b/src/foundation/subprocess.c @@ -27,6 +27,7 @@ #else #include #include +#include #include #ifdef __APPLE__ #include @@ -1434,7 +1435,15 @@ static void cbm_posix_reset_child_signals(void) { static bool g_force_close_range_enosys = false; #endif -static cbm_fd_close_strategy_t cbm_posix_close_fds_from(int lowfd, long max_fd) { +long cbm_posix_fd_close_cap(void) { + long max_fd = sysconf(_SC_OPEN_MAX); + if (max_fd < 0 || max_fd > 1048576L) { + max_fd = 65536L; + } + return max_fd; +} + +cbm_fd_close_strategy_t cbm_posix_close_fds_from(int lowfd, long max_fd) { #if defined(__linux__) && defined(SYS_close_range) bool range_unavailable = false; #ifdef CBM_ENABLE_TEST_SEAMS @@ -1448,6 +1457,13 @@ static cbm_fd_close_strategy_t cbm_posix_close_fds_from(int lowfd, long max_fd) closefrom(lowfd); return CBM_FD_CLOSEFROM; #endif + /* close() takes an int, so descriptors above INT_MAX cannot exist. Clamp + * inside the implementation: the exported function must not depend on a + * caller respecting the cap contract, or an int loop variable would wrap + * and never terminate in the fork child. */ + if (max_fd > INT_MAX) { + max_fd = INT_MAX; + } for (int fd = lowfd; fd < max_fd; fd++) { (void)close(fd); } @@ -1701,10 +1717,7 @@ static int cbm_subprocess_spawn_posix(cbm_subprocess_t *process) { } int child_output = stdout_capture >= 0 ? stdout_capture : output; - long max_fd = sysconf(_SC_OPEN_MAX); - if (max_fd < 0 || max_fd > 1048576L) { - max_fd = 65536L; - } + long max_fd = cbm_posix_fd_close_cap(); pid_t pid = -1; #ifdef __APPLE__ diff --git a/src/foundation/subprocess.h b/src/foundation/subprocess.h index 4dc8f18705..9ba4a75575 100644 --- a/src/foundation/subprocess.h +++ b/src/foundation/subprocess.h @@ -232,6 +232,20 @@ typedef enum { CBM_FD_CLOSE_LOOP /* close() per descriptor up to _SC_OPEN_MAX: O(RLIMIT_NOFILE) */ } cbm_fd_close_strategy_t; +#ifndef _WIN32 +/* Fallback loop bound for the descriptor sweep below: sysconf(_SC_OPEN_MAX), + * or 65536 when that is unavailable or above 1048576. Callers invoke this + * BEFORE fork(); sysconf may take libc locks. */ +long cbm_posix_fd_close_cap(void); + +/* Close every descriptor >= lowfd in a fork+exec child (#1484, #2796). + * async-signal-safe: a raw close_range syscall, closefrom(), or close() only — + * no allocation and no stdio, because this runs between fork and exec in a + * child of a possibly multi-threaded process. `max_fd` is the cap returned by + * cbm_posix_fd_close_cap() in the parent. */ +cbm_fd_close_strategy_t cbm_posix_close_fds_from(int lowfd, long max_fd); +#endif + #if defined(CBM_ENABLE_TEST_SEAMS) && !defined(_WIN32) /* Run the child's close-inherited-descriptors step in the CALLING process for * descriptors >= lowfd (callers pick a lowfd above everything they need) and diff --git a/src/main.c b/src/main.c index 3e658c8016..9da43d201a 100644 --- a/src/main.c +++ b/src/main.c @@ -2785,6 +2785,112 @@ static int main_run_daemon_ctl(int argc, char **argv, const cbm_daemon_ipc_endpo return ui_result; } +/* One path from an endpoint to a usable MCP client: bootstrap (including the + * one-shot upgrade retry) and the session context. The initial connection and + * the reconnect supplier share it so a future bootstrap or context change + * cannot land in only one of them. `session_live` selects bootstrap failure + * reporting: before the session exists stdout carries a JSON-RPC error + * (#1539); during a reconnect the session is live, and an unsolicited id:null + * frame would corrupt it, so diagnostics go to stderr only. A rejected + * session context keeps its pre-existing stderr-only report on both paths. */ +typedef struct { + cbm_daemon_process_role_t role; + const cbm_daemon_build_identity_t *identity; + const char *executable_path; + cbm_mcp_tool_profile_t tool_profile; + uint32_t startup_timeout_ms; + bool set_session_context; +} main_mcp_session_config_t; + +static void main_report_endpoint_failure(const main_mcp_session_config_t *config, + bool session_live) { + const char *why = cbm_daemon_ipc_validation_detail(); + char message[CBM_DAEMON_CONFLICT_MESSAGE_SIZE]; + (void)snprintf(message, sizeof(message), "secure daemon endpoint could not be created%s%s%s", + session_live ? " during reconnect" : "", (why && why[0]) ? ": " : "", + (why && why[0]) ? why : ""); + if (session_live) { + (void)fprintf(stderr, "codebase-memory-mcp: %s\n", message); + return; + } + main_report_client_failure(config->role, message); +} + +static cbm_daemon_runtime_client_t *main_establish_mcp_session( + const main_mcp_session_config_t *config, const cbm_daemon_ipc_endpoint_t *endpoint, + bool session_live) { + cbm_daemon_bootstrap_config_t bootstrap_config = { + .role = config->role, + .endpoint = endpoint, + .identity = config->identity, + .executable_path = config->executable_path, + .connect_timeout_ms = MAIN_CONNECT_TIMEOUT_MS, + .startup_timeout_ms = config->startup_timeout_ms, + }; + cbm_daemon_bootstrap_result_t result; + cbm_daemon_bootstrap_status_t status = + main_client_bootstrap_with_upgrade(&bootstrap_config, &result, true); + if (status != CBM_DAEMON_BOOTSTRAP_CONNECTED || !result.client) { + if (session_live) { + const char *message = result.message[0] + ? result.message + : "CBM daemon connection failed during reconnect"; + (void)fprintf(stderr, "codebase-memory-mcp: %s%s%s\n", message, + result.remedy[0] ? " " : "", result.remedy); + } else { + main_report_client_bootstrap_failure(config->role, &result); + } + return NULL; + } + /* Startup requests on the connected client wait within the startup budget, + * like the bootstrap before them: a 1 s bound on the reply (a probe + * budget) ended the server whenever the daemon's UI save ran long, which + * the smoke's `--ui=true --port` run saw as a binary without UI assets. */ + if (config->set_session_context && + !main_set_client_context(result.client, NULL, config->tool_profile, NULL, NULL, + config->startup_timeout_ms)) { + (void)fprintf(stderr, "codebase-memory-mcp: daemon session context was rejected\n"); + (void)cbm_daemon_runtime_client_close(result.client, MAIN_CLOSE_TIMEOUT_MS); + return NULL; + } + return result.client; +} + +/* #2796: supplier for the MCP frontend's daemon_lost callback. Re-runs the + * whole connect-or-spawn leg with the session context and hands the frontend + * the replacement. The dead client remains owned by the frontend (the stdin + * reader and maintenance monitor may still observe it; the frontend closes it + * after the swap and frees it once every frontend thread has stopped), so this + * callback only replaces. Runs on the frontend worker thread while the stdin + * reader and maintenance monitor are live; bootstrap's POSIX fork child keeps + * only async-signal-safe calls for that reason. It reports to stderr only and + * never touches the agent's stdio. UI flags are not re-applied: the daemon + * persists its UI configuration and a re-spawned daemon reads it back. */ +typedef struct { + main_mcp_session_config_t session; +} main_mcp_daemon_lost_context_t; + +static cbm_daemon_runtime_client_t *main_mcp_daemon_lost(void *context, + cbm_daemon_runtime_client_t *dead_client) { + main_mcp_daemon_lost_context_t *lost = context; + (void)dead_client; + if (atomic_load(&g_shutdown)) { + /* SIGTERM/SIGINT already closed stdin: do not cold-start a daemon for + * a session that is on its way out. */ + return NULL; + } + cbm_log_warn("mcp.frontend.daemon_lost_reconnect"); + cbm_daemon_ipc_endpoint_t *endpoint = main_daemon_endpoint_new(); + if (!endpoint) { + main_report_endpoint_failure(&lost->session, true); + return NULL; + } + cbm_daemon_runtime_client_t *replacement = + main_establish_mcp_session(&lost->session, endpoint, true); + cbm_daemon_ipc_endpoint_free(endpoint); + return replacement; +} + int main(int argc, char **argv) { /* Must remain the first statement: see allocator binding contract above. */ cbm_alloc_init(); @@ -3243,17 +3349,21 @@ int main(int argc, char **argv) { return result; } + main_mcp_session_config_t session_config = { + .role = role, + .identity = &identity, + .executable_path = executable_path, + .tool_profile = tool_profile, + .startup_timeout_ms = main_startup_timeout_ms(), + .set_session_context = role == CBM_DAEMON_PROCESS_MCP_CLIENT, + }; cbm_daemon_ipc_endpoint_t *endpoint = main_daemon_endpoint_new(); if (!endpoint) { /* #1582: this is where an ownership/ancestry refusal lands, and it was * the silent one — stderr only, so an MCP client saw a transport that * closed with no explanation. Include the validation detail, which * names the directory and the rule that refused. */ - const char *why = cbm_daemon_ipc_validation_detail(); - char message[CBM_DAEMON_CONFLICT_MESSAGE_SIZE]; - (void)snprintf(message, sizeof(message), "secure daemon endpoint could not be created%s%s", - (why && why[0]) ? ": " : "", (why && why[0]) ? why : ""); - main_report_client_failure(role, message); + main_report_endpoint_failure(&session_config, false); return EXIT_FAILURE; } @@ -3361,39 +3471,15 @@ int main(int argc, char **argv) { return EXIT_SUCCESS; } - cbm_daemon_bootstrap_config_t bootstrap_config = { - .role = role, - .endpoint = endpoint, - .identity = &identity, - .executable_path = executable_path, - .connect_timeout_ms = MAIN_CONNECT_TIMEOUT_MS, - .startup_timeout_ms = main_startup_timeout_ms(), - }; - cbm_daemon_bootstrap_result_t bootstrap_result; - cbm_daemon_bootstrap_status_t bootstrap_status = - main_client_bootstrap_with_upgrade(&bootstrap_config, &bootstrap_result, true); + cbm_daemon_runtime_client_t *session_client = + main_establish_mcp_session(&session_config, endpoint, false); cbm_daemon_ipc_endpoint_free(endpoint); - if (bootstrap_status != CBM_DAEMON_BOOTSTRAP_CONNECTED || !bootstrap_result.client) { - main_report_client_bootstrap_failure(role, &bootstrap_result); + if (!session_client) { (void)main_version_cohort_close(&client_cohort_lease, &client_cohort_manager); return EXIT_FAILURE; } - g_daemon_client = bootstrap_result.client; - - /* Startup requests on the connected client wait within the startup budget, - * like the bootstrap before them: a 1 s bound on the reply (a probe budget) - * ended the server whenever the daemon's UI save ran long, which the smoke's - * `--ui=true --port` run saw as a binary without UI assets. */ - if (role == CBM_DAEMON_PROCESS_MCP_CLIENT && - !main_set_client_context(g_daemon_client, NULL, tool_profile, NULL, NULL, - bootstrap_config.startup_timeout_ms)) { - (void)fprintf(stderr, "codebase-memory-mcp: daemon session context was rejected\n"); - (void)cbm_daemon_runtime_client_close(g_daemon_client, MAIN_CLOSE_TIMEOUT_MS); - g_daemon_client = NULL; - (void)main_version_cohort_close(&client_cohort_lease, &client_cohort_manager); - return EXIT_FAILURE; - } + g_daemon_client = session_client; /* Persist UI mutations only after the exact-build HELLO succeeds. A * conflicting binary must be observationally read-only: applying its @@ -3408,7 +3494,7 @@ int main(int argc, char **argv) { if (update_mask != 0 && cbm_daemon_application_client_set_ui_config( g_daemon_client, update_mask, ui_enabled, ui_port, - bootstrap_config.startup_timeout_ms) != CBM_DAEMON_RUNTIME_APPLICATION_OK) { + session_config.startup_timeout_ms) != CBM_DAEMON_RUNTIME_APPLICATION_OK) { (void)fprintf(stderr, "codebase-memory-mcp: daemon UI configuration update failed\n"); (void)cbm_daemon_runtime_client_close(g_daemon_client, MAIN_CLOSE_TIMEOUT_MS); g_daemon_client = NULL; @@ -3433,8 +3519,13 @@ int main(int argc, char **argv) { } setup_signal_handlers(); - int result = cbm_daemon_frontend_mcp_run(g_daemon_client, client_cohort_manager, stdin, stdout); - g_daemon_client = NULL; /* frontend consumed the handle */ + main_mcp_daemon_lost_context_t daemon_lost_context = { + .session = session_config, + }; + cbm_daemon_runtime_client_t *frontend_client = g_daemon_client; + g_daemon_client = NULL; /* frontend owns the handle from here on */ + int result = cbm_daemon_frontend_mcp_run(frontend_client, client_cohort_manager, stdin, stdout, + main_mcp_daemon_lost, &daemon_lost_context); bool client_cohort_cleanup = main_version_cohort_close(&client_cohort_lease, &client_cohort_manager); atomic_store(&g_shutdown, 1); diff --git a/tests/test_daemon_frontend.c b/tests/test_daemon_frontend.c index b91705f157..a055020ced 100644 --- a/tests/test_daemon_frontend.c +++ b/tests/test_daemon_frontend.c @@ -5,6 +5,7 @@ #include "test_helpers.h" #include "daemon/frontend.h" +#include "daemon/frontend_liveness.h" #include "daemon/ipc.h" #include "daemon/service.h" #include "daemon/version_cohort.h" @@ -22,6 +23,7 @@ #include #include #include +#include #include #include #endif @@ -213,7 +215,7 @@ static int frontend_test_close_fd(int fd) { static void *frontend_idle_run(void *opaque) { frontend_idle_run_t *run = opaque; run->result = - cbm_daemon_frontend_mcp_run(run->client, run->manager, run->input, run->output); + cbm_daemon_frontend_mcp_run(run->client, run->manager, run->input, run->output, NULL, NULL); return NULL; } #endif @@ -255,6 +257,10 @@ typedef struct { * of failing the session); the release is a state transition, not a * timeout, so the drain that follows is deterministic. */ atomic_bool release_first_request; + /* Reconnect regression mode: answer every request with CANCELLED so a + * frontend that retries anything other than a transport failure is + * observable (it would invoke the replacement callback). */ + atomic_bool cancel_requests; int request_observed_fd; int session_cancel_fd; } frontend_eof_application_context_t; @@ -294,6 +300,19 @@ static cbm_daemon_build_identity_t frontend_test_identity(void) { return identity; } +/* A service and every client that must reconnect to it share this identity; + * the replacement-connect is rejected when the build fingerprint differs. */ +static cbm_daemon_build_identity_t frontend_runtime_identity(const char *build) { + return (cbm_daemon_build_identity_t){ + .semantic_version = "2.4.0", + .build_fingerprint = build, + .cache_fingerprint = FRONTEND_TEST_CACHE, + .protocol_abi = 3, + .store_abi = 11, + .feature_abi = 7, + }; +} + static cbm_daemon_runtime_application_session_t *frontend_eof_application_session_open( void *opaque, cbm_daemon_client_id_t client_id, uint64_t authenticated_process_id) { frontend_eof_application_context_t *context = opaque; @@ -325,6 +344,9 @@ static cbm_daemon_runtime_application_status_t frontend_eof_application_request( *response_out = NULL; *response_length_out = 0; int request_index = atomic_fetch_add_explicit(&context->requests, 1, memory_order_relaxed); + if (atomic_load_explicit(&context->cancel_requests, memory_order_acquire)) { + return CBM_DAEMON_RUNTIME_APPLICATION_CANCELLED; + } if (request_index == 0 && atomic_load_explicit(&context->block_first_request, memory_order_acquire)) { atomic_store_explicit(&context->first_request_started, true, memory_order_release); @@ -464,6 +486,7 @@ static bool frontend_eof_fixture_start(frontend_eof_fixture_t *fixture, const ch atomic_init(&fixture->application.block_first_request, true); atomic_init(&fixture->application.first_request_started, false); atomic_init(&fixture->application.release_first_request, false); + atomic_init(&fixture->application.cancel_requests, false); fixture->application.request_observed_fd = -1; fixture->application.session_cancel_fd = -1; char key[CBM_DAEMON_KEY_SIZE]; @@ -477,14 +500,7 @@ static bool frontend_eof_fixture_start(frontend_eof_fixture_t *fixture, const ch } fixture->endpoint = cbm_daemon_ipc_endpoint_new(key, parent); fixture->manager = fixture->endpoint ? cbm_version_cohort_manager_new(fixture->endpoint) : NULL; - cbm_daemon_build_identity_t identity = { - .semantic_version = "2.4.0", - .build_fingerprint = build, - .cache_fingerprint = FRONTEND_TEST_CACHE, - .protocol_abi = 3, - .store_abi = 11, - .feature_abi = 7, - }; + cbm_daemon_build_identity_t identity = frontend_runtime_identity(build); cbm_daemon_runtime_application_callbacks_t callbacks = { .context = &fixture->application, .session_open = frontend_eof_application_session_open, @@ -579,7 +595,7 @@ static int frontend_eof_child_run(const char *parent, bool overflow) { cbm_daemon_runtime_client_t *frontend_client = fixture.client; fixture.client = NULL; /* cbm_daemon_frontend_mcp_run consumes it. */ - int result = cbm_daemon_frontend_mcp_run(frontend_client, fixture.manager, input, output); + int result = cbm_daemon_frontend_mcp_run(frontend_client, fixture.manager, input, output, NULL, NULL); bool joined = cbm_thread_join(&writer_thread) == 0; bool writer_ok = atomic_load_explicit(&writer.finished, memory_order_acquire) && atomic_load_explicit(&writer.succeeded, memory_order_acquire); @@ -660,6 +676,346 @@ static bool frontend_eof_run_isolated(const char *tag, bool overflow) { return child_ok && cleaned; } +/* #2796: a transport failure must move the session to a replacement daemon + * and re-issue the interrupted request; a daemon CANCELLED answer must reach + * the caller unchanged and must not trigger a reconnect; a supplier that + * cannot replace the client must fail-stop instead of hanging. */ +typedef enum { + FRONTEND_RECONNECT_OK, + FRONTEND_RECONNECT_CANCEL, + FRONTEND_RECONNECT_NULL, + FRONTEND_RECONNECT_RETURN_DEAD, + FRONTEND_RECONNECT_RETRY, +} frontend_reconnect_mode_t; + +typedef struct { + cbm_daemon_ipc_endpoint_t *endpoint; + cbm_daemon_build_identity_t identity; + char build[CBM_DAEMON_BUILD_FINGERPRINT_SIZE]; + char marker[FRONTEND_TEST_PATH_CAP]; + frontend_reconnect_mode_t mode; + /* RETRY: the first replacement is a client whose daemon is already gone; + * the second callback connects to endpoint instead. */ + cbm_daemon_runtime_client_t *retry_dead; + atomic_int calls; +} frontend_reconnect_callback_t; + +static void frontend_reconnect_mark(frontend_reconnect_callback_t *callback) { + int fd = open(callback->marker, O_WRONLY | O_CREAT | O_APPEND, 0600); + if (fd >= 0) { + (void)write(fd, "1", 1); + (void)close(fd); + } +} + +static cbm_daemon_runtime_client_t *frontend_reconnect_supply( + void *opaque, cbm_daemon_runtime_client_t *dead_client) { + frontend_reconnect_callback_t *callback = opaque; + int call = atomic_fetch_add_explicit(&callback->calls, 1, memory_order_relaxed); + if (callback->mode == FRONTEND_RECONNECT_NULL || + callback->mode == FRONTEND_RECONNECT_RETURN_DEAD) { + /* The worker fail-stops with _Exit, so the parent cannot read this + * counter; leave a marker the runner checks after waitpid. */ + frontend_reconnect_mark(callback); + return callback->mode == FRONTEND_RECONNECT_RETURN_DEAD ? dead_client : NULL; + } + if (callback->mode == FRONTEND_RECONNECT_RETRY && call == 0 && callback->retry_dead) { + cbm_daemon_runtime_client_t *client = callback->retry_dead; + callback->retry_dead = NULL; + return client; + } + if (callback->mode == FRONTEND_RECONNECT_RETRY && call != 1) { + return NULL; + } + cbm_daemon_runtime_connect_result_t result = {0}; + cbm_daemon_runtime_client_t *client = cbm_daemon_runtime_client_connect( + callback->endpoint, &callback->identity, FRONTEND_EOF_TEST_REQUEST_TIMEOUT_MS, &result); + if (!client || result.status != CBM_DAEMON_RUNTIME_CONNECT_ACCEPTED) { + if (client) { + (void)cbm_daemon_runtime_client_close(client, FRONTEND_EOF_TEST_REQUEST_TIMEOUT_MS); + } + return NULL; + } + return client; +} + +static bool frontend_reconnect_callback_init(frontend_reconnect_callback_t *callback, + cbm_daemon_ipc_endpoint_t *endpoint, + const char *marker, frontend_reconnect_mode_t mode) { + memset(callback, 0, sizeof(*callback)); + callback->endpoint = endpoint; + callback->mode = mode; + atomic_init(&callback->calls, 0); + int written = snprintf(callback->marker, sizeof(callback->marker), "%s", marker); + if (written <= 0 || written >= (int)sizeof(callback->marker) || + !cbm_daemon_runtime_process_build_fingerprint((uint64_t)getpid(), callback->build)) { + return false; + } + callback->identity = frontend_runtime_identity(callback->build); + return true; +} + +static bool frontend_reconnect_fixtures_finish(frontend_eof_fixture_t *first, + frontend_eof_fixture_t *replacement, + frontend_eof_fixture_t *second) { + bool ok = frontend_eof_fixture_finish(first); + ok = frontend_eof_fixture_finish(replacement) && ok; + if (second) { + ok = frontend_eof_fixture_finish(second) && ok; + } + return ok; +} + +static int frontend_reconnect_child_run(const char *parent, frontend_reconnect_mode_t mode) { + bool cancel_requests = mode == FRONTEND_RECONNECT_CANCEL; + bool retry_mode = mode == FRONTEND_RECONNECT_RETRY; + char parent_a[FRONTEND_TEST_PATH_CAP]; + char parent_b[FRONTEND_TEST_PATH_CAP]; + char parent_c[FRONTEND_TEST_PATH_CAP]; + /* AF_UNIX socket paths add cbm-daemon-/cbm-.sock below this + * parent; macOS sun_path is 104 bytes, so keep both components short. */ + int written = snprintf(parent_a, sizeof(parent_a), "%s/a", parent); + if (written <= 0 || written >= (int)sizeof(parent_a)) { + return 70; + } + written = snprintf(parent_b, sizeof(parent_b), "%s/b", parent); + if (written <= 0 || written >= (int)sizeof(parent_b)) { + return 70; + } + written = snprintf(parent_c, sizeof(parent_c), "%s/c", parent); + if (written <= 0 || written >= (int)sizeof(parent_c) || mkdir(parent_a, 0700) != 0 || + mkdir(parent_b, 0700) != 0 || (retry_mode && mkdir(parent_c, 0700) != 0)) { + return 71; + } + + frontend_eof_fixture_t first = {0}; + frontend_eof_fixture_t replacement = {0}; + frontend_eof_fixture_t second = {0}; + bool first_started = frontend_eof_fixture_start(&first, parent_a); + bool replacement_started = first_started && frontend_eof_fixture_start(&replacement, parent_b); + bool second_started = + replacement_started && (!retry_mode || frontend_eof_fixture_start(&second, parent_c)); + if (!second_started) { + (void)frontend_reconnect_fixtures_finish(&first, &replacement, retry_mode ? &second : NULL); + return 72; + } + atomic_store_explicit(&first.application.block_first_request, false, memory_order_release); + atomic_store_explicit(&replacement.application.block_first_request, false, + memory_order_release); + atomic_store_explicit(&first.application.cancel_requests, cancel_requests, + memory_order_release); + atomic_store_explicit(&replacement.application.cancel_requests, cancel_requests, + memory_order_release); + if (retry_mode) { + atomic_store_explicit(&second.application.block_first_request, false, memory_order_release); + atomic_store_explicit(&second.application.cancel_requests, false, memory_order_release); + } + + cbm_daemon_runtime_client_t *retry_dead = NULL; + bool services_stopped = true; + if (!cancel_requests) { + services_stopped = + cbm_daemon_runtime_service_stop(first.service, FRONTEND_EOF_TEST_REQUEST_TIMEOUT_MS) && + cbm_daemon_runtime_service_free(first.service); + if (services_stopped) { + first.service = NULL; + } + } + if (services_stopped && retry_mode) { + retry_dead = replacement.client; + services_stopped = cbm_daemon_runtime_service_stop(replacement.service, + FRONTEND_EOF_TEST_REQUEST_TIMEOUT_MS) && + cbm_daemon_runtime_service_free(replacement.service); + if (services_stopped) { + replacement.service = NULL; + replacement.client = NULL; /* the frontend owns the retry_dead handle */ + } + } + if (!services_stopped) { + (void)frontend_reconnect_fixtures_finish(&first, &replacement, retry_mode ? &second : NULL); + return 73; + } + + static const char request[] = + "{\"jsonrpc\":\"2.0\",\"id\":42,\"method\":\"reconnect-probe\"}\n"; + int input_pipe[2] = {-1, -1}; + bool request_written = false; + if (pipe(input_pipe) == 0) { + request_written = + write(input_pipe[1], request, sizeof(request) - 1) == (ssize_t)(sizeof(request) - 1); + (void)close(input_pipe[1]); + input_pipe[1] = -1; + } + FILE *input = request_written ? fdopen(input_pipe[0], "rb") : NULL; + if (!input) { + (void)close(input_pipe[0]); + } + FILE *output = tmpfile(); + char marker[FRONTEND_TEST_PATH_CAP]; + written = snprintf(marker, sizeof(marker), "%s/supplier-called", parent); + frontend_reconnect_callback_t callback; + cbm_daemon_ipc_endpoint_t *live_endpoint = retry_mode ? second.endpoint : replacement.endpoint; + bool callback_ready = input && output && written > 0 && written < (int)sizeof(marker) && + frontend_reconnect_callback_init(&callback, live_endpoint, marker, mode); + if (!callback_ready) { + if (input) { + (void)fclose(input); + } + if (output) { + (void)fclose(output); + } + if (retry_dead) { + /* The fixture no longer owns the extracted handle. */ + (void)cbm_daemon_runtime_client_close(retry_dead, FRONTEND_EOF_TEST_REQUEST_TIMEOUT_MS); + } + (void)frontend_reconnect_fixtures_finish(&first, &replacement, retry_mode ? &second : NULL); + return 74; + } + callback.retry_dead = retry_dead; + + cbm_daemon_runtime_client_t *frontend_client = first.client; + first.client = NULL; /* cbm_daemon_frontend_mcp_run consumes it. */ + int result = cbm_daemon_frontend_mcp_run(frontend_client, first.manager, input, output, + frontend_reconnect_supply, &callback); + if (callback.retry_dead) { + /* No transport failure consumed the spare replacement. */ + (void)cbm_daemon_runtime_client_close(callback.retry_dead, + FRONTEND_EOF_TEST_REQUEST_TIMEOUT_MS); + callback.retry_dead = NULL; + } + + bool rewound = fflush(output) == 0 && fseek(output, 0, SEEK_SET) == 0; + char response[4096]; + size_t response_length = rewound ? fread(response, 1, sizeof(response) - 1, output) : 0; + response[response_length] = '\0'; + bool streams_closed = fclose(input) == 0; + streams_closed = fclose(output) == 0 && streams_closed; + + int calls = atomic_load_explicit(&callback.calls, memory_order_acquire); + int first_requests = atomic_load_explicit(&first.application.requests, memory_order_acquire); + int replacement_requests = + atomic_load_explicit(&replacement.application.requests, memory_order_acquire); + int second_requests = + retry_mode ? atomic_load_explicit(&second.application.requests, memory_order_acquire) : 0; + bool fixtures_closed = + frontend_reconnect_fixtures_finish(&first, &replacement, retry_mode ? &second : NULL); + + const char *id_hit = strstr(response, "\"id\":42"); + bool single_response = id_hit && strstr(id_hit + 1, "\"id\":42") == NULL; + bool response_ok; + if (cancel_requests) { + response_ok = strstr(response, "Request cancelled") != NULL && single_response; + } else { + /* The fixture echoes the request body: requiring the unique method + * proves the final replacement daemon's OK payload reached the client, + * and the single id proves no duplicate/error frame was injected. */ + response_ok = strstr(response, "\"method\":\"reconnect-probe\"") != NULL && + strstr(response, "\"error\"") == NULL && single_response; + } + bool calls_ok; + bool requests_ok; + if (mode == FRONTEND_RECONNECT_CANCEL) { + calls_ok = calls == 0; + requests_ok = first_requests == 1 && replacement_requests == 0 && second_requests == 0; + } else if (mode == FRONTEND_RECONNECT_RETRY) { + calls_ok = calls == 2; + requests_ok = first_requests == 0 && replacement_requests == 0 && second_requests == 1; + } else if (mode == FRONTEND_RECONNECT_OK) { + calls_ok = calls == 1; + requests_ok = first_requests == 0 && replacement_requests == 1 && second_requests == 0; + } else { /* NULL: unreachable unless the fail-stop changes into a return */ + calls_ok = calls == 1; + requests_ok = false; + } + if (result == 0 && response_ok && calls_ok && requests_ok && streams_closed && + fixtures_closed) { + return 0; + } + (void)fprintf(stderr, + "frontend reconnect fixture: result=%d calls=%d first=%d replacement=%d " + "second=%d response_ok=%d streams=%d fixtures=%d response=%s\n", + result, calls, first_requests, replacement_requests, second_requests, response_ok, + streams_closed, fixtures_closed, response); + return 75; +} + +static bool frontend_reconnect_run_isolated(const char *tag, frontend_reconnect_mode_t mode) { + char parent[FRONTEND_TEST_PATH_CAP]; + int written = snprintf(parent, sizeof(parent), "%s/reconnect-XXXXXX", cbm_tmpdir()); + if (written <= 0 || written >= (int)sizeof(parent) || !cbm_mkdtemp(parent)) { + return false; + } + pid_t child = fork(); + if (child == 0) { + (void)signal(SIGALRM, SIG_DFL); + (void)alarm(FRONTEND_EOF_TEST_CATASTROPHIC_TIMEOUT_S); + _exit(frontend_reconnect_child_run(parent, mode)); + } + int status = 0; + pid_t waited; + do { + waited = child > 0 ? waitpid(child, &status, 0) : -1; + } while (waited < 0 && errno == EINTR); + char marker[FRONTEND_TEST_PATH_CAP]; + int marker_written = snprintf(marker, sizeof(marker), "%s/supplier-called", parent); + bool supplier_consulted = + marker_written > 0 && marker_written < (int)sizeof(marker) && access(marker, F_OK) == 0; + bool expect_failure = mode == FRONTEND_RECONNECT_NULL || mode == FRONTEND_RECONNECT_RETURN_DEAD; + bool marker_ok = !expect_failure || supplier_consulted; + bool cleaned = th_rmtree(parent) == 0; + int expected = expect_failure ? EXIT_FAILURE : EXIT_SUCCESS; + bool child_ok = child > 0 && waited == child && WIFEXITED(status) && + WEXITSTATUS(status) == expected && marker_ok; + if (!marker_ok) { + (void)fprintf(stderr, "frontend reconnect fixture %s: supplier was not consulted\n", tag); + } + if (!child_ok) { + if (child <= 0) { + (void)fprintf(stderr, "frontend reconnect fixture %s: fork failed\n", tag); + } else if (waited != child) { + (void)fprintf(stderr, "frontend reconnect fixture %s: waitpid failed errno=%d\n", tag, + errno); + } else if (WIFSIGNALED(status)) { + (void)fprintf(stderr, "frontend reconnect fixture %s: child signal=%d\n", tag, + WTERMSIG(status)); + } else if (WIFEXITED(status)) { + (void)fprintf(stderr, "frontend reconnect fixture %s: child exit=%d expected=%d\n", tag, + WEXITSTATUS(status), expected); + } else { + (void)fprintf(stderr, "frontend reconnect fixture %s: unexpected child status\n", tag); + } + } + if (!cleaned) { + (void)fprintf(stderr, "frontend reconnect fixture %s: cleanup failed\n", tag); + } + return child_ok && cleaned; +} + +TEST(daemon_frontend_reconnects_after_transport_failure) { + ASSERT_TRUE(frontend_reconnect_run_isolated("transport", FRONTEND_RECONNECT_OK)); + PASS(); +} + +TEST(daemon_frontend_does_not_retry_daemon_cancellation) { + ASSERT_TRUE(frontend_reconnect_run_isolated("cancelled", FRONTEND_RECONNECT_CANCEL)); + PASS(); +} + +TEST(daemon_frontend_fail_stops_without_a_replacement) { + ASSERT_TRUE(frontend_reconnect_run_isolated("no-supplier", FRONTEND_RECONNECT_NULL)); + PASS(); +} + +TEST(daemon_frontend_rejects_returning_the_dead_client) { + ASSERT_TRUE(frontend_reconnect_run_isolated("return-dead", FRONTEND_RECONNECT_RETURN_DEAD)); + PASS(); +} + +TEST(daemon_frontend_retries_across_two_replacements) { + ASSERT_TRUE(frontend_reconnect_run_isolated("retry", FRONTEND_RECONNECT_RETRY)); + PASS(); +} + static void frontend_test_release_lease(cbm_version_cohort_lease_t **lease) { while (lease && *lease && cbm_version_cohort_lease_release(lease) != CBM_PRIVATE_FILE_LOCK_OK) { cbm_usleep(1000); @@ -781,6 +1137,7 @@ static int frontend_backpressure_daemon_run(const char *parent, int ready_fd, in atomic_init(&application.block_first_request, false); atomic_init(&application.first_request_started, false); atomic_init(&application.release_first_request, false); + atomic_init(&application.cancel_requests, false); application.request_observed_fd = ready_fd; application.session_cancel_fd = cancel_fd; cbm_daemon_runtime_application_callbacks_t callbacks = { @@ -920,7 +1277,7 @@ static int frontend_backpressure_frontend_run(const char *parent, int input_fd, return 81; } - int result = cbm_daemon_frontend_mcp_run(client, manager, input, output); + int result = cbm_daemon_frontend_mcp_run(client, manager, input, output, NULL, NULL); bool joined = cbm_thread_join(&writer_thread) == 0; bool writer_ok = atomic_load_explicit(&writer.finished, memory_order_acquire) && atomic_load_explicit(&writer.succeeded, memory_order_acquire); @@ -1419,7 +1776,7 @@ TEST(daemon_frontend_maintenance_exits_while_stdio_reader_is_blocked) { _exit(70); } int result = cbm_daemon_frontend_mcp_run((cbm_daemon_runtime_client_t *)(uintptr_t)1, - manager, input, output); + manager, input, output, NULL, NULL); (void)result; _exit(71); } @@ -1677,7 +2034,98 @@ TEST(daemon_local_participant_monitor_joins_before_manager_teardown) { PASS(); } +TEST(daemon_frontend_liveness_treats_reconnect_as_progress) { + const uint64_t base = 1000; /* stand-in for the 15 s drain / 5 s watchdog */ + const uint64_t grace = CBM_DAEMON_FRONTEND_RECONNECT_GRACE_MS; + cbm_daemon_frontend_liveness_t liveness; + + /* A stuck request expires on the ordinary window. */ + cbm_daemon_frontend_liveness_init(&liveness, 0, 0); + ASSERT_TRUE(cbm_daemon_frontend_liveness_wait(&liveness, base - 1, 0, false, 0, base)); + ASSERT_FALSE(cbm_daemon_frontend_liveness_wait(&liveness, base, 0, false, 0, base)); + + /* A completed item restarts that window. */ + cbm_daemon_frontend_liveness_init(&liveness, 0, 0); + cbm_daemon_frontend_liveness_progress(&liveness, base - 1); + ASSERT_TRUE(cbm_daemon_frontend_liveness_wait(&liveness, base + base - 2, 0, false, 0, base)); + ASSERT_FALSE( + cbm_daemon_frontend_liveness_wait(&liveness, base + base - 1, 0, false, 0, base)); + + /* A spawn in progress stretches the deadline to its grace even when the + * attempt counter does not change (the first observation anchors it). */ + cbm_daemon_frontend_liveness_init(&liveness, 0, 1); + ASSERT_TRUE(cbm_daemon_frontend_liveness_wait(&liveness, base, 1, true, 0, base)); + ASSERT_TRUE(cbm_daemon_frontend_liveness_wait(&liveness, base + grace - 1, 1, true, 0, base)); + ASSERT_FALSE(cbm_daemon_frontend_liveness_wait(&liveness, base + grace, 1, true, 0, base)); + + /* The attempt's real start anchors the grace: a waiter that begins much + * later spends the same budget instead of anchoring a fresh one. */ + cbm_daemon_frontend_liveness_init(&liveness, 2 * grace, 1); + ASSERT_FALSE( + cbm_daemon_frontend_liveness_wait(&liveness, 2 * grace + base, 1, true, base, base)); + + /* The attempt ended: the retried exchange gets a fresh ordinary window + * instead of inheriting the spawn's remainder. */ + cbm_daemon_frontend_liveness_init(&liveness, 0, 0); + ASSERT_TRUE(cbm_daemon_frontend_liveness_wait(&liveness, 1, 1, true, 0, base)); + ASSERT_TRUE(cbm_daemon_frontend_liveness_wait(&liveness, 2, 1, false, 0, base)); + ASSERT_TRUE(cbm_daemon_frontend_liveness_wait(&liveness, 2 + base - 1, 1, false, 0, base)); + ASSERT_FALSE(cbm_daemon_frontend_liveness_wait(&liveness, 2 + base, 1, false, 0, base)); + + /* An attempt that began and ended between polls still restarts the + * window when its counter becomes visible. */ + cbm_daemon_frontend_liveness_init(&liveness, 0, 0); + ASSERT_TRUE(cbm_daemon_frontend_liveness_wait(&liveness, 10, 1, false, 0, base)); + ASSERT_TRUE(cbm_daemon_frontend_liveness_wait(&liveness, 10 + base - 1, 1, false, 0, base)); + ASSERT_FALSE(cbm_daemon_frontend_liveness_wait(&liveness, 10 + base, 1, false, 0, base)); + + /* A waiter initialized with a nonzero attempt count must not mistake its + * first poll for a counter change and restart the window. */ + cbm_daemon_frontend_liveness_init(&liveness, 0, 3); + ASSERT_TRUE(cbm_daemon_frontend_liveness_wait(&liveness, base - 1, 3, false, 0, base)); + ASSERT_FALSE(cbm_daemon_frontend_liveness_wait(&liveness, base, 3, false, 0, base)); + + /* Ordinary progress does not extend the hard per-attempt grace. */ + cbm_daemon_frontend_liveness_init(&liveness, 0, 1); + ASSERT_TRUE(cbm_daemon_frontend_liveness_wait(&liveness, base, 1, true, 0, base)); + cbm_daemon_frontend_liveness_progress(&liveness, base + 10); + ASSERT_FALSE(cbm_daemon_frontend_liveness_wait(&liveness, base + grace, 1, true, 0, base)); + + /* A new attempt that becomes visible while the previous one was observed + * restarts the grace at its own start instead of inheriting the old + * deadline. The new observation is deliberately after the old deadline. */ + cbm_daemon_frontend_liveness_init(&liveness, 0, 0); + ASSERT_TRUE(cbm_daemon_frontend_liveness_wait(&liveness, base, 1, true, 0, base)); + ASSERT_TRUE(cbm_daemon_frontend_liveness_wait(&liveness, base + grace + 1, 2, true, + base + grace + 1, base)); + ASSERT_FALSE(cbm_daemon_frontend_liveness_wait(&liveness, base + 2 * grace + 1, 2, true, + base + grace + 1, base)); + + /* A start newer than the sampled `now` (the reader raced the attempt's + * timestamp) falls back to `now` rather than moving the deadline. */ + cbm_daemon_frontend_liveness_init(&liveness, 0, 1); + ASSERT_TRUE(cbm_daemon_frontend_liveness_wait(&liveness, 100, 1, true, 200, base)); + ASSERT_FALSE(cbm_daemon_frontend_liveness_wait(&liveness, 100 + grace, 1, true, 200, base)); + + /* The late re-sample grants the attempt its grace and is idempotent: a + * repeated call must not re-anchor a hung spawn. */ + cbm_daemon_frontend_liveness_init(&liveness, 0, 0); + ASSERT_FALSE(cbm_daemon_frontend_liveness_wait(&liveness, base, 0, false, 0, base)); + ASSERT_TRUE(cbm_daemon_frontend_liveness_reconnect_late(&liveness, base, 1)); + ASSERT_FALSE(cbm_daemon_frontend_liveness_reconnect_late(&liveness, base + 1, 1)); + ASSERT_TRUE(cbm_daemon_frontend_liveness_wait(&liveness, 1 + grace - 1, 0, true, 0, base)); + ASSERT_FALSE(cbm_daemon_frontend_liveness_wait(&liveness, 1 + grace, 0, true, 0, base)); + + /* The deadline arithmetic saturates instead of wrapping. */ + cbm_daemon_frontend_liveness_init(&liveness, UINT64_MAX - 10, 0); + ASSERT_TRUE(cbm_daemon_frontend_liveness_wait(&liveness, UINT64_MAX - 1, 0, false, 0, base)); + ASSERT_FALSE(cbm_daemon_frontend_liveness_wait(&liveness, UINT64_MAX, 0, false, 0, base)); + + PASS(); +} + SUITE(daemon_frontend) { + RUN_TEST(daemon_frontend_liveness_treats_reconnect_as_progress); RUN_TEST(daemon_frontend_recognizes_exact_cancellation_notification); RUN_TEST(daemon_frontend_correlates_cancellation_to_exact_request); RUN_TEST(daemon_frontend_cancelled_index_reply_offers_async_issue2144); @@ -1692,6 +2140,11 @@ SUITE(daemon_frontend) { RUN_TEST(daemon_local_participant_monitor_allows_supervisor_containment_window); RUN_TEST(daemon_frontend_over_capacity_input_backpressures_without_loss); RUN_TEST(daemon_frontend_eof_drain_timeout_cancels_and_returns_success); + RUN_TEST(daemon_frontend_reconnects_after_transport_failure); + RUN_TEST(daemon_frontend_does_not_retry_daemon_cancellation); + RUN_TEST(daemon_frontend_fail_stops_without_a_replacement); + RUN_TEST(daemon_frontend_rejects_returning_the_dead_client); + RUN_TEST(daemon_frontend_retries_across_two_replacements); RUN_TEST(daemon_frontend_stdout_backpressure_eof_fail_stops_and_cancels_session); RUN_TEST(daemon_frontend_stdout_backpressure_maintenance_stops_and_cancels_session); #endif diff --git a/tests/test_subprocess.c b/tests/test_subprocess.c index c7f78b3d87..b8a0817756 100644 --- a/tests/test_subprocess.c +++ b/tests/test_subprocess.c @@ -1407,6 +1407,20 @@ TEST(subprocess_child_close_fds_uses_one_syscall_not_rlimit_loop) { #endif } +TEST(subprocess_fd_close_cap_is_bounded_for_fork_children) { +#ifdef _WIN32 + SKIP_PLATFORM("POSIX descriptor sweep cap; Windows uses a handle allow-list"); +#else + long cap = cbm_posix_fd_close_cap(); + /* The cap bounds a close() sweep in the fork child. A non-positive or + * oversized value would either skip the sweep (leaking inherited + * descriptors into the exec'd daemon) or make the int loop wrap. */ + ASSERT_TRUE(cap > STDERR_FILENO); + ASSERT_TRUE(cap <= 1048576L); + PASS(); +#endif +} + TEST(subprocess_root_exit_drains_surviving_descendant) { #ifdef _WIN32 SKIP_PLATFORM("POSIX process-group descendant probe; native Windows coverage pending"); @@ -2658,6 +2672,7 @@ SUITE(subprocess) { RUN_TEST(subprocess_final_log_drain_error_is_terminal_and_preserves_classification); RUN_TEST(subprocess_posix_child_closes_unrelated_descriptors); RUN_TEST(subprocess_child_close_fds_uses_one_syscall_not_rlimit_loop); + RUN_TEST(subprocess_fd_close_cap_is_bounded_for_fork_children); RUN_TEST(subprocess_root_exit_drains_surviving_descendant); RUN_TEST(subprocess_strip_git_repo_env_is_per_child); RUN_TEST(popen_git_strips_repo_env_and_reports_exit_status);