feat(547): in-flight slot expiry, backend.daemon.reclaim and typed admission reasons (slices 5–6)

control/crates/layerfs-control/tests/kernel_application.rs

@@ -56,6 +56,7 @@ fn actual_daemon_install_hello_bind_status_no_constructor_and_session_end() {
             connections: 2,
             handshake_ms: 3000,
             idle_ttl_ms: 60_000,
+            request_deadline_ms: 60_000,
             cache_bytes: 64 * 1024,
             namespaces: 8,
             pager_kib: 1024,

control/crates/layerfs-control/tests/observed_application.rs

@@ -78,6 +78,7 @@ fn configure(f: &support::Fixture, deployment: &Deployment) -> PathBuf {
             connections: 4,
             handshake_ms: 3000,
             idle_ttl_ms: 60_000,
+            request_deadline_ms: 60_000,
             cache_bytes: 64 * 1024,
             namespaces: 8,
             pager_kib: 1024,

control/crates/layerfs-control/tests/provider_ownership.rs

@@ -109,6 +109,7 @@ fn native_create(
             connections: 2,
             handshake_ms: 1000,
             idle_ttl_ms: 60_000,
+            request_deadline_ms: 60_000,
             cache_bytes: 65536,
             namespaces: 2,
             pager_kib: 1024,

control/crates/layerfs-control/tests/r5_1_actual_daemon_commit.rs

@@ -118,6 +118,7 @@ fn start(f: &support::Fixture, mounts: &Path) -> (Process, SocketAddr) {
             connections: 4,
             handshake_ms: 3000,
             idle_ttl_ms: 60_000,
+            request_deadline_ms: 60_000,
             cache_bytes: 64 * 1024,
             namespaces: 8,
             pager_kib: 1024,

control/crates/layerfs-sandbox/src/daemon/operator.rs

@@ -3,9 +3,10 @@ use crate::{internal::Control, SdkError, SdkResult};
 use layerfs_bridge::control::{SdkRefusal, SdkValue};
 
 /// Calls one daemon backend method: the read model that lists its
-/// Workspaces (`sandbox.list_workspaces`), or the operator view of control
+/// Workspaces (`sandbox.list_workspaces`), the operator view of control
 /// admission and retained connection failures
-/// (`backend.daemon.connection_failures`, #540). Any other method is refused
+/// (`backend.daemon.connection_failures`, #540), or the operator reclaim
+/// (`backend.daemon.reclaim`, #547 slice 5). Any other method is refused
 /// before it is sent.
 pub fn call(control: &mut Control, method: &str, args: SdkValue) -> SdkResult {
     match method {
@@ -14,6 +15,11 @@ pub fn call(control: &mut Control, method: &str, args: SdkValue) -> SdkResult {
         layerfs_bridge::control::OPERATION => {
             crate::daemon::call::exchange_call(control, method, args)
         }
+        // One generation-checked, one-shot reclaim of a named worker (#547
+        // slice 5): never restarts the daemon and never touches Workspaces.
+        layerfs_bridge::control::RECLAIM => {
+            crate::daemon::call::exchange_call(control, method, args)
+        }
         "sandbox.list_workspaces" => crate::daemon::call::exchange_call(control, method, args),
         _ => Err(SdkError::Refused(SdkRefusal {
             code: "UnknownMethod".into(),

control/crates/layerfs-sandbox/src/daemon/pool.rs

@@ -51,61 +51,53 @@ pub const ADMISSION_WAIT: Duration = Duration::from_secs(60);
 /// before it ends, and its `Busy` refusal is not retried.
 pub const LEND_WAIT: Duration = ADMISSION_WAIT;
 
-/// Why an open ended before the daemon's Hello reply. Every one means the
-/// call was not admitted and nothing was attempted, so opening again is
-/// safe; they name different observable causes (#540).
-#[derive(Clone, Copy, Debug, Eq, PartialEq)]
-enum NotAdmitted {
-    /// The daemon had no vacant pool slot. `retained` is exact when the
-    /// advisory capacity notice arrived, and unknown when only EOF was seen.
-    NoVacantSlot { retained: Option<u8> },
-    /// The connection itself failed before Hello (`BrokenPipe`,
-    /// `ConnectionReset`).
-    ConnectionFailed,
-}
-
-impl NotAdmitted {
-    fn code(self) -> &'static str {
-        match self {
-            Self::NoVacantSlot { .. } => "Capacity",
-            Self::ConnectionFailed => "Unavailable",
-        }
-    }
-
-    fn admission(self) -> &'static str {
-        match self {
-            Self::NoVacantSlot { .. } => "no-vacant-slot",
-            Self::ConnectionFailed => "connection-failed",
-        }
-    }
-}
-
-/// Classify a failed open that never reached the Hello reply. Through a port
-/// proxy the client reads the end (`UnexpectedEof`, a clean close by the
-/// daemon's accept loop); dialing the daemon directly, its handshake write can
-/// meet the closed socket first (`BrokenPipe`, `ConnectionReset`). The old
-/// message claimed "other processes hold all 8 connections"; that is not
-/// knowledge the client has (#540).
-fn not_admitted(refusal: &SdkRefusal) -> Option<(NotAdmitted, &'static str)> {
-    if refusal.code != "Failed" {
-        return None;
-    }
-    if refusal.message.contains("UnexpectedEof") {
-        return Some((
-            NotAdmitted::NoVacantSlot { retained: None },
-            "UnexpectedEof",
-        ));
-    }
-    for kind in ["BrokenPipe", "ConnectionReset"] {
-        if refusal.message.contains(kind) {
-            return Some((NotAdmitted::ConnectionFailed, kind));
+/// The exact refusal for a failed open that never reached the Hello reply,
+/// typed from the channel error rather than from refusal message text (#547
+/// slice 6). Nothing was attempted, so every one is safe to retry; they name
+/// different observable causes.
+fn opening_refusal(error: ChannelError) -> SdkRefusal {
+    match error {
+        // The daemon closed the connection before the handshake completed and
+        // sent no notice: the reason it did not admit this call is unknown, so
+        // the refusal says what was observed, never a guess at "no vacant
+        // slot" (#547 slice 6).
+        ChannelError::Io(io) if io.kind() == std::io::ErrorKind::UnexpectedEof => SdkRefusal {
+            code: "Unknown".to_owned(),
+            message: "the Sandbox's daemon closed the connection before it was admitted; the reason is unknown and the call was not admitted. Retrying is safe"
+                .to_owned(),
+            detail: SdkValue::map([
+                ("observed", SdkValue::text("UnexpectedEof")),
+                ("retryable", SdkValue::Bool(true)),
+                ("retry_after", SdkValue::Int(500)),
+            ]),
+        },
+        // Dialing the daemon directly, its handshake write can meet the closed
+        // socket first: the connection itself failed, not admission.
+        ChannelError::Io(io)
+            if matches!(
+                io.kind(),
+                std::io::ErrorKind::ConnectionReset | std::io::ErrorKind::BrokenPipe
+            ) =>
+        {
+            let kind = io.kind().to_string();
+            SdkRefusal {
+                code: "Unavailable".to_owned(),
+                message: format!(
+                    "the connection to the Sandbox's daemon failed before it was admitted ({kind}); nothing was attempted. Retrying is safe"
+                ),
+                detail: SdkValue::map([
+                    ("observed", SdkValue::text(kind)),
+                    ("retryable", SdkValue::Bool(true)),
+                    ("retry_after", SdkValue::Int(500)),
+                ]),
+            }
         }
+        other => failed(other),
     }
-    None
 }
 
-/// The exact refusal for a decoded advisory capacity notice. This is the
-/// precise form: it names the resource and the retained record count.
+/// The exact refusal for a decoded advisory capacity notice: it names the
+/// resource, the vacant slots and how long to wait (#540, #547 slice 6).
 fn notice_refusal(notice: AdmissionNotice) -> SdkRefusal {
     match notice.reason {
         AdmissionReason::NoPoolSlot => SdkRefusal {
@@ -123,6 +115,12 @@ fn notice_refusal(notice: AdmissionNotice) -> SdkRefusal {
                     SdkValue::Int(i64::from(notice.reserved_vacant)),
                 ),
                 ("retryable", SdkValue::Bool(true)),
+                // An idle pool slot becomes reclaimable after the daemon's
+                // `idle_ttl`, so retry after that, not sooner.
+                (
+                    "retry_after",
+                    SdkValue::Int(i64::from(crate::lifecycle::IDLE_TTL_MS)),
+                ),
                 ("admission", SdkValue::text(notice.reason.as_str())),
             ]),
         },
@@ -134,58 +132,34 @@ fn notice_refusal(notice: AdmissionNotice) -> SdkRefusal {
                 ("resource", SdkValue::Null),
                 ("retained", SdkValue::Null),
                 ("retryable", SdkValue::Bool(true)),
+                ("retry_after", SdkValue::Int(500)),
                 ("admission", SdkValue::text("starting")),
             ]),
         },
-    }
-}
-
-/// The exact refusal after `wait`: what the client observed, not a guess at
-/// what another process is doing.
-fn admission_refusal(wait: Duration, admission: NotAdmitted, kind: &str) -> SdkRefusal {
-    let message = match admission {
-        NotAdmitted::NoVacantSlot { retained } => format!(
-            "the Sandbox's daemon admitted no control connection for {}s: it closed the connection before Hello ({kind}), which it does when it has no vacant control slot{}. The call was not admitted; retrying is safe",
-            wait.as_secs(),
-            retained.map_or(String::new(), |count| format!(
-                " (it reported {count} retained failure record(s))"
-            ))
-        ),
-        NotAdmitted::ConnectionFailed => format!(
-            "the connection to the Sandbox's daemon failed before Hello for {}s ({kind}): the call was not admitted; retrying is safe",
-            wait.as_secs()
-        ),
-    };
-    let retained = match admission {
-        NotAdmitted::NoVacantSlot { retained } => {
-            retained.map_or(SdkValue::Null, |count| SdkValue::Int(i64::from(count)))
-        }
-        _ => SdkValue::Null,
-    };
-    SdkRefusal {
-        code: admission.code().to_owned(),
-        message,
-        detail: SdkValue::map([
-            (
-                "resource",
-                match admission {
-                    NotAdmitted::NoVacantSlot { .. } => SdkValue::text("daemon_control_slot"),
-                    _ => SdkValue::Null,
-                },
+        // A newer daemon's reason this client does not know: report what was
+        // observed, never guess (#547 slice 6 version skew).
+        other => SdkRefusal {
+            code: "Unknown".to_owned(),
+            message: format!(
+                "the Sandbox's daemon refused admission with a reason this client does not know ({:?}); the call was not admitted. Retrying is safe",
+                other
             ),
-            ("retained", retained),
-            ("retryable", SdkValue::Bool(true)),
-            ("admission", SdkValue::text(admission.admission())),
-            ("observed", SdkValue::text(kind)),
-        ]),
+            detail: SdkValue::map([
+                ("observed", SdkValue::text(other.as_str())),
+                ("retryable", SdkValue::Bool(true)),
+                ("retry_after", SdkValue::Int(500)),
+            ]),
+        },
     }
 }
 
 /// Whether this refusal says nothing was attempted and a later open may
 /// succeed. Only these exact refusals are retried.
 fn retryable(refusal: &SdkRefusal) -> bool {
-    matches!(refusal.code.as_str(), "Capacity" | "Unavailable")
-        && refusal.detail.get_bool("retryable")
+    matches!(
+        refusal.code.as_str(),
+        "Capacity" | "Unavailable" | "Unknown"
+    ) && refusal.detail.get_bool("retryable")
 }
 
 impl Dial {
@@ -224,7 +198,7 @@ impl Dial {
         let connection = match initiate_observed(stream, &self.secret, self.peer).result {
             Ok(connection) => connection,
             Err(ChannelError::Admission(notice)) => return Err(notice_refusal(notice)),
-            Err(error) => return Err(failed(error)),
+            Err(error) => return Err(opening_refusal(error)),
         };
         // the waits stay: the caller makes exactly one call and drops it
         let mut control = Control::new(connection);
@@ -249,17 +223,12 @@ impl Dial {
         loop {
             match self.open() {
                 Err(refusal) => {
-                    let legacy = not_admitted(&refusal);
-                    let may_retry = retryable(&refusal) || legacy.is_some();
-                    if may_retry && started.elapsed() < wait {
+                    if retryable(&refusal) && started.elapsed() < wait {
                         std::thread::sleep(pause);
                         pause = (pause * 2).min(Duration::from_millis(500));
                         continue;
                     }
-                    if let Some((admission, kind)) = legacy {
-                        return Err(admission_refusal(wait, admission, kind));
-                    }
-                    // A notice-based or transport refusal is already exact.
+                    // The refusal is already typed and exact (#547 slice 6).
                     return Err(refusal);
                 }
                 other => return other,
@@ -280,7 +249,7 @@ impl Dial {
         let connection = match initiate_observed(stream, &self.secret, self.peer).result {
             Ok(connection) => connection,
             Err(ChannelError::Admission(notice)) => return Err(notice_refusal(notice)),
-            Err(error) => return Err(failed(error)),
+            Err(error) => return Err(opening_refusal(error)),
         };
         connection.clear_io_waits().map_err(failed)?;
         let mut control = Control::new(connection);

control/crates/layerfs-sandbox/src/daemon/values.rs

@@ -24,6 +24,23 @@ pub fn refusal(failure: &OperationFailure) -> Option<SdkRefusal> {
             message: refused.detail.clone(),
             detail: SdkValue::map([("phase", SdkValue::text(refused.phase.as_str()))]),
         }),
+        // The request was sent but no reply arrived (the daemon reclaimed or
+        // expired the worker, or it died mid-operation): its effect is
+        // unknown, so the caller observes `Unavailable { outcome: unknown }`
+        // rather than a silent failure (#547 slice 5).
+        OperationCause::Exchange(exchange)
+            if exchange.phase == crate::daemon::internal::ControlPhase::Receive =>
+        {
+            Some(SdkRefusal {
+                code: "Unavailable".to_owned(),
+                message: "the Sandbox's daemon ended this connection before it answered; the operation's outcome is unknown"
+                    .to_owned(),
+                detail: SdkValue::map([
+                    ("outcome", SdkValue::text("unknown")),
+                    ("retryable", SdkValue::Bool(false)),
+                ]),
+            })
+        }
         _ => None,
     }
 }

control/crates/layerfs-sandbox/src/lifecycle.rs

@@ -49,6 +49,11 @@ pub const DAEMON_CONNECTIONS: u16 = 8;
 /// connection is likely abandoned, short enough that recovery stays bounded.
 pub const IDLE_TTL_MS: u32 = 60_000;
 
+/// A decoded request still in flight this long is expired (#547 slice 5):
+/// long enough that a 64 MiB Commit finishes normally, short enough that a
+/// fence/Store/diagnostic wait cannot hold a pool slot forever.
+pub const REQUEST_DEADLINE_MS: u32 = 300_000;
+
 pub fn setup(
     private: [u8; 32],
     peer: [u8; 32],
@@ -68,6 +73,7 @@ pub fn setup(
             connections: DAEMON_CONNECTIONS,
             handshake_ms: 2000,
             idle_ttl_ms: IDLE_TTL_MS,
+            request_deadline_ms: REQUEST_DEADLINE_MS,
             // The daemon's shared immutable-object cache: a fixed bound, the
             // daemon's own default. 64 KiB held too few directory and inode
             // pages for lookups in a large directory to stay resident

engine/crates/layerfs-bridge/src/control.rs

@@ -165,7 +165,7 @@ fn boolean(value: u8) -> Result<bool, ControlError> {
 }
 
 pub use crate::control_capacity::{
-    AdmissionModel, CapacityModel, RetainedFailureModel, RetainedModel, OPERATION,
+    AdmissionModel, CapacityModel, RetainedFailureModel, RetainedModel, OPERATION, RECLAIM,
 };
 pub use crate::control_forced::{
     AbortDisposition, CommitKnowledge, DetachDisposition, ForceUnmounted, ForcedCleanup,