Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
18 changes: 18 additions & 0 deletions crates/tracedecay/tests/common/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -783,11 +783,29 @@ impl Drop for TestChildProcess {
}

/// PID-directed stop: survives `process_group(0)` / `setsid` detachment.
///
/// The child is the leader of its own group. Killing only that pid leaves
/// helper children that still hold the listen socket, so the next spawn
/// observes a connectable daemon after this process has already been reaped.
fn terminate_and_reap(child: &mut Child) -> std::io::Result<ExitStatus> {
if let Ok(Some(status)) = child.try_wait() {
return Ok(status);
}

#[cfg(unix)]
{
let pid = child.id();
if pid != 0 {
// SAFETY: `pid` is this live child. Negating it targets the
// process group `process_group(0)` created with that pid as
// leader. ESRCH is ignored: setpgid may not have run yet, and the
// pid kill below still stops the leader.
unsafe {
libc::kill(-(pid as i32), libc::SIGKILL);
}
}
}

if let Err(kill_err) = child.kill() {
if let Some(status) = child.try_wait()? {
return Ok(status);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -1005,13 +1005,25 @@ async fn packaged_host_ingest_delivers_a_registered_advisory_cycle() {
.expect("registered daemon ingest response text"),
)
.expect("registered daemon ingest payload");
if payload["completed"] != false {
// `completed: true` with `accepted_for_replay` means the catch-up
// sweep has not yet drained this admission. That is not a durable
// commit, so keep polling until a terminal that proves the
// transcript, or the deadline reports the last payload.
if matches!(
payload["status"].as_str(),
Some("committed" | "exact_duplicate")
) {
break output;
}
assert_eq!(
payload["admission"]["retryable"], true,
"incomplete ingest must carry a retryable admission: {response}"
);
if payload["completed"] != false && payload["status"] != "accepted_for_replay" {
break output;
}
if payload["completed"] == false {
assert_eq!(
payload["admission"]["retryable"], true,
"incomplete ingest must carry a retryable admission: {response}"
);
}
} else {
let stderr = String::from_utf8_lossy(&output.stderr).into_owned();
assert!(
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -377,6 +377,62 @@ fn problem_envelope(payload: &Value, context: &str) -> Value {
panic!("{context}: no typed problem envelope in the payload: {payload}")
}

/// True when the daemon has not published the project open yet.
///
/// The open wait on a connection is 500 ms. Past that, the surface answers
/// `unavailable` / `after_delay` and leaves the open running. The CLI rides
/// that refusal out; a raw HTTP or SDK call does not. A restart's first packet
/// can therefore be warming even when the store's recorded terminal is
/// `reset_required`.
fn retryable_pre_admission_unavailable(payload: &Value) -> bool {
let problem = &payload["problem"];
problem["kind"] == "unavailable"
&& problem["retry"] == "after_delay"
&& problem["terminality"] == "pre_admission"
}

/// Polls until the open publishes a non-retryable problem, then returns it.
///
/// Bounded by the same 15 s a CLI tool call gives a cold open. A refusal that
/// stays retryable past that bound is a real failure, not a slow open.
fn await_settled_problem(context: &str, mut fetch: impl FnMut() -> Value) -> Value {
let deadline = Instant::now() + Duration::from_secs(15);
loop {
let payload = fetch();
if !retryable_pre_admission_unavailable(&payload) {
return payload;
}
assert!(
Instant::now() < deadline,
"{context}: project open stayed retryable unavailable past the open grace: {payload}"
);
std::thread::sleep(Duration::from_millis(50));
}
}

fn await_sdk_reset_problem(
context: &str,
client: &Client,
request: &<ApplicationStorageStatus as tracedecay_sdk::operations::TypedOperation>::Request,
) -> (String, Value) {
let deadline = Instant::now() + Duration::from_secs(15);
loop {
let error = client
.execute::<ApplicationStorageStatus>(request)
.expect_err("a refused store must not read as a healthy status");
let (kind, envelope) = sdk_problem(error, context);
let payload = problem_envelope(&envelope, context);
if !retryable_pre_admission_unavailable(&payload) {
return (kind, envelope);
}
assert!(
Instant::now() < deadline,
"{context}: project open stayed retryable unavailable past the open grace: {payload}"
);
std::thread::sleep(Duration::from_millis(50));
}
}

/// Arms the daemon's one-shot fact-commit barrier, runs `request` on its own
/// thread, holds the committed effect there until the request's own deadline
/// has certainly expired, then releases it and returns what `request` produced.
Expand Down Expand Up @@ -665,25 +721,23 @@ fn reset_required_survives_http_mcp_and_rust_sdk_across_restart() {
"a typed HTTP terminal must not be reported as success: status {http_status}, body {http_body}"
);

let mcp_response = mcp_tool_call(
&home_path,
&project_path,
"tracedecay_storage_status",
&storage_status_body,
None,
);
super::assert_reset_required(
&problem_envelope(&mcp_payload(&mcp_response), "MCP reset required"),
"MCP stdio host, first observation",
);
let mcp_problem = await_settled_problem("MCP stdio host, first observation", || {
let mcp_response = mcp_tool_call(
&home_path,
&project_path,
"tracedecay_storage_status",
&storage_status_body,
None,
);
problem_envelope(&mcp_payload(&mcp_response), "MCP reset required")
});
super::assert_reset_required(&mcp_problem, "MCP stdio host, first observation");

let client = sdk_client(&mount, &identity);
let request =
serde_json::from_value(storage_status_body.clone()).expect("canonical storage status");
let sdk_error = client
.execute::<ApplicationStorageStatus>(&request)
.expect_err("a refused store must not read as a healthy status");
let (sdk_kind, sdk_envelope) = sdk_problem(sdk_error, "Rust SDK reset required");
let (sdk_kind, sdk_envelope) =
await_sdk_reset_problem("Rust SDK, first observation", &client, &request);
assert_eq!(
sdk_kind, "reset_required",
"the Rust SDK must classify the terminal as reset required: {sdk_envelope}"
Expand All @@ -708,36 +762,37 @@ fn reset_required_survives_http_mcp_and_rust_sdk_across_restart() {
);
let mount = http_mount(&home_path);

let (_, http_body_after) = post_application(
&mount,
&identity,
STORAGE_STATUS_ROUTE,
&storage_status_body,
None,
);
super::assert_reset_required(
&problem_envelope(&http_body_after, "HTTP reset required after restart"),
"HTTP mount, after a physical restart",
);

let mcp_after = mcp_tool_call(
&home_path,
&project_path,
"tracedecay_storage_status",
&storage_status_body,
None,
);
let http_problem_after = await_settled_problem("HTTP mount, after a physical restart", || {
let (_, body) = post_application(
&mount,
&identity,
STORAGE_STATUS_ROUTE,
&storage_status_body,
None,
);
problem_envelope(&body, "HTTP reset required after restart")
});
super::assert_reset_required(&http_problem_after, "HTTP mount, after a physical restart");

let mcp_problem_after =
await_settled_problem("MCP stdio host, after a physical restart", || {
let mcp_after = mcp_tool_call(
&home_path,
&project_path,
"tracedecay_storage_status",
&storage_status_body,
None,
);
problem_envelope(&mcp_payload(&mcp_after), "MCP reset required after restart")
});
super::assert_reset_required(
&problem_envelope(&mcp_payload(&mcp_after), "MCP reset required after restart"),
&mcp_problem_after,
"MCP stdio host, after a physical restart",
);

let client = sdk_client(&mount, &identity);
let sdk_error_after = client
.execute::<ApplicationStorageStatus>(&request)
.expect_err("a refused store must not read as a healthy status after a restart");
let (sdk_kind_after, sdk_envelope_after) =
sdk_problem(sdk_error_after, "Rust SDK reset required after restart");
await_sdk_reset_problem("Rust SDK reset required after restart", &client, &request);
assert_eq!(
sdk_kind_after, "reset_required",
"the Rust SDK must keep classifying the terminal as reset required: {sdk_envelope_after}"
Expand Down
Loading