From 307c3ffbc531abd672fa503b4a883383d1c66818 Mon Sep 17 00:00:00 2001 From: Jon Currey Date: Thu, 17 Sep 2026 15:36:22 -0400 Subject: [PATCH] testdrive: bound the prepare by the query timeout try_run_sql wraps the query in tokio::time::timeout(state.timeout) but prepares the statement first, outside it. When an attempt times out, dropping the query future does not stop the statement on the server, and tokio-postgres serializes statements on a connection, so the next attempt's prepare waits behind the stuck statement with no deadline. Release-qualification 1373, "Checks 0dt upgrade to a bumped version": the SELECT over alter_index_source in AlterIndex.validate() timed out after the 300 s platform-checks budget at 23:41 UTC, and the job then produced nothing until Buildkite cancelled it at 02:51. The retry loop is bounded by max_duration(state.timeout); the prepare it called was not (QAR-167). Wrap the prepare in the same timeout, so a connection held by a stuck statement fails the attempt within the budget and the failure is reported against the query rather than as a silent step timeout. The message names the phase so the two timeouts are distinguishable in a log. Co-Authored-By: Claude Fable 5.1 --- src/testdrive/src/action/sql.rs | 21 +++++++++++++++------ 1 file changed, 15 insertions(+), 6 deletions(-) diff --git a/src/testdrive/src/action/sql.rs b/src/testdrive/src/action/sql.rs index a7f09ce1d437e..33911d8bffa71 100644 --- a/src/testdrive/src/action/sql.rs +++ b/src/testdrive/src/action/sql.rs @@ -245,12 +245,21 @@ async fn try_run_sql( raw_output: bool, should_retry: bool, ) -> Result<(), anyhow::Error> { - let stmt = state - .materialize - .pgclient - .prepare(query) - .await - .context("preparing query failed")?; + // The prepare shares the connection with any statement a previous attempt + // left running on the server after its future was dropped, and + // tokio-postgres serializes statements on a connection, so without a + // deadline it waits behind that statement for as long as the server keeps + // it: on a dataflow that never produces, that is until the step is killed. + // Bound it by the same budget as the query itself. + let stmt = match tokio::time::timeout( + state.timeout.clone(), + state.materialize.pgclient.prepare(query), + ) + .await + { + Ok(stmt) => stmt.context("preparing query failed")?, + Err(_) => bail!("preparing query timed out\n"), + }; let query_with_timeout = tokio::time::timeout( state.timeout.clone(),