From 6ac8ba20d7983ef3e02bdad37f7c8e642b388ad9 Mon Sep 17 00:00:00 2001 From: Andrey Vasiliev Date: Wed, 29 Jul 2026 13:28:40 +0300 Subject: [PATCH 1/4] feat: auto-continue after server overload --- codex-rs/tui/src/chatwidget.rs | 35 ++++++++++++++----- codex-rs/tui/src/chatwidget/constructor.rs | 3 +- codex-rs/tui/src/chatwidget/input_flow.rs | 17 ++++++--- .../tui/src/chatwidget/input_submission.rs | 12 +++++-- codex-rs/tui/src/chatwidget/protocol.rs | 16 ++++++--- codex-rs/tui/src/chatwidget/replay.rs | 2 +- .../tui/src/chatwidget/safety_buffering.rs | 6 +++- codex-rs/tui/src/chatwidget/turn_runtime.rs | 3 ++ 8 files changed, 72 insertions(+), 22 deletions(-) diff --git a/codex-rs/tui/src/chatwidget.rs b/codex-rs/tui/src/chatwidget.rs index cdde557ddbb0..06a06c05d511 100644 --- a/codex-rs/tui/src/chatwidget.rs +++ b/codex-rs/tui/src/chatwidget.rs @@ -483,6 +483,12 @@ const DEFAULT_OPENAI_BASE_URL: &str = "https://api.openai.com/v1"; const DEFAULT_STATUS_LINE_ITEMS: [&str; 2] = ["model-with-reasoning", "current-dir"]; const DEFAULT_USAGE_LIMIT_RESUME_PROMPT: &str = "The usage limit has been reset, so you can resume from where you left off."; +const DEFAULT_SERVER_OVERLOADED_RESUME_PROMPT: &str = "Continue"; + +struct PendingLocalUserMessageEcho { + display: UserMessageDisplay, + turn_id: Option, +} /// Common initialization parameters shared by all `ChatWidget` constructors. pub(crate) struct ChatWidgetInit { @@ -764,10 +770,11 @@ pub(crate) struct ChatWidget { current_goal_status_indicator: Option, current_goal_status: Option, external_editor_state: ExternalEditorState, - last_rendered_user_message_display: Option, + pending_local_user_message_echo: Option, last_non_retry_error: Option<(String, String)>, pending_auth_reload_attempt: Option, pending_usage_limit_resume_turn: Option, + pending_server_overloaded_resume_turn: Option, usage_limit_resume_waiting_for_auth_reload: bool, } @@ -1268,7 +1275,12 @@ impl ChatWidget { self.request_redraw(); } - fn on_committed_user_message(&mut self, items: &[UserInput], from_replay: bool) { + fn on_committed_user_message( + &mut self, + items: &[UserInput], + turn_id: &str, + from_replay: bool, + ) { let display = Self::user_message_display_from_inputs(items); if from_replay { if self.review.is_review_mode { @@ -1299,21 +1311,28 @@ impl ChatWidget { let pending_display = user_message_display_for_history(pending.user_message, &pending.history_record); self.on_user_message_display(pending_display); - } else if self.last_rendered_user_message_display.as_ref() != Some(&display) { + } else { tracing::warn!( "pending steer matched compare key but queue was empty when rendering committed user message" ); self.on_user_message_display(display); } - } else if !self.review.is_review_mode - && self.last_rendered_user_message_display.as_ref() != Some(&display) - { - self.on_user_message_display(display); + } else if !self.review.is_review_mode { + let is_local_echo = self + .pending_local_user_message_echo + .as_ref() + .is_some_and(|pending| { + pending.turn_id.as_deref() == Some(turn_id) && pending.display == display + }); + if is_local_echo { + self.pending_local_user_message_echo = None; + } else { + self.on_user_message_display(display); + } } } fn on_user_message_display(&mut self, display: UserMessageDisplay) { - self.last_rendered_user_message_display = Some(display.clone()); if !display.message.trim().is_empty() || !display.text_elements.is_empty() || !display.local_images.is_empty() diff --git a/codex-rs/tui/src/chatwidget/constructor.rs b/codex-rs/tui/src/chatwidget/constructor.rs index dfa5c1b3fcb3..460215e5096c 100644 --- a/codex-rs/tui/src/chatwidget/constructor.rs +++ b/codex-rs/tui/src/chatwidget/constructor.rs @@ -247,10 +247,11 @@ impl ChatWidget { current_goal_status_indicator: None, current_goal_status: None, external_editor_state: ExternalEditorState::Closed, - last_rendered_user_message_display: None, + pending_local_user_message_echo: None, last_non_retry_error: None, pending_auth_reload_attempt: None, pending_usage_limit_resume_turn: None, + pending_server_overloaded_resume_turn: None, usage_limit_resume_waiting_for_auth_reload: false, }; diff --git a/codex-rs/tui/src/chatwidget/input_flow.rs b/codex-rs/tui/src/chatwidget/input_flow.rs index 0963faf29505..35de5b63bb31 100644 --- a/codex-rs/tui/src/chatwidget/input_flow.rs +++ b/codex-rs/tui/src/chatwidget/input_flow.rs @@ -132,10 +132,20 @@ impl ChatWidget { /// If idle and there are queued inputs, submit exactly one to start the next turn. pub(crate) fn maybe_send_next_queued_input(&mut self) -> bool { - if self.input_queue.suppress_queue_autosend { + if self.blocks_direct_input { return false; } - if self.blocks_direct_input { + if self.is_user_turn_pending_or_running() { + return false; + } + if let Some(user_message) = self.pending_server_overloaded_resume_turn.take() { + self.reasoning_buffer.clear(); + self.set_status_header(String::from("Working")); + self.submit_user_message(user_message); + self.refresh_pending_input_preview(); + return true; + } + if self.input_queue.suppress_queue_autosend { return false; } if self.pending_auth_reload_attempt.is_some() { @@ -146,9 +156,6 @@ impl ChatWidget { { return false; } - if self.is_user_turn_pending_or_running() { - return false; - } let mut submitted_follow_up = false; if let Some(user_message) = self.pending_usage_limit_resume_turn.take() { self.usage_limit_resume_waiting_for_auth_reload = false; diff --git a/codex-rs/tui/src/chatwidget/input_submission.rs b/codex-rs/tui/src/chatwidget/input_submission.rs index 830cd3eea667..2dcd53e2ca31 100644 --- a/codex-rs/tui/src/chatwidget/input_submission.rs +++ b/codex-rs/tui/src/chatwidget/input_submission.rs @@ -362,13 +362,21 @@ impl ChatWidget { let render_before_submit = render_in_history && matches!(&self.codex_op_target, CodexOpTarget::AppEvent); if render_before_submit { - self.on_user_message_display(user_message_display_for_history( + let display = user_message_display_for_history( submitted_message.clone(), &history_record, - )); + ); + self.pending_local_user_message_echo = Some(PendingLocalUserMessageEcho { + display: display.clone(), + turn_id: None, + }); + self.on_user_message_display(display); } if !self.submit_op(op.clone()) { + if render_before_submit { + self.pending_local_user_message_echo = None; + } return (false, None); } if render_in_history { diff --git a/codex-rs/tui/src/chatwidget/protocol.rs b/codex-rs/tui/src/chatwidget/protocol.rs index 79e2c3fe6e75..d74c83eec7c5 100644 --- a/codex-rs/tui/src/chatwidget/protocol.rs +++ b/codex-rs/tui/src/chatwidget/protocol.rs @@ -58,6 +58,11 @@ impl ChatWidget { self.on_thread_settings_updated(notification); } ServerNotification::TurnStarted(notification) => { + if let Some(pending) = self.pending_local_user_message_echo.as_mut() + && pending.turn_id.is_none() + { + pending.turn_id = Some(notification.turn.id.clone()); + } self.turn_lifecycle.last_turn_id = Some(notification.turn.id); self.last_non_retry_error = None; if !matches!(replay_kind, Some(ReplayKind::ResumeInitialMessages)) { @@ -237,10 +242,13 @@ impl ChatWidget { notification: TurnCompletedNotification, replay_kind: Option, ) { - // User-message dedupe only suppresses the app-server echo of a prompt - // this TUI already rendered locally. Once that turn ends, another - // client can submit the same text and it still needs its own user cell. - self.last_rendered_user_message_display = None; + if self + .pending_local_user_message_echo + .as_ref() + .is_some_and(|pending| pending.turn_id.as_deref() == Some(¬ification.turn.id)) + { + self.pending_local_user_message_echo = None; + } match notification.turn.status { TurnStatus::Completed => { let last_agent_message = diff --git a/codex-rs/tui/src/chatwidget/replay.rs b/codex-rs/tui/src/chatwidget/replay.rs index fad6eb8473e2..0bdc62681955 100644 --- a/codex-rs/tui/src/chatwidget/replay.rs +++ b/codex-rs/tui/src/chatwidget/replay.rs @@ -87,7 +87,7 @@ impl ChatWidget { let replay_kind = render_source.replay_kind(); match item { ThreadItem::UserMessage { content, .. } => { - self.on_committed_user_message(&content, from_replay); + self.on_committed_user_message(&content, &turn_id, from_replay); } ThreadItem::AgentMessage { id, diff --git a/codex-rs/tui/src/chatwidget/safety_buffering.rs b/codex-rs/tui/src/chatwidget/safety_buffering.rs index be65e5ee581b..3c41bc467daa 100644 --- a/codex-rs/tui/src/chatwidget/safety_buffering.rs +++ b/codex-rs/tui/src/chatwidget/safety_buffering.rs @@ -64,13 +64,17 @@ impl ChatWidget { } pub(crate) fn prepare_safety_buffered_retry_submission(&mut self, prompt: UserMessage) { - self.last_rendered_user_message_display = None; + self.pending_local_user_message_echo = None; self.finalize_turn(); self.safety_buffering_prompt = Some(prompt); self.input_queue.user_turn_pending_start = true; } pub(crate) fn commit_safety_buffered_retry_submission(&mut self, display: UserMessageDisplay) { + self.pending_local_user_message_echo = Some(PendingLocalUserMessageEcho { + display: display.clone(), + turn_id: None, + }); self.on_user_message_display(display); } diff --git a/codex-rs/tui/src/chatwidget/turn_runtime.rs b/codex-rs/tui/src/chatwidget/turn_runtime.rs index 36c05299472b..ea478183e057 100644 --- a/codex-rs/tui/src/chatwidget/turn_runtime.rs +++ b/codex-rs/tui/src/chatwidget/turn_runtime.rs @@ -343,6 +343,9 @@ impl ChatWidget { pub(super) fn on_server_overloaded_error(&mut self, message: String) { self.input_queue.submit_pending_steers_after_interrupt = false; + self.pending_server_overloaded_resume_turn = Some(UserMessage::from( + DEFAULT_SERVER_OVERLOADED_RESUME_PROMPT, + )); self.finalize_turn(); let message = if message.trim().is_empty() { From 149a3f529b4366a894ad7894b505aaa8d8ac77ff Mon Sep 17 00:00:00 2001 From: Andrey Vasiliev Date: Wed, 29 Jul 2026 17:54:24 +0300 Subject: [PATCH 2/4] feat: add server overload auto-resume toggle --- codex-rs/config/src/types.rs | 5 +++++ codex-rs/core/config.schema.json | 5 +++++ codex-rs/core/src/config/mod.rs | 8 ++++++++ codex-rs/tui/src/chatwidget/turn_runtime.rs | 8 +++++--- 4 files changed, 23 insertions(+), 3 deletions(-) diff --git a/codex-rs/config/src/types.rs b/codex-rs/config/src/types.rs index f1a0cdc405cf..1e9ae8d9880f 100644 --- a/codex-rs/config/src/types.rs +++ b/codex-rs/config/src/types.rs @@ -785,6 +785,11 @@ pub struct Tui { #[serde(default)] pub usage_limit_resume_prompt: Option, + /// Automatically submit `Continue` after a turn fails with `ServerOverloaded`. + /// Defaults to `true`. + #[serde(default = "default_true")] + pub server_overloaded_resume_enabled: bool, + /// Startup tooltip availability NUX state persisted by the TUI. #[serde(default)] pub model_availability_nux: ModelAvailabilityNuxConfig, diff --git a/codex-rs/core/config.schema.json b/codex-rs/core/config.schema.json index d7f85ff8740b..067b91b3e153 100644 --- a/codex-rs/core/config.schema.json +++ b/codex-rs/core/config.schema.json @@ -3423,6 +3423,11 @@ "default": null, "description": "Working directory to use when resuming or forking a session. When unset, prompt if the current and session directories differ." }, + "server_overloaded_resume_enabled": { + "default": true, + "description": "Automatically submit `Continue` after a turn fails with `ServerOverloaded`. Defaults to `true`.", + "type": "boolean" + }, "session_picker_view": { "allOf": [ { diff --git a/codex-rs/core/src/config/mod.rs b/codex-rs/core/src/config/mod.rs index f43bc2f3371b..f566e41ddc06 100644 --- a/codex-rs/core/src/config/mod.rs +++ b/codex-rs/core/src/config/mod.rs @@ -811,6 +811,9 @@ pub struct Config { /// automatic recovery turn. pub tui_usage_limit_resume_prompt: Option, + /// Whether to automatically submit `Continue` after a turn fails with `ServerOverloaded`. + pub tui_server_overloaded_resume_enabled: bool, + /// The absolute directory that should be treated as the current working /// directory for the session. All relative paths inside the business-logic /// layer are resolved against this path. @@ -4227,6 +4230,11 @@ impl Config { .tui .as_ref() .and_then(|t| t.usage_limit_resume_prompt.clone()), + tui_server_overloaded_resume_enabled: cfg + .tui + .as_ref() + .map(|t| t.server_overloaded_resume_enabled) + .unwrap_or(true), otel, }; Ok(config) diff --git a/codex-rs/tui/src/chatwidget/turn_runtime.rs b/codex-rs/tui/src/chatwidget/turn_runtime.rs index ea478183e057..0fa886804907 100644 --- a/codex-rs/tui/src/chatwidget/turn_runtime.rs +++ b/codex-rs/tui/src/chatwidget/turn_runtime.rs @@ -343,9 +343,11 @@ impl ChatWidget { pub(super) fn on_server_overloaded_error(&mut self, message: String) { self.input_queue.submit_pending_steers_after_interrupt = false; - self.pending_server_overloaded_resume_turn = Some(UserMessage::from( - DEFAULT_SERVER_OVERLOADED_RESUME_PROMPT, - )); + if self.config.tui_server_overloaded_resume_enabled { + self.pending_server_overloaded_resume_turn = Some(UserMessage::from( + DEFAULT_SERVER_OVERLOADED_RESUME_PROMPT, + )); + } self.finalize_turn(); let message = if message.trim().is_empty() { From 0d0b55ff8141d853e766b60ebb327735b6a268a4 Mon Sep 17 00:00:00 2001 From: Andrey Vasiliev Date: Tue, 4 Aug 2026 13:26:31 +0300 Subject: [PATCH 3/4] fix: back off server overload auto-resume --- codex-rs/tui/src/app/event_dispatch.rs | 7 +++ codex-rs/tui/src/app_event.rs | 6 +++ codex-rs/tui/src/chatwidget.rs | 2 + codex-rs/tui/src/chatwidget/constructor.rs | 2 + codex-rs/tui/src/chatwidget/turn_runtime.rs | 54 +++++++++++++++++++-- 5 files changed, 68 insertions(+), 3 deletions(-) diff --git a/codex-rs/tui/src/app/event_dispatch.rs b/codex-rs/tui/src/app/event_dispatch.rs index f13f75dbca16..18b189d2a876 100644 --- a/codex-rs/tui/src/app/event_dispatch.rs +++ b/codex-rs/tui/src/app/event_dispatch.rs @@ -878,6 +878,13 @@ impl App { AppEvent::AuthFileChangedRetry { attempt } => { self.handle_auth_file_changed(app_server, attempt).await; } + AppEvent::ServerOverloadedRetry { + attempt, + generation, + } => { + self.chat_widget + .on_server_overloaded_retry(attempt, generation); + } AppEvent::SendAddCreditsNudgeEmail { credit_type } => { if self .chat_widget diff --git a/codex-rs/tui/src/app_event.rs b/codex-rs/tui/src/app_event.rs index 0c9f74eb724f..307917bc2f3b 100644 --- a/codex-rs/tui/src/app_event.rs +++ b/codex-rs/tui/src/app_event.rs @@ -371,6 +371,12 @@ pub(crate) enum AppEvent { attempt: u8, }, + /// Retry a server-overloaded turn after its backoff delay. + ServerOverloadedRetry { + attempt: u8, + generation: u64, + }, + /// Result of refreshing rate limits. RateLimitsLoaded { origin: RateLimitRefreshOrigin, diff --git a/codex-rs/tui/src/chatwidget.rs b/codex-rs/tui/src/chatwidget.rs index 06a06c05d511..4f95e42e9bb1 100644 --- a/codex-rs/tui/src/chatwidget.rs +++ b/codex-rs/tui/src/chatwidget.rs @@ -775,6 +775,8 @@ pub(crate) struct ChatWidget { pending_auth_reload_attempt: Option, pending_usage_limit_resume_turn: Option, pending_server_overloaded_resume_turn: Option, + server_overloaded_resume_attempts: u8, + server_overloaded_resume_generation: u64, usage_limit_resume_waiting_for_auth_reload: bool, } diff --git a/codex-rs/tui/src/chatwidget/constructor.rs b/codex-rs/tui/src/chatwidget/constructor.rs index 460215e5096c..bcc7cd505a80 100644 --- a/codex-rs/tui/src/chatwidget/constructor.rs +++ b/codex-rs/tui/src/chatwidget/constructor.rs @@ -252,6 +252,8 @@ impl ChatWidget { pending_auth_reload_attempt: None, pending_usage_limit_resume_turn: None, pending_server_overloaded_resume_turn: None, + server_overloaded_resume_attempts: 0, + server_overloaded_resume_generation: 0, usage_limit_resume_waiting_for_auth_reload: false, }; diff --git a/codex-rs/tui/src/chatwidget/turn_runtime.rs b/codex-rs/tui/src/chatwidget/turn_runtime.rs index 0fa886804907..dd639d93db04 100644 --- a/codex-rs/tui/src/chatwidget/turn_runtime.rs +++ b/codex-rs/tui/src/chatwidget/turn_runtime.rs @@ -9,6 +9,13 @@ const LEGACY_SAFETY_ACCESS_BLOCK_PREFIX: &str = "Invalid prompt: we've limited access to this content for safety reasons."; const BIO_POLICY_SAFETY_ACCESS_BLOCK_PREFIX: &str = "This content was flagged for possible biological risk."; +const SERVER_OVERLOADED_RESUME_DELAYS: [Duration; 5] = [ + Duration::from_secs(30), + Duration::from_secs(60), + Duration::from_secs(120), + Duration::from_secs(240), + Duration::from_secs(480), +]; fn is_safety_access_block_message(message: &str) -> bool { message.starts_with(LEGACY_SAFETY_ACCESS_BLOCK_PREFIX) @@ -108,6 +115,12 @@ impl ChatWidget { from_replay: bool, ) { self.input_queue.submit_pending_steers_after_interrupt = false; + if !from_replay { + self.pending_server_overloaded_resume_turn = None; + self.server_overloaded_resume_attempts = 0; + self.server_overloaded_resume_generation = + self.server_overloaded_resume_generation.wrapping_add(1); + } let sanitized_last_agent_message = last_agent_message.as_deref().map(|message| { parse_assistant_markdown(message, self.config.cwd.as_path()).visible_markdown }); @@ -344,9 +357,36 @@ impl ChatWidget { pub(super) fn on_server_overloaded_error(&mut self, message: String) { self.input_queue.submit_pending_steers_after_interrupt = false; if self.config.tui_server_overloaded_resume_enabled { - self.pending_server_overloaded_resume_turn = Some(UserMessage::from( - DEFAULT_SERVER_OVERLOADED_RESUME_PROMPT, - )); + let next_attempt = self.server_overloaded_resume_attempts.saturating_add(1); + if let Some(retry_delay) = SERVER_OVERLOADED_RESUME_DELAYS + .get(usize::from(next_attempt.saturating_sub(1))) + .copied() + { + self.server_overloaded_resume_attempts = next_attempt; + self.server_overloaded_resume_generation = + self.server_overloaded_resume_generation.wrapping_add(1); + let generation = self.server_overloaded_resume_generation; + self.pending_server_overloaded_resume_turn = Some(UserMessage::from( + DEFAULT_SERVER_OVERLOADED_RESUME_PROMPT, + )); + let app_event_tx = self.app_event_tx.clone(); + tokio::spawn(async move { + tokio::time::sleep(retry_delay).await; + app_event_tx.send(AppEvent::ServerOverloadedRetry { + attempt: next_attempt, + generation, + }); + }); + } else { + self.pending_server_overloaded_resume_turn = None; + self.server_overloaded_resume_generation = + self.server_overloaded_resume_generation.wrapping_add(1); + } + } else { + self.pending_server_overloaded_resume_turn = None; + self.server_overloaded_resume_attempts = 0; + self.server_overloaded_resume_generation = + self.server_overloaded_resume_generation.wrapping_add(1); } self.finalize_turn(); @@ -358,6 +398,14 @@ impl ChatWidget { self.add_to_history(history_cell::new_warning_event(message)); self.request_redraw(); + } + + pub(crate) fn on_server_overloaded_retry(&mut self, attempt: u8, generation: u64) { + if self.server_overloaded_resume_attempts != attempt + || self.server_overloaded_resume_generation != generation + { + return; + } self.maybe_send_next_queued_input(); } From 72c301bf407b40493a3cf8fc5590212baf27534d Mon Sep 17 00:00:00 2001 From: Andrey Vasiliev Date: Tue, 4 Aug 2026 13:37:53 +0300 Subject: [PATCH 4/4] fix: adjust overload retry backoff --- codex-rs/tui/src/chatwidget/turn_runtime.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/codex-rs/tui/src/chatwidget/turn_runtime.rs b/codex-rs/tui/src/chatwidget/turn_runtime.rs index dd639d93db04..2341d587c959 100644 --- a/codex-rs/tui/src/chatwidget/turn_runtime.rs +++ b/codex-rs/tui/src/chatwidget/turn_runtime.rs @@ -10,11 +10,11 @@ const LEGACY_SAFETY_ACCESS_BLOCK_PREFIX: &str = const BIO_POLICY_SAFETY_ACCESS_BLOCK_PREFIX: &str = "This content was flagged for possible biological risk."; const SERVER_OVERLOADED_RESUME_DELAYS: [Duration; 5] = [ + Duration::from_secs(15), Duration::from_secs(30), Duration::from_secs(60), Duration::from_secs(120), Duration::from_secs(240), - Duration::from_secs(480), ]; fn is_safety_access_block_message(message: &str) -> bool {