From 6193e7c37dbc2f56e884f1957126b59350aeac96 Mon Sep 17 00:00:00 2001 From: NessZerra <90105158+Finesssee@users.noreply.github.com> Date: Tue, 22 Sep 2026 17:19:47 +0700 Subject: [PATCH 01/19] Exclude inherited Codex fork baselines --- rust/src/core/jsonl_scanner.rs | 7 + rust/src/core/jsonl_scanner/codex.rs | 61 +++++++- rust/src/core/jsonl_scanner/codex/parser.rs | 158 ++++++++++++++++++++ rust/src/core/jsonl_scanner/tests.rs | 2 + rust/src/cost_scanner/codex.rs | 28 +++- rust/src/cost_scanner/tests/paginated.rs | 157 +++++++++++++++++++ 6 files changed, 408 insertions(+), 5 deletions(-) diff --git a/rust/src/core/jsonl_scanner.rs b/rust/src/core/jsonl_scanner.rs index 883a31d89f..13e78bcfe9 100755 --- a/rust/src/core/jsonl_scanner.rs +++ b/rust/src/core/jsonl_scanner.rs @@ -373,6 +373,8 @@ pub(crate) struct CodexSessionMetadata { pub lineage: CodexSessionLineage, pub fork_timestamp: Option, pub history_base_thread_id: Option, + pub is_subagent: bool, + pub subagent_history_start_ordinal: Option, } /// Running totals for Codex token counting @@ -396,6 +398,10 @@ pub struct CodexForkAccountingState { pub inherited_totals: Option, #[serde(default, skip_serializing_if = "Option::is_none")] pub remaining_inherited_totals: Option, + /// True when the child log itself supplied enough copied-prefix history to + /// establish the inherited baseline without consulting a parent cache row. + #[serde(default, skip_serializing_if = "std::ops::Not::not")] + pub locally_resolved: bool, } /// Snapshot of the last validated cost report, persisted so spend surfaces keep @@ -456,6 +462,7 @@ pub struct CodexParseResult { pub fork_baseline: Option, /// Remaining inherited counters used when a fork emits last-only rows. pub remaining_inherited_totals: Option, + pub fork_baseline_locally_resolved: bool, } /// A billable Codex token-count delta. diff --git a/rust/src/core/jsonl_scanner/codex.rs b/rust/src/core/jsonl_scanner/codex.rs index 30f159c0ca..1d88cc3f18 100644 --- a/rust/src/core/jsonl_scanner/codex.rs +++ b/rust/src/core/jsonl_scanner/codex.rs @@ -13,8 +13,9 @@ use parser::CodexParserState; /// Persisted Codex cache schema version. Version 0 predates 64-bit totals; /// version 1 can retain a terminal pause after treating a paginated v2 /// subagent's independent counters as an inherited fork. Version 3 adds -/// persisted paginated-fork accounting state. Rebuild older artifacts. -pub(crate) const CODEX_CACHE_SCHEMA_VERSION: u32 = 3; +/// persisted paginated-fork accounting state. Version 4 reparses copied-prefix +/// subagents with locally inferred component baselines. Rebuild older artifacts. +pub(crate) const CODEX_CACHE_SCHEMA_VERSION: u32 = 4; /// Whether a persisted Codex cache artifact matches the current schema. /// A mismatched artifact (e.g. a pre-64-bit cache from an older release) is @@ -186,6 +187,12 @@ impl JsonlScanner { .pointer("/source/subagent/thread_spawn") .is_some_and(Value::is_object)) }); + let is_subagent = payload.is_some_and(|value| { + value.get("thread_source").and_then(Value::as_str) == Some("subagent") + || value + .pointer("/source/subagent/thread_spawn") + .is_some_and(Value::is_object) + }); let history_base_thread_id = payload .and_then(|value| value.get("history_base")) .filter(|value| value.is_object()) @@ -220,6 +227,10 @@ impl JsonlScanner { payload.and_then(|value| nonempty_json_string(value.get("timestamp"))) }), history_base_thread_id, + is_subagent, + subagent_history_start_ordinal: payload + .and_then(|value| value.get("subagent_history_start_ordinal")) + .and_then(Value::as_i64), }); } @@ -364,6 +375,8 @@ impl JsonlScanner { None, None, max_bytes_to_read, + false, + None, ) } @@ -400,6 +413,8 @@ impl JsonlScanner { None, scan_target_size, max_bytes_to_read, + false, + None, ) } @@ -431,6 +446,8 @@ impl JsonlScanner { None, None, max_bytes_to_read, + false, + None, ) } @@ -456,6 +473,35 @@ impl JsonlScanner { cancel, scan_target_size, max_bytes_to_read, + false, + None, + ) + } + + pub(crate) fn parse_codex_file_with_inferred_fork_baseline( + file_path: &Path, + range: &CostUsageDayRange, + subagent_history_start_ordinal: Option, + cancel: Option<&AtomicBool>, + scan_target_size: Option, + max_bytes_to_read: Option, + ) -> std::io::Result { + Self::parse_codex_file_with_state_bounded_internal( + file_path, + range, + 0, + None, + None, + None, + None, + cancel, + true, + false, + None, + scan_target_size, + max_bytes_to_read, + true, + subagent_history_start_ordinal, ) } @@ -473,6 +519,8 @@ impl JsonlScanner { cancel: Option<&AtomicBool>, scan_target_size: Option, max_bytes_to_read: Option, + infer_fork_baseline: bool, + subagent_history_start_ordinal: Option, ) -> std::io::Result { Self::parse_codex_file_with_state_bounded_internal( file_path, @@ -488,6 +536,8 @@ impl JsonlScanner { remaining_inherited_totals, scan_target_size, max_bytes_to_read, + infer_fork_baseline, + subagent_history_start_ordinal, ) } @@ -509,6 +559,8 @@ impl JsonlScanner { remaining_inherited_totals: Option, scan_target_size: Option, max_bytes_to_read: Option, + infer_fork_baseline: bool, + subagent_history_start_ordinal: Option, ) -> std::io::Result { let file = File::open(file_path)?; // Session JSONL files are bounded by the cache budget; sizes fit i64. @@ -538,6 +590,9 @@ impl JsonlScanner { paginated_continuation, remaining_inherited_totals, ); + if infer_fork_baseline { + parser.enable_fork_baseline_inference(subagent_history_start_ordinal); + } let mut parsed_bytes = safe_start_offset; let mut committed_bytes = safe_start_offset; let mut cancelled = false; @@ -626,6 +681,7 @@ impl JsonlScanner { }; let is_complete = !cancelled && !budget_exhausted && parsed_bytes >= effective_target_size; let bytes_read = parsed_bytes.saturating_sub(safe_start_offset).max(0); + let fork_baseline_locally_resolved = parser.fork_baseline_locally_resolved(); Ok(CodexParseResult { records: parser.records, parsed_bytes, @@ -644,6 +700,7 @@ impl JsonlScanner { fork_baseline_ambiguous: parser.fork_baseline_ambiguous, fork_baseline: parser.fork_baseline, remaining_inherited_totals: parser.remaining_inherited_totals, + fork_baseline_locally_resolved, }) } diff --git a/rust/src/core/jsonl_scanner/codex/parser.rs b/rust/src/core/jsonl_scanner/codex/parser.rs index 2ec70f9cf1..d81cace664 100644 --- a/rust/src/core/jsonl_scanner/codex/parser.rs +++ b/rust/src/core/jsonl_scanner/codex/parser.rs @@ -22,6 +22,119 @@ pub(super) struct CodexParserState { paginated_continuation: bool, paginated_baseline_checked: bool, pub(super) fork_baseline_ambiguous: bool, + fork_baseline_inference: Option, +} + +#[derive(Debug)] +struct ForkBaselineInference { + explicit_start_ordinal: Option, + baseline: Option, + boundary_open: bool, + inherited_opening: bool, + locally_confirmed: bool, + resolved: bool, +} + +impl ForkBaselineInference { + fn new(explicit_start_ordinal: Option) -> Self { + Self { + explicit_start_ordinal, + baseline: explicit_start_ordinal.map(|_| CodexTotals { + input: 0, + cached: 0, + output: 0, + reasoning: None, + }), + boundary_open: false, + inherited_opening: false, + locally_confirmed: explicit_start_ordinal.is_some(), + resolved: false, + } + } + + fn observe_non_token(&mut self, obj: &Value) { + if obj.get("type").and_then(Value::as_str) == Some("turn_context") && self.inherited_opening + { + self.boundary_open = true; + } + } + + /// Return the baseline when this is the first owned token event. `None` + /// means the event is still part of the copied prefix. + fn observe_token(&mut self, obj: &Value) -> Option { + let payload = token_count_payload(obj)?; + let info = payload.get("info")?; + let total = read_token_totals(info.get("total_token_usage")?); + let last = read_token_totals(info.get("last_token_usage")?); + let ordinal = obj.get("ordinal").and_then(Value::as_i64); + + if let Some(start) = self.explicit_start_ordinal { + if ordinal.is_some_and(|ordinal| ordinal < start) { + self.baseline = Some(total); + return None; + } + self.boundary_open = true; + } else if self.baseline.is_none() { + if totals_contain_usage(&total) && !totals_contain_usage(&last) { + self.baseline = Some(total); + self.inherited_opening = true; + self.locally_confirmed = true; + } + return None; + } else if !self.boundary_open { + let changed = self + .baseline + .as_ref() + .is_some_and(|baseline| baseline != &total); + if self.inherited_opening && changed && totals_contain_usage(&last) { + self.boundary_open = true; + } else { + return None; + } + } + + let baseline = self.baseline.clone().unwrap_or(CodexTotals { + input: 0, + cached: 0, + output: 0, + reasoning: None, + }); + if total == baseline { + return None; + } + let copied_snapshot = + totals_contain_usage(&baseline) && total == last && totals_at_least(&total, &baseline); + if copied_snapshot { + self.baseline = Some(total); + self.locally_confirmed = true; + return None; + } + + let owned_baseline = totals_delta(&last, &total); + self.baseline = Some(owned_baseline.clone()); + self.locally_confirmed = true; + self.resolved = true; + Some(owned_baseline) + } +} + +fn totals_contain_usage(totals: &CodexTotals) -> bool { + totals.input > 0 || totals.cached > 0 || totals.output > 0 +} + +fn totals_at_least(total: &CodexTotals, baseline: &CodexTotals) -> bool { + total.input >= baseline.input + && total.cached >= baseline.cached + && total.output >= baseline.output +} + +fn totals_delta(last: &CodexTotals, total: &CodexTotals) -> CodexTotals { + CodexTotals { + input: total.input.saturating_sub(last.input).max(0), + cached: total.cached.saturating_sub(last.cached).max(0), + output: total.output.saturating_sub(last.output).max(0), + reasoning: subtract_optional(total.reasoning, last.reasoning), + } } impl CodexParserState { @@ -95,9 +208,24 @@ impl CodexParserState { paginated_continuation, paginated_baseline_checked: false, fork_baseline_ambiguous: false, + fork_baseline_inference: None, } } + pub(super) fn enable_fork_baseline_inference(&mut self, start_ordinal: Option) { + self.fork_baseline = None; + self.remaining_inherited_totals = None; + self.previous_totals = None; + self.totals_watermark = None; + self.fork_baseline_inference = Some(ForkBaselineInference::new(start_ordinal)); + } + + pub(super) fn fork_baseline_locally_resolved(&self) -> bool { + self.fork_baseline_inference + .as_ref() + .is_some_and(|inference| inference.locally_confirmed) + } + pub(super) fn process_line(&mut self, line: &str, range: &CostUsageDayRange) { self.process_line_with_source_offset(line, range, 0); } @@ -108,6 +236,36 @@ impl CodexParserState { range: &CostUsageDayRange, source_end_offset: i64, ) { + if self + .fork_baseline_inference + .as_ref() + .is_some_and(|inference| !inference.resolved) + { + let Ok(obj) = serde_json::from_str::(line) else { + return; + }; + if token_count_payload(&obj).is_some() { + let baseline = self + .fork_baseline_inference + .as_mut() + .and_then(|inference| inference.observe_token(&obj)); + let Some(baseline) = baseline else { return }; + self.fork_baseline = Some(baseline.clone()); + self.remaining_inherited_totals = Some(baseline.clone()); + self.previous_totals = Some(baseline.clone()); + self.totals_watermark = Some(baseline); + } else { + self.fork_baseline_inference + .as_mut() + .expect("inference exists") + .observe_non_token(&obj); + if obj.get("type").and_then(Value::as_str) == Some("turn_context") { + self.update_current_model(&obj); + } + return; + } + } + let event_candidate = is_candidate_codex_line(line); let bare_candidate = !event_candidate && line.contains("\"usage\""); if !event_candidate && !bare_candidate { diff --git a/rust/src/core/jsonl_scanner/tests.rs b/rust/src/core/jsonl_scanner/tests.rs index bcb1129cb1..cc4b2156ef 100644 --- a/rust/src/core/jsonl_scanner/tests.rs +++ b/rust/src/core/jsonl_scanner/tests.rs @@ -1043,6 +1043,8 @@ fn session_meta_pre_read_accepts_snake_and_camel_fork_identity() { lineage: CodexSessionLineage::Child, fork_timestamp: Some("2026-05-31T10:00:00Z".to_string()), history_base_thread_id: Some("history-snake".to_string()), + is_subagent: false, + subagent_history_start_ordinal: None, } ); diff --git a/rust/src/cost_scanner/codex.rs b/rust/src/cost_scanner/codex.rs index f054c6eb8f..cb152611a2 100644 --- a/rust/src/cost_scanner/codex.rs +++ b/rust/src/cost_scanner/codex.rs @@ -42,6 +42,13 @@ fn summary_from_cached_report( } fn codex_fork_parent_is_safe(cache: &CostUsageCache, usage: &CostUsageFileUsage) -> bool { + if usage + .codex_fork_accounting_state + .as_ref() + .is_some_and(|state| state.locally_resolved) + { + return true; + } let uses_parent_baseline = usage.codex_lineage.uses_parent_baseline() || (matches!(usage.codex_lineage, CodexSessionLineage::Root) && usage.codex_forked_from_id.is_some()); @@ -456,6 +463,7 @@ impl CostScanner { })) }); let is_fork = codex_lineage.uses_parent_baseline(); + let locally_inferred_subagent = is_fork && session_metadata.is_subagent; let cached_fork_state_matches = cached_fork_accounting_state.as_ref().is_some_and(|state| { state.session_id == codex_session_id @@ -485,7 +493,7 @@ impl CostScanner { .as_deref() .is_some_and(|history_base| Some(history_base) != codex_forked_from_id.as_deref()); - if is_fork && fork_baseline.is_none() { + if is_fork && fork_baseline.is_none() && !locally_inferred_subagent { cache.files.insert( path_key, CostUsageFileUsage { @@ -626,7 +634,16 @@ impl CostScanner { let parse_target_size = cached .as_ref() .and_then(|entry| codex_resumable_scan_target_size(size, entry)); - let parse_result = match if let Some(baseline) = fork_baseline.clone() { + let parse_result = match if locally_inferred_subagent { + JsonlScanner::parse_codex_file_with_inferred_fork_baseline( + path, + range, + session_metadata.subagent_history_start_ordinal, + cancel, + parse_target_size, + max_bytes_to_read, + ) + } else if let Some(baseline) = fork_baseline.clone() { JsonlScanner::parse_codex_file_with_state_bounded_fork_target_with_accounting( path, range, @@ -636,6 +653,8 @@ impl CostScanner { cancel, parse_target_size, max_bytes_to_read, + false, + None, ) } else { JsonlScanner::parse_codex_file_with_state_bounded( @@ -656,7 +675,9 @@ impl CostScanner { stats.token_timestamp_comparisons = stats .token_timestamp_comparisons .saturating_add(parse_result.token_timestamp_comparisons); - if parse_result.fork_baseline_ambiguous { + if parse_result.fork_baseline_ambiguous + || (locally_inferred_subagent && !parse_result.fork_baseline_locally_resolved) + { cache.files.insert( path_key, CostUsageFileUsage { @@ -707,6 +728,7 @@ impl CostScanner { fork_timestamp: codex_fork_timestamp.clone(), inherited_totals: Some(inherited_totals), remaining_inherited_totals: parse_result.remaining_inherited_totals.clone(), + locally_resolved: parse_result.fork_baseline_locally_resolved, }) } else { None diff --git a/rust/src/cost_scanner/tests/paginated.rs b/rust/src/cost_scanner/tests/paginated.rs index 592d484823..7e61f7df55 100644 --- a/rust/src/cost_scanner/tests/paginated.rs +++ b/rust/src/cost_scanner/tests/paginated.rs @@ -135,6 +135,163 @@ fn write_codex_paginated_continuation_fixture( path } +fn write_copied_prefix_subagent_fixture( + sessions_root: &Path, + name: &str, + base: DateTime, + owned: bool, +) -> PathBuf { + let day = base.with_timezone(&Local).date_naive(); + let day_dir = sessions_root + .join(day.format("%Y").to_string()) + .join(day.format("%m").to_string()) + .join(day.format("%d").to_string()); + std::fs::create_dir_all(&day_dir).unwrap(); + let path = day_dir.join(name); + let mut lines = vec![ + serde_json::json!({ + "type": "session_meta", "ordinal": 0, "timestamp": base.to_rfc3339(), + "payload": { + "id": "child-id", "forked_from_id": "missing-parent", + "subagent_history_start_ordinal": 10, + "thread_source": "subagent", + "source": {"subagent": {"thread_spawn": {"parent_thread_id": "missing-parent"}}} + } + }), + token_row(base, 2, [1_000, 900, 100], [0, 0, 0], "gpt-5.6-sol"), + serde_json::json!({ + "type": "turn_context", "ordinal": 10, "timestamp": base.to_rfc3339(), + "payload": {"model": "gpt-5.6-sol"} + }), + token_row( + base, + 12, + [1_000, 900, 100], + [1_000, 900, 100], + "gpt-5.6-sol", + ), + token_row( + base, + 13, + [5_000, 3_900, 500], + [5_000, 3_900, 500], + "gpt-5.6-sol", + ), + ]; + if owned { + lines.extend([ + token_row(base, 19, [5_050, 3_910, 505], [50, 10, 5], "gpt-5.6-sol"), + token_row( + base + Duration::seconds(1), + 20, + [5_070, 3_915, 510], + [20, 5, 5], + "gpt-5.6-sol", + ), + token_row( + base + Duration::seconds(2), + 21, + [5_070, 3_915, 510], + [20, 5, 5], + "gpt-5.6-sol", + ), + ]); + } + let body = lines + .into_iter() + .map(|line| line.to_string()) + .collect::>() + .join("\n") + + "\n"; + std::fs::write(&path, body).unwrap(); + path +} + +fn token_row( + timestamp: DateTime, + ordinal: i64, + total: [i64; 3], + last: [i64; 3], + model: &str, +) -> serde_json::Value { + serde_json::json!({ + "type": "event_msg", "ordinal": ordinal, "timestamp": timestamp.to_rfc3339(), + "payload": {"type": "token_count", "info": { + "model": model, + "total_token_usage": { + "input_tokens": total[0], "cached_input_tokens": total[1], "output_tokens": total[2] + }, + "last_token_usage": { + "input_tokens": last[0], "cached_input_tokens": last[1], "output_tokens": last[2] + } + }} + }) +} + +#[test] +fn copied_prefix_subagent_infers_advancing_baseline_without_parent() { + let root = tempfile::tempdir().unwrap(); + let sessions = root.path().join("sessions"); + let cache_root = root.path().join("cache"); + let child = write_copied_prefix_subagent_fixture( + &sessions, + "child.jsonl", + Utc::now() - Duration::hours(1), + true, + ); + let scanner = CostScanner::new(7) + .with_options(CostScanOptions::app_driven()) + .with_cache_root(&cache_root) + .with_sessions_dirs(vec![sessions]); + + let (summary, _, cache) = scanner.scan_codex_detailed_with_cache(None); + assert_eq!(summary.input_tokens, 70); + assert_eq!(summary.cached_tokens, 15); + assert_eq!(summary.output_tokens, 10); + assert_eq!(summary.sessions_count, 1); + let usage = &cache.files[&child.to_string_lossy().to_string()]; + assert!(!usage.codex_unresolved_fork_parent); + assert!( + usage + .codex_fork_accounting_state + .as_ref() + .is_some_and(|state| state.locally_resolved) + ); + assert_eq!( + usage.days.values().next().unwrap()["gpt-5.6-sol"], + vec![70, 15, 10] + ); + + let (cached, stats, _) = scanner.scan_codex_detailed_with_cache(None); + assert_eq!(cached.input_tokens, 70); + assert!(stats.codex_history_read_paths.is_empty()); +} + +#[test] +fn copied_prefix_subagent_inherited_only_suffix_is_not_billed() { + let root = tempfile::tempdir().unwrap(); + let sessions = root.path().join("sessions"); + let cache_root = root.path().join("cache"); + let child = write_copied_prefix_subagent_fixture( + &sessions, + "child.jsonl", + Utc::now() - Duration::hours(1), + false, + ); + let scanner = CostScanner::new(7) + .with_options(CostScanOptions::app_driven()) + .with_cache_root(&cache_root) + .with_sessions_dirs(vec![sessions]); + + let (summary, _, cache) = scanner.scan_codex_detailed_with_cache(None); + assert_eq!(summary.input_tokens, 0); + assert_eq!(summary.output_tokens, 0); + assert_eq!(summary.sessions_count, 0); + let usage = &cache.files[&child.to_string_lossy().to_string()]; + assert!(usage.days.is_empty()); + assert!(!usage.codex_unresolved_fork_parent); +} + #[test] fn paginated_continuation_raises_inherited_baseline_from_total_last() { let root = tempfile::tempdir().unwrap(); From f7d37e8913c7b280ff0147c5ab8a134336357d2d Mon Sep 17 00:00:00 2001 From: NessZerra <90105158+Finesssee@users.noreply.github.com> Date: Tue, 22 Sep 2026 20:19:50 +0700 Subject: [PATCH 02/19] Model Codex parser modes explicitly --- rust/src/core/jsonl_scanner/codex.rs | 113 ++++-------- rust/src/core/jsonl_scanner/codex/parser.rs | 183 +++++++++++++------- rust/src/core/jsonl_scanner/tests.rs | 24 ++- rust/src/cost_scanner/codex.rs | 2 - 4 files changed, 158 insertions(+), 164 deletions(-) diff --git a/rust/src/core/jsonl_scanner/codex.rs b/rust/src/core/jsonl_scanner/codex.rs index 1d88cc3f18..108d4284d7 100644 --- a/rust/src/core/jsonl_scanner/codex.rs +++ b/rust/src/core/jsonl_scanner/codex.rs @@ -8,7 +8,7 @@ use helpers::{ BoundedJsonlLine, CODEX_JSONL_MAX_LINE_BYTES, nonempty_json_string, parse_rfc3339_timestamp, read_bounded_jsonl_line, read_bounded_jsonl_line_until, session_meta_field, }; -use parser::CodexParserState; +use parser::{CodexParseMode, CodexParserState}; /// Persisted Codex cache schema version. Version 0 predates 64-bit totals; /// version 1 can retain a terminal pause after treating a paginated v2 @@ -364,19 +364,16 @@ impl JsonlScanner { Self::parse_codex_file_with_state_bounded_internal( file_path, range, - start_offset, - initial_model, - initial_totals, - previous_token_timestamp, - token_timestamps_monotonic, cancel, - false, - false, - None, None, max_bytes_to_read, - false, - None, + CodexParseMode::Standard { + start_offset, + initial_model, + initial_totals, + previous_token_timestamp, + token_timestamps_monotonic, + }, ) } @@ -402,19 +399,16 @@ impl JsonlScanner { Self::parse_codex_file_with_state_bounded_internal( file_path, range, - start_offset, - initial_model, - initial_totals, - previous_token_timestamp, - token_timestamps_monotonic, cancel, - false, - false, - None, scan_target_size, max_bytes_to_read, - false, - None, + CodexParseMode::Standard { + start_offset, + initial_model, + initial_totals, + previous_token_timestamp, + token_timestamps_monotonic, + }, ) } @@ -435,19 +429,14 @@ impl JsonlScanner { Self::parse_codex_file_with_state_bounded_internal( file_path, range, - 0, - None, - Some(initial_totals), - None, - None, cancel, - true, - false, - None, None, max_bytes_to_read, - false, - None, + CodexParseMode::ParentBaseline { + baseline: initial_totals, + paginated_continuation: false, + remaining_inherited_totals: None, + }, ) } @@ -473,8 +462,6 @@ impl JsonlScanner { cancel, scan_target_size, max_bytes_to_read, - false, - None, ) } @@ -489,19 +476,12 @@ impl JsonlScanner { Self::parse_codex_file_with_state_bounded_internal( file_path, range, - 0, - None, - None, - None, - None, cancel, - true, - false, - None, scan_target_size, max_bytes_to_read, - true, - subagent_history_start_ordinal, + CodexParseMode::InferSubagent { + start_ordinal: subagent_history_start_ordinal, + }, ) } @@ -519,48 +499,28 @@ impl JsonlScanner { cancel: Option<&AtomicBool>, scan_target_size: Option, max_bytes_to_read: Option, - infer_fork_baseline: bool, - subagent_history_start_ordinal: Option, ) -> std::io::Result { Self::parse_codex_file_with_state_bounded_internal( file_path, range, - 0, - None, - Some(initial_totals), - None, - None, cancel, - true, - paginated_continuation, - remaining_inherited_totals, scan_target_size, max_bytes_to_read, - infer_fork_baseline, - subagent_history_start_ordinal, + CodexParseMode::ParentBaseline { + baseline: initial_totals, + paginated_continuation, + remaining_inherited_totals, + }, ) } - #[allow( - clippy::too_many_arguments, - reason = "resume state mirrors the persisted parser cache" - )] fn parse_codex_file_with_state_bounded_internal( file_path: &Path, range: &CostUsageDayRange, - start_offset: i64, - initial_model: Option, - initial_totals: Option, - previous_token_timestamp: Option, - token_timestamps_monotonic: Option, cancel: Option<&AtomicBool>, - fork_baseline_mode: bool, - paginated_continuation: bool, - remaining_inherited_totals: Option, scan_target_size: Option, max_bytes_to_read: Option, - infer_fork_baseline: bool, - subagent_history_start_ordinal: Option, + mode: CodexParseMode, ) -> std::io::Result { let file = File::open(file_path)?; // Session JSONL files are bounded by the cache budget; sizes fit i64. @@ -570,7 +530,7 @@ impl JsonlScanner { )] let file_size = file.metadata()?.len() as i64; - let safe_start_offset = start_offset.clamp(0, file_size); + let safe_start_offset = mode.start_offset().clamp(0, file_size); let requested_target_size = scan_target_size .unwrap_or(file_size) .max(safe_start_offset) @@ -581,18 +541,7 @@ impl JsonlScanner { reader.seek(SeekFrom::Start(safe_start_offset as u64))?; } - let mut parser = CodexParserState::with_timestamp_state_and_fork_options( - initial_model, - initial_totals, - previous_token_timestamp, - token_timestamps_monotonic, - fork_baseline_mode, - paginated_continuation, - remaining_inherited_totals, - ); - if infer_fork_baseline { - parser.enable_fork_baseline_inference(subagent_history_start_ordinal); - } + let mut parser = CodexParserState::from_mode(mode); let mut parsed_bytes = safe_start_offset; let mut committed_bytes = safe_start_offset; let mut cancelled = false; diff --git a/rust/src/core/jsonl_scanner/codex/parser.rs b/rust/src/core/jsonl_scanner/codex/parser.rs index d81cace664..c3aa7cb332 100644 --- a/rust/src/core/jsonl_scanner/codex/parser.rs +++ b/rust/src/core/jsonl_scanner/codex/parser.rs @@ -25,6 +25,33 @@ pub(super) struct CodexParserState { fork_baseline_inference: Option, } +pub(super) enum CodexParseMode { + Standard { + start_offset: i64, + initial_model: Option, + initial_totals: Option, + previous_token_timestamp: Option, + token_timestamps_monotonic: Option, + }, + ParentBaseline { + baseline: CodexTotals, + paginated_continuation: bool, + remaining_inherited_totals: Option, + }, + InferSubagent { + start_ordinal: Option, + }, +} + +impl CodexParseMode { + pub(super) fn start_offset(&self) -> i64 { + match self { + Self::Standard { start_offset, .. } => *start_offset, + Self::ParentBaseline { .. } | Self::InferSubagent { .. } => 0, + } + } +} + #[derive(Debug)] struct ForkBaselineInference { explicit_start_ordinal: Option, @@ -35,6 +62,11 @@ struct ForkBaselineInference { resolved: bool, } +enum ForkBaselineDecision { + SkipCopiedPrefix, + ProcessWithBaseline(CodexTotals), +} + impl ForkBaselineInference { fn new(explicit_start_ordinal: Option) -> Self { Self { @@ -59,19 +91,27 @@ impl ForkBaselineInference { } } - /// Return the baseline when this is the first owned token event. `None` - /// means the event is still part of the copied prefix. - fn observe_token(&mut self, obj: &Value) -> Option { - let payload = token_count_payload(obj)?; - let info = payload.get("info")?; - let total = read_token_totals(info.get("total_token_usage")?); - let last = read_token_totals(info.get("last_token_usage")?); + fn observe_token(&mut self, obj: &Value) -> ForkBaselineDecision { + let Some(payload) = token_count_payload(obj) else { + return ForkBaselineDecision::SkipCopiedPrefix; + }; + let Some(info) = payload.get("info") else { + return ForkBaselineDecision::SkipCopiedPrefix; + }; + let Some(total_usage) = info.get("total_token_usage") else { + return ForkBaselineDecision::SkipCopiedPrefix; + }; + let Some(last_usage) = info.get("last_token_usage") else { + return ForkBaselineDecision::SkipCopiedPrefix; + }; + let total = read_token_totals(total_usage); + let last = read_token_totals(last_usage); let ordinal = obj.get("ordinal").and_then(Value::as_i64); if let Some(start) = self.explicit_start_ordinal { if ordinal.is_some_and(|ordinal| ordinal < start) { self.baseline = Some(total); - return None; + return ForkBaselineDecision::SkipCopiedPrefix; } self.boundary_open = true; } else if self.baseline.is_none() { @@ -80,7 +120,7 @@ impl ForkBaselineInference { self.inherited_opening = true; self.locally_confirmed = true; } - return None; + return ForkBaselineDecision::SkipCopiedPrefix; } else if !self.boundary_open { let changed = self .baseline @@ -89,7 +129,7 @@ impl ForkBaselineInference { if self.inherited_opening && changed && totals_contain_usage(&last) { self.boundary_open = true; } else { - return None; + return ForkBaselineDecision::SkipCopiedPrefix; } } @@ -100,21 +140,21 @@ impl ForkBaselineInference { reasoning: None, }); if total == baseline { - return None; + return ForkBaselineDecision::SkipCopiedPrefix; } let copied_snapshot = totals_contain_usage(&baseline) && total == last && totals_at_least(&total, &baseline); if copied_snapshot { self.baseline = Some(total); self.locally_confirmed = true; - return None; + return ForkBaselineDecision::SkipCopiedPrefix; } let owned_baseline = totals_delta(&last, &total); self.baseline = Some(owned_baseline.clone()); self.locally_confirmed = true; self.resolved = true; - Some(owned_baseline) + ForkBaselineDecision::ProcessWithBaseline(owned_baseline) } } @@ -139,58 +179,74 @@ fn totals_delta(last: &CodexTotals, total: &CodexTotals) -> CodexTotals { impl CodexParserState { pub(super) fn new(initial_model: Option, initial_totals: Option) -> Self { - Self::with_timestamp_state(initial_model, initial_totals, None, None) - } - - fn with_timestamp_state( - initial_model: Option, - initial_totals: Option, - previous_token_timestamp: Option, - token_timestamps_monotonic: Option, - ) -> Self { - Self::with_timestamp_state_and_fork_mode( + Self::from_mode(CodexParseMode::Standard { + start_offset: 0, initial_model, initial_totals, - previous_token_timestamp, - token_timestamps_monotonic, - false, - ) + previous_token_timestamp: None, + token_timestamps_monotonic: None, + }) } - pub(super) fn with_timestamp_state_and_fork_mode( - initial_model: Option, - initial_totals: Option, - previous_token_timestamp: Option, - token_timestamps_monotonic: Option, - fork_baseline_mode: bool, - ) -> Self { - Self::with_timestamp_state_and_fork_options( + pub(super) fn from_mode(mode: CodexParseMode) -> Self { + let ( initial_model, initial_totals, previous_token_timestamp, token_timestamps_monotonic, - fork_baseline_mode, - false, - None, - ) - } - - pub(super) fn with_timestamp_state_and_fork_options( - initial_model: Option, - initial_totals: Option, - previous_token_timestamp: Option, - token_timestamps_monotonic: Option, - fork_baseline_mode: bool, - paginated_continuation: bool, - remaining_inherited_totals: Option, - ) -> Self { + fork_baseline, + paginated_continuation, + remaining_inherited_totals, + fork_baseline_inference, + ) = match mode { + CodexParseMode::Standard { + initial_model, + initial_totals, + previous_token_timestamp, + token_timestamps_monotonic, + .. + } => ( + initial_model, + initial_totals, + previous_token_timestamp, + token_timestamps_monotonic, + None, + false, + None, + None, + ), + CodexParseMode::ParentBaseline { + baseline, + paginated_continuation, + remaining_inherited_totals, + } => { + let remaining_inherited_totals = + remaining_inherited_totals.or_else(|| Some(baseline.clone())); + ( + None, + Some(baseline.clone()), + None, + None, + Some(baseline), + paginated_continuation, + remaining_inherited_totals, + None, + ) + } + CodexParseMode::InferSubagent { start_ordinal } => ( + None, + None, + None, + None, + None, + false, + None, + Some(ForkBaselineInference::new(start_ordinal)), + ), + }; let previous_token_timestamp_parsed = previous_token_timestamp .as_deref() .and_then(parse_rfc3339_timestamp); - let fork_baseline = fork_baseline_mode.then(|| initial_totals.clone()).flatten(); - let remaining_inherited_totals = fork_baseline - .as_ref() - .and_then(|baseline| remaining_inherited_totals.or_else(|| Some(baseline.clone()))); Self { current_model: initial_model, previous_totals: initial_totals.clone(), @@ -208,18 +264,10 @@ impl CodexParserState { paginated_continuation, paginated_baseline_checked: false, fork_baseline_ambiguous: false, - fork_baseline_inference: None, + fork_baseline_inference, } } - pub(super) fn enable_fork_baseline_inference(&mut self, start_ordinal: Option) { - self.fork_baseline = None; - self.remaining_inherited_totals = None; - self.previous_totals = None; - self.totals_watermark = None; - self.fork_baseline_inference = Some(ForkBaselineInference::new(start_ordinal)); - } - pub(super) fn fork_baseline_locally_resolved(&self) -> bool { self.fork_baseline_inference .as_ref() @@ -245,11 +293,14 @@ impl CodexParserState { return; }; if token_count_payload(&obj).is_some() { - let baseline = self + let decision = self .fork_baseline_inference .as_mut() - .and_then(|inference| inference.observe_token(&obj)); - let Some(baseline) = baseline else { return }; + .expect("inference exists") + .observe_token(&obj); + let ForkBaselineDecision::ProcessWithBaseline(baseline) = decision else { + return; + }; self.fork_baseline = Some(baseline.clone()); self.remaining_inherited_totals = Some(baseline.clone()); self.previous_totals = Some(baseline.clone()); diff --git a/rust/src/core/jsonl_scanner/tests.rs b/rust/src/core/jsonl_scanner/tests.rs index cc4b2156ef..02d9257753 100644 --- a/rust/src/core/jsonl_scanner/tests.rs +++ b/rust/src/core/jsonl_scanner/tests.rs @@ -77,13 +77,11 @@ fn fork_baseline_subtracts_known_reasoning_without_affecting_core_tokens() { output: 10, reasoning: Some(4), }; - let mut state = CodexParserState::with_timestamp_state_and_fork_mode( - None, - Some(baseline), - None, - None, - true, - ); + let mut state = CodexParserState::from_mode(CodexParseMode::ParentBaseline { + baseline, + paginated_continuation: false, + remaining_inherited_totals: None, + }); assert_eq!( state.apply_totals_delta(CodexTotals { input: 20, @@ -100,13 +98,11 @@ fn fork_baseline_subtracts_known_reasoning_without_affecting_core_tokens() { output: 10, reasoning: None, }; - let mut state = CodexParserState::with_timestamp_state_and_fork_mode( - None, - Some(baseline_without_reasoning), - None, - None, - true, - ); + let mut state = CodexParserState::from_mode(CodexParseMode::ParentBaseline { + baseline: baseline_without_reasoning, + paginated_continuation: false, + remaining_inherited_totals: None, + }); assert_eq!( state.apply_totals_delta(CodexTotals { input: 20, diff --git a/rust/src/cost_scanner/codex.rs b/rust/src/cost_scanner/codex.rs index cb152611a2..5e0c4a8b72 100644 --- a/rust/src/cost_scanner/codex.rs +++ b/rust/src/cost_scanner/codex.rs @@ -653,8 +653,6 @@ impl CostScanner { cancel, parse_target_size, max_bytes_to_read, - false, - None, ) } else { JsonlScanner::parse_codex_file_with_state_bounded( From 340de7dabe0c804346d2fc914bb804a2be30f76c Mon Sep 17 00:00:00 2001 From: NessZerra <90105158+Finesssee@users.noreply.github.com> Date: Tue, 22 Sep 2026 20:54:09 +0700 Subject: [PATCH 03/19] Persist inherited-only fork accounting --- rust/src/cost_scanner/codex.rs | 25 ++++++++++++------------ rust/src/cost_scanner/tests/paginated.rs | 9 +++++++++ 2 files changed, 21 insertions(+), 13 deletions(-) diff --git a/rust/src/cost_scanner/codex.rs b/rust/src/cost_scanner/codex.rs index 5e0c4a8b72..5b181311f9 100644 --- a/rust/src/cost_scanner/codex.rs +++ b/rust/src/cost_scanner/codex.rs @@ -715,19 +715,18 @@ impl CostScanner { bytes_read: parse_result.bytes_read, is_complete: parse_result.is_complete, }; - let codex_fork_accounting_state = if is_fork { - parse_result - .fork_baseline - .clone() - .map(|inherited_totals| CodexForkAccountingState { - session_id: codex_session_id.clone(), - forked_from_id: codex_forked_from_id.clone(), - history_base_thread_id: history_base_thread_id.clone(), - fork_timestamp: codex_fork_timestamp.clone(), - inherited_totals: Some(inherited_totals), - remaining_inherited_totals: parse_result.remaining_inherited_totals.clone(), - locally_resolved: parse_result.fork_baseline_locally_resolved, - }) + let codex_fork_accounting_state = if is_fork + && (parse_result.fork_baseline.is_some() || parse_result.fork_baseline_locally_resolved) + { + Some(CodexForkAccountingState { + session_id: codex_session_id.clone(), + forked_from_id: codex_forked_from_id.clone(), + history_base_thread_id: history_base_thread_id.clone(), + fork_timestamp: codex_fork_timestamp.clone(), + inherited_totals: parse_result.fork_baseline.clone(), + remaining_inherited_totals: parse_result.remaining_inherited_totals.clone(), + locally_resolved: parse_result.fork_baseline_locally_resolved, + }) } else { None }; diff --git a/rust/src/cost_scanner/tests/paginated.rs b/rust/src/cost_scanner/tests/paginated.rs index 7e61f7df55..c13c8255b9 100644 --- a/rust/src/cost_scanner/tests/paginated.rs +++ b/rust/src/cost_scanner/tests/paginated.rs @@ -290,6 +290,15 @@ fn copied_prefix_subagent_inherited_only_suffix_is_not_billed() { let usage = &cache.files[&child.to_string_lossy().to_string()]; assert!(usage.days.is_empty()); assert!(!usage.codex_unresolved_fork_parent); + let state = usage.codex_fork_accounting_state.as_ref().unwrap(); + assert!(state.locally_resolved); + assert!(state.inherited_totals.is_none()); + + let (cached, stats, _) = scanner.scan_codex_detailed_with_cache(None); + assert_eq!(cached.input_tokens, 0); + assert_eq!(cached.output_tokens, 0); + assert_eq!(cached.sessions_count, 0); + assert!(stats.codex_history_read_paths.is_empty()); } #[test] From 1badfe80a251a9b325275591ab44047a686d962c Mon Sep 17 00:00:00 2001 From: NessZerra <90105158+Finesssee@users.noreply.github.com> Date: Tue, 22 Sep 2026 22:32:45 +0700 Subject: [PATCH 04/19] Fix Codex fork accounting precedence --- rust/src/core/jsonl_scanner/codex/parser.rs | 10 +- rust/src/core/jsonl_scanner/tests.rs | 29 ++++ rust/src/cost_scanner/codex.rs | 140 ++++++++++++++------ rust/src/cost_scanner/tests/paginated.rs | 46 ++++++- 4 files changed, 181 insertions(+), 44 deletions(-) diff --git a/rust/src/core/jsonl_scanner/codex/parser.rs b/rust/src/core/jsonl_scanner/codex/parser.rs index c3aa7cb332..f33316e378 100644 --- a/rust/src/core/jsonl_scanner/codex/parser.rs +++ b/rust/src/core/jsonl_scanner/codex/parser.rs @@ -108,8 +108,14 @@ impl ForkBaselineInference { let last = read_token_totals(last_usage); let ordinal = obj.get("ordinal").and_then(Value::as_i64); - if let Some(start) = self.explicit_start_ordinal { - if ordinal.is_some_and(|ordinal| ordinal < start) { + if let Some(start) = self.explicit_start_ordinal + && !self.boundary_open + { + let Some(ordinal) = ordinal else { + self.baseline = Some(total); + return ForkBaselineDecision::SkipCopiedPrefix; + }; + if ordinal < start { self.baseline = Some(total); return ForkBaselineDecision::SkipCopiedPrefix; } diff --git a/rust/src/core/jsonl_scanner/tests.rs b/rust/src/core/jsonl_scanner/tests.rs index 02d9257753..9296583fb8 100644 --- a/rust/src/core/jsonl_scanner/tests.rs +++ b/rust/src/core/jsonl_scanner/tests.rs @@ -115,6 +115,35 @@ fn fork_baseline_subtracts_known_reasoning_without_affecting_core_tokens() { assert!(!state.fork_baseline_ambiguous); } +#[test] +fn inferred_fork_waits_for_present_explicit_start_ordinal() { + let range = CostUsageDayRange::new( + NaiveDate::from_ymd_opt(2026, 9, 22).unwrap(), + NaiveDate::from_ymd_opt(2026, 9, 22).unwrap(), + ); + let mut state = CodexParserState::from_mode(CodexParseMode::InferSubagent { + start_ordinal: Some(10), + }); + + state.process_line( + r#"{"timestamp":"2026-09-22T10:00:00Z","type":"event_msg","payload":{"type":"token_count","info":{"model":"gpt-5.6-sol","total_token_usage":{"input_tokens":100,"cached_input_tokens":20,"output_tokens":10},"last_token_usage":{"input_tokens":0,"cached_input_tokens":0,"output_tokens":0}}}}"#, + &range, + ); + + assert!(state.records.is_empty()); + assert!(state.fork_baseline.is_none()); + + state.process_line( + r#"{"ordinal":10,"timestamp":"2026-09-22T10:00:01Z","type":"event_msg","payload":{"type":"token_count","info":{"model":"gpt-5.6-sol","total_token_usage":{"input_tokens":110,"cached_input_tokens":22,"output_tokens":11},"last_token_usage":{"input_tokens":10,"cached_input_tokens":2,"output_tokens":1}}}}"#, + &range, + ); + + assert_eq!(state.records.len(), 1); + assert_eq!(state.records[0].input, 10); + assert_eq!(state.records[0].cached, 2); + assert_eq!(state.records[0].output, 1); +} + #[test] fn codex_token_pipeline_preserves_counts_above_i32_max() { let parsed = read_token_totals(&serde_json::json!({ diff --git a/rust/src/cost_scanner/codex.rs b/rust/src/cost_scanner/codex.rs index 5b181311f9..56a0f9724a 100644 --- a/rust/src/cost_scanner/codex.rs +++ b/rust/src/cost_scanner/codex.rs @@ -13,6 +13,41 @@ use pending_range::{ }; use reconciliation::*; +#[derive(Debug)] +enum CodexAccountingMode { + Standard, + ValidatedBaseline { + baseline: crate::core::CodexTotals, + paginated_continuation: bool, + remaining_inherited_totals: Option, + locally_resolved: bool, + }, + InferSubagent { + start_ordinal: Option, + }, + Unresolved, +} + +impl CodexAccountingMode { + fn is_unresolved(&self) -> bool { + matches!(self, Self::Unresolved) + } + + fn infers_subagent_baseline(&self) -> bool { + matches!(self, Self::InferSubagent { .. }) + } + + fn locally_resolved(&self) -> bool { + matches!( + self, + Self::ValidatedBaseline { + locally_resolved: true, + .. + } + ) + } +} + fn summary_from_cached_report( report: &CachedCostReport, period_start: NaiveDate, @@ -463,7 +498,6 @@ impl CostScanner { })) }); let is_fork = codex_lineage.uses_parent_baseline(); - let locally_inferred_subagent = is_fork && session_metadata.is_subagent; let cached_fork_state_matches = cached_fork_accounting_state.as_ref().is_some_and(|state| { state.session_id == codex_session_id @@ -471,29 +505,44 @@ impl CostScanner { && state.history_base_thread_id == history_base_thread_id && state.fork_timestamp == codex_fork_timestamp }); - let fork_baseline = cached_fork_accounting_state + let matching_cached_fork_state = cached_fork_accounting_state .as_ref() - .filter(|_| cached_fork_state_matches) - .and_then(|state| state.inherited_totals.clone()) - .or_else(|| { - is_fork - .then_some(codex_forked_from_id.as_deref()) - .flatten() - .and_then(|parent_id| { - codex_parent_baseline(cache, parent_id, codex_fork_timestamp.as_deref()) - }) + .filter(|_| cached_fork_state_matches); + let cached_fork_baseline = + matching_cached_fork_state.and_then(|state| state.inherited_totals.clone()); + let parent_fork_baseline = is_fork + .then_some(codex_forked_from_id.as_deref()) + .flatten() + .and_then(|parent_id| { + codex_parent_baseline(cache, parent_id, codex_fork_timestamp.as_deref()) }); - let remaining_inherited_totals = cached_fork_accounting_state - .as_ref() - .filter(|_| cached_fork_state_matches) - .and_then(|state| state.remaining_inherited_totals.clone()); + let fork_baseline = cached_fork_baseline.or(parent_fork_baseline); + let remaining_inherited_totals = + matching_cached_fork_state.and_then(|state| state.remaining_inherited_totals.clone()); let paginated_continuation = is_fork && codex_forked_from_id.is_some() && history_base_thread_id .as_deref() .is_some_and(|history_base| Some(history_base) != codex_forked_from_id.as_deref()); + let accounting_mode = if !is_fork { + CodexAccountingMode::Standard + } else if let Some(baseline) = fork_baseline { + CodexAccountingMode::ValidatedBaseline { + baseline, + paginated_continuation, + remaining_inherited_totals, + locally_resolved: matching_cached_fork_state + .is_some_and(|state| state.locally_resolved), + } + } else if session_metadata.is_subagent { + CodexAccountingMode::InferSubagent { + start_ordinal: session_metadata.subagent_history_start_ordinal, + } + } else { + CodexAccountingMode::Unresolved + }; - if is_fork && fork_baseline.is_none() && !locally_inferred_subagent { + if accounting_mode.is_unresolved() { cache.files.insert( path_key, CostUsageFileUsage { @@ -634,38 +683,46 @@ impl CostScanner { let parse_target_size = cached .as_ref() .and_then(|entry| codex_resumable_scan_target_size(size, entry)); - let parse_result = match if locally_inferred_subagent { - JsonlScanner::parse_codex_file_with_inferred_fork_baseline( + let parse_result = match match &accounting_mode { + CodexAccountingMode::Standard => JsonlScanner::parse_codex_file_with_state_bounded( path, range, - session_metadata.subagent_history_start_ordinal, + 0, + None, + None, + None, + None, cancel, - parse_target_size, max_bytes_to_read, - ) - } else if let Some(baseline) = fork_baseline.clone() { - JsonlScanner::parse_codex_file_with_state_bounded_fork_target_with_accounting( - path, - range, + ), + CodexAccountingMode::ValidatedBaseline { baseline, paginated_continuation, - remaining_inherited_totals.clone(), - cancel, - parse_target_size, - max_bytes_to_read, - ) - } else { - JsonlScanner::parse_codex_file_with_state_bounded( + remaining_inherited_totals, + .. + } => JsonlScanner::parse_codex_file_with_state_bounded_fork_target_with_accounting( path, range, - 0, - None, - None, - None, - None, + baseline.clone(), + *paginated_continuation, + remaining_inherited_totals.clone(), cancel, + parse_target_size, max_bytes_to_read, - ) + ), + CodexAccountingMode::InferSubagent { start_ordinal } => { + JsonlScanner::parse_codex_file_with_inferred_fork_baseline( + path, + range, + *start_ordinal, + cancel, + parse_target_size, + max_bytes_to_read, + ) + } + CodexAccountingMode::Unresolved => { + unreachable!("unresolved forks return before parsing") + } } { Ok(result) => result, Err(_) => return CodexFileScanOutcome::default(), @@ -674,7 +731,8 @@ impl CostScanner { .token_timestamp_comparisons .saturating_add(parse_result.token_timestamp_comparisons); if parse_result.fork_baseline_ambiguous - || (locally_inferred_subagent && !parse_result.fork_baseline_locally_resolved) + || (accounting_mode.infers_subagent_baseline() + && !parse_result.fork_baseline_locally_resolved) { cache.files.insert( path_key, @@ -715,6 +773,8 @@ impl CostScanner { bytes_read: parse_result.bytes_read, is_complete: parse_result.is_complete, }; + let locally_resolved = + accounting_mode.locally_resolved() || parse_result.fork_baseline_locally_resolved; let codex_fork_accounting_state = if is_fork && (parse_result.fork_baseline.is_some() || parse_result.fork_baseline_locally_resolved) { @@ -725,7 +785,7 @@ impl CostScanner { fork_timestamp: codex_fork_timestamp.clone(), inherited_totals: parse_result.fork_baseline.clone(), remaining_inherited_totals: parse_result.remaining_inherited_totals.clone(), - locally_resolved: parse_result.fork_baseline_locally_resolved, + locally_resolved, }) } else { None diff --git a/rust/src/cost_scanner/tests/paginated.rs b/rust/src/cost_scanner/tests/paginated.rs index c13c8255b9..a225233a7d 100644 --- a/rust/src/cost_scanner/tests/paginated.rs +++ b/rust/src/cost_scanner/tests/paginated.rs @@ -138,6 +138,7 @@ fn write_codex_paginated_continuation_fixture( fn write_copied_prefix_subagent_fixture( sessions_root: &Path, name: &str, + parent_id: &str, base: DateTime, owned: bool, ) -> PathBuf { @@ -152,10 +153,10 @@ fn write_copied_prefix_subagent_fixture( serde_json::json!({ "type": "session_meta", "ordinal": 0, "timestamp": base.to_rfc3339(), "payload": { - "id": "child-id", "forked_from_id": "missing-parent", + "id": "child-id", "forked_from_id": parent_id, "subagent_history_start_ordinal": 10, "thread_source": "subagent", - "source": {"subagent": {"thread_spawn": {"parent_thread_id": "missing-parent"}}} + "source": {"subagent": {"thread_spawn": {"parent_thread_id": parent_id}}} } }), token_row(base, 2, [1_000, 900, 100], [0, 0, 0], "gpt-5.6-sol"), @@ -236,6 +237,7 @@ fn copied_prefix_subagent_infers_advancing_baseline_without_parent() { let child = write_copied_prefix_subagent_fixture( &sessions, "child.jsonl", + "missing-parent", Utc::now() - Duration::hours(1), true, ); @@ -275,6 +277,7 @@ fn copied_prefix_subagent_inherited_only_suffix_is_not_billed() { let child = write_copied_prefix_subagent_fixture( &sessions, "child.jsonl", + "missing-parent", Utc::now() - Duration::hours(1), false, ); @@ -301,6 +304,45 @@ fn copied_prefix_subagent_inherited_only_suffix_is_not_billed() { assert!(stats.codex_history_read_paths.is_empty()); } +#[test] +fn copied_prefix_subagent_prefers_validated_parent_baseline() { + let root = tempfile::tempdir().unwrap(); + let sessions = root.path().join("sessions"); + let cache_root = root.path().join("cache"); + let base = Utc::now() - Duration::hours(1); + write_codex_fork_session_fixture( + &sessions, + "parent.jsonl", + "parent-id", + None, + base, + base, + &[1_000], + ); + let child = write_copied_prefix_subagent_fixture( + &sessions, + "child.jsonl", + "parent-id", + base + Duration::seconds(10), + true, + ); + let mut options = CostScanOptions::app_driven(); + options.prefer_newest_codex_sessions_first = false; + let scanner = CostScanner::new(7) + .with_options(options) + .with_cache_root(&cache_root) + .with_sessions_dirs(vec![sessions]); + + let (_, _, cache) = scanner.scan_codex_detailed_with_cache(None); + let state = cache.files[&child.to_string_lossy().to_string()] + .codex_fork_accounting_state + .as_ref() + .unwrap(); + + assert_eq!(state.inherited_totals.as_ref().unwrap().input, 1_000); + assert!(!state.locally_resolved); +} + #[test] fn paginated_continuation_raises_inherited_baseline_from_total_last() { let root = tempfile::tempdir().unwrap(); From d3be0188aa0c4672f9ad3115b6c516f24fa908f2 Mon Sep 17 00:00:00 2001 From: NessZerra <90105158+Finesssee@users.noreply.github.com> Date: Tue, 22 Sep 2026 22:47:15 +0700 Subject: [PATCH 05/19] Fix Codex fork accounting test access --- rust/src/core/jsonl_scanner/tests.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/rust/src/core/jsonl_scanner/tests.rs b/rust/src/core/jsonl_scanner/tests.rs index 9296583fb8..2614ebefd3 100644 --- a/rust/src/core/jsonl_scanner/tests.rs +++ b/rust/src/core/jsonl_scanner/tests.rs @@ -139,9 +139,9 @@ fn inferred_fork_waits_for_present_explicit_start_ordinal() { ); assert_eq!(state.records.len(), 1); - assert_eq!(state.records[0].input, 10); - assert_eq!(state.records[0].cached, 2); - assert_eq!(state.records[0].output, 1); + assert_eq!(state.records[0].0.input, 10); + assert_eq!(state.records[0].0.cached, 2); + assert_eq!(state.records[0].0.output, 1); } #[test] From f64861c0a4cf40919a215f6455673320e2c6f463 Mon Sep 17 00:00:00 2001 From: NessZerra <90105158+Finesssee@users.noreply.github.com> Date: Tue, 22 Sep 2026 23:19:56 +0700 Subject: [PATCH 06/19] Fix Codex fork baseline provenance --- rust/src/cost_scanner/codex.rs | 76 ++++++++++++++++++------ rust/src/cost_scanner/tests/paginated.rs | 59 ++++++++++++++++++ 2 files changed, 117 insertions(+), 18 deletions(-) diff --git a/rust/src/cost_scanner/codex.rs b/rust/src/cost_scanner/codex.rs index 56a0f9724a..7a9d3068dc 100644 --- a/rust/src/cost_scanner/codex.rs +++ b/rust/src/cost_scanner/codex.rs @@ -16,11 +16,11 @@ use reconciliation::*; #[derive(Debug)] enum CodexAccountingMode { Standard, - ValidatedBaseline { + Baseline { baseline: crate::core::CodexTotals, paginated_continuation: bool, remaining_inherited_totals: Option, - locally_resolved: bool, + provenance: CodexBaselineProvenance, }, InferSubagent { start_ordinal: Option, @@ -28,6 +28,13 @@ enum CodexAccountingMode { Unresolved, } +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum CodexBaselineProvenance { + ValidatedParent { replaces_cached_state: bool }, + CachedValidatedParent, + CachedLocalInference, +} + impl CodexAccountingMode { fn is_unresolved(&self) -> bool { matches!(self, Self::Unresolved) @@ -40,8 +47,20 @@ impl CodexAccountingMode { fn locally_resolved(&self) -> bool { matches!( self, - Self::ValidatedBaseline { - locally_resolved: true, + Self::Baseline { + provenance: CodexBaselineProvenance::CachedLocalInference, + .. + } + ) + } + + fn requires_cached_reparse(&self) -> bool { + matches!( + self, + Self::Baseline { + provenance: CodexBaselineProvenance::ValidatedParent { + replaces_cached_state: true + }, .. } ) @@ -508,17 +527,12 @@ impl CostScanner { let matching_cached_fork_state = cached_fork_accounting_state .as_ref() .filter(|_| cached_fork_state_matches); - let cached_fork_baseline = - matching_cached_fork_state.and_then(|state| state.inherited_totals.clone()); let parent_fork_baseline = is_fork .then_some(codex_forked_from_id.as_deref()) .flatten() .and_then(|parent_id| { codex_parent_baseline(cache, parent_id, codex_fork_timestamp.as_deref()) }); - let fork_baseline = cached_fork_baseline.or(parent_fork_baseline); - let remaining_inherited_totals = - matching_cached_fork_state.and_then(|state| state.remaining_inherited_totals.clone()); let paginated_continuation = is_fork && codex_forked_from_id.is_some() && history_base_thread_id @@ -526,13 +540,34 @@ impl CostScanner { .is_some_and(|history_base| Some(history_base) != codex_forked_from_id.as_deref()); let accounting_mode = if !is_fork { CodexAccountingMode::Standard - } else if let Some(baseline) = fork_baseline { - CodexAccountingMode::ValidatedBaseline { + } else if let Some(baseline) = parent_fork_baseline { + let reparse_cached_file = matching_cached_fork_state.is_some_and(|state| { + state.locally_resolved || state.inherited_totals.as_ref() != Some(&baseline) + }); + let cached_parent_state = matching_cached_fork_state.filter(|state| { + !state.locally_resolved && state.inherited_totals.as_ref() == Some(&baseline) + }); + CodexAccountingMode::Baseline { baseline, paginated_continuation, - remaining_inherited_totals, - locally_resolved: matching_cached_fork_state - .is_some_and(|state| state.locally_resolved), + remaining_inherited_totals: cached_parent_state + .and_then(|state| state.remaining_inherited_totals.clone()), + provenance: CodexBaselineProvenance::ValidatedParent { + replaces_cached_state: reparse_cached_file, + }, + } + } else if let Some(state) = matching_cached_fork_state + && let Some(baseline) = state.inherited_totals.clone() + { + CodexAccountingMode::Baseline { + baseline, + paginated_continuation, + remaining_inherited_totals: state.remaining_inherited_totals.clone(), + provenance: if state.locally_resolved { + CodexBaselineProvenance::CachedLocalInference + } else { + CodexBaselineProvenance::CachedValidatedParent + }, } } else if session_metadata.is_subagent { CodexAccountingMode::InferSubagent { @@ -575,6 +610,7 @@ impl CostScanner { && cached_codex_file_is_fresh(cache, entry, cache_covers_range, mtime_ms, size) && (entry.codex_file_identity.is_none() || identity_matches_cached(entry)) && !cached_identity_changed + && !accounting_mode.requires_cached_reparse() { let (session_cost, has_tokens) = add_codex_days_map_to_summary(summary, &entry.days, range); @@ -680,9 +716,13 @@ impl CostScanner { } } - let parse_target_size = cached - .as_ref() - .and_then(|entry| codex_resumable_scan_target_size(size, entry)); + let parse_target_size = (!accounting_mode.requires_cached_reparse()) + .then(|| { + cached + .as_ref() + .and_then(|entry| codex_resumable_scan_target_size(size, entry)) + }) + .flatten(); let parse_result = match match &accounting_mode { CodexAccountingMode::Standard => JsonlScanner::parse_codex_file_with_state_bounded( path, @@ -695,7 +735,7 @@ impl CostScanner { cancel, max_bytes_to_read, ), - CodexAccountingMode::ValidatedBaseline { + CodexAccountingMode::Baseline { baseline, paginated_continuation, remaining_inherited_totals, diff --git a/rust/src/cost_scanner/tests/paginated.rs b/rust/src/cost_scanner/tests/paginated.rs index a225233a7d..be8d3972e5 100644 --- a/rust/src/cost_scanner/tests/paginated.rs +++ b/rust/src/cost_scanner/tests/paginated.rs @@ -343,6 +343,65 @@ fn copied_prefix_subagent_prefers_validated_parent_baseline() { assert!(!state.locally_resolved); } +#[test] +fn copied_prefix_subagent_replaces_cached_inference_when_parent_appears() { + let root = tempfile::tempdir().unwrap(); + let sessions = root.path().join("sessions"); + let cache_root = root.path().join("cache"); + let base = Utc::now() - Duration::hours(1); + let child = write_copied_prefix_subagent_fixture( + &sessions, + "child.jsonl", + "parent-id", + base + Duration::seconds(10), + true, + ); + let mut options = CostScanOptions::app_driven(); + options.prefer_newest_codex_sessions_first = false; + let scanner = CostScanner::new(7) + .with_options(options) + .with_cache_root(&cache_root) + .with_sessions_dirs(vec![sessions.clone()]); + + let (_, _, inferred_cache) = scanner.scan_codex_detailed_with_cache(None); + let inferred_state = inferred_cache.files[&child.to_string_lossy().to_string()] + .codex_fork_accounting_state + .as_ref() + .unwrap(); + assert!(inferred_state.locally_resolved); + assert_eq!( + inferred_state.inherited_totals.as_ref().unwrap().input, + 5_000 + ); + + write_codex_fork_session_fixture( + &sessions, + "parent.jsonl", + "parent-id", + None, + base, + base, + &[1_000], + ); + + let (_, stats, validated_cache) = scanner.scan_codex_detailed_with_cache(None); + let validated_state = validated_cache.files[&child.to_string_lossy().to_string()] + .codex_fork_accounting_state + .as_ref() + .unwrap(); + assert!(!validated_state.locally_resolved); + assert_eq!( + validated_state.inherited_totals.as_ref().unwrap().input, + 1_000 + ); + assert!( + stats + .codex_history_read_paths + .contains(&child.to_string_lossy().to_string()), + "the unchanged child must be reparsed when baseline provenance changes" + ); +} + #[test] fn paginated_continuation_raises_inherited_baseline_from_total_last() { let root = tempfile::tempdir().unwrap(); From 3521a2389252a21d9b6a6639fd9a604350907f31 Mon Sep 17 00:00:00 2001 From: NessZerra <90105158+Finesssee@users.noreply.github.com> Date: Tue, 22 Sep 2026 23:33:53 +0700 Subject: [PATCH 07/19] Fix Codex parent baseline transition order --- rust/src/cost_scanner/codex.rs | 40 +++++++++++++------ rust/src/cost_scanner/codex/logical_target.rs | 32 +++++++++++++++ rust/src/cost_scanner/codex/scan.rs | 1 + rust/src/cost_scanner/tests/paginated.rs | 32 +++++++++++++-- 4 files changed, 88 insertions(+), 17 deletions(-) diff --git a/rust/src/cost_scanner/codex.rs b/rust/src/cost_scanner/codex.rs index 7a9d3068dc..1f6d41eb9b 100644 --- a/rust/src/cost_scanner/codex.rs +++ b/rust/src/cost_scanner/codex.rs @@ -96,24 +96,38 @@ fn summary_from_cached_report( } fn codex_fork_parent_is_safe(cache: &CostUsageCache, usage: &CostUsageFileUsage) -> bool { - if usage + let locally_resolved = usage .codex_fork_accounting_state .as_ref() - .is_some_and(|state| state.locally_resolved) - { - return true; - } + .is_some_and(|state| state.locally_resolved); let uses_parent_baseline = usage.codex_lineage.uses_parent_baseline() || (matches!(usage.codex_lineage, CodexSessionLineage::Root) && usage.codex_forked_from_id.is_some()); - !uses_parent_baseline - || usage - .codex_forked_from_id - .as_deref() - .is_some_and(|parent_id| { - codex_parent_baseline(cache, parent_id, usage.codex_fork_timestamp.as_deref()) - .is_some() - }) + if !uses_parent_baseline { + return true; + } + let parent_is_available = usage + .codex_forked_from_id + .as_deref() + .is_some_and(|parent_id| { + codex_parent_baseline(cache, parent_id, usage.codex_fork_timestamp.as_deref()).is_some() + }); + + // Local inference is safe only while no validated parent is available. + // Once the parent enters the cache, force the child through baseline + // replacement instead of accepting its unchanged-file fast path. + if locally_resolved { + !parent_is_available + } else { + parent_is_available + } +} + +fn codex_fork_uses_local_inference(usage: &CostUsageFileUsage) -> bool { + usage + .codex_fork_accounting_state + .as_ref() + .is_some_and(|state| state.locally_resolved) } /// Return a parent cumulative baseline only when exactly one cached session diff --git a/rust/src/cost_scanner/codex/logical_target.rs b/rust/src/cost_scanner/codex/logical_target.rs index a98fb1e63e..3738a85f87 100644 --- a/rust/src/cost_scanner/codex/logical_target.rs +++ b/rust/src/cost_scanner/codex/logical_target.rs @@ -42,10 +42,42 @@ pub(super) fn cached_codex_file_is_complete_for_range( && codex_scan_target_size(usage) == size && usage.parsed_bytes.unwrap_or(0) >= size && !usage.codex_unresolved_fork_parent + // Reconsider locally inferred children after this pass has + // had a chance to discover and cache their parent. + && !super::codex_fork_uses_local_inference(usage) && super::codex_fork_parent_is_safe(cache, usage) }) } +/// Process cached local-inference children after all other candidates. A +/// parent discovered in this pass must enter the cache before its unchanged +/// child can decide whether the inferred baseline is still authoritative. +pub(super) fn defer_codex_locally_inferred_candidates( + candidates: &mut Vec, + cache: &CostUsageCache, +) { + if candidates.len() < 2 { + return; + } + + let mut other = Vec::with_capacity(candidates.len()); + let mut locally_inferred = Vec::new(); + for candidate in candidates.drain(..) { + let path_key = candidate.path.to_string_lossy(); + if cache + .files + .get(path_key.as_ref()) + .is_some_and(super::codex_fork_uses_local_inference) + { + locally_inferred.push(candidate); + } else { + other.push(candidate); + } + } + other.extend(locally_inferred); + candidates.extend(other); +} + /// Give paths already in the durable queue their saved turn before newly /// discovered dirty paths. The scanner appends unfinished paths after this /// pass, making the queue a round-robin cursor instead of a newest-first loop. diff --git a/rust/src/cost_scanner/codex/scan.rs b/rust/src/cost_scanner/codex/scan.rs index e983adfd6a..ab91037a59 100644 --- a/rust/src/cost_scanner/codex/scan.rs +++ b/rust/src/cost_scanner/codex/scan.rs @@ -198,6 +198,7 @@ pub(super) fn scan_codex_detailed_with_cache( let mut pending_next = cache.codex_pending_paths.clone(); let pending_paths_before_pass = cache.codex_pending_paths.clone(); prioritize_codex_pending_candidates(&mut candidates, &pending_paths_before_pass); + defer_codex_locally_inferred_candidates(&mut candidates, &cache); if discovery_complete && !is_cancelled(cancel) { pending_next .retain(|path| !cached_codex_file_is_complete_for_range(&cache, path, scan_range)); diff --git a/rust/src/cost_scanner/tests/paginated.rs b/rust/src/cost_scanner/tests/paginated.rs index be8d3972e5..d377728c4f 100644 --- a/rust/src/cost_scanner/tests/paginated.rs +++ b/rust/src/cost_scanner/tests/paginated.rs @@ -326,6 +326,13 @@ fn copied_prefix_subagent_prefers_validated_parent_baseline() { base + Duration::seconds(10), true, ); + let now = std::time::SystemTime::now(); + std::fs::OpenOptions::new() + .write(true) + .open(&child) + .unwrap() + .set_modified(now - std::time::Duration::from_secs(20)) + .unwrap(); let mut options = CostScanOptions::app_driven(); options.prefer_newest_codex_sessions_first = false; let scanner = CostScanner::new(7) @@ -343,8 +350,9 @@ fn copied_prefix_subagent_prefers_validated_parent_baseline() { assert!(!state.locally_resolved); } -#[test] -fn copied_prefix_subagent_replaces_cached_inference_when_parent_appears() { +fn assert_cached_inference_is_replaced_when_parent_appears( + prefer_newest_codex_sessions_first: bool, +) { let root = tempfile::tempdir().unwrap(); let sessions = root.path().join("sessions"); let cache_root = root.path().join("cache"); @@ -357,7 +365,7 @@ fn copied_prefix_subagent_replaces_cached_inference_when_parent_appears() { true, ); let mut options = CostScanOptions::app_driven(); - options.prefer_newest_codex_sessions_first = false; + options.prefer_newest_codex_sessions_first = prefer_newest_codex_sessions_first; let scanner = CostScanner::new(7) .with_options(options) .with_cache_root(&cache_root) @@ -374,7 +382,7 @@ fn copied_prefix_subagent_replaces_cached_inference_when_parent_appears() { 5_000 ); - write_codex_fork_session_fixture( + let parent = write_codex_fork_session_fixture( &sessions, "parent.jsonl", "parent-id", @@ -383,6 +391,12 @@ fn copied_prefix_subagent_replaces_cached_inference_when_parent_appears() { base, &[1_000], ); + std::fs::OpenOptions::new() + .write(true) + .open(parent) + .unwrap() + .set_modified(now - std::time::Duration::from_secs(10)) + .unwrap(); let (_, stats, validated_cache) = scanner.scan_codex_detailed_with_cache(None); let validated_state = validated_cache.files[&child.to_string_lossy().to_string()] @@ -402,6 +416,16 @@ fn copied_prefix_subagent_replaces_cached_inference_when_parent_appears() { ); } +#[test] +fn copied_prefix_subagent_replaces_cached_inference_when_parent_is_visited_first() { + assert_cached_inference_is_replaced_when_parent_appears(true); +} + +#[test] +fn copied_prefix_subagent_replaces_cached_inference_when_child_would_be_visited_first() { + assert_cached_inference_is_replaced_when_parent_appears(false); +} + #[test] fn paginated_continuation_raises_inherited_baseline_from_total_last() { let root = tempfile::tempdir().unwrap(); From f7d1ad974c9e8d136610d1a59c55b5cde781266c Mon Sep 17 00:00:00 2001 From: NessZerra <90105158+Finesssee@users.noreply.github.com> Date: Tue, 22 Sep 2026 23:43:42 +0700 Subject: [PATCH 08/19] Fix fork cache replacement test --- rust/src/cost_scanner/tests/paginated.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/rust/src/cost_scanner/tests/paginated.rs b/rust/src/cost_scanner/tests/paginated.rs index d377728c4f..f4ce5fc138 100644 --- a/rust/src/cost_scanner/tests/paginated.rs +++ b/rust/src/cost_scanner/tests/paginated.rs @@ -391,6 +391,7 @@ fn assert_cached_inference_is_replaced_when_parent_appears( base, &[1_000], ); + let now = std::time::SystemTime::now(); std::fs::OpenOptions::new() .write(true) .open(parent) From 0735ff5a68a8ceb832b38916f89178eb7c8c410b Mon Sep 17 00:00:00 2001 From: NessZerra <90105158+Finesssee@users.noreply.github.com> Date: Wed, 23 Sep 2026 00:02:59 +0700 Subject: [PATCH 09/19] Reconcile inferred Codex forks in one scan --- rust/src/cost_scanner/codex/scan.rs | 72 ++++++++++++++++++++++++++++- 1 file changed, 70 insertions(+), 2 deletions(-) diff --git a/rust/src/cost_scanner/codex/scan.rs b/rust/src/cost_scanner/codex/scan.rs index ab91037a59..54eab62379 100644 --- a/rust/src/cost_scanner/codex/scan.rs +++ b/rust/src/cost_scanner/codex/scan.rs @@ -205,6 +205,7 @@ pub(super) fn scan_codex_detailed_with_cache( } let mut incomplete_processed = Vec::new(); + let mut locally_inferred_complete_paths = Vec::new(); for (index, candidate) in candidates.iter().enumerate() { if is_cancelled(cancel) || index >= candidate_limit @@ -272,8 +273,75 @@ pub(super) fn scan_codex_detailed_with_cache( if !outcome.is_complete || has_unconsumed_tail { incomplete_processed.push(key); stats.files_deferred = stats.files_deferred.saturating_add(1); - } else if let Some(plan) = codex_source_row_plan(&cache, &candidate.path, scan_range) { - apply_codex_source_row_plan(&mut cache, &key, plan); + } else { + if cache + .files + .get(&key) + .is_some_and(codex_fork_uses_local_inference) + { + locally_inferred_complete_paths.push(candidate.path.clone()); + } + if let Some(plan) = codex_source_row_plan(&cache, &candidate.path, scan_range) { + apply_codex_source_row_plan(&mut cache, &key, plan); + } + } + } + + // A child can be visited before its parent during a cold scan. Once the + // remaining candidates have populated the cache, replace that temporary + // local inference in the same refresh instead of publishing it for one + // cycle. Reconciliation still consumes the normal byte budget; work that + // no longer fits is queued for the next explicit refresh. + for path in locally_inferred_complete_paths { + let key = path.to_string_lossy().to_string(); + let parent_is_now_available = cache.files.get(&key).is_some_and(|usage| { + codex_fork_uses_local_inference(usage) && !codex_fork_parent_is_safe(&cache, usage) + }); + if !parent_is_now_available { + continue; + } + let allowance = + per_file_limit.min(refresh_byte_limit.saturating_sub(bytes_read_this_refresh)); + if is_cancelled(cancel) || allowance <= 0 { + if !pending_next.contains(&key) { + pending_next.push(key); + } + stats.files_deferred = stats.files_deferred.saturating_add(1); + continue; + } + + let outcome = scanner.parse_codex_file_bounded( + &path, + scan_range, + &mut summary, + &mut cache, + cancel, + &mut stats, + Some(allowance), + ); + bytes_read_this_refresh = bytes_read_this_refresh.saturating_add(outcome.bytes_read.max(0)); + stats.codex_bytes_read = stats + .codex_bytes_read + .saturating_add(u64::try_from(outcome.bytes_read.max(0)).unwrap_or(u64::MAX)); + pending_next.retain(|pending| pending != &key); + let observed_size = fs::metadata(&path) + .ok() + .map(|metadata| { + #[allow( + clippy::cast_possible_wrap, + reason = "file sizes are clamped to i64::MAX" + )] + let size = metadata.len().min(i64::MAX as u64) as i64; + size + }) + .unwrap_or(0); + let has_unconsumed_tail = cache + .files + .get(&key) + .is_some_and(|usage| codex_logical_target_has_unconsumed_tail(observed_size, usage)); + if !outcome.is_complete || has_unconsumed_tail { + incomplete_processed.push(key); + stats.files_deferred = stats.files_deferred.saturating_add(1); } } pending_next.extend(incomplete_processed); From 361655e80169d68270927107d0ec1dbce7d8b833 Mon Sep 17 00:00:00 2001 From: NessZerra <90105158+Finesssee@users.noreply.github.com> Date: Wed, 23 Sep 2026 00:22:47 +0700 Subject: [PATCH 10/19] Order Codex scan work by lineage --- rust/src/cost_scanner/codex.rs | 27 ++- rust/src/cost_scanner/codex/logical_target.rs | 55 +++++++ rust/src/cost_scanner/codex/scan.rs | 143 ++++++---------- rust/src/cost_scanner/tests/paginated.rs | 155 +++++++++++++++++- 4 files changed, 277 insertions(+), 103 deletions(-) diff --git a/rust/src/cost_scanner/codex.rs b/rust/src/cost_scanner/codex.rs index 1f6d41eb9b..39f96eea2c 100644 --- a/rust/src/cost_scanner/codex.rs +++ b/rust/src/cost_scanner/codex.rs @@ -1,5 +1,5 @@ use super::*; -use crate::core::{CodexForkAccountingState, CodexSessionLineage}; +use crate::core::{CodexForkAccountingState, CodexSessionLineage, CodexSessionMetadata}; mod cache_days; mod logical_target; @@ -204,6 +204,11 @@ struct CodexScanCandidate { mtime_unix_ms: i64, } +struct CodexPreparedCandidate { + path: PathBuf, + session_metadata: CodexSessionMetadata, +} + #[derive(Debug, Clone, Copy, Default)] struct CodexFileScanOutcome { bytes_read: i64, @@ -381,7 +386,8 @@ impl CostScanner { cancel: Option<&AtomicBool>, stats: &mut CostScanStats, ) { - let _ = self.parse_codex_file_bounded(path, range, summary, cache, cancel, stats, None); + let _ = + self.parse_codex_file_bounded(path, range, summary, cache, cancel, stats, None, None); } #[allow( @@ -397,11 +403,14 @@ impl CostScanner { cancel: Option<&AtomicBool>, stats: &mut CostScanStats, max_bytes_to_read: Option, + prepared_session_metadata: Option<&CodexSessionMetadata>, ) -> CodexFileScanOutcome { if is_cancelled(cancel) { return CodexFileScanOutcome::default(); } - stats.files_seen = stats.files_seen.saturating_add(1); + if prepared_session_metadata.is_none() { + stats.files_seen = stats.files_seen.saturating_add(1); + } let metadata = match fs::metadata(path) { Ok(metadata) => metadata, @@ -461,10 +470,14 @@ impl CostScanner { }; } - stats.codex_metadata_read_paths.push(path_key.clone()); - stats.codex_read_receipt.metadata_reads = - stats.codex_read_receipt.metadata_reads.saturating_add(1); - let session_metadata = JsonlScanner::read_codex_session_metadata(path).unwrap_or_default(); + let session_metadata = if let Some(prepared) = prepared_session_metadata { + prepared.clone() + } else { + stats.codex_metadata_read_paths.push(path_key.clone()); + stats.codex_read_receipt.metadata_reads = + stats.codex_read_receipt.metadata_reads.saturating_add(1); + JsonlScanner::read_codex_session_metadata(path).unwrap_or_default() + }; let cached_identity_matches = cached .as_ref() .is_some_and(|entry| entry.mtime_unix_ms == mtime_ms && entry.size == size); diff --git a/rust/src/cost_scanner/codex/logical_target.rs b/rust/src/cost_scanner/codex/logical_target.rs index 3738a85f87..d5b0350396 100644 --- a/rust/src/cost_scanner/codex/logical_target.rs +++ b/rust/src/cost_scanner/codex/logical_target.rs @@ -78,6 +78,61 @@ pub(super) fn defer_codex_locally_inferred_candidates( candidates.extend(other); } +/// Order one bounded work set so every uniquely identified parent is parsed +/// before its children. The sort is stable for unrelated candidates and falls +/// back to discovery order for duplicate identities or dependency cycles. +pub(super) fn order_codex_candidates_by_lineage(candidates: &mut Vec) { + if candidates.len() < 2 { + return; + } + + let mut session_owners = HashMap::>::new(); + for (index, candidate) in candidates.iter().enumerate() { + let Some(session_id) = candidate.session_metadata.session_id.as_ref() else { + continue; + }; + session_owners + .entry(session_id.clone()) + .and_modify(|owner| *owner = None) + .or_insert(Some(index)); + } + let parent_indices = candidates + .iter() + .map(|candidate| { + candidate + .session_metadata + .forked_from_id + .as_ref() + .and_then(|parent_id| session_owners.get(parent_id)) + .copied() + .flatten() + }) + .collect::>(); + let mut remaining = candidates.drain(..).map(Some).collect::>(); + let mut ordered = Vec::with_capacity(remaining.len()); + + loop { + let mut progressed = false; + for index in 0..remaining.len() { + if remaining[index].is_none() { + continue; + } + let parent_is_ready = + parent_indices[index].is_none_or(|parent_index| remaining[parent_index].is_none()); + if parent_is_ready { + ordered.push(remaining[index].take().expect("candidate checked above")); + progressed = true; + } + } + if !progressed { + break; + } + } + + ordered.extend(remaining.into_iter().flatten()); + candidates.extend(ordered); +} + /// Give paths already in the durable queue their saved turn before newly /// discovered dirty paths. The scanner appends unfinished paths after this /// pass, making the queue a round-robin cursor instead of a newest-first loop. diff --git a/rust/src/cost_scanner/codex/scan.rs b/rust/src/cost_scanner/codex/scan.rs index 54eab62379..b83fe08f6c 100644 --- a/rust/src/cost_scanner/codex/scan.rs +++ b/rust/src/cost_scanner/codex/scan.rs @@ -204,38 +204,49 @@ pub(super) fn scan_codex_detailed_with_cache( .retain(|path| !cached_codex_file_is_complete_for_range(&cache, path, scan_range)); } - let mut incomplete_processed = Vec::new(); - let mut locally_inferred_complete_paths = Vec::new(); - for (index, candidate) in candidates.iter().enumerate() { - if is_cancelled(cancel) - || index >= candidate_limit - || bytes_read_this_refresh >= refresh_byte_limit - { - for deferred in &candidates[index..] { - let key = deferred.path.to_string_lossy().to_string(); - if !pending_next.contains(&key) { - pending_next.push(key); - } - } - stats.files_deferred = stats.files_deferred.saturating_add( - u32::try_from((candidates.len() - index).min(u32::MAX as usize)) - .unwrap_or(u32::MAX), - ); - break; + // Admit one bounded set, inspect each admitted candidate once, and order + // that set by lineage before reading token history. This makes cold + // child-before-parent scans parent-first without a second parse pass. + let deferred_candidates = candidates.split_off(candidate_limit.min(candidates.len())); + let deferred_paths = deferred_candidates + .into_iter() + .map(|candidate| candidate.path) + .collect::>(); + let mut work_queue = Vec::with_capacity(candidates.len()); + let mut cancelled_during_preparation = Vec::new(); + for candidate in candidates { + if is_cancelled(cancel) { + cancelled_during_preparation.push(candidate.path); + continue; } + let key = candidate.path.to_string_lossy().to_string(); + stats.files_seen = stats.files_seen.saturating_add(1); + stats.codex_metadata_read_paths.push(key); + stats.codex_read_receipt.metadata_reads = + stats.codex_read_receipt.metadata_reads.saturating_add(1); + work_queue.push(CodexPreparedCandidate { + session_metadata: JsonlScanner::read_codex_session_metadata(&candidate.path) + .unwrap_or_default(), + path: candidate.path, + }); + } + let mut unprocessed = Vec::new(); + if !cancelled_during_preparation.is_empty() || is_cancelled(cancel) { + unprocessed.extend(work_queue.drain(..).map(|candidate| candidate.path)); + unprocessed.extend(cancelled_during_preparation); + } else { + order_codex_candidates_by_lineage(&mut work_queue); + } + let mut incomplete_processed = Vec::new(); + for (index, candidate) in work_queue.iter().enumerate() { let refresh_remaining = refresh_byte_limit.saturating_sub(bytes_read_this_refresh); let allowance = per_file_limit.min(refresh_remaining); - if allowance <= 0 { - for deferred in &candidates[index..] { - let key = deferred.path.to_string_lossy().to_string(); - if !pending_next.contains(&key) { - pending_next.push(key); - } - } - stats.files_deferred = stats.files_deferred.saturating_add( - u32::try_from((candidates.len() - index).min(u32::MAX as usize)) - .unwrap_or(u32::MAX), + if is_cancelled(cancel) || allowance <= 0 { + unprocessed.extend( + work_queue[index..] + .iter() + .map(|candidate| candidate.path.clone()), ); break; } @@ -248,6 +259,7 @@ pub(super) fn scan_codex_detailed_with_cache( cancel, &mut stats, Some(allowance), + Some(&candidate.session_metadata), ); bytes_read_this_refresh = bytes_read_this_refresh.saturating_add(outcome.bytes_read.max(0)); stats.codex_bytes_read = stats @@ -273,75 +285,18 @@ pub(super) fn scan_codex_detailed_with_cache( if !outcome.is_complete || has_unconsumed_tail { incomplete_processed.push(key); stats.files_deferred = stats.files_deferred.saturating_add(1); - } else { - if cache - .files - .get(&key) - .is_some_and(codex_fork_uses_local_inference) - { - locally_inferred_complete_paths.push(candidate.path.clone()); - } - if let Some(plan) = codex_source_row_plan(&cache, &candidate.path, scan_range) { - apply_codex_source_row_plan(&mut cache, &key, plan); - } + } else if let Some(plan) = codex_source_row_plan(&cache, &candidate.path, scan_range) { + apply_codex_source_row_plan(&mut cache, &key, plan); } } - - // A child can be visited before its parent during a cold scan. Once the - // remaining candidates have populated the cache, replace that temporary - // local inference in the same refresh instead of publishing it for one - // cycle. Reconciliation still consumes the normal byte budget; work that - // no longer fits is queued for the next explicit refresh. - for path in locally_inferred_complete_paths { + unprocessed.extend(deferred_paths); + stats.files_deferred = stats.files_deferred.saturating_add( + u32::try_from(unprocessed.len().min(u32::MAX as usize)).unwrap_or(u32::MAX), + ); + for path in unprocessed { let key = path.to_string_lossy().to_string(); - let parent_is_now_available = cache.files.get(&key).is_some_and(|usage| { - codex_fork_uses_local_inference(usage) && !codex_fork_parent_is_safe(&cache, usage) - }); - if !parent_is_now_available { - continue; - } - let allowance = - per_file_limit.min(refresh_byte_limit.saturating_sub(bytes_read_this_refresh)); - if is_cancelled(cancel) || allowance <= 0 { - if !pending_next.contains(&key) { - pending_next.push(key); - } - stats.files_deferred = stats.files_deferred.saturating_add(1); - continue; - } - - let outcome = scanner.parse_codex_file_bounded( - &path, - scan_range, - &mut summary, - &mut cache, - cancel, - &mut stats, - Some(allowance), - ); - bytes_read_this_refresh = bytes_read_this_refresh.saturating_add(outcome.bytes_read.max(0)); - stats.codex_bytes_read = stats - .codex_bytes_read - .saturating_add(u64::try_from(outcome.bytes_read.max(0)).unwrap_or(u64::MAX)); - pending_next.retain(|pending| pending != &key); - let observed_size = fs::metadata(&path) - .ok() - .map(|metadata| { - #[allow( - clippy::cast_possible_wrap, - reason = "file sizes are clamped to i64::MAX" - )] - let size = metadata.len().min(i64::MAX as u64) as i64; - size - }) - .unwrap_or(0); - let has_unconsumed_tail = cache - .files - .get(&key) - .is_some_and(|usage| codex_logical_target_has_unconsumed_tail(observed_size, usage)); - if !outcome.is_complete || has_unconsumed_tail { - incomplete_processed.push(key); - stats.files_deferred = stats.files_deferred.saturating_add(1); + if !pending_next.contains(&key) { + pending_next.push(key); } } pending_next.extend(incomplete_processed); diff --git a/rust/src/cost_scanner/tests/paginated.rs b/rust/src/cost_scanner/tests/paginated.rs index f4ce5fc138..a42ad77405 100644 --- a/rust/src/cost_scanner/tests/paginated.rs +++ b/rust/src/cost_scanner/tests/paginated.rs @@ -310,7 +310,7 @@ fn copied_prefix_subagent_prefers_validated_parent_baseline() { let sessions = root.path().join("sessions"); let cache_root = root.path().join("cache"); let base = Utc::now() - Duration::hours(1); - write_codex_fork_session_fixture( + let parent = write_codex_fork_session_fixture( &sessions, "parent.jsonl", "parent-id", @@ -335,12 +335,16 @@ fn copied_prefix_subagent_prefers_validated_parent_baseline() { .unwrap(); let mut options = CostScanOptions::app_driven(); options.prefer_newest_codex_sessions_first = false; + let parent_size = std::fs::metadata(&parent).unwrap().len(); + let child_size = std::fs::metadata(&child).unwrap().len(); + options.codex_max_session_file_bytes = + i64::try_from(parent_size.max(child_size)).expect("fixture size fits i64"); let scanner = CostScanner::new(7) .with_options(options) .with_cache_root(&cache_root) .with_sessions_dirs(vec![sessions]); - let (_, _, cache) = scanner.scan_codex_detailed_with_cache(None); + let (_, stats, cache) = scanner.scan_codex_detailed_with_cache(None); let state = cache.files[&child.to_string_lossy().to_string()] .codex_fork_accounting_state .as_ref() @@ -348,6 +352,153 @@ fn copied_prefix_subagent_prefers_validated_parent_baseline() { assert_eq!(state.inherited_totals.as_ref().unwrap().input, 1_000); assert!(!state.locally_resolved); + assert_eq!(stats.files_seen, 2); + assert_eq!(stats.codex_read_receipt.metadata_reads, 2); + assert_eq!(stats.codex_read_receipt.history_reads, 2); + assert_eq!( + stats.codex_bytes_read, + parent_size.saturating_add(child_size), + "one bounded parse per candidate must enforce the per-file allowance" + ); + assert_eq!( + stats.codex_history_read_paths, + vec![ + parent.to_string_lossy().to_string(), + child.to_string_lossy().to_string(), + ] + ); +} + +#[test] +fn candidate_limit_counts_each_child_parent_candidate_once() { + let root = tempfile::tempdir().unwrap(); + let sessions = root.path().join("sessions"); + let cache_root = root.path().join("cache"); + let base = Utc::now() - Duration::hours(1); + let parent = write_codex_fork_session_fixture( + &sessions, + "parent.jsonl", + "parent-id", + None, + base, + base, + &[1_000], + ); + let child = write_copied_prefix_subagent_fixture( + &sessions, + "child.jsonl", + "parent-id", + base + Duration::seconds(10), + true, + ); + let now = std::time::SystemTime::now(); + std::fs::OpenOptions::new() + .write(true) + .open(&child) + .unwrap() + .set_modified(now - std::time::Duration::from_secs(20)) + .unwrap(); + std::fs::OpenOptions::new() + .write(true) + .open(&parent) + .unwrap() + .set_modified(now - std::time::Duration::from_secs(10)) + .unwrap(); + let mut options = CostScanOptions::app_driven(); + options.prefer_newest_codex_sessions_first = false; + options.codex_candidate_limit = 1; + let scanner = CostScanner::new(7) + .with_options(options) + .with_cache_root(&cache_root) + .with_sessions_dirs(vec![sessions]); + + let (_, stats, cache) = scanner.scan_codex_detailed_with_cache(None); + + assert_eq!(stats.files_seen, 1); + assert_eq!(stats.codex_read_receipt.metadata_reads, 1); + assert_eq!(stats.codex_read_receipt.history_reads, 1); + assert_eq!( + stats.codex_metadata_read_paths, + vec![child.to_string_lossy().to_string()] + ); + assert_eq!( + cache.codex_pending_paths, + vec![parent.to_string_lossy().to_string()] + ); +} + +#[test] +fn cold_scan_orders_multi_level_parent_chain_before_children() { + let root = tempfile::tempdir().unwrap(); + let sessions = root.path().join("sessions"); + let cache_root = root.path().join("cache"); + let base = Utc::now() - Duration::hours(1); + let ancestor = write_codex_fork_session_fixture( + &sessions, + "ancestor.jsonl", + "ancestor-id", + None, + base, + base, + &[1_000], + ); + let parent = write_codex_fork_session_fixture( + &sessions, + "parent.jsonl", + "parent-id", + Some("ancestor-id"), + base + Duration::seconds(10), + base + Duration::seconds(10), + &[1_500], + ); + let child = write_codex_fork_session_fixture( + &sessions, + "child.jsonl", + "child-id", + Some("parent-id"), + base + Duration::seconds(20), + base + Duration::seconds(20), + &[2_000], + ); + let now = std::time::SystemTime::now(); + for (path, age) in [(&child, 30), (&parent, 20), (&ancestor, 10)] { + std::fs::OpenOptions::new() + .write(true) + .open(path) + .unwrap() + .set_modified(now - std::time::Duration::from_secs(age)) + .unwrap(); + } + let mut options = CostScanOptions::app_driven(); + options.prefer_newest_codex_sessions_first = false; + let scanner = CostScanner::new(7) + .with_options(options) + .with_cache_root(&cache_root) + .with_sessions_dirs(vec![sessions]); + + let (_, stats, cache) = scanner.scan_codex_detailed_with_cache(None); + + assert_eq!(stats.files_seen, 3); + assert_eq!(stats.codex_read_receipt.metadata_reads, 3); + assert_eq!(stats.codex_read_receipt.history_reads, 3); + assert_eq!( + stats.codex_history_read_paths, + vec![ + ancestor.to_string_lossy().to_string(), + parent.to_string_lossy().to_string(), + child.to_string_lossy().to_string(), + ] + ); + let parent_state = cache.files[&parent.to_string_lossy().to_string()] + .codex_fork_accounting_state + .as_ref() + .unwrap(); + let child_state = cache.files[&child.to_string_lossy().to_string()] + .codex_fork_accounting_state + .as_ref() + .unwrap(); + assert_eq!(parent_state.inherited_totals.as_ref().unwrap().input, 1_000); + assert_eq!(child_state.inherited_totals.as_ref().unwrap().input, 1_500); } fn assert_cached_inference_is_replaced_when_parent_appears( From 4d0b5993a5b772f491c950b68be2b79a6acdc2ee Mon Sep 17 00:00:00 2001 From: NessZerra <90105158+Finesssee@users.noreply.github.com> Date: Wed, 23 Sep 2026 00:40:49 +0700 Subject: [PATCH 11/19] Fail closed on ambiguous Codex lineage --- rust/src/cost_scanner/codex.rs | 25 +- rust/src/cost_scanner/codex/logical_target.rs | 61 +- rust/src/cost_scanner/codex/scan.rs | 3 +- rust/src/cost_scanner/tests.rs | 3 + rust/src/cost_scanner/tests/copied_prefix.rs | 578 ++++++++++++++++++ rust/src/cost_scanner/tests/paginated.rs | 443 -------------- 6 files changed, 641 insertions(+), 472 deletions(-) create mode 100644 rust/src/cost_scanner/tests/copied_prefix.rs diff --git a/rust/src/cost_scanner/codex.rs b/rust/src/cost_scanner/codex.rs index 39f96eea2c..aab7def35d 100644 --- a/rust/src/cost_scanner/codex.rs +++ b/rust/src/cost_scanner/codex.rs @@ -207,6 +207,14 @@ struct CodexScanCandidate { struct CodexPreparedCandidate { path: PathBuf, session_metadata: CodexSessionMetadata, + lineage_disposition: CodexLineageDisposition, +} + +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] +enum CodexLineageDisposition { + #[default] + Ready, + AmbiguousOrCyclic, } #[derive(Debug, Clone, Copy, Default)] @@ -403,12 +411,12 @@ impl CostScanner { cancel: Option<&AtomicBool>, stats: &mut CostScanStats, max_bytes_to_read: Option, - prepared_session_metadata: Option<&CodexSessionMetadata>, + prepared_candidate: Option<&CodexPreparedCandidate>, ) -> CodexFileScanOutcome { if is_cancelled(cancel) { return CodexFileScanOutcome::default(); } - if prepared_session_metadata.is_none() { + if prepared_candidate.is_none() { stats.files_seen = stats.files_seen.saturating_add(1); } @@ -454,6 +462,9 @@ impl CostScanner { // before reading even the bounded metadata prefix; raw token history // is only needed after freshness fails or a fork needs reconciliation. if let Some(entry) = cached.as_ref() + && prepared_candidate.is_none_or(|candidate| { + candidate.lineage_disposition == CodexLineageDisposition::Ready + }) && cache_entry_is_fresh(entry) && identity_matches_cached(entry) { @@ -470,8 +481,8 @@ impl CostScanner { }; } - let session_metadata = if let Some(prepared) = prepared_session_metadata { - prepared.clone() + let session_metadata = if let Some(prepared) = prepared_candidate { + prepared.session_metadata.clone() } else { stats.codex_metadata_read_paths.push(path_key.clone()); stats.codex_read_receipt.metadata_reads = @@ -565,7 +576,11 @@ impl CostScanner { && history_base_thread_id .as_deref() .is_some_and(|history_base| Some(history_base) != codex_forked_from_id.as_deref()); - let accounting_mode = if !is_fork { + let accounting_mode = if prepared_candidate.is_some_and(|candidate| { + candidate.lineage_disposition == CodexLineageDisposition::AmbiguousOrCyclic + }) { + CodexAccountingMode::Unresolved + } else if !is_fork { CodexAccountingMode::Standard } else if let Some(baseline) = parent_fork_baseline { let reparse_cached_file = matching_cached_fork_state.is_some_and(|state| { diff --git a/rust/src/cost_scanner/codex/logical_target.rs b/rust/src/cost_scanner/codex/logical_target.rs index d5b0350396..b22b7ee6c3 100644 --- a/rust/src/cost_scanner/codex/logical_target.rs +++ b/rust/src/cost_scanner/codex/logical_target.rs @@ -79,48 +79,60 @@ pub(super) fn defer_codex_locally_inferred_candidates( } /// Order one bounded work set so every uniquely identified parent is parsed -/// before its children. The sort is stable for unrelated candidates and falls -/// back to discovery order for duplicate identities or dependency cycles. +/// before its children. Duplicate identities, cycles, and every dependent +/// candidate are marked unsafe so parsing cannot accept or infer a baseline +/// from ambiguous lineage. pub(super) fn order_codex_candidates_by_lineage(candidates: &mut Vec) { - if candidates.len() < 2 { + if candidates.is_empty() { return; } - let mut session_owners = HashMap::>::new(); + let mut session_owners = HashMap::>::new(); for (index, candidate) in candidates.iter().enumerate() { let Some(session_id) = candidate.session_metadata.session_id.as_ref() else { continue; }; session_owners .entry(session_id.clone()) - .and_modify(|owner| *owner = None) - .or_insert(Some(index)); + .or_default() + .push(index); } - let parent_indices = candidates - .iter() - .map(|candidate| { - candidate - .session_metadata - .forked_from_id - .as_ref() - .and_then(|parent_id| session_owners.get(parent_id)) - .copied() - .flatten() - }) - .collect::>(); + let mut unsafe_lineage = vec![false; candidates.len()]; + for owners in session_owners.values().filter(|owners| owners.len() > 1) { + for &index in owners { + unsafe_lineage[index] = true; + } + } + let mut parent_indices = vec![None; candidates.len()]; + for (index, candidate) in candidates.iter().enumerate() { + let Some(parent_id) = candidate.session_metadata.forked_from_id.as_ref() else { + continue; + }; + match session_owners.get(parent_id).map(Vec::as_slice) { + Some([parent_index]) => parent_indices[index] = Some(*parent_index), + Some([]) | None => {} + Some(_) => unsafe_lineage[index] = true, + } + } + let mut remaining = candidates.drain(..).map(Some).collect::>(); let mut ordered = Vec::with_capacity(remaining.len()); + let mut completed = vec![false; remaining.len()]; loop { let mut progressed = false; for index in 0..remaining.len() { - if remaining[index].is_none() { + if remaining[index].is_none() || unsafe_lineage[index] { continue; } - let parent_is_ready = - parent_indices[index].is_none_or(|parent_index| remaining[parent_index].is_none()); + let parent_is_ready = parent_indices[index].is_none_or(|parent_index| { + completed[parent_index] && !unsafe_lineage[parent_index] + }); if parent_is_ready { - ordered.push(remaining[index].take().expect("candidate checked above")); + let mut candidate = remaining[index].take().expect("candidate checked above"); + candidate.lineage_disposition = CodexLineageDisposition::Ready; + ordered.push(candidate); + completed[index] = true; progressed = true; } } @@ -129,7 +141,10 @@ pub(super) fn order_codex_candidates_by_lineage(candidates: &mut Vec, + owned: bool, +) -> PathBuf { + let day = base.with_timezone(&Local).date_naive(); + let day_dir = sessions_root + .join(day.format("%Y").to_string()) + .join(day.format("%m").to_string()) + .join(day.format("%d").to_string()); + std::fs::create_dir_all(&day_dir).unwrap(); + let path = day_dir.join(name); + let mut lines = vec![ + serde_json::json!({ + "type": "session_meta", "ordinal": 0, "timestamp": base.to_rfc3339(), + "payload": { + "id": session_id, "forked_from_id": parent_id, + "subagent_history_start_ordinal": 10, + "thread_source": "subagent", + "source": {"subagent": {"thread_spawn": {"parent_thread_id": parent_id}}} + } + }), + token_row(base, 2, [1_000, 900, 100], [0, 0, 0], "gpt-5.6-sol"), + serde_json::json!({ + "type": "turn_context", "ordinal": 10, "timestamp": base.to_rfc3339(), + "payload": {"model": "gpt-5.6-sol"} + }), + token_row( + base, + 12, + [1_000, 900, 100], + [1_000, 900, 100], + "gpt-5.6-sol", + ), + token_row( + base, + 13, + [5_000, 3_900, 500], + [5_000, 3_900, 500], + "gpt-5.6-sol", + ), + ]; + if owned { + lines.extend([ + token_row(base, 19, [5_050, 3_910, 505], [50, 10, 5], "gpt-5.6-sol"), + token_row( + base + Duration::seconds(1), + 20, + [5_070, 3_915, 510], + [20, 5, 5], + "gpt-5.6-sol", + ), + token_row( + base + Duration::seconds(2), + 21, + [5_070, 3_915, 510], + [20, 5, 5], + "gpt-5.6-sol", + ), + ]); + } + let body = lines + .into_iter() + .map(|line| line.to_string()) + .collect::>() + .join("\n") + + "\n"; + std::fs::write(&path, body).unwrap(); + path +} + +fn token_row( + timestamp: DateTime, + ordinal: i64, + total: [i64; 3], + last: [i64; 3], + model: &str, +) -> serde_json::Value { + serde_json::json!({ + "type": "event_msg", "ordinal": ordinal, "timestamp": timestamp.to_rfc3339(), + "payload": {"type": "token_count", "info": { + "model": model, + "total_token_usage": { + "input_tokens": total[0], "cached_input_tokens": total[1], "output_tokens": total[2] + }, + "last_token_usage": { + "input_tokens": last[0], "cached_input_tokens": last[1], "output_tokens": last[2] + } + }} + }) +} + +#[test] +fn copied_prefix_subagent_infers_advancing_baseline_without_parent() { + let root = tempfile::tempdir().unwrap(); + let sessions = root.path().join("sessions"); + let cache_root = root.path().join("cache"); + let child = write_copied_prefix_subagent_fixture( + &sessions, + "child.jsonl", + "child-id", + "missing-parent", + Utc::now() - Duration::hours(1), + true, + ); + let scanner = CostScanner::new(7) + .with_options(CostScanOptions::app_driven()) + .with_cache_root(&cache_root) + .with_sessions_dirs(vec![sessions]); + + let (summary, _, cache) = scanner.scan_codex_detailed_with_cache(None); + assert_eq!(summary.input_tokens, 70); + assert_eq!(summary.cached_tokens, 15); + assert_eq!(summary.output_tokens, 10); + assert_eq!(summary.sessions_count, 1); + let usage = &cache.files[&child.to_string_lossy().to_string()]; + assert!(!usage.codex_unresolved_fork_parent); + assert!( + usage + .codex_fork_accounting_state + .as_ref() + .is_some_and(|state| state.locally_resolved) + ); + assert_eq!( + usage.days.values().next().unwrap()["gpt-5.6-sol"], + vec![70, 15, 10] + ); + + let (cached, stats, _) = scanner.scan_codex_detailed_with_cache(None); + assert_eq!(cached.input_tokens, 70); + assert!(stats.codex_history_read_paths.is_empty()); +} + +#[test] +fn copied_prefix_subagent_inherited_only_suffix_is_not_billed() { + let root = tempfile::tempdir().unwrap(); + let sessions = root.path().join("sessions"); + let cache_root = root.path().join("cache"); + let child = write_copied_prefix_subagent_fixture( + &sessions, + "child.jsonl", + "child-id", + "missing-parent", + Utc::now() - Duration::hours(1), + false, + ); + let scanner = CostScanner::new(7) + .with_options(CostScanOptions::app_driven()) + .with_cache_root(&cache_root) + .with_sessions_dirs(vec![sessions]); + + let (summary, _, cache) = scanner.scan_codex_detailed_with_cache(None); + assert_eq!(summary.input_tokens, 0); + assert_eq!(summary.output_tokens, 0); + assert_eq!(summary.sessions_count, 0); + let usage = &cache.files[&child.to_string_lossy().to_string()]; + assert!(usage.days.is_empty()); + assert!(!usage.codex_unresolved_fork_parent); + let state = usage.codex_fork_accounting_state.as_ref().unwrap(); + assert!(state.locally_resolved); + assert!(state.inherited_totals.is_none()); + + let (cached, stats, _) = scanner.scan_codex_detailed_with_cache(None); + assert_eq!(cached.input_tokens, 0); + assert_eq!(cached.output_tokens, 0); + assert_eq!(cached.sessions_count, 0); + assert!(stats.codex_history_read_paths.is_empty()); +} + +#[test] +fn copied_prefix_subagent_prefers_validated_parent_baseline() { + let root = tempfile::tempdir().unwrap(); + let sessions = root.path().join("sessions"); + let cache_root = root.path().join("cache"); + let base = Utc::now() - Duration::hours(1); + let parent = write_codex_fork_session_fixture( + &sessions, + "parent.jsonl", + "parent-id", + None, + base, + base, + &[1_000], + ); + let child = write_copied_prefix_subagent_fixture( + &sessions, + "child.jsonl", + "child-id", + "parent-id", + base + Duration::seconds(10), + true, + ); + let now = std::time::SystemTime::now(); + std::fs::OpenOptions::new() + .write(true) + .open(&child) + .unwrap() + .set_modified(now - std::time::Duration::from_secs(20)) + .unwrap(); + let mut options = CostScanOptions::app_driven(); + options.prefer_newest_codex_sessions_first = false; + let parent_size = std::fs::metadata(&parent).unwrap().len(); + let child_size = std::fs::metadata(&child).unwrap().len(); + options.codex_max_session_file_bytes = + i64::try_from(parent_size.max(child_size)).expect("fixture size fits i64"); + let scanner = CostScanner::new(7) + .with_options(options) + .with_cache_root(&cache_root) + .with_sessions_dirs(vec![sessions]); + + let (_, stats, cache) = scanner.scan_codex_detailed_with_cache(None); + let state = cache.files[&child.to_string_lossy().to_string()] + .codex_fork_accounting_state + .as_ref() + .unwrap(); + + assert_eq!(state.inherited_totals.as_ref().unwrap().input, 1_000); + assert!(!state.locally_resolved); + assert_eq!(stats.files_seen, 2); + assert_eq!(stats.codex_read_receipt.metadata_reads, 2); + assert_eq!(stats.codex_read_receipt.history_reads, 2); + assert_eq!( + stats.codex_bytes_read, + parent_size.saturating_add(child_size), + "one bounded parse per candidate must enforce the per-file allowance" + ); + assert_eq!( + stats.codex_history_read_paths, + vec![ + parent.to_string_lossy().to_string(), + child.to_string_lossy().to_string(), + ] + ); +} + +#[test] +fn candidate_limit_counts_each_child_parent_candidate_once() { + let root = tempfile::tempdir().unwrap(); + let sessions = root.path().join("sessions"); + let cache_root = root.path().join("cache"); + let base = Utc::now() - Duration::hours(1); + let parent = write_codex_fork_session_fixture( + &sessions, + "parent.jsonl", + "parent-id", + None, + base, + base, + &[1_000], + ); + let child = write_copied_prefix_subagent_fixture( + &sessions, + "child.jsonl", + "child-id", + "parent-id", + base + Duration::seconds(10), + true, + ); + let now = std::time::SystemTime::now(); + std::fs::OpenOptions::new() + .write(true) + .open(&child) + .unwrap() + .set_modified(now - std::time::Duration::from_secs(20)) + .unwrap(); + std::fs::OpenOptions::new() + .write(true) + .open(&parent) + .unwrap() + .set_modified(now - std::time::Duration::from_secs(10)) + .unwrap(); + let mut options = CostScanOptions::app_driven(); + options.prefer_newest_codex_sessions_first = false; + options.codex_candidate_limit = 1; + let scanner = CostScanner::new(7) + .with_options(options) + .with_cache_root(&cache_root) + .with_sessions_dirs(vec![sessions]); + + let (_, stats, cache) = scanner.scan_codex_detailed_with_cache(None); + + assert_eq!(stats.files_seen, 1); + assert_eq!(stats.codex_read_receipt.metadata_reads, 1); + assert_eq!(stats.codex_read_receipt.history_reads, 1); + assert_eq!( + stats.codex_metadata_read_paths, + vec![child.to_string_lossy().to_string()] + ); + assert_eq!( + cache.codex_pending_paths, + vec![parent.to_string_lossy().to_string()] + ); +} + +#[test] +fn cold_scan_orders_multi_level_parent_chain_before_children() { + let root = tempfile::tempdir().unwrap(); + let sessions = root.path().join("sessions"); + let cache_root = root.path().join("cache"); + let base = Utc::now() - Duration::hours(1); + let ancestor = write_codex_fork_session_fixture( + &sessions, + "ancestor.jsonl", + "ancestor-id", + None, + base, + base, + &[1_000], + ); + let parent = write_codex_fork_session_fixture( + &sessions, + "parent.jsonl", + "parent-id", + Some("ancestor-id"), + base + Duration::seconds(10), + base + Duration::seconds(10), + &[1_500], + ); + let child = write_codex_fork_session_fixture( + &sessions, + "child.jsonl", + "child-id", + Some("parent-id"), + base + Duration::seconds(20), + base + Duration::seconds(20), + &[2_000], + ); + let now = std::time::SystemTime::now(); + for (path, age) in [(&child, 30), (&parent, 20), (&ancestor, 10)] { + std::fs::OpenOptions::new() + .write(true) + .open(path) + .unwrap() + .set_modified(now - std::time::Duration::from_secs(age)) + .unwrap(); + } + let mut options = CostScanOptions::app_driven(); + options.prefer_newest_codex_sessions_first = false; + let scanner = CostScanner::new(7) + .with_options(options) + .with_cache_root(&cache_root) + .with_sessions_dirs(vec![sessions]); + + let (_, stats, cache) = scanner.scan_codex_detailed_with_cache(None); + + assert_eq!(stats.files_seen, 3); + assert_eq!(stats.codex_read_receipt.metadata_reads, 3); + assert_eq!(stats.codex_read_receipt.history_reads, 3); + assert_eq!( + stats.codex_history_read_paths, + vec![ + ancestor.to_string_lossy().to_string(), + parent.to_string_lossy().to_string(), + child.to_string_lossy().to_string(), + ] + ); + let parent_state = cache.files[&parent.to_string_lossy().to_string()] + .codex_fork_accounting_state + .as_ref() + .unwrap(); + let child_state = cache.files[&child.to_string_lossy().to_string()] + .codex_fork_accounting_state + .as_ref() + .unwrap(); + assert_eq!(parent_state.inherited_totals.as_ref().unwrap().input, 1_000); + assert_eq!(child_state.inherited_totals.as_ref().unwrap().input, 1_500); +} + +fn assert_unsafe_lineage_is_unresolved( + summary: &CostSummary, + stats: &CostScanStats, + cache: &CostUsageCache, + paths: &[&Path], +) { + assert_eq!(summary.sessions_count, 0); + assert_eq!(summary.input_tokens, 0); + assert_eq!( + stats.codex_read_receipt.metadata_reads, + u32::try_from(paths.len()).expect("fixture count fits u32") + ); + assert_eq!(stats.codex_read_receipt.history_reads, 0); + assert!(stats.codex_history_read_paths.is_empty()); + for path in paths { + let usage = &cache.files[&path.to_string_lossy().to_string()]; + assert!(usage.codex_unresolved_fork_parent); + assert!(usage.codex_fork_accounting_state.is_none()); + assert!(usage.days.is_empty()); + } +} + +#[test] +fn duplicate_parent_session_ids_fail_closed_with_their_child() { + let root = tempfile::tempdir().unwrap(); + let sessions = root.path().join("sessions"); + let cache_root = root.path().join("cache"); + let base = Utc::now() - Duration::hours(1); + let first_parent = write_codex_fork_session_fixture( + &sessions, + "first-parent.jsonl", + "parent-id", + None, + base, + base, + &[1_000], + ); + let second_parent = write_codex_fork_session_fixture( + &sessions, + "second-parent.jsonl", + "parent-id", + None, + base + Duration::seconds(1), + base + Duration::seconds(1), + &[2_000], + ); + let child = write_copied_prefix_subagent_fixture( + &sessions, + "child.jsonl", + "child-id", + "parent-id", + base + Duration::seconds(2), + true, + ); + let mut options = CostScanOptions::app_driven(); + options.prefer_newest_codex_sessions_first = false; + let scanner = CostScanner::new(7) + .with_options(options) + .with_cache_root(&cache_root) + .with_sessions_dirs(vec![sessions]); + + let (summary, stats, cache) = scanner.scan_codex_detailed_with_cache(None); + + assert_unsafe_lineage_is_unresolved( + &summary, + &stats, + &cache, + &[&first_parent, &second_parent, &child], + ); +} + +#[test] +fn two_node_subagent_cycle_fails_closed() { + let root = tempfile::tempdir().unwrap(); + let sessions = root.path().join("sessions"); + let cache_root = root.path().join("cache"); + let base = Utc::now() - Duration::hours(1); + let first = write_copied_prefix_subagent_fixture( + &sessions, + "first.jsonl", + "first-id", + "second-id", + base, + true, + ); + let second = write_copied_prefix_subagent_fixture( + &sessions, + "second.jsonl", + "second-id", + "first-id", + base + Duration::seconds(1), + true, + ); + let mut options = CostScanOptions::app_driven(); + options.prefer_newest_codex_sessions_first = false; + let scanner = CostScanner::new(7) + .with_options(options) + .with_cache_root(&cache_root) + .with_sessions_dirs(vec![sessions]); + + let (summary, stats, cache) = scanner.scan_codex_detailed_with_cache(None); + + assert_unsafe_lineage_is_unresolved(&summary, &stats, &cache, &[&first, &second]); +} + +#[test] +fn self_referential_subagent_fails_closed() { + let root = tempfile::tempdir().unwrap(); + let sessions = root.path().join("sessions"); + let cache_root = root.path().join("cache"); + let session = write_copied_prefix_subagent_fixture( + &sessions, + "self-cycle.jsonl", + "self-id", + "self-id", + Utc::now() - Duration::hours(1), + true, + ); + let scanner = CostScanner::new(7) + .with_options(CostScanOptions::app_driven()) + .with_cache_root(&cache_root) + .with_sessions_dirs(vec![sessions]); + + let (summary, stats, cache) = scanner.scan_codex_detailed_with_cache(None); + + assert_unsafe_lineage_is_unresolved(&summary, &stats, &cache, &[&session]); +} + +fn assert_cached_inference_is_replaced_when_parent_appears( + prefer_newest_codex_sessions_first: bool, +) { + let root = tempfile::tempdir().unwrap(); + let sessions = root.path().join("sessions"); + let cache_root = root.path().join("cache"); + let base = Utc::now() - Duration::hours(1); + let child = write_copied_prefix_subagent_fixture( + &sessions, + "child.jsonl", + "child-id", + "parent-id", + base + Duration::seconds(10), + true, + ); + let mut options = CostScanOptions::app_driven(); + options.prefer_newest_codex_sessions_first = prefer_newest_codex_sessions_first; + let scanner = CostScanner::new(7) + .with_options(options) + .with_cache_root(&cache_root) + .with_sessions_dirs(vec![sessions.clone()]); + + let (_, _, inferred_cache) = scanner.scan_codex_detailed_with_cache(None); + let inferred_state = inferred_cache.files[&child.to_string_lossy().to_string()] + .codex_fork_accounting_state + .as_ref() + .unwrap(); + assert!(inferred_state.locally_resolved); + assert_eq!( + inferred_state.inherited_totals.as_ref().unwrap().input, + 5_000 + ); + + let parent = write_codex_fork_session_fixture( + &sessions, + "parent.jsonl", + "parent-id", + None, + base, + base, + &[1_000], + ); + let now = std::time::SystemTime::now(); + std::fs::OpenOptions::new() + .write(true) + .open(parent) + .unwrap() + .set_modified(now - std::time::Duration::from_secs(10)) + .unwrap(); + + let (_, stats, validated_cache) = scanner.scan_codex_detailed_with_cache(None); + let validated_state = validated_cache.files[&child.to_string_lossy().to_string()] + .codex_fork_accounting_state + .as_ref() + .unwrap(); + assert!(!validated_state.locally_resolved); + assert_eq!( + validated_state.inherited_totals.as_ref().unwrap().input, + 1_000 + ); + assert!( + stats + .codex_history_read_paths + .contains(&child.to_string_lossy().to_string()), + "the unchanged child must be reparsed when baseline provenance changes" + ); +} + +#[test] +fn copied_prefix_subagent_replaces_cached_inference_when_parent_is_visited_first() { + assert_cached_inference_is_replaced_when_parent_appears(true); +} + +#[test] +fn copied_prefix_subagent_replaces_cached_inference_when_child_would_be_visited_first() { + assert_cached_inference_is_replaced_when_parent_appears(false); +} diff --git a/rust/src/cost_scanner/tests/paginated.rs b/rust/src/cost_scanner/tests/paginated.rs index a42ad77405..592d484823 100644 --- a/rust/src/cost_scanner/tests/paginated.rs +++ b/rust/src/cost_scanner/tests/paginated.rs @@ -135,449 +135,6 @@ fn write_codex_paginated_continuation_fixture( path } -fn write_copied_prefix_subagent_fixture( - sessions_root: &Path, - name: &str, - parent_id: &str, - base: DateTime, - owned: bool, -) -> PathBuf { - let day = base.with_timezone(&Local).date_naive(); - let day_dir = sessions_root - .join(day.format("%Y").to_string()) - .join(day.format("%m").to_string()) - .join(day.format("%d").to_string()); - std::fs::create_dir_all(&day_dir).unwrap(); - let path = day_dir.join(name); - let mut lines = vec![ - serde_json::json!({ - "type": "session_meta", "ordinal": 0, "timestamp": base.to_rfc3339(), - "payload": { - "id": "child-id", "forked_from_id": parent_id, - "subagent_history_start_ordinal": 10, - "thread_source": "subagent", - "source": {"subagent": {"thread_spawn": {"parent_thread_id": parent_id}}} - } - }), - token_row(base, 2, [1_000, 900, 100], [0, 0, 0], "gpt-5.6-sol"), - serde_json::json!({ - "type": "turn_context", "ordinal": 10, "timestamp": base.to_rfc3339(), - "payload": {"model": "gpt-5.6-sol"} - }), - token_row( - base, - 12, - [1_000, 900, 100], - [1_000, 900, 100], - "gpt-5.6-sol", - ), - token_row( - base, - 13, - [5_000, 3_900, 500], - [5_000, 3_900, 500], - "gpt-5.6-sol", - ), - ]; - if owned { - lines.extend([ - token_row(base, 19, [5_050, 3_910, 505], [50, 10, 5], "gpt-5.6-sol"), - token_row( - base + Duration::seconds(1), - 20, - [5_070, 3_915, 510], - [20, 5, 5], - "gpt-5.6-sol", - ), - token_row( - base + Duration::seconds(2), - 21, - [5_070, 3_915, 510], - [20, 5, 5], - "gpt-5.6-sol", - ), - ]); - } - let body = lines - .into_iter() - .map(|line| line.to_string()) - .collect::>() - .join("\n") - + "\n"; - std::fs::write(&path, body).unwrap(); - path -} - -fn token_row( - timestamp: DateTime, - ordinal: i64, - total: [i64; 3], - last: [i64; 3], - model: &str, -) -> serde_json::Value { - serde_json::json!({ - "type": "event_msg", "ordinal": ordinal, "timestamp": timestamp.to_rfc3339(), - "payload": {"type": "token_count", "info": { - "model": model, - "total_token_usage": { - "input_tokens": total[0], "cached_input_tokens": total[1], "output_tokens": total[2] - }, - "last_token_usage": { - "input_tokens": last[0], "cached_input_tokens": last[1], "output_tokens": last[2] - } - }} - }) -} - -#[test] -fn copied_prefix_subagent_infers_advancing_baseline_without_parent() { - let root = tempfile::tempdir().unwrap(); - let sessions = root.path().join("sessions"); - let cache_root = root.path().join("cache"); - let child = write_copied_prefix_subagent_fixture( - &sessions, - "child.jsonl", - "missing-parent", - Utc::now() - Duration::hours(1), - true, - ); - let scanner = CostScanner::new(7) - .with_options(CostScanOptions::app_driven()) - .with_cache_root(&cache_root) - .with_sessions_dirs(vec![sessions]); - - let (summary, _, cache) = scanner.scan_codex_detailed_with_cache(None); - assert_eq!(summary.input_tokens, 70); - assert_eq!(summary.cached_tokens, 15); - assert_eq!(summary.output_tokens, 10); - assert_eq!(summary.sessions_count, 1); - let usage = &cache.files[&child.to_string_lossy().to_string()]; - assert!(!usage.codex_unresolved_fork_parent); - assert!( - usage - .codex_fork_accounting_state - .as_ref() - .is_some_and(|state| state.locally_resolved) - ); - assert_eq!( - usage.days.values().next().unwrap()["gpt-5.6-sol"], - vec![70, 15, 10] - ); - - let (cached, stats, _) = scanner.scan_codex_detailed_with_cache(None); - assert_eq!(cached.input_tokens, 70); - assert!(stats.codex_history_read_paths.is_empty()); -} - -#[test] -fn copied_prefix_subagent_inherited_only_suffix_is_not_billed() { - let root = tempfile::tempdir().unwrap(); - let sessions = root.path().join("sessions"); - let cache_root = root.path().join("cache"); - let child = write_copied_prefix_subagent_fixture( - &sessions, - "child.jsonl", - "missing-parent", - Utc::now() - Duration::hours(1), - false, - ); - let scanner = CostScanner::new(7) - .with_options(CostScanOptions::app_driven()) - .with_cache_root(&cache_root) - .with_sessions_dirs(vec![sessions]); - - let (summary, _, cache) = scanner.scan_codex_detailed_with_cache(None); - assert_eq!(summary.input_tokens, 0); - assert_eq!(summary.output_tokens, 0); - assert_eq!(summary.sessions_count, 0); - let usage = &cache.files[&child.to_string_lossy().to_string()]; - assert!(usage.days.is_empty()); - assert!(!usage.codex_unresolved_fork_parent); - let state = usage.codex_fork_accounting_state.as_ref().unwrap(); - assert!(state.locally_resolved); - assert!(state.inherited_totals.is_none()); - - let (cached, stats, _) = scanner.scan_codex_detailed_with_cache(None); - assert_eq!(cached.input_tokens, 0); - assert_eq!(cached.output_tokens, 0); - assert_eq!(cached.sessions_count, 0); - assert!(stats.codex_history_read_paths.is_empty()); -} - -#[test] -fn copied_prefix_subagent_prefers_validated_parent_baseline() { - let root = tempfile::tempdir().unwrap(); - let sessions = root.path().join("sessions"); - let cache_root = root.path().join("cache"); - let base = Utc::now() - Duration::hours(1); - let parent = write_codex_fork_session_fixture( - &sessions, - "parent.jsonl", - "parent-id", - None, - base, - base, - &[1_000], - ); - let child = write_copied_prefix_subagent_fixture( - &sessions, - "child.jsonl", - "parent-id", - base + Duration::seconds(10), - true, - ); - let now = std::time::SystemTime::now(); - std::fs::OpenOptions::new() - .write(true) - .open(&child) - .unwrap() - .set_modified(now - std::time::Duration::from_secs(20)) - .unwrap(); - let mut options = CostScanOptions::app_driven(); - options.prefer_newest_codex_sessions_first = false; - let parent_size = std::fs::metadata(&parent).unwrap().len(); - let child_size = std::fs::metadata(&child).unwrap().len(); - options.codex_max_session_file_bytes = - i64::try_from(parent_size.max(child_size)).expect("fixture size fits i64"); - let scanner = CostScanner::new(7) - .with_options(options) - .with_cache_root(&cache_root) - .with_sessions_dirs(vec![sessions]); - - let (_, stats, cache) = scanner.scan_codex_detailed_with_cache(None); - let state = cache.files[&child.to_string_lossy().to_string()] - .codex_fork_accounting_state - .as_ref() - .unwrap(); - - assert_eq!(state.inherited_totals.as_ref().unwrap().input, 1_000); - assert!(!state.locally_resolved); - assert_eq!(stats.files_seen, 2); - assert_eq!(stats.codex_read_receipt.metadata_reads, 2); - assert_eq!(stats.codex_read_receipt.history_reads, 2); - assert_eq!( - stats.codex_bytes_read, - parent_size.saturating_add(child_size), - "one bounded parse per candidate must enforce the per-file allowance" - ); - assert_eq!( - stats.codex_history_read_paths, - vec![ - parent.to_string_lossy().to_string(), - child.to_string_lossy().to_string(), - ] - ); -} - -#[test] -fn candidate_limit_counts_each_child_parent_candidate_once() { - let root = tempfile::tempdir().unwrap(); - let sessions = root.path().join("sessions"); - let cache_root = root.path().join("cache"); - let base = Utc::now() - Duration::hours(1); - let parent = write_codex_fork_session_fixture( - &sessions, - "parent.jsonl", - "parent-id", - None, - base, - base, - &[1_000], - ); - let child = write_copied_prefix_subagent_fixture( - &sessions, - "child.jsonl", - "parent-id", - base + Duration::seconds(10), - true, - ); - let now = std::time::SystemTime::now(); - std::fs::OpenOptions::new() - .write(true) - .open(&child) - .unwrap() - .set_modified(now - std::time::Duration::from_secs(20)) - .unwrap(); - std::fs::OpenOptions::new() - .write(true) - .open(&parent) - .unwrap() - .set_modified(now - std::time::Duration::from_secs(10)) - .unwrap(); - let mut options = CostScanOptions::app_driven(); - options.prefer_newest_codex_sessions_first = false; - options.codex_candidate_limit = 1; - let scanner = CostScanner::new(7) - .with_options(options) - .with_cache_root(&cache_root) - .with_sessions_dirs(vec![sessions]); - - let (_, stats, cache) = scanner.scan_codex_detailed_with_cache(None); - - assert_eq!(stats.files_seen, 1); - assert_eq!(stats.codex_read_receipt.metadata_reads, 1); - assert_eq!(stats.codex_read_receipt.history_reads, 1); - assert_eq!( - stats.codex_metadata_read_paths, - vec![child.to_string_lossy().to_string()] - ); - assert_eq!( - cache.codex_pending_paths, - vec![parent.to_string_lossy().to_string()] - ); -} - -#[test] -fn cold_scan_orders_multi_level_parent_chain_before_children() { - let root = tempfile::tempdir().unwrap(); - let sessions = root.path().join("sessions"); - let cache_root = root.path().join("cache"); - let base = Utc::now() - Duration::hours(1); - let ancestor = write_codex_fork_session_fixture( - &sessions, - "ancestor.jsonl", - "ancestor-id", - None, - base, - base, - &[1_000], - ); - let parent = write_codex_fork_session_fixture( - &sessions, - "parent.jsonl", - "parent-id", - Some("ancestor-id"), - base + Duration::seconds(10), - base + Duration::seconds(10), - &[1_500], - ); - let child = write_codex_fork_session_fixture( - &sessions, - "child.jsonl", - "child-id", - Some("parent-id"), - base + Duration::seconds(20), - base + Duration::seconds(20), - &[2_000], - ); - let now = std::time::SystemTime::now(); - for (path, age) in [(&child, 30), (&parent, 20), (&ancestor, 10)] { - std::fs::OpenOptions::new() - .write(true) - .open(path) - .unwrap() - .set_modified(now - std::time::Duration::from_secs(age)) - .unwrap(); - } - let mut options = CostScanOptions::app_driven(); - options.prefer_newest_codex_sessions_first = false; - let scanner = CostScanner::new(7) - .with_options(options) - .with_cache_root(&cache_root) - .with_sessions_dirs(vec![sessions]); - - let (_, stats, cache) = scanner.scan_codex_detailed_with_cache(None); - - assert_eq!(stats.files_seen, 3); - assert_eq!(stats.codex_read_receipt.metadata_reads, 3); - assert_eq!(stats.codex_read_receipt.history_reads, 3); - assert_eq!( - stats.codex_history_read_paths, - vec![ - ancestor.to_string_lossy().to_string(), - parent.to_string_lossy().to_string(), - child.to_string_lossy().to_string(), - ] - ); - let parent_state = cache.files[&parent.to_string_lossy().to_string()] - .codex_fork_accounting_state - .as_ref() - .unwrap(); - let child_state = cache.files[&child.to_string_lossy().to_string()] - .codex_fork_accounting_state - .as_ref() - .unwrap(); - assert_eq!(parent_state.inherited_totals.as_ref().unwrap().input, 1_000); - assert_eq!(child_state.inherited_totals.as_ref().unwrap().input, 1_500); -} - -fn assert_cached_inference_is_replaced_when_parent_appears( - prefer_newest_codex_sessions_first: bool, -) { - let root = tempfile::tempdir().unwrap(); - let sessions = root.path().join("sessions"); - let cache_root = root.path().join("cache"); - let base = Utc::now() - Duration::hours(1); - let child = write_copied_prefix_subagent_fixture( - &sessions, - "child.jsonl", - "parent-id", - base + Duration::seconds(10), - true, - ); - let mut options = CostScanOptions::app_driven(); - options.prefer_newest_codex_sessions_first = prefer_newest_codex_sessions_first; - let scanner = CostScanner::new(7) - .with_options(options) - .with_cache_root(&cache_root) - .with_sessions_dirs(vec![sessions.clone()]); - - let (_, _, inferred_cache) = scanner.scan_codex_detailed_with_cache(None); - let inferred_state = inferred_cache.files[&child.to_string_lossy().to_string()] - .codex_fork_accounting_state - .as_ref() - .unwrap(); - assert!(inferred_state.locally_resolved); - assert_eq!( - inferred_state.inherited_totals.as_ref().unwrap().input, - 5_000 - ); - - let parent = write_codex_fork_session_fixture( - &sessions, - "parent.jsonl", - "parent-id", - None, - base, - base, - &[1_000], - ); - let now = std::time::SystemTime::now(); - std::fs::OpenOptions::new() - .write(true) - .open(parent) - .unwrap() - .set_modified(now - std::time::Duration::from_secs(10)) - .unwrap(); - - let (_, stats, validated_cache) = scanner.scan_codex_detailed_with_cache(None); - let validated_state = validated_cache.files[&child.to_string_lossy().to_string()] - .codex_fork_accounting_state - .as_ref() - .unwrap(); - assert!(!validated_state.locally_resolved); - assert_eq!( - validated_state.inherited_totals.as_ref().unwrap().input, - 1_000 - ); - assert!( - stats - .codex_history_read_paths - .contains(&child.to_string_lossy().to_string()), - "the unchanged child must be reparsed when baseline provenance changes" - ); -} - -#[test] -fn copied_prefix_subagent_replaces_cached_inference_when_parent_is_visited_first() { - assert_cached_inference_is_replaced_when_parent_appears(true); -} - -#[test] -fn copied_prefix_subagent_replaces_cached_inference_when_child_would_be_visited_first() { - assert_cached_inference_is_replaced_when_parent_appears(false); -} - #[test] fn paginated_continuation_raises_inherited_baseline_from_total_last() { let root = tempfile::tempdir().unwrap(); From 9b2361adc3c9702926158d74b1fc5cc43dabf1f0 Mon Sep 17 00:00:00 2001 From: NessZerra <90105158+Finesssee@users.noreply.github.com> Date: Wed, 23 Sep 2026 01:25:36 +0700 Subject: [PATCH 12/19] Validate Codex lineage across refreshes --- rust/src/cost_scanner/codex.rs | 190 ++++++++++------ rust/src/cost_scanner/codex/logical_target.rs | 163 +++++++++++--- rust/src/cost_scanner/codex/scan.rs | 15 +- rust/src/cost_scanner/tests.rs | 3 + rust/src/cost_scanner/tests/lineage_cache.rs | 212 ++++++++++++++++++ 5 files changed, 492 insertions(+), 91 deletions(-) create mode 100644 rust/src/cost_scanner/tests/lineage_cache.rs diff --git a/rust/src/cost_scanner/codex.rs b/rust/src/cost_scanner/codex.rs index aab7def35d..23022e2226 100644 --- a/rust/src/cost_scanner/codex.rs +++ b/rust/src/cost_scanner/codex.rs @@ -95,31 +95,42 @@ fn summary_from_cached_report( } } +#[derive(Debug, Clone, PartialEq, Eq)] +enum CodexParentResolution { + Absent, + Safe(crate::core::CodexTotals), + Unsafe, +} + +fn codex_usage_uses_parent(usage: &CostUsageFileUsage) -> bool { + usage.codex_lineage.uses_parent_baseline() + || (matches!(usage.codex_lineage, CodexSessionLineage::Root) + && usage.codex_forked_from_id.is_some()) +} + fn codex_fork_parent_is_safe(cache: &CostUsageCache, usage: &CostUsageFileUsage) -> bool { let locally_resolved = usage .codex_fork_accounting_state .as_ref() .is_some_and(|state| state.locally_resolved); - let uses_parent_baseline = usage.codex_lineage.uses_parent_baseline() - || (matches!(usage.codex_lineage, CodexSessionLineage::Root) - && usage.codex_forked_from_id.is_some()); - if !uses_parent_baseline { + if !codex_usage_uses_parent(usage) { return true; } - let parent_is_available = usage - .codex_forked_from_id - .as_deref() - .is_some_and(|parent_id| { - codex_parent_baseline(cache, parent_id, usage.codex_fork_timestamp.as_deref()).is_some() - }); + let parent_resolution = + usage + .codex_forked_from_id + .as_deref() + .map_or(CodexParentResolution::Unsafe, |parent_id| { + codex_parent_resolution(cache, parent_id, usage.codex_fork_timestamp.as_deref()) + }); - // Local inference is safe only while no validated parent is available. - // Once the parent enters the cache, force the child through baseline - // replacement instead of accepting its unchanged-file fast path. + // Local inference is safe only while the parent is genuinely absent. + // An owner that is ambiguous, stale, locally inferred, cyclic, or + // transitively unsafe must fail closed instead of looking absent. if locally_resolved { - !parent_is_available + matches!(parent_resolution, CodexParentResolution::Absent) } else { - parent_is_available + matches!(parent_resolution, CodexParentResolution::Safe(_)) } } @@ -130,51 +141,101 @@ fn codex_fork_uses_local_inference(usage: &CostUsageFileUsage) -> bool { .is_some_and(|state| state.locally_resolved) } -/// Return a parent cumulative baseline only when exactly one cached session -/// identity is current, complete, timestamp-ordered, and safe to trust. -fn codex_parent_baseline( +/// Resolve one parent identity through the persisted cache graph. Absence is +/// deliberately distinct from ambiguity or transitive unsafety so copied +/// prefixes may infer only when no owner exists at all. +fn codex_parent_resolution( + cache: &CostUsageCache, + parent_session_id: &str, + child_fork_timestamp: Option<&str>, +) -> CodexParentResolution { + codex_parent_resolution_inner( + cache, + parent_session_id, + child_fork_timestamp, + &mut HashSet::new(), + ) +} + +fn codex_parent_resolution_inner( cache: &CostUsageCache, parent_session_id: &str, child_fork_timestamp: Option<&str>, + visiting: &mut HashSet, +) -> CodexParentResolution { + let mut owners = cache + .files + .iter() + .filter(|(_, usage)| usage.codex_session_id.as_deref() == Some(parent_session_id)); + let Some((path_key, usage)) = owners.next() else { + return CodexParentResolution::Absent; + }; + if owners.next().is_some() || !visiting.insert(path_key.clone()) { + return CodexParentResolution::Unsafe; + } + + let resolution = + codex_parent_owner_baseline(cache, path_key, usage, child_fork_timestamp, visiting) + .map_or(CodexParentResolution::Unsafe, CodexParentResolution::Safe); + visiting.remove(path_key); + resolution +} + +fn codex_parent_owner_baseline( + cache: &CostUsageCache, + path_key: &str, + usage: &CostUsageFileUsage, + child_fork_timestamp: Option<&str>, + visiting: &mut HashSet, ) -> Option { - let mut baseline = None; - for (path_key, usage) in &cache.files { - if usage.codex_session_id.as_deref() != Some(parent_session_id) { - continue; - } - if usage.codex_unresolved_fork_parent - || usage.codex_token_timestamps_monotonic != Some(true) - { - return None; - } - let metadata = fs::metadata(path_key).ok()?; - if let (Some(expected), Some(actual)) = ( - usage.codex_file_identity.as_ref(), - JsonlScanner::codex_file_identity(Path::new(path_key), &metadata), - ) && expected != &actual - { - return None; - } - #[allow(clippy::cast_possible_wrap, reason = "session file sizes fit i64")] - let size = metadata.len().min(i64::MAX as u64) as i64; - if usage.mtime_unix_ms != system_time_to_unix_ms(metadata.modified().ok()) - || usage.size != size - || usage.parsed_bytes.unwrap_or(0) < size - { - return None; - } - let last_totals = usage.last_totals.clone()?; - let last_token_timestamp = usage.codex_last_token_timestamp.as_deref()?; - let child_fork_timestamp = child_fork_timestamp?; - if !JsonlScanner::codex_timestamp_at_or_before(last_token_timestamp, child_fork_timestamp) { - return None; - } - if baseline.replace(last_totals).is_some() { - // Duplicate identities make the dependency ambiguous. - return None; + if usage.codex_unresolved_fork_parent + || usage.codex_token_timestamps_monotonic != Some(true) + || codex_fork_uses_local_inference(usage) + { + return None; + } + + if codex_usage_uses_parent(usage) { + let parent_id = usage.codex_forked_from_id.as_deref()?; + let inherited = usage + .codex_fork_accounting_state + .as_ref()? + .inherited_totals + .as_ref()?; + match codex_parent_resolution_inner( + cache, + parent_id, + usage.codex_fork_timestamp.as_deref(), + visiting, + ) { + CodexParentResolution::Safe(baseline) if &baseline == inherited => {} + CodexParentResolution::Absent + | CodexParentResolution::Safe(_) + | CodexParentResolution::Unsafe => return None, } } - baseline + + let metadata = fs::metadata(path_key).ok()?; + if let (Some(expected), Some(actual)) = ( + usage.codex_file_identity.as_ref(), + JsonlScanner::codex_file_identity(Path::new(path_key), &metadata), + ) && expected != &actual + { + return None; + } + #[allow(clippy::cast_possible_wrap, reason = "session file sizes fit i64")] + let size = metadata.len().min(i64::MAX as u64) as i64; + if usage.mtime_unix_ms != system_time_to_unix_ms(metadata.modified().ok()) + || usage.size != size + || usage.parsed_bytes.unwrap_or(0) < size + { + return None; + } + let last_totals = usage.last_totals.clone()?; + let last_token_timestamp = usage.codex_last_token_timestamp.as_deref()?; + let child_fork_timestamp = child_fork_timestamp?; + JsonlScanner::codex_timestamp_at_or_before(last_token_timestamp, child_fork_timestamp) + .then_some(last_totals) } fn is_codex_path_in_scan_window( @@ -565,11 +626,11 @@ impl CostScanner { let matching_cached_fork_state = cached_fork_accounting_state .as_ref() .filter(|_| cached_fork_state_matches); - let parent_fork_baseline = is_fork + let parent_resolution = is_fork .then_some(codex_forked_from_id.as_deref()) .flatten() - .and_then(|parent_id| { - codex_parent_baseline(cache, parent_id, codex_fork_timestamp.as_deref()) + .map_or(CodexParentResolution::Unsafe, |parent_id| { + codex_parent_resolution(cache, parent_id, codex_fork_timestamp.as_deref()) }); let paginated_continuation = is_fork && codex_forked_from_id.is_some() @@ -582,15 +643,15 @@ impl CostScanner { CodexAccountingMode::Unresolved } else if !is_fork { CodexAccountingMode::Standard - } else if let Some(baseline) = parent_fork_baseline { + } else if let CodexParentResolution::Safe(baseline) = &parent_resolution { let reparse_cached_file = matching_cached_fork_state.is_some_and(|state| { - state.locally_resolved || state.inherited_totals.as_ref() != Some(&baseline) + state.locally_resolved || state.inherited_totals.as_ref() != Some(baseline) }); let cached_parent_state = matching_cached_fork_state.filter(|state| { - !state.locally_resolved && state.inherited_totals.as_ref() == Some(&baseline) + !state.locally_resolved && state.inherited_totals.as_ref() == Some(baseline) }); CodexAccountingMode::Baseline { - baseline, + baseline: baseline.clone(), paginated_continuation, remaining_inherited_totals: cached_parent_state .and_then(|state| state.remaining_inherited_totals.clone()), @@ -598,7 +659,8 @@ impl CostScanner { replaces_cached_state: reparse_cached_file, }, } - } else if let Some(state) = matching_cached_fork_state + } else if matches!(&parent_resolution, CodexParentResolution::Absent) + && let Some(state) = matching_cached_fork_state && let Some(baseline) = state.inherited_totals.clone() { CodexAccountingMode::Baseline { @@ -611,7 +673,9 @@ impl CostScanner { CodexBaselineProvenance::CachedValidatedParent }, } - } else if session_metadata.is_subagent { + } else if matches!(&parent_resolution, CodexParentResolution::Absent) + && session_metadata.is_subagent + { CodexAccountingMode::InferSubagent { start_ordinal: session_metadata.subagent_history_start_ordinal, } diff --git a/rust/src/cost_scanner/codex/logical_target.rs b/rust/src/cost_scanner/codex/logical_target.rs index b22b7ee6c3..25eb78d8e8 100644 --- a/rust/src/cost_scanner/codex/logical_target.rs +++ b/rust/src/cost_scanner/codex/logical_target.rs @@ -78,18 +78,77 @@ pub(super) fn defer_codex_locally_inferred_candidates( candidates.extend(other); } -/// Order one bounded work set so every uniquely identified parent is parsed -/// before its children. Duplicate identities, cycles, and every dependent -/// candidate are marked unsafe so parsing cannot accept or infer a baseline -/// from ambiguous lineage. -pub(super) fn order_codex_candidates_by_lineage(candidates: &mut Vec) { +struct CodexLineageNode { + path: String, + session_id: Option, + parent_id: Option, + candidate_index: Option, + may_infer_missing_parent: bool, + may_author_parent: bool, + initially_unsafe: bool, +} + +/// Order one bounded work set against both its admitted metadata and the +/// persisted cache graph. The returned cache paths became structurally unsafe +/// and must be invalidated even when the candidate limit deferred them. +pub(super) fn order_codex_candidates_by_lineage( + cache: &CostUsageCache, + candidates: &mut Vec, +) -> Vec { if candidates.is_empty() { - return; + return Vec::new(); + } + + let candidate_paths = candidates + .iter() + .map(|candidate| candidate.path.to_string_lossy().to_string()) + .collect::>(); + let mut cached_paths = cache + .files + .keys() + .filter(|path| !candidate_paths.contains(*path)) + .cloned() + .collect::>(); + cached_paths.sort(); + + let mut nodes = Vec::with_capacity(cached_paths.len() + candidates.len()); + for path in cached_paths { + let usage = &cache.files[&path]; + let uses_parent = super::codex_usage_uses_parent(usage); + let locally_inferred = super::codex_fork_uses_local_inference(usage); + nodes.push(CodexLineageNode { + path, + session_id: usage.codex_session_id.clone(), + parent_id: uses_parent + .then(|| usage.codex_forked_from_id.clone()) + .flatten(), + candidate_index: None, + may_infer_missing_parent: locally_inferred, + may_author_parent: !locally_inferred && !usage.codex_unresolved_fork_parent, + initially_unsafe: usage.codex_unresolved_fork_parent, + }); + } + let mut candidate_node_indices = Vec::with_capacity(candidates.len()); + for (candidate_index, candidate) in candidates.iter().enumerate() { + let uses_parent = candidate.session_metadata.lineage.uses_parent_baseline() + || candidate.session_metadata.forked_from_id.is_some(); + nodes.push(CodexLineageNode { + path: candidate.path.to_string_lossy().to_string(), + session_id: candidate.session_metadata.session_id.clone(), + parent_id: uses_parent + .then(|| candidate.session_metadata.forked_from_id.clone()) + .flatten(), + candidate_index: Some(candidate_index), + may_infer_missing_parent: candidate.session_metadata.is_subagent, + may_author_parent: true, + initially_unsafe: false, + }); + candidate_node_indices.push(nodes.len() - 1); } let mut session_owners = HashMap::>::new(); - for (index, candidate) in candidates.iter().enumerate() { - let Some(session_id) = candidate.session_metadata.session_id.as_ref() else { + for (index, node) in nodes.iter().enumerate() { + let Some(session_id) = node.session_id.as_ref() else { continue; }; session_owners @@ -97,42 +156,47 @@ pub(super) fn order_codex_candidates_by_lineage(candidates: &mut Vec>(); for owners in session_owners.values().filter(|owners| owners.len() > 1) { for &index in owners { unsafe_lineage[index] = true; } } - let mut parent_indices = vec![None; candidates.len()]; - for (index, candidate) in candidates.iter().enumerate() { - let Some(parent_id) = candidate.session_metadata.forked_from_id.as_ref() else { + let mut parent_indices = vec![None; nodes.len()]; + for (index, node) in nodes.iter().enumerate() { + let Some(parent_id) = node.parent_id.as_ref() else { continue; }; match session_owners.get(parent_id).map(Vec::as_slice) { Some([parent_index]) => parent_indices[index] = Some(*parent_index), - Some([]) | None => {} + Some([]) | None if node.may_infer_missing_parent => {} + Some([]) | None => unsafe_lineage[index] = true, Some(_) => unsafe_lineage[index] = true, } } - let mut remaining = candidates.drain(..).map(Some).collect::>(); - let mut ordered = Vec::with_capacity(remaining.len()); - let mut completed = vec![false; remaining.len()]; + let mut completed = vec![false; nodes.len()]; + let mut ordered_indices = Vec::with_capacity(candidates.len()); loop { let mut progressed = false; - for index in 0..remaining.len() { - if remaining[index].is_none() || unsafe_lineage[index] { + for index in 0..nodes.len() { + if completed[index] || unsafe_lineage[index] { continue; } let parent_is_ready = parent_indices[index].is_none_or(|parent_index| { - completed[parent_index] && !unsafe_lineage[parent_index] + completed[parent_index] + && !unsafe_lineage[parent_index] + && nodes[parent_index].may_author_parent }); if parent_is_ready { - let mut candidate = remaining[index].take().expect("candidate checked above"); - candidate.lineage_disposition = CodexLineageDisposition::Ready; - ordered.push(candidate); completed[index] = true; + if let Some(candidate_index) = nodes[index].candidate_index { + ordered_indices.push(candidate_index); + } progressed = true; } } @@ -141,11 +205,58 @@ pub(super) fn order_codex_candidates_by_lineage(candidates: &mut Vec>(); + for candidate_index in ordered_indices { + let node_index = candidate_node_indices[candidate_index]; + let mut candidate = remaining[candidate_index] + .take() + .expect("candidate is ordered once"); + candidate.lineage_disposition = if unsafe_lineage[node_index] { + CodexLineageDisposition::AmbiguousOrCyclic + } else { + CodexLineageDisposition::Ready + }; + candidates.push(candidate); + } + + nodes + .iter() + .zip(unsafe_lineage) + .filter(|(node, unsafe_lineage)| { + *unsafe_lineage + && cache + .files + .get(&node.path) + .is_some_and(|usage| !usage.codex_unresolved_fork_parent) + }) + .map(|(node, _)| node.path.clone()) + .collect() +} + +pub(super) fn invalidate_codex_unsafe_lineage(cache: &mut CostUsageCache, paths: &[String]) { + for path in paths { + let Some(usage) = cache.files.get_mut(path) else { + continue; + }; + usage.days.clear(); + usage.parsed_bytes = Some(0); + usage.codex_scan_target_size = None; + usage.last_model = None; + usage.last_totals = None; + usage.codex_token_timestamps_monotonic = None; + usage.codex_last_token_timestamp = None; + usage.codex_fork_accounting_state = None; + usage.codex_unresolved_fork_parent = true; } - candidates.extend(ordered); } /// Give paths already in the durable queue their saved turn before newly diff --git a/rust/src/cost_scanner/codex/scan.rs b/rust/src/cost_scanner/codex/scan.rs index 2353b80dad..6da7fb5188 100644 --- a/rust/src/cost_scanner/codex/scan.rs +++ b/rust/src/cost_scanner/codex/scan.rs @@ -197,6 +197,7 @@ pub(super) fn scan_codex_detailed_with_cache( let mut bytes_read_this_refresh = 0_i64; let mut pending_next = cache.codex_pending_paths.clone(); let pending_paths_before_pass = cache.codex_pending_paths.clone(); + let mut invalidated_unsafe_lineage = false; prioritize_codex_pending_candidates(&mut candidates, &pending_paths_before_pass); defer_codex_locally_inferred_candidates(&mut candidates, &cache); if discovery_complete && !is_cancelled(cancel) { @@ -236,7 +237,17 @@ pub(super) fn scan_codex_detailed_with_cache( unprocessed.extend(work_queue.drain(..).map(|candidate| candidate.path)); unprocessed.extend(cancelled_during_preparation); } else { - order_codex_candidates_by_lineage(&mut work_queue); + let unsafe_cached_paths = order_codex_candidates_by_lineage(&cache, &mut work_queue); + invalidated_unsafe_lineage = !unsafe_cached_paths.is_empty(); + if invalidated_unsafe_lineage { + cache.previous_report = None; + } + invalidate_codex_unsafe_lineage(&mut cache, &unsafe_cached_paths); + for path in unsafe_cached_paths { + if !pending_next.contains(&path) { + pending_next.push(path); + } + } } let mut incomplete_processed = Vec::new(); @@ -369,7 +380,7 @@ pub(super) fn scan_codex_detailed_with_cache( // the range so unchanged files stay on the cache fast path. cache.scan_since_key = Some(scan_range.scan_since_key.clone()); cache.scan_until_key = Some(scan_range.scan_until_key.clone()); - } else if cache.previous_report.is_none() { + } else if cache.previous_report.is_none() && !invalidated_unsafe_lineage { cache.previous_report = established_report_before_scan; } if !is_cancelled(cancel) { diff --git a/rust/src/cost_scanner/tests.rs b/rust/src/cost_scanner/tests.rs index fb592e7180..d8dc136047 100644 --- a/rust/src/cost_scanner/tests.rs +++ b/rust/src/cost_scanner/tests.rs @@ -3040,5 +3040,8 @@ fn incomplete_or_buffered_empty_codex_fragment_is_not_marked_complete() { #[path = "tests/copied_prefix.rs"] mod copied_prefix; #[cfg(test)] +#[path = "tests/lineage_cache.rs"] +mod lineage_cache; +#[cfg(test)] #[path = "tests/paginated.rs"] mod paginated; diff --git a/rust/src/cost_scanner/tests/lineage_cache.rs b/rust/src/cost_scanner/tests/lineage_cache.rs new file mode 100644 index 0000000000..18c742ba72 --- /dev/null +++ b/rust/src/cost_scanner/tests/lineage_cache.rs @@ -0,0 +1,212 @@ +use super::*; + +fn write_subagent( + sessions_root: &Path, + name: &str, + session_id: &str, + parent_id: &str, + timestamp: DateTime, +) -> PathBuf { + let day = timestamp.with_timezone(&Local).date_naive(); + let day_dir = sessions_root + .join(day.format("%Y").to_string()) + .join(day.format("%m").to_string()) + .join(day.format("%d").to_string()); + std::fs::create_dir_all(&day_dir).unwrap(); + let path = day_dir.join(name); + let rows = [ + serde_json::json!({ + "type": "session_meta", "ordinal": 0, "timestamp": timestamp.to_rfc3339(), + "payload": { + "id": session_id, + "forked_from_id": parent_id, + "subagent_history_start_ordinal": 10, + "thread_source": "subagent", + "source": {"subagent": {"thread_spawn": {"parent_thread_id": parent_id}}} + } + }), + lineage_token_row(timestamp, 2, 1_000, 0), + serde_json::json!({ + "type": "turn_context", "ordinal": 10, "timestamp": timestamp.to_rfc3339(), + "payload": {"model": "gpt-5.6-sol"} + }), + lineage_token_row(timestamp, 12, 1_000, 1_000), + lineage_token_row(timestamp + Duration::seconds(1), 20, 1_050, 50), + ]; + let body = rows + .into_iter() + .map(|row| row.to_string()) + .collect::>() + .join("\n") + + "\n"; + std::fs::write(&path, body).unwrap(); + path +} + +fn lineage_token_row( + timestamp: DateTime, + ordinal: i64, + total_input: i64, + last_input: i64, +) -> serde_json::Value { + serde_json::json!({ + "type": "event_msg", "ordinal": ordinal, "timestamp": timestamp.to_rfc3339(), + "payload": {"type": "token_count", "info": { + "model": "gpt-5.6-sol", + "total_token_usage": { + "input_tokens": total_input, "cached_input_tokens": 0, "output_tokens": 5 + }, + "last_token_usage": { + "input_tokens": last_input, "cached_input_tokens": 0, "output_tokens": 5 + } + }} + }) +} + +fn bounded_scanner(sessions: &Path, cache_root: &Path) -> CostScanner { + let mut options = CostScanOptions::app_driven(); + options.codex_candidate_limit = 1; + options.prefer_newest_codex_sessions_first = false; + CostScanner::new(7) + .with_options(options) + .with_cache_root(cache_root) + .with_sessions_dirs(vec![sessions.to_path_buf()]) +} + +fn assert_locally_inferred(cache: &CostUsageCache, path: &Path) { + let usage = &cache.files[&path.to_string_lossy().to_string()]; + assert!(!usage.codex_unresolved_fork_parent); + assert!( + usage + .codex_fork_accounting_state + .as_ref() + .is_some_and(|state| state.locally_resolved) + ); +} + +fn assert_unresolved(cache: &CostUsageCache, path: &Path) { + let usage = &cache.files[&path.to_string_lossy().to_string()]; + assert!(usage.codex_unresolved_fork_parent); + assert!(usage.days.is_empty()); + assert!(usage.codex_fork_accounting_state.is_none()); +} + +#[test] +fn bounded_refresh_detects_duplicate_parent_owners_across_cache_and_candidate() { + let root = tempfile::tempdir().unwrap(); + let sessions = root.path().join("sessions"); + let cache_root = root.path().join("cache"); + let base = Utc::now() - Duration::hours(1); + let child = write_subagent(&sessions, "child.jsonl", "child-id", "parent-id", base); + let scanner = bounded_scanner(&sessions, &cache_root); + + let (_, first_stats, first_cache) = scanner.scan_codex_detailed_with_cache(None); + assert_eq!(first_stats.codex_read_receipt.metadata_reads, 1); + assert_eq!(first_stats.codex_read_receipt.history_reads, 1); + assert_locally_inferred(&first_cache, &child); + + let first_parent = write_codex_fork_session_fixture( + &sessions, + "parent-a.jsonl", + "parent-id", + None, + base - Duration::seconds(2), + base - Duration::seconds(2), + &[1_000], + ); + let (_, second_stats, second_cache) = scanner.scan_codex_detailed_with_cache(None); + assert_eq!(second_stats.codex_read_receipt.metadata_reads, 1); + assert_eq!(second_stats.codex_read_receipt.history_reads, 1); + assert_locally_inferred(&second_cache, &child); + + let second_parent = write_codex_fork_session_fixture( + &sessions, + "parent-b.jsonl", + "parent-id", + None, + base - Duration::seconds(1), + base - Duration::seconds(1), + &[2_000], + ); + let (summary, third_stats, cache) = scanner.scan_codex_detailed_with_cache(None); + + assert_eq!(third_stats.codex_read_receipt.metadata_reads, 1); + assert_eq!(third_stats.codex_read_receipt.history_reads, 0); + assert_eq!(summary.sessions_count, 0); + assert_unresolved(&cache, &first_parent); + assert_unresolved(&cache, &second_parent); + assert_unresolved(&cache, &child); +} + +#[test] +fn bounded_refresh_detects_equal_timestamp_two_node_cycle() { + let root = tempfile::tempdir().unwrap(); + let sessions = root.path().join("sessions"); + let cache_root = root.path().join("cache"); + let base = Utc::now() - Duration::hours(1); + let first = write_subagent(&sessions, "first.jsonl", "first-id", "second-id", base); + let scanner = bounded_scanner(&sessions, &cache_root); + let (_, _, first_cache) = scanner.scan_codex_detailed_with_cache(None); + assert_locally_inferred(&first_cache, &first); + + let second = write_subagent(&sessions, "second.jsonl", "second-id", "first-id", base); + let (summary, stats, cache) = scanner.scan_codex_detailed_with_cache(None); + + assert_eq!(stats.codex_read_receipt.metadata_reads, 1); + assert_eq!(stats.codex_read_receipt.history_reads, 0); + assert_eq!(summary.sessions_count, 0); + assert_unresolved(&cache, &first); + assert_unresolved(&cache, &second); +} + +#[test] +fn bounded_refresh_rejects_self_cycle_migration() { + let root = tempfile::tempdir().unwrap(); + let sessions = root.path().join("sessions"); + let cache_root = root.path().join("cache"); + let base = Utc::now() - Duration::hours(1); + let session = write_subagent(&sessions, "self.jsonl", "self-id", "missing-id", base); + let scanner = bounded_scanner(&sessions, &cache_root); + let (_, _, first_cache) = scanner.scan_codex_detailed_with_cache(None); + assert_locally_inferred(&first_cache, &session); + + write_subagent( + &sessions, + "self.jsonl", + "self-id", + "self-id", + base + Duration::seconds(1), + ); + let (summary, stats, cache) = scanner.scan_codex_detailed_with_cache(None); + + assert_eq!(stats.codex_read_receipt.metadata_reads, 1); + assert_eq!(stats.codex_read_receipt.history_reads, 0); + assert_eq!(summary.sessions_count, 0); + assert_unresolved(&cache, &session); +} + +#[test] +fn bounded_refresh_rejects_dependent_of_locally_inferred_parent() { + let root = tempfile::tempdir().unwrap(); + let sessions = root.path().join("sessions"); + let cache_root = root.path().join("cache"); + let base = Utc::now() - Duration::hours(1); + let parent = write_subagent(&sessions, "parent.jsonl", "parent-id", "missing-id", base); + let scanner = bounded_scanner(&sessions, &cache_root); + let (_, _, first_cache) = scanner.scan_codex_detailed_with_cache(None); + assert_locally_inferred(&first_cache, &parent); + + let dependent = write_subagent( + &sessions, + "dependent.jsonl", + "dependent-id", + "parent-id", + base + Duration::seconds(1), + ); + let (_, stats, cache) = scanner.scan_codex_detailed_with_cache(None); + + assert_eq!(stats.codex_read_receipt.metadata_reads, 1); + assert_eq!(stats.codex_read_receipt.history_reads, 0); + assert_locally_inferred(&cache, &parent); + assert_unresolved(&cache, &dependent); +} From 4558dc2efb5d83e88ca348f62c000487cc010931 Mon Sep 17 00:00:00 2001 From: NessZerra <90105158+Finesssee@users.noreply.github.com> Date: Wed, 23 Sep 2026 05:23:56 +0700 Subject: [PATCH 13/19] Harden Codex fork lineage validation --- rust/src/core/jsonl_scanner/codex/parser.rs | 30 +- rust/src/core/jsonl_scanner/tests.rs | 40 +++ rust/src/cost_scanner/codex.rs | 231 ++------------- rust/src/cost_scanner/codex/logical_target.rs | 270 ++++++++++++++++-- rust/src/cost_scanner/codex/reconciliation.rs | 39 ++- rust/src/cost_scanner/codex/scan.rs | 4 +- rust/src/cost_scanner/tests/lineage_cache.rs | 195 +++++++++++++ 7 files changed, 562 insertions(+), 247 deletions(-) diff --git a/rust/src/core/jsonl_scanner/codex/parser.rs b/rust/src/core/jsonl_scanner/codex/parser.rs index f33316e378..728bd16f76 100644 --- a/rust/src/core/jsonl_scanner/codex/parser.rs +++ b/rust/src/core/jsonl_scanner/codex/parser.rs @@ -58,6 +58,7 @@ struct ForkBaselineInference { baseline: Option, boundary_open: bool, inherited_opening: bool, + missing_explicit_ordinal: bool, locally_confirmed: bool, resolved: bool, } @@ -79,11 +80,18 @@ impl ForkBaselineInference { }), boundary_open: false, inherited_opening: false, - locally_confirmed: explicit_start_ordinal.is_some(), + missing_explicit_ordinal: false, + locally_confirmed: false, resolved: false, } } + fn confirm_local_resolution(&mut self) { + if !self.missing_explicit_ordinal { + self.locally_confirmed = true; + } + } + fn observe_non_token(&mut self, obj: &Value) { if obj.get("type").and_then(Value::as_str) == Some("turn_context") && self.inherited_opening { @@ -108,13 +116,19 @@ impl ForkBaselineInference { let last = read_token_totals(last_usage); let ordinal = obj.get("ordinal").and_then(Value::as_i64); + if self.explicit_start_ordinal.is_some() && ordinal.is_none() { + self.missing_explicit_ordinal = true; + self.locally_confirmed = false; + if !self.boundary_open { + self.baseline = Some(total); + } + return ForkBaselineDecision::SkipCopiedPrefix; + } + if let Some(start) = self.explicit_start_ordinal && !self.boundary_open { - let Some(ordinal) = ordinal else { - self.baseline = Some(total); - return ForkBaselineDecision::SkipCopiedPrefix; - }; + let ordinal = ordinal.expect("missing explicit ordinals return above"); if ordinal < start { self.baseline = Some(total); return ForkBaselineDecision::SkipCopiedPrefix; @@ -124,7 +138,7 @@ impl ForkBaselineInference { if totals_contain_usage(&total) && !totals_contain_usage(&last) { self.baseline = Some(total); self.inherited_opening = true; - self.locally_confirmed = true; + self.confirm_local_resolution(); } return ForkBaselineDecision::SkipCopiedPrefix; } else if !self.boundary_open { @@ -152,13 +166,13 @@ impl ForkBaselineInference { totals_contain_usage(&baseline) && total == last && totals_at_least(&total, &baseline); if copied_snapshot { self.baseline = Some(total); - self.locally_confirmed = true; + self.confirm_local_resolution(); return ForkBaselineDecision::SkipCopiedPrefix; } let owned_baseline = totals_delta(&last, &total); self.baseline = Some(owned_baseline.clone()); - self.locally_confirmed = true; + self.confirm_local_resolution(); self.resolved = true; ForkBaselineDecision::ProcessWithBaseline(owned_baseline) } diff --git a/rust/src/core/jsonl_scanner/tests.rs b/rust/src/core/jsonl_scanner/tests.rs index 2614ebefd3..4abfce1c93 100644 --- a/rust/src/core/jsonl_scanner/tests.rs +++ b/rust/src/core/jsonl_scanner/tests.rs @@ -132,6 +132,7 @@ fn inferred_fork_waits_for_present_explicit_start_ordinal() { assert!(state.records.is_empty()); assert!(state.fork_baseline.is_none()); + assert!(!state.fork_baseline_locally_resolved()); state.process_line( r#"{"ordinal":10,"timestamp":"2026-09-22T10:00:01Z","type":"event_msg","payload":{"type":"token_count","info":{"model":"gpt-5.6-sol","total_token_usage":{"input_tokens":110,"cached_input_tokens":22,"output_tokens":11},"last_token_usage":{"input_tokens":10,"cached_input_tokens":2,"output_tokens":1}}}}"#, @@ -142,6 +143,45 @@ fn inferred_fork_waits_for_present_explicit_start_ordinal() { assert_eq!(state.records[0].0.input, 10); assert_eq!(state.records[0].0.cached, 2); assert_eq!(state.records[0].0.output, 1); + assert!(!state.fork_baseline_locally_resolved()); +} + +#[test] +fn inferred_fork_keeps_missing_ordinal_unresolved_after_boundary_opens() { + let range = CostUsageDayRange::new( + NaiveDate::from_ymd_opt(2026, 9, 22).unwrap(), + NaiveDate::from_ymd_opt(2026, 9, 22).unwrap(), + ); + let mut state = CodexParserState::from_mode(CodexParseMode::InferSubagent { + start_ordinal: Some(10), + }); + let token_line = |ordinal: Option, total: i64, last: i64| { + let mut value = serde_json::json!({ + "timestamp": "2026-09-22T10:00:00Z", + "type": "event_msg", + "payload": {"type": "token_count", "info": { + "model": "gpt-5.6-sol", + "total_token_usage": {"input_tokens": total, "cached_input_tokens": 0, "output_tokens": 0}, + "last_token_usage": {"input_tokens": last, "cached_input_tokens": 0, "output_tokens": 0} + }} + }); + if let Some(ordinal) = ordinal { + value["ordinal"] = serde_json::json!(ordinal); + } + value.to_string() + }; + + state.process_line(&token_line(Some(9), 100, 0), &range); + state.process_line(&token_line(Some(10), 100, 0), &range); + state.process_line(&token_line(Some(11), 110, 110), &range); + assert!(state.fork_baseline_locally_resolved()); + state.process_line(&token_line(None, 120, 10), &range); + assert!(!state.fork_baseline_locally_resolved()); + state.process_line(&token_line(Some(12), 130, 10), &range); + + assert_eq!(state.records.len(), 1); + assert_eq!(state.records[0].0.input, 10); + assert!(!state.fork_baseline_locally_resolved()); } #[test] diff --git a/rust/src/cost_scanner/codex.rs b/rust/src/cost_scanner/codex.rs index 23022e2226..7e522c9da9 100644 --- a/rust/src/cost_scanner/codex.rs +++ b/rust/src/cost_scanner/codex.rs @@ -95,13 +95,6 @@ fn summary_from_cached_report( } } -#[derive(Debug, Clone, PartialEq, Eq)] -enum CodexParentResolution { - Absent, - Safe(crate::core::CodexTotals), - Unsafe, -} - fn codex_usage_uses_parent(usage: &CostUsageFileUsage) -> bool { usage.codex_lineage.uses_parent_baseline() || (matches!(usage.codex_lineage, CodexSessionLineage::Root) @@ -109,29 +102,7 @@ fn codex_usage_uses_parent(usage: &CostUsageFileUsage) -> bool { } fn codex_fork_parent_is_safe(cache: &CostUsageCache, usage: &CostUsageFileUsage) -> bool { - let locally_resolved = usage - .codex_fork_accounting_state - .as_ref() - .is_some_and(|state| state.locally_resolved); - if !codex_usage_uses_parent(usage) { - return true; - } - let parent_resolution = - usage - .codex_forked_from_id - .as_deref() - .map_or(CodexParentResolution::Unsafe, |parent_id| { - codex_parent_resolution(cache, parent_id, usage.codex_fork_timestamp.as_deref()) - }); - - // Local inference is safe only while the parent is genuinely absent. - // An owner that is ambiguous, stale, locally inferred, cyclic, or - // transitively unsafe must fail closed instead of looking absent. - if locally_resolved { - matches!(parent_resolution, CodexParentResolution::Absent) - } else { - matches!(parent_resolution, CodexParentResolution::Safe(_)) - } + CodexLineagePlanner::new(cache).cached_usage_is_safe(usage) } fn codex_fork_uses_local_inference(usage: &CostUsageFileUsage) -> bool { @@ -141,103 +112,6 @@ fn codex_fork_uses_local_inference(usage: &CostUsageFileUsage) -> bool { .is_some_and(|state| state.locally_resolved) } -/// Resolve one parent identity through the persisted cache graph. Absence is -/// deliberately distinct from ambiguity or transitive unsafety so copied -/// prefixes may infer only when no owner exists at all. -fn codex_parent_resolution( - cache: &CostUsageCache, - parent_session_id: &str, - child_fork_timestamp: Option<&str>, -) -> CodexParentResolution { - codex_parent_resolution_inner( - cache, - parent_session_id, - child_fork_timestamp, - &mut HashSet::new(), - ) -} - -fn codex_parent_resolution_inner( - cache: &CostUsageCache, - parent_session_id: &str, - child_fork_timestamp: Option<&str>, - visiting: &mut HashSet, -) -> CodexParentResolution { - let mut owners = cache - .files - .iter() - .filter(|(_, usage)| usage.codex_session_id.as_deref() == Some(parent_session_id)); - let Some((path_key, usage)) = owners.next() else { - return CodexParentResolution::Absent; - }; - if owners.next().is_some() || !visiting.insert(path_key.clone()) { - return CodexParentResolution::Unsafe; - } - - let resolution = - codex_parent_owner_baseline(cache, path_key, usage, child_fork_timestamp, visiting) - .map_or(CodexParentResolution::Unsafe, CodexParentResolution::Safe); - visiting.remove(path_key); - resolution -} - -fn codex_parent_owner_baseline( - cache: &CostUsageCache, - path_key: &str, - usage: &CostUsageFileUsage, - child_fork_timestamp: Option<&str>, - visiting: &mut HashSet, -) -> Option { - if usage.codex_unresolved_fork_parent - || usage.codex_token_timestamps_monotonic != Some(true) - || codex_fork_uses_local_inference(usage) - { - return None; - } - - if codex_usage_uses_parent(usage) { - let parent_id = usage.codex_forked_from_id.as_deref()?; - let inherited = usage - .codex_fork_accounting_state - .as_ref()? - .inherited_totals - .as_ref()?; - match codex_parent_resolution_inner( - cache, - parent_id, - usage.codex_fork_timestamp.as_deref(), - visiting, - ) { - CodexParentResolution::Safe(baseline) if &baseline == inherited => {} - CodexParentResolution::Absent - | CodexParentResolution::Safe(_) - | CodexParentResolution::Unsafe => return None, - } - } - - let metadata = fs::metadata(path_key).ok()?; - if let (Some(expected), Some(actual)) = ( - usage.codex_file_identity.as_ref(), - JsonlScanner::codex_file_identity(Path::new(path_key), &metadata), - ) && expected != &actual - { - return None; - } - #[allow(clippy::cast_possible_wrap, reason = "session file sizes fit i64")] - let size = metadata.len().min(i64::MAX as u64) as i64; - if usage.mtime_unix_ms != system_time_to_unix_ms(metadata.modified().ok()) - || usage.size != size - || usage.parsed_bytes.unwrap_or(0) < size - { - return None; - } - let last_totals = usage.last_totals.clone()?; - let last_token_timestamp = usage.codex_last_token_timestamp.as_deref()?; - let child_fork_timestamp = child_fork_timestamp?; - JsonlScanner::codex_timestamp_at_or_before(last_token_timestamp, child_fork_timestamp) - .then_some(last_totals) -} - fn is_codex_path_in_scan_window( path: &Path, sessions_dirs: &[PathBuf], @@ -268,14 +142,7 @@ struct CodexScanCandidate { struct CodexPreparedCandidate { path: PathBuf, session_metadata: CodexSessionMetadata, - lineage_disposition: CodexLineageDisposition, -} - -#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] -enum CodexLineageDisposition { - #[default] - Ready, - AmbiguousOrCyclic, + lineage_gate: CodexLineageGate, } #[derive(Debug, Clone, Copy, Default)] @@ -511,21 +378,19 @@ impl CostScanner { let cache_entry_is_fresh = |entry: &CostUsageFileUsage| { cached_codex_file_is_fresh(cache, entry, cache_covers_range, mtime_ms, size) }; - let identity_matches_cached = |entry: &CostUsageFileUsage| match ( - entry.codex_file_identity.as_ref(), - file_identity.as_ref(), - ) { - (Some(expected), Some(actual)) => expected == actual, - _ => false, + let identity_matches_cached = |entry: &CostUsageFileUsage| { + codex_file_identity_matches( + entry.codex_file_identity.as_deref(), + file_identity.as_deref(), + ) }; // The compact cache is authoritative for an unchanged file. Do this // before reading even the bounded metadata prefix; raw token history // is only needed after freshness fails or a fork needs reconciliation. if let Some(entry) = cached.as_ref() - && prepared_candidate.is_none_or(|candidate| { - candidate.lineage_disposition == CodexLineageDisposition::Ready - }) + && prepared_candidate + .is_none_or(|candidate| candidate.lineage_gate == CodexLineageGate::Eligible) && cache_entry_is_fresh(entry) && identity_matches_cached(entry) { @@ -550,9 +415,9 @@ impl CostScanner { stats.codex_read_receipt.metadata_reads.saturating_add(1); JsonlScanner::read_codex_session_metadata(path).unwrap_or_default() }; - let cached_identity_matches = cached - .as_ref() - .is_some_and(|entry| entry.mtime_unix_ms == mtime_ms && entry.size == size); + // Cached lineage metadata belongs to a physical file, not merely a + // path/size/mtime tuple. Missing identity evidence fails closed. + let cached_identity_matches = cached.as_ref().is_some_and(identity_matches_cached); let codex_session_id = session_metadata.session_id.clone().or_else(|| { cached_identity_matches .then(|| cached.as_ref()?.codex_session_id.clone()) @@ -626,62 +491,25 @@ impl CostScanner { let matching_cached_fork_state = cached_fork_accounting_state .as_ref() .filter(|_| cached_fork_state_matches); - let parent_resolution = is_fork - .then_some(codex_forked_from_id.as_deref()) - .flatten() - .map_or(CodexParentResolution::Unsafe, |parent_id| { - codex_parent_resolution(cache, parent_id, codex_fork_timestamp.as_deref()) - }); let paginated_continuation = is_fork && codex_forked_from_id.is_some() && history_base_thread_id .as_deref() .is_some_and(|history_base| Some(history_base) != codex_forked_from_id.as_deref()); - let accounting_mode = if prepared_candidate.is_some_and(|candidate| { - candidate.lineage_disposition == CodexLineageDisposition::AmbiguousOrCyclic - }) { - CodexAccountingMode::Unresolved - } else if !is_fork { - CodexAccountingMode::Standard - } else if let CodexParentResolution::Safe(baseline) = &parent_resolution { - let reparse_cached_file = matching_cached_fork_state.is_some_and(|state| { - state.locally_resolved || state.inherited_totals.as_ref() != Some(baseline) - }); - let cached_parent_state = matching_cached_fork_state.filter(|state| { - !state.locally_resolved && state.inherited_totals.as_ref() == Some(baseline) - }); - CodexAccountingMode::Baseline { - baseline: baseline.clone(), - paginated_continuation, - remaining_inherited_totals: cached_parent_state - .and_then(|state| state.remaining_inherited_totals.clone()), - provenance: CodexBaselineProvenance::ValidatedParent { - replaces_cached_state: reparse_cached_file, - }, - } - } else if matches!(&parent_resolution, CodexParentResolution::Absent) - && let Some(state) = matching_cached_fork_state - && let Some(baseline) = state.inherited_totals.clone() - { - CodexAccountingMode::Baseline { - baseline, - paginated_continuation, - remaining_inherited_totals: state.remaining_inherited_totals.clone(), - provenance: if state.locally_resolved { - CodexBaselineProvenance::CachedLocalInference - } else { - CodexBaselineProvenance::CachedValidatedParent - }, - } - } else if matches!(&parent_resolution, CodexParentResolution::Absent) - && session_metadata.is_subagent - { - CodexAccountingMode::InferSubagent { - start_ordinal: session_metadata.subagent_history_start_ordinal, - } - } else { - CodexAccountingMode::Unresolved - }; + let lineage_gate = prepared_candidate + .map(|candidate| candidate.lineage_gate) + .unwrap_or_default(); + let lineage_decision = CodexLineagePlanner::new(cache).decision_for_scan( + is_fork, + lineage_gate, + codex_forked_from_id.as_deref(), + codex_fork_timestamp.as_deref(), + ); + let accounting_mode = lineage_decision.accounting_mode( + matching_cached_fork_state, + &session_metadata, + paginated_continuation, + ); if accounting_mode.is_unresolved() { cache.files.insert( @@ -714,7 +542,7 @@ impl CostScanner { if let Some(entry) = &cached && cached_codex_file_is_fresh(cache, entry, cache_covers_range, mtime_ms, size) - && (entry.codex_file_identity.is_none() || identity_matches_cached(entry)) + && identity_matches_cached(entry) && !cached_identity_changed && !accounting_mode.requires_cached_reparse() { @@ -724,11 +552,6 @@ impl CostScanner { summary.total_cost_usd += session_cost; summary.sessions_count += 1; } - if entry.codex_file_identity != file_identity { - let mut refreshed = entry.clone(); - refreshed.codex_file_identity = file_identity.clone(); - cache.files.insert(path_key.clone(), refreshed); - } stats.files_skipped = stats.files_skipped.saturating_add(1); return CodexFileScanOutcome { bytes_read: 0, diff --git a/rust/src/cost_scanner/codex/logical_target.rs b/rust/src/cost_scanner/codex/logical_target.rs index 25eb78d8e8..6245ea6c3b 100644 --- a/rust/src/cost_scanner/codex/logical_target.rs +++ b/rust/src/cost_scanner/codex/logical_target.rs @@ -1,5 +1,221 @@ use super::*; +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] +pub(super) enum CodexLineageGate { + #[default] + Eligible, + Unsafe, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub(super) enum CodexLineageDecision { + Root, + ParentAbsent, + ParentReady(crate::core::CodexTotals), + Unsafe, +} + +pub(super) struct CodexLineagePlanner<'a> { + cache: &'a CostUsageCache, +} + +impl<'a> CodexLineagePlanner<'a> { + pub(super) fn new(cache: &'a CostUsageCache) -> Self { + Self { cache } + } + + pub(super) fn decision_for_scan( + &self, + uses_parent: bool, + gate: CodexLineageGate, + parent_id: Option<&str>, + fork_timestamp: Option<&str>, + ) -> CodexLineageDecision { + if gate == CodexLineageGate::Unsafe { + return CodexLineageDecision::Unsafe; + } + if !uses_parent { + return CodexLineageDecision::Root; + } + parent_id.map_or(CodexLineageDecision::Unsafe, |parent_id| { + self.resolve_parent(parent_id, fork_timestamp) + }) + } + + pub(super) fn decision_for_usage(&self, usage: &CostUsageFileUsage) -> CodexLineageDecision { + if usage.codex_unresolved_fork_parent { + return CodexLineageDecision::Unsafe; + } + if !super::codex_usage_uses_parent(usage) { + return CodexLineageDecision::Root; + } + usage + .codex_forked_from_id + .as_deref() + .map_or(CodexLineageDecision::Unsafe, |parent_id| { + self.resolve_parent(parent_id, usage.codex_fork_timestamp.as_deref()) + }) + } + + pub(super) fn cached_usage_is_safe(&self, usage: &CostUsageFileUsage) -> bool { + let locally_resolved = super::codex_fork_uses_local_inference(usage); + match self.decision_for_usage(usage) { + CodexLineageDecision::Root => true, + CodexLineageDecision::ParentAbsent => locally_resolved, + CodexLineageDecision::ParentReady(_) => !locally_resolved, + CodexLineageDecision::Unsafe => false, + } + } + + /// Resolve one parent identity through the persisted graph. Absence is + /// distinct from ambiguity and transitive unsafety so local inference is + /// allowed only when no owner exists at all. + fn resolve_parent( + &self, + parent_session_id: &str, + child_fork_timestamp: Option<&str>, + ) -> CodexLineageDecision { + self.resolve_parent_inner(parent_session_id, child_fork_timestamp, &mut HashSet::new()) + } + + fn resolve_parent_inner( + &self, + parent_session_id: &str, + child_fork_timestamp: Option<&str>, + visiting: &mut HashSet, + ) -> CodexLineageDecision { + let mut owners = self + .cache + .files + .iter() + .filter(|(_, usage)| usage.codex_session_id.as_deref() == Some(parent_session_id)); + let Some((path_key, usage)) = owners.next() else { + return CodexLineageDecision::ParentAbsent; + }; + if owners.next().is_some() || !visiting.insert(path_key.clone()) { + return CodexLineageDecision::Unsafe; + } + + let decision = self + .parent_owner_baseline(path_key, usage, child_fork_timestamp, visiting) + .map_or( + CodexLineageDecision::Unsafe, + CodexLineageDecision::ParentReady, + ); + visiting.remove(path_key); + decision + } + + fn parent_owner_baseline( + &self, + path_key: &str, + usage: &CostUsageFileUsage, + child_fork_timestamp: Option<&str>, + visiting: &mut HashSet, + ) -> Option { + if usage.codex_unresolved_fork_parent + || usage.codex_token_timestamps_monotonic != Some(true) + || super::codex_fork_uses_local_inference(usage) + { + return None; + } + + if super::codex_usage_uses_parent(usage) { + let parent_id = usage.codex_forked_from_id.as_deref()?; + let inherited = usage + .codex_fork_accounting_state + .as_ref()? + .inherited_totals + .as_ref()?; + match self.resolve_parent_inner( + parent_id, + usage.codex_fork_timestamp.as_deref(), + visiting, + ) { + CodexLineageDecision::ParentReady(baseline) if &baseline == inherited => {} + CodexLineageDecision::Root + | CodexLineageDecision::ParentAbsent + | CodexLineageDecision::ParentReady(_) + | CodexLineageDecision::Unsafe => return None, + } + } + + let metadata = fs::metadata(path_key).ok()?; + let expected_identity = usage.codex_file_identity.as_ref()?; + let actual_identity = JsonlScanner::codex_file_identity(Path::new(path_key), &metadata)?; + if expected_identity != &actual_identity { + return None; + } + #[allow(clippy::cast_possible_wrap, reason = "session file sizes fit i64")] + let size = metadata.len().min(i64::MAX as u64) as i64; + if usage.mtime_unix_ms != system_time_to_unix_ms(metadata.modified().ok()) + || usage.size != size + || usage.parsed_bytes.unwrap_or(0) < size + { + return None; + } + let last_totals = usage.last_totals.clone()?; + let last_token_timestamp = usage.codex_last_token_timestamp.as_deref()?; + let child_fork_timestamp = child_fork_timestamp?; + JsonlScanner::codex_timestamp_at_or_before(last_token_timestamp, child_fork_timestamp) + .then_some(last_totals) + } +} + +impl CodexLineageDecision { + pub(super) fn accounting_mode( + &self, + matching_cached_state: Option<&CodexForkAccountingState>, + metadata: &CodexSessionMetadata, + paginated_continuation: bool, + ) -> CodexAccountingMode { + match self { + Self::Root => CodexAccountingMode::Standard, + Self::Unsafe => CodexAccountingMode::Unresolved, + Self::ParentReady(baseline) => { + let replaces_cached_state = matching_cached_state.is_some_and(|state| { + state.locally_resolved || state.inherited_totals.as_ref() != Some(baseline) + }); + let cached_parent_state = matching_cached_state.filter(|state| { + !state.locally_resolved && state.inherited_totals.as_ref() == Some(baseline) + }); + CodexAccountingMode::Baseline { + baseline: baseline.clone(), + paginated_continuation, + remaining_inherited_totals: cached_parent_state + .and_then(|state| state.remaining_inherited_totals.clone()), + provenance: CodexBaselineProvenance::ValidatedParent { + replaces_cached_state, + }, + } + } + Self::ParentAbsent => { + if let Some(state) = matching_cached_state + && let Some(baseline) = state.inherited_totals.clone() + { + return CodexAccountingMode::Baseline { + baseline, + paginated_continuation, + remaining_inherited_totals: state.remaining_inherited_totals.clone(), + provenance: if state.locally_resolved { + CodexBaselineProvenance::CachedLocalInference + } else { + CodexBaselineProvenance::CachedValidatedParent + }, + }; + } + if metadata.is_subagent { + CodexAccountingMode::InferSubagent { + start_ordinal: metadata.subagent_history_start_ordinal, + } + } else { + CodexAccountingMode::Unresolved + } + } + } + } +} + pub(super) fn cached_codex_file_is_fresh( cache: &CostUsageCache, entry: &CostUsageFileUsage, @@ -16,6 +232,12 @@ pub(super) fn cached_codex_file_is_fresh( && super::codex_fork_parent_is_safe(cache, entry) } +pub(super) fn codex_file_identity_matches(expected: Option<&str>, actual: Option<&str>) -> bool { + expected + .zip(actual) + .is_some_and(|(expected, actual)| expected == actual) +} + pub(super) fn cached_codex_file_is_complete_for_range( cache: &CostUsageCache, path_key: &str, @@ -26,14 +248,10 @@ pub(super) fn cached_codex_file_is_complete_for_range( let Ok(metadata) = fs::metadata(path_key) else { return false; }; - let identity_matches = match ( - usage.codex_file_identity.as_ref(), - JsonlScanner::codex_file_identity(Path::new(path_key), &metadata).as_ref(), - ) { - (Some(expected), Some(actual)) => expected == actual, - (Some(_), None) => false, - (None, _) => true, - }; + let identity_matches = codex_file_identity_matches( + usage.codex_file_identity.as_deref(), + JsonlScanner::codex_file_identity(Path::new(path_key), &metadata).as_deref(), + ); #[allow(clippy::cast_possible_wrap, reason = "session file sizes fit i64")] let size = metadata.len().min(i64::MAX as u64) as i64; identity_matches @@ -91,7 +309,7 @@ struct CodexLineageNode { /// Order one bounded work set against both its admitted metadata and the /// persisted cache graph. The returned cache paths became structurally unsafe /// and must be invalidated even when the candidate limit deferred them. -pub(super) fn order_codex_candidates_by_lineage( +pub(super) fn plan_codex_candidates_by_lineage( cache: &CostUsageCache, candidates: &mut Vec, ) -> Vec { @@ -156,13 +374,19 @@ pub(super) fn order_codex_candidates_by_lineage( .or_default() .push(index); } - let mut unsafe_lineage = nodes + let mut lineage_gates = nodes .iter() - .map(|node| node.initially_unsafe) + .map(|node| { + if node.initially_unsafe { + CodexLineageGate::Unsafe + } else { + CodexLineageGate::Eligible + } + }) .collect::>(); for owners in session_owners.values().filter(|owners| owners.len() > 1) { for &index in owners { - unsafe_lineage[index] = true; + lineage_gates[index] = CodexLineageGate::Unsafe; } } let mut parent_indices = vec![None; nodes.len()]; @@ -173,8 +397,8 @@ pub(super) fn order_codex_candidates_by_lineage( match session_owners.get(parent_id).map(Vec::as_slice) { Some([parent_index]) => parent_indices[index] = Some(*parent_index), Some([]) | None if node.may_infer_missing_parent => {} - Some([]) | None => unsafe_lineage[index] = true, - Some(_) => unsafe_lineage[index] = true, + Some([]) | None => lineage_gates[index] = CodexLineageGate::Unsafe, + Some(_) => lineage_gates[index] = CodexLineageGate::Unsafe, } } @@ -184,12 +408,12 @@ pub(super) fn order_codex_candidates_by_lineage( loop { let mut progressed = false; for index in 0..nodes.len() { - if completed[index] || unsafe_lineage[index] { + if completed[index] || lineage_gates[index] == CodexLineageGate::Unsafe { continue; } let parent_is_ready = parent_indices[index].is_none_or(|parent_index| { completed[parent_index] - && !unsafe_lineage[parent_index] + && lineage_gates[parent_index] == CodexLineageGate::Eligible && nodes[parent_index].may_author_parent }); if parent_is_ready { @@ -207,7 +431,7 @@ pub(super) fn order_codex_candidates_by_lineage( for index in 0..nodes.len() { if !completed[index] { - unsafe_lineage[index] = true; + lineage_gates[index] = CodexLineageGate::Unsafe; if let Some(candidate_index) = nodes[index].candidate_index { ordered_indices.push(candidate_index); } @@ -220,19 +444,15 @@ pub(super) fn order_codex_candidates_by_lineage( let mut candidate = remaining[candidate_index] .take() .expect("candidate is ordered once"); - candidate.lineage_disposition = if unsafe_lineage[node_index] { - CodexLineageDisposition::AmbiguousOrCyclic - } else { - CodexLineageDisposition::Ready - }; + candidate.lineage_gate = lineage_gates[node_index]; candidates.push(candidate); } nodes .iter() - .zip(unsafe_lineage) - .filter(|(node, unsafe_lineage)| { - *unsafe_lineage + .zip(lineage_gates) + .filter(|(node, gate)| { + *gate == CodexLineageGate::Unsafe && cache .files .get(&node.path) diff --git a/rust/src/cost_scanner/codex/reconciliation.rs b/rust/src/cost_scanner/codex/reconciliation.rs index b16d211b3b..8451c3f72b 100644 --- a/rust/src/cost_scanner/codex/reconciliation.rs +++ b/rust/src/cost_scanner/codex/reconciliation.rs @@ -108,14 +108,10 @@ fn codex_pending_path_affects_current_window( if codex_logical_target_has_unconsumed_tail(observed_size, usage) { return true; } - let identity_matches = match ( - usage.codex_file_identity.as_ref(), - JsonlScanner::codex_file_identity(Path::new(path_key), &metadata).as_ref(), - ) { - (Some(expected), Some(actual)) => expected == actual, - (Some(_), None) => false, - (None, _) => true, - }; + let identity_matches = super::codex_file_identity_matches( + usage.codex_file_identity.as_deref(), + JsonlScanner::codex_file_identity(Path::new(path_key), &metadata).as_deref(), + ); if !identity_matches || usage.mtime_unix_ms != system_time_to_unix_ms(metadata.modified().ok()) || usage.size != observed_size @@ -270,6 +266,7 @@ mod tests { let old_usage = cache.files.get_mut(&old_key).unwrap(); old_usage.mtime_unix_ms = system_time_to_unix_ms(metadata.modified().ok()); old_usage.size = i64::try_from(metadata.len()).unwrap(); + old_usage.codex_file_identity = JsonlScanner::codex_file_identity(&old_path, &metadata); let range = active_range(); assert!(codex_current_window_is_established(&cache, &range)); @@ -279,6 +276,32 @@ mod tests { assert_eq!(report.sessions_count, 1); } + #[test] + fn historical_pending_entry_without_or_mismatched_identity_blocks_publication() { + let root = tempfile::tempdir().unwrap(); + let old_path = root.path().join("old.jsonl"); + let current_path = root.path().join("current.jsonl"); + std::fs::write(&old_path, vec![0_u8; 100]).unwrap(); + std::fs::write(¤t_path, vec![0_u8; 100]).unwrap(); + let old_key = old_path.to_string_lossy().into_owned(); + let current_key = current_path.to_string_lossy().into_owned(); + let metadata = std::fs::metadata(&old_path).unwrap(); + let range = active_range(); + + for cached_identity in [None, Some("different-file".to_string())] { + let mut cache = historical_pending_cache(&old_key, ¤t_key); + let old_usage = cache.files.get_mut(&old_key).unwrap(); + old_usage.mtime_unix_ms = system_time_to_unix_ms(metadata.modified().ok()); + old_usage.size = i64::try_from(metadata.len()).unwrap(); + old_usage.codex_file_identity = cached_identity; + + assert!(codex_pending_path_affects_current_window( + &cache, &old_key, &range + )); + assert!(!codex_current_window_is_established(&cache, &range)); + } + } + #[test] fn metadata_failure_blocks_historical_pending_publication() { let old_path = r"C:\sessions\missing-old.jsonl"; diff --git a/rust/src/cost_scanner/codex/scan.rs b/rust/src/cost_scanner/codex/scan.rs index 6da7fb5188..8b95e31f0d 100644 --- a/rust/src/cost_scanner/codex/scan.rs +++ b/rust/src/cost_scanner/codex/scan.rs @@ -229,7 +229,7 @@ pub(super) fn scan_codex_detailed_with_cache( session_metadata: JsonlScanner::read_codex_session_metadata(&candidate.path) .unwrap_or_default(), path: candidate.path, - lineage_disposition: CodexLineageDisposition::Ready, + lineage_gate: CodexLineageGate::Eligible, }); } let mut unprocessed = Vec::new(); @@ -237,7 +237,7 @@ pub(super) fn scan_codex_detailed_with_cache( unprocessed.extend(work_queue.drain(..).map(|candidate| candidate.path)); unprocessed.extend(cancelled_during_preparation); } else { - let unsafe_cached_paths = order_codex_candidates_by_lineage(&cache, &mut work_queue); + let unsafe_cached_paths = plan_codex_candidates_by_lineage(&cache, &mut work_queue); invalidated_unsafe_lineage = !unsafe_cached_paths.is_empty(); if invalidated_unsafe_lineage { cache.previous_report = None; diff --git a/rust/src/cost_scanner/tests/lineage_cache.rs b/rust/src/cost_scanner/tests/lineage_cache.rs index 18c742ba72..8ff35834d3 100644 --- a/rust/src/cost_scanner/tests/lineage_cache.rs +++ b/rust/src/cost_scanner/tests/lineage_cache.rs @@ -63,6 +63,52 @@ fn lineage_token_row( }) } +fn write_missing_ordinal_subagent( + sessions_root: &Path, + name: &str, + base: DateTime, + include_owned_usage: bool, +) -> PathBuf { + let day = base.with_timezone(&Local).date_naive(); + let day_dir = sessions_root + .join(day.format("%Y").to_string()) + .join(day.format("%m").to_string()) + .join(day.format("%d").to_string()); + std::fs::create_dir_all(&day_dir).unwrap(); + let path = day_dir.join(name); + let mut missing_ordinal = lineage_token_row(base, 11, 100, 0); + missing_ordinal.as_object_mut().unwrap().remove("ordinal"); + let tail = if include_owned_usage { + lineage_token_row(base, 12, 120, 10) + } else { + lineage_token_row(base, 12, 100, 0) + }; + let rows = [ + serde_json::json!({ + "type": "session_meta", "ordinal": 0, "timestamp": base.to_rfc3339(), + "payload": { + "id": "child-id", + "forked_from_id": "absent-parent-id", + "subagent_history_start_ordinal": 10, + "thread_source": "subagent", + "source": {"subagent": {"thread_spawn": {"parent_thread_id": "absent-parent-id"}}} + } + }), + lineage_token_row(base, 9, 100, 0), + lineage_token_row(base, 10, 100, 0), + missing_ordinal, + tail, + ]; + let body = rows + .into_iter() + .map(|row| row.to_string()) + .collect::>() + .join("\n") + + "\n"; + std::fs::write(&path, body).unwrap(); + path +} + fn bounded_scanner(sessions: &Path, cache_root: &Path) -> CostScanner { let mut options = CostScanOptions::app_driven(); options.codex_candidate_limit = 1; @@ -91,6 +137,155 @@ fn assert_unresolved(cache: &CostUsageCache, path: &Path) { assert!(usage.codex_fork_accounting_state.is_none()); } +#[test] +fn replaced_parent_with_same_path_size_and_mtime_cannot_author_lineage() { + let root = tempfile::tempdir().unwrap(); + let sessions = root.path().join("sessions"); + let cache_root = root.path().join("cache"); + let base = Utc::now() - Duration::hours(1); + let parent = write_codex_fork_session_fixture( + &sessions, + "parent.jsonl", + "parent-id", + None, + base, + base, + &[1_000], + ); + let child = write_subagent( + &sessions, + "child.jsonl", + "child-id", + "parent-id", + base + Duration::seconds(10), + ); + let scanner = CostScanner::new(7) + .with_options(CostScanOptions::app_driven()) + .with_cache_root(&cache_root) + .with_sessions_dirs(vec![sessions.clone()]); + let (_, _, cache) = scanner.scan_codex_detailed_with_cache(None); + let parent_key = parent.to_string_lossy().to_string(); + let child_usage = &cache.files[&child.to_string_lossy().to_string()]; + assert!(matches!( + CodexLineagePlanner::new(&cache).decision_for_usage(child_usage), + CodexLineageDecision::ParentReady(_) + )); + + let old_identity = cache.files[&parent_key] + .codex_file_identity + .clone() + .expect("parent identity persisted"); + let old_metadata = std::fs::metadata(&parent).unwrap(); + let old_mtime = old_metadata.modified().unwrap(); + let old_size = old_metadata.len(); + let rotated = parent.with_extension("old"); + std::fs::rename(&parent, &rotated).unwrap(); + let replacement = write_codex_fork_session_fixture( + &sessions, + "parent.jsonl", + "parent-id", + None, + base, + base, + &[2_000], + ); + std::fs::OpenOptions::new() + .write(true) + .open(&replacement) + .unwrap() + .set_modified(old_mtime) + .unwrap(); + let replacement_metadata = std::fs::metadata(&replacement).unwrap(); + assert_eq!(replacement_metadata.len(), old_size); + let replacement_identity = + JsonlScanner::codex_file_identity(&replacement, &replacement_metadata) + .expect("replacement identity available"); + assert_ne!(replacement_identity, old_identity); + + assert_eq!( + CodexLineagePlanner::new(&cache).decision_for_usage(child_usage), + CodexLineageDecision::Unsafe + ); +} + +#[test] +fn missing_explicit_ordinal_keeps_subagent_cache_unresolved() { + let root = tempfile::tempdir().unwrap(); + let sessions = root.path().join("sessions"); + let cache_root = root.path().join("cache"); + let base = Utc::now() - Duration::hours(1); + let child = write_missing_ordinal_subagent(&sessions, "child.jsonl", base, true); + let scanner = CostScanner::new(7) + .with_options(CostScanOptions::app_driven()) + .with_cache_root(&cache_root) + .with_sessions_dirs(vec![sessions]); + + let (summary, _, cache) = scanner.scan_codex_detailed_with_cache(None); + + assert_eq!(summary.input_tokens, 0); + assert_eq!(summary.sessions_count, 0); + assert_unresolved(&cache, &child); +} + +#[test] +fn missing_ordinal_cannot_complete_zero_usage_subagent_cache() { + let root = tempfile::tempdir().unwrap(); + let sessions = root.path().join("sessions"); + let cache_root = root.path().join("cache"); + let base = Utc::now() - Duration::hours(1); + let child = write_missing_ordinal_subagent(&sessions, "child.jsonl", base, false); + let scanner = CostScanner::new(7) + .with_options(CostScanOptions::app_driven()) + .with_cache_root(&cache_root) + .with_sessions_dirs(vec![sessions]); + + let (summary, _, cache) = scanner.scan_codex_detailed_with_cache(None); + + assert_eq!(summary.input_tokens, 0); + assert_eq!(summary.sessions_count, 0); + assert_unresolved(&cache, &child); +} + +#[test] +fn legacy_cache_without_file_identity_is_reparsed() { + let root = tempfile::tempdir().unwrap(); + let sessions = root.path().join("sessions"); + let cache_root = root.path().join("cache"); + let base = Utc::now() - Duration::hours(1); + let session = write_codex_fork_session_fixture( + &sessions, + "session.jsonl", + "root-session-id", + None, + base, + base, + &[1_000], + ); + let scanner = CostScanner::new(7) + .with_options(CostScanOptions::app_driven()) + .with_cache_root(&cache_root) + .with_sessions_dirs(vec![sessions]); + + let (_, _, _) = scanner.scan_codex_detailed_with_cache(None); + let session_key = session.to_string_lossy().to_string(); + let mut legacy_cache = JsonlScanner::load_cache(ProviderId::Codex, Some(&cache_root)); + legacy_cache + .files + .get_mut(&session_key) + .unwrap() + .codex_file_identity = None; + JsonlScanner::save_cache(ProviderId::Codex, &mut legacy_cache, Some(&cache_root)); + + let (_, stats, refreshed_cache) = scanner.scan_codex_detailed_with_cache(None); + + assert!(stats.codex_history_read_paths.contains(&session_key)); + assert!( + refreshed_cache.files[&session_key] + .codex_file_identity + .is_some() + ); +} + #[test] fn bounded_refresh_detects_duplicate_parent_owners_across_cache_and_candidate() { let root = tempfile::tempdir().unwrap(); From a7f685ae4939d3253e1f0473c4fa260971be62d3 Mon Sep 17 00:00:00 2001 From: NessZerra <90105158+Finesssee@users.noreply.github.com> Date: Wed, 23 Sep 2026 05:56:21 +0700 Subject: [PATCH 14/19] Consolidate Codex lineage validation graph --- rust/src/cost_scanner/codex.rs | 4 + rust/src/cost_scanner/codex/logical_target.rs | 448 ++++++++++-------- rust/src/cost_scanner/codex/scan.rs | 4 +- 3 files changed, 245 insertions(+), 211 deletions(-) diff --git a/rust/src/cost_scanner/codex.rs b/rust/src/cost_scanner/codex.rs index 7e522c9da9..a64eb92a39 100644 --- a/rust/src/cost_scanner/codex.rs +++ b/rust/src/cost_scanner/codex.rs @@ -143,6 +143,7 @@ struct CodexPreparedCandidate { path: PathBuf, session_metadata: CodexSessionMetadata, lineage_gate: CodexLineageGate, + parent_owner_expected: bool, } #[derive(Debug, Clone, Copy, Default)] @@ -499,11 +500,14 @@ impl CostScanner { let lineage_gate = prepared_candidate .map(|candidate| candidate.lineage_gate) .unwrap_or_default(); + let parent_owner_expected = + prepared_candidate.is_some_and(|candidate| candidate.parent_owner_expected); let lineage_decision = CodexLineagePlanner::new(cache).decision_for_scan( is_fork, lineage_gate, codex_forked_from_id.as_deref(), codex_fork_timestamp.as_deref(), + parent_owner_expected, ); let accounting_mode = lineage_decision.accounting_mode( matching_cached_fork_state, diff --git a/rust/src/cost_scanner/codex/logical_target.rs b/rust/src/cost_scanner/codex/logical_target.rs index 6245ea6c3b..8382c0a098 100644 --- a/rust/src/cost_scanner/codex/logical_target.rs +++ b/rust/src/cost_scanner/codex/logical_target.rs @@ -15,13 +15,227 @@ pub(super) enum CodexLineageDecision { Unsafe, } +struct CodexLineageNode { + path: String, + session_id: Option, + parent_id: Option, + candidate_index: Option, + may_infer_missing_parent: bool, + may_author_parent: bool, + initially_unsafe: bool, +} + +struct CodexLineageGraph { + nodes: Vec, + session_owners: HashMap>, + parent_indices: Vec>, + gates: Vec, + candidate_node_indices: Vec, + ordered_candidate_indices: Vec, +} + +impl CodexLineageGraph { + fn new(cache: &CostUsageCache, candidates: Option<&[CodexPreparedCandidate]>) -> Self { + let candidate_paths = candidates + .into_iter() + .flatten() + .map(|candidate| candidate.path.to_string_lossy().to_string()) + .collect::>(); + let mut cached_paths = cache + .files + .keys() + .filter(|path| !candidate_paths.contains(*path)) + .cloned() + .collect::>(); + cached_paths.sort(); + + let mut nodes = Vec::with_capacity(cached_paths.len() + candidate_paths.len()); + for path in cached_paths { + let usage = &cache.files[&path]; + let uses_parent = codex_usage_uses_parent(usage); + let locally_inferred = codex_fork_uses_local_inference(usage); + nodes.push(CodexLineageNode { + path, + session_id: usage.codex_session_id.clone(), + parent_id: uses_parent + .then(|| usage.codex_forked_from_id.clone()) + .flatten(), + candidate_index: None, + may_infer_missing_parent: locally_inferred, + may_author_parent: !locally_inferred && !usage.codex_unresolved_fork_parent, + initially_unsafe: usage.codex_unresolved_fork_parent, + }); + } + + let mut candidate_node_indices = Vec::new(); + if let Some(candidates) = candidates { + candidate_node_indices.reserve(candidates.len()); + for (candidate_index, candidate) in candidates.iter().enumerate() { + let uses_parent = candidate.session_metadata.lineage.uses_parent_baseline() + || candidate.session_metadata.forked_from_id.is_some(); + nodes.push(CodexLineageNode { + path: candidate.path.to_string_lossy().to_string(), + session_id: candidate.session_metadata.session_id.clone(), + parent_id: uses_parent + .then(|| candidate.session_metadata.forked_from_id.clone()) + .flatten(), + candidate_index: Some(candidate_index), + may_infer_missing_parent: candidate.session_metadata.is_subagent, + may_author_parent: true, + initially_unsafe: false, + }); + candidate_node_indices.push(nodes.len() - 1); + } + } + + let mut session_owners = HashMap::>::new(); + for (index, node) in nodes.iter().enumerate() { + if let Some(session_id) = node.session_id.as_ref() { + session_owners + .entry(session_id.clone()) + .or_default() + .push(index); + } + } + + let mut gates = nodes + .iter() + .map(|node| { + if node.initially_unsafe { + CodexLineageGate::Unsafe + } else { + CodexLineageGate::Eligible + } + }) + .collect::>(); + for owners in session_owners.values().filter(|owners| owners.len() > 1) { + for &index in owners { + gates[index] = CodexLineageGate::Unsafe; + } + } + + let mut parent_indices = vec![None; nodes.len()]; + for (index, node) in nodes.iter().enumerate() { + let Some(parent_id) = node.parent_id.as_ref() else { + continue; + }; + match session_owners.get(parent_id) { + Some(owners) if owners.len() == 1 => parent_indices[index] = Some(owners[0]), + Some(_) => { + gates[index] = CodexLineageGate::Unsafe; + } + None if !node.may_infer_missing_parent => { + gates[index] = CodexLineageGate::Unsafe; + } + None => {} + } + } + + // This single topological pass both rejects cycles/unsafe ancestry and + // orders candidates. Cached-parent validation consumes the same gates. + let mut completed = vec![false; nodes.len()]; + let mut ordered_candidate_indices = Vec::with_capacity(candidate_node_indices.len()); + loop { + let mut progressed = false; + for index in 0..nodes.len() { + if completed[index] || gates[index] == CodexLineageGate::Unsafe { + continue; + } + let parent_is_ready = parent_indices[index].is_none_or(|parent_index| { + completed[parent_index] + && gates[parent_index] == CodexLineageGate::Eligible + && nodes[parent_index].may_author_parent + }); + if parent_is_ready { + completed[index] = true; + if let Some(candidate_index) = nodes[index].candidate_index { + ordered_candidate_indices.push(candidate_index); + } + progressed = true; + } + } + if !progressed { + break; + } + } + + for index in 0..nodes.len() { + if !completed[index] { + gates[index] = CodexLineageGate::Unsafe; + if let Some(candidate_index) = nodes[index].candidate_index { + ordered_candidate_indices.push(candidate_index); + } + } + } + + Self { + nodes, + session_owners, + parent_indices, + gates, + candidate_node_indices, + ordered_candidate_indices, + } + } + + fn unique_owner(&self, session_id: &str) -> Result, ()> { + match self.session_owners.get(session_id).map(Vec::as_slice) { + None | Some([]) => Ok(None), + Some([index]) => Ok(Some(*index)), + Some(_) => Err(()), + } + } + + fn apply_candidate_plan(&self, candidates: &mut Vec) -> Vec { + if candidates.is_empty() { + return Vec::new(); + } + for (candidate_index, candidate) in candidates.iter_mut().enumerate() { + let node_index = self.candidate_node_indices[candidate_index]; + candidate.lineage_gate = self.gates[node_index]; + candidate.parent_owner_expected = self.parent_indices[node_index].is_some(); + } + + let mut remaining = candidates.drain(..).map(Some).collect::>(); + for candidate_index in &self.ordered_candidate_indices { + candidates.push( + remaining[*candidate_index] + .take() + .expect("candidate is ordered once"), + ); + } + + self.nodes + .iter() + .zip(&self.gates) + .filter(|(node, gate)| { + node.candidate_index.is_none() + && **gate == CodexLineageGate::Unsafe + && !node.initially_unsafe + }) + .map(|(node, _)| node.path.clone()) + .collect() + } +} + pub(super) struct CodexLineagePlanner<'a> { cache: &'a CostUsageCache, + graph: CodexLineageGraph, } impl<'a> CodexLineagePlanner<'a> { pub(super) fn new(cache: &'a CostUsageCache) -> Self { - Self { cache } + Self { + cache, + graph: CodexLineageGraph::new(cache, None), + } + } + + pub(super) fn plan_candidates_by_lineage( + cache: &CostUsageCache, + candidates: &mut Vec, + ) -> Vec { + CodexLineageGraph::new(cache, Some(candidates)).apply_candidate_plan(candidates) } pub(super) fn decision_for_scan( @@ -30,6 +244,7 @@ impl<'a> CodexLineagePlanner<'a> { gate: CodexLineageGate, parent_id: Option<&str>, fork_timestamp: Option<&str>, + parent_owner_expected: bool, ) -> CodexLineageDecision { if gate == CodexLineageGate::Unsafe { return CodexLineageDecision::Unsafe; @@ -38,7 +253,7 @@ impl<'a> CodexLineagePlanner<'a> { return CodexLineageDecision::Root; } parent_id.map_or(CodexLineageDecision::Unsafe, |parent_id| { - self.resolve_parent(parent_id, fork_timestamp) + self.resolve_parent(parent_id, fork_timestamp, parent_owner_expected) }) } @@ -53,7 +268,7 @@ impl<'a> CodexLineagePlanner<'a> { .codex_forked_from_id .as_deref() .map_or(CodexLineageDecision::Unsafe, |parent_id| { - self.resolve_parent(parent_id, usage.codex_fork_timestamp.as_deref()) + self.resolve_parent(parent_id, usage.codex_fork_timestamp.as_deref(), false) }) } @@ -74,45 +289,30 @@ impl<'a> CodexLineagePlanner<'a> { &self, parent_session_id: &str, child_fork_timestamp: Option<&str>, + parent_owner_expected: bool, ) -> CodexLineageDecision { - self.resolve_parent_inner(parent_session_id, child_fork_timestamp, &mut HashSet::new()) - } - - fn resolve_parent_inner( - &self, - parent_session_id: &str, - child_fork_timestamp: Option<&str>, - visiting: &mut HashSet, - ) -> CodexLineageDecision { - let mut owners = self - .cache - .files - .iter() - .filter(|(_, usage)| usage.codex_session_id.as_deref() == Some(parent_session_id)); - let Some((path_key, usage)) = owners.next() else { - return CodexLineageDecision::ParentAbsent; + let node_index = match self.graph.unique_owner(parent_session_id) { + Ok(None) if !parent_owner_expected => return CodexLineageDecision::ParentAbsent, + Ok(None) | Err(()) => return CodexLineageDecision::Unsafe, + Ok(Some(index)) => index, }; - if owners.next().is_some() || !visiting.insert(path_key.clone()) { - return CodexLineageDecision::Unsafe; - } - - let decision = self - .parent_owner_baseline(path_key, usage, child_fork_timestamp, visiting) + self.parent_owner_baseline(node_index, child_fork_timestamp) .map_or( CodexLineageDecision::Unsafe, CodexLineageDecision::ParentReady, - ); - visiting.remove(path_key); - decision + ) } fn parent_owner_baseline( &self, - path_key: &str, - usage: &CostUsageFileUsage, + node_index: usize, child_fork_timestamp: Option<&str>, - visiting: &mut HashSet, ) -> Option { + let node = self.graph.nodes.get(node_index)?; + if self.graph.gates[node_index] == CodexLineageGate::Unsafe || !node.may_author_parent { + return None; + } + let usage = self.cache.files.get(&node.path)?; if usage.codex_unresolved_fork_parent || usage.codex_token_timestamps_monotonic != Some(true) || super::codex_fork_uses_local_inference(usage) @@ -121,28 +321,22 @@ impl<'a> CodexLineagePlanner<'a> { } if super::codex_usage_uses_parent(usage) { - let parent_id = usage.codex_forked_from_id.as_deref()?; let inherited = usage .codex_fork_accounting_state .as_ref()? .inherited_totals .as_ref()?; - match self.resolve_parent_inner( - parent_id, - usage.codex_fork_timestamp.as_deref(), - visiting, - ) { - CodexLineageDecision::ParentReady(baseline) if &baseline == inherited => {} - CodexLineageDecision::Root - | CodexLineageDecision::ParentAbsent - | CodexLineageDecision::ParentReady(_) - | CodexLineageDecision::Unsafe => return None, + let parent_index = self.graph.parent_indices[node_index]?; + let baseline = + self.parent_owner_baseline(parent_index, usage.codex_fork_timestamp.as_deref())?; + if &baseline != inherited { + return None; } } - let metadata = fs::metadata(path_key).ok()?; + let metadata = fs::metadata(&node.path).ok()?; let expected_identity = usage.codex_file_identity.as_ref()?; - let actual_identity = JsonlScanner::codex_file_identity(Path::new(path_key), &metadata)?; + let actual_identity = JsonlScanner::codex_file_identity(Path::new(&node.path), &metadata)?; if expected_identity != &actual_identity { return None; } @@ -296,172 +490,6 @@ pub(super) fn defer_codex_locally_inferred_candidates( candidates.extend(other); } -struct CodexLineageNode { - path: String, - session_id: Option, - parent_id: Option, - candidate_index: Option, - may_infer_missing_parent: bool, - may_author_parent: bool, - initially_unsafe: bool, -} - -/// Order one bounded work set against both its admitted metadata and the -/// persisted cache graph. The returned cache paths became structurally unsafe -/// and must be invalidated even when the candidate limit deferred them. -pub(super) fn plan_codex_candidates_by_lineage( - cache: &CostUsageCache, - candidates: &mut Vec, -) -> Vec { - if candidates.is_empty() { - return Vec::new(); - } - - let candidate_paths = candidates - .iter() - .map(|candidate| candidate.path.to_string_lossy().to_string()) - .collect::>(); - let mut cached_paths = cache - .files - .keys() - .filter(|path| !candidate_paths.contains(*path)) - .cloned() - .collect::>(); - cached_paths.sort(); - - let mut nodes = Vec::with_capacity(cached_paths.len() + candidates.len()); - for path in cached_paths { - let usage = &cache.files[&path]; - let uses_parent = super::codex_usage_uses_parent(usage); - let locally_inferred = super::codex_fork_uses_local_inference(usage); - nodes.push(CodexLineageNode { - path, - session_id: usage.codex_session_id.clone(), - parent_id: uses_parent - .then(|| usage.codex_forked_from_id.clone()) - .flatten(), - candidate_index: None, - may_infer_missing_parent: locally_inferred, - may_author_parent: !locally_inferred && !usage.codex_unresolved_fork_parent, - initially_unsafe: usage.codex_unresolved_fork_parent, - }); - } - let mut candidate_node_indices = Vec::with_capacity(candidates.len()); - for (candidate_index, candidate) in candidates.iter().enumerate() { - let uses_parent = candidate.session_metadata.lineage.uses_parent_baseline() - || candidate.session_metadata.forked_from_id.is_some(); - nodes.push(CodexLineageNode { - path: candidate.path.to_string_lossy().to_string(), - session_id: candidate.session_metadata.session_id.clone(), - parent_id: uses_parent - .then(|| candidate.session_metadata.forked_from_id.clone()) - .flatten(), - candidate_index: Some(candidate_index), - may_infer_missing_parent: candidate.session_metadata.is_subagent, - may_author_parent: true, - initially_unsafe: false, - }); - candidate_node_indices.push(nodes.len() - 1); - } - - let mut session_owners = HashMap::>::new(); - for (index, node) in nodes.iter().enumerate() { - let Some(session_id) = node.session_id.as_ref() else { - continue; - }; - session_owners - .entry(session_id.clone()) - .or_default() - .push(index); - } - let mut lineage_gates = nodes - .iter() - .map(|node| { - if node.initially_unsafe { - CodexLineageGate::Unsafe - } else { - CodexLineageGate::Eligible - } - }) - .collect::>(); - for owners in session_owners.values().filter(|owners| owners.len() > 1) { - for &index in owners { - lineage_gates[index] = CodexLineageGate::Unsafe; - } - } - let mut parent_indices = vec![None; nodes.len()]; - for (index, node) in nodes.iter().enumerate() { - let Some(parent_id) = node.parent_id.as_ref() else { - continue; - }; - match session_owners.get(parent_id).map(Vec::as_slice) { - Some([parent_index]) => parent_indices[index] = Some(*parent_index), - Some([]) | None if node.may_infer_missing_parent => {} - Some([]) | None => lineage_gates[index] = CodexLineageGate::Unsafe, - Some(_) => lineage_gates[index] = CodexLineageGate::Unsafe, - } - } - - let mut completed = vec![false; nodes.len()]; - let mut ordered_indices = Vec::with_capacity(candidates.len()); - - loop { - let mut progressed = false; - for index in 0..nodes.len() { - if completed[index] || lineage_gates[index] == CodexLineageGate::Unsafe { - continue; - } - let parent_is_ready = parent_indices[index].is_none_or(|parent_index| { - completed[parent_index] - && lineage_gates[parent_index] == CodexLineageGate::Eligible - && nodes[parent_index].may_author_parent - }); - if parent_is_ready { - completed[index] = true; - if let Some(candidate_index) = nodes[index].candidate_index { - ordered_indices.push(candidate_index); - } - progressed = true; - } - } - if !progressed { - break; - } - } - - for index in 0..nodes.len() { - if !completed[index] { - lineage_gates[index] = CodexLineageGate::Unsafe; - if let Some(candidate_index) = nodes[index].candidate_index { - ordered_indices.push(candidate_index); - } - } - } - - let mut remaining = candidates.drain(..).map(Some).collect::>(); - for candidate_index in ordered_indices { - let node_index = candidate_node_indices[candidate_index]; - let mut candidate = remaining[candidate_index] - .take() - .expect("candidate is ordered once"); - candidate.lineage_gate = lineage_gates[node_index]; - candidates.push(candidate); - } - - nodes - .iter() - .zip(lineage_gates) - .filter(|(node, gate)| { - *gate == CodexLineageGate::Unsafe - && cache - .files - .get(&node.path) - .is_some_and(|usage| !usage.codex_unresolved_fork_parent) - }) - .map(|(node, _)| node.path.clone()) - .collect() -} - pub(super) fn invalidate_codex_unsafe_lineage(cache: &mut CostUsageCache, paths: &[String]) { for path in paths { let Some(usage) = cache.files.get_mut(path) else { diff --git a/rust/src/cost_scanner/codex/scan.rs b/rust/src/cost_scanner/codex/scan.rs index 8b95e31f0d..3fb183eabb 100644 --- a/rust/src/cost_scanner/codex/scan.rs +++ b/rust/src/cost_scanner/codex/scan.rs @@ -230,6 +230,7 @@ pub(super) fn scan_codex_detailed_with_cache( .unwrap_or_default(), path: candidate.path, lineage_gate: CodexLineageGate::Eligible, + parent_owner_expected: false, }); } let mut unprocessed = Vec::new(); @@ -237,7 +238,8 @@ pub(super) fn scan_codex_detailed_with_cache( unprocessed.extend(work_queue.drain(..).map(|candidate| candidate.path)); unprocessed.extend(cancelled_during_preparation); } else { - let unsafe_cached_paths = plan_codex_candidates_by_lineage(&cache, &mut work_queue); + let unsafe_cached_paths = + CodexLineagePlanner::plan_candidates_by_lineage(&cache, &mut work_queue); invalidated_unsafe_lineage = !unsafe_cached_paths.is_empty(); if invalidated_unsafe_lineage { cache.previous_report = None; From ac88694a92fd5aa1093b949252c916d30d06f3ae Mon Sep 17 00:00:00 2001 From: NessZerra <90105158+Finesssee@users.noreply.github.com> Date: Wed, 23 Sep 2026 06:19:26 +0700 Subject: [PATCH 15/19] Reject late ordinal gaps in inferred forks --- rust/src/core/jsonl_scanner/codex/parser.rs | 29 +++++++++++++++-- rust/src/core/jsonl_scanner/tests.rs | 34 ++++++++++++++++++++ rust/src/cost_scanner/tests/lineage_cache.rs | 32 ++++++++++++++++++ 3 files changed, 93 insertions(+), 2 deletions(-) diff --git a/rust/src/core/jsonl_scanner/codex/parser.rs b/rust/src/core/jsonl_scanner/codex/parser.rs index 728bd16f76..bdce9c68d8 100644 --- a/rust/src/core/jsonl_scanner/codex/parser.rs +++ b/rust/src/core/jsonl_scanner/codex/parser.rs @@ -92,6 +92,11 @@ impl ForkBaselineInference { } } + fn mark_missing_explicit_ordinal(&mut self) { + self.missing_explicit_ordinal = true; + self.locally_confirmed = false; + } + fn observe_non_token(&mut self, obj: &Value) { if obj.get("type").and_then(Value::as_str) == Some("turn_context") && self.inherited_opening { @@ -117,8 +122,7 @@ impl ForkBaselineInference { let ordinal = obj.get("ordinal").and_then(Value::as_i64); if self.explicit_start_ordinal.is_some() && ordinal.is_none() { - self.missing_explicit_ordinal = true; - self.locally_confirmed = false; + self.mark_missing_explicit_ordinal(); if !self.boundary_open { self.baseline = Some(total); } @@ -338,6 +342,27 @@ impl CodexParserState { } let event_candidate = is_candidate_codex_line(line); + if event_candidate + && self + .fork_baseline_inference + .as_ref() + .is_some_and(|inference| { + inference.resolved && inference.explicit_start_ordinal.is_some() + }) + { + let Ok(obj) = serde_json::from_str::(line) else { + return; + }; + if token_count_payload(&obj).is_some() + && obj.get("ordinal").and_then(Value::as_i64).is_none() + { + self.fork_baseline_inference + .as_mut() + .expect("resolved inference exists") + .mark_missing_explicit_ordinal(); + return; + } + } let bare_candidate = !event_candidate && line.contains("\"usage\""); if !event_candidate && !bare_candidate { return; diff --git a/rust/src/core/jsonl_scanner/tests.rs b/rust/src/core/jsonl_scanner/tests.rs index 4abfce1c93..1aebd89e20 100644 --- a/rust/src/core/jsonl_scanner/tests.rs +++ b/rust/src/core/jsonl_scanner/tests.rs @@ -184,6 +184,40 @@ fn inferred_fork_keeps_missing_ordinal_unresolved_after_boundary_opens() { assert!(!state.fork_baseline_locally_resolved()); } +#[test] +fn inferred_fork_keeps_missing_ordinal_unresolved_after_local_resolution() { + let range = CostUsageDayRange::new( + NaiveDate::from_ymd_opt(2026, 9, 22).unwrap(), + NaiveDate::from_ymd_opt(2026, 9, 22).unwrap(), + ); + let mut state = CodexParserState::from_mode(CodexParseMode::InferSubagent { + start_ordinal: Some(10), + }); + let token_line = |ordinal: Option, total: i64, last: i64| { + let mut value = serde_json::json!({ + "timestamp": "2026-09-22T10:00:00Z", + "type": "event_msg", + "payload": {"type": "token_count", "info": { + "model": "gpt-5.6-sol", + "total_token_usage": {"input_tokens": total, "cached_input_tokens": 0, "output_tokens": 0}, + "last_token_usage": {"input_tokens": last, "cached_input_tokens": 0, "output_tokens": 0} + }} + }); + if let Some(ordinal) = ordinal { + value["ordinal"] = serde_json::json!(ordinal); + } + value.to_string() + }; + + state.process_line(&token_line(Some(9), 100, 0), &range); + state.process_line(&token_line(Some(10), 100, 0), &range); + state.process_line(&token_line(Some(11), 110, 10), &range); + assert!(state.fork_baseline_locally_resolved()); + state.process_line(&token_line(None, 120, 10), &range); + + assert!(!state.fork_baseline_locally_resolved()); +} + #[test] fn codex_token_pipeline_preserves_counts_above_i32_max() { let parsed = read_token_totals(&serde_json::json!({ diff --git a/rust/src/cost_scanner/tests/lineage_cache.rs b/rust/src/cost_scanner/tests/lineage_cache.rs index 8ff35834d3..facfae31cf 100644 --- a/rust/src/cost_scanner/tests/lineage_cache.rs +++ b/rust/src/cost_scanner/tests/lineage_cache.rs @@ -246,6 +246,38 @@ fn missing_ordinal_cannot_complete_zero_usage_subagent_cache() { assert_unresolved(&cache, &child); } +#[test] +fn missing_ordinal_after_local_resolution_keeps_subagent_cache_unresolved() { + use std::io::Write as _; + + let root = tempfile::tempdir().unwrap(); + let sessions = root.path().join("sessions"); + let cache_root = root.path().join("cache"); + let base = Utc::now() - Duration::hours(1); + let child = write_subagent( + &sessions, + "child.jsonl", + "child-id", + "missing-parent-id", + base, + ); + let mut missing_ordinal = lineage_token_row(base + Duration::seconds(2), 21, 1_060, 10); + missing_ordinal.as_object_mut().unwrap().remove("ordinal"); + std::fs::OpenOptions::new() + .append(true) + .open(&child) + .unwrap() + .write_all(format!("{missing_ordinal}\n").as_bytes()) + .unwrap(); + let scanner = bounded_scanner(&sessions, &cache_root); + + let (summary, _, cache) = scanner.scan_codex_detailed_with_cache(None); + + assert_eq!(summary.input_tokens, 0); + assert_eq!(summary.sessions_count, 0); + assert_unresolved(&cache, &child); +} + #[test] fn legacy_cache_without_file_identity_is_reparsed() { let root = tempfile::tempdir().unwrap(); From 775a4c4af9892a576d895e1fdd936cc50784ed4f Mon Sep 17 00:00:00 2001 From: NessZerra <90105158+Finesssee@users.noreply.github.com> Date: Wed, 23 Sep 2026 21:30:16 +0700 Subject: [PATCH 16/19] Reuse Codex lineage plan per refresh --- rust/src/cost_scanner/codex.rs | 21 +- rust/src/cost_scanner/codex/logical_target.rs | 224 +++++++++++++----- rust/src/cost_scanner/codex/scan.rs | 27 ++- rust/src/cost_scanner/tests.rs | 6 + rust/src/cost_scanner/tests/lineage_cache.rs | 4 +- 5 files changed, 204 insertions(+), 78 deletions(-) diff --git a/rust/src/cost_scanner/codex.rs b/rust/src/cost_scanner/codex.rs index a64eb92a39..23730eb28f 100644 --- a/rust/src/cost_scanner/codex.rs +++ b/rust/src/cost_scanner/codex.rs @@ -101,10 +101,6 @@ fn codex_usage_uses_parent(usage: &CostUsageFileUsage) -> bool { && usage.codex_forked_from_id.is_some()) } -fn codex_fork_parent_is_safe(cache: &CostUsageCache, usage: &CostUsageFileUsage) -> bool { - CodexLineagePlanner::new(cache).cached_usage_is_safe(usage) -} - fn codex_fork_uses_local_inference(usage: &CostUsageFileUsage) -> bool { usage .codex_fork_accounting_state @@ -202,6 +198,7 @@ impl CostScanner { sessions_dirs: &[PathBuf], range: &CostUsageDayRange, cache: &CostUsageCache, + planner: &CodexLineagePlanner, cancel: Option<&AtomicBool>, stats: &mut CostScanStats, ) -> (Vec, bool) { @@ -263,7 +260,7 @@ impl CostScanner { }; let mtime_unix_ms = system_time_to_unix_ms(metadata.modified().ok()); let unchanged_complete = - cached_codex_file_is_complete_for_range(cache, &path_key, range); + cached_codex_file_is_complete_for_range(cache, planner, &path_key, range); if unchanged_complete { stats.files_seen = stats.files_seen.saturating_add(1); stats.files_skipped = stats.files_skipped.saturating_add(1); @@ -323,8 +320,10 @@ impl CostScanner { cancel: Option<&AtomicBool>, stats: &mut CostScanStats, ) { - let _ = - self.parse_codex_file_bounded(path, range, summary, cache, cancel, stats, None, None); + let planner = CodexLineagePlanner::new(cache); + let _ = self.parse_codex_file_bounded( + path, range, summary, cache, cancel, stats, None, None, &planner, + ); } #[allow( @@ -341,6 +340,7 @@ impl CostScanner { stats: &mut CostScanStats, max_bytes_to_read: Option, prepared_candidate: Option<&CodexPreparedCandidate>, + planner: &CodexLineagePlanner, ) -> CodexFileScanOutcome { if is_cancelled(cancel) { return CodexFileScanOutcome::default(); @@ -377,7 +377,7 @@ impl CostScanner { }; } let cache_entry_is_fresh = |entry: &CostUsageFileUsage| { - cached_codex_file_is_fresh(cache, entry, cache_covers_range, mtime_ms, size) + cached_codex_file_is_fresh(cache, planner, entry, cache_covers_range, mtime_ms, size) }; let identity_matches_cached = |entry: &CostUsageFileUsage| { codex_file_identity_matches( @@ -502,7 +502,8 @@ impl CostScanner { .unwrap_or_default(); let parent_owner_expected = prepared_candidate.is_some_and(|candidate| candidate.parent_owner_expected); - let lineage_decision = CodexLineagePlanner::new(cache).decision_for_scan( + let lineage_decision = planner.decision_for_scan( + cache, is_fork, lineage_gate, codex_forked_from_id.as_deref(), @@ -545,7 +546,7 @@ impl CostScanner { } if let Some(entry) = &cached - && cached_codex_file_is_fresh(cache, entry, cache_covers_range, mtime_ms, size) + && cached_codex_file_is_fresh(cache, planner, entry, cache_covers_range, mtime_ms, size) && identity_matches_cached(entry) && !cached_identity_changed && !accounting_mode.requires_cached_reparse() diff --git a/rust/src/cost_scanner/codex/logical_target.rs b/rust/src/cost_scanner/codex/logical_target.rs index 8382c0a098..a559588de8 100644 --- a/rust/src/cost_scanner/codex/logical_target.rs +++ b/rust/src/cost_scanner/codex/logical_target.rs @@ -1,4 +1,10 @@ use super::*; +#[cfg(test)] +use std::cell::Cell; +use std::collections::VecDeque; + +#[cfg(test)] +thread_local! { static CODEX_LINEAGE_GRAPH_BUILDS: Cell = const { Cell::new(0) }; } #[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] pub(super) enum CodexLineageGate { @@ -36,6 +42,8 @@ struct CodexLineageGraph { impl CodexLineageGraph { fn new(cache: &CostUsageCache, candidates: Option<&[CodexPreparedCandidate]>) -> Self { + #[cfg(test)] + CODEX_LINEAGE_GRAPH_BUILDS.with(|builds| builds.set(builds.get() + 1)); let candidate_paths = candidates .into_iter() .flatten() @@ -71,18 +79,61 @@ impl CodexLineageGraph { if let Some(candidates) = candidates { candidate_node_indices.reserve(candidates.len()); for (candidate_index, candidate) in candidates.iter().enumerate() { - let uses_parent = candidate.session_metadata.lineage.uses_parent_baseline() - || candidate.session_metadata.forked_from_id.is_some(); + let cached = cache + .files + .get(&candidate.path.to_string_lossy().to_string()); + let cached_identity_matches = cached.is_some_and(|usage| { + let Ok(metadata) = fs::metadata(&candidate.path) else { + return false; + }; + let expected = usage.codex_file_identity.as_deref(); + let actual = JsonlScanner::codex_file_identity(&candidate.path, &metadata); + codex_file_identity_matches(expected, actual.as_deref()) + }); + let metadata_owns_identity = candidate.session_metadata.session_id.is_some(); + let uses_parent = if metadata_owns_identity { + candidate.session_metadata.lineage.uses_parent_baseline() + || candidate.session_metadata.forked_from_id.is_some() + } else if cached_identity_matches { + cached.is_some_and(super::codex_usage_uses_parent) + } else { + candidate.session_metadata.lineage.uses_parent_baseline() + || candidate.session_metadata.forked_from_id.is_some() + }; + let session_id = candidate.session_metadata.session_id.clone().or_else(|| { + cached_identity_matches + .then(|| cached.and_then(|usage| usage.codex_session_id.clone())) + .flatten() + }); + let parent_id = if metadata_owns_identity { + candidate.session_metadata.forked_from_id.clone() + } else { + candidate + .session_metadata + .forked_from_id + .clone() + .or_else(|| { + (cached_identity_matches && uses_parent) + .then(|| { + cached.and_then(|usage| usage.codex_forked_from_id.clone()) + }) + .flatten() + }) + }; + let cached_fallback = cached_identity_matches.then_some(cached).flatten(); nodes.push(CodexLineageNode { path: candidate.path.to_string_lossy().to_string(), - session_id: candidate.session_metadata.session_id.clone(), - parent_id: uses_parent - .then(|| candidate.session_metadata.forked_from_id.clone()) - .flatten(), + session_id, + parent_id: uses_parent.then_some(parent_id).flatten(), candidate_index: Some(candidate_index), - may_infer_missing_parent: candidate.session_metadata.is_subagent, - may_author_parent: true, - initially_unsafe: false, + may_infer_missing_parent: candidate.session_metadata.is_subagent + || cached_fallback.is_some_and(super::codex_fork_uses_local_inference), + may_author_parent: cached_fallback.is_none_or(|usage| { + !super::codex_fork_uses_local_inference(usage) + && !usage.codex_unresolved_fork_parent + }), + initially_unsafe: cached_fallback + .is_some_and(|usage| usage.codex_unresolved_fork_parent), }); candidate_node_indices.push(nodes.len() - 1); } @@ -135,28 +186,30 @@ impl CodexLineageGraph { // orders candidates. Cached-parent validation consumes the same gates. let mut completed = vec![false; nodes.len()]; let mut ordered_candidate_indices = Vec::with_capacity(candidate_node_indices.len()); - loop { - let mut progressed = false; - for index in 0..nodes.len() { - if completed[index] || gates[index] == CodexLineageGate::Unsafe { - continue; - } - let parent_is_ready = parent_indices[index].is_none_or(|parent_index| { - completed[parent_index] - && gates[parent_index] == CodexLineageGate::Eligible - && nodes[parent_index].may_author_parent - }); - if parent_is_ready { - completed[index] = true; - if let Some(candidate_index) = nodes[index].candidate_index { - ordered_candidate_indices.push(candidate_index); + let mut children = vec![Vec::new(); nodes.len()]; + let mut ready = VecDeque::new(); + for (index, parent) in parent_indices.iter().enumerate() { + match parent { + Some(parent_index) => children[*parent_index].push(index), + None if gates[index] == CodexLineageGate::Eligible => ready.push_back(index), + None => {} + } + } + while let Some(index) = ready.pop_front() { + if completed[index] || gates[index] == CodexLineageGate::Unsafe { + continue; + } + completed[index] = true; + if let Some(candidate_index) = nodes[index].candidate_index { + ordered_candidate_indices.push(candidate_index); + } + if nodes[index].may_author_parent { + for child in &children[index] { + if gates[*child] == CodexLineageGate::Eligible { + ready.push_back(*child); } - progressed = true; } } - if !progressed { - break; - } } for index in 0..nodes.len() { @@ -218,28 +271,69 @@ impl CodexLineageGraph { } } -pub(super) struct CodexLineagePlanner<'a> { - cache: &'a CostUsageCache, - graph: CodexLineageGraph, +pub(super) struct CodexLineagePlanner { + graph: Option, } -impl<'a> CodexLineagePlanner<'a> { - pub(super) fn new(cache: &'a CostUsageCache) -> Self { +impl CodexLineagePlanner { + pub(super) fn new(cache: &CostUsageCache) -> Self { Self { - cache, - graph: CodexLineageGraph::new(cache, None), + graph: Self::needs_graph(cache, None).then(|| CodexLineageGraph::new(cache, None)), } } pub(super) fn plan_candidates_by_lineage( cache: &CostUsageCache, candidates: &mut Vec, - ) -> Vec { - CodexLineageGraph::new(cache, Some(candidates)).apply_candidate_plan(candidates) + ) -> (Self, Vec) { + let graph = Self::needs_graph(cache, Some(candidates)) + .then(|| CodexLineageGraph::new(cache, Some(candidates))); + let unsafe_paths = graph + .as_ref() + .map_or_else(Vec::new, |graph| graph.apply_candidate_plan(candidates)); + (Self { graph }, unsafe_paths) + } + + fn needs_graph(cache: &CostUsageCache, candidates: Option<&[CodexPreparedCandidate]>) -> bool { + cache.files.values().any(super::codex_usage_uses_parent) + || candidates.is_some_and(|items| { + items.iter().any(|candidate| { + candidate.session_metadata.lineage.uses_parent_baseline() + || candidate.session_metadata.forked_from_id.is_some() + }) + }) + } + + #[cfg(test)] + pub(crate) fn reset_graph_build_count() { + CODEX_LINEAGE_GRAPH_BUILDS.with(|count| count.set(0)); + } + #[cfg(test)] + pub(crate) fn graph_build_count() -> usize { + CODEX_LINEAGE_GRAPH_BUILDS.with(Cell::get) + } + + fn graph(&self) -> Option<&CodexLineageGraph> { + self.graph.as_ref() + } + + pub(super) fn cached_usage_is_safe( + &self, + cache: &CostUsageCache, + usage: &CostUsageFileUsage, + ) -> bool { + let locally_resolved = super::codex_fork_uses_local_inference(usage); + match self.decision_for_usage(cache, usage) { + CodexLineageDecision::Root => true, + CodexLineageDecision::ParentAbsent => locally_resolved, + CodexLineageDecision::ParentReady(_) => !locally_resolved, + CodexLineageDecision::Unsafe => false, + } } pub(super) fn decision_for_scan( &self, + cache: &CostUsageCache, uses_parent: bool, gate: CodexLineageGate, parent_id: Option<&str>, @@ -253,11 +347,15 @@ impl<'a> CodexLineagePlanner<'a> { return CodexLineageDecision::Root; } parent_id.map_or(CodexLineageDecision::Unsafe, |parent_id| { - self.resolve_parent(parent_id, fork_timestamp, parent_owner_expected) + self.resolve_parent(cache, parent_id, fork_timestamp, parent_owner_expected) }) } - pub(super) fn decision_for_usage(&self, usage: &CostUsageFileUsage) -> CodexLineageDecision { + pub(super) fn decision_for_usage( + &self, + cache: &CostUsageCache, + usage: &CostUsageFileUsage, + ) -> CodexLineageDecision { if usage.codex_unresolved_fork_parent { return CodexLineageDecision::Unsafe; } @@ -268,35 +366,34 @@ impl<'a> CodexLineagePlanner<'a> { .codex_forked_from_id .as_deref() .map_or(CodexLineageDecision::Unsafe, |parent_id| { - self.resolve_parent(parent_id, usage.codex_fork_timestamp.as_deref(), false) + self.resolve_parent( + cache, + parent_id, + usage.codex_fork_timestamp.as_deref(), + false, + ) }) } - pub(super) fn cached_usage_is_safe(&self, usage: &CostUsageFileUsage) -> bool { - let locally_resolved = super::codex_fork_uses_local_inference(usage); - match self.decision_for_usage(usage) { - CodexLineageDecision::Root => true, - CodexLineageDecision::ParentAbsent => locally_resolved, - CodexLineageDecision::ParentReady(_) => !locally_resolved, - CodexLineageDecision::Unsafe => false, - } - } - /// Resolve one parent identity through the persisted graph. Absence is /// distinct from ambiguity and transitive unsafety so local inference is /// allowed only when no owner exists at all. fn resolve_parent( &self, + cache: &CostUsageCache, parent_session_id: &str, child_fork_timestamp: Option<&str>, parent_owner_expected: bool, ) -> CodexLineageDecision { - let node_index = match self.graph.unique_owner(parent_session_id) { + let Some(graph) = self.graph() else { + return CodexLineageDecision::Unsafe; + }; + let node_index = match graph.unique_owner(parent_session_id) { Ok(None) if !parent_owner_expected => return CodexLineageDecision::ParentAbsent, Ok(None) | Err(()) => return CodexLineageDecision::Unsafe, Ok(Some(index)) => index, }; - self.parent_owner_baseline(node_index, child_fork_timestamp) + self.parent_owner_baseline(cache, node_index, child_fork_timestamp) .map_or( CodexLineageDecision::Unsafe, CodexLineageDecision::ParentReady, @@ -305,14 +402,16 @@ impl<'a> CodexLineagePlanner<'a> { fn parent_owner_baseline( &self, + cache: &CostUsageCache, node_index: usize, child_fork_timestamp: Option<&str>, ) -> Option { - let node = self.graph.nodes.get(node_index)?; - if self.graph.gates[node_index] == CodexLineageGate::Unsafe || !node.may_author_parent { + let graph = self.graph()?; + let node = graph.nodes.get(node_index)?; + if graph.gates[node_index] == CodexLineageGate::Unsafe || !node.may_author_parent { return None; } - let usage = self.cache.files.get(&node.path)?; + let usage = cache.files.get(&node.path)?; if usage.codex_unresolved_fork_parent || usage.codex_token_timestamps_monotonic != Some(true) || super::codex_fork_uses_local_inference(usage) @@ -326,9 +425,12 @@ impl<'a> CodexLineagePlanner<'a> { .as_ref()? .inherited_totals .as_ref()?; - let parent_index = self.graph.parent_indices[node_index]?; - let baseline = - self.parent_owner_baseline(parent_index, usage.codex_fork_timestamp.as_deref())?; + let parent_index = graph.parent_indices[node_index]?; + let baseline = self.parent_owner_baseline( + cache, + parent_index, + usage.codex_fork_timestamp.as_deref(), + )?; if &baseline != inherited { return None; } @@ -412,6 +514,7 @@ impl CodexLineageDecision { pub(super) fn cached_codex_file_is_fresh( cache: &CostUsageCache, + planner: &CodexLineagePlanner, entry: &CostUsageFileUsage, cache_covers_range: bool, mtime_unix_ms: i64, @@ -423,7 +526,7 @@ pub(super) fn cached_codex_file_is_fresh( && entry.size == size && codex_scan_target_size(entry) == size && entry.parsed_bytes.unwrap_or(0) >= size - && super::codex_fork_parent_is_safe(cache, entry) + && planner.cached_usage_is_safe(cache, entry) } pub(super) fn codex_file_identity_matches(expected: Option<&str>, actual: Option<&str>) -> bool { @@ -434,6 +537,7 @@ pub(super) fn codex_file_identity_matches(expected: Option<&str>, actual: Option pub(super) fn cached_codex_file_is_complete_for_range( cache: &CostUsageCache, + planner: &CodexLineagePlanner, path_key: &str, range: &CostUsageDayRange, ) -> bool { @@ -457,7 +561,7 @@ pub(super) fn cached_codex_file_is_complete_for_range( // Reconsider locally inferred children after this pass has // had a chance to discover and cache their parent. && !super::codex_fork_uses_local_inference(usage) - && super::codex_fork_parent_is_safe(cache, usage) + && planner.cached_usage_is_safe(cache, usage) }) } diff --git a/rust/src/cost_scanner/codex/scan.rs b/rust/src/cost_scanner/codex/scan.rs index 3fb183eabb..7a60775d3c 100644 --- a/rust/src/cost_scanner/codex/scan.rs +++ b/rust/src/cost_scanner/codex/scan.rs @@ -177,8 +177,15 @@ pub(super) fn scan_codex_detailed_with_cache( cache.codex_pending_scan_root_paths = pending_scan.root_paths.clone(); cache.codex_pending_scan_timezone = Some(pending_scan.timezone.clone()); - let (mut candidates, discovery_complete) = - scanner.collect_codex_candidates(&sessions_dirs, scan_range, &cache, cancel, &mut stats); + let cached_lineage = CodexLineagePlanner::new(&cache); + let (mut candidates, discovery_complete) = scanner.collect_codex_candidates( + &sessions_dirs, + scan_range, + &cache, + &cached_lineage, + cancel, + &mut stats, + ); let candidate_limit = if scanner.options.codex_candidate_limit == 0 { usize::MAX } else { @@ -201,8 +208,9 @@ pub(super) fn scan_codex_detailed_with_cache( prioritize_codex_pending_candidates(&mut candidates, &pending_paths_before_pass); defer_codex_locally_inferred_candidates(&mut candidates, &cache); if discovery_complete && !is_cancelled(cancel) { - pending_next - .retain(|path| !cached_codex_file_is_complete_for_range(&cache, path, scan_range)); + pending_next.retain(|path| { + !cached_codex_file_is_complete_for_range(&cache, &cached_lineage, path, scan_range) + }); } // Admit one bounded set, inspect each admitted candidate once, and order @@ -234,12 +242,15 @@ pub(super) fn scan_codex_detailed_with_cache( }); } let mut unprocessed = Vec::new(); - if !cancelled_during_preparation.is_empty() || is_cancelled(cancel) { + let lineage_planner; + let cancelled_before_plan = !cancelled_during_preparation.is_empty() || is_cancelled(cancel); + if cancelled_before_plan { unprocessed.extend(work_queue.drain(..).map(|candidate| candidate.path)); unprocessed.extend(cancelled_during_preparation); } else { - let unsafe_cached_paths = + let (planner, unsafe_cached_paths) = CodexLineagePlanner::plan_candidates_by_lineage(&cache, &mut work_queue); + lineage_planner = planner; invalidated_unsafe_lineage = !unsafe_cached_paths.is_empty(); if invalidated_unsafe_lineage { cache.previous_report = None; @@ -251,6 +262,9 @@ pub(super) fn scan_codex_detailed_with_cache( } } } + if cancelled_before_plan { + lineage_planner = cached_lineage; + } let mut incomplete_processed = Vec::new(); for (index, candidate) in work_queue.iter().enumerate() { @@ -274,6 +288,7 @@ pub(super) fn scan_codex_detailed_with_cache( &mut stats, Some(allowance), Some(candidate), + &lineage_planner, ); bytes_read_this_refresh = bytes_read_this_refresh.saturating_add(outcome.bytes_read.max(0)); stats.codex_bytes_read = stats diff --git a/rust/src/cost_scanner/tests.rs b/rust/src/cost_scanner/tests.rs index d8dc136047..9a930a3e7b 100644 --- a/rust/src/cost_scanner/tests.rs +++ b/rust/src/cost_scanner/tests.rs @@ -1449,10 +1449,16 @@ fn cost_scan_second_pass_skips_unchanged_files_via_cache() { // Second pass with default debounce still inspects files but skips re-parse. // Use app_driven so we exercise per-file mtime skip rather than whole-scan debounce. + CodexLineagePlanner::reset_graph_build_count(); let (summary2, stats2) = scanner.scan_codex_detailed(None); assert_eq!(stats2.files_seen, 2); assert_eq!(stats2.files_skipped, 2, "cache hit skips re-parse"); assert_eq!(stats2.files_parsed, 0); + assert_eq!( + CodexLineagePlanner::graph_build_count(), + 0, + "warm root-only scan must bypass lineage graph construction" + ); assert!(stats2.codex_metadata_read_paths.is_empty()); assert!(stats2.codex_history_read_paths.is_empty()); assert_eq!(stats2.codex_read_receipt, Default::default()); diff --git a/rust/src/cost_scanner/tests/lineage_cache.rs b/rust/src/cost_scanner/tests/lineage_cache.rs index facfae31cf..3fbca1dbd1 100644 --- a/rust/src/cost_scanner/tests/lineage_cache.rs +++ b/rust/src/cost_scanner/tests/lineage_cache.rs @@ -167,7 +167,7 @@ fn replaced_parent_with_same_path_size_and_mtime_cannot_author_lineage() { let parent_key = parent.to_string_lossy().to_string(); let child_usage = &cache.files[&child.to_string_lossy().to_string()]; assert!(matches!( - CodexLineagePlanner::new(&cache).decision_for_usage(child_usage), + CodexLineagePlanner::new(&cache).decision_for_usage(&cache, child_usage), CodexLineageDecision::ParentReady(_) )); @@ -203,7 +203,7 @@ fn replaced_parent_with_same_path_size_and_mtime_cannot_author_lineage() { assert_ne!(replacement_identity, old_identity); assert_eq!( - CodexLineagePlanner::new(&cache).decision_for_usage(child_usage), + CodexLineagePlanner::new(&cache).decision_for_usage(&cache, child_usage), CodexLineageDecision::Unsafe ); } From bc1ac36c59d78ccf695694c5d82dbf8789f09ddb Mon Sep 17 00:00:00 2001 From: NessZerra <90105158+Finesssee@users.noreply.github.com> Date: Wed, 23 Sep 2026 21:41:31 +0700 Subject: [PATCH 17/19] Initialize lineage planner in one branch expression --- rust/src/cost_scanner/codex/scan.rs | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/rust/src/cost_scanner/codex/scan.rs b/rust/src/cost_scanner/codex/scan.rs index 7a60775d3c..758f887e44 100644 --- a/rust/src/cost_scanner/codex/scan.rs +++ b/rust/src/cost_scanner/codex/scan.rs @@ -242,15 +242,14 @@ pub(super) fn scan_codex_detailed_with_cache( }); } let mut unprocessed = Vec::new(); - let lineage_planner; let cancelled_before_plan = !cancelled_during_preparation.is_empty() || is_cancelled(cancel); - if cancelled_before_plan { + let lineage_planner = if cancelled_before_plan { unprocessed.extend(work_queue.drain(..).map(|candidate| candidate.path)); unprocessed.extend(cancelled_during_preparation); + cached_lineage } else { let (planner, unsafe_cached_paths) = CodexLineagePlanner::plan_candidates_by_lineage(&cache, &mut work_queue); - lineage_planner = planner; invalidated_unsafe_lineage = !unsafe_cached_paths.is_empty(); if invalidated_unsafe_lineage { cache.previous_report = None; @@ -261,10 +260,8 @@ pub(super) fn scan_codex_detailed_with_cache( pending_next.push(path); } } - } - if cancelled_before_plan { - lineage_planner = cached_lineage; - } + planner + }; let mut incomplete_processed = Vec::new(); for (index, candidate) in work_queue.iter().enumerate() { From c186e197179c1a9234d7c12a122f7eefd5684774 Mon Sep 17 00:00:00 2001 From: NessZerra <90105158+Finesssee@users.noreply.github.com> Date: Wed, 23 Sep 2026 21:50:47 +0700 Subject: [PATCH 18/19] Prefer current Codex lineage metadata --- rust/src/cost_scanner/codex/logical_target.rs | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/rust/src/cost_scanner/codex/logical_target.rs b/rust/src/cost_scanner/codex/logical_target.rs index a559588de8..567e1a44a9 100644 --- a/rust/src/cost_scanner/codex/logical_target.rs +++ b/rust/src/cost_scanner/codex/logical_target.rs @@ -120,7 +120,14 @@ impl CodexLineageGraph { .flatten() }) }; - let cached_fallback = cached_identity_matches.then_some(cached).flatten(); + // Freshly read identity-bearing metadata owns this candidate's + // current lineage state. Retain cached lineage flags only when + // the bounded metadata read could not establish an identity. + let cached_fallback = if metadata_owns_identity { + None + } else { + cached_identity_matches.then_some(cached).flatten() + }; nodes.push(CodexLineageNode { path: candidate.path.to_string_lossy().to_string(), session_id, From 5d2d39efe9803efbba24b265b516335984f345bd Mon Sep 17 00:00:00 2001 From: NessZerra <90105158+Finesssee@users.noreply.github.com> Date: Wed, 23 Sep 2026 22:04:11 +0700 Subject: [PATCH 19/19] Scope lineage invalidation to active dependencies --- rust/src/cost_scanner/codex/logical_target.rs | 75 +++++++++++++------ rust/src/cost_scanner/codex/scan.rs | 8 +- rust/src/cost_scanner/tests/lineage_cache.rs | 68 +++++++++++++++++ 3 files changed, 128 insertions(+), 23 deletions(-) diff --git a/rust/src/cost_scanner/codex/logical_target.rs b/rust/src/cost_scanner/codex/logical_target.rs index 567e1a44a9..b261f174fe 100644 --- a/rust/src/cost_scanner/codex/logical_target.rs +++ b/rust/src/cost_scanner/codex/logical_target.rs @@ -246,34 +246,65 @@ impl CodexLineageGraph { } } - fn apply_candidate_plan(&self, candidates: &mut Vec) -> Vec { - if candidates.is_empty() { - return Vec::new(); - } - for (candidate_index, candidate) in candidates.iter_mut().enumerate() { - let node_index = self.candidate_node_indices[candidate_index]; - candidate.lineage_gate = self.gates[node_index]; - candidate.parent_owner_expected = self.parent_indices[node_index].is_some(); + fn apply_candidate_plan( + &self, + candidates: &mut Vec, + sessions_dirs: &[PathBuf], + range: &CostUsageDayRange, + ) -> Vec { + if !candidates.is_empty() { + for (candidate_index, candidate) in candidates.iter_mut().enumerate() { + let node_index = self.candidate_node_indices[candidate_index]; + candidate.lineage_gate = self.gates[node_index]; + candidate.parent_owner_expected = self.parent_indices[node_index].is_some(); + } + + let mut remaining = candidates.drain(..).map(Some).collect::>(); + for candidate_index in &self.ordered_candidate_indices { + candidates.push( + remaining[*candidate_index] + .take() + .expect("candidate is ordered once"), + ); + } } - let mut remaining = candidates.drain(..).map(Some).collect::>(); - for candidate_index in &self.ordered_candidate_indices { - candidates.push( - remaining[*candidate_index] - .take() - .expect("candidate is ordered once"), - ); + // Keep the complete graph for parent resolution, but invalidate only + // unsafe cached nodes in the active range or in the ancestor closure + // required to resolve an active node. + let mut relevant = vec![false; self.nodes.len()]; + let mut pending = VecDeque::new(); + for (index, node) in self.nodes.iter().enumerate() { + if super::is_codex_path_in_scan_window(Path::new(&node.path), sessions_dirs, range) { + relevant[index] = true; + pending.push_back(index); + } + } + while let Some(index) = pending.pop_front() { + let Some(parent_id) = self.nodes[index].parent_id.as_deref() else { + continue; + }; + if let Some(owners) = self.session_owners.get(parent_id) { + for &owner in owners { + if !relevant[owner] { + relevant[owner] = true; + pending.push_back(owner); + } + } + } } self.nodes .iter() .zip(&self.gates) - .filter(|(node, gate)| { - node.candidate_index.is_none() + .enumerate() + .filter(|(index, (node, gate))| { + relevant[*index] + && node.candidate_index.is_none() && **gate == CodexLineageGate::Unsafe && !node.initially_unsafe }) - .map(|(node, _)| node.path.clone()) + .map(|(_, (node, _))| node.path.clone()) .collect() } } @@ -292,12 +323,14 @@ impl CodexLineagePlanner { pub(super) fn plan_candidates_by_lineage( cache: &CostUsageCache, candidates: &mut Vec, + sessions_dirs: &[PathBuf], + range: &CostUsageDayRange, ) -> (Self, Vec) { let graph = Self::needs_graph(cache, Some(candidates)) .then(|| CodexLineageGraph::new(cache, Some(candidates))); - let unsafe_paths = graph - .as_ref() - .map_or_else(Vec::new, |graph| graph.apply_candidate_plan(candidates)); + let unsafe_paths = graph.as_ref().map_or_else(Vec::new, |graph| { + graph.apply_candidate_plan(candidates, sessions_dirs, range) + }); (Self { graph }, unsafe_paths) } diff --git a/rust/src/cost_scanner/codex/scan.rs b/rust/src/cost_scanner/codex/scan.rs index 758f887e44..94cb91f701 100644 --- a/rust/src/cost_scanner/codex/scan.rs +++ b/rust/src/cost_scanner/codex/scan.rs @@ -248,8 +248,12 @@ pub(super) fn scan_codex_detailed_with_cache( unprocessed.extend(cancelled_during_preparation); cached_lineage } else { - let (planner, unsafe_cached_paths) = - CodexLineagePlanner::plan_candidates_by_lineage(&cache, &mut work_queue); + let (planner, unsafe_cached_paths) = CodexLineagePlanner::plan_candidates_by_lineage( + &cache, + &mut work_queue, + &sessions_dirs, + scan_range, + ); invalidated_unsafe_lineage = !unsafe_cached_paths.is_empty(); if invalidated_unsafe_lineage { cache.previous_report = None; diff --git a/rust/src/cost_scanner/tests/lineage_cache.rs b/rust/src/cost_scanner/tests/lineage_cache.rs index 3fbca1dbd1..19ac317153 100644 --- a/rust/src/cost_scanner/tests/lineage_cache.rs +++ b/rust/src/cost_scanner/tests/lineage_cache.rs @@ -437,3 +437,71 @@ fn bounded_refresh_rejects_dependent_of_locally_inferred_parent() { assert_locally_inferred(&cache, &parent); assert_unresolved(&cache, &dependent); } + +#[test] +fn current_refresh_scopes_unsafe_cache_invalidation_to_range_and_dependencies() { + let root = tempfile::tempdir().unwrap(); + let sessions = root.path().join("sessions"); + let cache_root = root.path().join("cache"); + let active_time = Utc::now() - Duration::hours(1); + let old_date = Local::now().date_naive() - Duration::days(30); + let old_day = old_date.format("%Y-%m-%d").to_string(); + let old_dir = sessions + .join(old_date.format("%Y").to_string()) + .join(old_date.format("%m").to_string()) + .join(old_date.format("%d").to_string()); + let mut cache = CostUsageCache::default(); + + { + let mut add_cached = |name: &str, session_id: &str, parent_id: Option<&str>| { + let path = old_dir.join(name).to_string_lossy().to_string(); + let mut usage = cached_usage_with_packed(&old_day, "gpt-5.6-sol", vec![100, 0, 5, 0]); + usage.codex_session_id = Some(session_id.to_string()); + usage.codex_forked_from_id = parent_id.map(str::to_string); + cache.files.insert(path, usage); + }; + add_cached("unrelated-a.jsonl", "unrelated-a", Some("unrelated-b")); + add_cached("unrelated-b.jsonl", "unrelated-b", Some("unrelated-a")); + add_cached("required-a.jsonl", "required-parent", None); + add_cached("required-b.jsonl", "required-parent", None); + } + JsonlScanner::save_cache(ProviderId::Codex, &mut cache, Some(&cache_root)); + + let active_child = write_codex_fork_session_fixture( + &sessions, + "active-child.jsonl", + "active-child", + Some("required-parent"), + active_time, + active_time + Duration::seconds(1), + &[1_000_000, 1_000_140], + ); + let scanner = CostScanner::new(7) + .with_options(CostScanOptions::app_driven()) + .with_cache_root(&cache_root) + .with_sessions_dirs(vec![sessions]); + + let (summary, _, refreshed) = scanner.scan_codex_detailed_with_cache(None); + let cached_path = |name: &str| old_dir.join(name).to_string_lossy().to_string(); + + for path in [ + cached_path("unrelated-a.jsonl"), + cached_path("unrelated-b.jsonl"), + ] { + let usage = refreshed + .files + .get(&path) + .expect("unrelated history retained"); + assert_eq!(usage.days[&old_day]["gpt-5.6-sol"], vec![100, 0, 5, 0]); + assert!(!usage.codex_unresolved_fork_parent); + } + for path in [ + cached_path("required-a.jsonl"), + cached_path("required-b.jsonl"), + ] { + assert_unresolved(&refreshed, Path::new(&path)); + } + assert_eq!(summary.input_tokens, 0); + assert_eq!(summary.sessions_count, 0); + assert_unresolved(&refreshed, &active_child); +}