From 2e41dfffea8e405d47b1d15d6dec06cf70ec16bb Mon Sep 17 00:00:00 2001 From: Hung Pham Date: Wed, 7 Oct 2026 15:53:20 +0700 Subject: [PATCH 1/3] perf(graph): skip chain-engine rebuild on reopen via version stamp MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit rebuild() luôn clear + re-insert toàn bộ func id vào chain trie dù trie đã persist trong rt_* — chiếm >99% thời gian mở index (đo được: ~11s@5k, ~113s@20k). Mirror version-stamp của name engine cho chain engine: ghi [version|count|sample_func] vào slot 0 của storage chính, reopen đọc stamp là bỏ qua rebuild. Kết quả (reopen 20k symbols, full shape): rebuild TOTAL 113.27s → 0.234s (~485×). chains_map vẫn decode đủ cho callees/flow; trie giữ nguyên phục vụ callers/search_flow. Thêm test hồi quy chain_engine_stamp_survives_reopen. --- crates/codegraph-graph/src/lib.rs | 59 +++++++++++++++++++++++++++- crates/codegraph-graph/src/shared.rs | 47 ++++++++++++++++++++++ 2 files changed, 104 insertions(+), 2 deletions(-) diff --git a/crates/codegraph-graph/src/lib.rs b/crates/codegraph-graph/src/lib.rs index d138a380f..a9cdf5fb3 100644 --- a/crates/codegraph-graph/src/lib.rs +++ b/crates/codegraph-graph/src/lib.rs @@ -890,7 +890,14 @@ impl GraphIndex { // Engines. let __t_e = cg_t!("rebuild_chain_engine"); - self.rebuild_chain_engine(None).await?; + // Chain trie đã persist trong `rt_*` — version khớp thì bỏ qua + // `clear()` + re-insert toàn bộ func id (chiếm >99% thời gian rebuild: + // ~11s@5k, ~113s@20k). `chains_map` đã decode ở trên nên `callees`/`flow` + // vẫn đủ; chỉ trie (dùng cho `callers`/`search_flow`) là giữ nguyên. + if !self.chain_engine_is_current().await { + self.rebuild_chain_engine(None).await?; + self.stamp_chain_engine().await; + } cg_d!(__t_e, "rebuild_chain_engine"); let __t_f = cg_t!("rebuild_name_engine"); // Name trie đã persist ở dataset riêng — nếu version khớp thì dựng @@ -1016,7 +1023,6 @@ impl GraphIndex { /// Slot record dành cho version stamp — name record bắt đầu từ 1 nên không /// va chạm. Stamp = `[version: u64 LE][count: u64 LE]`. const NAME_STAMP_SLOT: usize = 0; - /// Name trie trên đĩa còn đúng version + khớp số record đã ghi (chống dataset /// bị xoá giữa chừng → coi như stale). async fn name_engine_is_current(&self) -> bool { @@ -1052,6 +1058,52 @@ impl GraphIndex { } } + /// Slot record cho version stamp của **chain engine** — chain trie lưu trên + /// storage chính (`rt_*`), func id ≥ `SYMBOL_BASE` nên slot `0` không va chạm + /// với node nào. Stamp = `[version: u64 LE][chain_count: u64 LE][sample_func: + /// u64 LE]` (24 byte) — `sample_func` = func id nhỏ nhất, dùng để probe trie + /// còn sống (record id của chain engine là func id **thưa**, không phải 1..N). + const CHAIN_STAMP_SLOT: usize = 0; + + /// Chain trie trên đĩa còn đúng version + khớp số func + còn node mẫu (chống + /// `rt_*` bị xoá dở → coi như stale). Giống `name_engine_is_current` nhưng + /// đọc trên storage chính, và probe bằng `sample_func` (func id thật). + async fn chain_engine_is_current(&self) -> bool { + let guard = self.storage.read().await; + let Ok(Some(bytes)) = guard.get_meta(Self::CHAIN_STAMP_SLOT).await else { + return false; + }; + if bytes.len() != 24 { + return false; + } + let version = u64::from_le_bytes(bytes[0..8].try_into().unwrap_or([0; 8])); + let count = u64::from_le_bytes(bytes[8..16].try_into().unwrap_or([0; 8])) as usize; + let sample = u64::from_le_bytes(bytes[16..24].try_into().unwrap_or([0; 8])) as usize; + if version != self.version || count != self.chains_map.len() { + return false; + } + // Index rỗng là hợp lệ (không có chain nào). + if count == 0 { + return true; + } + // Triet còn sống nếu record mẫu (func id thật) còn độ dài key. + guard.get_key_len(sample).await.ok().flatten().is_some() + } + + /// Ghi stamp sau khi chain trie dựng lại — `version` + số func + func id nhỏ + /// nhất làm mẫu probe. + async fn stamp_chain_engine(&self) { + let mut bytes = [0u8; 24]; + bytes[0..8].copy_from_slice(&self.version.to_le_bytes()); + bytes[8..16].copy_from_slice(&(self.chains_map.len() as u64).to_le_bytes()); + let sample = self.chains_map.keys().min().copied().unwrap_or(0); + bytes[16..24].copy_from_slice(&sample.to_le_bytes()); + let mut guard = self.storage.write().await; + if let Err(e) = guard.set_meta(Self::CHAIN_STAMP_SLOT, &bytes).await { + eprintln!("[codegraph] chain engine stamp failed: {e}"); + } + } + /// Dựng lại `name_records` từ `name_index` (nằm sẵn trong RAM) — thay cho /// vòng trie insert khi đã bỏ qua rebuild. Mảng giữ thứ tự sort để vừa map /// record→tên vừa dùng cho scan Prefix/Suffix/Exact. @@ -1267,6 +1319,9 @@ impl GraphIndex { // đĩa, nên `rebuild()` lúc mở index chỉ cần đọc stamp là bỏ qua // `rebuild_name_engine` (11-32s) — thay vì dựng lại mỗi lần mở. self.stamp_name_engine().await; + // Chain trie cũng persist trong `rt_*` — stamp version để reopen bỏ qua + // `rebuild_chain_engine` (>99% thời gian rebuild: ~113s@20k). + self.stamp_chain_engine().await; { let mut st = self.storage.write().await; st.save_next_id(self.next_id).await.map_err(serr)?; diff --git a/crates/codegraph-graph/src/shared.rs b/crates/codegraph-graph/src/shared.rs index 2d4890a8a..5dee55e76 100644 --- a/crates/codegraph-graph/src/shared.rs +++ b/crates/codegraph-graph/src/shared.rs @@ -413,6 +413,53 @@ mod tests { assert!(idx.name_engine_is_current().await, "stamp mới sau ingest"); } + /// Chain trie persist trong `rt_*` + stamp version → mở lại index bỏ qua + /// `rebuild_chain_engine` (phần tốn nhất: ~113s@20k). Trie vẫn đọc lại được + /// cho `callers`/`search_flow`. + #[cfg(feature = "sqlite")] + #[tokio::test] + async fn chain_engine_stamp_survives_reopen() { + let dir = tempfile::tempdir().unwrap(); + let db_path = dir.path().join("db.sqlite"); + let db_str = format!("sqlite://{}", db_path.to_string_lossy()); + + { + let mut idx = GraphIndex::open(&db_str).await.unwrap(); + let r = mk_result( + "a.ts", + vec![sym("a", SYMBOL_BASE), sym("b", SYMBOL_BASE + 1)], + vec![SYMBOL_BASE, SYMBOL_BASE + 1], + ); + idx.ingest(&[r]).await.unwrap(); + assert!(idx.chain_engine_is_current().await, "stamp sau ingest"); + } + + // Mở lại → stamp khớp version → bỏ qua rebuild trie; trie vẫn sống. + let idx = GraphIndex::open(&db_str).await.unwrap(); + assert!( + idx.chain_engine_is_current().await, + "stamp phải khớp sau reopen" + ); + // `search_flow` đọc thẳng chain trie persist (plain `search`, không cần shortcuts). + let sf = idx.search_flow(&[SYMBOL_BASE + 1]).await.unwrap(); + assert_eq!(sf.len(), 1); + assert_eq!(sf[0].function_name, "a"); + // `callers` (substring search qua shortcuts) cũng phải sống lại. + let cers = idx.callers(SYMBOL_BASE + 1, 1).await.unwrap(); + assert_eq!(cers.len(), 1); + assert_eq!(cers[0].name, "a"); + + // Re-index → version bump → stamp cũ thành stale, rebuild lại. + let mut idx = GraphIndex::open(&db_str).await.unwrap(); + let r = mk_result( + "b.ts", + vec![sym("gamma", SYMBOL_BASE + 9)], + vec![SYMBOL_BASE + 9], + ); + idx.ingest(&[r]).await.unwrap(); + assert!(idx.chain_engine_is_current().await, "stamp mới sau ingest"); + } + #[tokio::test] async fn sqlite_stale_version_rebuilds() { let dir = tempfile::tempdir().unwrap(); From cfb2a8cf60d132680f2f6af0a2b39c3d354cebcf Mon Sep 17 00:00:00 2001 From: Hung Pham Date: Wed, 7 Oct 2026 16:46:44 +0700 Subject: [PATCH 2/3] refactor(graph): drop in-memory edges map, read edge metadata from call records MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit self.edges: HashMap<(u64,u64), EdgeMeta> (51k entries @ repo này) là bản sao dữ liệu đã persist trong call records — flow() trước giờ vẫn fallback về record khi miss. Xoá hẳn: - GraphIndex.edges + rebuild_edges() + resolve_arg_ids() (arg_ids không ai đọc) - EdgeMeta struct trong codegraph-core (rt_edges luôn rỗng, chưa từng được ghi) - edges_mem/edge_meta_heap trong memtrack; predicted_edges trong bench mem - rebuild(): bỏ fetch+deserialize all_call_records (~130ms@20k), đếm edge_count thẳng từ chains; flow() lấy condition/effect trực tiếp từ record - build_edges_from_calls -> build_call_indexes (đếm edges, không materialize) stats().edges giữ nguyên (u64 count) nên API/MCP/tests không đổi. --- crates/codegraph-bench/src/bin/mem.rs | 35 ++-- crates/codegraph-core/src/lib.rs | 2 +- crates/codegraph-core/src/semgraph.rs | 20 --- crates/codegraph-graph/src/lib.rs | 228 +++++++------------------ crates/codegraph-graph/src/memtrack.rs | 23 +-- crates/codegraph-graph/tests/sqlite.rs | 2 +- 6 files changed, 80 insertions(+), 230 deletions(-) diff --git a/crates/codegraph-bench/src/bin/mem.rs b/crates/codegraph-bench/src/bin/mem.rs index 493c677f4..90b913cea 100644 --- a/crates/codegraph-bench/src/bin/mem.rs +++ b/crates/codegraph-bench/src/bin/mem.rs @@ -15,9 +15,7 @@ use camino::Utf8Path; use clap::Parser; use codegraph_bench::{BenchOptions, Repo, extract, index_at, orchestrator, run_queries}; -use codegraph_core::{ - Annotation, CallRecord, EdgeMeta, EffectType, ScopeLevel, Symbol, SymbolKind, -}; +use codegraph_core::{Annotation, CallRecord, EffectType, ScopeLevel, Symbol, SymbolKind}; use codegraph_graph::meminfo::{MemTracker, fmt_bytes, rss_bytes}; use codegraph_graph::memtrack::MemBreakdown; use std::sync::OnceLock; @@ -127,7 +125,7 @@ struct Cli { /// Dựng index synthetic với N function thay vì đọc repo thật — deterministic, /// không phụ thuộc network. Mỗi function gọi `--fanout` function khác nên - /// `edges` + `call_names` (hai cấu trúc đang tối ưu) có quy mô đáng kể. + /// `call_names` (cấu trúc đang tối ưu) có quy mô đáng kể. #[arg(long, value_name = "N")] synthetic: Option, @@ -141,7 +139,7 @@ struct Cli { /// /// - `symbols` — chỉ symbol → `symbols` + `name_index` + **name engine** /// - `chains` — symbol + chain, không call record → thêm **chain engine** - /// - `full` — kèm call record → thêm `call_names` + `edges` + /// - `full` — kèm call record → thêm `call_names` (+ đếm `edge_count`) #[arg(long, value_enum, default_value_t = Shape::Full)] shape: Shape, @@ -212,8 +210,6 @@ struct RepoMem { rss_index_delta: u64, /// `symbols.len() × size_of::()` — phần trong HashMap `symbols`. predicted_symbols: u64, - /// `edges.len() × size_of::()` — phần trong HashMap `edges`. - predicted_edges: u64, /// Deep size từng cấu trúc, sort giảm dần. breakdown: Vec, /// Tổng bytes đã quy được về cấu trúc (chưa gồm allocator + radix engine). @@ -301,7 +297,6 @@ fn measure(repo: &Repo, opts: &BenchOptions) -> anyhow::Result { rss_peak: tracker.peak(), rss_index_delta: after_index.saturating_sub(after_extract), predicted_symbols: st.symbols * size_of::() as u64, - predicted_edges: st.edges * size_of::() as u64, accounted_total: breakdown.accounted_total(), caches: breakdown.caches.clone(), breakdown: breakdown_rows(&breakdown), @@ -311,10 +306,10 @@ fn measure(repo: &Repo, opts: &BenchOptions) -> anyhow::Result { /// Dựng `ParseResult` synthetic: `n` function, mỗi function gọi `fanout` /// function khác (id local tính từ `SYMBOL_BASE`). /// -/// Mục tiêu là **làm đầy `edges` + `call_names`** — hai `HashMap` đang tốn -/// nhiều RAM nhất trong `GraphIndex`. Call name cố tình trùng lặp (chỉ vài -/// tên lib giả) để `call_names` có nhiều key chứa nhiều site, đúng hình dạng -/// repo thật. +/// Mục tiêu là **làm đầy `call_names`** — `HashMap` đang tốn nhiều RAM nhất +/// trong `GraphIndex` (edge metadata không còn materialize thành map riêng). +/// Call name cố tình trùng lặp (chỉ vài tên lib giả) để `call_names` có nhiều +/// key chứa nhiều site, đúng hình dạng repo thật. fn synthetic_parse_result(n: usize, fanout: usize, shape: Shape) -> codegraph_graph::ParseResult { // `n = 0` sẽ làm `% n` panic ở vòng sinh chain — chặn sớm, báo rõ. assert!(n > 0, "--synthetic cần N > 0"); @@ -423,7 +418,6 @@ fn measure_synthetic(n: usize, fanout: usize, shape: Shape) -> anyhow::Result() as u64, - predicted_edges: st.edges * size_of::() as u64, accounted_total: breakdown.accounted_total(), caches: breakdown.caches.clone(), breakdown: breakdown_rows(&breakdown), @@ -488,7 +482,6 @@ fn measure_reopen(n: usize, fanout: usize, shape: Shape) -> anyhow::Result() as u64, - predicted_edges: st.edges * size_of::() as u64, accounted_total: breakdown.accounted_total(), caches: breakdown.caches.clone(), breakdown: breakdown_rows(&breakdown), @@ -534,19 +527,12 @@ fn main() -> anyhow::Result<()> { } println!( - "{:<14} {:>8} {:>8} {:>10} {:>10} {:>10} {:>12} {:>12}", - "repo", - "symbols", - "edges", - "rss extract", - "rss index", - "Δ index", - "pred symbols", - "pred edges" + "{:<14} {:>8} {:>8} {:>10} {:>10} {:>10} {:>12}", + "repo", "symbols", "edges", "rss extract", "rss index", "Δ index", "pred symbols", ); for r in &results { println!( - "{:<14} {:>8} {:>8} {:>10} {:>10} {:>10} {:>12} {:>12}", + "{:<14} {:>8} {:>8} {:>10} {:>10} {:>10} {:>12}", r.repo, r.symbols, r.edges, @@ -554,7 +540,6 @@ fn main() -> anyhow::Result<()> { fmt_bytes(r.rss_after_index), fmt_bytes(r.rss_index_delta), fmt_bytes(r.predicted_symbols), - fmt_bytes(r.predicted_edges), ); } for r in &results { diff --git a/crates/codegraph-core/src/lib.rs b/crates/codegraph-core/src/lib.rs index 2bc9b085b..dbc2aba63 100644 --- a/crates/codegraph-core/src/lib.rs +++ b/crates/codegraph-core/src/lib.rs @@ -12,7 +12,7 @@ pub use error::{Error, Result}; pub use route::StorageRoute; pub use semgraph::{ is_marker, marker_id, marker_name, Annotation, BranchLabel, CallRecord, CallSite, - CallSiteResult, ClassInfo, DbStats as SemgraphStats, DependenciesReport, Dependency, EdgeMeta, + CallSiteResult, ClassInfo, DbStats as SemgraphStats, DependenciesReport, Dependency, EffectCallPattern, EffectRule, EffectType, FileInfo, FlowCall, FlowResult, FunctionScope, MemberInfo, ResolveResult, ScopeLevel, SearchFlowResult, Symbol, SymbolId, SymbolKind, SymbolMatch, MARKER_BRANCH_END, MARKER_BREAK, MARKER_CONTINUE, MARKER_IF_FALSE, MARKER_IF_TRUE, diff --git a/crates/codegraph-core/src/semgraph.rs b/crates/codegraph-core/src/semgraph.rs index 984b15350..8bf0ba209 100644 --- a/crates/codegraph-core/src/semgraph.rs +++ b/crates/codegraph-core/src/semgraph.rs @@ -316,26 +316,6 @@ pub struct Annotation { pub line: u32, } -/// Metadata của 1 call edge — serialized thành edge data (edge stream). -/// -/// `(caller_id, callee_id)` là chiều chuẩn; `position` = index của callee trong -/// chain của caller (để nối với CallRecord khi render flow). -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct EdgeMeta { - pub caller_id: SymbolId, - pub callee_id: SymbolId, - /// Index trong chain của caller mà callee xuất hiện. - pub position: usize, - /// Guard text của if bao quanh (nếu có). - pub condition: Option, - pub effect: EffectType, - pub effect_desc: Option, - #[serde(default)] - pub arg_ids: Vec, - pub is_loop_body: bool, - pub is_recursive: bool, -} - /// Call record thô — persist để render flow khi call không resolve được. /// /// Vị trí `0` trong chain là placeholder, được thay bằng id thật khi resolve. diff --git a/crates/codegraph-graph/src/lib.rs b/crates/codegraph-graph/src/lib.rs index a9cdf5fb3..eeb1040e7 100644 --- a/crates/codegraph-graph/src/lib.rs +++ b/crates/codegraph-graph/src/lib.rs @@ -33,8 +33,9 @@ //! toàn bộ symbol (id global) + remap (idMap bỏ `0` — placeholder phải giữ 0) → //! `resolve_calls` (thay placeholder 0 trong chain bằng id thật: structural hint //! → exact name → short name → best-candidate: @Override +10 / has-chain +5 / -//! same-file +3) → `build_edges_from_calls` (edge = chain[position], CallSite + -//! var-type alias, gom SaveCallRecords) → files → rebuild engines → bump version. +//! same-file +3) → `build_call_indexes` (đếm edge = chain[position], +//! CallSite + var-type alias, gom SaveCallRecords) → files → rebuild engines → +//! bump version. use crate::embeddings::{EmbeddingBackend, default_backend, embedding_enabled, make_backend}; pub use crate::radix::Element; @@ -64,9 +65,9 @@ pub use crate::storage::{ use crate::vector_index::VectorIndex; use codegraph_core::{ BranchLabel, CallRecord, CallSite, CallSiteResult, ClassInfo, DependenciesReport, Dependency, - EdgeMeta, EffectType, Error, FileInfo, FlowCall, FlowResult, FunctionScope, MemberInfo, - ResolveResult, SYMBOL_BASE, SearchFlowResult, SemgraphStats, StorageRoute, Symbol, SymbolKind, - SymbolMatch, is_marker, marker_name, + Error, FileInfo, FlowCall, FlowResult, FunctionScope, MemberInfo, ResolveResult, SYMBOL_BASE, + SearchFlowResult, SemgraphStats, StorageRoute, Symbol, SymbolKind, SymbolMatch, is_marker, + marker_name, }; use serde::{Deserialize, Serialize}; use std::collections::{BTreeMap, HashMap, HashSet}; @@ -331,8 +332,10 @@ pub struct GraphIndex { chains_map: HashMap>, /// call name (lowercase, kèm alias type-qualified) → call sites. call_names: HashMap>, - /// `(caller, callee)` → edge meta (last-wins, rebuild từ chains + records). - edges: HashMap<(u64, u64), EdgeMeta>, + /// Số edge `(caller, callee)` — đếm từ chains lúc ingest/rebuild. + /// Metadata của edge (condition/effect/...) KHÔNG materialize thành map: + /// đã nằm trong call records persist, `flow()` đọc thẳng từ đó. + edge_count: u64, /// Files trong graph. files: Vec, /// next_id của registry. @@ -771,7 +774,7 @@ impl GraphIndex { scope_index: HashMap::new(), chains_map: HashMap::new(), call_names: HashMap::new(), - edges: HashMap::new(), + edge_count: 0, files: Vec::new(), next_id: SYMBOL_BASE, version: 0, @@ -829,15 +832,6 @@ impl GraphIndex { .await .map_err(serr)?; cg_d!(__t_all_call_name_indexes, "all_call_name_indexes"); - let __t_all_call_records = cg_t!("all_call_records"); - let call_records_raw = self - .storage - .read() - .await - .all_call_records() - .await - .map_err(serr)?; - cg_d!(__t_all_call_records, "all_call_records"); let __t_load_all_files = cg_t!("load_all_files"); self.files = self .storage @@ -876,17 +870,21 @@ impl GraphIndex { } } - // Edges — rebuild từ chains + call records (không persist riêng). - let mut recs: HashMap> = HashMap::new(); - for (func, bytes) in call_records_raw { - if let Ok(r) = serde_json::from_slice::>(&bytes) { - recs.insert(func, r); - } - } + // Edge count — đếm thẳng từ chains (không materialize EdgeMeta: + // metadata đã nằm trong call records persist, `flow()` đọc từ đó). + // Không cần `all_call_records` ở đây (đó là JSON deserialize ~80ms). + self.edge_count = self + .chains_map + .values() + .map(|chain| { + chain + .iter() + .skip(1) + .filter(|&&e| e != 0 && !is_marker(e)) + .count() as u64 + }) + .sum(); cg_d!(__t_c, "call_names_json"); - let __t_d = cg_t!("rebuild_edges"); - self.rebuild_edges(&recs); - cg_d!(__t_d, "rebuild_edges"); // Engines. let __t_e = cg_t!("rebuild_chain_engine"); @@ -918,7 +916,7 @@ impl GraphIndex { st.set_stats(IndexCounts { symbols: self.symbols.len() as u64, chains: self.chains_map.len() as u64, - edges: self.edges.len() as u64, + edges: self.edge_count, files: self.files.len() as u64, next_id: self.next_id, }) @@ -959,43 +957,6 @@ impl GraphIndex { } } - /// Rebuild edges từ chains + call records (nhanh — chỉ dùng khi reopen). - #[cfg_attr( - not(any(feature = "sqlite", feature = "lmdb", feature = "redis")), - allow(dead_code) - )] - // chỉ rebuild() dùng — không backend thì không ai gọi. - fn rebuild_edges(&mut self, recs: &HashMap>) { - self.edges.clear(); - for (&func_id, chain) in &self.chains_map { - let rec_by_pos: HashMap = recs - .get(&func_id) - .map(|r| r.iter().map(|c| (c.position, c)).collect()) - .unwrap_or_default(); - for (i, &e) in chain.iter().enumerate() { - // Vị trí 0 = owner — skip như build_edges_from_calls (ingest). - if i == 0 || is_marker(e) || e == 0 { - continue; - } - let rec = rec_by_pos.get(&i); - self.edges.insert( - (func_id, e), - EdgeMeta { - caller_id: func_id, - callee_id: e, - position: i, - condition: rec.and_then(|r| r.condition.clone()), - effect: rec.map(|r| r.effect).unwrap_or_default(), - effect_desc: rec.and_then(|r| r.effect_desc.clone()), - arg_ids: Vec::new(), - is_loop_body: rec.map(|r| r.is_loop_body).unwrap_or(false), - is_recursive: e == func_id, - }, - ); - } - } - } - /// Rebuild chain engine từ `chains_map` (clear + insert tuần tự). async fn rebuild_chain_engine(&mut self, progress: Option<&dyn IngestProgress>) -> Result<()> { self.chains.clear().await.map_err(serr_search)?; @@ -1202,9 +1163,8 @@ impl GraphIndex { // ── Ingest (full re-index — pipeline 2 phase như semgraph) ── /// Ingest toàn bộ parse results — **full re-index**: xoá dữ liệu cũ, register - /// symbol (id global) + remap, resolve placeholder 0, build edges + call-name - /// index, persist + bump version. Không báo tiến độ — dùng - /// [`ingest_with_progress`](Self::ingest_with_progress) nếu cần. + /// symbol (id global) + remap, resolve placeholder 0, build call-name index + /// + đếm edges, persist + bump version (bản có tiến độ: `ingest_with_progress`). pub async fn ingest(&mut self, results: &[ParseResult]) -> Result<()> { self.ingest_with_progress(results, None).await } @@ -1233,7 +1193,7 @@ impl GraphIndex { self.scope_index.clear(); self.chains_map.clear(); self.call_names.clear(); - self.edges.clear(); + self.edge_count = 0; self.files.clear(); self.name_records.clear(); self.next_id = SYMBOL_BASE; @@ -1284,8 +1244,8 @@ impl GraphIndex { // ── Phase 2: resolve placeholder 0 trong chains ── self.resolve_calls(&all_calls); - // ── Phase 3: build edges + call records + call-name index ── - self.build_edges_from_calls(&all_calls, p).await?; + // ── Phase 3: call records + call-name index + edge count ── + self.build_call_indexes(&all_calls, p).await?; // ── Phase 4: files ── if let Some(p) = p { @@ -1329,7 +1289,7 @@ impl GraphIndex { st.set_stats(IndexCounts { symbols: self.symbols.len() as u64, chains: self.chains_map.len() as u64, - edges: self.edges.len() as u64, + edges: self.edge_count, files: self.files.len() as u64, next_id: self.next_id, }) @@ -1600,13 +1560,15 @@ impl GraphIndex { None } - /// Build edges từ chains (đã resolve) + call records; persist call records + - /// call-name index (kèm alias type-qualified `svc.validate` → `type.validate`). + /// Build call-name index từ chains (đã resolve) + call records; persist call + /// records + call-name index (kèm alias type-qualified `svc.validate` → + /// `type.validate`). Đồng thời đếm `edge_count` cho stats. /// - /// Edge model: mọi symbol element trong chain là một callee (thống nhất với - /// `rebuild_edges` khi reopen) — call record chỉ bổ sung metadata theo - /// position. Chain dựng thẳng (không qua placeholder) vẫn sinh edge đủ. - async fn build_edges_from_calls( + /// Edge model: mọi symbol element trong chain là một callee — call record + /// chỉ bổ sung metadata theo position. Chain dựng thẳng (không qua + /// placeholder) vẫn sinh edge đủ. Metadata KHÔNG materialize thành map + /// (xem `edge_count`): `flow()` đọc thẳng từ call records persist. + async fn build_call_indexes( &mut self, calls: &[CallRef<'_>], progress: Option<&dyn IngestProgress>, @@ -1646,41 +1608,20 @@ impl GraphIndex { } } - // Edges từ mọi chain — rec lookup theo position cho metadata. - for (&caller, chain) in &self.chains_map { - let mut rec_by_pos: HashMap = HashMap::new(); - if let Some(idxs) = recs_by_caller.get(&caller) { - for &i in idxs { - let rec = calls[i as usize].rec; - rec_by_pos.insert(rec.position, rec); - } - } - for (i, &e) in chain.iter().enumerate() { - // Vị trí 0 = chính func id (owner) — không phải call. Recursion - // thật xuất hiện ở vị trí > 0 (vẫn giữ là edge is_recursive). - if i == 0 || is_marker(e) || e == 0 { - continue; - } - let rec = rec_by_pos.get(&i); - let arg_ids = rec - .map(|r| self.resolve_arg_ids(caller, &r.arg_exprs)) - .unwrap_or_default(); - self.edges.insert( - (caller, e), - EdgeMeta { - caller_id: caller, - callee_id: e, - position: i, - condition: rec.and_then(|r| r.condition.clone()), - effect: rec.map(|r| r.effect).unwrap_or_default(), - effect_desc: rec.and_then(|r| r.effect_desc.clone()), - arg_ids, - is_loop_body: rec.map(|r| r.is_loop_body).unwrap_or(false), - is_recursive: e == caller, - }, - ); - } - } + // Đếm edges từ mọi chain — metadata không materialize (đã nằm trong + // call records persist; `flow()` đọc thẳng từ đó). + // Vị trí 0 = chính func id (owner) — không phải call. + self.edge_count = self + .chains_map + .values() + .map(|chain| { + chain + .iter() + .skip(1) + .filter(|&&e| e != 0 && !is_marker(e)) + .count() as u64 + }) + .sum(); // Persist call records (gom theo caller). if let Some(p) = progress @@ -1691,9 +1632,9 @@ impl GraphIndex { for (caller, idxs) in recs_by_caller { let recs: Vec<&CallRecord> = idxs.iter().map(|&i| calls[i as usize].rec).collect(); // `rec.caller_id` ghi ra đây là id **local** của file, không phải - // id global — vô hại: mọi reader (`flow`, `rebuild_edges`, - // `bingraph`) lấy caller id từ **key** của blob, không đọc field - // này. Nếu sau này cần id global thì đừng đọc field — dùng key. + // id global — vô hại: mọi reader (`flow`, `bingraph`) lấy caller id + // từ **key** của blob, không đọc field này. Nếu sau này cần id + // global thì đừng đọc field — dùng key. let bytes = serde_json::to_vec(&recs).map_err(|e| Error::Search(e.to_string()))?; self.storage .write() @@ -1753,32 +1694,6 @@ impl GraphIndex { None } - /// Resolve arg expr (tên var/param) về symbol id trong scope của caller. - fn resolve_arg_ids(&self, caller_id: u64, arg_exprs: &[String]) -> Vec { - let mut scopes = vec![caller_id]; - if let Some(s) = self.symbols.get(&caller_id) - && s.scope_id != 0 - { - scopes.push(s.scope_id); - } - let mut out = Vec::with_capacity(arg_exprs.len()); - for expr in arg_exprs { - let mut found = 0; - 'outer: for sid in &scopes { - if let Some(ids) = self.scope_index.get(sid) { - for id in ids { - if self.symbols.get(id).is_some_and(|s| s.name == *expr) { - found = *id; - break 'outer; - } - } - } - } - out.push(found); - } - out - } - // ── Queries (mọi search đều qua `search_symbol_paged_resumable` — resumable, // deadline-aware; các hàm tiện ích còn lại chỉ wrap nó, không có implementation // song song) ── @@ -2078,30 +1993,17 @@ impl GraphIndex { let Some(callee) = self.symbols.get(&e) else { continue; }; - let meta = self.edges.get(&(id, e)); let rec = rec_by_pos.get(&i); - let mut cond = meta.and_then(|m| m.condition.clone()); - let mut effect = meta.map(|m| m.effect).unwrap_or_default(); - let mut effect_desc = meta.and_then(|m| m.effect_desc.clone()); - if let Some(r) = rec { - if cond.is_none() { - cond = r.condition.clone(); - } - if effect == EffectType::None { - effect = r.effect; - } - if effect_desc.is_none() { - effect_desc = r.effect_desc.clone(); - } - } + // Metadata (condition/effect/...) đọc thẳng từ call record — cùng + // nguồn mà EdgeMeta cũ cũng dùng (không còn map trung gian). calls.push(FlowCall { position: i, to_name: callee.name.clone(), to_id: Some(e), line: rec.map(|r| r.line).unwrap_or(0), - condition: cond, - effect, - effect_desc, + condition: rec.and_then(|r| r.condition.clone()), + effect: rec.map(|r| r.effect).unwrap_or_default(), + effect_desc: rec.and_then(|r| r.effect_desc.clone()), args: rec.map(|r| r.arg_exprs.clone()).unwrap_or_default(), }); } @@ -2933,7 +2835,6 @@ impl GraphIndex { symbols: mt::symbols_mem(&self.symbols), chains_map: mt::chains_map_mem(&self.chains_map), call_names: mt::call_names_mem(&self.call_names), - edges: mt::edges_mem(&self.edges), name_index: mt::name_index_mem(&self.name_index), scope_index: mt::scope_index_mem(&self.scope_index), name_keys: mt::name_keys_mem(&self.name_records), @@ -2947,7 +2848,7 @@ impl GraphIndex { SemgraphStats { symbols: self.symbols.len() as u64, chains: self.chains_map.len() as u64, - edges: self.edges.len() as u64, + edges: self.edge_count, files: self.files.len() as u64, next_id: self.next_id, } @@ -2965,7 +2866,8 @@ impl GraphIndex { mod tests { use super::*; use codegraph_core::{ - Annotation, MARKER_BRANCH_END, MARKER_IF_TRUE, MARKER_LOOP, MARKER_LOOP_BACK, ScopeLevel, + Annotation, EffectType, MARKER_BRANCH_END, MARKER_IF_TRUE, MARKER_LOOP, MARKER_LOOP_BACK, + ScopeLevel, }; #[test] diff --git a/crates/codegraph-graph/src/memtrack.rs b/crates/codegraph-graph/src/memtrack.rs index 74286093a..3e486089b 100644 --- a/crates/codegraph-graph/src/memtrack.rs +++ b/crates/codegraph-graph/src/memtrack.rs @@ -20,7 +20,7 @@ use std::collections::HashMap; use std::hash::Hash; -use codegraph_core::{Annotation, CallSite, EdgeMeta, FileInfo, Symbol}; +use codegraph_core::{Annotation, CallSite, FileInfo, Symbol}; /// Một cấu trúc: số phần tử + bytes đã cấp (fixed + heap). #[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] @@ -106,11 +106,6 @@ pub fn call_site_heap(site: &CallSite) -> u64 { + site.arg_exprs.iter().map(str_bytes).sum::() } -/// Deep size của `EdgeMeta` (chưa tính slot trong HashMap). -pub fn edge_meta_heap(m: &EdgeMeta) -> u64 { - opt_str_bytes(&m.condition) + opt_str_bytes(&m.effect_desc) + u64_vec_bytes(&m.arg_ids) -} - // ── Tổng hợp theo cấu trúc ── /// RAM của `symbols: HashMap`. @@ -147,15 +142,6 @@ pub fn call_names_mem(m: &HashMap>) -> StructureMem { } } -/// RAM của `edges: HashMap<(u64, u64), EdgeMeta>`. -pub fn edges_mem(m: &HashMap<(u64, u64), EdgeMeta>) -> StructureMem { - StructureMem { - entries: m.len() as u64, - fixed_bytes: map_bucket_bytes(m), - heap_bytes: m.values().map(edge_meta_heap).sum(), - } -} - /// RAM của `name_index: HashMap>` (key + buffer, không tính /// phần tử `u64` vì đã nằm trong `u64_vec_bytes` của từng value). pub fn name_index_mem(m: &HashMap>) -> StructureMem { @@ -197,7 +183,6 @@ pub struct MemBreakdown { pub symbols: StructureMem, pub chains_map: StructureMem, pub call_names: StructureMem, - pub edges: StructureMem, pub name_index: StructureMem, pub scope_index: StructureMem, pub name_keys: StructureMem, @@ -213,7 +198,6 @@ impl MemBreakdown { self.symbols.total_bytes(), self.chains_map.total_bytes(), self.call_names.total_bytes(), - self.edges.total_bytes(), self.name_index.total_bytes(), self.scope_index.total_bytes(), self.name_keys.total_bytes(), @@ -229,7 +213,6 @@ impl MemBreakdown { ("symbols", self.symbols), ("chains_map", self.chains_map), ("call_names", self.call_names), - ("edges", self.edges), ("name_index", self.name_index), ("scope_index", self.scope_index), ("name_keys", self.name_keys), @@ -297,7 +280,7 @@ mod tests { fixed_bytes: 0, heap_bytes: 100, }, - edges: StructureMem { + call_names: StructureMem { entries: 1, fixed_bytes: 0, heap_bytes: 10, @@ -306,7 +289,7 @@ mod tests { }; let r = b.ranked(); assert_eq!(r[0].0, "symbols"); - assert_eq!(r[1].0, "edges"); + assert_eq!(r[1].0, "call_names"); assert_eq!(b.accounted_total(), 110); } } diff --git a/crates/codegraph-graph/tests/sqlite.rs b/crates/codegraph-graph/tests/sqlite.rs index ccd428ae7..b450ba2b1 100644 --- a/crates/codegraph-graph/tests/sqlite.rs +++ b/crates/codegraph-graph/tests/sqlite.rs @@ -56,7 +56,7 @@ fn result( } /// Ingest → reopen: mọi entity (symbols/chains/files/version) + query surface -/// sống lại từ file; edges tái dựng từ chains + call records. +/// sống lại từ file; edge count đếm lại từ chains (metadata đọc từ call records). #[tokio::test] async fn index_ingest_reopen_roundtrip() { let dir = tempfile::tempdir().unwrap(); From f34b8b937d517081e7d4f2ac9131e74309498d91 Mon Sep 17 00:00:00 2001 From: Hung Pham Date: Wed, 7 Oct 2026 17:40:31 +0700 Subject: [PATCH 3/3] Bump version to v2.2.8 --- Cargo.lock | 30 ++++++++++++------------- Cargo.toml | 2 +- packaging/aur/codegraph-rs-bin/PKGBUILD | 2 +- packaging/choco/codegraph.nuspec | 2 +- packaging/winget/codegraph.yaml | 4 ++-- scripts/install.ps1 | 4 ++-- 6 files changed, 22 insertions(+), 22 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 1f793ca72..04647c40e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -720,7 +720,7 @@ checksum = "c8d4a3bb8b1e0c1050499d1815f5ab16d04f0959b233085fb31653fbfc9d98f9" [[package]] name = "codegraph" -version = "2.2.7" +version = "2.2.8" dependencies = [ "anyhow", "camino", @@ -743,7 +743,7 @@ dependencies = [ [[package]] name = "codegraph-api" -version = "2.2.7" +version = "2.2.8" dependencies = [ "anyhow", "camino", @@ -761,7 +761,7 @@ dependencies = [ [[package]] name = "codegraph-bench" -version = "2.2.7" +version = "2.2.8" dependencies = [ "anyhow", "camino", @@ -780,7 +780,7 @@ dependencies = [ [[package]] name = "codegraph-binary" -version = "2.2.7" +version = "2.2.8" dependencies = [ "camino", "codegraph-core", @@ -797,7 +797,7 @@ dependencies = [ [[package]] name = "codegraph-context" -version = "2.2.7" +version = "2.2.8" dependencies = [ "camino", "codegraph-core", @@ -812,7 +812,7 @@ dependencies = [ [[package]] name = "codegraph-core" -version = "2.2.7" +version = "2.2.8" dependencies = [ "async-graphql", "camino", @@ -823,7 +823,7 @@ dependencies = [ [[package]] name = "codegraph-docs" -version = "2.2.7" +version = "2.2.8" dependencies = [ "anyhow", "codegraph-core", @@ -841,7 +841,7 @@ dependencies = [ [[package]] name = "codegraph-extract" -version = "2.2.7" +version = "2.2.8" dependencies = [ "camino", "codegraph-binary", @@ -880,7 +880,7 @@ dependencies = [ [[package]] name = "codegraph-graph" -version = "2.2.7" +version = "2.2.8" dependencies = [ "async-trait", "bincode", @@ -910,7 +910,7 @@ dependencies = [ [[package]] name = "codegraph-graphql" -version = "2.2.7" +version = "2.2.8" dependencies = [ "anyhow", "async-graphql", @@ -936,7 +936,7 @@ dependencies = [ [[package]] name = "codegraph-installer" -version = "2.2.7" +version = "2.2.8" dependencies = [ "anyhow", "camino", @@ -952,7 +952,7 @@ dependencies = [ [[package]] name = "codegraph-mcp" -version = "2.2.7" +version = "2.2.8" dependencies = [ "anyhow", "axum", @@ -976,7 +976,7 @@ dependencies = [ [[package]] name = "codegraph-sboxes" -version = "2.2.7" +version = "2.2.8" dependencies = [ "camino", "codegraph-core", @@ -998,7 +998,7 @@ dependencies = [ [[package]] name = "codegraph-source" -version = "2.2.7" +version = "2.2.8" dependencies = [ "async-trait", "camino", @@ -1011,7 +1011,7 @@ dependencies = [ [[package]] name = "codegraph-web" -version = "2.2.7" +version = "2.2.8" dependencies = [ "axum", "mime_guess", diff --git a/Cargo.toml b/Cargo.toml index e345dfba7..973357cb4 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -19,7 +19,7 @@ members = [ ] [workspace.package] -version = "2.2.7" +version = "2.2.8" edition = "2021" rust-version = "1.80" license = "MIT" diff --git a/packaging/aur/codegraph-rs-bin/PKGBUILD b/packaging/aur/codegraph-rs-bin/PKGBUILD index a8ac08cde..b3b71a4e9 100644 --- a/packaging/aur/codegraph-rs-bin/PKGBUILD +++ b/packaging/aur/codegraph-rs-bin/PKGBUILD @@ -1,6 +1,6 @@ # Maintainer: Hung Pham pkgname=codegraph-rs-bin -pkgver=2.2.7 +pkgver=2.2.8 pkgrel=1 pkgdesc="Local-first code intelligence: tree-sitter knowledge graph + MCP server (prebuilt binary)" arch=('x86_64' 'aarch64') diff --git a/packaging/choco/codegraph.nuspec b/packaging/choco/codegraph.nuspec index 706becfda..58b5bb9ac 100644 --- a/packaging/choco/codegraph.nuspec +++ b/packaging/choco/codegraph.nuspec @@ -2,7 +2,7 @@ codegraph - 2.2.7 + 2.2.8 codegraph Hung Pham https://github.com/hungpham10/codegraph-rs diff --git a/packaging/winget/codegraph.yaml b/packaging/winget/codegraph.yaml index 0b5ef8f09..3fb59275c 100644 --- a/packaging/winget/codegraph.yaml +++ b/packaging/winget/codegraph.yaml @@ -6,7 +6,7 @@ # release time (or automate it in the release pipeline before submitting to # microsoft/winget-pkgs). PackageIdentifier: hungpham10.codegraph -PackageVersion: 2.2.7 +PackageVersion: 2.2.8 PackageName: codegraph Publisher: Hung Pham PublisherUrl: https://github.com/hungpham10/codegraph-rs @@ -18,7 +18,7 @@ PackageUrl: https://github.com/hungpham10/codegraph-rs InstallerType: zip Installers: - Architecture: x64 - InstallerUrl: https://github.com/hungpham10/codegraph-rs/releases/download/v2.2.7/codegraph-x86_64-pc-windows-msvc.zip + InstallerUrl: https://github.com/hungpham10/codegraph-rs/releases/download/v2.2.8/codegraph-x86_64-pc-windows-msvc.zip InstallerSha256: 0000000000000000000000000000000000000000000000000000000000000000 InstallerType: zip ManifestType: singleton diff --git a/scripts/install.ps1 b/scripts/install.ps1 index c410c820e..4145e2c5f 100644 --- a/scripts/install.ps1 +++ b/scripts/install.ps1 @@ -5,11 +5,11 @@ # # Usage (pin a version / download the script first): # irm https://raw.githubusercontent.com/hungpham10/codegraph-rs/main/scripts/install.ps1 -OutFile install.ps1 -# .\install.ps1 -Version 2.2.7 +# .\install.ps1 -Version 2.2.8 [CmdletBinding()] param( - # Pin a specific version, e.g. "2.2.7". Empty = latest release. + # Pin a specific version, e.g. "2.2.8". Empty = latest release. [string]$Version )