diff --git a/Cargo.lock b/Cargo.lock index dab8310..f05a1d2 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -112,6 +112,7 @@ dependencies = [ "cfg-if", "constant_time_eq", "cpufeatures", + "memmap2", ] [[package]] @@ -357,7 +358,7 @@ checksum = "460fbee9c2c2f33933d720630a6a0bac33ba7053db5344fac858d4b8952d77d5" [[package]] name = "diskghost-cli" -version = "0.4.0" +version = "0.5.0" dependencies = [ "clap", "diskghost-core", @@ -366,7 +367,7 @@ dependencies = [ [[package]] name = "diskghost-core" -version = "0.4.0" +version = "0.5.0" dependencies = [ "blake3", "criterion", @@ -572,6 +573,15 @@ version = "2.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "88904434abc2901f197fe8cc55f0445e7ded921dba5911dad2e2b39b48e663c4" +[[package]] +name = "memmap2" +version = "0.9.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d1219ed1b7f229ee7104d281dd01d6802fe28bb6e95d292942c4daacdeb798c0" +dependencies = [ + "libc", +] + [[package]] name = "num-traits" version = "0.2.19" diff --git a/Cargo.toml b/Cargo.toml index fcefae3..0191f4d 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -4,7 +4,7 @@ exclude = ["gui"] resolver = "2" [workspace.package] -version = "0.4.0" +version = "0.5.0" edition = "2021" license = "MIT" authors = ["kingchenc"] diff --git a/crates/diskghost-core/Cargo.toml b/crates/diskghost-core/Cargo.toml index 561fa5b..fbcb960 100644 --- a/crates/diskghost-core/Cargo.toml +++ b/crates/diskghost-core/Cargo.toml @@ -11,7 +11,7 @@ description = "Fast parallel disk scanning and duplicate detection — the engin [dependencies] jwalk = "0.8" rayon = "1" -blake3 = "1" +blake3 = { version = "1", features = ["mmap"] } same-file = "1" globset = "0.4" trash = "5" diff --git a/crates/diskghost-core/src/lib.rs b/crates/diskghost-core/src/lib.rs index 429ec72..d4ec6ee 100644 --- a/crates/diskghost-core/src/lib.rs +++ b/crates/diskghost-core/src/lib.rs @@ -199,23 +199,36 @@ fn is_excluded(glob: &globset::GlobSet, path: &Path) -> bool { fn walk(root: &Path, opts: &Options, progress: &Progress) -> Walk { let glob = Arc::new(build_globset(&opts.exclude)); - let mut wd = jwalk::WalkDir::new(root) + // Each file's size is cached in the entry's client_state (Some(size), or + // None if the stat failed) so it can be read on the parallel walk threads + // instead of one-at-a-time in the serial consumer below. This is the + // scan's hot path: statting hundreds of thousands of files serially was + // the whole cost; jwalk already walks in parallel, so we stat there too. + let mut wd = jwalk::WalkDirGeneric::<((), Option)>::new(root) .skip_hidden(false) .follow_links(opts.follow_symlinks); if let Some(d) = opts.max_depth { wd = wd.max_depth(d); } - if !glob.is_empty() { + let glob = glob.clone(); + let wd = wd.process_read_dir(move |_depth, _path, _state, children| { // Prune excluded entries *before* descending, so an excluded directory // (e.g. node_modules) costs no I/O for its entire subtree. - let glob = glob.clone(); - wd = wd.process_read_dir(move |_depth, _path, _state, children| { + if !glob.is_empty() { children.retain(|res| match res { Ok(e) => !is_excluded(&glob, &e.path()), Err(_) => true, }); - }); - } + } + // Stat every file here (on the walk's rayon worker), caching its size. + for res in children.iter_mut() { + if let Ok(e) = res { + if e.file_type().is_file() { + e.client_state = std::fs::metadata(e.path()).ok().map(|m| m.len()); + } + } + } + }); let mut files = Vec::new(); let mut dirs = 0u64; @@ -233,9 +246,8 @@ fn walk(root: &Path, opts: &Options, progress: &Progress) -> Walk { dirs += 1; // don't count the root itself } } else if ft.is_file() { - match e.metadata() { - Ok(m) => { - let size = m.len(); + match e.client_state { + Some(size) => { progress.files.fetch_add(1, Ordering::Relaxed); progress.bytes.fetch_add(size, Ordering::Relaxed); files.push(FileEntry { @@ -243,7 +255,7 @@ fn walk(root: &Path, opts: &Options, progress: &Progress) -> Walk { size, }); } - Err(_) => skipped += 1, + None => skipped += 1, } } } @@ -587,12 +599,28 @@ pub fn remove_path( .collect(); errors.extend(file_errors); - // Remove directories deepest-first so each is empty when removed. + // Remove directories deepest-first. Within a single depth level no + // dir contains another, so each level is removed in parallel; the + // levels run deepest → shallowest so a dir is always empty when we + // reach it. dirs.sort_by_key(|d| std::cmp::Reverse(d.components().count())); - for d in &dirs { - if let Err(e) = std::fs::remove_dir(d) { - errors.push(format!("{}: {e}", d.display())); + let mut i = 0; + while i < dirs.len() { + let depth = dirs[i].components().count(); + let mut j = i; + while j < dirs.len() && dirs[j].components().count() == depth { + j += 1; } + let level_errors: Vec = dirs[i..j] + .par_iter() + .filter_map(|d| { + std::fs::remove_dir(d) + .err() + .map(|e| format!("{}: {e}", d.display())) + }) + .collect(); + errors.extend(level_errors); + i = j; } } } @@ -644,6 +672,14 @@ fn hash_prefix(path: &Path, len: usize) -> std::io::Result<[u8; 32]> { } fn hash_file(path: &Path) -> std::io::Result { + // Memory-map the file so BLAKE3 reads straight from the page cache with no + // buffered-read syscalls or intermediate copies — the fast path for the + // large files that dominate full-hash cost. + let mut hasher = blake3::Hasher::new(); + if hasher.update_mmap(path).is_ok() { + return Ok(hasher.finalize().to_hex().to_string()); + } + // Fallback: stream the file (mmap unsupported, empty, or a special file). let mut hasher = blake3::Hasher::new(); let mut file = std::fs::File::open(path)?; std::io::copy(&mut file, &mut hasher)?; diff --git a/gui/src-tauri/Cargo.lock b/gui/src-tauri/Cargo.lock index 4d5390f..a26a322 100644 --- a/gui/src-tauri/Cargo.lock +++ b/gui/src-tauri/Cargo.lock @@ -663,7 +663,7 @@ dependencies = [ [[package]] name = "diskghost-core" -version = "0.4.0" +version = "0.5.0" dependencies = [ "blake3", "fs4", @@ -677,7 +677,7 @@ dependencies = [ [[package]] name = "diskghost-gui" -version = "0.4.0" +version = "0.5.0" dependencies = [ "diskghost-core", "serde", @@ -4159,7 +4159,7 @@ dependencies = [ [[package]] name = "winapi-i686-pc-windows-gnu" -version = "0.4.0" +version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" @@ -4174,7 +4174,7 @@ dependencies = [ [[package]] name = "winapi-x86_64-pc-windows-gnu" -version = "0.4.0" +version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" diff --git a/gui/src-tauri/Cargo.toml b/gui/src-tauri/Cargo.toml index 344fc71..03e19f4 100644 --- a/gui/src-tauri/Cargo.toml +++ b/gui/src-tauri/Cargo.toml @@ -4,7 +4,7 @@ [package] name = "diskghost-gui" -version = "0.4.0" +version = "0.5.0" edition = "2021" license = "MIT" description = "Diskghost desktop GUI (Tauri) — modern disk usage & duplicate finder." diff --git a/gui/src-tauri/tauri.conf.json b/gui/src-tauri/tauri.conf.json index 26d5d25..4f8333f 100644 --- a/gui/src-tauri/tauri.conf.json +++ b/gui/src-tauri/tauri.conf.json @@ -1,7 +1,7 @@ { "$schema": "https://schema.tauri.app/config/2", "productName": "Diskghost", - "version": "0.4.0", + "version": "0.5.0", "identifier": "org.diskghost.app", "build": { "frontendDist": "../ui"