diff --git a/Cargo.lock b/Cargo.lock index 0d96351..43bc27d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -358,7 +358,7 @@ checksum = "460fbee9c2c2f33933d720630a6a0bac33ba7053db5344fac858d4b8952d77d5" [[package]] name = "diskghost-cli" -version = "0.6.0" +version = "0.6.1" dependencies = [ "clap", "diskghost-core", @@ -367,7 +367,7 @@ dependencies = [ [[package]] name = "diskghost-core" -version = "0.6.0" +version = "0.6.1" dependencies = [ "blake3", "criterion", diff --git a/Cargo.toml b/Cargo.toml index f23d811..a3bbe82 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -4,7 +4,7 @@ exclude = ["gui"] resolver = "2" [workspace.package] -version = "0.6.0" +version = "0.6.1" edition = "2021" license = "MIT" authors = ["kingchenc"] diff --git a/crates/diskghost-core/src/lib.rs b/crates/diskghost-core/src/lib.rs index 13277b9..dc3426b 100644 --- a/crates/diskghost-core/src/lib.rs +++ b/crates/diskghost-core/src/lib.rs @@ -64,7 +64,15 @@ impl Progress { #[derive(Debug, Clone, Serialize)] pub struct FileEntry { pub path: PathBuf, + /// Allocated size on disk — what the file actually occupies. This is what + /// disk-usage scanning reports, so OneDrive/dehydrated placeholders and + /// sparse files count their real footprint (often ~0), not their logical + /// length. See [`on_disk_size`]. pub size: u64, + /// Logical (content) length from the file metadata. Used by duplicate + /// detection, where byte-identical files must group by content size + /// regardless of how they are allocated on disk. + pub logical_size: u64, } /// Aggregated size of a directory. @@ -194,6 +202,80 @@ fn is_excluded(glob: &globset::GlobSet, path: &Path) -> bool { path.components().any(|c| glob.is_match(c.as_os_str())) } +/// Bytes a file actually occupies on disk, as opposed to its logical length +/// ([`std::fs::Metadata::len`]). The two differ for sparse files, filesystem +/// compression, and — the case that matters most here — cloud placeholders +/// (OneDrive "online-only" files) that advertise their full cloud size in the +/// metadata but store ~0 bytes locally. A disk-usage tool must count the real +/// footprint, or a folder full of placeholders is reported wildly too large. +#[cfg(windows)] +fn on_disk_size(meta: &std::fs::Metadata, path: &Path) -> u64 { + use std::os::windows::fs::MetadataExt; + // Only these attributes can make on-disk size diverge from the logical + // length. For the overwhelmingly common normal file we skip the extra + // syscall and use the length we already have, keeping the scan hot path + // exactly as cheap as before. + const SPARSE: u32 = 0x0000_0200; + const REPARSE_POINT: u32 = 0x0000_0400; + const COMPRESSED: u32 = 0x0000_0800; + const OFFLINE: u32 = 0x0000_1000; + const RECALL_ON_OPEN: u32 = 0x0004_0000; + const RECALL_ON_DATA_ACCESS: u32 = 0x0040_0000; + const WEIRD: u32 = + SPARSE | REPARSE_POINT | COMPRESSED | OFFLINE | RECALL_ON_OPEN | RECALL_ON_DATA_ACCESS; + + if meta.file_attributes() & WEIRD != 0 { + if let Some(actual) = compressed_size(path) { + return actual; + } + } + meta.file_size() +} + +/// Actual allocated size via `GetCompressedFileSizeW`. Returns `None` on error +/// so the caller can fall back to the logical length. +#[cfg(windows)] +fn compressed_size(path: &Path) -> Option { + use std::os::windows::ffi::OsStrExt; + #[link(name = "kernel32")] + extern "system" { + fn GetCompressedFileSizeW(lp_file_name: *const u16, lp_file_size_high: *mut u32) -> u32; + fn GetLastError() -> u32; + } + let wide: Vec = path + .as_os_str() + .encode_wide() + .chain(std::iter::once(0)) + .collect(); + let mut high: u32 = 0; + // SAFETY: `wide` is a valid NUL-terminated UTF-16 string kept alive for the + // call, and `high` is a valid out-pointer to a u32. + let low = unsafe { GetCompressedFileSizeW(wide.as_ptr(), &mut high) }; + if low == u32::MAX { + // INVALID_FILE_SIZE is only an error when GetLastError != NO_ERROR; + // otherwise it is a legitimate low dword equal to 0xFFFFFFFF. + // SAFETY: no preconditions. + if unsafe { GetLastError() } != 0 { + return None; + } + } + Some(((high as u64) << 32) | low as u64) +} + +/// Bytes a file actually occupies on disk. On Unix `st_blocks` (512-byte units) +/// reflects the real allocation, so sparse files and holes count correctly. +#[cfg(unix)] +fn on_disk_size(meta: &std::fs::Metadata, _path: &Path) -> u64 { + use std::os::unix::fs::MetadataExt; + meta.blocks().saturating_mul(512) +} + +/// Fallback for platforms without an allocation query: use the logical length. +#[cfg(not(any(windows, unix)))] +fn on_disk_size(meta: &std::fs::Metadata, _path: &Path) -> u64 { + meta.len() +} + /// Walk `root` recursively in a single pass: collect files (with size), count /// directories, and count unreadable entries, honouring [`Options`]. fn walk(root: &Path, opts: &Options, progress: &Progress) -> Walk { @@ -204,7 +286,7 @@ fn walk(root: &Path, opts: &Options, progress: &Progress) -> Walk { // instead of one-at-a-time in the serial consumer below. This is the // scan's hot path: statting hundreds of thousands of files serially was // the whole cost; jwalk already walks in parallel, so we stat there too. - let mut wd = jwalk::WalkDirGeneric::<((), Option)>::new(root) + let mut wd = jwalk::WalkDirGeneric::<((), Option<(u64, u64)>)>::new(root) .skip_hidden(false) .follow_links(opts.follow_symlinks); if let Some(d) = opts.max_depth { @@ -220,10 +302,14 @@ fn walk(root: &Path, opts: &Options, progress: &Progress) -> Walk { Err(_) => true, }); } - // Stat every file here (on the walk's rayon worker), caching its size. + // Stat every file here (on the walk's rayon worker), caching both its + // logical length and its real on-disk size as (logical, on_disk). for e in children.iter_mut().flatten() { if e.file_type().is_file() { - e.client_state = std::fs::metadata(e.path()).ok().map(|m| m.len()); + let p = e.path(); + e.client_state = std::fs::metadata(&p) + .ok() + .map(|m| (m.len(), on_disk_size(&m, &p))); } } }); @@ -245,12 +331,14 @@ fn walk(root: &Path, opts: &Options, progress: &Progress) -> Walk { } } else if ft.is_file() { match e.client_state { - Some(size) => { + Some((logical, on_disk)) => { progress.files.fetch_add(1, Ordering::Relaxed); - progress.bytes.fetch_add(size, Ordering::Relaxed); + // Progress and totals track real on-disk bytes. + progress.bytes.fetch_add(on_disk, Ordering::Relaxed); files.push(FileEntry { path: e.path(), - size, + size: on_disk, + logical_size: logical, }); } None => skipped += 1, @@ -376,15 +464,17 @@ pub fn find_duplicates_with_progress( opts: &Options, progress: &Progress, ) -> Vec { + // Duplicate detection groups by *logical* (content) size — two byte-identical + // files are duplicates regardless of how they happen to be allocated on disk. let files: Vec = walk(root, opts, progress) .files .into_iter() - .filter(|f| f.size >= min_size && f.size > 0) + .filter(|f| f.logical_size >= min_size && f.logical_size > 0) .collect(); let mut by_size: HashMap> = HashMap::new(); for f in files { - by_size.entry(f.size).or_default().push(f.path); + by_size.entry(f.logical_size).or_default().push(f.path); } let candidates: Vec<(u64, Vec)> = @@ -539,17 +629,18 @@ pub fn remove_path( if e.file_type().is_dir() { dirs.push(e.path()); } else { + let p = e.path(); if let Ok(m) = e.metadata() { - bytes = bytes.saturating_add(m.len()); + bytes = bytes.saturating_add(on_disk_size(&m, &p)); } - files.push(e.path()); + files.push(p); } } } } else { // A single file (or a symlink): remove just that entry. if let Ok(m) = &top { - bytes = m.len(); + bytes = on_disk_size(m, path); } files.push(path.to_path_buf()); } @@ -728,13 +819,22 @@ mod tests { write(&d.join("a/2.bin"), &[0u8; 500]); write(&d.join("b/3.bin"), &[0u8; 200]); let r = scan(&d, 10); - assert_eq!(r.total_size, 1700); + // Totals are on-disk (allocated) bytes, which round up to the block size + // on filesystems like ext4, so assert a lower bound on the logical sum. + assert!(r.total_size >= 1700, "total_size {} < 1700", r.total_size); assert_eq!(r.total_files, 3); assert_eq!( r.children[0].path.file_name().unwrap().to_str().unwrap(), "a" ); - assert_eq!(r.children[0].size, 1500); + assert!( + r.children[0].size >= 1500, + "children[0].size {} < 1500", + r.children[0].size + ); + // Logical sizes remain exact regardless of allocation. + let logical: u64 = walk_files(&d).iter().map(|f| f.logical_size).sum(); + assert_eq!(logical, 1700); std::fs::remove_dir_all(&d).ok(); } @@ -749,9 +849,13 @@ mod tests { .iter() .all(|c| c.path.file_name().unwrap() != "loose.bin")); assert_eq!(r.root_files_count, 1); - assert_eq!(r.root_files_size, 777); + assert!( + r.root_files_size >= 777, + "root_files_size {} < 777", + r.root_files_size + ); assert_eq!(r.children.len(), 1); - assert_eq!(r.total_size, 1077); + assert!(r.total_size >= 1077, "total_size {} < 1077", r.total_size); std::fs::remove_dir_all(&d).ok(); } diff --git a/gui/src-tauri/Cargo.lock b/gui/src-tauri/Cargo.lock index 05c2e03..187a936 100644 --- a/gui/src-tauri/Cargo.lock +++ b/gui/src-tauri/Cargo.lock @@ -664,7 +664,7 @@ dependencies = [ [[package]] name = "diskghost-core" -version = "0.6.0" +version = "0.6.1" dependencies = [ "blake3", "fs4", @@ -678,7 +678,7 @@ dependencies = [ [[package]] name = "diskghost-gui" -version = "0.6.0" +version = "0.6.1" dependencies = [ "diskghost-core", "serde", diff --git a/gui/src-tauri/Cargo.toml b/gui/src-tauri/Cargo.toml index 80a01c1..6a6c9a9 100644 --- a/gui/src-tauri/Cargo.toml +++ b/gui/src-tauri/Cargo.toml @@ -4,7 +4,7 @@ [package] name = "diskghost-gui" -version = "0.6.0" +version = "0.6.1" edition = "2021" license = "MIT" description = "Diskghost desktop GUI (Tauri) — modern disk usage & duplicate finder." diff --git a/gui/src-tauri/tauri.conf.json b/gui/src-tauri/tauri.conf.json index 43535f7..2b748c7 100644 --- a/gui/src-tauri/tauri.conf.json +++ b/gui/src-tauri/tauri.conf.json @@ -1,7 +1,7 @@ { "$schema": "https://schema.tauri.app/config/2", "productName": "Diskghost", - "version": "0.6.0", + "version": "0.6.1", "identifier": "org.diskghost.app", "build": { "frontendDist": "../ui"