measure disk size of directories if available (#7120)

generall · xzfc · web-flow · commit 5235d8643552 · 2025-08-25T00:17:27.000+02:00
* measure disk size of directories if available

* review suggestions

* Update comment

---------

Co-authored-by: xzfc &lt;xzfcpw@gmail.com&gt;
diff --git a/lib/collection/src/collection_manager/optimizers/segment_optimizer.rs b/lib/collection/src/collection_manager/optimizers/segment_optimizer.rs
@@ -6,7 +6,7 @@ use std::sync::atomic::AtomicBool;
 
 use common::budget::{ResourceBudget, ResourcePermit};
 use common::counter::hardware_counter::HardwareCounterCell;
-use common::disk::dir_size;
+use common::disk::dir_disk_size;
 use io::storage_version::StorageVersion;
 use itertools::Itertools;
 use parking_lot::lock_api::RwLockWriteGuard;
@@ -117,7 +117,7 @@ pub trait SegmentOptimizer {
             let locked_segment = segment.read();
 
             space_occupied =
-                space_occupied.and_then(|acc| match dir_size(locked_segment.data_path()) {
+                space_occupied.and_then(|acc| match dir_disk_size(locked_segment.data_path()) {
                     Ok(size) => Some(size + acc),
                     Err(err) => {
                         log::debug!(
@@ -157,10 +157,13 @@ pub trait SegmentOptimizer {
 
         match (space_available, space_needed) {
             (Some(space_available), Some(space_needed)) => {
+                log::debug!(
+                    "Available space: {space_available}, needed for optimization: {space_needed}",
+                );
                 if space_available < space_needed {
-                    return Err(CollectionError::service_error(
-                        "Not enough space available for optimization".to_string(),
-                    ));
+                    return Err(CollectionError::service_error(format!(
+                        "Not enough space available for optimization, needed: {space_needed}, available: {space_available}"
+                    )));
                 }
             }
             _ => {
diff --git a/lib/common/common/src/disk.rs b/lib/common/common/src/disk.rs
@@ -2,20 +2,34 @@ use std::path::{Path, PathBuf};
 
 use walkdir::WalkDir;
 
-/// How many bytes a directory takes.
-pub fn dir_size(path: impl Into<PathBuf>) -> std::io::Result<u64> {
-    fn dir_size(mut dir: std::fs::ReadDir) -> std::io::Result<u64> {
+/// How many bytes a directory takes on disk.
+///
+/// Note: on non-unix systems, this function returns the apparent/logical
+/// directory size rather than actual disk usage.
+pub fn dir_disk_size(path: impl Into<PathBuf>) -> std::io::Result<u64> {
+    fn dir_disk_size(mut dir: std::fs::ReadDir) -> std::io::Result<u64> {
         dir.try_fold(0, |acc, file| {
             let file = file?;
-            let size = match file.metadata()? {
-                data if data.is_dir() => dir_size(std::fs::read_dir(file.path())?)?,
-                data => data.len(),
+            let metadata = file.metadata()?;
+            let size = if metadata.is_dir() {
+                dir_disk_size(std::fs::read_dir(file.path())?)?
+            } else {
+                #[cfg(unix)]
+                {
+                    const BLOCK_SIZE: u64 = 512; // aka DEV_BSIZE
+                    use std::os::unix::fs::MetadataExt;
+                    metadata.blocks() * BLOCK_SIZE
+                }
+                #[cfg(not(unix))]
+                {
+                    metadata.len()
+                }
             };
             Ok(acc + size)
         })
     }
 
-    dir_size(std::fs::read_dir(path.into())?)
+    dir_disk_size(std::fs::read_dir(path.into())?)
 }
 
 /// List all files in the given directory recursively.
