src/infra/src/file_list/mod.rs (1)

Line range hint 436-459: Consider improving the garbage collection implementation.

A few suggestions to enhance the robustness of the GC implementation:

1. Consider making the interval configurable rather than hardcoding 3600 seconds
2. Add a mechanism to gracefully shutdown the GC task
3. Consider retrying on errors with backoff instead of just logging them

Here's a suggested implementation:

 pub async fn local_cache_gc() -> Result<()> {
+    const DEFAULT_GC_INTERVAL_SECS: u64 = 3600;
+    let gc_interval = std::env::var("LOCAL_CACHE_GC_INTERVAL_SECS")
+        .ok()
+        .and_then(|s| s.parse().ok())
+        .unwrap_or(DEFAULT_GC_INTERVAL_SECS);
+
+    let handle = tokio::task::spawn(async move {
         let cfg = config::get_config();
         if cfg.common.local_mode || !cfg.common.meta_store_external {
             return;
         }
 
-        // gc every hour
-        let mut interval = tokio::time::interval(tokio::time::Duration::from_secs(3600));
+        let mut interval = tokio::time::interval(tokio::time::Duration::from_secs(gc_interval));
         interval.tick().await; // the first tick is immediate
+
+        let mut backoff = tokio::time::Duration::from_secs(1);
         loop {
             if let Ok(min_id) = get_min_pk_value().await {
                 if min_id > 0 {
                     match LOCAL_CACHE.clean_by_min_pk_value(min_id).await {
-                        Ok(_) => log::info!("[file_list] local cache gc done"),
-                        Err(e) => log::error!("[file_list] local cache gc failed: {}", e),
+                        Ok(_) => {
+                            log::info!("[file_list] local cache gc done");
+                            backoff = tokio::time::Duration::from_secs(1);
+                        }
+                        Err(e) => {
+                            log::error!("[file_list] local cache gc failed: {}", e);
+                            tokio::time::sleep(backoff).await;
+                            backoff = std::cmp::min(backoff * 2, tokio::time::Duration::from_secs(60));
+                        }
                     }
                 }
             }
             interval.tick().await;
         }
-    });
+    });
+
+    // Store the handle somewhere for cleanup during shutdown
+    Ok(())
 }

src/config/src/config.rs (2)

Line range hint 1349-1500: Consider adding documentation for file list configurations

The file list configurations (file_list_id_batch_size and file_list_multi_thread) would benefit from documentation explaining:

• The impact of batch size on memory usage and performance
• When to enable/disable multi-threading
• Recommended values for different scenarios

---

Line range hint 1349-1500: Add memory-based validation for file list batch size

Consider adding validation in check_memory_config to ensure file_list_id_batch_size doesn't exceed a reasonable percentage of available memory. This would prevent potential memory issues in resource-constrained environments.

Example validation to add in check_memory_config:

 fn check_memory_config(cfg: &mut Config) -> Result<(), anyhow::Error> {
     let mem_total = cgroup::get_memory_limit();
     cfg.limit.mem_total = mem_total;
+    
+    // Validate file_list_id_batch_size
+    let max_batch_size = mem_total / (1024 * 1024 * 10); // Max 10% of memory
+    if cfg.limit.file_list_id_batch_size > max_batch_size {
+        cfg.limit.file_list_id_batch_size = max_batch_size;
+        log::warn!("Reducing file_list_id_batch_size to {} to prevent memory issues", max_batch_size);
+    }

Configuration used: CodeRabbit UI
Review profile: CHILL

#!/bin/bash
# Search for dotenv_config usage patterns that might need updates
rg -A 5 'dotenv_config::'

#!/bin/bash
# Let's check the actual usage of EnvConfig to understand potential breaking changes
ast-grep --pattern 'use dotenv_config::EnvConfig;
$$$'

#!/bin/bash
# Let's check the actual implementation in config.rs to understand the usage
cat src/config/src/config.rs