src/infra/src/storage/mod.rs (1)

83-86: Add documentation and improve error handling

As this is a public API, consider these improvements:

1. Add rustdoc documentation with examples
2. Wrap errors with more context using .context()
3. Consider validating range values

Here's a suggested improvement:

+/// Retrieves a specific byte range from a file in storage.
+///
+/// # Arguments
+/// * `file` - The path to the file
+/// * `range` - The byte range to retrieve
+///
+/// # Examples
+/// ```
+/// let data = storage::get_range("path/to/file", 0..100).await?;
+/// ```
 pub async fn get_range(file: &str, range: Range<usize>) -> Result<bytes::Bytes, anyhow::Error> {
-    let data = DEFAULT.get_range(&file.into(), range).await?;
+    let data = DEFAULT
+        .get_range(&file.into(), range)
+        .await
+        .with_context(|| format!("Failed to get range from file: {}", file))?;
     Ok(data)
 }

src/service/search/cluster/flight.rs (1)

Line range hint 590-614: Consider optimizing node indexing in partition_file_by_hash.

The current implementation uses string-based HashMap lookups in a potentially hot path. Consider using integer IDs or a more efficient indexing mechanism to improve performance.

     let mut node_idx = HashMap::with_capacity(nodes.len());
     let mut idx = 0;
     for node in nodes {
         if !node.is_querier() {
             continue;
         }
-        node_idx.insert(&node.name, idx);
+        node_idx.insert(node.id, idx);  // Use node.id instead of node.name
         idx += 1;
     }
     let mut partitions = vec![Vec::new(); idx];
     for fk in file_id_list {
         let node_name =
             infra_cluster::get_node_from_consistent_hash(&fk.id.to_string(), &Role::Querier, group)
                 .await
                 .expect("there is no querier node in consistent hash ring");
-        let idx = match node_idx.get(&node_name) {
+        let node_id = nodes.iter()
+            .find(|n| n.name == node_name)
+            .map(|n| n.id)
+            .expect("node not found");
+        let idx = match node_idx.get(&node_id) {
             Some(idx) => *idx,
             None => {
                 log::error!(
                     "partition_file_by_hash: {} not found in node_idx",
                     node_name
                 );
                 0
             }
         };
         partitions[idx].push(fk.id);
     }

src/service/compact/merge.rs (1)

649-650: Avoid variable shadowing of new_file_size.

The variable new_file_size is redefined here after being previously used to accumulate file sizes before caching. This could lead to confusion or potential bugs. Consider renaming the variable to something like total_original_size for clarity.

Configuration used: CodeRabbit UI
Review profile: CHILL

#!/bin/bash
# Description: Search for any remaining references to `merge_parquet_files_by_datafusion` in the codebase.

# Test: Use ripgrep to find any occurrences. Expect: No matches found.
rg 'merge_parquet_files_by_datafusion'

#!/bin/bash
# Let's verify both the old and new function names to understand the changes better
# First, check for the old function definition and usage
ast-grep --pattern 'fn merge_parquet_files_by_datafusion'

# Then check for the new function definition and usage
ast-grep --pattern 'fn merge_parquet_files'

# Also search with ripgrep for any string occurrences including in comments and tests
rg -A 2 'merge_parquet_files'

deploy/build/buildspec-tag-arm64-pi.yml (1)

19-21: Consider improving feature management in build scripts.

The build script currently uses multiple sed commands to modify features. Consider creating a dedicated configuration file or using build arguments to manage feature flags more maintainably.

Example approach:

      # disable gxhash
-     - sed -i 's/default = \[\"gxhash\"\]/default = []/g' src/config/Cargo.toml
-     - sed -i 's/+aes,//g' .cargo/config.toml
+     # Create a features.sh script
+     - |
+       cat > scripts/configure_features.sh << 'EOF'
+       #!/bin/bash
+       # Disable specific features for ARM64 Pi builds
+       features_to_disable=(
+         "s/default = \[\"gxhash\"\]/default = []/g:src/config/Cargo.toml"
+         "s/+aes,//g:.cargo/config.toml"
+       )
+       for feature in "${features_to_disable[@]}"; do
+         IFS=: read -r pattern file <<< "$feature"
+         sed -i "$pattern" "$file"
+       done
+       EOF
+     - chmod +x scripts/configure_features.sh
+     - ./scripts/configure_features.sh

🧰 Tools

🪛 yamllint

[error] 21-21: trailing spaces

(trailing-spaces)

src/config/src/meta/inverted_index/search.rs (1)

Line range hint 111-129: Consider adding error handling for bitmap size mismatches.

While the changes effectively implement the streaming approach, consider adding explicit error handling for cases where bitmap sizes might not match expectations. This could prevent potential runtime issues.

 let bitmap = index_reader.get_bitmap(value).await?;
 
 // Resize if the res map is smaller than the bitmap
 if res.len() < bitmap.len() {
+    // Consider adding a size sanity check here
+    if bitmap.len() > column_index_meta.max_bitmap_size {
+        return Err(anyhow::anyhow!("Bitmap size exceeds maximum allowed size"));
+    }
     res.resize(bitmap.len(), false);
 }

src/config/src/meta/inverted_index/writer.rs (2)

58-58: Document the rationale for bitmap size increment.

The change from segment_id + 1 to segment_id + 64 appears to be a performance optimization to reduce bitmap resizing frequency. However, the choice of 64 as the increment value should be documented.

Consider adding a comment explaining why 64 was chosen as the growth factor:

-            bitmap.resize(segment_id + 64, false);
+            // Pre-allocate space for 64 additional segments to reduce resize operations
+            bitmap.resize(segment_id + 64, false);

---

94-100: Consider pre-allocating writer capacity.

The implementation correctly serializes and writes metadata. However, since we know we'll be writing both the metadata and its size, consider pre-allocating the required capacity to avoid potential reallocations.

         // write meta into writer buffer
+        writer.reserve(std::mem::size_of::<u32>() + meta_bytes.len());
         let meta_bytes = serde_json::to_vec(&self.meta)?;
         writer.write_all(&meta_bytes)?;
         let metas_size = meta_bytes.len() as u32;
         writer.write_all(&metas_size.to_le_bytes())?;
         writer.flush()?;

src/config/src/utils/parquet.rs (2)

121-139: Add documentation for the new streaming reader functions

The implementation looks good, but these public functions would benefit from documentation comments explaining:

• Their purpose in the streaming context
• The difference from their non-streaming counterparts
• Memory usage benefits

Add rustdoc comments like this:

/// Returns a streaming reader for parquet data from bytes, optimized for memory efficiency.
/// 
/// This function enables processing large parquet files without loading the entire dataset
/// into memory at once.
pub async fn get_recordbatch_reader_from_bytes(

---

141-154: Add performance considerations to documentation

The implementation correctly provides backward compatibility while enabling streaming capabilities. Consider adding documentation that helps users choose between streaming (get_recordbatch_reader_*) and batch collection (read_recordbatch_*) based on their memory constraints.

Add a note about memory usage:

/// Collects all record batches into memory. For large files, consider using
/// `get_recordbatch_reader_from_bytes` instead to process data in a streaming fashion.
pub async fn read_recordbatch_from_bytes(

src/cli/basic/cli.rs (2)

294-294: Fix grammatical error in success message.

The message should use past tense for better grammar.

-    println!("command {name} execute successfully");
+    println!("command {name} executed successfully");

---

Line range hint 266-270: Consider enhancing error handling consistency.

The error handling could be improved in two ways:

1. Add file existence validation before attempting deletion
2. Use consistent error message formatting across all commands (prefer using format! macro)

Example improvement:

match file_list::delete_parquet_file(file, true).await {
    Ok(_) => {
        println!("delete parquet file {} successfully", file);
    }
    Err(e) => {
        println!("{}", format!("delete parquet file {} failed: {}", file, e));
    }
}

src/job/files/idx.rs (1)

Line range hint 1-329: LGTM: Successful implementation of streaming parquet writing.

The changes effectively implement streaming parquet writing, which should significantly reduce memory usage during compaction. The code structure is clean and maintainable.

Consider adding metrics to monitor:

1. Memory usage during compaction
2. Time taken for index file generation
3. Success/failure rates of file uploads

This will help validate the performance improvements and identify potential bottlenecks.

src/service/dashboards/reports.rs (3)

382-382: Consider validating the email subject.

While the simplified subject assignment is good, consider adding validation to ensure the title is not empty before using it as the email subject. This would prevent emails with empty subjects which might affect deliverability or user experience.

-            .subject(self.title.to_string());
+            .subject(if self.title.is_empty() {
+                "Dashboard Report".to_string()
+            } else {
+                self.title.to_string()
+            });

---

Line range hint 516-516: Replace fixed sleep durations with proper wait conditions.

The current implementation uses fixed sleep durations which could be unreliable across different environments and load conditions. Consider using proper wait conditions or exponential backoff instead.

-    tokio::time::sleep(Duration::from_secs(5)).await;
+    // Wait for the dashboard to be fully loaded
+    page.wait_for_element("div#dashboard-container").await?;
+    page.wait_for_function("() => !document.querySelector('.loading-indicator')").await?;

-    tokio::time::sleep(Duration::from_secs(2)).await;
+    // Wait for organization switch to complete
+    page.wait_for_function("() => window.location.href.includes(arguments[0])", vec![org_id.into()]).await?;

Also applies to: 595-595

---

Line range hint 489-499: Improve browser automation error handling.

Consider implementing proper cleanup using Drop trait and providing more specific error messages during page navigation failures.

struct BrowserCleanup {
    browser: Browser,
    handle: tokio::task::JoinHandle<()>,
}

impl Drop for BrowserCleanup {
    fn drop(&mut self) {
        let browser = std::mem::replace(&mut self.browser, unsafe { std::mem::zeroed() });
        let handle = std::mem::replace(&mut self.handle, unsafe { std::mem::zeroed() });
        
        tokio::spawn(async move {
            if let Err(e) = browser.close().await {
                log::error!("Failed to close browser: {}", e);
            }
            if let Err(e) = handle.await {
                log::error!("Failed to join browser handler: {}", e);
            }
        });
    }
}

src/service/search/grpc/storage.rs (2)

Line range hint 519-541: Consider enhancing error context and optimizing string clone

1. The error handling at line 540 could provide more context about the task failure.
2. The trace_id string clone at line 519 could be avoided by using a reference.

Consider this improvement:

-        let trace_id = query.trace_id.to_string();
+        let trace_id = &query.trace_id;
         let permit = semaphore.clone().acquire_owned().await.unwrap();
         // Spawn a task for each file, wherein full text search and
         // secondary index search queries are executed
         let task = tokio::task::spawn(async move {

And enhance the error context:

-        .map_err(|e| Error::ErrorCode(ErrorCodes::ServerInternalError(e.to_string())))?;
+        .map_err(|e| Error::ErrorCode(ErrorCodes::ServerInternalError(
+            format!("Failed to join inverted index search tasks: {}", e)
+        )))?;

---

632-647: Consider improving error handling and search type constants

1. Field access error handling could be more descriptive.
2. Search type strings could be defined as constants.

Consider these improvements:

1. Enhance field access error handling:

-    if let Some(mut field_reader) = index_reader.field(INDEX_FIELD_NAME_FOR_ALL).await? {
+    let mut field_reader = index_reader.field(INDEX_FIELD_NAME_FOR_ALL).await?
+        .ok_or_else(|| anyhow::anyhow!("Field {} not found in index", INDEX_FIELD_NAME_FOR_ALL))?;

2. Define search type constants:

const SEARCH_TYPE_EXACT: &str = "eq";
const SEARCH_TYPE_CONTAINS: &str = "contains";
const SEARCH_TYPE_PREFIX: &str = "prefix";

Also applies to: 670-673

src/service/compact/merge.rs (5)

433-434: Consider optimizing the FileSize strategy implementation.

The FileSize strategy's early break could lead to suboptimal merging when files are sorted by size. Consider accumulating files until reaching the optimal chunk size instead of breaking early.

-if job_strategy == MergeStrategy::FileSize {
-    break;
-}
-new_file_size = 0;
-new_file_list.clear();
-continue; // this batch don't need to merge, skip
+// Continue accumulating files even if we exceed the size limit
+// This ensures we get closer to the optimal chunk size
+new_file_list.push(file.clone());
+new_file_size += file.meta.original_size;

Also applies to: 456-461

---

657-658: Consider using fold for efficient record counting.

The separate mapping operations for records and file size could be combined into a single fold operation for better performance.

-let total_records = new_file_list.iter().map(|f| f.meta.records).sum();
-let new_file_size = new_file_list.iter().map(|f| f.meta.original_size).sum();
+let (total_records, new_file_size) = new_file_list.iter().fold((0, 0), |(records, size), f| {
+    (records + f.meta.records, size + f.meta.original_size)
+});

Also applies to: 668-668

---

1195-1211: Add input validation for schema diff generation.

Consider validating input parameters and handling edge cases.

 fn generate_schema_diff(
     schema: &Schema,
     schema_latest_map: &HashMap<&String, &Arc<Field>>,
 ) -> Result<HashMap<String, DataType>, anyhow::Error> {
+    if schema.fields().is_empty() {
+        return Err(anyhow::anyhow!("Empty schema provided"));
+    }
+
     let mut diff_fields = HashMap::new();
     // ... rest of the implementation

---

798-868: Improve error handling in FST index generation.

Consider adding more detailed error context when FST generation fails.

-generate_fst_inverted_index(
+generate_fst_inverted_index(
     &new_file_key,
     &full_text_search_fields,
     &index_fields,
     Some(&retain_file_list),
     schema,
     &mut reader,
-)
-.await?;
+)
+.await
+.map_err(|e| anyhow::anyhow!("Failed to generate FST index for {}: {}", new_file_key, e))?;

---

Line range hint 1113-1129: Optimize RecordBatch concatenation.

Consider pre-allocating the capacity for the concatenated batch to improve performance.

 let new_batch = if inverted_idx_batches.len() == 1 {
     inverted_idx_batches.remove(0)
 } else {
     let new_schema = inverted_idx_batches.first().unwrap().schema();
+    // Pre-calculate total rows for better memory allocation
+    let total_rows = inverted_idx_batches.iter().map(|b| b.num_rows()).sum();
+    // TODO: Add capacity hint to concat_batches function
     concat_batches(new_schema, inverted_idx_batches).map_err(anyhow::Error::from)?
 };

src/job/files/parquet.rs (3)

696-705: Consider enhancing error handling in merge operation.

The error handling could be improved by adding context information to the error chain. This would help in debugging issues when the merge operation fails.

-    let merge_result = merge_parquet_files_by_datafusion(
+    let merge_result = merge_parquet_files_by_datafusion(
         stream_type,
         &stream_name,
         schema,
         tables,
         &bloom_filter_fields,
         &new_file_meta,
-    ).await;
+    ).await.with_context(|| format!(
+        "Failed to merge parquet files for stream: {}/{}", 
+        stream_type, stream_name
+    ));

---

1147-1177: Consider optimizing term processing for better performance.

The current implementation processes terms sequentially. Consider using parallel processing for large datasets:

-            let terms = (0..num_rows)
-                .flat_map(|i| {
-                    split_token(column_data.value(i), &cfg.common.inverted_index_split_chars)
-                        .into_iter()
-                        .map(|s| (s, i))
-                        .collect::<Vec<_>>()
-                })
-                .collect::<Vec<_>>();
+            let terms = (0..num_rows)
+                .into_par_iter()
+                .flat_map(|i| {
+                    split_token(column_data.value(i), &cfg.common.inverted_index_split_chars)
+                        .into_par_iter()
+                        .map(|s| (s, i))
+                        .collect::<Vec<_>>()
+                })
+                .collect::<Vec<_>>();

This would require adding rayon as a dependency for parallel iteration support.

---

1494-1506: Consider using a pre-allocated buffer for better memory efficiency.

The current implementation creates new buffers for each column. Consider pre-allocating a reusable buffer:

+    let mut reusable_buf = Vec::with_capacity(1024 * 1024); // Pre-allocate 1MB
     for (column_name, indexer) in indexers {
         if indexer.is_empty() {
             continue;
         }
-        let mut buf = Vec::new();
+        reusable_buf.clear(); // Reuse the buffer
         let _index_meta = indexer.write(&mut buf).context(format!(
             "Error constructing FST ColumnIndex for field {}",
             column_name
         ))?;
         original_size += buf.len();
-        puffin_writer.add_blob(column_name, buf)?;
+        puffin_writer.add_blob(column_name, reusable_buf.clone())?;
     }

src/handler/http/request/search/mod.rs (1)

150-169: Consider adding unit tests for the parameter setting logic.

The new parameter handling logic would benefit from unit tests to verify:

1. Parameters are not overwritten when already set
2. Error handling works as expected
3. Empty/None cases are handled correctly

Would you like me to help create unit tests for these scenarios?

src/config/src/config.rs (1)

1590-1590: Improve error message clarity.

The error message could be more descriptive by including:

1. The current invalid value
2. The purpose of this configuration

Consider updating the error message to be more informative:

-            "ZO_INVERTED_INDEX_STORE_FORMAT must be one of both, parquet."
+            "Invalid ZO_INVERTED_INDEX_STORE_FORMAT value. This setting controls how inverted indices are stored. Allowed values are: 'both' or 'parquet' (default)."

src/config/src/meta/puffin/writer.rs (2)

171-174: Typographical error: variable file_metdadata should be file_metadata

There is a typo in the variable name file_metdadata. Correcting it to file_metadata will improve code readability and prevent potential confusion.

Apply this diff to correct the typo:

     fn get_payload(&mut self) -> Result<Vec<u8>> {
-        let file_metdadata = PuffinMeta {
+        let file_metadata = PuffinMeta {
             blobs: mem::take(&mut self.blob_metadata),
             properties: mem::take(&mut self.file_properties),
         };
-        serde_json::to_vec(&file_metdadata).context("Error serializing puffin metadata")
+        serde_json::to_vec(&file_metadata).context("Error serializing puffin metadata")
     }

---

171-174: Inconsistent naming of blob metadata fields across structs

There is inconsistency in the naming of blob metadata fields:

• In PuffinBytesWriter, the field is blobs_metadata.
• In PuffinFooterWriter, it's blob_metadata.
• In PuffinMeta, it's blobs.

For consistency and improved readability, consider standardizing the field names across these structs.

src/common/utils/stream.rs (1)

92-101: Optimize min and max computation using Arrow compute functions

Manually iterating over each value to find min and max can be inefficient. Utilize Arrow's compute::min and compute::max functions for better performance.

Apply this diff to use compute functions:

+        use arrow::compute::{min as compute_min, max as compute_max};
         // ...
-        for i in 0..num_row {
-            if min_col.is_valid(i) {
-                let val = min_col.value(i);
-                if val < min_val {
-                    min_val = val;
-                }
-            }
-            if max_col.is_valid(i) {
-                let val = max_col.value(i);
-                if val > max_val {
-                    max_val = val;
-                }
-            }
-        }
+        if let Some(batch_min) = compute_min(min_col)? {
+            if batch_min < min_val {
+                min_val = batch_min;
+            }
+        }
+        if let Some(batch_max) = compute_max(max_col)? {
+            if batch_max > max_val {
+                max_val = batch_max;
+            }
+        }

src/config/src/meta/puffin/reader.rs (1)

222-227: Validate blob offsets without assuming continuity.

In the validate_payload function, the code assumes that each blob.offset is equal to the cumulative offset, which may not hold if blobs are not stored sequentially without gaps. This could lead to false positives when validating blob offsets. Consider verifying that each blob's offset and length are within the valid range without assuming continuity.

Consider modifying the validation logic as follows:

             for blob in &puffin_metadata.blobs {
-                ensure!(
-                    blob.offset == offset,
-                    anyhow!("Blob payload offset mismatch")
-                );
-                offset += blob.length;
+                ensure!(
+                    blob.offset + blob.length <= self.head_magic_offset(),
+                    anyhow!("Blob payload exceeds file boundaries")
+                );
             }

src/config/src/meta/search.rs (1)

Line range hint 711-743: Consider using the strum crate to derive Display and FromStr implementations

The manual implementations of the Display and FromStr traits for SearchEventType can be simplified by leveraging the strum crate's Display and EnumString derives. This reduces boilerplate code and improves maintainability.

Apply this diff to implement the change:

+use strum_macros::{Display, EnumString};

 #[derive(Hash, Clone, Copy, Debug, Eq, PartialEq, Serialize, Deserialize, ToSchema)]
-#[serde(rename_all = "lowercase")]
+#[derive(Display, EnumString)]
+#[strum(serialize_all = "lowercase")]
 pub enum SearchEventType {
     UI,
     Dashboards,
     Reports,
     Alerts,
-    Values,
+    #[strum(serialize = "_values", serialize = "values")]
     Values,
     Other,
     RUM,
-    DerivedStream,
+    #[strum(serialize = "derived_stream", serialize = "derivedstream")]
+    DerivedStream,
 }
 
-impl std::fmt::Display for SearchEventType {
-    fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result {
-        match self {
-            SearchEventType::UI => write!(f, "ui"),
-            SearchEventType::Dashboards => write!(f, "dashboards"),
-            SearchEventType::Reports => write!(f, "reports"),
-            SearchEventType::Alerts => write!(f, "alerts"),
-            SearchEventType::Values => write!(f, "_values"),
-            SearchEventType::Other => write!(f, "other"),
-            SearchEventType::RUM => write!(f, "rum"),
-            SearchEventType::DerivedStream => write!(f, "derived_stream"),
-        }
-    }
-}
-
-impl FromStr for SearchEventType {
-    type Err = String;
-    fn from_str(s: &str) -> std::result::Result<Self, Self::Err> {
-        let s = s.to_lowercase();
-        match s.as_str() {
-            "ui" => Ok(SearchEventType::UI),
-            "dashboards" => Ok(SearchEventType::Dashboards),
-            "reports" => Ok(SearchEventType::Reports),
-            "alerts" => Ok(SearchEventType::Alerts),
-            "values" | "_values" => Ok(SearchEventType::Values),
-            "other" => Ok(SearchEventType::Other),
-            "rum" => Ok(SearchEventType::RUM),
-            "derived_stream" | "derivedstream" => Ok(SearchEventType::DerivedStream),
-            _ => Err(format!("Invalid search event type: {s}")),
-        }
-    }
-}

src/infra/src/file_list/postgres.rs (2)

Line range hint 82-84: Implement or remove the batch_add_with_id method

The batch_add_with_id method is currently unimplemented. If this method is intended for future use, please implement it. Otherwise, consider removing it to maintain code clarity and avoid potential confusion.

---

1520-1520: Adjust log level for successful operation

The log message indicates a successful operation but uses log::warn!. Consider changing it to log::info! to reflect the successful nature of the action.

Apply this diff to adjust the log level:

-            log::warn!("[POSTGRES] create table index(file_list_stream_file_idx) successfully");
+            log::info!("[POSTGRES] create table index(file_list_stream_file_idx) successfully");

Configuration used: CodeRabbit UI
Review profile: CHILL

#!/bin/bash
# Description: Check for any cryptographic implementations that might be affected
# by the removal of AES CPU feature

# Look for potential crypto/AES usage in the codebase
rg -l 'aes|crypto|cipher' --type rust

# Check if there are any relevant configuration or feature flags in Cargo.toml files
fd Cargo.toml --exec grep -l 'aes\|crypto\|cipher' {}

#!/bin/bash
# Let's try a broader search to understand the context better

# 1. First, let's see what's in the .cargo/config.toml to understand the AES config
fd -g '.cargo/config.toml' --exec cat {}

# 2. Check for any dependencies in Cargo.toml that might use crypto features
fd Cargo.toml --exec cat {}

# 3. Look for any encryption-related code with broader terms
rg -l 'encrypt|decrypt|cipher|hash|crypto' --type-not lock

#!/bin/bash
# Verify that all calls to ExactSearch::search have been updated
rg -p "ExactSearch.*?search" --type rust

#!/bin/bash
# Verify that all calls to PrefixSearch::search have been updated
rg -p "PrefixSearch.*?search" --type rust

#!/bin/bash
# Verify that all calls to SubstringSearch::search have been updated
rg -p "SubstringSearch.*?search" --type rust

#!/bin/bash
# Let's try a broader search to find all usages of search method in the codebase
rg -p "search.*?FieldReader" --type rust -A 3 -B 3

# Also search for SubstringSearch struct definition and implementations
ast-grep --pattern 'impl $_ for SubstringSearch'

# And direct usage of SubstringSearch
rg "SubstringSearch" --type rust -A 3 -B 3

#!/bin/bash
# Search for other usages of index_size to understand the impact
rg -p "index_size" --type rust

#!/bin/bash
# Get the full implementation context of the writer
ast-grep --pattern 'impl$_ InvertedIndexWriter {
  $$$
}'

#!/bin/bash
# Get more context around the index_size usage in writer.rs
rg -p -B 10 -A 10 "index_size" src/config/src/meta/inverted_index/writer.rs

#!/bin/bash
# Look for validation in IndexReader implementation
ast-grep --pattern 'impl IndexReader {
  $$$
  fn new($$$) {
    $$$
  }
  $$$
}'

#!/bin/bash
# Search for any remaining anyhow error handling patterns
rg -A 2 'anyhow!|ensure!' src/config/src/utils/

#!/bin/bash
# Let's check the error handling patterns in the inverted_index.rs file
rg -A 5 'Result<' src/config/src/utils/inverted_index.rs

# Also check for any error propagation using '?'
rg -A 2 '\?' src/config/src/utils/inverted_index.rs

# Look for the implementation of create_index_reader_from_puffin_bytes
ast-grep --pattern 'fn create_index_reader_from_puffin_bytes($$$) -> Result<$$$> {
  $$$
}'

#!/bin/bash
# Let's look at the complete implementation of the file to understand the error handling
cat src/config/src/utils/inverted_index.rs

# Also check for any other files that might be using this module to ensure error handling is consistent
rg -l "use.*inverted_index" src/config/src/

-        populate_file_meta(&[&batch], &mut file_meta, Some("min_ts"), Some("max_ts")).await?;
+        if let Err(e) = populate_file_meta(&[&batch], &mut file_meta, Some("min_ts"), Some("max_ts")).await {
+            log::warn!("[JOB:IDX] Failed to populate timestamp metadata: {}", e);
+            // Continue without timestamp metadata
+        }

#!/bin/bash
# Description: Search for negative assignments or comparisons involving the affected fields.

# Search for negative assignments
rg --type rust --pcre2 '(fields|snapshot_id|sequence_number|offset|length)\s*=\s*-\d+'

# Search for negative comparisons
rg --type rust --pcre2 '(fields|snapshot_id|sequence_number|offset|length)\s*[<>=!]=\s*-'

#!/bin/bash
# Let's perform a more comprehensive analysis:

# 1. Find all usages of these fields in the codebase
rg -A 2 -B 2 '\b(fields|snapshot_id|sequence_number|offset|length)\b' --type rust

# 2. Look for type declarations or casts involving these fields
ast-grep --pattern 'let $name: i32 = $value' --lang rust
ast-grep --pattern 'let $name: i64 = $value' --lang rust

# 3. Look for arithmetic operations that might assume signed integers
rg '\b(fields|snapshot_id|sequence_number|offset|length)\s*[-+*/]\s*' --type rust

# 4. Check for any test files that might reveal expected behavior
fd -e rs test -e spec -e tests --exec grep -l 'fields\|snapshot_id\|sequence_number\|offset\|length' {}

buf.extend_from_slice(&PuffinFooterFlags::DEFAULT.bits().to_le_bytes());