Skip to content

Commit e346776

Browse files
schenksjclaude
andcommitted
perf(contrib-delta) P7m: O(1) partition-value lookup in build_delta_partitioned_files
Perf-audit apache#137 finding #1. The inner `partition_schema.fields()` loop was calling `.iter().find()` on `task.partition_values` for every field -- O(width × values) per task. Pre-build a per-task HashMap<&str, &str> once, then O(1) gets. The map is reused across tasks via clear() so the allocation amortises across all DeltaScanTasks in the scan. Co-Authored-By: Claude Opus 4.7 (1M context) <[email protected]>
1 parent a805f81 commit e346776

1 file changed

Lines changed: 14 additions & 6 deletions

File tree

‎contrib/delta/native/src/planner.rs‎

Lines changed: 14 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -106,6 +106,12 @@ pub fn build_delta_partitioned_files(
106106
) -> Result<Vec<PartitionedFile>, String> {
107107
let parsed_tz = SessionTimezone::parse(session_tz);
108108
let mut files = Vec::with_capacity(tasks.len());
109+
// Reused scratch map for per-task partition-value lookup. Without it, the inner
110+
// `partition_schema.fields()` loop walks `task.partition_values` with `.iter().find()`
111+
// for every field -- O(width × values) per task. With it, build the map once per task
112+
// and do O(1) gets. `clear()` keeps the allocation across tasks.
113+
let mut partition_values_by_name: std::collections::HashMap<&str, &str> =
114+
std::collections::HashMap::new();
109115
for task in tasks {
110116
let url = Url::parse(task.file_path.as_ref())
111117
.map_err(|e| format!("Invalid Delta file URL: {e}"))?;
@@ -125,13 +131,15 @@ pub fn build_delta_partitioned_files(
125131

126132
let mut partition_values: Vec<ScalarValue> =
127133
Vec::with_capacity(partition_schema.fields().len());
134+
partition_values_by_name.clear();
135+
for pv in &task.partition_values {
136+
if let Some(v) = pv.value.as_deref() {
137+
partition_values_by_name.insert(pv.name.as_str(), v);
138+
}
139+
}
128140
for field in partition_schema.fields() {
129-
let proto_value = task
130-
.partition_values
131-
.iter()
132-
.find(|p| p.name == *field.name());
133-
let scalar = match proto_value.and_then(|p| p.value.clone()) {
134-
Some(s) => parse_delta_partition_scalar(&s, field.data_type(), &parsed_tz, session_tz)
141+
let scalar = match partition_values_by_name.get(field.name().as_str()).copied() {
142+
Some(s) => parse_delta_partition_scalar(s, field.data_type(), &parsed_tz, session_tz)
135143
.map_err(|e| {
136144
format!(
137145
"Failed to parse Delta partition value for column '{}': {e}",

0 commit comments

Comments
 (0)