Skip to content

Commit adad560

Browse files
committed
Only allocate coerced fields once something actually changes
`coerce_fields_by_name` rebuilt every file field into a new `Fields` even when no coercion applied, which cost an allocation and a reference count bump per field for schemas that need no coercion at all. Build the output lazily instead: the fields before the first change are copied over when that change happens, and schemas that need no coercion are walked without allocating or touching the file fields' reference counts. The same helper is now used for map entries, so the laziness applies at every nesting level rather than only at the top as the previous `needs_*` pre-scan did. Co-Authored-By: Claude Opus 5 <[email protected]> Claude-Session: https://claude.ai/code/session_01PL624RiDwff7RVJ2BKSvS5
1 parent 2a6cbd0 commit adad560

1 file changed

Lines changed: 39 additions & 31 deletions

File tree

‎datafusion/datasource-parquet/src/schema_coercion.rs‎

Lines changed: 39 additions & 31 deletions
Original file line numberDiff line numberDiff line change
@@ -85,25 +85,44 @@ fn coerce_fields_by_name(table_fields: &Fields, file_fields: &Fields) -> Option<
8585
.map(|f| (f.name(), f.data_type()))
8686
.collect();
8787

88-
let mut changed = false;
89-
let fields: Fields = file_fields
90-
.iter()
91-
.map(|field| {
92-
match table_types
93-
.get(field.name())
94-
.and_then(|table_type| coerce_data_type(table_type, field.data_type()))
95-
{
96-
Some(new_type) => {
97-
changed = true;
98-
field_with_new_type(field, new_type)
88+
coerce_fields(file_fields, |_, field| {
89+
let table_type = table_types.get(field.name())?;
90+
coerce_data_type(table_type, field.data_type())
91+
.map(|new_type| field_with_new_type(field, new_type))
92+
})
93+
}
94+
95+
/// Rebuild `file_fields`, replacing every field for which `coerce` returns a
96+
/// new one. Returns `None` if no field changed.
97+
///
98+
/// The output is only allocated once a field actually changes, so schemas
99+
/// needing no coercion at all (the common case) are walked without allocating
100+
/// or touching the reference counts of the file fields.
101+
fn coerce_fields(
102+
file_fields: &Fields,
103+
mut coerce: impl FnMut(usize, &FieldRef) -> Option<FieldRef>,
104+
) -> Option<Fields> {
105+
let mut coerced: Option<Vec<FieldRef>> = None;
106+
for (idx, field) in file_fields.iter().enumerate() {
107+
match coerce(idx, field) {
108+
Some(new_field) => coerced
109+
.get_or_insert_with(|| {
110+
// The fields before the first change are carried over as is
111+
let mut fields = Vec::with_capacity(file_fields.len());
112+
fields.extend_from_slice(&file_fields[..idx]);
113+
fields
114+
})
115+
.push(new_field),
116+
// Unchanged fields are only copied once something else changed
117+
None => {
118+
if let Some(coerced) = &mut coerced {
119+
coerced.push(Arc::clone(field));
99120
}
100-
// If no transformation is needed, keep the original field
101-
None => Arc::clone(field),
102121
}
103-
})
104-
.collect();
122+
}
123+
}
105124

106-
changed.then_some(fields)
125+
coerced.map(Fields::from)
107126
}
108127

109128
/// Coerce `file_type` towards `table_type`, recursing into nested types.
@@ -175,22 +194,11 @@ fn coerce_map_entries(
175194
return None;
176195
}
177196

178-
let mut changed = false;
179-
let fields: Fields = table_fields
180-
.iter()
181-
.zip(file_fields.iter())
182-
.map(
183-
|(table_child, file_child)| match coerce_child(table_child, file_child) {
184-
Some(child) => {
185-
changed = true;
186-
child
187-
}
188-
None => Arc::clone(file_child),
189-
},
190-
)
191-
.collect();
197+
let fields = coerce_fields(file_fields, |idx, file_child| {
198+
coerce_child(&table_fields[idx], file_child)
199+
})?;
192200

193-
changed.then(|| field_with_new_type(file_entries, DataType::Struct(fields)))
201+
Some(field_with_new_type(file_entries, DataType::Struct(fields)))
194202
}
195203

196204
/// Coerces the file schema's Timestamps to the provided TimeUnit if the

0 commit comments

Comments
 (0)