Skip to content

Commit 0b39348

Browse files
strawgatediegoQuinas
authored andcommitted
Avoid formatting errors for expected missing physical columns (apache#25664)
## Which issue does this PR close? Closes apache#25663. ## Rationale for this change Schema adaptation can legitimately encounter columns absent from an older physical schema. A missing-column lookup currently formats an error listing every physical field, then discards it. This adds avoidable latency as schemas widen. ## What changes are included in this PR? Use the non-error-producing field lookup in the physical expression adapter. Add a Criterion benchmark for present and missing columns in 16- and 128-field schemas. ## What is the testing strategy for this PR? The existing adapter tests cover missing nullable and non-nullable columns and present columns. The full extended workspace test command passed locally, as did formatting and strict package-level Clippy. Full-workspace Clippy currently reports two warnings in unchanged `datafusion/physical-plan/benches/multi_group_by.rs`. On Apple Silicon, the missing-column benchmark changed from 537 to 256 ns for 16 fields and 1.83 us to 302 ns for 128 fields. Present-column timings were within run-to-run variation. These are microbenchmarks, not end-to-end query measurements. ## Are there any user-facing changes? No behavior or API changes; this reduces work for expected missing columns.
1 parent 105585e commit 0b39348

4 files changed

Lines changed: 59 additions & 1 deletion

File tree

‎Cargo.lock‎

Lines changed: 1 addition & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎datafusion/physical-expr-adapter/Cargo.toml‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -28,3 +28,8 @@ itertools = { workspace = true }
2828
workspace = true
2929

3030
[dev-dependencies]
31+
criterion = { workspace = true }
32+
33+
[[bench]]
34+
name = "schema_rewrite"
35+
harness = false
Lines changed: 51 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,51 @@
1+
// Licensed to the Apache Software Foundation (ASF) under one
2+
// or more contributor license agreements. See the NOTICE file
3+
// distributed with this work for additional information
4+
// regarding copyright ownership. The ASF licenses this file
5+
// to you under the Apache License, Version 2.0 (the
6+
// "License"); you may not use this file except in compliance
7+
// with the License. You may obtain a copy of the License at
8+
//
9+
// http://www.apache.org/licenses/LICENSE-2.0
10+
//
11+
// Unless required by applicable law or agreed to in writing,
12+
// software distributed under the License is distributed on an
13+
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
// KIND, either express or implied. See the License for the
15+
// specific language governing permissions and limitations
16+
// under the License.
17+
18+
use std::hint::black_box;
19+
use std::sync::Arc;
20+
21+
use arrow::datatypes::{DataType, Field, Schema};
22+
use criterion::{Criterion, criterion_group, criterion_main};
23+
use datafusion_physical_expr::expressions::Column;
24+
use datafusion_physical_expr_adapter::{DefaultPhysicalExprAdapter, PhysicalExprAdapter};
25+
26+
fn bench_schema_rewrite(c: &mut Criterion) {
27+
for field_count in [16, 128] {
28+
let physical_fields: Vec<_> = (0..field_count)
29+
.map(|index| Field::new(format!("field_{index}"), DataType::Utf8, true))
30+
.collect();
31+
let mut logical_fields = physical_fields.clone();
32+
logical_fields.push(Field::new("optional_new_field", DataType::Utf8, true));
33+
34+
let adapter = DefaultPhysicalExprAdapter::new(
35+
Arc::new(Schema::new(logical_fields)),
36+
Arc::new(Schema::new(physical_fields)),
37+
);
38+
let missing = Arc::new(Column::new("optional_new_field", field_count));
39+
let present = Arc::new(Column::new("field_0", 0));
40+
41+
c.bench_function(&format!("schema_rewrite/missing/{field_count}"), |b| {
42+
b.iter(|| black_box(adapter.rewrite(black_box(missing.clone())).unwrap()))
43+
});
44+
c.bench_function(&format!("schema_rewrite/present/{field_count}"), |b| {
45+
b.iter(|| black_box(adapter.rewrite(black_box(present.clone())).unwrap()))
46+
});
47+
}
48+
}
49+
50+
criterion_group!(benches, bench_schema_rewrite);
51+
criterion_main!(benches);

‎datafusion/physical-expr-adapter/src/schema_rewriter.rs‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -772,7 +772,8 @@ impl DefaultPhysicalExprAdapterRewriter {
772772
// resolving by name ensures we match the correct physical slot. Once we know the
773773
// proper index we rebuild the `Column` with `new_with_schema` so callers can rely
774774
// on `column.index()` later without having to re-query the schema.
775-
let Ok(physical_column_index) = self.physical_file_schema.index_of(column.name())
775+
let Some((physical_column_index, _)) =
776+
self.physical_file_schema.fields().find(column.name())
776777
else {
777778
return Ok(None);
778779
};

0 commit comments

Comments
 (0)