Skip to content

Commit 5f1da98

Browse files
author
Gene Bordegaray
committed
Add range partitioning sqllogictest fixture (apache#22607)
## Which issue does this PR close? - Part of apache#22397. - Discussion: apache#21992. ## Rationale for this change This adds a focused sqllogictest fixture for source-provided `Range` partitioning before changing optimizer behavior. It follows the direction discussed in apache#21992 and gives later planning PRs stable baselines for current behavior. ## What changes are included in this PR? - Registers a `range_partitioned` test table for `range_partitioning.slt`. - Adds a sqllogictest-only source wrapper that reports `Range` partitioning when `range_key` is projected, and `UnknownPartitioning` when it is not. - Adds baselines for grouping on the range key, grouping on a non-range key, joining on the range key, and `UNION ALL` over range-partitioned inputs. ## Are these changes tested? Yes. - `cargo fmt --all` - `cargo test -p datafusion-sqllogictest --test sqllogictests range_partitioning` - `cargo clippy --all-targets --all-features -- -D warnings` ## Are there any user-facing changes? No. This is test-only infrastructure and sqllogictest coverage.
1 parent 9baa581 commit 5f1da98

6 files changed

Lines changed: 399 additions & 0 deletions

File tree

‎Cargo.lock‎

Lines changed: 1 addition & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎datafusion/physical-expr/src/partitioning.rs‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -171,6 +171,11 @@ impl Display for Partitioning {
171171
/// Values equal to split point `i` belong to partition `i + 1`, so interior
172172
/// partitions are lower-inclusive and upper-exclusive.
173173
///
174+
/// Like other user-specified data properties such as sortedness, if a source
175+
/// declares range partitioning, it is responsible for placing each row in the
176+
/// partition described by the split points. DataFusion will not validate this is
177+
/// upheld.
178+
///
174179
/// For a single range key:
175180
///
176181
/// ```text

‎datafusion/sqllogictest/Cargo.toml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -47,6 +47,7 @@ bytes = { workspace = true, optional = true }
4747
chrono = { workspace = true, optional = true }
4848
clap = { version = "4.5.60", features = ["derive", "env"] }
4949
datafusion = { workspace = true, default-features = true, features = ["avro"] }
50+
datafusion-datasource = { workspace = true }
5051
datafusion-spark = { workspace = true, features = ["core"] }
5152
datafusion-substrait = { workspace = true, default-features = true, optional = true }
5253
futures = { workspace = true }

‎datafusion/sqllogictest/src/test_context.rs‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -53,6 +53,8 @@ use datafusion::{
5353
use datafusion_spark::SessionStateBuilderSpark;
5454

5555
use crate::is_spark_path;
56+
use range_partitioning::register_range_partitioned_table;
57+
5658
use async_trait::async_trait;
5759
use datafusion::common::cast::as_float64_array;
5860
use datafusion::execution::SessionStateBuilder;
@@ -61,6 +63,8 @@ use log::info;
6163
use sqlparser::ast;
6264
use tempfile::TempDir;
6365

66+
mod range_partitioning;
67+
6468
/// Context for running tests
6569
pub struct TestContext {
6670
/// Context for running queries
@@ -167,6 +171,10 @@ impl TestContext {
167171
info!("Registering table with many types");
168172
register_table_with_many_types(test_ctx.session_ctx()).await;
169173
}
174+
"range_partitioning.slt" => {
175+
info!("Registering range partitioned table");
176+
register_range_partitioned_table(test_ctx.session_ctx());
177+
}
170178
"metadata.slt" | "arrow_field.slt" => {
171179
info!("Registering metadata table tables");
172180
register_metadata_tables(test_ctx.session_ctx()).await;
Lines changed: 250 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,250 @@
1+
// Licensed to the Apache Software Foundation (ASF) under one
2+
// or more contributor license agreements. See the NOTICE file
3+
// distributed with this work for additional information
4+
// regarding copyright ownership. The ASF licenses this file
5+
// to you under the Apache License, Version 2.0 (the
6+
// "License"); you may not use this file except in compliance
7+
// with the License. You may obtain a copy of the License at
8+
//
9+
// http://www.apache.org/licenses/LICENSE-2.0
10+
//
11+
// Unless required by applicable law or agreed to in writing,
12+
// software distributed under the License is distributed on an
13+
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
// KIND, either express or implied. See the License for the
15+
// specific language governing permissions and limitations
16+
// under the License.
17+
18+
use std::fmt;
19+
use std::sync::Arc;
20+
21+
use arrow::array::Int32Array;
22+
use arrow::compute::SortOptions;
23+
use arrow::datatypes::{DataType, Field, Schema, SchemaRef};
24+
use arrow::record_batch::RecordBatch;
25+
use async_trait::async_trait;
26+
use datafusion::catalog::Session;
27+
use datafusion::common::{Result, ScalarValue, project_schema};
28+
use datafusion::datasource::source::{DataSource, DataSourceExec};
29+
use datafusion::datasource::{TableProvider, TableType};
30+
use datafusion::execution::context::TaskContext;
31+
use datafusion::logical_expr::Expr;
32+
use datafusion::physical_expr::EquivalenceProperties;
33+
use datafusion::physical_expr::expressions::col as physical_col;
34+
use datafusion::physical_expr::{LexOrdering, PhysicalSortExpr};
35+
use datafusion::physical_plan::execution_plan::SchedulingType;
36+
use datafusion::physical_plan::projection::ProjectionExprs;
37+
use datafusion::physical_plan::{
38+
DisplayFormatType, ExecutionPlan, Partitioning, RangePartitioning,
39+
SendableRecordBatchStream, SplitPoint, Statistics,
40+
};
41+
use datafusion::prelude::SessionContext;
42+
use datafusion_datasource::memory::MemorySourceConfig;
43+
44+
// ==============================================================================
45+
// Range Partitioned Table (sqllogictest-only)
46+
// ==============================================================================
47+
48+
/// Simple range-partitioned table for testing before declaring such tables is
49+
/// supported via SQL.
50+
#[derive(Debug)]
51+
struct RangePartitionedTable {
52+
schema: SchemaRef,
53+
partitions: Vec<Vec<RecordBatch>>,
54+
range_column_index: usize,
55+
split_points: Vec<SplitPoint>,
56+
}
57+
58+
#[async_trait]
59+
impl TableProvider for RangePartitionedTable {
60+
fn schema(&self) -> SchemaRef {
61+
Arc::clone(&self.schema)
62+
}
63+
64+
fn table_type(&self) -> TableType {
65+
TableType::Base
66+
}
67+
68+
async fn scan(
69+
&self,
70+
state: &dyn Session,
71+
projection: Option<&Vec<usize>>,
72+
_filters: &[Expr],
73+
_limit: Option<usize>,
74+
) -> Result<Arc<dyn ExecutionPlan>> {
75+
let projected_schema = project_schema(&self.schema, projection)?;
76+
let mut source = MemorySourceConfig::try_new(
77+
&self.partitions,
78+
Arc::clone(&self.schema),
79+
projection.cloned(),
80+
)?;
81+
source = source.with_show_sizes(state.config_options().explain.show_sizes);
82+
83+
let output_partitioning =
84+
self.output_partitioning(projection, &projected_schema)?;
85+
let source = RangePartitionedSource {
86+
inner: source,
87+
output_partitioning,
88+
};
89+
90+
Ok(DataSourceExec::from_data_source(source))
91+
}
92+
}
93+
94+
impl RangePartitionedTable {
95+
fn output_partitioning(
96+
&self,
97+
projection: Option<&Vec<usize>>,
98+
projected_schema: &SchemaRef,
99+
) -> Result<Partitioning> {
100+
let Some(projected_range_index) =
101+
projected_index(self.range_column_index, projection)
102+
else {
103+
return Ok(Partitioning::UnknownPartitioning(self.partitions.len()));
104+
};
105+
106+
let range_column = projected_schema.field(projected_range_index).name();
107+
let ordering = LexOrdering::new(vec![PhysicalSortExpr::new(
108+
physical_col(range_column, projected_schema)?,
109+
SortOptions::default(),
110+
)])
111+
.expect("range ordering should not be empty");
112+
113+
Ok(Partitioning::Range(RangePartitioning::try_new(
114+
ordering,
115+
self.split_points.clone(),
116+
)?))
117+
}
118+
}
119+
120+
fn projected_index(
121+
column_index: usize,
122+
projection: Option<&Vec<usize>>,
123+
) -> Option<usize> {
124+
projection
125+
.map(|projection| projection.iter().position(|idx| *idx == column_index))
126+
.unwrap_or(Some(column_index))
127+
}
128+
129+
#[derive(Clone, Debug)]
130+
struct RangePartitionedSource {
131+
inner: MemorySourceConfig,
132+
output_partitioning: Partitioning,
133+
}
134+
135+
impl DataSource for RangePartitionedSource {
136+
fn open(
137+
&self,
138+
partition: usize,
139+
context: Arc<TaskContext>,
140+
) -> Result<SendableRecordBatchStream> {
141+
self.inner.open(partition, context)
142+
}
143+
144+
fn fmt_as(&self, t: DisplayFormatType, f: &mut fmt::Formatter) -> fmt::Result {
145+
self.inner.fmt_as(t, f)?;
146+
match t {
147+
DisplayFormatType::Default | DisplayFormatType::Verbose => {
148+
write!(f, ", output_partitioning={}", self.output_partitioning)
149+
}
150+
DisplayFormatType::TreeRender => Ok(()),
151+
}
152+
}
153+
154+
fn output_partitioning(&self) -> Partitioning {
155+
self.output_partitioning.clone()
156+
}
157+
158+
fn eq_properties(&self) -> EquivalenceProperties {
159+
self.inner.eq_properties()
160+
}
161+
162+
fn scheduling_type(&self) -> SchedulingType {
163+
self.inner.scheduling_type()
164+
}
165+
166+
fn partition_statistics(&self, partition: Option<usize>) -> Result<Arc<Statistics>> {
167+
self.inner.partition_statistics(partition)
168+
}
169+
170+
fn with_fetch(&self, limit: Option<usize>) -> Option<Arc<dyn DataSource>> {
171+
Some(Arc::new(Self {
172+
inner: self.inner.clone().with_limit(limit),
173+
output_partitioning: self.output_partitioning.clone(),
174+
}))
175+
}
176+
177+
fn fetch(&self) -> Option<usize> {
178+
self.inner.fetch()
179+
}
180+
181+
fn try_swapping_with_projection(
182+
&self,
183+
_projection: &ProjectionExprs,
184+
) -> Result<Option<Arc<dyn DataSource>>> {
185+
// Range partitioning metadata is projection-sensitive. This fixture
186+
// computes it in TableProvider::scan, so do not rewrite later
187+
// ProjectionExec nodes into the source.
188+
Ok(None)
189+
}
190+
}
191+
192+
pub(super) fn register_range_partitioned_table(ctx: &SessionContext) {
193+
let schema = Arc::new(Schema::new(vec![
194+
Field::new("range_key", DataType::Int32, false),
195+
Field::new("non_range_key", DataType::Int32, false),
196+
Field::new("value", DataType::Int32, false),
197+
]));
198+
let partitions = vec![
199+
vec![range_partition_batch(&schema, &[1, 5], &[1, 2], &[10, 50])],
200+
vec![range_partition_batch(
201+
&schema,
202+
&[10, 15],
203+
&[1, 2],
204+
&[100, 150],
205+
)],
206+
vec![range_partition_batch(
207+
&schema,
208+
&[20, 25],
209+
&[1, 2],
210+
&[200, 250],
211+
)],
212+
vec![range_partition_batch(
213+
&schema,
214+
&[30, 35],
215+
&[1, 2],
216+
&[300, 350],
217+
)],
218+
];
219+
let split_points = vec![
220+
SplitPoint::new(vec![ScalarValue::Int32(Some(10))]),
221+
SplitPoint::new(vec![ScalarValue::Int32(Some(20))]),
222+
SplitPoint::new(vec![ScalarValue::Int32(Some(30))]),
223+
];
224+
let table = RangePartitionedTable {
225+
schema,
226+
partitions,
227+
range_column_index: 0,
228+
split_points,
229+
};
230+
231+
ctx.register_table("range_partitioned", Arc::new(table))
232+
.expect("range partitioned table registration should succeed");
233+
}
234+
235+
fn range_partition_batch(
236+
schema: &SchemaRef,
237+
range_key: &[i32],
238+
non_range_key: &[i32],
239+
value: &[i32],
240+
) -> RecordBatch {
241+
RecordBatch::try_new(
242+
Arc::clone(schema),
243+
vec![
244+
Arc::new(Int32Array::from(range_key.to_vec())),
245+
Arc::new(Int32Array::from(non_range_key.to_vec())),
246+
Arc::new(Int32Array::from(value.to_vec())),
247+
],
248+
)
249+
.expect("range partition batch should be valid")
250+
}

0 commit comments

Comments
 (0)