Skip to content

Commit 39be5f2

Browse files
authored
fix(substrait): Preserve pushed-down table scan offsets and limits (apache#26035)
## Which issue does this PR close? - Closes apache#26034. ## Rationale for this change Substrait roundtripping loses pushed-down TableScan offsets and limits when no parent Limit exists. A Projection between the scan and a parent Limit also bypasses the existing offset compensation, changing query results. ## What changes are included in this PR? - Encode a scan's skip/fetch in a FetchRel wrapping its ReadRel. - Serialize each parent Limit's own offset without adding the scan offset again. - Use checked conversions for scan offsets and fetch counts; values above `i64::MAX` return an explicit unsupported-range error instead of being encoded as negative values. - Add execution-based roundtrip coverage for 96 combinations of scan offsets, fetch counts, optional projections, and parent limits, including zero limits and offsets beyond the input. ## What is the testing strategy for this PR? Verified the offset-only regression against upstream 6047791: the original scan returns [2, 3, 4, 5], while the roundtripped scan incorrectly returns all six rows. The new regression test passes with the fix. An additional regression test verifies that the pushed-down fetch from `SELECT * FROM t LIMIT 9223372036854775807 OFFSET 1` is rejected with an explicit unsupported-range error. Passed: - cargo fmt --all - cargo clippy --all-targets --all-features -- -D warnings - All Substrait integration tests: 227 passed, 6 ignored. - Required extended workspace tests: 12,388 passed, 0 failed, 8 ignored; all 526 SQLLogicTests files completed. - Contributor lint suite through formatting, its feature-specific Clippy check, TOML formatting, license headers, typos, documentation formatting and generation, workflow checks, large-file checks, Markdown links, and security audit. - Rust documentation check. No relevant local Substrait producer benchmark is defined. ## Are there any user-facing changes? Substrait roundtrips preserve pushed-down table-scan offsets and limits, including across projections and nested limits. No public API changes.
1 parent 46bff84 commit 39be5f2

5 files changed

Lines changed: 157 additions & 25 deletions

File tree

‎Cargo.lock‎

Lines changed: 1 addition & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎datafusion/substrait/Cargo.toml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -54,6 +54,7 @@ datafusion-functions-aggregate = { workspace = true }
5454
serde_json = { workspace = true }
5555
tokio = { workspace = true }
5656
insta = { workspace = true }
57+
tempfile = { workspace = true }
5758

5859
[features]
5960
default = ["physical"]

‎datafusion/substrait/src/logical_plan/producer/rel/fetch_rel.rs‎

Lines changed: 5 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,7 @@
1717

1818
use crate::logical_plan::producer::SubstraitProducer;
1919
use datafusion::common::DFSchema;
20-
use datafusion::logical_expr::{Limit, LogicalPlan, lit};
20+
use datafusion::logical_expr::Limit;
2121
use std::sync::Arc;
2222
use substrait::proto::rel::RelType;
2323
use substrait::proto::{FetchRel, Rel, fetch_rel};
@@ -29,26 +29,10 @@ pub fn from_limit(
2929
let input = producer.handle_plan(limit.input.as_ref())?;
3030
let empty_schema = Arc::new(DFSchema::empty());
3131

32-
// A provider-level skip pushdown (see `push_down_limit`) may have moved
33-
// some or all of this `Limit`'s skip into the child `TableScan::skip`,
34-
// reducing `limit.skip` accordingly (down to `None`/0 when the scan
35-
// handles the whole skip itself). Substrait's `ReadRel` has no field of
36-
// its own for it, so it must be folded back into this `FetchRel`'s
37-
// offset — otherwise it is silently lost when the plan is serialized.
38-
let scan_offset = match limit.input.as_ref() {
39-
LogicalPlan::TableScan(scan) => scan.skip,
40-
_ => None,
41-
};
42-
let skip_expr = match (limit.skip.as_deref(), scan_offset) {
43-
(Some(limit_skip), Some(scan_skip)) => {
44-
Some(limit_skip.clone() + lit(scan_skip as i64))
45-
}
46-
(Some(skip), None) => Some(skip.clone()),
47-
(None, Some(skip)) => Some(lit(skip as i64)),
48-
(None, None) => None,
49-
};
50-
let offset_mode = skip_expr
51-
.map(|expr| producer.handle_expr(&expr, &empty_schema))
32+
let offset_mode = limit
33+
.skip
34+
.as_ref()
35+
.map(|expr| producer.handle_expr(expr.as_ref(), &empty_schema))
5236
.transpose()?
5337
.map(Box::new)
5438
.map(fetch_rel::OffsetMode::OffsetExpr);

‎datafusion/substrait/src/logical_plan/producer/rel/read_rel.rs‎

Lines changed: 36 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -16,17 +16,19 @@
1616
// under the License.
1717

1818
use crate::logical_plan::producer::{SubstraitProducer, to_substrait_named_struct};
19-
use datafusion::common::{DFSchema, ToDFSchema, substrait_datafusion_err};
19+
use datafusion::common::{
20+
DFSchema, ToDFSchema, not_impl_datafusion_err, substrait_datafusion_err,
21+
};
2022
use datafusion::logical_expr::utils::conjunction;
21-
use datafusion::logical_expr::{EmptyRelation, Expr, TableScan, Values};
23+
use datafusion::logical_expr::{EmptyRelation, Expr, TableScan, Values, lit};
2224
use datafusion::scalar::ScalarValue;
2325
use std::sync::Arc;
2426
use substrait::proto::expression::MaskExpression;
2527
use substrait::proto::expression::mask_expression::{StructItem, StructSelect};
2628
use substrait::proto::expression::nested::Struct as NestedStruct;
2729
use substrait::proto::read_rel::{NamedTable, ReadType, VirtualTable};
2830
use substrait::proto::rel::RelType;
29-
use substrait::proto::{ReadRel, Rel};
31+
use substrait::proto::{FetchRel, ReadRel, Rel, fetch_rel};
3032

3133
/// Converts rows of arbitrary expressions into Substrait nested structs.
3234
///
@@ -61,6 +63,12 @@ pub fn from_table_scan(
6163
producer: &mut impl SubstraitProducer,
6264
scan: &TableScan,
6365
) -> datafusion::common::Result<Box<Rel>> {
66+
let skip = scan.skip.map(i64::try_from).transpose().map_err(|_| {
67+
not_impl_datafusion_err!("TableScan offset exceeds the supported Int64 range")
68+
})?;
69+
let fetch = scan.fetch.map(i64::try_from).transpose().map_err(|_| {
70+
not_impl_datafusion_err!("TableScan fetch exceeds the supported Int64 range")
71+
})?;
6472
let projection = scan.projection.as_ref().map(|p| {
6573
p.iter()
6674
.map(|i| StructItem {
@@ -95,7 +103,7 @@ pub fn from_table_scan(
95103
Some(Box::new(filter_expr))
96104
};
97105

98-
Ok(Box::new(Rel {
106+
let read = Box::new(Rel {
99107
rel_type: Some(RelType::Read(Box::new(ReadRel {
100108
common: None,
101109
base_schema: Some(base_schema),
@@ -108,6 +116,30 @@ pub fn from_table_scan(
108116
advanced_extension: None,
109117
})),
110118
}))),
119+
});
120+
121+
// ReadRel cannot represent pushed-down offsets or limits. Keep them with
122+
// the scan, even when there is no parent Limit or a Projection separates it.
123+
if scan.skip.is_none() && scan.fetch.is_none() {
124+
return Ok(read);
125+
}
126+
let empty_schema = Arc::new(DFSchema::empty());
127+
let offset_mode = skip
128+
.map(|skip| producer.handle_expr(&lit(skip), &empty_schema))
129+
.transpose()?
130+
.map(|expr| fetch_rel::OffsetMode::OffsetExpr(Box::new(expr)));
131+
let count_mode = fetch
132+
.map(|fetch| producer.handle_expr(&lit(fetch), &empty_schema))
133+
.transpose()?
134+
.map(|expr| fetch_rel::CountMode::CountExpr(Box::new(expr)));
135+
Ok(Box::new(Rel {
136+
rel_type: Some(RelType::Fetch(Box::new(FetchRel {
137+
common: None,
138+
input: Some(read),
139+
offset_mode,
140+
count_mode,
141+
advanced_extension: None,
142+
}))),
111143
}))
112144
}
113145

‎datafusion/substrait/tests/cases/roundtrip_logical_plan.rs‎

Lines changed: 114 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -320,6 +320,120 @@ async fn select_without_limit() -> Result<()> {
320320
roundtrip_fill_na("SELECT * FROM data OFFSET 5+7-2").await
321321
}
322322

323+
#[tokio::test]
324+
async fn roundtrip_table_scan_offset_and_fetch() -> Result<()> {
325+
use datafusion::arrow::array::Int64Array;
326+
use datafusion::dataframe::DataFrame;
327+
use datafusion::datasource::provider_as_source;
328+
use datafusion::logical_expr::TableScanBuilder;
329+
330+
// A single CSV partition preserves input order and supports exact skip pushdown.
331+
let temp = tempfile::tempdir()?;
332+
let path = temp.path().join("data.csv");
333+
std::fs::write(&path, "x\n0\n1\n2\n3\n4\n5\n")?;
334+
let ctx = SessionContext::new();
335+
ctx.register_csv("t", path.to_str().unwrap(), CsvReadOptions::new())
336+
.await?;
337+
let source = provider_as_source(ctx.table_provider("t").await?);
338+
339+
for skip in [Some(2), None, Some(0), Some(8)] {
340+
for fetch in [None, Some(0), Some(2)] {
341+
for project in [false, true] {
342+
for parent in [
343+
None,
344+
Some((0, Some(2))),
345+
Some((1, Some(3))),
346+
Some((1, None)),
347+
] {
348+
let scan = TableScanBuilder::new("t", source.clone())
349+
.with_skip(skip)
350+
.with_fetch(fetch)
351+
.build()?;
352+
let mut builder =
353+
LogicalPlanBuilder::from(LogicalPlan::TableScan(scan));
354+
if project {
355+
builder = builder.project(vec![col("x")])?;
356+
}
357+
if let Some((skip, fetch)) = parent {
358+
builder = builder.limit(skip, fetch)?;
359+
}
360+
let plan = builder.build()?;
361+
let proto = to_substrait_plan(&plan, &ctx.state())?;
362+
let restored = from_substrait_plan(&ctx.state(), &proto).await?;
363+
let mut expected = (0_i64..6)
364+
.skip(skip.unwrap_or(0))
365+
.take(fetch.unwrap_or(usize::MAX))
366+
.collect::<Vec<_>>();
367+
if let Some((skip, fetch)) = parent {
368+
expected = expected
369+
.into_iter()
370+
.skip(skip)
371+
.take(fetch.unwrap_or(usize::MAX))
372+
.collect();
373+
}
374+
for (stage, plan) in [("original", plan), ("roundtrip", restored)] {
375+
let batches = DataFrame::new(ctx.state(), plan).collect().await?;
376+
let actual = batches
377+
.iter()
378+
.flat_map(|batch| {
379+
batch
380+
.column(0)
381+
.as_any()
382+
.downcast_ref::<Int64Array>()
383+
.unwrap()
384+
.values()
385+
.iter()
386+
.copied()
387+
})
388+
.collect::<Vec<_>>();
389+
assert_eq!(
390+
actual, expected,
391+
"{stage}: skip={skip:?}, fetch={fetch:?}, project={project}, parent={parent:?}"
392+
);
393+
}
394+
}
395+
}
396+
}
397+
}
398+
Ok(())
399+
}
400+
401+
// Limit expressions use Int64, but limit pushdown can produce a larger usize fetch.
402+
#[tokio::test]
403+
#[cfg(target_pointer_width = "64")]
404+
async fn reject_table_scan_fetch_above_i64_max() -> Result<()> {
405+
use datafusion::arrow::array::Int64Array;
406+
use datafusion::arrow::record_batch::RecordBatch;
407+
408+
let ctx = SessionContext::new();
409+
ctx.register_batch(
410+
"t",
411+
RecordBatch::try_from_iter(vec![(
412+
"x",
413+
Arc::new(Int64Array::from(vec![0, 1, 2])) as ArrayRef,
414+
)])?,
415+
)?;
416+
let plan = ctx
417+
.sql("SELECT * FROM t LIMIT 9223372036854775807 OFFSET 1")
418+
.await?
419+
.into_optimized_plan()?;
420+
let LogicalPlan::Limit(limit) = &plan else {
421+
panic!("expected Limit")
422+
};
423+
let LogicalPlan::TableScan(scan) = limit.input.as_ref() else {
424+
panic!("expected TableScan")
425+
};
426+
assert_eq!(scan.fetch, Some(i64::MAX as usize + 1));
427+
let error = to_substrait_plan(&plan, &ctx.state())
428+
.unwrap_err()
429+
.to_string();
430+
assert!(
431+
error.contains("TableScan fetch exceeds the supported Int64 range"),
432+
"{error}"
433+
);
434+
Ok(())
435+
}
436+
323437
#[tokio::test]
324438
async fn select_with_limit_offset() -> Result<()> {
325439
roundtrip("SELECT * FROM data LIMIT 200 OFFSET 10").await?;

0 commit comments

Comments
 (0)