Skip to content

Commit f4fa266

Browse files
committed
fix: Reverse row selection should respect the row group index (apache#19557)
- Closes [apache#19535](apache#19535) Reverse row selection should respect the row group index, this PR will fix the issue. Reverse row selection should respect the row group index, this PR will fix the issue. Yes No (cherry picked from commit 27de50d)
1 parent a4b686c commit f4fa266

2 files changed

Lines changed: 873 additions & 134 deletions

File tree

‎datafusion/datasource-parquet/src/opener.rs‎

Lines changed: 133 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -119,16 +119,16 @@ pub(super) struct ParquetOpener {
119119
}
120120

121121
/// Represents a prepared access plan with optional row selection
122-
struct PreparedAccessPlan {
122+
pub(crate) struct PreparedAccessPlan {
123123
/// Row group indexes to read
124-
row_group_indexes: Vec<usize>,
124+
pub(crate) row_group_indexes: Vec<usize>,
125125
/// Optional row selection for filtering within row groups
126-
row_selection: Option<parquet::arrow::arrow_reader::RowSelection>,
126+
pub(crate) row_selection: Option<parquet::arrow::arrow_reader::RowSelection>,
127127
}
128128

129129
impl PreparedAccessPlan {
130130
/// Create a new prepared access plan from a ParquetAccessPlan
131-
fn from_access_plan(
131+
pub(crate) fn from_access_plan(
132132
access_plan: ParquetAccessPlan,
133133
rg_metadata: &[RowGroupMetaData],
134134
) -> Result<Self> {
@@ -142,17 +142,23 @@ impl PreparedAccessPlan {
142142
}
143143

144144
/// Reverse the access plan for reverse scanning
145-
fn reverse(
145+
pub(crate) fn reverse(
146146
mut self,
147147
file_metadata: &parquet::file::metadata::ParquetMetaData,
148148
) -> Result<Self> {
149+
// Get the row group indexes before reversing
150+
let row_groups_to_scan = self.row_group_indexes.clone();
151+
149152
// Reverse the row group indexes
150153
self.row_group_indexes = self.row_group_indexes.into_iter().rev().collect();
151154

152155
// If we have a row selection, reverse it to match the new row group order
153156
if let Some(row_selection) = self.row_selection {
154-
self.row_selection =
155-
Some(reverse_row_selection(&row_selection, file_metadata)?);
157+
self.row_selection = Some(reverse_row_selection(
158+
&row_selection,
159+
file_metadata,
160+
&row_groups_to_scan, // Pass the original (non-reversed) row group indexes
161+
)?);
156162
}
157163

158164
Ok(self)
@@ -826,8 +832,9 @@ fn should_enable_page_index(
826832

827833
#[cfg(test)]
828834
mod test {
829-
use crate::{opener::ParquetOpener, DefaultParquetFileReaderFactory};
835+
use std::sync::Arc;
830836
use arrow::compute::cast;
837+
use crate::{DefaultParquetFileReaderFactory, RowGroupAccess, opener::ParquetOpener};
831838
use arrow::datatypes::{DataType, Field, Schema, SchemaRef};
832839
use bytes::{BufMut, BytesMut};
833840
use datafusion_common::{
@@ -851,7 +858,6 @@ mod test {
851858
use object_store::{memory::InMemory, path::Path, ObjectStore};
852859
use parquet::arrow::ArrowWriter;
853860
use parquet::file::properties::WriterProperties;
854-
use std::sync::Arc;
855861

856862
async fn count_batches_and_rows(
857863
mut stream: std::pin::Pin<
@@ -1726,4 +1732,122 @@ mod test {
17261732
"Reverse scan should reverse row group order while maintaining correct RowSelection for each group"
17271733
);
17281734
}
1735+
1736+
#[tokio::test]
1737+
async fn test_reverse_scan_with_non_contiguous_row_groups() {
1738+
use parquet::file::properties::WriterProperties;
1739+
1740+
let store = Arc::new(InMemory::new()) as Arc<dyn ObjectStore>;
1741+
1742+
// Create 4 batches (4 row groups)
1743+
let batch0 = record_batch!(("a", Int32, vec![Some(1), Some(2)])).unwrap();
1744+
let batch1 = record_batch!(("a", Int32, vec![Some(3), Some(4)])).unwrap();
1745+
let batch2 = record_batch!(("a", Int32, vec![Some(5), Some(6)])).unwrap();
1746+
let batch3 = record_batch!(("a", Int32, vec![Some(7), Some(8)])).unwrap();
1747+
1748+
let props = WriterProperties::builder()
1749+
.set_max_row_group_size(2)
1750+
.build();
1751+
1752+
let data_len = write_parquet_batches(
1753+
Arc::clone(&store),
1754+
"test.parquet",
1755+
vec![batch0.clone(), batch1, batch2, batch3],
1756+
Some(props),
1757+
)
1758+
.await;
1759+
1760+
let schema = batch0.schema();
1761+
1762+
use crate::ParquetAccessPlan;
1763+
use parquet::arrow::arrow_reader::{RowSelection, RowSelector};
1764+
1765+
// KEY: Skip RG1 (non-contiguous!)
1766+
// Only scan row groups: [0, 2, 3]
1767+
let mut access_plan = ParquetAccessPlan::new(vec![
1768+
RowGroupAccess::Scan, // RG0
1769+
RowGroupAccess::Skip, // RG1 - SKIPPED!
1770+
RowGroupAccess::Scan, // RG2
1771+
RowGroupAccess::Scan, // RG3
1772+
]);
1773+
1774+
// Add RowSelection for each scanned row group
1775+
// RG0: select first row (1), skip second (2)
1776+
access_plan.scan_selection(
1777+
0,
1778+
RowSelection::from(vec![RowSelector::select(1), RowSelector::skip(1)]),
1779+
);
1780+
// RG1: skipped, no selection needed
1781+
// RG2: select first row (5), skip second (6)
1782+
access_plan.scan_selection(
1783+
2,
1784+
RowSelection::from(vec![RowSelector::select(1), RowSelector::skip(1)]),
1785+
);
1786+
// RG3: select first row (7), skip second (8)
1787+
access_plan.scan_selection(
1788+
3,
1789+
RowSelection::from(vec![RowSelector::select(1), RowSelector::skip(1)]),
1790+
);
1791+
1792+
let file = PartitionedFile::new(
1793+
"test.parquet".to_string(),
1794+
u64::try_from(data_len).unwrap(),
1795+
)
1796+
.with_extensions(Arc::new(access_plan));
1797+
1798+
let make_opener = |reverse_scan: bool| ParquetOpener {
1799+
projection: Arc::new([0]),
1800+
partition_index: 0,
1801+
batch_size: 1024,
1802+
limit: None,
1803+
predicate: None,
1804+
metadata_size_hint: None,
1805+
metrics: ExecutionPlanMetricsSet::new(),
1806+
parquet_file_reader_factory: Arc::new(DefaultParquetFileReaderFactory::new(
1807+
Arc::clone(&store),
1808+
)),
1809+
partition_fields: vec![],
1810+
pushdown_filters: true,
1811+
reorder_filters: false,
1812+
enable_page_index: false,
1813+
enable_bloom_filter: false,
1814+
enable_limit_pruning: false,
1815+
schema_adapter_factory: Arc::new(DefaultSchemaAdapterFactory),
1816+
enable_row_group_stats_pruning: false,
1817+
coerce_int96: None,
1818+
#[cfg(feature = "parquet_encryption")]
1819+
file_decryption_properties: None,
1820+
expr_adapter_factory: Some(Arc::new(DefaultPhysicalExprAdapterFactory)),
1821+
#[cfg(feature = "parquet_encryption")]
1822+
encryption_factory: None,
1823+
max_predicate_cache_size: None,
1824+
reverse_row_groups: reverse_scan,
1825+
logical_file_schema: schema.clone(),
1826+
};
1827+
1828+
// Forward scan: RG0(1), RG2(5), RG3(7)
1829+
// Note: RG1 is completely skipped
1830+
let opener = make_opener(false);
1831+
let stream = opener.open(file.clone()).unwrap().await.unwrap();
1832+
let forward_values = collect_int32_values(stream).await;
1833+
1834+
assert_eq!(
1835+
forward_values,
1836+
vec![1, 5, 7],
1837+
"Forward scan with non-contiguous row groups"
1838+
);
1839+
1840+
// Reverse scan: RG3(7), RG2(5), RG0(1)
1841+
// WITHOUT the bug fix, this would return WRONG values
1842+
// because the RowSelection would be incorrectly mapped
1843+
let opener = make_opener(true);
1844+
let stream = opener.open(file).unwrap().await.unwrap();
1845+
let reverse_values = collect_int32_values(stream).await;
1846+
1847+
assert_eq!(
1848+
reverse_values,
1849+
vec![7, 5, 1],
1850+
"Reverse scan with non-contiguous row groups should correctly map RowSelection"
1851+
);
1852+
}
17291853
}

0 commit comments

Comments
 (0)