Skip to content

Commit 601afda

Browse files
authored
Merge branch 'main' into rich-T-kid/substr-dict-encoding-preservation
2 parents c11a73c + b295a67 commit 601afda

43 files changed

Lines changed: 3590 additions & 336 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎datafusion-examples/.examples-docs-check.E3y1ZS/README.md‎

Whitespace-only changes.

‎datafusion/common/src/config.rs‎

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -925,6 +925,29 @@ config_namespace! {
925925
/// Support for build_side.num_rows() >= u32::MAX will be added in the future.
926926
pub perfect_hash_join_min_key_density: f64, default = 0.15
927927

928+
/// Enable an exact integer membership prefilter for hash joins with a single
929+
/// integer key when the build side uses a hash table and the join is not
930+
/// null-aware. It skips lookups for absent keys while retaining NULLs and
931+
/// probe rows for the original join logic.
932+
pub enable_join_integer_prefilter: bool, default = false
933+
934+
/// Maximum number of integer values covered by the join prefilter bitmap
935+
/// (`max_key - min_key + 1`). Zero disables bitmap construction.
936+
/// Larger ranges fall back to normal hash table lookups.
937+
pub join_integer_prefilter_max_key_range: u64, default = 262144
938+
939+
/// The integer prefilter must prune more than this fraction of all sampled
940+
/// probe rows, including NULLs, to keep filtering. Must be finite and in
941+
/// [0, 1], validated when executing the join.
942+
pub join_integer_prefilter_min_pruning_ratio: f64, default = 0.5
943+
944+
/// Minimum input probe rows in each integer prefilter sampling window,
945+
/// including NULLs. Must be greater than zero. Windows end at batch boundaries.
946+
/// Unprofitable samples initially pause for 10 times this many input rows;
947+
/// consecutive failures double the pause up to 128 times. A profitable
948+
/// sample resets the pause to 10 times this many rows.
949+
pub join_integer_prefilter_sample_rows: ConfigNonZeroUsize, default = non_zero_usize_default(12288)
950+
928951
/// When set to true, record batches will be examined between each operator and
929952
/// small batches will be coalesced into larger batches. This is helpful when there
930953
/// are highly selective filters or joins that could produce tiny output batches. The

‎datafusion/common/src/hash_utils.rs‎

Lines changed: 15 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -543,23 +543,27 @@ fn hash_dictionary_scatter<
543543
if let Some(key) = key {
544544
let idx = key.as_usize();
545545
if !HAS_NULL_VALUES || dict_values.is_valid(idx) {
546-
if MULTI_COL {
547-
*hash = combine_hashes(dict_hashes[idx], *hash);
546+
// SAFETY: `idx` is a valid Arrow dict key, in-bounds for `dict_hashes`.
547+
let dict_hash = unsafe { *dict_hashes.get_unchecked(idx) };
548+
*hash = if MULTI_COL {
549+
combine_hashes(dict_hash, *hash)
548550
} else {
549-
*hash = dict_hashes[idx];
550-
}
551+
dict_hash
552+
};
551553
}
552554
}
553555
}
554556
} else {
555557
for (hash, key) in hashes_buffer.iter_mut().zip(array.keys().values()) {
556558
let idx = key.as_usize();
557559
if !HAS_NULL_VALUES || dict_values.is_valid(idx) {
558-
if MULTI_COL {
559-
*hash = combine_hashes(dict_hashes[idx], *hash);
560+
// SAFETY: `idx` is a valid Arrow dict key, in-bounds for `dict_hashes`.
561+
let dict_hash = unsafe { *dict_hashes.get_unchecked(idx) };
562+
*hash = if MULTI_COL {
563+
combine_hashes(dict_hash, *hash)
560564
} else {
561-
*hash = dict_hashes[idx];
562-
}
565+
dict_hash
566+
};
563567
}
564568
}
565569
}
@@ -1304,6 +1308,9 @@ mod tests {
13041308
use std::hash::{BuildHasherDefault, Hasher};
13051309
use std::sync::Arc;
13061310

1311+
#[cfg(not(feature = "force_hash_collisions"))]
1312+
use arrow::buffer::{NullBuffer, ScalarBuffer};
1313+
13071314
use super::*;
13081315

13091316
#[cfg(not(feature = "force_hash_collisions"))]
@@ -1856,8 +1863,6 @@ mod tests {
18561863
// Tests actual values of hashes, which are different if forcing collisions
18571864
#[cfg(not(feature = "force_hash_collisions"))]
18581865
fn create_hashes_for_list_view_arrays() {
1859-
use arrow::buffer::{NullBuffer, ScalarBuffer};
1860-
18611866
// Create values array: [0, 1, 2, 3, null, 5]
18621867
let values = Arc::new(Int32Array::from(vec![
18631868
Some(0),
@@ -1906,8 +1911,6 @@ mod tests {
19061911
// Tests actual values of hashes, which are different if forcing collisions
19071912
#[cfg(not(feature = "force_hash_collisions"))]
19081913
fn create_hashes_for_large_list_view_arrays() {
1909-
use arrow::buffer::{NullBuffer, ScalarBuffer};
1910-
19111914
// Create values array: [0, 1, 2, 3, null, 5]
19121915
let values = Arc::new(Int32Array::from(vec![
19131916
Some(0),

‎datafusion/core/tests/fuzz_cases/join_fuzz.rs‎

Lines changed: 44 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -1487,7 +1487,7 @@ fn make_staggered_batches_binary(
14871487
//
14881488
// What only randomization reaches: `LeftSemi`/`LeftAnti` record matches in a shared
14891489
// `AtomicUsize` watermark and emit once, from whichever streamed partition finishes last. The
1490-
// streamed side below is spread round-robin over several partitions as one-row batches, so
1490+
// streamed side below is spread round-robin over several partitions in small batches, so
14911491
// batches arrive in an order no static test pins down, and the counter that gates the final
14921492
// pass is seeded from a partition count that deliberately disagrees with the `num_partitions`
14931493
// argument. `RightSemi`/`RightAnti` take the mirror path -- every partition emits its own
@@ -1575,25 +1575,39 @@ fn pwmj_single_exec<K: PwmjFuzzKey>(
15751575
.unwrap()
15761576
}
15771577

1578-
/// Rows spread round-robin across `nparts` partitions, one row per batch, with any partition
1579-
/// that draws no row left holding a single empty batch. Used for the streamed side throughout,
1580-
/// and for the buffered side of the right existence joins, which place no single-partition
1581-
/// requirement on it.
1578+
/// Rows spread round-robin across `nparts` partitions, with any partition that draws no row
1579+
/// left holding a single empty batch. With `stagger_seed`, each partition's rows are cut into
1580+
/// random-size batches, empty ones included, by [`stagger_batch_with_seed`]; without it, one
1581+
/// row per batch. Used for the streamed side throughout, and for the buffered side of the
1582+
/// right existence joins, which place no single-partition requirement on it.
15821583
fn pwmj_parts_exec<K: PwmjFuzzKey>(
15831584
ids: &[i32],
15841585
keys: &[Option<K>],
15851586
nparts: usize,
1587+
stagger_seed: Option<u64>,
15861588
) -> Arc<dyn ExecutionPlan> {
15871589
let nparts = nparts.max(1);
1588-
let mut partitions: Vec<Vec<RecordBatch>> = vec![Vec::new(); nparts];
1590+
let mut part_rows: Vec<(Vec<i32>, Vec<Option<K>>)> = vec![Default::default(); nparts];
15891591
for (row, (&id, key)) in ids.iter().zip(keys.iter()).enumerate() {
1590-
partitions[row % nparts].push(pwmj_kv_batch(&[id], &[*key]));
1591-
}
1592-
for p in partitions.iter_mut() {
1593-
if p.is_empty() {
1594-
p.push(pwmj_kv_batch::<K>(&[], &[]));
1595-
}
1596-
}
1592+
part_rows[row % nparts].0.push(id);
1593+
part_rows[row % nparts].1.push(*key);
1594+
}
1595+
let partitions: Vec<Vec<RecordBatch>> = part_rows
1596+
.iter()
1597+
.enumerate()
1598+
.map(|(part, (ids, keys))| match stagger_seed {
1599+
// `stagger_batch_with_seed` panics on a batch with no rows.
1600+
_ if ids.is_empty() => vec![pwmj_kv_batch::<K>(&[], &[])],
1601+
Some(seed) => {
1602+
stagger_batch_with_seed(pwmj_kv_batch(ids, keys), seed + part as u64)
1603+
}
1604+
None => ids
1605+
.iter()
1606+
.zip(keys)
1607+
.map(|(&id, key)| pwmj_kv_batch(&[id], &[*key]))
1608+
.collect(),
1609+
})
1610+
.collect();
15971611
MemorySourceConfig::try_new_exec(&partitions, pwmj_kv_schema::<K>(), None).unwrap()
15981612
}
15991613

@@ -1820,6 +1834,9 @@ async fn run_pwmj_fuzz<K: PwmjFuzzKey + std::fmt::Debug>(task_ctx: &Arc<TaskCont
18201834
let left_keys = gen_keys(left_len, &mut rng);
18211835
let right_ids: Vec<i32> = (0..right_len as i32).collect();
18221836
let right_keys = gen_keys(right_len, &mut rng);
1837+
// Staggered streamed batches mix rows that can and cannot match, so the classic joins
1838+
// set some aside before the sort and map the rest back to their rows.
1839+
let stagger_seed = Some(seed);
18231840

18241841
for op in ops {
18251842
for join_type in join_types {
@@ -1828,7 +1845,7 @@ async fn run_pwmj_fuzz<K: PwmjFuzzKey + std::fmt::Debug>(task_ctx: &Arc<TaskCont
18281845
// single-partition shape below never reaches.
18291846
let buffered = match join_type {
18301847
JoinType::RightSemi | JoinType::RightAnti | JoinType::RightMark => {
1831-
pwmj_parts_exec(&left_ids, &left_keys, buffered_nparts)
1848+
pwmj_parts_exec(&left_ids, &left_keys, buffered_nparts, None)
18321849
}
18331850
_ => pwmj_single_exec(&left_ids, &left_keys),
18341851
};
@@ -1840,7 +1857,12 @@ async fn run_pwmj_fuzz<K: PwmjFuzzKey + std::fmt::Debug>(task_ctx: &Arc<TaskCont
18401857
let got = pwmj_collect_mark_pairs(
18411858
pwmj_plan(
18421859
buffered,
1843-
pwmj_parts_exec(&right_ids, &right_keys, nparts),
1860+
pwmj_parts_exec(
1861+
&right_ids,
1862+
&right_keys,
1863+
nparts,
1864+
stagger_seed,
1865+
),
18441866
op,
18451867
join_type,
18461868
),
@@ -1872,7 +1894,7 @@ async fn run_pwmj_fuzz<K: PwmjFuzzKey + std::fmt::Debug>(task_ctx: &Arc<TaskCont
18721894
let got = pwmj_collect_id_pairs(
18731895
pwmj_plan(
18741896
buffered,
1875-
pwmj_parts_exec(&right_ids, &right_keys, nparts),
1897+
pwmj_parts_exec(&right_ids, &right_keys, nparts, stagger_seed),
18761898
op,
18771899
join_type,
18781900
),
@@ -2000,7 +2022,12 @@ async fn pwmj_watermark_nulls_duplicates_multi_partition() {
20002022
let got = pwmj_collect_id_pairs(
20012023
pwmj_plan(
20022024
pwmj_single_exec(&left_ids, &left_keys),
2003-
pwmj_parts_exec(&case.streamed_ids, &case.streamed_keys, nparts),
2025+
pwmj_parts_exec(
2026+
&case.streamed_ids,
2027+
&case.streamed_keys,
2028+
nparts,
2029+
None,
2030+
),
20042031
Operator::Lt,
20052032
join_type,
20062033
),

‎datafusion/core/tests/parquet/schema.rs‎

Lines changed: 94 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,8 @@
1818
//! Tests for parquet schema handling
1919
use std::{collections::HashMap, fs, path::Path};
2020

21+
use arrow::array::ArrayRef;
22+
2123
use tempfile::TempDir;
2224

2325
use super::*;
@@ -185,6 +187,71 @@ async fn schema_merge_can_preserve_metadata() {
185187
assert_metadata(&actual, &expected_metadata);
186188
}
187189

190+
/// A column that only some files carry must be inferred as nullable, because
191+
/// reading the files without it yields nulls, even when every file that has
192+
/// the column declares it required.
193+
#[tokio::test]
194+
async fn schema_merge_marks_partially_present_columns_nullable() {
195+
let tmp_dir = TempDir::new().unwrap();
196+
let table_dir = tmp_dir.path().join("parquet_test");
197+
198+
let id = Field::new("id", DataType::Int32, false);
199+
let name = Field::new("name", DataType::Utf8, false);
200+
let extra = Field::new("extra", DataType::Int32, false);
201+
let opt = Field::new("opt", DataType::Int32, true);
202+
203+
let schemas = vec![
204+
// required `extra` and nullable `opt` in the first file only
205+
Schema::new(vec![id.clone(), name.clone(), extra.clone(), opt]),
206+
// neither column
207+
Schema::new(vec![id.clone(), name.clone()]),
208+
// required `extra` again, in a different position
209+
Schema::new(vec![extra, id, name]),
210+
];
211+
write_files_with_columns(table_dir.as_path(), schemas);
212+
let table_path = table_dir.to_str().unwrap().to_string();
213+
214+
for skip_metadata in [true, false] {
215+
let options = ParquetReadOptions::default().skip_metadata(skip_metadata);
216+
let ctx = SessionContext::new();
217+
let df = ctx.read_parquet(&table_path, options).await.unwrap();
218+
219+
let nullability: Vec<(&str, bool)> = df
220+
.schema()
221+
.fields()
222+
.iter()
223+
.map(|f| (f.name().as_str(), f.is_nullable()))
224+
.collect();
225+
assert_eq!(
226+
nullability,
227+
vec![
228+
("id", false),
229+
("name", false),
230+
("extra", true),
231+
("opt", true)
232+
],
233+
"skip_metadata={skip_metadata}"
234+
);
235+
236+
let actual = df.collect().await.unwrap();
237+
let expected = [
238+
"+----+------+-------+-----+",
239+
"| id | name | extra | opt |",
240+
"+----+------+-------+-----+",
241+
"| 0 | test | 0 | |",
242+
"| 1 | test | | |",
243+
"| 2 | test | 2 | |",
244+
"+----+------+-------+-----+",
245+
]
246+
.join("\n");
247+
assert_eq!(
248+
batches_to_sort_string(&actual).trim(),
249+
expected,
250+
"skip_metadata={skip_metadata}"
251+
);
252+
}
253+
}
254+
188255
fn make_meta(k: impl Into<String>, v: impl Into<String>) -> HashMap<String, String> {
189256
let mut meta = HashMap::new();
190257
meta.insert(k.into(), v.into());
@@ -214,6 +281,33 @@ fn write_files(table_path: &Path, schemas: Vec<Schema>) {
214281
}
215282
}
216283

284+
/// Writes one file per schema, filling each column from its name: `id` and
285+
/// `extra` get the file index, `name` gets "test", anything else is null.
286+
fn write_files_with_columns(table_path: &Path, schemas: Vec<Schema>) {
287+
fs::create_dir(table_path).expect("Error creating temp dir");
288+
289+
for (i, schema) in schemas.into_iter().enumerate() {
290+
let schema = Arc::new(schema);
291+
let path = table_path.join(format!("part-{i}.parquet"));
292+
let file = fs::File::create(path).unwrap();
293+
let mut writer = ArrowWriter::try_new(file, schema.clone(), None).unwrap();
294+
295+
let columns: Vec<ArrayRef> = schema
296+
.fields()
297+
.iter()
298+
.map(|field| match field.name().as_str() {
299+
"id" | "extra" => Arc::new(Int32Array::from(vec![i as i32])) as ArrayRef,
300+
"name" => Arc::new(StringArray::from(vec!["test"])),
301+
_ => Arc::new(Int32Array::from(vec![None::<i32>])),
302+
})
303+
.collect();
304+
let rec_batch = RecordBatch::try_new(schema, columns).unwrap();
305+
306+
writer.write(&rec_batch).unwrap();
307+
writer.close().unwrap();
308+
}
309+
}
310+
217311
fn assert_no_metadata(batches: &[RecordBatch]) {
218312
// all batches should have no metadata
219313
for batch in batches {

‎datafusion/datasource-csv/src/source.rs‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -174,6 +174,10 @@ impl CsvSource {
174174
}
175175

176176
/// Whether values may contain newline characters
177+
#[deprecated(
178+
since = "56.0.0",
179+
note = "No replacement getter exists. Retain CsvOptions and read its newlines_in_values field with unwrap_or(false)."
180+
)]
177181
pub fn newlines_in_values(&self) -> bool {
178182
self.options.newlines_in_values.unwrap_or(false)
179183
}
@@ -354,7 +358,7 @@ impl FileSource for CsvSource {
354358
)
355359
})
356360
.transpose()?,
357-
newlines_in_values: self.newlines_in_values(),
361+
newlines_in_values: self.options.newlines_in_values.unwrap_or(false),
358362
truncate_rows: self.truncate_rows(),
359363
terminator: self.terminator().map(|terminator| vec![terminator]),
360364
};

0 commit comments

Comments
 (0)