Skip to content

Commit bdd253c

Browse files
committed
fix(bench): read Spark's NULL partition directories as NULL
Spark writes rows whose partition key is NULL to a `<column>=__HIVE_DEFAULT_PARTITION__/` directory, and TPC-DS fact tables have rows with a NULL date key. DataFusion casts each directory's value to the partition column's type, so with `ss_sold_date_sk` declared `Int32` every query that scanned such a table failed with a cast error (apache/datafusion#18083). When a partition column has that directory, register the files as `<table>_raw` with the column as a string, and register `<table>` as a view that maps the directory to NULL and casts the other values back. Filters on the column still skip directories. Tables without that directory are registered as before.
1 parent adaff3c commit bdd253c

3 files changed

Lines changed: 199 additions & 20 deletions

File tree

‎benchmarks/README.md‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -411,6 +411,15 @@ directory names, such as `ss_sold_date_sk` in data written with Spark's
411411
`partitionBy`, is read as an integer, because TPC-DS partitions its fact tables
412412
by date surrogate key.
413413

414+
Some fact table rows have a NULL date key. Spark writes them to a directory
415+
such as `store_sales/ss_sold_date_sk=__HIVE_DEFAULT_PARTITION__/`, which
416+
DataFusion can't read as an integer yet
417+
([apache/datafusion#18083](https://github.com/apache/datafusion/issues/18083)).
418+
For a table with such a directory, the runner registers the files as
419+
`<table>_raw`, with that column as a string, and `<table>` as a view that reads
420+
the directory as NULL and casts the other values to integers. Filters on the
421+
column still skip directories.
422+
414423
`tpcgen-cli` names three columns differently from the TPC-DS spec, which the
415424
queries follow, so the runner rewrites those references when the tables have
416425
the `tpcgen-cli` names. Data from `dsdgen` or Spark uses the spec names and the

‎benchmarks/src/bin/tpcds.rs‎

Lines changed: 92 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -533,21 +533,23 @@ mod tests {
533533
assert!(!replace_word(&joined, "r_reason_desc", "?").contains('?'));
534534
}
535535

536-
// Spark's `partitionBy` layout: `ss_sold_date_sk` is only a directory
537-
// level. It must register as an integer, so it joins `d_date_sk` and a
538-
// filter on it skips the other partitions.
539-
#[tokio::test]
540-
async fn path_only_date_sk_partition_is_an_integer_and_prunes() -> Result<()> {
536+
/// Writes a one-column `store_sales` table under `dir` in the layout of
537+
/// Spark's `partitionBy`, where `ss_sold_date_sk` is only a directory
538+
/// level, and registers it on a new session. Each `(item, date_sk)` pair
539+
/// becomes the one row of `store_sales/ss_sold_date_sk=<date_sk>/`.
540+
async fn spark_store_sales(
541+
dir: &std::path::Path,
542+
rows: &[(i32, &str)],
543+
) -> Result<SessionContext> {
541544
use datafusion::arrow::array::{ArrayRef, Int32Array};
542545
use datafusion::parquet::arrow::ArrowWriter;
543546
use std::fs;
544547
use std::sync::Arc;
545548

546-
let dir = tempfile::tempdir().unwrap();
547-
for (item, date_sk) in [(1, 2451000), (2, 2451001)] {
548-
let item: ArrayRef = Arc::new(Int32Array::from(vec![item]));
549+
for (item, date_sk) in rows {
550+
let item: ArrayRef = Arc::new(Int32Array::from(vec![*item]));
549551
let batch = RecordBatch::try_from_iter([("ss_item_sk", item)]).unwrap();
550-
let path = dir.path().join(format!(
552+
let path = dir.join(format!(
551553
"store_sales/ss_sold_date_sk={date_sk}/part-0.parquet"
552554
));
553555
fs::create_dir_all(path.parent().unwrap()).unwrap();
@@ -562,7 +564,7 @@ mod tests {
562564
}
563565

564566
let ctx = SessionContext::new();
565-
let path = format!("{}/store_sales", dir.path().display());
567+
let path = format!("{}/store_sales", dir.display());
566568
ballista_benchmarks::register_parquet_table(
567569
&ctx,
568570
"store_sales",
@@ -571,6 +573,26 @@ mod tests {
571573
tpcds_path_column_type,
572574
)
573575
.await?;
576+
Ok(ctx)
577+
}
578+
579+
async fn physical_plan(df: &datafusion::prelude::DataFrame) -> Result<String> {
580+
Ok(format!(
581+
"{}",
582+
datafusion::physical_plan::displayable(
583+
df.clone().create_physical_plan().await?.as_ref()
584+
)
585+
.indent(false)
586+
))
587+
}
588+
589+
// The date key must register as an integer, so it joins `d_date_sk`, and
590+
// a filter on it skips the other partitions.
591+
#[tokio::test]
592+
async fn path_only_date_sk_partition_is_an_integer_and_prunes() -> Result<()> {
593+
let dir = tempfile::tempdir().unwrap();
594+
let ctx =
595+
spark_store_sales(dir.path(), &[(1, "2451000"), (2, "2451001")]).await?;
574596

575597
let schema = ctx.table_provider("store_sales").await?.schema();
576598
assert_eq!(
@@ -581,13 +603,7 @@ mod tests {
581603
let df = ctx
582604
.sql("SELECT ss_item_sk FROM store_sales WHERE ss_sold_date_sk = 2451001")
583605
.await?;
584-
let plan = format!(
585-
"{}",
586-
datafusion::physical_plan::displayable(
587-
df.clone().create_physical_plan().await?.as_ref()
588-
)
589-
.indent(false)
590-
);
606+
let plan = physical_plan(&df).await?;
591607
assert!(plan.contains("ss_sold_date_sk=2451001"), "{plan}");
592608
assert!(!plan.contains("ss_sold_date_sk=2451000"), "{plan}");
593609
datafusion::assert_batches_eq!(
@@ -602,4 +618,63 @@ mod tests {
602618
);
603619
Ok(())
604620
}
621+
622+
// Spark writes the rows whose partition key is NULL to a
623+
// `__HIVE_DEFAULT_PARTITION__` directory, and TPC-DS fact tables have
624+
// rows with a NULL date key. They must read back as NULL integers
625+
// instead of failing the scan, and a filter on the key must still skip
626+
// that directory.
627+
#[tokio::test]
628+
async fn null_date_sk_partition_reads_as_null() -> Result<()> {
629+
let dir = tempfile::tempdir().unwrap();
630+
let ctx = spark_store_sales(
631+
dir.path(),
632+
&[(1, "2451000"), (2, "__HIVE_DEFAULT_PARTITION__")],
633+
)
634+
.await?;
635+
636+
let schema = ctx.table_provider("store_sales").await?.schema();
637+
assert_eq!(
638+
schema.field_with_name("ss_sold_date_sk")?.data_type(),
639+
&DataType::Int32
640+
);
641+
642+
let all = ctx
643+
.sql(
644+
"SELECT ss_item_sk, ss_sold_date_sk FROM store_sales ORDER BY ss_item_sk",
645+
)
646+
.await?;
647+
datafusion::assert_batches_eq!(
648+
[
649+
"+------------+-----------------+",
650+
"| ss_item_sk | ss_sold_date_sk |",
651+
"+------------+-----------------+",
652+
"| 1 | 2451000 |",
653+
"| 2 | |",
654+
"+------------+-----------------+",
655+
],
656+
&all.collect().await?
657+
);
658+
659+
let df = ctx
660+
.sql("SELECT ss_item_sk FROM store_sales WHERE ss_sold_date_sk = 2451000")
661+
.await?;
662+
let plan = physical_plan(&df).await?;
663+
assert!(plan.contains("ss_sold_date_sk=2451000"), "{plan}");
664+
assert!(
665+
!plan.contains("ss_sold_date_sk=__HIVE_DEFAULT_PARTITION__"),
666+
"{plan}"
667+
);
668+
datafusion::assert_batches_eq!(
669+
[
670+
"+------------+",
671+
"| ss_item_sk |",
672+
"+------------+",
673+
"| 1 |",
674+
"+------------+",
675+
],
676+
&df.collect().await?
677+
);
678+
Ok(())
679+
}
605680
}

‎benchmarks/src/lib.rs‎

Lines changed: 98 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -36,8 +36,12 @@ use datafusion::datasource::listing::{ListingOptions, ListingTableUrl};
3636
use datafusion::error::{DataFusionError, Result};
3737
use datafusion::execution::context::SessionState;
3838
use datafusion::execution::options::ReadOptions;
39-
use datafusion::prelude::{ParquetReadOptions, SessionConfig, SessionContext};
39+
use datafusion::prelude::{
40+
Expr, ParquetReadOptions, SessionConfig, SessionContext, cast, ident, lit, nullif,
41+
};
42+
use futures::TryStreamExt;
4043
use futures::future::try_join_all;
44+
use std::collections::HashSet;
4145
use std::fs;
4246
use std::path::Path;
4347
use std::sync::Arc;
@@ -259,6 +263,10 @@ pub async fn execute_query_capturing_answer(
259263
/// names, given the table and column name, or `None` to read it as a string.
260264
pub type PathColumnType = fn(&str, &str) -> Option<DataType>;
261265

266+
/// The value that Hive and Spark put in a partition directory's name when the
267+
/// partition key is NULL, as in `ss_sold_date_sk=__HIVE_DEFAULT_PARTITION__/`.
268+
const HIVE_DEFAULT_PARTITION: &str = "__HIVE_DEFAULT_PARTITION__";
269+
262270
/// A Parquet table's location and layout. Inferring the layout reads the
263271
/// files, so the runners do it once per table and register the result on each
264272
/// per-query session, which then reads nothing.
@@ -267,6 +275,9 @@ pub struct ParquetTable {
267275
pub path: String,
268276
pub schema: SchemaRef,
269277
pub partition_cols: Vec<(String, DataType)>,
278+
/// The partition columns that have a `__HIVE_DEFAULT_PARTITION__`
279+
/// directory (see [`ParquetTable::register`]).
280+
pub null_partition_cols: Vec<String>,
270281
}
271282

272283
impl ParquetTable {
@@ -294,25 +305,109 @@ impl ParquetTable {
294305
path_column_type,
295306
)
296307
.await?;
308+
let null_partition_cols =
309+
null_partition_cols(&state, &table_url, &options, &partition_cols).await?;
297310
Ok(Self {
298311
name: name.to_string(),
299312
path: path.to_string(),
300313
schema,
301314
partition_cols,
315+
null_partition_cols,
302316
})
303317
}
304318

305319
/// Registers the table on `ctx` with the inferred layout. Filters on the
306320
/// partition columns then skip whole directories at planning time instead
307321
/// of reading every file's footer.
322+
///
323+
/// DataFusion casts each partition directory's value to the column's type,
324+
/// and `__HIVE_DEFAULT_PARTITION__` only casts to a string
325+
/// (<https://github.com/apache/datafusion/issues/18083>). So when a column
326+
/// has that directory, the files are registered as `<name>_raw` with the
327+
/// column as a string, and `<name>` is a view that maps that directory to
328+
/// NULL and casts the other values back. Filters on the column still skip
329+
/// directories.
308330
pub async fn register(&self, ctx: &SessionContext) -> Result<()> {
331+
if self.null_partition_cols.is_empty() {
332+
return self
333+
.register_files(ctx, &self.name, self.partition_cols.clone())
334+
.await;
335+
}
336+
337+
let mut raw_partition_cols = vec![];
338+
let mut columns: Vec<Expr> = self
339+
.schema
340+
.fields()
341+
.iter()
342+
.map(|field| ident(field.name()))
343+
.collect();
344+
for (name, data_type) in &self.partition_cols {
345+
if self.null_partition_cols.contains(name) {
346+
raw_partition_cols.push((name.clone(), DataType::Utf8));
347+
let value = nullif(ident(name), lit(HIVE_DEFAULT_PARTITION));
348+
columns.push(cast(value, data_type.clone()).alias(name));
349+
} else {
350+
raw_partition_cols.push((name.clone(), data_type.clone()));
351+
columns.push(ident(name));
352+
}
353+
}
354+
355+
let raw_name = format!("{}_raw", self.name);
356+
self.register_files(ctx, &raw_name, raw_partition_cols)
357+
.await?;
358+
let view = ctx.table(raw_name.as_str()).await?.select(columns)?;
359+
ctx.register_table(self.name.as_str(), view.into_view())?;
360+
Ok(())
361+
}
362+
363+
/// Registers the files as table `name` with the given partition columns.
364+
async fn register_files(
365+
&self,
366+
ctx: &SessionContext,
367+
name: &str,
368+
partition_cols: Vec<(String, DataType)>,
369+
) -> Result<()> {
309370
let options = ParquetReadOptions::default()
310371
.schema(&self.schema)
311-
.table_partition_cols(self.partition_cols.clone());
312-
ctx.register_parquet(&self.name, &self.path, options).await
372+
.table_partition_cols(partition_cols);
373+
ctx.register_parquet(name, &self.path, options).await
313374
}
314375
}
315376

377+
/// The columns in `partition_cols` that have a `__HIVE_DEFAULT_PARTITION__`
378+
/// directory anywhere under `table_url`. Finding them lists every file, so an
379+
/// unpartitioned table skips the listing.
380+
async fn null_partition_cols(
381+
state: &SessionState,
382+
table_url: &ListingTableUrl,
383+
options: &ListingOptions,
384+
partition_cols: &[(String, DataType)],
385+
) -> Result<Vec<String>> {
386+
if partition_cols.is_empty() {
387+
return Ok(vec![]);
388+
}
389+
let null_dir_suffix = format!("={HIVE_DEFAULT_PARTITION}");
390+
let store = state.runtime_env().object_store(table_url)?;
391+
let mut files = table_url
392+
.list_all_files(state, store.as_ref(), &options.file_extension)
393+
.await?;
394+
let mut null_cols = HashSet::new();
395+
while let Some(file) = files.try_next().await? {
396+
let dirs = table_url.strip_prefix(&file.location).into_iter().flatten();
397+
for dir in dirs {
398+
if let Some(name) = dir.strip_suffix(&null_dir_suffix) {
399+
null_cols.insert(name.to_string());
400+
}
401+
}
402+
}
403+
Ok(partition_cols
404+
.iter()
405+
.map(|(name, _)| name)
406+
.filter(|name| null_cols.contains(*name))
407+
.cloned()
408+
.collect())
409+
}
410+
316411
/// Infers the layout of each named table under `path`, concurrently. Works for
317412
/// both a single `<table>.parquet` file and a partitioned `<table>/` directory
318413
/// (see `find_path`).

0 commit comments

Comments
 (0)