Skip to content

Commit 7274de3

Browse files
committed
Merge branch 'main' into feat/hash-join-dynamic-filter-with-range-partition
2 parents f9f1f0d + a05388e commit 7274de3

43 files changed

Lines changed: 8337 additions & 6101 deletions

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎Cargo.lock‎

Lines changed: 1 addition & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎datafusion/common/src/test_util.rs‎

Lines changed: 23 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -621,7 +621,29 @@ pub mod array_conversion {
621621
}
622622
}
623623

624-
//#TODO add impl for f16
624+
impl IntoArrayRef for Vec<half::f16> {
625+
fn into_array_ref(self) -> ArrayRef {
626+
create_array!(Float16, self)
627+
}
628+
}
629+
630+
impl IntoArrayRef for Vec<Option<half::f16>> {
631+
fn into_array_ref(self) -> ArrayRef {
632+
create_array!(Float16, self)
633+
}
634+
}
635+
636+
impl IntoArrayRef for &[half::f16] {
637+
fn into_array_ref(self) -> ArrayRef {
638+
create_array!(Float16, self.to_vec())
639+
}
640+
}
641+
642+
impl IntoArrayRef for &[Option<half::f16>] {
643+
fn into_array_ref(self) -> ArrayRef {
644+
create_array!(Float16, self.to_vec())
645+
}
646+
}
625647

626648
impl IntoArrayRef for Vec<f32> {
627649
fn into_array_ref(self) -> ArrayRef {

‎datafusion/core/Cargo.toml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -173,6 +173,7 @@ bytes = { workspace = true }
173173
env_logger = { workspace = true }
174174
glob = { workspace = true }
175175
insta = { workspace = true }
176+
half = { workspace = true }
176177
rand = { workspace = true, features = ["small_rng"] }
177178
rand_distr = "0.5"
178179
recursive = { workspace = true }

‎datafusion/core/tests/dataframe/mod.rs‎

Lines changed: 148 additions & 29 deletions
Original file line numberDiff line numberDiff line change
@@ -21,9 +21,10 @@ mod describe;
2121

2222
use arrow::array::{
2323
Array, ArrayRef, BooleanArray, DictionaryArray, FixedSizeListArray,
24-
FixedSizeListBuilder, Float32Array, Float64Array, Int8Array, Int32Array,
25-
Int32Builder, LargeListArray, ListArray, ListBuilder, RecordBatch, StringArray,
26-
StringBuilder, StructBuilder, UInt32Array, UInt32Builder, UnionArray, record_batch,
24+
FixedSizeListBuilder, Float16Array, Float32Array, Float64Array, Int8Array,
25+
Int16Array, Int32Array, Int32Builder, Int64Array, LargeListArray, ListArray,
26+
ListBuilder, RecordBatch, StringArray, StringBuilder, StructBuilder, UInt8Array,
27+
UInt16Array, UInt32Array, UInt32Builder, UInt64Array, UnionArray, record_batch,
2728
};
2829
use arrow::buffer::ScalarBuffer;
2930
use arrow::datatypes::{
@@ -6941,24 +6942,80 @@ async fn test_insert_into_casting_support() -> Result<()> {
69416942

69426943
#[tokio::test]
69436944
async fn test_dataframe_from_columns() -> Result<()> {
6944-
let a: ArrayRef = Arc::new(Int32Array::from(vec![1, 2, 3]));
6945-
let b: ArrayRef = Arc::new(BooleanArray::from(vec![true, true, false]));
6946-
let c: ArrayRef = Arc::new(StringArray::from(vec![Some("foo"), Some("bar"), None]));
6947-
let df = DataFrame::from_columns(vec![("a", a), ("b", b), ("c", c)])?;
6945+
let bools: ArrayRef = Arc::new(BooleanArray::from(vec![true, false, true]));
6946+
let i8s: ArrayRef = Arc::new(Int8Array::from(vec![-1, 0, 1]));
6947+
let i16s: ArrayRef = Arc::new(Int16Array::from(vec![-1, 0, 1]));
6948+
let i32s: ArrayRef = Arc::new(Int32Array::from(vec![-1, 0, 1]));
6949+
let i64s: ArrayRef = Arc::new(Int64Array::from(vec![-1, 0, 1]));
6950+
6951+
let u8s: ArrayRef = Arc::new(UInt8Array::from(vec![0, 1, 2]));
6952+
let u16s: ArrayRef = Arc::new(UInt16Array::from(vec![0, 1, 2]));
6953+
let u32s: ArrayRef = Arc::new(UInt32Array::from(vec![0, 1, 2]));
6954+
let u64s: ArrayRef = Arc::new(UInt64Array::from(vec![0, 1, 2]));
6955+
6956+
let f16s: ArrayRef = Arc::new(Float16Array::from(vec![
6957+
half::f16::from_f64(1.0),
6958+
half::f16::from_f64(2.0),
6959+
half::f16::from_f64(3.0),
6960+
]));
6961+
let f32s: ArrayRef = Arc::new(Float32Array::from(vec![1.0, 2.0, 3.0]));
6962+
let f64s: ArrayRef = Arc::new(Float64Array::from(vec![1.0, 2.0, 3.0]));
6963+
6964+
let strings: ArrayRef =
6965+
Arc::new(StringArray::from(vec![Some("foo"), Some("bar"), None]));
6966+
6967+
let df = DataFrame::from_columns(vec![
6968+
("bool", bools),
6969+
("i8", i8s),
6970+
("i16", i16s),
6971+
("i32", i32s),
6972+
("i64", i64s),
6973+
("u8", u8s),
6974+
("u16", u16s),
6975+
("u32", u32s),
6976+
("u64", u64s),
6977+
("f16", f16s),
6978+
("f32", f32s),
6979+
("f64", f64s),
6980+
("str", strings),
6981+
])?;
69486982

6949-
assert_eq!(df.schema().fields().len(), 3);
6983+
assert_eq!(df.schema().fields().len(), 13);
69506984
assert_eq!(df.clone().count().await?, 3);
69516985

6952-
let rows = df.sort(vec![col("a").sort(true, true)])?;
6986+
let expected_types = [
6987+
("bool", DataType::Boolean),
6988+
("i8", DataType::Int8),
6989+
("i16", DataType::Int16),
6990+
("i32", DataType::Int32),
6991+
("i64", DataType::Int64),
6992+
("u8", DataType::UInt8),
6993+
("u16", DataType::UInt16),
6994+
("u32", DataType::UInt32),
6995+
("u64", DataType::UInt64),
6996+
("f16", DataType::Float16),
6997+
("f32", DataType::Float32),
6998+
("f64", DataType::Float64),
6999+
("str", DataType::Utf8),
7000+
];
7001+
7002+
let schema = df.schema();
7003+
7004+
for (name, data_type) in expected_types {
7005+
assert_eq!(schema.field_with_name(None, name)?.data_type(), &data_type);
7006+
}
7007+
7008+
let rows = df.sort(vec![col("i32").sort(true, true)])?;
7009+
69537010
assert_batches_eq!(
69547011
&[
6955-
"+---+-------+-----+",
6956-
"| a | b | c |",
6957-
"+---+-------+-----+",
6958-
"| 1 | true | foo |",
6959-
"| 2 | true | bar |",
6960-
"| 3 | false | |",
6961-
"+---+-------+-----+",
7012+
"+-------+----+-----+-----+-----+----+-----+-----+-----+-----+-----+-----+-----+",
7013+
"| bool | i8 | i16 | i32 | i64 | u8 | u16 | u32 | u64 | f16 | f32 | f64 | str |",
7014+
"+-------+----+-----+-----+-----+----+-----+-----+-----+-----+-----+-----+-----+",
7015+
"| true | -1 | -1 | -1 | -1 | 0 | 0 | 0 | 0 | 1 | 1.0 | 1.0 | foo |",
7016+
"| false | 0 | 0 | 0 | 0 | 1 | 1 | 1 | 1 | 2 | 2.0 | 2.0 | bar |",
7017+
"| true | 1 | 1 | 1 | 1 | 2 | 2 | 2 | 2 | 3 | 3.0 | 3.0 | |",
7018+
"+-------+----+-----+-----+-----+----+-----+-----+-----+-----+-----+-----+-----+",
69627019
],
69637020
&rows.collect().await?
69647021
);
@@ -6968,25 +7025,86 @@ async fn test_dataframe_from_columns() -> Result<()> {
69687025

69697026
#[tokio::test]
69707027
async fn test_dataframe_macro() -> Result<()> {
7028+
let bools = [true, false, true];
7029+
let i8s = [-1_i8, 0, 1];
7030+
let i16s = [-1_i16, 0, 1];
7031+
let i32s = [-1_i32, 0, 1];
7032+
let i64s = [-1_i64, 0, 1];
7033+
7034+
let u8s = [0_u8, 1, 2];
7035+
let u16s = [0_u16, 1, 2];
7036+
let u32s = [0_u32, 1, 2];
7037+
let u64s = [0_u64, 1, 2];
7038+
7039+
let f16s = [
7040+
half::f16::from_f64(1.0),
7041+
half::f16::from_f64(2.0),
7042+
half::f16::from_f64(3.0),
7043+
];
7044+
let f32s = [1.0_f32, 2.0, 3.0];
7045+
let f64s = [1.0_f64, 2.0, 3.0];
7046+
7047+
let strings = ["foo", "bar", "baz"];
7048+
69717049
let df = dataframe!(
6972-
"a" => [1, 2, 3],
6973-
"b" => [true, true, false],
6974-
"c" => [Some("foo"), Some("bar"), None]
7050+
// Vec<T>
7051+
"bool" => bools.to_vec(),
7052+
"i8" => i8s.to_vec(),
7053+
"i16" => i16s.to_vec(),
7054+
"i32" => i32s.to_vec(),
7055+
7056+
// Vec<Option<T>>
7057+
"i64" => vec![Some(i64s[0]), None, Some(i64s[2])],
7058+
"u8" => vec![Some(u8s[0]), None, Some(u8s[2])],
7059+
"u16" => vec![Some(u16s[0]), None, Some(u16s[2])],
7060+
7061+
// &[T]
7062+
"u32" => &u32s,
7063+
"u64" => &u64s,
7064+
"f16" => &f16s,
7065+
7066+
// &[Option<T>]
7067+
"f32" => &[Some(f32s[0]), None, Some(f32s[2])],
7068+
"f64" => &[Some(f64s[0]), None, Some(f64s[2])],
7069+
"str" => &[Some(strings[0]), None, Some(strings[2])],
69757070
)?;
69767071

6977-
assert_eq!(df.schema().fields().len(), 3);
7072+
assert_eq!(df.schema().fields().len(), 13);
69787073
assert_eq!(df.clone().count().await?, 3);
69797074

6980-
let rows = df.sort(vec![col("a").sort(true, true)])?;
7075+
let expected_types = [
7076+
("bool", DataType::Boolean),
7077+
("i8", DataType::Int8),
7078+
("i16", DataType::Int16),
7079+
("i32", DataType::Int32),
7080+
("i64", DataType::Int64),
7081+
("u8", DataType::UInt8),
7082+
("u16", DataType::UInt16),
7083+
("u32", DataType::UInt32),
7084+
("u64", DataType::UInt64),
7085+
("f16", DataType::Float16),
7086+
("f32", DataType::Float32),
7087+
("f64", DataType::Float64),
7088+
("str", DataType::Utf8),
7089+
];
7090+
7091+
let schema = df.schema();
7092+
7093+
for (name, data_type) in expected_types {
7094+
assert_eq!(schema.field_with_name(None, name)?.data_type(), &data_type);
7095+
}
7096+
7097+
let rows = df.sort(vec![col("i32").sort(true, true)])?;
7098+
69817099
assert_batches_eq!(
69827100
&[
6983-
"+---+-------+-----+",
6984-
"| a | b | c |",
6985-
"+---+-------+-----+",
6986-
"| 1 | true | foo |",
6987-
"| 2 | true | bar |",
6988-
"| 3 | false | |",
6989-
"+---+-------+-----+",
7101+
"+-------+----+-----+-----+-----+----+-----+-----+-----+-----+-----+-----+-----+",
7102+
"| bool | i8 | i16 | i32 | i64 | u8 | u16 | u32 | u64 | f16 | f32 | f64 | str |",
7103+
"+-------+----+-----+-----+-----+----+-----+-----+-----+-----+-----+-----+-----+",
7104+
"| true | -1 | -1 | -1 | -1 | 0 | 0 | 0 | 0 | 1 | 1.0 | 1.0 | foo |",
7105+
"| false | 0 | 0 | 0 | | | | 1 | 1 | 2 | | | |",
7106+
"| true | 1 | 1 | 1 | 1 | 2 | 2 | 2 | 2 | 3 | 3.0 | 3.0 | baz |",
7107+
"+-------+----+-----+-----+-----+----+-----+-----+-----+-----+-----+-----+-----+",
69907108
],
69917109
&rows.collect().await?
69927110
);
@@ -7075,7 +7193,8 @@ async fn test_copy_to_preserves_order() -> Result<()> {
70757193
DataSinkExec: sink=CsvSink(file_groups=[])
70767194
SortExec: expr=[column1@0 DESC], preserve_partitioning=[false]
70777195
DataSourceExec: partitions=1, partition_sizes=[1]
7078-
DataSourceExec: partitions=1, partition_sizes=[1]
7196+
ProjectionExec: expr=[CAST(column1@0 AS UInt64) as count]
7197+
DataSourceExec: partitions=1, partition_sizes=[1]
70797198
"
70807199
);
70817200
Ok(())

‎datafusion/core/tests/parquet/ordering.rs‎

Lines changed: 80 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -101,3 +101,83 @@ async fn test_create_table_with_order_writes_sorting_columns() -> Result<()> {
101101

102102
Ok(())
103103
}
104+
105+
/// Test that partition columns are removed and remaining column indices are
106+
/// remapped when writing sorting_columns to Parquet metadata.
107+
#[tokio::test]
108+
async fn test_partitioned_table_remaps_sorting_columns() -> Result<()> {
109+
use parquet::file::reader::FileReader;
110+
use parquet::file::serialized_reader::SerializedFileReader;
111+
use std::fs::File;
112+
113+
let ctx = SessionContext::new();
114+
let tmp_dir = tempdir()?;
115+
let table_path = tmp_dir.path().join("sorted_partitioned_table");
116+
std::fs::create_dir_all(&table_path)?;
117+
118+
let create_table_sql = format!(
119+
"CREATE EXTERNAL TABLE sorted_partitioned_data (a INT, b VARCHAR, part VARCHAR) \
120+
STORED AS PARQUET \
121+
LOCATION '{}' \
122+
PARTITIONED BY (part) \
123+
WITH ORDER (part ASC NULLS FIRST, a ASC NULLS FIRST, b DESC NULLS LAST)",
124+
table_path.display()
125+
);
126+
ctx.sql(&create_table_sql).await?;
127+
128+
ctx.sql(
129+
"INSERT INTO sorted_partitioned_data VALUES \
130+
(2, 'c', 'x'), (1, 'a', 'x'), (1, 'b', 'x')",
131+
)
132+
.await?
133+
.collect()
134+
.await?;
135+
136+
let parquet_file = find_parquet_file(&table_path)?
137+
.expect("expected a Parquet file in the partition directory");
138+
139+
let file = File::open(parquet_file)?;
140+
let reader = SerializedFileReader::new(file)?;
141+
let metadata = reader.metadata();
142+
let parquet_schema = metadata.file_metadata().schema_descr();
143+
assert_eq!(parquet_schema.num_columns(), 2);
144+
assert_eq!(parquet_schema.column(0).name(), "a");
145+
assert_eq!(parquet_schema.column(1).name(), "b");
146+
147+
let sorting = metadata
148+
.row_group(0)
149+
.sorting_columns()
150+
.expect("expected sorting_columns in row group metadata");
151+
assert_eq!(sorting.len(), 2);
152+
153+
assert_eq!(sorting[0].column_idx, 0);
154+
assert!(!sorting[0].descending);
155+
assert!(sorting[0].nulls_first);
156+
157+
assert_eq!(sorting[1].column_idx, 1);
158+
assert!(sorting[1].descending);
159+
assert!(!sorting[1].nulls_first);
160+
161+
Ok(())
162+
}
163+
164+
fn find_parquet_file(
165+
path: &std::path::Path,
166+
) -> std::io::Result<Option<std::path::PathBuf>> {
167+
for entry in std::fs::read_dir(path)? {
168+
let entry = entry?;
169+
let path = entry.path();
170+
if path.is_dir() {
171+
if let Some(path) = find_parquet_file(&path)? {
172+
return Ok(Some(path));
173+
}
174+
} else if path
175+
.extension()
176+
.is_some_and(|extension| extension == "parquet")
177+
{
178+
return Ok(Some(path));
179+
}
180+
}
181+
182+
Ok(None)
183+
}

‎datafusion/datasource-parquet/src/file_format.rs‎

Lines changed: 9 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -50,6 +50,7 @@ use datafusion_common::{
5050
use datafusion_datasource::file::FileSource;
5151
use datafusion_datasource::file_scan_config::{FileScanConfig, FileScanConfigBuilder};
5252
use datafusion_datasource::sink::DataSinkExec;
53+
use datafusion_datasource::write::get_writer_schema;
5354
use datafusion_expr::dml::InsertOp;
5455
use datafusion_physical_expr_common::sort_expr::{LexOrdering, LexRequirement};
5556
use datafusion_physical_plan::ExecutionPlan;
@@ -534,11 +535,18 @@ impl FileFormat for ParquetFormat {
534535
// Convert ordering requirements to Parquet SortingColumns for file metadata
535536
let sorting_columns = if let Some(ref requirements) = order_requirements {
536537
let ordering: LexOrdering = requirements.clone().into();
538+
let writer_schema = get_writer_schema(&conf);
537539
// In cases like `COPY (... ORDER BY ...) TO ...` the ORDER BY clause
538540
// may not be compatible with Parquet sorting columns (e.g. ordering on `random()`).
539541
// So if we cannot create a Parquet sorting column from the ordering requirement,
540542
// we skip setting sorting columns on the Parquet sink.
541-
lex_ordering_to_sorting_columns(&ordering).ok()
543+
lex_ordering_to_sorting_columns(
544+
&ordering,
545+
conf.output_schema(),
546+
&writer_schema,
547+
)
548+
.ok()
549+
.filter(|columns| !columns.is_empty())
542550
} else {
543551
None
544552
};

0 commit comments

Comments
 (0)