Skip to content

Commit b15cca0

Browse files
fix[duckdb]: handle dict with constant value with validity exporter (#4544)
Signed-off-by: Joe Isaacs <[email protected]>
1 parent dbbd62f commit b15cca0

5 files changed

Lines changed: 112 additions & 18 deletions

File tree

‎vortex-duckdb/src/exporter/constant.rs‎

Lines changed: 24 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -2,20 +2,40 @@
22
// SPDX-FileCopyrightText: Copyright the Vortex contributors
33

44
use vortex::arrays::ConstantArray;
5-
use vortex::dtype::DType;
65
use vortex::error::VortexResult;
76
use vortex::mask::Mask;
87

98
use crate::convert::ToDuckDBScalar;
109
use crate::duckdb::{Value, Vector};
11-
use crate::exporter::{ColumnExporter, VectorExt};
10+
use crate::exporter::{ColumnExporter, ConversionCache, VectorExt, new_array_exporter, validity};
1211

1312
struct ConstantExporter {
1413
value: Option<Value>,
1514
}
1615

16+
pub fn new_exporter_with_mask(
17+
array: &ConstantArray,
18+
mask: Mask,
19+
cache: &ConversionCache,
20+
) -> VortexResult<Box<dyn ColumnExporter>> {
21+
if mask.all_false() {
22+
return Ok(Box::new(ConstantExporter { value: None }));
23+
}
24+
25+
// duckdb cannot have a nullable constant vector, so we create primitive vector with validity mask
26+
if !mask.all_true() {
27+
// TODO(joe): we can splat the constant in a specific exporter and save a copy.
28+
return Ok(validity::new_exporter(
29+
mask,
30+
new_array_exporter(array.to_canonical().as_ref(), cache)?,
31+
));
32+
}
33+
34+
new_exporter(array)
35+
}
36+
1737
pub(crate) fn new_exporter(array: &ConstantArray) -> VortexResult<Box<dyn ColumnExporter>> {
18-
let value = if array.scalar().is_null() && !matches!(array.dtype(), DType::Null) {
38+
let value = if array.scalar().is_null() {
1939
// If the scalar is null and _not_ of type Null, then we cannot assign a null DuckDB value
2040
// to a constant vector since DuckDB will complain about a type-mismatch. In these cases,
2141
// we need to create an all-null flat vector instead.
@@ -25,6 +45,7 @@ pub(crate) fn new_exporter(array: &ConstantArray) -> VortexResult<Box<dyn Column
2545
// constant all-null DuckDB vector.
2646
Some(array.scalar().try_to_duckdb_scalar()?)
2747
};
48+
2849
Ok(Box::new(ConstantExporter { value }))
2950
}
3051

‎vortex-duckdb/src/exporter/dict.rs‎

Lines changed: 45 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -7,15 +7,15 @@ use std::sync::Arc;
77
use bitvec::macros::internal::funty::Fundamental;
88
use num_traits::AsPrimitive;
99
use parking_lot::Mutex;
10-
use vortex::arrays::PrimitiveArray;
10+
use vortex::arrays::{ConstantArray, ConstantVTable, PrimitiveArray};
1111
use vortex::dtype::{NativePType, match_each_integer_ptype};
1212
use vortex::encodings::dict::DictArray;
1313
use vortex::error::VortexResult;
1414
use vortex::{Array, ToCanonical};
1515

1616
use crate::duckdb::{SelectionVector, Vector};
1717
use crate::exporter::cache::ConversionCache;
18-
use crate::exporter::{ColumnExporter, new_array_exporter};
18+
use crate::exporter::{ColumnExporter, constant, new_array_exporter};
1919

2020
struct DictExporter<I: NativePType> {
2121
// Store the dictionary values once and export the same dictionary with each codes chunk.
@@ -33,6 +33,14 @@ pub(crate) fn new_exporter(
3333
) -> VortexResult<Box<dyn ColumnExporter>> {
3434
// Grab the cache dictionary values.
3535
let values = array.values();
36+
if let Some(constant) = values.as_opt::<ConstantVTable>() {
37+
return constant::new_exporter_with_mask(
38+
&ConstantArray::new(constant.scalar().clone(), array.codes().len()),
39+
array.codes().validity_mask(),
40+
cache,
41+
);
42+
}
43+
3644
let values_key = Arc::as_ptr(values).addr();
3745

3846
// Check if we have a cached vector and extract it if we do.
@@ -95,3 +103,38 @@ impl<I: NativePType + AsPrimitive<u32>> ColumnExporter for DictExporter<I> {
95103
Ok(())
96104
}
97105
}
106+
107+
#[cfg(test)]
108+
mod tests {
109+
use vortex::IntoArray;
110+
use vortex::arrays::{ConstantArray, PrimitiveArray};
111+
use vortex::encodings::dict::DictArray;
112+
113+
use crate::cpp;
114+
use crate::duckdb::{DataChunk, LogicalType};
115+
use crate::exporter::ConversionCache;
116+
use crate::exporter::dict::new_exporter;
117+
118+
#[test]
119+
fn test_constant_dict() {
120+
let arr = DictArray::new(
121+
PrimitiveArray::from_option_iter([None, Some(0u32)]).into_array(),
122+
ConstantArray::new(10, 1).into_array(),
123+
);
124+
125+
let mut chunk = DataChunk::new([LogicalType::new(cpp::duckdb_type::DUCKDB_TYPE_INTEGER)]);
126+
127+
new_exporter(&arr, &ConversionCache::default())
128+
.unwrap()
129+
.export(0, 2, &mut chunk.get_vector(0))
130+
.unwrap();
131+
chunk.set_len(2);
132+
133+
assert_eq!(
134+
format!("{}", String::try_from(&chunk).unwrap()),
135+
r#"Chunk - [1 Columns]
136+
- FLAT INTEGER: 2 = [ NULL, 10]
137+
"#
138+
);
139+
}
140+
}

‎vortex-duckdb/src/exporter/mod.rs‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@ mod primitive;
1111
mod run_end;
1212
mod sequence;
1313
mod temporal;
14+
mod validity;
1415
mod varbinview;
1516

1617
use std::sync::Arc;

‎vortex-duckdb/src/exporter/primitive.rs‎

Lines changed: 9 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -6,35 +6,31 @@ use std::marker::PhantomData;
66
use vortex::arrays::PrimitiveArray;
77
use vortex::dtype::{NativePType, match_each_native_ptype};
88
use vortex::error::VortexResult;
9-
use vortex::mask::Mask;
109

1110
use crate::duckdb::Vector;
12-
use crate::exporter::{ColumnExporter, VectorExt};
11+
use crate::exporter::{ColumnExporter, validity};
1312

1413
struct PrimitiveExporter<T: NativePType> {
1514
array: PrimitiveArray,
1615
array_type: PhantomData<T>,
17-
validity: Mask,
1816
}
1917

2018
pub(crate) fn new_exporter(array: &PrimitiveArray) -> VortexResult<Box<dyn ColumnExporter>> {
21-
Ok(match_each_native_ptype!(array.ptype(), |T| {
19+
let prim = match_each_native_ptype!(array.ptype(), |T| {
2220
Box::new(PrimitiveExporter {
2321
array: array.clone(),
2422
array_type: PhantomData::<T>,
25-
validity: array.validity_mask(),
26-
})
27-
}))
23+
}) as Box<dyn ColumnExporter>
24+
});
25+
Ok(if array.dtype().is_nullable() {
26+
validity::new_exporter(array.validity_mask(), prim)
27+
} else {
28+
prim
29+
})
2830
}
2931

3032
impl<T: NativePType> ColumnExporter for PrimitiveExporter<T> {
3133
fn export(&self, offset: usize, len: usize, vector: &mut Vector) -> VortexResult<()> {
32-
// Set validity if necessary.
33-
if unsafe { vector.set_validity(&self.validity, offset, len) } {
34-
// All values are null, so no point copying the data.
35-
return Ok(());
36-
}
37-
3834
// Copy the values from the Vortex array to the DuckDB vector.
3935
unsafe { vector.as_slice_mut(len) }
4036
.copy_from_slice(&self.array.as_slice::<T>()[offset..offset + len]);
Lines changed: 33 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,33 @@
1+
// SPDX-License-Identifier: Apache-2.0
2+
// SPDX-FileCopyrightText: Copyright the Vortex contributors
3+
4+
use vortex::error::VortexResult;
5+
use vortex::mask::Mask;
6+
7+
use crate::duckdb::Vector;
8+
use crate::exporter::{ColumnExporter, VectorExt};
9+
10+
struct ValidityExporter {
11+
mask: Mask,
12+
exporter: Box<dyn ColumnExporter>,
13+
}
14+
15+
pub(crate) fn new_exporter(
16+
mask: Mask,
17+
exporter: Box<dyn ColumnExporter>,
18+
) -> Box<dyn ColumnExporter> {
19+
Box::new(ValidityExporter { mask, exporter })
20+
}
21+
22+
impl ColumnExporter for ValidityExporter {
23+
fn export(&self, offset: usize, len: usize, vector: &mut Vector) -> VortexResult<()> {
24+
if unsafe { vector.set_validity(&self.mask, offset, len) } {
25+
// All values are null, so no point copying the data.
26+
return Ok(());
27+
}
28+
29+
self.exporter.export(offset, len, vector)?;
30+
31+
Ok(())
32+
}
33+
}

0 commit comments

Comments
 (0)