Skip to content

Commit d19c6ac

Browse files
committed
adding safe support to to_date and to_timestamp functions. Fix ListingTable to not allow duplicated partitions.
1 parent 5411b4c commit d19c6ac

4 files changed

Lines changed: 360 additions & 87 deletions

File tree

‎datafusion/core/src/datasource/listing/table.rs‎

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -990,7 +990,14 @@ impl ListingTable {
990990
// Add the partition columns to the file schema
991991
let mut builder = SchemaBuilder::from(file_schema.as_ref().to_owned());
992992
for (part_col_name, part_col_type) in &options.table_partition_cols {
993-
builder.push(Field::new(part_col_name, part_col_type.clone(), false));
993+
// only add the partition if it is not already in the file_schema
994+
if !file_schema
995+
.fields
996+
.iter()
997+
.any(|f| f.name().eq_ignore_ascii_case(part_col_name))
998+
{
999+
builder.push(Field::new(part_col_name, part_col_type.clone(), false));
1000+
}
9941001
}
9951002

9961003
let table_schema = Arc::new(

‎datafusion/functions/src/datetime/common.rs‎

Lines changed: 45 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -183,6 +183,7 @@ pub(crate) fn handle<O, F, S>(
183183
args: &[ColumnarValue],
184184
op: F,
185185
name: &str,
186+
safe: bool,
186187
) -> Result<ColumnarValue>
187188
where
188189
O: ArrowPrimitiveType,
@@ -195,26 +196,35 @@ where
195196
unary_string_to_primitive_function::<&StringViewArray, O, _>(
196197
a.as_ref().as_string_view(),
197198
op,
199+
safe,
198200
)?,
199201
))),
200202
DataType::LargeUtf8 => Ok(ColumnarValue::Array(Arc::new(
201203
unary_string_to_primitive_function::<&GenericStringArray<i64>, O, _>(
202204
a.as_ref().as_string::<i64>(),
203205
op,
206+
safe,
204207
)?,
205208
))),
206209
DataType::Utf8 => Ok(ColumnarValue::Array(Arc::new(
207210
unary_string_to_primitive_function::<&GenericStringArray<i32>, O, _>(
208211
a.as_ref().as_string::<i32>(),
209212
op,
213+
safe,
210214
)?,
211215
))),
212216
other => exec_err!("Unsupported data type {other:?} for function {name}"),
213217
},
214218
ColumnarValue::Scalar(scalar) => match scalar.try_as_str() {
215219
Some(a) => {
216-
let result = a.as_ref().map(|x| op(x)).transpose()?;
217-
Ok(ColumnarValue::Scalar(S::scalar(result)))
220+
let result = a.as_ref().map(|x| op(x)).transpose();
221+
if let Ok(v) = result {
222+
Ok(ColumnarValue::Scalar(S::scalar(v)))
223+
} else if safe {
224+
Ok(ColumnarValue::Scalar(S::scalar(None)))
225+
} else {
226+
Err(result.err().unwrap())
227+
}
218228
}
219229
_ => exec_err!("Unsupported data type {scalar:?} for function {name}"),
220230
},
@@ -229,6 +239,7 @@ pub(crate) fn handle_multiple<O, F, S, M>(
229239
op: F,
230240
op2: M,
231241
name: &str,
242+
safe: bool,
232243
) -> Result<ColumnarValue>
233244
where
234245
O: ArrowPrimitiveType,
@@ -260,7 +271,7 @@ where
260271
}
261272

262273
Ok(ColumnarValue::Array(Arc::new(
263-
strings_to_primitive_function::<O, _, _>(args, op, op2, name)?,
274+
strings_to_primitive_function::<O, _, _>(args, op, op2, name, safe)?,
264275
)))
265276
}
266277
other => {
@@ -270,12 +281,12 @@ where
270281
// if the first argument is a scalar utf8 all arguments are expected to be scalar utf8
271282
ColumnarValue::Scalar(scalar) => match scalar.try_as_str() {
272283
Some(a) => {
284+
let mut val: Option<Result<ColumnarValue>> = None;
285+
let mut err: Option<DataFusionError> = None;
273286
let a = a.as_ref();
274287
// ASK: Why do we trust `a` to be non-null at this point?
275288
let a = unwrap_or_internal_err!(a);
276289

277-
let mut ret = None;
278-
279290
for (pos, v) in args.iter().enumerate().skip(1) {
280291
let ColumnarValue::Scalar(
281292
ScalarValue::Utf8View(x)
@@ -289,17 +300,26 @@ where
289300
if let Some(s) = x {
290301
match op(a, s.as_str()) {
291302
Ok(r) => {
292-
ret = Some(Ok(ColumnarValue::Scalar(S::scalar(Some(
303+
val = Some(Ok(ColumnarValue::Scalar(S::scalar(Some(
293304
op2(r),
294305
)))));
295306
break;
296307
}
297-
Err(e) => ret = Some(Err(e)),
308+
Err(e) => err = Some(e),
298309
}
299310
}
300311
}
301312

302-
unwrap_or_internal_err!(ret)
313+
if let Some(v) = val {
314+
v
315+
} else if safe {
316+
Ok(ColumnarValue::Scalar(S::scalar(None)))
317+
} else {
318+
match err {
319+
Some(e) => Err(e),
320+
None => Ok(ColumnarValue::Scalar(S::scalar(None))),
321+
}
322+
}
303323
}
304324
other => {
305325
exec_err!("Unsupported data type {other:?} for function {name}")
@@ -317,12 +337,13 @@ where
317337
/// # Errors
318338
/// This function errors iff:
319339
/// * the number of arguments is not > 1 or
320-
/// * the function `op` errors for all input
340+
/// * the function `op` errors for all input and safe is false
321341
pub(crate) fn strings_to_primitive_function<O, F, F2>(
322342
args: &[ColumnarValue],
323343
op: F,
324344
op2: F2,
325345
name: &str,
346+
safe: bool,
326347
) -> Result<PrimitiveArray<O>>
327348
where
328349
O: ArrowPrimitiveType,
@@ -346,6 +367,7 @@ where
346367
&args[1..],
347368
op,
348369
op2,
370+
safe,
349371
)
350372
}
351373
DataType::LargeUtf8 => {
@@ -355,6 +377,7 @@ where
355377
&args[1..],
356378
op,
357379
op2,
380+
safe,
358381
)
359382
}
360383
DataType::Utf8 => {
@@ -364,6 +387,7 @@ where
364387
&args[1..],
365388
op,
366389
op2,
390+
safe,
367391
)
368392
}
369393
other => exec_err!(
@@ -383,6 +407,7 @@ fn handle_array_op<'a, O, V, F, F2>(
383407
args: &[ColumnarValue],
384408
op: F,
385409
op2: F2,
410+
safe: bool,
386411
) -> Result<PrimitiveArray<O>>
387412
where
388413
V: StringArrayType<'a>,
@@ -422,6 +447,7 @@ where
422447
};
423448

424449
val.transpose()
450+
.or_else(|e| if safe { Ok(None) } else { Err(e) })
425451
})
426452
.collect()
427453
}
@@ -432,16 +458,24 @@ where
432458
/// # Errors
433459
/// This function errors iff:
434460
/// * the number of arguments is not 1 or
435-
/// * the function `op` errors
461+
/// * the function `op` errors and safe is false
436462
fn unary_string_to_primitive_function<'a, StringArrType, O, F>(
437463
array: StringArrType,
438464
op: F,
465+
safe: bool,
439466
) -> Result<PrimitiveArray<O>>
440467
where
441468
StringArrType: StringArrayType<'a>,
442469
O: ArrowPrimitiveType,
443470
F: Fn(&'a str) -> Result<O::Native>,
444471
{
445472
// first map is the iterator, second is for the `Option<_>`
446-
array.iter().map(|x| x.map(&op).transpose()).collect()
473+
array
474+
.iter()
475+
.map(|x| {
476+
x.map(&op)
477+
.transpose()
478+
.or_else(|e| if safe { Ok(None) } else { Err(e) })
479+
})
480+
.collect()
447481
}

‎datafusion/functions/src/datetime/to_date.rs‎

Lines changed: 17 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,7 @@
1616
// under the License.
1717

1818
use crate::datetime::common::*;
19+
use arrow::compute::CastOptions;
1920
use arrow::datatypes::DataType;
2021
use arrow::datatypes::DataType::*;
2122
use arrow::error::ArrowError::ParseError;
@@ -66,6 +67,8 @@ Additional examples can be found [here](https://github.com/apache/datafusion/blo
6667
#[derive(Debug, PartialEq, Eq, Hash)]
6768
pub struct ToDateFunc {
6869
signature: Signature,
70+
/// how to handle cast or parsing failures, either return NULL (safe=true) or return ERR (safe=false)
71+
safe: bool,
6972
}
7073

7174
impl Default for ToDateFunc {
@@ -78,6 +81,14 @@ impl ToDateFunc {
7881
pub fn new() -> Self {
7982
Self {
8083
signature: Signature::variadic_any(Volatility::Immutable),
84+
safe: false,
85+
}
86+
}
87+
88+
pub fn new_with_safe(safe: bool) -> Self {
89+
Self {
90+
signature: Signature::variadic_any(Volatility::Immutable),
91+
safe,
8192
}
8293
}
8394

@@ -93,6 +104,7 @@ impl ToDateFunc {
93104
)),
94105
},
95106
"to_date",
107+
self.safe,
96108
),
97109
2.. => handle_multiple::<Date32Type, _, Date32Type, _>(
98110
args,
@@ -107,6 +119,7 @@ impl ToDateFunc {
107119
},
108120
|n| n,
109121
"to_date",
122+
self.safe,
110123
),
111124
0 => exec_err!("Unsupported 0 argument count for function to_date"),
112125
}
@@ -145,9 +158,10 @@ impl ScalarUDFImpl for ToDateFunc {
145158
}
146159

147160
match args[0].data_type() {
148-
Int32 | Int64 | Null | Float64 | Date32 | Date64 => {
149-
args[0].cast_to(&Date32, None)
150-
}
161+
Int32 | Int64 | Null | Float64 | Date32 | Date64 => match self.safe {
162+
true => args[0].cast_to(&Date32, Some(&CastOptions::default())),
163+
false => args[0].cast_to(&Date32, None),
164+
},
151165
Utf8View | LargeUtf8 | Utf8 => self.to_date(&args),
152166
other => {
153167
exec_err!("Unsupported data type {:?} for function to_date", other)

0 commit comments

Comments
 (0)