Skip to content

Commit e4f9b03

Browse files
mikamikasukimac
andauthored
fix: validate temporal display formats (apache#26045)
## Which issue does this PR close? - Closes apache#24909 ## Rationale for this change An invalid temporal display format such as '%' can reach Arrow's table formatter and panic when the DataFusion CLI renders a time value. Invalid formats should return a configuration error before rendering. ## What changes are included in this PR? Validate configured date, datetime, timestamp, timestamp-with-timezone, and time formats when converting DataFusion format options to Arrow format options. Invalid values now return a configuration error that identifies the datafusion.format.* setting. ## What is the testing strategy for this PR? Added invalid_temporal_format_is_rejected_before_rendering, covering all five settings and confirming defaults remain valid. The CLI reproducer now returns a configuration error instead of panicking. The datafusion-common test suite, extended workspace tests, full workspace Clippy with -D warnings, formatting, and git diff --check passed. ## Are there any user-facing changes? Invalid temporal format settings now return a clear configuration error instead of panicking while query results are rendered. --------- Co-authored-by: mac <[email protected]> Co-authored-by: mika <[email protected]>
1 parent 14aa185 commit e4f9b03

2 files changed

Lines changed: 226 additions & 2 deletions

File tree

‎datafusion-cli/src/print_format.rs‎

Lines changed: 44 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -206,8 +206,8 @@ mod tests {
206206
use super::*;
207207
use std::sync::Arc;
208208

209-
use arrow::array::Int32Array;
210-
use arrow::datatypes::{DataType, Field, Schema};
209+
use arrow::array::{Int32Array, Time64NanosecondArray};
210+
use arrow::datatypes::{DataType, Field, Schema, TimeUnit};
211211
use insta::{allow_duplicates, assert_snapshot};
212212

213213
#[test]
@@ -489,6 +489,48 @@ mod tests {
489489
assert_eq!(output, "")
490490
}
491491

492+
#[test]
493+
fn invalid_temporal_format_errors_before_table_output() {
494+
let schema = Arc::new(Schema::new(vec![Field::new(
495+
"time",
496+
DataType::Time64(TimeUnit::Nanosecond),
497+
false,
498+
)]));
499+
let batch = RecordBatch::try_new(
500+
schema.clone(),
501+
vec![Arc::new(Time64NanosecondArray::from(vec![1_000, 2_000]))],
502+
)
503+
.unwrap();
504+
let empty_batch = RecordBatch::new_empty(schema.clone());
505+
let format_options = FormatOptions {
506+
time_format: Some("%Y".to_owned()),
507+
..Default::default()
508+
};
509+
510+
for batches in [vec![batch], vec![empty_batch]] {
511+
let mut output = Vec::new();
512+
let error = PrintFormat::Table
513+
.print_batches(
514+
&mut output,
515+
schema.clone(),
516+
&batches,
517+
MaxRows::Limited(1),
518+
true,
519+
&format_options,
520+
)
521+
.expect_err("incompatible time formats must fail before writing output");
522+
523+
assert!(
524+
error.to_string().contains("datafusion.format.time_format"),
525+
"unexpected error: {error}"
526+
);
527+
assert!(
528+
output.is_empty(),
529+
"format errors must not write partial output"
530+
);
531+
}
532+
}
533+
492534
#[derive(Debug)]
493535
struct PrintBatchesTest {
494536
format: PrintFormat,

‎datafusion/common/src/config.rs‎

Lines changed: 182 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2133,6 +2133,62 @@ config_namespace! {
21332133
impl<'a> TryFrom<&'a FormatOptions> for arrow::util::display::FormatOptions<'a> {
21342134
type Error = DataFusionError;
21352135
fn try_from(options: &'a FormatOptions) -> Result<Self> {
2136+
for (name, format) in [
2137+
("date_format", options.date_format.as_deref()),
2138+
("datetime_format", options.datetime_format.as_deref()),
2139+
("timestamp_format", options.timestamp_format.as_deref()),
2140+
(
2141+
"timestamp_tz_format",
2142+
options.timestamp_tz_format.as_deref(),
2143+
),
2144+
("time_format", options.time_format.as_deref()),
2145+
] {
2146+
if let Some(format) = format {
2147+
let items = chrono::format::StrftimeItems::new(format).parse().map_err(
2148+
|error| {
2149+
DataFusionError::Configuration(format!(
2150+
"Invalid datafusion.format.{name}: {error}"
2151+
))
2152+
},
2153+
)?;
2154+
2155+
// Parsing validates directives, but some directives require fields
2156+
// that are absent from a particular temporal type (for example,
2157+
// `%H` for a date). Chrono reports those errors while formatting.
2158+
let mut rendered = String::new();
2159+
let result = match name {
2160+
"date_format" => chrono::NaiveDate::from_ymd_opt(2001, 2, 3)
2161+
.expect("valid sample date")
2162+
.format_with_items(items.iter())
2163+
.write_to(&mut rendered),
2164+
"datetime_format" | "timestamp_format" => {
2165+
chrono::NaiveDate::from_ymd_opt(2001, 2, 3)
2166+
.expect("valid sample date")
2167+
.and_hms_opt(4, 5, 6)
2168+
.expect("valid sample time")
2169+
.format_with_items(items.iter())
2170+
.write_to(&mut rendered)
2171+
}
2172+
"timestamp_tz_format" => {
2173+
chrono::DateTime::from_timestamp(981_173_106, 0)
2174+
.expect("valid sample timestamp")
2175+
.format_with_items(items.iter())
2176+
.write_to(&mut rendered)
2177+
}
2178+
"time_format" => chrono::NaiveTime::from_hms_opt(4, 5, 6)
2179+
.expect("valid sample time")
2180+
.format_with_items(items.iter())
2181+
.write_to(&mut rendered),
2182+
_ => unreachable!("all temporal format names are handled above"),
2183+
};
2184+
if result.is_err() {
2185+
return Err(DataFusionError::Configuration(format!(
2186+
"Invalid datafusion.format.{name}: format is incompatible with its temporal type"
2187+
)));
2188+
}
2189+
}
2190+
}
2191+
21362192
Ok(Self::new()
21372193
.with_display_error(options.safe)
21382194
.with_null(&options.null)
@@ -4242,6 +4298,132 @@ mod tests {
42424298
ConfigEntry, ConfigExtension, ConfigField, ConfigFileType, ExtensionOptions,
42434299
Extensions, TableOptions,
42444300
};
4301+
4302+
#[test]
4303+
fn invalid_temporal_format_is_rejected_before_rendering() {
4304+
use crate::config::FormatOptions;
4305+
4306+
let invalid_options = [
4307+
(
4308+
"date_format",
4309+
FormatOptions {
4310+
date_format: Some("%".to_owned()),
4311+
..Default::default()
4312+
},
4313+
),
4314+
(
4315+
"datetime_format",
4316+
FormatOptions {
4317+
datetime_format: Some("%".to_owned()),
4318+
..Default::default()
4319+
},
4320+
),
4321+
(
4322+
"timestamp_format",
4323+
FormatOptions {
4324+
timestamp_format: Some("%".to_owned()),
4325+
..Default::default()
4326+
},
4327+
),
4328+
(
4329+
"timestamp_tz_format",
4330+
FormatOptions {
4331+
timestamp_tz_format: Some("%".to_owned()),
4332+
..Default::default()
4333+
},
4334+
),
4335+
(
4336+
"time_format",
4337+
FormatOptions {
4338+
time_format: Some("%".to_owned()),
4339+
..Default::default()
4340+
},
4341+
),
4342+
];
4343+
4344+
for (name, options) in invalid_options {
4345+
let error = arrow::util::display::FormatOptions::try_from(&options)
4346+
.expect_err("invalid format strings must be rejected before rendering");
4347+
assert!(
4348+
error
4349+
.to_string()
4350+
.contains(&format!("datafusion.format.{name}")),
4351+
"unexpected validation error: {error}"
4352+
);
4353+
}
4354+
4355+
let incompatible_options = [
4356+
(
4357+
"time_format",
4358+
FormatOptions {
4359+
time_format: Some("%Y".to_owned()),
4360+
..Default::default()
4361+
},
4362+
),
4363+
(
4364+
"date_format",
4365+
FormatOptions {
4366+
date_format: Some("%H".to_owned()),
4367+
..Default::default()
4368+
},
4369+
),
4370+
(
4371+
"timestamp_format",
4372+
FormatOptions {
4373+
timestamp_format: Some("%+".to_owned()),
4374+
..Default::default()
4375+
},
4376+
),
4377+
];
4378+
4379+
for (name, options) in incompatible_options {
4380+
let error = arrow::util::display::FormatOptions::try_from(&options)
4381+
.expect_err(
4382+
"type-incompatible formats must be rejected before rendering",
4383+
);
4384+
assert!(
4385+
error
4386+
.to_string()
4387+
.contains(&format!("datafusion.format.{name}")),
4388+
"unexpected validation error: {error}"
4389+
);
4390+
}
4391+
4392+
arrow::util::display::FormatOptions::try_from(&FormatOptions::default())
4393+
.expect("the default temporal formats must remain valid");
4394+
4395+
let options = FormatOptions {
4396+
timestamp_tz_format: Some("%+".to_owned()),
4397+
..Default::default()
4398+
};
4399+
let options: arrow::util::display::FormatOptions = (&options)
4400+
.try_into()
4401+
.expect("the timezone-aware ISO 8601 format must be valid");
4402+
let schema = arrow::datatypes::Schema::new(vec![arrow::datatypes::Field::new(
4403+
"timestamp",
4404+
arrow::datatypes::DataType::Timestamp(
4405+
arrow::datatypes::TimeUnit::Nanosecond,
4406+
Some("UTC".into()),
4407+
),
4408+
false,
4409+
)]);
4410+
let batch = arrow::record_batch::RecordBatch::try_new(
4411+
std::sync::Arc::new(schema),
4412+
vec![std::sync::Arc::new(
4413+
arrow::array::TimestampNanosecondArray::from(vec![0])
4414+
.with_timezone("UTC"),
4415+
)],
4416+
)
4417+
.expect("valid timestamp batch");
4418+
let formatted =
4419+
arrow::util::pretty::pretty_format_batches_with_options(&[batch], &options)
4420+
.expect("valid timezone-aware format must render")
4421+
.to_string();
4422+
assert!(
4423+
formatted.contains("1970-01-01T00:00:00+00:00"),
4424+
"unexpected formatted output: {formatted}"
4425+
);
4426+
}
42454427
use std::any::Any;
42464428
use std::collections::HashMap;
42474429

0 commit comments

Comments
 (0)