Skip to content

Commit ad83cde

Browse files
kumarUjjawalAdamGS
authored andcommitted
refactor: centralize SQL dialect metadata (apache#22840)
## Which issue does this PR close? <!-- We generally require a GitHub issue to be filed for all bug fixes and enhancements and this helps us generate change logs for our releases. You can link an issue to this PR using the GitHub syntax. For example `Closes apache#123` indicates that this PR will close issue apache#123. --> - Closes apache#22818. ## Rationale for this change DataFusion kept SQL dialect metadata in several separate places, which made it easy for enum variants, canonical names, aliases, display names, docs, and error messages to drift. <!-- Why are you proposing this change? If this is already explained clearly in the issue then this section is not needed. Explaining clearly why changes are proposed helps reviewers understand your changes and offer better suggestions for fixes. --> ## What changes are included in this PR? - Centralize dialect metadata in one macro-driven source. - Generate the `Dialect` enum, metadata table, canonical names, aliases, display names, and available-dialect string from that source. - Replace `Dialect::AVAILABLE` with `Dialect::available()`. - Add `Dialect::metadata()` for iterating supported dialects. - Update dialect error messages and generated config docs to use the centralized metadata. - Add an upgrade note for the removed public constant. <!-- There is no need to duplicate the description in the issue here but it is sometimes worth providing a summary of the individual changes in this PR. --> ## Are these changes tested? Yes <!-- We typically require tests for all PRs in order to: 1. Prevent the code from being accidentally broken by subsequent changes 2. Serve as another way to document the expected behavior of the code If tests are not included in your PR, please explain why (for example, are they covered by existing tests)? --> ## Are there any user-facing changes? Yes. `Dialect::AVAILABLE` is removed and replaced by `Dialect::available()`. <!-- If there are user-facing changes then we may require documentation to be updated before approving the PR. --> <!-- If there are any breaking changes to public APIs, please add the `api change` label. -->
1 parent 990c3e0 commit ad83cde

6 files changed

Lines changed: 276 additions & 75 deletions

File tree

‎datafusion-cli/src/exec.rs‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -224,7 +224,7 @@ pub(super) async fn exec_and_print(
224224
let dialect = dialect_from_str(dialect).ok_or_else(|| {
225225
plan_datafusion_err!(
226226
"Unsupported SQL dialect: {dialect}. Available dialects: {}.",
227-
Dialect::AVAILABLE
227+
Dialect::available()
228228
)
229229
})?;
230230

@@ -613,7 +613,7 @@ mod tests {
613613
let dialect = dialect_from_str(dialect).ok_or_else(|| {
614614
plan_datafusion_err!(
615615
"Unsupported SQL dialect: {dialect}. Available dialects: {}.",
616-
Dialect::AVAILABLE
616+
Dialect::available()
617617
)
618618
})?;
619619
for location in locations {

‎datafusion/common/src/config.rs‎

Lines changed: 253 additions & 69 deletions
Original file line numberDiff line numberDiff line change
@@ -278,8 +278,8 @@ config_namespace! {
278278
/// are normalized automatically.
279279
pub enable_options_value_normalization: bool, warn = "`enable_options_value_normalization` is deprecated and ignored", default = false
280280

281-
/// Configure the SQL dialect used by DataFusion's parser; supported values include: Generic,
282-
/// MySQL, PostgreSQL, Hive, SQLite, Snowflake, Redshift, MsSQL, ClickHouse, BigQuery, Ansi, DuckDB, Databricks and Spark.
281+
/// Configure the SQL dialect used by DataFusion's parser.
282+
/// The configuration reference lists the supported values from [`Dialect::available`].
283283
pub dialect: Dialect, default = Dialect::Generic
284284
// no need to lowercase because `sqlparser::dialect_from_str`] is case-insensitive
285285

@@ -323,87 +323,204 @@ config_namespace! {
323323
}
324324
}
325325

326-
/// This is the SQL dialect used by DataFusion's parser.
327-
/// This mirrors [sqlparser::dialect::Dialect](https://docs.rs/sqlparser/latest/sqlparser/dialect/trait.Dialect.html)
328-
/// trait in order to offer an easier API and avoid adding the `sqlparser` dependency
329-
#[derive(Debug, Default, Clone, Copy, PartialEq, Eq)]
330-
pub enum Dialect {
331-
#[default]
332-
Generic,
333-
MySQL,
334-
PostgreSQL,
335-
Hive,
336-
SQLite,
337-
Snowflake,
338-
Redshift,
339-
MsSQL,
340-
ClickHouse,
341-
BigQuery,
342-
Ansi,
343-
DuckDB,
344-
Databricks,
345-
Spark,
326+
/// Metadata for a SQL dialect supported by DataFusion configuration.
327+
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
328+
#[non_exhaustive]
329+
pub struct DialectInfo {
330+
pub dialect: Dialect,
331+
pub canonical_name: &'static str,
332+
pub display_name: &'static str,
333+
pub aliases: &'static [&'static str],
334+
}
335+
336+
// Keep this key in sync with the `SqlParserOptions::dialect` config path.
337+
const SQL_PARSER_DIALECT_CONFIG_KEY: &str = "datafusion.sql_parser.dialect";
338+
339+
macro_rules! dialect_display_list {
340+
($($display_name:literal),+ $(,)?) => {
341+
dialect_display_list!(@acc [] $($display_name),+)
342+
};
343+
(@acc [$($acc:tt)*] $last:literal) => {
344+
concat!($($acc)* $last)
345+
};
346+
(@acc [$($acc:tt)*] $next:literal, $($rest:literal),+) => {
347+
dialect_display_list!(@acc [$($acc)* $next, ", ",] $($rest),+)
348+
};
349+
}
350+
351+
macro_rules! dialect_metadata {
352+
(
353+
default: $default_variant:ident;
354+
$(
355+
$variant:ident {
356+
canonical_name: $canonical_name:literal,
357+
display_name: $display_name:literal,
358+
aliases: [$($alias:literal),* $(,)?],
359+
}
360+
),+ $(,)?
361+
) => {
362+
/// This is the SQL dialect used by DataFusion's parser.
363+
/// This mirrors [sqlparser::dialect::Dialect](https://docs.rs/sqlparser/latest/sqlparser/dialect/trait.Dialect.html)
364+
/// trait in order to offer an easier API and avoid adding the `sqlparser` dependency
365+
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
366+
pub enum Dialect {
367+
$($variant,)+
368+
}
369+
370+
impl Default for Dialect {
371+
fn default() -> Self {
372+
Self::$default_variant
373+
}
374+
}
375+
376+
const DIALECT_INFOS: &[DialectInfo] = &[
377+
$(
378+
DialectInfo {
379+
dialect: Dialect::$variant,
380+
canonical_name: $canonical_name,
381+
display_name: $display_name,
382+
aliases: &[$($alias),*],
383+
},
384+
)+
385+
];
386+
387+
const AVAILABLE_DIALECTS: &str = dialect_display_list!($($display_name),+);
388+
const DIALECT_CONFIG_DESCRIPTION: &str = concat!(
389+
"Configure the SQL dialect used by DataFusion's parser; supported values include: ",
390+
dialect_display_list!($($display_name),+),
391+
"."
392+
);
393+
};
394+
}
395+
396+
dialect_metadata! {
397+
default: Generic;
398+
Generic {
399+
canonical_name: "generic",
400+
display_name: "Generic",
401+
aliases: [],
402+
},
403+
MySQL {
404+
canonical_name: "mysql",
405+
display_name: "MySQL",
406+
aliases: [],
407+
},
408+
PostgreSQL {
409+
canonical_name: "postgresql",
410+
display_name: "PostgreSQL",
411+
aliases: ["postgres"],
412+
},
413+
Hive {
414+
canonical_name: "hive",
415+
display_name: "Hive",
416+
aliases: [],
417+
},
418+
SQLite {
419+
canonical_name: "sqlite",
420+
display_name: "SQLite",
421+
aliases: [],
422+
},
423+
Snowflake {
424+
canonical_name: "snowflake",
425+
display_name: "Snowflake",
426+
aliases: [],
427+
},
428+
Redshift {
429+
canonical_name: "redshift",
430+
display_name: "Redshift",
431+
aliases: [],
432+
},
433+
MsSQL {
434+
canonical_name: "mssql",
435+
display_name: "MsSQL",
436+
aliases: [],
437+
},
438+
ClickHouse {
439+
canonical_name: "clickhouse",
440+
display_name: "ClickHouse",
441+
aliases: [],
442+
},
443+
BigQuery {
444+
canonical_name: "bigquery",
445+
display_name: "BigQuery",
446+
aliases: [],
447+
},
448+
Ansi {
449+
canonical_name: "ansi",
450+
display_name: "Ansi",
451+
aliases: [],
452+
},
453+
DuckDB {
454+
canonical_name: "duckdb",
455+
display_name: "DuckDB",
456+
aliases: [],
457+
},
458+
Databricks {
459+
canonical_name: "databricks",
460+
display_name: "Databricks",
461+
aliases: [],
462+
},
463+
Spark {
464+
canonical_name: "spark",
465+
display_name: "Spark",
466+
aliases: ["sparksql"],
467+
},
346468
}
347469

348470
impl Dialect {
349-
/// List of all supported dialect names, for use in error messages.
350-
pub const AVAILABLE: &'static str = "Generic, MySQL, PostgreSQL, Hive, SQLite, Snowflake, Redshift, \
351-
MsSQL, ClickHouse, BigQuery, Ansi, DuckDB, Databricks, Spark";
471+
/// Return metadata for all supported dialects.
472+
pub fn metadata() -> &'static [DialectInfo] {
473+
DIALECT_INFOS
474+
}
475+
476+
/// Return all supported dialect names, for use in error messages.
477+
pub fn available() -> &'static str {
478+
AVAILABLE_DIALECTS
479+
}
480+
481+
fn info(&self) -> &'static DialectInfo {
482+
DIALECT_INFOS
483+
.iter()
484+
.find(|info| info.dialect == *self)
485+
.expect("all Dialect variants are listed in DIALECT_INFOS")
486+
}
352487
}
353488

354489
impl AsRef<str> for Dialect {
355490
fn as_ref(&self) -> &str {
356-
match self {
357-
Self::Generic => "generic",
358-
Self::MySQL => "mysql",
359-
Self::PostgreSQL => "postgresql",
360-
Self::Hive => "hive",
361-
Self::SQLite => "sqlite",
362-
Self::Snowflake => "snowflake",
363-
Self::Redshift => "redshift",
364-
Self::MsSQL => "mssql",
365-
Self::ClickHouse => "clickhouse",
366-
Self::BigQuery => "bigquery",
367-
Self::Ansi => "ansi",
368-
Self::DuckDB => "duckdb",
369-
Self::Databricks => "databricks",
370-
Self::Spark => "spark",
371-
}
491+
self.info().canonical_name
372492
}
373493
}
374494

375495
impl FromStr for Dialect {
376496
type Err = DataFusionError;
377497

378498
fn from_str(s: &str) -> Result<Self, Self::Err> {
379-
let value = match s.to_ascii_lowercase().as_str() {
380-
"generic" => Self::Generic,
381-
"mysql" => Self::MySQL,
382-
"postgresql" | "postgres" => Self::PostgreSQL,
383-
"hive" => Self::Hive,
384-
"sqlite" => Self::SQLite,
385-
"snowflake" => Self::Snowflake,
386-
"redshift" => Self::Redshift,
387-
"mssql" => Self::MsSQL,
388-
"clickhouse" => Self::ClickHouse,
389-
"bigquery" => Self::BigQuery,
390-
"ansi" => Self::Ansi,
391-
"duckdb" => Self::DuckDB,
392-
"databricks" => Self::Databricks,
393-
"spark" | "sparksql" => Self::Spark,
394-
other => {
395-
return Err(DataFusionError::Configuration(format!(
396-
"Invalid Dialect: {other}. Expected one of: {}",
397-
Self::AVAILABLE
398-
)));
499+
for info in DIALECT_INFOS {
500+
if info.canonical_name.eq_ignore_ascii_case(s)
501+
|| info
502+
.aliases
503+
.iter()
504+
.any(|alias| alias.eq_ignore_ascii_case(s))
505+
{
506+
return Ok(info.dialect);
399507
}
400-
};
401-
Ok(value)
508+
}
509+
510+
Err(DataFusionError::Configuration(format!(
511+
"Invalid Dialect: {s}. Expected one of: {}",
512+
Self::available()
513+
)))
402514
}
403515
}
404516

405517
impl ConfigField for Dialect {
406518
fn visit<V: Visit>(&self, v: &mut V, key: &str, description: &'static str) {
519+
let description = if key == SQL_PARSER_DIALECT_CONFIG_KEY {
520+
DIALECT_CONFIG_DESCRIPTION
521+
} else {
522+
description
523+
};
407524
v.some(key, self, description)
408525
}
409526

@@ -4182,15 +4299,82 @@ mod tests {
41824299
}
41834300

41844301
#[test]
4185-
fn test_dialect_spark_roundtrip() {
4302+
fn test_dialect_metadata_roundtrip() {
4303+
use crate::config::Dialect;
4304+
use std::str::FromStr;
4305+
4306+
assert_eq!(Dialect::default(), Dialect::Generic);
4307+
assert!(!Dialect::metadata().is_empty());
4308+
4309+
for info in Dialect::metadata() {
4310+
let dialect = info.dialect;
4311+
4312+
assert_eq!(Dialect::from_str(info.canonical_name).unwrap(), dialect);
4313+
assert_eq!(
4314+
Dialect::from_str(&info.canonical_name.to_ascii_uppercase()).unwrap(),
4315+
dialect
4316+
);
4317+
assert_eq!(dialect.as_ref(), info.canonical_name);
4318+
assert_eq!(dialect.to_string(), info.canonical_name);
4319+
}
4320+
}
4321+
4322+
#[test]
4323+
fn test_dialect_aliases() {
4324+
use crate::config::Dialect;
4325+
use std::str::FromStr;
4326+
4327+
for info in Dialect::metadata() {
4328+
for alias in info.aliases {
4329+
assert_eq!(Dialect::from_str(alias).unwrap(), info.dialect);
4330+
assert_eq!(
4331+
Dialect::from_str(&alias.to_ascii_uppercase()).unwrap(),
4332+
info.dialect
4333+
);
4334+
}
4335+
}
4336+
}
4337+
4338+
#[test]
4339+
fn test_available_dialects_includes_each_display_name_once() {
4340+
use crate::config::Dialect;
4341+
use std::collections::BTreeSet;
4342+
4343+
let available = Dialect::available();
4344+
let listed: Vec<_> = available.split(", ").collect();
4345+
let display_names: Vec<_> = Dialect::metadata()
4346+
.iter()
4347+
.map(|info| info.display_name)
4348+
.collect();
4349+
let unique_display_names: BTreeSet<_> = display_names.iter().copied().collect();
4350+
4351+
assert_eq!(display_names.len(), unique_display_names.len());
4352+
assert_eq!(listed, display_names);
4353+
}
4354+
4355+
#[test]
4356+
fn test_dialect_config_description_uses_metadata() {
4357+
use crate::config::{ConfigOptions, Dialect, SQL_PARSER_DIALECT_CONFIG_KEY};
4358+
4359+
let description = ConfigOptions::default()
4360+
.entries()
4361+
.into_iter()
4362+
.find(|entry| entry.key == SQL_PARSER_DIALECT_CONFIG_KEY)
4363+
.unwrap()
4364+
.description;
4365+
4366+
assert!(description.contains(Dialect::available()));
4367+
}
4368+
4369+
#[test]
4370+
fn test_invalid_dialect_error_lists_available_dialects() {
41864371
use crate::config::Dialect;
41874372
use std::str::FromStr;
41884373

4189-
assert_eq!(Dialect::from_str("spark").unwrap(), Dialect::Spark);
4190-
assert_eq!(Dialect::from_str("sparksql").unwrap(), Dialect::Spark);
4191-
assert_eq!(Dialect::from_str("SPARK").unwrap(), Dialect::Spark);
4192-
assert_eq!(Dialect::Spark.as_ref(), "spark");
4193-
assert_eq!(Dialect::Spark.to_string(), "spark");
4374+
let error = Dialect::from_str("notadialect").unwrap_err().to_string();
4375+
4376+
assert!(error.contains("Invalid Dialect: notadialect"));
4377+
assert!(error.contains(Dialect::available()));
41944378
}
41954379

41964380
#[test]

0 commit comments

Comments
 (0)