Skip to content

Commit 7127f8b

Browse files
kumarUjjawalalamb
andauthored
feat: Add Spark SQL parser dialect config (apache#22529)
## Which issue does this PR close? <!-- We generally require a GitHub issue to be filed for all bug fixes and enhancements and this helps us generate change logs for our releases. You can link an issue to this PR using the GitHub syntax. For example `Closes apache#123` indicates that this PR will close issue apache#123. --> - Closes apache#21653. ## Rationale for this change This lets users combine Spark-compatible functions with Spark SQL parsing when they choose the dialect. <!-- Why are you proposing this change? If this is already explained clearly in the issue then this section is not needed. Explaining clearly why changes are proposed helps reviewers understand your changes and offer better suggestions for fixes. --> ## What changes are included in this PR? - Add `Spark` to the SQL parser dialect config enum. - Accept `spark` and `sparksql` as dialect config values. - Use Spark dialect for Spark sqllogictests. - Keep `with_spark_features()` config-neutral. - Update dialect docs and expected config output. - Add tests for Spark dialect parsing and Spark SQL execution with Spark functions. <!-- There is no need to duplicate the description in the issue here but it is sometimes worth providing a summary of the individual changes in this PR. --> ## Are these changes tested? Yes <!-- We typically require tests for all PRs in order to: 1. Prevent the code from being accidentally broken by subsequent changes 2. Serve as another way to document the expected behavior of the code If tests are not included in your PR, please explain why (for example, are they covered by existing tests)? --> ## Are there any user-facing changes? Users can now set `datafusion.sql_parser.dialect` to `spark` or `sparksql`. <!-- If there are user-facing changes then we may require documentation to be updated before approving the PR. --> <!-- If there are any breaking changes to public APIs, please add the `api change` label. --> --------- Co-authored-by: Andrew Lamb <[email protected]>
1 parent 0d3fb77 commit 7127f8b

11 files changed

Lines changed: 86 additions & 22 deletions

File tree

‎Cargo.lock‎

Lines changed: 1 addition & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎datafusion-cli/src/exec.rs‎

Lines changed: 5 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -28,7 +28,7 @@ use crate::{
2828
};
2929
use datafusion::common::instant::Instant;
3030
use datafusion::common::{plan_datafusion_err, plan_err};
31-
use datafusion::config::ConfigFileType;
31+
use datafusion::config::{ConfigFileType, Dialect};
3232
use datafusion::datasource::listing::ListingTableUrl;
3333
use datafusion::error::{DataFusionError, Result};
3434
use datafusion::execution::memory_pool::MemoryConsumer;
@@ -223,9 +223,8 @@ pub(super) async fn exec_and_print(
223223
let dialect = &options.sql_parser.dialect;
224224
let dialect = dialect_from_str(dialect).ok_or_else(|| {
225225
plan_datafusion_err!(
226-
"Unsupported SQL dialect: {dialect}. Available dialects: \
227-
Generic, MySQL, PostgreSQL, Hive, SQLite, Snowflake, Redshift, \
228-
MsSQL, ClickHouse, BigQuery, Ansi, DuckDB, Databricks."
226+
"Unsupported SQL dialect: {dialect}. Available dialects: {}.",
227+
Dialect::AVAILABLE
229228
)
230229
})?;
231230

@@ -613,9 +612,8 @@ mod tests {
613612
let dialect = &task_ctx.session_config().options().sql_parser.dialect;
614613
let dialect = dialect_from_str(dialect).ok_or_else(|| {
615614
plan_datafusion_err!(
616-
"Unsupported SQL dialect: {dialect}. Available dialects: \
617-
Generic, MySQL, PostgreSQL, Hive, SQLite, Snowflake, Redshift, \
618-
MsSQL, ClickHouse, BigQuery, Ansi, DuckDB, Databricks."
615+
"Unsupported SQL dialect: {dialect}. Available dialects: {}.",
616+
Dialect::AVAILABLE
619617
)
620618
})?;
621619
for location in locations {

‎datafusion/common/src/config.rs‎

Lines changed: 26 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -279,7 +279,7 @@ config_namespace! {
279279
pub enable_options_value_normalization: bool, warn = "`enable_options_value_normalization` is deprecated and ignored", default = false
280280

281281
/// Configure the SQL dialect used by DataFusion's parser; supported values include: Generic,
282-
/// MySQL, PostgreSQL, Hive, SQLite, Snowflake, Redshift, MsSQL, ClickHouse, BigQuery, Ansi, DuckDB and Databricks.
282+
/// MySQL, PostgreSQL, Hive, SQLite, Snowflake, Redshift, MsSQL, ClickHouse, BigQuery, Ansi, DuckDB, Databricks and Spark.
283283
pub dialect: Dialect, default = Dialect::Generic
284284
// no need to lowercase because `sqlparser::dialect_from_str`] is case-insensitive
285285

@@ -342,6 +342,13 @@ pub enum Dialect {
342342
Ansi,
343343
DuckDB,
344344
Databricks,
345+
Spark,
346+
}
347+
348+
impl Dialect {
349+
/// List of all supported dialect names, for use in error messages.
350+
pub const AVAILABLE: &'static str = "Generic, MySQL, PostgreSQL, Hive, SQLite, Snowflake, Redshift, \
351+
MsSQL, ClickHouse, BigQuery, Ansi, DuckDB, Databricks, Spark";
345352
}
346353

347354
impl AsRef<str> for Dialect {
@@ -360,6 +367,7 @@ impl AsRef<str> for Dialect {
360367
Self::Ansi => "ansi",
361368
Self::DuckDB => "duckdb",
362369
Self::Databricks => "databricks",
370+
Self::Spark => "spark",
363371
}
364372
}
365373
}
@@ -382,11 +390,12 @@ impl FromStr for Dialect {
382390
"ansi" => Self::Ansi,
383391
"duckdb" => Self::DuckDB,
384392
"databricks" => Self::Databricks,
393+
"spark" | "sparksql" => Self::Spark,
385394
other => {
386-
let error_message = format!(
387-
"Invalid Dialect: {other}. Expected one of: Generic, MySQL, PostgreSQL, Hive, SQLite, Snowflake, Redshift, MsSQL, ClickHouse, BigQuery, Ansi, DuckDB, Databricks"
388-
);
389-
return Err(DataFusionError::Configuration(error_message));
395+
return Err(DataFusionError::Configuration(format!(
396+
"Invalid Dialect: {other}. Expected one of: {}",
397+
Self::AVAILABLE
398+
)));
390399
}
391400
};
392401
Ok(value)
@@ -4161,6 +4170,18 @@ mod tests {
41614170
assert_eq!(cdc.norm_level, 0);
41624171
}
41634172

4173+
#[test]
4174+
fn test_dialect_spark_roundtrip() {
4175+
use crate::config::Dialect;
4176+
use std::str::FromStr;
4177+
4178+
assert_eq!(Dialect::from_str("spark").unwrap(), Dialect::Spark);
4179+
assert_eq!(Dialect::from_str("sparksql").unwrap(), Dialect::Spark);
4180+
assert_eq!(Dialect::from_str("SPARK").unwrap(), Dialect::Spark);
4181+
assert_eq!(Dialect::Spark.as_ref(), "spark");
4182+
assert_eq!(Dialect::Spark.to_string(), "spark");
4183+
}
4184+
41644185
#[test]
41654186
fn max_row_group_bytes_rejects_zero() {
41664187
use crate::config::MaxRowGroupBytes;

‎datafusion/core/src/execution/session_state.rs‎

Lines changed: 4 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -439,9 +439,8 @@ impl SessionState {
439439
) -> datafusion_common::Result<Statement> {
440440
let dialect = dialect_from_str(dialect).ok_or_else(|| {
441441
plan_datafusion_err!(
442-
"Unsupported SQL dialect: {dialect}. Available dialects: \
443-
Generic, MySQL, PostgreSQL, Hive, SQLite, Snowflake, Redshift, \
444-
MsSQL, ClickHouse, BigQuery, Ansi, DuckDB, Databricks."
442+
"Unsupported SQL dialect: {dialect}. Available dialects: {}.",
443+
Dialect::AVAILABLE
445444
)
446445
})?;
447446

@@ -488,9 +487,8 @@ impl SessionState {
488487
) -> datafusion_common::Result<SQLExprWithAlias> {
489488
let dialect = dialect_from_str(dialect).ok_or_else(|| {
490489
plan_datafusion_err!(
491-
"Unsupported SQL dialect: {dialect}. Available dialects: \
492-
Generic, MySQL, PostgreSQL, Hive, SQLite, Snowflake, Redshift, \
493-
MsSQL, ClickHouse, BigQuery, Ansi, DuckDB, Databricks."
490+
"Unsupported SQL dialect: {dialect}. Available dialects: {}.",
491+
Dialect::AVAILABLE
494492
)
495493
})?;
496494

‎datafusion/spark/Cargo.toml‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -71,7 +71,8 @@ url = { workspace = true }
7171
arrow = { workspace = true, features = ["test_utils"] }
7272
criterion = { workspace = true }
7373
# for SessionStateBuilderSpark tests
74-
datafusion = { workspace = true, default-features = false }
74+
datafusion = { workspace = true, default-features = false, features = ["sql"] }
75+
tokio = { workspace = true, features = ["rt"] }
7576

7677
[[bench]]
7778
harness = false

‎datafusion/spark/src/session_state.rs‎

Lines changed: 36 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -88,6 +88,9 @@ impl SessionStateBuilderSpark for SessionStateBuilder {
8888
#[cfg(test)]
8989
mod tests {
9090
use super::*;
91+
use datafusion::common::config::Dialect;
92+
use datafusion::prelude::SessionConfig;
93+
use datafusion::prelude::SessionContext;
9194

9295
#[test]
9396
fn test_session_state_with_spark_features() {
@@ -108,4 +111,37 @@ mod tests {
108111
"Apache Spark expr planners should be registered"
109112
);
110113
}
114+
115+
#[tokio::test]
116+
async fn test_spark_dialect_with_spark_functions() {
117+
let query = "SELECT sha2('abc', 256), CAST(1 AS LONG)";
118+
119+
let mut config = SessionConfig::new();
120+
config.options_mut().sql_parser.dialect = Dialect::Spark;
121+
let state = SessionStateBuilder::new()
122+
.with_config(config)
123+
.with_default_features()
124+
.with_spark_features()
125+
.build();
126+
let ctx = SessionContext::new_with_state(state);
127+
128+
let result = ctx.sql(query).await.unwrap().collect().await.unwrap();
129+
assert_eq!(result.len(), 1);
130+
assert_eq!(result[0].num_rows(), 1);
131+
132+
let mut config = SessionConfig::new();
133+
config.options_mut().sql_parser.dialect = Dialect::Generic;
134+
let state = SessionStateBuilder::new()
135+
.with_config(config)
136+
.with_default_features()
137+
.with_spark_features()
138+
.build();
139+
let ctx = SessionContext::new_with_state(state);
140+
141+
let err = ctx.sql(query).await.unwrap_err().to_string();
142+
assert!(
143+
err.contains("Unsupported SQL type LONG"),
144+
"unexpected error: {err}"
145+
);
146+
}
111147
}

‎datafusion/sqllogictest/src/test_context.rs‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -36,6 +36,7 @@ use arrow::record_batch::RecordBatch;
3636
use datafusion::catalog::{
3737
CatalogProvider, MemoryCatalogProvider, MemorySchemaProvider, SchemaProvider, Session,
3838
};
39+
use datafusion::common::config::Dialect;
3940
use datafusion::common::{DataFusionError, Result, not_impl_err};
4041
use datafusion::functions::math::abs;
4142
use datafusion::logical_expr::async_udf::{AsyncScalarUDF, AsyncScalarUDFImpl};
@@ -151,6 +152,9 @@ impl TestContext {
151152

152153
if is_spark_path(relative_path) {
153154
state_builder = state_builder.with_spark_features();
155+
if let Some(config) = state_builder.config() {
156+
config.options_mut().sql_parser.dialect = Dialect::Spark;
157+
}
154158
}
155159

156160
if matches!(

‎datafusion/sqllogictest/test_files/information_schema.slt‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -505,7 +505,7 @@ datafusion.runtime.temp_directory NULL The path to the temporary file directory.
505505
datafusion.spark.map_key_dedup_policy EXCEPTION Policy for handling duplicate keys in Spark-compatible map-construction functions (`map_from_arrays`, `map_from_entries`, `str_to_map`). Mirrors Spark's [`spark.sql.mapKeyDedupPolicy`](https://github.com/apache/spark/blob/cf3a34e19dfcf70e2d679217ff1ba21302212472/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala#L4961): - `EXCEPTION` (default): raise `[DUPLICATED_MAP_KEY]` at runtime on any duplicate key. - `LAST_WIN`: keep the last occurrence of each duplicate key. Values are case-insensitive.
506506
datafusion.sql_parser.collect_spans false When set to true, the source locations relative to the original SQL query (i.e. [`Span`](https://docs.rs/sqlparser/latest/sqlparser/tokenizer/struct.Span.html)) will be collected and recorded in the logical plan nodes.
507507
datafusion.sql_parser.default_null_ordering nulls_max Specifies the default null ordering for query results. There are 4 options: - `nulls_max`: Nulls appear last in ascending order. - `nulls_min`: Nulls appear first in ascending order. - `nulls_first`: Nulls always be first in any order. - `nulls_last`: Nulls always be last in any order. By default, `nulls_max` is used to follow Postgres's behavior. postgres rule: <https://www.postgresql.org/docs/current/queries-order.html>
508-
datafusion.sql_parser.dialect generic Configure the SQL dialect used by DataFusion's parser; supported values include: Generic, MySQL, PostgreSQL, Hive, SQLite, Snowflake, Redshift, MsSQL, ClickHouse, BigQuery, Ansi, DuckDB and Databricks.
508+
datafusion.sql_parser.dialect generic Configure the SQL dialect used by DataFusion's parser; supported values include: Generic, MySQL, PostgreSQL, Hive, SQLite, Snowflake, Redshift, MsSQL, ClickHouse, BigQuery, Ansi, DuckDB, Databricks and Spark.
509509
datafusion.sql_parser.enable_ident_normalization true When set to true, SQL parser will normalize ident (convert ident to lowercase when not quoted)
510510
datafusion.sql_parser.enable_options_value_normalization false When set to true, SQL parser will normalize options value (convert value to lowercase). Note that this option is ignored and will be removed in the future. All case-insensitive values are normalized automatically.
511511
datafusion.sql_parser.enable_subquery_sort_elimination true When set to true, DataFusion may remove `ORDER BY` clauses from subqueries or CTEs during SQL planning when their ordering cannot affect the result, such as when no `LIMIT` or other order-sensitive operator depends on them. Disable this option to preserve explicit subquery ordering in the planned query.

‎datafusion/sqllogictest/test_files/spark/collection/size.slt‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -84,7 +84,7 @@ SELECT size(make_array(1, NULL, 3));
8484

8585
# NULL array returns -1 (Spark behavior)
8686
query I
87-
SELECT size(NULL::int[]);
87+
SELECT size(CAST(NULL AS ARRAY<INT>));
8888
----
8989
-1
9090

‎docs/source/library-user-guide/upgrading/54.0.0.md‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -932,3 +932,8 @@ match register_function {
932932
RegisterFunction::Table(name, table) => {},
933933
}
934934
```
935+
936+
### New `Dialect::Spark` variant
937+
938+
The `Dialect` enum in `datafusion_common::config` now includes a `Spark` variant.
939+
If you match exhaustively on `Dialect`, add a `Dialect::Spark` arm.

0 commit comments

Comments
 (0)