Skip to content

Commit 571477c

Browse files
authored
feat(cli): add opt-in Spark function registration (apache#24179)
## Which issue does this PR close? - Closes apache#24146. ## Rationale for this change `datafusion-cli` does not currently enable the Spark SQL dialect or register the `datafusion-spark` function library, so Spark-compatible syntax and functions cannot be explored interactively. Enabling them by default could override core DataFusion functions with the same names and change existing CLI behavior. ## What changes are included in this PR? - Add a `--spark` CLI flag. - Select the Spark SQL dialect while constructing the session configuration. - Register Spark-compatible scalar, aggregate, and window functions only when the flag is enabled. - Add CLI integration coverage for both default and opt-in behavior. - Document the flag and its function-override behavior in the CLI and Spark documentation. ## Are these changes tested? Yes. The following checks pass: - `cargo test -p datafusion-cli --test cli_integration spark_features_require_opt_in` - `cargo test -p datafusion-cli --all-targets` - `cargo fmt --all -- --check` - `cargo clippy --all-targets --all-features -- -D warnings` - `./ci/scripts/doc_prettier_check.sh` ## Are there any user-facing changes? Yes. Passing `--spark` enables the Spark SQL dialect, Spark expression planner, and Spark-compatible functions for the CLI session. The option is disabled by default, so existing CLI behavior remains unchanged.
1 parent 00eba79 commit 571477c

7 files changed

Lines changed: 85 additions & 2 deletions

File tree

‎Cargo.lock‎

Lines changed: 1 addition & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎datafusion-cli/Cargo.toml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -56,6 +56,7 @@ datafusion = { workspace = true, features = [
5656
"unicode_expressions",
5757
] }
5858
datafusion-common = { workspace = true }
59+
datafusion-spark = { workspace = true, features = ["core"] }
5960
dirs = "6.0.0"
6061
env_logger = { workspace = true }
6162
futures = { workspace = true }

‎datafusion-cli/src/main.rs‎

Lines changed: 22 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,7 @@ use std::process::ExitCode;
2323
use std::sync::{Arc, LazyLock};
2424

2525
use datafusion::error::{DataFusionError, Result};
26+
use datafusion::execution::SessionStateBuilder;
2627
use datafusion::execution::context::SessionConfig;
2728
use datafusion::execution::memory_pool::{
2829
FairSpillPool, GreedyMemoryPool, MemoryPool, TrackConsumersPool,
@@ -46,9 +47,11 @@ use datafusion_cli::{
4647
};
4748

4849
use clap::Parser;
50+
use datafusion::common::config::Dialect;
4951
use datafusion::common::config_err;
5052
use datafusion::config::ConfigOptions;
5153
use datafusion::execution::disk_manager::{DiskManagerBuilder, DiskManagerMode};
54+
use datafusion_spark::SessionStateBuilderSpark;
5255
use mimalloc::MiMalloc;
5356

5457
#[global_allocator]
@@ -143,6 +146,12 @@ struct Args {
143146
#[clap(long, help = "Enables console syntax highlighting")]
144147
color: bool,
145148

149+
#[clap(
150+
long,
151+
help = "Enable Apache Spark-compatible SQL dialect, functions, and expression planning"
152+
)]
153+
spark: bool,
154+
146155
#[clap(
147156
short = 'd',
148157
long,
@@ -243,9 +252,16 @@ async fn main_inner() -> Result<()> {
243252

244253
let runtime_env = rt_builder.build_arc()?;
245254

255+
let mut state_builder = SessionStateBuilder::new()
256+
.with_config(session_config)
257+
.with_runtime_env(runtime_env)
258+
.with_default_features();
259+
if args.spark {
260+
state_builder = state_builder.with_spark_features();
261+
}
262+
246263
// enable dynamic file query
247-
let ctx = SessionContext::new_with_config_rt(session_config, runtime_env)
248-
.enable_url_table();
264+
let ctx = SessionContext::new_with_state(state_builder.build()).enable_url_table();
249265
ctx.refresh_catalogs().await?;
250266
// install dynamic catalog provider that can register required object stores
251267
ctx.register_catalog_list(Arc::new(DynamicObjectStoreCatalog::new(
@@ -350,6 +366,10 @@ fn get_session_config(args: &Args) -> Result<SessionConfig> {
350366
config_options.format.null = String::from("NULL");
351367
}
352368

369+
if args.spark {
370+
config_options.sql_parser.dialect = Dialect::Spark;
371+
}
372+
353373
let mut session_config =
354374
SessionConfig::from(config_options).with_information_schema(true);
355375

‎datafusion-cli/tests/cli_integration.rs‎

Lines changed: 31 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -173,6 +173,37 @@ fn cli_quick_test<'a>(
173173
assert_cmd_snapshot!(cmd);
174174
}
175175

176+
#[test]
177+
fn spark_features_require_opt_in() {
178+
let dialect_query = "SELECT CAST(1 AS LONG);";
179+
let spark_query = "SELECT concat_ws(',', array(1, 2)), CAST(1 AS LONG);";
180+
181+
let output = cli()
182+
.args(["-q", "--command", dialect_query])
183+
.output()
184+
.expect("failed to run datafusion-cli without --spark");
185+
let stdout = String::from_utf8_lossy(&output.stdout);
186+
assert!(!output.status.success(), "query unexpectedly succeeded");
187+
assert!(
188+
stdout.contains("Unsupported SQL type LONG"),
189+
"expected Spark SQL syntax to be unavailable without --spark, got:\n{stdout}"
190+
);
191+
192+
let output = cli()
193+
.args(["-q", "--spark", "--format", "csv", "--command", spark_query])
194+
.output()
195+
.expect("failed to run datafusion-cli with --spark");
196+
let stdout = String::from_utf8_lossy(&output.stdout);
197+
assert!(
198+
output.status.success(),
199+
"query failed with --spark, got:\n{stdout}"
200+
);
201+
assert!(
202+
stdout.lines().any(|line| line == "\"1,2\",1"),
203+
"expected Spark planner and dialect result, got:\n{stdout}"
204+
);
205+
}
206+
176207
/// Read data piped into the CLI via the `/dev/stdin` pseudo-path.
177208
///
178209
/// Unix-only: `/dev/stdin` does not exist on Windows. This drives the real

‎datafusion/spark/README.md‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,18 @@ under the License.
2323

2424
This crate is a submodule of DataFusion that provides [Apache Spark] compatible expressions for use with DataFusion.
2525

26+
## Using the DataFusion CLI
27+
28+
Pass `--spark` to `datafusion-cli` to enable the Spark SQL dialect and register
29+
the Spark-compatible scalar, aggregate, and window functions for the session:
30+
31+
```shell
32+
datafusion-cli --spark
33+
```
34+
35+
The option is disabled by default because Spark-compatible functions can
36+
override DataFusion functions with the same name.
37+
2638
[apache arrow]: https://arrow.apache.org/
2739
[apache datafusion]: https://datafusion.apache.org/
2840
[apache spark]: https://spark.apache.org/

‎datafusion/sqllogictest/test_files/spark/README.md‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -52,6 +52,10 @@ statement ok
5252
set datafusion.execution.enable_ansi_mode = false;
5353
```
5454

55+
Run `datafusion-cli --spark` to explore Spark-compatible functions
56+
interactively. This enables the Spark SQL dialect and registers the
57+
Spark-compatible function library for the CLI session.
58+
5559
### Finding Test Cases
5660

5761
To verify and compare function behavior at a minimum, you can refer to the following documentation sources:

‎docs/source/user-guide/cli/usage.md‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -52,6 +52,8 @@ Options:
5252
[possible values: numbers(0/10/...), inf(no limit)] [default: 40]
5353
--color
5454
Enables console syntax highlighting
55+
--spark
56+
Enable Apache Spark-compatible SQL dialect, functions, and expression planning
5557
-d, --disk-limit <DISK_LIMIT>
5658
Available disk space for spilling queries (e.g. '10g'), default to None (uses DataFusion's default value of '100g')
5759
--object-store-profiling <OBJECT_STORE_PROFILING>
@@ -63,6 +65,18 @@ Options:
6365
Print version
6466
```
6567
68+
## Apache Spark compatibility
69+
70+
Use `--spark` to enable the Apache Spark SQL dialect, expression planner, and
71+
function library for the CLI session:
72+
73+
```bash
74+
datafusion-cli --spark -c "SELECT next_day('2015-07-27'::DATE, 'Sun'::STRING);"
75+
```
76+
77+
Spark compatibility is disabled by default. Enabling it overrides DataFusion
78+
functions with the same name for the duration of the session.
79+
6680
## Commands
6781
6882
Available commands inside DataFusion CLI are:

0 commit comments

Comments
 (0)