Skip to content

Commit 98360e3

Browse files
KassaSanaalamb
andauthored
fix(cli): respect sql_parser.recursion_limit when parsing and validating input (apache#25807)
## Which issue does this PR close? - Closes apache#24913. ## Rationale for this change `datafusion.sql_parser.recursion_limit` has no effect in `datafusion-cli`. After `SET datafusion.sql_parser.recursion_limit = 200` (or `DATAFUSION_SQL_PARSER_RECURSION_LIMIT=200`), a query with 60 nested `abs(...)` calls still fails with `RecursionLimitExceeded (current limit: 51)` via `-c`, `-f` and the interactive shell, while the same query works through `SessionContext::sql`. Lowering the limit is ignored too. The CLI parses statements itself instead of going through `SessionState::sql_to_statement`, and only reads the dialect from the session config. There are two parse sites: - `exec_and_print` (`-c`, `-f`, rc files, and REPL execution) - `CliHelper::validate_input`, the REPL's rustyline validator, which rejects the line before it is executed The earlier attempt in apache#24914 fixed only the first one, so the REPL stayed broken. ## What changes are included in this PR? - `exec_and_print` parses with `DFParserBuilder::with_recursion_limit` using the session's limit, the same way `SessionState::sql_to_statement` does. - `CliHelper` gets a `recursion_limit` (default taken from `SqlParserOptions`) and a new `set_parser_options(&SqlParserOptions)` method that sets both the dialect and the recursion limit. `exec_from_repl` calls it once at the top of the REPL loop, before every read, so changes from SQL input, `\i` files and other backslash commands all reach the validator. `CliHelper::new` and `set_dialect` are unchanged, so this is additive. Not changed: - `is_open_quote_for_location` in `helper.rs` (tab completion for `LOCATION '...`) still uses the default parser. It already ignores the dialect, and a parse failure there only means no filename completion. - A single `-c` string or REPL line containing `SET ...; <query>` is parsed as a whole before the `SET` runs, so the new limit does not apply to the query in that same string. This matches how the dialect already behaves. Separate `-c` arguments, file lines, or REPL entries work. Something I noticed and did not special-case: with a limit of 1 or 2, even a `SET` statement fails to parse, so in the REPL you have to restart to recover (3 and above is fine). `SessionContext::sql` behaves the same way; the CLI just used to ignore the setting. ## What is the testing strategy for this PR? - Extended the existing `tests/sql/select.sql` fixture (`cli_quick_test` case `files`, `[email protected]`): it raises the limit and runs a 60-deep query, then lowers it to 5 and runs a 10-deep query. On `main` the snapshot shows the reverse (the deep query fails at 51, the shallow one succeeds). - New `helper.rs` unit test `sql_recursion_limit`, modelled on `sql_dialect`, for the REPL validator. - Manually checked a debug build with `-c`, the env var, and the interactive shell (via `script` for a pty). - Manual pty check of the REPL after `\i` (rustyline needs a terminal, so this is not an sqllogictest): `\i` a file that sets the limit to 100, then run a 60-deep query. Before the follow-up commit the query was held as incomplete and never ran; after it, the query runs. A `SET` typed directly and the default-limit rejection behave as before. Commands run: ```bash cargo test -p datafusion-cli --lib helper::tests cargo test -p datafusion-cli --test cli_integration cli_quick_test cargo test -p datafusion-cli cargo fmt --all cargo clippy -p datafusion-cli --all-targets --all-features -- -D warnings ``` ## Are there any user-facing changes? Yes: `datafusion.sql_parser.recursion_limit` is now respected by `datafusion-cli`. The only API change is the new public `CliHelper::set_parser_options` method (additive). --------- Co-authored-by: Andrew Lamb <[email protected]>
1 parent 8733e2e commit 98360e3

4 files changed

Lines changed: 74 additions & 11 deletions

File tree

‎datafusion-cli/src/exec.rs‎

Lines changed: 13 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -23,7 +23,7 @@ use datafusion::logical_expr::{DdlStatement, LogicalPlan};
2323
use datafusion::physical_plan::execution_plan::EmissionType;
2424
use datafusion::physical_plan::spill::get_record_batch_memory_size;
2525
use datafusion::physical_plan::{ExecutionPlanProperties, execute_stream};
26-
use datafusion::sql::parser::{DFParser, Statement};
26+
use datafusion::sql::parser::{DFParserBuilder, Statement};
2727
use datafusion::sql::sqlparser;
2828
use datafusion::sql::sqlparser::dialect::dialect_from_str;
2929
use futures::StreamExt;
@@ -120,6 +120,12 @@ pub async fn exec_from_repl(
120120
rl.load_history(".history").ok();
121121

122122
loop {
123+
// Parser options can change through any input, including `\i` files,
124+
// so sync the validator before every read.
125+
rl.helper_mut()
126+
.unwrap()
127+
.set_parser_options(&ctx.task_ctx().session_config().options().sql_parser);
128+
123129
match rl.readline("> ") {
124130
Ok(line) if line.starts_with('\\') => {
125131
rl.add_history_entry(line.trim_end())?;
@@ -167,13 +173,8 @@ pub async fn exec_from_repl(
167173
},
168174
_ = signal::ctrl_c() => {
169175
println!("^C");
170-
continue
171176
},
172177
}
173-
// dialect might have changed
174-
rl.helper_mut().unwrap().set_dialect(
175-
&ctx.task_ctx().session_config().options().sql_parser.dialect,
176-
);
177178
}
178179
}
179180
Err(ReadlineError::Interrupted) => {
@@ -209,7 +210,11 @@ pub(super) async fn exec_and_print(
209210
)
210211
})?;
211212

212-
let statements = DFParser::parse_sql_with_dialect(&sql, dialect.as_ref())?;
213+
let statements = DFParserBuilder::new(sql.as_str())
214+
.with_dialect(dialect.as_ref())
215+
.with_recursion_limit(options.sql_parser.recursion_limit.get())
216+
.build()?
217+
.parse_statements()?;
213218
for statement in statements {
214219
StatementExecutor::new(statement)
215220
.execute(ctx, print_options)
@@ -506,6 +511,7 @@ mod tests {
506511
use super::*;
507512

508513
use datafusion::common::plan_err;
514+
use datafusion::sql::parser::DFParser;
509515

510516
use datafusion::prelude::SessionContext;
511517
use datafusion_common::assert_contains;

‎datafusion-cli/src/helper.rs‎

Lines changed: 49 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -10,9 +10,9 @@ use std::cell::Cell;
1010

1111
use crate::highlighter::{Color, NoSyntaxHighlighter, SyntaxHighlighter};
1212

13-
use datafusion::sql::parser::{DFParser, Statement};
13+
use datafusion::sql::parser::{DFParser, DFParserBuilder, Statement};
1414
use datafusion::sql::sqlparser::dialect::dialect_from_str;
15-
use datafusion_common::config::Dialect;
15+
use datafusion_common::config::{Dialect, SqlParserOptions};
1616

1717
use rustyline::completion::{Completer, FilenameCompleter, Pair};
1818
use rustyline::error::ReadlineError;
@@ -27,6 +27,7 @@ const DEFAULT_HINT_SUGGESTION: &str = " \\? for help, \\q to quit";
2727
pub struct CliHelper {
2828
completer: FilenameCompleter,
2929
dialect: Dialect,
30+
recursion_limit: usize,
3031
highlighter: Box<dyn Highlighter>,
3132
/// Tracks whether to show the default hint. Set to `false` once the user
3233
/// types anything, so the hint doesn't reappear after deleting back to
@@ -44,6 +45,7 @@ impl CliHelper {
4445
Self {
4546
completer: FilenameCompleter::new(),
4647
dialect: *dialect,
48+
recursion_limit: SqlParserOptions::default().recursion_limit.get(),
4749
highlighter,
4850
show_hint: Cell::new(true),
4951
}
@@ -55,6 +57,13 @@ impl CliHelper {
5557
}
5658
}
5759

60+
/// Sets the parser options (dialect and recursion limit) used when
61+
/// validating input, see `datafusion.sql_parser.*`.
62+
pub fn set_parser_options(&mut self, options: &SqlParserOptions) {
63+
self.set_dialect(&options.dialect);
64+
self.recursion_limit = options.recursion_limit.get();
65+
}
66+
5867
/// Re-enable the default hint for the next prompt.
5968
pub fn reset_hint(&self) {
6069
self.show_hint.set(true);
@@ -70,7 +79,12 @@ impl CliHelper {
7079
};
7180
let lines = split_from_semicolon(sql);
7281
for line in lines {
73-
match DFParser::parse_sql_with_dialect(&line, dialect.as_ref()) {
82+
let statements = DFParserBuilder::new(line.as_str())
83+
.with_dialect(dialect.as_ref())
84+
.with_recursion_limit(self.recursion_limit)
85+
.build()
86+
.and_then(|mut parser| parser.parse_statements());
87+
match statements {
7488
Ok(statements) if statements.is_empty() => {
7589
return Ok(ValidationResult::Invalid(Some(
7690
" 🤔 You entered an empty statement".to_string(),
@@ -201,6 +215,8 @@ mod tests {
201215

202216
use super::*;
203217

218+
use datafusion_common::config::ConfigNonZeroUsize;
219+
204220
fn readline_direct(
205221
mut reader: impl BufRead,
206222
validator: &CliHelper,
@@ -305,6 +321,36 @@ mod tests {
305321
Ok(())
306322
}
307323

324+
#[test]
325+
fn sql_recursion_limit() -> Result<()> {
326+
let mut validator = CliHelper::default();
327+
let nested = |depth: usize| {
328+
format!("select {}1{};", "abs(".repeat(depth), ")".repeat(depth))
329+
};
330+
let with_limit = |limit: usize| SqlParserOptions {
331+
recursion_limit: ConfigNonZeroUsize::try_new(limit).unwrap(),
332+
..Default::default()
333+
};
334+
335+
// exceeds the default recursion limit
336+
let result = readline_direct(Cursor::new(nested(60)), &validator)?;
337+
assert!(
338+
matches!(result, ValidationResult::Invalid(Some(e)) if e.contains("RecursionLimitExceeded"))
339+
);
340+
341+
validator.set_parser_options(&with_limit(100));
342+
let result = readline_direct(Cursor::new(nested(60)), &validator)?;
343+
assert!(matches!(result, ValidationResult::Valid(None)));
344+
345+
validator.set_parser_options(&with_limit(5));
346+
let result = readline_direct(Cursor::new(nested(10)), &validator)?;
347+
assert!(
348+
matches!(result, ValidationResult::Invalid(Some(e)) if e.contains("RecursionLimitExceeded"))
349+
);
350+
351+
Ok(())
352+
}
353+
308354
#[test]
309355
fn test_split_from_semicolon() {
310356
let sql = "SELECT 1; SELECT 2;";

‎datafusion-cli/tests/snapshots/[email protected]‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -15,5 +15,11 @@ exit_code: 0
1515
+----------+
1616
| 1 |
1717
+----------+
18+
+------+
19+
| deep |
20+
+------+
21+
| 1 |
22+
+------+
1823

1924
----- stderr -----
25+
SQL error: RecursionLimitExceeded (current limit: 5)
Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1 +1,6 @@
1-
select 1;
1+
select 1;
2+
-- datafusion.sql_parser.recursion_limit is respected when parsing (#24913)
3+
set datafusion.sql_parser.recursion_limit = 100;
4+
select abs(abs(abs(abs(abs(abs(abs(abs(abs(abs(abs(abs(abs(abs(abs(abs(abs(abs(abs(abs(abs(abs(abs(abs(abs(abs(abs(abs(abs(abs(abs(abs(abs(abs(abs(abs(abs(abs(abs(abs(abs(abs(abs(abs(abs(abs(abs(abs(abs(abs(abs(abs(abs(abs(abs(abs(abs(abs(abs(abs(1)))))))))))))))))))))))))))))))))))))))))))))))))))))))))))) as deep;
5+
set datafusion.sql_parser.recursion_limit = 5;
6+
select abs(abs(abs(abs(abs(abs(abs(abs(abs(abs(1)))))))))) as shallow;

0 commit comments

Comments
 (0)