@@ -92,9 +92,16 @@ pub trait ContextProvider {
9292 fn get_config_option ( & self , variable : & str ) -> Option < ScalarValue > ;
9393}
9494
95+ /// SQL parser options
96+ #[ derive( Debug , Default ) ]
97+ pub struct ParserOptions {
98+ parse_float_as_decimal : bool ,
99+ }
100+
95101/// SQL query planner
96102pub struct SqlToRel < ' a , S : ContextProvider > {
97103 schema_provider : & ' a S ,
104+ options : ParserOptions ,
98105}
99106
100107fn plan_key ( key : SQLExpr ) -> Result < ScalarValue > {
@@ -137,7 +144,15 @@ fn plan_indexed(expr: Expr, mut keys: Vec<SQLExpr>) -> Result<Expr> {
137144impl < ' a , S : ContextProvider > SqlToRel < ' a , S > {
138145 /// Create a new query planner
139146 pub fn new ( schema_provider : & ' a S ) -> Self {
140- SqlToRel { schema_provider }
147+ Self :: new_with_options ( schema_provider, ParserOptions :: default ( ) )
148+ }
149+
150+ /// Create a new query planner
151+ pub fn new_with_options ( schema_provider : & ' a S , options : ParserOptions ) -> Self {
152+ SqlToRel {
153+ schema_provider,
154+ options,
155+ }
141156 }
142157
143158 /// Generate a logical plan from an DataFusion SQL statement
@@ -1699,7 +1714,7 @@ impl<'a, S: ContextProvider> SqlToRel<'a, S> {
16991714 . map ( |row| {
17001715 row. into_iter ( )
17011716 . map ( |v| match v {
1702- SQLExpr :: Value ( Value :: Number ( n, _) ) => parse_sql_number ( & n) ,
1717+ SQLExpr :: Value ( Value :: Number ( n, _) ) => self . parse_sql_number ( & n) ,
17031718 SQLExpr :: Value (
17041719 Value :: SingleQuotedString ( s) | Value :: DoubleQuotedString ( s) ,
17051720 ) => Ok ( lit ( s) ) ,
@@ -1753,7 +1768,7 @@ impl<'a, S: ContextProvider> SqlToRel<'a, S> {
17531768 ctes : & mut HashMap < String , LogicalPlan > ,
17541769 ) -> Result < Expr > {
17551770 match sql {
1756- SQLExpr :: Value ( Value :: Number ( n, _) ) => parse_sql_number ( & n) ,
1771+ SQLExpr :: Value ( Value :: Number ( n, _) ) => self . parse_sql_number ( & n) ,
17571772 SQLExpr :: Value ( Value :: SingleQuotedString ( ref s) | Value :: DoubleQuotedString ( ref s) ) => Ok ( lit ( s. clone ( ) ) ) ,
17581773 SQLExpr :: Value ( Value :: Boolean ( n) ) => Ok ( lit ( n) ) ,
17591774 SQLExpr :: Value ( Value :: Null ) => Ok ( Expr :: Literal ( ScalarValue :: Null ) ) ,
@@ -2668,6 +2683,51 @@ impl<'a, S: ContextProvider> SqlToRel<'a, S> {
26682683 }
26692684 }
26702685
2686+ /// Parse number in sql string, convert to Expr::Literal
2687+ fn parse_sql_number ( & self , n : & str ) -> Result < Expr > {
2688+ if n. find ( 'E' ) . is_some ( ) {
2689+ // not implemented yet
2690+ // https://github.com/apache/arrow-datafusion/issues/3448
2691+ Err ( DataFusionError :: NotImplemented (
2692+ "sql numeric literals in scientific notation are not supported"
2693+ . to_string ( ) ,
2694+ ) )
2695+ } else if let Ok ( n) = n. parse :: < i64 > ( ) {
2696+ Ok ( lit ( n) )
2697+ } else if self . options . parse_float_as_decimal {
2698+ // remove leading zeroes
2699+ let str = n. trim_start_matches ( '0' ) ;
2700+ if let Some ( i) = str. find ( '.' ) {
2701+ let p = str. len ( ) - 1 ;
2702+ let s = str. len ( ) - i - 1 ;
2703+ let str = str. replace ( '.' , "" ) ;
2704+ let n = str. parse :: < i128 > ( ) . map_err ( |_| {
2705+ DataFusionError :: from ( ParserError ( format ! (
2706+ "Cannot parse {} as i128 when building decimal" ,
2707+ str
2708+ ) ) )
2709+ } ) ?;
2710+ Ok ( Expr :: Literal ( ScalarValue :: Decimal128 (
2711+ Some ( n) ,
2712+ p as u8 ,
2713+ s as u8 ,
2714+ ) ) )
2715+ } else {
2716+ let number = n. parse :: < i128 > ( ) . map_err ( |_| {
2717+ DataFusionError :: from ( ParserError ( format ! (
2718+ "Cannot parse {} as i128 when building decimal" ,
2719+ n
2720+ ) ) )
2721+ } ) ?;
2722+ Ok ( Expr :: Literal ( ScalarValue :: Decimal128 ( Some ( number) , 38 , 0 ) ) )
2723+ }
2724+ } else {
2725+ n. parse :: < f64 > ( ) . map ( lit) . map_err ( |_| {
2726+ DataFusionError :: from ( ParserError ( format ! ( "Cannot parse {} as f64" , n) ) )
2727+ } )
2728+ }
2729+ }
2730+
26712731 fn convert_data_type ( & self , sql_type : & SQLDataType ) -> Result < DataType > {
26722732 match sql_type {
26732733 SQLDataType :: Array ( inner_sql_type) => {
@@ -2919,28 +2979,40 @@ fn extract_possible_join_keys(
29192979 }
29202980}
29212981
2922- // Parse number in sql string, convert to Expr::Literal
2923- fn parse_sql_number ( n : & str ) -> Result < Expr > {
2924- // parse first as i64
2925- n. parse :: < i64 > ( )
2926- . map ( lit)
2927- // if parsing as i64 fails try f64
2928- . or_else ( |_| n. parse :: < f64 > ( ) . map ( lit) )
2929- . map_err ( |_| {
2930- DataFusionError :: from ( ParserError ( format ! (
2931- "Cannot parse {} as i64 or f64" ,
2932- n
2933- ) ) )
2934- } )
2935- }
2936-
29372982#[ cfg( test) ]
29382983mod tests {
29392984 use super :: * ;
29402985 use datafusion_common:: assert_contains;
29412986 use sqlparser:: dialect:: { Dialect , GenericDialect , HiveDialect , MySqlDialect } ;
29422987 use std:: any:: Any ;
29432988
2989+ #[ test]
2990+ fn parse_decimals ( ) {
2991+ let test_data = [
2992+ ( "1" , "Int64(1)" ) ,
2993+ ( "001" , "Int64(1)" ) ,
2994+ ( "0.1" , "Decimal128(Some(1),1,1)" ) ,
2995+ ( "0.01" , "Decimal128(Some(1),2,2)" ) ,
2996+ ( "1.0" , "Decimal128(Some(10),2,1)" ) ,
2997+ ( "10.01" , "Decimal128(Some(1001),4,2)" ) ,
2998+ (
2999+ "10000000000000000000.00" ,
3000+ "Decimal128(Some(1000000000000000000000),22,2)" ,
3001+ ) ,
3002+ ] ;
3003+ for ( a, b) in test_data {
3004+ let sql = format ! ( "SELECT {}" , a) ;
3005+ let expected = format ! ( "Projection: {}\n EmptyRelation" , b) ;
3006+ quick_test_with_options (
3007+ & sql,
3008+ & expected,
3009+ ParserOptions {
3010+ parse_float_as_decimal : true ,
3011+ } ,
3012+ ) ;
3013+ }
3014+ }
3015+
29443016 #[ test]
29453017 fn select_no_relation ( ) {
29463018 quick_test (
@@ -4913,8 +4985,15 @@ mod tests {
49134985 }
49144986
49154987 fn logical_plan ( sql : & str ) -> Result < LogicalPlan > {
4988+ logical_plan_with_options ( sql, ParserOptions :: default ( ) )
4989+ }
4990+
4991+ fn logical_plan_with_options (
4992+ sql : & str ,
4993+ options : ParserOptions ,
4994+ ) -> Result < LogicalPlan > {
49164995 let dialect = & GenericDialect { } ;
4917- logical_plan_with_dialect ( sql, dialect)
4996+ logical_plan_with_dialect_and_options ( sql, dialect, options )
49184997 }
49194998
49204999 fn logical_plan_with_dialect (
@@ -4927,12 +5006,28 @@ mod tests {
49275006 planner. statement_to_plan ( ast. pop_front ( ) . unwrap ( ) )
49285007 }
49295008
5009+ fn logical_plan_with_dialect_and_options (
5010+ sql : & str ,
5011+ dialect : & dyn Dialect ,
5012+ options : ParserOptions ,
5013+ ) -> Result < LogicalPlan > {
5014+ let planner = SqlToRel :: new_with_options ( & MockContextProvider { } , options) ;
5015+ let result = DFParser :: parse_sql_with_dialect ( sql, dialect) ;
5016+ let mut ast = result?;
5017+ planner. statement_to_plan ( ast. pop_front ( ) . unwrap ( ) )
5018+ }
5019+
49305020 /// Create logical plan, write with formatter, compare to expected output
49315021 fn quick_test ( sql : & str , expected : & str ) {
49325022 let plan = logical_plan ( sql) . unwrap ( ) ;
49335023 assert_eq ! ( format!( "{:?}" , plan) , expected) ;
49345024 }
49355025
5026+ fn quick_test_with_options ( sql : & str , expected : & str , options : ParserOptions ) {
5027+ let plan = logical_plan_with_options ( sql, options) . unwrap ( ) ;
5028+ assert_eq ! ( format!( "{:?}" , plan) , expected) ;
5029+ }
5030+
49365031 struct MockContextProvider { }
49375032
49385033 impl ContextProvider for MockContextProvider {
0 commit comments