1616// under the License.
1717
1818//! This example demonstrates extending the DataFusion SQL parser to support
19- //! custom DDL statements, specifically `CREATE EXTERNAL CATALOG`.
19+ //! custom DDL statements, specifically `CREATE FOREIGN CATALOG`.
20+ //!
21+ //! Note: DataFusion supports `CREATE EXTERNAL CATALOG` out-of-the-box making use of
22+ //! [`CatalogProviderFactory`](datafusion::catalog::CatalogProviderFactory). This example
23+ //! is a partial reimplementation of the existing functionality to demonstrate how to extend the
24+ //! SQL parser.
2025//!
2126//! ### Custom Syntax
2227//! ```sql
23- //! CREATE EXTERNAL CATALOG my_catalog
28+ //! CREATE FOREIGN CATALOG my_catalog
2429//! STORED AS ICEBERG
2530//! LOCATION 's3://my-bucket/warehouse/'
2631//! OPTIONS (
@@ -59,7 +64,7 @@ use object_store::local::LocalFileSystem;
5964
6065/// Entry point for the example.
6166pub async fn custom_sql_parser ( ) -> Result < ( ) > {
62- // Use standard Parquet testing data as our "external " source.
67+ // Use standard Parquet testing data as our "foreign " source.
6368 let base_path = datafusion:: common:: test_util:: parquet_test_data ( ) ;
6469 let base_path = std:: path:: Path :: new ( & base_path) . canonicalize ( ) ?;
6570
@@ -71,7 +76,7 @@ pub async fn custom_sql_parser() -> Result<()> {
7176 . unwrap_or_else ( |_| base_path. to_string_lossy ( ) . to_string ( ) ) ;
7277
7378 let create_catalog_sql = format ! (
74- "CREATE EXTERNAL CATALOG parquet_testing
79+ "CREATE FOREIGN CATALOG parquet_testing
7580 STORED AS parquet
7681 LOCATION 'local://workspace/{location}'
7782 OPTIONS (
@@ -90,10 +95,10 @@ pub async fn custom_sql_parser() -> Result<()> {
9095 let err = ctx_standard
9196 . sql ( & create_catalog_sql)
9297 . await
93- . expect_err ( "Expected the standard parser to reject CREATE EXTERNAL CATALOG (custom DDL syntax)" ) ;
98+ . expect_err ( "Expected the standard parser to reject CREATE FOREIGN CATALOG (custom DDL syntax)" ) ;
9499
95100 println ! ( "Error: {err}\n " ) ;
96- assert_snapshot ! ( err. to_string( ) , @r#"SQL error: ParserError("Expected: TABLE , found: CATALOG at Line: 1, Column: 17 ")"# ) ;
101+ assert_snapshot ! ( err. to_string( ) , @r#"SQL error: ParserError("Expected: an object type after CREATE , found: FOREIGN at Line: 1, Column: 8 ")"# ) ;
97102
98103 // =========================================================================
99104 // Part 2: Custom parser handles the statement
@@ -106,11 +111,11 @@ pub async fn custom_sql_parser() -> Result<()> {
106111 let mut parser = CustomParser :: new ( & create_catalog_sql) ?;
107112 let statement = parser. parse_statement ( ) ?;
108113 match statement {
109- CustomStatement :: CreateExternalCatalog ( stmt) => {
110- handle_create_external_catalog ( & ctx, stmt) . await ?;
114+ CustomStatement :: CreateForeignCatalog ( stmt) => {
115+ handle_create_foreign_catalog ( & ctx, stmt) . await ?;
111116 }
112117 CustomStatement :: DFStatement ( _) => {
113- panic ! ( "Expected CreateExternalCatalog statement" ) ;
118+ panic ! ( "Expected CreateForeignCatalog statement" ) ;
114119 }
115120 }
116121
@@ -141,10 +146,10 @@ async fn execute_sql(ctx: &SessionContext, sql: &str) -> Result<String> {
141146 Ok ( arrow:: util:: pretty:: pretty_format_batches ( & batches) ?. to_string ( ) )
142147}
143148
144- /// Custom handler for the `CREATE EXTERNAL CATALOG` statement.
145- async fn handle_create_external_catalog (
149+ /// Custom handler for the `CREATE FOREIGN CATALOG` statement.
150+ async fn handle_create_foreign_catalog (
146151 ctx : & SessionContext ,
147- stmt : CreateExternalCatalog ,
152+ stmt : CreateForeignCatalog ,
148153) -> Result < ( ) > {
149154 let factory = ListingTableFactory :: new ( ) ;
150155 let catalog = Arc :: new ( MemoryCatalogProvider :: new ( ) ) ;
@@ -240,13 +245,13 @@ async fn handle_create_external_catalog(
240245pub enum CustomStatement {
241246 /// Standard DataFusion statement
242247 DFStatement ( Box < Statement > ) ,
243- /// Custom `CREATE EXTERNAL CATALOG` statement
244- CreateExternalCatalog ( CreateExternalCatalog ) ,
248+ /// Custom `CREATE FOREIGN CATALOG` statement
249+ CreateForeignCatalog ( CreateForeignCatalog ) ,
245250}
246251
247- /// Data structure for `CREATE EXTERNAL CATALOG`.
252+ /// Data structure for `CREATE FOREIGN CATALOG`.
248253#[ derive( Debug , Clone ) ]
249- pub struct CreateExternalCatalog {
254+ pub struct CreateForeignCatalog {
250255 pub name : ObjectName ,
251256 pub catalog_type : String ,
252257 pub location : String ,
@@ -257,16 +262,16 @@ impl Display for CustomStatement {
257262 fn fmt ( & self , f : & mut std:: fmt:: Formatter < ' _ > ) -> std:: fmt:: Result {
258263 match self {
259264 Self :: DFStatement ( s) => write ! ( f, "{s}" ) ,
260- Self :: CreateExternalCatalog ( s) => write ! ( f, "{s}" ) ,
265+ Self :: CreateForeignCatalog ( s) => write ! ( f, "{s}" ) ,
261266 }
262267 }
263268}
264269
265- impl Display for CreateExternalCatalog {
270+ impl Display for CreateForeignCatalog {
266271 fn fmt ( & self , f : & mut std:: fmt:: Formatter < ' _ > ) -> std:: fmt:: Result {
267272 write ! (
268273 f,
269- "CREATE EXTERNAL CATALOG {} STORED AS {} LOCATION '{}'" ,
274+ "CREATE FOREIGN CATALOG {} STORED AS {} LOCATION '{}'" ,
270275 self . name, self . catalog_type, self . location
271276 ) ?;
272277 if !self . options . is_empty ( ) {
@@ -296,28 +301,23 @@ impl<'a> CustomParser<'a> {
296301 }
297302
298303 pub fn parse_statement ( & mut self ) -> Result < CustomStatement > {
299- if self . is_create_external_catalog ( ) {
300- return self . parse_create_external_catalog ( ) ;
304+ if let Some ( cfc ) = self . parse_create_foreign_catalog ( ) ? {
305+ return Ok ( CustomStatement :: CreateForeignCatalog ( cfc ) ) ;
301306 }
307+
302308 Ok ( CustomStatement :: DFStatement ( Box :: new (
303309 self . df_parser . parse_statement ( ) ?,
304310 ) ) )
305311 }
306312
307- fn is_create_external_catalog ( & self ) -> bool {
308- let t1 = & self . df_parser . parser . peek_nth_token ( 0 ) . token ;
309- let t2 = & self . df_parser . parser . peek_nth_token ( 1 ) . token ;
310- let t3 = & self . df_parser . parser . peek_nth_token ( 2 ) . token ;
311-
312- matches ! ( t1, Token :: Word ( w) if w. keyword == Keyword :: CREATE )
313- && matches ! ( t2, Token :: Word ( w) if w. keyword == Keyword :: EXTERNAL )
314- && matches ! ( t3, Token :: Word ( w) if w. value. to_uppercase( ) == "CATALOG" )
315- }
316-
317- fn parse_create_external_catalog ( & mut self ) -> Result < CustomStatement > {
318- // Consume prefix tokens: CREATE EXTERNAL CATALOG
319- for _ in 0 ..3 {
320- self . df_parser . parser . next_token ( ) ;
313+ /// Parses a `CREATE FOREIGN CATALOG` statement.
314+ fn parse_create_foreign_catalog ( & mut self ) -> Result < Option < CreateForeignCatalog > > {
315+ if !self . df_parser . parser . parse_keywords ( & [
316+ Keyword :: CREATE ,
317+ Keyword :: FOREIGN ,
318+ Keyword :: CATALOG ,
319+ ] ) {
320+ return Ok ( None ) ;
321321 }
322322
323323 let name = self
@@ -373,16 +373,13 @@ impl<'a> CustomParser<'a> {
373373 }
374374 }
375375
376- Ok ( CustomStatement :: CreateExternalCatalog (
377- CreateExternalCatalog {
378- name,
379- catalog_type : catalog_type
380- . ok_or_else ( || plan_datafusion_err ! ( "Missing STORED AS" ) ) ?,
381- location : location
382- . ok_or_else ( || plan_datafusion_err ! ( "Missing LOCATION" ) ) ?,
383- options,
384- } ,
385- ) )
376+ Ok ( Some ( CreateForeignCatalog {
377+ name,
378+ catalog_type : catalog_type
379+ . ok_or_else ( || plan_datafusion_err ! ( "Missing STORED AS" ) ) ?,
380+ location : location. ok_or_else ( || plan_datafusion_err ! ( "Missing LOCATION" ) ) ?,
381+ options,
382+ } ) )
386383 }
387384
388385 /// Parse options in the form: (key [=] value, key [=] value, ...)
0 commit comments