Skip to content

Commit e3812d0

Browse files
pepijnvealamb
authored andcommitted
Add an extension point to allow SQL based registration of external catalogs (apache#25112)
## Which issue does this PR close? - Closes apache#25111. ## Rationale for this change Adding a `CatalogProviderFactory` trait, similar to `TableProviderFactory`, makes it possible for extensions to register support for external catalogs that can then be used by consumers without requiring additional code. This is quite convenient when working with open table formats via catalogs. ## What changes are included in this PR? - Adds a `CatalogProviderFactory` trait similar to `TableProviderFactory` - Adds `CREATE EXTERNAL CATALOG` and `DROP CATALOG` support ## What is the testing strategy for this PR? - Covered by unit and integration tests ## Are there any user-facing changes? Yes, the supported DDL has been extended. Documentation has been updated to reflect this. --------- Co-authored-by: Andrew Lamb <[email protected]>
1 parent 8253ffe commit e3812d0

24 files changed

Lines changed: 1176 additions & 101 deletions

File tree

‎datafusion-cli/src/catalog.rs‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -57,6 +57,14 @@ impl CatalogProviderList for DynamicObjectStoreCatalog {
5757
self.inner.register_catalog(name, catalog)
5858
}
5959

60+
fn deregister_catalog(
61+
&self,
62+
name: &str,
63+
cascade: bool,
64+
) -> Result<Option<Arc<dyn CatalogProvider>>> {
65+
self.inner.deregister_catalog(name, cascade)
66+
}
67+
6068
fn catalog_names(&self) -> Vec<String> {
6169
self.inner.catalog_names()
6270
}

‎datafusion-examples/examples/data_io/catalog.rs‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -279,6 +279,21 @@ impl CatalogProviderList for CustomCatalogProviderList {
279279
Some(catalog)
280280
}
281281

282+
fn deregister_catalog(
283+
&self,
284+
name: &str,
285+
cascade: bool,
286+
) -> Result<Option<Arc<dyn CatalogProvider>>> {
287+
let mut cats = self.catalogs.write().unwrap();
288+
let Some(cat) = cats.get_mut(name) else {
289+
return Ok(None);
290+
};
291+
292+
cat.prepare_deregister_catalog(cascade)?;
293+
294+
Ok(cats.remove(name))
295+
}
296+
282297
/// Retrieves the list of available catalog names
283298
fn catalog_names(&self) -> Vec<String> {
284299
let cats = self.catalogs.read().unwrap();

‎datafusion-examples/examples/sql_ops/custom_sql_parser.rs‎

Lines changed: 42 additions & 45 deletions
Original file line numberDiff line numberDiff line change
@@ -16,11 +16,16 @@
1616
// under the License.
1717

1818
//! This example demonstrates extending the DataFusion SQL parser to support
19-
//! custom DDL statements, specifically `CREATE EXTERNAL CATALOG`.
19+
//! custom DDL statements, specifically `CREATE FOREIGN CATALOG`.
20+
//!
21+
//! Note: DataFusion supports `CREATE EXTERNAL CATALOG` out-of-the-box making use of
22+
//! [`CatalogProviderFactory`](datafusion::catalog::CatalogProviderFactory). This example
23+
//! is a partial reimplementation of the existing functionality to demonstrate how to extend the
24+
//! SQL parser.
2025
//!
2126
//! ### Custom Syntax
2227
//! ```sql
23-
//! CREATE EXTERNAL CATALOG my_catalog
28+
//! CREATE FOREIGN CATALOG my_catalog
2429
//! STORED AS ICEBERG
2530
//! LOCATION 's3://my-bucket/warehouse/'
2631
//! OPTIONS (
@@ -59,7 +64,7 @@ use object_store::local::LocalFileSystem;
5964

6065
/// Entry point for the example.
6166
pub async fn custom_sql_parser() -> Result<()> {
62-
// Use standard Parquet testing data as our "external" source.
67+
// Use standard Parquet testing data as our "foreign" source.
6368
let base_path = datafusion::common::test_util::parquet_test_data();
6469
let base_path = std::path::Path::new(&base_path).canonicalize()?;
6570

@@ -71,7 +76,7 @@ pub async fn custom_sql_parser() -> Result<()> {
7176
.unwrap_or_else(|_| base_path.to_string_lossy().to_string());
7277

7378
let create_catalog_sql = format!(
74-
"CREATE EXTERNAL CATALOG parquet_testing
79+
"CREATE FOREIGN CATALOG parquet_testing
7580
STORED AS parquet
7681
LOCATION 'local://workspace/{location}'
7782
OPTIONS (
@@ -90,10 +95,10 @@ pub async fn custom_sql_parser() -> Result<()> {
9095
let err = ctx_standard
9196
.sql(&create_catalog_sql)
9297
.await
93-
.expect_err("Expected the standard parser to reject CREATE EXTERNAL CATALOG (custom DDL syntax)");
98+
.expect_err("Expected the standard parser to reject CREATE FOREIGN CATALOG (custom DDL syntax)");
9499

95100
println!("Error: {err}\n");
96-
assert_snapshot!(err.to_string(), @r#"SQL error: ParserError("Expected: TABLE, found: CATALOG at Line: 1, Column: 17")"#);
101+
assert_snapshot!(err.to_string(), @r#"SQL error: ParserError("Expected: an object type after CREATE, found: FOREIGN at Line: 1, Column: 8")"#);
97102

98103
// =========================================================================
99104
// Part 2: Custom parser handles the statement
@@ -106,11 +111,11 @@ pub async fn custom_sql_parser() -> Result<()> {
106111
let mut parser = CustomParser::new(&create_catalog_sql)?;
107112
let statement = parser.parse_statement()?;
108113
match statement {
109-
CustomStatement::CreateExternalCatalog(stmt) => {
110-
handle_create_external_catalog(&ctx, stmt).await?;
114+
CustomStatement::CreateForeignCatalog(stmt) => {
115+
handle_create_foreign_catalog(&ctx, stmt).await?;
111116
}
112117
CustomStatement::DFStatement(_) => {
113-
panic!("Expected CreateExternalCatalog statement");
118+
panic!("Expected CreateForeignCatalog statement");
114119
}
115120
}
116121

@@ -141,10 +146,10 @@ async fn execute_sql(ctx: &SessionContext, sql: &str) -> Result<String> {
141146
Ok(arrow::util::pretty::pretty_format_batches(&batches)?.to_string())
142147
}
143148

144-
/// Custom handler for the `CREATE EXTERNAL CATALOG` statement.
145-
async fn handle_create_external_catalog(
149+
/// Custom handler for the `CREATE FOREIGN CATALOG` statement.
150+
async fn handle_create_foreign_catalog(
146151
ctx: &SessionContext,
147-
stmt: CreateExternalCatalog,
152+
stmt: CreateForeignCatalog,
148153
) -> Result<()> {
149154
let factory = ListingTableFactory::new();
150155
let catalog = Arc::new(MemoryCatalogProvider::new());
@@ -240,13 +245,13 @@ async fn handle_create_external_catalog(
240245
pub enum CustomStatement {
241246
/// Standard DataFusion statement
242247
DFStatement(Box<Statement>),
243-
/// Custom `CREATE EXTERNAL CATALOG` statement
244-
CreateExternalCatalog(CreateExternalCatalog),
248+
/// Custom `CREATE FOREIGN CATALOG` statement
249+
CreateForeignCatalog(CreateForeignCatalog),
245250
}
246251

247-
/// Data structure for `CREATE EXTERNAL CATALOG`.
252+
/// Data structure for `CREATE FOREIGN CATALOG`.
248253
#[derive(Debug, Clone)]
249-
pub struct CreateExternalCatalog {
254+
pub struct CreateForeignCatalog {
250255
pub name: ObjectName,
251256
pub catalog_type: String,
252257
pub location: String,
@@ -257,16 +262,16 @@ impl Display for CustomStatement {
257262
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
258263
match self {
259264
Self::DFStatement(s) => write!(f, "{s}"),
260-
Self::CreateExternalCatalog(s) => write!(f, "{s}"),
265+
Self::CreateForeignCatalog(s) => write!(f, "{s}"),
261266
}
262267
}
263268
}
264269

265-
impl Display for CreateExternalCatalog {
270+
impl Display for CreateForeignCatalog {
266271
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
267272
write!(
268273
f,
269-
"CREATE EXTERNAL CATALOG {} STORED AS {} LOCATION '{}'",
274+
"CREATE FOREIGN CATALOG {} STORED AS {} LOCATION '{}'",
270275
self.name, self.catalog_type, self.location
271276
)?;
272277
if !self.options.is_empty() {
@@ -296,28 +301,23 @@ impl<'a> CustomParser<'a> {
296301
}
297302

298303
pub fn parse_statement(&mut self) -> Result<CustomStatement> {
299-
if self.is_create_external_catalog() {
300-
return self.parse_create_external_catalog();
304+
if let Some(cfc) = self.parse_create_foreign_catalog()? {
305+
return Ok(CustomStatement::CreateForeignCatalog(cfc));
301306
}
307+
302308
Ok(CustomStatement::DFStatement(Box::new(
303309
self.df_parser.parse_statement()?,
304310
)))
305311
}
306312

307-
fn is_create_external_catalog(&self) -> bool {
308-
let t1 = &self.df_parser.parser.peek_nth_token(0).token;
309-
let t2 = &self.df_parser.parser.peek_nth_token(1).token;
310-
let t3 = &self.df_parser.parser.peek_nth_token(2).token;
311-
312-
matches!(t1, Token::Word(w) if w.keyword == Keyword::CREATE)
313-
&& matches!(t2, Token::Word(w) if w.keyword == Keyword::EXTERNAL)
314-
&& matches!(t3, Token::Word(w) if w.value.to_uppercase() == "CATALOG")
315-
}
316-
317-
fn parse_create_external_catalog(&mut self) -> Result<CustomStatement> {
318-
// Consume prefix tokens: CREATE EXTERNAL CATALOG
319-
for _ in 0..3 {
320-
self.df_parser.parser.next_token();
313+
/// Parses a `CREATE FOREIGN CATALOG` statement.
314+
fn parse_create_foreign_catalog(&mut self) -> Result<Option<CreateForeignCatalog>> {
315+
if !self.df_parser.parser.parse_keywords(&[
316+
Keyword::CREATE,
317+
Keyword::FOREIGN,
318+
Keyword::CATALOG,
319+
]) {
320+
return Ok(None);
321321
}
322322

323323
let name = self
@@ -373,16 +373,13 @@ impl<'a> CustomParser<'a> {
373373
}
374374
}
375375

376-
Ok(CustomStatement::CreateExternalCatalog(
377-
CreateExternalCatalog {
378-
name,
379-
catalog_type: catalog_type
380-
.ok_or_else(|| plan_datafusion_err!("Missing STORED AS"))?,
381-
location: location
382-
.ok_or_else(|| plan_datafusion_err!("Missing LOCATION"))?,
383-
options,
384-
},
385-
))
376+
Ok(Some(CreateForeignCatalog {
377+
name,
378+
catalog_type: catalog_type
379+
.ok_or_else(|| plan_datafusion_err!("Missing STORED AS"))?,
380+
location: location.ok_or_else(|| plan_datafusion_err!("Missing LOCATION"))?,
381+
options,
382+
}))
386383
}
387384

388385
/// Parse options in the form: (key [=] value, key [=] value, ...)

‎datafusion/catalog/src/catalog.rs‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,9 @@
1616
// under the License.
1717

1818
// Re-export from this module for backwards compatibility.
19-
pub use datafusion_session::{CatalogProvider, CatalogProviderList};
19+
pub use datafusion_session::{
20+
CatalogProvider, CatalogProviderFactory, CatalogProviderList,
21+
};
2022
// Re-export so users can access this type through `datafusion_catalog` and
2123
// `datafusion::catalog` without depending directly on `datafusion_session`.
2224
pub use datafusion_session::EmptyCatalogProviderList;

‎datafusion/catalog/src/dynamic_file/catalog.rs‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -49,6 +49,14 @@ impl CatalogProviderList for DynamicFileCatalog {
4949
self.inner.register_catalog(name, catalog)
5050
}
5151

52+
fn deregister_catalog(
53+
&self,
54+
name: &str,
55+
cascade: bool,
56+
) -> datafusion_common::Result<Option<Arc<dyn CatalogProvider>>> {
57+
self.inner.deregister_catalog(name, cascade)
58+
}
59+
5260
fn catalog_names(&self) -> Vec<String> {
5361
self.inner.catalog_names()
5462
}

‎datafusion/catalog/src/memory/catalog.rs‎

Lines changed: 34 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,7 @@
2020
2121
use crate::{CatalogProvider, CatalogProviderList, SchemaProvider};
2222
use dashmap::DashMap;
23-
use datafusion_common::exec_err;
23+
use datafusion_common::{Result, exec_err};
2424
use std::sync::Arc;
2525

2626
/// Simple in-memory list of catalogs
@@ -54,6 +54,24 @@ impl CatalogProviderList for MemoryCatalogProviderList {
5454
self.catalogs.insert(name, catalog)
5555
}
5656

57+
fn deregister_catalog(
58+
&self,
59+
name: &str,
60+
cascade: bool,
61+
) -> Result<Option<Arc<dyn CatalogProvider>>> {
62+
let Some(mut entry) = self.catalogs.get_mut(name) else {
63+
return Ok(None);
64+
};
65+
66+
let catalog = entry.value_mut();
67+
catalog.prepare_deregister_catalog(cascade)?;
68+
69+
// Drop entry to release its lock
70+
drop(entry);
71+
72+
Ok(self.catalogs.remove(name).map(|(_, catalog)| catalog))
73+
}
74+
5775
fn catalog_names(&self) -> Vec<String> {
5876
self.catalogs.iter().map(|c| c.key().clone()).collect()
5977
}
@@ -97,15 +115,15 @@ impl CatalogProvider for MemoryCatalogProvider {
97115
&self,
98116
name: &str,
99117
schema: Arc<dyn SchemaProvider>,
100-
) -> datafusion_common::Result<Option<Arc<dyn SchemaProvider>>> {
118+
) -> Result<Option<Arc<dyn SchemaProvider>>> {
101119
Ok(self.schemas.insert(name.into(), schema))
102120
}
103121

104122
fn deregister_schema(
105123
&self,
106124
name: &str,
107125
cascade: bool,
108-
) -> datafusion_common::Result<Option<Arc<dyn SchemaProvider>>> {
126+
) -> Result<Option<Arc<dyn SchemaProvider>>> {
109127
if let Some(schema) = self.schema(name) {
110128
let table_names = schema.table_names();
111129
match (table_names.is_empty(), cascade) {
@@ -114,7 +132,7 @@ impl CatalogProvider for MemoryCatalogProvider {
114132
Ok(Some(removed))
115133
}
116134
(false, false) => exec_err!(
117-
"Cannot drop schema {} because other tables depend on it: {}",
135+
"Cannot drop schema {} containing tables: {}",
118136
name,
119137
itertools::join(table_names.iter(), ", ")
120138
),
@@ -123,4 +141,16 @@ impl CatalogProvider for MemoryCatalogProvider {
123141
Ok(None)
124142
}
125143
}
144+
145+
fn prepare_deregister_catalog(&self, cascade: bool) -> Result<()> {
146+
if !cascade && !self.schemas.is_empty() {
147+
return exec_err!(
148+
"Cannot drop catalog containing schemas: {}",
149+
itertools::join(self.schema_names().iter(), ", ")
150+
);
151+
}
152+
153+
self.schemas.clear();
154+
Ok(())
155+
}
126156
}

0 commit comments

Comments
 (0)