Skip to content

Commit 910e8ee

Browse files
pepijnveVaibhaveSalamb
authored
Support partitioning of information schema content per catalog (apache#25887)
## Which issue does this PR close? - Closes apache#15960. - Supersedes / closes apache#24200. - Prerequisite for PR apache#26057. ## Rationale for this change Catalog-qualified information schema queries currently return metadata for object from every registered catalog. If we follow the letter of the SQL specification, `my_catalog.information_schema.tables` should describe only `my_catalog`. This is consistent with PostgreSQL's information schema representing the current database. When working with multiple catalogs, for instance multiple Iceberg REST catalogs, it is more practical to be able to query just the objects defined in one specific catalog without further filtering. A special 'system' catalog is optionally provided to be able to query across catalogs. This can be seen as an approximation of SQL's definition schema. ## What changes are included in this PR? - The catalog list provided to `InformationSchemaProvider` is now constructed dynamically based on the catalog being queried. An optional 'system' catalog provides the existing global catalog behaviour. All other catalogs are restricted to just their own content. - Unqualified information_schema queries now use the configured default catalog instead of returning metadata from all catalogs. ## What is the testing strategy for this PR? Added multi-catalog SQL logic coverage for qualified and unqualified information schema queries. ## Are there any user-facing changes? Yes. Information schema results are now scoped to the resolved catalog. --------- Co-authored-by: Vaibhave Sekar <[email protected]> Co-authored-by: Andrew Lamb <[email protected]>
1 parent da11e68 commit 910e8ee

10 files changed

Lines changed: 613 additions & 137 deletions

File tree

‎datafusion/catalog/src/information_schema.rs‎

Lines changed: 162 additions & 34 deletions
Original file line numberDiff line numberDiff line change
@@ -44,6 +44,7 @@ use datafusion_expr::{TableType, Volatility};
4444
use datafusion_physical_plan::SendableRecordBatchStream;
4545
use datafusion_physical_plan::stream::RecordBatchStreamAdapter;
4646
use datafusion_physical_plan::streaming::PartitionStream;
47+
use datafusion_session::CatalogProvider;
4748
use itertools::Itertools;
4849
use std::collections::{BTreeSet, HashMap, HashSet};
4950
use std::fmt::Debug;
@@ -81,10 +82,58 @@ pub struct InformationSchemaProvider {
8182
}
8283

8384
impl InformationSchemaProvider {
85+
/// Creates a new [`InformationSchemaProvider`] for the provided `catalog`
86+
pub fn new_from_catalog(
87+
catalog_name: String,
88+
catalog: Arc<dyn CatalogProvider>,
89+
) -> Self {
90+
#[derive(Debug)]
91+
struct SingleCatalogList {
92+
catalog_name: String,
93+
catalog: Arc<dyn CatalogProvider>,
94+
}
95+
96+
impl CatalogProviderList for SingleCatalogList {
97+
fn register_catalog(
98+
&self,
99+
_: String,
100+
_: Arc<dyn CatalogProvider>,
101+
) -> Option<Arc<dyn CatalogProvider>> {
102+
None
103+
}
104+
105+
fn catalog_names(&self) -> Vec<String> {
106+
vec![self.catalog_name.clone()]
107+
}
108+
109+
fn catalog(&self, name: &str) -> Option<Arc<dyn CatalogProvider>> {
110+
if name == self.catalog_name {
111+
Some(Arc::clone(&self.catalog))
112+
} else {
113+
None
114+
}
115+
}
116+
}
117+
118+
let catalog_list = Arc::new(SingleCatalogList {
119+
catalog_name,
120+
catalog,
121+
});
122+
123+
Self {
124+
config: InformationSchemaConfig {
125+
system_catalog_name: None,
126+
catalog_list,
127+
table_functions: HashMap::new(),
128+
},
129+
}
130+
}
131+
84132
/// Creates a new [`InformationSchemaProvider`] for the provided `catalog_list`
85133
pub fn new(catalog_list: Arc<dyn CatalogProviderList>) -> Self {
86134
Self {
87135
config: InformationSchemaConfig {
136+
system_catalog_name: None,
88137
catalog_list,
89138
table_functions: HashMap::new(),
90139
},
@@ -100,10 +149,16 @@ impl InformationSchemaProvider {
100149
self.config.table_functions = table_functions;
101150
self
102151
}
152+
153+
pub fn with_system_catalog(mut self, catalog_name: String) -> Self {
154+
self.config.system_catalog_name = Some(catalog_name);
155+
self
156+
}
103157
}
104158

105159
#[derive(Clone, Debug)]
106160
struct InformationSchemaConfig {
161+
system_catalog_name: Option<String>,
107162
catalog_list: Arc<dyn CatalogProviderList>,
108163
table_functions: HashMap<String, Arc<TableFunction>>,
109164
}
@@ -115,9 +170,10 @@ impl InformationSchemaConfig {
115170
builder: &mut InformationSchemaTablesBuilder,
116171
) -> Result<(), DataFusionError> {
117172
// create a mem table with the names of tables
173+
let catalog_names = self.catalog_list.catalog_names();
118174

119-
for catalog_name in self.catalog_list.catalog_names() {
120-
let catalog = self.catalog_list.catalog(&catalog_name).unwrap();
175+
for catalog_name in &catalog_names {
176+
let catalog = self.catalog_list.catalog(catalog_name).unwrap();
121177

122178
for schema_name in catalog.schema_names() {
123179
if schema_name != INFORMATION_SCHEMA {
@@ -128,7 +184,7 @@ impl InformationSchemaConfig {
128184
schema.table_type(&table_name).await?
129185
{
130186
builder.add_table(
131-
&catalog_name,
187+
catalog_name,
132188
&schema_name,
133189
&table_name,
134190
table_type,
@@ -142,7 +198,20 @@ impl InformationSchemaConfig {
142198
// Add a final list for the information schema tables themselves
143199
for table_name in INFORMATION_SCHEMA_TABLES {
144200
builder.add_table(
145-
&catalog_name,
201+
catalog_name,
202+
INFORMATION_SCHEMA,
203+
table_name,
204+
TableType::View,
205+
);
206+
}
207+
}
208+
209+
if let Some(system_catalog_name) = &self.system_catalog_name
210+
&& !catalog_names.contains(system_catalog_name)
211+
{
212+
for table_name in INFORMATION_SCHEMA_TABLES {
213+
builder.add_table(
214+
system_catalog_name,
146215
INFORMATION_SCHEMA,
147216
table_name,
148217
TableType::View,
@@ -179,16 +248,13 @@ impl InformationSchemaConfig {
179248
if schema_name != INFORMATION_SCHEMA {
180249
// schema name may not exist in the catalog, so we need to check
181250
if let Some(schema) = catalog.schema(&schema_name) {
182-
for table_name in schema.table_names() {
183-
if let Some(table) = schema.table(&table_name).await? {
184-
builder.add_view(
185-
&catalog_name,
186-
&schema_name,
187-
&table_name,
188-
table.get_table_definition(),
189-
)
190-
}
191-
}
251+
Self::add_views(
252+
builder,
253+
&catalog_name,
254+
&schema_name,
255+
schema.as_ref(),
256+
)
257+
.await?;
192258
}
193259
}
194260
}
@@ -197,6 +263,25 @@ impl InformationSchemaConfig {
197263
Ok(())
198264
}
199265

266+
async fn add_views(
267+
builder: &mut InformationSchemaViewBuilder,
268+
catalog_name: &str,
269+
schema_name: &str,
270+
schema: &dyn SchemaProvider,
271+
) -> Result<(), DataFusionError> {
272+
for table_name in schema.table_names() {
273+
if let Some(table) = schema.table(&table_name).await? {
274+
builder.add_view(
275+
catalog_name,
276+
schema_name,
277+
&table_name,
278+
table.get_table_definition(),
279+
)
280+
}
281+
}
282+
Ok(())
283+
}
284+
200285
/// Construct the `information_schema.columns` virtual table
201286
async fn make_columns(
202287
&self,
@@ -209,21 +294,13 @@ impl InformationSchemaConfig {
209294
if schema_name != INFORMATION_SCHEMA {
210295
// schema name may not exist in the catalog, so we need to check
211296
if let Some(schema) = catalog.schema(&schema_name) {
212-
for table_name in schema.table_names() {
213-
if let Some(table) = schema.table(&table_name).await? {
214-
for (field_position, field) in
215-
table.schema().fields().iter().enumerate()
216-
{
217-
builder.add_column(
218-
&catalog_name,
219-
&schema_name,
220-
&table_name,
221-
field_position,
222-
field,
223-
)
224-
}
225-
}
226-
}
297+
Self::add_columns(
298+
builder,
299+
&catalog_name,
300+
&schema_name,
301+
schema.as_ref(),
302+
)
303+
.await?;
227304
}
228305
}
229306
}
@@ -232,6 +309,29 @@ impl InformationSchemaConfig {
232309
Ok(())
233310
}
234311

312+
async fn add_columns(
313+
builder: &mut InformationSchemaColumnsBuilder,
314+
catalog_name: &str,
315+
schema_name: &str,
316+
schema: &dyn SchemaProvider,
317+
) -> Result<(), DataFusionError> {
318+
for table_name in schema.table_names() {
319+
if let Some(table) = schema.table(&table_name).await? {
320+
for (field_position, field) in table.schema().fields().iter().enumerate()
321+
{
322+
builder.add_column(
323+
catalog_name,
324+
schema_name,
325+
&table_name,
326+
field_position,
327+
field,
328+
)
329+
}
330+
}
331+
}
332+
Ok(())
333+
}
334+
235335
/// Construct the `information_schema.df_settings` virtual table
236336
fn make_df_settings(
237337
&self,
@@ -561,6 +661,24 @@ fn remove_native_type_prefix(native_type: &NativeType) -> String {
561661

562662
#[async_trait]
563663
impl SchemaProvider for InformationSchemaProvider {
664+
fn table_names(&self) -> Vec<String> {
665+
self.config.table_names()
666+
}
667+
668+
async fn table(
669+
&self,
670+
name: &str,
671+
) -> Result<Option<Arc<dyn TableProvider>>, DataFusionError> {
672+
self.config.get_table(name)
673+
}
674+
675+
fn table_exist(&self, name: &str) -> bool {
676+
self.config.table_exist(name)
677+
}
678+
}
679+
680+
#[async_trait]
681+
impl SchemaProvider for InformationSchemaConfig {
564682
fn table_names(&self) -> Vec<String> {
565683
INFORMATION_SCHEMA_TABLES
566684
.iter()
@@ -572,7 +690,20 @@ impl SchemaProvider for InformationSchemaProvider {
572690
&self,
573691
name: &str,
574692
) -> Result<Option<Arc<dyn TableProvider>>, DataFusionError> {
575-
let config = self.config.clone();
693+
self.get_table(name)
694+
}
695+
696+
fn table_exist(&self, name: &str) -> bool {
697+
INFORMATION_SCHEMA_TABLES.contains(&name.to_ascii_lowercase().as_str())
698+
}
699+
}
700+
701+
impl InformationSchemaConfig {
702+
fn get_table(
703+
&self,
704+
name: &str,
705+
) -> Result<Option<Arc<dyn TableProvider>>, DataFusionError> {
706+
let config = self.clone();
576707
let table: Arc<dyn PartitionStream> = match name.to_ascii_lowercase().as_str() {
577708
TABLES => Arc::new(InformationSchemaTables::new(config)),
578709
COLUMNS => Arc::new(InformationSchemaColumns::new(config)),
@@ -588,10 +719,6 @@ impl SchemaProvider for InformationSchemaProvider {
588719
StreamingTable::try_new(Arc::clone(table.schema()), vec![table]).unwrap(),
589720
)))
590721
}
591-
592-
fn table_exist(&self, name: &str) -> bool {
593-
INFORMATION_SCHEMA_TABLES.contains(&name.to_ascii_lowercase().as_str())
594-
}
595722
}
596723

597724
#[derive(Debug)]
@@ -1553,6 +1680,7 @@ mod tests {
15531680
#[tokio::test]
15541681
async fn make_tables_uses_table_type() {
15551682
let config = InformationSchemaConfig {
1683+
system_catalog_name: None,
15561684
catalog_list: Arc::new(Fixture),
15571685
table_functions: HashMap::new(),
15581686
};

‎datafusion/common/src/config.rs‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -242,6 +242,12 @@ config_namespace! {
242242
/// virtual tables for displaying schema information
243243
pub information_schema: bool, default = false
244244

245+
/// The system catalog name - when set together with `information_schema`, the
246+
/// `information_schema` for each catalog will only describe objects contained in the
247+
/// catalog itself. The global, cross-catalog `information_schema` is available under
248+
/// the virtual catalog named by this option.
249+
pub system_catalog: Option<String>, default = None
250+
245251
/// Location scanned to load tables for `default` schema
246252
pub location: Option<String>, default = None
247253

‎datafusion/core/src/execution/context/mod.rs‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -2859,7 +2859,9 @@ mod tests {
28592859
async fn sql_create_catalog() -> Result<()> {
28602860
// the information schema used to introduce cyclic Arcs
28612861
let ctx = SessionContext::new_with_config(
2862-
SessionConfig::new().with_information_schema(true),
2862+
SessionConfig::new()
2863+
.with_information_schema(true)
2864+
.with_system_catalog(Some(String::from("system"))),
28632865
);
28642866

28652867
// Create catalog
@@ -2875,7 +2877,7 @@ mod tests {
28752877
.await?;
28762878

28772879
// Check table exists in schema
2878-
let results = ctx.sql("SELECT * FROM information_schema.tables WHERE table_catalog='test' AND table_schema='abc' AND table_name = 'y'").await.unwrap().collect().await.unwrap();
2880+
let results = ctx.sql("SELECT * FROM system.information_schema.tables WHERE table_catalog='test' AND table_schema='abc' AND table_name = 'y'").await.unwrap().collect().await.unwrap();
28792881

28802882
assert_eq!(results[0].num_rows(), 1);
28812883
Ok(())

0 commit comments

Comments
 (0)