Skip to content

Commit c96cdd0

Browse files
authored
perf: stream and prune SST manifest scans (#9931)
* refactor: add reusable metadata pruning and projected SST batches Signed-off-by: WenyXu <[email protected]> * perf(mito2): stream and prune manifest SST metadata Signed-off-by: WenyXu <[email protected]> * perf(datanode): stream inspection queries through reusable providers Signed-off-by: WenyXu <[email protected]> * perf(catalog): push manifest filters through inspection queries Signed-off-by: WenyXu <[email protected]> * refactor: align manifest column constants with module conventions Signed-off-by: WenyXu <[email protected]> * perf: reduce manifest inspection allocations Signed-off-by: WenyXu <[email protected]> * refactor: narrow manifest inspection API visibility Signed-off-by: WenyXu <[email protected]> * refactor: retain residual filters for manifest inspection Signed-off-by: WenyXu <[email protected]> --------- Signed-off-by: WenyXu <[email protected]>
1 parent 513b9ce commit c96cdd0

17 files changed

Lines changed: 2055 additions & 178 deletions

File tree

‎src/catalog/src/information_extension.rs‎

Lines changed: 119 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -310,3 +310,122 @@ impl InformationExtension for DistributedInformationExtension {
310310
))))
311311
}
312312
}
313+
314+
#[cfg(test)]
315+
mod tests {
316+
use common_meta::node_manager::{DatanodeManager, DatanodeRef};
317+
use common_meta::peer::Peer;
318+
use datafusion::common::tree_node::TreeNode;
319+
use datafusion::logical_expr::LogicalPlan;
320+
use datafusion::physical_plan::sorts::sort::SortExec;
321+
use meta_client::client::MetaClientBuilder;
322+
use store_api::sst_entry::ManifestSstEntry;
323+
use store_api::storage::ScanRequest;
324+
325+
use super::*;
326+
use crate::information_schema::DatanodeInspectKind;
327+
328+
struct UnusedDatanodeManager;
329+
330+
#[async_trait::async_trait]
331+
impl DatanodeManager for UnusedDatanodeManager {
332+
async fn datanode(&self, _node: &Peer) -> DatanodeRef {
333+
panic!("plan tests must not contact datanodes")
334+
}
335+
}
336+
337+
#[derive(Debug)]
338+
struct ManifestScanProvider;
339+
340+
#[async_trait::async_trait]
341+
impl datafusion::catalog::TableProvider for ManifestScanProvider {
342+
fn schema(&self) -> ArrowSchemaRef {
343+
ManifestSstEntry::schema().arrow_schema().clone()
344+
}
345+
346+
fn table_type(&self) -> datafusion::logical_expr::TableType {
347+
datafusion::logical_expr::TableType::View
348+
}
349+
350+
fn supports_filters_pushdown(
351+
&self,
352+
filters: &[&datafusion::logical_expr::Expr],
353+
) -> Result<Vec<datafusion::logical_expr::TableProviderFilterPushDown>> {
354+
Ok(vec![datafusion::logical_expr::TableProviderFilterPushDown::Inexact; filters.len()])
355+
}
356+
357+
async fn scan(
358+
&self,
359+
_state: &dyn datafusion::catalog::Session,
360+
projection: Option<&Vec<usize>>,
361+
filters: &[datafusion::logical_expr::Expr],
362+
limit: Option<usize>,
363+
) -> Result<Arc<dyn ExecutionPlan>> {
364+
let schema = ManifestSstEntry::schema();
365+
let schema = match projection {
366+
Some(projection) => Arc::new(schema.try_project(projection).unwrap()),
367+
None => schema,
368+
};
369+
Ok(Arc::new(DistributedInspectExec::new(
370+
DatanodeInspectRequest {
371+
kind: DatanodeInspectKind::SstManifest,
372+
scan: ScanRequest {
373+
projection: projection.cloned(),
374+
filters: filters.to_vec(),
375+
limit,
376+
..Default::default()
377+
},
378+
},
379+
schema,
380+
Arc::new(MetaClientBuilder::new(0, api::v1::meta::Role::Frontend).build()),
381+
Arc::new(UnusedDatanodeManager),
382+
)))
383+
}
384+
}
385+
386+
#[tokio::test]
387+
async fn test_sql_manifest_sort_stays_on_frontend() {
388+
let ctx = datafusion::prelude::SessionContext::new();
389+
ctx.register_table("ssts_manifest", Arc::new(ManifestScanProvider))
390+
.unwrap();
391+
for sql in [
392+
"SELECT region_id, file_id, min_ts, max_ts, file_size FROM ssts_manifest WHERE region_id = 42 ORDER BY min_ts LIMIT 100",
393+
"SELECT file_id FROM ssts_manifest WHERE region_id = 42 ORDER BY max_ts DESC LIMIT 10 OFFSET 3",
394+
"SELECT min_ts AS ts, file_id FROM ssts_manifest WHERE region_id = 42 ORDER BY ts DESC NULLS FIRST LIMIT 10",
395+
"SELECT file_id FROM ssts_manifest WHERE region_id = 42 ORDER BY min_ts, max_ts DESC LIMIT 10",
396+
] {
397+
let plan = ctx
398+
.sql(sql)
399+
.await
400+
.unwrap()
401+
.create_physical_plan()
402+
.await
403+
.unwrap();
404+
let mut found_scan = false;
405+
let mut found_sort = false;
406+
plan.apply(|plan| {
407+
found_sort |= plan.is::<SortExec>();
408+
if let Some(scan) = plan.downcast_ref::<DistributedInspectExec>() {
409+
found_scan = true;
410+
// All matching rows must reach the frontend global sort.
411+
assert_eq!(scan.request.scan.limit, None);
412+
let remote = scan.request.clone().build_plan().unwrap();
413+
remote.apply(|node| {
414+
assert!(!matches!(
415+
node,
416+
LogicalPlan::Sort(_) | LogicalPlan::Limit(_)
417+
));
418+
Ok(TreeNodeRecursion::Continue)
419+
})?;
420+
}
421+
Ok(TreeNodeRecursion::Continue)
422+
})
423+
.unwrap();
424+
assert!(
425+
found_scan && found_sort,
426+
"{}",
427+
datafusion::physical_plan::displayable(plan.as_ref()).indent(true)
428+
);
429+
}
430+
}
431+
}

‎src/catalog/src/system_schema/information_schema/ssts.rs‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -349,6 +349,13 @@ mod tests {
349349
.unwrap()
350350
.unwrap();
351351

352+
let filter =
353+
datafusion::logical_expr::col("region_id").eq(datafusion::logical_expr::lit(42_u64));
354+
assert_eq!(
355+
table.supports_filters_pushdown(&[&filter]).unwrap(),
356+
vec![table::metadata::FilterPushDownType::Inexact],
357+
);
358+
352359
let request = ScanRequest {
353360
projection: Some(vec![0]),
354361
..Default::default()

‎src/common/recordbatch/src/filter.rs‎

Lines changed: 37 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -19,10 +19,11 @@ mod predicate;
1919
mod tests;
2020
mod timestamp;
2121

22+
use std::borrow::Borrow;
2223
use std::sync::Arc;
2324

2425
use datafusion::error::Result as DfResult;
25-
use datafusion::logical_expr::{Expr, Literal, Operator};
26+
use datafusion::logical_expr::{Expr, Literal, Operator, utils};
2627
use datafusion::physical_plan::PhysicalExpr;
2728
use datafusion_common::arrow::array::{ArrayRef, Datum};
2829
use datafusion_common::arrow::buffer::BooleanBuffer;
@@ -41,6 +42,41 @@ pub use self::timestamp::timestamp_scalar_value;
4142
use crate::error::{Result, ToArrowScalarSnafu};
4243
use crate::filter::predicate::SimplePredicate;
4344

45+
/// Prunes candidates using known metadata columns. Missing columns and unsupported
46+
/// predicates cannot exclude a candidate. Complete SQL predicates must still run
47+
/// after pruning.
48+
#[derive(Debug, Clone, Default)]
49+
pub struct MetadataFilter {
50+
filters: Arc<[SimpleFilterEvaluator]>,
51+
}
52+
53+
impl MetadataFilter {
54+
/// Compiles supported conjuncts once for reuse across scan stages. OR is
55+
/// never split: an unknown alternative might match the candidate.
56+
pub fn new(expressions: &[Expr]) -> Self {
57+
Self {
58+
filters: expressions
59+
.iter()
60+
.flat_map(utils::split_conjunction)
61+
.filter_map(SimpleFilterEvaluator::try_new)
62+
.collect(),
63+
}
64+
}
65+
66+
/// Returns false only when known metadata proves a conjunct cannot match.
67+
/// `None` means unavailable metadata, distinct from a known SQL NULL scalar.
68+
/// Values may be borrowed so repeated predicates can reuse allocated strings.
69+
pub fn matches<T: Borrow<ScalarValue>>(
70+
&self,
71+
mut value: impl FnMut(&str) -> Option<T>,
72+
) -> bool {
73+
self.filters.iter().all(|filter| {
74+
value(filter.column_name())
75+
.is_none_or(|value| filter.evaluate_scalar(value.borrow()).unwrap_or(true))
76+
})
77+
}
78+
}
79+
4480
/// Evaluates supported single-column predicates with precompiled regular expressions.
4581
/// Compound predicates preserve SQL null semantics until conversion to a scan mask.
4682
#[derive(Debug, Clone)]

‎src/common/recordbatch/src/filter/tests.rs‎

Lines changed: 88 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -70,3 +70,91 @@ fn test_nullable_boolean_predicate_becomes_scan_mask() {
7070
boolean_array_to_scan_mask(&predicate)
7171
);
7272
}
73+
74+
#[test]
75+
fn test_partial_metadata() {
76+
use datafusion_common::ScalarValue;
77+
78+
use crate::filter::MetadataFilter;
79+
80+
let node = col("node_id").eq(lit(7u64));
81+
let path = col("file_path").eq(lit("table/region/file.parquet"));
82+
let filters = MetadataFilter::new(&[node.clone().and(path.clone())]);
83+
assert!(!filters.matches(|name| (name == "node_id").then_some(ScalarValue::UInt64(Some(8)))));
84+
assert!(filters.matches(|name| (name == "node_id").then_some(ScalarValue::UInt64(Some(7)))));
85+
assert!(!filters.matches(|name| {
86+
(name == "file_path").then(|| ScalarValue::Utf8(Some("other/file.parquet".into())))
87+
}));
88+
assert!(filters.matches(|name| {
89+
(name == "file_path").then(|| ScalarValue::Utf8(Some("table/region/file.parquet".into())))
90+
}));
91+
// Unknown alternatives cannot be discarded from OR, nor extracted from NOT.
92+
let disjunction = MetadataFilter::new(&[node.clone().or(path.clone())]);
93+
assert!(
94+
disjunction.matches(|name| (name == "node_id").then_some(ScalarValue::UInt64(Some(8))))
95+
);
96+
let negation = MetadataFilter::new(&[datafusion::logical_expr::Expr::Not(Box::new(
97+
node.and(path),
98+
))]);
99+
assert!(negation.matches(|_| None::<ScalarValue>));
100+
// Unsupported LIKE remains a residual even when its column is available.
101+
let residual = MetadataFilter::new(&[col("file_path").like(lit("table/%"))]);
102+
assert!(residual.matches(|_| Some(ScalarValue::Utf8(Some("other/file".into())))));
103+
// Missing metadata differs from known NULL; type errors conservatively pass.
104+
let null_filter = MetadataFilter::new(&[col("node_id").is_not_null()]);
105+
assert!(null_filter.matches(|_| None::<ScalarValue>));
106+
assert!(!null_filter.matches(|_| Some(ScalarValue::UInt64(None))));
107+
let typed = MetadataFilter::new(&[col("node_id").eq(lit(7u64))]);
108+
assert!(typed.matches(|_| Some(ScalarValue::Utf8(Some("invalid".into())))));
109+
}
110+
111+
#[test]
112+
fn test_borrowed_and_owned_values() {
113+
use datafusion_common::ScalarValue;
114+
115+
use crate::filter::MetadataFilter;
116+
117+
let pruner = MetadataFilter::new(&[col("path").gt_eq(lit("a")).and(col("path").lt(lit("z")))]);
118+
for (value, expected) in [
119+
(ScalarValue::Utf8(Some("a".into())), true),
120+
(ScalarValue::Utf8(Some("z".into())), false),
121+
(ScalarValue::Utf8(None), false),
122+
(ScalarValue::UInt64(Some(1)), true),
123+
] {
124+
assert_eq!(pruner.matches(|_| Some(&value)), expected);
125+
assert_eq!(pruner.matches(|_| Some(value.clone())), expected);
126+
}
127+
}
128+
129+
#[test]
130+
fn test_borrowed_lazy_metadata() {
131+
use std::cell::OnceCell;
132+
133+
use datafusion_common::ScalarValue;
134+
135+
use crate::filter::MetadataFilter;
136+
137+
let pruner = MetadataFilter::new(&[col("path").gt(lit("a")).and(col("path").lt(lit("z")))]);
138+
let path = OnceCell::new();
139+
let mut builds = 0;
140+
assert!(pruner.matches(|name| {
141+
(name == "path").then(|| {
142+
path.get_or_init(|| {
143+
builds += 1;
144+
ScalarValue::Utf8(Some("region/file.parquet".into()))
145+
})
146+
})
147+
}));
148+
assert_eq!(builds, 1);
149+
assert_eq!(
150+
path.into_inner(),
151+
Some(ScalarValue::Utf8(Some("region/file.parquet".into())))
152+
);
153+
154+
let path = OnceCell::new();
155+
assert!(
156+
MetadataFilter::default()
157+
.matches(|_| { Some(path.get_or_init(|| ScalarValue::Utf8(Some("unused".into())))) })
158+
);
159+
assert!(path.get().is_none());
160+
}

‎src/datanode/src/region_server/catalog.rs‎

Lines changed: 9 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,8 @@
1212
// See the License for the specific language governing permissions and
1313
// limitations under the License.
1414

15+
mod inspect;
16+
1517
use std::collections::HashMap;
1618
use std::sync::Arc;
1719

@@ -33,6 +35,7 @@ use store_api::storage::RegionId;
3335

3436
use crate::error::{DataFusionSnafu, ListStorageSstsSnafu, Result, UnexpectedSnafu};
3537
use crate::region_server::RegionServer;
38+
use crate::region_server::catalog::inspect::{InspectTable, ManifestSstSource};
3639

3740
/// Reserved internal table kinds used.
3841
/// These are recognized by reserved table names and mapped to providers.
@@ -75,7 +78,7 @@ impl InternalTableKind {
7578
}
7679

7780
impl RegionServer {
78-
/// Expose SSTs listed in Manifest as an in-memory table for inspection.
81+
/// Exposes manifest SSTs through a lazy table with metadata predicate pruning.
7982
pub async fn inspect_sst_manifest_provider(&self) -> Result<Arc<dyn TableProvider>> {
8083
let mito = {
8184
let guard = self.inner.mito_engine.read().unwrap();
@@ -84,14 +87,9 @@ impl RegionServer {
8487
})?
8588
};
8689

87-
let entries = mito.all_ssts_from_manifest().await;
88-
let schema = ManifestSstEntry::schema().arrow_schema().clone();
89-
let batch = ManifestSstEntry::to_record_batch(&entries)
90-
.map_err(DataFusionError::from)
91-
.context(DataFusionSnafu)?;
92-
93-
let table = MemTable::try_new(schema, vec![vec![batch]]).context(DataFusionSnafu)?;
94-
Ok(Arc::new(table))
90+
Ok(Arc::new(InspectTable::new(Arc::new(
91+
ManifestSstSource::new(mito),
92+
))))
9593
}
9694

9795
/// Expose SSTs found in storage as an in-memory table for inspection.
@@ -355,6 +353,8 @@ impl TreeNodeRewriter for NameAwareDataSourceInjector {
355353

356354
#[cfg(test)]
357355
mod tests {
356+
mod manifest;
357+
358358
use std::sync::Arc;
359359

360360
use datafusion::catalog::MemTable as DfMemTable;

0 commit comments

Comments
 (0)