Skip to content

Commit 92e4758

Browse files
committed
[core] Keep exact index hits when candidate rows are excluded or refined
1 parent 2f11ddb commit 92e4758

4 files changed

Lines changed: 133 additions & 29 deletions

File tree

‎crates/paimon/src/table/de_vector_read.rs‎

Lines changed: 31 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -634,36 +634,45 @@ async fn scalar_matched_row_ids(
634634
return Ok(None);
635635
};
636636
let ranges = intersect_sorted_ranges(&ranges, vector_ranges);
637-
// Candidates outside the rows this search ranks do not make it inexact.
638-
if exact && intersect_sorted_ranges(&candidate_ranges, &ranges).is_empty() {
639-
let mut row_ids = RoaringTreemap::new();
640-
for range in ranges {
641-
let start = u64::try_from(range.from()).map_err(|_| crate::Error::DataInvalid {
642-
message: "Scalar vector index returned a negative row ID".to_string(),
643-
source: None,
644-
})?;
645-
let end = u64::try_from(range.to()).map_err(|_| crate::Error::DataInvalid {
646-
message: "Scalar vector index returned a negative row ID".to_string(),
647-
source: None,
648-
})?;
649-
row_ids.insert_range(start..=end);
650-
}
651-
return Ok(Some(row_ids));
637+
// With an exact answer, only rows in `candidate_ranges` still need the data;
638+
// the rest are proven matches whether or not refinement is enabled.
639+
let (mut matched, candidates) = if exact {
640+
let candidates = intersect_sorted_ranges(&candidate_ranges, &ranges);
641+
let mut matched = scalar_row_ids(&ranges)?;
642+
matched -= scalar_row_ids(&candidates)?;
643+
(matched, candidates)
644+
} else {
645+
(RoaringTreemap::new(), ranges)
646+
};
647+
if candidates.is_empty() {
648+
return Ok(Some(matched));
652649
}
653650
if !core.global_index_filter_refine_from_data() {
654651
log::warn!("Scalar global index only provides candidates for {filter:?}; global-index.filter.refine-from-data is false, so indexed candidates are excluded");
655-
return Ok(Some(RoaringTreemap::new()));
656-
}
657-
if ranges.is_empty() {
658-
return Ok(Some(RoaringTreemap::new()));
652+
return Ok(Some(matched));
659653
}
660654
let table = evaluation.table.ok_or_else(|| crate::Error::DataInvalid {
661655
message: "Scalar vector filter refinement requires table context".to_string(),
662656
source: None,
663657
})?;
664-
Ok(Some(
665-
matching_row_ids_for_filter(table, filter, Some(ranges)).await?,
666-
))
658+
matched |= matching_row_ids_for_filter(table, filter, Some(candidates)).await?;
659+
Ok(Some(matched))
660+
}
661+
662+
fn scalar_row_ids(ranges: &[RowRange]) -> crate::Result<RoaringTreemap> {
663+
let mut row_ids = RoaringTreemap::new();
664+
for range in ranges {
665+
let start = u64::try_from(range.from()).map_err(|_| crate::Error::DataInvalid {
666+
message: "Scalar vector index returned a negative row ID".to_string(),
667+
source: None,
668+
})?;
669+
let end = u64::try_from(range.to()).map_err(|_| crate::Error::DataInvalid {
670+
message: "Scalar vector index returned a negative row ID".to_string(),
671+
source: None,
672+
})?;
673+
row_ids.insert_range(start..=end);
674+
}
675+
Ok(row_ids)
667676
}
668677

669678
#[cfg(test)]

‎crates/paimon/src/table/de_vector_scan/tests.rs‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1129,6 +1129,18 @@ async fn top1_after_bitmap_then_btree(
11291129
.collect()
11301130
}
11311131

1132+
#[tokio::test]
1133+
async fn bitmap_candidates_keep_exact_btree_hits_without_refinement() {
1134+
// Bitmap rows 0/1 are only candidates for `f > 1.5`, BTree row 2 is an
1135+
// exact hit. Without refinement the candidates drop out, row 2 stays.
1136+
let ids = top1_after_bitmap_then_btree(vec![Some(0.0), Some(1.0)], |pb| {
1137+
pb.greater_than("f", crate::spec::Datum::Double(1.5))
1138+
.unwrap()
1139+
})
1140+
.await;
1141+
assert_eq!(ids, vec![2]);
1142+
}
1143+
11321144
#[tokio::test]
11331145
async fn pruned_null_bitmap_keeps_btree_answer_exact() {
11341146
// The bitmap shard covers only NULL rows, so `f > 1.5` prunes it and the

‎crates/paimon/src/table/full_text_search_builder/filter.rs‎

Lines changed: 14 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -204,17 +204,24 @@ pub(super) async fn matching_indexed_rows(
204204
};
205205
match answer {
206206
Some((answer, exact, candidate_ranges)) => {
207-
let candidates = row_ids(&answer)? & &decided;
208-
let exact = exact && (row_ids(&candidate_ranges)? & &candidates).is_empty();
209-
if exact {
207+
let answer = row_ids(&answer)? & &decided;
208+
// With an exact answer, only rows in `candidate_ranges` still need
209+
// the data; the rest are proven matches.
210+
let (proven, candidates) = if exact {
211+
let candidates = row_ids(&candidate_ranges)? & &answer;
212+
(&answer - &candidates, candidates)
213+
} else {
214+
(RoaringTreemap::new(), answer)
215+
};
216+
if !proven.is_empty() {
210217
matched |= if core.data_evolution_enabled() {
211-
candidates
218+
proven
212219
} else {
213220
// An exact index match does not establish snapshot visibility.
214-
matching_rows(evaluation, &Predicate::AlwaysTrue, ranges(&candidates)?)
215-
.await?
221+
matching_rows(evaluation, &Predicate::AlwaysTrue, ranges(&proven)?).await?
216222
};
217-
} else if core.global_index_filter_refine_from_data() {
223+
}
224+
if !candidates.is_empty() && core.global_index_filter_refine_from_data() {
218225
matched |= matching_rows(evaluation, filter, ranges(&candidates)?).await?;
219226
}
220227
}

‎crates/paimon/src/table/full_text_search_builder/parity_tests.rs‎

Lines changed: 76 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -329,6 +329,82 @@ async fn scalar_modes_and_refinement_follow_java_before_top_k() {
329329
assert_eq!(ids(search.execute().await.unwrap()), vec![5]);
330330
}
331331

332+
#[tokio::test]
333+
async fn bitmap_candidates_keep_exact_btree_hits_in_full_text_filters() {
334+
// Rows 0/1 have a bitmap on the double `f`, so they are only candidates for
335+
// `f > 1.5`; rows 2/3 have a BTree, so rows 2 and 3 are exact hits.
336+
let schema = Schema::builder()
337+
.column("id", DataType::Int(IntType::new()))
338+
.column("text", DataType::VarChar(VarCharType::string_type()))
339+
.column("f", DataType::Double(crate::spec::DoubleType::new()))
340+
.options(HashMap::from([
341+
("bucket".into(), "-1".into()),
342+
("row-tracking.enabled".into(), "true".into()),
343+
("data-evolution.enabled".into(), "true".into()),
344+
("global-index.enabled".into(), "true".into()),
345+
("scalar-index.search-mode".into(), "full".into()),
346+
("full-text-index.search-mode".into(), "full".into()),
347+
]))
348+
.build()
349+
.unwrap();
350+
let table = Table::new(
351+
FileIOBuilder::new("memory").build().unwrap(),
352+
Identifier::new("default", "text_mixed_scalar"),
353+
format!("memory:/text-mixed-scalar-{}", uuid::Uuid::new_v4()),
354+
TableSchema::new(0, &schema),
355+
None,
356+
);
357+
let append = |start: i32, f: Vec<f64>| {
358+
let table = table.clone();
359+
async move {
360+
let rows = f.len() as i32;
361+
let schema = Arc::new(ArrowSchema::new(vec![
362+
Field::new("id", ArrowType::Int32, true),
363+
Field::new("text", ArrowType::Utf8, true),
364+
Field::new("f", ArrowType::Float64, true),
365+
]));
366+
let columns: Vec<ArrayRef> = vec![
367+
Arc::new(Int32Array::from_iter_values(start..start + rows)),
368+
Arc::new(StringArray::from(vec!["paimon"; f.len()])),
369+
Arc::new(arrow_array::Float64Array::from(f)),
370+
];
371+
let mut write = TableWrite::new(&table, "data".into()).unwrap();
372+
write
373+
.write_arrow_batch(&RecordBatch::try_new(schema, columns).unwrap())
374+
.await
375+
.unwrap();
376+
TableCommit::new(table.clone(), "data".into())
377+
.commit(write.prepare_commit().await.unwrap())
378+
.await
379+
.unwrap();
380+
}
381+
};
382+
append(0, vec![0.0, 1.0]).await;
383+
index(&table, "bitmap", "f").await;
384+
append(2, vec![2.0, 3.0]).await;
385+
index(&table, "btree", "f").await;
386+
index(&table, "full-text", "text").await;
387+
388+
let filter = PredicateBuilder::new(table.schema().fields())
389+
.greater_than("f", Datum::Double(1.5))
390+
.unwrap();
391+
for refine in [false, true] {
392+
let table = table.copy_with_options(HashMap::from([(
393+
"global-index.filter.refine-from-data".into(),
394+
refine.to_string(),
395+
)]));
396+
assert_eq!(
397+
ids(query(&table)
398+
.with_filter(filter.clone())
399+
.execute()
400+
.await
401+
.unwrap()),
402+
vec![2, 3],
403+
"refine={refine}"
404+
);
405+
}
406+
}
407+
332408
#[tokio::test]
333409
async fn empty_string_filters_preserve_java_search_refinement() {
334410
for kind in ["btree", "bitmap"] {

0 commit comments

Comments
 (0)