Repository navigation
Expand file tree
/
Copy pathscanner.rs
More file actions
10310 lines (9342 loc) · 384 KB
/
Copy pathscanner.rs
File metadata and controls
10310 lines (9342 loc) · 384 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
// SPDX-License-Identifier: Apache-2.0
// SPDX-FileCopyrightText: Copyright The Lance Authors
use std::ops::Range;
use std::pin::Pin;
use std::sync::{Arc, LazyLock};
use std::task::{Context, Poll};
use arrow::array::AsArray;
use arrow_array::{Array, Float32Array, Int64Array, RecordBatch};
use arrow_schema::{DataType, Field as ArrowField, Schema as ArrowSchema, SchemaRef, SortOptions};
use arrow_select::concat::concat_batches;
use async_recursion::async_recursion;
use chrono::Utc;
use datafusion::common::{DFSchema, JoinType, NullEquality, SchemaExt, exec_datafusion_err};
use datafusion::functions_aggregate;
use datafusion::logical_expr::{Expr, ScalarUDF, col, lit};
use datafusion::physical_expr::PhysicalSortExpr;
use datafusion::physical_plan::coalesce_batches::CoalesceBatchesExec;
use datafusion::physical_plan::expressions;
use datafusion::physical_plan::projection::ProjectionExec as DFProjectionExec;
use datafusion::physical_plan::sorts::sort::SortExec;
use datafusion::physical_plan::{
ExecutionPlan, SendableRecordBatchStream,
aggregates::{AggregateExec, AggregateMode, PhysicalGroupBy},
display::DisplayableExecutionPlan,
limit::GlobalLimitExec,
repartition::RepartitionExec,
union::UnionExec,
};
use datafusion::scalar::ScalarValue;
use datafusion_expr::ExprSchemable;
use datafusion_expr::execution_props::ExecutionProps;
use datafusion_functions::core::getfield::GetFieldFunc;
use datafusion_physical_expr::expressions::Column;
use datafusion_physical_expr::{LexOrdering, Partitioning, PhysicalExpr, create_physical_expr};
use datafusion_physical_plan::joins::PartitionMode;
use datafusion_physical_plan::projection::ProjectionExec;
use datafusion_physical_plan::stream::RecordBatchStreamAdapter;
use datafusion_physical_plan::{empty::EmptyExec, joins::HashJoinExec};
use futures::future::BoxFuture;
use futures::stream::{Stream, StreamExt};
use futures::{FutureExt, TryStreamExt};
use lance_arrow::floats::{FloatType, coerce_float_vector};
use lance_arrow::{DataTypeExt, SchemaExt as ArrowSchemaExt};
use lance_core::datatypes::{
BlobHandling, Field, OnMissing, Projection, escape_field_path_for_project, format_field_path,
};
use lance_core::error::LanceOptionExt;
use lance_core::utils::address::RowAddress;
use lance_core::utils::mask::{RowAddrMask, RowAddrTreeMap};
use lance_core::utils::tokio::get_num_compute_intensive_cpus;
use lance_core::{ROW_ADDR, ROW_ID, ROW_OFFSET};
use lance_datafusion::aggregate::Aggregate;
use lance_datafusion::exec::{
LanceExecutionOptions, OneShotExec, StrictBatchSizeExec, analyze_plan, execute_plan,
};
use lance_datafusion::expr::safe_coerce_scalar;
use lance_datafusion::projection::ProjectionPlan;
use lance_file::reader::FileReaderOptions;
use lance_index::IndexCriteria;
use lance_index::scalar::FullTextSearchQuery;
use lance_index::scalar::expression::{INDEX_EXPR_RESULT_SCHEMA, IndexExprResult, PlannerIndexExt};
use lance_index::scalar::inverted::query::{
FtsQuery, FtsQueryNode, FtsSearchParams, MatchQuery, PhraseQuery, fill_fts_query_column,
};
use lance_index::scalar::inverted::{SCORE_COL, SCORE_FIELD};
use lance_index::vector::{DIST_COL, Query};
use lance_index::{DatasetIndexExt, scalar::expression::ScalarIndexExpr};
use lance_index::{metrics::NoOpMetricsCollector, scalar::inverted::FTS_SCHEMA};
use lance_io::stream::RecordBatchStream;
use lance_linalg::distance::MetricType;
use lance_table::format::{Fragment, IndexMetadata};
use roaring::RoaringBitmap;
use tracing::{Span, info_span, instrument};
use super::Dataset;
use crate::dataset::row_offsets_to_row_addresses;
use crate::dataset::utils::SchemaAdapter;
use crate::index::DatasetIndexInternalExt;
use crate::index::vector::utils::{
default_distance_type_for, get_vector_dim, get_vector_type, validate_distance_type_for,
};
use crate::io::exec::filtered_read::{FilteredReadExec, FilteredReadOptions};
use crate::io::exec::fts::{
BoostQueryExec, FlatMatchFilterExec, FlatMatchQueryExec, MatchQueryExec, PhraseQueryExec,
};
use crate::io::exec::knn::MultivectorScoringExec;
use crate::io::exec::scalar_index::{MaterializeIndexExec, ScalarIndexExec};
use crate::io::exec::{
AddRowAddrExec, FilterPlan as ExprFilterPlan, KNNVectorDistanceExec, LancePushdownScanExec,
LanceScanExec, Planner, PreFilterSource, ScanConfig, TakeExec,
knn::{KNN_INDEX_SCHEMA, new_knn_exec},
project,
};
use crate::io::exec::{AddRowOffsetExec, LanceFilterExec, LanceScanConfig, get_physical_optimizer};
use crate::{Error, Result};
use crate::{datatypes::Schema, io::exec::fts::BooleanQueryExec};
pub use lance_datafusion::exec::{ExecutionStatsCallback, ExecutionSummaryCounts};
#[cfg(feature = "substrait")]
use lance_datafusion::substrait::parse_substrait;
pub(crate) const BATCH_SIZE_FALLBACK: usize = 8192;
/// Parse an environment variable as a specific type, logging a warning on parse failure.
fn parse_env_var<T: std::str::FromStr>(env_var_name: &str, default_val: &str) -> Option<T>
where
T::Err: std::fmt::Display,
{
std::env::var(env_var_name)
.ok()
.and_then(|val| match val.parse() {
Ok(value) => Some(value),
Err(e) => {
log::warn!(
"Failed to parse the environment variable {}='{}': {}, the default value is: {}.",
env_var_name,
val,
e,
default_val
);
None
}
})
}
// For backwards compatibility / historical reasons we re-calculate the default batch size
// on each call
pub fn get_default_batch_size() -> Option<usize> {
parse_env_var("LANCE_DEFAULT_BATCH_SIZE", &BATCH_SIZE_FALLBACK.to_string())
}
pub const LEGACY_DEFAULT_FRAGMENT_READAHEAD: usize = 4;
pub static DEFAULT_FRAGMENT_READAHEAD: LazyLock<Option<usize>> = LazyLock::new(|| {
parse_env_var(
"LANCE_DEFAULT_FRAGMENT_READAHEAD",
&LEGACY_DEFAULT_FRAGMENT_READAHEAD.to_string(),
)
});
const DEFAULT_XTR_OVERFETCH_VALUE: u32 = 10;
pub static DEFAULT_XTR_OVERFETCH: LazyLock<u32> = LazyLock::new(|| {
parse_env_var(
"LANCE_XTR_OVERFETCH",
&DEFAULT_XTR_OVERFETCH_VALUE.to_string(),
)
.unwrap_or(DEFAULT_XTR_OVERFETCH_VALUE)
});
// We want to support ~256 concurrent reads to maximize throughput on cloud storage systems
// Our typical page size is 8MiB (though not all reads are this large yet due to offset buffers, validity buffers, etc.)
// So we want to support 256 * 8MiB ~= 2GiB of queued reads
const DEFAULT_IO_BUFFER_SIZE_VALUE: u64 = 2 * 1024 * 1024 * 1024;
pub static DEFAULT_IO_BUFFER_SIZE: LazyLock<u64> = LazyLock::new(|| {
parse_env_var(
"LANCE_DEFAULT_IO_BUFFER_SIZE",
&DEFAULT_IO_BUFFER_SIZE_VALUE.to_string(),
)
.unwrap_or(DEFAULT_IO_BUFFER_SIZE_VALUE)
});
/// Defines an ordering for a single column
///
/// Floats are sorted using the IEEE 754 total ordering
/// Strings are sorted using UTF-8 lexicographic order (i.e. we sort the binary)
#[derive(Debug, Clone)]
pub struct ColumnOrdering {
pub ascending: bool,
pub nulls_first: bool,
pub column_name: String,
}
impl ColumnOrdering {
pub fn asc_nulls_first(column_name: String) -> Self {
Self {
ascending: true,
nulls_first: true,
column_name,
}
}
pub fn asc_nulls_last(column_name: String) -> Self {
Self {
ascending: true,
nulls_first: false,
column_name,
}
}
pub fn desc_nulls_first(column_name: String) -> Self {
Self {
ascending: false,
nulls_first: true,
column_name,
}
}
pub fn desc_nulls_last(column_name: String) -> Self {
Self {
ascending: false,
nulls_first: false,
column_name,
}
}
}
/// Materialization style for the scanner
///
/// This only affects columns that are not used in a filter
///
/// Early materialization will fetch the entire column and throw
/// away the rows that are not needed. This fetches more data but
/// uses fewer I/O requests.
///
/// Late materialization will only fetch the rows that are needed.
/// This fetches less data but uses more I/O requests.
///
/// This parameter only affects scans. Vector search and full text search
/// always use late materialization.
#[derive(Clone)]
pub enum MaterializationStyle {
/// Heuristic-based materialization style
///
/// The default approach depends on the type of object storage. For
/// cloud storage (e.g. S3, GCS, etc.) we only use late materialization
/// for columns that are more than 1000 bytes in size.
///
/// For local storage we use late materialization for columns that are
/// more than 10 bytes in size.
///
/// These values are based on experimentation and the assumption that a
/// filter will be selecting ~0.1% of the rows in a column.
Heuristic,
/// All columns will be fetched with late materialization where possible
AllLate,
/// All columns will be fetched with early materialization where possible
AllEarly,
/// All columns will be fetched with late materialization except for the specified columns
AllEarlyExcept(Vec<u32>),
}
impl MaterializationStyle {
pub fn all_early_except(columns: &[impl AsRef<str>], schema: &Schema) -> Result<Self> {
let field_ids = schema
.project(columns)?
.field_ids()
.into_iter()
.map(|id| id as u32)
.collect();
Ok(Self::AllEarlyExcept(field_ids))
}
}
#[derive(Debug)]
struct PlannedFilteredScan {
plan: Arc<dyn ExecutionPlan>,
limit_pushed_down: bool,
filter_pushed_down: bool,
}
pub struct FilterPlan {
// Query filter plan
query_filter: Option<QueryFilter>,
refine_query_filter: bool,
// Expr filter plan
expr_filter_plan: ExprFilterPlan,
}
impl FilterPlan {
pub fn new(query_filter: Option<QueryFilter>, expr_filter_plan: ExprFilterPlan) -> Self {
Self {
query_filter,
refine_query_filter: false,
expr_filter_plan,
}
}
pub fn disable_refine(&mut self) {
self.expr_filter_plan = ExprFilterPlan::default();
self.refine_query_filter = false;
}
pub fn make_refine_only(&mut self) {
self.expr_filter_plan.make_refine_only();
self.refine_query_filter = true;
}
pub fn fts_filter(&self) -> Option<FullTextSearchQuery> {
match &self.query_filter {
Some(QueryFilter::Fts(query)) => Some(query.clone()),
_ => None,
}
}
pub fn vector_filter(&self) -> Option<Query> {
match &self.query_filter {
Some(QueryFilter::Vector(query)) => Some(query.clone()),
_ => None,
}
}
pub fn has_refine(&self) -> bool {
self.expr_filter_plan.has_refine() || self.refine_query_filter
}
pub async fn refine_columns(&self, dataset: &Arc<Dataset>) -> Result<Vec<String>> {
let mut columns = vec![];
if self.expr_filter_plan.has_refine() {
columns.extend(self.expr_filter_plan.refine_columns());
}
if self.refine_query_filter {
match &self.query_filter {
Some(QueryFilter::Fts(fts_query)) => {
let cols = if fts_query.columns().is_empty() {
let indexed_columns = fts_indexed_columns(dataset.clone()).await?;
let q = fill_fts_query_column(&fts_query.query, &indexed_columns, false)?;
q.columns()
} else {
fts_query.columns()
};
// Add refine column for match query since it supports `FlatMatchQueryExec`.
// Other fts query use join so we don't need to add refine column.
if let FtsQuery::Match(_) = &fts_query.query {
columns.extend(cols.iter().cloned().collect::<Vec<_>>());
}
}
Some(QueryFilter::Vector(vector_query)) => {
columns.push(vector_query.column.clone());
}
None => {}
}
}
Ok(columns)
}
pub async fn refine_filter(
&self,
input: Arc<dyn ExecutionPlan>,
scanner: &Scanner,
) -> Result<Arc<dyn ExecutionPlan>> {
let mut plan = input;
if self.refine_query_filter {
match &self.query_filter {
Some(QueryFilter::Fts(fts_query)) => {
plan = scanner.flat_fts_filter(plan, fts_query).await?;
}
Some(QueryFilter::Vector(vector_query)) => {
plan = scanner.flat_knn(plan, vector_query)?;
}
None => {}
}
}
if let Some(refine_expr) = &self.expr_filter_plan.refine_expr {
// We create a new planner specific to the node's schema, since
// physical expressions reference column by index rather than by name.
plan = Arc::new(LanceFilterExec::try_new(refine_expr.clone(), plan)?);
}
Ok(plan)
}
}
#[derive(Debug, Clone, Default)]
pub struct LanceFilter {
query_filter: Option<QueryFilter>,
expr_filter: Option<ExprFilter>,
}
impl LanceFilter {
pub fn is_none(&self) -> bool {
self.query_filter.is_none() && self.expr_filter.is_none()
}
}
/// Query filter for filtering rows
#[derive(Debug, Clone)]
pub enum QueryFilter {
Fts(FullTextSearchQuery),
Vector(Query),
}
/// Expr filter for filtering rows
#[derive(Debug, Clone)]
pub enum ExprFilter {
/// The filter is an SQL string
Sql(String),
/// The filter is a Substrait expression
Substrait(Vec<u8>),
/// The filter is a Datafusion expression
Datafusion(Expr),
}
impl ExprFilter {
/// Converts the filter to a Datafusion expression
///
/// The schema for this conversion should be the full schema available to
/// the filter (`full_schema`). However, due to a limitation in the way
/// we do Substrait conversion today we can only do Substrait conversion with
/// the dataset schema (`dataset_schema`). This means that Substrait will
/// not be able to access columns that are not in the dataset schema (e.g.
/// _rowid, _rowaddr, etc.)
#[allow(unused)]
#[instrument(level = "trace", name = "filter_to_df", skip_all)]
pub fn to_datafusion(&self, dataset_schema: &Schema, full_schema: &Schema) -> Result<Expr> {
match self {
Self::Sql(sql) => {
let schema = Arc::new(ArrowSchema::from(full_schema));
let planner = Planner::new(schema.clone());
let filter = planner.parse_filter(sql)?;
let df_schema = DFSchema::try_from(schema)?;
let ret_field = filter.to_field(&df_schema)?.1;
let ret_type = ret_field.data_type();
if ret_type != &DataType::Boolean {
return Err(Error::invalid_input_source(
format!("The filter {} does not return a boolean", filter).into(),
));
}
let optimized = planner.optimize_expr(filter).map_err(|e| {
Error::invalid_input(format!("Error optimizing sql filter: {sql} ({e})"))
})?;
Ok(optimized)
}
#[cfg(feature = "substrait")]
Self::Substrait(expr) => {
use lance_datafusion::exec::{LanceExecutionOptions, get_session_context};
let ctx = get_session_context(&LanceExecutionOptions::default());
let state = ctx.state();
let schema = Arc::new(ArrowSchema::from(dataset_schema));
let expr = parse_substrait(expr, schema.clone(), &ctx.state())
.now_or_never()
.expect("could not parse the Substrait filter in a synchronous fashion")?;
let planner = Planner::new(schema);
planner.optimize_expr(expr.clone()).map_err(|e| {
Error::invalid_input(format!(
"Error optimizing substrait filter: {expr:?} ({e})"
))
})
}
#[cfg(not(feature = "substrait"))]
Self::Substrait(_) => Err(Error::not_supported_source(
"Substrait filter is not supported in this build".into(),
)),
Self::Datafusion(expr) => Ok(expr.clone()),
}
}
}
/// Aggregate expression from Substrait or DataFusion.
#[derive(Debug, Clone)]
pub enum AggregateExpr {
#[cfg(feature = "substrait")]
Substrait(Vec<u8>),
Datafusion {
group_by: Vec<Expr>,
aggregates: Vec<Expr>,
},
}
impl AggregateExpr {
/// Create a new builder for aggregate expressions.
///
/// # Example
/// ```ignore
/// let agg = AggregateExpr::builder()
/// .group_by("category")
/// .count_star().alias("total_count")
/// .sum("amount").alias("total_amount")
/// .avg("price")
/// .build();
/// scanner.aggregate(agg);
/// ```
pub fn builder() -> AggregateExprBuilder<false> {
AggregateExprBuilder::new()
}
/// Create from Substrait Plan bytes.
#[cfg(feature = "substrait")]
pub fn substrait(bytes: impl Into<Vec<u8>>) -> Self {
Self::Substrait(bytes.into())
}
/// Create from DataFusion expressions.
/// Use `.alias()` on expressions to set output column names.
pub fn datafusion(group_by: Vec<Expr>, aggregates: Vec<Expr>) -> Self {
Self::Datafusion {
group_by,
aggregates,
}
}
/// Parse into a unified Aggregate structure.
///
/// For Substrait, this parses the bytes into DataFusion expressions.
/// For DataFusion, this just wraps the expressions.
///
/// The schema is used to resolve field references in Substrait expressions.
fn parse(self, #[allow(unused_variables)] schema: Arc<ArrowSchema>) -> Result<Aggregate> {
match self {
#[cfg(feature = "substrait")]
Self::Substrait(bytes) => {
use lance_datafusion::exec::{LanceExecutionOptions, get_session_context};
use lance_datafusion::substrait::parse_substrait_aggregate;
let ctx = get_session_context(&LanceExecutionOptions::default());
parse_substrait_aggregate(&bytes, schema, &ctx.state())
.now_or_never()
.expect("could not parse the Substrait aggregate in a synchronous fashion")
}
Self::Datafusion {
group_by,
aggregates,
} => Ok(Aggregate::new(group_by, aggregates)),
}
}
}
/// Builder for creating aggregate expressions without using DataFusion or Substrait directly.
///
/// The const generic `HAS_PENDING` tracks whether there's a pending aggregate that can be aliased.
/// When `HAS_PENDING` is `true`, the last item in `aggregates` is the pending aggregate.
#[derive(Debug, Clone)]
pub struct AggregateExprBuilder<const HAS_PENDING: bool> {
group_by: Vec<Expr>,
aggregates: Vec<Expr>,
}
impl Default for AggregateExprBuilder<false> {
fn default() -> Self {
Self {
group_by: Vec::new(),
aggregates: Vec::new(),
}
}
}
impl AggregateExprBuilder<false> {
/// Create a new builder.
pub fn new() -> Self {
Self::default()
}
/// Build the aggregate expression.
pub fn build(self) -> AggregateExpr {
AggregateExpr::Datafusion {
group_by: self.group_by,
aggregates: self.aggregates,
}
}
}
impl<const HAS_PENDING: bool> AggregateExprBuilder<HAS_PENDING> {
/// Add a column to group by.
///
/// Multiple invocations will add to the list (not replace it).
/// E.g. `.group_by("x").group_by("y")` will group by both `x` and `y`.
pub fn group_by(mut self, column: impl Into<String>) -> AggregateExprBuilder<false> {
self.group_by.push(col(column.into()));
AggregateExprBuilder {
group_by: self.group_by,
aggregates: self.aggregates,
}
}
/// Add multiple columns to group by.
///
/// Multiple invocations will add to the list (not replace it).
/// E.g. `.group_by("x").group_by_columns(["y", "z"])` will group by `x`, `y`, and `z`.
pub fn group_by_columns(
mut self,
columns: impl IntoIterator<Item = impl Into<String>>,
) -> AggregateExprBuilder<false> {
for column in columns {
self.group_by.push(col(column.into()));
}
AggregateExprBuilder {
group_by: self.group_by,
aggregates: self.aggregates,
}
}
/// Add COUNT(*) aggregate that counts all rows.
pub fn count_star(mut self) -> AggregateExprBuilder<true> {
self.aggregates
.push(functions_aggregate::count::count(lit(1)));
AggregateExprBuilder {
group_by: self.group_by,
aggregates: self.aggregates,
}
}
/// Add COUNT(column) aggregate.
///
/// Unlike `count_star`, this will only count the number of rows where `column`
/// is not NULL.
pub fn count(mut self, column: impl Into<String>) -> AggregateExprBuilder<true> {
self.aggregates
.push(functions_aggregate::count::count(col(column.into())));
AggregateExprBuilder {
group_by: self.group_by,
aggregates: self.aggregates,
}
}
/// Add SUM(column) aggregate.
pub fn sum(mut self, column: impl Into<String>) -> AggregateExprBuilder<true> {
self.aggregates
.push(functions_aggregate::sum::sum(col(column.into())));
AggregateExprBuilder {
group_by: self.group_by,
aggregates: self.aggregates,
}
}
/// Add AVG(column) aggregate.
pub fn avg(mut self, column: impl Into<String>) -> AggregateExprBuilder<true> {
self.aggregates
.push(functions_aggregate::average::avg(col(column.into())));
AggregateExprBuilder {
group_by: self.group_by,
aggregates: self.aggregates,
}
}
/// Add MIN(column) aggregate.
pub fn min(mut self, column: impl Into<String>) -> AggregateExprBuilder<true> {
self.aggregates
.push(functions_aggregate::min_max::min(col(column.into())));
AggregateExprBuilder {
group_by: self.group_by,
aggregates: self.aggregates,
}
}
/// Add MAX(column) aggregate.
pub fn max(mut self, column: impl Into<String>) -> AggregateExprBuilder<true> {
self.aggregates
.push(functions_aggregate::min_max::max(col(column.into())));
AggregateExprBuilder {
group_by: self.group_by,
aggregates: self.aggregates,
}
}
}
impl AggregateExprBuilder<true> {
/// Set an alias for the pending aggregate (the last added aggregate).
pub fn alias(mut self, name: impl Into<String>) -> AggregateExprBuilder<false> {
let pending = self.aggregates.pop().expect("pending aggregate must exist");
self.aggregates.push(pending.alias(name.into()));
AggregateExprBuilder {
group_by: self.group_by,
aggregates: self.aggregates,
}
}
/// Build the aggregate expression.
pub fn build(self) -> AggregateExpr {
AggregateExpr::Datafusion {
group_by: self.group_by,
aggregates: self.aggregates,
}
}
}
/// Dataset Scanner
///
/// ```rust,ignore
/// let dataset = Dataset::open(uri).await.unwrap();
/// let stream = dataset.scan()
/// .project(&["col", "col2.subfield"]).unwrap()
/// .limit(10)
/// .into_stream();
/// stream
/// .map(|batch| batch.num_rows())
/// .buffered(16)
/// .sum()
/// ```
#[derive(Clone)]
pub struct Scanner {
dataset: Arc<Dataset>,
/// The projection plan for the scanner
///
/// This includes
/// - The physical projection that must be read from the dataset
/// - Dynamic expressions that are evaluated after the physical projection
/// - The names of the output columns
projection_plan: ProjectionPlan,
blob_handling: BlobHandling,
/// If true then the filter will be applied before an index scan
prefilter: bool,
/// Materialization style controls when columns are fetched
materialization_style: MaterializationStyle,
/// Filter.
filter: LanceFilter,
/// Optional full text search query
full_text_query: Option<FullTextSearchQuery>,
/// The batch size controls the maximum size of rows to return for each read.
batch_size: Option<usize>,
/// Number of batches to prefetch
batch_readahead: usize,
/// Number of fragments to read concurrently
fragment_readahead: Option<usize>,
/// Number of bytes to allow to queue up in the I/O buffer
io_buffer_size: Option<u64>,
limit: Option<i64>,
offset: Option<i64>,
/// If Some then results will be ordered by the provided ordering
///
/// If there are multiple columns the results will first be ordered
/// by the first column. Then, any values whose first column is equal
/// will be sorted by the next column, and so on.
///
/// If this is Some then the value of `ordered` is ignored. The scan
/// will always be unordered since we are just going to reorder it anyways.
ordering: Option<Vec<ColumnOrdering>>,
nearest: Option<Query>,
/// If false, do not use any scalar indices for the scan
///
/// This can be used to pick a more efficient plan for certain queries where
/// scalar indices do not work well (though we should also improve our planning
/// to handle this better in the future as well)
use_scalar_index: bool,
/// Whether to use statistics to optimize the scan (default: true)
///
/// This is used for debugging or benchmarking purposes.
use_stats: bool,
/// Whether to scan in deterministic order (default: true)
///
/// This field is ignored if `ordering` is defined
ordered: bool,
/// If set, this scanner serves only these fragments.
fragments: Option<Vec<Fragment>>,
/// Only search the data being indexed (weak consistency search).
///
/// Default value is false.
///
/// This is essentially a weak consistency search. Users can run index or optimize index
/// to make the index catch up with the latest data.
fast_search: bool,
/// If true, the scanner will emit deleted rows
include_deleted_rows: bool,
/// If set, this callback will be called after the scan with summary statistics
scan_stats_callback: Option<ExecutionStatsCallback>,
/// Whether the result returned by the scanner must be of the size of the batch_size.
/// By default, it is false.
/// Mainly, if the result is returned strictly according to the batch_size,
/// batching and waiting are required, and the performance will decrease.
strict_batch_size: bool,
/// File reader options to use when reading data files.
file_reader_options: Option<FileReaderOptions>,
aggregate: Option<Aggregate>,
// Legacy fields to help migrate some old projection behavior to new behavior
//
// There are two behaviors we are moving away from:
//
// First, the old behavior used methods like with_row_id and with_row_addr to add
// "system" columns. The new behavior is to specify them in the projection like any
// other column. The only difference between a system column and a regular column is
// that system columns are not returned in the schema and are not returned by default
// (i.e. "SELECT *")
//
// Second, the old behavior would _always_ add the _score or _distance columns to the
// output and there was no way for the user to opt out. The new behavior treats the
// _score and _distance as regular output columns of the "search table function". If
// the user does not specify a projection (i.e. "SELECT *") then we will add the _score
// and _distance columns to the end. If the user does specify a projection then they
// must request those columns for them to show up.
//
// --------------------------------------------------------------------------
/// Whether the user wants the row id on top of the projection, will always come last
/// except possibly before _rowaddr
legacy_with_row_id: bool,
/// Whether the user wants the row address on top of the projection, will always come last
legacy_with_row_addr: bool,
/// Whether the user explicitly requested a projection. If they did then we will warn them
/// if they do not specify _score / _distance unless legacy_projection_behavior is set to false
explicit_projection: bool,
/// Whether the user wants to use the legacy projection behavior.
autoproject_scoring_columns: bool,
}
/// Represents a user-requested take operation
#[derive(Debug, Clone)]
pub enum TakeOperation {
/// Take rows by row id
RowIds(Vec<u64>),
/// Take rows by row address
RowAddrs(Vec<u64>),
/// Take rows by row offset
///
/// The row offset is the offset of the row in the dataset. This can
/// be converted to row addresses using the fragment sizes.
RowOffsets(Vec<u64>),
}
impl TakeOperation {
fn extract_u64_list(list: &[Expr]) -> Option<Vec<u64>> {
let mut u64s = Vec::with_capacity(list.len());
for expr in list {
if let Expr::Literal(lit, _) = expr {
if let Some(ScalarValue::UInt64(Some(val))) =
safe_coerce_scalar(lit, &DataType::UInt64)
{
u64s.push(val);
} else {
return None;
}
} else {
return None;
}
}
Some(u64s)
}
fn merge(self, other: Self) -> Option<Self> {
match (self, other) {
(Self::RowIds(mut left), Self::RowIds(right)) => {
left.extend(right);
Some(Self::RowIds(left))
}
(Self::RowAddrs(mut left), Self::RowAddrs(right)) => {
left.extend(right);
Some(Self::RowAddrs(left))
}
(Self::RowOffsets(mut left), Self::RowOffsets(right)) => {
left.extend(right);
Some(Self::RowOffsets(left))
}
_ => None,
}
}
/// Attempts to create a take operation from an expression. This will succeed if the expression
/// has one of the following forms:
/// - `_rowid = 10`
/// - `_rowid = 10 OR _rowid = 20 OR _rowid = 30`
/// - `_rowid IN (10, 20, 30)`
/// - `_rowaddr = 10`
/// - `_rowaddr = 10 OR _rowaddr = 20 OR _rowaddr = 30`
/// - `_rowaddr IN (10, 20, 30)`
/// - `_rowoffset = 10`
/// - `_rowoffset = 10 OR _rowoffset = 20 OR _rowoffset = 30`
/// - `_rowoffset IN (10, 20, 30)`
///
/// The _rowid / _rowaddr / _rowoffset determine if we are taking by row id, address, or offset.
///
/// If a take expression is combined with some other filter via an AND then the remainder will be
/// returned as well. For example, `_rowid = 10` will return (take_op, None) and
/// `_rowid = 10 AND x > 70` will return (take_op, Some(x > 70)).
fn try_from_expr(expr: &Expr) -> Option<(Self, Option<Expr>)> {
if let Expr::BinaryExpr(binary) = expr {
match binary.op {
datafusion_expr::Operator::And => {
let left_take = Self::try_from_expr(&binary.left);
let right_take = Self::try_from_expr(&binary.right);
match (left_take, right_take) {
(Some(_), Some(_)) => {
// This is something like...
//
// _rowid = 10 AND _rowid = 20
//
// ...which is kind of nonsensical. Better to just return None.
return None;
}
(Some((left_op, left_rem)), None) => {
let remainder = match left_rem {
// If there is a remainder on the left side we combine it. This _should_
// be something like converting (_rowid = 10 AND x > 70) AND y > 80
// to (_rowid = 10) AND (x > 70 AND y > 80) which should be valid
Some(expr) => Expr::and(expr, binary.right.as_ref().clone()),
None => binary.right.as_ref().clone(),
};
return Some((left_op, Some(remainder)));
}
(None, Some((right_op, right_rem))) => {
let remainder = match right_rem {
Some(expr) => Expr::and(expr, binary.left.as_ref().clone()),
None => binary.left.as_ref().clone(),
};
return Some((right_op, Some(remainder)));
}
(None, None) => {
return None;
}
}
}
datafusion_expr::Operator::Eq => {
// Check for _rowid = literal
if let (Expr::Column(col), Expr::Literal(lit, _)) =
(binary.left.as_ref(), binary.right.as_ref())
&& let Some(ScalarValue::UInt64(Some(val))) =
safe_coerce_scalar(lit, &DataType::UInt64)
{
if col.name == ROW_ID {
return Some((Self::RowIds(vec![val]), None));
} else if col.name == ROW_ADDR {
return Some((Self::RowAddrs(vec![val]), None));
} else if col.name == ROW_OFFSET {
return Some((Self::RowOffsets(vec![val]), None));
}
}
}
datafusion_expr::Operator::Or => {
let left_take = Self::try_from_expr(&binary.left);
let right_take = Self::try_from_expr(&binary.right);
if let (Some(left), Some(right)) = (left_take, right_take) {
if left.1.is_some() || right.1.is_some() {
// This would be something like...
//
// (_rowid = 10 AND x > 70) OR _rowid = 20
//
// I don't think it's correct to convert this into a take operation
// which would give us (_rowid = 10 OR _rowid = 20) AND x > 70
return None;
}
return left.0.merge(right.0).map(|op| (op, None));
}
}
_ => {}
}
} else if let Expr::InList(in_expr) = expr
&& let Expr::Column(col) = in_expr.expr.as_ref()
&& let Some(u64s) = Self::extract_u64_list(&in_expr.list)
{
if col.name == ROW_ID {
return Some((Self::RowIds(u64s), None));
} else if col.name == ROW_ADDR {
return Some((Self::RowAddrs(u64s), None));
} else if col.name == ROW_OFFSET {
return Some((Self::RowOffsets(u64s), None));
}
}
None
}
}
impl Scanner {
pub fn new(dataset: Arc<Dataset>) -> Self {
let projection_plan = ProjectionPlan::full(dataset.clone()).unwrap();
let file_reader_options = dataset.file_reader_options.clone();
let mut scanner = Self {
dataset,
projection_plan,
blob_handling: BlobHandling::default(),
prefilter: false,
materialization_style: MaterializationStyle::Heuristic,
filter: LanceFilter::default(),
full_text_query: None,
batch_size: None,
batch_readahead: get_num_compute_intensive_cpus(),
fragment_readahead: None,
io_buffer_size: None,
limit: None,
offset: None,
ordering: None,
nearest: None,
use_stats: true,
ordered: true,
fragments: None,
fast_search: false,
use_scalar_index: true,
include_deleted_rows: false,
scan_stats_callback: None,
strict_batch_size: false,