Repository navigation
Expand file tree
/
Copy pathmod.rs
More file actions
4627 lines (4196 loc) · 180 KB
/
Copy pathmod.rs
File metadata and controls
4627 lines (4196 loc) · 180 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
// SPDX-License-Identifier: Apache-2.0
// SPDX-FileCopyrightText: See the NOTICE file distributed with this work for additional information regarding copyright ownership
// SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements
//! [`FileScanConfig`] to configure scanning of possibly partitioned
//! file sources.
pub(crate) mod sort_pushdown;
/// Shared `FileScanConfig` <-> proto conversion, gated on the `proto` feature.
/// Attaches inherent `try_to_proto` / `try_from_proto` /
/// `parse_table_schema_from_proto` helpers to [`FileScanConfig`] used by every
/// file source's `try_to_proto` hook.
#[cfg(feature = "proto")]
mod proto;
use crate::file_groups::FileGroup;
use crate::{
PartitionedFile, display::FileGroupsDisplay, file::FileSource,
file_compression_type::FileCompressionType, file_stream::FileStreamBuilder,
file_stream::work_source::SharedWorkSource, source::DataSource,
statistics::MinMaxStatistics,
};
use arrow::datatypes::Fields;
use arrow::datatypes::{DataType, Schema, SchemaRef};
use datafusion_common::config::ConfigOptions;
use datafusion_common::tree_node::TreeNodeRecursion;
use datafusion_common::{
Constraint, Constraints, Result, ScalarValue, SplitPoint, Statistics,
internal_datafusion_err, internal_err,
};
use datafusion_execution::{
SendableRecordBatchStream, TaskContext, object_store::ObjectStoreUrl,
};
use datafusion_expr::Operator;
use crate::source::OpenArgs;
use datafusion_common::stats::{Precision, is_known_empty};
use datafusion_physical_expr::expressions::{BinaryExpr, Column};
use datafusion_physical_expr::projection::{ProjectionExprs, ProjectionMapping};
use datafusion_physical_expr::utils::reassign_expr_columns;
use datafusion_physical_expr::{
EquivalenceProperties, Partitioning, RangePartitioning, split_conjunction,
};
use datafusion_physical_expr_adapter::PhysicalExprAdapterFactory;
use datafusion_physical_expr_common::physical_expr::{PhysicalExpr, is_volatile};
use datafusion_physical_expr_common::sort_expr::{LexOrdering, PhysicalSortExpr};
use datafusion_physical_plan::SortOrderPushdownResult;
use datafusion_physical_plan::coop::cooperative;
use datafusion_physical_plan::execution_plan::SchedulingType;
use datafusion_physical_plan::{
DisplayAs, DisplayFormatType,
display::{ProjectSchemaDisplay, display_orderings},
filter_pushdown::FilterPushdownPropagation,
metrics::ExecutionPlanMetricsSet,
};
use log::{debug, warn};
use std::any::Any;
use std::{fmt::Debug, fmt::Formatter, fmt::Result as FmtResult, sync::Arc};
/// [`FileScanConfig`] represents scanning data from a group of files
///
/// `FileScanConfig` is used to create a [`DataSourceExec`], the physical plan
/// for scanning files with a particular file format.
///
/// The [`FileSource`] (e.g. `ParquetSource`, `CsvSource`, etc.) is responsible
/// for creating the actual execution plan to read the files based on a
/// `FileScanConfig`. Fields in a `FileScanConfig` such as Statistics represent
/// information about the files **before** any projection or filtering is
/// applied in the file source.
///
/// Use [`FileScanConfigBuilder`] to construct a `FileScanConfig`.
///
/// Use [`DataSourceExec::from_data_source`] to create a [`DataSourceExec`] from
/// a `FileScanConfig`.
///
/// # Example
/// ```
/// # use std::sync::Arc;
/// # use arrow::datatypes::{Field, Fields, DataType, Schema, SchemaRef};
/// # use object_store::ObjectStore;
/// # use datafusion_common::Result;
/// # use datafusion_common::tree_node::TreeNodeRecursion;
/// # use datafusion_datasource::file::FileSource;
/// # use datafusion_physical_plan::PhysicalExpr;
/// # use datafusion_datasource::file_groups::FileGroup;
/// # use datafusion_datasource::PartitionedFile;
/// # use datafusion_datasource::file_scan_config::{FileScanConfig, FileScanConfigBuilder};
/// # use datafusion_datasource::file_stream::FileOpener;
/// # use datafusion_datasource::source::DataSourceExec;
/// # use datafusion_datasource::table_schema::TableSchema;
/// # use datafusion_execution::object_store::ObjectStoreUrl;
/// # use datafusion_physical_expr::projection::ProjectionExprs;
/// # use datafusion_physical_plan::ExecutionPlan;
/// # use datafusion_physical_plan::metrics::ExecutionPlanMetricsSet;
/// # let file_schema = Arc::new(Schema::new(vec![
/// # Field::new("c1", DataType::Int32, false),
/// # Field::new("c2", DataType::Int32, false),
/// # Field::new("c3", DataType::Int32, false),
/// # Field::new("c4", DataType::Int32, false),
/// # ]));
/// # // Note: crate mock ParquetSource, as ParquetSource is not in the datasource crate
/// #[derive(Clone)]
/// # struct ParquetSource {
/// # table_schema: TableSchema,
/// # };
/// # impl FileSource for ParquetSource {
/// # fn create_file_opener(&self, _: Arc<dyn ObjectStore>, _: &FileScanConfig, _: usize) -> Result<Arc<dyn FileOpener>> { unimplemented!() }
/// # fn table_schema(&self) -> &TableSchema { &self.table_schema }
/// # fn with_batch_size(&self, _: usize) -> Arc<dyn FileSource> { unimplemented!() }
/// # fn metrics(&self) -> &ExecutionPlanMetricsSet { unimplemented!() }
/// # fn file_type(&self) -> &str { "parquet" }
/// # // Note that this implementation drops the projection on the floor, it is not complete!
/// # fn try_pushdown_projection(&self, projection: &ProjectionExprs) -> Result<Option<Arc<dyn FileSource>>> { Ok(Some(Arc::new(self.clone()) as Arc<dyn FileSource>)) }
/// # fn apply_expressions(&self, _f: &mut dyn FnMut(&Arc<dyn PhysicalExpr>) -> Result<TreeNodeRecursion>) -> Result<TreeNodeRecursion> { Ok(TreeNodeRecursion::Continue) }
/// # }
/// # impl ParquetSource {
/// # fn new(table_schema: impl Into<TableSchema>) -> Self { Self {table_schema: table_schema.into()} }
/// # }
/// // create FileScan config for reading parquet files from file://
/// let object_store_url = ObjectStoreUrl::local_filesystem();
/// let file_source = Arc::new(ParquetSource::new(file_schema.clone()));
/// let config = FileScanConfigBuilder::new(object_store_url, file_source)
/// .with_limit(Some(1000)) // read only the first 1000 records
/// .with_projection_indices(Some(vec![2, 3])) // project columns 2 and 3
/// .expect("Failed to push down projection")
/// // Read /tmp/file1.parquet with known size of 1234 bytes in a single group
/// .with_file(PartitionedFile::new("file1.parquet", 1234))
/// // Read /tmp/file2.parquet 56 bytes and /tmp/file3.parquet 78 bytes
/// // in a single row group
/// .with_file_group(FileGroup::new(vec![
/// PartitionedFile::new("file2.parquet", 56),
/// PartitionedFile::new("file3.parquet", 78),
/// ])).build();
/// // create an execution plan from the config
/// let plan: Arc<dyn ExecutionPlan> = DataSourceExec::from_data_source(config);
/// ```
///
/// [`DataSourceExec`]: crate::source::DataSourceExec
/// [`DataSourceExec::from_data_source`]: crate::source::DataSourceExec::from_data_source
#[derive(Clone)]
pub struct FileScanConfig {
/// Object store URL, used to get an [`ObjectStore`] instance from
/// [`RuntimeEnv::object_store`]
///
/// This `ObjectStoreUrl` should be the prefix of the absolute url for files
/// as `file://` or `s3://my_bucket`. It should not include the path to the
/// file itself. The relevant URL prefix must be registered via
/// [`RuntimeEnv::register_object_store`]
///
/// [`ObjectStore`]: object_store::ObjectStore
/// [`RuntimeEnv::register_object_store`]: datafusion_execution::runtime_env::RuntimeEnv::register_object_store
/// [`RuntimeEnv::object_store`]: datafusion_execution::runtime_env::RuntimeEnv::object_store
pub object_store_url: ObjectStoreUrl,
/// List of files to be processed, grouped into partitions
///
/// Each file must have a schema of `file_schema` or a subset. If
/// a particular file has a subset, the missing columns are
/// padded with NULLs.
///
/// DataFusion may attempt to read each partition of files
/// concurrently, however files *within* a partition will be read
/// sequentially, one after the next.
///
/// Note that when `datafusion.execution.enable_file_stream_work_stealing`
/// is enabled (the default), files may be reassigned to a different
/// partition at runtime unless `preserve_order` or
/// `partitioned_by_file_group` is set, so a file is not guaranteed to be
/// read by the partition it is grouped under here.
pub file_groups: Vec<FileGroup>,
/// Table constraints
pub constraints: Constraints,
/// The maximum number of records to read from this plan. If `None`,
/// all records after filtering are returned.
pub limit: Option<usize>,
/// Read files in the order of [`Self::file_groups`].
///
/// [`FileScanConfigBuilder::build`] sets this when the scan has an output
/// ordering. Readers use [`Self::preserves_file_order`], which also checks
/// the current [`Self::output_ordering`], so clearing this flag never
/// reorders a scan that advertises an ordering.
pub preserve_order: bool,
/// All equivalent lexicographical output orderings of this file scan, in terms of
/// [`FileSource::table_schema`]. See [`FileScanConfigBuilder::with_output_ordering`] for more
/// details.
///
/// [`Self::eq_properties`] uses this information along with projection
/// and filtering information to compute the effective
/// [`EquivalenceProperties`]
pub output_ordering: Vec<LexOrdering>,
/// File compression type
pub file_compression_type: FileCompressionType,
/// File source such as `ParquetSource`, `CsvSource`, `JsonSource`, etc.
pub file_source: Arc<dyn FileSource>,
/// Batch size while creating new batches
/// Defaults to [`datafusion_common::config::ExecutionOptions`] batch_size.
pub batch_size: Option<usize>,
/// Expression adapter used to adapt filters and projections that are pushed down into the scan
/// from the logical schema to the physical schema of the file.
pub expr_adapter_factory: Option<Arc<dyn PhysicalExprAdapterFactory>>,
/// Statistics for the entire table (file schema + partition columns).
/// See [`FileScanConfigBuilder::with_statistics`] for more details.
///
/// The effective statistics are computed on-demand via
/// [`ProjectionExprs::project_statistics`].
///
/// Note that this field is pub(crate) because accessing it directly from outside
/// would be incorrect if there are filters being applied, thus this should be accessed
/// via [`FileScanConfig::statistics`].
pub(crate) statistics: Statistics,
/// Declared physical output partitioning for this scan.
///
/// Expressions are against the full table schema, before scan projection or
/// filtering. `ListingTable` validates partition count before building the
/// scan, and direct builders with mismatched counts fall back to
/// `UnknownPartitioning`.
pub output_partitioning: Option<Partitioning>,
}
/// A builder for [`FileScanConfig`]'s.
///
/// Example:
///
/// ```rust
/// # use std::sync::Arc;
/// # use arrow::datatypes::{DataType, Field, Schema};
/// # use datafusion_datasource::file_scan_config::{FileScanConfigBuilder, FileScanConfig};
/// # use datafusion_datasource::file_compression_type::FileCompressionType;
/// # use datafusion_datasource::file_groups::FileGroup;
/// # use datafusion_datasource::PartitionedFile;
/// # use datafusion_datasource::table_schema::TableSchema;
/// # use datafusion_execution::object_store::ObjectStoreUrl;
/// # use datafusion_common::Statistics;
/// # use datafusion_datasource::file::FileSource;
///
/// # fn main() {
/// # fn with_source(file_source: Arc<dyn FileSource>) {
/// // Create a schema for our Parquet files
/// let file_schema = Arc::new(Schema::new(vec![
/// Field::new("id", DataType::Int32, false),
/// Field::new("value", DataType::Utf8, false),
/// ]));
///
/// // Create partition columns
/// let partition_cols = vec![
/// Arc::new(Field::new("date", DataType::Utf8, false)),
/// ];
///
/// // Create table schema with file schema and partition columns
/// let table_schema = TableSchema::builder(file_schema)
/// .with_table_partition_cols(partition_cols)
/// .build();
///
/// // Create a builder for scanning Parquet files from a local filesystem
/// let config = FileScanConfigBuilder::new(
/// ObjectStoreUrl::local_filesystem(),
/// file_source,
/// )
/// // Set a limit of 1000 rows
/// .with_limit(Some(1000))
/// // Project only the first column
/// .with_projection_indices(Some(vec![0]))
/// .expect("Failed to push down projection")
/// // Add a file group with two files
/// .with_file_group(FileGroup::new(vec![
/// PartitionedFile::new("data/date=2024-01-01/file1.parquet", 1024),
/// PartitionedFile::new("data/date=2024-01-01/file2.parquet", 2048),
/// ]))
/// // Set compression type
/// .with_file_compression_type(FileCompressionType::UNCOMPRESSED)
/// // Build the final config
/// .build();
/// # }
/// # }
/// ```
#[derive(Clone)]
pub struct FileScanConfigBuilder {
object_store_url: ObjectStoreUrl,
file_source: Arc<dyn FileSource>,
limit: Option<usize>,
preserve_order: bool,
constraints: Option<Constraints>,
file_groups: Vec<FileGroup>,
statistics: Option<Statistics>,
output_ordering: Vec<LexOrdering>,
output_partitioning: Option<Partitioning>,
file_compression_type: Option<FileCompressionType>,
batch_size: Option<usize>,
expr_adapter_factory: Option<Arc<dyn PhysicalExprAdapterFactory>>,
}
impl FileScanConfigBuilder {
/// Create a new [`FileScanConfigBuilder`] with default settings for scanning files.
///
/// # Parameters:
/// * `object_store_url`: See [`FileScanConfig::object_store_url`]
/// * `file_source`: See [`FileScanConfig::file_source`]. The file source must have
/// a schema set via its constructor.
pub fn new(
object_store_url: ObjectStoreUrl,
file_source: Arc<dyn FileSource>,
) -> Self {
Self {
object_store_url,
file_source,
file_groups: vec![],
statistics: None,
output_ordering: vec![],
output_partitioning: None,
file_compression_type: None,
limit: None,
preserve_order: false,
constraints: None,
batch_size: None,
expr_adapter_factory: None,
}
}
/// Set the maximum number of records to read from this plan.
///
/// If `None`, all records after filtering are returned.
pub fn with_limit(mut self, limit: Option<usize>) -> Self {
self.limit = limit;
self
}
/// Set whether files must be read in the order of the file groups even
/// when the scan has no output ordering. See [`FileScanConfig::preserve_order`].
pub fn with_preserve_order(mut self, preserve_order: bool) -> Self {
self.preserve_order = preserve_order;
self
}
/// Set the file source for scanning files.
///
/// This method allows you to change the file source implementation (e.g.
/// ParquetSource, CsvSource, etc.) after the builder has been created.
pub fn with_source(mut self, file_source: Arc<dyn FileSource>) -> Self {
self.file_source = file_source;
self
}
/// Return the table schema
pub fn table_schema(&self) -> &SchemaRef {
self.file_source.table_schema().table_schema()
}
/// Set the columns on which to project the data. Indexes that are higher than the
/// number of columns of `file_schema` refer to `table_partition_cols`.
///
/// # Deprecated
/// Use [`Self::with_projection_indices`] instead. This method will be removed in a future release.
#[deprecated(since = "51.0.0", note = "Use with_projection_indices instead")]
pub fn with_projection(self, indices: Option<Vec<usize>>) -> Self {
match self.clone().with_projection_indices(indices) {
Ok(builder) => builder,
Err(e) => {
warn!(
"Failed to push down projection in FileScanConfigBuilder::with_projection: {e}"
);
self
}
}
}
/// Set the columns on which to project the data using column indices.
///
/// This method attempts to push down the projection to the underlying file
/// source if supported. If the file source does not support projection
/// pushdown, an error is returned.
///
/// Indexes that are higher than the number of columns of `file_schema`
/// refer to `table_partition_cols`.
pub fn with_projection_indices(
mut self,
indices: Option<Vec<usize>>,
) -> Result<Self> {
let projection_exprs = indices.map(|indices| {
ProjectionExprs::from_indices(
&indices,
self.file_source.table_schema().table_schema(),
)
});
let Some(projection_exprs) = projection_exprs else {
return Ok(self);
};
let new_source = self
.file_source
.try_pushdown_projection(&projection_exprs)
.map_err(|e| {
internal_datafusion_err!(
"Failed to push down projection in FileScanConfigBuilder::build: {e}"
)
})?;
if let Some(new_source) = new_source {
self.file_source = new_source;
} else {
internal_err!(
"FileSource {} does not support projection pushdown",
self.file_source.file_type()
)?;
}
Ok(self)
}
/// Set the table constraints
pub fn with_constraints(mut self, constraints: Constraints) -> Self {
self.constraints = Some(constraints);
self
}
/// Set the statistics of the files, including partition
/// columns. Defaults to [`Statistics::new_unknown`].
///
/// These statistics are for the entire table (file schema + partition
/// columns) before any projection or filtering is applied. Projections are
/// applied when statistics are retrieved, and if a filter is present,
/// [`FileScanConfig::statistics`] will mark the statistics as inexact
/// (counts are not adjusted).
///
/// Projections and filters may be applied by the file source, either by
/// [`Self::with_projection_indices`] or a preexisting
/// [`FileSource::projection`] or [`FileSource::filter`].
pub fn with_statistics(mut self, statistics: Statistics) -> Self {
self.statistics = Some(statistics);
self
}
/// Set the list of files to be processed, grouped into partitions.
///
/// Each file must have a schema of `file_schema` or a subset. If
/// a particular file has a subset, the missing columns are
/// padded with NULLs.
///
/// DataFusion may attempt to read each partition of files
/// concurrently, however files *within* a partition will be read
/// sequentially, one after the next.
pub fn with_file_groups(mut self, file_groups: Vec<FileGroup>) -> Self {
self.file_groups = file_groups;
self
}
/// Add a new file group
///
/// See [`Self::with_file_groups`] for more information
pub fn with_file_group(mut self, file_group: FileGroup) -> Self {
self.file_groups.push(file_group);
self
}
/// Add a file as a single group
///
/// See [`Self::with_file_groups`] for more information.
pub fn with_file(self, partitioned_file: PartitionedFile) -> Self {
self.with_file_group(FileGroup::new(vec![partitioned_file]))
}
/// Set the output ordering of the files
///
/// The expressions are in terms of the entire table schema (file schema +
/// partition columns), before any projection or filtering from the file
/// scan is applied.
///
/// This is used for optimization purposes, e.g. to determine if a file scan
/// can satisfy an `ORDER BY` without an additional sort.
pub fn with_output_ordering(mut self, output_ordering: Vec<LexOrdering>) -> Self {
self.output_ordering = output_ordering;
self
}
/// Set declared physical output partitioning for this scan.
pub fn with_output_partitioning(
mut self,
output_partitioning: Option<Partitioning>,
) -> Self {
self.output_partitioning = output_partitioning;
self
}
/// Set the file compression type
pub fn with_file_compression_type(
mut self,
file_compression_type: FileCompressionType,
) -> Self {
self.file_compression_type = Some(file_compression_type);
self
}
/// Set the batch_size property
pub fn with_batch_size(mut self, batch_size: Option<usize>) -> Self {
self.batch_size = batch_size;
self
}
/// Register an expression adapter used to adapt filters and projections that are pushed down into the scan
/// from the logical schema to the physical schema of the file.
/// This can include things like:
/// - Column ordering changes
/// - Handling of missing columns
/// - Rewriting expression to use pre-computed values or file format specific optimizations
pub fn with_expr_adapter(
mut self,
expr_adapter: Option<Arc<dyn PhysicalExprAdapterFactory>>,
) -> Self {
self.expr_adapter_factory = expr_adapter;
self
}
/// Build the final [`FileScanConfig`] with all the configured settings.
///
/// This method takes ownership of the builder and returns the constructed `FileScanConfig`.
/// Any unset optional fields will use their default values.
///
/// # Errors
/// Returns an error if projection pushdown fails or if schema operations fail.
pub fn build(self) -> FileScanConfig {
let Self {
object_store_url,
file_source,
limit,
preserve_order,
constraints,
file_groups,
statistics,
output_ordering,
output_partitioning,
file_compression_type,
batch_size,
expr_adapter_factory: expr_adapter,
} = self;
let constraints = constraints.unwrap_or_default();
let mut statistics = statistics.unwrap_or_else(|| {
Statistics::new_unknown(file_source.table_schema().table_schema())
});
add_key_distinct_counts(&constraints, &mut statistics);
let file_compression_type =
file_compression_type.unwrap_or(FileCompressionType::UNCOMPRESSED);
// If there is an output ordering, we should preserve it.
let preserve_order = preserve_order || !output_ordering.is_empty();
FileScanConfig {
object_store_url,
file_source,
limit,
preserve_order,
constraints,
file_groups,
output_ordering,
file_compression_type,
batch_size,
expr_adapter_factory: expr_adapter,
statistics,
output_partitioning,
}
}
}
/// Records that a key column holds one distinct value per row, which no file format
/// stores. Single-column keys only: a composite key says nothing about its columns.
fn add_key_distinct_counts(constraints: &Constraints, statistics: &mut Statistics) {
let num_rows = statistics.num_rows;
for constraint in constraints.iter() {
let (Constraint::PrimaryKey(indices) | Constraint::Unique(indices)) = constraint;
let [index] = indices[..] else {
continue;
};
let Some(column) = statistics.column_statistics.get_mut(index) else {
continue;
};
if column.distinct_count != Precision::Absent {
continue;
}
// A NULL is not a distinct value. A primary key has none; a unique column may
// repeat them, so an unknown count leaves the result inexact.
let nulls = match (constraint, column.null_count) {
(Constraint::PrimaryKey(_), Precision::Absent) => Precision::Exact(0),
(_, Precision::Absent) => Precision::Inexact(0),
(_, nulls) => nulls,
};
column.distinct_count = num_rows.sub(&nulls);
}
}
impl From<FileScanConfig> for FileScanConfigBuilder {
fn from(config: FileScanConfig) -> Self {
Self {
object_store_url: config.object_store_url,
file_source: Arc::<dyn FileSource>::clone(&config.file_source),
file_groups: config.file_groups,
statistics: Some(config.statistics),
output_ordering: config.output_ordering,
output_partitioning: config.output_partitioning,
file_compression_type: Some(config.file_compression_type),
limit: config.limit,
preserve_order: config.preserve_order,
constraints: Some(config.constraints),
batch_size: config.batch_size,
expr_adapter_factory: config.expr_adapter_factory,
}
}
}
/// Builds `Partitioning::Hash` over `partition_cols` (resolved to their indices in
/// `schema`). Returns `None` when there are no partition columns.
#[deprecated(
since = "56.0.0",
note = "Use `output_range_partitioning_from_split_points` instead"
)]
pub fn output_partitioning_from_partition_fields(
schema: &Schema,
partition_cols: &Fields,
partition_count: usize,
) -> Option<Partitioning> {
let exprs = partition_column_exprs(schema, partition_cols)?;
Some(Partitioning::Hash(exprs, partition_count))
}
/// Builds the [`Partitioning::Range`] for file groups from
/// [`FileGroup::group_by_partition_values_with_split_points`], each column sorted with
/// `SortOptions::default()` to match how the groups were cut. Returns `None` when there
/// are no partition columns or one is missing from `schema`, and errors when
/// `split_points` are not strictly ordered. Split point values must have the partition
/// column types in `schema`.
pub fn output_range_partitioning_from_split_points(
schema: &Schema,
partition_cols: &Fields,
split_points: Vec<SplitPoint>,
) -> Result<Option<Partitioning>> {
let Some(exprs) = partition_column_exprs(schema, partition_cols) else {
return Ok(None);
};
let sort_exprs = exprs
.into_iter()
.map(PhysicalSortExpr::new_default)
.collect::<Vec<_>>();
let Some(ordering) = LexOrdering::new(sort_exprs) else {
return Ok(None);
};
let range = RangePartitioning::try_new(ordering, split_points)?;
Ok(Some(Partitioning::Range(range)))
}
/// Resolves `partition_cols` to `Column` expressions in `schema`, or `None` when there
/// are no partition columns or one is missing.
fn partition_column_exprs(
schema: &Schema,
partition_cols: &Fields,
) -> Option<Vec<Arc<dyn PhysicalExpr>>> {
if partition_cols.is_empty() {
return None;
}
let mut exprs: Vec<Arc<dyn PhysicalExpr>> = Vec::with_capacity(partition_cols.len());
for partition_col in partition_cols {
let name = partition_col.name();
let idx = schema
.fields()
.iter()
.position(|field| field.name() == name)?;
exprs.push(Arc::new(Column::new(name, idx)));
}
Some(exprs)
}
fn project_output_partitioning(
partitioning: &Partitioning,
mapping: &ProjectionMapping,
input_schema: &SchemaRef,
partition_count: usize,
) -> Partitioning {
let input_eq_properties = EquivalenceProperties::new(Arc::clone(input_schema));
match partitioning {
Partitioning::Hash(exprs, _) => {
let projected_exprs = input_eq_properties
.project_expressions(exprs, mapping)
.collect::<Option<Vec<_>>>();
projected_exprs
.map(|exprs| Partitioning::Hash(exprs, partition_count))
.unwrap_or_else(|| Partitioning::UnknownPartitioning(partition_count))
}
Partitioning::Range(_)
| Partitioning::RoundRobinBatch(_)
| Partitioning::UnknownPartitioning(_) => {
partitioning.project(mapping, &input_eq_properties)
}
}
}
/// Returns `true` if merging `outer` into `inner` would duplicate a volatile or
/// non-trivial expression that CSE deduplicated; the caller should then decline
/// the merge.
///
/// Merging substitutes each `inner` expression into every `outer` reference to
/// it. Since the logical optimizer extracts a repeated expression into a single
/// `inner` entry referenced by column, re-inlining it at more than one
/// reference site undoes that deduplication. An `inner` expression referenced
/// more than once is therefore blocked when it is either:
///
/// - **volatile** (e.g. `random()`) — evaluating it independently at each site
/// makes references that should share one "locked-in" value diverge (the
/// correctness guard the physical `ProjectionPushdown` and `FilterPushdown`
/// rules also apply via
/// `datafusion_physical_expr_common::physical_expr::is_volatile`); or
/// - **not cheap to recompute** — its placement is not push-to-leaves
/// (`KeepInPlace`: arithmetic, casts, most scalar functions). Leaf-pushable
/// expressions (columns, `get_field`, `input_file_name`) still merge. This
/// matches `try_collapse_projection_chain`.
///
/// References are counted with multiplicity, so `r + r` counts as two; an
/// expression referenced exactly once has nothing to duplicate.
fn would_duplicate_costly_exprs(
inner: &ProjectionExprs,
outer: &ProjectionExprs,
) -> bool {
use datafusion_common::tree_node::{TreeNode, TreeNodeRecursion};
let inner_exprs = inner.as_ref();
let mut ref_counts = vec![0usize; inner_exprs.len()];
for proj_expr in outer.as_ref() {
proj_expr
.expr
.apply(|e| {
if let Some(col) = e.as_ref().downcast_ref::<Column>()
&& let Some(count) = ref_counts.get_mut(col.index())
{
*count += 1;
}
Ok(TreeNodeRecursion::Continue)
})
.expect("infallible closure should not fail");
}
ref_counts.iter().enumerate().any(|(idx, &count)| {
let expr = &inner_exprs[idx].expr;
count > 1 && (is_volatile(expr) || !expr.placement().should_push_to_leaves())
})
}
impl DataSource for FileScanConfig {
fn open(
&self,
partition: usize,
context: Arc<TaskContext>,
) -> Result<SendableRecordBatchStream> {
self.open_with_args(OpenArgs::new(partition, context))
}
fn open_with_args(&self, args: OpenArgs) -> Result<SendableRecordBatchStream> {
let OpenArgs {
partition,
context,
sibling_state,
} = args;
let object_store = context.runtime_env().object_store(&self.object_store_url)?;
let batch_size = self
.batch_size
.unwrap_or_else(|| context.session_config().batch_size());
let source = self.file_source.with_batch_size(batch_size);
let source = source
.with_memory_pool(context.memory_pool())
.unwrap_or(source);
let morselizer = source.create_morselizer(object_store, self, partition)?;
// Extract the shared work source from the sibling state if it exists.
// This allows multiple sibling streams to steal work from a single
// shared queue of unopened files.
let shared_work_source = sibling_state
.as_ref()
.and_then(|state| state.downcast_ref::<SharedWorkSource>())
.cloned();
let stream = FileStreamBuilder::new(self)
.with_partition(partition)
.with_shared_work_source(shared_work_source)
.with_morselizer(morselizer)
.with_metrics(source.metrics())
.build()?;
Ok(Box::pin(cooperative(stream)))
}
fn fmt_as(&self, t: DisplayFormatType, f: &mut Formatter) -> FmtResult {
match t {
DisplayFormatType::Default | DisplayFormatType::Verbose => {
let schema = self.projected_schema().map_err(|_| std::fmt::Error {})?;
let orderings =
sort_pushdown::get_projected_output_ordering(self, &schema);
write!(f, "file_groups=")?;
FileGroupsDisplay(&self.file_groups).fmt_as(t, f)?;
if !schema.fields().is_empty() {
if let Some(projection) = self.file_source.projection() {
// This matches what ProjectionExec does.
// TODO: can we put this into ProjectionExprs so that it's shared code?
let expr: Vec<String> = projection
.as_ref()
.iter()
.map(|proj_expr| {
if let Some(column) =
proj_expr.expr.downcast_ref::<Column>()
{
if column.name() == proj_expr.alias {
column.name().to_string()
} else {
format!(
"{} as {}",
proj_expr.expr, proj_expr.alias
)
}
} else {
format!("{} as {}", proj_expr.expr, proj_expr.alias)
}
})
.collect();
write!(f, ", projection=[{}]", expr.join(", "))?;
} else {
write!(f, ", projection={}", ProjectSchemaDisplay(&schema))?;
}
}
if let Some(limit) = self.limit {
write!(f, ", limit={limit}")?;
}
display_orderings(f, &orderings)?;
if self.output_partitioning.is_some() {
write!(f, ", output_partitioning={}", self.output_partitioning())?;
}
if !self.constraints.is_empty() {
write!(f, ", {}", self.constraints)?;
}
self.fmt_file_source(t, f)
}
DisplayFormatType::TreeRender => {
writeln!(f, "format={}", self.file_source.file_type())?;
self.file_source.fmt_extra(t, f)?;
let num_files = self.file_groups.iter().map(|fg| fg.len()).sum::<usize>();
writeln!(f, "files={num_files}")?;
Ok(())
}
}
}
/// If supported by the underlying [`FileSource`], redistribute files across partitions according to their size.
fn repartitioned(
&self,
target_partitions: usize,
repartition_file_min_size: usize,
output_ordering: Option<LexOrdering>,
) -> Result<Option<Arc<dyn DataSource>>> {
// When file groups define output partitioning, repartitioning files
// would invalidate the partition-to-file-group mapping.
if self.output_partitioning.is_some() {
return Ok(None);
}
// An ordered scan can use the size-balanced split of an unordered
// scan when every new group is still sorted. That split keeps the
// files in group order and only moves the group boundaries, so groups
// like `[f1, f2], [f3, f4]` with a large `f1` become
// `[f1 head], [f1 tail, f2], ...`.
if output_ordering.is_some()
&& let Some(balanced) = self.file_source.repartitioned(
target_partitions,
repartition_file_min_size,
None,
self,
)?
&& balanced.validated_output_ordering() == self.validated_output_ordering()
{
return Ok(Some(Arc::new(balanced)));
}
let source = self.file_source.repartitioned(
target_partitions,
repartition_file_min_size,
output_ordering,
self,
)?;
Ok(source.map(|s| Arc::new(s) as _))
}
/// Returns the output partitioning for this file scan.
///
/// When `output_partitioning` is set, this returns the declared partitioning
/// after applying scan projection, allowing the optimizer to skip hash
/// repartitioning for aggregates and joins on the partitioning columns.
///
/// If projection or partition count validation fails, this returns
/// `UnknownPartitioning`.
///
/// Tradeoffs
/// - Benefit: Eliminates `RepartitionExec` and `SortExec` for queries whose
/// required distribution is satisfied by the scan's output partitioning.
/// - Cost: Files are grouped by partition values rather than split by byte
/// ranges, which may reduce I/O parallelism when partition sizes are uneven.
/// For simple aggregations without `ORDER BY`, this cost may outweigh the benefit.
///
/// Follow-up Work
/// - Idea: Could allow byte-range splitting within partition-aware groups,
/// preserving I/O parallelism while maintaining partition semantics.
fn output_partitioning(&self) -> Partitioning {
let Some(output_partitioning) = self.output_partitioning.clone() else {
return Partitioning::UnknownPartitioning(self.file_groups.len());
};
if output_partitioning.partition_count() != self.file_groups.len() {
warn!(
"Declared output partitioning has {} partitions, but file scan has {} file groups. Falling back to UnknownPartitioning.",
output_partitioning.partition_count(),
self.file_groups.len()
);
return Partitioning::UnknownPartitioning(self.file_groups.len());
}
if let Some(projection) = self.file_source.projection() {
let schema = self.file_source.table_schema().table_schema();
return match projection.projection_mapping(schema) {
Ok(mapping) => project_output_partitioning(
&output_partitioning,
&mapping,
schema,
self.file_groups.len(),
),
Err(e) => {
debug!(
"Could not project output partitioning, falling back to UnknownPartitioning: {e}"
);
Partitioning::UnknownPartitioning(self.file_groups.len())
}
};
}
output_partitioning
}
/// Computes the effective equivalence properties of this file scan, taking
/// into account the file schema, any projections or filters applied by the
/// file source, and the output ordering.
fn eq_properties(&self) -> EquivalenceProperties {
let schema = self.file_source.table_schema().table_schema();
let mut eq_properties = EquivalenceProperties::new_with_orderings(
Arc::clone(schema),
self.validated_output_ordering(),
)
.with_constraints(self.constraints.clone());
// Only a filter that the source applies exactly gives valid equivalences.
// A filter that is used only for pruning lets non-matching rows through.
if let Some(filter) = self.file_source.exact_filter() {
// We need to remap column indexes to match the projected schema since that's what the equivalence properties deal with.
// Note that this will *ignore* any non-projected columns: these don't factor into ordering / equivalence.
match Self::add_filter_equivalence_info(&filter, &mut eq_properties, schema) {
Ok(()) => {}
Err(e) => {
warn!("Failed to add filter equivalence info: {e}");
#[cfg(debug_assertions)]
panic!("Failed to add filter equivalence info: {e}");
}
}
}
if let Some(projection) = self.file_source.projection() {
match (
projection.project_schema(schema),
projection.projection_mapping(schema),
) {
(Ok(output_schema), Ok(mapping)) => {
eq_properties =
eq_properties.project(&mapping, Arc::new(output_schema));
}
(Err(e), _) | (_, Err(e)) => {
warn!("Failed to project equivalence properties: {e}");
#[cfg(debug_assertions)]
panic!("Failed to project equivalence properties: {e}");
}
}
}
eq_properties
}
fn scheduling_type(&self) -> SchedulingType {
SchedulingType::Cooperative
}
fn partition_statistics(&self, partition: Option<usize>) -> Result<Arc<Statistics>> {
if let Some(partition) = partition {
// Get statistics for a specific partition
// Note: FileGroup statistics include partition columns (computed from partition_values)
if let Some(file_group) = self.file_groups.get(partition)
&& let Some(stat) = file_group.file_statistics(None)