@@ -34,7 +34,8 @@ use datafusion_expr::dml::InsertOp;
3434use datafusion_expr::execution_props::ExecutionProps;
3535use datafusion_expr::physical_planning_context::PhysicalPlanningContext;
3636use datafusion_expr::{
37- Expr, Partitioning as LogicalPartitioning, TableProviderFilterPushDown, TableType,
37+ Expr, Operator, Partitioning as LogicalPartitioning, TableProviderFilterPushDown,
38+ TableType,
3839};
3940use datafusion_physical_expr::{create_lex_ordering, create_physical_partitioning};
4041use datafusion_physical_expr_adapter::PhysicalExprAdapterFactory;
@@ -627,29 +628,81 @@ impl TableProvider for ListingTable {
627628}
628629
629630impl ListingTable {
631+ fn filter_supports_dictionary_column(expr: &Expr, column_name: &str) -> bool {
632+ if !expr
633+ .column_refs()
634+ .iter()
635+ .any(|column| column.name == column_name)
636+ {
637+ return true;
638+ }
639+
640+ match expr {
641+ Expr::BinaryExpr(binary) => match binary.op {
642+ Operator::And | Operator::Or => {
643+ Self::filter_supports_dictionary_column(
644+ binary.left.as_ref(),
645+ column_name,
646+ ) && Self::filter_supports_dictionary_column(
647+ binary.right.as_ref(),
648+ column_name,
649+ )
650+ }
651+ Operator::Eq => {
652+ let is_column = |expr: &Expr| matches!(expr, Expr::Column(column) if column.name == column_name);
653+ (is_column(binary.left.as_ref())
654+ && matches!(binary.right.as_ref(), Expr::Literal(_, _)))
655+ || (is_column(binary.right.as_ref())
656+ && matches!(binary.left.as_ref(), Expr::Literal(_, _)))
657+ }
658+ _ => false,
659+ },
660+ Expr::InList(in_list) => {
661+ !in_list.negated
662+ && !in_list.list.is_empty()
663+ && matches!(in_list.expr.as_ref(), Expr::Column(column) if column.name == column_name)
664+ && in_list
665+ .list
666+ .iter()
667+ .all(|expr| matches!(expr, Expr::Literal(_, _)))
668+ }
669+ Expr::Like(like) => {
670+ like.negated
671+ && !like.case_insensitive
672+ && like.escape_char.is_none()
673+ && matches!(like.expr.as_ref(), Expr::Column(column) if column.name == column_name)
674+ && matches!(like.pattern.as_ref(), Expr::Literal(_, _))
675+ }
676+ _ => false,
677+ }
678+ }
679+
630680 fn physical_statistics_required(
631681 &self,
632682 state: &dyn Session,
633683 projection: Option<&[usize]>,
684+ excluded_columns: &HashSet<&str>,
634685 ) -> bool {
635686 if !state
636687 .config_options()
637688 .optimizer
638- .enable_dictionary_aggregation
689+ .enable_dictionary_preservation
639690 {
640691 return false;
641692 }
642693
643694 let file_fields = self.file_schema.fields();
644695 match projection {
645696 Some(projection) => projection.iter().any(|index| {
646- file_fields
647- .get(*index)
648- .is_some_and(|field| field.data_type() == &DataType::Utf8View)
697+ file_fields.get(*index).is_some_and(|field| {
698+ field.data_type() == &DataType::Utf8View
699+ && !excluded_columns.contains(field.name().as_str())
700+ })
701+ }),
702+ None => file_fields.iter().any(|field| {
703+ field.data_type() == &DataType::Utf8View
704+ && !excluded_columns.contains(field.name().as_str())
649705 }),
650- None => file_fields
651- .iter()
652- .any(|field| field.data_type() == &DataType::Utf8View),
653706 }
654707 }
655708
@@ -693,12 +746,31 @@ impl ListingTable {
693746 filters.iter().cloned().partition(|filter| {
694747 can_be_evaluated_for_partition_pruning(&table_partition_col_names, filter)
695748 });
696- // The dictionary aggregation rule rejects scans with data filters or
697- // limits, so avoid collecting its optional file metadata for plans it
698- // cannot rewrite. Partition-only filters remain eligible.
699- let physical_statistics_required = filters.is_empty()
700- && limit.is_none()
701- && self.physical_statistics_required(state, projection.as_deref());
749+ // A data filter can remain transparent to a dictionary grouping column
750+ // when it does not reference that column. Case-sensitive literal
751+ // `NOT LIKE` filters can keep the dictionary through grouping, while
752+ // equality and `IN` filters can benefit when the string is discarded
753+ // immediately afterwards.
754+ // Exclude only columns whose filter use cannot support either rewrite.
755+ let filtered_columns: HashSet<_> = filters
756+ .iter()
757+ .flat_map(Expr::column_refs)
758+ .map(|column| column.name.as_str())
759+ .collect();
760+ let excluded_columns: HashSet<_> = filtered_columns
761+ .into_iter()
762+ .filter(|column_name| {
763+ filters.iter().any(|filter| {
764+ !Self::filter_supports_dictionary_column(filter, column_name)
765+ })
766+ })
767+ .collect();
768+ let physical_statistics_required = limit.is_none()
769+ && self.physical_statistics_required(
770+ state,
771+ projection.as_deref(),
772+ &excluded_columns,
773+ );
702774
703775 let declared_output_partitioning = self.options.output_partitioning.as_ref();
704776
@@ -979,7 +1051,8 @@ impl ListingTable {
9791051 filters: &'a [Expr],
9801052 limit: Option<usize>,
9811053 ) -> datafusion_common::Result<ListFilesResult> {
982- let physical_statistics_required = self.physical_statistics_required(ctx, None);
1054+ let physical_statistics_required =
1055+ self.physical_statistics_required(ctx, None, &HashSet::new());
9831056 self.list_files_for_scan_inner(
9841057 ctx,
9851058 filters,
@@ -1041,15 +1114,7 @@ impl ListingTable {
10411114 .await?;
10421115 let meta_fetch_concurrency =
10431116 ctx.config_options().execution.meta_fetch_concurrency.get();
1044- // Table paths can overlap, for example when one path is a directory and
1045- // another names a file inside it. A ListingTable uses one object store,
1046- // so the object path uniquely identifies a file within this scan.
1047- let mut seen_files = HashSet::new();
1048- let file_list = stream::iter(file_list)
1049- .flatten_unordered(meta_fetch_concurrency)
1050- .try_filter(move |file| {
1051- future::ready(seen_files.insert(file.object_meta.location.clone()))
1052- });
1117+ let file_list = stream::iter(file_list).flatten_unordered(meta_fetch_concurrency);
10531118 // collect the statistics and ordering if required by the config
10541119 let files = file_list
10551120 .map(|part_file| async {
@@ -1517,6 +1582,49 @@ mod tests {
15171582 ListingTable::try_new(config).unwrap()
15181583 }
15191584
1585+ #[test]
1586+ fn dictionary_filter_support_matches_q16_candidates() {
1587+ use datafusion_expr::{col, lit};
1588+
1589+ let filter = col("p_brand")
1590+ .not_eq(lit("Brand#45"))
1591+ .and(col("p_type").not_like(lit("MEDIUM POLISHED%")));
1592+ assert!(ListingTable::filter_supports_dictionary_column(
1593+ &filter, "p_type"
1594+ ));
1595+ assert!(!ListingTable::filter_supports_dictionary_column(
1596+ &filter, "p_brand"
1597+ ));
1598+ assert!(!ListingTable::filter_supports_dictionary_column(
1599+ &col("p_type").like(col("pattern")),
1600+ "p_type"
1601+ ));
1602+ assert!(!ListingTable::filter_supports_dictionary_column(
1603+ &col("p_type").like(lit("MEDIUM POLISHED%")),
1604+ "p_type"
1605+ ));
1606+ assert!(!ListingTable::filter_supports_dictionary_column(
1607+ &col("p_type").not_ilike(lit("MEDIUM POLISHED%")),
1608+ "p_type"
1609+ ));
1610+
1611+ let q19_filter = col("l_shipinstruct")
1612+ .eq(lit("DELIVER IN PERSON"))
1613+ .and(col("l_shipmode").in_list(vec![lit("AIR"), lit("AIR REG")], false));
1614+ assert!(ListingTable::filter_supports_dictionary_column(
1615+ &q19_filter,
1616+ "l_shipinstruct"
1617+ ));
1618+ assert!(ListingTable::filter_supports_dictionary_column(
1619+ &q19_filter,
1620+ "l_shipmode"
1621+ ));
1622+ assert!(!ListingTable::filter_supports_dictionary_column(
1623+ &col("l_shipmode").in_list(vec![lit("AIR")], true),
1624+ "l_shipmode"
1625+ ));
1626+ }
1627+
15201628 /// Helper to create a PhysicalSortExpr
15211629 fn sort_expr(
15221630 name: &str,
0 commit comments