22
33#include " duckdb/common/uhugeint.hpp"
44#include " duckdb/execution/expression_executor.hpp"
5- #include " duckdb/optimizer/filter_combiner.hpp"
65#include " duckdb/planner/expression/bound_columnref_expression.hpp"
76#include " duckdb/planner/expression/bound_constant_expression.hpp"
87#include " duckdb/planner/expression/bound_reference_expression.hpp"
1211
1312namespace duckdb {
1413
15- static unordered_map<column_t , string> GetKnownColumnValues (string &filename,
16- unordered_map<string, column_t > &column_map,
17- bool filename_col,
18- bool hive_partition_cols) {
19- unordered_map<column_t , string> result;
14+ struct PartitioningColumnValue {
15+ explicit PartitioningColumnValue (string value_p) : value(std::move(value_p)) {
16+ }
17+ PartitioningColumnValue (string key_p, string value_p) : key(std::move(key_p)), value(std::move(value_p)) {
18+ }
19+
20+ string key;
21+ string value;
22+ };
23+
24+ static unordered_map<column_t , PartitioningColumnValue>
25+ GetKnownColumnValues (const string &filename, const HivePartitioningFilterInfo &filter_info) {
26+ unordered_map<column_t , PartitioningColumnValue> result;
2027
21- if (filename_col) {
28+ auto &column_map = filter_info.column_map ;
29+ if (filter_info.filename_enabled ) {
2230 auto lookup_column_id = column_map.find (" filename" );
2331 if (lookup_column_id != column_map.end ()) {
24- result[ lookup_column_id->second ] = filename;
32+ result. insert ( make_pair ( lookup_column_id->second , PartitioningColumnValue ( filename))) ;
2533 }
2634 }
2735
28- if (hive_partition_cols ) {
36+ if (filter_info. hive_enabled ) {
2937 auto partitions = HivePartitioning::Parse (filename);
3038 for (auto &partition : partitions) {
3139 auto lookup_column_id = column_map.find (partition.first );
3240 if (lookup_column_id != column_map.end ()) {
33- result[lookup_column_id->second ] = partition.second ;
41+ result.insert (
42+ make_pair (lookup_column_id->second , PartitioningColumnValue (partition.first , partition.second )));
3443 }
3544 }
3645 }
@@ -39,8 +48,9 @@ static unordered_map<column_t, string> GetKnownColumnValues(string &filename,
3948}
4049
4150// Takes an expression and converts a list of known column_refs to constants
42- static void ConvertKnownColRefToConstants (unique_ptr<Expression> &expr,
43- unordered_map<column_t , string> &known_column_values, idx_t table_index) {
51+ static void ConvertKnownColRefToConstants (ClientContext &context, unique_ptr<Expression> &expr,
52+ const unordered_map<column_t , PartitioningColumnValue> &known_column_values,
53+ idx_t table_index) {
4454 if (expr->type == ExpressionType::BOUND_COLUMN_REF ) {
4555 auto &bound_colref = expr->Cast <BoundColumnRefExpression>();
4656
@@ -51,11 +61,21 @@ static void ConvertKnownColRefToConstants(unique_ptr<Expression> &expr,
5161
5262 auto lookup = known_column_values.find (bound_colref.binding .column_index );
5363 if (lookup != known_column_values.end ()) {
54- expr = make_uniq<BoundConstantExpression>(Value (lookup->second ).DefaultCastAs (bound_colref.return_type ));
64+ auto &partition_val = lookup->second ;
65+ Value result_val;
66+ if (partition_val.key .empty ()) {
67+ // filename column - use directly
68+ result_val = Value (partition_val.value );
69+ } else {
70+ // hive partitioning column - cast the value to the target type
71+ result_val = HivePartitioning::GetValue (context, partition_val.key , partition_val.value ,
72+ bound_colref.return_type );
73+ }
74+ expr = make_uniq<BoundConstantExpression>(std::move (result_val));
5575 }
5676 } else {
5777 ExpressionIterator::EnumerateChildren (*expr, [&](unique_ptr<Expression> &child) {
58- ConvertKnownColRefToConstants (child, known_column_values, table_index);
78+ ConvertKnownColRefToConstants (context, child, known_column_values, table_index);
5979 });
6080 }
6181}
@@ -69,7 +89,7 @@ std::map<string, string> HivePartitioning::Parse(const string &filename) {
6989 idx_t equality_sign = 0 ;
7090 bool candidate_partition = true ;
7191 std::map<string, string> result;
72- for (idx_t c = 0 ; c < filename.size (); c++) {
92+ for (idx_t c = 0 ; c < filename.size (); c++) {
7393 if (filename[c] == ' ?' || filename[c] == ' \n ' ) {
7494 // get parameter or newline - not a partition
7595 candidate_partition = false ;
@@ -95,34 +115,56 @@ std::map<string, string> HivePartitioning::Parse(const string &filename) {
95115 return result;
96116}
97117
118+ Value HivePartitioning::GetValue (ClientContext &context, const string &key, const string &str_val,
119+ const LogicalType &type) {
120+ // Handle nulls
121+ if (StringUtil::CIEquals (str_val, " NULL" )) {
122+ return Value (type);
123+ }
124+ if (type.id () == LogicalTypeId::VARCHAR ) {
125+ // for string values we can directly return the type
126+ return Value (str_val);
127+ }
128+ if (str_val.empty ()) {
129+ // empty strings are NULL for non-string types
130+ return Value (type);
131+ }
132+
133+ // cast to the target type
134+ Value value (str_val);
135+ if (!value.TryCastAs (context, type)) {
136+ throw InvalidInputException (" Unable to cast '%s' (from hive partition column '%s') to: '%s'" , value.ToString (),
137+ StringUtil::Upper (key), type.ToString ());
138+ }
139+ return value;
140+ }
141+
98142// TODO: this can still be improved by removing the parts of filter expressions that are true for all remaining files.
99143// currently, only expressions that cannot be evaluated during pushdown are removed.
100144void HivePartitioning::ApplyFiltersToFileList (ClientContext &context, vector<string> &files,
101145 vector<unique_ptr<Expression>> &filters,
102- unordered_map<string, column_t > &column_map, MultiFilePushdownInfo &info ,
103- bool hive_enabled, bool filename_enabled ) {
146+ const HivePartitioningFilterInfo &filter_info ,
147+ MultiFilePushdownInfo &info ) {
104148
105149 vector<string> pruned_files;
106150 vector<bool > have_preserved_filter (filters.size (), false );
107151 vector<unique_ptr<Expression>> pruned_filters;
108152 unordered_set<idx_t > filters_applied_to_files;
109153 auto table_index = info.table_index ;
110154
111- if ((!filename_enabled && !hive_enabled) || filters.empty ()) {
155+ if ((!filter_info. filename_enabled && !filter_info. hive_enabled ) || filters.empty ()) {
112156 return ;
113157 }
114158
115159 for (idx_t i = 0 ; i < files.size (); i++) {
116160 auto &file = files[i];
117161 bool should_prune_file = false ;
118- auto known_values = GetKnownColumnValues (file, column_map, filename_enabled, hive_enabled);
119-
120- FilterCombiner combiner (context);
162+ auto known_values = GetKnownColumnValues (file, filter_info);
121163
122164 for (idx_t j = 0 ; j < filters.size (); j++) {
123165 auto &filter = filters[j];
124166 unique_ptr<Expression> filter_copy = filter->Copy ();
125- ConvertKnownColRefToConstants (filter_copy, known_values, table_index);
167+ ConvertKnownColRefToConstants (context, filter_copy, known_values, table_index);
126168 // Evaluate the filter, if it can be evaluated here, we can not prune this filter
127169 Value result_value;
128170
@@ -133,7 +175,7 @@ void HivePartitioning::ApplyFiltersToFileList(ClientContext &context, vector<str
133175 pruned_filters.emplace_back (filter->Copy ());
134176 have_preserved_filter[j] = true ;
135177 }
136- } else if (!result_value.GetValue <bool >()) {
178+ } else if (result_value. IsNull () || !result_value.GetValue <bool >()) {
137179 // filter evaluates to false
138180 should_prune_file = true ;
139181 // convert the filter to a table filter.
0 commit comments