Skip to content

Commit 4622cca

Browse files
committed
Fix duckdb#12921: correctly deal with filters on NULL values in hive partition columns
1 parent 0b749b6 commit 4622cca

7 files changed

Lines changed: 111 additions & 52 deletions

File tree

Binary file not shown.

‎src/common/hive_partitioning.cpp‎

Lines changed: 65 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,6 @@
22

33
#include "duckdb/common/uhugeint.hpp"
44
#include "duckdb/execution/expression_executor.hpp"
5-
#include "duckdb/optimizer/filter_combiner.hpp"
65
#include "duckdb/planner/expression/bound_columnref_expression.hpp"
76
#include "duckdb/planner/expression/bound_constant_expression.hpp"
87
#include "duckdb/planner/expression/bound_reference_expression.hpp"
@@ -12,25 +11,35 @@
1211

1312
namespace duckdb {
1413

15-
static unordered_map<column_t, string> GetKnownColumnValues(string &filename,
16-
unordered_map<string, column_t> &column_map,
17-
bool filename_col,
18-
bool hive_partition_cols) {
19-
unordered_map<column_t, string> result;
14+
struct PartitioningColumnValue {
15+
explicit PartitioningColumnValue(string value_p) : value(std::move(value_p)) {
16+
}
17+
PartitioningColumnValue(string key_p, string value_p) : key(std::move(key_p)), value(std::move(value_p)) {
18+
}
19+
20+
string key;
21+
string value;
22+
};
23+
24+
static unordered_map<column_t, PartitioningColumnValue>
25+
GetKnownColumnValues(const string &filename, const HivePartitioningFilterInfo &filter_info) {
26+
unordered_map<column_t, PartitioningColumnValue> result;
2027

21-
if (filename_col) {
28+
auto &column_map = filter_info.column_map;
29+
if (filter_info.filename_enabled) {
2230
auto lookup_column_id = column_map.find("filename");
2331
if (lookup_column_id != column_map.end()) {
24-
result[lookup_column_id->second] = filename;
32+
result.insert(make_pair(lookup_column_id->second, PartitioningColumnValue(filename)));
2533
}
2634
}
2735

28-
if (hive_partition_cols) {
36+
if (filter_info.hive_enabled) {
2937
auto partitions = HivePartitioning::Parse(filename);
3038
for (auto &partition : partitions) {
3139
auto lookup_column_id = column_map.find(partition.first);
3240
if (lookup_column_id != column_map.end()) {
33-
result[lookup_column_id->second] = partition.second;
41+
result.insert(
42+
make_pair(lookup_column_id->second, PartitioningColumnValue(partition.first, partition.second)));
3443
}
3544
}
3645
}
@@ -39,8 +48,9 @@ static unordered_map<column_t, string> GetKnownColumnValues(string &filename,
3948
}
4049

4150
// Takes an expression and converts a list of known column_refs to constants
42-
static void ConvertKnownColRefToConstants(unique_ptr<Expression> &expr,
43-
unordered_map<column_t, string> &known_column_values, idx_t table_index) {
51+
static void ConvertKnownColRefToConstants(ClientContext &context, unique_ptr<Expression> &expr,
52+
const unordered_map<column_t, PartitioningColumnValue> &known_column_values,
53+
idx_t table_index) {
4454
if (expr->type == ExpressionType::BOUND_COLUMN_REF) {
4555
auto &bound_colref = expr->Cast<BoundColumnRefExpression>();
4656

@@ -51,11 +61,21 @@ static void ConvertKnownColRefToConstants(unique_ptr<Expression> &expr,
5161

5262
auto lookup = known_column_values.find(bound_colref.binding.column_index);
5363
if (lookup != known_column_values.end()) {
54-
expr = make_uniq<BoundConstantExpression>(Value(lookup->second).DefaultCastAs(bound_colref.return_type));
64+
auto &partition_val = lookup->second;
65+
Value result_val;
66+
if (partition_val.key.empty()) {
67+
// filename column - use directly
68+
result_val = Value(partition_val.value);
69+
} else {
70+
// hive partitioning column - cast the value to the target type
71+
result_val = HivePartitioning::GetValue(context, partition_val.key, partition_val.value,
72+
bound_colref.return_type);
73+
}
74+
expr = make_uniq<BoundConstantExpression>(std::move(result_val));
5575
}
5676
} else {
5777
ExpressionIterator::EnumerateChildren(*expr, [&](unique_ptr<Expression> &child) {
58-
ConvertKnownColRefToConstants(child, known_column_values, table_index);
78+
ConvertKnownColRefToConstants(context, child, known_column_values, table_index);
5979
});
6080
}
6181
}
@@ -69,7 +89,7 @@ std::map<string, string> HivePartitioning::Parse(const string &filename) {
6989
idx_t equality_sign = 0;
7090
bool candidate_partition = true;
7191
std::map<string, string> result;
72-
for(idx_t c = 0; c < filename.size(); c++) {
92+
for (idx_t c = 0; c < filename.size(); c++) {
7393
if (filename[c] == '?' || filename[c] == '\n') {
7494
// get parameter or newline - not a partition
7595
candidate_partition = false;
@@ -95,34 +115,56 @@ std::map<string, string> HivePartitioning::Parse(const string &filename) {
95115
return result;
96116
}
97117

118+
Value HivePartitioning::GetValue(ClientContext &context, const string &key, const string &str_val,
119+
const LogicalType &type) {
120+
// Handle nulls
121+
if (StringUtil::CIEquals(str_val, "NULL")) {
122+
return Value(type);
123+
}
124+
if (type.id() == LogicalTypeId::VARCHAR) {
125+
// for string values we can directly return the type
126+
return Value(str_val);
127+
}
128+
if (str_val.empty()) {
129+
// empty strings are NULL for non-string types
130+
return Value(type);
131+
}
132+
133+
// cast to the target type
134+
Value value(str_val);
135+
if (!value.TryCastAs(context, type)) {
136+
throw InvalidInputException("Unable to cast '%s' (from hive partition column '%s') to: '%s'", value.ToString(),
137+
StringUtil::Upper(key), type.ToString());
138+
}
139+
return value;
140+
}
141+
98142
// TODO: this can still be improved by removing the parts of filter expressions that are true for all remaining files.
99143
// currently, only expressions that cannot be evaluated during pushdown are removed.
100144
void HivePartitioning::ApplyFiltersToFileList(ClientContext &context, vector<string> &files,
101145
vector<unique_ptr<Expression>> &filters,
102-
unordered_map<string, column_t> &column_map, MultiFilePushdownInfo &info,
103-
bool hive_enabled, bool filename_enabled) {
146+
const HivePartitioningFilterInfo &filter_info,
147+
MultiFilePushdownInfo &info) {
104148

105149
vector<string> pruned_files;
106150
vector<bool> have_preserved_filter(filters.size(), false);
107151
vector<unique_ptr<Expression>> pruned_filters;
108152
unordered_set<idx_t> filters_applied_to_files;
109153
auto table_index = info.table_index;
110154

111-
if ((!filename_enabled && !hive_enabled) || filters.empty()) {
155+
if ((!filter_info.filename_enabled && !filter_info.hive_enabled) || filters.empty()) {
112156
return;
113157
}
114158

115159
for (idx_t i = 0; i < files.size(); i++) {
116160
auto &file = files[i];
117161
bool should_prune_file = false;
118-
auto known_values = GetKnownColumnValues(file, column_map, filename_enabled, hive_enabled);
119-
120-
FilterCombiner combiner(context);
162+
auto known_values = GetKnownColumnValues(file, filter_info);
121163

122164
for (idx_t j = 0; j < filters.size(); j++) {
123165
auto &filter = filters[j];
124166
unique_ptr<Expression> filter_copy = filter->Copy();
125-
ConvertKnownColRefToConstants(filter_copy, known_values, table_index);
167+
ConvertKnownColRefToConstants(context, filter_copy, known_values, table_index);
126168
// Evaluate the filter, if it can be evaluated here, we can not prune this filter
127169
Value result_value;
128170

@@ -133,7 +175,7 @@ void HivePartitioning::ApplyFiltersToFileList(ClientContext &context, vector<str
133175
pruned_filters.emplace_back(filter->Copy());
134176
have_preserved_filter[j] = true;
135177
}
136-
} else if (!result_value.GetValue<bool>()) {
178+
} else if (result_value.IsNull() || !result_value.GetValue<bool>()) {
137179
// filter evaluates to false
138180
should_prune_file = true;
139181
// convert the filter to a table filter.

‎src/common/multi_file_list.cpp‎

Lines changed: 5 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -25,16 +25,17 @@ MultiFilePushdownInfo::MultiFilePushdownInfo(idx_t table_index, const vector<str
2525
// Helper method to do Filter Pushdown into a MultiFileList
2626
bool PushdownInternal(ClientContext &context, const MultiFileReaderOptions &options, MultiFilePushdownInfo &info,
2727
vector<unique_ptr<Expression>> &filters, vector<string> &expanded_files) {
28-
unordered_map<string, column_t> column_map;
28+
HivePartitioningFilterInfo filter_info;
2929
for (idx_t i = 0; i < info.column_ids.size(); i++) {
3030
if (!IsRowIdColumnId(info.column_ids[i])) {
31-
column_map.insert({info.column_names[info.column_ids[i]], i});
31+
filter_info.column_map.insert({info.column_names[info.column_ids[i]], i});
3232
}
3333
}
34+
filter_info.hive_enabled = options.hive_partitioning;
35+
filter_info.filename_enabled = options.filename;
3436

3537
auto start_files = expanded_files.size();
36-
HivePartitioning::ApplyFiltersToFileList(context, expanded_files, filters, column_map, info,
37-
options.hive_partitioning, options.filename);
38+
HivePartitioning::ApplyFiltersToFileList(context, expanded_files, filters, filter_info, info);
3839

3940
if (expanded_files.size() != start_files) {
4041
return true;

‎src/common/multi_file_reader.cpp‎

Lines changed: 5 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -457,7 +457,7 @@ bool MultiFileReaderOptions::AutoDetectHivePartitioningInternal(MultiFileList &f
457457
// partition count mismatch
458458
return false;
459459
}
460-
for(auto &part : new_partitions) {
460+
for (auto &part : new_partitions) {
461461
auto entry = partitions.find(part.first);
462462
if (entry == partitions.end()) {
463463
// differing partitions between files
@@ -549,28 +549,13 @@ bool MultiFileReaderOptions::AnySet() {
549549
return filename || hive_partitioning || union_by_name;
550550
}
551551

552-
Value MultiFileReaderOptions::GetHivePartitionValue(const string &base, const string &entry,
552+
Value MultiFileReaderOptions::GetHivePartitionValue(const string &value, const string &key,
553553
ClientContext &context) const {
554-
Value value(base);
555-
auto it = hive_types_schema.find(entry);
554+
auto it = hive_types_schema.find(key);
556555
if (it == hive_types_schema.end()) {
557-
return value;
556+
return HivePartitioning::GetValue(context, key, value, LogicalType::VARCHAR);
558557
}
559-
560-
// Handle nulls
561-
if (StringUtil::CIEquals(base, "NULL")) {
562-
return Value(it->second);
563-
}
564-
if (it->second.id() != LogicalTypeId::VARCHAR && base.empty()) {
565-
// empty strings are NULL for non-string types
566-
return Value(it->second);
567-
}
568-
569-
if (!value.TryCastAs(context, it->second)) {
570-
throw InvalidInputException("Unable to cast '%s' (from hive partition column '%s') to: '%s'", value.ToString(),
571-
StringUtil::Upper(it->first), it->second.ToString());
572-
}
573-
return value;
558+
return HivePartitioning::GetValue(context, key, value, it->second);
574559
}
575560

576561
} // namespace duckdb

‎src/include/duckdb/common/hive_partitioning.hpp‎

Lines changed: 11 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,12 @@
2121
namespace duckdb {
2222
struct MultiFilePushdownInfo;
2323

24+
struct HivePartitioningFilterInfo {
25+
unordered_map<string, column_t> column_map;
26+
bool hive_enabled;
27+
bool filename_enabled;
28+
};
29+
2430
class HivePartitioning {
2531
public:
2632
//! Parse a filename that follows the hive partitioning scheme
@@ -30,9 +36,11 @@ class HivePartitioning {
3036
//! evaluate to true.
3137
DUCKDB_API static void ApplyFiltersToFileList(ClientContext &context, vector<string> &files,
3238
vector<unique_ptr<Expression>> &filters,
33-
unordered_map<string, column_t> &column_map,
34-
MultiFilePushdownInfo &info, bool hive_enabled,
35-
bool filename_enabled);
39+
const HivePartitioningFilterInfo &filter_info,
40+
MultiFilePushdownInfo &info);
41+
42+
DUCKDB_API static Value GetValue(ClientContext &context, const string &key, const string &value,
43+
const LogicalType &type);
3644
};
3745

3846
struct HivePartitionKey {

‎test/sql/copy/parquet/parquet_hive_empty.test‎

Lines changed: 24 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -6,9 +6,32 @@ require parquet
66

77
query II
88
select *
9-
from parquet_scan('data/parquet-testing/hive-partitioning/empty_string/*/*.parquet', hive_partitioning=1)
9+
from parquet_scan('data/parquet-testing/hive-partitioning/empty_string/*/*.parquet')
1010
ORDER BY ALL
1111
----
1212
a a
1313
b (empty)
14+
c NULL
1415

16+
# filter on hive partitioning with NULL values
17+
query II
18+
select *
19+
from parquet_scan('data/parquet-testing/hive-partitioning/empty_string/*/*.parquet')
20+
WHERE key IS NULL
21+
----
22+
c NULL
23+
24+
25+
query II
26+
select *
27+
from parquet_scan('data/parquet-testing/hive-partitioning/empty_string/*/*.parquet')
28+
WHERE key='a'
29+
----
30+
a a
31+
32+
query II
33+
select *
34+
from parquet_scan('data/parquet-testing/hive-partitioning/empty_string/*/*.parquet')
35+
WHERE key=''
36+
----
37+
b (empty)

‎test/sql/copy/partitioned/hive_partition_duplicate_name.test‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
# name: test/sql/copy/partitioned/hive_partition_duplicate_name.test
22
# description: Test partitioning names with duplicate keys
3-
# group: [parquet]
3+
# group: [partitioned]
44

55
require parquet
66

0 commit comments

Comments
 (0)