Skip to content

Commit 0419942

Browse files
authored
[arrow] Fix pushdown errors (duckdb#27319)
Fixes duckdb/duckdb-python#663, duckdb/duckdb-python#665, duckdb/duckdb-python#666 and the engine half of duckdb#664. The `arrow_scan` function refuses filter pushdown per column through `supports_pushdown_type`, because the Arrow producer applies the pushed filter itself and never gets it re-checked. pyarrow cannot evaluate filters on string or binary view arrays, on run-end encoded arrays, or on narrow decimals. The gate missed three cases, so filters reached pyarrow and failed loud, in two cases with an INTERNAL error that invalidates the database. This PR closes the three gaps: - A dictionary column was judged by its index type, so a dictionary of `string_view`, `binary_view` or a narrow decimal was pushed although the plain type is refused. The gate now judges a dictionary by its value type. A dictionary of view values does not turn pushdown off for the table, since pyarrow filters a dictionary by its indices. - A run-end encoded column kept the `run_ends, values` struct as its type info while its duck type became the values' type, so reading the string info at planning threw an INTERNAL error for any filter or Top-N on the table. The gate now checks run-end encoding before it reads any type info, and treats it like a view type: pushdown is off for the whole table. - Join filter pushdown never consulted `supports_pushdown_type`, so dynamic join filters reached the producer for columns the static filter path keeps above the scan. `GetPushdownFilterTargets` now drops the refused columns and keeps the rest; a scan with no accepted column gets no target. This applies to every caller, Top-N included. Top-N's filter above the scan was an optional filter that passed every row, so nothing is lost; measured parity. Tests hand-build the Arrow columns DuckDB's own export never produces and scan them through a producer that records which filters it receives. Notes: - The run-end encoded type info inconsistency also breaks binding of run-end encoded lists and mistypes run-end encoded structs. That is a separate fix. - None of these is a regression; all reproduce on 1.5.6.
2 parents c710433 + 4e57747 commit 0419942

3 files changed

Lines changed: 421 additions & 12 deletions

File tree

‎src/function/table/arrow.cpp‎

Lines changed: 21 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -213,6 +213,12 @@ OperatorPartitionData ArrowTableFunction::ArrowGetPartitionData(ClientContext &c
213213
}
214214

215215
static bool CanPushdown(const ArrowType &type) {
216+
if (type.RunEndEncoded()) {
217+
return false;
218+
}
219+
if (type.HasDictionary()) {
220+
return CanPushdown(type.GetDictionary());
221+
}
216222
auto duck_type = type.GetDuckType();
217223
switch (duck_type.id()) {
218224
case LogicalTypeId::BOOLEAN:
@@ -266,7 +272,13 @@ static bool CanPushdown(const ArrowType &type) {
266272
return false;
267273
}
268274
}
269-
static bool HasViewType(const ArrowType &type) {
275+
276+
// PyArrow's array_filter kernel has no implementation for string view, binary view and run-end encoded arrays.
277+
// It filters a dictionary by its indices, so a dictionary of view values is fine here.
278+
static bool HasUnfilterableType(const ArrowType &type) {
279+
if (type.RunEndEncoded()) {
280+
return true;
281+
}
270282
auto duck_type = type.GetDuckType();
271283
switch (duck_type.id()) {
272284
case LogicalTypeId::VARCHAR:
@@ -277,7 +289,7 @@ static bool HasViewType(const ArrowType &type) {
277289
case LogicalTypeId::UNION: {
278290
const auto &struct_info = type.GetTypeInfo<ArrowStructInfo>();
279291
for (idx_t i = 0; i < struct_info.ChildCount(); i++) {
280-
if (HasViewType(struct_info.GetChild(i))) {
292+
if (HasUnfilterableType(struct_info.GetChild(i))) {
281293
return true;
282294
}
283295
}
@@ -286,20 +298,20 @@ static bool HasViewType(const ArrowType &type) {
286298
case LogicalTypeId::LIST:
287299
case LogicalTypeId::MAP: {
288300
const auto &list_info = type.GetTypeInfo<ArrowListInfo>();
289-
return HasViewType(list_info.GetChild());
301+
return HasUnfilterableType(list_info.GetChild());
290302
}
291303
case LogicalTypeId::ARRAY: {
292304
const auto &array_info = type.GetTypeInfo<ArrowArrayInfo>();
293-
return HasViewType(array_info.GetChild());
305+
return HasUnfilterableType(array_info.GetChild());
294306
}
295307
default:
296308
return false;
297309
}
298310
}
299311

300-
static bool TableHasViewTypes(const arrow_column_map_t &column_info) {
312+
static bool TableHasUnfilterableTypes(const arrow_column_map_t &column_info) {
301313
for (const auto &col : column_info) {
302-
if (HasViewType(*col.second)) {
314+
if (HasUnfilterableType(*col.second)) {
303315
return true;
304316
}
305317
}
@@ -309,11 +321,9 @@ static bool TableHasViewTypes(const arrow_column_map_t &column_info) {
309321
bool ArrowTableFunction::ArrowPushdownType(const FunctionData &bind_data, idx_t col_idx) {
310322
auto &arrow_bind_data = bind_data.Cast<ArrowScanFunctionData>();
311323
const auto &column_info = arrow_bind_data.arrow_table.GetColumns();
312-
// PyArrow's array_filter kernel doesn't support string_view/binary_view types.
313-
// The filter is applied to ALL columns in a record batch, so if any column has a
314-
// view type, we must disable filter pushdown for the entire table.
315-
// See https://github.com/duckdb/duckdb-python/issues/227
316-
if (TableHasViewTypes(column_info)) {
324+
// The producer applies a pushed filter to every column of a record batch, so one column that cannot be
325+
// filtered turns pushdown off for the entire table.
326+
if (TableHasUnfilterableTypes(column_info)) {
317327
return false;
318328
}
319329
const auto column_type = column_info.at(col_idx);

‎src/optimizer/join_filter_pushdown_optimizer.cpp‎

Lines changed: 12 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -233,11 +233,18 @@ void JoinFilterPushdownOptimizer::GetPushdownFilterTargets(LogicalOperator &op,
233233
}
234234
get.ResolveOperatorTypes();
235235
const auto bindings = get.GetColumnBindings();
236+
vector<JoinFilterPushdownColumn> accepted_columns;
236237
for (auto &filter : columns) {
237238
if (filter.probe_column_index.table_index != get.table_index) {
238239
// the filter does not apply to the probe side here - bail-out
239240
return;
240241
}
242+
if (get.function.supports_pushdown_type) {
243+
auto column_id = get.GetColumnIndex(filter.probe_column_index).GetPrimaryIndex();
244+
if (!get.function.supports_pushdown_type(*get.bind_data, column_id)) {
245+
continue;
246+
}
247+
}
241248
// add storage type to columns
242249
for (idx_t i = 0; i < bindings.size(); i++) {
243250
if (filter.probe_column_index.column_index == bindings[i].column_index) {
@@ -254,8 +261,12 @@ void JoinFilterPushdownOptimizer::GetPushdownFilterTargets(LogicalOperator &op,
254261
reconstructed_type.id() == LogicalTypeId::VARIANT) {
255262
return;
256263
}
264+
accepted_columns.push_back(std::move(filter));
265+
}
266+
if (accepted_columns.empty()) {
267+
return;
257268
}
258-
targets.emplace_back(get, std::move(columns));
269+
targets.emplace_back(get, std::move(accepted_columns));
259270
break;
260271
}
261272
case LogicalOperatorType::LOGICAL_PROJECTION: {

0 commit comments

Comments
 (0)