Skip to content

Commit 584c07d

Browse files
authored
Disable pushdown for arrow tables that contain view columns (#20703)
This is an attempt at a workaround / fix for duckdb/duckdb-python#227. ### Problem Pyarrow's `array_filter` kernel throws `NotImplemented` when it's applied on a schema that has a view type, even if the filter itself is on a different type. Since we evaluate whether we can pushdown a filter based on the type of the column we filter on (`supports_pushdown_type`), we might still pushdown filters that fail during the scan. Note: this is an issue for binary and string views. List views do not seem to have this problem. ### Approach We refuse to push down filters if the table contains a view type. This is a coarse solution, but for now it solves the problem and given that Arrow's own compute kernels refuse to filter tables with views, it's not unreasonable. ### Alternatives * **Gating pushdown with a config option.** We could add a config option (`arrow_filter_pushdown_with_view_types`, default = false) that gates pushdown on schemas that includes a view type. By default, it checks whether `ArrowScanFunctionData::arrow_table` contains columns with view typers, and if so, disables pushdown. Users can enable pushdown by setting `arrow_filter_pushdown_with_view_types = true`. We deiced against this because we already have `arrow_output_list_view`, `produce_arrow_string_view` and `arrow_output_version` that all play together in producing views. We should really clean that up. * **Adding pushdown capabilities to the scanner.** We could extend `TableFunctionData` with some notion of pushdown capabilities that can be checked by the table function when we decide whether we can pushdown a filter. This is a nicer option that allows more control per scanner, so that when we use e.g. pyarrow to push down we can set some conditions. It's also more invasive and relatively complex, so for now we don't do this.
2 parents 2b4138f + 7d57546 commit 584c07d

3 files changed

Lines changed: 173 additions & 3 deletions

File tree

‎src/function/table/arrow.cpp‎

Lines changed: 48 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,7 @@
1515
#include "utf8proc_wrapper.hpp"
1616
#include "duckdb/common/extra_type_info.hpp"
1717
#include "duckdb/common/arrow/schema_metadata.hpp"
18+
#include "duckdb/main/settings.hpp"
1819

1920
namespace duckdb {
2021

@@ -276,10 +277,56 @@ static bool CanPushdown(const ArrowType &type) {
276277
return false;
277278
}
278279
}
280+
static bool HasViewType(const ArrowType &type) {
281+
auto duck_type = type.GetDuckType();
282+
switch (duck_type.id()) {
283+
case LogicalTypeId::VARCHAR:
284+
case LogicalTypeId::BLOB:
285+
return type.GetTypeInfo<ArrowStringInfo>().GetSizeType() == ArrowVariableSizeType::VIEW;
286+
case LogicalTypeId::STRUCT:
287+
case LogicalTypeId::UNION: {
288+
const auto &struct_info = type.GetTypeInfo<ArrowStructInfo>();
289+
for (idx_t i = 0; i < struct_info.ChildCount(); i++) {
290+
if (HasViewType(struct_info.GetChild(i))) {
291+
return true;
292+
}
293+
}
294+
return false;
295+
}
296+
case LogicalTypeId::LIST:
297+
case LogicalTypeId::MAP: {
298+
const auto &list_info = type.GetTypeInfo<ArrowListInfo>();
299+
return HasViewType(list_info.GetChild());
300+
}
301+
case LogicalTypeId::ARRAY: {
302+
const auto &array_info = type.GetTypeInfo<ArrowArrayInfo>();
303+
return HasViewType(array_info.GetChild());
304+
}
305+
default:
306+
return false;
307+
}
308+
}
309+
310+
static bool TableHasViewTypes(const arrow_column_map_t &column_info) {
311+
for (const auto &col : column_info) {
312+
if (HasViewType(*col.second)) {
313+
return true;
314+
}
315+
}
316+
return false;
317+
}
318+
279319
bool ArrowTableFunction::ArrowPushdownType(const FunctionData &bind_data, idx_t col_idx) {
280320
auto &arrow_bind_data = bind_data.Cast<ArrowScanFunctionData>();
281321
const auto &column_info = arrow_bind_data.arrow_table.GetColumns();
282-
auto column_type = column_info.at(col_idx);
322+
// PyArrow's array_filter kernel doesn't support string_view/binary_view types.
323+
// The filter is applied to ALL columns in a record batch, so if any column has a
324+
// view type, we must disable filter pushdown for the entire table.
325+
// See https://github.com/duckdb/duckdb-python/issues/227
326+
if (TableHasViewTypes(column_info)) {
327+
return false;
328+
}
329+
const auto column_type = column_info.at(col_idx);
283330
return CanPushdown(*column_type);
284331
}
285332

‎test/arrow/CMakeLists.txt‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
1-
add_library_unity(test_arrow_roundtrip OBJECT arrow_test_helper.cpp
2-
arrow_roundtrip.cpp arrow_move_children.cpp)
1+
add_library_unity(
2+
test_arrow_roundtrip OBJECT arrow_test_helper.cpp arrow_roundtrip.cpp
3+
arrow_move_children.cpp arrow_filter_pushdown.cpp)
34
set(ALL_OBJECT_FILES
45
${ALL_OBJECT_FILES} $<TARGET_OBJECTS:test_arrow_roundtrip>
56
PARENT_SCOPE)
Lines changed: 122 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,122 @@
1+
#include <regex>
2+
3+
#include "catch.hpp"
4+
5+
#include "arrow/arrow_test_helper.hpp"
6+
7+
using namespace duckdb;
8+
9+
// Helper: create an ArrowTestFactory from a query result (all rows, no filter)
10+
static unique_ptr<ArrowTestFactory> MakeArrowFactory(Connection &con, const string &query, bool use_string_view) {
11+
if (use_string_view) {
12+
REQUIRE(!con.Query("SET produce_arrow_string_view = true")->HasError());
13+
REQUIRE(!con.Query("SET arrow_output_version = '1.5'")->HasError());
14+
} else {
15+
REQUIRE(!con.Query("SET produce_arrow_string_view = false")->HasError());
16+
REQUIRE(!con.Query("SET arrow_output_version = '1.0'")->HasError());
17+
}
18+
auto client_properties = con.context->GetClientProperties();
19+
auto result = con.context->Query(query, false);
20+
REQUIRE(!result->HasError());
21+
auto types = result->types;
22+
auto names = result->names;
23+
return make_uniq<ArrowTestFactory>(std::move(types), std::move(names), std::move(result), false, client_properties,
24+
*con.context);
25+
}
26+
27+
// Helper: get the EXPLAIN output for an arrow_scan with a filter
28+
static string GetExplainForFilter(Connection &con, ArrowTestFactory &factory, const string &filter_expr) {
29+
const auto params = ArrowTestHelper::ConstructArrowScan(factory);
30+
const auto rel = con.TableFunction("arrow_scan", params)->Filter(filter_expr);
31+
const auto explain_result = rel->Explain();
32+
REQUIRE(!explain_result->HasError());
33+
auto &mat = explain_result->Cast<MaterializedQueryResult>();
34+
return mat.GetValue(1, 0).ToString();
35+
}
36+
37+
// Helper: regexp search for a filter node
38+
static bool StandaloneFilter(const std::string &explain_str) {
39+
// This is meant to match e.g.:
40+
//
41+
// ┌─────────────┴─────────────┐
42+
// │ FILTER │
43+
// │ ──────────────────── │
44+
// │ ((a > 25) AND (b > 25)) │
45+
// │ │
46+
// │ ~20 rows │
47+
// └─────────────┬─────────────┘
48+
//
49+
const std::regex re_filter(R"(│ +FILTER +│)");
50+
return std::regex_search(explain_str, re_filter);
51+
}
52+
53+
// Helper: regexp search for a scan node with filter
54+
static bool FilterInScan(const std::string &explain_str) {
55+
// This is meant to match e.g.:
56+
//
57+
// ┌─────────────┴─────────────┐
58+
// │ ARROW_SCAN │
59+
// │ ──────────────────── │
60+
// │ Function: ARROW_SCAN │
61+
// │ │
62+
// │ Filters: │
63+
// │ a>25 │
64+
// │ b>25 │
65+
// │ │
66+
// │ ~1 row │
67+
// └───────────────────────────┘
68+
//
69+
const std::regex re_block(R"(│[ \t]*Function:[ \t]*ARROW_SCAN[ \t]*│[\s\S]*?│[ \t]*Filters:[ \t]*([^│]*?)[ \t]*│)");
70+
return std::regex_search(explain_str, re_block);
71+
}
72+
73+
TEST_CASE("Arrow filter pushdown - view types disable pushdown", "[arrow]") {
74+
DuckDB db;
75+
Connection con(db);
76+
77+
// Create a test table with an INT id column and a VARCHAR name column
78+
REQUIRE(!con.Query("CREATE TABLE src AS SELECT i AS id, i::VARCHAR AS name FROM range(10) tbl(i)")->HasError());
79+
80+
SECTION("String view column: filter above scan (not pushed)") {
81+
auto factory = MakeArrowFactory(con, "SELECT * FROM src", true);
82+
auto explain_str = GetExplainForFilter(con, *factory, "id > 5");
83+
REQUIRE(StandaloneFilter(explain_str));
84+
REQUIRE(!FilterInScan(explain_str));
85+
}
86+
87+
SECTION("Regular string column: filter pushed into scan") {
88+
auto factory = MakeArrowFactory(con, "SELECT * FROM src", false);
89+
auto explain_str = GetExplainForFilter(con, *factory, "id > 5");
90+
REQUIRE(!StandaloneFilter(explain_str));
91+
REQUIRE(FilterInScan(explain_str));
92+
}
93+
94+
SECTION("Integer-only table: filter pushed into scan") {
95+
auto factory = MakeArrowFactory(con, "SELECT id FROM src", false);
96+
auto explain_str = GetExplainForFilter(con, *factory, "id > 5");
97+
REQUIRE(!StandaloneFilter(explain_str));
98+
REQUIRE(FilterInScan(explain_str));
99+
}
100+
}
101+
102+
TEST_CASE("Arrow filter pushdown - nested view types disable pushdown", "[arrow]") {
103+
DuckDB db;
104+
Connection con(db);
105+
106+
REQUIRE(!con.Query("CREATE TABLE src AS SELECT i AS id, 'val_' || i::VARCHAR AS name FROM range(10) tbl(i)")
107+
->HasError());
108+
109+
SECTION("Struct containing string_view") {
110+
auto factory = MakeArrowFactory(con, "SELECT id, {'s': name} AS nested FROM src", true);
111+
auto explain_str = GetExplainForFilter(con, *factory, "id > 5");
112+
REQUIRE(StandaloneFilter(explain_str));
113+
REQUIRE(!FilterInScan(explain_str));
114+
}
115+
116+
SECTION("List containing string_view") {
117+
auto factory = MakeArrowFactory(con, "SELECT id, [name] AS names FROM src", true);
118+
auto explain_str = GetExplainForFilter(con, *factory, "id > 5");
119+
REQUIRE(StandaloneFilter(explain_str));
120+
REQUIRE(!FilterInScan(explain_str));
121+
}
122+
}

0 commit comments

Comments
 (0)