Skip to content

Commit 2d55a21

Browse files
Replace the Arrow scan with a port of the neo branch's scan (duckdb#693)
Registered Arrow objects are wrapped in a pyarrow `Scanner` that evaluates pushed filters with pyarrow kernels on every batch and reads one shot streams far ahead of the query. That caused the kernel failures of duckdb#663 to duckdb#666, the memory growth of duckdb#691, wrong rows for struct columns in sliced batches, and a hang at exit with an unread generator reader. This PR replaces that path with a port of the neo branch's scan: one table function, `python_arrow_scan`, over a private source adapter. - The engine offers each filter to the source, which takes only what it applies exactly (a pyarrow expression on Datasets, a polars one on LazyFrames); the rest stays above the scan. In memory objects never receive filters. - Only the columns a query reads are converted. Every array is validated before adoption; a stream whose schema differs from the declared one is refused. - Streams are read one batch per thread, without read ahead and without holding the GIL; a one shot reader read twice gets a clear error. - `pa.Array`, `ChunkedArray`, pandas and polars Series and polars LazyFrames are accepted everywhere; every Python error from a source is an `InvalidInputException` naming the object. - Interpreter exit waits for a pull inside Python and refuses new scans. Notes: - No longer pushed, the engine filters instead: `LIKE`, NaN, nanosecond and infinite constants, date64 columns, and float predicates NaN can satisfy on Datasets and LazyFrames (parquet row group statistics leave NaN out). In memory row counts are estimates. - A struct below a struct in a sliced batch is refused until the engine folds the parent offset; it read wrong rows before.
2 parents 18f1b20 + a3d8e3c commit 2d55a21

59 files changed

Lines changed: 13595 additions & 2313 deletions

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎_duckdb-stubs/__init__.pyi‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -443,6 +443,7 @@ class DuckDBPyConnection:
443443
compression: ParquetCompression | None = None,
444444
) -> DuckDBPyRelation: ...
445445
def register(self, view_name: str, python_object: object) -> DuckDBPyConnection: ...
446+
def _register_arrow_source(self, view_name: str, source: object) -> DuckDBPyConnection: ...
446447
def register_filesystem(self, filesystem: fsspec.AbstractFileSystem) -> None: ...
447448
def remove_function(self, name: str) -> DuckDBPyConnection: ...
448449
def rollback(self) -> DuckDBPyConnection: ...
@@ -1243,6 +1244,14 @@ def table_function(
12431244
connection: DuckDBPyConnection | None = None,
12441245
) -> DuckDBPyRelation: ...
12451246
def tf(*, connection: DuckDBPyConnection | None = None) -> dict[str, typing.Any]: ...
1247+
def _set_arrow_sources(sources: object) -> None: ...
1248+
def _translate_predicate(
1249+
connection: DuckDBPyConnection,
1250+
predicate: str,
1251+
columns: object = None,
1252+
schema: object = None,
1253+
select: str = "*",
1254+
) -> lst[tuple[str, object | None]]: ...
12461255
def tokenize(query: str) -> lst[tuple[int, token_type]]: ...
12471256
def torch(*, connection: DuckDBPyConnection | None = None) -> dict[str, typing.Any]: ...
12481257
def type(type_str: str, *, connection: DuckDBPyConnection | None = None) -> sqltypes.DuckDBPyType: ...

‎duckdb/__init__.py‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@
99
- Future versions may move them into submodules with deprecation warnings.
1010
"""
1111

12+
import _duckdb
1213
from _duckdb import (
1314
Appender,
1415
BinderException,
@@ -158,6 +159,7 @@
158159
write_csv,
159160
)
160161

162+
from duckdb import _sources
161163
from duckdb._dbapi_type_object import (
162164
BINARY,
163165
DATETIME,
@@ -208,6 +210,8 @@
208210
Value,
209211
)
210212

213+
_duckdb._set_arrow_sources(_sources)
214+
211215
__all__: list[str] = [
212216
"Appender",
213217
"BinaryValue",

‎duckdb/_sources/__init__.py‎

Lines changed: 59 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,59 @@
1+
"""The Python objects the Arrow scan reads, each behind a source that exports an Arrow C stream for it.
2+
3+
The scan calls `adapt` for an Arrow shaped object and then speaks only to the source: `rows()`, `pull_under_gil`,
4+
`accepts(predicate)`, `stream(columns, filters)` and, when present, `__arrow_c_schema__()`; `one_shot` is
5+
informational, since the scan decides single use from the underlying object.
6+
Every module of the package is imported here, since importing one while a query runs could load the extension a
7+
second time; pyarrow, polars and pandas are imported only when an object's class, or a class it derives from, is
8+
defined in them, and their classes are then recognised by identity.
9+
"""
10+
11+
from __future__ import annotations
12+
13+
import datetime
14+
15+
from duckdb._sources import nodes, to_arrow, to_polars
16+
from duckdb._sources.arrow import (
17+
ArrowArraySource,
18+
ArrowCapsuleSource,
19+
ArrowScanSource,
20+
ArrowStreamSource,
21+
Source,
22+
class_modules,
23+
)
24+
from duckdb._sources.polars import polars_source
25+
from duckdb._sources.pyarrow import DatasetSource, ScannerSource, pyarrow_source
26+
27+
__all__ = ["ArrowScanSource", "Source", "adapt", "nodes", "to_arrow", "to_polars"]
28+
29+
_CAPSULE = type(datetime.datetime_CAPI)
30+
31+
32+
def adapt(obj: object) -> Source:
33+
"""The source for an Arrow shaped object; an existing `Source` is returned unchanged."""
34+
if isinstance(obj, Source):
35+
return obj
36+
if type(obj) is _CAPSULE:
37+
return ArrowCapsuleSource(obj)
38+
modules = class_modules(obj)
39+
libraries = {module.partition(".")[0] for module in modules}
40+
source = None
41+
if "pyarrow" in libraries:
42+
source = pyarrow_source(obj, modules)
43+
if source is None and "polars" in libraries:
44+
source = polars_source(obj)
45+
if source is not None:
46+
return source
47+
if hasattr(obj, "__arrow_c_stream__"):
48+
return ArrowStreamSource(obj)
49+
if hasattr(obj, "__arrow_c_array__"):
50+
return ArrowArraySource(obj)
51+
if hasattr(obj, "to_reader") and hasattr(obj, "projected_schema"):
52+
return ScannerSource(obj)
53+
if hasattr(obj, "scanner") and hasattr(obj, "schema"):
54+
return DatasetSource(obj)
55+
message = (
56+
"an Arrow object must export __arrow_c_stream__ or __arrow_c_array__, or be an Arrow stream capsule, a "
57+
f"pyarrow Dataset or Scanner, or a polars DataFrame or LazyFrame; {type(obj).__name__} is none of these"
58+
)
59+
raise TypeError(message)

‎duckdb/_sources/arrow.py‎

Lines changed: 163 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,163 @@
1+
"""The source protocol, and the sources for any object exporting the Arrow PyCapsule interface or a bare capsule."""
2+
3+
from __future__ import annotations
4+
5+
import ctypes
6+
from typing import TYPE_CHECKING, Any
7+
8+
if TYPE_CHECKING:
9+
from collections.abc import Sequence
10+
11+
import pyarrow as pa
12+
13+
from duckdb._sources.nodes import Node
14+
15+
16+
class Source:
17+
"""An object wrapped for a scan."""
18+
19+
#: Informational only: the scan decides from the underlying object whether it can be read once.
20+
one_shot = False
21+
#: Whether the schema the object declares cannot change while it is registered, so the scan may cache it.
22+
schema_is_stable = False
23+
24+
def __init__(self, obj: object) -> None:
25+
self.obj: Any = obj
26+
27+
def rows(self) -> int | None:
28+
"""How many rows a scan produces, when the object knows without reading itself, and None otherwise."""
29+
return None
30+
31+
32+
class ArrowScanSource(Source):
33+
"""A source the Arrow scan reads through the stream `stream()` exports, optionally with `__arrow_c_schema__`."""
34+
35+
#: An opt in asking the scan to hold the GIL for pulls and release callbacks; no adapter here needs it, since a
36+
#: producer that reaches Python takes the GIL itself, as pyarrow, ctypes, cffi and Cython callbacks do.
37+
pull_under_gil = False
38+
39+
def accepts(self, predicate: Node) -> bool:
40+
"""Whether every stream will hold only rows where `predicate` is true by the engine's semantics."""
41+
return False
42+
43+
def stream(self, columns: Sequence[int] | None, filters: Sequence[Node]) -> tuple[object, bool]:
44+
"""A stream capsule or a (schema, array) capsule pair, and whether it holds exactly `columns` in order.
45+
46+
`columns` are declared column indexes, None for all; a source that cannot narrow returns every declared
47+
column in declared order and False. `filters` are the accepted predicates, applied before narrowing.
48+
"""
49+
return self.obj.__arrow_c_stream__(), False
50+
51+
52+
class _ArrowSchema(ctypes.Structure):
53+
"""The C struct an object's `_export_to_c` writes a schema into."""
54+
55+
_fields_ = (
56+
("format", ctypes.c_char_p),
57+
("name", ctypes.c_char_p),
58+
("metadata", ctypes.c_char_p),
59+
("flags", ctypes.c_int64),
60+
("n_children", ctypes.c_int64),
61+
("children", ctypes.c_void_p),
62+
("dictionary", ctypes.c_void_p),
63+
("release", ctypes.c_void_p),
64+
("private_data", ctypes.c_void_p),
65+
)
66+
67+
68+
def class_modules(obj: object) -> set[str]:
69+
"""The modules defining the class of `obj` and the classes it derives from; a class may have no module."""
70+
return {module for cls in type(obj).__mro__ if isinstance(module := getattr(cls, "__module__", None), str)}
71+
72+
73+
def _exported(exporter: object) -> pa.Schema | pa.Field:
74+
"""What `exporter._export_to_c` writes: a Schema for a struct, otherwise a Field keeping name and metadata."""
75+
import pyarrow
76+
77+
exported = _ArrowSchema()
78+
address = ctypes.addressof(exported)
79+
exporter._export_to_c(address) # type: ignore[attr-defined]
80+
if exported.format == b"+s":
81+
return pyarrow.Schema._import_from_c(address)
82+
return pyarrow.Field._import_from_c(address)
83+
84+
85+
def arrow_schema(schema: object) -> pa.Schema:
86+
"""A pyarrow Schema for a schema object exporting through `__arrow_c_schema__` or the older `_export_to_c`."""
87+
import pyarrow
88+
89+
if isinstance(schema, pyarrow.Schema):
90+
return schema
91+
hook = getattr(schema, "__arrow_c_schema__", None)
92+
if callable(hook):
93+
return pyarrow.Schema._import_from_c_capsule(hook())
94+
exported = _exported(schema)
95+
if not isinstance(exported, pyarrow.Schema):
96+
message = f"a table schema must be a struct, not {exported.type}"
97+
raise TypeError(message)
98+
return exported
99+
100+
101+
def _schema_capsule(exporter: object) -> object:
102+
"""The schema capsule of a schema or type object exporting through `__arrow_c_schema__` or `_export_to_c`."""
103+
hook = getattr(exporter, "__arrow_c_schema__", None)
104+
if callable(hook):
105+
return hook()
106+
return _exported(exporter).__arrow_c_schema__()
107+
108+
109+
def _schema_hook(source: ArrowScanSource, attribute: str) -> None:
110+
"""Give `source` a `__arrow_c_schema__`, so bind reads the schema without opening a stream.
111+
112+
The hook reads the object's own `__arrow_c_schema__`, or else `attribute`, at every call, so an object whose
113+
schema changes reports the new one.
114+
"""
115+
obj = source.obj
116+
if callable(getattr(obj, "__arrow_c_schema__", None)):
117+
source.__dict__["__arrow_c_schema__"] = lambda: obj.__arrow_c_schema__()
118+
return
119+
exporter = getattr(obj, attribute, None)
120+
if callable(getattr(exporter, "__arrow_c_schema__", None)) or callable(getattr(exporter, "_export_to_c", None)):
121+
source.__dict__["__arrow_c_schema__"] = lambda: _schema_capsule(getattr(obj, attribute))
122+
123+
124+
class ArrowStreamSource(ArrowScanSource):
125+
"""Anything with `__arrow_c_stream__`: every column, its schema from `__arrow_c_schema__` or `schema`."""
126+
127+
#: The attribute that holds the object's schema when the object has no `__arrow_c_schema__` of its own.
128+
schema_attribute = "schema"
129+
130+
def __init__(self, obj: object) -> None:
131+
super().__init__(obj)
132+
_schema_hook(self, self.schema_attribute)
133+
134+
def rows(self) -> int | None:
135+
"""The length of a pandas Series or DataFrame, which is its row count; unknown for anything else."""
136+
if "pandas" not in {module.partition(".")[0] for module in class_modules(self.obj)}:
137+
return None
138+
import pandas
139+
140+
return len(self.obj) if isinstance(self.obj, pandas.Series | pandas.DataFrame) else None
141+
142+
143+
class ArrowArraySource(ArrowScanSource):
144+
"""Anything with `__arrow_c_array__` and no stream: one batch, read as often as asked."""
145+
146+
def __init__(self, obj: object) -> None:
147+
super().__init__(obj)
148+
_schema_hook(self, "type")
149+
150+
def stream(self, columns: Sequence[int] | None, filters: Sequence[Node]) -> tuple[object, bool]:
151+
"""The schema and array capsules of the array, every column."""
152+
return self.obj.__arrow_c_array__(), False
153+
154+
155+
class ArrowCapsuleSource(ArrowScanSource):
156+
"""A bare capsule; the scan checks that it is an unreleased `arrow_array_stream`."""
157+
158+
one_shot = True
159+
schema_is_stable = True
160+
161+
def stream(self, columns: Sequence[int] | None, filters: Sequence[Node]) -> tuple[object, bool]:
162+
"""The capsule itself, every column."""
163+
return self.obj, False

0 commit comments

Comments
 (0)