Skip to content

Commit 1dd1ab5

Browse files
committed
pt3
1 parent 3af5d61 commit 1dd1ab5

83 files changed

Lines changed: 1168 additions & 1157 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎_duckdb-stubs/__init__.pyi‎

Lines changed: 3 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -87,6 +87,7 @@ __all__: lst[str] = [
8787
"ProgrammingError",
8888
"PythonExceptionHandling",
8989
"RenderMode",
90+
"ResourceInUseException",
9091
"SQLExpression",
9192
"SequenceException",
9293
"SerializationException",
@@ -514,7 +515,6 @@ class DuckDBPyRelation:
514515
def df(self, *, date_as_object: bool = False) -> pandas.DataFrame: ...
515516
def distinct(self) -> DuckDBPyRelation: ...
516517
def except_(self, other_rel: Self) -> DuckDBPyRelation: ...
517-
def execute(self) -> DuckDBPyRelation: ...
518518
def explain(
519519
self, type: ExplainType | ExplainTypeLiteral = ExplainType.STANDARD, format: str | None = None
520520
) -> str: ...
@@ -527,17 +527,12 @@ class DuckDBPyRelation:
527527
def fetch_arrow_table(self, batch_size: typing.SupportsInt = 131072) -> pyarrow.lib.Table:
528528
"""Deprecated: use to_arrow_table() instead."""
529529
...
530-
def fetch_df_chunk(
531-
self, vectors_per_chunk: typing.SupportsInt = 1, *, date_as_object: bool = False
532-
) -> pandas.DataFrame: ...
533530
def fetch_record_batch(self, rows_per_batch: typing.SupportsInt = 131072) -> pyarrow.lib.RecordBatchReader:
534531
"""Deprecated: use to_arrow_reader() instead."""
535532
...
536533
def fetchall(self) -> lst[tuple[typing.Any, ...]]: ...
537534
def fetchdf(self, *, date_as_object: bool = False) -> pandas.DataFrame: ...
538-
def fetchmany(self, size: typing.SupportsInt = 1) -> lst[tuple[typing.Any, ...]]: ...
539535
def fetchnumpy(self) -> dict[str, np.typing.NDArray[typing.Any] | pandas.Categorical]: ...
540-
def fetchone(self) -> tuple[typing.Any, ...] | None: ...
541536
def filter(self, filter_expr: IntoExprColumn) -> DuckDBPyRelation: ...
542537
def first(self, expression: str, groups: str = "", projected_columns: str = "") -> DuckDBPyRelation: ...
543538
def first_value(self, expression: str, window_spec: str = "", projected_columns: str = "") -> DuckDBPyRelation: ...
@@ -579,6 +574,7 @@ class DuckDBPyRelation:
579574
def map(
580575
self, map_function: Callable[..., typing.Any], *, schema: dict[str, sqltypes.DuckDBPyType] | None = None
581576
) -> DuckDBPyRelation: ...
577+
def materialize(self) -> DuckDBPyRelation: ...
582578
def max(
583579
self, expression: str, groups: str = "", window_spec: str = "", projected_columns: str = ""
584580
) -> DuckDBPyRelation: ...
@@ -814,6 +810,7 @@ class OutOfRangeException(DataError): ...
814810
class ParserException(ProgrammingError): ...
815811
class PermissionException(DatabaseError): ...
816812
class ProgrammingError(DatabaseError): ...
813+
class ResourceInUseException(OperationalError): ...
817814
class SequenceException(DatabaseError): ...
818815
class SerializationException(OperationalError): ...
819816

‎benchmarks/test_produce_numpy_perf.py‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -99,16 +99,16 @@ def test_fetchnumpy_numeric_with_nulls(benchmark: BenchmarkFixture, con: duckdb.
9999
@pytest.mark.informational # per-chunk streaming drain (GIL-per-chunk), not gated
100100
def test_fetch_df_chunk_loop(benchmark: BenchmarkFixture, con: duckdb.DuckDBPyConnection) -> None:
101101
def run() -> int:
102-
rel = con.sql(Q_NUM)
102+
con.execute(Q_NUM)
103103
rows = 0
104104
while True:
105-
chunk = rel.fetch_df_chunk()
105+
chunk = con.fetch_df_chunk()
106106
if len(chunk) == 0:
107107
break
108108
rows += len(chunk)
109109
return rows
110110

111-
con.sql(Q_NUM).fetch_df_chunk() # warm
111+
con.execute(Q_NUM).fetch_df_chunk() # warm
112112
benchmark(run)
113113

114114

‎duckdb/__init__.py‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -51,6 +51,7 @@
5151
ProgrammingError,
5252
PythonExceptionHandling,
5353
RenderMode,
54+
ResourceInUseException,
5455
SequenceException,
5556
SerializationException,
5657
SQLExpression,
@@ -261,6 +262,7 @@
261262
"ProgrammingError",
262263
"PythonExceptionHandling",
263264
"RenderMode",
265+
"ResourceInUseException",
264266
"SQLExpression",
265267
"SequenceException",
266268
"SerializationException",

‎duckdb/experimental/spark/sql/dataframe.py‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1362,7 +1362,7 @@ def count(self) -> int:
13621362
3
13631363
"""
13641364
count_rel = self.relation.count("*")
1365-
return int(count_rel.fetchone()[0])
1365+
return int(count_rel.fetchall()[0][0])
13661366

13671367
def _cast_types(self, *types) -> "DataFrame":
13681368
existing_columns = self.relation.columns
@@ -1427,7 +1427,7 @@ def cache(self) -> "DataFrame":
14271427
== Physical Plan ==
14281428
InMemoryTableScan ...
14291429
"""
1430-
cached_relation = self.relation.execute()
1430+
cached_relation = self.relation.materialize()
14311431
return DataFrame(cached_relation, self.session)
14321432

14331433

‎duckdb/polars_io.py‎

Lines changed: 11 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -390,11 +390,16 @@ def source_generator(
390390
relation_final = relation_final.filter(duck_predicate)
391391
results = relation_final.to_arrow_reader() if batch_size is None else relation_final.to_arrow_reader(batch_size)
392392

393-
for record_batch in iter(results.read_next_batch, None):
394-
if fallback_predicate is not None:
395-
# We have a predicate, but did not manage to push it down, we fallback here
396-
yield pl.from_arrow(record_batch).filter(fallback_predicate) # type: ignore[arg-type,misc,unused-ignore]
397-
else:
398-
yield pl.from_arrow(record_batch) # type: ignore[misc,unused-ignore]
393+
# The reader holds the connection until it is read to the end or closed, so a source polars stops reading
394+
# early must close it, or the connection stays refused
395+
try:
396+
for record_batch in iter(results.read_next_batch, None):
397+
if fallback_predicate is not None:
398+
# We have a predicate, but did not manage to push it down, we fallback here
399+
yield pl.from_arrow(record_batch).filter(fallback_predicate) # type: ignore[arg-type,misc,unused-ignore]
400+
else:
401+
yield pl.from_arrow(record_batch) # type: ignore[misc,unused-ignore]
402+
finally:
403+
results.close()
399404

400405
return register_io_source(source_generator, schema=schema, is_pure=True)

‎src/common/exceptions.cpp‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -120,6 +120,12 @@ class PyOutOfMemoryException : public OperationalError {
120120
}
121121
};
122122

123+
class PyResourceInUseException : public OperationalError {
124+
public:
125+
explicit PyResourceInUseException(const string &err) : OperationalError(err) {
126+
}
127+
};
128+
123129
class PyConnectionException : public OperationalError {
124130
public:
125131
explicit PyConnectionException(const string &err) : OperationalError(err) {
@@ -287,6 +293,8 @@ void PyThrowException(ErrorData &error, PyObject *http_exception) {
287293
throw PyOutOfMemoryException(error.Message());
288294
case ExceptionType::CONNECTION:
289295
throw PyConnectionException(error.Message());
296+
case ExceptionType::RESOURCE_IN_USE:
297+
throw PyResourceInUseException(error.Message());
290298
case ExceptionType::SERIALIZATION:
291299
throw PySerializationException(error.Message());
292300
case ExceptionType::CONSTRAINT:
@@ -347,6 +355,7 @@ void RegisterExceptions(const nb::module_ &m) {
347355
nb::exception<PyTransactionException>(m, "TransactionException", operational_error);
348356
nb::exception<PyOutOfMemoryException>(m, "OutOfMemoryException", operational_error);
349357
nb::exception<PyConnectionException>(m, "ConnectionException", operational_error);
358+
nb::exception<PyResourceInUseException>(m, "ResourceInUseException", operational_error);
350359
// no object size error
351360
// no null pointer errors
352361
auto io_exception = nb::exception<PyIOException>(m, "IOException", operational_error).ptr();

‎src/engine/connection.cpp‎

Lines changed: 70 additions & 46 deletions
Original file line numberDiff line numberDiff line change
@@ -3,10 +3,7 @@
33
#include "duckdb/main/relation.hpp"
44
#include "duckdb/main/statement_iterator.hpp"
55
#include "duckdb/main/valid_checker.hpp"
6-
#include "duckdb/parser/expression/star_expression.hpp"
7-
#include "duckdb/parser/query_node/select_node.hpp"
86
#include "duckdb/parser/statement/select_statement.hpp"
9-
#include "duckdb/parser/tableref/column_data_ref.hpp"
107
#include "duckdb/transaction/meta_transaction.hpp"
118

129
namespace duckdb {
@@ -16,20 +13,52 @@ shared_ptr<ConnectionState> ConnectionState::Get(ClientContext &context) {
1613
return context.registered_state->GetOrCreate<ConnectionState>("python_connection_state");
1714
}
1815

19-
void ConnectionState::SetOpen(const shared_ptr<Supersedable> &result) {
20-
lock_guard<mutex> guard(lock);
21-
open = result;
16+
static constexpr const char *BUSY_ERROR =
17+
"connection is busy with another call; it refuses instead of waiting, so use a separate connection from cursor()";
18+
static constexpr const char *CALLBACK_ERROR =
19+
"code running inside a query cannot use that query's connection; use a separate connection from cursor()";
20+
21+
static thread_local CallbackScope *innermost_callback = nullptr;
22+
23+
CallbackScope::CallbackScope(const ClientContext &context_p) : context(context_p), outer(innermost_callback) {
24+
innermost_callback = this;
2225
}
2326

24-
void ConnectionState::SupersedeOpen() {
25-
shared_ptr<Supersedable> result;
26-
{
27-
lock_guard<mutex> guard(lock);
28-
result = open.lock();
29-
open.reset();
27+
CallbackScope::~CallbackScope() {
28+
innermost_callback = outer;
29+
}
30+
31+
bool CallbackScope::Inside(const ClientContext &context) {
32+
for (auto scope = innermost_callback; scope; scope = scope->outer) {
33+
if (&scope->context == &context) {
34+
return true;
35+
}
36+
}
37+
return false;
38+
}
39+
40+
ConnectionClaim::ConnectionClaim(ConnectionState &state_p, const ClientContext &context) : state(state_p) {
41+
if (CallbackScope::Inside(context)) {
42+
throw ResourceInUseException(CALLBACK_ERROR);
3043
}
31-
if (result) {
32-
result->Supersede();
44+
if (state.busy.exchange(true)) {
45+
throw ResourceInUseException(BUSY_ERROR);
46+
}
47+
}
48+
49+
ConnectionClaim::ConnectionClaim(ClientContext &context) : ConnectionClaim(*ConnectionState::Get(context), context) {
50+
}
51+
52+
ConnectionClaim::~ConnectionClaim() {
53+
state.busy = false;
54+
}
55+
56+
void ConnectionState::ThrowIfBusy(const ClientContext &context) const {
57+
if (CallbackScope::Inside(context)) {
58+
throw ResourceInUseException(CALLBACK_ERROR);
59+
}
60+
if (busy) {
61+
throw ResourceInUseException(BUSY_ERROR);
3362
}
3463
}
3564

@@ -111,11 +140,29 @@ class PinGuard {
111140

112141
} // namespace
113142

143+
ClaimingContextWrapper::ClaimingContextWrapper(shared_ptr<ClientContextWrapper> inner_p)
144+
: ClientContextWrapper(inner_p->TryGetContext()), inner(std::move(inner_p)) {
145+
}
146+
147+
void ClaimingContextWrapper::TryBindRelation(Relation &relation, vector<ColumnDefinition> &columns) {
148+
auto context = GetContext();
149+
ConnectionClaim claim(*context);
150+
inner->TryBindRelation(relation, columns);
151+
}
152+
153+
void ClaimingContextWrapper::Install(Relation &relation) {
154+
if (!relation.context || dynamic_cast<ClaimingContextWrapper *>(relation.context.get())) {
155+
return;
156+
}
157+
relation.context = make_shared_ptr<ClaimingContextWrapper>(relation.context);
158+
}
159+
114160
Session::Session(shared_ptr<ClientContext> context_p)
115161
: context(std::move(context_p)), state(ConnectionState::Get(*context)) {
116162
}
117163

118164
vector<unique_ptr<SQLStatement>> Session::Parse(const string &sql) {
165+
ConnectionClaim claim(*state, *context);
119166
// The iterator also runs the preprocessor, without which a PRAGMA cannot be prepared
120167
auto iterator = context->IterateStatements(sql);
121168
vector<unique_ptr<SQLStatement>> statements;
@@ -128,7 +175,7 @@ vector<unique_ptr<SQLStatement>> Session::Parse(const string &sql) {
128175
}
129176

130177
Prepared Session::Prepare(unique_ptr<SQLStatement> statement) {
131-
state->SupersedeOpen();
178+
ConnectionClaim claim(*state, *context);
132179
Prepared prepared;
133180
{
134181
ResolvingGuard guard(*state);
@@ -141,63 +188,40 @@ Prepared Session::Prepare(unique_ptr<SQLStatement> statement) {
141188
return prepared;
142189
}
143190

144-
shared_ptr<Result> Session::Track(unique_ptr<QueryResult> handle, const Format &format, shared_ptr<Relation> relation) {
145-
auto result = make_shared_ptr<Result>(std::move(handle), format, std::move(relation));
146-
state->SetOpen(result);
147-
return result;
148-
}
149-
150191
shared_ptr<Result> Session::Submit(unique_ptr<SQLStatement> statement, identifier_map_t<BoundParameterData> &values,
151192
const Format &format) {
152-
state->SupersedeOpen();
193+
ConnectionClaim claim(*state, *context);
153194
auto handle = context->Submit(std::move(statement), values, QueryParameters(format.EngineFormat()));
154-
return Track(std::move(handle), format, nullptr);
195+
return make_shared_ptr<Result>(std::move(handle), format, nullptr);
155196
}
156197

157198
shared_ptr<Result> Session::Submit(Prepared &prepared, identifier_map_t<BoundParameterData> &values,
158199
const Format &format) {
159-
state->SupersedeOpen();
200+
ConnectionClaim claim(*state, *context);
160201
unique_ptr<QueryResult> handle;
161202
{
162203
PinGuard guard(*state, prepared.tables);
163204
handle = prepared.statement->Submit(values, QueryParameters(format.EngineFormat()));
164205
}
165-
return Track(std::move(handle), format, nullptr);
206+
return make_shared_ptr<Result>(std::move(handle), format, nullptr);
166207
}
167208

168209
shared_ptr<Result> Session::Submit(const shared_ptr<Relation> &relation, const Format &format) {
169-
state->SupersedeOpen();
210+
ConnectionClaim claim(*state, *context);
170211
QueryParameters parameters(format.EngineFormat());
171212
if (!relation->IsReadOnly()) {
172213
auto handle = context->Submit(relation, parameters);
173-
return Track(std::move(handle), format, relation);
214+
return make_shared_ptr<Result>(std::move(handle), format, relation);
174215
}
175216
auto select = make_uniq<SelectStatement>();
176217
select->node = relation->GetQueryNode();
177218
select->query = relation->GetQuery();
178219
auto handle = context->Submit(std::move(select), parameters);
179-
return Track(std::move(handle), format, relation);
180-
}
181-
182-
shared_ptr<Result> Session::Submit(unique_ptr<ColumnDataCollection> collection, const vector<Identifier> &names,
183-
const Format &format) {
184-
// The binder rejects duplicate column names; the caller keeps the originals
185-
auto deduplicated_names = names;
186-
QueryResult::DeduplicateColumns(deduplicated_names);
187-
auto table_ref = make_uniq<ColumnDataRef>(std::move(collection), std::move(deduplicated_names));
188-
// Binding asserts on an unset alias
189-
table_ref->alias = "materialized";
190-
auto select_node = make_uniq<SelectNode>();
191-
select_node->select_list.push_back(make_uniq<StarExpression>());
192-
select_node->from_table = std::move(table_ref);
193-
auto select = make_uniq<SelectStatement>();
194-
select->node = std::move(select_node);
195-
identifier_map_t<BoundParameterData> no_values;
196-
return Submit(std::move(select), no_values, format);
220+
return make_shared_ptr<Result>(std::move(handle), format, relation);
197221
}
198222

199223
void Session::Run(unique_ptr<SQLStatement> statement, const InterruptCheck &check) {
200-
state->SupersedeOpen();
224+
ConnectionClaim claim(*state, *context);
201225
auto handle = context->Submit(std::move(statement), QueryParameters());
202226
Complete(*handle, check);
203227
}

0 commit comments

Comments
 (0)