Skip to content

Commit efa73f3

Browse files
authored
duckdb: more const for FFI (#9173)
- Remove unused `duckdb_vx_set_dictionary_vector_length`. - Move comments about duckdb callbacks from C++ to Rust side. - Add const modifiers to some callbacks. - Add helper `get_ffi_*` functions for C++ side. - Move structs to header file #8628 Signed-off-by: Mikhail Kot <[email protected]>
1 parent 4ebe623 commit efa73f3

8 files changed

Lines changed: 115 additions & 127 deletions

File tree

‎vortex-duckdb/cpp/include/table_function.hpp‎

Lines changed: 46 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -3,33 +3,68 @@
33

44
#pragma once
55

6+
#include "data.hpp"
67
#include "duckdb.h"
78
#include "duckdb/function/function.hpp"
89
#include "duckdb/function/table_function.hpp"
910

11+
using namespace duckdb;
12+
1013
static_assert(sizeof(idx_t) == 8);
1114

1215
// We need this exposed to compare function addresses in optimizer.cpp
13-
duckdb::unique_ptr<duckdb::FunctionData>
14-
duckdb_vx_table_function_bind(duckdb::ClientContext &context,
15-
duckdb::TableFunctionBindInput &input,
16-
duckdb::vector<duckdb::LogicalType> &return_types,
17-
duckdb::vector<duckdb::string> &names);
16+
unique_ptr<FunctionData> duckdb_vx_table_function_bind(ClientContext &context,
17+
TableFunctionBindInput &input,
18+
vector<LogicalType> &return_types,
19+
vector<string> &names);
1820

1921
struct TableFunctionProjectionExpressionInput {
20-
const duckdb::LogicalGet &get;
21-
const duckdb::Expression &expression;
22+
const LogicalGet &get;
23+
const Expression &expression;
2224
idx_t projection_idx;
2325
};
2426

2527
// true if we can push down the expression, false otherwise
26-
bool projection_expression_pushdown(duckdb::ClientContext &context,
28+
bool projection_expression_pushdown(ClientContext &context,
2729
const TableFunctionProjectionExpressionInput &input);
2830

2931
struct TableFunctionUngroupedAggregateInput {
30-
const duckdb::LogicalGet &get;
32+
const LogicalGet &get;
3133
// Column scan index -> aggregate expression
32-
const duckdb::vector<std::pair<idx_t, const duckdb::Expression &>> &projections;
34+
const vector<std::pair<idx_t, const Expression &>> &projections;
35+
};
36+
37+
bool aggregate_pushdown(ClientContext &context, const TableFunctionUngroupedAggregateInput &input);
38+
39+
struct VortexBindData final : FunctionData {
40+
VortexBindData(unique_ptr<CData> ffi_data, const vector<LogicalType> &types)
41+
: ffi_data(std::move(ffi_data)), types(types) {
42+
}
43+
unique_ptr<FunctionData> Copy() const override;
44+
bool Equals(const FunctionData &other) const override;
45+
46+
unique_ptr<CData> ffi_data;
47+
vector<LogicalType> types;
48+
};
49+
50+
struct VortexGlobalData final : GlobalTableFunctionState {
51+
explicit VortexGlobalData(unique_ptr<CData> ffi_data) : ffi_data(std::move(ffi_data)) {
52+
}
53+
54+
idx_t MaxThreads() const override {
55+
return GlobalTableFunctionState::MAX_THREADS;
56+
}
57+
58+
unique_ptr<CData> ffi_data;
3359
};
3460

35-
bool aggregate_pushdown(duckdb::ClientContext &context, const TableFunctionUngroupedAggregateInput &input);
61+
struct VortexLocalData final : LocalTableFunctionState {
62+
explicit VortexLocalData(unique_ptr<CData> ffi_data) : ffi_data(std::move(ffi_data)) {
63+
}
64+
unique_ptr<CData> ffi_data;
65+
};
66+
67+
struct VortexBindResults {
68+
vector<LogicalType> &return_types;
69+
vector<string> &names;
70+
};

‎vortex-duckdb/cpp/include/vector.h‎

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -22,8 +22,6 @@ void duckdb_vx_vector_dictionary(duckdb_vector ffi_vector,
2222
duckdb_selection_vector ffi_sel_vec,
2323
idx_t count);
2424

25-
void duckdb_vx_set_dictionary_vector_length(duckdb_vector dict, unsigned int len);
26-
2725
// Reset vector's validity mask to nullptr, making all vector's elements valid.
2826
// vector must not be a DictionaryVector or a SequenceVector
2927
void duckdb_vx_vector_set_all_valid(duckdb_vector ffi_vector);

‎vortex-duckdb/cpp/table_function.cpp‎

Lines changed: 49 additions & 99 deletions
Original file line numberDiff line numberDiff line change
@@ -20,55 +20,43 @@
2020
#include "duckdb/planner/operator/logical_get.hpp"
2121

2222
using namespace std::string_literals;
23-
using namespace duckdb;
2423
constexpr column_t COLUMN_IDENTIFIER_FILE_INDEX = MultiFileReader::COLUMN_IDENTIFIER_FILE_INDEX;
2524
constexpr column_t COLUMN_IDENTIFIER_FILE_ROW_NUMBER = MultiFileReader::COLUMN_IDENTIFIER_FILE_ROW_NUMBER;
2625

27-
struct CTableBindData final : FunctionData {
28-
CTableBindData(unique_ptr<CData> ffi_data_p, const vector<LogicalType> &types)
29-
: ffi_data(std::move(ffi_data_p)), types(types) {
30-
}
31-
unique_ptr<FunctionData> Copy() const override;
32-
bool Equals(const FunctionData &other_base) const override;
33-
34-
unique_ptr<CData> ffi_data;
35-
vector<LogicalType> types;
36-
};
37-
38-
unique_ptr<FunctionData> CTableBindData::Copy() const {
26+
unique_ptr<FunctionData> VortexBindData::Copy() const {
3927
const auto copied_ffi_data = duckdb_table_function_bind_data_clone(ffi_data->DataPtr());
4028
auto ffi_data_p = unique_ptr<CData>(reinterpret_cast<CData *>(copied_ffi_data));
41-
return make_uniq<CTableBindData>(std::move(ffi_data_p), types);
29+
return make_uniq<VortexBindData>(std::move(ffi_data_p), types);
4230
}
4331

44-
bool CTableBindData::Equals(const FunctionData &other_base) const {
45-
const CTableBindData &other = other_base.Cast<CTableBindData>();
32+
bool VortexBindData::Equals(const FunctionData &other_base) const {
33+
const VortexBindData &other = other_base.Cast<VortexBindData>();
4634
// if "types" are different, "ffi_data" would also be different as it
4735
// contains types inside, so omit "types" from comparison.
4836
return ffi_data.get() == other.ffi_data.get();
4937
}
5038

51-
struct CTableGlobalData final : GlobalTableFunctionState {
52-
explicit CTableGlobalData(unique_ptr<CData> ffi_data) : ffi_data(std::move(ffi_data)) {
53-
}
54-
55-
idx_t MaxThreads() const override {
56-
return GlobalTableFunctionState::MAX_THREADS;
57-
}
39+
// This is a flaw of Duckdb API which doesn't allow passing non-const
40+
// expressions. We never modify the value on Rust side.
41+
static duckdb_vx_expr get_ffi_expr(const Expression &expr) {
42+
return reinterpret_cast<duckdb_vx_expr>(const_cast<Expression *>(&expr));
43+
}
5844

59-
unique_ptr<CData> ffi_data;
60-
};
45+
static void *get_ffi_bind(const FunctionData *bind_data) {
46+
return bind_data->Cast<VortexBindData>().ffi_data->DataPtr();
47+
}
6148

62-
struct CTableLocalData final : LocalTableFunctionState {
63-
explicit CTableLocalData(unique_ptr<CData> ffi_data) : ffi_data(std::move(ffi_data)) {
64-
}
49+
static void *get_ffi_global(GlobalTableFunctionState *state) {
50+
return state->Cast<VortexGlobalData>().ffi_data->DataPtr();
51+
}
6552

66-
unique_ptr<CData> ffi_data;
67-
};
53+
static void *get_ffi_local(LocalTableFunctionState *state) {
54+
return state->Cast<VortexLocalData>().ffi_data->DataPtr();
55+
}
6856

6957
double
7058
table_scan_progress(ClientContext &, const FunctionData *, const GlobalTableFunctionState *global_state) {
71-
void *const c_global_state = global_state->Cast<CTableGlobalData>().ffi_data->DataPtr();
59+
void *const c_global_state = global_state->Cast<VortexGlobalData>().ffi_data->DataPtr();
7260
return duckdb_table_function_scan_progress(c_global_state);
7361
}
7462

@@ -125,8 +113,8 @@ unique_ptr<BaseStatistics> statistics(ClientContext &, const FunctionData *bind_
125113
return {};
126114
}
127115

128-
const auto &bind = bind_data->Cast<CTableBindData>();
129-
void *const ffi_bind = bind.ffi_data->DataPtr();
116+
const auto &bind = bind_data->Cast<VortexBindData>();
117+
const void *const ffi_bind = get_ffi_bind(bind_data);
130118

131119
duckdb_column_statistics statistics = {};
132120
if (!duckdb_table_function_statistics(ffi_bind, column_index, &statistics)) {
@@ -167,22 +155,9 @@ unique_ptr<BaseStatistics> statistics(ClientContext &, const FunctionData *bind_
167155
}
168156
}
169157

170-
struct CTableBindResult {
171-
vector<LogicalType> &return_types;
172-
vector<string> &names;
173-
};
174-
175-
// This is a flaw of Duckdb API which doesn't allow passing non-const
176-
// expressions. We never modify the value on Rust side.
177-
static duckdb_vx_expr get_ffi_expr(const Expression &expr) {
178-
return reinterpret_cast<duckdb_vx_expr>(const_cast<Expression *>(&expr));
179-
}
180-
181158
bool projection_expression_pushdown(ClientContext &, const TableFunctionProjectionExpressionInput &input) {
182-
const auto &bind = input.get.bind_data->Cast<CTableBindData>();
183-
184159
duckdb_vx_expr ffi_expr = get_ffi_expr(input.expression);
185-
void *const ffi_bind = bind.ffi_data->DataPtr();
160+
void *const ffi_bind = get_ffi_bind(input.get.bind_data.get());
186161
duckdb_vx_error error_out = nullptr;
187162

188163
const bool ret = duckdb_table_function_pushdown_projection_expression( //
@@ -211,8 +186,7 @@ duckdb_vx_expr duckdb_vx_aggregate_at(duckdb_vx_agg_input ffi_input, idx_t i, id
211186
}
212187

213188
bool aggregate_pushdown(ClientContext &, const TableFunctionUngroupedAggregateInput &input) {
214-
const auto &bind = input.get.bind_data->Cast<CTableBindData>();
215-
void *const ffi_bind = bind.ffi_data->DataPtr();
189+
void *const ffi_bind = get_ffi_bind(input.get.bind_data.get());
216190
duckdb_vx_error error_out = nullptr;
217191
const auto ffi_input =
218192
reinterpret_cast<duckdb_vx_agg_input>(const_cast<TableFunctionUngroupedAggregateInput *>(&input));
@@ -223,34 +197,29 @@ bool aggregate_pushdown(ClientContext &, const TableFunctionUngroupedAggregateIn
223197
return res;
224198
}
225199

226-
/**
227-
* Called for every new query. For example, if there is a VIEW over *.vortex,
228-
* and after a query another file is added matching the glob, for second query
229-
* bind() will be called again.
230-
*/
231200
unique_ptr<FunctionData> duckdb_vx_table_function_bind(ClientContext &,
232201
TableFunctionBindInput &input,
233202
vector<LogicalType> &return_types,
234203
vector<string> &names) {
235-
CTableBindResult result = {return_types, names};
204+
VortexBindResults result = {return_types, names};
236205

237206
duckdb_vx_error error_out = nullptr;
238-
auto ffi_bind_data = duckdb_table_function_bind(reinterpret_cast<duckdb_vx_tfunc_bind_input>(&input),
239-
reinterpret_cast<duckdb_vx_tfunc_bind_result>(&result),
240-
&error_out);
207+
duckdb_vx_tfunc_bind_input bind_input = reinterpret_cast<duckdb_vx_tfunc_bind_input>(&input);
208+
duckdb_vx_tfunc_bind_result bind_result = reinterpret_cast<duckdb_vx_tfunc_bind_result>(&result);
209+
duckdb_vx_data ffi_bind_data = duckdb_table_function_bind(bind_input, bind_result, &error_out);
241210
if (error_out) {
242211
throw BinderException(IntoErrString(error_out));
243212
}
244213

245214
auto cdata = unique_ptr<CData>(reinterpret_cast<CData *>(ffi_bind_data));
246-
return make_uniq<CTableBindData>(std::move(cdata), return_types);
215+
return make_uniq<VortexBindData>(std::move(cdata), return_types);
247216
}
248217

249-
unique_ptr<GlobalTableFunctionState> c_init_global(ClientContext &context, TableFunctionInitInput &input) {
250-
const CTableBindData &bind = input.bind_data->Cast<CTableBindData>();
218+
unique_ptr<GlobalTableFunctionState> init_global(ClientContext &context, TableFunctionInitInput &input) {
219+
const void *const ffi_bind = get_ffi_bind(input.bind_data.get());
251220

252221
duckdb_vx_tfunc_init_input ffi_input = {
253-
.bind_data = bind.ffi_data->DataPtr(),
222+
.bind_data = ffi_bind,
254223
.column_ids = input.column_ids.data(),
255224
.column_ids_count = input.column_ids.size(),
256225
.projection_ids = input.projection_ids.data(),
@@ -266,22 +235,22 @@ unique_ptr<GlobalTableFunctionState> c_init_global(ClientContext &context, Table
266235
}
267236

268237
auto cdata = unique_ptr<CData>(reinterpret_cast<CData *>(ffi_global_data));
269-
return make_uniq<CTableGlobalData>(std::move(cdata));
238+
return make_uniq<VortexGlobalData>(std::move(cdata));
270239
}
271240

272241
unique_ptr<LocalTableFunctionState>
273242
init_local(ExecutionContext &, TableFunctionInitInput &input, GlobalTableFunctionState *global_state) {
274-
const void *const ffi_bind = input.bind_data->Cast<CTableBindData>().ffi_data->DataPtr();
275-
void *const ffi_global = global_state->Cast<CTableGlobalData>().ffi_data->DataPtr();
243+
const void *const ffi_bind = get_ffi_bind(input.bind_data.get());
244+
void *const ffi_global = get_ffi_global(global_state);
276245

277246
duckdb_vx_data ffi_local_data = duckdb_table_function_init_local(ffi_bind, ffi_global);
278247
auto cdata = unique_ptr<CData>(reinterpret_cast<CData *>(ffi_local_data));
279-
return make_uniq<CTableLocalData>(std::move(cdata));
248+
return make_uniq<VortexLocalData>(std::move(cdata));
280249
}
281250

282251
void function(ClientContext &, TableFunctionInput &input, DataChunk &output) {
283-
void *const ffi_global = input.global_state->Cast<CTableGlobalData>().ffi_data->DataPtr();
284-
void *const ffi_local = input.local_state->Cast<CTableLocalData>().ffi_data->DataPtr();
252+
void *const ffi_global = get_ffi_global(input.global_state.get());
253+
void *const ffi_local = get_ffi_local(input.local_state.get());
285254

286255
duckdb_data_chunk chunk = reinterpret_cast<duckdb_data_chunk>(&output);
287256
duckdb_vx_error error_out = nullptr;
@@ -293,21 +262,8 @@ void function(ClientContext &, TableFunctionInput &input, DataChunk &output) {
293262

294263
using FilterVec = vector<unique_ptr<Expression>>;
295264

296-
/*
297-
* Table filter pushdown is used for two tasks in duckdb:
298-
*
299-
* 1. Prune files based on filename or hive partitioning, see Parquet
300-
* filter pushdown. We don't use this because we do own file-level pruning in
301-
* FileStatsLayoutReader, and we don't support hive partitioning yet.
302-
*
303-
* 2. Avoid reading unused file data. Filter expressions are pushed to Vortex,
304-
* converted to Vortex expressions and used during the scan.
305-
* Duckdb pushes a subset of expressions i.e. equality operators, and also
306-
* expressions which return true in pushdown_expression.
307-
*/
308265
void pushdown_complex_filter(const FunctionData &bind_data, FilterVec &filters) {
309-
const auto &bind = bind_data.Cast<CTableBindData>();
310-
void *const ffi_bind = bind.ffi_data->DataPtr();
266+
void *const ffi_bind = get_ffi_bind(&bind_data);
311267
duckdb_vx_error error_out = nullptr;
312268

313269
for (auto iter = filters.begin(); iter != filters.end();) {
@@ -321,11 +277,11 @@ void pushdown_complex_filter(const FunctionData &bind_data, FilterVec &filters)
321277
}
322278
}
323279

324-
unique_ptr<NodeStatistics> c_cardinality(ClientContext &, const FunctionData *bind_data) {
325-
auto &bind = bind_data->Cast<CTableBindData>();
280+
unique_ptr<NodeStatistics> cardinality(ClientContext &, const FunctionData *bind_data) {
281+
const void *const ffi_bind = get_ffi_bind(bind_data);
326282

327283
duckdb_vx_node_statistics stats = {};
328-
duckdb_table_function_cardinality(bind.ffi_data->DataPtr(), &stats);
284+
duckdb_table_function_cardinality(ffi_bind, &stats);
329285

330286
auto out = make_uniq<NodeStatistics>();
331287
out->has_estimated_cardinality = stats.has_estimated_cardinality;
@@ -350,7 +306,7 @@ extern "C" void duckdb_vx_tfunc_bind_result_add_column(duckdb_vx_tfunc_bind_resu
350306
D_ASSERT(ffi_result);
351307
D_ASSERT(name_str);
352308
D_ASSERT(ffi_type);
353-
const CTableBindResult &result = *reinterpret_cast<CTableBindResult *>(ffi_result);
309+
const VortexBindResults &result = *reinterpret_cast<VortexBindResults *>(ffi_result);
354310
const LogicalType logical_type = *reinterpret_cast<LogicalType *>(ffi_type);
355311

356312
result.names.emplace_back(name_str, name_len);
@@ -373,15 +329,9 @@ TablePartitionInfo get_partition_info(ClientContext &, TableFunctionPartitionInp
373329
: TablePartitionInfo::NOT_PARTITIONED;
374330
}
375331

376-
/**
377-
* Duckdb requests this function after exporting the chunk. We answer with
378-
* partition_index we have exported as well as information about constant
379-
* columns in this partition. As data is partitioned by array exporters, in
380-
* each partition ~ exported array file_index is constant.
381-
*/
382332
OperatorPartitionData get_partition_data(ClientContext &, TableFunctionGetPartitionInput &input) {
383-
void *const ffi_global = input.global_state->Cast<CTableGlobalData>().ffi_data->DataPtr();
384-
void *const ffi_local = input.local_state->Cast<CTableLocalData>().ffi_data->DataPtr();
333+
void *const ffi_global = get_ffi_global(input.global_state.get());
334+
void *const ffi_local = get_ffi_local(input.local_state.get());
385335
duckdb_vx_partition_data partition_data;
386336
duckdb_table_function_get_partition_data(ffi_global, ffi_local, &partition_data);
387337

@@ -408,16 +358,16 @@ extern "C" void duckdb_vx_string_map_insert(duckdb_vx_string_map map, const char
408358
reinterpret_cast<InsertionOrderPreservingMap<string> *>(map)->insert(key, value);
409359
}
410360

411-
InsertionOrderPreservingMap<string> c_to_string(TableFunctionToStringInput &input) {
361+
InsertionOrderPreservingMap<string> to_string(TableFunctionToStringInput &input) {
412362
InsertionOrderPreservingMap<string> result;
413363
duckdb_vx_string_map ffi_map = reinterpret_cast<duckdb_vx_string_map>(&result);
414-
void *const ffi_bind = input.bind_data->Cast<CTableBindData>().ffi_data->DataPtr();
364+
const void *const ffi_bind = get_ffi_bind(input.bind_data.get());
415365
duckdb_table_function_to_string(ffi_bind, ffi_map);
416366
return result;
417367
}
418368

419369
duckdb_state register_table_function(DatabaseInstance &db, LogicalType parameter, const std::string &name) {
420-
TableFunction tf(name, {}, function, duckdb_vx_table_function_bind, c_init_global, init_local);
370+
TableFunction tf(name, {}, function, duckdb_vx_table_function_bind, init_global, init_local);
421371

422372
tf.projection_pushdown = true;
423373
tf.filter_pushdown = true;
@@ -430,10 +380,10 @@ duckdb_state register_table_function(DatabaseInstance &db, LogicalType parameter
430380
tf.pushdown_complex_filter = [](auto &, auto &, FunctionData *bind_data, FilterVec &filters) {
431381
pushdown_complex_filter(*bind_data, filters);
432382
};
433-
tf.cardinality = c_cardinality;
383+
tf.cardinality = cardinality;
434384
tf.get_partition_info = get_partition_info;
435385
tf.get_partition_data = get_partition_data;
436-
tf.to_string = c_to_string;
386+
tf.to_string = to_string;
437387
tf.table_scan_progress = table_scan_progress;
438388
tf.statistics = statistics;
439389

‎vortex-duckdb/cpp/vector.cpp‎

Lines changed: 0 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -26,11 +26,6 @@ extern "C" void duckdb_vx_vector_dictionary(duckdb_vector ffi_vector,
2626
vector->Dictionary(*dict, dictionary_size, *sel_vec, count);
2727
}
2828

29-
extern "C" void duckdb_vx_set_dictionary_vector_length(duckdb_vector dict, unsigned int len) {
30-
auto ddict = reinterpret_cast<duckdb::Vector *>(dict);
31-
ddict->GetBuffer()->Cast<DictionaryBuffer>().SetDictionarySize(len);
32-
}
33-
3429
extern "C" void
3530
duckdb_vx_sequence_vector(duckdb_vector c_vector, int64_t start, int64_t step, idx_t capacity) {
3631
auto vector = reinterpret_cast<Vector *>(c_vector);

0 commit comments

Comments
 (0)