11// SPDX-License-Identifier: Apache-2.0
22// SPDX-FileCopyrightText: Copyright the Vortex contributors
3+
4+ #include " copy_function.hpp"
35#include " data.hpp"
46#include " error.hpp"
57#include " vortex_duckdb.h"
68#include " table_function.h"
79#include " vortex.h"
8- #include " duckdb/function/copy_function .hpp"
10+ #include " duckdb/common/types/column/column_data_collection .hpp"
911#include " duckdb/main/capi/capi_internal.hpp"
1012#include " duckdb/main/client_context.hpp"
1113#include " duckdb/main/connection.hpp"
1214#include " duckdb/parser/parsed_data/create_copy_function_info.hpp"
1315
14- using namespace duckdb ;
15-
16- struct CopyBindData final : TableFunctionData {
17- CopyBindData (unique_ptr<CData> ffi_data) : ffi_data(std::move(ffi_data)) {
18- }
19- unique_ptr<CData> ffi_data;
20- };
21-
22- struct CopyGlobalData final : GlobalFunctionData {
23- CopyGlobalData (unique_ptr<CData> ffi_data) : ffi_data(std::move(ffi_data)) {
24- }
25-
26- unique_ptr<CData> ffi_data;
27- };
28-
2916unique_ptr<FunctionData> copy_to_bind (ClientContext &,
3017 CopyFunctionBindInput &,
31- const vector<string> &column_names ,
32- const vector<LogicalType> &column_types ) {
33- vector<const char *> ffi_column_names (column_names .size ());
34- for (size_t i = 0 ; i < column_names .size (); ++i) {
35- ffi_column_names [i] = column_names [i].c_str ();
18+ const vector<string> &names ,
19+ const vector<LogicalType> &types ) {
20+ vector<const char *> ffi_names (names .size ());
21+ for (size_t i = 0 ; i < names .size (); ++i) {
22+ ffi_names [i] = names [i].c_str ();
3623 }
3724
38- vector<duckdb_logical_type> ffi_column_types (column_types .size ());
39- for (size_t i = 0 ; i < column_types .size (); ++i) {
25+ vector<duckdb_logical_type> ffi_types (types .size ());
26+ for (size_t i = 0 ; i < types .size (); ++i) {
4027 // duckdb C api doesn't allow passing const LogicalTypes. We never
4128 // modify input in copy function.
42- ffi_column_types[i] =
43- reinterpret_cast <duckdb_logical_type>(const_cast <LogicalType *>(&column_types[i]));
29+ ffi_types[i] = reinterpret_cast <duckdb_logical_type>(const_cast <LogicalType *>(&types[i]));
4430 }
4531
4632 duckdb_vx_error error_out = nullptr ;
47- const duckdb_vx_data ffi_bind_data = duckdb_copy_function_copy_to_bind (ffi_column_names .data (),
48- ffi_column_names .size (),
49- ffi_column_types .data (),
50- ffi_column_types .size (),
33+ const duckdb_vx_data ffi_bind_data = duckdb_copy_function_copy_to_bind (ffi_names .data (),
34+ ffi_names .size (),
35+ ffi_types .data (),
36+ ffi_types .size (),
5137 &error_out);
5238 if (error_out) {
5339 throw BinderException (IntoErrString (error_out));
5440 }
5541 auto cdata = unique_ptr<CData>(reinterpret_cast <CData *>(ffi_bind_data));
56- return make_uniq<CopyBindData >(std::move (cdata));
42+ return make_uniq<VortexCopyBindData >(std::move (cdata));
5743}
5844
5945unique_ptr<GlobalFunctionData>
6046copy_to_initialize_global (ClientContext &, FunctionData &bind_data, const string &file_path) {
61- void *const ffi_bind = bind_data.Cast <CopyBindData>().ffi_data ->DataPtr ();
47+ const VortexCopyBindData &bind = bind_data.Cast <VortexCopyBindData>();
48+ const void *const ffi_bind = bind.ffi_bind ->DataPtr ();
6249
6350 duckdb_vx_error error_out = nullptr ;
6451 const duckdb_vx_data ffi_global =
@@ -68,17 +55,21 @@ copy_to_initialize_global(ClientContext &, FunctionData &bind_data, const string
6855 }
6956
7057 auto cdata = unique_ptr<CData>(reinterpret_cast <CData *>(ffi_global));
71- return make_uniq<CopyGlobalData >(std::move (cdata));
58+ return make_uniq<VortexCopyGlobalState >(std::move (cdata));
7259}
7360
7461void copy_to_sink (ExecutionContext &,
7562 FunctionData &bind_data,
7663 GlobalFunctionData &gstate,
7764 LocalFunctionData &,
7865 DataChunk &input) {
79- void *const ffi_bind = bind_data.Cast <CopyBindData>().ffi_data ->DataPtr ();
80- void *const ffi_global = gstate.Cast <CopyGlobalData>().ffi_data ->DataPtr ();
81- auto ffi_chunk = reinterpret_cast <duckdb_data_chunk>(&input);
66+ const VortexCopyBindData &bind = bind_data.Cast <VortexCopyBindData>();
67+ const VortexCopyGlobalState &global = gstate.Cast <VortexCopyGlobalState>();
68+
69+ const void *const ffi_bind = bind.ffi_bind ->DataPtr ();
70+ const void *const ffi_global = global.ffi_global ->DataPtr ();
71+
72+ duckdb_data_chunk ffi_chunk = reinterpret_cast <duckdb_data_chunk>(&input);
8273 duckdb_vx_error error_out = nullptr ;
8374 duckdb_copy_function_copy_to_sink (ffi_bind, ffi_global, ffi_chunk, &error_out);
8475 if (error_out) {
@@ -87,14 +78,52 @@ void copy_to_sink(ExecutionContext &,
8778}
8879
8980void copy_to_finalize (ClientContext &, FunctionData &, GlobalFunctionData &gstate) {
90- void *const ffi_global = gstate.Cast <CopyGlobalData>().ffi_data ->DataPtr ();
81+ const VortexCopyGlobalState &global = gstate.Cast <VortexCopyGlobalState>();
82+
83+ void *const ffi_global = global.ffi_global ->DataPtr ();
9184 duckdb_vx_error error_out = nullptr ;
9285 duckdb_copy_function_copy_to_finalize (ffi_global, &error_out);
9386 if (error_out) {
9487 throw ExecutorException (IntoErrString (error_out));
9588 }
9689}
9790
91+ unique_ptr<PreparedBatchData> copy_to_prepare_batch (ClientContext &,
92+ FunctionData &bind_data,
93+ GlobalFunctionData &,
94+ unique_ptr<ColumnDataCollection> collection) {
95+ const VortexCopyBindData &bind = bind_data.Cast <VortexCopyBindData>();
96+
97+ const void *const ffi_bind = bind.ffi_bind ->DataPtr ();
98+ auto ffi_batch = unique_ptr<CData>(reinterpret_cast <CData *>(duckdb_copy_function_prepare_batch_new ()));
99+ duckdb_vx_error error_out = nullptr ;
100+
101+ for (DataChunk &chunk : collection->Chunks ()) {
102+ duckdb_data_chunk ffi_chunk = reinterpret_cast <duckdb_data_chunk>(&chunk);
103+ duckdb_copy_function_prepare_batch_push (ffi_bind, ffi_batch->DataPtr (), ffi_chunk, &error_out);
104+ if (error_out) {
105+ throw ExecutorException (IntoErrString (error_out));
106+ }
107+ }
108+ return make_uniq<VortexCopyPreparedBatchData>(std::move (ffi_batch));
109+ }
110+
111+ void copy_to_flush_batch (ClientContext &,
112+ FunctionData &,
113+ GlobalFunctionData &gstate,
114+ PreparedBatchData &batch_data) {
115+ const VortexCopyGlobalState &global = gstate.Cast <VortexCopyGlobalState>();
116+ const VortexCopyPreparedBatchData &batch = batch_data.Cast <VortexCopyPreparedBatchData>();
117+
118+ const void *const ffi_global = global.ffi_global ->DataPtr ();
119+ const void *const ffi_batch = batch.ffi_copy_prepared ->DataPtr ();
120+ duckdb_vx_error error_out = nullptr ;
121+ duckdb_copy_function_flush_batch (ffi_global, ffi_batch, &error_out);
122+ if (error_out) {
123+ throw ExecutorException (IntoErrString (error_out));
124+ }
125+ }
126+
98127extern " C" duckdb_state duckdb_vx_register_copy_function (duckdb_database ffi_db) {
99128 D_ASSERT (ffi_db);
100129 const DatabaseWrapper &wrapper = *reinterpret_cast <DatabaseWrapper *>(ffi_db);
@@ -103,18 +132,29 @@ extern "C" duckdb_state duckdb_vx_register_copy_function(duckdb_database ffi_db)
103132 CopyFunction fn (" vortex" );
104133 fn.copy_to_bind = copy_to_bind;
105134 fn.copy_to_initialize_global = copy_to_initialize_global;
135+ // required by duckdb
106136 fn.copy_to_initialize_local = [](auto &, auto &) {
107137 return make_uniq<LocalFunctionData>();
108138 };
109139 fn.copy_to_sink = copy_to_sink;
140+ // required by duckdb for PARTITION_BY
141+ fn.copy_to_combine = [](ExecutionContext &, FunctionData &, GlobalFunctionData &, LocalFunctionData &) {
142+ };
110143 fn.copy_to_finalize = copy_to_finalize;
144+ fn.prepare_batch = copy_to_prepare_batch;
145+ fn.flush_batch = copy_to_flush_batch;
111146 fn.extension = " vortex" ;
112147
113- // TODO(joe): expose this via c our api
114- fn.execution_mode = [](bool , bool ) {
115- return CopyFunctionExecutionMode::REGULAR_COPY_TO_FILE ;
148+ fn.execution_mode = [](bool preserve_insertion_order, bool supports_batch_index) {
149+ using enum CopyFunctionExecutionMode;
150+ if (!preserve_insertion_order) {
151+ return PARALLEL_COPY_TO_FILE ;
152+ }
153+ if (supports_batch_index) {
154+ return BATCH_COPY_TO_FILE ;
155+ }
156+ return REGULAR_COPY_TO_FILE ;
116157 };
117- // TODO(joe): handle parameters as in table_function
118158
119159 try {
120160 Catalog &system_catalog = Catalog::GetSystemCatalog (db);
0 commit comments