Skip to content

Commit 6f10740

Browse files
authored
Configure DuckDB native threads for benchmarking (#6512)
Benchmarking takes a "threads" flag. We used to only set this on the Vortex table function, not the rest of DuckDB. This was different to how the same flag behaves in DataFusion which configures the entire Tokio runtime with n threads. Signed-off-by: Nicholas Gates <[email protected]>
1 parent 82d578c commit 6f10740

3 files changed

Lines changed: 9 additions & 109 deletions

File tree

‎benchmarks/duckdb-bench/src/lib.rs‎

Lines changed: 5 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,6 @@ use vortex_bench::generate_duckdb_registration_sql;
1717
use vortex_duckdb::duckdb::Config;
1818
use vortex_duckdb::duckdb::Connection;
1919
use vortex_duckdb::duckdb::Database;
20-
use vortex_duckdb::register_extension_options;
2120

2221
/// DuckDB context for benchmarks.
2322
pub struct DuckClient {
@@ -67,10 +66,12 @@ impl DuckClient {
6766
path: Option<PathBuf>,
6867
threads: Option<usize>,
6968
) -> Result<(Database, Connection)> {
70-
let config = Config::new().vortex_expect("failed to create duckdb config");
69+
let mut config = Config::new().vortex_expect("failed to create duckdb config");
7170

72-
// Register Vortex extension options before creating connection
73-
register_extension_options(&config);
71+
// Set DuckDB thread count if specified
72+
if let Some(thread_count) = threads {
73+
config.set("threads", &format!("{}", thread_count))?;
74+
}
7475

7576
let db = match path {
7677
Some(path) => Database::open_with_config(path, config),
@@ -91,11 +92,6 @@ impl DuckClient {
9192
// parquet_metadata_cache" when running DuckDB in debug mode.
9293
connection.query("SET parquet_metadata_cache = true")?;
9394

94-
// Set vortex_max_threads if specified
95-
if let Some(thread_count) = threads {
96-
connection.query(&format!("SET vortex_max_threads = {}", thread_count))?;
97-
}
98-
9995
Ok((db, connection))
10096
}
10197

‎vortex-duckdb/src/lib.rs‎

Lines changed: 0 additions & 73 deletions
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,6 @@ use vortex::io::session::RuntimeSessionExt;
1616
use vortex::session::VortexSession;
1717

1818
use crate::copy::VortexCopyFunction;
19-
use crate::duckdb::Config;
2019
pub use crate::duckdb::Connection;
2120
pub use crate::duckdb::Database;
2221
pub use crate::duckdb::LogicalType;
@@ -44,36 +43,6 @@ static RUNTIME: LazyLock<CurrentThreadRuntime> = LazyLock::new(CurrentThreadRunt
4443
static SESSION: LazyLock<VortexSession> =
4544
LazyLock::new(|| VortexSession::default().with_handle(RUNTIME.handle()));
4645

47-
/// Register Vortex extension configuration options with DuckDB.
48-
/// This must be called before `register_table_functions` to take effect.
49-
pub fn register_extension_options(config: &Config) {
50-
let logical_type = LogicalType::uint64();
51-
52-
let default_threads = std::thread::available_parallelism()
53-
.map(|n| n.get() as u64)
54-
.unwrap_or(1);
55-
let default_value = Value::from(default_threads);
56-
57-
// Register the vortex_max_threads extension option
58-
// SAFETY: We're passing valid pointers for database, logical_type, and default_value
59-
// The C++ code will copy the LogicalType and Value, so we can safely drop them after this call
60-
let result = unsafe {
61-
cpp::duckdb_vx_add_extension_option(
62-
config.as_ptr(),
63-
c"vortex_max_threads".as_ptr(),
64-
c"Maximum number of threads for Vortex table scans".as_ptr(),
65-
logical_type.as_ptr(),
66-
default_value.as_ptr(),
67-
)
68-
};
69-
70-
assert_eq!(
71-
result,
72-
cpp::duckdb_state::DuckDBSuccess,
73-
"Failed to register vortex_max_threads extension option"
74-
);
75-
}
76-
7746
/// Initialize the Vortex extension by registering the extension functions.
7847
/// Note: This also registers extension options. If you want to register options
7948
/// separately (e.g., before creating connections), call `register_extension_options` first.
@@ -124,45 +93,3 @@ pub extern "C" fn vortex_extension_version_rust() -> *const c_char {
12493
}
12594
.as_ptr()
12695
}
127-
128-
#[cfg(test)]
129-
mod tests {
130-
use std::ffi::CString;
131-
132-
use super::*;
133-
use crate::duckdb::Config;
134-
use crate::duckdb::Database;
135-
136-
#[test]
137-
fn test_vortex_max_threads_option_registration() {
138-
let config = Config::new().expect("Failed to create config");
139-
register_extension_options(&config);
140-
let db = Database::open_in_memory_with_config(config).expect("Failed to open database");
141-
142-
let conn = db.connect().expect("Failed to connect");
143-
144-
let _result1 = conn
145-
.query("SET vortex_max_threads = 4")
146-
.expect("Failed to set vortex_max_threads - option may not be registered");
147-
148-
let max_threads_cstr = CString::new("vortex_max_threads").unwrap();
149-
let ctx = conn.client_context().vortex_expect("ctx exists");
150-
assert_eq!(
151-
ctx.try_get_current_setting(&max_threads_cstr)
152-
.unwrap()
153-
.to_string(),
154-
"4"
155-
);
156-
157-
let _result2 = conn
158-
.query("SET vortex_max_threads = 8")
159-
.expect("Failed to set vortex_max_threads to 8");
160-
161-
assert_eq!(
162-
ctx.try_get_current_setting(&max_threads_cstr)
163-
.unwrap()
164-
.to_string(),
165-
"8"
166-
);
167-
}
168-
}

‎vortex-duckdb/src/scan.rs‎

Lines changed: 4 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,6 @@
22
// SPDX-FileCopyrightText: Copyright the Vortex contributors
33

44
use std::cmp::max;
5-
use std::ffi::CString;
65
use std::fmt;
76
use std::fmt::Debug;
87
use std::fmt::Formatter;
@@ -62,7 +61,6 @@ use crate::duckdb::BindResult;
6261
use crate::duckdb::Cardinality;
6362
use crate::duckdb::ClientContext;
6463
use crate::duckdb::DataChunk;
65-
use crate::duckdb::ExtractedValue;
6664
use crate::duckdb::LogicalType;
6765
use crate::duckdb::TableFunction;
6866
use crate::duckdb::TableInitInput;
@@ -79,7 +77,6 @@ pub struct VortexBindData {
7977
file_urls: Vec<Url>,
8078
column_names: Vec<String>,
8179
column_types: Vec<LogicalType>,
82-
max_threads: u64,
8380
}
8481

8582
impl Clone for VortexBindData {
@@ -92,7 +89,6 @@ impl Clone for VortexBindData {
9289
file_urls: self.file_urls.clone(),
9390
column_names: self.column_names.clone(),
9491
column_types: self.column_types.clone(),
95-
max_threads: self.max_threads,
9692
}
9793
}
9894
}
@@ -260,24 +256,6 @@ impl TableFunction for VortexTableFunction {
260256
.get_parameter(0)
261257
.ok_or_else(|| vortex_err!("Missing file glob parameter"))?;
262258

263-
// Read the vortex_max_threads setting from DuckDB configuration
264-
let max_threads_cstr = CString::new("vortex_max_threads")
265-
.map_err(|e| vortex_err!("Invalid setting name: {}", e))?;
266-
let max_threads = ctx
267-
.try_get_current_setting(&max_threads_cstr)
268-
.and_then(|v| match v.as_ref().extract() {
269-
ExtractedValue::UBigInt(val) => usize::try_from(val).ok(),
270-
ExtractedValue::BigInt(val) if val > 0 => usize::try_from(val as u64).ok(),
271-
_ => None,
272-
})
273-
.unwrap_or_else(|| {
274-
std::thread::available_parallelism()
275-
.map(|n| n.get())
276-
.unwrap_or(1)
277-
});
278-
279-
tracing::trace!("running scan with max_threads {max_threads}");
280-
281259
let (file_urls, _metadata) = RUNTIME.block_on(Compat::new(expand_glob(
282260
file_glob_string.as_ref().as_string(),
283261
)))?;
@@ -309,7 +287,6 @@ impl TableFunction for VortexTableFunction {
309287
filter_exprs: vec![],
310288
column_names,
311289
column_types,
312-
max_threads: max_threads as u64,
313290
})
314291
}
315292

@@ -389,13 +366,13 @@ impl TableFunction for VortexTableFunction {
389366
.map_or_else(|| "true".to_string(), |f| f.to_string())
390367
);
391368

392-
// Use the max_threads from bind_data (read from vortex_max_threads setting)
393-
#[expect(clippy::cast_possible_truncation, reason = "max_threads fits in usize")]
394-
let num_workers = bind_data.max_threads as usize;
395-
396369
let client_context = init_input.client_context()?;
397370
let object_cache = client_context.object_cache();
398371

372+
let num_workers = std::thread::available_parallelism()
373+
.map(|n| n.get())
374+
.unwrap_or(1);
375+
399376
let handle = RUNTIME.handle();
400377
let first_file = bind_data.first_file.clone();
401378
let scan_streams = stream::iter(bind_data.file_urls.clone())

0 commit comments

Comments
 (0)