Skip to content

Commit 5455409

Browse files
authored
Drop DuckDB file scan state outside the multi-file lock (#10486)
DuckDB releases its last reference to a file reader while holding the global multi-file scan lock, so dropping a Vortex file reader stalled every scan thread. The layout reader and conversion cache now live in an `Arc` shared by the local states that scan the file; under the lock we only move these references, and the local state drops them outside it.
1 parent 3c731b6 commit 5455409

3 files changed

Lines changed: 73 additions & 14 deletions

File tree

‎vortex-duckdb/src/ffi.rs‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -316,11 +316,12 @@ pub unsafe extern "C-unwind" fn duckdb_reader_scan(
316316
chunk: cpp::duckdb_data_chunk,
317317
error: *mut cpp::duckdb_vx_error,
318318
) -> bool {
319-
let file = unsafe { file.cast::<OpenFileReader>().as_ref() }.vortex_expect("null pointer");
319+
// The local state holds the scan state of its current file, so the file is not needed.
320+
let _ = file;
320321
let global = unsafe { global.cast::<GlobalState>().as_ref() }.vortex_expect("null pointer");
321322
let local = unsafe { local.cast::<LocalState>().as_mut() }.vortex_expect("null pointer");
322323
let chunk = unsafe { DataChunk::borrow_mut(chunk) };
323-
try_or(error, || reader_scan(file, global, local, chunk))
324+
try_or(error, || reader_scan(global, local, chunk))
324325
}
325326

326327
#[unsafe(no_mangle)]

‎vortex-duckdb/src/file_reader.rs‎

Lines changed: 61 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -98,34 +98,56 @@ fn resolve_filesystem(url: &Url) -> VortexResult<(FileSystemRef, String)> {
9898
}
9999

100100
pub struct OpenFileReader {
101-
pub reader: LayoutReaderRef,
101+
/// The layout reader and conversion cache of the file. `None` after the file is skipped or
102+
/// all its splits are taken.
103+
///
104+
/// DuckDB releases its last reference to a file while it holds the global lock of the
105+
/// multi-file scan, and dropping this state takes long. So every local state that scans the
106+
/// file holds a reference too, and the file gives its own reference to a local state before
107+
/// DuckDB releases the file. The last local state drops the state outside the lock.
108+
scan_state: Option<Arc<FileScanState>>,
102109
/// File splits stored in inverse order
103110
pub splits: Vec<Split>,
104-
pub cache: ConversionCache,
105111
total_splits: usize,
106112
}
107113

114+
/// The state of an open file that the threads scanning it share.
115+
pub struct FileScanState {
116+
reader: LayoutReaderRef,
117+
cache: ConversionCache,
118+
}
119+
108120
impl OpenFileReader {
109121
async fn open(path: String) -> VortexResult<Self> {
110122
let (fs, fs_path) = resolve_filesystem(&parse_uri_or_path(&path)?)?;
111123
let source = fs.open_read(&fs_path).await?;
112124
let file = open_cached(&SESSION, Some(&path), source, None, &|options| options).await?;
113125
Ok(OpenFileReader {
114-
reader: file.layout_reader()?,
115-
cache: ConversionCache::default(),
126+
scan_state: Some(Arc::new(FileScanState {
127+
reader: file.layout_reader()?,
128+
cache: ConversionCache::default(),
129+
})),
116130
splits: vec![],
117131
total_splits: 0,
118132
})
119133
}
120134

135+
fn reader(&self) -> &LayoutReaderRef {
136+
&self
137+
.scan_state
138+
.as_ref()
139+
.vortex_expect("file used after its scan state was released")
140+
.reader
141+
}
142+
121143
fn can_skip(&self, filter: &Filter) -> VortexResult<bool> {
122144
let Some(filter) = &filter.filter else {
123145
return Ok(false);
124146
};
125-
let row_count = self.reader.row_count();
147+
let row_count = self.reader().row_count();
126148
let row_range = 0..row_count;
127149
let mask = Mask::new_true(usize::try_from(row_count).unwrap_or(usize::MAX));
128-
let evaluation = self.reader.pruning_evaluation(&row_range, filter, mask)?;
150+
let evaluation = self.reader().pruning_evaluation(&row_range, filter, mask)?;
129151
match evaluation.now_or_never() {
130152
Some(mask) => mask.map(|mask| mask.all_false()),
131153
None => Ok(false),
@@ -144,7 +166,7 @@ pub fn reader_open(file_path: &str) -> VortexResult<OpenFileReader> {
144166
/// support schema evolution, so if any file schema doesn't match first schema,
145167
/// we break.
146168
pub fn reader_bind(file: &OpenFileReader, result: &mut BindResultRef) -> VortexResult<BindState> {
147-
let dtype = file.reader.dtype().clone();
169+
let dtype = file.reader().dtype().clone();
148170
let columns = extract_schema_from_dtype(&dtype)?;
149171

150172
for column in &columns {
@@ -153,7 +175,7 @@ pub fn reader_bind(file: &OpenFileReader, result: &mut BindResultRef) -> VortexR
153175

154176
Ok(BindState {
155177
dtype,
156-
first_file_row_count: file.reader.row_count(),
178+
first_file_row_count: file.reader().row_count(),
157179
filters: vec![],
158180
columns,
159181
has_non_optional_filter: AtomicBool::new(false),
@@ -167,12 +189,15 @@ pub fn reader_bind(file: &OpenFileReader, result: &mut BindResultRef) -> VortexR
167189
/// duckdb closes the file and doesn't call reader_try_initialize_scan on it.
168190
pub fn reader_initialize(file: &mut OpenFileReader, global: &GlobalState) -> VortexResult<bool> {
169191
if file.can_skip(&global.filter)? {
192+
// DuckDB releases a skipped file under the global lock. Here only the
193+
// file-local lock is held, so drop the scan state now.
194+
file.scan_state = None;
170195
return Ok(true);
171196
}
172197

173198
// Getting splits is non-trivial work so we prefer doing it here under file
174199
// lock and not in reader_try_initialize_scan under global lock.
175-
let reader = Arc::clone(&file.reader);
200+
let reader = Arc::clone(file.reader());
176201
let filter = &global.filter;
177202
let builder = ScanBuilder::new(SESSION.clone(), reader)
178203
.with_projection(global.projection.clone())
@@ -192,10 +217,31 @@ pub fn reader_initialize(file: &mut OpenFileReader, global: &GlobalState) -> Vor
192217
/// thread calls reader_scan on this file. If this function returns false,
193218
/// duckdb thinks file is exhausted, closes the file, and the first thread to
194219
/// get "false" switches to next file.
220+
///
221+
/// This function must not drop a file's scan state, because the caller holds
222+
/// the global lock. It only moves references between the file and the local
223+
/// state, and reader_scan drops the retired ones.
195224
pub fn reader_try_initialize_scan(file: &mut OpenFileReader, local: &mut LocalState) -> bool {
196225
let Some(split) = file.splits.pop() else {
226+
// DuckDB releases the file after this call, under the global lock.
227+
// Give the reference of the file to this thread so it drops outside.
228+
local.retired_files.extend(file.scan_state.take());
197229
return false;
198230
};
231+
232+
let scan_state = file
233+
.scan_state
234+
.as_ref()
235+
.vortex_expect("file has splits but no scan state");
236+
if !local
237+
.file
238+
.as_ref()
239+
.is_some_and(|current| Arc::ptr_eq(current, scan_state))
240+
{
241+
let previous = local.file.replace(Arc::clone(scan_state));
242+
local.retired_files.extend(previous);
243+
}
244+
199245
local.split = Some(split);
200246
true
201247
}
@@ -204,11 +250,13 @@ pub fn reader_try_initialize_scan(file: &mut OpenFileReader, local: &mut LocalSt
204250
/// returns false, duckdb closes the file, and first thread to get "false"
205251
/// switches to next file.
206252
pub fn reader_scan(
207-
file: &OpenFileReader,
208253
global: &GlobalState,
209254
local: &mut LocalState,
210255
chunk: &mut DataChunkRef,
211256
) -> VortexResult<bool> {
257+
// No lock is held here, so drop the scan states of files this thread left.
258+
local.retired_files.clear();
259+
212260
if !global.aggregates.is_empty() {
213261
return reader_scan_aggregate(global, local);
214262
}
@@ -223,6 +271,7 @@ pub fn reader_scan(
223271
};
224272
let mut ctx = SESSION.create_execution_ctx();
225273
let array = convert_result(array, &mut ctx)?;
274+
let file = local.file.as_ref().vortex_expect("split without a file");
226275
local.exporter = Some(ArrayExporter::try_new(&array, &file.cache, ctx)?);
227276
}
228277
let exporter = local.exporter.as_mut().vortex_expect("no exporter");
@@ -270,11 +319,11 @@ pub fn reader_get_statistics(
270319
}
271320

272321
let reader = file
273-
.reader
322+
.reader()
274323
.as_any()
275324
.downcast_ref::<FileStatsLayoutReader>()?;
276325

277-
let DType::Struct(fields, _) = &file.reader.dtype() else {
326+
let DType::Struct(fields, _) = &file.reader().dtype() else {
278327
return None;
279328
};
280329
let index = fields.find(column)?;

‎vortex-duckdb/src/table_function.rs‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@
44
use std::cmp::max;
55
use std::fmt::Formatter;
66
use std::fmt::{self};
7+
use std::sync::Arc;
78
use std::sync::atomic::AtomicBool;
89
use std::sync::atomic::AtomicU64;
910
use std::sync::atomic::Ordering;
@@ -51,6 +52,7 @@ use crate::duckdb::TableFilterSetRef;
5152
use crate::duckdb::TableInitInput;
5253
use crate::duckdb::Value;
5354
use crate::exporter::ArrayExporter;
55+
use crate::file_reader::FileScanState;
5456
use crate::projection::DuckdbField;
5557
use crate::projection::Filter;
5658
use crate::projection::Projection;
@@ -165,6 +167,11 @@ pub type Partials = Vec<(usize, Box<dyn DynAccumulator>)>;
165167
pub struct LocalState {
166168
pub exporter: Option<ArrayExporter>,
167169
pub split: Option<Split>,
170+
/// The scan state of the file that this thread scans.
171+
pub(crate) file: Option<Arc<FileScanState>>,
172+
/// Scan states that this thread no longer uses. They are dropped outside
173+
/// DuckDB's global multi-file lock; see `OpenFileReader`.
174+
pub(crate) retired_files: Vec<Arc<FileScanState>>,
168175

169176
/// Empty for non-aggregate scan
170177
pub partials: Partials,
@@ -363,6 +370,8 @@ pub fn init_local(bind_data: &BindState, global: &GlobalState) -> LocalState {
363370
exporter: None,
364371
partials,
365372
split: None,
373+
file: None,
374+
retired_files: Vec::new(),
366375
finished: false,
367376
}
368377
}

0 commit comments

Comments
 (0)