@@ -98,34 +98,56 @@ fn resolve_filesystem(url: &Url) -> VortexResult<(FileSystemRef, String)> {
9898}
9999
100100pub struct OpenFileReader {
101- pub reader : LayoutReaderRef ,
101+ /// The layout reader and conversion cache of the file. `None` after the file is skipped or
102+ /// all its splits are taken.
103+ ///
104+ /// DuckDB releases its last reference to a file while it holds the global lock of the
105+ /// multi-file scan, and dropping this state takes long. So every local state that scans the
106+ /// file holds a reference too, and the file gives its own reference to a local state before
107+ /// DuckDB releases the file. The last local state drops the state outside the lock.
108+ scan_state : Option < Arc < FileScanState > > ,
102109 /// File splits stored in inverse order
103110 pub splits : Vec < Split > ,
104- pub cache : ConversionCache ,
105111 total_splits : usize ,
106112}
107113
114+ /// The state of an open file that the threads scanning it share.
115+ pub struct FileScanState {
116+ reader : LayoutReaderRef ,
117+ cache : ConversionCache ,
118+ }
119+
108120impl OpenFileReader {
109121 async fn open ( path : String ) -> VortexResult < Self > {
110122 let ( fs, fs_path) = resolve_filesystem ( & parse_uri_or_path ( & path) ?) ?;
111123 let source = fs. open_read ( & fs_path) . await ?;
112124 let file = open_cached ( & SESSION , Some ( & path) , source, None , & |options| options) . await ?;
113125 Ok ( OpenFileReader {
114- reader : file. layout_reader ( ) ?,
115- cache : ConversionCache :: default ( ) ,
126+ scan_state : Some ( Arc :: new ( FileScanState {
127+ reader : file. layout_reader ( ) ?,
128+ cache : ConversionCache :: default ( ) ,
129+ } ) ) ,
116130 splits : vec ! [ ] ,
117131 total_splits : 0 ,
118132 } )
119133 }
120134
135+ fn reader ( & self ) -> & LayoutReaderRef {
136+ & self
137+ . scan_state
138+ . as_ref ( )
139+ . vortex_expect ( "file used after its scan state was released" )
140+ . reader
141+ }
142+
121143 fn can_skip ( & self , filter : & Filter ) -> VortexResult < bool > {
122144 let Some ( filter) = & filter. filter else {
123145 return Ok ( false ) ;
124146 } ;
125- let row_count = self . reader . row_count ( ) ;
147+ let row_count = self . reader ( ) . row_count ( ) ;
126148 let row_range = 0 ..row_count;
127149 let mask = Mask :: new_true ( usize:: try_from ( row_count) . unwrap_or ( usize:: MAX ) ) ;
128- let evaluation = self . reader . pruning_evaluation ( & row_range, filter, mask) ?;
150+ let evaluation = self . reader ( ) . pruning_evaluation ( & row_range, filter, mask) ?;
129151 match evaluation. now_or_never ( ) {
130152 Some ( mask) => mask. map ( |mask| mask. all_false ( ) ) ,
131153 None => Ok ( false ) ,
@@ -144,7 +166,7 @@ pub fn reader_open(file_path: &str) -> VortexResult<OpenFileReader> {
144166/// support schema evolution, so if any file schema doesn't match first schema,
145167/// we break.
146168pub fn reader_bind ( file : & OpenFileReader , result : & mut BindResultRef ) -> VortexResult < BindState > {
147- let dtype = file. reader . dtype ( ) . clone ( ) ;
169+ let dtype = file. reader ( ) . dtype ( ) . clone ( ) ;
148170 let columns = extract_schema_from_dtype ( & dtype) ?;
149171
150172 for column in & columns {
@@ -153,7 +175,7 @@ pub fn reader_bind(file: &OpenFileReader, result: &mut BindResultRef) -> VortexR
153175
154176 Ok ( BindState {
155177 dtype,
156- first_file_row_count : file. reader . row_count ( ) ,
178+ first_file_row_count : file. reader ( ) . row_count ( ) ,
157179 filters : vec ! [ ] ,
158180 columns,
159181 has_non_optional_filter : AtomicBool :: new ( false ) ,
@@ -167,12 +189,15 @@ pub fn reader_bind(file: &OpenFileReader, result: &mut BindResultRef) -> VortexR
167189/// duckdb closes the file and doesn't call reader_try_initialize_scan on it.
168190pub fn reader_initialize ( file : & mut OpenFileReader , global : & GlobalState ) -> VortexResult < bool > {
169191 if file. can_skip ( & global. filter ) ? {
192+ // DuckDB releases a skipped file under the global lock. Here only the
193+ // file-local lock is held, so drop the scan state now.
194+ file. scan_state = None ;
170195 return Ok ( true ) ;
171196 }
172197
173198 // Getting splits is non-trivial work so we prefer doing it here under file
174199 // lock and not in reader_try_initialize_scan under global lock.
175- let reader = Arc :: clone ( & file. reader ) ;
200+ let reader = Arc :: clone ( file. reader ( ) ) ;
176201 let filter = & global. filter ;
177202 let builder = ScanBuilder :: new ( SESSION . clone ( ) , reader)
178203 . with_projection ( global. projection . clone ( ) )
@@ -192,10 +217,31 @@ pub fn reader_initialize(file: &mut OpenFileReader, global: &GlobalState) -> Vor
192217/// thread calls reader_scan on this file. If this function returns false,
193218/// duckdb thinks file is exhausted, closes the file, and the first thread to
194219/// get "false" switches to next file.
220+ ///
221+ /// This function must not drop a file's scan state, because the caller holds
222+ /// the global lock. It only moves references between the file and the local
223+ /// state, and reader_scan drops the retired ones.
195224pub fn reader_try_initialize_scan ( file : & mut OpenFileReader , local : & mut LocalState ) -> bool {
196225 let Some ( split) = file. splits . pop ( ) else {
226+ // DuckDB releases the file after this call, under the global lock.
227+ // Give the reference of the file to this thread so it drops outside.
228+ local. retired_files . extend ( file. scan_state . take ( ) ) ;
197229 return false ;
198230 } ;
231+
232+ let scan_state = file
233+ . scan_state
234+ . as_ref ( )
235+ . vortex_expect ( "file has splits but no scan state" ) ;
236+ if !local
237+ . file
238+ . as_ref ( )
239+ . is_some_and ( |current| Arc :: ptr_eq ( current, scan_state) )
240+ {
241+ let previous = local. file . replace ( Arc :: clone ( scan_state) ) ;
242+ local. retired_files . extend ( previous) ;
243+ }
244+
199245 local. split = Some ( split) ;
200246 true
201247}
@@ -204,11 +250,13 @@ pub fn reader_try_initialize_scan(file: &mut OpenFileReader, local: &mut LocalSt
204250/// returns false, duckdb closes the file, and first thread to get "false"
205251/// switches to next file.
206252pub fn reader_scan (
207- file : & OpenFileReader ,
208253 global : & GlobalState ,
209254 local : & mut LocalState ,
210255 chunk : & mut DataChunkRef ,
211256) -> VortexResult < bool > {
257+ // No lock is held here, so drop the scan states of files this thread left.
258+ local. retired_files . clear ( ) ;
259+
212260 if !global. aggregates . is_empty ( ) {
213261 return reader_scan_aggregate ( global, local) ;
214262 }
@@ -223,6 +271,7 @@ pub fn reader_scan(
223271 } ;
224272 let mut ctx = SESSION . create_execution_ctx ( ) ;
225273 let array = convert_result ( array, & mut ctx) ?;
274+ let file = local. file . as_ref ( ) . vortex_expect ( "split without a file" ) ;
226275 local. exporter = Some ( ArrayExporter :: try_new ( & array, & file. cache , ctx) ?) ;
227276 }
228277 let exporter = local. exporter . as_mut ( ) . vortex_expect ( "no exporter" ) ;
@@ -270,11 +319,11 @@ pub fn reader_get_statistics(
270319 }
271320
272321 let reader = file
273- . reader
322+ . reader ( )
274323 . as_any ( )
275324 . downcast_ref :: < FileStatsLayoutReader > ( ) ?;
276325
277- let DType :: Struct ( fields, _) = & file. reader . dtype ( ) else {
326+ let DType :: Struct ( fields, _) = & file. reader ( ) . dtype ( ) else {
278327 return None ;
279328 } ;
280329 let index = fields. find ( column) ?;
0 commit comments