|
17 | 17 |
|
18 | 18 | #include <chrono> |
19 | 19 | #include <cstring> |
| 20 | +#include <memory> |
20 | 21 | #include <sstream> |
21 | 22 | #include <utility> |
22 | 23 |
|
|
29 | 30 | #include <sys/stat.h> |
30 | 31 | #endif |
31 | 32 |
|
| 33 | +#include "arrow/filesystem/filesystem.h" |
32 | 34 | #include "arrow/filesystem/localfs.h" |
33 | 35 | #include "arrow/filesystem/path_util.h" |
| 36 | +#include "arrow/filesystem/type_fwd.h" |
34 | 37 | #include "arrow/filesystem/util_internal.h" |
35 | 38 | #include "arrow/io/file.h" |
| 39 | +#include "arrow/io/type_fwd.h" |
| 40 | +#include "arrow/util/async_generator.h" |
36 | 41 | #include "arrow/util/io_util.h" |
37 | | -#include "arrow/util/logging.h" |
38 | 42 | #include "arrow/util/uri.h" |
39 | 43 | #include "arrow/util/windows_fixup.h" |
40 | 44 |
|
@@ -243,7 +247,8 @@ LocalFileSystemOptions LocalFileSystemOptions::Defaults() { |
243 | 247 | } |
244 | 248 |
|
245 | 249 | bool LocalFileSystemOptions::Equals(const LocalFileSystemOptions& other) const { |
246 | | - return use_mmap == other.use_mmap; |
| 250 | + return use_mmap == other.use_mmap && directory_readahead == other.directory_readahead && |
| 251 | + file_info_batch_size == other.file_info_batch_size; |
247 | 252 | } |
248 | 253 |
|
249 | 254 | Result<LocalFileSystemOptions> LocalFileSystemOptions::FromUri( |
@@ -309,6 +314,241 @@ Result<std::vector<FileInfo>> LocalFileSystem::GetFileInfo(const FileSelector& s |
309 | 314 | return results; |
310 | 315 | } |
311 | 316 |
|
| 317 | +namespace { |
| 318 | + |
| 319 | +/// Workhorse for streaming async implementation of `GetFileInfo` |
| 320 | +/// (`GetFileInfoGenerator`). |
| 321 | +/// |
| 322 | +/// There are two variants of async discovery functions suported: |
| 323 | +/// 1. `DiscoverDirectoryFiles`, which parallelizes traversal of individual directories |
| 324 | +/// so that each directory results are yielded as a separate `FileInfoGenerator` via |
| 325 | +/// an underlying `DiscoveryImplIterator`, which delivers items in chunks (default size |
| 326 | +/// is 1K items). |
| 327 | +/// 2. `DiscoverDirectoriesFlattened`, which forwards execution to the |
| 328 | +/// `DiscoverDirectoryFiles`, with the difference that the results from individual |
| 329 | +/// sub-directory iterators are merged into the single FileInfoGenerator stream. |
| 330 | +/// |
| 331 | +/// The implementation makes use of additional attributes in `LocalFileSystemOptions`, |
| 332 | +/// such as `directory_readahead`, which can be used to tune algorithm |
| 333 | +/// behavior and adjust how many directories can be processed in parallel. |
| 334 | +/// This option is disabled by default, so that individual directories are processed |
| 335 | +/// in serial manner via `MakeConcatenatedGenerator` under the hood. |
| 336 | +class AsyncStatSelector { |
| 337 | + public: |
| 338 | + using FileInfoGeneratorProducer = PushGenerator<FileInfoGenerator>::Producer; |
| 339 | + |
| 340 | + /// Discovery state, which is shared among all `DiscoveryImplGenerator`:s, |
| 341 | + /// spawned by a single discovery operation (`DiscoverDirectoryFiles()`). |
| 342 | + /// |
| 343 | + /// The sole purpose of this struct is to handle automatic closing the |
| 344 | + /// producer side of the resulting `FileInfoGenerator`. I.e. the producer |
| 345 | + /// is kept alive until all discovery iterators are exhausted, in which case |
| 346 | + /// `producer.Close()` is called automatically when ref-count for the state |
| 347 | + /// reaches zero (which is equivalent to finishing the file discovery |
| 348 | + /// process). |
| 349 | + struct DiscoveryState { |
| 350 | + FileInfoGeneratorProducer producer; |
| 351 | + |
| 352 | + explicit DiscoveryState(FileInfoGeneratorProducer p) : producer(std::move(p)) {} |
| 353 | + ~DiscoveryState() { producer.Close(); } |
| 354 | + }; |
| 355 | + |
| 356 | + /// The main procedure to start async streaming discovery using a given `FileSelector`. |
| 357 | + /// |
| 358 | + /// The result is a two-level generator, i.e. "generator of FileInfoGenerator:s", |
| 359 | + /// where each individual generator represents an FileInfo item stream from coming an |
| 360 | + /// individual sub-directory under the selector's `base_dir`. |
| 361 | + static Result<AsyncGenerator<FileInfoGenerator>> DiscoverDirectoryFiles( |
| 362 | + FileSelector selector, LocalFileSystemOptions fs_opts, |
| 363 | + const io::IOContext& io_context) { |
| 364 | + PushGenerator<FileInfoGenerator> file_gen; |
| 365 | + |
| 366 | + ARROW_ASSIGN_OR_RAISE( |
| 367 | + auto base_dir, arrow::internal::PlatformFilename::FromString(selector.base_dir)); |
| 368 | + ARROW_RETURN_NOT_OK(DoDiscovery(std::move(base_dir), 0, std::move(selector), |
| 369 | + std::make_shared<DiscoveryState>(file_gen.producer()), |
| 370 | + io_context, fs_opts.file_info_batch_size)); |
| 371 | + |
| 372 | + return file_gen; |
| 373 | + } |
| 374 | + |
| 375 | + /// Version of `DiscoverDirectoryFiles` which flattens the stream of generators |
| 376 | + /// into a single FileInfoGenerator stream. |
| 377 | + /// Makes use of `LocalFileSystemOptions::directory_readahead` to determine how much |
| 378 | + /// readahead should happen. |
| 379 | + static arrow::Result<FileInfoGenerator> DiscoverDirectoriesFlattened( |
| 380 | + FileSelector selector, LocalFileSystemOptions fs_opts, |
| 381 | + const io::IOContext& io_context) { |
| 382 | + int32_t dir_readahead = fs_opts.directory_readahead; |
| 383 | + ARROW_ASSIGN_OR_RAISE( |
| 384 | + auto part_gen, |
| 385 | + DiscoverDirectoryFiles(std::move(selector), std::move(fs_opts), io_context)); |
| 386 | + return dir_readahead > 1 |
| 387 | + ? MakeSequencedMergedGenerator(std::move(part_gen), dir_readahead) |
| 388 | + : MakeConcatenatedGenerator(std::move(part_gen)); |
| 389 | + } |
| 390 | + |
| 391 | + private: |
| 392 | + /// The class, which implements iterator interface to traverse a given |
| 393 | + /// directory at the fixed nesting depth, and possibly recurses into |
| 394 | + /// sub-directories (if specified by the selector), spawning more |
| 395 | + /// `DiscoveryImplIterators`, which feed their data into a single producer. |
| 396 | + class DiscoveryImplIterator { |
| 397 | + const PlatformFilename dir_fn_; |
| 398 | + const int32_t nesting_depth_; |
| 399 | + const FileSelector selector_; |
| 400 | + const uint32_t file_info_batch_size_; |
| 401 | + |
| 402 | + const io::IOContext& io_context_; |
| 403 | + std::shared_ptr<DiscoveryState> discovery_state_; |
| 404 | + FileInfoVector current_chunk_; |
| 405 | + std::vector<PlatformFilename> child_fns_; |
| 406 | + size_t idx_ = 0; |
| 407 | + bool initialized_ = false; |
| 408 | + |
| 409 | + public: |
| 410 | + DiscoveryImplIterator(PlatformFilename dir_fn, int32_t nesting_depth, |
| 411 | + FileSelector selector, |
| 412 | + std::shared_ptr<DiscoveryState> discovery_state, |
| 413 | + const io::IOContext& io_context, uint32_t file_info_batch_size) |
| 414 | + : dir_fn_(std::move(dir_fn)), |
| 415 | + nesting_depth_(nesting_depth), |
| 416 | + selector_(std::move(selector)), |
| 417 | + file_info_batch_size_(file_info_batch_size), |
| 418 | + io_context_(io_context), |
| 419 | + discovery_state_(std::move(discovery_state)) {} |
| 420 | + |
| 421 | + /// Pre-initialize the iterator by listing directory contents and caching |
| 422 | + /// in the current instance. |
| 423 | + Status Initialize() { |
| 424 | + auto result = arrow::internal::ListDir(dir_fn_); |
| 425 | + if (!result.ok()) { |
| 426 | + auto status = result.status(); |
| 427 | + if (selector_.allow_not_found && status.IsIOError()) { |
| 428 | + ARROW_ASSIGN_OR_RAISE(bool exists, FileExists(dir_fn_)); |
| 429 | + if (!exists) { |
| 430 | + return Status::OK(); |
| 431 | + } |
| 432 | + } |
| 433 | + return status; |
| 434 | + } |
| 435 | + child_fns_ = result.MoveValueUnsafe(); |
| 436 | + |
| 437 | + const size_t dirent_count = child_fns_.size(); |
| 438 | + current_chunk_.reserve(dirent_count >= file_info_batch_size_ ? file_info_batch_size_ |
| 439 | + : dirent_count); |
| 440 | + |
| 441 | + initialized_ = true; |
| 442 | + return Status::OK(); |
| 443 | + } |
| 444 | + |
| 445 | + Result<FileInfoVector> Next() { |
| 446 | + if (!initialized_) { |
| 447 | + auto init = Initialize(); |
| 448 | + if (!init.ok()) { |
| 449 | + return Finish(init); |
| 450 | + } |
| 451 | + } |
| 452 | + while (idx_ < child_fns_.size()) { |
| 453 | + auto full_fn = dir_fn_.Join(child_fns_[idx_++]); |
| 454 | + auto res = StatFile(full_fn.ToNative()); |
| 455 | + if (!res.ok()) { |
| 456 | + return Finish(res.status()); |
| 457 | + } |
| 458 | + |
| 459 | + auto info = res.MoveValueUnsafe(); |
| 460 | + |
| 461 | + // Try to recurse into subdirectories, if needed. |
| 462 | + if (info.type() == FileType::Directory && |
| 463 | + nesting_depth_ < selector_.max_recursion && selector_.recursive) { |
| 464 | + auto status = DoDiscovery(std::move(full_fn), nesting_depth_ + 1, selector_, |
| 465 | + discovery_state_, io_context_, file_info_batch_size_); |
| 466 | + if (!status.ok()) { |
| 467 | + return Finish(status); |
| 468 | + } |
| 469 | + } |
| 470 | + // Everything is ok. Add the item to the current chunk of data. |
| 471 | + current_chunk_.emplace_back(std::move(info)); |
| 472 | + // Keep `current_chunk_` as large, as `batch_size_`. |
| 473 | + // Otherwise, yield the complete chunk to the caller. |
| 474 | + if (current_chunk_.size() == file_info_batch_size_) { |
| 475 | + FileInfoVector yield_vec = std::move(current_chunk_); |
| 476 | + const size_t items_left = child_fns_.size() - idx_; |
| 477 | + current_chunk_.reserve( |
| 478 | + items_left >= file_info_batch_size_ ? file_info_batch_size_ : items_left); |
| 479 | + return yield_vec; |
| 480 | + } |
| 481 | + } // while (idx_ < child_fns_.size()) |
| 482 | + |
| 483 | + // Flush out remaining items |
| 484 | + if (!current_chunk_.empty()) { |
| 485 | + return std::move(current_chunk_); |
| 486 | + } |
| 487 | + return Finish(); |
| 488 | + } |
| 489 | + |
| 490 | + private: |
| 491 | + /// Release reference to shared discovery state and return iteration end |
| 492 | + /// marker to indicate that this iterator is exhausted. |
| 493 | + Result<FileInfoVector> Finish(Status status = Status::OK()) { |
| 494 | + discovery_state_.reset(); |
| 495 | + ARROW_RETURN_NOT_OK(status); |
| 496 | + return IterationEnd<FileInfoVector>(); |
| 497 | + } |
| 498 | + }; |
| 499 | + |
| 500 | + /// Create an instance of `DiscoveryImplIterator` under the hood for the |
| 501 | + /// specified directory, wrap it in the `BackgroundGenerator` and feed |
| 502 | + /// the results to the main producer queue. |
| 503 | + /// |
| 504 | + /// Each `DiscoveryImplIterator` maintains a reference to `DiscoveryState`, |
| 505 | + /// which simply wraps the producer to keep it alive for the lifetime |
| 506 | + /// of this iterator. When all references to `DiscoveryState` are invalidated, |
| 507 | + /// the producer is closed automatically. |
| 508 | + static Status DoDiscovery(const PlatformFilename& dir_fn, int32_t nesting_depth, |
| 509 | + FileSelector selector, |
| 510 | + std::shared_ptr<DiscoveryState> discovery_state, |
| 511 | + const io::IOContext& io_context, |
| 512 | + int32_t file_info_batch_size) { |
| 513 | + ARROW_RETURN_IF(discovery_state->producer.is_closed(), |
| 514 | + arrow::Status::Cancelled("Discovery cancelled")); |
| 515 | + |
| 516 | + // Note, that here we use `MakeTransferredGenerator()` with the same |
| 517 | + // target executor (io executor) as the current iterator is running on. |
| 518 | + // |
| 519 | + // This is done on purpose, since typically the user of |
| 520 | + // `GetFileInfoGenerator()` would want to perform some more IO on the |
| 521 | + // produced results (e.g. read the files, examine metadata etc.). |
| 522 | + // So, it is preferable to execute the attached continuations on the same |
| 523 | + // executor, which belongs to the IO thread pool. |
| 524 | + ARROW_ASSIGN_OR_RAISE( |
| 525 | + auto gen, |
| 526 | + MakeBackgroundGenerator(Iterator<FileInfoVector>(DiscoveryImplIterator( |
| 527 | + std::move(dir_fn), nesting_depth, std::move(selector), |
| 528 | + discovery_state, io_context, file_info_batch_size)), |
| 529 | + io_context.executor())); |
| 530 | + gen = MakeTransferredGenerator(std::move(gen), io_context.executor()); |
| 531 | + ARROW_RETURN_IF(!discovery_state->producer.Push(std::move(gen)), |
| 532 | + arrow::Status::Cancelled("Discovery cancelled")); |
| 533 | + return arrow::Status::OK(); |
| 534 | + } |
| 535 | +}; |
| 536 | + |
| 537 | +} // anonymous namespace |
| 538 | + |
| 539 | +FileInfoGenerator LocalFileSystem::GetFileInfoGenerator(const FileSelector& select) { |
| 540 | + auto path_status = ValidatePath(select.base_dir); |
| 541 | + if (!path_status.ok()) { |
| 542 | + return MakeFailingGenerator<FileInfoVector>(path_status); |
| 543 | + } |
| 544 | + auto fileinfo_gen = |
| 545 | + AsyncStatSelector::DiscoverDirectoriesFlattened(select, options(), io_context_); |
| 546 | + if (!fileinfo_gen.ok()) { |
| 547 | + return MakeFailingGenerator<FileInfoVector>(fileinfo_gen.status()); |
| 548 | + } |
| 549 | + return fileinfo_gen.MoveValueUnsafe(); |
| 550 | +} |
| 551 | + |
312 | 552 | Status LocalFileSystem::CreateDir(const std::string& path, bool recursive) { |
313 | 553 | RETURN_NOT_OK(ValidatePath(path)); |
314 | 554 | ARROW_ASSIGN_OR_RAISE(auto fn, PlatformFilename::FromString(path)); |
|
0 commit comments