Skip to content

Commit acd69f9

Browse files
authored
ARROW-17614: [CI][Python] test test_write_dataset_max_rows_per_file is producing several nightly build failures (#14199)
Authored-by: Weston Pace <[email protected]> Signed-off-by: Weston Pace <[email protected]>
1 parent 67f5ac8 commit acd69f9

1 file changed

Lines changed: 15 additions & 8 deletions

File tree

‎cpp/src/arrow/dataset/dataset_writer.cc‎

Lines changed: 15 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -329,9 +329,9 @@ class DatasetWriterDirectoryQueue {
329329
};
330330
util::AsyncTaskScheduler* file_scheduler =
331331
scheduler_->MakeSubScheduler(std::move(file_finish_task), throttle_view);
332-
if (init_task_) {
333-
file_scheduler->AddSimpleTask(init_task_);
334-
init_task_ = {};
332+
if (init_future_.is_valid()) {
333+
file_scheduler->AddSimpleTask(
334+
[init_future = init_future_]() { return init_future; });
335335
}
336336
file_queue_view->Start(file_scheduler, filename);
337337
return file_queue_view;
@@ -343,21 +343,28 @@ class DatasetWriterDirectoryQueue {
343343
if (directory_.empty() || !write_options_.create_dir) {
344344
return;
345345
}
346+
init_future_ = Future<>::Make();
346347
auto create_dir_cb = [this] {
347-
return DeferNotOk(write_options_.filesystem->io_context().executor()->Submit(
348-
[this]() { return write_options_.filesystem->CreateDir(directory_); }));
348+
return DeferNotOk(
349+
write_options_.filesystem->io_context().executor()->Submit([this]() {
350+
ARROW_RETURN_NOT_OK(write_options_.filesystem->CreateDir(directory_));
351+
init_future_.MarkFinished();
352+
return Status::OK();
353+
}));
349354
};
355+
std::function<Future<>()> init_task;
350356
if (write_options_.existing_data_behavior ==
351357
ExistingDataBehavior::kDeleteMatchingPartitions) {
352-
init_task_ = [this, create_dir_cb] {
358+
init_task = [this, create_dir_cb] {
353359
return write_options_.filesystem
354360
->DeleteDirContentsAsync(directory_,
355361
/*missing_dir_ok=*/true)
356362
.Then(create_dir_cb);
357363
};
358364
} else {
359-
init_task_ = [create_dir_cb] { return create_dir_cb(); };
365+
init_task = [create_dir_cb] { return create_dir_cb(); };
360366
}
367+
scheduler_->AddSimpleTask(std::move(init_task));
361368
}
362369

363370
static Result<std::unique_ptr<DatasetWriterDirectoryQueue>> Make(
@@ -388,7 +395,7 @@ class DatasetWriterDirectoryQueue {
388395
std::shared_ptr<Schema> schema_;
389396
const FileSystemDatasetWriteOptions& write_options_;
390397
DatasetWriterState* writer_state_;
391-
std::function<Future<>()> init_task_;
398+
Future<> init_future_;
392399
std::string current_filename_;
393400
DatasetWriterFileQueue* latest_open_file_ = nullptr;
394401
uint64_t rows_written_ = 0;

0 commit comments

Comments
 (0)