|
| 1 | +/* |
| 2 | + * Licensed to the Apache Software Foundation (ASF) under one |
| 3 | + * or more contributor license agreements. See the NOTICE file |
| 4 | + * distributed with this work for additional information |
| 5 | + * regarding copyright ownership. The ASF licenses this file |
| 6 | + * to you under the Apache License, Version 2.0 (the |
| 7 | + * "License"); you may not use this file except in compliance |
| 8 | + * with the License. You may obtain a copy of the License at |
| 9 | + * |
| 10 | + * http://www.apache.org/licenses/LICENSE-2.0 |
| 11 | + * |
| 12 | + * Unless required by applicable law or agreed to in writing, software |
| 13 | + * distributed under the License is distributed on an "AS IS" BASIS, |
| 14 | + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| 15 | + * See the License for the specific language governing permissions and |
| 16 | + * limitations under the License. |
| 17 | + */ |
| 18 | + |
| 19 | +#include "paimon/core/operation/append_only_file_store_write.h" |
| 20 | + |
| 21 | +#include <vector> |
| 22 | + |
| 23 | +#include "paimon/common/data/binary_row.h" |
| 24 | +#include "paimon/common/table/special_fields.h" |
| 25 | +#include "paimon/common/utils/arrow/arrow_utils.h" |
| 26 | +#include "paimon/core/append/append_only_writer.h" |
| 27 | +#include "paimon/core/append/bucketed_append_compact_manager.h" |
| 28 | +#include "paimon/core/compact/noop_compact_manager.h" |
| 29 | +#include "paimon/core/core_options.h" |
| 30 | +#include "paimon/core/io/data_file_meta.h" |
| 31 | +#include "paimon/core/io/data_file_path_factory.h" |
| 32 | +#include "paimon/core/io/data_file_writer.h" |
| 33 | +#include "paimon/core/io/rolling_file_writer.h" |
| 34 | +#include "paimon/core/manifest/manifest_file.h" |
| 35 | +#include "paimon/core/manifest/manifest_list.h" |
| 36 | +#include "paimon/core/operation/append_only_file_store_scan.h" |
| 37 | +#include "paimon/core/operation/file_store_scan.h" |
| 38 | +#include "paimon/core/operation/internal_read_context.h" |
| 39 | +#include "paimon/core/operation/raw_file_split_read.h" |
| 40 | +#include "paimon/core/operation/restore_files.h" |
| 41 | +#include "paimon/core/schema/table_schema.h" |
| 42 | +#include "paimon/core/snapshot.h" |
| 43 | +#include "paimon/core/utils/file_store_path_factory.h" |
| 44 | +#include "paimon/core/utils/snapshot_manager.h" |
| 45 | +#include "paimon/executor.h" |
| 46 | +#include "paimon/logging.h" |
| 47 | +#include "paimon/read_context.h" |
| 48 | +#include "paimon/result.h" |
| 49 | +namespace arrow { |
| 50 | +class Schema; |
| 51 | +} // namespace arrow |
| 52 | + |
| 53 | +namespace paimon { |
| 54 | +class DataFilePathFactory; |
| 55 | +class MemoryPool; |
| 56 | +class SchemaManager; |
| 57 | + |
| 58 | +AppendOnlyFileStoreWrite::AppendOnlyFileStoreWrite( |
| 59 | + const std::shared_ptr<FileStorePathFactory>& file_store_path_factory, |
| 60 | + const std::shared_ptr<SnapshotManager>& snapshot_manager, |
| 61 | + const std::shared_ptr<SchemaManager>& schema_manager, const std::string& commit_user, |
| 62 | + const std::string& root_path, const std::shared_ptr<TableSchema>& table_schema, |
| 63 | + const std::shared_ptr<arrow::Schema>& schema, |
| 64 | + const std::shared_ptr<arrow::Schema>& write_schema, |
| 65 | + const std::shared_ptr<arrow::Schema>& partition_schema, |
| 66 | + const std::shared_ptr<BucketedDvMaintainer::Factory>& dv_maintainer_factory, |
| 67 | + const std::shared_ptr<IOManager>& io_manager, const CoreOptions& options, |
| 68 | + bool ignore_previous_files, bool is_streaming_mode, bool ignore_num_bucket_check, |
| 69 | + const std::shared_ptr<Executor>& executor, const std::shared_ptr<MemoryPool>& pool) |
| 70 | + : AbstractFileStoreWrite(file_store_path_factory, snapshot_manager, schema_manager, commit_user, |
| 71 | + root_path, table_schema, schema, write_schema, partition_schema, |
| 72 | + dv_maintainer_factory, io_manager, options, ignore_previous_files, |
| 73 | + is_streaming_mode, ignore_num_bucket_check, executor, pool), |
| 74 | + logger_(Logger::GetLogger("AppendOnlyFileStoreWrite")) { |
| 75 | + write_cols_ = write_schema->field_names(); |
| 76 | + auto schemas = BlobUtils::SeparateBlobSchema(schema_); |
| 77 | + if (schemas.blob_schema && schemas.blob_schema->num_fields() > 0) { |
| 78 | + with_blob_ = true; |
| 79 | + } |
| 80 | + // optimize write_cols to null in following cases: |
| 81 | + // 1. write_schema contains all columns |
| 82 | + // 2. TODO(xinyu.lxy) write_schema contains all columns and append _ROW_ID & _SEQUENCE_NUMBER |
| 83 | + // cols |
| 84 | + if (schema->Equals(write_schema)) { |
| 85 | + write_cols_ = std::nullopt; |
| 86 | + } |
| 87 | +} |
| 88 | + |
| 89 | +AppendOnlyFileStoreWrite::~AppendOnlyFileStoreWrite() = default; |
| 90 | + |
| 91 | +Result<std::unique_ptr<FileStoreScan>> AppendOnlyFileStoreWrite::CreateFileStoreScan( |
| 92 | + const std::shared_ptr<ScanFilter>& scan_filter) const { |
| 93 | + PAIMON_ASSIGN_OR_RAISE( |
| 94 | + std::shared_ptr<ManifestList> manifest_list, |
| 95 | + ManifestList::Create(options_.GetFileSystem(), options_.GetManifestFormat(), |
| 96 | + options_.GetManifestCompression(), file_store_path_factory_, pool_)); |
| 97 | + PAIMON_ASSIGN_OR_RAISE( |
| 98 | + std::shared_ptr<ManifestFile> manifest_file, |
| 99 | + ManifestFile::Create(options_.GetFileSystem(), options_.GetManifestFormat(), |
| 100 | + options_.GetManifestCompression(), file_store_path_factory_, |
| 101 | + options_.GetManifestTargetFileSize(), pool_, options_, |
| 102 | + partition_schema_)); |
| 103 | + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<FileStoreScan> scan, |
| 104 | + AppendOnlyFileStoreScan::Create( |
| 105 | + snapshot_manager_, schema_manager_, manifest_list, manifest_file, |
| 106 | + table_schema_, schema_, scan_filter, options_, executor_, pool_)); |
| 107 | + return scan; |
| 108 | +} |
| 109 | + |
| 110 | +Result<std::vector<std::shared_ptr<DataFileMeta>>> AppendOnlyFileStoreWrite::CompactRewrite( |
| 111 | + const BinaryRow& partition, int32_t bucket, DeletionVector::Factory dv_factory, |
| 112 | + const std::vector<std::shared_ptr<DataFileMeta>>& to_compact, |
| 113 | + const std::shared_ptr<CancellationController>& cancellation_controller) { |
| 114 | + if (to_compact.empty()) { |
| 115 | + return std::vector<std::shared_ptr<DataFileMeta>>{}; |
| 116 | + } |
| 117 | + |
| 118 | + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<BatchReader> reader, |
| 119 | + CreateFilesReader(partition, bucket, dv_factory, to_compact)); |
| 120 | + auto rewriter = |
| 121 | + std::make_unique<RollingFileWriter<::ArrowArray*, std::shared_ptr<DataFileMeta>>>( |
| 122 | + options_.GetTargetFileSize(/*has_primary_key=*/false), |
| 123 | + GetDataFileWriterCreator(partition, bucket, write_schema_, write_cols_, to_compact)); |
| 124 | + |
| 125 | + ScopeGuard reader_guard([&]() { |
| 126 | + if (reader) { |
| 127 | + reader->Close(); |
| 128 | + } |
| 129 | + }); |
| 130 | + |
| 131 | + ScopeGuard rewriter_guard([&]() { |
| 132 | + if (rewriter) { |
| 133 | + (void)rewriter->Close(); |
| 134 | + } |
| 135 | + }); |
| 136 | + |
| 137 | + while (true) { |
| 138 | + if (cancellation_controller->IsCancelled()) { |
| 139 | + return Status::Cancelled("Compaction cancelled while rewriting files."); |
| 140 | + } |
| 141 | + PAIMON_ASSIGN_OR_RAISE(BatchReader::ReadBatch batch, reader->NextBatch()); |
| 142 | + if (BatchReader::IsEofBatch(batch)) { |
| 143 | + break; |
| 144 | + } |
| 145 | + auto& [c_array, c_schema] = batch; |
| 146 | + PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::shared_ptr<arrow::Array> arrow_array, |
| 147 | + arrow::ImportArray(c_array.get(), c_schema.get())); |
| 148 | + auto struct_array = std::dynamic_pointer_cast<arrow::StructArray>(arrow_array); |
| 149 | + if (!struct_array) { |
| 150 | + return Status::Invalid( |
| 151 | + "cannot cast array to StructArray in CompleteRowKindBatchReader"); |
| 152 | + } |
| 153 | + PAIMON_ASSIGN_OR_RAISE(struct_array, ArrowUtils::RemoveFieldFromStructArray( |
| 154 | + struct_array, SpecialFields::ValueKind().Name())); |
| 155 | + PAIMON_RETURN_NOT_OK_FROM_ARROW( |
| 156 | + arrow::ExportArray(*struct_array, c_array.get(), c_schema.get())); |
| 157 | + ArrowSchemaRelease(c_schema.get()); |
| 158 | + ScopeGuard guard([array = c_array.get()]() { ArrowArrayRelease(array); }); |
| 159 | + PAIMON_RETURN_NOT_OK(rewriter->Write(c_array.get())); |
| 160 | + guard.Release(); |
| 161 | + } |
| 162 | + rewriter_guard.Release(); |
| 163 | + PAIMON_RETURN_NOT_OK(rewriter->Close()); |
| 164 | + return rewriter->GetResult(); |
| 165 | +} |
| 166 | + |
| 167 | +Result<std::shared_ptr<BatchWriter>> AppendOnlyFileStoreWrite::CreateWriter( |
| 168 | + const BinaryRow& partition, int32_t bucket, |
| 169 | + const std::vector<std::shared_ptr<DataFileMeta>>& restore_data_files, |
| 170 | + int64_t restore_max_seq_number, const std::shared_ptr<BucketedDvMaintainer>& dv_maintainer) { |
| 171 | + PAIMON_LOG_DEBUG(logger_, "Creating append only writer for partition %s, bucket %d", |
| 172 | + partition.ToString().c_str(), bucket); |
| 173 | + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<DataFilePathFactory> data_file_path_factory, |
| 174 | + file_store_path_factory_->CreateDataFilePathFactory(partition, bucket)); |
| 175 | + |
| 176 | + std::shared_ptr<CompactManager> compact_manager; |
| 177 | + auto schemas = BlobUtils::SeparateBlobSchema(write_schema_); |
| 178 | + if (options_.WriteOnly() || options_.DataEvolutionEnabled() || options_.GetBucket() == -1 || |
| 179 | + with_blob_) { |
| 180 | + compact_manager = std::make_shared<NoopCompactManager>(); |
| 181 | + } else { |
| 182 | + auto dv_factory = |
| 183 | + [dv_maintainer]( |
| 184 | + const std::string& file_name) -> Result<std::shared_ptr<DeletionVector>> { |
| 185 | + if (dv_maintainer) { |
| 186 | + return dv_maintainer->DeletionVectorOf(file_name).value_or( |
| 187 | + std::shared_ptr<DeletionVector>()); |
| 188 | + } |
| 189 | + return std::shared_ptr<DeletionVector>(); |
| 190 | + }; |
| 191 | + auto cancellation_controller = std::make_shared<CancellationController>(); |
| 192 | + |
| 193 | + auto rewriter = [this, partition, bucket, dv_factory, cancellation_controller]( |
| 194 | + const std::vector<std::shared_ptr<DataFileMeta>>& to_compact) |
| 195 | + -> Result<std::vector<std::shared_ptr<DataFileMeta>>> { |
| 196 | + return CompactRewrite(partition, bucket, dv_factory, to_compact, |
| 197 | + cancellation_controller); |
| 198 | + }; |
| 199 | + |
| 200 | + compact_manager = std::make_shared<BucketedAppendCompactManager>( |
| 201 | + compact_executor_, restore_data_files, dv_maintainer, |
| 202 | + options_.GetCompactionMinFileNum(), |
| 203 | + options_.GetTargetFileSize(/*has_primary_key=*/false), |
| 204 | + options_.GetCompactionFileSize(/*has_primary_key=*/false), |
| 205 | + options_.CompactionForceRewriteAllFiles(), rewriter, |
| 206 | + compaction_metrics_->CreateReporter(partition, bucket), cancellation_controller); |
| 207 | + } |
| 208 | + |
| 209 | + auto writer = std::make_shared<AppendOnlyWriter>( |
| 210 | + options_, table_schema_->Id(), write_schema_, write_cols_, restore_max_seq_number, |
| 211 | + data_file_path_factory, compact_manager, pool_); |
| 212 | + return std::shared_ptr<BatchWriter>(writer); |
| 213 | +} |
| 214 | + |
| 215 | +AppendOnlyFileStoreWrite::SingleFileWriterCreator |
| 216 | +AppendOnlyFileStoreWrite::GetDataFileWriterCreator( |
| 217 | + const BinaryRow& partition, int32_t bucket, const std::shared_ptr<arrow::Schema>& schema, |
| 218 | + const std::optional<std::vector<std::string>>& write_cols, |
| 219 | + const std::vector<std::shared_ptr<DataFileMeta>>& to_compact) const { |
| 220 | + return |
| 221 | + [this, partition, bucket, schema, write_cols, to_compact]() |
| 222 | + -> Result< |
| 223 | + std::unique_ptr<SingleFileWriter<::ArrowArray*, std::shared_ptr<DataFileMeta>>>> { |
| 224 | + ::ArrowSchema arrow_schema; |
| 225 | + ScopeGuard guard([&arrow_schema]() { ArrowSchemaRelease(&arrow_schema); }); |
| 226 | + PAIMON_RETURN_NOT_OK_FROM_ARROW(arrow::ExportSchema(*schema, &arrow_schema)); |
| 227 | + auto format = options_.GetFileFormat(); |
| 228 | + PAIMON_ASSIGN_OR_RAISE( |
| 229 | + std::shared_ptr<WriterBuilder> writer_builder, |
| 230 | + format->CreateWriterBuilder(&arrow_schema, options_.GetWriteBatchSize())); |
| 231 | + writer_builder->WithMemoryPool(pool_); |
| 232 | + |
| 233 | + PAIMON_RETURN_NOT_OK_FROM_ARROW(arrow::ExportSchema(*schema, &arrow_schema)); |
| 234 | + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<FormatStatsExtractor> stats_extractor, |
| 235 | + format->CreateStatsExtractor(&arrow_schema)); |
| 236 | + PAIMON_ASSIGN_OR_RAISE( |
| 237 | + std::shared_ptr<DataFilePathFactory> data_file_path_factory, |
| 238 | + file_store_path_factory_->CreateDataFilePathFactory(partition, bucket)); |
| 239 | + auto writer = std::make_unique<DataFileWriter>( |
| 240 | + options_.GetFileCompression(), std::function<Status(ArrowArray*, ArrowArray*)>(), |
| 241 | + table_schema_->Id(), |
| 242 | + std::make_shared<LongCounter>(to_compact[0]->min_sequence_number), |
| 243 | + FileSource::Compact(), stats_extractor, data_file_path_factory->IsExternalPath(), |
| 244 | + write_cols, pool_); |
| 245 | + PAIMON_RETURN_NOT_OK(writer->Init(options_.GetFileSystem(), |
| 246 | + data_file_path_factory->NewPath(), writer_builder)); |
| 247 | + return writer; |
| 248 | + }; |
| 249 | +} |
| 250 | + |
| 251 | +Result<std::unique_ptr<BatchReader>> AppendOnlyFileStoreWrite::CreateFilesReader( |
| 252 | + const BinaryRow& partition, int32_t bucket, DeletionVector::Factory dv_factory, |
| 253 | + const std::vector<std::shared_ptr<DataFileMeta>>& files) const { |
| 254 | + ReadContextBuilder context_builder(root_path_); |
| 255 | + context_builder.SetOptions(options_.ToMap()) |
| 256 | + .WithFileSystem(options_.GetFileSystem()) |
| 257 | + .EnablePrefetch(true) |
| 258 | + .SetPrefetchMaxParallelNum(1) |
| 259 | + .SetPrefetchBatchCount(3) |
| 260 | + .WithMemoryPool(pool_); |
| 261 | + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<ReadContext> read_context, context_builder.Finish()); |
| 262 | + std::map<std::string, std::string> options = options_.ToMap(); |
| 263 | + // TODO(xinyu.lxy): temporarily disabled pre-buffer for parquet, which may cause high |
| 264 | + // memory usage during compaction. Will fix via parquet format refactor. |
| 265 | + auto new_options = options; |
| 266 | + if (new_options.find("parquet.read.enable-pre-buffer") == new_options.end()) { |
| 267 | + new_options["parquet.read.enable-pre-buffer"] = "false"; |
| 268 | + } |
| 269 | + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<InternalReadContext> internal_read_context, |
| 270 | + InternalReadContext::Create(read_context, table_schema_, new_options)); |
| 271 | + auto read = std::make_unique<RawFileSplitRead>(file_store_path_factory_, internal_read_context, |
| 272 | + pool_, compact_executor_); |
| 273 | + |
| 274 | + return read->CreateReader(partition, bucket, files, dv_factory); |
| 275 | +} |
| 276 | + |
| 277 | +} // namespace paimon |
0 commit comments