Skip to content

Commit abf9e66

Browse files
authored
feat(core): add append-only/key-value file store write and compaction metrics (alibaba#92)
1 parent 0476932 commit abf9e66

8 files changed

Lines changed: 1628 additions & 0 deletions
Lines changed: 277 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,277 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one
3+
* or more contributor license agreements. See the NOTICE file
4+
* distributed with this work for additional information
5+
* regarding copyright ownership. The ASF licenses this file
6+
* to you under the Apache License, Version 2.0 (the
7+
* "License"); you may not use this file except in compliance
8+
* with the License. You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing, software
13+
* distributed under the License is distributed on an "AS IS" BASIS,
14+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15+
* See the License for the specific language governing permissions and
16+
* limitations under the License.
17+
*/
18+
19+
#include "paimon/core/operation/append_only_file_store_write.h"
20+
21+
#include <vector>
22+
23+
#include "paimon/common/data/binary_row.h"
24+
#include "paimon/common/table/special_fields.h"
25+
#include "paimon/common/utils/arrow/arrow_utils.h"
26+
#include "paimon/core/append/append_only_writer.h"
27+
#include "paimon/core/append/bucketed_append_compact_manager.h"
28+
#include "paimon/core/compact/noop_compact_manager.h"
29+
#include "paimon/core/core_options.h"
30+
#include "paimon/core/io/data_file_meta.h"
31+
#include "paimon/core/io/data_file_path_factory.h"
32+
#include "paimon/core/io/data_file_writer.h"
33+
#include "paimon/core/io/rolling_file_writer.h"
34+
#include "paimon/core/manifest/manifest_file.h"
35+
#include "paimon/core/manifest/manifest_list.h"
36+
#include "paimon/core/operation/append_only_file_store_scan.h"
37+
#include "paimon/core/operation/file_store_scan.h"
38+
#include "paimon/core/operation/internal_read_context.h"
39+
#include "paimon/core/operation/raw_file_split_read.h"
40+
#include "paimon/core/operation/restore_files.h"
41+
#include "paimon/core/schema/table_schema.h"
42+
#include "paimon/core/snapshot.h"
43+
#include "paimon/core/utils/file_store_path_factory.h"
44+
#include "paimon/core/utils/snapshot_manager.h"
45+
#include "paimon/executor.h"
46+
#include "paimon/logging.h"
47+
#include "paimon/read_context.h"
48+
#include "paimon/result.h"
49+
namespace arrow {
50+
class Schema;
51+
} // namespace arrow
52+
53+
namespace paimon {
54+
class DataFilePathFactory;
55+
class MemoryPool;
56+
class SchemaManager;
57+
58+
AppendOnlyFileStoreWrite::AppendOnlyFileStoreWrite(
59+
const std::shared_ptr<FileStorePathFactory>& file_store_path_factory,
60+
const std::shared_ptr<SnapshotManager>& snapshot_manager,
61+
const std::shared_ptr<SchemaManager>& schema_manager, const std::string& commit_user,
62+
const std::string& root_path, const std::shared_ptr<TableSchema>& table_schema,
63+
const std::shared_ptr<arrow::Schema>& schema,
64+
const std::shared_ptr<arrow::Schema>& write_schema,
65+
const std::shared_ptr<arrow::Schema>& partition_schema,
66+
const std::shared_ptr<BucketedDvMaintainer::Factory>& dv_maintainer_factory,
67+
const std::shared_ptr<IOManager>& io_manager, const CoreOptions& options,
68+
bool ignore_previous_files, bool is_streaming_mode, bool ignore_num_bucket_check,
69+
const std::shared_ptr<Executor>& executor, const std::shared_ptr<MemoryPool>& pool)
70+
: AbstractFileStoreWrite(file_store_path_factory, snapshot_manager, schema_manager, commit_user,
71+
root_path, table_schema, schema, write_schema, partition_schema,
72+
dv_maintainer_factory, io_manager, options, ignore_previous_files,
73+
is_streaming_mode, ignore_num_bucket_check, executor, pool),
74+
logger_(Logger::GetLogger("AppendOnlyFileStoreWrite")) {
75+
write_cols_ = write_schema->field_names();
76+
auto schemas = BlobUtils::SeparateBlobSchema(schema_);
77+
if (schemas.blob_schema && schemas.blob_schema->num_fields() > 0) {
78+
with_blob_ = true;
79+
}
80+
// optimize write_cols to null in following cases:
81+
// 1. write_schema contains all columns
82+
// 2. TODO(xinyu.lxy) write_schema contains all columns and append _ROW_ID & _SEQUENCE_NUMBER
83+
// cols
84+
if (schema->Equals(write_schema)) {
85+
write_cols_ = std::nullopt;
86+
}
87+
}
88+
89+
AppendOnlyFileStoreWrite::~AppendOnlyFileStoreWrite() = default;
90+
91+
Result<std::unique_ptr<FileStoreScan>> AppendOnlyFileStoreWrite::CreateFileStoreScan(
92+
const std::shared_ptr<ScanFilter>& scan_filter) const {
93+
PAIMON_ASSIGN_OR_RAISE(
94+
std::shared_ptr<ManifestList> manifest_list,
95+
ManifestList::Create(options_.GetFileSystem(), options_.GetManifestFormat(),
96+
options_.GetManifestCompression(), file_store_path_factory_, pool_));
97+
PAIMON_ASSIGN_OR_RAISE(
98+
std::shared_ptr<ManifestFile> manifest_file,
99+
ManifestFile::Create(options_.GetFileSystem(), options_.GetManifestFormat(),
100+
options_.GetManifestCompression(), file_store_path_factory_,
101+
options_.GetManifestTargetFileSize(), pool_, options_,
102+
partition_schema_));
103+
PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<FileStoreScan> scan,
104+
AppendOnlyFileStoreScan::Create(
105+
snapshot_manager_, schema_manager_, manifest_list, manifest_file,
106+
table_schema_, schema_, scan_filter, options_, executor_, pool_));
107+
return scan;
108+
}
109+
110+
Result<std::vector<std::shared_ptr<DataFileMeta>>> AppendOnlyFileStoreWrite::CompactRewrite(
111+
const BinaryRow& partition, int32_t bucket, DeletionVector::Factory dv_factory,
112+
const std::vector<std::shared_ptr<DataFileMeta>>& to_compact,
113+
const std::shared_ptr<CancellationController>& cancellation_controller) {
114+
if (to_compact.empty()) {
115+
return std::vector<std::shared_ptr<DataFileMeta>>{};
116+
}
117+
118+
PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<BatchReader> reader,
119+
CreateFilesReader(partition, bucket, dv_factory, to_compact));
120+
auto rewriter =
121+
std::make_unique<RollingFileWriter<::ArrowArray*, std::shared_ptr<DataFileMeta>>>(
122+
options_.GetTargetFileSize(/*has_primary_key=*/false),
123+
GetDataFileWriterCreator(partition, bucket, write_schema_, write_cols_, to_compact));
124+
125+
ScopeGuard reader_guard([&]() {
126+
if (reader) {
127+
reader->Close();
128+
}
129+
});
130+
131+
ScopeGuard rewriter_guard([&]() {
132+
if (rewriter) {
133+
(void)rewriter->Close();
134+
}
135+
});
136+
137+
while (true) {
138+
if (cancellation_controller->IsCancelled()) {
139+
return Status::Cancelled("Compaction cancelled while rewriting files.");
140+
}
141+
PAIMON_ASSIGN_OR_RAISE(BatchReader::ReadBatch batch, reader->NextBatch());
142+
if (BatchReader::IsEofBatch(batch)) {
143+
break;
144+
}
145+
auto& [c_array, c_schema] = batch;
146+
PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::shared_ptr<arrow::Array> arrow_array,
147+
arrow::ImportArray(c_array.get(), c_schema.get()));
148+
auto struct_array = std::dynamic_pointer_cast<arrow::StructArray>(arrow_array);
149+
if (!struct_array) {
150+
return Status::Invalid(
151+
"cannot cast array to StructArray in CompleteRowKindBatchReader");
152+
}
153+
PAIMON_ASSIGN_OR_RAISE(struct_array, ArrowUtils::RemoveFieldFromStructArray(
154+
struct_array, SpecialFields::ValueKind().Name()));
155+
PAIMON_RETURN_NOT_OK_FROM_ARROW(
156+
arrow::ExportArray(*struct_array, c_array.get(), c_schema.get()));
157+
ArrowSchemaRelease(c_schema.get());
158+
ScopeGuard guard([array = c_array.get()]() { ArrowArrayRelease(array); });
159+
PAIMON_RETURN_NOT_OK(rewriter->Write(c_array.get()));
160+
guard.Release();
161+
}
162+
rewriter_guard.Release();
163+
PAIMON_RETURN_NOT_OK(rewriter->Close());
164+
return rewriter->GetResult();
165+
}
166+
167+
Result<std::shared_ptr<BatchWriter>> AppendOnlyFileStoreWrite::CreateWriter(
168+
const BinaryRow& partition, int32_t bucket,
169+
const std::vector<std::shared_ptr<DataFileMeta>>& restore_data_files,
170+
int64_t restore_max_seq_number, const std::shared_ptr<BucketedDvMaintainer>& dv_maintainer) {
171+
PAIMON_LOG_DEBUG(logger_, "Creating append only writer for partition %s, bucket %d",
172+
partition.ToString().c_str(), bucket);
173+
PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<DataFilePathFactory> data_file_path_factory,
174+
file_store_path_factory_->CreateDataFilePathFactory(partition, bucket));
175+
176+
std::shared_ptr<CompactManager> compact_manager;
177+
auto schemas = BlobUtils::SeparateBlobSchema(write_schema_);
178+
if (options_.WriteOnly() || options_.DataEvolutionEnabled() || options_.GetBucket() == -1 ||
179+
with_blob_) {
180+
compact_manager = std::make_shared<NoopCompactManager>();
181+
} else {
182+
auto dv_factory =
183+
[dv_maintainer](
184+
const std::string& file_name) -> Result<std::shared_ptr<DeletionVector>> {
185+
if (dv_maintainer) {
186+
return dv_maintainer->DeletionVectorOf(file_name).value_or(
187+
std::shared_ptr<DeletionVector>());
188+
}
189+
return std::shared_ptr<DeletionVector>();
190+
};
191+
auto cancellation_controller = std::make_shared<CancellationController>();
192+
193+
auto rewriter = [this, partition, bucket, dv_factory, cancellation_controller](
194+
const std::vector<std::shared_ptr<DataFileMeta>>& to_compact)
195+
-> Result<std::vector<std::shared_ptr<DataFileMeta>>> {
196+
return CompactRewrite(partition, bucket, dv_factory, to_compact,
197+
cancellation_controller);
198+
};
199+
200+
compact_manager = std::make_shared<BucketedAppendCompactManager>(
201+
compact_executor_, restore_data_files, dv_maintainer,
202+
options_.GetCompactionMinFileNum(),
203+
options_.GetTargetFileSize(/*has_primary_key=*/false),
204+
options_.GetCompactionFileSize(/*has_primary_key=*/false),
205+
options_.CompactionForceRewriteAllFiles(), rewriter,
206+
compaction_metrics_->CreateReporter(partition, bucket), cancellation_controller);
207+
}
208+
209+
auto writer = std::make_shared<AppendOnlyWriter>(
210+
options_, table_schema_->Id(), write_schema_, write_cols_, restore_max_seq_number,
211+
data_file_path_factory, compact_manager, pool_);
212+
return std::shared_ptr<BatchWriter>(writer);
213+
}
214+
215+
AppendOnlyFileStoreWrite::SingleFileWriterCreator
216+
AppendOnlyFileStoreWrite::GetDataFileWriterCreator(
217+
const BinaryRow& partition, int32_t bucket, const std::shared_ptr<arrow::Schema>& schema,
218+
const std::optional<std::vector<std::string>>& write_cols,
219+
const std::vector<std::shared_ptr<DataFileMeta>>& to_compact) const {
220+
return
221+
[this, partition, bucket, schema, write_cols, to_compact]()
222+
-> Result<
223+
std::unique_ptr<SingleFileWriter<::ArrowArray*, std::shared_ptr<DataFileMeta>>>> {
224+
::ArrowSchema arrow_schema;
225+
ScopeGuard guard([&arrow_schema]() { ArrowSchemaRelease(&arrow_schema); });
226+
PAIMON_RETURN_NOT_OK_FROM_ARROW(arrow::ExportSchema(*schema, &arrow_schema));
227+
auto format = options_.GetFileFormat();
228+
PAIMON_ASSIGN_OR_RAISE(
229+
std::shared_ptr<WriterBuilder> writer_builder,
230+
format->CreateWriterBuilder(&arrow_schema, options_.GetWriteBatchSize()));
231+
writer_builder->WithMemoryPool(pool_);
232+
233+
PAIMON_RETURN_NOT_OK_FROM_ARROW(arrow::ExportSchema(*schema, &arrow_schema));
234+
PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<FormatStatsExtractor> stats_extractor,
235+
format->CreateStatsExtractor(&arrow_schema));
236+
PAIMON_ASSIGN_OR_RAISE(
237+
std::shared_ptr<DataFilePathFactory> data_file_path_factory,
238+
file_store_path_factory_->CreateDataFilePathFactory(partition, bucket));
239+
auto writer = std::make_unique<DataFileWriter>(
240+
options_.GetFileCompression(), std::function<Status(ArrowArray*, ArrowArray*)>(),
241+
table_schema_->Id(),
242+
std::make_shared<LongCounter>(to_compact[0]->min_sequence_number),
243+
FileSource::Compact(), stats_extractor, data_file_path_factory->IsExternalPath(),
244+
write_cols, pool_);
245+
PAIMON_RETURN_NOT_OK(writer->Init(options_.GetFileSystem(),
246+
data_file_path_factory->NewPath(), writer_builder));
247+
return writer;
248+
};
249+
}
250+
251+
Result<std::unique_ptr<BatchReader>> AppendOnlyFileStoreWrite::CreateFilesReader(
252+
const BinaryRow& partition, int32_t bucket, DeletionVector::Factory dv_factory,
253+
const std::vector<std::shared_ptr<DataFileMeta>>& files) const {
254+
ReadContextBuilder context_builder(root_path_);
255+
context_builder.SetOptions(options_.ToMap())
256+
.WithFileSystem(options_.GetFileSystem())
257+
.EnablePrefetch(true)
258+
.SetPrefetchMaxParallelNum(1)
259+
.SetPrefetchBatchCount(3)
260+
.WithMemoryPool(pool_);
261+
PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<ReadContext> read_context, context_builder.Finish());
262+
std::map<std::string, std::string> options = options_.ToMap();
263+
// TODO(xinyu.lxy): temporarily disabled pre-buffer for parquet, which may cause high
264+
// memory usage during compaction. Will fix via parquet format refactor.
265+
auto new_options = options;
266+
if (new_options.find("parquet.read.enable-pre-buffer") == new_options.end()) {
267+
new_options["parquet.read.enable-pre-buffer"] = "false";
268+
}
269+
PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<InternalReadContext> internal_read_context,
270+
InternalReadContext::Create(read_context, table_schema_, new_options));
271+
auto read = std::make_unique<RawFileSplitRead>(file_store_path_factory_, internal_read_context,
272+
pool_, compact_executor_);
273+
274+
return read->CreateReader(partition, bucket, files, dv_factory);
275+
}
276+
277+
} // namespace paimon

0 commit comments

Comments
 (0)