Skip to content

Commit e2561c2

Browse files
committed
[feature](paimon) Add Paimon table write runtime
Issue Number: close apache#65086 Related PR: apache#65868, apache#66612, apache#66810 Problem Summary: Add the BE Paimon sink, JNI writer backend, Java writer, commit payload transport, and worktree-local spill and memory lifecycle support required to write Paimon tables from master. Support writing Apache Paimon tables through the native Doris execution pipeline. - Test: Static validation - BE clang-format/check-format and FE checkstyle; full build and tests are deferred until all forward-port picks are complete - Behavior changed: Yes, adds Paimon table writes - Does this need documentation: Yes, documentation can follow separately
1 parent 4872fed commit e2561c2

39 files changed

Lines changed: 5510 additions & 15 deletions

‎be/src/common/config.cpp‎

Lines changed: 35 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1782,6 +1782,41 @@ DEFINE_mInt64(paimon_jni_writer_memory_pool_limit_bytes, "536870912"); // 512MB
17821782
DEFINE_Validator(paimon_jni_writer_memory_pool_limit_bytes,
17831783
[](int64_t bytes) -> bool { return bytes > 0; });
17841784

1785+
// URI scheme to Doris file type mappings used by paimon-cpp DorisFileSystem.
1786+
// Each entry uses the format "<scheme>=<file_type>", and file_type must be one of:
1787+
// local, hdfs, s3, http, broker.
1788+
DEFINE_Strings(paimon_file_system_scheme_mappings,
1789+
"file=local,hdfs=hdfs,viewfs=hdfs,local=hdfs,jfs=hdfs,"
1790+
"s3=s3,s3a=s3,s3n=s3,oss=s3,obs=s3,cos=s3,cosn=s3,gs=s3,"
1791+
"abfs=s3,abfss=s3,wasb=s3,wasbs=s3,http=http,https=http,"
1792+
"ofs=broker,gfs=broker");
1793+
DEFINE_Validator(paimon_file_system_scheme_mappings,
1794+
([](const std::vector<std::string>& mappings) -> bool {
1795+
doris::StringCaseUnorderedSet seen_schemes;
1796+
static const doris::StringCaseUnorderedSet supported_types = {
1797+
"local", "hdfs", "s3", "http", "broker"};
1798+
for (const auto& raw_entry : mappings) {
1799+
std::string_view entry = doris::trim(raw_entry);
1800+
size_t separator = entry.find('=');
1801+
if (separator == std::string_view::npos) {
1802+
return false;
1803+
}
1804+
std::string scheme = std::string(doris::trim(entry.substr(0, separator)));
1805+
std::string file_type =
1806+
std::string(doris::trim(entry.substr(separator + 1)));
1807+
if (scheme.empty() || file_type.empty()) {
1808+
return false;
1809+
}
1810+
if (supported_types.find(file_type) == supported_types.end()) {
1811+
return false;
1812+
}
1813+
if (!seen_schemes.insert(scheme).second) {
1814+
return false;
1815+
}
1816+
}
1817+
return true;
1818+
}));
1819+
17851820
DEFINE_mInt32(thrift_client_open_num_tries, "1");
17861821

17871822
// Default HTTP scheme used by S3Client when the endpoint has no scheme.

‎be/src/common/config.h‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1804,6 +1804,9 @@ DECLARE_mInt64(iceberg_sink_max_file_size);
18041804
// Hard upper bound for Doris-managed Paimon write-buffer memory per JNI writer.
18051805
DECLARE_mInt64(paimon_jni_writer_memory_pool_limit_bytes);
18061806

1807+
/** Paimon file system configurations **/
1808+
DECLARE_Strings(paimon_file_system_scheme_mappings);
1809+
18071810
// Number of open tries, default 1 means only try to open once.
18081811
// Retry the Open num_retries time waiting 100 milliseconds between retries.
18091812
DECLARE_mInt32(thrift_client_open_num_tries);

‎be/src/exec/operator/operator.cpp‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -64,6 +64,7 @@
6464
#include "exec/operator/olap_scan_operator.h"
6565
#include "exec/operator/olap_table_sink_operator.h"
6666
#include "exec/operator/olap_table_sink_v2_operator.h"
67+
#include "exec/operator/paimon_table_sink_operator.h"
6768
#include "exec/operator/partition_sort_sink_operator.h"
6869
#include "exec/operator/partition_sort_source_operator.h"
6970
#include "exec/operator/partitioned_aggregation_sink_operator.h"
@@ -846,6 +847,7 @@ DECLARE_OPERATOR(OlapTableSinkV2LocalState)
846847
DECLARE_OPERATOR(HiveTableSinkLocalState)
847848
DECLARE_OPERATOR(TVFTableSinkLocalState)
848849
DECLARE_OPERATOR(IcebergTableSinkLocalState)
850+
DECLARE_OPERATOR(PaimonTableSinkLocalState)
849851
DECLARE_OPERATOR(SpillIcebergTableSinkLocalState)
850852
DECLARE_OPERATOR(IcebergDeleteSinkLocalState)
851853
DECLARE_OPERATOR(IcebergMergeSinkLocalState)
Lines changed: 83 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,83 @@
1+
// Licensed to the Apache Software Foundation (ASF) under one
2+
// or more contributor license agreements. See the NOTICE file
3+
// distributed with this work for additional information
4+
// regarding copyright ownership. The ASF licenses this file
5+
// to you under the Apache License, Version 2.0 (the
6+
// "License"); you may not use this file except in compliance
7+
// with the License. You may obtain a copy of the License at
8+
//
9+
// http://www.apache.org/licenses/LICENSE-2.0
10+
//
11+
// Unless required by applicable law or agreed to in writing,
12+
// software distributed under the License is distributed on an
13+
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
// KIND, either express or implied. See the License for the
15+
// specific language governing permissions and limitations
16+
// under the License.
17+
18+
#include "exec/operator/paimon_table_sink_operator.h"
19+
20+
#include "common/logging.h"
21+
22+
namespace doris {
23+
24+
Status PaimonTableSinkLocalState::init(RuntimeState* state, LocalSinkStateInfo& info) {
25+
RETURN_IF_ERROR(Base::init(state, info));
26+
_writer = std::make_unique<PaimonTableWriter>(info.tsink, _output_vexpr_ctxs);
27+
return Status::OK();
28+
}
29+
30+
Status PaimonTableSinkLocalState::open(RuntimeState* state) {
31+
SCOPED_TIMER(exec_time_counter());
32+
SCOPED_TIMER(_open_timer);
33+
RETURN_IF_ERROR(Base::open(state));
34+
35+
auto& parent = _parent->cast<Parent>();
36+
_output_vexpr_ctxs.resize(parent._output_vexpr_ctxs.size());
37+
for (size_t i = 0; i < _output_vexpr_ctxs.size(); ++i) {
38+
RETURN_IF_ERROR(parent._output_vexpr_ctxs[i]->clone(state, _output_vexpr_ctxs[i]));
39+
}
40+
return _writer->open(state, operator_profile());
41+
}
42+
43+
Status PaimonTableSinkLocalState::close(RuntimeState* state, Status exec_status) {
44+
if (_closed) {
45+
return Status::OK();
46+
}
47+
48+
SCOPED_TIMER(exec_time_counter());
49+
SCOPED_TIMER(_close_timer);
50+
51+
Status final_status = exec_status;
52+
if (_writer) {
53+
Status writer_status = _writer->close(exec_status);
54+
if (final_status.ok() && !writer_status.ok()) {
55+
final_status = writer_status;
56+
}
57+
_writer.reset();
58+
}
59+
60+
Status base_status = Base::close(state, final_status);
61+
if (final_status.ok() && !base_status.ok()) {
62+
final_status = base_status;
63+
}
64+
return final_status;
65+
}
66+
67+
Status PaimonTableSinkOperatorX::sink_impl(RuntimeState* state, Block* in_block, bool /*eos*/) {
68+
auto& local_state = get_local_state(state);
69+
SCOPED_TIMER(local_state.exec_time_counter());
70+
COUNTER_UPDATE(local_state.rows_input_counter(), static_cast<int64_t>(in_block->rows()));
71+
72+
if (in_block->rows() == 0) {
73+
return Status::OK();
74+
}
75+
76+
// This is a synchronous SDK call. The LocalState is marked blockable, so
77+
// the whole pipeline task (including open and close) runs on the blocking
78+
// scheduler instead of occupying a regular pipeline worker.
79+
DCHECK(local_state._writer);
80+
return local_state._writer->write(state, *in_block);
81+
}
82+
83+
} // namespace doris
Lines changed: 102 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,102 @@
1+
// Licensed to the Apache Software Foundation (ASF) under one
2+
// or more contributor license agreements. See the NOTICE file
3+
// distributed with this work for additional information
4+
// regarding copyright ownership. The ASF licenses this file
5+
// to you under the Apache License, Version 2.0 (the
6+
// "License"); you may not use this file except in compliance
7+
// with the License. You may obtain a copy of the License at
8+
//
9+
// http://www.apache.org/licenses/LICENSE-2.0
10+
//
11+
// Unless required by applicable law or agreed to in writing,
12+
// software distributed under the License is distributed on an
13+
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
// KIND, either express or implied. See the License for the
15+
// specific language governing permissions and limitations
16+
// under the License.
17+
18+
#pragma once
19+
20+
#include <gen_cpp/DataSinks_types.h>
21+
22+
#include <memory>
23+
24+
#include "common/status.h"
25+
#include "core/block/block.h"
26+
#include "exec/operator/operator.h"
27+
#include "exec/sink/writer/paimon/paimon_table_writer.h"
28+
#include "runtime/runtime_state.h"
29+
30+
namespace doris {
31+
32+
/// Paimon table sink operator.
33+
///
34+
/// Each pipeline instance (LocalState) owns one PaimonTableWriter, which in
35+
/// turn owns one IPaimonWriteBackend + IPaimonWriter. Pipeline parallelism
36+
/// determines the number of concurrent Paimon writer sessions per table.
37+
/// Paimon writes are synchronous: sink_impl() returns only after the SDK has
38+
/// consumed the input Block. The LocalState is therefore always blockable so
39+
/// that open, write, and close run on the pipeline blocking scheduler.
40+
/// Doris-owned Arrow buffers remain under the query MemTracker, while Paimon
41+
/// pages are allocated lazily under DorisMemorySegmentPool's fixed cap. The
42+
/// sink uses the generic pipeline minimum reservation only as an admission
43+
/// guard; it does not try to predict Paimon's future page demand.
44+
///
45+
/// The upstream sink Exchange may reproduce Paimon's stateless HASH_FIXED
46+
/// selector to establish unique writer ownership. The writer still passes
47+
/// complete Blocks to the SDK, which independently computes partition and
48+
/// bucket values for file writing; no routing column is appended to the row.
49+
class PaimonTableSinkOperatorX;
50+
51+
class PaimonTableSinkLocalState final : public PipelineXSinkLocalState<FakeSharedState> {
52+
public:
53+
using Base = PipelineXSinkLocalState<FakeSharedState>;
54+
using Parent = PaimonTableSinkOperatorX;
55+
ENABLE_FACTORY_CREATOR(PaimonTableSinkLocalState);
56+
PaimonTableSinkLocalState(DataSinkOperatorXBase* parent, RuntimeState* state)
57+
: Base(parent, state) {}
58+
59+
Status init(RuntimeState* state, LocalSinkStateInfo& info) override;
60+
Status open(RuntimeState* state) override;
61+
Status close(RuntimeState* state, Status exec_status) override;
62+
63+
[[nodiscard]] bool is_blockable() const override { return true; }
64+
65+
private:
66+
friend class PaimonTableSinkOperatorX;
67+
68+
VExprContextSPtrs _output_vexpr_ctxs;
69+
std::unique_ptr<PaimonTableWriter> _writer;
70+
};
71+
72+
class PaimonTableSinkOperatorX final : public DataSinkOperatorX<PaimonTableSinkLocalState> {
73+
public:
74+
using Base = DataSinkOperatorX<PaimonTableSinkLocalState>;
75+
PaimonTableSinkOperatorX(int operator_id, const RowDescriptor& row_desc,
76+
const std::vector<TExpr>& t_output_expr)
77+
: Base(operator_id, 0, 0), _row_desc(row_desc), _t_output_expr(t_output_expr) {}
78+
79+
Status init(const TDataSink& thrift_sink) override {
80+
RETURN_IF_ERROR(Base::init(thrift_sink));
81+
DCHECK(thrift_sink.__isset.paimon_table_sink);
82+
RETURN_IF_ERROR(VExpr::create_expr_trees(_t_output_expr, _output_vexpr_ctxs));
83+
return Status::OK();
84+
}
85+
86+
Status prepare(RuntimeState* state) override {
87+
RETURN_IF_ERROR(Base::prepare(state));
88+
RETURN_IF_ERROR(VExpr::prepare(_output_vexpr_ctxs, state, _row_desc));
89+
return VExpr::open(_output_vexpr_ctxs, state);
90+
}
91+
92+
Status sink_impl(RuntimeState* state, Block* in_block, bool eos) override;
93+
94+
private:
95+
friend class PaimonTableSinkLocalState;
96+
97+
const RowDescriptor& _row_desc;
98+
VExprContextSPtrs _output_vexpr_ctxs;
99+
const std::vector<TExpr>& _t_output_expr;
100+
};
101+
102+
} // namespace doris

‎be/src/exec/pipeline/pipeline_fragment_context.cpp‎

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -88,6 +88,7 @@
8888
#include "exec/operator/olap_scan_operator.h"
8989
#include "exec/operator/olap_table_sink_operator.h"
9090
#include "exec/operator/olap_table_sink_v2_operator.h"
91+
#include "exec/operator/paimon_table_sink_operator.h"
9192
#include "exec/operator/partition_sort_sink_operator.h"
9293
#include "exec/operator/partition_sort_source_operator.h"
9394
#include "exec/operator/partitioned_aggregation_sink_operator.h"
@@ -1371,6 +1372,14 @@ Status PipelineFragmentContext::_create_data_sink(ObjectPool* pool, const TDataS
13711372
output_exprs);
13721373
break;
13731374
}
1375+
case TDataSinkType::PAIMON_TABLE_SINK: {
1376+
if (!thrift_sink.__isset.paimon_table_sink) {
1377+
return Status::InternalError("Missing paimon table sink.");
1378+
}
1379+
_sink = std::make_shared<PaimonTableSinkOperatorX>(next_sink_operator_id(), row_desc,
1380+
output_exprs);
1381+
break;
1382+
}
13741383
case TDataSinkType::JDBC_TABLE_SINK: {
13751384
if (!thrift_sink.__isset.jdbc_table_sink) {
13761385
return Status::InternalError("Missing data jdbc sink.");
@@ -2533,6 +2542,20 @@ void PipelineFragmentContext::_coordinator_callback(const ReportStatusRequest& r
25332542
}
25342543
_append_external_file_commit_data(req, &params);
25352544

2545+
if (auto pcm = req.runtime_state->paimon_commit_messages(); !pcm.empty()) {
2546+
params.__isset.paimon_commit_messages = true;
2547+
params.paimon_commit_messages.insert(params.paimon_commit_messages.end(), pcm.begin(),
2548+
pcm.end());
2549+
} else if (!req.runtime_states.empty()) {
2550+
for (auto* rs : req.runtime_states) {
2551+
if (auto rs_pcm = rs->paimon_commit_messages(); !rs_pcm.empty()) {
2552+
params.__isset.paimon_commit_messages = true;
2553+
params.paimon_commit_messages.insert(params.paimon_commit_messages.end(),
2554+
rs_pcm.begin(), rs_pcm.end());
2555+
}
2556+
}
2557+
}
2558+
25362559
req.runtime_state->get_unreported_errors(&(params.error_log));
25372560
params.__isset.error_log = (!params.error_log.empty());
25382561

Lines changed: 34 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,34 @@
1+
// Licensed to the Apache Software Foundation (ASF) under one
2+
// or more contributor license agreements. See the NOTICE file
3+
// distributed with this work for additional information
4+
// regarding copyright ownership. The ASF licenses this file
5+
// to you under the Apache License, Version 2.0 (the
6+
// "License"); you may not use this file except in compliance
7+
// with the License. You may obtain a copy of the License at
8+
//
9+
// http://www.apache.org/licenses/LICENSE-2.0
10+
//
11+
// Unless required by applicable law or agreed to in writing,
12+
// software distributed under the License is distributed on an
13+
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
// KIND, either express or implied. See the License for the
15+
// specific language governing permissions and limitations
16+
// under the License.
17+
18+
#include "exec/sink/writer/paimon/ffi_paimon_write_backend.h"
19+
20+
namespace doris {
21+
22+
Status FfiPaimonWriteBackend::open(const TPaimonTableSink&, RuntimeState*, RuntimeProfile*) {
23+
return Status::NotSupported("Paimon Rust FFI writer is not implemented");
24+
}
25+
26+
Status FfiPaimonWriteBackend::create_writer(std::unique_ptr<IPaimonWriter>*) {
27+
return Status::NotSupported("Paimon Rust FFI writer is not implemented");
28+
}
29+
30+
Status FfiPaimonWriteBackend::close() {
31+
return Status::OK();
32+
}
33+
34+
} // namespace doris
Lines changed: 36 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,36 @@
1+
// Licensed to the Apache Software Foundation (ASF) under one
2+
// or more contributor license agreements. See the NOTICE file
3+
// distributed with this work for additional information
4+
// regarding copyright ownership. The ASF licenses this file
5+
// to you under the Apache License, Version 2.0 (the
6+
// "License"); you may not use this file except in compliance
7+
// with the License. You may obtain a copy of the License at
8+
//
9+
// http://www.apache.org/licenses/LICENSE-2.0
10+
//
11+
// Unless required by applicable law or agreed to in writing,
12+
// software distributed under the License is distributed on an
13+
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
// KIND, either express or implied. See the License for the
15+
// specific language governing permissions and limitations
16+
// under the License.
17+
18+
#pragma once
19+
20+
#include "exec/sink/writer/paimon/paimon_write_backend.h"
21+
22+
namespace doris {
23+
24+
/// Placeholder for the future paimon-rust writer implementation. Keeping this
25+
/// backend in the factory makes the integration boundary explicit without
26+
/// introducing a BE commit contract that the Rust writer will not own.
27+
class FfiPaimonWriteBackend final : public IPaimonWriteBackend {
28+
public:
29+
Status open(const TPaimonTableSink& sink, RuntimeState* state,
30+
RuntimeProfile* profile) override;
31+
Status create_writer(std::unique_ptr<IPaimonWriter>* writer) override;
32+
Status close() override;
33+
PaimonBackendType type() const override { return PaimonBackendType::FFI; }
34+
};
35+
36+
} // namespace doris

0 commit comments

Comments
 (0)