|
| 1 | +// Licensed to the Apache Software Foundation (ASF) under one |
| 2 | +// or more contributor license agreements. See the NOTICE file |
| 3 | +// distributed with this work for additional information |
| 4 | +// regarding copyright ownership. The ASF licenses this file |
| 5 | +// to you under the Apache License, Version 2.0 (the |
| 6 | +// "License"); you may not use this file except in compliance |
| 7 | +// with the License. You may obtain a copy of the License at |
| 8 | +// |
| 9 | +// http://www.apache.org/licenses/LICENSE-2.0 |
| 10 | +// |
| 11 | +// Unless required by applicable law or agreed to in writing, |
| 12 | +// software distributed under the License is distributed on an |
| 13 | +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
| 14 | +// KIND, either express or implied. See the License for the |
| 15 | +// specific language governing permissions and limitations |
| 16 | +// under the License. |
| 17 | + |
| 18 | +#pragma once |
| 19 | + |
| 20 | +#include <gen_cpp/DataSinks_types.h> |
| 21 | + |
| 22 | +#include <memory> |
| 23 | +#include <string> |
| 24 | + |
| 25 | +#include "common/status.h" |
| 26 | +#include "core/block/block.h" |
| 27 | +#include "exec/operator/operator.h" |
| 28 | +#include "exec/sink/writer/paimon/paimon_table_writer.h" |
| 29 | +#include "runtime/runtime_state.h" |
| 30 | + |
| 31 | +namespace doris { |
| 32 | + |
| 33 | +/// Paimon table sink operator — simple pass-through to AsyncWriterSink. |
| 34 | +/// |
| 35 | +/// Each pipeline instance (LocalState) owns one PaimonTableWriter, which in |
| 36 | +/// turn owns one IPaimonWriteBackend + IPaimonWriter. Pipeline parallelism |
| 37 | +/// determines the number of concurrent Paimon writer sessions per table. |
| 38 | +/// |
| 39 | +/// Partition and bucket routing is performed internally by the Paimon SDK |
| 40 | +/// (Java via JNI, or Rust via FFI). Doris does not compute partition values |
| 41 | +/// or bucket ids; it passes complete Blocks through the backend to the SDK, |
| 42 | +/// where each row is routed via getPartition(row) + getBucket(row). |
| 43 | +/// |
| 44 | +/// This mirrors Iceberg's approach: IcebergTableSinkOperatorX delegates to |
| 45 | +/// AsyncWriterSink<VIcebergTableWriter>, with partition routing inside |
| 46 | +/// VIcebergTableWriter::write(). |
| 47 | +class PaimonTableSinkOperatorX; |
| 48 | + |
| 49 | +class PaimonTableSinkLocalState final |
| 50 | + : public AsyncWriterSink<PaimonTableWriter, PaimonTableSinkOperatorX> { |
| 51 | +public: |
| 52 | + using Base = AsyncWriterSink<PaimonTableWriter, PaimonTableSinkOperatorX>; |
| 53 | + using Parent = PaimonTableSinkOperatorX; |
| 54 | + ENABLE_FACTORY_CREATOR(PaimonTableSinkLocalState); |
| 55 | + PaimonTableSinkLocalState(DataSinkOperatorXBase* parent, RuntimeState* state) |
| 56 | + : Base(parent, state) {} |
| 57 | + Status init(RuntimeState* state, LocalSinkStateInfo& info) override; |
| 58 | + Status open(RuntimeState* state) override { |
| 59 | + SCOPED_TIMER(exec_time_counter()); |
| 60 | + SCOPED_TIMER(_open_timer); |
| 61 | + return Base::open(state); |
| 62 | + } |
| 63 | + |
| 64 | + friend class PaimonTableSinkOperatorX; |
| 65 | +}; |
| 66 | + |
| 67 | +class PaimonTableSinkOperatorX final : public DataSinkOperatorX<PaimonTableSinkLocalState> { |
| 68 | +public: |
| 69 | + using Base = DataSinkOperatorX<PaimonTableSinkLocalState>; |
| 70 | + PaimonTableSinkOperatorX(ObjectPool* pool, int operator_id, const RowDescriptor& row_desc, |
| 71 | + const std::vector<TExpr>& t_output_expr) |
| 72 | + : Base(operator_id, 0, 0), |
| 73 | + _row_desc(row_desc), |
| 74 | + _t_output_expr(t_output_expr), |
| 75 | + _pool(pool) {} |
| 76 | + |
| 77 | + Status init(const TDataSink& thrift_sink) override { |
| 78 | + RETURN_IF_ERROR(Base::init(thrift_sink)); |
| 79 | + DCHECK(thrift_sink.__isset.paimon_table_sink); |
| 80 | + RETURN_IF_ERROR(VExpr::create_expr_trees(_t_output_expr, _output_vexpr_ctxs)); |
| 81 | + return Status::OK(); |
| 82 | + } |
| 83 | + |
| 84 | + Status prepare(RuntimeState* state) override { |
| 85 | + RETURN_IF_ERROR(Base::prepare(state)); |
| 86 | + RETURN_IF_ERROR(VExpr::prepare(_output_vexpr_ctxs, state, _row_desc)); |
| 87 | + return VExpr::open(_output_vexpr_ctxs, state); |
| 88 | + } |
| 89 | + |
| 90 | + Status sink_impl(RuntimeState* state, Block* in_block, bool eos) override; |
| 91 | + |
| 92 | +private: |
| 93 | + friend class PaimonTableSinkLocalState; |
| 94 | + template <typename Writer, typename Parent> |
| 95 | + requires(std::is_base_of_v<AsyncResultWriter, Writer>) |
| 96 | + friend class AsyncWriterSink; |
| 97 | + |
| 98 | + const RowDescriptor& _row_desc; |
| 99 | + VExprContextSPtrs _output_vexpr_ctxs; |
| 100 | + const std::vector<TExpr>& _t_output_expr; |
| 101 | + ObjectPool* _pool = nullptr; |
| 102 | +}; |
| 103 | + |
| 104 | +} // namespace doris |
0 commit comments