Skip to content

Commit 22f82f8

Browse files
committed
[feature](be) Support Paimon row-location metadata in V2 scans
### What problem does this PR solve? Issue Number: close apache#66816 Related PR: None Problem Summary: Add FileScannerV2 metadata columns for Iceberg and Paimon native Parquet/ORC scans. Preserve the original file path and materialize the physical row position, while rejecting unsupported JNI and V1 reader paths. Add focused FE unit coverage and external regression suites for cross-file path/position semantics and reserved metadata-column names. ### Release note Expose Iceberg _file/_pos and Paimon __paimon_file_path/__paimon_row_index for supported native scans. ### Check List (For Author) - Test: Unit Test / Manual test - Targeted Iceberg and Paimon connector unit tests; private Iceberg cluster manual validation. External Docker regression is pending a retained CI environment. - Behavior changed: Yes (supported native external scans expose file-location metadata columns) - Does this need documentation: No
1 parent 6519448 commit 22f82f8

24 files changed

Lines changed: 733 additions & 42 deletions

‎be/src/common/consts.h‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -33,6 +33,8 @@ const std::string VIRTUAL_COLUMN_PREFIX = "__DORIS_VIRTUAL_COL__";
3333
const std::string ICEBERG_ROWID_COL = "__DORIS_ICEBERG_ROWID_COL__";
3434
const std::string ICEBERG_FILE_PATH_COL = "_file";
3535
const std::string ICEBERG_ROW_POSITION_COL = "_pos";
36+
const std::string PAIMON_FILE_PATH_COL = "__paimon_file_path";
37+
const std::string PAIMON_ROW_POSITION_COL = "__paimon_row_index";
3638

3739
/// The maximum precision representable by a 4-byte decimal (Decimal4Value)
3840
constexpr int MAX_DECIMAL32_PRECISION = 9;

‎be/src/exec/scan/file_scanner_v2.cpp‎

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -198,7 +198,9 @@ bool is_partition_slot(const TFileScanSlotInfo& slot_info, const std::string& co
198198
if (column_name.starts_with(BeConsts::GLOBAL_ROWID_COL) ||
199199
column_name == BeConsts::ICEBERG_ROWID_COL ||
200200
column_name == BeConsts::ICEBERG_FILE_PATH_COL ||
201-
column_name == BeConsts::ICEBERG_ROW_POSITION_COL) {
201+
column_name == BeConsts::ICEBERG_ROW_POSITION_COL ||
202+
column_name == BeConsts::PAIMON_FILE_PATH_COL ||
203+
column_name == BeConsts::PAIMON_ROW_POSITION_COL) {
202204
return false;
203205
}
204206
return slot_info.__isset.category ? slot_info.category == TColumnCategory::PARTITION_KEY
@@ -209,7 +211,9 @@ bool is_data_file_slot(const TFileScanSlotInfo& slot_info, const std::string& co
209211
if (column_name.starts_with(BeConsts::GLOBAL_ROWID_COL) ||
210212
column_name == BeConsts::ICEBERG_ROWID_COL ||
211213
column_name == BeConsts::ICEBERG_FILE_PATH_COL ||
212-
column_name == BeConsts::ICEBERG_ROW_POSITION_COL) {
214+
column_name == BeConsts::ICEBERG_ROW_POSITION_COL ||
215+
column_name == BeConsts::PAIMON_FILE_PATH_COL ||
216+
column_name == BeConsts::PAIMON_ROW_POSITION_COL) {
213217
return false;
214218
}
215219
// CSV and other non-self-describing formats need FE slot descriptors for only the columns that

‎be/src/format_v2/column_mapper.cpp‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -201,6 +201,10 @@ std::string virtual_column_type_to_string(TableVirtualColumnType type) {
201201
return "ICEBERG_FILE_PATH";
202202
case TableVirtualColumnType::ICEBERG_ROW_POSITION:
203203
return "ICEBERG_ROW_POSITION";
204+
case TableVirtualColumnType::PAIMON_FILE_PATH:
205+
return "PAIMON_FILE_PATH";
206+
case TableVirtualColumnType::PAIMON_ROW_POSITION:
207+
return "PAIMON_ROW_POSITION";
204208
}
205209
return "UNKNOWN";
206210
}
@@ -433,6 +437,7 @@ std::string TableColumnMapperOptions::debug_string() const {
433437
<< ", allow_idless_complex_wrapper_projection=" << allow_idless_complex_wrapper_projection
434438
<< ", enable_row_lineage_virtual_columns=" << enable_row_lineage_virtual_columns
435439
<< ", enable_iceberg_metadata_virtual_columns=" << enable_iceberg_metadata_virtual_columns
440+
<< ", enable_paimon_metadata_virtual_columns=" << enable_paimon_metadata_virtual_columns
436441
<< "}";
437442
return out.str();
438443
}
@@ -2254,6 +2259,18 @@ Status TableColumnMapper::_create_mapping_for_column(const ColumnDefinition& tab
22542259
}
22552260
return TableVirtualColumnType::INVALID;
22562261
}();
2262+
const auto paimon_metadata_type = [&] {
2263+
if (!_options.enable_paimon_metadata_virtual_columns) {
2264+
return TableVirtualColumnType::INVALID;
2265+
}
2266+
if (iequal(table_column.name, BeConsts::PAIMON_FILE_PATH_COL)) {
2267+
return TableVirtualColumnType::PAIMON_FILE_PATH;
2268+
}
2269+
if (iequal(table_column.name, BeConsts::PAIMON_ROW_POSITION_COL)) {
2270+
return TableVirtualColumnType::PAIMON_ROW_POSITION;
2271+
}
2272+
return TableVirtualColumnType::INVALID;
2273+
}();
22572274
// Row-lineage names are Iceberg metadata contracts, not reserved names in generic Hive,
22582275
// Hudi, or Paimon schemas. Only the Iceberg reader may opt into virtual synthesis.
22592276
const auto row_lineage_type =
@@ -2265,6 +2282,9 @@ Status TableColumnMapper::_create_mapping_for_column(const ColumnDefinition& tab
22652282
// physical field with the same name. The connector rejects such schema collisions; keeping
22662283
// this branch first also makes the BE safe during an FE rolling upgrade.
22672284
mapping->virtual_column_type = iceberg_metadata_type;
2285+
} else if (paimon_metadata_type != TableVirtualColumnType::INVALID) {
2286+
// Paimon metadata is carried by RawFile and is never a physical Parquet/ORC field.
2287+
mapping->virtual_column_type = paimon_metadata_type;
22682288
} else if (const auto* partition_value = find_partition_value(table_column, _partition_values);
22692289
table_column.is_partition_key && partition_value != nullptr) {
22702290
// Partition values are split constants and must take precedence over defaults.

‎be/src/format_v2/column_mapper.h‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -67,6 +67,10 @@ enum TableVirtualColumnType {
6767
ICEBERG_FILE_PATH = 4,
6868
// Iceberg absolute physical row position metadata column `_pos`.
6969
ICEBERG_ROW_POSITION = 5,
70+
// Paimon RawFile path metadata column `__paimon_file_path`.
71+
PAIMON_FILE_PATH = 6,
72+
// Paimon absolute physical row position metadata column `__paimon_row_index`.
73+
PAIMON_ROW_POSITION = 7,
7074
};
7175

7276
enum class FilterConversionType {
@@ -169,6 +173,7 @@ struct TableColumnMapperOptions {
169173
bool allow_idless_complex_wrapper_projection = false;
170174
bool enable_row_lineage_virtual_columns = false;
171175
bool enable_iceberg_metadata_virtual_columns = false;
176+
bool enable_paimon_metadata_virtual_columns = false;
172177

173178
std::string debug_string() const;
174179
};

‎be/src/format_v2/table/iceberg_reader.cpp‎

Lines changed: 8 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1113,6 +1113,10 @@ Status IcebergTableReader::materialize_virtual_columns(Block* table_block) {
11131113
case format::TableVirtualColumnType::ICEBERG_ROW_POSITION:
11141114
RETURN_IF_ERROR(_materialize_iceberg_row_position(table_block, column_idx));
11151115
break;
1116+
case format::TableVirtualColumnType::PAIMON_FILE_PATH:
1117+
case format::TableVirtualColumnType::PAIMON_ROW_POSITION:
1118+
DORIS_CHECK(false);
1119+
break;
11161120
case format::TableVirtualColumnType::INVALID:
11171121
break;
11181122
}
@@ -1913,14 +1917,12 @@ Status IcebergTableReader::_materialize_iceberg_file_path(Block* table_block, si
19131917
DORIS_CHECK(string_column != nullptr);
19141918

19151919
const auto file_path = _data_file_path();
1916-
string_column->reserve(row_position_column.size());
1917-
for (size_t row = 0; row < row_position_column.size(); ++row) {
1918-
string_column->insert_data(file_path.data(), file_path.size());
1919-
}
1920+
string_column->insert_data(file_path.data(), file_path.size());
19201921
if (nullable_column != nullptr) {
1921-
nullable_column->get_null_map_data().resize_fill(row_position_column.size(), 0);
1922+
nullable_column->get_null_map_data().resize_fill(1, 0);
19221923
}
1923-
table_block->replace_by_position(column_idx, std::move(column));
1924+
table_block->replace_by_position(
1925+
column_idx, ColumnConst::create(std::move(column), row_position_column.size()));
19241926
return Status::OK();
19251927
}
19261928

‎be/src/format_v2/table/paimon_reader.cpp‎

Lines changed: 107 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -24,10 +24,17 @@
2424
#include <string>
2525
#include <utility>
2626

27+
#include "common/consts.h"
28+
#include "core/assert_cast.h"
29+
#include "core/column/column_const.h"
30+
#include "core/column/column_nullable.h"
31+
#include "core/column/column_string.h"
32+
#include "core/column/column_vector.h"
2733
#include "core/data_type/data_type_array.h"
2834
#include "core/data_type/data_type_factory.hpp"
2935
#include "core/data_type/data_type_map.h"
3036
#include "core/data_type/data_type_nullable.h"
37+
#include "core/data_type/data_type_number.h"
3138
#include "core/data_type/data_type_struct.h"
3239
#include "core/data_type/data_type_variant_v2.h"
3340
#include "exprs/vexpr_context.h"
@@ -37,6 +44,7 @@
3744
#include "format_v2/jni/paimon_jni_reader.h"
3845
#include "format_v2/table/schema_history_util.h"
3946
#include "gen_cpp/PlanNodes_types.h"
47+
#include "util/string_util.h"
4048

4149
namespace doris::format::paimon {
4250
namespace {
@@ -229,10 +237,14 @@ Status PaimonReader::prepare_split(const format::SplitReadOptions& options) {
229237
SCOPED_TIMER(_profile.total_timer);
230238
SCOPED_TIMER(_profile.prepare_split_timer);
231239
_split_schema_id = -1;
240+
_original_file_path.clear();
232241
const auto& paimon_params = options.current_range.table_format_params.paimon_params;
233242
if (paimon_params.__isset.schema_id) {
234243
_split_schema_id = paimon_params.schema_id;
235244
}
245+
if (paimon_params.__isset.original_file_path) {
246+
_original_file_path = paimon_params.original_file_path;
247+
}
236248
}
237249
RETURN_IF_ERROR(format::TableReader::prepare_split(options));
238250
SCOPED_TIMER(_profile.total_timer);
@@ -280,10 +292,105 @@ Status PaimonReader::annotate_file_schema(std::vector<format::ColumnDefinition>*
280292
Status PaimonReader::customize_file_scan_request(format::FileScanRequest* file_request) {
281293
DORIS_CHECK(file_request != nullptr);
282294
RETURN_IF_ERROR(format::TableReader::customize_file_scan_request(file_request));
295+
if (_need_metadata_columns()) {
296+
if (_original_file_path.empty()) {
297+
return Status::InvalidArgument(
298+
"Paimon metadata columns require a FileScannerV2 native Parquet/ORC split with "
299+
"the original RawFile path");
300+
}
301+
RETURN_IF_ERROR(_append_row_position_output_column(file_request));
302+
}
283303
file_request->variant_schema_overrides = _variant_schema_overrides;
284304
return Status::OK();
285305
}
286306

307+
Status PaimonReader::materialize_virtual_columns(Block* table_block) {
308+
DORIS_CHECK(table_block != nullptr);
309+
for (size_t column_idx = 0; column_idx < _data_reader.column_mapper->mappings().size();
310+
++column_idx) {
311+
const auto& mapping = _data_reader.column_mapper->mappings()[column_idx];
312+
switch (mapping.virtual_column_type) {
313+
case format::TableVirtualColumnType::PAIMON_FILE_PATH:
314+
RETURN_IF_ERROR(_materialize_file_path(table_block, column_idx));
315+
break;
316+
case format::TableVirtualColumnType::PAIMON_ROW_POSITION:
317+
RETURN_IF_ERROR(_materialize_row_position(table_block, column_idx));
318+
break;
319+
default:
320+
break;
321+
}
322+
}
323+
return Status::OK();
324+
}
325+
326+
std::string PaimonReader::_data_file_path() const {
327+
DORIS_CHECK(!_original_file_path.empty());
328+
return _original_file_path;
329+
}
330+
331+
Status PaimonReader::_append_row_position_output_column(format::FileScanRequest* request) {
332+
const auto row_position_column_id = format::LocalColumnId(format::ROW_POSITION_COLUMN_ID);
333+
_append_file_scan_column(request, row_position_column_id, &request->non_predicate_columns);
334+
_row_position_block_position = request->local_positions.at(row_position_column_id).value();
335+
return Status::OK();
336+
}
337+
338+
Status PaimonReader::_materialize_file_path(Block* table_block, size_t column_idx) {
339+
DORIS_CHECK(_row_position_block_position < _data_reader.block_template.columns());
340+
const auto& row_position_column = assert_cast<const ColumnInt64&>(
341+
*_data_reader.block_template.get_by_position(_row_position_block_position).column);
342+
auto column = table_block->get_by_position(column_idx).type->create_column();
343+
auto* nullable_column = check_and_get_column<ColumnNullable>(*column);
344+
auto* string_column = nullable_column != nullptr
345+
? check_and_get_column<ColumnString>(
346+
nullable_column->get_nested_column_ptr().get())
347+
: check_and_get_column<ColumnString>(column.get());
348+
DORIS_CHECK(string_column != nullptr);
349+
const auto file_path = _data_file_path();
350+
string_column->insert_data(file_path.data(), file_path.size());
351+
if (nullable_column != nullptr) {
352+
nullable_column->get_null_map_data().resize_fill(1, 0);
353+
}
354+
table_block->replace_by_position(
355+
column_idx, ColumnConst::create(std::move(column), row_position_column.size()));
356+
return Status::OK();
357+
}
358+
359+
Status PaimonReader::_materialize_row_position(Block* table_block, size_t column_idx) {
360+
DORIS_CHECK(_row_position_block_position < _data_reader.block_template.columns());
361+
const auto& row_position_column = assert_cast<const ColumnInt64&>(
362+
*_data_reader.block_template.get_by_position(_row_position_block_position).column);
363+
auto column = table_block->get_by_position(column_idx).type->create_column();
364+
auto* nullable_column = check_and_get_column<ColumnNullable>(*column);
365+
auto* int_column = nullable_column != nullptr
366+
? check_and_get_column<ColumnInt64>(
367+
nullable_column->get_nested_column_ptr().get())
368+
: check_and_get_column<ColumnInt64>(column.get());
369+
DORIS_CHECK(int_column != nullptr);
370+
int_column->get_data().assign(row_position_column.get_data().begin(),
371+
row_position_column.get_data().end());
372+
if (nullable_column != nullptr) {
373+
nullable_column->get_null_map_data().resize_fill(row_position_column.size(), 0);
374+
}
375+
table_block->replace_by_position(column_idx, std::move(column));
376+
return Status::OK();
377+
}
378+
379+
bool PaimonReader::_need_metadata_columns() const {
380+
if (_data_reader.column_mapper != nullptr) {
381+
return std::ranges::any_of(_data_reader.column_mapper->mappings(), [](const auto& mapping) {
382+
return mapping.virtual_column_type ==
383+
format::TableVirtualColumnType::PAIMON_FILE_PATH ||
384+
mapping.virtual_column_type ==
385+
format::TableVirtualColumnType::PAIMON_ROW_POSITION;
386+
});
387+
}
388+
return std::ranges::any_of(_projected_columns, [](const auto& column) {
389+
return iequal(column.name, BeConsts::PAIMON_FILE_PATH_COL) ||
390+
iequal(column.name, BeConsts::PAIMON_ROW_POSITION_COL);
391+
});
392+
}
393+
287394
Status PaimonReader::_parse_deletion_vector_file(const TTableFormatFileDesc& t_desc,
288395
DeleteFileDesc* desc, bool* has_delete_file) {
289396
DORIS_CHECK(desc != nullptr);

‎be/src/format_v2/table/paimon_reader.h‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -53,15 +53,27 @@ class PaimonReader final : public format::TableReader {
5353
#endif
5454

5555
protected:
56+
void configure_mapper_options(format::TableColumnMapperOptions* options) const override {
57+
options->enable_paimon_metadata_virtual_columns = true;
58+
}
5659
format::TableColumnMappingMode mapping_mode() const override;
5760
Status annotate_file_schema(std::vector<format::ColumnDefinition>* file_schema) override;
5861
Status customize_file_scan_request(format::FileScanRequest* file_request) override;
62+
Status materialize_virtual_columns(Block* table_block) override;
5963

6064
Status _parse_deletion_vector_file(const TTableFormatFileDesc& t_desc, DeleteFileDesc* desc,
6165
bool* has_delete_file) override;
6266

6367
private:
68+
std::string _data_file_path() const;
69+
Status _append_row_position_output_column(format::FileScanRequest* request);
70+
Status _materialize_file_path(Block* table_block, size_t column_idx);
71+
Status _materialize_row_position(Block* table_block, size_t column_idx);
72+
bool _need_metadata_columns() const;
73+
6474
int64_t _split_schema_id = -1;
75+
size_t _row_position_block_position = 0;
76+
std::string _original_file_path;
6577
std::vector<format::LocalColumnIndex> _variant_schema_overrides;
6678
};
6779

‎fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergConnectorMetadata.java‎

Lines changed: 10 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -731,7 +731,7 @@ public Map<String, ConnectorColumnHandle> getColumnHandles(
731731
// metadata-table columns (t$snapshots -> committed_at/...) so the generic scan node can look up
732732
// its pruned sys-table slots by name; a data handle resolves the base table's columns.
733733
Table table = iceHandle.isSystemTable() ? loadSysTable(session, iceHandle) : loadTable(session, iceHandle);
734-
return buildColumnHandles(table.schema());
734+
return buildColumnHandles(table.schema(), !iceHandle.isSystemTable());
735735
}
736736

737737
@Override
@@ -746,21 +746,27 @@ public Map<String, ConnectorColumnHandle> getColumnHandles(
746746
Schema schema = table.currentSnapshot() == null
747747
? table.schema() : table.schemas().get((int) snapshot.getSchemaId());
748748
// Keep the handle-schema fallback identical to getTableSchema so slots and handles cannot diverge.
749-
return buildColumnHandles(schema == null ? table.schema() : schema);
749+
return buildColumnHandles(schema == null ? table.schema() : schema, true);
750750
}
751751

752752
@Override
753753
public boolean supportsColumnHandleSnapshotPin(ConnectorSession session) {
754754
return true;
755755
}
756756

757-
private static Map<String, ConnectorColumnHandle> buildColumnHandles(Schema schema) {
757+
private static Map<String, ConnectorColumnHandle> buildColumnHandles(
758+
Schema schema, boolean appendDataFileMetadataColumns) {
758759
List<Types.NestedField> fields = schema.columns();
759-
Map<String, ConnectorColumnHandle> handles = new LinkedHashMap<>(fields.size());
760+
Map<String, ConnectorColumnHandle> handles = new LinkedHashMap<>(
761+
fields.size() + (appendDataFileMetadataColumns ? 2 : 0));
760762
for (Types.NestedField field : fields) {
761763
String name = field.name();
762764
handles.put(name, new IcebergColumnHandle(name, field.fieldId()));
763765
}
766+
if (appendDataFileMetadataColumns) {
767+
handles.put(ICEBERG_FILE_PATH_COL, new IcebergColumnHandle(ICEBERG_FILE_PATH_COL, -1));
768+
handles.put(ICEBERG_ROW_POSITION_COL, new IcebergColumnHandle(ICEBERG_ROW_POSITION_COL, -1));
769+
}
764770
return handles;
765771
}
766772

0 commit comments

Comments
 (0)