Skip to content

Commit ad4e9d7

Browse files
authored
[feature](be) Support file metadata columns in V2 scans (#67207)
### What problem does this PR solve? Issue Number: close #66816 Related PR: None Problem Summary: FileScannerV2 native Parquet/ORC scans did not expose stable file-location metadata for Iceberg and Paimon. This change adds Iceberg `_file`/`_pos` and Paimon `__paimon_file_path`/`__paimon_row_index`, preserves original source paths, and rejects unsupported JNI or V1 reader paths. ### Release note Expose file path and physical row position metadata columns for supported native Iceberg and Paimon scans. ### Check List (For Author) - Test: Unit Test / Manual test / Regression test pending - Targeted Iceberg and Paimon connector unit tests passed. - Private Iceberg cluster manual validation passed. - External Docker regression will be run through `run buildall` and a retained CI environment. - Behavior changed: Yes (supported native external scans expose file-location metadata columns) - Does this need documentation: No
1 parent dc848b3 commit ad4e9d7

36 files changed

Lines changed: 1519 additions & 96 deletions

‎be/src/common/consts.h‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,10 @@ const std::string DYNAMIC_COLUMN_NAME = "__DORIS_DYNAMIC_COL__";
3030
const std::string PARTIAL_UPDATE_AUTO_INC_COL = "__PARTIAL_UPDATE_AUTO_INC_COLUMN__";
3131
const std::string VIRTUAL_COLUMN_PREFIX = "__DORIS_VIRTUAL_COL__";
3232
const std::string ICEBERG_ROWID_COL = "__DORIS_ICEBERG_ROWID_COL__";
33+
const std::string ICEBERG_FILE_PATH_COL = "_file";
34+
const std::string ICEBERG_ROW_POSITION_COL = "_pos";
35+
const std::string PAIMON_FILE_PATH_COL = "__paimon_file_path";
36+
const std::string PAIMON_ROW_POSITION_COL = "__paimon_row_index";
3337

3438
/// The maximum precision representable by a 4-byte decimal (Decimal4Value)
3539
constexpr int MAX_DECIMAL32_PRECISION = 9;

‎be/src/exec/scan/file_scanner_v2.cpp‎

Lines changed: 12 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -75,6 +75,7 @@
7575
#include "runtime/runtime_state.h"
7676
#include "service/backend_options.h"
7777
#include "storage/id_manager.h"
78+
#include "util/string_util.h"
7879

7980
namespace doris {
8081
namespace {
@@ -190,20 +191,19 @@ bool is_wal_format(TFileFormatType::type format_type) {
190191
return format_type == TFileFormatType::FORMAT_WAL;
191192
}
192193

194+
bool is_legacy_virtual_slot(const std::string& column_name) {
195+
return column_name.starts_with(BeConsts::GLOBAL_ROWID_COL) ||
196+
iequal(column_name, BeConsts::ICEBERG_ROWID_COL);
197+
}
198+
193199
bool is_partition_slot(const TFileScanSlotInfo& slot_info, const std::string& column_name) {
194-
if (column_name.starts_with(BeConsts::GLOBAL_ROWID_COL) ||
195-
column_name == BeConsts::ICEBERG_ROWID_COL) {
196-
return false;
200+
if (slot_info.__isset.category) {
201+
return slot_info.category == TColumnCategory::PARTITION_KEY;
197202
}
198-
return slot_info.__isset.category ? slot_info.category == TColumnCategory::PARTITION_KEY
199-
: !slot_info.is_file_slot;
203+
return !slot_info.is_file_slot && !is_legacy_virtual_slot(column_name);
200204
}
201205

202206
bool is_data_file_slot(const TFileScanSlotInfo& slot_info, const std::string& column_name) {
203-
if (column_name.starts_with(BeConsts::GLOBAL_ROWID_COL) ||
204-
column_name == BeConsts::ICEBERG_ROWID_COL) {
205-
return false;
206-
}
207207
// CSV and other non-self-describing formats need FE slot descriptors for only the columns that
208208
// are physically read from the file. Partition/default/virtual columns stay in TableReader's
209209
// mapping layer and are materialized after the file-local block is read. New FE provides an
@@ -212,7 +212,7 @@ bool is_data_file_slot(const TFileScanSlotInfo& slot_info, const std::string& co
212212
return slot_info.category == TColumnCategory::REGULAR ||
213213
slot_info.category == TColumnCategory::GENERATED;
214214
}
215-
return slot_info.is_file_slot;
215+
return slot_info.is_file_slot && !is_legacy_virtual_slot(column_name);
216216
}
217217

218218
Status rewrite_slot_refs_to_global_index(
@@ -813,6 +813,8 @@ Status FileScannerV2::_build_projected_columns(const format::TableReader& table_
813813
slot_info.slot_id);
814814
}
815815
auto column = _build_table_column(it->second);
816+
column.is_synthesized =
817+
slot_info.__isset.category && slot_info.category == TColumnCategory::SYNTHESIZED;
816818
_has_variant_projection = _has_variant_projection || contains_variant_type(column.type);
817819
build_context.slot_desc = it->second;
818820
if (column.name.starts_with(BeConsts::GLOBAL_ROWID_COL)) {

‎be/src/format_v2/column_data.h‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -291,6 +291,10 @@ struct ColumnDefinition {
291291
// Partition columns are constants from split metadata and should not be matched against file
292292
// schema unless table-format logic explicitly asks for it.
293293
bool is_partition_key = false;
294+
// Synthesized table columns are produced by the table reader rather than matched against a
295+
// physical file field. This comes from the explicit FE slot category; old FE plans leave it
296+
// false so a physical column that shares a metadata spelling keeps its legacy behavior.
297+
bool is_synthesized = false;
294298
// File-local column kind. For table/global columns this remains DATA_COLUMN.
295299
ColumnType column_type = ColumnType::DATA_COLUMN;
296300

‎be/src/format_v2/column_mapper.cpp‎

Lines changed: 49 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -197,6 +197,14 @@ std::string virtual_column_type_to_string(TableVirtualColumnType type) {
197197
return "LAST_UPDATED_SEQUENCE_NUMBER";
198198
case TableVirtualColumnType::ICEBERG_ROWID:
199199
return "ICEBERG_ROWID";
200+
case TableVirtualColumnType::ICEBERG_FILE_PATH:
201+
return "ICEBERG_FILE_PATH";
202+
case TableVirtualColumnType::ICEBERG_ROW_POSITION:
203+
return "ICEBERG_ROW_POSITION";
204+
case TableVirtualColumnType::PAIMON_FILE_PATH:
205+
return "PAIMON_FILE_PATH";
206+
case TableVirtualColumnType::PAIMON_ROW_POSITION:
207+
return "PAIMON_ROW_POSITION";
200208
}
201209
return "UNKNOWN";
202210
}
@@ -427,7 +435,10 @@ std::string TableColumnMapperOptions::debug_string() const {
427435
out << "TableColumnMapperOptions{mode=" << mapping_mode_to_string(mode)
428436
<< ", reject_missing_required_field=" << reject_missing_required_field
429437
<< ", allow_idless_complex_wrapper_projection=" << allow_idless_complex_wrapper_projection
430-
<< ", enable_row_lineage_virtual_columns=" << enable_row_lineage_virtual_columns << "}";
438+
<< ", enable_row_lineage_virtual_columns=" << enable_row_lineage_virtual_columns
439+
<< ", enable_iceberg_metadata_virtual_columns=" << enable_iceberg_metadata_virtual_columns
440+
<< ", enable_paimon_metadata_virtual_columns=" << enable_paimon_metadata_virtual_columns
441+
<< "}";
431442
return out.str();
432443
}
433444

@@ -454,7 +465,8 @@ std::string ColumnDefinition::debug_string() const {
454465
} else {
455466
out << "unknown";
456467
}
457-
out << ", is_partition_key=" << is_partition_key << "}";
468+
out << ", is_partition_key=" << is_partition_key << ", is_synthesized=" << is_synthesized
469+
<< "}";
458470
return out.str();
459471
}
460472

@@ -2259,14 +2271,47 @@ Status TableColumnMapper::_create_mapping_for_column(const ColumnDefinition& tab
22592271
mapping->table_column_name = table_column.name;
22602272
mapping->table_type = table_column.type;
22612273
mapping->variant_access_paths = table_column.variant_access_paths;
2274+
const auto iceberg_metadata_type = [&] {
2275+
if (!_options.enable_iceberg_metadata_virtual_columns || !table_column.is_synthesized) {
2276+
return TableVirtualColumnType::INVALID;
2277+
}
2278+
if (iequal(table_column.name, BeConsts::ICEBERG_FILE_PATH_COL)) {
2279+
return TableVirtualColumnType::ICEBERG_FILE_PATH;
2280+
}
2281+
if (iequal(table_column.name, BeConsts::ICEBERG_ROW_POSITION_COL)) {
2282+
return TableVirtualColumnType::ICEBERG_ROW_POSITION;
2283+
}
2284+
return TableVirtualColumnType::INVALID;
2285+
}();
2286+
const auto paimon_metadata_type = [&] {
2287+
if (!_options.enable_paimon_metadata_virtual_columns || !table_column.is_synthesized) {
2288+
return TableVirtualColumnType::INVALID;
2289+
}
2290+
if (iequal(table_column.name, BeConsts::PAIMON_FILE_PATH_COL)) {
2291+
return TableVirtualColumnType::PAIMON_FILE_PATH;
2292+
}
2293+
if (iequal(table_column.name, BeConsts::PAIMON_ROW_POSITION_COL)) {
2294+
return TableVirtualColumnType::PAIMON_ROW_POSITION;
2295+
}
2296+
return TableVirtualColumnType::INVALID;
2297+
}();
22622298
// Row-lineage names are Iceberg metadata contracts, not reserved names in generic Hive,
22632299
// Hudi, or Paimon schemas. Only the Iceberg reader may opt into virtual synthesis.
22642300
const auto row_lineage_type =
22652301
_options.enable_row_lineage_virtual_columns
22662302
? row_lineage_virtual_column_type(table_column, _options.mode)
22672303
: TableVirtualColumnType::INVALID;
2268-
if (const auto* partition_value = find_partition_value(table_column, _partition_values);
2269-
table_column.is_partition_key && partition_value != nullptr) {
2304+
if (iceberg_metadata_type != TableVirtualColumnType::INVALID) {
2305+
// Iceberg `_file` and `_pos` are metadata contracts only when the current FE explicitly
2306+
// classifies the slot as synthesized. Old FE plans can still read physical fields with the
2307+
// same spelling during a rolling upgrade.
2308+
mapping->virtual_column_type = iceberg_metadata_type;
2309+
} else if (paimon_metadata_type != TableVirtualColumnType::INVALID) {
2310+
// Paimon metadata is carried by RawFile. The explicit synthesized marker prevents a
2311+
// physical same-name field from being reinterpreted during a rolling upgrade.
2312+
mapping->virtual_column_type = paimon_metadata_type;
2313+
} else if (const auto* partition_value = find_partition_value(table_column, _partition_values);
2314+
table_column.is_partition_key && partition_value != nullptr) {
22702315
// Partition values are split constants and must take precedence over defaults.
22712316
_set_constant_mapping(mapping, VExprContext::create_shared(VLiteral::create_shared(
22722317
mapping->table_type, *partition_value)));

‎be/src/format_v2/column_mapper.h‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -63,6 +63,14 @@ enum TableVirtualColumnType {
6363
// Doris internal Iceberg row locator column `__DORIS_ICEBERG_ROWID_COL__`.
6464
// It is a struct used by delete/update/merge, not the Iceberg `_row_id`.
6565
ICEBERG_ROWID = 3,
66+
// Iceberg data-file path metadata column `_file`.
67+
ICEBERG_FILE_PATH = 4,
68+
// Iceberg absolute physical row position metadata column `_pos`.
69+
ICEBERG_ROW_POSITION = 5,
70+
// Paimon RawFile path metadata column `__paimon_file_path`.
71+
PAIMON_FILE_PATH = 6,
72+
// Paimon absolute physical row position metadata column `__paimon_row_index`.
73+
PAIMON_ROW_POSITION = 7,
6674
};
6775

6876
enum class FilterConversionType {
@@ -164,6 +172,8 @@ struct TableColumnMapperOptions {
164172
bool reject_missing_required_field = false;
165173
bool allow_idless_complex_wrapper_projection = false;
166174
bool enable_row_lineage_virtual_columns = false;
175+
bool enable_iceberg_metadata_virtual_columns = false;
176+
bool enable_paimon_metadata_virtual_columns = false;
167177

168178
std::string debug_string() const;
169179
};

‎be/src/format_v2/table/iceberg_reader.cpp‎

Lines changed: 77 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -183,6 +183,11 @@ static bool is_projected_iceberg_rowid(const format::ColumnDefinition& column) {
183183
return column.name == BeConsts::ICEBERG_ROWID_COL;
184184
}
185185

186+
static bool is_projected_iceberg_metadata(const format::ColumnDefinition& column) {
187+
return iequal(column.name, BeConsts::ICEBERG_FILE_PATH_COL) ||
188+
iequal(column.name, BeConsts::ICEBERG_ROW_POSITION_COL);
189+
}
190+
186191
static int iceberg_hex_value(char value) {
187192
if (value >= '0' && value <= '9') {
188193
return value - '0';
@@ -1102,6 +1107,16 @@ Status IcebergTableReader::materialize_virtual_columns(Block* table_block) {
11021107
case format::TableVirtualColumnType::ICEBERG_ROWID:
11031108
RETURN_IF_ERROR(_materialize_iceberg_rowid(table_block, column_idx));
11041109
break;
1110+
case format::TableVirtualColumnType::ICEBERG_FILE_PATH:
1111+
RETURN_IF_ERROR(_materialize_iceberg_file_path(table_block, column_idx));
1112+
break;
1113+
case format::TableVirtualColumnType::ICEBERG_ROW_POSITION:
1114+
RETURN_IF_ERROR(_materialize_iceberg_row_position(table_block, column_idx));
1115+
break;
1116+
case format::TableVirtualColumnType::PAIMON_FILE_PATH:
1117+
case format::TableVirtualColumnType::PAIMON_ROW_POSITION:
1118+
DORIS_CHECK(false);
1119+
break;
11051120
case format::TableVirtualColumnType::INVALID:
11061121
break;
11071122
}
@@ -1112,7 +1127,7 @@ Status IcebergTableReader::materialize_virtual_columns(Block* table_block) {
11121127
Status IcebergTableReader::customize_file_scan_request(format::FileScanRequest* file_request) {
11131128
RETURN_IF_ERROR(TableReader::customize_file_scan_request(file_request));
11141129
if ((_row_lineage_columns.first_row_id >= 0 && _need_row_lineage_row_id()) ||
1115-
_need_iceberg_rowid()) {
1130+
_need_iceberg_rowid() || _need_iceberg_metadata()) {
11161131
RETURN_IF_ERROR(_append_row_position_output_column(file_request));
11171132
}
11181133
RETURN_IF_ERROR(_append_equality_delete_predicates(file_request));
@@ -1887,6 +1902,54 @@ Status IcebergTableReader::_materialize_iceberg_rowid(Block* table_block, size_t
18871902
return Status::OK();
18881903
}
18891904

1905+
Status IcebergTableReader::_materialize_iceberg_file_path(Block* table_block, size_t column_idx) {
1906+
DORIS_CHECK(_row_position_block_position < _data_reader.block_template.columns());
1907+
const auto& row_position_column = assert_cast<const ColumnInt64&>(
1908+
*_data_reader.block_template.get_by_position(_row_position_block_position).column);
1909+
DORIS_CHECK(row_position_column.size() == table_block->rows());
1910+
1911+
auto column = table_block->get_by_position(column_idx).type->create_column();
1912+
auto* nullable_column = check_and_get_column<ColumnNullable>(*column);
1913+
auto* string_column = nullable_column != nullptr
1914+
? check_and_get_column<ColumnString>(
1915+
nullable_column->get_nested_column_ptr().get())
1916+
: check_and_get_column<ColumnString>(column.get());
1917+
DORIS_CHECK(string_column != nullptr);
1918+
1919+
const auto file_path = _data_file_path();
1920+
string_column->insert_data(file_path.data(), file_path.size());
1921+
if (nullable_column != nullptr) {
1922+
nullable_column->get_null_map_data().resize_fill(1, 0);
1923+
}
1924+
table_block->replace_by_position(
1925+
column_idx, ColumnConst::create(std::move(column), row_position_column.size()));
1926+
return Status::OK();
1927+
}
1928+
1929+
Status IcebergTableReader::_materialize_iceberg_row_position(Block* table_block,
1930+
size_t column_idx) {
1931+
DORIS_CHECK(_row_position_block_position < _data_reader.block_template.columns());
1932+
const auto& row_position_column = assert_cast<const ColumnInt64&>(
1933+
*_data_reader.block_template.get_by_position(_row_position_block_position).column);
1934+
DORIS_CHECK(row_position_column.size() == table_block->rows());
1935+
1936+
auto column = table_block->get_by_position(column_idx).type->create_column();
1937+
auto* nullable_column = check_and_get_column<ColumnNullable>(*column);
1938+
auto* int_column = nullable_column != nullptr
1939+
? check_and_get_column<ColumnInt64>(
1940+
nullable_column->get_nested_column_ptr().get())
1941+
: check_and_get_column<ColumnInt64>(column.get());
1942+
DORIS_CHECK(int_column != nullptr);
1943+
1944+
int_column->get_data().assign(row_position_column.get_data().begin(),
1945+
row_position_column.get_data().end());
1946+
if (nullable_column != nullptr) {
1947+
nullable_column->get_null_map_data().resize_fill(row_position_column.size(), 0);
1948+
}
1949+
table_block->replace_by_position(column_idx, std::move(column));
1950+
return Status::OK();
1951+
}
1952+
18901953
Status IcebergTableReader::_materialize_row_lineage_last_updated_sequence_number(
18911954
Block* table_block, size_t column_idx) {
18921955
if (_row_lineage_columns.last_updated_sequence_number < 0) {
@@ -1931,4 +1994,17 @@ bool IcebergTableReader::_need_iceberg_rowid() const {
19311994
return std::ranges::any_of(_projected_columns, is_projected_iceberg_rowid);
19321995
}
19331996

1997+
bool IcebergTableReader::_need_iceberg_metadata() const {
1998+
if (_data_reader.column_mapper != nullptr) {
1999+
for (const auto& mapping : _data_reader.column_mapper->mappings()) {
2000+
if (mapping.virtual_column_type == format::TableVirtualColumnType::ICEBERG_FILE_PATH ||
2001+
mapping.virtual_column_type ==
2002+
format::TableVirtualColumnType::ICEBERG_ROW_POSITION) {
2003+
return true;
2004+
}
2005+
}
2006+
}
2007+
return std::ranges::any_of(_projected_columns, is_projected_iceberg_metadata);
2008+
}
2009+
19342010
} // namespace doris::format::iceberg

‎be/src/format_v2/table/iceberg_reader.h‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -81,6 +81,7 @@ class IcebergTableReader : public format::TableReader {
8181

8282
void configure_mapper_options(format::TableColumnMapperOptions* options) const override {
8383
options->enable_row_lineage_virtual_columns = true;
84+
options->enable_iceberg_metadata_virtual_columns = true;
8485
options->reject_missing_required_field = supports_iceberg_scan_semantics_v2(_scan_params);
8586
options->allow_idless_complex_wrapper_projection =
8687
supports_iceberg_scan_semantics_v1(_scan_params) && _format == FileFormat::PARQUET;
@@ -182,6 +183,8 @@ class IcebergTableReader : public format::TableReader {
182183

183184
// Materialize row lineage virtual columns based on the position delete file.
184185
Status _materialize_iceberg_rowid(Block* table_block, size_t column_idx);
186+
Status _materialize_iceberg_file_path(Block* table_block, size_t column_idx);
187+
Status _materialize_iceberg_row_position(Block* table_block, size_t column_idx);
185188
Status _materialize_row_lineage_row_id(Block* table_block, size_t column_idx);
186189
Status _materialize_row_lineage_last_updated_sequence_number(Block* table_block,
187190
size_t column_idx);
@@ -206,6 +209,7 @@ class IcebergTableReader : public format::TableReader {
206209

207210
bool _need_row_lineage_row_id() const;
208211
bool _need_iceberg_rowid() const;
212+
bool _need_iceberg_metadata() const;
209213
};
210214

211215
} // namespace doris::format::iceberg

0 commit comments

Comments
 (0)