Skip to content

Commit 68e9df4

Browse files
committed
[fix](connector) Preserve external metadata column semantics
### What problem does this PR solve? Issue Number: close apache#66816 Related PR: apache#67207 Problem Summary: External Paimon tables may contain names used by Doris virtual metadata columns, and Iceberg streaming scans could bypass the FileScannerV2 reader validation. This could overwrite a physical Paimon column with a synthetic handle or route Iceberg metadata-column queries to an unsupported reader. Reject Paimon name collisions safely, validate Iceberg streaming scans before empty-snapshot handling, and keep BE metadata classification and materialization consistent. ### Release note Paimon external tables with reserved physical metadata names now fail safely; Iceberg streaming metadata-column scans reject unsupported readers. ### Check List (For Author) - Test: Unit Test\n - Java connector tests: IcebergScanPlanProviderTest, PaimonConnectorMetadataTest, and PaimonScanPlanProviderTest\n - BE unit tests: FileScannerV2 classification tests and Paimon reader tests\n- Behavior changed: Yes (safe error on Paimon physical metadata-name collisions and unsupported Iceberg streaming readers)\n- Does this need documentation: No
1 parent 22f82f8 commit 68e9df4

9 files changed

Lines changed: 139 additions & 33 deletions

File tree

‎be/src/exec/scan/file_scanner_v2.cpp‎

Lines changed: 11 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -75,6 +75,7 @@
7575
#include "runtime/runtime_state.h"
7676
#include "service/backend_options.h"
7777
#include "storage/id_manager.h"
78+
#include "util/string_util.h"
7879

7980
namespace doris {
8081
namespace {
@@ -196,11 +197,11 @@ bool is_wal_format(TFileFormatType::type format_type) {
196197

197198
bool is_partition_slot(const TFileScanSlotInfo& slot_info, const std::string& column_name) {
198199
if (column_name.starts_with(BeConsts::GLOBAL_ROWID_COL) ||
199-
column_name == BeConsts::ICEBERG_ROWID_COL ||
200-
column_name == BeConsts::ICEBERG_FILE_PATH_COL ||
201-
column_name == BeConsts::ICEBERG_ROW_POSITION_COL ||
202-
column_name == BeConsts::PAIMON_FILE_PATH_COL ||
203-
column_name == BeConsts::PAIMON_ROW_POSITION_COL) {
200+
iequal(column_name, BeConsts::ICEBERG_ROWID_COL) ||
201+
iequal(column_name, BeConsts::ICEBERG_FILE_PATH_COL) ||
202+
iequal(column_name, BeConsts::ICEBERG_ROW_POSITION_COL) ||
203+
iequal(column_name, BeConsts::PAIMON_FILE_PATH_COL) ||
204+
iequal(column_name, BeConsts::PAIMON_ROW_POSITION_COL)) {
204205
return false;
205206
}
206207
return slot_info.__isset.category ? slot_info.category == TColumnCategory::PARTITION_KEY
@@ -209,11 +210,11 @@ bool is_partition_slot(const TFileScanSlotInfo& slot_info, const std::string& co
209210

210211
bool is_data_file_slot(const TFileScanSlotInfo& slot_info, const std::string& column_name) {
211212
if (column_name.starts_with(BeConsts::GLOBAL_ROWID_COL) ||
212-
column_name == BeConsts::ICEBERG_ROWID_COL ||
213-
column_name == BeConsts::ICEBERG_FILE_PATH_COL ||
214-
column_name == BeConsts::ICEBERG_ROW_POSITION_COL ||
215-
column_name == BeConsts::PAIMON_FILE_PATH_COL ||
216-
column_name == BeConsts::PAIMON_ROW_POSITION_COL) {
213+
iequal(column_name, BeConsts::ICEBERG_ROWID_COL) ||
214+
iequal(column_name, BeConsts::ICEBERG_FILE_PATH_COL) ||
215+
iequal(column_name, BeConsts::ICEBERG_ROW_POSITION_COL) ||
216+
iequal(column_name, BeConsts::PAIMON_FILE_PATH_COL) ||
217+
iequal(column_name, BeConsts::PAIMON_ROW_POSITION_COL)) {
217218
return false;
218219
}
219220
// CSV and other non-self-describing formats need FE slot descriptors for only the columns that

‎be/src/format_v2/table/paimon_reader.cpp‎

Lines changed: 9 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -323,7 +323,7 @@ Status PaimonReader::materialize_virtual_columns(Block* table_block) {
323323
return Status::OK();
324324
}
325325

326-
std::string PaimonReader::_data_file_path() const {
326+
const std::string& PaimonReader::_data_file_path() const {
327327
DORIS_CHECK(!_original_file_path.empty());
328328
return _original_file_path;
329329
}
@@ -339,27 +339,30 @@ Status PaimonReader::_materialize_file_path(Block* table_block, size_t column_id
339339
DORIS_CHECK(_row_position_block_position < _data_reader.block_template.columns());
340340
const auto& row_position_column = assert_cast<const ColumnInt64&>(
341341
*_data_reader.block_template.get_by_position(_row_position_block_position).column);
342+
const auto rows = table_block->rows();
343+
DORIS_CHECK(row_position_column.size() == rows);
342344
auto column = table_block->get_by_position(column_idx).type->create_column();
343345
auto* nullable_column = check_and_get_column<ColumnNullable>(*column);
344346
auto* string_column = nullable_column != nullptr
345347
? check_and_get_column<ColumnString>(
346348
nullable_column->get_nested_column_ptr().get())
347349
: check_and_get_column<ColumnString>(column.get());
348350
DORIS_CHECK(string_column != nullptr);
349-
const auto file_path = _data_file_path();
351+
const auto& file_path = _data_file_path();
350352
string_column->insert_data(file_path.data(), file_path.size());
351353
if (nullable_column != nullptr) {
352354
nullable_column->get_null_map_data().resize_fill(1, 0);
353355
}
354-
table_block->replace_by_position(
355-
column_idx, ColumnConst::create(std::move(column), row_position_column.size()));
356+
table_block->replace_by_position(column_idx, ColumnConst::create(std::move(column), rows));
356357
return Status::OK();
357358
}
358359

359360
Status PaimonReader::_materialize_row_position(Block* table_block, size_t column_idx) {
360361
DORIS_CHECK(_row_position_block_position < _data_reader.block_template.columns());
361362
const auto& row_position_column = assert_cast<const ColumnInt64&>(
362363
*_data_reader.block_template.get_by_position(_row_position_block_position).column);
364+
const auto rows = table_block->rows();
365+
DORIS_CHECK(row_position_column.size() == rows);
363366
auto column = table_block->get_by_position(column_idx).type->create_column();
364367
auto* nullable_column = check_and_get_column<ColumnNullable>(*column);
365368
auto* int_column = nullable_column != nullptr
@@ -368,9 +371,9 @@ Status PaimonReader::_materialize_row_position(Block* table_block, size_t column
368371
: check_and_get_column<ColumnInt64>(column.get());
369372
DORIS_CHECK(int_column != nullptr);
370373
int_column->get_data().assign(row_position_column.get_data().begin(),
371-
row_position_column.get_data().end());
374+
row_position_column.get_data().begin() + rows);
372375
if (nullable_column != nullptr) {
373-
nullable_column->get_null_map_data().resize_fill(row_position_column.size(), 0);
376+
nullable_column->get_null_map_data().resize_fill(rows, 0);
374377
}
375378
table_block->replace_by_position(column_idx, std::move(column));
376379
return Status::OK();

‎be/src/format_v2/table/paimon_reader.h‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -65,7 +65,7 @@ class PaimonReader final : public format::TableReader {
6565
bool* has_delete_file) override;
6666

6767
private:
68-
std::string _data_file_path() const;
68+
const std::string& _data_file_path() const;
6969
Status _append_row_position_output_column(format::FileScanRequest* request);
7070
Status _materialize_file_path(Block* table_block, size_t column_idx);
7171
Status _materialize_row_position(Block* table_block, size_t column_idx);

‎be/test/exec/scan/file_scanner_v2_test.cpp‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -904,6 +904,10 @@ TEST(FileScannerV2Test, PartitionSlotClassificationMatrix) {
904904
BeConsts::ICEBERG_FILE_PATH_COL));
905905
EXPECT_FALSE(FileScannerV2::TEST_is_partition_slot(legacy_partition,
906906
BeConsts::ICEBERG_ROW_POSITION_COL));
907+
EXPECT_FALSE(FileScannerV2::TEST_is_partition_slot(legacy_partition, "_FILE"));
908+
EXPECT_FALSE(FileScannerV2::TEST_is_partition_slot(legacy_partition, "_POS"));
909+
EXPECT_FALSE(FileScannerV2::TEST_is_partition_slot(legacy_partition, "__PAIMON_FILE_PATH"));
910+
EXPECT_FALSE(FileScannerV2::TEST_is_partition_slot(legacy_partition, "__PAIMON_ROW_INDEX"));
907911
}
908912

909913
// Scenario: data-file slots are the complement of partition/default/synthesized columns for
@@ -944,6 +948,10 @@ TEST(FileScannerV2Test, DataFileSlotClassificationMatrix) {
944948
FileScannerV2::TEST_is_data_file_slot(legacy_file, BeConsts::ICEBERG_FILE_PATH_COL));
945949
EXPECT_FALSE(
946950
FileScannerV2::TEST_is_data_file_slot(legacy_file, BeConsts::ICEBERG_ROW_POSITION_COL));
951+
EXPECT_FALSE(FileScannerV2::TEST_is_data_file_slot(legacy_file, "_FILE"));
952+
EXPECT_FALSE(FileScannerV2::TEST_is_data_file_slot(legacy_file, "_POS"));
953+
EXPECT_FALSE(FileScannerV2::TEST_is_data_file_slot(legacy_file, "__PAIMON_FILE_PATH"));
954+
EXPECT_FALSE(FileScannerV2::TEST_is_data_file_slot(legacy_file, "__PAIMON_ROW_INDEX"));
947955
}
948956

949957
// Scenario: table conjuncts are cloned into global-index space before they are handed to

‎fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergScanPlanProvider.java‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -526,6 +526,7 @@ public boolean canServeMetadataOnlyCount(ConnectorSession session, ConnectorTabl
526526
public ConnectorSplitSource streamSplits(ConnectorSession session, ConnectorTableHandle handle,
527527
List<ConnectorColumnHandle> columns, Optional<ConnectorExpression> filter, long limit) {
528528
IcebergTableHandle iceHandle = (IcebergTableHandle) handle;
529+
validateMetadataColumnReader(session, columns);
529530
if (iceHandle.isResolvedEmptySnapshot()) {
530531
// The batch decision is made before the engine pins MVCC; once pinned empty, streaming must
531532
// preserve that boundary instead of interpreting Iceberg's sentinel as the latest snapshot.

‎fe/fe-connector/fe-connector-iceberg/src/test/java/org/apache/doris/connector/iceberg/IcebergScanPlanProviderTest.java‎

Lines changed: 35 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -936,6 +936,41 @@ public void metadataColumnsRejectDisabledFileScannerV2DuringPlanning() {
936936
ex.getMessage());
937937
}
938938

939+
@Test
940+
public void metadataColumnsRejectForceJniInStreamingPath() throws IOException {
941+
IcebergScanPlanProvider provider = providerOver(
942+
createTable("stream_metadata", SCHEMA, PartitionSpec.unpartitioned()));
943+
ConnectorSession session = new FakeScanSession("UTC",
944+
Collections.singletonMap("force_jni_scanner", "true"));
945+
List<ConnectorColumnHandle> columns = Collections.singletonList(new IcebergColumnHandle("_file", -1));
946+
947+
DorisConnectorException ex = Assertions.assertThrows(DorisConnectorException.class,
948+
() -> provider.streamSplits(session,
949+
new IcebergTableHandle("db1", "stream_metadata"), columns,
950+
Optional.empty(), -1L));
951+
Assertions.assertEquals(
952+
"Iceberg metadata columns are only supported by FileScannerV2 native Parquet/ORC reader; "
953+
+ "actual reader is JNI",
954+
ex.getMessage());
955+
}
956+
957+
@Test
958+
public void metadataColumnsRejectDisabledFileScannerV2InStreamingPath() throws IOException {
959+
IcebergScanPlanProvider provider = providerOver(
960+
createTable("stream_metadata_v2", SCHEMA, PartitionSpec.unpartitioned()));
961+
ConnectorSession session = new FakeScanSession("UTC",
962+
Collections.singletonMap("enable_file_scanner_v2", "false"));
963+
List<ConnectorColumnHandle> columns = Collections.singletonList(new IcebergColumnHandle("_pos", -1));
964+
965+
DorisConnectorException ex = Assertions.assertThrows(DorisConnectorException.class,
966+
() -> provider.streamSplits(session,
967+
new IcebergTableHandle("db1", "stream_metadata_v2"), columns,
968+
Optional.empty(), -1L));
969+
Assertions.assertEquals(
970+
"Iceberg metadata columns require FileScannerV2 native Parquet/ORC reader",
971+
ex.getMessage());
972+
}
973+
939974
@Test
940975
public void ordinaryColumnsAllowDisabledFileScannerV2InProperties() {
941976
IcebergScanPlanProvider provider = providerOver(

‎fe/fe-connector/fe-connector-paimon/src/main/java/org/apache/doris/connector/paimon/PaimonConnectorMetadata.java‎

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -381,6 +381,9 @@ private Table resolveSystemTableAt(ConnectorSession session,
381381
*/
382382
private ConnectorTableSchema buildTableSchema(String tableName, Table table, List<DataField> fields,
383383
List<String> partitionKeys, List<String> primaryKeys, boolean appendDataFileMetadataColumns) {
384+
if (appendDataFileMetadataColumns) {
385+
rejectMetadataColumnCollisions(fields);
386+
}
384387
List<ConnectorColumn> columns = mapFields(fields, primaryKeys);
385388
if (appendDataFileMetadataColumns) {
386389
columns.add(new ConnectorColumn(PAIMON_FILE_PATH_COL, ConnectorType.of("STRING"),
@@ -1249,6 +1252,9 @@ public boolean supportsColumnHandleSnapshotPin(ConnectorSession session) {
12491252

12501253
private static Map<String, ConnectorColumnHandle> buildColumnHandles(List<DataField> fields,
12511254
boolean appendDataFileMetadataColumns) {
1255+
if (appendDataFileMetadataColumns) {
1256+
rejectMetadataColumnCollisions(fields);
1257+
}
12521258
Map<String, ConnectorColumnHandle> handles = new LinkedHashMap<>(fields.size() + 2);
12531259
for (int i = 0; i < fields.size(); i++) {
12541260
String name = fields.get(i).name();
@@ -1262,6 +1268,25 @@ private static Map<String, ConnectorColumnHandle> buildColumnHandles(List<DataFi
12621268
return handles;
12631269
}
12641270

1271+
/**
1272+
* Paimon does not reserve the metadata column names. An externally-created table can therefore
1273+
* contain a real physical column with one of these names. Appending a synthetic handle in that
1274+
* case would overwrite the physical handle and make a query return file metadata instead of the
1275+
* stored value. Fail while exposing the data-table schema/handles so the collision cannot reach
1276+
* the scan planner.
1277+
*/
1278+
private static void rejectMetadataColumnCollisions(List<DataField> fields) {
1279+
for (DataField field : fields) {
1280+
String name = field.name();
1281+
if (PAIMON_FILE_PATH_COL.equalsIgnoreCase(name)
1282+
|| PAIMON_ROW_POSITION_COL.equalsIgnoreCase(name)) {
1283+
throw new DorisConnectorException(
1284+
"Paimon physical column '" + name
1285+
+ "' conflicts with a reserved metadata column name");
1286+
}
1287+
}
1288+
}
1289+
12651290
@Override
12661291
public List<String> listPartitionNames(ConnectorSession session, ConnectorTableHandle handle) {
12671292
List<ConnectorPartitionInfo> partitions = cachedPartitions((PaimonTableHandle) handle);

‎fe/fe-connector/fe-connector-paimon/src/main/java/org/apache/doris/connector/paimon/PaimonScanPlanProvider.java‎

Lines changed: 28 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -609,6 +609,31 @@ private boolean isIncrementalBinlogScan(PaimonTableHandle handle, Map<String, St
609609
|| scanOptions.containsKey("incremental-between-timestamp"));
610610
}
611611

612+
private static void validateMetadataColumnReader(ConnectorSession session,
613+
PaimonTableHandle handle, boolean requiresMetadataColumns) {
614+
if (!requiresMetadataColumns) {
615+
return;
616+
}
617+
if (handle.isForceJni() || isForceJniScannerEnabled(session)) {
618+
throw new DorisConnectorException(
619+
"Paimon metadata columns are only supported by FileScannerV2 native Parquet/ORC "
620+
+ "reader; actual reader is JNI");
621+
}
622+
if (!isFileScannerV2Enabled(session)) {
623+
throw new DorisConnectorException(
624+
"Paimon metadata columns require FileScannerV2 native Parquet/ORC reader");
625+
}
626+
}
627+
628+
private static void validateMetadataColumnReader(boolean requiresMetadataColumns,
629+
boolean nativeReader) {
630+
if (requiresMetadataColumns && !nativeReader) {
631+
throw new DorisConnectorException(
632+
"Paimon metadata columns are only supported by FileScannerV2 native Parquet/ORC "
633+
+ "reader; actual reader is JNI");
634+
}
635+
}
636+
612637
private List<ConnectorScanRange> planScanInternal(
613638
ConnectorSession session,
614639
ConnectorTableHandle handle,
@@ -622,6 +647,7 @@ private List<ConnectorScanRange> planScanInternal(
622647
.map(column -> ((PaimonColumnHandle) column).getName())
623648
.anyMatch(name -> PAIMON_FILE_PATH_COL.equalsIgnoreCase(name)
624649
|| PAIMON_ROW_POSITION_COL.equalsIgnoreCase(name));
650+
validateMetadataColumnReader(session, paimonHandle, requiresMetadataColumns);
625651
Map<String, String> pinnedOptions = paimonHandle.getScanOptions();
626652
boolean optionsPin = PaimonScanParams.isOptionsPin(pinnedOptions);
627653
if (countPushdown && isIncrementalBinlogScan(paimonHandle, pinnedOptions)) {
@@ -637,16 +663,6 @@ private List<ConnectorScanRange> planScanInternal(
637663
}
638664
Table table = resolveScanTable(paimonHandle);
639665

640-
if (requiresMetadataColumns
641-
&& (paimonHandle.isForceJni() || isForceJniScannerEnabled(session))) {
642-
throw new DorisConnectorException(
643-
"Paimon metadata columns are only supported by FileScannerV2 native Parquet/ORC "
644-
+ "reader; actual reader is JNI");
645-
}
646-
if (requiresMetadataColumns && !isFileScannerV2Enabled(session)) {
647-
throw new DorisConnectorException(
648-
"Paimon metadata columns require FileScannerV2 native Parquet/ORC reader");
649-
}
650666
// Metadata predicates must be evaluated against actual rows. A merged-row-count range has no
651667
// file path or physical row positions, so it cannot satisfy the predicate correctly.
652668
if (requiresMetadataColumns) {
@@ -761,9 +777,7 @@ private List<ConnectorScanRange> planScanInternal(
761777
continue;
762778
}
763779
if (requiresMetadataColumns) {
764-
throw new DorisConnectorException(
765-
"Paimon metadata columns are only supported by FileScannerV2 native Parquet/ORC "
766-
+ "reader; actual reader is JNI");
780+
validateMetadataColumnReader(true, false);
767781
}
768782
if (hasVariantProjection) {
769783
throw new DorisConnectorException(
@@ -850,9 +864,7 @@ private List<ConnectorScanRange> planScanInternal(
850864
continue;
851865
}
852866
if (requiresMetadataColumns) {
853-
throw new DorisConnectorException(
854-
"Paimon metadata columns are only supported by FileScannerV2 native Parquet/ORC "
855-
+ "reader; actual reader is JNI");
867+
validateMetadataColumnReader(true, false);
856868
}
857869
if (hasVariantProjection) {
858870
throw new DorisConnectorException(

‎fe/fe-connector/fe-connector-paimon/src/test/java/org/apache/doris/connector/paimon/PaimonConnectorMetadataTest.java‎

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@
1919

2020
import org.apache.doris.connector.spi.ConnectorColumn;
2121
import org.apache.doris.connector.spi.ConnectorTableSchema;
22+
import org.apache.doris.connector.spi.DorisConnectorException;
2223
import org.apache.doris.connector.spi.handle.ConnectorColumnHandle;
2324
import org.apache.doris.connector.spi.handle.ConnectorTableHandle;
2425
import org.apache.doris.connector.spi.mvcc.ConnectorMvccSnapshot;
@@ -215,6 +216,26 @@ public void getColumnHandlesUsesTransientTableWithoutReload() {
215216
"with a present transient table, no remote getTable reload must happen");
216217
}
217218

219+
@Test
220+
public void externalPhysicalMetadataColumnNameFailsSafely() {
221+
for (String physicalName : Arrays.asList("__paimon_file_path", "__PAIMON_ROW_INDEX")) {
222+
RecordingPaimonCatalogOps ops = new RecordingPaimonCatalogOps();
223+
ops.table = new FakePaimonTable(
224+
"external_table",
225+
rowType("id", physicalName),
226+
Collections.emptyList(),
227+
Collections.emptyList());
228+
PaimonConnectorMetadata metadata = metadataWith(ops);
229+
ConnectorTableHandle handle = metadata.getTableHandle(null, "db1", "external_table").get();
230+
231+
DorisConnectorException schemaError = Assertions.assertThrows(DorisConnectorException.class,
232+
() -> metadata.getTableSchema(null, handle));
233+
Assertions.assertTrue(schemaError.getMessage().contains("conflicts with a reserved metadata"));
234+
Assertions.assertThrows(DorisConnectorException.class,
235+
() -> metadata.getColumnHandles(null, handle));
236+
}
237+
}
238+
218239
@Test
219240
public void disablesCastPredicatePushdown() {
220241
PaimonConnectorMetadata metadata =

0 commit comments

Comments
 (0)