Skip to content

Commit 9e83bc7

Browse files
committed
[fix](regression) Align Paimon metadata fixture warehouse
### What problem does this PR solve? Issue Number: close apache#66816 Related PR: apache#67207 Problem Summary: The cross-file Paimon fixture was created through the Spark paimon catalog, whose warehouse is the external MinIO service, while the Doris catalog used by the suite points to the preloaded HDFS warehouse. Doris therefore could not discover the suite-owned database even though Spark created it successfully. Keep the HDFS catalog for the existing deletion-vector fixtures and add an isolated MinIO-backed Doris catalog for the Spark-created cross-file fixture. ### Release note None ### Check List (For Author) - Test: Regression test - External Regression build 1036742 reproduced the warehouse mismatch; git diff --check passed - Full Paimon Docker regression requires the community CI environment - Behavior changed: No - Does this need documentation: No
1 parent c913b2a commit 9e83bc7

1 file changed

Lines changed: 18 additions & 2 deletions

File tree

‎regression-test/suites/external_table_p0/paimon/test_paimon_file_metadata_columns.groovy‎

Lines changed: 18 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -23,8 +23,10 @@ suite("test_paimon_file_metadata_columns", "p0,external") {
2323
}
2424

2525
String catalogName = "test_paimon_file_metadata_columns"
26+
String crossFileCatalogName = "test_paimon_file_metadata_columns_cross_file"
2627
String dbName = "paimon_file_metadata_columns_db"
2728
String hdfsPort = context.config.otherConfigs.get("hive2HdfsPort")
29+
String minioPort = context.config.otherConfigs.get("iceberg_minio_port")
2830
String externalEnvIp = context.config.otherConfigs.get("externalEnvIp")
2931
String crossFileTable = "file_metadata_cross_file_positions"
3032

@@ -79,6 +81,16 @@ suite("test_paimon_file_metadata_columns", "p0,external") {
7981
"warehouse" = "hdfs://${externalEnvIp}:${hdfsPort}/user/doris/paimon1",
8082
"meta.cache.paimon.table.ttl-second" = "0"
8183
)"""
84+
sql """drop catalog if exists ${crossFileCatalogName}"""
85+
sql """create catalog ${crossFileCatalogName} properties (
86+
"type" = "paimon",
87+
"warehouse" = "s3://warehouse/wh",
88+
"s3.endpoint" = "http://${externalEnvIp}:${minioPort}",
89+
"s3.access_key" = "admin",
90+
"s3.secret_key" = "password",
91+
"s3.path.style.access" = "true",
92+
"meta.cache.paimon.table.ttl-second" = "0"
93+
)"""
8294

8395
try {
8496
// Two independent append commits create two physical data files. The selected rows omit
@@ -99,7 +111,6 @@ suite("test_paimon_file_metadata_columns", "p0,external") {
99111
(4, 'second-1'), (5, 'second-2'), (6, 'second-3');
100112
"""
101113

102-
sql """refresh catalog ${catalogName}"""
103114
sql """switch ${catalogName}"""
104115
sql """use db1"""
105116
sql """set enable_file_scanner_v2=true"""
@@ -110,9 +121,11 @@ suite("test_paimon_file_metadata_columns", "p0,external") {
110121
verifyMetadata("deletion_vector_orc")
111122
verifyMetadata("deletion_vector_parquet")
112123

124+
sql """switch ${crossFileCatalogName}"""
125+
sql """use ${dbName}"""
113126
def crossFileRows = sql """
114127
select id, `__paimon_file_path`, `__paimon_row_index`
115-
from ${dbName}.${crossFileTable}
128+
from ${crossFileTable}
116129
where id in (2, 3, 5, 6)
117130
order by id
118131
"""
@@ -126,6 +139,8 @@ suite("test_paimon_file_metadata_columns", "p0,external") {
126139
assertEquals(secondFile, crossFileRows[3][1].toString())
127140
assertTrue(firstFile != secondFile, "two append commits must retain distinct source-file paths")
128141

142+
sql """switch ${catalogName}"""
143+
sql """use db1"""
129144
sql """set force_jni_scanner=true"""
130145
test {
131146
sql """select `__paimon_file_path` from deletion_vector_parquet"""
@@ -142,6 +157,7 @@ suite("test_paimon_file_metadata_columns", "p0,external") {
142157
sql """unset variable file_split_size"""
143158
sql """set force_jni_scanner=false"""
144159
sql """set enable_file_scanner_v2=true"""
160+
sql """drop catalog if exists ${crossFileCatalogName}"""
145161
sql """drop catalog if exists ${catalogName}"""
146162
}
147163
}

0 commit comments

Comments
 (0)