Skip to content

Commit 4e60704

Browse files
committed
[test](iceberg) Cover scan schema carrier names
### What problem does this PR solve? Issue Number: N/A Problem Summary: Add an external Iceberg regression suite for mixed-case full-schema and snapshot-pinned scans. The suite covers Top-N, nested structs, partition schema evolution, partition predicates, and count pushdown. ### Release note None ### Check List (For Author) - Test: Regression test - test_iceberg_scan_schema_carrier_names - Behavior changed: No - Does this need documentation: No
1 parent 4bb8a35 commit 4e60704

2 files changed

Lines changed: 203 additions & 0 deletions

File tree

Lines changed: 37 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,37 @@
1+
-- This file is automatically generated. You should know what you did if you want to edit this
2+
-- !mixed_case_scan --
3+
1 one
4+
2 two
5+
3 three
6+
7+
-- !mixed_case_topn --
8+
one
9+
two
10+
11+
-- !mixed_case_nested --
12+
1 11 alpha
13+
2 22 beta
14+
3 33 gamma
15+
16+
-- !pinned_partition_schema --
17+
1 Beijing
18+
2 Beijing
19+
20+
-- !pinned_partition_filter --
21+
1 Beijing
22+
2 Beijing
23+
24+
-- !pinned_partition_topn --
25+
1 Beijing
26+
27+
-- !current_partition_schema --
28+
1 Beijing
29+
2 Beijing
30+
3 Shanghai
31+
32+
-- !pinned_count --
33+
2
34+
35+
-- !current_count --
36+
3
37+
Lines changed: 166 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,166 @@
1+
// Licensed to the Apache Software Foundation (ASF) under one
2+
// or more contributor license agreements. See the NOTICE file
3+
// distributed with this work for additional information
4+
// regarding copyright ownership. The ASF licenses this file
5+
// to you under the Apache License, Version 2.0 (the
6+
// "License"); you may not use this file except in compliance
7+
// with the License. You may obtain a copy of the License at
8+
//
9+
// http://www.apache.org/licenses/LICENSE-2.0
10+
//
11+
// Unless required by applicable law or agreed to in writing,
12+
// software distributed under the License is distributed on an
13+
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
// KIND, either express or implied. See the License for the
15+
// specific language governing permissions and limitations
16+
// under the License.
17+
18+
suite("test_iceberg_scan_schema_carrier_names",
19+
"p0,external,iceberg,external_docker,external_docker_iceberg") {
20+
String enabled = context.config.otherConfigs.get("enableIcebergTest")
21+
if (enabled == null || !enabled.equalsIgnoreCase("true")) {
22+
logger.info("disable iceberg test")
23+
return
24+
}
25+
26+
String restPort = context.config.otherConfigs.get("iceberg_rest_uri_port")
27+
String minioPort = context.config.otherConfigs.get("iceberg_minio_port")
28+
String externalEnvIp = context.config.otherConfigs.get("externalEnvIp")
29+
30+
sql """drop catalog if exists test_iceberg_scan_schema_carrier_names"""
31+
32+
spark_iceberg_multi """
33+
SET spark.sql.shuffle.partitions=1;
34+
CREATE DATABASE IF NOT EXISTS demo.iceberg_scan_schema_carrier_names_db;
35+
DROP TABLE IF EXISTS demo.iceberg_scan_schema_carrier_names_db.mixed_case_orc;
36+
CREATE TABLE demo.iceberg_scan_schema_carrier_names_db.mixed_case_orc (
37+
ID INT,
38+
Name STRING,
39+
Payload STRUCT<Child: INT, Other: STRING>
40+
) USING iceberg
41+
TBLPROPERTIES (
42+
'format-version'='2',
43+
'write.format.default'='orc'
44+
);
45+
INSERT INTO demo.iceberg_scan_schema_carrier_names_db.mixed_case_orc
46+
SELECT /*+ COALESCE(1) */ id, name, payload
47+
FROM VALUES
48+
(1, 'one', named_struct('Child', 11, 'Other', 'alpha')),
49+
(2, 'two', named_struct('Child', 22, 'Other', 'beta')),
50+
(3, 'three', named_struct('Child', 33, 'Other', 'gamma'))
51+
AS t(id, name, payload);
52+
53+
DROP TABLE IF EXISTS demo.iceberg_scan_schema_carrier_names_db.mixed_case_partition_orc;
54+
CREATE TABLE demo.iceberg_scan_schema_carrier_names_db.mixed_case_partition_orc (
55+
id INT,
56+
City STRING
57+
) USING iceberg
58+
PARTITIONED BY (City)
59+
TBLPROPERTIES (
60+
'format-version'='2',
61+
'write.format.default'='orc'
62+
);
63+
INSERT INTO demo.iceberg_scan_schema_carrier_names_db.mixed_case_partition_orc
64+
VALUES (1, 'Beijing'), (2, 'Beijing');
65+
"""
66+
67+
sql """
68+
CREATE CATALOG test_iceberg_scan_schema_carrier_names PROPERTIES (
69+
'type'='iceberg',
70+
'iceberg.catalog.type'='rest',
71+
'uri'='http://${externalEnvIp}:${restPort}',
72+
's3.access_key'='admin',
73+
's3.secret_key'='password',
74+
's3.endpoint'='http://${externalEnvIp}:${minioPort}',
75+
's3.region'='us-east-1',
76+
'meta.cache.iceberg.table.ttl-second'='0',
77+
'meta.cache.iceberg.schema.ttl-second'='0'
78+
)
79+
"""
80+
81+
try {
82+
sql """
83+
ALTER TABLE
84+
`test_iceberg_scan_schema_carrier_names`.`iceberg_scan_schema_carrier_names_db`.`mixed_case_partition_orc`
85+
CREATE TAG mixed_case_initial
86+
"""
87+
88+
spark_iceberg_multi """
89+
ALTER TABLE demo.iceberg_scan_schema_carrier_names_db.mixed_case_partition_orc
90+
RENAME COLUMN City TO city_name;
91+
INSERT INTO demo.iceberg_scan_schema_carrier_names_db.mixed_case_partition_orc
92+
VALUES (3, 'Shanghai');
93+
"""
94+
sql """refresh catalog test_iceberg_scan_schema_carrier_names"""
95+
96+
qt_mixed_case_scan """
97+
SELECT ID, Name
98+
FROM
99+
`test_iceberg_scan_schema_carrier_names`.`iceberg_scan_schema_carrier_names_db`.`mixed_case_orc`
100+
ORDER BY ID
101+
"""
102+
103+
qt_mixed_case_topn """
104+
SELECT Name
105+
FROM
106+
`test_iceberg_scan_schema_carrier_names`.`iceberg_scan_schema_carrier_names_db`.`mixed_case_orc`
107+
ORDER BY ID
108+
LIMIT 2
109+
"""
110+
111+
qt_mixed_case_nested """
112+
SELECT ID, Payload.Child, Payload.Other
113+
FROM
114+
`test_iceberg_scan_schema_carrier_names`.`iceberg_scan_schema_carrier_names_db`.`mixed_case_orc`
115+
ORDER BY ID
116+
"""
117+
118+
qt_pinned_partition_schema """
119+
SELECT id, City
120+
FROM
121+
`test_iceberg_scan_schema_carrier_names`.`iceberg_scan_schema_carrier_names_db`.`mixed_case_partition_orc`
122+
FOR VERSION AS OF 'mixed_case_initial'
123+
ORDER BY id
124+
"""
125+
126+
qt_pinned_partition_filter """
127+
SELECT id, City
128+
FROM
129+
`test_iceberg_scan_schema_carrier_names`.`iceberg_scan_schema_carrier_names_db`.`mixed_case_partition_orc`
130+
FOR VERSION AS OF 'mixed_case_initial'
131+
WHERE City = 'Beijing'
132+
ORDER BY id
133+
"""
134+
135+
qt_pinned_partition_topn """
136+
SELECT id, City
137+
FROM
138+
`test_iceberg_scan_schema_carrier_names`.`iceberg_scan_schema_carrier_names_db`.`mixed_case_partition_orc`
139+
FOR VERSION AS OF 'mixed_case_initial'
140+
ORDER BY id
141+
LIMIT 1
142+
"""
143+
144+
qt_current_partition_schema """
145+
SELECT id, city_name
146+
FROM
147+
`test_iceberg_scan_schema_carrier_names`.`iceberg_scan_schema_carrier_names_db`.`mixed_case_partition_orc`
148+
ORDER BY id
149+
"""
150+
151+
qt_pinned_count """
152+
SELECT COUNT(*)
153+
FROM
154+
`test_iceberg_scan_schema_carrier_names`.`iceberg_scan_schema_carrier_names_db`.`mixed_case_partition_orc`
155+
FOR VERSION AS OF 'mixed_case_initial'
156+
"""
157+
158+
qt_current_count """
159+
SELECT COUNT(*)
160+
FROM
161+
`test_iceberg_scan_schema_carrier_names`.`iceberg_scan_schema_carrier_names_db`.`mixed_case_partition_orc`
162+
"""
163+
} finally {
164+
sql """drop catalog if exists test_iceberg_scan_schema_carrier_names"""
165+
}
166+
}

0 commit comments

Comments
 (0)