Skip to content

Commit 934233d

Browse files
authored
[fix](regression) Stabilize external profile and VARIANT cases (#68165)
### What problem does this PR solve? Two external regression cases rely on assumptions that no longer hold: - The Parquet join runtime filter case requires exactly two counter matches, so repeated profile sections fail despite sufficient filtering. Wait for profile completion, scope counters to the fact scan, and check the largest merged or instance value against the original thresholds without summing duplicates. - The remote Doris VARIANT case implicitly casts JSON text to VARIANT and compares against obsolete output. String casts now preserve strings. Explicitly parse valid JSON fixtures, cast numeric predicates, and compare all eight non-Arrow queries with the internal source table. Two known predicate-result checks prevent empty nested-path results from passing both sides. Keep the Arrow error assertion and remove the unused golden file. Only regression cases and their expected-data file change. ### Release note None ### Check List (For Author) - Test - [ ] Regression test - [ ] Unit Test - [x] Manual test Local Groovy checks reproduced the old assertions and validated the changes: - Profile parsing: normal, repeated, merged, multiple-instance, missing-counter, unrelated-table and insufficient-filtering inputs, plus archived profiles and normalized whitespace. - Polling: delayed completion, transient read errors and timeouts. The full Parquet suite DSL ran with simulated SQL/profile responses for all four queries and three wait settings. - VARIANT suite DSL: explicit JSON parsing, SQL NULL and malformed-string fixtures, eight matching catalog/source query pairs, two semantic checks, ordering and the Arrow error assertion. - Current framework result-comparison code: matching values pass; stale output, SQL/JSON null confusion, missing/extra/reordered rows and changed boolean values fail. - `git diff --check` passed. Live Doris/Hive regression was not run; CI is requested. - Behavior changed: - [x] No production behavior changes. - [ ] Yes. - Does this need documentation? - [x] No. - [ ] Yes. ### Check List (For Reviewer who merge this PR) - [ ] Confirm the release note - [ ] Confirm test cases - [ ] Confirm document - [ ] Add branch pick label
1 parent d07e97b commit 934233d

3 files changed

Lines changed: 76 additions & 150 deletions

File tree

‎regression-test/data/external_table_p0/remote_doris/test_remote_doris_variant_select.out‎

Lines changed: 0 additions & 81 deletions
This file was deleted.

‎regression-test/suites/external_table_p0/hive/test_parquet_join_runtime_filter.groovy‎

Lines changed: 27 additions & 48 deletions
Original file line numberDiff line numberDiff line change
@@ -15,55 +15,34 @@
1515
// specific language governing permissions and limitations
1616
// under the License.
1717

18-
import groovy.json.JsonSlurper
18+
import org.apache.doris.regression.action.ProfileAction
1919

2020
suite("test_parquet_join_runtime_filter", "p0,external") {
2121

22-
def getProfileList = {
23-
def dst = 'http://' + context.config.feHttpAddress
24-
def conn = new URL(dst + "/rest/v1/query_profile").openConnection()
25-
conn.setRequestMethod("GET")
26-
def encoding = Base64.getEncoder().encodeToString((context.config.feHttpUser + ":" +
27-
(context.config.feHttpPassword == null ? "" : context.config.feHttpPassword)).getBytes("UTF-8"))
28-
conn.setRequestProperty("Authorization", "Basic ${encoding}")
29-
return conn.getInputStream().getText()
30-
}
31-
32-
def getProfile = { id ->
33-
def dst = 'http://' + context.config.feHttpAddress
34-
def conn = new URL(dst + "/api/profile/text/?query_id=$id").openConnection()
35-
conn.setRequestMethod("GET")
36-
def encoding = Base64.getEncoder().encodeToString((context.config.feHttpUser + ":" +
37-
(context.config.feHttpPassword == null ? "" : context.config.feHttpPassword)).getBytes("UTF-8"))
38-
conn.setRequestProperty("Authorization", "Basic ${encoding}")
39-
return conn.getInputStream().getText()
40-
}
41-
22+
def profileAction = new ProfileAction(context)
4223

4324
def extractFilteredGroupsValue = { String profileText ->
44-
def values = (profileText =~ /RowGroupsFiltered:\s*(\d+)/).collect { it[1].toLong() }
25+
def values = []
26+
boolean inFactScan = false
27+
profileText.eachLine { line ->
28+
if (line =~ /^\s*[A-Z_]+_OPERATOR\(/) {
29+
inFactScan = (line =~ /^\s*FILE_SCAN_OPERATOR\([^)]*\btable_name=fact_big\):/).find()
30+
}
31+
if (inFactScan) {
32+
def counter = (line =~ /RowGroupsFiltered:\s*(?:sum\s+)?(\d+)/)
33+
if (counter.find()) {
34+
values.add(counter.group(1).toLong())
35+
}
36+
}
37+
}
38+
// A profile can repeat scan sections. Use the largest fact scan count, not the
39+
// number or sum of matches, so duplicates cannot inflate the filtering result.
4540
return values.sort { a, b -> b <=> a }
4641
}
4742

4843
def getProfileWithToken = { token ->
49-
String profileId = ""
50-
int attempts = 0
51-
while (attempts < 10 && (profileId == null || profileId == "")) {
52-
List profileData = new JsonSlurper().parseText(getProfileList()).data.rows
53-
for (def profileItem in profileData) {
54-
if (profileItem["Sql Statement"].toString().contains(token)) {
55-
profileId = profileItem["Profile ID"].toString()
56-
break
57-
}
58-
}
59-
if (profileId == null || profileId == "") {
60-
Thread.sleep(300)
61-
}
62-
attempts++
63-
}
64-
assertTrue(profileId != null && profileId != "")
65-
Thread.sleep(800)
66-
return getProfile(profileId).toString()
44+
// Wait for asynchronous profile collection instead of assuming a fixed delay is enough.
45+
return profileAction.getProfileBySql(token, ["table_name=fact_big)", "RowGroupsFiltered:"])
6746
}
6847
// session vars
6948
sql "unset variable all;"
@@ -108,8 +87,8 @@ suite("test_parquet_join_runtime_filter", "p0,external") {
10887
logger.info("sql_result = ${sql_result}");
10988
logger.info("filter_result = ${filter_result}");
11089

111-
assertTrue(filter_result.size() == 2)
112-
assertTrue(filter_result[0] > 40)
90+
assertFalse(filter_result.isEmpty(), "Missing RowGroupsFiltered for fact_big")
91+
assertTrue(filter_result[0] > 40, "Insufficient fact_big filtering: ${filter_result}")
11392
}
11493

11594

@@ -123,8 +102,8 @@ suite("test_parquet_join_runtime_filter", "p0,external") {
123102
logger.info("sql_result = ${sql_result}");
124103
logger.info("filter_result = ${filter_result}");
125104

126-
assertTrue(filter_result.size() == 2)
127-
assertTrue(filter_result[0] > 30)
105+
assertFalse(filter_result.isEmpty(), "Missing RowGroupsFiltered for fact_big")
106+
assertTrue(filter_result[0] > 30, "Insufficient fact_big filtering: ${filter_result}")
128107
}
129108

130109

@@ -139,8 +118,8 @@ suite("test_parquet_join_runtime_filter", "p0,external") {
139118
logger.info("sql_result = ${sql_result}");
140119
logger.info("filter_result = ${filter_result}");
141120

142-
assertTrue(filter_result.size() == 2)
143-
assertTrue(filter_result[0] > 30)
121+
assertFalse(filter_result.isEmpty(), "Missing RowGroupsFiltered for fact_big")
122+
assertTrue(filter_result[0] > 30, "Insufficient fact_big filtering: ${filter_result}")
144123
}
145124

146125

@@ -154,8 +133,8 @@ suite("test_parquet_join_runtime_filter", "p0,external") {
154133
logger.info("sql_result = ${sql_result}");
155134
logger.info("filter_result = ${filter_result}");
156135

157-
assertTrue(filter_result.size() == 2)
158-
assertTrue(filter_result[0] > 40)
136+
assertFalse(filter_result.isEmpty(), "Missing RowGroupsFiltered for fact_big")
137+
assertTrue(filter_result[0] > 40, "Insufficient fact_big filtering: ${filter_result}")
159138
}
160139

161140

‎regression-test/suites/external_table_p0/remote_doris/test_remote_doris_variant_select.groovy‎

Lines changed: 49 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -54,20 +54,21 @@ suite("test_remote_doris_variant_select", "p0,external") {
5454
);
5555
"""
5656

57+
// String-to-VARIANT casts preserve strings; parse JSON explicitly to exercise nested paths.
5758
sql """
5859
INSERT INTO `${db_name}`.`test_remote_doris_variant_select_t` values
59-
(1, 'null'),
60+
(1, parse_to_variant('null')),
6061
(2, NULL),
61-
(3, 'true'),
62-
(4, '-17'),
63-
(5, '123.12'),
64-
(6, '1.912'),
65-
(7, '"A quote"'),
66-
(8, '[-1, 12, false]'),
67-
(9, '{ "x": "abc", "y": false, "z": 10 }'),
68-
(10, '"2021-01-01"'),
69-
(11, '{"a":"a", "b":"0.1", "c":{"c1":"c1", "c2":"1"}}'),
70-
(12, '{"a":"b", "b":"0.1X", "c":{"c1":"2", "c2":"2"}}'),
62+
(3, parse_to_variant('true')),
63+
(4, parse_to_variant('-17')),
64+
(5, parse_to_variant('123.12')),
65+
(6, parse_to_variant('1.912')),
66+
(7, parse_to_variant('"A quote"')),
67+
(8, parse_to_variant('[-1, 12, false]')),
68+
(9, parse_to_variant('{ "x": "abc", "y": false, "z": 10 }')),
69+
(10, parse_to_variant('"2021-01-01"')),
70+
(11, parse_to_variant('{"a":"a", "b":"0.1", "c":{"c1":"c1", "c2":"1"}}')),
71+
(12, parse_to_variant('{"a":"b", "b":"0.1X", "c":{"c1":"2", "c2":"2"}}')),
7172
(13, '{"a":"ab\\"cde", "b":NULL, "d":{"d1":NULL, "d2":NULL}}'),
7273
(14, '{"a":"ab{{c\\"de\\"}}"'),
7374
(15, '{"a":"abc{{{de"');
@@ -104,8 +105,21 @@ suite("test_remote_doris_variant_select", "p0,external") {
104105
);
105106
"""
106107

107-
qt_sql """
108+
// Keep nested-path coverage non-empty before comparing catalog reads with the source table.
109+
check_sqls_result_equal """
110+
select id from `internal`.`${db_name}`.`test_remote_doris_variant_select_t`
111+
where cast(v['b'] as double) >= 0 order by id
112+
""", "select 11"
113+
check_sqls_result_equal """
114+
select id from `internal`.`${db_name}`.`test_remote_doris_variant_select_t`
115+
where cast(v['c']['c2'] as int) > 1 order by id
116+
""", "select 12"
117+
118+
// Compare with the source instead of golden text tied to an older VARIANT representation.
119+
check_sqls_result_equal """
108120
select * from `${catalog_name}`.`${db_name}`.`test_remote_doris_variant_select_t` order by id
121+
""", """
122+
select * from `internal`.`${db_name}`.`test_remote_doris_variant_select_t` order by id
109123
"""
110124
test {
111125
sql """
@@ -116,32 +130,46 @@ suite("test_remote_doris_variant_select", "p0,external") {
116130
+ "file format ARROW is not supported")
117131
}
118132

119-
qt_sql """
120-
select * from `${catalog_name}`.`${db_name}`.`test_remote_doris_variant_select_t` where v['b'] >= 0 order by id
133+
check_sqls_result_equal """
134+
select * from `${catalog_name}`.`${db_name}`.`test_remote_doris_variant_select_t` where cast(v['b'] as double) >= 0 order by id
135+
""", """
136+
select * from `internal`.`${db_name}`.`test_remote_doris_variant_select_t` where cast(v['b'] as double) >= 0 order by id
121137
"""
122138

123-
qt_sql """
124-
select * from `${catalog_name}`.`${db_name}`.`test_remote_doris_variant_select_t` where v['c']['c2'] > 1 order by id
139+
check_sqls_result_equal """
140+
select * from `${catalog_name}`.`${db_name}`.`test_remote_doris_variant_select_t` where cast(v['c']['c2'] as int) > 1 order by id
141+
""", """
142+
select * from `internal`.`${db_name}`.`test_remote_doris_variant_select_t` where cast(v['c']['c2'] as int) > 1 order by id
125143
"""
126144

127-
qt_sql """
145+
check_sqls_result_equal """
128146
select v['a'] from `${catalog_name}`.`${db_name}`.`test_remote_doris_variant_select_t` order by id
147+
""", """
148+
select v['a'] from `internal`.`${db_name}`.`test_remote_doris_variant_select_t` order by id
129149
"""
130150

131-
qt_sql """
151+
check_sqls_result_equal """
132152
select v['b'] from `${catalog_name}`.`${db_name}`.`test_remote_doris_variant_select_t` order by id
153+
""", """
154+
select v['b'] from `internal`.`${db_name}`.`test_remote_doris_variant_select_t` order by id
133155
"""
134156

135-
qt_sql """
157+
check_sqls_result_equal """
136158
select sum(cast(v['b'] as double)) from `${catalog_name}`.`${db_name}`.`test_remote_doris_variant_select_t`
159+
""", """
160+
select sum(cast(v['b'] as double)) from `internal`.`${db_name}`.`test_remote_doris_variant_select_t`
137161
"""
138162

139-
qt_sql """
163+
check_sqls_result_equal """
140164
select v['c']['c1'] from `${catalog_name}`.`${db_name}`.`test_remote_doris_variant_select_t` order by id
165+
""", """
166+
select v['c']['c1'] from `internal`.`${db_name}`.`test_remote_doris_variant_select_t` order by id
141167
"""
142168

143-
qt_sql """
169+
check_sqls_result_equal """
144170
select sum(cast(v['c']['c1'] as double)) from `${catalog_name}`.`${db_name}`.`test_remote_doris_variant_select_t`
171+
""", """
172+
select sum(cast(v['c']['c1'] as double)) from `internal`.`${db_name}`.`test_remote_doris_variant_select_t`
145173
"""
146174

147175
sql """ DROP DATABASE IF EXISTS `${db_name}` """

0 commit comments

Comments
 (0)