Skip to content

Commit b391936

Browse files
committed
Fix Flink regression fixture for native Arrow Variant output
1 parent 6741189 commit b391936

1 file changed

Lines changed: 22 additions & 7 deletions

File tree

‎regression-test/suites/flink_connector_p0/flink_connector_type.groovy‎

Lines changed: 22 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -24,13 +24,15 @@ import org.awaitility.Awaitility
2424

2525
suite("flink_connector_type") {
2626

27+
def inputTable = "test_types_input"
2728
def tableName1 = "test_types_source"
2829
def tableName2 = "test_types_sink"
30+
sql """DROP TABLE IF EXISTS ${inputTable}"""
2931
sql """DROP TABLE IF EXISTS ${tableName1}"""
3032
sql """DROP TABLE IF EXISTS ${tableName2}"""
3133

3234
sql """
33-
CREATE TABLE `test_types_source` (
35+
CREATE TABLE `${inputTable}` (
3436
`id` int,
3537
`c1` boolean,
3638
`c2` tinyint,
@@ -59,9 +61,9 @@ PROPERTIES (
5961
);
6062
""";
6163

62-
sql """CREATE TABLE `test_types_sink` like `test_types_source` """
64+
sql """CREATE TABLE `${tableName2}` like `${inputTable}` """
6365

64-
sql """ INSERT INTO `test_types_source`
66+
sql """ INSERT INTO `${inputTable}`
6567
VALUES
6668
(
6769
1,
@@ -106,6 +108,17 @@ VALUES
106108
'{"B":"variant_value1"}'
107109
);"""
108110

111+
// The connector declares c18 as STRING and cannot decode native Arrow Variant.
112+
// Materialize an explicit text projection while retaining Variant ingestion in the sink.
113+
sql """
114+
CREATE TABLE `${tableName1}`
115+
DISTRIBUTED BY HASH(`id`) BUCKETS 1
116+
PROPERTIES ("replication_num" = "1")
117+
AS SELECT id, c1, c2, c3, c4, c5, c6, c7, c8, c9,
118+
c10, c11, c12, c13, c14, c15, c16, c17, CAST(c18 AS STRING) AS c18
119+
FROM `${inputTable}`
120+
"""
121+
109122
def thisDb = sql """select database()""";
110123
thisDb = thisDb[0][0];
111124
logger.info("current database is ${thisDb}");
@@ -143,19 +156,21 @@ VALUES
143156
run_cmd.addAll(addOpens.tokenize())
144157
run_cmd.addAll(["-cp", jarName, "org.apache.doris.FlinkConnectorTypeCase",
145158
"--doris-fe-address", context.config.feHttpAddress,
146-
"--doris-database", "regression_test_flink_connector_p0",
159+
"--doris-database", thisDb,
147160
"--doris-user", context.config.feHttpUser,
148161
"--doris-password", context.config.feHttpPassword])
149162
run_cmd.addAll(getDorisConnectorTlsArgs())
150163
logger.info("run_cmd : ${run_cmd.join(' ')}")
151-
def run_flink_jar = run_cmd.execute().getText()
152-
logger.info("result: $run_flink_jar")
164+
// Drain both streams and check the exit code so a failed Flink job cannot look successful.
165+
def flinkProcess = new ProcessBuilder(run_cmd.collect { it.toString() }).redirectErrorStream(true).start()
166+
logger.info("result: ${flinkProcess.text}")
167+
assertEquals(0, flinkProcess.waitFor())
153168
// The publish in the commit phase is asynchronous
154169
Awaitility.await().atMost(30, SECONDS).pollInterval(1, SECONDS).await().until(
155170
{
156171
def resultTbl = sql """ select count(1) from test_types_sink"""
157172
logger.info("retry test_types_sink count: $resultTbl")
158-
resultTbl.size() >= 1
173+
resultTbl[0][0] == 2
159174
})
160175

161176
logger.info("flink job execute finished.");

0 commit comments

Comments
 (0)