Skip to content

Commit a9f0018

Browse files
morningmanclaude
andcommitted
[fix](jdbc) Keep the VARBINARY elements of a remote Doris array as STRING
### What problem does this PR solve? Issue Number: None Related PR: #62183, #55651 Problem Summary: Mapping a remote Doris VARBINARY to VARBINARY also reached the element of an ARRAY column, because the Doris dialect maps an array element through the same function. A remote view with array(cast('abc' as varbinary(16))) then became ARRAY<VARBINARY>, and reading it failed with "Unsupported array child type: VARBINARY": the remote returns an array as text that spells each VARBINARY element as 0x hex ([0x616263, null, 0x]), and MySQLTypeHandler decodes array elements from that text with no case for VARBINARY. Before the mapping change the element was STRING and the column read as that text. branch-4.x maps such a column to ARRAY<VARBINARY> as well and fails the same way when it is read. A VARBINARY element of a remote Doris array, nested or not, maps to STRING again; only a top-level VARBINARY maps to VARBINARY. ### Release note None ### Check List (For Author) - Test: Unit Test / Regression test - JdbcLogicalTypeMappingTest.remoteDorisVarbinaryArrayElementsStayString: ARRAY<VARBINARY>, ARRAY<VARBINARY(16)> and ARRAY<ARRAY<VARBINARY(16)>> keep STRING elements; all fe-connector-jdbc tests (251) pass. - test_doris_jdbc_catalog_varbinary also reads a remote view with a VARBINARY array; the .out was generated by the script. - Behavior changed: No (compared with master) - Does this need documentation: No Co-Authored-By: Claude Opus 5.5 (1M context) <[email protected]>
1 parent 370410c commit a9f0018

4 files changed

Lines changed: 58 additions & 7 deletions

File tree

‎fe/fe-connector/fe-connector-jdbc/src/main/java/org/apache/doris/connector/jdbc/client/JdbcMySQLConnectorClient.java‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -332,6 +332,11 @@ private ConnectorType dorisTypeToConnectorType(String rawType, JdbcFieldInfo fie
332332
if (upperType.startsWith("ARRAY")) {
333333
// ARRAY<INNER_TYPE> -> extract inner type and recursively convert
334334
String innerType = upperType.substring(6, upperType.length() - 1).trim();
335+
if (innerType.startsWith("VARBINARY")) {
336+
// The remote returns an array as text that spells each VARBINARY element as 0x hex, and the scanner
337+
// decodes array elements only into strings, so such an element stays STRING.
338+
return ConnectorType.arrayOf(ConnectorType.of("STRING"));
339+
}
335340
return ConnectorType.arrayOf(dorisTypeToConnectorType(innerType, null));
336341
}
337342

‎fe/fe-connector/fe-connector-jdbc/src/test/java/org/apache/doris/connector/jdbc/client/JdbcLogicalTypeMappingTest.java‎

Lines changed: 33 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -52,13 +52,24 @@ void binaryTypesKeepBytesAndDeclaredLength() {
5252
@Test
5353
void remoteDorisVarbinaryKeepsBytesAndDeclaredLength() throws ReflectiveOperationException {
5454
for (boolean flag : new boolean[] {false, true}) {
55-
JdbcMySQLConnectorClient doris = new JdbcMySQLConnectorClient("test", JdbcDbType.MYSQL,
56-
"jdbc:mysql://localhost/test", false, Collections.emptyMap(), Collections.emptyMap(), flag, flag);
57-
// A remote Doris is detected through the connection, which a unit test does not have.
58-
Field isDoris = JdbcMySQLConnectorClient.class.getDeclaredField("isDoris");
59-
isDoris.setAccessible(true);
60-
isDoris.setBoolean(doris, true);
61-
assertVarbinary(doris, "VARBINARY");
55+
assertVarbinary(remoteDoris(flag), "VARBINARY");
56+
}
57+
}
58+
59+
@Test
60+
void remoteDorisVarbinaryArrayElementsStayString() throws ReflectiveOperationException {
61+
// The scanner decodes the elements of a remote Doris array only into strings, so a VARBINARY element,
62+
// nested or not, keeps the STRING it had before the top-level VARBINARY mapping.
63+
for (boolean flag : new boolean[] {false, true}) {
64+
JdbcConnectorClient doris = remoteDoris(flag);
65+
for (String typeName : new String[] {"ARRAY<VARBINARY>", "ARRAY<VARBINARY(16)>"}) {
66+
ConnectorType type = doris.jdbcTypeToConnectorType(arrayField(typeName));
67+
Assertions.assertEquals("ARRAY", type.getTypeName(), typeName);
68+
Assertions.assertEquals("STRING", type.getChildren().get(0).getTypeName(), typeName);
69+
}
70+
ConnectorType nested = doris.jdbcTypeToConnectorType(arrayField("ARRAY<ARRAY<VARBINARY(16)>>"));
71+
Assertions.assertEquals("ARRAY", nested.getChildren().get(0).getTypeName());
72+
Assertions.assertEquals("STRING", nested.getChildren().get(0).getChildren().get(0).getTypeName());
6273
}
6374
}
6475

@@ -102,6 +113,21 @@ void postgresInstantsAllowDecoderCreatedNulls() {
102113
Assertions.assertFalse(local.isAllowNull());
103114
}
104115

116+
private static JdbcConnectorClient remoteDoris(boolean flag) throws ReflectiveOperationException {
117+
JdbcMySQLConnectorClient doris = new JdbcMySQLConnectorClient("test", JdbcDbType.MYSQL,
118+
"jdbc:mysql://localhost/test", false, Collections.emptyMap(), Collections.emptyMap(), flag, flag);
119+
// A remote Doris is detected through the connection, which a unit test does not have.
120+
Field isDoris = JdbcMySQLConnectorClient.class.getDeclaredField("isDoris");
121+
isDoris.setAccessible(true);
122+
isDoris.setBoolean(doris, true);
123+
return doris;
124+
}
125+
126+
private static JdbcFieldInfo arrayField(String typeName) {
127+
return new JdbcFieldInfo("a", Optional.of(typeName), Types.OTHER,
128+
Optional.of(65535), Optional.of(0), Optional.empty());
129+
}
130+
105131
private static JdbcConnectorClient postgres(boolean flag) {
106132
return new JdbcPostgreSQLConnectorClient("test", JdbcDbType.POSTGRESQL, "jdbc:postgresql://localhost/test",
107133
false, Collections.emptyMap(), Collections.emptyMap(), flag, flag);

‎regression-test/data/external_table_p0/jdbc/test_doris_jdbc_catalog_varbinary.out‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -11,3 +11,11 @@ id tinyint No true \N
1111
1 0x616263 616263
1212
2 \N \N
1313

14+
-- !array_desc --
15+
a array<text> Yes true \N
16+
id tinyint No true \N
17+
18+
-- !array_select --
19+
1 ["0x616263", null, "0x"] 0x616263
20+
2 \N \N
21+

‎regression-test/suites/external_table_p0/jdbc/test_doris_jdbc_catalog_varbinary.groovy‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -37,6 +37,13 @@ suite("test_doris_jdbc_catalog_varbinary", "p0,external") {
3737
select 2, cast(null as varbinary(16))
3838
"""
3939
order_qt_internal """select id, b from internal.${internal_db_name}.varbinary_view order by id"""
40+
sql """drop view if exists ${internal_db_name}.varbinary_array_view"""
41+
sql """
42+
create view ${internal_db_name}.varbinary_array_view as
43+
select 1 as id, array(cast('abc' as varbinary(16)), cast(null as varbinary(16)), cast('' as varbinary(16))) as a
44+
union all
45+
select 2, cast(null as array<varbinary(16)>)
46+
"""
4047

4148
sql """drop catalog if exists ${catalog_name}"""
4249
sql """CREATE CATALOG `${catalog_name}` PROPERTIES (
@@ -53,4 +60,9 @@ suite("test_doris_jdbc_catalog_varbinary", "p0,external") {
5360
order_qt_desc """desc ${catalog_name}.${internal_db_name}.varbinary_view"""
5461
// The bytes read through the catalog are the remote ones.
5562
order_qt_select """select id, b, hex(b) from ${catalog_name}.${internal_db_name}.varbinary_view order by id"""
63+
64+
// The remote returns an array as text that spells each VARBINARY element as 0x hex, and the scanner decodes
65+
// array elements only into strings, so a VARBINARY element stays STRING and reads as that text.
66+
order_qt_array_desc """desc ${catalog_name}.${internal_db_name}.varbinary_array_view"""
67+
order_qt_array_select """select id, a, a[1] from ${catalog_name}.${internal_db_name}.varbinary_array_view order by id"""
5668
}

0 commit comments

Comments
 (0)