Skip to content

Commit 81d106e

Browse files
committed
[fix](hive) Match OpenCSV reader escape and character constraints
### What problem does this PR solve? Related PR: #68027 Problem Summary: An explicit double-quote escape property is Hive's writer-default sentinel, but its reader uses backslash for that value. Passing the raw sentinel to the BE corrupts escaped fields. Resolve the sentinel before emitting scan properties and reject NUL separators or duplicate active parser characters, while allowing disabled NUL quote/escape characters. ### Release note Honor Hive's explicit default CSV escape and reject invalid CSV character combinations before scanning. ### Check List (For Author) - Test: 479 Hive connector tests (1,249 including upstream modules), eight FE Core Thrift tests, real Hive reader comparisons for 343 character tuples with three property sources, FE Checkstyle, and Groovy compilation. HiveServer2 end-to-end execution remains pending. - Behavior changed: Yes. Match Hive's reader-effective escape and character constraints. - Does this need documentation: No.
1 parent 9c7e82e commit 81d106e

6 files changed

Lines changed: 272 additions & 4 deletions

File tree

‎fe/fe-connector/fe-connector-hive/pom.xml‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -114,6 +114,14 @@ under the License.
114114
<artifactId>log4j-api</artifactId>
115115
</dependency>
116116

117+
<!-- Match the parser used by Hive 3.1.3 for reader-compatibility tests only. -->
118+
<dependency>
119+
<groupId>net.sf.opencsv</groupId>
120+
<artifactId>opencsv</artifactId>
121+
<version>2.3</version>
122+
<scope>test</scope>
123+
</dependency>
124+
117125
<dependency>
118126
<groupId>org.junit.jupiter</groupId>
119127
<artifactId>junit-jupiter</artifactId>
@@ -146,6 +154,13 @@ under the License.
146154
<build>
147155
<finalName>doris-fe-connector-hive</finalName>
148156
<plugins>
157+
<plugin>
158+
<artifactId>maven-surefire-plugin</artifactId>
159+
<configuration>
160+
<!-- Hive 3's real SerDe initializes StringInternUtils, which reflects on URI on Java 17. -->
161+
<argLine>@{argLine} --add-opens=java.base/java.net=ALL-UNNAMED</argLine>
162+
</configuration>
163+
</plugin>
149164
<plugin>
150165
<artifactId>maven-assembly-plugin</artifactId>
151166
<configuration>

‎fe/fe-connector/fe-connector-hive/src/main/java/org/apache/doris/connector/hive/HiveTextProperties.java‎

Lines changed: 19 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -164,18 +164,33 @@ private static void extractCsvSerDeProps(Map<String, String> params,
164164
Map<String, String> tableParams, Map<String, String> result) {
165165
// Trino stores CSV settings in table parameters. Honor Hive's table-over-SerDe precedence
166166
// so valid CSV files are not silently split with the default delimiter and quote characters.
167-
result.put(ScanNodePropertyKeys.TEXT_COLUMN_SEPARATOR,
168-
getCsvCharacter(params, tableParams, SEPARATOR_CHAR, ','));
167+
String separator = getCsvCharacter(params, tableParams, SEPARATOR_CHAR, ',');
168+
String quoteChar = getCsvCharacter(params, tableParams, QUOTE_CHAR, '"');
169+
String escapeChar = getCsvCharacter(params, tableParams, ESCAPE_CHAR, '"');
170+
// Hive treats the writer-default double quote as a sentinel: newReader selects the constructor
171+
// whose effective escape is backslash. Resolve it before validating the parser's character tuple.
172+
if ("\"".equals(escapeChar)) {
173+
escapeChar = "\\";
174+
}
175+
if ("\0".equals(separator)) {
176+
throw new DorisConnectorException("Invalid OpenCSVSerde property 'separatorChar': must not be NUL");
177+
}
178+
// OpenCSV requires distinct active characters; NUL disables quote/escape and may be shared by both.
179+
if (separator.equals(quoteChar) || separator.equals(escapeChar)
180+
|| (!"\0".equals(quoteChar) && quoteChar.equals(escapeChar))) {
181+
throw new DorisConnectorException("Invalid OpenCSVSerde configuration: "
182+
+ "separatorChar, quoteChar and escapeChar must be distinct when non-NUL");
183+
}
184+
result.put(ScanNodePropertyKeys.TEXT_COLUMN_SEPARATOR, separator);
169185
// OpenCSVSerde does not use table-level line.delim to frame records. Preserve the existing
170186
// SerDe-only override; applying a table property here can merge otherwise valid newline records.
171187
String lineDelimiter = params == null ? null : params.get(LINE_DELIM);
172188
result.put(ScanNodePropertyKeys.TEXT_LINE_DELIMITER,
173189
lineDelimiter == null ? DEFAULT_LINE_DELIM : lineDelimiter);
174-
String quoteChar = getCsvCharacter(params, tableParams, QUOTE_CHAR, '"');
175190
result.put(ScanNodePropertyKeys.TEXT_ENCLOSE, quoteChar);
176191
// BE's extra double-quote trimming is valid only for the effective double-quote enclosure.
177192
result.put(ScanNodePropertyKeys.TEXT_TRIM_DOUBLE_QUOTES, String.valueOf("\"".equals(quoteChar)));
178-
result.put(ScanNodePropertyKeys.TEXT_ESCAPE, getCsvCharacter(params, tableParams, ESCAPE_CHAR, '\\'));
193+
result.put(ScanNodePropertyKeys.TEXT_ESCAPE, escapeChar);
179194
result.put(ScanNodePropertyKeys.TEXT_NULL_FORMAT, "");
180195
}
181196

Lines changed: 105 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,105 @@
1+
// Licensed to the Apache Software Foundation (ASF) under one
2+
// or more contributor license agreements. See the NOTICE file
3+
// distributed with this work for additional information
4+
// regarding copyright ownership. The ASF licenses this file
5+
// to you under the Apache License, Version 2.0 (the
6+
// "License"); you may not use this file except in compliance
7+
// with the License. You may obtain a copy of the License at
8+
//
9+
// http://www.apache.org/licenses/LICENSE-2.0
10+
//
11+
// Unless required by applicable law or agreed to in writing,
12+
// software distributed under the License is distributed on an
13+
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
// KIND, either express or implied. See the License for the
15+
// specific language governing permissions and limitations
16+
// under the License.
17+
18+
package org.apache.doris.connector.hive;
19+
20+
import org.apache.doris.connector.spi.DorisConnectorException;
21+
import org.apache.doris.connector.spi.scan.ScanNodePropertyKeys;
22+
23+
import au.com.bytecode.opencsv.CSVReader;
24+
import org.apache.hadoop.hive.serde2.OpenCSVSerde;
25+
import org.apache.hadoop.hive.serde2.SerDeException;
26+
import org.apache.hadoop.io.Text;
27+
import org.junit.jupiter.api.Assertions;
28+
import org.junit.jupiter.api.Test;
29+
30+
import java.io.StringReader;
31+
import java.util.Arrays;
32+
import java.util.List;
33+
import java.util.Map;
34+
import java.util.Properties;
35+
36+
class HiveCsvReaderCompatibilityTest {
37+
@Test
38+
void testExplicitDefaultEscapeReadsLikeHive() throws Exception {
39+
for (String quote : new String[] {"\"", "q"}) {
40+
String record = quote + "left\\" + quote + "middle" + quote + ",tail";
41+
Map<String, String> properties = Map.of("quoteChar", quote, "escapeChar", "\"suffix");
42+
List<?> expected = (List<?>) hiveSerde(properties).deserialize(new Text(record));
43+
Assertions.assertEquals(List.of("left" + quote + "middle", "tail"), expected);
44+
for (Map<String, String> extracted : List.of(
45+
HiveTextProperties.extract(HiveTextProperties.HIVE_OPEN_CSV_SERDE, Map.of(), properties),
46+
HiveTextProperties.extract(HiveTextProperties.HIVE_OPEN_CSV_SERDE, properties, Map.of()))) {
47+
Assertions.assertEquals("\\", extracted.get(ScanNodePropertyKeys.TEXT_ESCAPE));
48+
try (CSVReader reader = new CSVReader(new StringReader(record),
49+
extracted.get(ScanNodePropertyKeys.TEXT_COLUMN_SEPARATOR).charAt(0),
50+
extracted.get(ScanNodePropertyKeys.TEXT_ENCLOSE).charAt(0),
51+
extracted.get(ScanNodePropertyKeys.TEXT_ESCAPE).charAt(0))) {
52+
Assertions.assertEquals(expected, Arrays.asList(reader.readNext()));
53+
}
54+
}
55+
}
56+
}
57+
58+
@Test
59+
void testCharacterTuplesMatchHiveReaderAcceptance() throws Exception {
60+
// Use Hive's real reader as the oracle, including its writer-default escape sentinel and NUL rules.
61+
char[] characters = {'\0', ',', '|', '"', '\\', 'q', 'e'};
62+
for (char separator : characters) {
63+
for (char quote : characters) {
64+
for (char escape : characters) {
65+
Map<String, String> properties = Map.of("separatorChar", String.valueOf(separator),
66+
"quoteChar", String.valueOf(quote), "escapeChar", String.valueOf(escape));
67+
OpenCSVSerde serde = hiveSerde(properties);
68+
boolean accepted;
69+
try {
70+
serde.deserialize(new Text(""));
71+
accepted = true;
72+
} catch (SerDeException e) {
73+
Assertions.assertInstanceOf(UnsupportedOperationException.class, e.getCause());
74+
accepted = false;
75+
}
76+
assertAcceptance(accepted, Map.of(), properties);
77+
assertAcceptance(accepted, properties, Map.of());
78+
assertAcceptance(accepted, Map.of("quoteChar", String.valueOf(quote)),
79+
Map.of("separatorChar", String.valueOf(separator), "escapeChar", String.valueOf(escape)));
80+
}
81+
}
82+
}
83+
}
84+
85+
private static void assertAcceptance(boolean accepted, Map<String, String> serdeProperties,
86+
Map<String, String> tableProperties) {
87+
if (accepted) {
88+
Assertions.assertDoesNotThrow(() -> HiveTextProperties.extract(
89+
HiveTextProperties.HIVE_OPEN_CSV_SERDE, serdeProperties, tableProperties));
90+
} else {
91+
Assertions.assertThrows(DorisConnectorException.class, () -> HiveTextProperties.extract(
92+
HiveTextProperties.HIVE_OPEN_CSV_SERDE, serdeProperties, tableProperties));
93+
}
94+
}
95+
96+
private static OpenCSVSerde hiveSerde(Map<String, String> properties) throws SerDeException {
97+
Properties serdeProperties = new Properties();
98+
serdeProperties.setProperty("columns", "first,second");
99+
serdeProperties.setProperty("columns.types", "string:string");
100+
serdeProperties.putAll(properties);
101+
OpenCSVSerde serde = new OpenCSVSerde();
102+
serde.initialize(null, serdeProperties);
103+
return serde;
104+
}
105+
}

‎fe/fe-connector/fe-connector-hive/src/test/java/org/apache/doris/connector/hive/HiveTextPropertiesTest.java‎

Lines changed: 52 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -283,6 +283,58 @@ public void testCsvValidatesOnlyTheEffectiveCharacterProperties() {
283283
"s", "q", "e", "false");
284284
}
285285

286+
@Test
287+
public void testCsvDefaultEscapeSentinelFromEitherPropertySource() {
288+
for (String value : new String[] {"\"", "\"suffix"}) {
289+
assertCsvProperties(HiveTextProperties.extract(OPEN_CSV_SERDE, sd(), sd("escapeChar", value)),
290+
",", "\"", "\\", "true");
291+
assertCsvProperties(HiveTextProperties.extract(OPEN_CSV_SERDE, sd("escapeChar", value), sd()),
292+
",", "\"", "\\", "true");
293+
}
294+
assertCsvProperties(HiveTextProperties.extract(OPEN_CSV_SERDE,
295+
sd("escapeChar", "e"), sd("escapeChar", "\"", "quoteChar", "q")),
296+
",", "q", "\\", "false");
297+
assertCsvProperties(HiveTextProperties.extract(OPEN_CSV_SERDE,
298+
sd("escapeChar", "\""), sd("escapeChar", "e")), ",", "\"", "e", "true");
299+
}
300+
301+
@Test
302+
public void testCsvRejectsConflictingEffectiveCharacters() {
303+
String[][] pairs = {{"separatorChar", "quoteChar"}, {"separatorChar", "escapeChar"},
304+
{"quoteChar", "escapeChar"}};
305+
for (String[] pair : pairs) {
306+
Map<String, String> properties = sd(pair[0], "|first", pair[1], "|second");
307+
Assertions.assertThrows(DorisConnectorException.class,
308+
() -> HiveTextProperties.extract(OPEN_CSV_SERDE, sd(), properties));
309+
Assertions.assertThrows(DorisConnectorException.class,
310+
() -> HiveTextProperties.extract(OPEN_CSV_SERDE, properties, sd()));
311+
Assertions.assertThrows(DorisConnectorException.class,
312+
() -> HiveTextProperties.extract(OPEN_CSV_SERDE, sd(pair[0], "|"), sd(pair[1], "|")));
313+
}
314+
// Resolve the writer-default escape sentinel before checking parser-character conflicts.
315+
Assertions.assertThrows(DorisConnectorException.class,
316+
() -> HiveTextProperties.extract(OPEN_CSV_SERDE,
317+
sd("separatorChar", "\\"), sd("escapeChar", "\"")));
318+
Assertions.assertThrows(DorisConnectorException.class,
319+
() -> HiveTextProperties.extract(OPEN_CSV_SERDE,
320+
sd("quoteChar", "\\"), sd("escapeChar", "\"")));
321+
assertCsvProperties(HiveTextProperties.extract(OPEN_CSV_SERDE, sd(),
322+
sd("separatorChar", "\"", "quoteChar", "q", "escapeChar", "\"")),
323+
"\"", "q", "\\", "false");
324+
}
325+
326+
@Test
327+
public void testCsvRejectsNulSeparatorButAllowsDisabledQuoteAndEscape() {
328+
Assertions.assertThrows(DorisConnectorException.class,
329+
() -> HiveTextProperties.extract(OPEN_CSV_SERDE, sd(), sd("separatorChar", "\0")));
330+
Assertions.assertThrows(DorisConnectorException.class,
331+
() -> HiveTextProperties.extract(OPEN_CSV_SERDE, sd("separatorChar", "\0"), sd()));
332+
assertCsvProperties(HiveTextProperties.extract(OPEN_CSV_SERDE, sd(),
333+
sd("quoteChar", "\0", "escapeChar", "\0")), ",", "\0", "\0", "false");
334+
assertCsvProperties(HiveTextProperties.extract(OPEN_CSV_SERDE,
335+
sd("quoteChar", "\0", "escapeChar", "\0"), sd()), ",", "\0", "\0", "false");
336+
}
337+
286338
private static void assertCsvProperties(Map<String, String> result,
287339
String separator, String quote, String escape, String trimDoubleQuotes) {
288340
Assertions.assertAll(

‎fe/fe-core/src/test/java/org/apache/doris/datasource/scan/PluginDrivenScanNodeCsvPropertiesTest.java‎

Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,7 @@
3131
import org.junit.jupiter.api.Test;
3232
import org.mockito.Mockito;
3333

34+
import java.util.List;
3435
import java.util.Map;
3536

3637
class PluginDrivenScanNodeCsvPropertiesTest {
@@ -76,6 +77,35 @@ void testInvalidCharactersFailBeforeThriftConstruction() {
7677
() -> toWire(Map.of(), Map.of("separatorChar", "😀")));
7778
}
7879

80+
@Test
81+
void testDefaultEscapeSentinelSurvivesThriftAsBackslash() throws Exception {
82+
for (String quote : new String[] {"\"", "q"}) {
83+
Map<String, String> properties = Map.of("quoteChar", quote, "escapeChar", "\"suffix");
84+
assertWireProperties(toWire(Map.of(), properties), ",", quote.charAt(0), '\\', "\"".equals(quote));
85+
assertWireProperties(toWire(properties, Map.of()), ",", quote.charAt(0), '\\', "\"".equals(quote));
86+
}
87+
}
88+
89+
@Test
90+
void testCharacterConflictsFailBeforeThriftConstruction() {
91+
for (Map<String, String> properties : List.of(
92+
Map.of("separatorChar", "|", "quoteChar", "|"),
93+
Map.of("separatorChar", "|", "escapeChar", "|"),
94+
Map.of("quoteChar", "q", "escapeChar", "q"),
95+
Map.of("separatorChar", "\\", "escapeChar", "\""),
96+
Map.of("separatorChar", "\0"))) {
97+
Assertions.assertThrows(DorisConnectorException.class, () -> toWire(Map.of(), properties));
98+
Assertions.assertThrows(DorisConnectorException.class, () -> toWire(properties, Map.of()));
99+
}
100+
}
101+
102+
@Test
103+
void testDisabledQuoteAndEscapeSurviveThrift() throws Exception {
104+
Map<String, String> properties = Map.of("quoteChar", "\0", "escapeChar", "\0");
105+
assertWireProperties(toWire(Map.of(), properties), ",", '\0', '\0', false);
106+
assertWireProperties(toWire(properties, Map.of()), ",", '\0', '\0', false);
107+
}
108+
79109
private static TFileAttributes toWire(Map<String, String> serdeProperties,
80110
Map<String, String> tableProperties) throws Exception {
81111
PluginDrivenScanNode node = Mockito.mock(PluginDrivenScanNode.class, Mockito.CALLS_REAL_METHODS);

‎regression-test/suites/external_table_p0/hive/test_csv_table_properties.groovy‎

Lines changed: 51 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -129,5 +129,56 @@ suite("test_csv_table_properties", "p0,external,hive,external_docker,external_do
129129
)
130130
"""
131131
}
132+
// Write with an explicit backslash, then change only the metadata to Hive's double-quote sentinel.
133+
// This leaves backslash-escaped embedded quotes in the file and exercises Hive's reader constructor choice.
134+
String sqlBackslash = "\\\\"
135+
for (boolean tableProperties : [true, false]) {
136+
String table = tableProperties ? "csv_default_escape_table" : "csv_default_escape_serde"
137+
hive_docker "DROP TABLE IF EXISTS csv_table_properties_db.${table}"
138+
hive_docker """
139+
CREATE TABLE csv_table_properties_db.${table} (label STRING, payload STRING)
140+
ROW FORMAT SERDE 'org.apache.hadoop.hive.serde2.OpenCSVSerde'
141+
${tableProperties ? '' : "WITH SERDEPROPERTIES ('escapeChar'='${sqlBackslash}')"}
142+
STORED AS TEXTFILE
143+
${tableProperties ? "TBLPROPERTIES ('escapeChar'='${sqlBackslash}')" : ''}
144+
"""
145+
hive_docker """
146+
INSERT INTO csv_table_properties_db.${table}
147+
SELECT label, payload FROM csv_table_properties_db.source_rows
148+
"""
149+
for (String sentinel : ['"', '"suffix']) {
150+
hive_docker """
151+
ALTER TABLE csv_table_properties_db.${table}
152+
SET ${tableProperties ? 'TBLPROPERTIES' : 'SERDEPROPERTIES'} ('escapeChar'='${sentinel}')
153+
"""
154+
sql "REFRESH DATABASE csv_table_properties_catalog.csv_table_properties_db"
155+
def expected = hive_docker "SELECT label, payload FROM csv_table_properties_db.source_rows ORDER BY label"
156+
def hiveRows = hive_docker "SELECT label, payload FROM csv_table_properties_db.${table} ORDER BY label"
157+
assertEquals(expected, hiveRows)
158+
assertEquals(hiveRows, sql("SELECT label, payload FROM ${table} ORDER BY label"))
159+
}
160+
}
161+
162+
// These metadata values initialize successfully in Hive, but its reader rejects the effective tuple.
163+
for (String properties : [
164+
"'separatorChar'='q', 'quoteChar'='q', 'escapeChar'='e'",
165+
"'separatorChar'='e', 'quoteChar'='q', 'escapeChar'='e'",
166+
"'separatorChar'='s', 'quoteChar'='e', 'escapeChar'='e'",
167+
"'separatorChar'='${sqlBackslash}', 'quoteChar'='q', 'escapeChar'='\"'"
168+
]) {
169+
hive_docker """
170+
ALTER TABLE csv_table_properties_db.csv_two_columns SET TBLPROPERTIES (${properties})
171+
"""
172+
sql "REFRESH DATABASE csv_table_properties_catalog.csv_table_properties_db"
173+
test {
174+
sql "SELECT label, payload FROM csv_two_columns ORDER BY label"
175+
exception "separatorChar, quoteChar and escapeChar must be distinct when non-NUL"
176+
}
177+
}
178+
hive_docker """
179+
ALTER TABLE csv_table_properties_db.csv_two_columns SET TBLPROPERTIES (
180+
'separatorChar'='s', 'quoteChar'='q', 'escapeChar'='e'
181+
)
182+
"""
132183
sql "REFRESH DATABASE csv_table_properties_catalog.csv_table_properties_db"
133184
}

0 commit comments

Comments
 (0)