Skip to content

Commit 9c7e82e

Browse files
committed
[fix](hive) Validate CSV characters and preserve record framing
1 parent bdf7dc6 commit 9c7e82e

4 files changed

Lines changed: 253 additions & 28 deletions

File tree

‎fe/fe-connector/fe-connector-hive/src/main/java/org/apache/doris/connector/hive/HiveTextProperties.java‎

Lines changed: 36 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@
1717

1818
package org.apache.doris.connector.hive;
1919

20+
import org.apache.doris.connector.spi.DorisConnectorException;
2021
import org.apache.doris.connector.spi.scan.ScanNodePropertyKeys;
2122

2223
import java.util.HashMap;
@@ -37,8 +38,8 @@
3738
* {@link HiveCatalogProperties} (per catalog) and {@link HmsConf} (per FE). Its input keys are remote HMS
3839
* table parameters and its output keys are a BE payload, so both sets belong to the single class that
3940
* reads them, which is this one. That is also why its two numeric parses stay lenient: a garbage
40-
* {@code skip.header.line.count} or delimiter in somebody else's Hive table must not fail the query the
41-
* way a garbage value a user typed into {@code CREATE CATALOG} now does.</p>
41+
* {@code skip.header.line.count} or LazySimpleSerDe delimiter keeps its historical fallback. CSV
42+
* character properties instead follow OpenCSVSerde validation and the BE wire-format limits.</p>
4243
*/
4344
public final class HiveTextProperties {
4445

@@ -103,7 +104,7 @@ private HiveTextProperties() {
103104
*
104105
* @param serDeLib the SerDe library class name
105106
* @param sdParams the StorageDescriptor / SerDeInfo parameters
106-
* @param tableParams the table-level parameters (for skip.header.line.count)
107+
* @param tableParams the table-level SerDe properties and skip.header.line.count
107108
* @return map of text properties, empty if not a text-based format
108109
*/
109110
public static Map<String, String> extract(String serDeLib,
@@ -164,19 +165,17 @@ private static void extractCsvSerDeProps(Map<String, String> params,
164165
// Trino stores CSV settings in table parameters. Honor Hive's table-over-SerDe precedence
165166
// so valid CSV files are not silently split with the default delimiter and quote characters.
166167
result.put(ScanNodePropertyKeys.TEXT_COLUMN_SEPARATOR,
167-
getParamOrDefault(params, tableParams, SEPARATOR_CHAR, ","));
168+
getCsvCharacter(params, tableParams, SEPARATOR_CHAR, ','));
169+
// OpenCSVSerde does not use table-level line.delim to frame records. Preserve the existing
170+
// SerDe-only override; applying a table property here can merge otherwise valid newline records.
171+
String lineDelimiter = params == null ? null : params.get(LINE_DELIM);
168172
result.put(ScanNodePropertyKeys.TEXT_LINE_DELIMITER,
169-
getParamOrDefault(params, tableParams, LINE_DELIM, DEFAULT_LINE_DELIM));
170-
String quoteChar = getParamOrDefault(params, tableParams, QUOTE_CHAR, "\"");
173+
lineDelimiter == null ? DEFAULT_LINE_DELIM : lineDelimiter);
174+
String quoteChar = getCsvCharacter(params, tableParams, QUOTE_CHAR, '"');
171175
result.put(ScanNodePropertyKeys.TEXT_ENCLOSE, quoteChar);
172-
// #65501: BE strips the wrapping quotes only when the enclose char is exactly the double-quote '"'.
173-
// The connector owns this CSV serde semantics, so decide here and pass an explicit flag; the generic
174-
// PluginDrivenScanNode then just applies it instead of trimming for any enclose char. Compare the
175-
// first byte, matching how the node sets enclose (enclose.getBytes()[0]) and BE's getEnclose() == '"'.
176-
boolean trimDoubleQuotes = !quoteChar.isEmpty() && quoteChar.getBytes()[0] == (byte) '"';
177-
result.put(ScanNodePropertyKeys.TEXT_TRIM_DOUBLE_QUOTES, String.valueOf(trimDoubleQuotes));
178-
String escapeChar = getParamOrDefault(params, tableParams, ESCAPE_CHAR, "\\");
179-
result.put(ScanNodePropertyKeys.TEXT_ESCAPE, escapeChar);
176+
// BE's extra double-quote trimming is valid only for the effective double-quote enclosure.
177+
result.put(ScanNodePropertyKeys.TEXT_TRIM_DOUBLE_QUOTES, String.valueOf("\"".equals(quoteChar)));
178+
result.put(ScanNodePropertyKeys.TEXT_ESCAPE, getCsvCharacter(params, tableParams, ESCAPE_CHAR, '\\'));
180179
result.put(ScanNodePropertyKeys.TEXT_NULL_FORMAT, "");
181180
}
182181

@@ -256,9 +255,28 @@ private static int getSkipHeaderCount(Map<String, String> tableParams) {
256255
}
257256
}
258257

259-
private static String getParamOrDefault(Map<String, String> params, Map<String, String> tableParams,
260-
String key, String defaultVal) {
261-
String val = serdeVal(params, tableParams, key);
262-
return (val != null) ? val : defaultVal;
258+
private static String getCsvCharacter(Map<String, String> params, Map<String, String> tableParams,
259+
String key, char defaultValue) {
260+
String value = serdeVal(params, tableParams, key);
261+
if (value == null) {
262+
return Character.toString(defaultValue);
263+
}
264+
if (value.isEmpty()) {
265+
throw new DorisConnectorException("Invalid OpenCSVSerde property '" + key + "': value must not be empty");
266+
}
267+
// Hive uses charAt(0), not a complete string, Unicode code point, or numeric byte value.
268+
// Validate only that effective character, after resolving table-over-SerDe precedence.
269+
char character = value.charAt(0);
270+
if (Character.isSurrogate(character)) {
271+
throw new DorisConnectorException("Unsupported OpenCSVSerde property '" + key
272+
+ "': the first Java character must not be a surrogate");
273+
}
274+
// Separator is a Thrift string, but quote/escape are i8 fields consumed as single UTF-8 bytes.
275+
// Reject unrepresentable characters instead of silently passing only their first encoded byte.
276+
if (!SEPARATOR_CHAR.equals(key) && character > 0x7f) {
277+
throw new DorisConnectorException("Unsupported OpenCSVSerde property '" + key
278+
+ "': the first character must be ASCII (a single UTF-8 byte)");
279+
}
280+
return Character.toString(character);
263281
}
264282
}

‎fe/fe-connector/fe-connector-hive/src/test/java/org/apache/doris/connector/hive/HiveTextPropertiesTest.java‎

Lines changed: 64 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,8 @@
1717

1818
package org.apache.doris.connector.hive;
1919

20+
import org.apache.doris.connector.spi.DorisConnectorException;
21+
2022
import org.junit.jupiter.api.Assertions;
2123
import org.junit.jupiter.api.Test;
2224

@@ -216,10 +218,69 @@ public void testCsvDefaultsWithMissingParameterMaps() {
216218
}
217219

218220
@Test
219-
public void testCsvExplicitEmptyTableParametersOverrideSerdeParameters() {
221+
public void testCsvIgnoresTableRecordDelimiter() {
222+
for (String value : new String[] {"|", "", "\r\n"}) {
223+
Map<String, String> result = HiveTextProperties.extract(OPEN_CSV_SERDE, sd(), sd("line.delim", value));
224+
Assertions.assertEquals("\n", result.get(PREFIX + "line_delimiter"));
225+
}
220226
Map<String, String> result = HiveTextProperties.extract(OPEN_CSV_SERDE,
221-
sd("quoteChar", "q", "escapeChar", "e"), sd("quoteChar", "", "escapeChar", ""));
222-
assertCsvProperties(result, ",", "", "", "false");
227+
sd("line.delim", "\r\n"), sd("line.delim", "|"));
228+
Assertions.assertEquals("\r\n", result.get(PREFIX + "line_delimiter"));
229+
}
230+
231+
@Test
232+
public void testCsvUsesFirstJavaCharacterFromEitherPropertySource() {
233+
Map<String, String> properties = sd("separatorChar", "ss", "quoteChar", "qq", "escapeChar", "ee");
234+
assertCsvProperties(HiveTextProperties.extract(OPEN_CSV_SERDE, sd(), properties),
235+
"s", "q", "e", "false");
236+
assertCsvProperties(HiveTextProperties.extract(OPEN_CSV_SERDE, properties, sd()),
237+
"s", "q", "e", "false");
238+
assertCsvProperties(HiveTextProperties.extract(OPEN_CSV_SERDE, sd(),
239+
sd("separatorChar", "99", "quoteChar", "\"suffix", "escapeChar", "eé")),
240+
"9", "\"", "e", "true");
241+
}
242+
243+
@Test
244+
public void testCsvRejectsEmptyCharacterPropertiesInsteadOfFallingBack() {
245+
for (String key : new String[] {"separatorChar", "quoteChar", "escapeChar"}) {
246+
DorisConnectorException tableError = Assertions.assertThrows(DorisConnectorException.class,
247+
() -> HiveTextProperties.extract(OPEN_CSV_SERDE, sd(key, "x"), sd(key, "")));
248+
Assertions.assertTrue(tableError.getMessage().contains(key));
249+
Assertions.assertTrue(tableError.getMessage().contains("empty"));
250+
Assertions.assertThrows(DorisConnectorException.class,
251+
() -> HiveTextProperties.extract(OPEN_CSV_SERDE, sd(key, ""), sd()));
252+
}
253+
}
254+
255+
@Test
256+
public void testCsvRejectsNonAsciiQuoteAndEscapeFromEitherSource() {
257+
for (String key : new String[] {"quoteChar", "escapeChar"}) {
258+
for (String value : new String[] {"é", "中", "😀"}) {
259+
Assertions.assertThrows(DorisConnectorException.class,
260+
() -> HiveTextProperties.extract(OPEN_CSV_SERDE, sd(), sd(key, value)), key);
261+
Assertions.assertThrows(DorisConnectorException.class,
262+
() -> HiveTextProperties.extract(OPEN_CSV_SERDE, sd(key, value), sd()), key);
263+
}
264+
}
265+
}
266+
267+
@Test
268+
public void testCsvPreservesUtf8SeparatorButRejectsSurrogates() {
269+
assertCsvProperties(HiveTextProperties.extract(OPEN_CSV_SERDE, sd(), sd("separatorChar", "ésuffix")),
270+
"é", "\"", "\\", "true");
271+
for (String value : new String[] {"😀", String.valueOf(Character.MIN_HIGH_SURROGATE),
272+
String.valueOf(Character.MIN_LOW_SURROGATE)}) {
273+
Assertions.assertThrows(DorisConnectorException.class,
274+
() -> HiveTextProperties.extract(OPEN_CSV_SERDE, sd(), sd("separatorChar", value)));
275+
}
276+
}
277+
278+
@Test
279+
public void testCsvValidatesOnlyTheEffectiveCharacterProperties() {
280+
assertCsvProperties(HiveTextProperties.extract(OPEN_CSV_SERDE,
281+
sd("separatorChar", "", "quoteChar", "é", "escapeChar", ""),
282+
sd("separatorChar", "ss", "quoteChar", "qq", "escapeChar", "ee")),
283+
"s", "q", "e", "false");
223284
}
224285

225286
private static void assertCsvProperties(Map<String, String> result,
Lines changed: 110 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,110 @@
1+
// Licensed to the Apache Software Foundation (ASF) under one
2+
// or more contributor license agreements. See the NOTICE file
3+
// distributed with this work for additional information
4+
// regarding copyright ownership. The ASF licenses this file
5+
// to you under the Apache License, Version 2.0 (the
6+
// "License"); you may not use this file except in compliance
7+
// with the License. You may obtain a copy of the License at
8+
//
9+
// http://www.apache.org/licenses/LICENSE-2.0
10+
//
11+
// Unless required by applicable law or agreed to in writing,
12+
// software distributed under the License is distributed on an
13+
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
// KIND, either express or implied. See the License for the
15+
// specific language governing permissions and limitations
16+
// under the License.
17+
18+
package org.apache.doris.datasource.scan;
19+
20+
import org.apache.doris.common.jmockit.Deencapsulation;
21+
import org.apache.doris.connector.hive.HiveTextProperties;
22+
import org.apache.doris.connector.spi.DorisConnectorException;
23+
import org.apache.doris.qe.SessionVariable;
24+
import org.apache.doris.thrift.TFileAttributes;
25+
import org.apache.doris.thrift.TFileTextScanRangeParams;
26+
27+
import org.apache.thrift.TDeserializer;
28+
import org.apache.thrift.TSerializer;
29+
import org.apache.thrift.protocol.TCompactProtocol;
30+
import org.junit.jupiter.api.Assertions;
31+
import org.junit.jupiter.api.Test;
32+
import org.mockito.Mockito;
33+
34+
import java.util.Map;
35+
36+
class PluginDrivenScanNodeCsvPropertiesTest {
37+
private static final String CSV_SERDE = "org.apache.hadoop.hive.serde2.OpenCSVSerde";
38+
39+
@Test
40+
void testNormalizedCharactersSurviveScanNodeAndThrift() throws Exception {
41+
Map<String, String> properties = Map.of("separatorChar", "ss", "quoteChar", "qq", "escapeChar", "ee",
42+
"line.delim", "|");
43+
assertWireProperties(toWire(Map.of(), properties), "s", 'q', 'e', false);
44+
assertWireProperties(toWire(properties, Map.of("line.delim", "|")), "s", 'q', 'e', false, "|");
45+
}
46+
47+
@Test
48+
void testEffectiveTablePropertiesOverrideInvalidSerdeProperties() throws Exception {
49+
assertWireProperties(toWire(Map.of("separatorChar", "", "quoteChar", "é", "escapeChar", ""),
50+
Map.of("separatorChar", "ss", "quoteChar", "qq", "escapeChar", "ee")),
51+
"s", 'q', 'e', false);
52+
}
53+
54+
@Test
55+
void testDefaultsAndNormalizedDoubleQuote() throws Exception {
56+
assertWireProperties(toWire(Map.of(), Map.of()), ",", '"', '\\', true);
57+
assertWireProperties(toWire(Map.of(),
58+
Map.of("separatorChar", "99", "quoteChar", "\"suffix", "escapeChar", "eé")),
59+
"9", '"', 'e', true);
60+
}
61+
62+
@Test
63+
void testUtf8SeparatorRemainsACompleteCharacter() throws Exception {
64+
assertWireProperties(toWire(Map.of(), Map.of("separatorChar", "ésuffix")), "é", '"', '\\', true);
65+
}
66+
67+
@Test
68+
void testInvalidCharactersFailBeforeThriftConstruction() {
69+
for (String key : new String[] {"separatorChar", "quoteChar", "escapeChar"}) {
70+
Assertions.assertThrows(DorisConnectorException.class, () -> toWire(Map.of(), Map.of(key, "")));
71+
}
72+
for (String key : new String[] {"quoteChar", "escapeChar"}) {
73+
Assertions.assertThrows(DorisConnectorException.class, () -> toWire(Map.of(), Map.of(key, "é")));
74+
}
75+
Assertions.assertThrows(DorisConnectorException.class,
76+
() -> toWire(Map.of(), Map.of("separatorChar", "😀")));
77+
}
78+
79+
private static TFileAttributes toWire(Map<String, String> serdeProperties,
80+
Map<String, String> tableProperties) throws Exception {
81+
PluginDrivenScanNode node = Mockito.mock(PluginDrivenScanNode.class, Mockito.CALLS_REAL_METHODS);
82+
Deencapsulation.setField(node, "scanNodeProperties",
83+
HiveTextProperties.extract(CSV_SERDE, serdeProperties, tableProperties));
84+
Deencapsulation.setField(node, "sessionVariable", new SessionVariable());
85+
// Exercise the real scan-node consumer and wire encoding: map assertions cannot detect byte truncation.
86+
TFileAttributes attributes = node.getFileAttributes();
87+
TFileAttributes decoded = new TFileAttributes();
88+
new TDeserializer(new TCompactProtocol.Factory()).deserialize(decoded,
89+
new TSerializer(new TCompactProtocol.Factory()).serialize(attributes));
90+
return decoded;
91+
}
92+
93+
private static void assertWireProperties(TFileAttributes attributes, String separator,
94+
char quote, char escape, boolean trim) {
95+
assertWireProperties(attributes, separator, quote, escape, trim, "\n");
96+
}
97+
98+
private static void assertWireProperties(TFileAttributes attributes, String separator,
99+
char quote, char escape, boolean trim, String lineDelimiter) {
100+
TFileTextScanRangeParams text = attributes.getTextParams();
101+
Assertions.assertEquals(separator, text.getColumnSeparator());
102+
Assertions.assertEquals(lineDelimiter, text.getLineDelimiter());
103+
Assertions.assertTrue(text.isSetEnclose());
104+
Assertions.assertEquals((byte) quote, text.getEnclose());
105+
Assertions.assertTrue(text.isSetEscape());
106+
Assertions.assertEquals((byte) escape, text.getEscape());
107+
Assertions.assertEquals("", text.getNullFormat());
108+
Assertions.assertEquals(trim, attributes.isTrimDoubleQuotes());
109+
}
110+
}

‎regression-test/suites/external_table_p0/hive/test_csv_table_properties.groovy‎

Lines changed: 43 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -35,7 +35,7 @@ suite("test_csv_table_properties", "p0,external,hive,external_docker,external_do
3535
hive_docker """
3636
INSERT INTO csv_table_properties_db.source_rows VALUES
3737
('1', 'Alpha', 'quiet sequoias, escape q and e', 1),
38-
('2', 'Omega', 'edge, requests', 4),
38+
('2', 'Omega', 'edge | requests, end', 4),
3939
('3', 'Beta', '"literal quotes", ss qq ee', 4),
4040
('4', 'Empty', '', 1)
4141
"""
@@ -66,13 +66,14 @@ suite("test_csv_table_properties", "p0,external,hive,external_docker,external_do
6666
]
6767
for (def layout : layouts) {
6868
hive_docker "DROP TABLE IF EXISTS csv_table_properties_db.${layout.table}"
69-
// Keep these settings ONLY in TBLPROPERTIES: putting them in SERDEPROPERTIES hides the bug.
69+
// Keep character settings ONLY in TBLPROPERTIES: SERDEPROPERTIES would hide the lookup bug.
70+
// Hive ignores table-level line.delim for this SerDe; it must not turn payload pipes into records.
7071
hive_docker """
7172
CREATE TABLE csv_table_properties_db.${layout.table} (${layout.columns})
7273
${layout.partitioned ? 'PARTITIONED BY (group_id INT)' : ''}
7374
ROW FORMAT SERDE 'org.apache.hadoop.hive.serde2.OpenCSVSerde'
7475
STORED AS TEXTFILE
75-
TBLPROPERTIES ('separatorChar'='s', 'quoteChar'='q', 'escapeChar'='e')
76+
TBLPROPERTIES ('separatorChar'='s', 'quoteChar'='q', 'escapeChar'='e', 'line.delim'='|')
7677
"""
7778
hive_docker """
7879
INSERT INTO csv_table_properties_db.${layout.table}
@@ -88,10 +89,45 @@ suite("test_csv_table_properties", "p0,external,hive,external_docker,external_do
8889
"SELECT ${layout.aggregate} FROM %s",
8990
"SELECT ${layout.projection} FROM %s WHERE label = 'Omega' ORDER BY label"
9091
]
91-
for (String query : queries) {
92-
def expected = hive_docker(String.format(query, "csv_table_properties_db.source_rows"))
93-
def actual = sql(String.format(query, layout.table))
94-
assertEquals(expected, actual)
92+
// The same files must remain readable after a metadata-only change to multi-character values:
93+
// OpenCSVSerde takes the first Java character of each property.
94+
for (boolean multiCharacter : [false, true]) {
95+
if (multiCharacter) {
96+
hive_docker """
97+
ALTER TABLE csv_table_properties_db.${layout.table} SET TBLPROPERTIES (
98+
'separatorChar'='ss', 'quoteChar'='qq', 'escapeChar'='ee'
99+
)
100+
"""
101+
sql "REFRESH DATABASE csv_table_properties_catalog.csv_table_properties_db"
102+
}
103+
for (String query : queries) {
104+
def expected = hive_docker(String.format(query, "csv_table_properties_db.source_rows"))
105+
def actual = sql(String.format(query, layout.table))
106+
assertEquals(expected, actual)
107+
}
95108
}
96109
}
110+
111+
// Hive accepts these characters, but Doris must reject them before truncating their UTF-8 bytes.
112+
// Empty values are covered by unit tests: Hive itself rejects them while validating ALTER TABLE.
113+
for (def invalid : [
114+
[key: "quoteChar", value: "é", message: "the first character must be ASCII"],
115+
[key: "escapeChar", value: "é", message: "the first character must be ASCII"]
116+
]) {
117+
hive_docker """
118+
ALTER TABLE csv_table_properties_db.csv_two_columns
119+
SET TBLPROPERTIES ('${invalid.key}'='${invalid.value}')
120+
"""
121+
sql "REFRESH DATABASE csv_table_properties_catalog.csv_table_properties_db"
122+
test {
123+
sql "SELECT label, payload FROM csv_two_columns ORDER BY label"
124+
exception "OpenCSVSerde property '${invalid.key}': ${invalid.message}"
125+
}
126+
hive_docker """
127+
ALTER TABLE csv_table_properties_db.csv_two_columns SET TBLPROPERTIES (
128+
'separatorChar'='s', 'quoteChar'='q', 'escapeChar'='e'
129+
)
130+
"""
131+
}
132+
sql "REFRESH DATABASE csv_table_properties_catalog.csv_table_properties_db"
97133
}

0 commit comments

Comments
 (0)