|
| 1 | +// Licensed to the Apache Software Foundation (ASF) under one |
| 2 | +// or more contributor license agreements. See the NOTICE file |
| 3 | +// distributed with this work for additional information |
| 4 | +// regarding copyright ownership. The ASF licenses this file |
| 5 | +// to you under the Apache License, Version 2.0 (the |
| 6 | +// "License"); you may not use this file except in compliance |
| 7 | +// with the License. You may obtain a copy of the License at |
| 8 | +// |
| 9 | +// http://www.apache.org/licenses/LICENSE-2.0 |
| 10 | +// |
| 11 | +// Unless required by applicable law or agreed to in writing, |
| 12 | +// software distributed under the License is distributed on an |
| 13 | +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
| 14 | +// KIND, either express or implied. See the License for the |
| 15 | +// specific language governing permissions and limitations |
| 16 | +// under the License. |
| 17 | + |
| 18 | +package org.apache.doris.datasource.scan; |
| 19 | + |
| 20 | +import org.apache.doris.common.jmockit.Deencapsulation; |
| 21 | +import org.apache.doris.connector.hive.HiveTextProperties; |
| 22 | +import org.apache.doris.connector.spi.DorisConnectorException; |
| 23 | +import org.apache.doris.qe.SessionVariable; |
| 24 | +import org.apache.doris.thrift.TFileAttributes; |
| 25 | +import org.apache.doris.thrift.TFileTextScanRangeParams; |
| 26 | + |
| 27 | +import org.apache.thrift.TDeserializer; |
| 28 | +import org.apache.thrift.TSerializer; |
| 29 | +import org.apache.thrift.protocol.TCompactProtocol; |
| 30 | +import org.junit.jupiter.api.Assertions; |
| 31 | +import org.junit.jupiter.api.Test; |
| 32 | +import org.mockito.Mockito; |
| 33 | + |
| 34 | +import java.util.Map; |
| 35 | + |
| 36 | +class PluginDrivenScanNodeCsvPropertiesTest { |
| 37 | + private static final String CSV_SERDE = "org.apache.hadoop.hive.serde2.OpenCSVSerde"; |
| 38 | + |
| 39 | + @Test |
| 40 | + void testNormalizedCharactersSurviveScanNodeAndThrift() throws Exception { |
| 41 | + Map<String, String> properties = Map.of("separatorChar", "ss", "quoteChar", "qq", "escapeChar", "ee", |
| 42 | + "line.delim", "|"); |
| 43 | + assertWireProperties(toWire(Map.of(), properties), "s", 'q', 'e', false); |
| 44 | + assertWireProperties(toWire(properties, Map.of("line.delim", "|")), "s", 'q', 'e', false, "|"); |
| 45 | + } |
| 46 | + |
| 47 | + @Test |
| 48 | + void testEffectiveTablePropertiesOverrideInvalidSerdeProperties() throws Exception { |
| 49 | + assertWireProperties(toWire(Map.of("separatorChar", "", "quoteChar", "é", "escapeChar", ""), |
| 50 | + Map.of("separatorChar", "ss", "quoteChar", "qq", "escapeChar", "ee")), |
| 51 | + "s", 'q', 'e', false); |
| 52 | + } |
| 53 | + |
| 54 | + @Test |
| 55 | + void testDefaultsAndNormalizedDoubleQuote() throws Exception { |
| 56 | + assertWireProperties(toWire(Map.of(), Map.of()), ",", '"', '\\', true); |
| 57 | + assertWireProperties(toWire(Map.of(), |
| 58 | + Map.of("separatorChar", "99", "quoteChar", "\"suffix", "escapeChar", "eé")), |
| 59 | + "9", '"', 'e', true); |
| 60 | + } |
| 61 | + |
| 62 | + @Test |
| 63 | + void testUtf8SeparatorRemainsACompleteCharacter() throws Exception { |
| 64 | + assertWireProperties(toWire(Map.of(), Map.of("separatorChar", "ésuffix")), "é", '"', '\\', true); |
| 65 | + } |
| 66 | + |
| 67 | + @Test |
| 68 | + void testInvalidCharactersFailBeforeThriftConstruction() { |
| 69 | + for (String key : new String[] {"separatorChar", "quoteChar", "escapeChar"}) { |
| 70 | + Assertions.assertThrows(DorisConnectorException.class, () -> toWire(Map.of(), Map.of(key, ""))); |
| 71 | + } |
| 72 | + for (String key : new String[] {"quoteChar", "escapeChar"}) { |
| 73 | + Assertions.assertThrows(DorisConnectorException.class, () -> toWire(Map.of(), Map.of(key, "é"))); |
| 74 | + } |
| 75 | + Assertions.assertThrows(DorisConnectorException.class, |
| 76 | + () -> toWire(Map.of(), Map.of("separatorChar", "😀"))); |
| 77 | + } |
| 78 | + |
| 79 | + private static TFileAttributes toWire(Map<String, String> serdeProperties, |
| 80 | + Map<String, String> tableProperties) throws Exception { |
| 81 | + PluginDrivenScanNode node = Mockito.mock(PluginDrivenScanNode.class, Mockito.CALLS_REAL_METHODS); |
| 82 | + Deencapsulation.setField(node, "scanNodeProperties", |
| 83 | + HiveTextProperties.extract(CSV_SERDE, serdeProperties, tableProperties)); |
| 84 | + Deencapsulation.setField(node, "sessionVariable", new SessionVariable()); |
| 85 | + // Exercise the real scan-node consumer and wire encoding: map assertions cannot detect byte truncation. |
| 86 | + TFileAttributes attributes = node.getFileAttributes(); |
| 87 | + TFileAttributes decoded = new TFileAttributes(); |
| 88 | + new TDeserializer(new TCompactProtocol.Factory()).deserialize(decoded, |
| 89 | + new TSerializer(new TCompactProtocol.Factory()).serialize(attributes)); |
| 90 | + return decoded; |
| 91 | + } |
| 92 | + |
| 93 | + private static void assertWireProperties(TFileAttributes attributes, String separator, |
| 94 | + char quote, char escape, boolean trim) { |
| 95 | + assertWireProperties(attributes, separator, quote, escape, trim, "\n"); |
| 96 | + } |
| 97 | + |
| 98 | + private static void assertWireProperties(TFileAttributes attributes, String separator, |
| 99 | + char quote, char escape, boolean trim, String lineDelimiter) { |
| 100 | + TFileTextScanRangeParams text = attributes.getTextParams(); |
| 101 | + Assertions.assertEquals(separator, text.getColumnSeparator()); |
| 102 | + Assertions.assertEquals(lineDelimiter, text.getLineDelimiter()); |
| 103 | + Assertions.assertTrue(text.isSetEnclose()); |
| 104 | + Assertions.assertEquals((byte) quote, text.getEnclose()); |
| 105 | + Assertions.assertTrue(text.isSetEscape()); |
| 106 | + Assertions.assertEquals((byte) escape, text.getEscape()); |
| 107 | + Assertions.assertEquals("", text.getNullFormat()); |
| 108 | + Assertions.assertEquals(trim, attributes.isTrimDoubleQuotes()); |
| 109 | + } |
| 110 | +} |
0 commit comments