|
| 1 | +// Licensed to the Apache Software Foundation (ASF) under one |
| 2 | +// or more contributor license agreements. See the NOTICE file |
| 3 | +// distributed with this work for additional information |
| 4 | +// regarding copyright ownership. The ASF licenses this file |
| 5 | +// to you under the Apache License, Version 2.0 (the |
| 6 | +// "License"); you may not use this file except in compliance |
| 7 | +// with the License. You may obtain a copy of the License at |
| 8 | +// |
| 9 | +// http://www.apache.org/licenses/LICENSE-2.0 |
| 10 | +// |
| 11 | +// Unless required by applicable law or agreed to in writing, |
| 12 | +// software distributed under the License is distributed on an |
| 13 | +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
| 14 | +// KIND, either express or implied. See the License for the |
| 15 | +// specific language governing permissions and limitations |
| 16 | +// under the License. |
| 17 | + |
| 18 | +package org.apache.doris.connector.hive; |
| 19 | + |
| 20 | +import org.apache.doris.connector.spi.DorisConnectorException; |
| 21 | +import org.apache.doris.connector.spi.scan.ScanNodePropertyKeys; |
| 22 | + |
| 23 | +import au.com.bytecode.opencsv.CSVReader; |
| 24 | +import org.apache.hadoop.hive.serde2.OpenCSVSerde; |
| 25 | +import org.apache.hadoop.hive.serde2.SerDeException; |
| 26 | +import org.apache.hadoop.io.Text; |
| 27 | +import org.junit.jupiter.api.Assertions; |
| 28 | +import org.junit.jupiter.api.Test; |
| 29 | + |
| 30 | +import java.io.StringReader; |
| 31 | +import java.util.Arrays; |
| 32 | +import java.util.List; |
| 33 | +import java.util.Map; |
| 34 | +import java.util.Properties; |
| 35 | + |
| 36 | +class HiveCsvReaderCompatibilityTest { |
| 37 | + @Test |
| 38 | + void testExplicitDefaultEscapeReadsLikeHive() throws Exception { |
| 39 | + for (String quote : new String[] {"\"", "q"}) { |
| 40 | + String record = quote + "left\\" + quote + "middle" + quote + ",tail"; |
| 41 | + Map<String, String> properties = Map.of("quoteChar", quote, "escapeChar", "\"suffix"); |
| 42 | + List<?> expected = (List<?>) hiveSerde(properties).deserialize(new Text(record)); |
| 43 | + Assertions.assertEquals(List.of("left" + quote + "middle", "tail"), expected); |
| 44 | + for (Map<String, String> extracted : List.of( |
| 45 | + HiveTextProperties.extract(HiveTextProperties.HIVE_OPEN_CSV_SERDE, Map.of(), properties), |
| 46 | + HiveTextProperties.extract(HiveTextProperties.HIVE_OPEN_CSV_SERDE, properties, Map.of()))) { |
| 47 | + Assertions.assertEquals("\\", extracted.get(ScanNodePropertyKeys.TEXT_ESCAPE)); |
| 48 | + try (CSVReader reader = new CSVReader(new StringReader(record), |
| 49 | + extracted.get(ScanNodePropertyKeys.TEXT_COLUMN_SEPARATOR).charAt(0), |
| 50 | + extracted.get(ScanNodePropertyKeys.TEXT_ENCLOSE).charAt(0), |
| 51 | + extracted.get(ScanNodePropertyKeys.TEXT_ESCAPE).charAt(0))) { |
| 52 | + Assertions.assertEquals(expected, Arrays.asList(reader.readNext())); |
| 53 | + } |
| 54 | + } |
| 55 | + } |
| 56 | + } |
| 57 | + |
| 58 | + @Test |
| 59 | + void testCharacterTuplesMatchHiveReaderAcceptance() throws Exception { |
| 60 | + // Use Hive's real reader as the oracle, including its writer-default escape sentinel and NUL rules. |
| 61 | + char[] characters = {'\0', ',', '|', '"', '\\', 'q', 'e'}; |
| 62 | + for (char separator : characters) { |
| 63 | + for (char quote : characters) { |
| 64 | + for (char escape : characters) { |
| 65 | + Map<String, String> properties = Map.of("separatorChar", String.valueOf(separator), |
| 66 | + "quoteChar", String.valueOf(quote), "escapeChar", String.valueOf(escape)); |
| 67 | + OpenCSVSerde serde = hiveSerde(properties); |
| 68 | + boolean accepted; |
| 69 | + try { |
| 70 | + serde.deserialize(new Text("")); |
| 71 | + accepted = true; |
| 72 | + } catch (SerDeException e) { |
| 73 | + Assertions.assertInstanceOf(UnsupportedOperationException.class, e.getCause()); |
| 74 | + accepted = false; |
| 75 | + } |
| 76 | + assertAcceptance(accepted, Map.of(), properties); |
| 77 | + assertAcceptance(accepted, properties, Map.of()); |
| 78 | + assertAcceptance(accepted, Map.of("quoteChar", String.valueOf(quote)), |
| 79 | + Map.of("separatorChar", String.valueOf(separator), "escapeChar", String.valueOf(escape))); |
| 80 | + } |
| 81 | + } |
| 82 | + } |
| 83 | + } |
| 84 | + |
| 85 | + private static void assertAcceptance(boolean accepted, Map<String, String> serdeProperties, |
| 86 | + Map<String, String> tableProperties) { |
| 87 | + if (accepted) { |
| 88 | + Assertions.assertDoesNotThrow(() -> HiveTextProperties.extract( |
| 89 | + HiveTextProperties.HIVE_OPEN_CSV_SERDE, serdeProperties, tableProperties)); |
| 90 | + } else { |
| 91 | + Assertions.assertThrows(DorisConnectorException.class, () -> HiveTextProperties.extract( |
| 92 | + HiveTextProperties.HIVE_OPEN_CSV_SERDE, serdeProperties, tableProperties)); |
| 93 | + } |
| 94 | + } |
| 95 | + |
| 96 | + private static OpenCSVSerde hiveSerde(Map<String, String> properties) throws SerDeException { |
| 97 | + Properties serdeProperties = new Properties(); |
| 98 | + serdeProperties.setProperty("columns", "first,second"); |
| 99 | + serdeProperties.setProperty("columns.types", "string:string"); |
| 100 | + serdeProperties.putAll(properties); |
| 101 | + OpenCSVSerde serde = new OpenCSVSerde(); |
| 102 | + serde.initialize(null, serdeProperties); |
| 103 | + return serde; |
| 104 | + } |
| 105 | +} |
0 commit comments