|
24 | 24 | #include <fstream> |
25 | 25 | #include <memory> |
26 | 26 |
|
| 27 | +#include "common/config.h" |
27 | 28 | #include "common/consts.h" |
28 | 29 | #include "common/object_pool.h" |
29 | 30 | #include "core/assert_cast.h" |
|
45 | 46 | #include "runtime/runtime_profile.h" |
46 | 47 | #include "testutil/desc_tbl_builder.h" |
47 | 48 | #include "testutil/mock/mock_runtime_state.h" |
| 49 | +#include "util/debug_points.h" |
| 50 | +#include "util/defer_op.h" |
48 | 51 |
|
49 | 52 | namespace doris::format::csv { |
50 | 53 | namespace { |
@@ -847,6 +850,87 @@ TEST_F(CsvV2ReaderTest, MatchingEscapeAndEncloseStillSplitQuotedFields) { |
847 | 850 | EXPECT_EQ(nullable_int_at(*block.get_by_position(2).column, 0), 10); |
848 | 851 | } |
849 | 852 |
|
| 853 | +// A column separator that overlaps the line delimiter must not be recorded because CsvReader |
| 854 | +// splits a Slice that excludes the line delimiter bytes. |
| 855 | +TEST_F(CsvV2ReaderTest, ColumnSeparatorOverlappingLineDelimiterStaysOutsideReturnedLine) { |
| 856 | + const auto overlap_path = (_test_dir / "overlapping_delimiters.csv").string(); |
| 857 | + std::ofstream output(overlap_path, std::ios::binary); |
| 858 | + output << "value|\n"; |
| 859 | + output.close(); |
| 860 | + |
| 861 | + auto value_slot = make_test_slot(&_pool, 10, 0, |
| 862 | + make_nullable(std::make_shared<DataTypeString>()), "value"); |
| 863 | + std::vector<SlotDescriptor*> slots {value_slot}; |
| 864 | + _params.__set_column_idxs({0}); |
| 865 | + _params.file_attributes.__isset.header_type = false; |
| 866 | + _params.file_attributes.text_params.__set_column_separator("|\n"); |
| 867 | + _params.file_attributes.text_params.__set_enclose('"'); |
| 868 | + _params.file_attributes.text_params.__set_escape('\\'); |
| 869 | + auto reader = create_reader(overlap_path, &_params, slots, &_state, &_profile); |
| 870 | + std::vector<ColumnDefinition> schema; |
| 871 | + ASSERT_TRUE(reader->get_schema(&schema).ok()); |
| 872 | + |
| 873 | + auto request = std::make_shared<FileScanRequest>(); |
| 874 | + request->non_predicate_columns = {LocalColumnIndex::top_level(LocalColumnId(0))}; |
| 875 | + request->local_positions.emplace(LocalColumnId(0), LocalIndex(0)); |
| 876 | + ASSERT_TRUE(reader->open(request).ok()); |
| 877 | + |
| 878 | + auto block = make_block(schema, {0}); |
| 879 | + size_t rows = 0; |
| 880 | + bool eof = false; |
| 881 | + ASSERT_TRUE(reader->get_block(&block, &rows, &eof).ok()); |
| 882 | + ASSERT_EQ(rows, 1); |
| 883 | + EXPECT_EQ(nullable_string_at(*block.get_by_position(0).column, 0), "value|"); |
| 884 | +} |
| 885 | + |
| 886 | +// When a logical row is refilled, partial-separator backtracking must not cross the end of a |
| 887 | +// separator that was already accepted from the previous buffer contents. |
| 888 | +TEST_F(CsvV2ReaderTest, MultiCharacterSeparatorAcrossOutputBufferRefill) { |
| 889 | + const auto refill_path = (_test_dir / "separator_refill.csv").string(); |
| 890 | + std::ofstream output(refill_path, std::ios::binary); |
| 891 | + output << std::string(28, 'a') << "|||||b\n"; |
| 892 | + output.close(); |
| 893 | + |
| 894 | + auto first_slot = make_test_slot(&_pool, 10, 0, |
| 895 | + make_nullable(std::make_shared<DataTypeString>()), "first"); |
| 896 | + auto second_slot = make_test_slot(&_pool, 11, 1, |
| 897 | + make_nullable(std::make_shared<DataTypeString>()), "second"); |
| 898 | + std::vector<SlotDescriptor*> slots {first_slot, second_slot}; |
| 899 | + _params.__set_column_idxs({0, 1}); |
| 900 | + _params.file_attributes.__isset.header_type = false; |
| 901 | + _params.file_attributes.text_params.__set_column_separator("|||"); |
| 902 | + _params.file_attributes.text_params.__set_enclose('"'); |
| 903 | + _params.file_attributes.text_params.__set_escape('\\'); |
| 904 | + |
| 905 | + const bool enable_debug_points = config::enable_debug_points; |
| 906 | + config::enable_debug_points = true; |
| 907 | + DebugPoints::instance()->add_with_params("NewPlainTextLineReader.shrink_output_buf", |
| 908 | + {{"output_buf_size", "32"}}); |
| 909 | + Defer restore_debug_point([enable_debug_points]() { |
| 910 | + DebugPoints::instance()->remove("NewPlainTextLineReader.shrink_output_buf"); |
| 911 | + config::enable_debug_points = enable_debug_points; |
| 912 | + }); |
| 913 | + |
| 914 | + auto reader = create_reader(refill_path, &_params, slots, &_state, &_profile); |
| 915 | + std::vector<ColumnDefinition> schema; |
| 916 | + ASSERT_TRUE(reader->get_schema(&schema).ok()); |
| 917 | + |
| 918 | + auto request = std::make_shared<FileScanRequest>(); |
| 919 | + request->non_predicate_columns = {LocalColumnIndex::top_level(LocalColumnId(0)), |
| 920 | + LocalColumnIndex::top_level(LocalColumnId(1))}; |
| 921 | + request->local_positions.emplace(LocalColumnId(0), LocalIndex(0)); |
| 922 | + request->local_positions.emplace(LocalColumnId(1), LocalIndex(1)); |
| 923 | + ASSERT_TRUE(reader->open(request).ok()); |
| 924 | + |
| 925 | + auto block = make_block(schema, {0, 1}); |
| 926 | + size_t rows = 0; |
| 927 | + bool eof = false; |
| 928 | + ASSERT_TRUE(reader->get_block(&block, &rows, &eof).ok()); |
| 929 | + ASSERT_EQ(rows, 1); |
| 930 | + EXPECT_EQ(nullable_string_at(*block.get_by_position(0).column, 0), std::string(28, 'a')); |
| 931 | + EXPECT_EQ(nullable_string_at(*block.get_by_position(1).column, 0), "||b"); |
| 932 | +} |
| 933 | + |
850 | 934 | // Scenario: when the CSV row has fewer fields than the FE-provided file slot list, v2 fills the |
851 | 935 | // missing requested field with NULL instead of failing or shifting later columns. |
852 | 936 | TEST_F(CsvV2ReaderTest, MissingRequestedFieldUsesNullFormat) { |
|
0 commit comments