Skip to content

Commit 0be70b5

Browse files
committed
[fix](be) Handle CSV separator boundary cases
### What problem does this PR solve? Issue Number: None Related PR: #65501 Problem Summary: CSV V2 could record column separators that overlap line delimiter bytes or re-record an accepted multi-character separator after an output-buffer refill, leading to invalid split offsets and BE checks. The local TVF regression suite also skipped fixture copies for single-BE environments. Limit separator searches to the returned logical line, clamp refill backtracking after the last accepted separator, copy fixtures for every backend, and add focused unit coverage. ### Release note Prevent CSV V2 failures when column separators overlap line delimiters or cross line-reader refill boundaries. ### Check List (For Author) - Test: Unit Test - CsvV2ReaderTest: 34 tests passed - Behavior changed: Yes. CSV V2 no longer records separator offsets outside the returned logical line or duplicates accepted separators across refills. - Does this need documentation: No
1 parent 7d2dd9d commit 0be70b5

3 files changed

Lines changed: 96 additions & 10 deletions

File tree

‎be/src/format/file_reader/new_plain_text_line_reader.cpp‎

Lines changed: 8 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -67,7 +67,10 @@ const uint8_t* EncloseCsvLineReaderCtx::read_line_impl(const uint8_t* start, con
6767
// causing parse column separator error.
6868
if (_state.curr_state == ReaderState::NORMAL ||
6969
_state.curr_state == ReaderState::MATCH_ENCLOSE) {
70-
_idx -= std::min(_column_sep_len - 1, _idx);
70+
const size_t last_column_sep_end =
71+
_column_sep_positions.empty() ? 0 : _column_sep_positions.back() + _column_sep_len;
72+
DORIS_CHECK_LE(last_column_sep_end, _idx);
73+
_idx -= std::min(_column_sep_len - 1, _idx - last_column_sep_end);
7174
}
7275
_total_len = length;
7376
size_t bound = update_reading_bound(start);
@@ -151,7 +154,8 @@ void EncloseCsvLineReaderCtx::_on_start(const uint8_t* start, size_t& len) {
151154

152155
void EncloseCsvLineReaderCtx::_on_normal(const uint8_t* start, size_t& len) {
153156
const uint8_t* curr_start = start + _idx;
154-
size_t curr_len = len - _idx;
157+
const size_t field_search_bound = _result == nullptr ? len : _result - start;
158+
const size_t curr_len = field_search_bound - _idx;
155159
const uint8_t* col_sep_pos =
156160
find_col_sep_func(curr_start, curr_len, _column_sep.c_str(), _column_sep_len);
157161

@@ -207,7 +211,8 @@ void EncloseCsvLineReaderCtx::_on_pre_match_enclose(const uint8_t* start, size_t
207211

208212
void EncloseCsvLineReaderCtx::_on_match_enclose(const uint8_t* start, size_t& len) {
209213
const uint8_t* curr_start = start + _idx;
210-
size_t curr_len = len - _idx;
214+
const size_t field_search_bound = _result == nullptr ? len : _result - start;
215+
const size_t curr_len = field_search_bound - _idx;
211216
const uint8_t* delim_pos =
212217
find_col_sep_func(curr_start, curr_len, _column_sep.c_str(), _column_sep_len);
213218

‎be/test/format_v2/delimited_text/csv_reader_test.cpp‎

Lines changed: 84 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,7 @@
2424
#include <fstream>
2525
#include <memory>
2626

27+
#include "common/config.h"
2728
#include "common/consts.h"
2829
#include "common/object_pool.h"
2930
#include "core/assert_cast.h"
@@ -45,6 +46,8 @@
4546
#include "runtime/runtime_profile.h"
4647
#include "testutil/desc_tbl_builder.h"
4748
#include "testutil/mock/mock_runtime_state.h"
49+
#include "util/debug_points.h"
50+
#include "util/defer_op.h"
4851

4952
namespace doris::format::csv {
5053
namespace {
@@ -847,6 +850,87 @@ TEST_F(CsvV2ReaderTest, MatchingEscapeAndEncloseStillSplitQuotedFields) {
847850
EXPECT_EQ(nullable_int_at(*block.get_by_position(2).column, 0), 10);
848851
}
849852

853+
// A column separator that overlaps the line delimiter must not be recorded because CsvReader
854+
// splits a Slice that excludes the line delimiter bytes.
855+
TEST_F(CsvV2ReaderTest, ColumnSeparatorOverlappingLineDelimiterStaysOutsideReturnedLine) {
856+
const auto overlap_path = (_test_dir / "overlapping_delimiters.csv").string();
857+
std::ofstream output(overlap_path, std::ios::binary);
858+
output << "value|\n";
859+
output.close();
860+
861+
auto value_slot = make_test_slot(&_pool, 10, 0,
862+
make_nullable(std::make_shared<DataTypeString>()), "value");
863+
std::vector<SlotDescriptor*> slots {value_slot};
864+
_params.__set_column_idxs({0});
865+
_params.file_attributes.__isset.header_type = false;
866+
_params.file_attributes.text_params.__set_column_separator("|\n");
867+
_params.file_attributes.text_params.__set_enclose('"');
868+
_params.file_attributes.text_params.__set_escape('\\');
869+
auto reader = create_reader(overlap_path, &_params, slots, &_state, &_profile);
870+
std::vector<ColumnDefinition> schema;
871+
ASSERT_TRUE(reader->get_schema(&schema).ok());
872+
873+
auto request = std::make_shared<FileScanRequest>();
874+
request->non_predicate_columns = {LocalColumnIndex::top_level(LocalColumnId(0))};
875+
request->local_positions.emplace(LocalColumnId(0), LocalIndex(0));
876+
ASSERT_TRUE(reader->open(request).ok());
877+
878+
auto block = make_block(schema, {0});
879+
size_t rows = 0;
880+
bool eof = false;
881+
ASSERT_TRUE(reader->get_block(&block, &rows, &eof).ok());
882+
ASSERT_EQ(rows, 1);
883+
EXPECT_EQ(nullable_string_at(*block.get_by_position(0).column, 0), "value|");
884+
}
885+
886+
// When a logical row is refilled, partial-separator backtracking must not cross the end of a
887+
// separator that was already accepted from the previous buffer contents.
888+
TEST_F(CsvV2ReaderTest, MultiCharacterSeparatorAcrossOutputBufferRefill) {
889+
const auto refill_path = (_test_dir / "separator_refill.csv").string();
890+
std::ofstream output(refill_path, std::ios::binary);
891+
output << std::string(28, 'a') << "|||||b\n";
892+
output.close();
893+
894+
auto first_slot = make_test_slot(&_pool, 10, 0,
895+
make_nullable(std::make_shared<DataTypeString>()), "first");
896+
auto second_slot = make_test_slot(&_pool, 11, 1,
897+
make_nullable(std::make_shared<DataTypeString>()), "second");
898+
std::vector<SlotDescriptor*> slots {first_slot, second_slot};
899+
_params.__set_column_idxs({0, 1});
900+
_params.file_attributes.__isset.header_type = false;
901+
_params.file_attributes.text_params.__set_column_separator("|||");
902+
_params.file_attributes.text_params.__set_enclose('"');
903+
_params.file_attributes.text_params.__set_escape('\\');
904+
905+
const bool enable_debug_points = config::enable_debug_points;
906+
config::enable_debug_points = true;
907+
DebugPoints::instance()->add_with_params("NewPlainTextLineReader.shrink_output_buf",
908+
{{"output_buf_size", "32"}});
909+
Defer restore_debug_point([enable_debug_points]() {
910+
DebugPoints::instance()->remove("NewPlainTextLineReader.shrink_output_buf");
911+
config::enable_debug_points = enable_debug_points;
912+
});
913+
914+
auto reader = create_reader(refill_path, &_params, slots, &_state, &_profile);
915+
std::vector<ColumnDefinition> schema;
916+
ASSERT_TRUE(reader->get_schema(&schema).ok());
917+
918+
auto request = std::make_shared<FileScanRequest>();
919+
request->non_predicate_columns = {LocalColumnIndex::top_level(LocalColumnId(0)),
920+
LocalColumnIndex::top_level(LocalColumnId(1))};
921+
request->local_positions.emplace(LocalColumnId(0), LocalIndex(0));
922+
request->local_positions.emplace(LocalColumnId(1), LocalIndex(1));
923+
ASSERT_TRUE(reader->open(request).ok());
924+
925+
auto block = make_block(schema, {0, 1});
926+
size_t rows = 0;
927+
bool eof = false;
928+
ASSERT_TRUE(reader->get_block(&block, &rows, &eof).ok());
929+
ASSERT_EQ(rows, 1);
930+
EXPECT_EQ(nullable_string_at(*block.get_by_position(0).column, 0), std::string(28, 'a'));
931+
EXPECT_EQ(nullable_string_at(*block.get_by_position(1).column, 0), "||b");
932+
}
933+
850934
// Scenario: when the CSV row has fewer fields than the FE-provided file slot list, v2 fills the
851935
// missing requested field with NULL instead of failing or shifting later columns.
852936
TEST_F(CsvV2ReaderTest, MissingRequestedFieldUsesNullFormat) {

‎regression-test/suites/external_table_p0/tvf/test_local_tvf_csv_enclose_consistency.groovy‎

Lines changed: 4 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -20,19 +20,16 @@ suite("test_local_tvf_csv_enclose_consistency", "p0,external") {
2020
assertTrue(backends.size() > 0)
2121
def beId = backends[0][0]
2222
def dataPath = context.config.dataPath + "/external_table_p0/tvf"
23-
def filePath = dataPath
23+
def filePath = "/"
2424

2525
def files = [
2626
"csv_enclose_state.csv",
2727
"csv_matching_escape_enclose.csv",
2828
"csv_quoted_null.csv"
2929
]
30-
if (backends.size() > 1) {
31-
filePath = "/"
32-
for (def backend : backends) {
33-
for (def file : files) {
34-
scpFiles("root", backend[1], "${dataPath}/${file}", filePath, false)
35-
}
30+
for (def backend : backends) {
31+
for (def file : files) {
32+
scpFiles("root", backend[1], "${dataPath}/${file}", filePath, false)
3633
}
3734
}
3835

0 commit comments

Comments
 (0)