Skip to content

Commit 7b50e05

Browse files
branch-4.1: [fix](scan) Fix lost and duplicated rows when splitting CSV/JSON on multi-character line delimiters #68539 (#68600)
Cherry-picked from #68539 Co-authored-by: hui lai <[email protected]>
1 parent 5e1f8f4 commit 7b50e05

13 files changed

Lines changed: 598 additions & 18 deletions

File tree

‎be/src/format/csv/csv_reader.cpp‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -345,6 +345,14 @@ Status CsvReader::get_next_block(Block* block, size_t* read_rows, bool* eof) {
345345

346346
bool success = false;
347347
bool is_remove_bom = false;
348+
if (_range.start_offset != 0 && _skip_lines > 0 && _enclose == 0 &&
349+
_file_format_type == TFileFormatType::FORMAT_CSV_PLAIN) {
350+
auto* text_reader = assert_cast<NewPlainTextLineReader*>(_line_reader.get());
351+
RETURN_IF_ERROR(text_reader->skip_split_prefix(_range.start_offset, _line_delimiter,
352+
&_line_reader_eof, _io_ctx));
353+
_skip_lines = 0;
354+
is_remove_bom = true;
355+
}
348356
if (_push_down_agg_type == TPushAggOp::type::COUNT) {
349357
while (rows < batch_size && !_line_reader_eof) {
350358
const uint8_t* ptr = nullptr;

‎be/src/format/file_reader/new_plain_text_line_reader.cpp‎

Lines changed: 77 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,7 @@
2525
#include <immintrin.h>
2626
#endif
2727
#include <algorithm>
28+
#include <array>
2829
#include <cstddef>
2930
#include <cstring>
3031
#include <ostream>
@@ -287,6 +288,82 @@ inline bool NewPlainTextLineReader::update_eof() {
287288
return _eof;
288289
}
289290

291+
Status NewPlainTextLineReader::skip_split_prefix(size_t split_start, const std::string& delimiter,
292+
bool* eof, const io::IOContext* io_ctx,
293+
size_t* skipped_lines) {
294+
DCHECK_EQ(_total_read_bytes, 0);
295+
DCHECK_EQ(_output_buf_limit, 0);
296+
bool overlaps = false;
297+
if (delimiter.size() > 1) {
298+
// Compute the KMP prefix function once for this split in linear time. A nonempty
299+
// proper prefix that is also a suffix of the whole delimiter permits overlapping matches.
300+
std::vector<size_t> prefix_lengths(delimiter.size());
301+
for (size_t i = 1, matched = 0; i < delimiter.size(); ++i) {
302+
while (matched > 0 && delimiter[i] != delimiter[matched]) {
303+
matched = prefix_lengths[matched - 1];
304+
}
305+
if (delimiter[i] == delimiter[matched]) {
306+
++matched;
307+
}
308+
prefix_lengths[i] = matched;
309+
}
310+
overlaps = prefix_lengths.back() > 0;
311+
}
312+
313+
if (overlaps && _decompressor == nullptr) {
314+
// A fixed lookbehind can start in an overlapping delimiter chain (e.g. three newlines
315+
// with a two-newline delimiter). Find a byte that cannot belong to any delimiter, then
316+
// replay greedy matches from immediately after it. Keep scratch bounded even for long
317+
// delimiter runs; ordinary delimiters do not need this extra I/O.
318+
std::array<bool, 256> delimiter_bytes {};
319+
for (unsigned char byte : delimiter) {
320+
delimiter_bytes[byte] = true;
321+
}
322+
constexpr size_t max_lookbehind_size = 64 * 1024;
323+
std::vector<char> buffer(1024);
324+
size_t sync_offset = _current_offset;
325+
bool synchronized = false;
326+
while (sync_offset > 0 && !synchronized) {
327+
const size_t length = std::min(sync_offset, buffer.size());
328+
const size_t offset = sync_offset - length;
329+
size_t bytes_read = 0;
330+
RETURN_IF_ERROR(_file_reader->read_at(offset, Slice(buffer.data(), length), &bytes_read,
331+
io_ctx));
332+
if (bytes_read != length) {
333+
return Status::IOError("Short read while aligning text split at offset {}",
334+
split_start);
335+
}
336+
sync_offset = offset;
337+
for (size_t i = length; i > 0; --i) {
338+
if (!delimiter_bytes[static_cast<unsigned char>(buffer[i - 1])]) {
339+
sync_offset = offset + i;
340+
synchronized = true;
341+
break;
342+
}
343+
}
344+
if (!synchronized && sync_offset > 0) {
345+
// Extend backward into new bytes; do not reread the already searched suffix.
346+
buffer.resize(std::min(buffer.size() * 2, max_lookbehind_size));
347+
}
348+
}
349+
_min_length += _current_offset - sync_offset;
350+
_current_offset = sync_offset;
351+
}
352+
353+
const size_t prefix_length = split_start - _current_offset;
354+
const uint8_t* line = nullptr;
355+
size_t size = 0;
356+
size_t skipped = 0;
357+
do {
358+
RETURN_IF_ERROR(read_line(&line, &size, eof, io_ctx));
359+
skipped += !*eof;
360+
} while (!*eof && _total_read_bytes < prefix_length);
361+
if (skipped_lines != nullptr) {
362+
*skipped_lines = skipped;
363+
}
364+
return Status::OK();
365+
}
366+
290367
// extend input buf if necessary only when _more_input_bytes > 0
291368
void NewPlainTextLineReader::extend_input_buf() {
292369
DCHECK(_more_input_bytes > 0);

‎be/src/format/file_reader/new_plain_text_line_reader.h‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -248,6 +248,12 @@ class NewPlainTextLineReader : public LineReader {
248248
Status read_line(const uint8_t** ptr, size_t* size, bool* eof,
249249
const io::IOContext* io_ctx) override;
250250

251+
// Called before the first read of a non-first plain-text split. Discard records owned by the
252+
// preceding split, preserving greedy delimiter matching. Not suitable for enclosed CSV:
253+
// finding a delimiter synchronization point does not recover quote/escape state.
254+
Status skip_split_prefix(size_t split_start, const std::string& delimiter, bool* eof,
255+
const io::IOContext* io_ctx, size_t* skipped_lines = nullptr);
256+
251257
inline TextLineReaderCtxPtr text_line_reader_ctx() { return _line_reader_ctx; }
252258

253259
void close() override;

‎be/src/format/json/new_json_reader.cpp‎

Lines changed: 9 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -152,6 +152,8 @@ NewJsonReader::NewJsonReader(RuntimeProfile* profile, const TFileScanRangeParams
152152
_init_file_description();
153153
}
154154

155+
NewJsonReader::~NewJsonReader() = default;
156+
155157
void NewJsonReader::_init_system_properties() {
156158
if (_range.__isset.file_type) {
157159
// for compatibility
@@ -211,9 +213,8 @@ Status NewJsonReader::get_next_block(Block* block, size_t* read_rows, bool* eof)
211213

212214
while (block->rows() < batch_size && !_reader_eof && (block->bytes() < max_block_bytes)) {
213215
if (UNLIKELY(_read_json_by_line && _skip_first_line)) {
214-
size_t size = 0;
215-
const uint8_t* line_ptr = nullptr;
216-
RETURN_IF_ERROR(_line_reader->read_line(&line_ptr, &size, &_reader_eof, _io_ctx));
216+
RETURN_IF_ERROR(_line_reader->skip_split_prefix(_range.start_offset, _line_delimiter,
217+
&_reader_eof, _io_ctx));
217218
_skip_first_line = false;
218219
continue;
219220
}
@@ -445,7 +446,9 @@ void json_reader_detail::pop_back_last_inserted_value(Block& block, size_t colum
445446
Status NewJsonReader::_open_file_reader(bool need_schema) {
446447
int64_t start_offset = _range.start_offset;
447448
if (start_offset != 0) {
448-
start_offset -= 1;
449+
// Include the whole delimiter when the split starts inside it, so skipping the first
450+
// partial line cannot discard the next complete JSON record.
451+
start_offset -= std::min<int64_t>(start_offset, _line_delimiter_length);
449452
}
450453

451454
_current_offset = start_offset;
@@ -482,8 +485,8 @@ Status NewJsonReader::_open_file_reader(bool need_schema) {
482485
Status NewJsonReader::_open_line_reader() {
483486
int64_t size = _range.size;
484487
if (_range.start_offset != 0) {
485-
// When we fetch range doesn't start from 0, size will += 1.
486-
size += 1;
488+
// Preserve the original range end after moving the start backwards.
489+
size += _range.start_offset - _current_offset;
487490
_skip_first_line = true;
488491
} else {
489492
_skip_first_line = false;

‎be/src/format/json/new_json_reader.h‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -62,6 +62,7 @@ struct IOContext;
6262
struct ScannerCounter;
6363
class Block;
6464
class IColumn;
65+
class NewPlainTextLineReader;
6566

6667
namespace json_reader_detail {
6768
Status append_null_for_malformed_json(Block& block);
@@ -83,7 +84,7 @@ class NewJsonReader : public GenericReader {
8384
const TFileRangeDesc& range, const std::vector<SlotDescriptor*>& file_slot_descs,
8485
size_t batch_size, io::IOContext* io_ctx,
8586
std::shared_ptr<io::IOContext> io_ctx_holder = nullptr);
86-
~NewJsonReader() override = default;
87+
~NewJsonReader() override;
8788

8889
Status init_reader(
8990
const std::unordered_map<std::string, VExprContextSPtr>& col_default_value_ctx,
@@ -200,7 +201,7 @@ class NewJsonReader : public GenericReader {
200201
const std::vector<SlotDescriptor*>& _file_slot_descs;
201202

202203
io::FileReaderSPtr _file_reader;
203-
std::unique_ptr<LineReader> _line_reader;
204+
std::unique_ptr<NewPlainTextLineReader> _line_reader;
204205
bool _reader_eof;
205206
std::unique_ptr<Decompressor> _decompressor;
206207
TFileCompressType::type _file_compress_type;

‎be/src/format_v2/delimited_text/csv_reader.cpp‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -128,11 +128,13 @@ Status CsvReader::_create_decompressor() {
128128
}
129129

130130
Status CsvReader::_create_line_reader() {
131+
_align_split_prefix = false;
131132
if (is_csv_text_format(_file_format_type)) {
132133
std::shared_ptr<TextLineReaderContextIf> text_line_reader_ctx;
133134
if (_enclose == 0) {
134135
text_line_reader_ctx = std::make_shared<PlainTextLineReaderCtx>(
135136
_line_delimiter, _line_delimiter.size(), _keep_cr);
137+
_align_split_prefix = _file_description->range_start_offset != 0;
136138
} else {
137139
const size_t col_sep_num =
138140
_source_file_slot_descs.size() > 1 ? _source_file_slot_descs.size() - 1 : 0;

‎be/src/format_v2/delimited_text/delimited_text_reader.cpp‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,7 @@
3232
#include "core/data_type/data_type_nullable.h"
3333
#include "core/data_type/data_type_string.h"
3434
#include "core/data_type/data_type_struct.h"
35+
#include "format/file_reader/new_plain_text_line_reader.h"
3536
#include "format/line_reader.h"
3637
#include "format_v2/column_mapper.h"
3738
#include "format_v2/materialized_reader_util.h"
@@ -555,6 +556,19 @@ Status DelimitedTextReader::_open_file() {
555556
Status DelimitedTextReader::_read_next_line(Slice* line, bool* eof) {
556557
DORIS_CHECK(line != nullptr);
557558
DORIS_CHECK(eof != nullptr);
559+
if (_align_split_prefix) {
560+
SCOPED_TIMER(_text_profile.read_line_time);
561+
DCHECK_EQ(_skip_lines, 1);
562+
size_t skipped_lines = 0;
563+
auto* text_reader = assert_cast<NewPlainTextLineReader*>(_line_reader.get());
564+
RETURN_IF_ERROR(text_reader->skip_split_prefix(_file_description->range_start_offset,
565+
_line_delimiter, &_line_reader_eof,
566+
_io_ctx.get(), &skipped_lines));
567+
_align_split_prefix = false;
568+
_skip_lines = 0;
569+
_bom_removed = true;
570+
update_counter(_text_profile.skipped_lines, skipped_lines);
571+
}
558572
while (true) {
559573
const uint8_t* ptr = nullptr;
560574
size_t size = 0;

‎be/src/format_v2/delimited_text/delimited_text_reader.h‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -156,6 +156,8 @@ class DelimitedTextReader : public FileReader {
156156
int64_t _start_offset = 0;
157157
int64_t _size = -1;
158158
int _skip_lines = 0;
159+
// Enabled only by readers using plain, quote-independent delimiter matching.
160+
bool _align_split_prefix = false;
159161
char _escape = 0;
160162
bool _line_reader_eof = false;
161163
bool _bom_removed = false;

‎be/src/format_v2/json/json_reader.cpp‎

Lines changed: 7 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -314,10 +314,8 @@ Status JsonReader::get_block(Block* file_block, size_t* rows, bool* eof) {
314314
while (file_block->rows() < batch_size && !_reader_eof &&
315315
file_block->bytes() < max_block_bytes) {
316316
if (_read_json_by_line && _skip_first_line) {
317-
size_t skipped_size = 0;
318-
const uint8_t* skipped_line = nullptr;
319-
RETURN_IF_ERROR(_line_reader->read_line(&skipped_line, &skipped_size, &_reader_eof,
320-
_io_ctx.get()));
317+
RETURN_IF_ERROR(_line_reader->skip_split_prefix(
318+
_reader_range.start_offset, _line_delimiter, &_reader_eof, _io_ctx.get()));
321319
_skip_first_line = false;
322320
continue;
323321
}
@@ -445,7 +443,9 @@ TFileRangeDesc JsonReader::_json_range() const {
445443
Status JsonReader::_open_file_reader() {
446444
_current_offset = _reader_range.start_offset;
447445
if (_current_offset != 0) {
448-
--_current_offset;
446+
// Include the whole delimiter when the split starts inside it, so skipping the first
447+
// partial line cannot discard the next complete JSON record.
448+
_current_offset -= std::min<int64_t>(_current_offset, _line_delimiter_length);
449449
}
450450
if (_scan_params->file_type == TFileType::FILE_STREAM) {
451451
if (!_stream_load_id.has_value()) {
@@ -478,9 +478,8 @@ Status JsonReader::_create_decompressor() {
478478
Status JsonReader::_create_line_reader() {
479479
int64_t size = _reader_range.size;
480480
if (_reader_range.start_offset != 0) {
481-
// Start one byte earlier and discard the first partial line, matching split semantics used
482-
// by text readers.
483-
++size;
481+
// Preserve the original range end after moving the start backwards.
482+
size += _reader_range.start_offset - _current_offset;
484483
_skip_first_line = true;
485484
} else {
486485
_skip_first_line = false;

‎be/src/format_v2/json/json_reader.h‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -35,7 +35,7 @@
3535

3636
namespace doris {
3737
class Decompressor;
38-
class LineReader;
38+
class NewPlainTextLineReader;
3939
class SlotDescriptor;
4040
class IColumn;
4141
} // namespace doris
@@ -188,7 +188,7 @@ class JsonReader final : public FileReader {
188188

189189
io::FileReaderSPtr _physical_file_reader;
190190
std::unique_ptr<Decompressor> _decompressor;
191-
std::unique_ptr<LineReader> _line_reader;
191+
std::unique_ptr<NewPlainTextLineReader> _line_reader;
192192
int64_t _current_offset = 0;
193193
bool _reader_eof = false;
194194
bool _skip_first_line = false;

0 commit comments

Comments
 (0)