Skip to content

Commit 192131b

Browse files
committed
[fix](orc) Address ORC reader review comments
1 parent 16a9a90 commit 192131b

4 files changed

Lines changed: 223 additions & 90 deletions

File tree

‎be/src/format_v2/expr/delete_predicate.cpp‎

Lines changed: 2 additions & 35 deletions
Original file line numberDiff line numberDiff line change
@@ -30,8 +30,6 @@
3030
#include "core/block/column_numbers.h"
3131
#include "core/block/column_with_type_and_name.h"
3232
#include "core/block/columns_with_type_and_name.h"
33-
#include "core/column/column_nullable.h"
34-
#include "core/column/column_vector.h"
3533

3634
namespace doris::format {
3735

@@ -82,27 +80,9 @@ Status DeletePredicate::execute(VExprContext* context, Block* block, int* result
8280
"DeletePredicate row id child returned invalid column id {}, block has {} columns",
8381
slot, block->columns());
8482
}
85-
const auto& row_id_column = *block->get_by_position(slot).column;
86-
const ColumnInt64* row_id_data_column = nullptr;
87-
const NullMap* null_map = nullptr;
88-
if (const auto* nullable_column = check_and_get_column<ColumnNullable>(row_id_column)) {
89-
row_id_data_column =
90-
check_and_get_column<ColumnInt64>(nullable_column->get_nested_column());
91-
null_map = &nullable_column->get_null_map_data();
92-
} else {
93-
row_id_data_column = check_and_get_column<ColumnInt64>(row_id_column);
94-
}
95-
if (row_id_data_column == nullptr) {
96-
return Status::InternalError("DeletePredicate row id child must be Int64, but got {}",
97-
row_id_column.get_name());
98-
}
99-
const auto& row_ids = row_id_data_column->get_data();
83+
const auto& row_ids =
84+
assert_cast<const ColumnInt64&>(*block->get_by_position(slot).column).get_data();
10085
const auto count = row_ids.size();
101-
if (null_map != nullptr && null_map->size() != count) {
102-
return Status::InternalError(
103-
"DeletePredicate nullable row id column has {} values but {} null flags", count,
104-
null_map->size());
105-
}
10686
auto res_col = ColumnBool::create(count, 0);
10787
if (_deleted_rows.empty()) {
10888
block->insert({std::move(res_col), std::make_shared<DataTypeBool>(), expr_name()});
@@ -114,19 +94,6 @@ Status DeletePredicate::execute(VExprContext* context, Block* block, int* result
11494
*result_column_id = static_cast<int>(block->get_columns().size() - 1);
11595
return Status::OK();
11696
}
117-
if (null_map != nullptr && std::any_of(null_map->begin(), null_map->end(),
118-
[](UInt8 is_null) { return is_null != 0; })) {
119-
auto& result_data = res_col->get_data();
120-
for (size_t row = 0; row < count; ++row) {
121-
if ((*null_map)[row] == 0 &&
122-
std::binary_search(_deleted_rows.begin(), _deleted_rows.end(), row_ids[row])) {
123-
result_data[row] = true;
124-
}
125-
}
126-
block->insert({std::move(res_col), std::make_shared<DataTypeBool>(), expr_name()});
127-
*result_column_id = static_cast<int>(block->get_columns().size() - 1);
128-
return Status::OK();
129-
}
13097
const int64_t* delete_rows = _deleted_rows.data();
13198
const int64_t* delete_rows_end = delete_rows + _deleted_rows.size();
13299
const int64_t* start_pos = std::lower_bound(delete_rows, delete_rows_end, row_ids[0]);

‎be/src/format_v2/orc/orc_reader.cpp‎

Lines changed: 46 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,7 @@
2828
#include <cmath>
2929
#include <cstdint>
3030
#include <cstring>
31+
#include <limits>
3132
#include <list>
3233
#include <map>
3334
#include <memory>
@@ -112,6 +113,10 @@ uint64_t orc_read_row_count(const ::orc::ReaderMetrics& metrics) {
112113
return OrcReadRowCountMetric<::orc::ReaderMetrics>::value(metrics);
113114
}
114115

116+
bool is_orc_stop(const io::IOContext* io_ctx, const std::exception& e) {
117+
return io_ctx != nullptr && io_ctx->should_stop && std::string_view(e.what()) == "stop";
118+
}
119+
115120
bool is_hour_offset_timezone(std::string_view timezone) {
116121
return timezone.size() == 6 && (timezone[0] == '+' || timezone[0] == '-') &&
117122
std::isdigit(static_cast<unsigned char>(timezone[1])) &&
@@ -161,6 +166,9 @@ class DorisOrcInputStream final : public ::orc::InputStream {
161166
uint64_t bytes_read = 0;
162167
auto* out = static_cast<uint8_t*>(buf);
163168
while (bytes_read < length) {
169+
if (_io_ctx != nullptr && _io_ctx->should_stop) {
170+
throw ::orc::ParseError("stop");
171+
}
164172
size_t loop_read = 0;
165173
Status st = _file_reader->read_at(
166174
static_cast<size_t>(offset + bytes_read),
@@ -1126,9 +1134,7 @@ void OrcReader::_collect_profile() const {
11261134
}
11271135

11281136
format::ColumnDefinition OrcReader::row_position_column_definition() {
1129-
auto field = format::row_position_column_definition();
1130-
field.type = make_nullable(field.type);
1131-
return field;
1137+
return format::row_position_column_definition();
11321138
}
11331139

11341140
Status OrcReader::init(RuntimeState* state) {
@@ -1150,6 +1156,9 @@ Status OrcReader::init(RuntimeState* state) {
11501156
_state->reader = ::orc::createReader(std::move(input_stream), options);
11511157
_state->root_type = &_state->reader->getType();
11521158
} catch (const std::exception& e) {
1159+
if (is_orc_stop(_io_ctx.get(), e)) {
1160+
return Status::EndOfFile("stop");
1161+
}
11531162
return Status::InternalError("Failed to open ORC file {}: {}", _file_description->path,
11541163
e.what());
11551164
}
@@ -1723,7 +1732,10 @@ Status OrcReader::_create_row_reader() {
17231732
_state->orc_lazy_selection_valid = false;
17241733
_state->orc_lazy_selected_rows.clear();
17251734
_state->orc_lazy_input_rows = 0;
1726-
_state->orc_lazy_next_batch_first_row = 0;
1735+
const auto initial_row_number = _state->row_reader->getRowNumber();
1736+
_state->orc_lazy_next_batch_first_row =
1737+
initial_row_number == std::numeric_limits<uint64_t>::max() ? 0
1738+
: initial_row_number + 1;
17271739
_state->column_to_selected_batch_index.clear();
17281740
size_t physical_read_column_count = 0;
17291741
for (const auto file_column_id : _state->read_columns) {
@@ -1767,18 +1779,28 @@ Status OrcReader::_filter_orc_batch(::orc::ColumnVectorBatch& data, uint16_t* se
17671779
return Status::InternalError("New ORC lazy filter expects struct row batch");
17681780
}
17691781

1782+
std::vector<std::optional<format::LocalColumnId>> position_to_column(
1783+
_request->local_positions.size());
1784+
for (const auto& [file_column_id, local_position] : _request->local_positions) {
1785+
const auto position = local_position.value();
1786+
if (position >= position_to_column.size()) {
1787+
return Status::InvalidArgument("ORC scan local position {} is out of range {}",
1788+
position, position_to_column.size());
1789+
}
1790+
if (position_to_column[position].has_value()) {
1791+
return Status::InvalidArgument("ORC scan local position {} is duplicated", position);
1792+
}
1793+
position_to_column[position] = file_column_id;
1794+
}
1795+
17701796
Block filter_block;
1771-
filter_block.reserve(_request->local_positions.size());
1772-
for (size_t position = 0; position < _request->local_positions.size(); ++position) {
1773-
const auto local_position = format::LocalIndex(position);
1774-
const auto entry_it = std::ranges::find_if(
1775-
_request->local_positions,
1776-
[&](const auto& entry) { return entry.second == local_position; });
1777-
if (entry_it == _request->local_positions.end()) {
1797+
filter_block.reserve(position_to_column.size());
1798+
for (size_t position = 0; position < position_to_column.size(); ++position) {
1799+
if (!position_to_column[position].has_value()) {
17781800
return Status::InvalidArgument("ORC scan local positions are not dense at {}",
17791801
position);
17801802
}
1781-
const auto file_column_id = entry_it->first;
1803+
const auto file_column_id = *position_to_column[position];
17821804
if (is_row_position_column(file_column_id)) {
17831805
auto field = row_position_column_definition();
17841806
filter_block.insert({field.type->create_column(), field.type, field.name});
@@ -2053,6 +2075,11 @@ Status OrcReader::get_block(Block* file_block, size_t* rows, bool* eof) {
20532075
}
20542076
*rows = 0;
20552077
file_block->clear_column_data(file_block->columns());
2078+
if (_io_ctx != nullptr && _io_ctx->should_stop) {
2079+
*eof = true;
2080+
_eof = true;
2081+
return Status::OK();
2082+
}
20562083
if (_eof) {
20572084
*eof = true;
20582085
return Status::OK();
@@ -2067,9 +2094,15 @@ Status OrcReader::get_block(Block* file_block, size_t* rows, bool* eof) {
20672094
_state->orc_lazy_selection_valid = false;
20682095
_state->orc_lazy_selected_rows.clear();
20692096
_state->orc_lazy_input_rows = 0;
2070-
_state->orc_lazy_next_batch_first_row = _state->row_reader->getRowNumber() + 1;
20712097
has_next = _state->row_reader->next(*_state->batch);
20722098
} catch (const std::exception& e) {
2099+
if (is_orc_stop(_io_ctx.get(), e)) {
2100+
file_block->clear_column_data(file_block->columns());
2101+
*rows = 0;
2102+
*eof = true;
2103+
_eof = true;
2104+
return Status::OK();
2105+
}
20732106
return Status::InternalError("Failed to read ORC batch: {}", e.what());
20742107
}
20752108
if (has_next) {

‎be/test/format_v2/expr/delete_predicate_test.cpp‎

Lines changed: 0 additions & 39 deletions
Original file line numberDiff line numberDiff line change
@@ -25,9 +25,7 @@
2525

2626
#include "common/status.h"
2727
#include "core/block/block.h"
28-
#include "core/column/column_nullable.h"
2928
#include "core/column/column_vector.h"
30-
#include "core/data_type/data_type_nullable.h"
3129
#include "core/data_type/data_type_number.h"
3230
#include "exprs/vexpr_context.h"
3331
#include "runtime/descriptors.h"
@@ -48,21 +46,6 @@ class DeletePredicateTest : public testing::Test {
4846
return block;
4947
}
5048

51-
static Block make_nullable_block(const std::vector<int64_t>& row_ids,
52-
const std::vector<UInt8>& nulls) {
53-
auto column = ColumnInt64::create();
54-
auto null_map = ColumnUInt8::create();
55-
for (size_t row = 0; row < row_ids.size(); ++row) {
56-
column->insert_value(row_ids[row]);
57-
null_map->insert_value(nulls[row]);
58-
}
59-
60-
Block block;
61-
block.insert({ColumnNullable::create(std::move(column), std::move(null_map)),
62-
make_nullable(std::make_shared<DataTypeInt64>()), "row_id"});
63-
return block;
64-
}
65-
6649
static std::vector<UInt8> result_column_data(const Block& block, int result_column_id) {
6750
const auto& result_column =
6851
assert_cast<const ColumnBool&>(*block.get_by_position(result_column_id).column);
@@ -116,28 +99,6 @@ TEST_F(DeletePredicateTest, DeletedRowsOutsideInputRangeReturnAllFalse) {
11699
EXPECT_EQ(result_column_data(block, result_column_id), std::vector<UInt8>({0, 0, 0}));
117100
}
118101

119-
TEST_F(DeletePredicateTest, NullableRowIdColumnMatchesDeletedRows) {
120-
const std::vector<int64_t> deleted_rows {1, 3};
121-
auto block = make_nullable_block({0, 1, 2, 3, 4}, {0, 0, 0, 0, 0});
122-
123-
int result_column_id = -1;
124-
auto status = execute_delete_predicate(deleted_rows, &block, &result_column_id);
125-
ASSERT_TRUE(status.ok()) << status;
126-
127-
EXPECT_EQ(result_column_data(block, result_column_id), std::vector<UInt8>({0, 1, 0, 1, 0}));
128-
}
129-
130-
TEST_F(DeletePredicateTest, NullableRowIdColumnIgnoresNullRows) {
131-
const std::vector<int64_t> deleted_rows {1, 2, 4};
132-
auto block = make_nullable_block({0, 1, 2, 3, 4}, {0, 1, 0, 0, 0});
133-
134-
int result_column_id = -1;
135-
auto status = execute_delete_predicate(deleted_rows, &block, &result_column_id);
136-
ASSERT_TRUE(status.ok()) << status;
137-
138-
EXPECT_EQ(result_column_data(block, result_column_id), std::vector<UInt8>({0, 0, 1, 0, 1}));
139-
}
140-
141102
TEST_F(DeletePredicateTest, EmptyRowIdColumnAppendsEmptyResultColumn) {
142103
const std::vector<int64_t> deleted_rows {1, 2, 3};
143104
auto block = make_block({});

0 commit comments

Comments
 (0)