Skip to content

Commit 0d3907c

Browse files
authored
branch-3.0:[fix](core) normalize DeleteSubPredicatePB operator on read path #62052 (#61961)
Problem Summary: pick: #62052 ## compaction core ``` #0 0x0000000000000000 in ?? () #1 0x00005561c333fa74 in doris::parse_to_predicate (column=..., index=6, condition=..., arena=0x7ff95ae1b710, opposite=true) at /root/be/src/olap/predicate_creator.h:329 #2 0x00005561c333f351 in doris::DeleteHandler::_parse_column_pred<doris::DeleteSubPredicatePB> (this=this@entry=0x7ff89bc0bb08, complete_schema=..., delete_pred_related_schema=..., sub_pred_list=..., delete_conditions=delete_conditions@entry=0x7ff89bc0b330) at /root/be/src/olap/delete_handler.cpp:375 #3 0x00005561c333c57f in doris::DeleteHandler::init (this=0x7ff89bc0bb08, tablet_schema=..., delete_preds=..., version=15) at /root/be/src/olap/delete_handler.cpp:410 #4 0x00005561c3c504d7 in doris::TabletReader::_init_delete_condition (this=this@entry=0x7ff89bc0b848, read_params=...) at /root/be/src/olap/tablet_reader.cpp:637 #5 0x00005561c3c4e1fb in doris::TabletReader::_init_params (this=this@entry=0x7ff89bc0b848, read_params=...) at /root/be/src/olap/tablet_reader.cpp:297 #6 0x00005561c3c4dda6 in doris::TabletReader::init (this=0x7ff89bc0b848, read_params=...) at /root/be/src/olap/tablet_reader.cpp:126 #7 0x00005561cd62a27a in doris::vectorized::VerticalBlockReader::init (this=0x7ff89bc0b848, read_params=..., sample_info=0x7ff89bc0c5e0) at /root/be/src/vec/olap/vertical_block_reader.cpp:224 #8 0x00005561c3ac6f0f in doris::Merger::vertical_compact_one_group (tablet=..., reader_type=reader_type@entry=doris::ReaderType::READER_BASE_COMPACTION, tablet_schema=..., is_key=true, column_group=..., row_source_buf=0x7ff89bc0c590, src_rowset_readers=..., dst_rowset_writer=0x7ff928685500, max_rows_per_segment=460833, stats_output=0x7ff89bc0cb70, key_group_cluster_key_idxes=..., batch_size=992, sample_info=0x7ff89bc0c5e0) at /root/be/src/olap/merger.cpp:252 #9 0x00005561c3ac8e02 in doris::Merger::vertical_merge_rowsets (tablet=..., reader_type=<optimized out>, tablet_schema=..., src_rowset_readers=..., dst_rowset_writer=0x7ff928685500, max_rows_per_segment=460833, merge_way_num=1, stats_output=0x7ff89bc0cb70) at /root/be/src/olap/merger.cpp:427 #10 0x00005561c3aacade in doris::Compaction::do_compaction_impl (this=this@entry=0x7ff92e8a9290, permits=permits@entry=3) at /root/be/src/olap/compaction.cpp:392 #11 0x00005561c3aabcac in doris::Compaction::do_compaction (this=0x7ff92e8a9290, permits=3) at /root/be/src/olap/compaction.cpp:140 #12 0x00005561c3c07d92 in doris::BaseCompaction::execute_compact_impl (this=0x7ff92e8a9290) at /root/be/src/olap/base_compaction.cpp:79 #13 0x00005561c3aab8d3 in doris::Compaction::execute_compact (this=0x6) at /root/be/src/olap/compaction.cpp:122 #14 0x00005561c3bddf1b in doris::Tablet::execute_compaction(doris::Compaction&)::$_0::operator()() const (this=<optimized out>) at /root/be/src/olap/tablet.cpp:2093 #15 doris::Tablet::execute_compaction (this=0x7ff8e54f5610, compaction=...) at /root/be/src/olap/tablet.cpp:2093 #16 0x00005561c3a7e336 in doris::StorageEngine::_submit_compaction_task(std::shared_ptr<doris::Tablet>, doris::CompactionType, bool)::$_0::operator()() const (this=0x7ff92bc351c0) at /root/be/src/olap/olap_server.cpp:1068 #17 std::__invoke_impl<void, doris::StorageEngine::_submit_compaction_task(std::shared_ptr<doris::Tablet>, doris::CompactionType, bool)::$_0&>(std::__invoke_other, doris::StorageEngine::_submit_compaction_task(std::shared_ptr<doris::Tablet>, doris::CompactionType, bool)::$_0&) (__f=...) at /var/local/ldb-toolchain/bin/../lib/gcc/x86_64-linux-gnu/11/../../../../include/c++/11/bits/invoke.h:61 #18 std::__invoke_r<void, doris::StorageEngine::_submit_compaction_task(std::shared_ptr<doris::Tablet>, doris::CompactionType, bool)::$_0&>(doris::StorageEngine::_submit_compaction_task(std::shared_ptr<doris::Tablet>, doris::CompactionType, bool)::$_0&) (__fn=...) at /var/local/ldb-toolchain/bin/../lib/gcc/x86_64-linux-gnu/11/../../../../include/c++/11/bits/invoke.h:111 #19 std::_Function_handler<void (), doris::StorageEngine::_submit_compaction_task(std::shared_ptr<doris::Tablet>, doris::CompactionType, bool)::$_0>::_M_invoke(std::_Any_data const&) (__functor=...) at /var/local/ldb-toolchain/bin/../lib/gcc/x86_64-linux-gnu/11/../../../../include/c++/11/bits/std_function.h:291 #20 0x00005561c4109fb8 in doris::ThreadPool::dispatch_thread (this=0x7ff92b66da00) at /root/be/src/util/threadpool.cpp:544 #21 0x00005561c40ff341 in std::function<void ()>::operator()() const (this=0x7ff9e3e65e10) at /var/local/ldb-toolchain/bin/../lib/gcc/x86_64-linux-gnu/11/../../../../include/c++/11/bits/std_function.h:560 #22 doris::Thread::supervise_thread (arg=0x7ff9dd49b860) at /root/be/src/util/thread.cpp:498 #23 0x00007ffa0a7a2215 in start_thread () from /lib64/libc.so.6 #24 0x00007ffa0a824bdc in clone3 () from /lib64/libc.so.6 ``` ## query core ``` #0 0x0000000000000000 in ?? () #1 0x000055c0b109da74 in doris::parse_to_predicate (column=..., index=6, condition=..., arena=0x7f8357530270, opposite=true) at /root/be/src/olap/predicate_creator.h:329 #2 0x000055c0b109d351 in doris::DeleteHandler::_parse_column_pred<doris::DeleteSubPredicatePB> (this=this@entry=0x7f837ab178c0, complete_schema=..., delete_pred_related_schema=..., sub_pred_list=..., delete_conditions=delete_conditions@entry=0x7f7f53dc5b00) at /root/be/src/olap/delete_handler.cpp:375 #3 0x000055c0b109a57f in doris::DeleteHandler::init (this=0x7f837ab178c0, tablet_schema=..., delete_preds=..., version=16) at /root/be/src/olap/delete_handler.cpp:410 #4 0x000055c0b19ae4d7 in doris::TabletReader::_init_delete_condition (this=this@entry=0x7f837ab17600, read_params=...) at /root/be/src/olap/tablet_reader.cpp:637 #5 0x000055c0b19ac1fb in doris::TabletReader::_init_params (this=this@entry=0x7f837ab17600, read_params=...) at /root/be/src/olap/tablet_reader.cpp:297 #6 0x000055c0b19abda6 in doris::TabletReader::init (this=0x7f837ab17600, read_params=...) at /root/be/src/olap/tablet_reader.cpp:126 #7 0x000055c0bb36da9f in doris::vectorized::BlockReader::init (this=0x6, read_params=...) at /root/be/src/vec/olap/block_reader.cpp:201 #8 0x000055c0b61ddffc in doris::vectorized::NewOlapScanner::open (this=0x7f7fac09a810, state=<optimized out>) at /root/be/src/vec/exec/scan/new_olap_scanner.cpp:228 #9 0x000055c0b61eb5e0 in doris::vectorized::ScannerScheduler::_scanner_scan (ctx=..., scan_task=...) at /root/be/src/vec/exec/scan/scanner_scheduler.cpp:249 #10 0x000055c0b61ec2cd in doris::vectorized::ScannerScheduler::submit(std::shared_ptr<doris::vectorized::ScannerContext>, std::shared_ptr<doris::vectorized::ScanTask>)::$_1::operator()() const::{lambda()#1}::operator()() const::{lambda()#1}::operator()() const (this=<optimized out>) at /root/be/src/vec/exec/scan/scanner_scheduler.cpp:170 #11 doris::vectorized::ScannerScheduler::submit(std::shared_ptr<doris::vectorized::ScannerContext>, std::shared_ptr<doris::vectorized::ScanTask>)::$_1::operator()() const::{lambda()#1}::operator()() const (this=0x7f802ccdef20) at /root/be/src/vec/exec/scan/scanner_scheduler.cpp:169 #12 std::__invoke_impl<void, doris::vectorized::ScannerScheduler::submit(std::shared_ptr<doris::vectorized::ScannerContext>, std::shared_ptr<doris::vectorized::ScanTask>)::$_1::operator()() const::{lambda()#1}&>(std::__invoke_other, doris::vectorized::ScannerScheduler::submit(std::shared_ptr<doris::vectorized::ScannerContext>, std::shared_ptr<doris::vectorized::ScanTask>)::$_1::operator()() const::{lambda()#1}&) (__f=...) at /var/local/ldb-toolchain/bin/../lib/gcc/x86_64-linux-gnu/11/../../../../include/c++/11/bits/invoke.h:61 #13 std::__invoke_r<void, doris::vectorized::ScannerScheduler::submit(std::shared_ptr<doris::vectorized::ScannerContext>, std::shared_ptr<doris::vectorized::ScanTask>)::$_1::operator()() const::{lambda()#1}&>(doris::vectorized::ScannerScheduler::submit(std::shared_ptr<doris::vectorized::ScannerContext>, std::shared_ptr<doris::vectorized::ScanTask>)::$_1::operator()() const::{lambda()#1}&) (__fn=...) at /var/local/ldb-toolchain/bin/../lib/gcc/x86_64-linux-gnu/11/../../../../include/c++/11/bits/invoke.h:111 #14 std::_Function_handler<void (), doris::vectorized::ScannerScheduler::submit(std::shared_ptr<doris::vectorized::ScannerContext>, std::shared_ptr<doris::vectorized::ScanTask>)::$_1::operator()() const::{lambda()#1}>::_M_invoke(std::_Any_data const&) (__functor=...) at /var/local/ldb-toolchain/bin/../lib/gcc/x86_64-linux-gnu/11/../../../../include/c++/11/bits/std_function.h:291 #15 0x000055c0b1e67fb8 in doris::ThreadPool::dispatch_thread (this=0x7f8368f68200) at /root/be/src/util/threadpool.cpp:544 #16 0x000055c0b1e5d341 in std::function<void ()>::operator()() const (this=0x7f837ab06f90) at /var/local/ldb-toolchain/bin/../lib/gcc/x86_64-linux-gnu/11/../../../../include/c++/11/bits/std_function.h:560 #17 doris::Thread::supervise_thread (arg=0x7f8247529280) at /root/be/src/util/thread.cpp:498 #18 0x00007f837d5a2215 in start_thread () from /lib64/libc.so.6 #19 0x00007f837d624bdc in clone3 () from /lib64/libc.so.6 ``` ### Summary 1. This PR fixes a BE coredump caused by legacy DeleteSubPredicatePB.op encoding during delete predicate parsing 2. The issue affects both query scan and compaction paths and is rooted in inconsistent operator encoding in historical rowset metadata ### Root Cause 1. DeleteSubPredicatePB read path did not normalize op before predicate creation 2. Historical metadata may contain legacy operators such as - < - *= 3. But predicate creation logic expects normalized internal operators - << - = 4. When condition_op is not recognized, parse_to_predicate() leaves creator function pointer null and then calls it, causing coredump ### Temporary Contingency Plan 1. set skip_delete_predicate = true; 2. disable compaction ### Reproduction Steps 1. set delete_without_partition = true 2. DELETE FROM ${tableName} WHERE k1 <=> 1 3. Because the metadata contains the `<=>` operator, but BE does not implement the corresponding logic ### Release note None ### Check List (For Author) - Test <!-- At least one of them must be included. --> - [ ] Regression test - [x] Unit Test - [ ] Manual test (add detailed scripts or steps below) - [ ] No need to test or manual test. Explain why: - [ ] This is a refactor/code format and no logic has been changed. - [ ] Previous test can cover this change. - [ ] No code files have been changed. - [ ] Other reason <!-- Add your reason? --> - Behavior changed: - [ ] No. - [ ] Yes. <!-- Explain the behavior change --> - Does this need documentation? - [x] No. - [ ] Yes. <!-- Add document PR link here. eg: apache/doris-website#1214 --> ### Check List (For Reviewer who merge this PR) - [ ] Confirm the release note - [ ] Confirm test cases - [ ] Confirm document - [ ] Add branch pick label <!-- Add branch pick label that this PR should merge into -->
1 parent f635ac3 commit 0d3907c

3 files changed

Lines changed: 63 additions & 2 deletions

File tree

‎be/src/olap/delete_handler.cpp‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -67,7 +67,7 @@ std::string construct_sub_predicate(const TCondition& condition) {
6767
}
6868

6969
// make operators from FE adaptive to BE
70-
std::string trans_op(const std::string& opt) {
70+
std::string trans_op(const std::string& opt, const std::string& condition_value = "") {
7171
std::string op = string(opt);
7272
if (op == "<") {
7373
op += "<";
@@ -79,6 +79,8 @@ std::string trans_op(const std::string& opt) {
7979
op = "=";
8080
} else if (op == "!*=") {
8181
op = "!=";
82+
} else if (op == "<=>") {
83+
op = iequal(condition_value, "NULL") ? "IS" : "=";
8284
}
8385
}
8486
return op;
@@ -296,7 +298,7 @@ Status DeleteHandler::parse_condition(const DeleteSubPredicatePB& sub_cond, TCon
296298
condition->column_unique_id = sub_cond.column_unique_id();
297299
}
298300
condition->column_name = sub_cond.column_name();
299-
condition->condition_op = sub_cond.op();
301+
condition->condition_op = trans_op(sub_cond.op(), sub_cond.cond_value());
300302
condition->condition_values.push_back(sub_cond.cond_value());
301303
return Status::OK();
302304
}
Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
-- This file is automatically generated. You should know what you did if you want to edit this
2+
-- !after_delete --
3+
2 20
4+
Lines changed: 55 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,55 @@
1+
// Licensed to the Apache Software Foundation (ASF) under one
2+
// or more contributor license agreements. See the NOTICE file
3+
// distributed with this work for additional information
4+
// regarding copyright ownership. The ASF licenses this file
5+
// to you under the Apache License, Version 2.0 (the
6+
// "License"); you may not use this file except in compliance
7+
// with the License. You may obtain a copy of the License at
8+
//
9+
// http://www.apache.org/licenses/LICENSE-2.0
10+
//
11+
// Unless required by applicable law or agreed to in writing,
12+
// software distributed under the License is distributed on an
13+
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
// KIND, either express or implied. See the License for the
15+
// specific language governing permissions and limitations
16+
// under the License.
17+
18+
suite("test_delete_nullsafe_eq", "delete_p0") {
19+
def dbName = "repro_db_delete_nullsafe_eq"
20+
def tableName = "t_core"
21+
22+
sql """ DROP DATABASE IF EXISTS ${dbName} """
23+
sql """ CREATE DATABASE ${dbName} """
24+
sql """ USE ${dbName} """
25+
26+
try {
27+
sql """
28+
CREATE TABLE ${tableName} (
29+
k1 INT,
30+
v1 INT
31+
)
32+
DUPLICATE KEY(k1)
33+
PARTITION BY RANGE(k1) (
34+
PARTITION p1 VALUES LESS THAN ("100")
35+
)
36+
DISTRIBUTED BY HASH(k1) BUCKETS 1
37+
PROPERTIES(
38+
"replication_num" = "1",
39+
"disable_auto_compaction" = "true"
40+
)
41+
"""
42+
43+
sql """ INSERT INTO ${tableName}(k1, v1) VALUES (1, 10) """
44+
sql """ INSERT INTO ${tableName}(k1, v1) VALUES (2, 20) """
45+
46+
sql """ set delete_without_partition = true """
47+
sql """ DELETE FROM ${tableName} WHERE k1 <=> 1 """
48+
sql """ sync """
49+
50+
qt_after_delete """ SELECT * FROM ${tableName} ORDER BY k1 """
51+
} finally {
52+
sql """ set delete_without_partition = false """
53+
sql """ DROP DATABASE IF EXISTS ${dbName} """
54+
}
55+
}

0 commit comments

Comments
 (0)