Skip to content

Commit fc34411

Browse files
authored
branch-4.1.4: [fix](expr) Canonicalize logical OR results (#68401)
### What problem does this PR solve? This backports #68308 to branch-4.1.4.
1 parent 3661553 commit fc34411

5 files changed

Lines changed: 345 additions & 3 deletions

File tree

‎be/src/exprs/function/functions_logical.h‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -84,7 +84,9 @@ struct AndImpl {
8484
struct OrImpl {
8585
using ResultType = UInt8;
8686

87-
static inline constexpr ResultType apply(UInt8 a, UInt8 b) { return a | b; }
87+
// A NULL row may carry an arbitrary nested byte. Canonicalize logical OR to 0 or 1 so that
88+
// such a byte cannot become visible when the other operand makes the result non-NULL.
89+
static inline constexpr ResultType apply(UInt8 a, UInt8 b) { return (a | b) != 0; }
8890
static inline constexpr ResultType apply_null(UInt8 a, UInt8 l_null, UInt8 b, UInt8 r_null) {
8991
// (<> || true) is true, (false || NULL) is NULL
9092
return (l_null & r_null) | (r_null & (r_null ^ a)) | (l_null & (l_null ^ b));

‎be/src/exprs/vcompound_pred.h‎

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -610,7 +610,8 @@ class VCompoundPred : public VectorizedFnCall {
610610
if constexpr (is_and) {
611611
lhs[i] &= rhs[i];
612612
} else {
613-
lhs[i] |= rhs[i];
613+
// Logical OR must produce a canonical Boolean instead of preserving input bits.
614+
lhs[i] = (lhs[i] | rhs[i]) != 0;
614615
}
615616
}
616617
}
@@ -633,7 +634,9 @@ class VCompoundPred : public VectorizedFnCall {
633634
res_data[i] = lhs_data[i] & rhs_data[i];
634635
} else {
635636
res_null[i] = apply_or_null(lhs_data[i], lhs_null[i], rhs_data[i], rhs_null[i]);
636-
res_data[i] = lhs_data[i] | rhs_data[i];
637+
// A NULL row may carry an arbitrary nested byte. If the result remains NULL the
638+
// byte is ignored; otherwise normalization prevents it from becoming visible.
639+
res_data[i] = (lhs_data[i] | rhs_data[i]) != 0;
637640
}
638641
}
639642
}
Lines changed: 143 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,143 @@
1+
// Licensed to the Apache Software Foundation (ASF) under one
2+
// or more contributor license agreements. See the NOTICE file
3+
// distributed with this work for additional information
4+
// regarding copyright ownership. The ASF licenses this file
5+
// to you under the Apache License, Version 2.0 (the
6+
// "License"); you may not use this file except in compliance
7+
// with the License. You may obtain a copy of the License at
8+
//
9+
// http://www.apache.org/licenses/LICENSE-2.0
10+
//
11+
// Unless required by applicable law or agreed to in writing,
12+
// software distributed under the License is distributed on an
13+
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
// KIND, either express or implied. See the License for the
15+
// specific language governing permissions and limitations
16+
// under the License.
17+
18+
#include "exprs/function/functions_logical.h"
19+
20+
#include <gen_cpp/Exprs_types.h>
21+
#include <gtest/gtest.h>
22+
23+
#include <string>
24+
#include <utility>
25+
#include <vector>
26+
27+
#include "core/block/block.h"
28+
#include "core/column/column_const.h"
29+
#include "core/column/column_nullable.h"
30+
#include "core/data_type/data_type_nullable.h"
31+
#include "core/data_type/data_type_number.h"
32+
#include "core/data_type/primitive_type.h"
33+
#include "exprs/vcompound_pred.h"
34+
#include "exprs/vexpr_context.h"
35+
#include "testutil/column_helper.h"
36+
37+
namespace doris {
38+
39+
namespace {
40+
41+
ColumnWithTypeAndName nullable_boolean_column(std::vector<UInt8> data, std::vector<UInt8> null_map,
42+
std::string name) {
43+
return {ColumnHelper::create_nullable_column<DataTypeUInt8>(data, null_map),
44+
make_nullable(std::make_shared<DataTypeUInt8>()), std::move(name)};
45+
}
46+
47+
ColumnPtr execute_or(ColumnWithTypeAndName left, ColumnWithTypeAndName right, size_t rows) {
48+
auto result_type = make_nullable(std::make_shared<DataTypeUInt8>());
49+
Block block({std::move(left), std::move(right), {nullptr, result_type, "result"}});
50+
auto status = FunctionOr::create()->execute_impl(nullptr, block, {0, 1}, 2, rows);
51+
EXPECT_TRUE(status.ok()) << status.to_string();
52+
return block.get_by_position(2).column;
53+
}
54+
55+
class ColumnExpr final : public VExpr {
56+
public:
57+
ColumnExpr(ColumnPtr column, DataTypePtr type)
58+
: VExpr(std::move(type), false), _column(std::move(column)) {}
59+
60+
bool is_constant() const override { return false; }
61+
62+
const std::string& expr_name() const override {
63+
static const std::string name = "ColumnExpr";
64+
return name;
65+
}
66+
67+
Status execute_column_impl(VExprContext*, const Block*, const Selector*, size_t,
68+
ColumnPtr& result_column) const override {
69+
result_column = _column;
70+
return Status::OK();
71+
}
72+
73+
private:
74+
ColumnPtr _column;
75+
};
76+
77+
ColumnPtr execute_compound_or(ColumnWithTypeAndName left, ColumnWithTypeAndName right,
78+
size_t rows) {
79+
TExprNode node;
80+
node.__set_type(create_type_desc(TYPE_BOOLEAN));
81+
node.__set_node_type(TExprNodeType::COMPOUND_PRED);
82+
node.__set_opcode(TExprOpcode::COMPOUND_OR);
83+
node.__set_num_children(2);
84+
node.__set_is_nullable(true);
85+
86+
auto compound = VCompoundPred::create_shared(node);
87+
compound->add_child(std::make_shared<ColumnExpr>(std::move(left.column), left.type));
88+
compound->add_child(std::make_shared<ColumnExpr>(std::move(right.column), right.type));
89+
VExprContext context(compound);
90+
ColumnPtr result;
91+
auto status = compound->execute_column(&context, nullptr, nullptr, rows, result);
92+
EXPECT_TRUE(status.ok()) << status.to_string();
93+
return result;
94+
}
95+
96+
void expect_boolean(const ColumnNullable& result, size_t row, UInt8 value) {
97+
EXPECT_FALSE(result.is_null_at(row));
98+
EXPECT_EQ(assert_cast<const ColumnUInt8&>(result.get_nested_column()).get_data()[row], value);
99+
}
100+
101+
} // namespace
102+
103+
TEST(FunctionsLogicalTest, NullableOrIgnoresNullPayload) {
104+
auto result =
105+
execute_or(nullable_boolean_column({65, 0, 65, 1, 65}, {1, 0, 1, 0, 1}, "left"),
106+
nullable_boolean_column({1, 0, 0, 65, 127}, {0, 0, 0, 1, 1}, "right"), 5);
107+
108+
const auto& nullable_result = assert_cast<const ColumnNullable&>(*result);
109+
expect_boolean(nullable_result, 0, 1);
110+
expect_boolean(nullable_result, 1, 0);
111+
EXPECT_TRUE(nullable_result.is_null_at(2));
112+
expect_boolean(nullable_result, 3, 1);
113+
EXPECT_TRUE(nullable_result.is_null_at(4));
114+
}
115+
116+
TEST(FunctionsLogicalTest, NullableOrWithTrueConstantProducesCanonicalTrue) {
117+
constexpr size_t rows = 4;
118+
auto left = nullable_boolean_column({3, 65, 0, 255}, {1, 1, 1, 1}, "left");
119+
ColumnWithTypeAndName right {
120+
ColumnConst::create(ColumnHelper::create_column<DataTypeUInt8>({1}), rows),
121+
std::make_shared<DataTypeUInt8>(), "right"};
122+
123+
auto result = execute_or(std::move(left), std::move(right), rows);
124+
const auto& nullable_result = assert_cast<const ColumnNullable&>(*result);
125+
for (size_t row = 0; row < rows; ++row) {
126+
expect_boolean(nullable_result, row, 1);
127+
}
128+
}
129+
130+
TEST(FunctionsLogicalTest, CompoundNullableOrIgnoresNullPayload) {
131+
auto result = execute_compound_or(
132+
nullable_boolean_column({65, 0, 65, 1, 65}, {1, 0, 1, 0, 1}, "left"),
133+
nullable_boolean_column({1, 0, 0, 65, 127}, {0, 0, 0, 1, 1}, "right"), 5);
134+
135+
const auto& nullable_result = assert_cast<const ColumnNullable&>(*result);
136+
expect_boolean(nullable_result, 0, 1);
137+
expect_boolean(nullable_result, 1, 0);
138+
EXPECT_TRUE(nullable_result.is_null_at(2));
139+
expect_boolean(nullable_result, 3, 1);
140+
EXPECT_TRUE(nullable_result.is_null_at(4));
141+
}
142+
143+
} // namespace doris
Lines changed: 41 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,41 @@
1+
-- This file is automatically generated. You should know what you did if you want to edit this
2+
-- !or_payload_no_short_circuit --
3+
\N 66000
4+
true 34000
5+
6+
-- !and_or_payload_no_short_circuit --
7+
\N 66000
8+
true 34000
9+
10+
-- !case_count_no_short_circuit --
11+
100000
12+
13+
-- !case_branches_no_short_circuit --
14+
then_a 66000
15+
then_b 34000
16+
17+
-- !or_payload_short_circuit --
18+
\N 66000
19+
true 34000
20+
21+
-- !and_or_payload_short_circuit --
22+
\N 66000
23+
true 34000
24+
25+
-- !case_count_short_circuit --
26+
100000
27+
28+
-- !case_branches_short_circuit --
29+
then_a 66000
30+
then_b 34000
31+
32+
-- !nullable_or_truth_table --
33+
false false false
34+
false null \N
35+
false true true
36+
null false \N
37+
null null \N
38+
null true true
39+
true false true
40+
true null true
41+
true true true
Lines changed: 153 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,153 @@
1+
// Licensed to the Apache Software Foundation (ASF) under one
2+
// or more contributor license agreements. See the NOTICE file
3+
// distributed with this work for additional information
4+
// regarding copyright ownership. The ASF licenses this file
5+
// to you under the Apache License, Version 2.0 (the
6+
// "License"); you may not use this file except in compliance
7+
// with the License. You may obtain a copy of the License at
8+
//
9+
// http://www.apache.org/licenses/LICENSE-2.0
10+
//
11+
// Unless required by applicable law or agreed to in writing,
12+
// software distributed under the License is distributed on an
13+
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
// KIND, either express or implied. See the License for the
15+
// specific language governing permissions and limitations
16+
// under the License.
17+
18+
suite("test_json_extract_bool_null_payload") {
19+
sql "DROP TABLE IF EXISTS test_json_extract_bool_null_payload"
20+
sql """
21+
CREATE TABLE test_json_extract_bool_null_payload (
22+
id BIGINT,
23+
j STRING,
24+
a BIGINT,
25+
b BIGINT
26+
) DUPLICATE KEY(id)
27+
DISTRIBUTED BY HASH(id) BUCKETS 1
28+
PROPERTIES ("replication_num" = "1")
29+
"""
30+
sql """
31+
INSERT INTO test_json_extract_bool_null_payload
32+
SELECT number,
33+
CASE WHEN number % 100 = 0
34+
THEN '{"flag": true}'
35+
ELSE CONCAT('{"n": ', number, '}')
36+
END,
37+
number + CASE number % 3
38+
WHEN 0 THEN 10
39+
WHEN 1 THEN -100
40+
ELSE -200000
41+
END,
42+
number
43+
FROM numbers("number" = "100000")
44+
"""
45+
46+
sql "SET enable_sql_cache = false"
47+
sql "SET short_circuit_evaluation = false"
48+
49+
order_qt_or_payload_no_short_circuit """
50+
SELECT v, count(*) FROM (
51+
SELECT JSON_EXTRACT_BOOL(JSON_PARSE_ERROR_TO_NULL(j), '\$.flag') OR (a > b) AS v
52+
FROM test_json_extract_bool_null_payload
53+
) t
54+
GROUP BY v
55+
ORDER BY v
56+
"""
57+
58+
order_qt_and_or_payload_no_short_circuit """
59+
SELECT v, count(*) FROM (
60+
SELECT (JSON_PARSE_ERROR_TO_NULL(j) IS NOT NULL
61+
AND JSON_EXTRACT_BOOL(JSON_PARSE_ERROR_TO_NULL(j), '\$.flag'))
62+
OR (a > b) AS v
63+
FROM test_json_extract_bool_null_payload
64+
) t
65+
GROUP BY v
66+
ORDER BY v
67+
"""
68+
69+
qt_case_count_no_short_circuit """
70+
SELECT count(*)
71+
FROM test_json_extract_bool_null_payload
72+
WHERE (CASE
73+
WHEN JSON_EXTRACT_BOOL(JSON_PARSE_ERROR_TO_NULL(j), '\$.flag') OR a > b THEN b
74+
WHEN a < b THEN a
75+
END) IS NOT NULL
76+
"""
77+
78+
order_qt_case_branches_no_short_circuit """
79+
SELECT CASE
80+
WHEN JSON_EXTRACT_BOOL(JSON_PARSE_ERROR_TO_NULL(j), '\$.flag') OR a > b
81+
THEN 'then_b'
82+
WHEN a < b THEN 'then_a'
83+
ELSE 'else'
84+
END AS branch,
85+
count(*)
86+
FROM test_json_extract_bool_null_payload
87+
GROUP BY branch
88+
ORDER BY branch
89+
"""
90+
91+
sql "SET short_circuit_evaluation = true"
92+
93+
order_qt_or_payload_short_circuit """
94+
SELECT v, count(*) FROM (
95+
SELECT JSON_EXTRACT_BOOL(JSON_PARSE_ERROR_TO_NULL(j), '\$.flag') OR (a > b) AS v
96+
FROM test_json_extract_bool_null_payload
97+
) t
98+
GROUP BY v
99+
ORDER BY v
100+
"""
101+
102+
order_qt_and_or_payload_short_circuit """
103+
SELECT v, count(*) FROM (
104+
SELECT (JSON_PARSE_ERROR_TO_NULL(j) IS NOT NULL
105+
AND JSON_EXTRACT_BOOL(JSON_PARSE_ERROR_TO_NULL(j), '\$.flag'))
106+
OR (a > b) AS v
107+
FROM test_json_extract_bool_null_payload
108+
) t
109+
GROUP BY v
110+
ORDER BY v
111+
"""
112+
113+
qt_case_count_short_circuit """
114+
SELECT count(*)
115+
FROM test_json_extract_bool_null_payload
116+
WHERE (CASE
117+
WHEN JSON_EXTRACT_BOOL(JSON_PARSE_ERROR_TO_NULL(j), '\$.flag') OR a > b THEN b
118+
WHEN a < b THEN a
119+
END) IS NOT NULL
120+
"""
121+
122+
order_qt_case_branches_short_circuit """
123+
SELECT CASE
124+
WHEN JSON_EXTRACT_BOOL(JSON_PARSE_ERROR_TO_NULL(j), '\$.flag') OR a > b
125+
THEN 'then_b'
126+
WHEN a < b THEN 'then_a'
127+
ELSE 'else'
128+
END AS branch,
129+
count(*)
130+
FROM test_json_extract_bool_null_payload
131+
GROUP BY branch
132+
ORDER BY branch
133+
"""
134+
135+
order_qt_nullable_or_truth_table """
136+
SELECT lhs.label, rhs.label, lhs.v OR rhs.v AS result
137+
FROM (
138+
SELECT 'false' AS label, CAST(false AS BOOLEAN) AS v
139+
UNION ALL
140+
SELECT 'null', CAST(NULL AS BOOLEAN)
141+
UNION ALL
142+
SELECT 'true', CAST(true AS BOOLEAN)
143+
) lhs
144+
CROSS JOIN (
145+
SELECT 'false' AS label, CAST(false AS BOOLEAN) AS v
146+
UNION ALL
147+
SELECT 'null', CAST(NULL AS BOOLEAN)
148+
UNION ALL
149+
SELECT 'true', CAST(true AS BOOLEAN)
150+
) rhs
151+
ORDER BY lhs.label, rhs.label
152+
"""
153+
}

0 commit comments

Comments
 (0)