Skip to content

Commit 0451443

Browse files
authored
[fix](function) Enforce input boundaries in IP functions (#68160)
IP functions did not consistently respect the explicit boundaries of `ColumnString` values. IPv4 and IPv6 text conversions accepted an otherwise valid address followed by an embedded NUL byte and trailing data because unbounded parser overloads stopped at the first NUL. In addition, `is_ipv4_compat` and `is_ipv4_mapped` discarded each binary string's length and unconditionally read 16 bytes, so short values could consume bytes from the following row and produce results that changed with filtering, while oversized values were accepted based only on their first 16 bytes. Root cause: both paths reduced length-aware string values to raw pointers before parsing or inspection. This change uses bounded parsers throughout, removes the unsafe null-terminated parser overloads, maps parsed IPv4 values directly to IPv6, and requires binary IPv6 inputs to be exactly 16 bytes before inspecting them. Unit coverage includes throw, default, nullable, vector, constant, short, valid, and oversized input paths. ### Release note None ### Check List (For Author) - Test <!-- At least one of them must be included. --> - [ ] Regression test - [ ] Unit Test - [ ] Manual test (add detailed scripts or steps below) - [ ] No need to test or manual test. Explain why: - [ ] This is a refactor/code format and no logic has been changed. - [ ] Previous test can cover this change. - [ ] No code files have been changed. - [ ] Other reason <!-- Add your reason? --> - Behavior changed: - [ ] No. - [ ] Yes. <!-- Explain the behavior change --> - Does this need documentation? - [ ] No. - [ ] Yes. <!-- Add document PR link here. eg: apache/doris-website#1214 --> ### Check List (For Reviewer who merge this PR) - [ ] Confirm the release note - [ ] Confirm test cases - [ ] Confirm document - [ ] Add branch pick label <!-- Add branch pick label that this PR should merge into -->
1 parent 1271f00 commit 0451443

4 files changed

Lines changed: 107 additions & 83 deletions

File tree

‎be/src/exec/common/format_ip.h‎

Lines changed: 0 additions & 29 deletions
Original file line numberDiff line numberDiff line change
@@ -179,21 +179,6 @@ inline bool parse_ipv4_whole(const char* src, const char* end, unsigned char* ds
179179
return parse_ipv4(src, end, dst) == end;
180180
}
181181

182-
/// returns pointer to the right after parsed sequence or null on failed parsing
183-
inline const char* parse_ipv4(const char* src, unsigned char* dst) {
184-
if (parse_ipv4(
185-
src, []() { return false; }, dst)) {
186-
return src;
187-
}
188-
return nullptr;
189-
}
190-
191-
/// returns true if whole null-terminated string was parsed successfully
192-
inline bool parse_ipv4_whole(const char* src, unsigned char* dst) {
193-
const char* end = parse_ipv4(src, dst);
194-
return end != nullptr && *end == '\0';
195-
}
196-
197182
/// integer logarithm, return ceil(log(value, base)) (the smallest integer greater or equal than log(value, base)
198183
inline constexpr UInt32 int_log(const UInt32 value, const UInt32 base, const bool carry) {
199184
return value >= base ? 1 + int_log(value / base, base, value % base || carry)
@@ -480,18 +465,4 @@ inline bool parse_ipv6_whole(const char* src, const char* end, unsigned char* ds
480465
return parse_ipv6(src, end, dst) == end;
481466
}
482467

483-
/// returns pointer to the right after parsed sequence or null on failed parsing
484-
inline const char* parse_ipv6(const char* src, unsigned char* dst) {
485-
if (parse_ipv6(
486-
src, []() { return false; }, dst))
487-
return src;
488-
return nullptr;
489-
}
490-
491-
/// returns true if whole null-terminated string was parsed successfully
492-
inline bool parse_ipv6_whole(const char* src, unsigned char* dst) {
493-
const char* end = parse_ipv6(src, dst);
494-
return end != nullptr && *end == '\0';
495-
}
496-
497468
} // namespace doris

‎be/src/exprs/function/function_ip.h‎

Lines changed: 24 additions & 52 deletions
Original file line numberDiff line numberDiff line change
@@ -43,6 +43,7 @@
4343
#include "core/field.h"
4444
#include "core/types.h"
4545
#include "core/value/ip_address_cidr.h"
46+
#include "core/value/ipv4_value.h"
4647
#include "exec/common/endian.h"
4748
#include "exec/common/format_ip.h"
4849
#include "exec/common/ipv6_to_binary.h"
@@ -135,8 +136,8 @@ class FunctionIPv4NumToString : public IFunction {
135136
/// Since IPExceptionMode means wider scope, we use more specific name here.
136137
enum class IPConvertExceptionMode : uint8_t { Throw, Default, Null };
137138

138-
static inline bool try_parse_ipv4(const char* pos, Int64& result_value) {
139-
return parse_ipv4_whole(pos, reinterpret_cast<unsigned char*>(&result_value));
139+
static inline bool try_parse_ipv4(const char* begin, const char* end, Int64& result_value) {
140+
return parse_ipv4_whole(begin, end, reinterpret_cast<unsigned char*>(&result_value));
140141
}
141142

142143
template <IPConvertExceptionMode exception_mode, typename ToColumn>
@@ -156,10 +157,6 @@ ColumnPtr convert_to_ipv4(ColumnPtr column, const PaddedPODArray<UInt8>* null_ma
156157
auto col_res = ToColumn::create(column_size, 0);
157158
auto& vec_res = col_res->get_data();
158159

159-
const ColumnString::Chars& vec_src = column_string->get_chars();
160-
const ColumnString::Offsets& offsets_src = column_string->get_offsets();
161-
size_t prev_offset = 0;
162-
163160
for (size_t i = 0; i < vec_res.size(); ++i) {
164161
if (null_map && (*null_map)[i]) {
165162
if constexpr (exception_mode == IPConvertExceptionMode::Throw) {
@@ -169,17 +166,13 @@ ColumnPtr convert_to_ipv4(ColumnPtr column, const PaddedPODArray<UInt8>* null_ma
169166
"like '0.0.0.0' first");
170167
}
171168
vec_res[i] = 0;
172-
prev_offset = offsets_src[i];
173169
if constexpr (exception_mode == IPConvertExceptionMode::Null) {
174170
(*vec_null_map_to)[i] = true;
175171
}
176172
continue;
177173
}
178-
const char* src_start = reinterpret_cast<const char*>(&vec_src[prev_offset]);
179-
size_t src_length = (i < vec_res.size() - 1) ? (offsets_src[i] - prev_offset)
180-
: (vec_src.size() - prev_offset);
181-
std::string src(src_start, src_length);
182-
bool parse_result = try_parse_ipv4(src.c_str(), vec_res[i]);
174+
const auto src = column_string->get_data_at(i);
175+
bool parse_result = try_parse_ipv4(src.begin(), src.end(), vec_res[i]);
183176

184177
if (!parse_result) {
185178
if constexpr (exception_mode == IPConvertExceptionMode::Throw) {
@@ -191,8 +184,6 @@ ColumnPtr convert_to_ipv4(ColumnPtr column, const PaddedPODArray<UInt8>* null_ma
191184
vec_res[i] = 0;
192185
}
193186
}
194-
195-
prev_offset = offsets_src[i];
196187
}
197188

198189
if constexpr (exception_mode == IPConvertExceptionMode::Null) {
@@ -384,15 +375,6 @@ ColumnPtr convert_to_ipv6(const StringColumnType& string_column,
384375
auto col_res = column_create(column_size);
385376
auto& vec_res = get_vector(col_res, column_size);
386377

387-
using Chars = typename StringColumnType::Chars;
388-
const Chars& vec_src = string_column.get_chars();
389-
390-
size_t src_offset = 0;
391-
392-
/// ColumnString contains not null terminated strings. But functions parseIPv6, parseIPv4 expect null terminated string.
393-
/// TODO fix this - now parseIPv6/parseIPv4 accept end iterator, so can be parsed in-place
394-
std::string string_buffer;
395-
396378
int offset_inc = 1;
397379
ColumnString* column_string = nullptr;
398380
if constexpr (std::is_same_v<ToColumn, ColumnString>) {
@@ -401,20 +383,11 @@ ColumnPtr convert_to_ipv6(const StringColumnType& string_column,
401383
}
402384

403385
for (size_t out_offset = 0, i = 0; i < column_size; out_offset += offset_inc, ++i) {
404-
char src_ipv4_buf[sizeof("::ffff:") + IPV4_MAX_TEXT_LENGTH + 1] = "::ffff:";
405-
size_t src_next_offset = src_offset;
406-
407-
const char* src_value = nullptr;
386+
const auto src = string_column.get_data_at(i);
387+
const char* src_value = src.begin();
388+
const char* src_end = src.end();
408389
auto* res_value = reinterpret_cast<unsigned char*>(&vec_res[out_offset]);
409390

410-
if constexpr (std::is_same_v<StringColumnType, ColumnString>) {
411-
src_value = reinterpret_cast<const char*>(&vec_src[src_offset]);
412-
src_next_offset = string_column.get_offsets()[i];
413-
414-
string_buffer.assign(src_value, src_next_offset - src_offset);
415-
src_value = string_buffer.c_str();
416-
}
417-
418391
if (null_map && (*null_map)[i]) {
419392
if (exception_mode == IPConvertExceptionMode::Throw) {
420393
throw Exception(
@@ -431,7 +404,6 @@ ColumnPtr convert_to_ipv6(const StringColumnType& string_column,
431404
DCHECK(column_string != nullptr);
432405
column_string->get_offsets().push_back((i + 1) * IPV6_BINARY_LENGTH);
433406
}
434-
src_offset = src_next_offset;
435407
continue;
436408
}
437409

@@ -442,13 +414,13 @@ ColumnPtr convert_to_ipv6(const StringColumnType& string_column,
442414

443415
/// If the source IP address is parsable as an IPv4 address, then transform it into a valid IPv6 address.
444416
/// Keeping it simple by just prefixing `::ffff:` to the IPv4 address to represent it as a valid IPv6 address.
445-
size_t string_length = src_next_offset - src_offset;
417+
size_t string_length = src.size;
446418
if (string_length != 0) {
447-
if (try_parse_ipv4(src_value, dummy_result)) {
448-
strncat(src_ipv4_buf, src_value, sizeof(src_ipv4_buf) - strlen(src_ipv4_buf) - 1);
449-
parse_result = parse_ipv6_whole(src_ipv4_buf, res_value);
419+
if (try_parse_ipv4(src_value, src_end, dummy_result)) {
420+
map_ipv4_to_ipv6(static_cast<IPv4>(dummy_result), res_value);
421+
parse_result = true;
450422
} else {
451-
parse_result = parse_ipv6_whole(src_value, res_value);
423+
parse_result = parse_ipv6_whole(src_value, src_end, res_value);
452424
}
453425
}
454426

@@ -479,7 +451,6 @@ ColumnPtr convert_to_ipv6(const StringColumnType& string_column,
479451
(*vec_null_map_to)[i] = true;
480452
}
481453
}
482-
src_offset = src_next_offset;
483454
}
484455

485456
if constexpr (exception_mode == IPConvertExceptionMode::Null) {
@@ -1018,8 +989,8 @@ class FunctionIsIPv4Compat : public IFunction {
1018989
auto& col_res_data = col_res->get_data();
1019990

1020991
for (size_t i = 0; i < col_size; ++i) {
1021-
auto ipv4_in = col_in->get_data_at(i);
1022-
if (is_ipv4_compat(reinterpret_cast<const UInt8*>(ipv4_in.data))) {
992+
const auto address = col_in->get_data_at(i);
993+
if (is_ipv4_compat(address)) {
1023994
col_res_data[i] = 1;
1024995
}
1025996
}
@@ -1029,9 +1000,10 @@ class FunctionIsIPv4Compat : public IFunction {
10291000
}
10301001

10311002
private:
1032-
static bool is_ipv4_compat(const UInt8* address) {
1033-
return (LittleEndian::Load64(address) == 0) && (LittleEndian::Load32(address + 8) == 0) &&
1034-
(LittleEndian::Load32(address + 12) != 0);
1003+
static bool is_ipv4_compat(const StringRef& address) {
1004+
return address.size == IPV6_BINARY_LENGTH && (LittleEndian::Load64(address.data) == 0) &&
1005+
(LittleEndian::Load32(address.data + 8) == 0) &&
1006+
(LittleEndian::Load32(address.data + 12) != 0);
10351007
}
10361008
};
10371009

@@ -1058,8 +1030,8 @@ class FunctionIsIPv4Mapped : public IFunction {
10581030
auto& col_res_data = col_res->get_data();
10591031

10601032
for (size_t i = 0; i < col_size; ++i) {
1061-
auto ipv4_in = col_in->get_data_at(i);
1062-
if (is_ipv4_mapped(reinterpret_cast<const UInt8*>(ipv4_in.data))) {
1033+
const auto address = col_in->get_data_at(i);
1034+
if (is_ipv4_mapped(address)) {
10631035
col_res_data[i] = 1;
10641036
}
10651037
}
@@ -1069,9 +1041,9 @@ class FunctionIsIPv4Mapped : public IFunction {
10691041
}
10701042

10711043
private:
1072-
static bool is_ipv4_mapped(const UInt8* address) {
1073-
return (LittleEndian::Load64(address) == 0) &&
1074-
((LittleEndian::Load64(address + 8) & 0x00000000FFFFFFFFULL) ==
1044+
static bool is_ipv4_mapped(const StringRef& address) {
1045+
return address.size == IPV6_BINARY_LENGTH && (LittleEndian::Load64(address.data) == 0) &&
1046+
((LittleEndian::Load64(address.data + 8) & 0x00000000FFFFFFFFULL) ==
10751047
0x00000000FFFF0000ULL);
10761048
}
10771049
};

‎be/test/core/column/column_ip_test.cpp‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -303,8 +303,9 @@ TEST_F(ColumnIPTest, IPv4ValueFromStringTest) {
303303
TEST_F(ColumnIPTest, IPv4Parse) {
304304
std::string ipv4_str = "127.0.0.1";
305305
Int64 result_value = 0;
306-
ASSERT_EQ(try_parse_ipv4(ipv4_str.data(), result_value), true);
306+
ASSERT_EQ(try_parse_ipv4(ipv4_str.data(), ipv4_str.data() + ipv4_str.size(), result_value),
307+
true);
307308
ASSERT_EQ(2130706433, result_value);
308309
};
309310

310-
} // namespace doris
311+
} // namespace doris

‎be/test/exprs/function/function_ip_test.cpp‎

Lines changed: 80 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -39,6 +39,86 @@ TEST(FunctionIpTest, IPAddressVariantTypeTest) {
3939
EXPECT_TRUE(ipv6_zero.is_v6());
4040
}
4141

42+
TEST(FunctionIpTest, StringToNumRejectsEmbeddedNullTail) {
43+
std::string invalid_ipv4 = "192.168.0.1";
44+
invalid_ipv4.push_back('\0');
45+
invalid_ipv4.append("tail");
46+
47+
std::string invalid_ipv6 = "2001:db8::1";
48+
invalid_ipv6.push_back('\0');
49+
invalid_ipv6.append("tail");
50+
51+
const DataSet ipv4_error_data = {{{invalid_ipv4}, int64_t {0}}};
52+
const DataSet ipv4_default_data = {{{invalid_ipv4}, int64_t {0}}};
53+
const DataSet ipv4_null_data = {{{invalid_ipv4}, Null()}};
54+
const DataSet ipv6_error_data = {{{invalid_ipv6}, std::string {}}};
55+
const DataSet ipv6_default_data = {{{invalid_ipv6}, std::string(IPV6_BINARY_LENGTH, '\0')},
56+
{{invalid_ipv4}, std::string(IPV6_BINARY_LENGTH, '\0')}};
57+
const DataSet ipv6_null_data = {{{invalid_ipv6}, Null()}, {{invalid_ipv4}, Null()}};
58+
59+
for (const auto& input_types : {InputTypeSet {PrimitiveType::TYPE_VARCHAR},
60+
InputTypeSet {Consted {PrimitiveType::TYPE_VARCHAR}}}) {
61+
auto status = check_function<DataTypeInt64>("ipv4_string_to_num", input_types,
62+
ipv4_error_data, -1, -1, true);
63+
EXPECT_EQ(status.code(), ErrorCode::INVALID_ARGUMENT);
64+
65+
status = check_function<DataTypeString>("ipv6_string_to_num", input_types, ipv6_error_data,
66+
-1, -1, true);
67+
EXPECT_EQ(status.code(), ErrorCode::INVALID_ARGUMENT);
68+
}
69+
70+
const InputTypeSet input_types = {PrimitiveType::TYPE_VARCHAR};
71+
check_function_all_arg_comb<DataTypeInt64>("ipv4_string_to_num_or_default", input_types,
72+
ipv4_default_data);
73+
check_function_all_arg_comb<DataTypeInt64, true>("ipv4_string_to_num_or_null", input_types,
74+
ipv4_null_data);
75+
check_function_all_arg_comb<DataTypeInt64, true>("inet_aton", input_types, ipv4_null_data);
76+
check_function_all_arg_comb<DataTypeString>("ipv6_string_to_num_or_default", input_types,
77+
ipv6_default_data);
78+
check_function_all_arg_comb<DataTypeString, true>("ipv6_string_to_num_or_null", input_types,
79+
ipv6_null_data);
80+
check_function_all_arg_comb<DataTypeString, true>("inet6_aton", input_types, ipv6_null_data);
81+
}
82+
83+
TEST(FunctionIpTest, StringToIPv6AcceptsLongIPv4Spellings) {
84+
std::string mapped_ipv4_zero(IPV6_BINARY_LENGTH, '\0');
85+
mapped_ipv4_zero[10] = static_cast<char>(0xff);
86+
mapped_ipv4_zero[11] = static_cast<char>(0xff);
87+
88+
for (const auto& [input, input_type] :
89+
{std::pair {std::string("0000.0000.0000.0"), InputTypeSet {PrimitiveType::TYPE_VARCHAR}},
90+
std::pair {std::string("0000.0000.0000.0000"),
91+
InputTypeSet {Consted {PrimitiveType::TYPE_VARCHAR}}}}) {
92+
const DataSet data = {{{input}, mapped_ipv4_zero}};
93+
static_cast<void>(check_function<DataTypeString>("ipv6_string_to_num", input_type, data));
94+
}
95+
}
96+
97+
TEST(FunctionIpTest, IPv4CompatAndMappedRequireIPv6BinaryLength) {
98+
const std::string ipv4_address = {static_cast<char>(0xc0), static_cast<char>(0xa8), '\0',
99+
static_cast<char>(0x01)};
100+
101+
const std::string compat_prefix(IPV6_BINARY_LENGTH - IPV4_BINARY_LENGTH, '\0');
102+
const std::string compat_address = compat_prefix + ipv4_address;
103+
const DataSet compat_data = {{{compat_prefix}, uint8_t {0}},
104+
{{ipv4_address}, uint8_t {0}},
105+
{{compat_address}, uint8_t {1}},
106+
{{compat_address + '\0'}, uint8_t {0}}};
107+
108+
const std::string mapped_marker = {static_cast<char>(0xff), static_cast<char>(0xff)};
109+
const std::string mapped_prefix(IPV6_BINARY_LENGTH - IPV4_BINARY_LENGTH - mapped_marker.size(),
110+
'\0');
111+
const std::string mapped_address = mapped_prefix + mapped_marker + ipv4_address;
112+
const DataSet mapped_data = {{{mapped_prefix}, uint8_t {0}},
113+
{{mapped_marker}, uint8_t {0}},
114+
{{mapped_address}, uint8_t {1}},
115+
{{mapped_address + '\0'}, uint8_t {0}}};
116+
117+
const InputTypeSet input_types = {PrimitiveType::TYPE_VARCHAR};
118+
check_function_all_arg_comb<DataTypeUInt8, true>("is_ipv4_compat", input_types, compat_data);
119+
check_function_all_arg_comb<DataTypeUInt8, true>("is_ipv4_mapped", input_types, mapped_data);
120+
}
121+
42122
TEST(FunctionIpTest, FunctionIsIPAddressInRangeTest) {
43123
std::string func_name = "is_ip_address_in_range";
44124

0 commit comments

Comments
 (0)