|
17 | 17 |
|
18 | 18 | // T08 -- wildcard matcher scratch reuse. |
19 | 19 | // |
20 | | -// Proves the request-scoped internal::WildcardMatcher (a) matches bit-for-bit |
21 | | -// identically to the former per-call DP in wildcard_query.cpp, and (b) reuses its |
22 | | -// two DP scratch rows across every visited term so a whole-dictionary scan |
23 | | -// performs O(1) heap allocations (<= 2) instead of O(2N). A header-only |
24 | | -// CountingAllocator gives the deterministic allocation counts; a byte-for-byte |
25 | | -// copy of the original DP serves as the equivalence oracle. |
| 20 | +// Proves the request-scoped internal::WildcardMatcher preserves the former ASCII |
| 21 | +// behavior while matching UTF-8 code points, and reuses its two DP scratch rows |
| 22 | +// across every visited term so a whole-dictionary scan performs O(1) heap |
| 23 | +// allocations (<= 2) instead of O(2N). A header-only CountingAllocator gives the |
| 24 | +// deterministic allocation counts; a byte-for-byte copy of the original DP |
| 25 | +// serves as the ASCII equivalence oracle. |
26 | 26 |
|
27 | 27 | #include <gtest/gtest.h> |
28 | 28 |
|
@@ -155,7 +155,7 @@ std::vector<std::string> make_varied_length_terms(size_t count, size_t max_len) |
155 | 155 | return terms; |
156 | 156 | } |
157 | 157 |
|
158 | | -// W-EQ-DP: the optimized matcher reproduces the reference DP bit-for-bit over an |
| 158 | +// W-EQ-DP: the optimized matcher reproduces the reference DP for ASCII over an |
159 | 159 | // exhaustive small-alphabet battery (covers "", leading/trailing '*'/'?', |
160 | 160 | // consecutive "**", '?' interplay) plus realistic dictionary patterns/terms. One |
161 | 161 | // matcher is reused across all terms of a pattern, so this also proves scratch |
@@ -212,12 +212,69 @@ TEST(SniiWildcardQueryTest, StarMatchesEverything) { |
212 | 212 | EXPECT_TRUE(matcher("xyz")); |
213 | 213 | } |
214 | 214 |
|
215 | | -// W-QMARK: "?" matches exactly one byte. |
216 | | -TEST(SniiWildcardQueryTest, QuestionMarkMatchesExactlyOneByte) { |
| 215 | +// W-QMARK: "?" matches exactly one UTF-8 code point. |
| 216 | +TEST(SniiWildcardQueryTest, QuestionMarkMatchesExactlyOneUtf8CodePoint) { |
217 | 217 | internal::WildcardMatcher<> matcher("?"); |
218 | 218 | EXPECT_FALSE(matcher("")); |
219 | 219 | EXPECT_TRUE(matcher("a")); |
| 220 | + EXPECT_TRUE(matcher("猫")); |
| 221 | + EXPECT_TRUE(matcher("🔥")); |
220 | 222 | EXPECT_FALSE(matcher("ab")); |
| 223 | + |
| 224 | + internal::WildcardMatcher<> surrounded("a?b"); |
| 225 | + EXPECT_TRUE(surrounded("a猫b")); |
| 226 | + EXPECT_TRUE(surrounded("a🔥b")); |
| 227 | + EXPECT_FALSE(surrounded("a猫猫b")); |
| 228 | + |
| 229 | + internal::WildcardMatcher<> three("a???b"); |
| 230 | + EXPECT_FALSE(three("a猫b")); |
| 231 | + EXPECT_TRUE(three("a猫🔥éb")); |
| 232 | + |
| 233 | + internal::WildcardMatcher<> star_then_two("a*??b"); |
| 234 | + EXPECT_FALSE(star_then_two("a猫b")); |
| 235 | + EXPECT_TRUE(star_then_two("a猫🔥b")); |
| 236 | +} |
| 237 | + |
| 238 | +// W-UTF8-LITERAL: non-ASCII literals are compared as complete code points. |
| 239 | +TEST(SniiWildcardQueryTest, Utf8LiteralsMatchCompleteCodePoints) { |
| 240 | + internal::WildcardMatcher<> matcher("猫?火"); |
| 241 | + EXPECT_TRUE(matcher("猫🔥火")); |
| 242 | + EXPECT_FALSE(matcher("猫🔥🔥火")); |
| 243 | + EXPECT_FALSE(matcher("狗🔥火")); |
| 244 | +} |
| 245 | + |
| 246 | +// W-INVALID-UTF8: patterns remain strict UTF-8, while malformed raw keyword |
| 247 | +// terms retain the byte-wise semantics used before code-point matching. |
| 248 | +TEST(SniiWildcardQueryTest, MalformedTermsRetainByteCompatibleMatching) { |
| 249 | + const std::string invalid_lead("\xff", 1); |
| 250 | + const std::string truncated("\xe7\x8c", 2); |
| 251 | + const std::string invalid_continuation("\xe7x\xab", 3); |
| 252 | + |
| 253 | + internal::WildcardMatcher<> any("*"); |
| 254 | + EXPECT_TRUE(any(invalid_lead)); |
| 255 | + EXPECT_TRUE(any(truncated)); |
| 256 | + EXPECT_TRUE(any(invalid_continuation)); |
| 257 | + |
| 258 | + internal::WildcardMatcher<> two_bytes("??"); |
| 259 | + EXPECT_FALSE(two_bytes(invalid_lead)); |
| 260 | + EXPECT_TRUE(two_bytes(truncated)); |
| 261 | + EXPECT_FALSE(two_bytes(invalid_continuation)); |
| 262 | + |
| 263 | + internal::WildcardMatcher<> invalid_pattern(invalid_lead); |
| 264 | + EXPECT_FALSE(invalid_pattern(invalid_lead)); |
| 265 | + EXPECT_FALSE(invalid_pattern("猫")); |
| 266 | +} |
| 267 | + |
| 268 | +TEST(SniiWildcardQueryTest, InvalidUtf8PatternReturnsInvalidArgument) { |
| 269 | + MemoryFile file; |
| 270 | + reader::SniiSegmentReader segment_reader; |
| 271 | + reader::LogicalIndexReader index_reader; |
| 272 | + assert_ok(build_reader(&file, &segment_reader, &index_reader)); |
| 273 | + |
| 274 | + const std::string invalid_pattern("\xff*", 2); |
| 275 | + std::vector<uint32_t> docids; |
| 276 | + EXPECT_TRUE(wildcard_query(index_reader, invalid_pattern, &docids) |
| 277 | + .is<doris::ErrorCode::INVALID_ARGUMENT>()); |
221 | 278 | } |
222 | 279 |
|
223 | 280 | // W-CONSEC-STAR: consecutive '*' degrade gracefully. |
|
0 commit comments