Skip to content

Commit 5c6bf72

Browse files
HappenLeeHappenLee
committed
[improvement](be) Batch consecutive ASCII rows in case conversion
### What problem does this PR solve? Related PR: #69223 Problem Summary: Mixed ASCII/Unicode batches convert each ASCII row separately, which leaves short rows below SIMD width and repeats output resizing and length checks. Group consecutive ASCII rows into one contiguous conversion, then rebase their individual offsets on the current result position. Keep full ICU mapping for Unicode rows, including expansions and context-sensitive casing. A run of 32 two-byte rows now needs one 64-byte conversion instead of 32 conversions; runtime speedup has not been measured. Add coverage for runs around Unicode expansions/contractions, empty rows, NULLs and embedded zero bytes. ### Release note Reduce per-row overhead for lower and upper on mixed ASCII/Unicode batches. ### Check List (For Author) - Test: Unit Test added; focused ASAN run attempted but CMake could not find OpenMP_C, so tests did not execute. Formatting, diff and build hygiene checks passed. Clang-tidy was blocked by the missing compilation database. - Behavior changed: No (SQL results and Unicode semantics are unchanged) - Does this need documentation: No
1 parent 2824ba6 commit 5c6bf72

2 files changed

Lines changed: 85 additions & 24 deletions

File tree

‎be/src/exprs/function/function_string.cpp‎

Lines changed: 38 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -589,7 +589,7 @@ struct TransferImpl {
589589
return Status::OK();
590590
}
591591

592-
// Mixed path: decide per row. ASCII rows use SIMD; only non-ASCII rows go through ICU.
592+
// Mixed path: combine consecutive ASCII rows for SIMD; only non-ASCII rows use ICU.
593593
// Preserve full Unicode case mappings for non-ASCII rows.
594594
execute_mixed(data, offsets, res_data, res_offsets);
595595
return Status::OK();
@@ -599,37 +599,51 @@ struct TransferImpl {
599599
ColumnString::Chars& res_data, ColumnString::Offsets& res_offsets) {
600600
size_t offset_size = offsets.size();
601601
res_data.reserve(data.size());
602+
auto process_ascii_run = [&](int64_t first_row, int64_t last_row) {
603+
if (first_row == last_row) {
604+
return;
605+
}
606+
const size_t src_offset = offsets[first_row - 1];
607+
const size_t size = offsets[last_row - 1] - src_offset;
608+
const size_t res_offset = res_data.size();
609+
ColumnString::check_chars_length(res_offset + size, res_offsets.size());
610+
res_data.resize(res_offset + size);
611+
if constexpr (std::is_same_v<OpName, NameToUpper>) {
612+
simd::VStringFunctions::to_upper(data.data() + src_offset, size,
613+
res_data.data() + res_offset);
614+
} else if constexpr (std::is_same_v<OpName, NameToLower>) {
615+
simd::VStringFunctions::to_lower(data.data() + src_offset, size,
616+
res_data.data() + res_offset);
617+
}
618+
// Earlier Unicode rows can change length, so rebase each offset on this run's output.
619+
for (int64_t row = first_row; row < last_row; ++row) {
620+
res_offsets[row] = (ColumnString::Offset)(res_offset + offsets[row] - src_offset);
621+
}
622+
};
623+
624+
int64_t ascii_run_start = 0;
602625
std::string result;
603626
for (int64_t i = 0; i < offset_size; ++i) {
604627
const uint8_t* begin = &data[offsets[i - 1]];
605628
uint32_t size = offsets[i] - offsets[i - 1];
606-
607-
if (size == 0) {
608-
res_offsets[i] = (ColumnString::Offset)res_data.size();
629+
if (size == 0 ||
630+
simd::VStringFunctions::is_ascii({reinterpret_cast<const char*>(begin), size})) {
609631
continue;
610632
}
611-
if (simd::VStringFunctions::is_ascii({reinterpret_cast<const char*>(begin), size})) {
612-
// ASCII row: SIMD case conversion in place (length preserved)
613-
size_t res_offset = res_data.size();
614-
ColumnString::check_chars_length(res_offset + size, res_offsets.size());
615-
res_data.resize(res_offset + size);
616-
if constexpr (std::is_same_v<OpName, NameToUpper>) {
617-
simd::VStringFunctions::to_upper(begin, size, res_data.data() + res_offset);
618-
} else if constexpr (std::is_same_v<OpName, NameToLower>) {
619-
simd::VStringFunctions::to_lower(begin, size, res_data.data() + res_offset);
620-
}
621-
res_offsets[i] = (ColumnString::Offset)res_data.size();
622-
} else {
623-
// Non-ASCII row: ICU (case mapping may change length)
624-
result.clear();
625-
if constexpr (std::is_same_v<OpName, NameToUpper>) {
626-
to_upper_utf8(reinterpret_cast<const char*>(begin), size, result);
627-
} else if constexpr (std::is_same_v<OpName, NameToLower>) {
628-
to_lower_utf8(reinterpret_cast<const char*>(begin), size, result);
629-
}
630-
StringOP::push_value_string(result, i, res_data, res_offsets);
633+
634+
// E.g. ["a", "b", "你", "c"] converts "ab" together, keeping short rows SIMD-friendly.
635+
process_ascii_run(ascii_run_start, i);
636+
ascii_run_start = i + 1;
637+
// Preserve full Unicode mappings, including expansions and context-sensitive casing.
638+
result.clear();
639+
if constexpr (std::is_same_v<OpName, NameToUpper>) {
640+
to_upper_utf8(reinterpret_cast<const char*>(begin), size, result);
641+
} else if constexpr (std::is_same_v<OpName, NameToLower>) {
642+
to_lower_utf8(reinterpret_cast<const char*>(begin), size, result);
631643
}
644+
StringOP::push_value_string(result, i, res_data, res_offsets);
632645
}
646+
process_ascii_run(ascii_run_start, offset_size);
633647
}
634648

635649
static void to_upper_utf8(const char* data, uint32_t size, std::string& result) {

‎be/test/exprs/function/function_string_test.cpp‎

Lines changed: 47 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -747,6 +747,53 @@ TEST(function_string_test, function_string_upper_test) {
747747
}
748748
}
749749

750+
TEST(function_string_test, function_string_case_mixed_ascii_runs_test) {
751+
InputTypeSet input_types = {PrimitiveType::TYPE_VARCHAR};
752+
DataSet lower_data = {{{std::string("")}, std::string("")}};
753+
DataSet upper_data = lower_data;
754+
// Each row is shorter than a SIMD register, but the consecutive run spans multiple registers.
755+
for (int i = 0; i < 32; ++i) {
756+
lower_data.push_back({{std::string("aB")}, std::string("ab")});
757+
upper_data.push_back({{std::string("aB")}, std::string("AB")});
758+
}
759+
lower_data.insert(
760+
lower_data.end(),
761+
{
762+
{{std::string("\u0130")},
763+
std::string("i\u0307")}, // UTF-8 grows from 2 to 3 bytes.
764+
{{std::string("")}, std::string("")},
765+
{{std::string("AFTER")}, std::string("after")},
766+
{{Null()}, Null()},
767+
{{std::string("EXPANSION")}, std::string("expansion")},
768+
{{std::string("\u212A")}, std::string("k")}, // UTF-8 shrinks from 3 to 1 byte.
769+
{{std::string("")}, std::string("")}, // An empty-only run between Unicode rows.
770+
{{std::string("ΟΣ")},
771+
std::string("ος")}, // Keep contextual final sigma mapping.
772+
{{std::string("你好A")}, std::string("你好a")}, // Consecutive Unicode rows.
773+
{{std::string("A\0Z", 3)}, std::string("a\0z", 3)},
774+
{{std::string("TAIL")}, std::string("tail")},
775+
{{std::string("")}, std::string("")},
776+
});
777+
upper_data.insert(
778+
upper_data.end(),
779+
{
780+
{{std::string("\u0390")}, std::string("\u0399\u0308\u0301")}, // 2 to 6 bytes.
781+
{{std::string("")}, std::string("")},
782+
{{std::string("after")}, std::string("AFTER")},
783+
{{Null()}, Null()},
784+
{{std::string("expansion")}, std::string("EXPANSION")},
785+
{{std::string("\u0131")}, std::string("I")}, // UTF-8 shrinks from 2 to 1 byte.
786+
{{std::string("")}, std::string("")},
787+
{{std::string("straße")}, std::string("STRASSE")},
788+
{{std::string("你好a")}, std::string("你好A")},
789+
{{std::string("a\0z", 3)}, std::string("A\0Z", 3)},
790+
{{std::string("tail")}, std::string("TAIL")},
791+
{{std::string("")}, std::string("")},
792+
});
793+
check_function_all_arg_comb<DataTypeString, true>("lower", input_types, lower_data);
794+
check_function_all_arg_comb<DataTypeString, true>("upper", input_types, upper_data);
795+
}
796+
750797
// LOWER/UPPER/INITCAP case-map through the ICU root locale, independent of the
751798
// process default locale. Under a Turkish (tr_TR) default locale ICU maps ASCII
752799
// 'I' -> 'ı' (U+0131) and 'i' -> 'İ' (U+0130); the root locale keeps the plain

0 commit comments

Comments
 (0)