Skip to content

Commit d437841

Browse files
authored
[fix](be) Skip NULL-masked payload in strict cast (apache#67981)
### What problem does this PR solve? Issue Number: close #xxx Related PR: #xxx Problem Summary: ### Release note None ### Check List (For Author) - Test <!-- At least one of them must be included. --> - [ ] Regression test - [ ] Unit Test - [ ] Manual test (add detailed scripts or steps below) - [ ] No need to test or manual test. Explain why: - [ ] This is a refactor/code format and no logic has been changed. - [ ] Previous test can cover this change. - [ ] No code files have been changed. - [ ] Other reason <!-- Add your reason? --> - Behavior changed: - [ ] No. - [ ] Yes. <!-- Explain the behavior change --> - Does this need documentation? - [ ] No. - [ ] Yes. <!-- Add document PR link here. eg: apache/doris-website#1214 --> ### Check List (For Reviewer who merge this PR) - [ ] Confirm the release note - [ ] Confirm test cases - [ ] Confirm document - [ ] Add branch pick label <!-- Add branch pick label that this PR should merge into -->
1 parent ccf23fa commit d437841

42 files changed

Lines changed: 1895 additions & 178 deletions

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎be/src/core/data_type_serde/data_type_date_or_datetime_serde.cpp‎

Lines changed: 65 additions & 31 deletions
Original file line numberDiff line numberDiff line change
@@ -505,12 +505,16 @@ Status DataTypeDateSerDe<T>::from_int_batch(const typename IntDataType::ColumnTy
505505
template <PrimitiveType T>
506506
template <typename IntDataType>
507507
Status DataTypeDateSerDe<T>::from_int_strict_mode_batch(
508-
const typename IntDataType::ColumnType& int_col, IColumn& target_col) const {
508+
const typename IntDataType::ColumnType& int_col, IColumn& target_col,
509+
const NullMap::value_type* null_map) const {
509510
auto& col_data = assert_cast<ColumnType&>(target_col);
510511
col_data.resize(int_col.size());
511512

512513
CastParameters params {.status = Status::OK(), .is_strict = true};
513514
for (size_t i = 0; i < int_col.size(); ++i) {
515+
if (null_map && null_map[i]) {
516+
continue;
517+
}
514518
CppType val;
515519
CastToDateOrDatetime::from_integer<DatelikeParseMode::STRICT,
516520
IsDatetime ? DatelikeTargetType::DATE_TIME
@@ -556,12 +560,16 @@ Status DataTypeDateSerDe<T>::from_float_batch(const typename FloatDataType::Colu
556560
template <PrimitiveType T>
557561
template <typename FloatDataType>
558562
Status DataTypeDateSerDe<T>::from_float_strict_mode_batch(
559-
const typename FloatDataType::ColumnType& float_col, IColumn& target_col) const {
563+
const typename FloatDataType::ColumnType& float_col, IColumn& target_col,
564+
const NullMap::value_type* null_map) const {
560565
auto& col_data = assert_cast<ColumnType&>(target_col);
561566
col_data.resize(float_col.size());
562567

563568
CastParameters params {.status = Status::OK(), .is_strict = true};
564569
for (size_t i = 0; i < float_col.size(); ++i) {
570+
if (null_map && null_map[i]) {
571+
continue;
572+
}
565573
CppType val;
566574
CastToDateOrDatetime::from_float<DatelikeParseMode::STRICT,
567575
IsDatetime ? DatelikeTargetType::DATE_TIME
@@ -609,12 +617,16 @@ Status DataTypeDateSerDe<T>::from_decimal_batch(
609617
template <PrimitiveType T>
610618
template <typename DecimalDataType>
611619
Status DataTypeDateSerDe<T>::from_decimal_strict_mode_batch(
612-
const typename DecimalDataType::ColumnType& decimal_col, IColumn& target_col) const {
620+
const typename DecimalDataType::ColumnType& decimal_col, IColumn& target_col,
621+
const NullMap::value_type* null_map) const {
613622
auto& col_data = assert_cast<ColumnType&>(target_col);
614623
col_data.resize(decimal_col.size());
615624

616625
CastParameters params {.status = Status::OK(), .is_strict = true};
617626
for (size_t i = 0; i < decimal_col.size(); ++i) {
627+
if (null_map && null_map[i]) {
628+
continue;
629+
}
618630
CppType val;
619631
CastToDateOrDatetime::from_decimal<DatelikeParseMode::STRICT,
620632
IsDatetime ? DatelikeTargetType::DATE_TIME
@@ -665,23 +677,30 @@ template Status DataTypeDateSerDe<TYPE_DATE>::from_int_batch<DataTypeInt64>(
665677
template Status DataTypeDateSerDe<TYPE_DATE>::from_int_batch<DataTypeInt128>(
666678
const DataTypeInt128::ColumnType& int_col, ColumnNullable& target_col) const;
667679
template Status DataTypeDateSerDe<TYPE_DATE>::from_int_strict_mode_batch<DataTypeInt8>(
668-
const DataTypeInt8::ColumnType& int_col, IColumn& target_col) const;
680+
const DataTypeInt8::ColumnType& int_col, IColumn& target_col,
681+
const NullMap::value_type* null_map) const;
669682
template Status DataTypeDateSerDe<TYPE_DATE>::from_int_strict_mode_batch<DataTypeInt16>(
670-
const DataTypeInt16::ColumnType& int_col, IColumn& target_col) const;
683+
const DataTypeInt16::ColumnType& int_col, IColumn& target_col,
684+
const NullMap::value_type* null_map) const;
671685
template Status DataTypeDateSerDe<TYPE_DATE>::from_int_strict_mode_batch<DataTypeInt32>(
672-
const DataTypeInt32::ColumnType& int_col, IColumn& target_col) const;
686+
const DataTypeInt32::ColumnType& int_col, IColumn& target_col,
687+
const NullMap::value_type* null_map) const;
673688
template Status DataTypeDateSerDe<TYPE_DATE>::from_int_strict_mode_batch<DataTypeInt64>(
674-
const DataTypeInt64::ColumnType& int_col, IColumn& target_col) const;
689+
const DataTypeInt64::ColumnType& int_col, IColumn& target_col,
690+
const NullMap::value_type* null_map) const;
675691
template Status DataTypeDateSerDe<TYPE_DATE>::from_int_strict_mode_batch<DataTypeInt128>(
676-
const DataTypeInt128::ColumnType& int_col, IColumn& target_col) const;
692+
const DataTypeInt128::ColumnType& int_col, IColumn& target_col,
693+
const NullMap::value_type* null_map) const;
677694
template Status DataTypeDateSerDe<TYPE_DATE>::from_float_batch<DataTypeFloat32>(
678695
const DataTypeFloat32::ColumnType& float_col, ColumnNullable& target_col) const;
679696
template Status DataTypeDateSerDe<TYPE_DATE>::from_float_batch<DataTypeFloat64>(
680697
const DataTypeFloat64::ColumnType& float_col, ColumnNullable& target_col) const;
681698
template Status DataTypeDateSerDe<TYPE_DATE>::from_float_strict_mode_batch<DataTypeFloat32>(
682-
const DataTypeFloat32::ColumnType& float_col, IColumn& target_col) const;
699+
const DataTypeFloat32::ColumnType& float_col, IColumn& target_col,
700+
const NullMap::value_type* null_map) const;
683701
template Status DataTypeDateSerDe<TYPE_DATE>::from_float_strict_mode_batch<DataTypeFloat64>(
684-
const DataTypeFloat64::ColumnType& float_col, IColumn& target_col) const;
702+
const DataTypeFloat64::ColumnType& float_col, IColumn& target_col,
703+
const NullMap::value_type* null_map) const;
685704
template Status DataTypeDateSerDe<TYPE_DATE>::from_decimal_batch<DataTypeDecimal32>(
686705
const DataTypeDecimal32::ColumnType& decimal_col, ColumnNullable& target_col) const;
687706
template Status DataTypeDateSerDe<TYPE_DATE>::from_decimal_batch<DataTypeDecimal64>(
@@ -693,15 +712,20 @@ template Status DataTypeDateSerDe<TYPE_DATE>::from_decimal_batch<DataTypeDecimal
693712
template Status DataTypeDateSerDe<TYPE_DATE>::from_decimal_batch<DataTypeDecimal256>(
694713
const DataTypeDecimal256::ColumnType& decimal_col, ColumnNullable& target_col) const;
695714
template Status DataTypeDateSerDe<TYPE_DATE>::from_decimal_strict_mode_batch<DataTypeDecimal32>(
696-
const DataTypeDecimal32::ColumnType& decimal_col, IColumn& target_col) const;
715+
const DataTypeDecimal32::ColumnType& decimal_col, IColumn& target_col,
716+
const NullMap::value_type* null_map) const;
697717
template Status DataTypeDateSerDe<TYPE_DATE>::from_decimal_strict_mode_batch<DataTypeDecimal64>(
698-
const DataTypeDecimal64::ColumnType& decimal_col, IColumn& target_col) const;
718+
const DataTypeDecimal64::ColumnType& decimal_col, IColumn& target_col,
719+
const NullMap::value_type* null_map) const;
699720
template Status DataTypeDateSerDe<TYPE_DATE>::from_decimal_strict_mode_batch<DataTypeDecimalV2>(
700-
const DataTypeDecimalV2::ColumnType& decimal_col, IColumn& target_col) const;
721+
const DataTypeDecimalV2::ColumnType& decimal_col, IColumn& target_col,
722+
const NullMap::value_type* null_map) const;
701723
template Status DataTypeDateSerDe<TYPE_DATE>::from_decimal_strict_mode_batch<DataTypeDecimal128>(
702-
const DataTypeDecimal128::ColumnType& decimal_col, IColumn& target_col) const;
724+
const DataTypeDecimal128::ColumnType& decimal_col, IColumn& target_col,
725+
const NullMap::value_type* null_map) const;
703726
template Status DataTypeDateSerDe<TYPE_DATE>::from_decimal_strict_mode_batch<DataTypeDecimal256>(
704-
const DataTypeDecimal256::ColumnType& decimal_col, IColumn& target_col) const;
727+
const DataTypeDecimal256::ColumnType& decimal_col, IColumn& target_col,
728+
const NullMap::value_type* null_map) const;
705729

706730
template Status DataTypeDateSerDe<TYPE_DATETIME>::from_int_batch<DataTypeInt8>(
707731
const DataTypeInt8::ColumnType& int_col, ColumnNullable& target_col) const;
@@ -714,23 +738,30 @@ template Status DataTypeDateSerDe<TYPE_DATETIME>::from_int_batch<DataTypeInt64>(
714738
template Status DataTypeDateSerDe<TYPE_DATETIME>::from_int_batch<DataTypeInt128>(
715739
const DataTypeInt128::ColumnType& int_col, ColumnNullable& target_col) const;
716740
template Status DataTypeDateSerDe<TYPE_DATETIME>::from_int_strict_mode_batch<DataTypeInt8>(
717-
const DataTypeInt8::ColumnType& int_col, IColumn& target_col) const;
741+
const DataTypeInt8::ColumnType& int_col, IColumn& target_col,
742+
const NullMap::value_type* null_map) const;
718743
template Status DataTypeDateSerDe<TYPE_DATETIME>::from_int_strict_mode_batch<DataTypeInt16>(
719-
const DataTypeInt16::ColumnType& int_col, IColumn& target_col) const;
744+
const DataTypeInt16::ColumnType& int_col, IColumn& target_col,
745+
const NullMap::value_type* null_map) const;
720746
template Status DataTypeDateSerDe<TYPE_DATETIME>::from_int_strict_mode_batch<DataTypeInt32>(
721-
const DataTypeInt32::ColumnType& int_col, IColumn& target_col) const;
747+
const DataTypeInt32::ColumnType& int_col, IColumn& target_col,
748+
const NullMap::value_type* null_map) const;
722749
template Status DataTypeDateSerDe<TYPE_DATETIME>::from_int_strict_mode_batch<DataTypeInt64>(
723-
const DataTypeInt64::ColumnType& int_col, IColumn& target_col) const;
750+
const DataTypeInt64::ColumnType& int_col, IColumn& target_col,
751+
const NullMap::value_type* null_map) const;
724752
template Status DataTypeDateSerDe<TYPE_DATETIME>::from_int_strict_mode_batch<DataTypeInt128>(
725-
const DataTypeInt128::ColumnType& int_col, IColumn& target_col) const;
753+
const DataTypeInt128::ColumnType& int_col, IColumn& target_col,
754+
const NullMap::value_type* null_map) const;
726755
template Status DataTypeDateSerDe<TYPE_DATETIME>::from_float_batch<DataTypeFloat32>(
727756
const DataTypeFloat32::ColumnType& float_col, ColumnNullable& target_col) const;
728757
template Status DataTypeDateSerDe<TYPE_DATETIME>::from_float_batch<DataTypeFloat64>(
729758
const DataTypeFloat64::ColumnType& float_col, ColumnNullable& target_col) const;
730759
template Status DataTypeDateSerDe<TYPE_DATETIME>::from_float_strict_mode_batch<DataTypeFloat32>(
731-
const DataTypeFloat32::ColumnType& float_col, IColumn& target_col) const;
760+
const DataTypeFloat32::ColumnType& float_col, IColumn& target_col,
761+
const NullMap::value_type* null_map) const;
732762
template Status DataTypeDateSerDe<TYPE_DATETIME>::from_float_strict_mode_batch<DataTypeFloat64>(
733-
const DataTypeFloat64::ColumnType& float_col, IColumn& target_col) const;
763+
const DataTypeFloat64::ColumnType& float_col, IColumn& target_col,
764+
const NullMap::value_type* null_map) const;
734765
template Status DataTypeDateSerDe<TYPE_DATETIME>::from_decimal_batch<DataTypeDecimal32>(
735766
const DataTypeDecimal32::ColumnType& decimal_col, ColumnNullable& target_col) const;
736767
template Status DataTypeDateSerDe<TYPE_DATETIME>::from_decimal_batch<DataTypeDecimal64>(
@@ -742,15 +773,18 @@ template Status DataTypeDateSerDe<TYPE_DATETIME>::from_decimal_batch<DataTypeDec
742773
template Status DataTypeDateSerDe<TYPE_DATETIME>::from_decimal_batch<DataTypeDecimal256>(
743774
const DataTypeDecimal256::ColumnType& decimal_col, ColumnNullable& target_col) const;
744775
template Status DataTypeDateSerDe<TYPE_DATETIME>::from_decimal_strict_mode_batch<DataTypeDecimal32>(
745-
const DataTypeDecimal32::ColumnType& decimal_col, IColumn& target_col) const;
776+
const DataTypeDecimal32::ColumnType& decimal_col, IColumn& target_col,
777+
const NullMap::value_type* null_map) const;
746778
template Status DataTypeDateSerDe<TYPE_DATETIME>::from_decimal_strict_mode_batch<DataTypeDecimal64>(
747-
const DataTypeDecimal64::ColumnType& decimal_col, IColumn& target_col) const;
779+
const DataTypeDecimal64::ColumnType& decimal_col, IColumn& target_col,
780+
const NullMap::value_type* null_map) const;
748781
template Status DataTypeDateSerDe<TYPE_DATETIME>::from_decimal_strict_mode_batch<DataTypeDecimalV2>(
749-
const DataTypeDecimalV2::ColumnType& decimal_col, IColumn& target_col) const;
750-
template Status
751-
DataTypeDateSerDe<TYPE_DATETIME>::from_decimal_strict_mode_batch<DataTypeDecimal128>(
752-
const DataTypeDecimal128::ColumnType& decimal_col, IColumn& target_col) const;
753-
template Status
754-
DataTypeDateSerDe<TYPE_DATETIME>::from_decimal_strict_mode_batch<DataTypeDecimal256>(
755-
const DataTypeDecimal256::ColumnType& decimal_col, IColumn& target_col) const;
782+
const DataTypeDecimalV2::ColumnType& decimal_col, IColumn& target_col,
783+
const NullMap::value_type* null_map) const;
784+
template Status DataTypeDateSerDe<TYPE_DATETIME>::from_decimal_strict_mode_batch<
785+
DataTypeDecimal128>(const DataTypeDecimal128::ColumnType& decimal_col, IColumn& target_col,
786+
const NullMap::value_type* null_map) const;
787+
template Status DataTypeDateSerDe<TYPE_DATETIME>::from_decimal_strict_mode_batch<
788+
DataTypeDecimal256>(const DataTypeDecimal256::ColumnType& decimal_col, IColumn& target_col,
789+
const NullMap::value_type* null_map) const;
756790
} // namespace doris

‎be/src/core/data_type_serde/data_type_date_or_datetime_serde.h‎

Lines changed: 6 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -69,21 +69,24 @@ class DataTypeDateSerDe : public DataTypeNumberSerDe<T> {
6969
ColumnNullable& target_col) const;
7070
template <typename IntDataType>
7171
Status from_int_strict_mode_batch(const typename IntDataType::ColumnType& int_col,
72-
IColumn& target_col) const;
72+
IColumn& target_col,
73+
const NullMap::value_type* null_map = nullptr) const;
7374

7475
template <typename FloatDataType>
7576
Status from_float_batch(const typename FloatDataType::ColumnType& float_col,
7677
ColumnNullable& target_col) const;
7778
template <typename FloatDataType>
7879
Status from_float_strict_mode_batch(const typename FloatDataType::ColumnType& float_col,
79-
IColumn& target_col) const;
80+
IColumn& target_col,
81+
const NullMap::value_type* null_map = nullptr) const;
8082

8183
template <typename DecimalDataType>
8284
Status from_decimal_batch(const typename DecimalDataType::ColumnType& decimal_col,
8385
ColumnNullable& target_col) const;
8486
template <typename DecimalDataType>
8587
Status from_decimal_strict_mode_batch(const typename DecimalDataType::ColumnType& decimal_col,
86-
IColumn& target_col) const;
88+
IColumn& target_col,
89+
const NullMap::value_type* null_map = nullptr) const;
8790

8891
Status serialize_one_cell_to_json(
8992
const IColumn& column, int64_t row_num, BufferWritable& bw,

0 commit comments

Comments
 (0)