2222#include < fmt/core.h>
2323
2424#include < cstdint>
25+ #include < limits>
2526#include < vector>
2627
2728#include " common/config.h"
4243namespace doris {
4344
4445#include " common/compile_check_begin.h"
45- // This number represents the number of days from 0000-01-01 to 1970-01-01.
46- static constexpr int32_t date_threshold = 719528 ;
4746
4847namespace {
4948
50- constexpr int32_t DORIS_DATE_EPOCH_DAYNR = 719528 ;
51-
5249Status decode_date_orc_values (const DataTypeSerDe& serde, IColumn& column,
5350 const OrcDecodedColumnView& orc_view) {
5451 const auto * orc_batch = dynamic_cast <const ::orc::LongVectorBatch*>(orc_view.batch );
@@ -65,22 +62,44 @@ Status decode_date_orc_values(const DataTypeSerDe& serde, IColumn& column,
6562 orc_serde_utils::orc_decode_row_count (orc_view.rows , orc_view.selected_rows );
6663 std::vector<int32_t > date_values;
6764 date_values.resize (output_rows);
68- auto & date_dict = date_day_offset_dict::get ();
6965 for (size_t row = 0 ; row < output_rows; ++row) {
7066 const auto source_row = orc_serde_utils::orc_source_row_at (row, orc_view.selected_rows );
71- const auto date = date_dict[cast_set<int >(orc_batch->data [source_row])];
72- date_values[row] = cast_set<int32_t >(date.daynr () - DORIS_DATE_EPOCH_DAYNR );
67+ // The payload of a null slot is undefined, so it is zeroed rather than range-checked;
68+ // `read_column_from_decoded_values` never decodes it.
69+ if (view.null_map != nullptr && view.null_map [row] != 0 ) {
70+ date_values[row] = 0 ;
71+ continue ;
72+ }
73+ // ORC DATE is days since 1970-01-01 in the proleptic Gregorian calendar, the same encoding
74+ // `decode_epoch_days()` expects, but ORC hands it over in an int64 batch. Reject a value
75+ // that does not fit the int32 view here, while the real file value is still available for
76+ // the message, instead of narrowing it blindly.
77+ const int64_t file_days = orc_batch->data [source_row];
78+ if (file_days < std::numeric_limits<int32_t >::min () ||
79+ file_days > std::numeric_limits<int32_t >::max ()) {
80+ return Status::DataQualityError (
81+ " DATE value {} is outside the Doris DATE range (0000-01-01..9999-12-31, "
82+ " 0000-02-29 excluded)" ,
83+ file_days);
84+ }
85+ date_values[row] = static_cast <int32_t >(file_days);
7386 }
7487 view.values = reinterpret_cast <const uint8_t *>(date_values.data ());
7588 RETURN_IF_ERROR (orc_serde_utils::read_decoded_values (serde, column, &view));
7689 return Status::OK ();
7790}
7891
79- Status decode_parquet_date (int32_t encoded_date, DateV2Value<DateV2ValueType>* value) {
92+ // Shared by every "days since 1970-01-01" source: Parquet, ORC and Arrow date32/date64. The
93+ // message must therefore not name one format, and it carries the value so a broken file can be
94+ // identified without re-reading it.
95+ Status decode_epoch_days (int64_t encoded_date, DateV2Value<DateV2ValueType>* value) {
8096 DORIS_CHECK (value != nullptr );
81- const int64_t day_number = static_cast <int64_t >(encoded_date) + date_threshold;
82- if (day_number < 0 || !value->get_date_from_daynr (static_cast <uint64_t >(day_number))) {
83- return Status::DataQualityError (" Parquet DATE value is out of range" );
97+ const int64_t day_number = epoch_days_to_daynr (encoded_date);
98+ if (day_number == 0 || !value->get_date_from_daynr (static_cast <uint64_t >(day_number))) {
99+ return Status::DataQualityError (
100+ " DATE value {} is outside the Doris DATE range (0000-01-01..9999-12-31, "
101+ " 0000-02-29 excluded)" ,
102+ encoded_date);
84103 }
85104 return Status::OK ();
86105}
@@ -100,7 +119,7 @@ class DateV2ParquetConsumer final : public ParquetFixedValueConsumer {
100119 _data.resize (old_size + num_values);
101120 for (size_t row = 0 ; row < num_values; ++row) {
102121 DateV2Value<DateV2ValueType> value;
103- const auto status = decode_parquet_date (
122+ const auto status = decode_epoch_days (
104123 unaligned_load<int32_t >(values + row * sizeof (int32_t )), &value);
105124 if (!status.ok ()) {
106125 if (_state != nullptr && _state->mark_conversion_failure (old_size + row)) {
@@ -217,13 +236,12 @@ Status DataTypeDateV2SerDe::write_column_to_arrow(const IColumn& column, const N
217236 const auto & col_data = static_cast <const ColumnDateV2&>(column).get_data ();
218237 auto & date32_builder = assert_cast<arrow::Date32Builder&>(*array_builder);
219238 for (size_t i = start; i < end; ++i) {
220- auto daynr = col_data[i].daynr () - date_threshold;
221239 if (null_map && (*null_map)[i]) {
222240 RETURN_IF_ERROR (checkArrowStatus (date32_builder.AppendNull (), column, *array_builder));
223241 } else {
224- RETURN_IF_ERROR (
225- checkArrowStatus ( date32_builder.Append (cast_set< int , int64_t , false >( daynr)) ,
226- column, *array_builder));
242+ RETURN_IF_ERROR (checkArrowStatus (
243+ date32_builder.Append (daynr_to_epoch_days (col_data[i]. daynr ())), column ,
244+ *array_builder));
227245 }
228246 }
229247 return Status::OK ();
@@ -258,10 +276,9 @@ Status DataTypeDateV2SerDe::read_column_from_arrow(IColumn& column, const arrow:
258276 " Arrow Date64 value must contain whole days: row={}, milliseconds={}" ,
259277 value_i, milliseconds);
260278 }
261- const int64_t daynr = milliseconds / MILLISECONDS_PER_DAY + date_threshold;
262279 DateV2Value<DateV2ValueType> value;
263- if (daynr <= 0 || daynr > DATE_MAX_DAYNR ||
264- !value. get_date_from_daynr ( static_cast < uint64_t >(daynr) )) {
280+ if (const auto status = decode_epoch_days (milliseconds / MILLISECONDS_PER_DAY , &value);
281+ !status. ok ( )) {
265282 return Status::InvalidArgument (
266283 " Arrow Date64 value is outside the Doris DATE range: "
267284 " row={}, milliseconds={}" ,
@@ -278,12 +295,20 @@ Status DataTypeDateV2SerDe::read_column_from_arrow(IColumn& column, const arrow:
278295 const auto * base_ptr = reinterpret_cast <const uint8_t *>(concrete_array->raw_values ());
279296 const size_t element_size = sizeof (int32_t );
280297 for (auto value_i = start; value_i < end; ++value_i) {
281- int32_t date_value = 0 ;
298+ // A null slot has no calendar value and its payload may be outside the DATE range.
299+ if (concrete_array->IsNull (value_i)) {
300+ col_data.emplace_back (DateV2Value<DateV2ValueType>());
301+ continue ;
302+ }
282303 const uint8_t * raw_byte_ptr = base_ptr + value_i * element_size;
283- memcpy (& date_value, raw_byte_ptr, element_size );
304+ auto date_value = unaligned_load< int32_t >(raw_byte_ptr );
284305
285306 DateV2Value<DateV2ValueType> v;
286- v.get_date_from_daynr (date_value + date_threshold);
307+ if (const auto status = decode_epoch_days (date_value, &v); !status.ok ()) {
308+ return Status::InvalidArgument (
309+ " Arrow Date32 value is outside the Doris DATE range: row={}, days={}" ,
310+ value_i, date_value);
311+ }
287312 col_data.emplace_back (v);
288313 }
289314 } else {
@@ -311,7 +336,7 @@ Status DataTypeDateV2SerDe::read_column_from_decoded_values(IColumn& column,
311336 continue ;
312337 }
313338 DateV2Value<DateV2ValueType> date_v2;
314- const auto status = decode_parquet_date (values[row], &date_v2);
339+ const auto status = decode_epoch_days (values[row], &date_v2);
315340 if (!status.ok ()) {
316341 // Decoded values back both metadata conversion and native materialization. Preserve
317342 // strict errors while allowing nullable non-strict scans to mark only the bad row.
@@ -403,7 +428,7 @@ Status DataTypeDateV2SerDe::write_column_to_orc(const std::string& timezone, con
403428 if (cur_batch->notNull [row_id] == 0 ) {
404429 continue ;
405430 }
406- cur_batch->data [row_id] = col_data[row_id].daynr () - date_threshold ;
431+ cur_batch->data [row_id] = daynr_to_epoch_days ( col_data[row_id].daynr ()) ;
407432 }
408433 cur_batch->numElements = end - start;
409434 return Status::OK ();
0 commit comments