Skip to content

Commit 5945bab

Browse files
committed
[fix](arrow) correct year-zero DATE conversion at format boundaries
1 parent c6536cf commit 5945bab

9 files changed

Lines changed: 879 additions & 37 deletions

File tree

‎be/src/core/data_type_serde/data_type_datev2_serde.cpp‎

Lines changed: 49 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,7 @@
2222
#include <fmt/core.h>
2323

2424
#include <cstdint>
25+
#include <limits>
2526
#include <vector>
2627

2728
#include "common/config.h"
@@ -42,13 +43,9 @@
4243
namespace doris {
4344

4445
#include "common/compile_check_begin.h"
45-
// This number represents the number of days from 0000-01-01 to 1970-01-01.
46-
static constexpr int32_t date_threshold = 719528;
4746

4847
namespace {
4948

50-
constexpr int32_t DORIS_DATE_EPOCH_DAYNR = 719528;
51-
5249
Status decode_date_orc_values(const DataTypeSerDe& serde, IColumn& column,
5350
const OrcDecodedColumnView& orc_view) {
5451
const auto* orc_batch = dynamic_cast<const ::orc::LongVectorBatch*>(orc_view.batch);
@@ -65,22 +62,44 @@ Status decode_date_orc_values(const DataTypeSerDe& serde, IColumn& column,
6562
orc_serde_utils::orc_decode_row_count(orc_view.rows, orc_view.selected_rows);
6663
std::vector<int32_t> date_values;
6764
date_values.resize(output_rows);
68-
auto& date_dict = date_day_offset_dict::get();
6965
for (size_t row = 0; row < output_rows; ++row) {
7066
const auto source_row = orc_serde_utils::orc_source_row_at(row, orc_view.selected_rows);
71-
const auto date = date_dict[cast_set<int>(orc_batch->data[source_row])];
72-
date_values[row] = cast_set<int32_t>(date.daynr() - DORIS_DATE_EPOCH_DAYNR);
67+
// The payload of a null slot is undefined, so it is zeroed rather than range-checked;
68+
// `read_column_from_decoded_values` never decodes it.
69+
if (view.null_map != nullptr && view.null_map[row] != 0) {
70+
date_values[row] = 0;
71+
continue;
72+
}
73+
// ORC DATE is days since 1970-01-01 in the proleptic Gregorian calendar, the same encoding
74+
// `decode_epoch_days()` expects, but ORC hands it over in an int64 batch. Reject a value
75+
// that does not fit the int32 view here, while the real file value is still available for
76+
// the message, instead of narrowing it blindly.
77+
const int64_t file_days = orc_batch->data[source_row];
78+
if (file_days < std::numeric_limits<int32_t>::min() ||
79+
file_days > std::numeric_limits<int32_t>::max()) {
80+
return Status::DataQualityError(
81+
"DATE value {} is outside the Doris DATE range (0000-01-01..9999-12-31, "
82+
"0000-02-29 excluded)",
83+
file_days);
84+
}
85+
date_values[row] = static_cast<int32_t>(file_days);
7386
}
7487
view.values = reinterpret_cast<const uint8_t*>(date_values.data());
7588
RETURN_IF_ERROR(orc_serde_utils::read_decoded_values(serde, column, &view));
7689
return Status::OK();
7790
}
7891

79-
Status decode_parquet_date(int32_t encoded_date, DateV2Value<DateV2ValueType>* value) {
92+
// Shared by every "days since 1970-01-01" source: Parquet, ORC and Arrow date32/date64. The
93+
// message must therefore not name one format, and it carries the value so a broken file can be
94+
// identified without re-reading it.
95+
Status decode_epoch_days(int64_t encoded_date, DateV2Value<DateV2ValueType>* value) {
8096
DORIS_CHECK(value != nullptr);
81-
const int64_t day_number = static_cast<int64_t>(encoded_date) + date_threshold;
82-
if (day_number < 0 || !value->get_date_from_daynr(static_cast<uint64_t>(day_number))) {
83-
return Status::DataQualityError("Parquet DATE value is out of range");
97+
const int64_t day_number = epoch_days_to_daynr(encoded_date);
98+
if (day_number == 0 || !value->get_date_from_daynr(static_cast<uint64_t>(day_number))) {
99+
return Status::DataQualityError(
100+
"DATE value {} is outside the Doris DATE range (0000-01-01..9999-12-31, "
101+
"0000-02-29 excluded)",
102+
encoded_date);
84103
}
85104
return Status::OK();
86105
}
@@ -100,7 +119,7 @@ class DateV2ParquetConsumer final : public ParquetFixedValueConsumer {
100119
_data.resize(old_size + num_values);
101120
for (size_t row = 0; row < num_values; ++row) {
102121
DateV2Value<DateV2ValueType> value;
103-
const auto status = decode_parquet_date(
122+
const auto status = decode_epoch_days(
104123
unaligned_load<int32_t>(values + row * sizeof(int32_t)), &value);
105124
if (!status.ok()) {
106125
if (_state != nullptr && _state->mark_conversion_failure(old_size + row)) {
@@ -217,13 +236,12 @@ Status DataTypeDateV2SerDe::write_column_to_arrow(const IColumn& column, const N
217236
const auto& col_data = static_cast<const ColumnDateV2&>(column).get_data();
218237
auto& date32_builder = assert_cast<arrow::Date32Builder&>(*array_builder);
219238
for (size_t i = start; i < end; ++i) {
220-
auto daynr = col_data[i].daynr() - date_threshold;
221239
if (null_map && (*null_map)[i]) {
222240
RETURN_IF_ERROR(checkArrowStatus(date32_builder.AppendNull(), column, *array_builder));
223241
} else {
224-
RETURN_IF_ERROR(
225-
checkArrowStatus(date32_builder.Append(cast_set<int, int64_t, false>(daynr)),
226-
column, *array_builder));
242+
RETURN_IF_ERROR(checkArrowStatus(
243+
date32_builder.Append(daynr_to_epoch_days(col_data[i].daynr())), column,
244+
*array_builder));
227245
}
228246
}
229247
return Status::OK();
@@ -258,10 +276,9 @@ Status DataTypeDateV2SerDe::read_column_from_arrow(IColumn& column, const arrow:
258276
"Arrow Date64 value must contain whole days: row={}, milliseconds={}",
259277
value_i, milliseconds);
260278
}
261-
const int64_t daynr = milliseconds / MILLISECONDS_PER_DAY + date_threshold;
262279
DateV2Value<DateV2ValueType> value;
263-
if (daynr <= 0 || daynr > DATE_MAX_DAYNR ||
264-
!value.get_date_from_daynr(static_cast<uint64_t>(daynr))) {
280+
if (const auto status = decode_epoch_days(milliseconds / MILLISECONDS_PER_DAY, &value);
281+
!status.ok()) {
265282
return Status::InvalidArgument(
266283
"Arrow Date64 value is outside the Doris DATE range: "
267284
"row={}, milliseconds={}",
@@ -278,12 +295,20 @@ Status DataTypeDateV2SerDe::read_column_from_arrow(IColumn& column, const arrow:
278295
const auto* base_ptr = reinterpret_cast<const uint8_t*>(concrete_array->raw_values());
279296
const size_t element_size = sizeof(int32_t);
280297
for (auto value_i = start; value_i < end; ++value_i) {
281-
int32_t date_value = 0;
298+
// A null slot has no calendar value and its payload may be outside the DATE range.
299+
if (concrete_array->IsNull(value_i)) {
300+
col_data.emplace_back(DateV2Value<DateV2ValueType>());
301+
continue;
302+
}
282303
const uint8_t* raw_byte_ptr = base_ptr + value_i * element_size;
283-
memcpy(&date_value, raw_byte_ptr, element_size);
304+
auto date_value = unaligned_load<int32_t>(raw_byte_ptr);
284305

285306
DateV2Value<DateV2ValueType> v;
286-
v.get_date_from_daynr(date_value + date_threshold);
307+
if (const auto status = decode_epoch_days(date_value, &v); !status.ok()) {
308+
return Status::InvalidArgument(
309+
"Arrow Date32 value is outside the Doris DATE range: row={}, days={}",
310+
value_i, date_value);
311+
}
287312
col_data.emplace_back(v);
288313
}
289314
} else {
@@ -311,7 +336,7 @@ Status DataTypeDateV2SerDe::read_column_from_decoded_values(IColumn& column,
311336
continue;
312337
}
313338
DateV2Value<DateV2ValueType> date_v2;
314-
const auto status = decode_parquet_date(values[row], &date_v2);
339+
const auto status = decode_epoch_days(values[row], &date_v2);
315340
if (!status.ok()) {
316341
// Decoded values back both metadata conversion and native materialization. Preserve
317342
// strict errors while allowing nullable non-strict scans to mark only the bad row.
@@ -403,7 +428,7 @@ Status DataTypeDateV2SerDe::write_column_to_orc(const std::string& timezone, con
403428
if (cur_batch->notNull[row_id] == 0) {
404429
continue;
405430
}
406-
cur_batch->data[row_id] = col_data[row_id].daynr() - date_threshold;
431+
cur_batch->data[row_id] = daynr_to_epoch_days(col_data[row_id].daynr());
407432
}
408433
cur_batch->numElements = end - start;
409434
return Status::OK();

‎be/src/core/value/vdatetime_value.h‎

Lines changed: 44 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1697,8 +1697,6 @@ class date_day_offset_dict {
16971697

16981698
static constexpr int START_YEAR = 1900; // 1900-01-01
16991699
static constexpr int END_YEAR = 2039; // 2039-10-24
1700-
static constexpr int DAY_OFFSET_CAL_START_POINT_DAYNR =
1701-
719528; // 1970-01-01 (start from 0000-01-01, 0000-01-01 is day 1, returns 1)
17021700

17031701
static std::array<DateV2Value<DateV2ValueType>, DICT_DAYS> DATE_DAY_OFFSET_ITEMS;
17041702
static std::array<std::array<std::array<int, 31>, 12>, 140> DATE_DAY_OFFSET_DICT;
@@ -1713,6 +1711,9 @@ class date_day_offset_dict {
17131711
date_day_offset_dict& operator=(const date_day_offset_dict&) = default;
17141712

17151713
public:
1714+
static constexpr int DAY_OFFSET_CAL_START_POINT_DAYNR =
1715+
719528; // 1970-01-01 (start from 0000-01-01, 0000-01-01 is day 1, returns 1)
1716+
17161717
static bool can_speed_up_calc_daynr(int year) { return year >= START_YEAR && year <= END_YEAR; }
17171718

17181719
static int get_offset_by_daynr(int daynr) { return daynr - DAY_OFFSET_CAL_START_POINT_DAYNR; }
@@ -1773,6 +1774,47 @@ inline uint32_t calc_daynr(uint16_t year, uint8_t month, uint8_t day) {
17731774
return delsum + y / 4 - y / 100 + y / 400;
17741775
}
17751776

1777+
// Doris follows MySQL's calendar, in which year 0 is NOT a leap year (see `is_leap()`): 0000-02-29
1778+
// does not exist and `calc_daynr()` therefore numbers 0000-01-01 .. 0000-02-28 one day ahead of the
1779+
// proleptic Gregorian calendar. Arrow `date32`, Parquet/ORC `DATE` and the Iceberg spec all define
1780+
// their day ordinal in the proleptic Gregorian calendar, where year 0 IS a leap year. The two
1781+
// numberings coincide from 0000-03-01 (daynr 60) onwards, so the whole difference is the missing
1782+
// 0000-02-29. Every conversion between a Doris DATE and an external "days since 1970-01-01" value
1783+
// must go through the two helpers below instead of adding/subtracting the epoch daynr directly.
1784+
inline constexpr int64_t DAYNR_OF_UNIX_EPOCH =
1785+
date_day_offset_dict::DAY_OFFSET_CAL_START_POINT_DAYNR;
1786+
inline constexpr int64_t DAYNR_OF_0000_03_01 = 60; // first daynr shared by both calendars
1787+
1788+
// Doris daynr -> days since 1970-01-01 in the proleptic Gregorian calendar.
1789+
inline constexpr int32_t daynr_to_epoch_days(int64_t daynr) {
1790+
return static_cast<int32_t>(daynr - DAYNR_OF_UNIX_EPOCH -
1791+
(daynr < DAYNR_OF_0000_03_01 ? 1 : 0));
1792+
}
1793+
1794+
// The three boundaries of the external ordinal domain, derived from the daynr domain rather than
1795+
// typed out, so a change to either calendar constant cannot leave them behind.
1796+
inline constexpr int64_t EPOCH_DAYS_MIN = daynr_to_epoch_days(1); // 0000-01-01, smallest Doris DATE
1797+
inline constexpr int64_t EPOCH_DAYS_0000_02_29 =
1798+
daynr_to_epoch_days(DAYNR_OF_0000_03_01) - 1; // exists in proleptic Gregorian only
1799+
inline constexpr int64_t EPOCH_DAYS_MAX =
1800+
daynr_to_epoch_days(DATE_MAX_DAYNR); // 9999-12-31, the largest Doris DATE
1801+
1802+
static_assert(DAYNR_OF_UNIX_EPOCH == 719528, "calc_daynr(1970, 1, 1)");
1803+
static_assert(EPOCH_DAYS_MIN == -719528);
1804+
static_assert(EPOCH_DAYS_0000_02_29 == -719469);
1805+
static_assert(EPOCH_DAYS_MAX == 2932896);
1806+
1807+
// Inverse of `daynr_to_epoch_days()`. Returns 0, which is never a valid daynr, when the value has
1808+
// no Doris DATE representation: before 0000-01-01, after 9999-12-31, or the proleptic-only
1809+
// 0000-02-29. Callers must reject 0 instead of feeding it to `get_date_from_daynr()`.
1810+
inline constexpr int64_t epoch_days_to_daynr(int64_t epoch_days) {
1811+
if (epoch_days < EPOCH_DAYS_MIN || epoch_days > EPOCH_DAYS_MAX ||
1812+
epoch_days == EPOCH_DAYS_0000_02_29) {
1813+
return 0;
1814+
}
1815+
return epoch_days + DAYNR_OF_UNIX_EPOCH + (epoch_days < EPOCH_DAYS_0000_02_29 ? 1 : 0);
1816+
}
1817+
17761818
// DAY / WEEK fast path. Real workloads hold dates that cluster in a narrow range, so the two
17771819
// dictionary lookups below (`daynr(y, m, d)` and `daynr -> date`) are L1-resident; measured about
17781820
// 3x faster than the generic `date_add_interval<DAY>` (no TimeInterval, no second-level

‎be/src/format_v2/orc/orc_reader.cpp‎

Lines changed: 29 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -487,10 +487,26 @@ bool set_date_zone_map(const ::orc::ColumnStatistics& statistics, segment_v2::Zo
487487
!date_statistics->hasMaximum()) {
488488
return false;
489489
}
490-
auto& date_dict = date_day_offset_dict::get();
491-
return set_validated_zone_map(
492-
Field::create_field<TYPE_DATEV2>(date_dict[date_statistics->getMinimum()]),
493-
Field::create_field<TYPE_DATEV2>(date_dict[date_statistics->getMaximum()]), zone_map);
490+
// ORC DATE statistics are proleptic-Gregorian day ordinals, the same domain the row decoder
491+
// (DataTypeDateV2SerDe::read_column_from_orc) interprets. Converting them through
492+
// `date_day_offset_dict` instead would put the year-zero window one day off the rows and let a
493+
// pushed-down MIN/MAX report a value no row holds. A bound with no Doris DATE disables the
494+
// statistics, so MIN/MAX falls back to scanning rows.
495+
const auto to_date = [](int64_t epoch_days) -> std::optional<DateV2Value<DateV2ValueType>> {
496+
const int64_t daynr = epoch_days_to_daynr(epoch_days);
497+
DateV2Value<DateV2ValueType> value;
498+
if (daynr == 0 || !value.get_date_from_daynr(static_cast<uint64_t>(daynr))) {
499+
return std::nullopt;
500+
}
501+
return value;
502+
};
503+
const auto min_value = to_date(date_statistics->getMinimum());
504+
const auto max_value = to_date(date_statistics->getMaximum());
505+
if (!min_value.has_value() || !max_value.has_value()) {
506+
return false;
507+
}
508+
return set_validated_zone_map(Field::create_field<TYPE_DATEV2>(*min_value),
509+
Field::create_field<TYPE_DATEV2>(*max_value), zone_map);
494510
}
495511

496512
std::optional<DateV2Value<DateTimeV2ValueType>> datetime_v2_from_orc_millis(
@@ -2409,8 +2425,15 @@ Status OrcReader::_decode_column_into_block(const ::orc::StructVectorBatch& stru
24092425
const auto* selected_type = _state->selected_type->getSubtype(selected_batch_idx);
24102426
DORIS_CHECK(selected_type != nullptr);
24112427
auto column = file_block->get_by_position(block_position.value()).column->assert_mutable();
2412-
RETURN_IF_ERROR(_decode_column(*type, *selected_type, *struct_batch.fields[selected_batch_idx],
2413-
column, rows, selected_rows));
2428+
auto status = _decode_column(*type, *selected_type, *struct_batch.fields[selected_batch_idx],
2429+
column, rows, selected_rows);
2430+
if (!status.ok()) {
2431+
// The decoders work on a bare value buffer and cannot name the column themselves; without
2432+
// this the user only learns that some DATE/TIMESTAMP in some file is unrepresentable.
2433+
return status.prepend(fmt::format(
2434+
"Failed to decode ORC column '{}': ",
2435+
_state->root_type->getFieldName(static_cast<uint64_t>(file_column_id.value()))));
2436+
}
24142437
file_block->replace_by_position(block_position.value(), std::move(column));
24152438
return Status::OK();
24162439
}

0 commit comments

Comments
 (0)