Skip to content

Commit baad197

Browse files
authored
fix: Overflow when reading Timestamp from parquet file (apache#542)
* fix: Overflow when reading Timestamp from parquet file * Add helper method
1 parent a4e268c commit baad197

2 files changed

Lines changed: 37 additions & 33 deletions

File tree

‎core/src/parquet/read/values.rs‎

Lines changed: 28 additions & 32 deletions
Original file line numberDiff line numberDiff line change
@@ -727,6 +727,17 @@ const INT96_SRC_BYTE_WIDTH: usize = 12;
727727
// We convert INT96 to micros and store using i64
728728
const INT96_DST_BYTE_WIDTH: usize = 8;
729729

730+
fn int96_to_microsecond(v: &[u8]) -> i64 {
731+
let nanos = &v[..INT96_DST_BYTE_WIDTH] as *const [u8] as *const u8 as *const i64;
732+
let day = &v[INT96_DST_BYTE_WIDTH..] as *const [u8] as *const u8 as *const i32;
733+
734+
unsafe {
735+
((day.read_unaligned() - JULIAN_DAY_OF_EPOCH) as i64)
736+
.wrapping_mul(MICROS_PER_DAY)
737+
.wrapping_add(nanos.read_unaligned() / 1000)
738+
}
739+
}
740+
730741
/// Decode timestamps represented as INT96 into i64 with micros precision
731742
impl PlainDecoding for Int96TimestampMicrosType {
732743
#[inline]
@@ -736,51 +747,36 @@ impl PlainDecoding for Int96TimestampMicrosType {
736747
if !src.read_options.use_legacy_date_timestamp_or_ntz {
737748
let mut offset = src.offset;
738749
for _ in 0..num {
739-
let v = &src_data[offset..offset + INT96_SRC_BYTE_WIDTH];
740-
let nanos = &v[..INT96_DST_BYTE_WIDTH] as *const [u8] as *const u8 as *const i64;
741-
let day = &v[INT96_DST_BYTE_WIDTH..] as *const [u8] as *const u8 as *const i32;
742-
743750
// TODO: optimize this further as checking value one by one is not very efficient
744-
unsafe {
745-
let micros = (day.read_unaligned() - JULIAN_DAY_OF_EPOCH) as i64
746-
* MICROS_PER_DAY
747-
+ nanos.read_unaligned() / 1000;
751+
let micros = int96_to_microsecond(&src_data[offset..offset + INT96_SRC_BYTE_WIDTH]);
748752

749-
if unlikely(micros < JULIAN_GREGORIAN_SWITCH_OFF_TS) {
750-
panic!(
751-
"Encountered timestamp value {}, which is before 1582-10-15 (counting \
753+
if unlikely(micros < JULIAN_GREGORIAN_SWITCH_OFF_TS) {
754+
panic!(
755+
"Encountered timestamp value {}, which is before 1582-10-15 (counting \
752756
backwards from Unix eopch date 1970-01-01), and could be ambigous \
753757
depending on whether a legacy Julian/Gregorian hybrid calendar is used, \
754758
or a Proleptic Gregorian calendar is used.",
755-
micros
756-
);
757-
}
758-
759-
offset += INT96_SRC_BYTE_WIDTH;
759+
micros
760+
);
760761
}
762+
763+
offset += INT96_SRC_BYTE_WIDTH;
761764
}
762765
}
763766

764767
let mut offset = src.offset;
765768
let mut dst_offset = INT96_DST_BYTE_WIDTH * dst.num_values;
766-
unsafe {
767-
for _ in 0..num {
768-
let v = &src_data[offset..offset + INT96_SRC_BYTE_WIDTH];
769-
let nanos = &v[..INT96_DST_BYTE_WIDTH] as *const [u8] as *const u8 as *const i64;
770-
let day = &v[INT96_DST_BYTE_WIDTH..] as *const [u8] as *const u8 as *const i32;
771-
772-
let micros = (day.read_unaligned() - JULIAN_DAY_OF_EPOCH) as i64 * MICROS_PER_DAY
773-
+ nanos.read_unaligned() / 1000;
769+
for _ in 0..num {
770+
let micros = int96_to_microsecond(&src_data[offset..offset + INT96_SRC_BYTE_WIDTH]);
774771

775-
bit::memcpy_value(
776-
&micros,
777-
INT96_DST_BYTE_WIDTH,
778-
&mut dst.value_buffer[dst_offset..],
779-
);
772+
bit::memcpy_value(
773+
&micros,
774+
INT96_DST_BYTE_WIDTH,
775+
&mut dst.value_buffer[dst_offset..],
776+
);
780777

781-
offset += INT96_SRC_BYTE_WIDTH;
782-
dst_offset += INT96_DST_BYTE_WIDTH;
783-
}
778+
offset += INT96_SRC_BYTE_WIDTH;
779+
dst_offset += INT96_DST_BYTE_WIDTH;
784780
}
785781

786782
src.offset = offset;

‎spark/src/test/scala/org/apache/comet/CometCastSuite.scala‎

Lines changed: 9 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -780,7 +780,7 @@ class CometCastSuite extends CometTestBase with AdaptiveSparkPlanHelper {
780780

781781
test("cast TimestampType to LongType") {
782782
assume(CometSparkSessionExtensions.isSpark33Plus)
783-
castTest(generateTimestamps(), DataTypes.LongType)
783+
castTest(generateTimestampsExtended(), DataTypes.LongType)
784784
}
785785

786786
ignore("cast TimestampType to FloatType") {
@@ -884,6 +884,14 @@ class CometCastSuite extends CometTestBase with AdaptiveSparkPlanHelper {
884884
withNulls(values).toDF("b").withColumn("a", col("b").cast(DataTypes.DateType)).drop("b")
885885
}
886886

887+
// Extended values are Timestamps that are outside dates supported chrono::DateTime and
888+
// therefore not supported by operations using it.
889+
private def generateTimestampsExtended(): DataFrame = {
890+
val values = Seq("290000-12-31T01:00:00+02:00")
891+
generateTimestamps().unionByName(
892+
values.toDF("str").select(col("str").cast(DataTypes.TimestampType).as("a")))
893+
}
894+
887895
private def generateTimestamps(): DataFrame = {
888896
val values =
889897
Seq(

0 commit comments

Comments
 (0)