Skip to content

Commit bdf9491

Browse files
committed
fix: route Spark UTF8 binary formatter through JVM-compatible decoder
Addresses review feedback on the CAST(binary AS string) UB fix: - BinaryOutputStyle::Utf8 previously rendered via String::from_utf8(..).unwrap(), panicking the executor on non-UTF-8 input. With spark.sql.binaryOutputStyle=UTF8 (Spark 4.0+), a ToPrettyString/show() over a binary column with invalid bytes hit this. Decode lossily instead so invalid bytes become U+FFFD, matching Spark's new String(bytes, UTF_8). - Build cast_binary_to_string with a GenericStringBuilder and append the decoder's borrowed Cow, so the common valid-UTF-8 path copies once instead of allocating a throwaway String and copying twice. - Move decode_utf8_spark_lossy into datafusion-comet-common next to bytes_to_i128 so the shuffle crate no longer reaches into spark-expr. - Document why the char::from_u32 unwraps in the decoder are infallible.
1 parent 8e2dcd2 commit bdf9491

5 files changed

Lines changed: 229 additions & 191 deletions

File tree

‎native/common/src/lib.rs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -22,4 +22,4 @@ mod utils;
2222

2323
pub use error::{decimal_overflow_error, SparkError, SparkErrorWithContext, SparkResult};
2424
pub use query_context::{create_query_context_map, QueryContext, QueryContextMap};
25-
pub use utils::bytes_to_i128;
25+
pub use utils::{bytes_to_i128, decode_utf8_spark_lossy};

‎native/common/src/utils.rs‎

Lines changed: 174 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,8 @@
1515
// specific language governing permissions and limitations
1616
// under the License.
1717

18+
use std::borrow::Cow;
19+
1820
/// Converts a slice of bytes to i128. The bytes are serialized in big-endian order by
1921
/// `BigInteger.toByteArray()` in Java.
2022
pub fn bytes_to_i128(slice: &[u8]) -> i128 {
@@ -35,3 +37,175 @@ pub fn bytes_to_i128(slice: &[u8]) -> i128 {
3537

3638
i128::from_le_bytes(bytes)
3739
}
40+
41+
/// Decode `bytes` as UTF-8 the way Spark renders `StringType` -- `new String(bytes, UTF_8)` on the
42+
/// JVM -- replacing each ill-formed sequence with a single `U+FFFD` and skipping the same number of
43+
/// bytes the JDK's UTF-8 `CharsetDecoder` (action REPLACE) would. Valid UTF-8 is returned as a
44+
/// zero-cost borrow.
45+
///
46+
/// This intentionally differs from `str::from_utf8_lossy` for surrogate-range three-byte sequences
47+
/// (`ED A0..BF ..`, e.g. CESU-8 / Java modified-UTF-8 supplementary chars) and for some other
48+
/// ill-formed multi-byte units: `from_utf8_lossy` follows the Unicode "maximal subpart" rule and
49+
/// can emit one `U+FFFD` per byte, whereas the JDK collapses certain ill-formed units into a single
50+
/// `U+FFFD`. Matching the JDK byte-for-byte means Comet renders arbitrary bytes identically to
51+
/// Spark -- whether they arrive via a columnar shuffle or a `CAST(binary AS string)`. The
52+
/// per-class malformed lengths below (E0/ED overlong & surrogate handling, F0/F4 range checks)
53+
/// match the observable replacement behavior of the JDK UTF-8 decoder; they were determined from
54+
/// observed `new String(bytes, UTF_8)` output, not by reviewing the OpenJDK source.
55+
pub fn decode_utf8_spark_lossy(bytes: &[u8]) -> Cow<'_, str> {
56+
// Fast path: well-formed UTF-8 borrows with zero copy (the overwhelmingly common case).
57+
if let Ok(s) = std::str::from_utf8(bytes) {
58+
return Cow::Borrowed(s);
59+
}
60+
61+
const RC: char = '\u{FFFD}';
62+
let n = bytes.len();
63+
let mut out = String::with_capacity(n);
64+
let mut i = 0;
65+
while i < n {
66+
let b1 = bytes[i];
67+
if b1 < 0x80 {
68+
out.push(b1 as char);
69+
i += 1;
70+
} else if (0xC2..=0xDF).contains(&b1) {
71+
// 2-byte lead. Bad/absent continuation -> single FFFD, skip 1.
72+
if i + 1 < n && (bytes[i + 1] & 0xC0) == 0x80 {
73+
let cp = (((b1 as u32) & 0x1F) << 6) | ((bytes[i + 1] as u32) & 0x3F);
74+
// guards above keep cp a valid scalar (2-byte range, never a surrogate)
75+
out.push(char::from_u32(cp).unwrap());
76+
i += 2;
77+
} else {
78+
out.push(RC);
79+
i += 1;
80+
}
81+
} else if (0xE0..=0xEF).contains(&b1) {
82+
// 3-byte lead.
83+
if i + 1 >= n {
84+
out.push(RC); // truncated lead at EOF
85+
i = n;
86+
} else {
87+
let b2 = bytes[i + 1];
88+
if (b1 == 0xE0 && (b2 & 0xE0) == 0x80) || (b2 & 0xC0) != 0x80 {
89+
// overlong (E0 80..9F) or b2 not a continuation -> skip 1
90+
out.push(RC);
91+
i += 1;
92+
} else if i + 2 >= n {
93+
out.push(RC); // truncated after a valid b2 at EOF
94+
i = n;
95+
} else {
96+
let b3 = bytes[i + 2];
97+
if (b3 & 0xC0) != 0x80 {
98+
out.push(RC); // b3 not a continuation -> skip 2
99+
i += 2;
100+
} else {
101+
let cp = (((b1 as u32) & 0x0F) << 12)
102+
| (((b2 as u32) & 0x3F) << 6)
103+
| ((b3 as u32) & 0x3F);
104+
if (0xD800..=0xDFFF).contains(&cp) {
105+
// surrogate (e.g. ED A0 80) -> JDK skips all 3, single FFFD
106+
out.push(RC);
107+
i += 3;
108+
} else {
109+
// surrogate range excluded above -> cp is a valid scalar
110+
out.push(char::from_u32(cp).unwrap());
111+
i += 3;
112+
}
113+
}
114+
}
115+
}
116+
} else if (0xF0..=0xF4).contains(&b1) {
117+
// 4-byte lead.
118+
if i + 1 >= n {
119+
out.push(RC);
120+
i = n;
121+
} else {
122+
let b2 = bytes[i + 1];
123+
if (b1 == 0xF0 && !(0x90..=0xBF).contains(&b2))
124+
|| (b1 == 0xF4 && (b2 & 0xF0) != 0x80)
125+
|| (b2 & 0xC0) != 0x80
126+
{
127+
out.push(RC); // bad b2 -> skip 1
128+
i += 1;
129+
} else if i + 2 >= n {
130+
out.push(RC);
131+
i = n;
132+
} else if (bytes[i + 2] & 0xC0) != 0x80 {
133+
out.push(RC); // b3 not a continuation -> skip 2
134+
i += 2;
135+
} else if i + 3 >= n {
136+
out.push(RC);
137+
i = n;
138+
} else if (bytes[i + 3] & 0xC0) != 0x80 {
139+
out.push(RC); // b4 not a continuation -> skip 3
140+
i += 3;
141+
} else {
142+
let cp = (((b1 as u32) & 0x07) << 18)
143+
| (((b2 as u32) & 0x3F) << 12)
144+
| (((bytes[i + 2] as u32) & 0x3F) << 6)
145+
| ((bytes[i + 3] as u32) & 0x3F);
146+
// F0/F4 range guards above keep cp within U+10000..=U+10FFFF -> a valid scalar
147+
out.push(char::from_u32(cp).unwrap());
148+
i += 4;
149+
}
150+
}
151+
} else {
152+
// Lone continuation (0x80..0xBF), overlong 2-byte leads (0xC0/0xC1), or out-of-range
153+
// 4-byte leads (0xF5..0xFF): each is a single ill-formed byte -> skip 1.
154+
out.push(RC);
155+
i += 1;
156+
}
157+
}
158+
Cow::Owned(out)
159+
}
160+
161+
#[cfg(test)]
162+
mod tests {
163+
use super::*;
164+
165+
/// Oracle = JDK 17 `new String(bytes, StandardCharsets.UTF_8)` (the renderer Spark uses for
166+
/// StringType). Each row's expected output was verified against the JVM. The decoder must match
167+
/// it byte-for-byte -- including the surrogate-range case where `str::from_utf8_lossy` differs.
168+
#[test]
169+
fn decode_utf8_spark_lossy_matches_jvm_replacement_granularity() {
170+
let cases: &[(&[u8], &str)] = &[
171+
(&[0xFF, 0xFE, 0x41], "\u{FFFD}\u{FFFD}A"),
172+
(&[0x80, 0x42], "\u{FFFD}B"),
173+
(&[0xE0, 0x80], "\u{FFFD}\u{FFFD}"),
174+
(&[0xF0, 0x80, 0x80, 0x41], "\u{FFFD}\u{FFFD}\u{FFFD}A"),
175+
(&[0xC0, 0xAF], "\u{FFFD}\u{FFFD}"),
176+
// The parity case: Rust's from_utf8_lossy would give three U+FFFD here.
177+
(&[0xED, 0xA0, 0x80], "\u{FFFD}"),
178+
(
179+
&[0xF4, 0x90, 0x80, 0x80],
180+
"\u{FFFD}\u{FFFD}\u{FFFD}\u{FFFD}",
181+
),
182+
];
183+
for (bytes, expected) in cases {
184+
assert_eq!(
185+
decode_utf8_spark_lossy(bytes),
186+
*expected,
187+
"bytes {bytes:02x?} should render like the JVM"
188+
);
189+
}
190+
}
191+
192+
#[test]
193+
fn decode_utf8_spark_lossy_valid_utf8_is_borrowed_zero_copy() {
194+
let s = "café — 日本語 🦀";
195+
match decode_utf8_spark_lossy(s.as_bytes()) {
196+
Cow::Borrowed(b) => assert_eq!(b, s),
197+
Cow::Owned(_) => panic!("valid UTF-8 must borrow, not allocate"),
198+
}
199+
}
200+
201+
#[test]
202+
fn decode_utf8_spark_lossy_valid_multibyte_around_invalid_bytes_decodes() {
203+
// 'a' | é (C3 A9) | stray 0xFF | 'b' | 🦀 (F0 9F A6 80) -> valid chars preserved, one FFFD.
204+
let mut bytes = vec![b'a'];
205+
bytes.extend_from_slice("é".as_bytes());
206+
bytes.push(0xFF);
207+
bytes.push(b'b');
208+
bytes.extend_from_slice("🦀".as_bytes());
209+
assert_eq!(decode_utf8_spark_lossy(&bytes), "aé\u{FFFD}b🦀");
210+
}
211+
}

‎native/shuffle/src/spark_unsafe/unsafe_object.rs‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -18,8 +18,7 @@
1818
use super::list::SparkUnsafeArray;
1919
use super::map::SparkUnsafeMap;
2020
use super::row::SparkUnsafeRow;
21-
use datafusion_comet_common::bytes_to_i128;
22-
use datafusion_comet_spark_expr::utils::decode_utf8_spark_lossy;
21+
use datafusion_comet_common::{bytes_to_i128, decode_utf8_spark_lossy};
2322
use std::borrow::Cow;
2423

2524
const MAX_LONG_DIGITS: u8 = 18;

‎native/spark-expr/src/conversion_funcs/cast.rs‎

Lines changed: 53 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -36,13 +36,11 @@ use crate::conversion_funcs::temporal::{
3636
is_df_cast_from_timestamp_spark_compatible,
3737
};
3838
use crate::conversion_funcs::utils::spark_cast_postprocess;
39-
use crate::utils::{
40-
array_with_timezone, cast_timestamp_to_ntz, decode_utf8_spark_lossy, timestamp_ntz_to_timestamp,
41-
};
39+
use crate::utils::{array_with_timezone, cast_timestamp_to_ntz, timestamp_ntz_to_timestamp};
4240
use crate::EvalMode::Legacy;
4341
use crate::{cast_whole_num_to_binary, BinaryOutputStyle};
4442
use crate::{EvalMode, SparkError};
45-
use arrow::array::builder::StringBuilder;
43+
use arrow::array::builder::{GenericStringBuilder, StringBuilder};
4644
use arrow::array::{
4745
new_null_array, BinaryBuilder, DictionaryArray, GenericByteArray, ListArray, MapArray,
4846
StringArray, StructArray,
@@ -52,8 +50,8 @@ use arrow::datatypes::{Field, Fields, GenericBinaryType};
5250
use arrow::error::ArrowError;
5351
use arrow::{
5452
array::{
55-
cast::AsArray, types::Int32Type, Array, ArrayRef, GenericStringArray, Int16Array,
56-
Int32Array, Int64Array, Int8Array, OffsetSizeTrait, PrimitiveArray,
53+
cast::AsArray, types::Int32Type, Array, ArrayRef, Int16Array, Int32Array, Int64Array,
54+
Int8Array, OffsetSizeTrait, PrimitiveArray,
5755
},
5856
compute::{cast_with_options, take, CastOptions},
5957
record_batch::RecordBatch,
@@ -64,6 +62,8 @@ use base64::Engine;
6462
use datafusion::common::{internal_err, DataFusionError, Result as DataFusionResult, ScalarValue};
6563
use datafusion::physical_expr::PhysicalExpr;
6664
use datafusion::physical_plan::ColumnarValue;
65+
use datafusion_comet_common::decode_utf8_spark_lossy;
66+
use std::borrow::Cow;
6767
use std::{
6868
fmt::{Debug, Display, Formatter},
6969
hash::Hash,
@@ -803,21 +803,22 @@ fn cast_binary_to_string<O: OffsetSizeTrait>(
803803
.downcast_ref::<GenericByteArray<GenericBinaryType<O>>>()
804804
.unwrap();
805805

806-
fn binary_formatter(value: &[u8], spark_cast_options: &SparkCastOptions) -> String {
807-
match spark_cast_options.binary_output_style {
808-
Some(s) => spark_binary_formatter(value, s),
809-
None => cast_binary_formatter(value),
806+
// Build with a GenericStringBuilder and append &str straight from the decoder's Cow so the
807+
// common valid-UTF-8 cast path copies the bytes once (into the Arrow value buffer) instead of
808+
// twice (an owned String, then a copy into the buffer).
809+
let mut builder = GenericStringBuilder::<O>::new();
810+
for value in input.iter() {
811+
match value {
812+
Some(value) => match spark_cast_options.binary_output_style {
813+
// ToPrettyString styles (Spark 4.0+) build owned strings, which is unavoidable.
814+
Some(s) => builder.append_value(spark_binary_formatter(value, s)),
815+
// Default CAST(binary AS string): borrows in the valid path, appends once.
816+
None => builder.append_value(cast_binary_formatter(value)),
817+
},
818+
None => builder.append_null(),
810819
}
811820
}
812-
813-
let output_array = input
814-
.iter()
815-
.map(|value| match value {
816-
Some(value) => Ok(Some(binary_formatter(value, spark_cast_options))),
817-
_ => Ok(None),
818-
})
819-
.collect::<Result<GenericStringArray<O>, ArrowError>>()?;
820-
Ok(Arc::new(output_array))
821+
Ok(Arc::new(builder.finish()))
821822
}
822823

823824
/// This function mimics the [BinaryFormatter]: https://github.com/apache/spark/blob/v4.0.0/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/ToStringBase.scala#L449-L468
@@ -826,7 +827,10 @@ fn cast_binary_to_string<O: OffsetSizeTrait>(
826827
/// Before Spark 4.0.0, the default is SPACE_DELIMITED_UPPERCASE_HEX
827828
fn spark_binary_formatter(value: &[u8], binary_output_style: BinaryOutputStyle) -> String {
828829
match binary_output_style {
829-
BinaryOutputStyle::Utf8 => String::from_utf8(value.to_vec()).unwrap(),
830+
// Spark's UTF8 BinaryFormatter renders via `UTF8String.fromBytes(bytes).toString`, i.e.
831+
// `new String(bytes, UTF_8)`. Route through the shared JVM-compatible decoder so invalid
832+
// bytes become U+FFFD (matching Spark) instead of panicking on non-UTF-8 input (#4488).
833+
BinaryOutputStyle::Utf8 => decode_utf8_spark_lossy(value).into_owned(),
830834
BinaryOutputStyle::Basic => {
831835
format!(
832836
"{:?}",
@@ -855,7 +859,7 @@ fn spark_binary_formatter(value: &[u8], binary_output_style: BinaryOutputStyle)
855859
}
856860
}
857861

858-
fn cast_binary_formatter(value: &[u8]) -> String {
862+
fn cast_binary_formatter(value: &[u8]) -> Cow<'_, str> {
859863
// CAST(binary AS string) reinterprets the bytes as UTF-8, like Spark's UTF8String.fromBytes.
860864
// Spark keeps the raw bytes, but Arrow's Utf8 type requires valid UTF-8, and building a String
861865
// from non-UTF-8 bytes is undefined behaviour (#4488). Decode JVM-compatibly-lossily instead:
@@ -864,8 +868,9 @@ fn cast_binary_formatter(value: &[u8]) -> String {
864868
// result is memory-safe valid UTF-8, never feeds invalid bytes into downstream native string
865869
// kernels, and matches Spark's rendered output byte-for-byte. It diverges from Spark only under
866870
// byte-level round-trips such as CAST(CAST(x AS string) AS binary), where Spark still has the
867-
// original bytes and Comet has the U+FFFD replacements.
868-
decode_utf8_spark_lossy(value).into_owned()
871+
// original bytes and Comet has the U+FFFD replacements. Returning `Cow` lets the valid path
872+
// borrow so the caller appends without an intermediate allocation.
873+
decode_utf8_spark_lossy(value)
869874
}
870875

871876
#[cfg(test)]
@@ -901,6 +906,31 @@ mod tests {
901906
assert_eq!(strings.value(3), "\u{FFFD}");
902907
}
903908

909+
#[test]
910+
fn test_cast_binary_to_string_utf8_output_style_replaces_invalid_utf8() {
911+
// Spark's `binaryOutputStyle=UTF8` (Spark 4.0+) ToPrettyString formatter renders binary via
912+
// `new String(bytes, UTF_8)`. Previously this arm called `String::from_utf8(..).unwrap()`,
913+
// which panicked the executor on non-UTF-8 input. It must now decode JVM-compatibly-lossily,
914+
// matching the default cast path's replacement behavior.
915+
let input = BinaryArray::from_opt_vec(vec![
916+
Some(&[0xFFu8, 0xFE][..]),
917+
None,
918+
Some("abc".as_bytes()),
919+
Some(&[0xEDu8, 0xA0, 0x80][..]),
920+
]);
921+
let mut cast_options = SparkCastOptions::new(EvalMode::Legacy, "UTC", false);
922+
cast_options.binary_output_style = Some(BinaryOutputStyle::Utf8);
923+
924+
let result = cast_binary_to_string::<i32>(&input, &cast_options).unwrap();
925+
926+
let strings = result.as_string::<i32>();
927+
assert_eq!(strings.len(), 4);
928+
assert_eq!(strings.value(0), "\u{FFFD}\u{FFFD}");
929+
assert!(strings.is_null(1));
930+
assert_eq!(strings.value(2), "abc");
931+
assert_eq!(strings.value(3), "\u{FFFD}");
932+
}
933+
904934
#[test]
905935
fn test_cast_unsupported_timestamp_to_date() {
906936
// Since datafusion uses chrono::Datetime internally not all dates representable by TimestampMicrosecondType are supported

0 commit comments

Comments
 (0)