@@ -36,13 +36,11 @@ use crate::conversion_funcs::temporal::{
3636 is_df_cast_from_timestamp_spark_compatible,
3737} ;
3838use crate :: conversion_funcs:: utils:: spark_cast_postprocess;
39- use crate :: utils:: {
40- array_with_timezone, cast_timestamp_to_ntz, decode_utf8_spark_lossy, timestamp_ntz_to_timestamp,
41- } ;
39+ use crate :: utils:: { array_with_timezone, cast_timestamp_to_ntz, timestamp_ntz_to_timestamp} ;
4240use crate :: EvalMode :: Legacy ;
4341use crate :: { cast_whole_num_to_binary, BinaryOutputStyle } ;
4442use crate :: { EvalMode , SparkError } ;
45- use arrow:: array:: builder:: StringBuilder ;
43+ use arrow:: array:: builder:: { GenericStringBuilder , StringBuilder } ;
4644use arrow:: array:: {
4745 new_null_array, BinaryBuilder , DictionaryArray , GenericByteArray , ListArray , MapArray ,
4846 StringArray , StructArray ,
@@ -52,8 +50,8 @@ use arrow::datatypes::{Field, Fields, GenericBinaryType};
5250use arrow:: error:: ArrowError ;
5351use arrow:: {
5452 array:: {
55- cast:: AsArray , types:: Int32Type , Array , ArrayRef , GenericStringArray , Int16Array ,
56- Int32Array , Int64Array , Int8Array , OffsetSizeTrait , PrimitiveArray ,
53+ cast:: AsArray , types:: Int32Type , Array , ArrayRef , Int16Array , Int32Array , Int64Array ,
54+ Int8Array , OffsetSizeTrait , PrimitiveArray ,
5755 } ,
5856 compute:: { cast_with_options, take, CastOptions } ,
5957 record_batch:: RecordBatch ,
@@ -64,6 +62,8 @@ use base64::Engine;
6462use datafusion:: common:: { internal_err, DataFusionError , Result as DataFusionResult , ScalarValue } ;
6563use datafusion:: physical_expr:: PhysicalExpr ;
6664use datafusion:: physical_plan:: ColumnarValue ;
65+ use datafusion_comet_common:: decode_utf8_spark_lossy;
66+ use std:: borrow:: Cow ;
6767use std:: {
6868 fmt:: { Debug , Display , Formatter } ,
6969 hash:: Hash ,
@@ -803,21 +803,22 @@ fn cast_binary_to_string<O: OffsetSizeTrait>(
803803 . downcast_ref :: < GenericByteArray < GenericBinaryType < O > > > ( )
804804 . unwrap ( ) ;
805805
806- fn binary_formatter ( value : & [ u8 ] , spark_cast_options : & SparkCastOptions ) -> String {
807- match spark_cast_options. binary_output_style {
808- Some ( s) => spark_binary_formatter ( value, s) ,
809- None => cast_binary_formatter ( value) ,
806+ // Build with a GenericStringBuilder and append &str straight from the decoder's Cow so the
807+ // common valid-UTF-8 cast path copies the bytes once (into the Arrow value buffer) instead of
808+ // twice (an owned String, then a copy into the buffer).
809+ let mut builder = GenericStringBuilder :: < O > :: new ( ) ;
810+ for value in input. iter ( ) {
811+ match value {
812+ Some ( value) => match spark_cast_options. binary_output_style {
813+ // ToPrettyString styles (Spark 4.0+) build owned strings, which is unavoidable.
814+ Some ( s) => builder. append_value ( spark_binary_formatter ( value, s) ) ,
815+ // Default CAST(binary AS string): borrows in the valid path, appends once.
816+ None => builder. append_value ( cast_binary_formatter ( value) ) ,
817+ } ,
818+ None => builder. append_null ( ) ,
810819 }
811820 }
812-
813- let output_array = input
814- . iter ( )
815- . map ( |value| match value {
816- Some ( value) => Ok ( Some ( binary_formatter ( value, spark_cast_options) ) ) ,
817- _ => Ok ( None ) ,
818- } )
819- . collect :: < Result < GenericStringArray < O > , ArrowError > > ( ) ?;
820- Ok ( Arc :: new ( output_array) )
821+ Ok ( Arc :: new ( builder. finish ( ) ) )
821822}
822823
823824/// This function mimics the [BinaryFormatter]: https://github.com/apache/spark/blob/v4.0.0/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/ToStringBase.scala#L449-L468
@@ -826,7 +827,10 @@ fn cast_binary_to_string<O: OffsetSizeTrait>(
826827/// Before Spark 4.0.0, the default is SPACE_DELIMITED_UPPERCASE_HEX
827828fn spark_binary_formatter ( value : & [ u8 ] , binary_output_style : BinaryOutputStyle ) -> String {
828829 match binary_output_style {
829- BinaryOutputStyle :: Utf8 => String :: from_utf8 ( value. to_vec ( ) ) . unwrap ( ) ,
830+ // Spark's UTF8 BinaryFormatter renders via `UTF8String.fromBytes(bytes).toString`, i.e.
831+ // `new String(bytes, UTF_8)`. Route through the shared JVM-compatible decoder so invalid
832+ // bytes become U+FFFD (matching Spark) instead of panicking on non-UTF-8 input (#4488).
833+ BinaryOutputStyle :: Utf8 => decode_utf8_spark_lossy ( value) . into_owned ( ) ,
830834 BinaryOutputStyle :: Basic => {
831835 format ! (
832836 "{:?}" ,
@@ -855,7 +859,7 @@ fn spark_binary_formatter(value: &[u8], binary_output_style: BinaryOutputStyle)
855859 }
856860}
857861
858- fn cast_binary_formatter ( value : & [ u8 ] ) -> String {
862+ fn cast_binary_formatter ( value : & [ u8 ] ) -> Cow < ' _ , str > {
859863 // CAST(binary AS string) reinterprets the bytes as UTF-8, like Spark's UTF8String.fromBytes.
860864 // Spark keeps the raw bytes, but Arrow's Utf8 type requires valid UTF-8, and building a String
861865 // from non-UTF-8 bytes is undefined behaviour (#4488). Decode JVM-compatibly-lossily instead:
@@ -864,8 +868,9 @@ fn cast_binary_formatter(value: &[u8]) -> String {
864868 // result is memory-safe valid UTF-8, never feeds invalid bytes into downstream native string
865869 // kernels, and matches Spark's rendered output byte-for-byte. It diverges from Spark only under
866870 // byte-level round-trips such as CAST(CAST(x AS string) AS binary), where Spark still has the
867- // original bytes and Comet has the U+FFFD replacements.
868- decode_utf8_spark_lossy ( value) . into_owned ( )
871+ // original bytes and Comet has the U+FFFD replacements. Returning `Cow` lets the valid path
872+ // borrow so the caller appends without an intermediate allocation.
873+ decode_utf8_spark_lossy ( value)
869874}
870875
871876#[ cfg( test) ]
@@ -901,6 +906,31 @@ mod tests {
901906 assert_eq ! ( strings. value( 3 ) , "\u{FFFD} " ) ;
902907 }
903908
909+ #[ test]
910+ fn test_cast_binary_to_string_utf8_output_style_replaces_invalid_utf8 ( ) {
911+ // Spark's `binaryOutputStyle=UTF8` (Spark 4.0+) ToPrettyString formatter renders binary via
912+ // `new String(bytes, UTF_8)`. Previously this arm called `String::from_utf8(..).unwrap()`,
913+ // which panicked the executor on non-UTF-8 input. It must now decode JVM-compatibly-lossily,
914+ // matching the default cast path's replacement behavior.
915+ let input = BinaryArray :: from_opt_vec ( vec ! [
916+ Some ( & [ 0xFFu8 , 0xFE ] [ ..] ) ,
917+ None ,
918+ Some ( "abc" . as_bytes( ) ) ,
919+ Some ( & [ 0xEDu8 , 0xA0 , 0x80 ] [ ..] ) ,
920+ ] ) ;
921+ let mut cast_options = SparkCastOptions :: new ( EvalMode :: Legacy , "UTC" , false ) ;
922+ cast_options. binary_output_style = Some ( BinaryOutputStyle :: Utf8 ) ;
923+
924+ let result = cast_binary_to_string :: < i32 > ( & input, & cast_options) . unwrap ( ) ;
925+
926+ let strings = result. as_string :: < i32 > ( ) ;
927+ assert_eq ! ( strings. len( ) , 4 ) ;
928+ assert_eq ! ( strings. value( 0 ) , "\u{FFFD} \u{FFFD} " ) ;
929+ assert ! ( strings. is_null( 1 ) ) ;
930+ assert_eq ! ( strings. value( 2 ) , "abc" ) ;
931+ assert_eq ! ( strings. value( 3 ) , "\u{FFFD} " ) ;
932+ }
933+
904934 #[ test]
905935 fn test_cast_unsupported_timestamp_to_date ( ) {
906936 // Since datafusion uses chrono::Datetime internally not all dates representable by TimestampMicrosecondType are supported
0 commit comments