|
20 | 20 | package org.apache.comet |
21 | 21 |
|
22 | 22 | import java.io.File |
| 23 | +import java.nio.charset.StandardCharsets |
23 | 24 |
|
24 | 25 | import scala.collection.mutable.ListBuffer |
25 | 26 | import scala.jdk.CollectionConverters._ |
@@ -1310,6 +1311,39 @@ class CometCastSuite extends CometTestBase with AdaptiveSparkPlanHelper { |
1310 | 1311 | castTest(gen.generateStrings(dataSize, numericPattern, 8).toDF("a"), DataTypes.StringType) |
1311 | 1312 | } |
1312 | 1313 |
|
| 1314 | + test("cast BinaryType to StringType - invalid UTF-8 bytes are replaced") { |
| 1315 | + // Arrow strings must be valid UTF-8, so Comet decodes the bytes the way the JVM's |
| 1316 | + // `new String(bytes, UTF_8)` does, replacing each ill-formed sequence with U+FFFD (#4488). |
| 1317 | + // Spark applies the same replacement when the string is materialized, so the results match, |
| 1318 | + // but pin the replacement explicitly here so a change in the native decoder is caught. |
| 1319 | + val values: Seq[Array[Byte]] = Seq( |
| 1320 | + "abc".getBytes(StandardCharsets.UTF_8), // valid, preserved as is |
| 1321 | + Array[Byte](0xff.toByte, 0xfe.toByte), // two invalid bytes |
| 1322 | + Array[Byte](0xed.toByte, 0xa0.toByte, 0x80.toByte), // surrogate range, one replacement |
| 1323 | + Array[Byte]('a'.toByte, 0xc3.toByte), // truncated two-byte sequence |
| 1324 | + Array.emptyByteArray) |
| 1325 | + castTest(values.toDF("a"), DataTypes.StringType) |
| 1326 | + |
| 1327 | + withTempPath { dir => |
| 1328 | + // Carry an explicit index so the rows can be put back in input order after the round-trip. |
| 1329 | + val indexed = values.zipWithIndex.toDF("a", "i") |
| 1330 | + val df = roundtripParquet(indexed, dir) |
| 1331 | + .coalesce(1) |
| 1332 | + .orderBy(col("i")) |
| 1333 | + .select(col("a").cast(DataTypes.StringType).as("casted")) |
| 1334 | + checkSparkAnswerAndOperator(df) |
| 1335 | + |
| 1336 | + val actual = df.collect().map(_.getString(0)) |
| 1337 | + // The JVM decoder is the oracle: Comet must produce the same string Spark renders. |
| 1338 | + assert(actual.toSeq === values.map(new String(_, StandardCharsets.UTF_8))) |
| 1339 | + assert(actual(0) === "abc") |
| 1340 | + assert(actual(1) === "\uFFFD\uFFFD") |
| 1341 | + assert(actual(2) === "\uFFFD") |
| 1342 | + assert(actual(3) === "a\uFFFD") |
| 1343 | + assert(actual(4) === "") |
| 1344 | + } |
| 1345 | + } |
| 1346 | + |
1313 | 1347 | // CAST from DateType |
1314 | 1348 |
|
1315 | 1349 | // Date to Boolean/Byte/Short/Int/Long/Float/Double/Decimal casts always return NULL |
|
0 commit comments