Skip to content

Commit 96c7f07

Browse files
committed
test: ignore the nested BINARY invalid UTF-8 tests on DataFusion 56
DataFusion 56 coerces nested BINARY file fields to Utf8 before the parquet reader, which skips UTF-8 validation for unannotated BINARY columns, so the native scan panics in debug builds instead of decoding to U+FFFD. Ignore the Rust and JVM tests from #7106 until apache/datafusion#27065 is fixed (#7469).
1 parent 56a88c9 commit 96c7f07

2 files changed

Lines changed: 6 additions & 1 deletion

File tree

‎native/core/src/parquet/parquet_exec.rs‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -841,6 +841,7 @@ mod tests {
841841
/// Spark's string type as those types, not as `Binary`. A malformed value must still decode
842842
/// to U+FFFD.
843843
#[tokio::test]
844+
#[ignore = "DataFusion 56 coerces nested BINARY to unvalidated Utf8: https://github.com/apache/datafusion-comet/issues/7469"]
844845
async fn nested_arrow_binary_types_read_as_string_decode_invalid_utf8() {
845846
use arrow::array::{Array, AsArray, BinaryArray, ListArray};
846847
use arrow::buffer::OffsetBuffer;

‎spark/src/test/scala/org/apache/comet/parquet/ParquetReadSuite.scala‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -147,7 +147,11 @@ abstract class ParquetReadSuite extends CometTestBase {
147147
}
148148
}
149149

150-
test("native scan decodes invalid UTF-8 in nested BINARY read as string") {
150+
// Ignored while Comet is on DataFusion 56: its file-schema coercion reads nested BINARY as
151+
// Utf8 without validating UTF-8, so the native scan panics in debug builds.
152+
// Re-enable once https://github.com/apache/datafusion/issues/27065 is fixed.
153+
// https://github.com/apache/datafusion-comet/issues/7469
154+
ignore("native scan decodes invalid UTF-8 in nested BINARY read as string") {
151155
withTempDir { dir =>
152156
val path = new Path(dir.toURI.toString, "nested-binary.parquet")
153157
val parquetSchema = MessageTypeParser.parseMessageType("""message root {

0 commit comments

Comments
 (0)