Skip to content

Commit d197efd

Browse files
fix(spark): read every column as nullable (#11020)
- Tracking Issue: #11019 A table's schema can't promise that every file is free of nulls. Inference reads only one file, and a user or catalog schema can declare a column `NOT NULL` that some file still has nulls in. Reading such a table fails with a `NullPointerException` in Spark's generated code. Spark's own file sources avoid this by making every column nullable, so this does the same. Signed-off-by: Christopher Wirt <[email protected]>
1 parent da5587f commit d197efd

5 files changed

Lines changed: 177 additions & 46 deletions

File tree

‎java/vortex-spark/src/main/java/dev/vortex/spark/VortexDataSourceV2.java‎

Lines changed: 26 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -23,7 +23,10 @@
2323
import org.apache.spark.sql.connector.expressions.Expressions;
2424
import org.apache.spark.sql.connector.expressions.Transform;
2525
import org.apache.spark.sql.sources.DataSourceRegister;
26+
import org.apache.spark.sql.types.ArrayType;
2627
import org.apache.spark.sql.types.DataType;
28+
import org.apache.spark.sql.types.DataTypes;
29+
import org.apache.spark.sql.types.MapType;
2730
import org.apache.spark.sql.types.Metadata;
2831
import org.apache.spark.sql.types.StructField;
2932
import org.apache.spark.sql.types.StructType;
@@ -123,6 +126,26 @@ public StructType inferSchema(CaseInsensitiveStringMap options) {
123126
return dataSchema;
124127
}
125128

129+
/** Returns {@code type} with every field, array element and map value nullable, at any depth. */
130+
static StructType asNullable(StructType type) {
131+
return (StructType) asNullable((DataType) type);
132+
}
133+
134+
private static DataType asNullable(DataType type) {
135+
if (type instanceof StructType struct) {
136+
return new StructType(Stream.of(struct.fields())
137+
.map(f -> new StructField(f.name(), asNullable(f.dataType()), true, f.metadata()))
138+
.toArray(StructField[]::new));
139+
}
140+
if (type instanceof ArrayType array) {
141+
return DataTypes.createArrayType(asNullable(array.elementType()), true);
142+
}
143+
if (type instanceof MapType map) {
144+
return DataTypes.createMapType(asNullable(map.keyType()), asNullable(map.valueType()), true);
145+
}
146+
return type;
147+
}
148+
126149
/**
127150
* Infers partition transforms by inspecting Hive-style {@code key=value} segments in the first listed file path.
128151
*
@@ -179,7 +202,9 @@ public Transform[] inferPartitioning(CaseInsensitiveStringMap options) {
179202
public Table getTable(StructType schema, Transform[] partitioning, Map<String, String> properties) {
180203
var uncased = new CaseInsensitiveStringMap(properties);
181204
ImmutableList<String> paths = getPathsOrEmpty(uncased);
182-
return new VortexTable(paths, schema, buildDataSourceOptions(properties), partitioning);
205+
// Whether inferred from one file, given by the user or stored in a catalog, the schema cannot promise that
206+
// every file is free of nulls, so like Spark's file sources the table reads every field as nullable.
207+
return new VortexTable(paths, asNullable(schema), buildDataSourceOptions(properties), partitioning);
183208
}
184209

185210
/**

‎java/vortex-spark/src/test/java/dev/vortex/spark/VortexDataSourceInferSchemaTest.java‎

Lines changed: 137 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3,19 +3,31 @@
33

44
package dev.vortex.spark;
55

6+
import static org.junit.jupiter.api.Assertions.assertEquals;
7+
import static org.junit.jupiter.api.Assertions.assertFalse;
68
import static org.junit.jupiter.api.Assertions.assertThrows;
79
import static org.junit.jupiter.api.Assertions.assertTrue;
810

911
import java.nio.file.Path;
12+
import java.util.Arrays;
13+
import java.util.List;
14+
import java.util.stream.Collectors;
15+
import org.apache.spark.sql.Column;
16+
import org.apache.spark.sql.Dataset;
17+
import org.apache.spark.sql.Row;
18+
import org.apache.spark.sql.SaveMode;
1019
import org.apache.spark.sql.SparkSession;
20+
import org.apache.spark.sql.functions;
21+
import org.apache.spark.sql.types.DataTypes;
22+
import org.apache.spark.sql.types.StructType;
1123
import org.junit.jupiter.api.AfterAll;
1224
import org.junit.jupiter.api.BeforeAll;
1325
import org.junit.jupiter.api.DisplayName;
1426
import org.junit.jupiter.api.Test;
1527
import org.junit.jupiter.api.TestInstance;
1628
import org.junit.jupiter.api.io.TempDir;
1729

18-
/** Tests for schema inference failure reporting in {@link VortexDataSourceV2}. */
30+
/** Tests for the schema {@link VortexDataSourceV2} reads tables with. */
1931
@TestInstance(TestInstance.Lifecycle.PER_CLASS)
2032
public final class VortexDataSourceInferSchemaTest {
2133

@@ -56,6 +68,130 @@ public void inferSchemaFailureNamesThePath() {
5668
assertTrue(allMessages.contains(emptyDir), "error should name the offending path, got: " + allMessages);
5769
}
5870

71+
@Test
72+
@DisplayName("asNullable makes fields nullable at every depth, keeping names and types")
73+
public void asNullableReachesEveryDepth() {
74+
StructType key = new StructType().add("k", DataTypes.IntegerType, false);
75+
StructType element = new StructType().add("e", DataTypes.LongType, false);
76+
StructType schema = new StructType()
77+
.add("id", DataTypes.LongType, false)
78+
.add("arr", DataTypes.createArrayType(element, false), false)
79+
.add(
80+
"m",
81+
DataTypes.createMapType(key, DataTypes.createArrayType(DataTypes.StringType, false), false),
82+
false);
83+
84+
StructType nullable = VortexDataSourceV2.asNullable(schema);
85+
86+
assertEquals(
87+
"struct<id:bigint,arr:array<struct<e:bigint>>,m:map<struct<k:int>,array<string>>>",
88+
nullable.simpleString());
89+
assertNoNonNullableFields(nullable);
90+
}
91+
92+
@Test
93+
@DisplayName("A null struct reads correctly when schema inference picks a file without nulls")
94+
public void inferFromFileWithoutNulls() {
95+
assertFilesDifferingInNullabilityRead(false);
96+
}
97+
98+
@Test
99+
@DisplayName("A null struct reads correctly when schema inference picks the file with nulls")
100+
public void inferFromFileWithNulls() {
101+
assertFilesDifferingInNullabilityRead(true);
102+
}
103+
104+
@Test
105+
@DisplayName("A null struct reads correctly under a user schema that declares it non-nullable")
106+
public void userSchemaDeclaringNonNullable() {
107+
Column s = functions.struct(
108+
functions.col("id").plus(100).alias("a"),
109+
functions.col("id").plus(200).alias("b"));
110+
StructType nonNullable =
111+
spark.range(0, 1).select(functions.col("id"), s.alias("s")).schema();
112+
String withNull = write(
113+
spark.range(2, 4)
114+
.select(
115+
functions.col("id"),
116+
functions
117+
.when(functions.col("id").equalTo(3), functions.lit(null))
118+
.otherwise(s)
119+
.alias("s")),
120+
"user-schema");
121+
122+
Dataset<Row> table = spark.read().schema(nonNullable).format("vortex").load(withNull);
123+
124+
assertEquals(
125+
"struct<id:bigint,s:struct<a:bigint,b:bigint>>", table.schema().simpleString());
126+
assertNoNonNullableFields(table.schema());
127+
assertEquals(
128+
Arrays.asList(Arrays.asList(2L, 202L, false), Arrays.asList(3L, null, true)),
129+
rowsOf(table.select(
130+
functions.col("id"),
131+
functions.col("s.b"),
132+
functions.col("s").isNull())));
133+
}
134+
135+
/**
136+
* Reads two files that differ only in whether struct {@code s} has nulls. Schema inference reads the last path, so
137+
* {@code inferFromNulls} picks the file it infers from.
138+
*/
139+
private void assertFilesDifferingInNullabilityRead(boolean inferFromNulls) {
140+
Column s = functions.struct(
141+
functions.col("id").plus(100).alias("a"),
142+
functions.col("id").plus(200).alias("b"));
143+
String noNulls =
144+
write(spark.range(0, 2).select(functions.col("id"), s.alias("s")), "no-nulls-" + inferFromNulls);
145+
String withNull = write(
146+
spark.range(2, 4)
147+
.select(
148+
functions.col("id"),
149+
functions
150+
.when(functions.col("id").equalTo(3), functions.lit(null))
151+
.otherwise(s)
152+
.alias("s")),
153+
"with-null-" + inferFromNulls);
154+
155+
Dataset<Row> table = inferFromNulls
156+
? spark.read().format("vortex").load(noNulls, withNull)
157+
: spark.read().format("vortex").load(withNull, noNulls);
158+
159+
assertEquals(
160+
"struct<id:bigint,s:struct<a:bigint,b:bigint>>", table.schema().simpleString());
161+
assertNoNonNullableFields(table.schema());
162+
assertEquals(
163+
Arrays.asList(
164+
Arrays.asList(0L, 200L, false),
165+
Arrays.asList(1L, 201L, false),
166+
Arrays.asList(2L, 202L, false),
167+
Arrays.asList(3L, null, true)),
168+
rowsOf(table.select(
169+
functions.col("id"),
170+
functions.col("s.b"),
171+
functions.col("s").isNull())));
172+
}
173+
174+
/** Asserts that no field, array element or map value in {@code schema} is non-nullable. */
175+
static void assertNoNonNullableFields(StructType schema) {
176+
String json = schema.json();
177+
for (String flag :
178+
new String[] {"\"nullable\":false", "\"containsNull\":false", "\"valueContainsNull\":false"}) {
179+
assertFalse(json.contains(flag), "expected every field to be nullable: " + json);
180+
}
181+
}
182+
183+
private String write(Dataset<Row> rows, String name) {
184+
String path = tempDir.resolve(name).toUri().toString();
185+
rows.coalesce(1).write().format("vortex").mode(SaveMode.Overwrite).save(path);
186+
return path;
187+
}
188+
189+
private static List<List<Object>> rowsOf(Dataset<Row> rows) {
190+
return rows.orderBy("id").collectAsList().stream()
191+
.map(row -> Arrays.asList(row.get(0), row.get(1), row.get(2)))
192+
.collect(Collectors.toList());
193+
}
194+
59195
/** Concatenates the messages of the whole cause chain, since Spark may wrap data source exceptions. */
60196
private static String messagesOf(Throwable thrown) {
61197
StringBuilder sb = new StringBuilder();

‎java/vortex-spark/src/test/java/dev/vortex/spark/VortexDataSourceS3MockTest.java‎

Lines changed: 5 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,6 @@
1010
import org.apache.spark.sql.Row;
1111
import org.apache.spark.sql.SaveMode;
1212
import org.apache.spark.sql.SparkSession;
13-
import org.apache.spark.sql.types.StructField;
1413
import org.apache.spark.sql.types.StructType;
1514
import org.junit.jupiter.api.AfterAll;
1615
import org.junit.jupiter.api.BeforeAll;
@@ -92,7 +91,7 @@ public void testWriteAndReadVortexFilesFromS3Mock() {
9291
Dataset<Row> readDf = spark.read().format("vortex").load(s3Path);
9392

9493
// Verify schema is preserved
95-
assertSchemaEquals(originalDf.schema(), readDf.schema());
94+
assertReadBackSchema(originalDf.schema(), readDf.schema());
9695

9796
// Verify row count
9897
assertEquals(numRows, readDf.count(), "Read DataFrame should have same number of rows as original");
@@ -137,24 +136,10 @@ private Dataset<Row> createTestDataFrame(int numRows) {
137136
"array('Alpha', 'Bravo', 'Charlie') AS elements");
138137
}
139138

140-
/** Verifies that two schemas are equal. */
141-
private void assertSchemaEquals(StructType expected, StructType actual) {
142-
assertEquals(expected.fields().length, actual.fields().length, "Schemas should have same number of fields");
143-
144-
for (int i = 0; i < expected.fields().length; i++) {
145-
StructField expectedField = expected.fields()[i];
146-
StructField actualField = actual.fields()[i];
147-
148-
assertEquals(expectedField.name(), actualField.name(), "Field names should match at position " + i);
149-
assertEquals(
150-
expectedField.dataType(),
151-
actualField.dataType(),
152-
"Field types should match for field: " + expectedField.name());
153-
assertEquals(
154-
expectedField.nullable(),
155-
actualField.nullable(),
156-
"Field nullability should match for field: " + expectedField.name());
157-
}
139+
/** Verifies that {@code actual} is the {@code written} schema, read back with every field nullable. */
140+
private void assertReadBackSchema(StructType written, StructType actual) {
141+
assertEquals(written.simpleString(), actual.simpleString());
142+
VortexDataSourceInferSchemaTest.assertNoNonNullableFields(actual);
158143
}
159144

160145
/** Verifies that the data content of two DataFrames is identical. */

‎java/vortex-spark/src/test/java/dev/vortex/spark/VortexDataSourceWriteTest.java‎

Lines changed: 8 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -26,7 +26,6 @@
2626
import org.apache.spark.sql.SparkSession;
2727
import org.apache.spark.sql.types.DataTypes;
2828
import org.apache.spark.sql.types.MapType;
29-
import org.apache.spark.sql.types.StructField;
3029
import org.apache.spark.sql.types.StructType;
3130
import org.junit.jupiter.api.AfterAll;
3231
import org.junit.jupiter.api.AfterEach;
@@ -123,7 +122,7 @@ public void testWriteAndReadVortexFiles() throws IOException {
123122
.load();
124123

125124
// Then: Verify schema is preserved
126-
assertSchemaEquals(originalDf.schema(), readDf.schema());
125+
assertReadBackSchema(originalDf.schema(), readDf.schema());
127126

128127
// Verify row count
129128
assertEquals(numRows, readDf.count(), "Read DataFrame should have same number of rows as original");
@@ -153,7 +152,7 @@ public void testWriteAndReadWithBarePath() throws IOException {
153152
Dataset<Row> readDf =
154153
spark.read().format("vortex").option("path", barePath).load();
155154

156-
assertSchemaEquals(originalDf.schema(), readDf.schema());
155+
assertReadBackSchema(originalDf.schema(), readDf.schema());
157156
assertEquals(numRows, readDf.count(), "Read DataFrame should have same number of rows as original");
158157
verifyDataContent(originalDf, readDf);
159158

@@ -196,7 +195,7 @@ public void testWriteEmptyDataFrame() throws IOException {
196195
.load();
197196

198197
assertEquals(0, readDf.count(), "Empty DataFrame should remain empty after write/read");
199-
assertSchemaEquals(emptyDf.schema(), readDf.schema());
198+
assertReadBackSchema(emptyDf.schema(), readDf.schema());
200199
}
201200

202201
@Test
@@ -443,7 +442,7 @@ ELSE CAST(NULL AS TIMESTAMP_NTZ) END AS event_ntz""", """
443442

444443
assertFalse(findVortexFiles(outputPath).isEmpty(), "TimestampNTZ write should create Vortex files");
445444
Dataset<Row> actual = spark.read().format("vortex").load(outputPath.toString());
446-
assertSchemaEquals(timestampNtzDf.schema(), actual.schema());
445+
assertReadBackSchema(timestampNtzDf.schema(), actual.schema());
447446
verifyDataContent(timestampNtzDf, actual);
448447
}
449448

@@ -567,24 +566,10 @@ private List<Path> findVortexFiles(Path directory) throws IOException {
567566
}
568567
}
569568

570-
/** Verifies that two schemas are equal. */
571-
private void assertSchemaEquals(StructType expected, StructType actual) {
572-
assertEquals(expected.fields().length, actual.fields().length, "Schemas should have same number of fields");
573-
574-
for (int i = 0; i < expected.fields().length; i++) {
575-
StructField expectedField = expected.fields()[i];
576-
StructField actualField = actual.fields()[i];
577-
578-
assertEquals(expectedField.name(), actualField.name(), "Field names should match at position " + i);
579-
assertEquals(
580-
expectedField.dataType(),
581-
actualField.dataType(),
582-
"Field types should match for field: " + expectedField.name());
583-
assertEquals(
584-
expectedField.nullable(),
585-
actualField.nullable(),
586-
"Field nullability should match for field: " + expectedField.name());
587-
}
569+
/** Verifies that {@code actual} is the {@code written} schema, read back with every field nullable. */
570+
private void assertReadBackSchema(StructType written, StructType actual) {
571+
assertEquals(written.simpleString(), actual.simpleString());
572+
VortexDataSourceInferSchemaTest.assertNoNonNullableFields(actual);
588573
}
589574

590575
/** Verifies that the data content of two DataFrames is identical. */

‎java/vortex-spark/src/test/java/dev/vortex/spark/VortexFilterPushdownTest.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -378,7 +378,7 @@ public void testPushedFiltersInPlan() throws IOException {
378378
SparkPlan plan = qe.executedPlan();
379379
String planString = plan.toString();
380380
assertTrue(
381-
planString.contains("pushedPredicates=[id > 1]"),
381+
planString.contains("pushedPredicates=[id IS NOT NULL, id > 1]"),
382382
"Expected pushed predicate for id > 1 in the executed plan: " + planString);
383383
}
384384

0 commit comments

Comments
 (0)