Skip to content

Commit 9f21db7

Browse files
ogolden-devOtis Goldenclaude
authored
fix(spark): honor session vortex.splitFiles in V1 scans (#79)
## Summary `VortexFileFormat.isSplitable` passed `VortexOptions.of(options.asJava)` straight to `VortexScan.isSplitable`. The other V1 entry points (`inferSchema`, `prepareWrite` and both reader builders) resolve session defaults first, but this one didn't. As a result, V1 and catalog scans ignored `vortex.splitFiles=false` when it was set in the session, under either `vortex.splitFiles` or `spark.datasource.vortex.splitFiles`, and split large files anyway. The V2 path was unaffected because `VortexTable` already resolves session defaults. The fix resolves session defaults in `isSplitable` using the class's existing `withSessionDefaults` helper. Found while reviewing #35. Closes #51 ## Tests Added `VortexFileSplitTest.sessionSplitFilesTurnsSplittingOff`, which runs for both V2 and V1. For each session key form it checks two things: - `false` in the session plans one partition per file and reads every row. - An explicit `.option("vortex.splitFiles", "true")` overrides the session setting and splits the file as usual. ## Checks - `./gradlew -DsparkVersions=4.0 :vortex-spark-4.0_2.13:test --tests 'dev.vortex.spark.VortexFileSplitTest'` passes. - With the fix reverted, the new test's `v1` case fails (`AssertionFailedError` at `VortexFileSplitTest.java:266`) and its `v2` case passes. - Not run: the Spark 3.5 modules, formatting, and lint. 🤖 Generated with [Claude Code](https://claude.com/claude-code) Signed-off-by: Otis Golden <[email protected]> Co-authored-by: Otis Golden <[email protected]> Co-authored-by: Claude Opus 5.5 <[email protected]>
1 parent fd929e9 commit 9f21db7

2 files changed

Lines changed: 31 additions & 2 deletions

File tree

‎java/vortex-spark/common/src/main/scala/dev/vortex/spark/VortexFileFormat.scala‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -69,7 +69,8 @@ final class VortexFileFormat
6969
sparkSession: SparkSession,
7070
options: Map[String, String],
7171
path: Path
72-
): Boolean = VortexScan.isSplitable(VortexOptions.of(options.asJava), path)
72+
): Boolean =
73+
VortexScan.isSplitable(withSessionDefaults(sparkSession, options), path)
7374

7475
override def supportBatch(
7576
sparkSession: SparkSession,

‎java/vortex-spark/common/src/test/java/dev/vortex/spark/VortexFileSplitTest.java‎

Lines changed: 29 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -69,7 +69,9 @@ public void resetConf() {
6969
"spark.sql.files.minPartitionNum",
7070
"spark.sql.files.maxPartitionNum",
7171
"spark.sql.codegen.wholeStage",
72-
"spark.sql.sources.useV1SourceList")) {
72+
"spark.sql.sources.useV1SourceList",
73+
"vortex.splitFiles",
74+
"spark.datasource.vortex.splitFiles")) {
7375
spark.conf().unset(key);
7476
}
7577
}
@@ -249,6 +251,32 @@ void splitFilesOptionTurnsSplittingOff(String mode) {
249251
}
250252
}
251253

254+
@ParameterizedTest(name = "{0}")
255+
@ValueSource(strings = {"v2", "v1"})
256+
@DisplayName("A session vortex.splitFiles=false turns splitting off, and a read option overrides it")
257+
void sessionSplitFilesTurnsSplittingOff(String mode) {
258+
Path output = write(rows(ROWS).coalesce(1), "session_option", null);
259+
List<Long> sizes = fileSizes(output);
260+
long splitSize = sizes.get(0) / 4;
261+
for (String sessionKey : List.of("vortex.splitFiles", "spark.datasource.vortex.splitFiles")) {
262+
spark.conf().set(sessionKey, "false");
263+
String context = "mode=" + mode + " sessionKey=" + sessionKey;
264+
265+
Dataset<Row> read = read(output, mode, splitSize);
266+
assertEquals(1, read.rdd().getNumPartitions(), context);
267+
assertEquals(ROWS, read.count(), context);
268+
269+
Dataset<Row> overridden = spark.read()
270+
.format("vortex")
271+
.option("vortex.splitFiles", "true")
272+
.load(output.toString());
273+
assertEquals(expectedPartitions(sizes, splitSize), overridden.rdd().getNumPartitions(), context);
274+
assertEquals(ROWS, overridden.count(), context);
275+
276+
spark.conf().unset(sessionKey);
277+
}
278+
}
279+
252280
private Dataset<Row> rows(long count) {
253281
return spark.range(count)
254282
.selectExpr(

0 commit comments

Comments
 (0)