Skip to content

Commit 16bfaa3

Browse files
committed
fix(spark): read split options in any case
Spark lowercases scan option keys, so vortex.read.splitFiles and vortex.read.maxSplitBytes were only found in the exact spelling of the table options, and a table option could win over a scan option that differed only in case. Look the options up case-insensitively, and let a scan option replace the table option it matches in any case, under the table's spelling so case-sensitive lookups such as vortex.workerThreads still find it. Signed-off-by: gitRasheed <[email protected]>
1 parent 29f6efa commit 16bfaa3

3 files changed

Lines changed: 71 additions & 5 deletions

File tree

‎java/vortex-spark/src/main/java/dev/vortex/spark/VortexTable.java‎

Lines changed: 10 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -58,9 +58,16 @@ public VortexTable(
5858
*/
5959
@Override
6060
public ScanBuilder newScanBuilder(CaseInsensitiveStringMap options) {
61-
Map<String, String> opts = Maps.newHashMap();
62-
opts.putAll(formatOptions);
63-
opts.putAll(options);
61+
Map<String, String> opts = Maps.newHashMap(formatOptions);
62+
// Spark lowercases scan option keys. A scan option replaces the table option it matches in any case, under the
63+
// table's spelling, which is the one case-sensitive lookups such as vortex.workerThreads find.
64+
for (Map.Entry<String, String> option : options.entrySet()) {
65+
String key = opts.keySet().stream()
66+
.filter(option.getKey()::equalsIgnoreCase)
67+
.findFirst()
68+
.orElse(option.getKey());
69+
opts.put(key, option.getValue());
70+
}
6471
return new VortexScanBuilder(opts, partitionTransforms)
6572
.addAllPaths(paths)
6673
.addAllColumns(toColumns(schema));

‎java/vortex-spark/src/main/java/dev/vortex/spark/read/FileSplitPlanner.java‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@
1212
import java.util.Map;
1313
import org.apache.spark.sql.SparkSession;
1414
import org.apache.spark.sql.internal.SQLConf;
15+
import org.apache.spark.sql.util.CaseInsensitiveStringMap;
1516
import scala.Option;
1617

1718
/**
@@ -48,7 +49,7 @@ record FileSize(long rows, long bytes) {}
4849
private FileSplitPlanner() {}
4950

5051
static boolean enabled(Map<String, String> options) {
51-
String value = options.getOrDefault(SPLIT_FILES_OPTION, "false");
52+
String value = new CaseInsensitiveStringMap(options).getOrDefault(SPLIT_FILES_OPTION, "false");
5253
if (value.equalsIgnoreCase("true")) {
5354
return true;
5455
}
@@ -58,7 +59,7 @@ static boolean enabled(Map<String, String> options) {
5859

5960
/** The byte budget of one split, from the option or Spark's file-source formula. */
6061
static long maxSplitBytes(Map<String, String> options, Collection<FileSize> files) {
61-
String configured = options.get(MAX_SPLIT_BYTES_OPTION);
62+
String configured = new CaseInsensitiveStringMap(options).get(MAX_SPLIT_BYTES_OPTION);
6263
if (configured != null) {
6364
long bytes;
6465
try {

‎java/vortex-spark/src/test/java/dev/vortex/spark/VortexSplitReadTest.java‎

Lines changed: 58 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5,14 +5,18 @@
55

66
import static org.junit.jupiter.api.Assertions.assertEquals;
77

8+
import com.google.common.collect.ImmutableList;
89
import java.io.IOException;
910
import java.nio.file.Files;
1011
import java.nio.file.Path;
12+
import java.util.Map;
1113
import java.util.stream.Stream;
1214
import org.apache.spark.sql.Dataset;
1315
import org.apache.spark.sql.Row;
1416
import org.apache.spark.sql.SaveMode;
1517
import org.apache.spark.sql.SparkSession;
18+
import org.apache.spark.sql.connector.expressions.Transform;
19+
import org.apache.spark.sql.util.CaseInsensitiveStringMap;
1620
import org.junit.jupiter.api.AfterAll;
1721
import org.junit.jupiter.api.BeforeAll;
1822
import org.junit.jupiter.api.DisplayName;
@@ -96,4 +100,58 @@ void splitFileIsReadExactlyOnce() throws IOException {
96100
df.filter("name is null").select("id").orderBy("id").collectAsList(),
97101
read.filter("name is null").select("id").orderBy("id").collectAsList());
98102
}
103+
104+
@Test
105+
@DisplayName("Split options match in any case, and scan options override table options")
106+
void splitOptionsIgnoreCase() throws IOException {
107+
Path outputPath = tempDir.resolve("cased");
108+
spark.range(ROWS)
109+
.selectExpr("id", "concat('n', id) as name")
110+
.coalesce(1)
111+
.write()
112+
.format("vortex")
113+
.option("path", outputPath.toUri().toString())
114+
.mode(SaveMode.Overwrite)
115+
.save();
116+
long fileBytes;
117+
try (Stream<Path> files = Files.walk(outputPath)) {
118+
fileBytes = files.filter(p -> p.toString().endsWith(".vortex"))
119+
.mapToLong(p -> p.toFile().length())
120+
.sum();
121+
}
122+
// A third of the file is 33333 rows, rounded down to 32768: four ranges.
123+
spark.conf().set("spark.sql.files.maxPartitionBytes", fileBytes / 3);
124+
spark.conf().set("spark.sql.files.openCostInBytes", 0);
125+
var schema = spark.read()
126+
.format("vortex")
127+
.option("path", outputPath.toUri().toString())
128+
.load()
129+
.schema();
130+
ImmutableList<String> paths = ImmutableList.of(outputPath.toUri().toString());
131+
132+
VortexTable off = new VortexTable(paths, schema, Map.of("vortex.read.splitFiles", "false"), new Transform[0]);
133+
assertEquals(4, partitions(off, Map.of("vortex.read.splitfiles", "true")));
134+
135+
VortexTable on = new VortexTable(paths, schema, Map.of("VORTEX.READ.SPLITFILES", "true"), new Transform[0]);
136+
assertEquals(1, partitions(on, Map.of("vortex.read.splitFiles", "false")));
137+
assertEquals(4, partitions(on, Map.of()));
138+
assertEquals(1, partitions(on, Map.of("Vortex.Read.MaxSplitBytes", Long.toString(Long.MAX_VALUE))));
139+
140+
// Other options keep the table's spelling, which their case-sensitive lookups rely on.
141+
VortexTable threads = new VortexTable(paths, schema, Map.of("vortex.workerThreads", "2"), new Transform[0]);
142+
var partition = (VortexFilePartition)
143+
threads.newScanBuilder(new CaseInsensitiveStringMap(Map.of("vortex.workerThreads", "8")))
144+
.build()
145+
.toBatch()
146+
.planInputPartitions()[0];
147+
assertEquals("8", partition.formatOptions().get("vortex.workerThreads"));
148+
}
149+
150+
private static int partitions(VortexTable table, Map<String, String> scanOptions) {
151+
return table.newScanBuilder(new CaseInsensitiveStringMap(scanOptions))
152+
.build()
153+
.toBatch()
154+
.planInputPartitions()
155+
.length;
156+
}
99157
}

0 commit comments

Comments
 (0)