|
5 | 5 |
|
6 | 6 | import static org.junit.jupiter.api.Assertions.assertEquals; |
7 | 7 |
|
| 8 | +import com.google.common.collect.ImmutableList; |
8 | 9 | import java.io.IOException; |
9 | 10 | import java.nio.file.Files; |
10 | 11 | import java.nio.file.Path; |
| 12 | +import java.util.Map; |
11 | 13 | import java.util.stream.Stream; |
12 | 14 | import org.apache.spark.sql.Dataset; |
13 | 15 | import org.apache.spark.sql.Row; |
14 | 16 | import org.apache.spark.sql.SaveMode; |
15 | 17 | import org.apache.spark.sql.SparkSession; |
| 18 | +import org.apache.spark.sql.connector.expressions.Transform; |
| 19 | +import org.apache.spark.sql.util.CaseInsensitiveStringMap; |
16 | 20 | import org.junit.jupiter.api.AfterAll; |
17 | 21 | import org.junit.jupiter.api.BeforeAll; |
18 | 22 | import org.junit.jupiter.api.DisplayName; |
@@ -96,4 +100,58 @@ void splitFileIsReadExactlyOnce() throws IOException { |
96 | 100 | df.filter("name is null").select("id").orderBy("id").collectAsList(), |
97 | 101 | read.filter("name is null").select("id").orderBy("id").collectAsList()); |
98 | 102 | } |
| 103 | + |
| 104 | + @Test |
| 105 | + @DisplayName("Split options match in any case, and scan options override table options") |
| 106 | + void splitOptionsIgnoreCase() throws IOException { |
| 107 | + Path outputPath = tempDir.resolve("cased"); |
| 108 | + spark.range(ROWS) |
| 109 | + .selectExpr("id", "concat('n', id) as name") |
| 110 | + .coalesce(1) |
| 111 | + .write() |
| 112 | + .format("vortex") |
| 113 | + .option("path", outputPath.toUri().toString()) |
| 114 | + .mode(SaveMode.Overwrite) |
| 115 | + .save(); |
| 116 | + long fileBytes; |
| 117 | + try (Stream<Path> files = Files.walk(outputPath)) { |
| 118 | + fileBytes = files.filter(p -> p.toString().endsWith(".vortex")) |
| 119 | + .mapToLong(p -> p.toFile().length()) |
| 120 | + .sum(); |
| 121 | + } |
| 122 | + // A third of the file is 33333 rows, rounded down to 32768: four ranges. |
| 123 | + spark.conf().set("spark.sql.files.maxPartitionBytes", fileBytes / 3); |
| 124 | + spark.conf().set("spark.sql.files.openCostInBytes", 0); |
| 125 | + var schema = spark.read() |
| 126 | + .format("vortex") |
| 127 | + .option("path", outputPath.toUri().toString()) |
| 128 | + .load() |
| 129 | + .schema(); |
| 130 | + ImmutableList<String> paths = ImmutableList.of(outputPath.toUri().toString()); |
| 131 | + |
| 132 | + VortexTable off = new VortexTable(paths, schema, Map.of("vortex.read.splitFiles", "false"), new Transform[0]); |
| 133 | + assertEquals(4, partitions(off, Map.of("vortex.read.splitfiles", "true"))); |
| 134 | + |
| 135 | + VortexTable on = new VortexTable(paths, schema, Map.of("VORTEX.READ.SPLITFILES", "true"), new Transform[0]); |
| 136 | + assertEquals(1, partitions(on, Map.of("vortex.read.splitFiles", "false"))); |
| 137 | + assertEquals(4, partitions(on, Map.of())); |
| 138 | + assertEquals(1, partitions(on, Map.of("Vortex.Read.MaxSplitBytes", Long.toString(Long.MAX_VALUE)))); |
| 139 | + |
| 140 | + // Other options keep the table's spelling, which their case-sensitive lookups rely on. |
| 141 | + VortexTable threads = new VortexTable(paths, schema, Map.of("vortex.workerThreads", "2"), new Transform[0]); |
| 142 | + var partition = (VortexFilePartition) |
| 143 | + threads.newScanBuilder(new CaseInsensitiveStringMap(Map.of("vortex.workerThreads", "8"))) |
| 144 | + .build() |
| 145 | + .toBatch() |
| 146 | + .planInputPartitions()[0]; |
| 147 | + assertEquals("8", partition.formatOptions().get("vortex.workerThreads")); |
| 148 | + } |
| 149 | + |
| 150 | + private static int partitions(VortexTable table, Map<String, String> scanOptions) { |
| 151 | + return table.newScanBuilder(new CaseInsensitiveStringMap(scanOptions)) |
| 152 | + .build() |
| 153 | + .toBatch() |
| 154 | + .planInputPartitions() |
| 155 | + .length; |
| 156 | + } |
99 | 157 | } |
0 commit comments