|
4 | 4 | package dev.vortex.spark.write; |
5 | 5 |
|
6 | 6 | import static org.junit.jupiter.api.Assertions.assertEquals; |
| 7 | +import static org.junit.jupiter.api.Assertions.assertThrows; |
7 | 8 | import static org.junit.jupiter.api.Assertions.assertTrue; |
8 | 9 |
|
9 | 10 | import dev.vortex.api.DataSource; |
|
16 | 17 | import dev.vortex.relocated.org.apache.arrow.vector.ipc.ArrowReader; |
17 | 18 | import dev.vortex.spark.VortexSparkSession; |
18 | 19 | import java.io.IOException; |
| 20 | +import java.math.BigDecimal; |
19 | 21 | import java.nio.file.Path; |
20 | 22 | import java.util.ArrayList; |
21 | 23 | import java.util.Arrays; |
|
27 | 29 | import org.apache.spark.sql.catalyst.util.ArrayBasedMapData; |
28 | 30 | import org.apache.spark.sql.catalyst.util.GenericArrayData; |
29 | 31 | import org.apache.spark.sql.types.DataTypes; |
| 32 | +import org.apache.spark.sql.types.Decimal; |
30 | 33 | import org.apache.spark.sql.types.StructType; |
31 | 34 | import org.apache.spark.sql.util.CaseInsensitiveStringMap; |
32 | 35 | import org.apache.spark.unsafe.Platform; |
@@ -142,6 +145,51 @@ void consumesReusedRowsAtWriteTime() throws IOException { |
142 | 145 | assertEquals(expected, readRows(fileUri, options)); |
143 | 146 | } |
144 | 147 |
|
| 148 | + /** A write that fails part-way leaves the batch misaligned; commit must then fail and release the writer. */ |
| 149 | + @Test |
| 150 | + void commitAfterFailedWriteReleasesTheWriter() throws IOException { |
| 151 | + StructType schema = |
| 152 | + new StructType().add("id", DataTypes.LongType, false).add("s", DataTypes.StringType, true); |
| 153 | + String fileUri = tempDir.resolve("failed.vortex").toUri().toString(); |
| 154 | + VortexDataWriter writer = new VortexDataWriter(fileUri, schema, new CaseInsensitiveStringMap(Map.of())); |
| 155 | + writer.write(new GenericInternalRow(new Object[] {1L, UTF8String.fromString("a")})); |
| 156 | + assertThrows(ClassCastException.class, () -> writer.write(new GenericInternalRow(new Object[] {2L, "a"}))); |
| 157 | + |
| 158 | + assertThrows(IOException.class, writer::commit); |
| 159 | + assertEquals(0, writer.allocatedBytes()); |
| 160 | + } |
| 161 | + |
| 162 | + @Test |
| 163 | + void decimalsRoundTrip() throws IOException { |
| 164 | + StructType schema = new StructType() |
| 165 | + .add("small", DataTypes.createDecimalType(18, 4), true) |
| 166 | + .add("wide", DataTypes.createDecimalType(38, 10), true); |
| 167 | + UnsafeProjection projection = UnsafeProjection.create(schema); |
| 168 | + String fileUri = tempDir.resolve("decimals.vortex").toUri().toString(); |
| 169 | + CaseInsensitiveStringMap options = new CaseInsensitiveStringMap(Map.of()); |
| 170 | + String[][] values = { |
| 171 | + {"0.0000", "0E-10"}, |
| 172 | + {"-12345678901234.5678", "-1234567890123456789012345678.0123456789"}, |
| 173 | + {"99999999999999.9999", "9999999999999999999999999999.9999999999"}, |
| 174 | + {null, null}, |
| 175 | + }; |
| 176 | + |
| 177 | + VortexDataWriter writer = new VortexDataWriter(fileUri, schema, options); |
| 178 | + List<String> expected = new ArrayList<>(); |
| 179 | + for (int i = 0; i < values.length * 2; i++) { |
| 180 | + String[] row = values[i % values.length]; |
| 181 | + GenericInternalRow generic = new GenericInternalRow(new Object[] { |
| 182 | + row[0] == null ? null : Decimal.apply(new BigDecimal(row[0]), 18, 4), |
| 183 | + row[1] == null ? null : Decimal.apply(new BigDecimal(row[1]), 38, 10), |
| 184 | + }); |
| 185 | + writer.write(i < values.length ? generic : projection.apply(generic)); |
| 186 | + expected.add(row[0] + "\t" + row[1]); |
| 187 | + } |
| 188 | + writer.commit(); |
| 189 | + |
| 190 | + assertEquals(expected, readRows(fileUri, options)); |
| 191 | + } |
| 192 | + |
145 | 193 | private static List<String> readRows(String fileUri, CaseInsensitiveStringMap options) throws IOException { |
146 | 194 | List<String> rows = new ArrayList<>(); |
147 | 195 | try (BufferAllocator allocator = new RootAllocator()) { |
|
0 commit comments