Skip to content

Commit 910abcc

Browse files
committed
fix(spark): release a failed writer on commit, and write small decimals as longs
A commit after a failed write threw before any cleanup, and PartitionedVortexDataWriter does not abort a child whose commit threw, so the native writer, allocator, vectors and exported schema leaked. Abort first, then throw. Decimals of up to 18 digits are written from their unscaled long instead of a BigDecimal per value. The capacity comment is shorter and notes that Arrow keeps the largest capacity a batch needed. Signed-off-by: gitRasheed <[email protected]>
1 parent 67fa7eb commit 910abcc

3 files changed

Lines changed: 63 additions & 9 deletions

File tree

‎java/vortex-spark/src/main/java/dev/vortex/spark/write/ArrowFieldWriter.java‎

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,7 @@
3030
import org.apache.spark.sql.types.ByteType;
3131
import org.apache.spark.sql.types.DataType;
3232
import org.apache.spark.sql.types.DateType;
33+
import org.apache.spark.sql.types.Decimal;
3334
import org.apache.spark.sql.types.DecimalType;
3435
import org.apache.spark.sql.types.DoubleType;
3536
import org.apache.spark.sql.types.FloatType;
@@ -321,7 +322,12 @@ private static final class DecimalWriter extends ArrowFieldWriter {
321322

322323
@Override
323324
void setValue(SpecializedGetters row, int ordinal) {
324-
decimals.setSafe(count, row.getDecimal(ordinal, precision, scale).toJavaBigDecimal());
325+
Decimal value = row.getDecimal(ordinal, precision, scale);
326+
if (precision <= Decimal.MAX_LONG_DIGITS()) {
327+
decimals.setSafe(count, value.toUnscaledLong());
328+
} else {
329+
decimals.setSafe(count, value.toJavaBigDecimal());
330+
}
325331
}
326332
}
327333

‎java/vortex-spark/src/main/java/dev/vortex/spark/write/VortexDataWriter.java‎

Lines changed: 8 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -94,13 +94,9 @@ public final class VortexDataWriter implements DataWriter<InternalRow>, AutoClos
9494
.options(options.asCaseSensitiveMap())
9595
.build();
9696
this.vectorSchemaRoot = VectorSchemaRoot.create(arrowSchema, allocator);
97-
// Arrow's default capacity is a few thousand values per vector regardless of how many
98-
// rows a batch holds, and the native writer keeps every batch it receives until a row
99-
// block is complete, so the batch bounds the capacity. Start well below it, though:
100-
// the buffers are allocated at the first row of every batch, and a partition writer
101-
// that sees a handful of rows must not hold a whole batch of direct memory for its
102-
// lifetime. Arrow doubles on demand and remembers the grown capacity for the next
103-
// allocateNew(), so a writer that fills its batches reaches the batch size once.
97+
// Small start so writers that see few rows hold little memory; Arrow grows on demand.
98+
// It also keeps the largest capacity a batch needed, so one batch of huge values makes
99+
// later batches allocate as much.
104100
int initialCapacity = Math.min(batchSize, INITIAL_VECTOR_CAPACITY);
105101
for (FieldVector vector : vectorSchemaRoot.getFieldVectors()) {
106102
vector.setInitialCapacity(initialCapacity);
@@ -197,7 +193,11 @@ long allocatedBytes() {
197193
*/
198194
@Override
199195
public WriterCommitMessage commit() throws IOException {
200-
checkNotFailed();
196+
if (failed) {
197+
// PartitionedVortexDataWriter never aborts a child whose commit threw, so release it here.
198+
abort();
199+
checkNotFailed();
200+
}
201201
if (!closed) {
202202
IOException exception = null;
203203

‎java/vortex-spark/src/test/java/dev/vortex/spark/write/VortexDataWriterTest.java‎

Lines changed: 48 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@
44
package dev.vortex.spark.write;
55

66
import static org.junit.jupiter.api.Assertions.assertEquals;
7+
import static org.junit.jupiter.api.Assertions.assertThrows;
78
import static org.junit.jupiter.api.Assertions.assertTrue;
89

910
import dev.vortex.api.DataSource;
@@ -16,6 +17,7 @@
1617
import dev.vortex.relocated.org.apache.arrow.vector.ipc.ArrowReader;
1718
import dev.vortex.spark.VortexSparkSession;
1819
import java.io.IOException;
20+
import java.math.BigDecimal;
1921
import java.nio.file.Path;
2022
import java.util.ArrayList;
2123
import java.util.Arrays;
@@ -27,6 +29,7 @@
2729
import org.apache.spark.sql.catalyst.util.ArrayBasedMapData;
2830
import org.apache.spark.sql.catalyst.util.GenericArrayData;
2931
import org.apache.spark.sql.types.DataTypes;
32+
import org.apache.spark.sql.types.Decimal;
3033
import org.apache.spark.sql.types.StructType;
3134
import org.apache.spark.sql.util.CaseInsensitiveStringMap;
3235
import org.apache.spark.unsafe.Platform;
@@ -142,6 +145,51 @@ void consumesReusedRowsAtWriteTime() throws IOException {
142145
assertEquals(expected, readRows(fileUri, options));
143146
}
144147

148+
/** A write that fails part-way leaves the batch misaligned; commit must then fail and release the writer. */
149+
@Test
150+
void commitAfterFailedWriteReleasesTheWriter() throws IOException {
151+
StructType schema =
152+
new StructType().add("id", DataTypes.LongType, false).add("s", DataTypes.StringType, true);
153+
String fileUri = tempDir.resolve("failed.vortex").toUri().toString();
154+
VortexDataWriter writer = new VortexDataWriter(fileUri, schema, new CaseInsensitiveStringMap(Map.of()));
155+
writer.write(new GenericInternalRow(new Object[] {1L, UTF8String.fromString("a")}));
156+
assertThrows(ClassCastException.class, () -> writer.write(new GenericInternalRow(new Object[] {2L, "a"})));
157+
158+
assertThrows(IOException.class, writer::commit);
159+
assertEquals(0, writer.allocatedBytes());
160+
}
161+
162+
@Test
163+
void decimalsRoundTrip() throws IOException {
164+
StructType schema = new StructType()
165+
.add("small", DataTypes.createDecimalType(18, 4), true)
166+
.add("wide", DataTypes.createDecimalType(38, 10), true);
167+
UnsafeProjection projection = UnsafeProjection.create(schema);
168+
String fileUri = tempDir.resolve("decimals.vortex").toUri().toString();
169+
CaseInsensitiveStringMap options = new CaseInsensitiveStringMap(Map.of());
170+
String[][] values = {
171+
{"0.0000", "0E-10"},
172+
{"-12345678901234.5678", "-1234567890123456789012345678.0123456789"},
173+
{"99999999999999.9999", "9999999999999999999999999999.9999999999"},
174+
{null, null},
175+
};
176+
177+
VortexDataWriter writer = new VortexDataWriter(fileUri, schema, options);
178+
List<String> expected = new ArrayList<>();
179+
for (int i = 0; i < values.length * 2; i++) {
180+
String[] row = values[i % values.length];
181+
GenericInternalRow generic = new GenericInternalRow(new Object[] {
182+
row[0] == null ? null : Decimal.apply(new BigDecimal(row[0]), 18, 4),
183+
row[1] == null ? null : Decimal.apply(new BigDecimal(row[1]), 38, 10),
184+
});
185+
writer.write(i < values.length ? generic : projection.apply(generic));
186+
expected.add(row[0] + "\t" + row[1]);
187+
}
188+
writer.commit();
189+
190+
assertEquals(expected, readRows(fileUri, options));
191+
}
192+
145193
private static List<String> readRows(String fileUri, CaseInsensitiveStringMap options) throws IOException {
146194
List<String> rows = new ArrayList<>();
147195
try (BufferAllocator allocator = new RootAllocator()) {

0 commit comments

Comments
 (0)