Skip to content

Commit dcada48

Browse files
authored
feat(jni): let Java and Spark writers use the compact strategy (#11406)
The JNI writer always uses the default write strategy, so Java and Spark users cannot select the compact one that Python (`VortexWriteOptions.compact()`) and `vx convert --strategy compact` already offer. #11374 measures 169.95 MB default vs 106.52 MB compact (37% smaller) on 3M rows. Closes #11374. Covers the compression part of #7725. Signed-off-by: jackylee-ch <[email protected]>
1 parent bb1b744 commit dcada48

10 files changed

Lines changed: 191 additions & 7 deletions

File tree

‎docs/user-guide/spark.md‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -137,6 +137,7 @@ derive the column and partition by it, for example `withColumn("year", year(col(
137137
|---------------------------|---------|-------------|
138138
| `batch.size` | `2048` | Rows buffered per write batch; range 1–65536. |
139139
| `vortex.write.batch.size` | — | Rows buffered per write batch for Vortex alone, overriding `batch.size`. |
140+
| `vortex.write.compact` | `false` | Write with the compact strategy: smaller files, slower reads. |
140141

141142
## Spark SQL
142143

‎java/vortex-jni/src/main/java/dev/vortex/api/VortexWriter.java‎

Lines changed: 14 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -92,6 +92,7 @@ public static final class Builder {
9292
private final BufferAllocator allocator;
9393
private final Map<String, String> options = new HashMap<>();
9494
private final Map<String, byte[]> metadata = new HashMap<>();
95+
private boolean compact;
9596

9697
private Builder(
9798
Session session, String uri, NativeWritable writable, Schema arrowSchema, BufferAllocator allocator) {
@@ -145,6 +146,16 @@ public Builder putMetadata(String key, byte[] value) {
145146
return this;
146147
}
147148

149+
/**
150+
* Write with the compact strategy, which prefers a smaller file over read throughput and latency. It is what
151+
* {@code vortex.io.VortexWriteOptions.compact()} selects in Python, and is off by default. The file is read
152+
* like any other.
153+
*/
154+
public Builder compact(boolean compact) {
155+
this.compact = compact;
156+
return this;
157+
}
158+
148159
/**
149160
* Open the writer.
150161
*
@@ -158,9 +169,10 @@ public VortexWriter build() throws IOException {
158169
final long pointer;
159170
try {
160171
pointer = uri != null
161-
? NativeWriter.create(session.nativePointer(), uri, ffi.memoryAddress(), options, metadata)
172+
? NativeWriter.create(
173+
session.nativePointer(), uri, ffi.memoryAddress(), options, metadata, compact)
162174
: NativeWriter.createStream(
163-
session.nativePointer(), writable, ffi.memoryAddress(), metadata);
175+
session.nativePointer(), writable, ffi.memoryAddress(), metadata, compact);
164176
} catch (RuntimeException e) {
165177
// Native failures arrive as RuntimeException. Wrap them so callers can handle every
166178
// failure to open a writer as an IOException, as writeBatch and finish already do.

‎java/vortex-jni/src/main/java/dev/vortex/jni/NativeWriter.java‎

Lines changed: 9 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -18,22 +18,29 @@ private NativeWriter() {}
1818
* Open a writer at {@code uri} that accepts batches matching the Arrow schema at {@code arrowSchemaAddress}.
1919
*
2020
* @param metadata user-defined metadata segments to store in the file footer (may be null)
21+
* @param compact write with the compact strategy, which prefers a smaller file over read throughput
2122
*/
2223
public static native long create(
2324
long sessionPointer,
2425
String uri,
2526
long arrowSchemaAddress,
2627
Map<String, String> options,
27-
Map<String, byte[]> metadata);
28+
Map<String, byte[]> metadata,
29+
boolean compact);
2830

2931
/**
3032
* Open a writer that streams the file into a caller-provided {@link dev.vortex.io.NativeWritable}. The native side
3133
* writes and flushes but never closes the writable; the caller must close it after {@link #close}.
3234
*
3335
* @param metadata user-defined metadata segments to store in the file footer (may be null)
36+
* @param compact write with the compact strategy, which prefers a smaller file over read throughput
3437
*/
3538
public static native long createStream(
36-
long sessionPointer, Object writable, long arrowSchemaAddress, Map<String, byte[]> metadata);
39+
long sessionPointer,
40+
Object writable,
41+
long arrowSchemaAddress,
42+
Map<String, byte[]> metadata,
43+
boolean compact);
3744

3845
/**
3946
* Write a batch directly from Arrow C Data Interface addresses.

‎java/vortex-jni/src/test/java/dev/vortex/jni/IndependentBuffersTest.java‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -119,7 +119,8 @@ public void failedImportReleasesRetains() throws Exception {
119119
directory.resolve("failure.vortex").toString(),
120120
ffiSchema.memoryAddress(),
121121
Map.of(),
122-
Map.of());
122+
Map.of(),
123+
false);
123124
try {
124125
Data.exportVectorSchemaRoot(allocator, root, null, array);
125126
// Omit the struct validity buffer while retaining its child's allocations.

‎java/vortex-jni/src/test/java/dev/vortex/jni/JNIWriterTest.java‎

Lines changed: 77 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -20,13 +20,16 @@
2020
import dev.vortex.api.VortexWriteSummary;
2121
import dev.vortex.api.VortexWriter;
2222
import dev.vortex.arrow.ArrowAllocation;
23+
import dev.vortex.io.NativeWritable;
2324
import java.io.IOException;
25+
import java.io.OutputStream;
2426
import java.nio.file.Files;
2527
import java.nio.file.Path;
2628
import java.util.ArrayList;
2729
import java.util.HashMap;
2830
import java.util.List;
2931
import java.util.Map;
32+
import java.util.Random;
3033
import org.apache.arrow.c.ArrowArray;
3134
import org.apache.arrow.c.ArrowSchema;
3235
import org.apache.arrow.c.Data;
@@ -61,6 +64,7 @@ public final class JNIWriterTest {
6164
private static final byte[] VARIANT_METADATA = new byte[] {0x01, 0x00};
6265
private static final byte[] VARIANT_INT8_42 = new byte[] {0x0c, 0x2a};
6366
private static final byte[] VARIANT_TRUE = new byte[] {0x04};
67+
private static final long PROSE_ROWS = 20_000;
6468

6569
@TempDir
6670
Path tempDir;
@@ -492,6 +496,79 @@ public void testLegacyEditionWritesLegacyStatistics() throws IOException {
492496
assertEquals(new DataSource.RowCount.Exact(3L), ds.rowCount());
493497
}
494498

499+
/** The compact strategy trades read throughput for size, and the file it writes reads like any other. */
500+
@ParameterizedTest
501+
@ValueSource(booleans = {false, true})
502+
public void testCompactWritesASmallerFile(boolean toStream) throws IOException {
503+
BufferAllocator allocator = ArrowAllocation.rootAllocator();
504+
VortexWriteSummary standard = writeProse(allocator, "standard.vortex", false, toStream);
505+
VortexWriteSummary compact = writeProse(allocator, "compact.vortex", true, toStream);
506+
507+
assertEquals(PROSE_ROWS, compact.rowCount());
508+
assertTrue(
509+
compact.fileSize() < standard.fileSize(),
510+
"compact " + compact.fileSize() + " bytes, standard " + standard.fileSize() + " bytes");
511+
assertEquals(
512+
standard.columnStatistics().get(0).lowerBound(),
513+
compact.columnStatistics().get(0).lowerBound());
514+
assertEquals(
515+
standard.columnStatistics().get(0).upperBound(),
516+
compact.columnStatistics().get(0).upperBound());
517+
518+
DataSource ds = DataSource.open(
519+
Session.create(), tempDir.resolve("compact.vortex").toUri().toString());
520+
assertEquals(new DataSource.RowCount.Exact(PROSE_ROWS), ds.rowCount());
521+
}
522+
523+
/** Writes rows of random words, text that a general-purpose compressor shrinks further than the default does. */
524+
private VortexWriteSummary writeProse(BufferAllocator allocator, String name, boolean compact, boolean toStream)
525+
throws IOException {
526+
Schema schema = new Schema(List.of(Field.notNullable("text", new ArrowType.Utf8())));
527+
Path path = tempDir.resolve(name);
528+
Random random = new Random(7);
529+
String[] words = new String[4096];
530+
for (int i = 0; i < words.length; i++) {
531+
words[i] = Long.toString(random.nextLong() & Long.MAX_VALUE, 36);
532+
}
533+
try (VectorSchemaRoot root = VectorSchemaRoot.create(schema, allocator);
534+
OutputStream out = Files.newOutputStream(path)) {
535+
VarCharVector text = (VarCharVector) root.getVector("text");
536+
text.allocateNew((int) PROSE_ROWS);
537+
for (int i = 0; i < PROSE_ROWS; i++) {
538+
StringBuilder row = new StringBuilder();
539+
for (int w = 0; w < 12; w++) {
540+
row.append(words[random.nextInt(words.length)]).append(' ');
541+
}
542+
text.setSafe(i, row.toString().getBytes(UTF_8));
543+
}
544+
root.setRowCount((int) PROSE_ROWS);
545+
546+
Session session = Session.create();
547+
NativeWritable sink = new NativeWritable() {
548+
@Override
549+
public void write(byte[] buffer, int offset, int length) throws IOException {
550+
out.write(buffer, offset, length);
551+
}
552+
553+
@Override
554+
public void flush() throws IOException {
555+
out.flush();
556+
}
557+
558+
@Override
559+
public void close() {}
560+
};
561+
VortexWriter.Builder builder = toStream
562+
? VortexWriter.builder(session, sink, schema, allocator)
563+
: VortexWriter.builder(
564+
session, path.toAbsolutePath().toUri().toString(), schema, allocator);
565+
try (VortexWriter writer = builder.compact(compact).build()) {
566+
writer.writeBatch(root, allocator);
567+
return writer.finish();
568+
}
569+
}
570+
}
571+
495572
private static void writePeopleBatch(VortexWriter writer, BufferAllocator allocator) throws IOException {
496573
try (VectorSchemaRoot root = VectorSchemaRoot.create(personSchema(), allocator)) {
497574
VarCharVector nameVec = (VarCharVector) root.getVector("name");

‎java/vortex-spark/common/src/main/java/dev/vortex/spark/write/VortexOutputWriter.java‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -38,6 +38,12 @@ public final class VortexOutputWriter extends OutputWriter {
3838
/** Option sizing the write batch for Vortex alone, overriding {@value #BATCH_SIZE_OPTION}. */
3939
public static final String WRITE_BATCH_SIZE_OPTION = "vortex.write.batch.size";
4040

41+
/**
42+
* Boolean option that writes with the compact strategy, which prefers a smaller file over read throughput. Off by
43+
* default.
44+
*/
45+
public static final String COMPACT_OPTION = "vortex.write.compact";
46+
4147
private static final int DEFAULT_BATCH_SIZE = 2048;
4248
private static final int MIN_BATCH_SIZE = 1;
4349
private static final int MAX_BATCH_SIZE = 65536; // 64K rows max per batch
@@ -107,6 +113,7 @@ public VortexOutputWriter(
107113
this.session = VortexSparkSession.get(options);
108114
this.vortexWriter = VortexWriter.builder(session, writable, arrowSchema, allocator)
109115
.metadata(metadata)
116+
.compact(options.getBoolean(COMPACT_OPTION, false))
110117
.build();
111118
this.vectorSchemaRoot = VectorSchemaRoot.create(arrowSchema, allocator);
112119
// The native writer holds several batches at once, so a batch of few rows must not take Arrow's default

‎java/vortex-spark/common/src/main/java/dev/vortex/spark/write/VortexOutputWriterFactory.java‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,8 @@ public final class VortexOutputWriterFactory extends OutputWriterFactory {
2424
private final VortexOptions options;
2525

2626
public VortexOutputWriterFactory(StructType schema, VortexOptions options) {
27+
// Spark builds the factory on the driver, so a malformed value fails once there rather than in every task.
28+
options.getBoolean(VortexOutputWriter.COMPACT_OPTION, false);
2729
this.schema = schema;
2830
this.options = options;
2931
}

‎java/vortex-spark/common/src/test/java/dev/vortex/spark/VortexDataSourceWriteTest.java‎

Lines changed: 44 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -12,11 +12,13 @@
1212
import java.io.IOException;
1313
import java.nio.file.Files;
1414
import java.nio.file.Path;
15+
import java.util.ArrayList;
1516
import java.util.Arrays;
1617
import java.util.Comparator;
1718
import java.util.HashMap;
1819
import java.util.List;
1920
import java.util.Map;
21+
import java.util.Random;
2022
import java.util.stream.Collectors;
2123
import java.util.stream.Stream;
2224
import org.apache.spark.sql.Dataset;
@@ -195,6 +197,48 @@ void nullInNullableColumnRoundTrips() {
195197
assertTrue(rows.get(1).isNullAt(1), "expected a null for the second row");
196198
}
197199

200+
@Test
201+
@DisplayName("vortex.write.compact writes smaller files that read back the same")
202+
public void testCompactWriteIsSmallerAndReadsBack() throws IOException {
203+
Random random = new Random(7);
204+
String[] words = new String[4096];
205+
for (int i = 0; i < words.length; i++) {
206+
words[i] = Long.toString(random.nextLong() & Long.MAX_VALUE, 36);
207+
}
208+
List<Row> rows = new ArrayList<>();
209+
for (int i = 0; i < 20_000; i++) {
210+
StringBuilder text = new StringBuilder();
211+
for (int w = 0; w < 12; w++) {
212+
text.append(words[random.nextInt(words.length)]).append(' ');
213+
}
214+
rows.add(RowFactory.create(i, text.toString()));
215+
}
216+
StructType schema =
217+
new StructType().add("id", DataTypes.IntegerType, false).add("text", DataTypes.StringType, false);
218+
Dataset<Row> df = spark.createDataFrame(rows, schema).coalesce(1);
219+
220+
Path standard = tempDir.resolve("standard");
221+
Path compact = tempDir.resolve("compact");
222+
df.write().format("vortex").save(standard.toUri().toString());
223+
df.write()
224+
.format("vortex")
225+
.option("vortex.write.compact", "true")
226+
.save(compact.toUri().toString());
227+
228+
long standardBytes = totalBytes(findVortexFiles(standard));
229+
long compactBytes = totalBytes(findVortexFiles(compact));
230+
assertTrue(compactBytes < standardBytes, "compact " + compactBytes + " bytes, standard " + standardBytes);
231+
verifyDataContent(df, spark.read().format("vortex").load(compact.toUri().toString()));
232+
}
233+
234+
private static long totalBytes(List<Path> files) throws IOException {
235+
long total = 0;
236+
for (Path file : files) {
237+
total += Files.size(file);
238+
}
239+
return total;
240+
}
241+
198242
@Test
199243
@DisplayName("Write empty DataFrame as Vortex")
200244
public void testWriteEmptyDataFrame() throws IOException {

‎java/vortex-spark/common/src/test/java/dev/vortex/spark/write/VortexOutputWriterTest.java‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -87,6 +87,17 @@ void aBatchSizeOfTheWrongShapeNamesItsOption() {
8787
.contains("vortex.write.batch.size"));
8888
}
8989

90+
@Test
91+
void aMalformedCompactOptionFailsWhenSparkBuildsTheFactory() {
92+
StructType schema = new StructType().add("id", DataTypes.LongType, false);
93+
VortexOptions options = options(Map.of("vortex.write.compact", "maybe"));
94+
95+
// Spark builds the factory on the driver, so a malformed value fails before any task starts.
96+
assertTrue(assertThrows(IllegalArgumentException.class, () -> new VortexOutputWriterFactory(schema, options))
97+
.getMessage()
98+
.contains("vortex.write.compact"));
99+
}
100+
90101
/**
91102
* The native writer keeps every batch it receives until a row block is complete, so the memory each batch takes
92103
* must be proportional to its rows, not to Arrow's default vector capacity of a few thousand values.

‎vortex-jni/src/writer.rs‎

Lines changed: 24 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -36,12 +36,15 @@ use object_store::ObjectStore;
3636
use object_store::path::Path as ObjectStorePath;
3737
use vortex::array::ArrayRef;
3838
use vortex::array::stream::ArrayStreamAdapter;
39+
use vortex::compressor::BtrBlocksCompressorBuilder;
3940
use vortex::dtype::DType;
4041
use vortex::error::VortexError;
4142
use vortex::error::VortexResult;
4243
use vortex::error::vortex_err;
4344
use vortex::file::CountingVortexWrite;
45+
use vortex::file::VortexWriteOptions;
4446
use vortex::file::WriteOptionsSessionExt;
47+
use vortex::file::WriteStrategyBuilder;
4548
use vortex::file::WriteSummary;
4649
use vortex::file::multi::parse_uri_or_path;
4750
use vortex::io::VortexWrite;
@@ -94,6 +97,21 @@ fn resolve_store(
9497
}
9598
}
9699

100+
/// Write options for `session`: its defaults, or the compact strategy, which trades read throughput
101+
/// for file size as Python's `VortexWriteOptions.compact()` does.
102+
fn session_write_options(session: &VortexSession, compact: bool) -> VortexWriteOptions {
103+
let options = session.write_options();
104+
if !compact {
105+
return options;
106+
}
107+
let compressor = BtrBlocksCompressorBuilder::from_session(session).with_compact();
108+
options.with_strategy(
109+
WriteStrategyBuilder::from_session(session)
110+
.with_btrblocks_builder(compressor)
111+
.build(),
112+
)
113+
}
114+
97115
/// Native writer holding a write-task handle and a sender that Java pushes batches into.
98116
pub struct NativeWriter {
99117
handle: Option<Task<VortexResult<WriteSummary>>>,
@@ -204,6 +222,7 @@ pub extern "system" fn Java_dev_vortex_jni_NativeWriter_create(
204222
arrow_schema_addr: jlong,
205223
options: JObject,
206224
metadata: JObject,
225+
compact: jboolean,
207226
) -> jlong {
208227
try_or_throw(&mut env, |env| {
209228
if session_ptr == 0 {
@@ -226,7 +245,8 @@ pub extern "system" fn Java_dev_vortex_jni_NativeWriter_create(
226245
let resolved = resolve_store(&file_path, &properties)?;
227246
let (tx, rx) = mpsc::channel(WRITE_CHANNEL_CAPACITY);
228247
let stream = ArrayStreamAdapter::new(write_schema.clone(), rx);
229-
let write_options = session.write_options().with_metadata_segments(metadata);
248+
let write_options =
249+
session_write_options(session, compact).with_metadata_segments(metadata);
230250
// The same check runs inside `write`, but only once the write task is under way, where
231251
// it would surface as an opaque send failure on the first batch.
232252
write_options.validate_metadata()?;
@@ -290,6 +310,7 @@ pub extern "system" fn Java_dev_vortex_jni_NativeWriter_createStream(
290310
writable: JObject,
291311
arrow_schema_addr: jlong,
292312
metadata: JObject,
313+
compact: jboolean,
293314
) -> jlong {
294315
try_or_throw(&mut env, |env| {
295316
if session_ptr == 0 {
@@ -314,7 +335,8 @@ pub extern "system" fn Java_dev_vortex_jni_NativeWriter_createStream(
314335
let writable = Arc::new(env.new_global_ref(&writable)?);
315336
let (tx, rx) = mpsc::channel(WRITE_CHANNEL_CAPACITY);
316337
let stream = ArrayStreamAdapter::new(write_schema.clone(), rx);
317-
let write_options = session.write_options().with_metadata_segments(metadata);
338+
let write_options =
339+
session_write_options(session, compact).with_metadata_segments(metadata);
318340
// See the note in `create`: validate before the write task can start.
319341
write_options.validate_metadata()?;
320342
let buffered_bytes = write_options.buffered_bytes_tracker();

0 commit comments

Comments
 (0)