Skip to content

Commit 29348ee

Browse files
refactor(jni): simplify stream output staging and pin its chunking (#74)
1 parent 9956b11 commit 29348ee

5 files changed

Lines changed: 201 additions & 43 deletions

File tree

‎java/README.md‎

Lines changed: 0 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -7,13 +7,6 @@ We provide two interfaces for working with Vortex from Java:
77
[vortex-spark/README.md](vortex-spark/README.md) for how to load the connector into Spark
88
and query Vortex files from the DataFrame API or Spark SQL.
99

10-
## Compatibility notes
11-
12-
Native stream output now reuses the `byte[]` passed to `NativeWritable.write`.
13-
Implementations must consume the requested range synchronously and must not retain the array.
14-
Asynchronous sinks must copy that range into memory they own before returning.
15-
Native `ByteBufferWritable` buffers and derived views have the same borrowed lifetime.
16-
1710
## Publishing
1811

1912
We publish the JNI artifacts and three Spark connector variants from this repo to Maven Central:

‎java/vortex-jni/src/test/java/dev/vortex/io/NativeOutputBufferTest.java‎

Lines changed: 24 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,8 @@
2222
import java.nio.file.Files;
2323
import java.nio.file.Path;
2424
import java.nio.file.StandardOpenOption;
25+
import java.util.ArrayList;
26+
import java.util.Arrays;
2527
import java.util.List;
2628
import java.util.Map;
2729
import java.util.Random;
@@ -45,12 +47,24 @@ static void loadLibrary() {
4547
NativeLoader.loadJni();
4648
}
4749

50+
/** Random bytes, so each 64 KiB chunk is distinct and can be recognized in a write. */
4851
private static byte[] payload() {
4952
byte[] bytes = new byte[3 * 65_536 + 17];
5053
new Random(0).nextBytes(bytes);
5154
return bytes;
5255
}
5356

57+
/** Index of the payload chunk that {@code buffer[offset, offset + length)} holds exactly, or -1. */
58+
private static int payloadChunk(byte[] payload, byte[] buffer, int offset, int length) {
59+
for (int start = 0, index = 0; start < payload.length; start += 65_536, index++) {
60+
int end = Math.min(start + 65_536, payload.length);
61+
if (end - start == length && Arrays.equals(payload, start, end, buffer, offset, offset + length)) {
62+
return index;
63+
}
64+
}
65+
return -1;
66+
}
67+
5468
private static long writeMetadata(NativeWritable sink, byte[] payload) throws IOException {
5569
try (BufferAllocator allocator = new RootAllocator();
5670
VortexWriter writer = VortexWriter.builder(Session.create(), sink, new Schema(List.of()), allocator)
@@ -130,7 +144,7 @@ public void close() {
130144
void legacyWritesReuseOneBoundedArrayAcrossLargeBuffers() throws IOException {
131145
Path path = directory.resolve("legacy.vortex");
132146
byte[] payload = payload();
133-
AtomicInteger calls = new AtomicInteger();
147+
List<Integer> chunks = new ArrayList<>();
134148
try (OutputStream output = Files.newOutputStream(path)) {
135149
NativeWritable sink = new NativeWritable() {
136150
private byte[] firstArray;
@@ -143,7 +157,10 @@ public void write(byte[] buffer, int offset, int length) throws IOException {
143157
assertSame(firstArray, buffer, "reuse staging across upcalls and native buffers");
144158
assertEquals(65_536, buffer.length);
145159
assertTrue(length > 0 && length <= buffer.length);
146-
calls.incrementAndGet();
160+
int chunk = payloadChunk(payload, buffer, offset, length);
161+
if (chunk >= 0) {
162+
chunks.add(chunk);
163+
}
147164
output.write(buffer, offset, length);
148165
}
149166

@@ -158,7 +175,7 @@ public void close() {
158175
}
159176
};
160177
assertEquals(writeMetadata(sink, payload), Files.size(path));
161-
assertTrue(calls.get() > 4, "large native buffers must be split into bounded writes");
178+
assertEquals(List.of(0, 1, 2, 3), chunks, "the payload is written as 64 KiB chunks in order");
162179
}
163180
assertArrayEquals(
164181
payload,
@@ -262,7 +279,7 @@ public void close() {}
262279

263280
@Test
264281
void legacyWriteExceptionsStopChunkingAndReachTheCaller() {
265-
AtomicInteger fullChunks = new AtomicInteger();
282+
byte[] payload = payload();
266283
AtomicInteger callsAfterFailure = new AtomicInteger();
267284
AtomicBoolean failed = new AtomicBoolean();
268285
NativeWritable sink = new NativeWritable() {
@@ -271,8 +288,8 @@ public void write(byte[] buffer, int offset, int length) throws IOException {
271288
if (failed.get()) {
272289
callsAfterFailure.incrementAndGet();
273290
}
274-
// Fail inside the payload's chunk loop, after one full chunk succeeded.
275-
if (length == buffer.length && fullChunks.incrementAndGet() == 2) {
291+
// Fail inside the payload's chunk loop, after its first chunk succeeded.
292+
if (payloadChunk(payload, buffer, offset, length) == 1) {
276293
failed.set(true);
277294
throw new IOException("injected array write failure");
278295
}
@@ -284,7 +301,7 @@ public void flush() {}
284301
@Override
285302
public void close() {}
286303
};
287-
IOException error = assertThrows(IOException.class, () -> writeMetadata(sink, payload()));
304+
IOException error = assertThrows(IOException.class, () -> writeMetadata(sink, payload));
288305
assertTrue(failed.get());
289306
assertEquals(0, callsAfterFailure.get());
290307
assertTrue(error.getCause().getMessage().contains("injected array write failure"), error.toString());
Lines changed: 153 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,153 @@
1+
// SPDX-License-Identifier: Apache-2.0
2+
// SPDX-FileCopyrightText: Copyright the Vortex contributors
3+
4+
package dev.vortex.spark.bench;
5+
6+
import dev.vortex.api.Session;
7+
import dev.vortex.api.VortexWriter;
8+
import dev.vortex.relocated.org.apache.arrow.memory.BufferAllocator;
9+
import dev.vortex.relocated.org.apache.arrow.memory.RootAllocator;
10+
import dev.vortex.relocated.org.apache.arrow.vector.BigIntVector;
11+
import dev.vortex.relocated.org.apache.arrow.vector.Float8Vector;
12+
import dev.vortex.relocated.org.apache.arrow.vector.VarBinaryVector;
13+
import dev.vortex.relocated.org.apache.arrow.vector.VectorSchemaRoot;
14+
import dev.vortex.relocated.org.apache.arrow.vector.types.FloatingPointPrecision;
15+
import dev.vortex.relocated.org.apache.arrow.vector.types.pojo.ArrowType;
16+
import dev.vortex.relocated.org.apache.arrow.vector.types.pojo.Field;
17+
import dev.vortex.relocated.org.apache.arrow.vector.types.pojo.FieldType;
18+
import dev.vortex.relocated.org.apache.arrow.vector.types.pojo.Schema;
19+
import dev.vortex.spark.io.HadoopWritable;
20+
import java.io.IOException;
21+
import java.lang.management.ManagementFactory;
22+
import java.lang.management.MemoryPoolMXBean;
23+
import java.nio.file.Files;
24+
import java.nio.file.Path;
25+
import java.util.List;
26+
import java.util.Random;
27+
import java.util.concurrent.TimeUnit;
28+
import org.apache.hadoop.conf.Configuration;
29+
import org.openjdk.jmh.annotations.AuxCounters;
30+
import org.openjdk.jmh.annotations.Benchmark;
31+
import org.openjdk.jmh.annotations.BenchmarkMode;
32+
import org.openjdk.jmh.annotations.Fork;
33+
import org.openjdk.jmh.annotations.Level;
34+
import org.openjdk.jmh.annotations.Measurement;
35+
import org.openjdk.jmh.annotations.Mode;
36+
import org.openjdk.jmh.annotations.OutputTimeUnit;
37+
import org.openjdk.jmh.annotations.Param;
38+
import org.openjdk.jmh.annotations.Scope;
39+
import org.openjdk.jmh.annotations.Setup;
40+
import org.openjdk.jmh.annotations.State;
41+
import org.openjdk.jmh.annotations.TearDown;
42+
import org.openjdk.jmh.annotations.Warmup;
43+
44+
/**
45+
* Cost of writing a Vortex file through {@link HadoopWritable}, the sink Spark's file-format writer streams native
46+
* output into.
47+
*
48+
* <p>Run with {@code -prof gc} for heap allocated per file. {@code oldGenPeakMiB} is the peak old-generation use in an
49+
* iteration, where G1 places arrays larger than half a region. JMH sums it over iterations in the summary, so read the
50+
* per-iteration values, for example from {@code -rf json}.
51+
*/
52+
@State(Scope.Benchmark)
53+
@BenchmarkMode(Mode.AverageTime)
54+
@OutputTimeUnit(TimeUnit.MILLISECONDS)
55+
@Warmup(iterations = 5)
56+
@Measurement(iterations = 10)
57+
@Fork(
58+
value = 2,
59+
jvmArgsAppend = {"-Xmx2g", "-XX:+UseG1GC"})
60+
public class HadoopWriteBenchmark {
61+
private static final int BATCH_ROWS = 8192;
62+
63+
/** Rows per file: the larger file has bigger native output buffers. */
64+
@Param({"262144", "4194304"})
65+
private int rows;
66+
67+
private Path directory;
68+
private Configuration conf;
69+
private Session session;
70+
private BufferAllocator allocator;
71+
private Schema schema;
72+
private VectorSchemaRoot batch;
73+
private String path;
74+
75+
@Setup(Level.Trial)
76+
public void setUp() throws IOException {
77+
directory = Files.createTempDirectory("vortex-hadoop-write-bench");
78+
path = directory.resolve("file.vortex").toString();
79+
conf = new Configuration();
80+
session = Session.create();
81+
allocator = new RootAllocator();
82+
schema = new Schema(List.of(
83+
new Field("id", FieldType.notNullable(new ArrowType.Int(64, true)), null),
84+
new Field("payload", FieldType.notNullable(new ArrowType.Binary()), null),
85+
new Field(
86+
"measure",
87+
FieldType.notNullable(new ArrowType.FloatingPoint(FloatingPointPrecision.DOUBLE)),
88+
null)));
89+
batch = VectorSchemaRoot.create(schema, allocator);
90+
// Random values, so encoded output is close to the input size rather than compressed away.
91+
Random random = new Random(0);
92+
BigIntVector id = (BigIntVector) batch.getVector("id");
93+
VarBinaryVector payload = (VarBinaryVector) batch.getVector("payload");
94+
Float8Vector measure = (Float8Vector) batch.getVector("measure");
95+
byte[] value = new byte[32];
96+
for (int i = 0; i < BATCH_ROWS; i++) {
97+
id.setSafe(i, random.nextLong());
98+
random.nextBytes(value);
99+
payload.setSafe(i, value);
100+
measure.setSafe(i, random.nextDouble());
101+
}
102+
batch.setRowCount(BATCH_ROWS);
103+
}
104+
105+
@TearDown(Level.Trial)
106+
public void tearDown() throws IOException {
107+
batch.close();
108+
allocator.close();
109+
SparkScanBenchmark.deleteRecursively(directory);
110+
}
111+
112+
/** Peak G1 old-generation use per iteration, reset before each one. */
113+
@State(Scope.Thread)
114+
@AuxCounters(AuxCounters.Type.EVENTS)
115+
public static class OldGenPeak {
116+
private MemoryPoolMXBean oldGen;
117+
public double oldGenPeakMiB;
118+
119+
@Setup(Level.Trial)
120+
public void findPool() {
121+
oldGen = ManagementFactory.getMemoryPoolMXBeans().stream()
122+
.filter(pool -> pool.getName().equals("G1 Old Gen"))
123+
.findFirst()
124+
.orElseThrow();
125+
}
126+
127+
@Setup(Level.Iteration)
128+
public void reset() {
129+
System.gc();
130+
oldGen.resetPeakUsage();
131+
oldGenPeakMiB = 0;
132+
}
133+
134+
void record() {
135+
oldGenPeakMiB = Math.max(oldGenPeakMiB, oldGen.getPeakUsage().getUsed() / (1024.0 * 1024.0));
136+
}
137+
}
138+
139+
@Benchmark
140+
public long writeFile(OldGenPeak peak) throws IOException {
141+
long size;
142+
try (HadoopWritable sink = HadoopWritable.create(conf, path);
143+
VortexWriter writer =
144+
VortexWriter.builder(session, sink, schema, allocator).build()) {
145+
for (int written = 0; written < rows; written += BATCH_ROWS) {
146+
writer.writeBatch(batch, allocator);
147+
}
148+
size = writer.finish().fileSize();
149+
}
150+
peak.record();
151+
return size;
152+
}
153+
}

‎vortex-jni/src/io/write.rs‎

Lines changed: 23 additions & 28 deletions
Original file line numberDiff line numberDiff line change
@@ -4,24 +4,28 @@
44
use std::io;
55
use std::sync::Arc;
66

7+
use jni::Env;
78
use jni::JValue;
89
use jni::JavaVM;
910
use jni::objects::JByteArray;
1011
use jni::objects::JObject;
1112
use jni::refs::Global;
13+
use jni::sys::jint;
1214
use vortex::error::VortexExpect;
1315
use vortex::error::vortex_err;
1416
use vortex::io::IoBuf;
1517
use vortex::io::VortexWrite;
1618

19+
use crate::errors::JNIError;
1720
use crate::io::with_jvm;
1821

19-
/// Bound heap staging independently of the size of a native output buffer.
20-
const WRITE_SCRATCH_BYTES: usize = 65_536;
22+
/// Size of the `byte[]` each array-mode writer reuses to stage native output for Java, so its
23+
/// heap use stays fixed however large a native output buffer is.
24+
const WRITE_SCRATCH_BYTES: usize = 64 * 1024;
2125

2226
enum WriteMode {
2327
Direct,
24-
Array(Option<Global<JByteArray<'static>>>),
28+
Array(Global<JByteArray<'static>>),
2529
}
2630

2731
/// A [`VortexWrite`] backed by a Java object implementing `dev.vortex.io.NativeWritable`.
@@ -39,23 +43,22 @@ pub(crate) struct JavaWrite {
3943

4044
impl JavaWrite {
4145
pub(crate) fn new(
46+
env: &mut Env,
4247
vm: JavaVM,
4348
writable: Arc<Global<JObject<'static>>>,
4449
direct_buffers: bool,
45-
) -> Self {
46-
Self {
47-
vm,
48-
writable,
49-
mode: if direct_buffers {
50-
WriteMode::Direct
51-
} else {
52-
WriteMode::Array(None)
53-
},
54-
}
50+
) -> Result<Self, JNIError> {
51+
let mode = if direct_buffers {
52+
WriteMode::Direct
53+
} else {
54+
let array = JByteArray::new(env, WRITE_SCRATCH_BYTES)?;
55+
WriteMode::Array(env.new_global_ref(&array)?)
56+
};
57+
Ok(Self { vm, writable, mode })
5558
}
5659

57-
fn write_slice(&mut self, bytes: &[u8]) -> io::Result<()> {
58-
match &mut self.mode {
60+
fn write_slice(&self, bytes: &[u8]) -> io::Result<()> {
61+
match &self.mode {
5962
WriteMode::Direct => {
6063
for chunk in bytes.chunks(i32::MAX as usize) {
6164
with_jvm(&self.vm, |env| {
@@ -100,20 +103,10 @@ impl JavaWrite {
100103
.map_err(io::Error::other)?;
101104
}
102105
}
103-
WriteMode::Array(scratch) => {
104-
if bytes.is_empty() {
105-
return Ok(());
106-
}
106+
WriteMode::Array(array) => {
107107
with_jvm(&self.vm, |env| {
108-
let array = match scratch {
109-
Some(array) => array,
110-
None => {
111-
let array = JByteArray::new(env, WRITE_SCRATCH_BYTES)?;
112-
scratch.insert(env.new_global_ref(&array)?)
113-
}
114-
};
115108
for chunk in bytes.chunks(WRITE_SCRATCH_BYTES) {
116-
let jlen = i32::try_from(chunk.len()).vortex_expect("64 KiB fits in i32");
109+
let jlen = jint::try_from(chunk.len()).vortex_expect("64 KiB fits in jint");
117110
// SAFETY: u8 and i8 have the same size and alignment; only the bit
118111
// representation is copied into Java's signed byte array.
119112
let signed = unsafe {
@@ -213,7 +206,9 @@ mod tests {
213206

214207
let thread_vm = vm.clone();
215208
std::thread::spawn(move || {
216-
let mut write = JavaWrite::new(thread_vm.clone(), Arc::new(writable), false);
209+
let write = with_jvm(&thread_vm, |env| {
210+
JavaWrite::new(env, thread_vm.clone(), Arc::new(writable), false)
211+
})?;
217212
write.write_slice(b"vortex")?;
218213
write.flush_upcall()?;
219214
// `with_jvm` attaches permanently: the thread stays attached until it

‎vortex-jni/src/writer.rs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -368,7 +368,7 @@ pub extern "system" fn Java_dev_vortex_jni_NativeWriter_createStream(
368368
write_options.validate_metadata()?;
369369
let buffered_bytes = write_options.buffered_bytes_tracker();
370370

371-
let mut write = CountingVortexWrite::new(JavaWrite::new(vm, writable, direct_buffers));
371+
let mut write = CountingVortexWrite::new(JavaWrite::new(env, vm, writable, direct_buffers)?);
372372
let bytes_written = write.counter();
373373
let handle = session.handle().spawn(async move {
374374
let summary = write_options.write(&mut write, stream).await?;

0 commit comments

Comments
 (0)