Skip to content

Commit 54b045b

Browse files
authored
feat(jni): bridge caller-provided Java I/O into VortexReadAt/VortexWrite (#8748)
Implement VortexReadAt and VortexWrite using jvm native interfaces to be able to use java readers and writers natively in vortex
1 parent 01e5eb0 commit 54b045b

17 files changed

Lines changed: 1268 additions & 0 deletions

File tree

‎Cargo.lock‎

Lines changed: 2 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎java/vortex-jni/src/main/java/dev/vortex/api/DataSource.java‎

Lines changed: 46 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@
55

66
import com.google.common.base.Preconditions;
77
import dev.vortex.VortexCleaner;
8+
import dev.vortex.io.NativeReadable;
89
import dev.vortex.jni.NativeDataSource;
910
import java.util.Arrays;
1011
import java.util.Collections;
@@ -71,6 +72,51 @@ public static DataSource open(Session session, List<String> uris, Map<String, St
7172
return new DataSource(session, pointer);
7273
}
7374

75+
/** Open a single file through a caller-provided byte source. See {@link #open(Session, List, int)}. */
76+
public static DataSource open(Session session, NativeReadable readable) {
77+
return open(session, List.of(readable), 0);
78+
}
79+
80+
/** Open files through caller-provided byte sources with the default read concurrency. */
81+
public static DataSource open(Session session, List<NativeReadable> readables) {
82+
return open(session, readables, 0);
83+
}
84+
85+
/**
86+
* Open one or more files through caller-provided byte sources instead of native storage clients. Every read the
87+
* scan performs becomes an upcall into the corresponding {@link NativeReadable}, so this is the integration point
88+
* for external I/O abstractions (for example Iceberg's {@code FileIO}). Each file is identified by
89+
* {@link NativeReadable#name()}, which must be unique within {@code readables}.
90+
*
91+
* <p>The readables must remain open until this data source and all scans created from it are closed; native code
92+
* never closes them.
93+
*
94+
* @param session open session
95+
* @param readables byte sources
96+
* @param readConcurrency maximum in-flight {@code readFully} calls across all files of this data source; {@code 0}
97+
* selects the default. Each in-flight read occupies one native thread and typically one stream on its readable.
98+
*/
99+
public static DataSource open(Session session, List<NativeReadable> readables, int readConcurrency) {
100+
Objects.requireNonNull(session, "session");
101+
Objects.requireNonNull(readables, "readables");
102+
Preconditions.checkArgument(!readables.isEmpty(), "at least one readable is required");
103+
Preconditions.checkArgument(readConcurrency >= 0, "readConcurrency must not be negative");
104+
Object[] readableArray = readables.toArray();
105+
String[] names = new String[readableArray.length];
106+
long[] lengths = new long[readableArray.length];
107+
for (int i = 0; i < readableArray.length; i++) {
108+
Preconditions.checkArgument(readableArray[i] != null, "readables must not contain null values");
109+
NativeReadable readable = readables.get(i);
110+
names[i] = readable.name();
111+
Preconditions.checkArgument(names[i] != null, "readable at index %s returned a null name", i);
112+
lengths[i] = readable.length();
113+
Preconditions.checkArgument(lengths[i] >= 0, "readable for %s reported negative length", names[i]);
114+
}
115+
long pointer =
116+
NativeDataSource.openFiles(session.nativePointer(), readableArray, names, lengths, readConcurrency);
117+
return new DataSource(session, pointer);
118+
}
119+
74120
/** Arrow schema of the data source (and of scans produced from it). */
75121
public Schema arrowSchema(BufferAllocator allocator) {
76122
try (ArrowSchema schema = ArrowSchema.allocateNew(allocator)) {

‎java/vortex-jni/src/main/java/dev/vortex/api/VortexWriter.java‎

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@
55

66
import com.google.common.base.Preconditions;
77
import dev.vortex.VortexCleaner;
8+
import dev.vortex.io.NativeWritable;
89
import dev.vortex.jni.NativeWriter;
910
import java.io.IOException;
1011
import java.util.Map;
@@ -66,6 +67,33 @@ public static VortexWriter create(
6667
}
6768
}
6869

70+
/**
71+
* Create a writer that streams the file into a caller-provided byte sink instead of a native storage client. This
72+
* is the integration point for external I/O abstractions (for example Iceberg's {@code PositionOutputStream}).
73+
*
74+
* <p>The native side writes and flushes the sink but never closes it: after {@link #close()} returns, all bytes
75+
* have been written and flushed, and the caller must close the sink to finalize the file.
76+
*/
77+
public static VortexWriter create(
78+
Session session, NativeWritable writable, Schema arrowSchema, BufferAllocator allocator)
79+
throws IOException {
80+
Objects.requireNonNull(session, "session");
81+
Objects.requireNonNull(writable, "writable");
82+
Objects.requireNonNull(arrowSchema, "arrowSchema");
83+
Objects.requireNonNull(allocator, "allocator");
84+
ArrowSchema ffi = ArrowSchema.allocateNew(allocator);
85+
try {
86+
Data.exportSchema(allocator, arrowSchema, null, ffi);
87+
long ptr = NativeWriter.createStream(session.nativePointer(), writable, ffi.memoryAddress());
88+
if (ptr <= 0) {
89+
throw new IOException("failed to create stream writer (ptr=" + ptr + ")");
90+
}
91+
return new VortexWriter(ptr);
92+
} finally {
93+
ffi.close();
94+
}
95+
}
96+
6997
/** Write a batch directly from Arrow C Data Interface addresses. */
7098
public void writeBatch(long arrowArrayAddr, long arrowSchemaAddr) throws IOException {
7199
Preconditions.checkState(!closed.get(), "writer already closed");
Lines changed: 47 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,47 @@
1+
// SPDX-License-Identifier: Apache-2.0
2+
// SPDX-FileCopyrightText: Copyright the Vortex contributors
3+
4+
package dev.vortex.io;
5+
6+
import java.io.Closeable;
7+
import java.io.IOException;
8+
import java.nio.ByteBuffer;
9+
10+
/**
11+
* A random-access byte source supplied by the caller and read from native code.
12+
*
13+
* <p>Implementations bridge external I/O abstractions — for example Iceberg's {@code FileIO} input streams — into the
14+
* Vortex native reader. The native side invokes {@link #readFully(long, ByteBuffer)} from its own worker threads,
15+
* potentially many calls concurrently, so implementations must be safe for concurrent positional reads (typically by
16+
* opening one underlying stream per concurrent call, or delegating to a positional-read API).
17+
*
18+
* <p>Lifecycle: the creator owns this object. Native code never calls {@link #close()}; close it only after every data
19+
* source or scan built on top of it has been closed.
20+
*/
21+
public interface NativeReadable extends Closeable {
22+
/**
23+
* Unique name of this source, typically its original location (URI or file path). Used to key per-session caches
24+
* and in native error messages, so it must be stable and unique across sources; it must not contain glob characters
25+
* ({@code *?[}).
26+
*/
27+
String name();
28+
29+
/** Total length of the source in bytes. Must be cheap; called once when the source is registered. */
30+
long length();
31+
32+
/**
33+
* Read exactly {@code buffer.remaining()} bytes starting at absolute {@code position} into {@code buffer}.
34+
*
35+
* <p>Unlike {@link java.io.InputStream#read}, short reads are not permitted: implementations must either fill the
36+
* buffer completely ({@code buffer.remaining() == 0} on return) or throw.
37+
*
38+
* <p>{@code buffer} is a <em>direct</em> {@link ByteBuffer} wrapping native memory owned by the caller, so bytes
39+
* written to it land in the native reader without an extra copy. It is valid only for the duration of this call:
40+
* implementations must not retain a reference to it after returning. Implementations backed by {@code byte[]} APIs
41+
* can bulk-{@link ByteBuffer#put(byte[], int, int) put} into it.
42+
*
43+
* @throws java.io.EOFException if the range extends past the end of the source
44+
* @throws IOException if the underlying storage fails
45+
*/
46+
void readFully(long position, ByteBuffer buffer) throws IOException;
47+
}
Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,26 @@
1+
// SPDX-License-Identifier: Apache-2.0
2+
// SPDX-FileCopyrightText: Copyright the Vortex contributors
3+
4+
package dev.vortex.io;
5+
6+
import java.io.Closeable;
7+
import java.io.IOException;
8+
9+
/**
10+
* A sequential byte sink supplied by the caller and written to from native code.
11+
*
12+
* <p>Implementations bridge external output abstractions — for example Iceberg's {@code PositionOutputStream} — into
13+
* the Vortex native writer. Writes are sequential and single-threaded: the native side never issues concurrent
14+
* {@code write} calls against the same instance.
15+
*
16+
* <p>Lifecycle: the creator owns this object. Native code calls {@link #write} and {@link #flush} but never
17+
* {@link #close()}; after {@link dev.vortex.api.VortexWriter#close()} returns, all bytes have been written and flushed,
18+
* and the creator must close the underlying stream to finalize it.
19+
*/
20+
public interface NativeWritable extends Closeable {
21+
/** Append {@code length} bytes from {@code buffer} starting at {@code offset}. */
22+
void write(byte[] buffer, int offset, int length) throws IOException;
23+
24+
/** Flush buffered bytes to the underlying storage. */
25+
void flush() throws IOException;
26+
}

‎java/vortex-jni/src/main/java/dev/vortex/jni/NativeDataSource.java‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,21 @@ private NativeDataSource() {}
2222
*/
2323
public static native long open(long sessionPointer, String[] uris, Map<String, String> options);
2424

25+
/**
26+
* Open a data source over caller-provided {@link dev.vortex.io.NativeReadable} objects. All reads become upcalls
27+
* into the supplied readables; no native storage client is created.
28+
*
29+
* @param sessionPointer pointer from {@link NativeSession#newSession()}
30+
* @param readables one {@link dev.vortex.io.NativeReadable} per file
31+
* @param names unique name per file (from {@link dev.vortex.io.NativeReadable#name()}), parallel to
32+
* {@code readables}
33+
* @param lengths file sizes in bytes, parallel to {@code readables}
34+
* @param readConcurrency maximum in-flight {@code readFully} upcalls across all files of this data source;
35+
* {@code <= 0} selects the default
36+
*/
37+
public static native long openFiles(
38+
long sessionPointer, Object[] readables, String[] names, long[] lengths, int readConcurrency);
39+
2540
/** Free a data source pointer. */
2641
public static native void free(long pointer);
2742

‎java/vortex-jni/src/main/java/dev/vortex/jni/NativeWriter.java‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,12 @@ private NativeWriter() {}
1818
public static native long create(
1919
long sessionPointer, String uri, long arrowSchemaAddress, Map<String, String> options);
2020

21+
/**
22+
* Open a writer that streams the file into a caller-provided {@link dev.vortex.io.NativeWritable}. The native side
23+
* writes and flushes but never closes the writable; the caller must close it after {@link #close}.
24+
*/
25+
public static native long createStream(long sessionPointer, Object writable, long arrowSchemaAddress);
26+
2127
/**
2228
* Write a batch directly from Arrow C Data Interface addresses.
2329
*

0 commit comments

Comments
 (0)