Skip to content

Commit 56108c8

Browse files
feat(jni): make Java read coalescing configurable (#11369)
- Tracking Issue: #11366 - Fixes #11366 Java-backed readers always use `CoalesceConfig::object_storage()` (a 1 MiB gap, a 16 MiB maximum merged span). Callers can't change it. The right values depend on the storage underneath. Local SSD, spinning disk and object stores each trade request count against extra bytes read differently. This PR lets Java and Spark callers set them, keeping the current 1 MiB / 16 MiB as the defaults. --------- Signed-off-by: Christopher Wirt <[email protected]>
1 parent 37e3e9f commit 56108c8

11 files changed

Lines changed: 358 additions & 38 deletions

File tree

‎java/vortex-jni/src/main/java/dev/vortex/io/NativeReadable.java‎

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,31 @@
1919
* source or scan built on top of it has been closed.
2020
*/
2121
public interface NativeReadable extends Closeable {
22+
/** Default maximum gap between coalesced reads, in bytes (1 MiB), matching the native object-storage policy. */
23+
int DEFAULT_COALESCE_DISTANCE = 1024 * 1024;
24+
25+
/** Default maximum span of a coalesced read, in bytes (16 MiB). */
26+
int DEFAULT_COALESCE_MAX_SIZE = 16 * 1024 * 1024;
27+
28+
/**
29+
* Maximum gap in bytes between reads that native code may merge. Must be non-negative; zero merges only adjacent or
30+
* overlapping reads. Captured once when the source is registered.
31+
*
32+
* <p>Larger gaps reduce storage requests but may re-read segments already held in the file's segment cache.
33+
*/
34+
default int coalesceDistance() {
35+
return DEFAULT_COALESCE_DISTANCE;
36+
}
37+
38+
/**
39+
* Maximum payload span in bytes of a merged read, excluding leading alignment padding. Must be positive. Individual
40+
* reads may exceed this size. Captured once when the source is registered. Native code caps values near
41+
* Integer.MAX_VALUE to reserve alignment headroom within the Java ByteBuffer limit.
42+
*/
43+
default int coalesceMaxSize() {
44+
return DEFAULT_COALESCE_MAX_SIZE;
45+
}
46+
2247
/**
2348
* Unique name of this source, typically its original location (URI or file path). Used to key per-session caches
2449
* and in native error messages, so it must be stable and unique across sources; it must not contain glob characters

‎java/vortex-jni/src/test/java/dev/vortex/io/NativeIOBridgeTest.java‎

Lines changed: 70 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -48,6 +48,8 @@
4848
import org.junit.jupiter.api.BeforeAll;
4949
import org.junit.jupiter.api.Test;
5050
import org.junit.jupiter.api.io.TempDir;
51+
import org.junit.jupiter.params.ParameterizedTest;
52+
import org.junit.jupiter.params.provider.CsvSource;
5153

5254
/** Round-trip tests for the caller-provided I/O bridge ({@link NativeReadable} / {@link NativeWritable}). */
5355
public final class NativeIOBridgeTest {
@@ -70,17 +72,39 @@ private static final class FileChannelReadable implements NativeReadable {
7072
private final String name;
7173
private final FileChannel channel;
7274
private final long length;
75+
private final int coalesceDistance;
76+
private final int coalesceMaxSize;
77+
private int distanceCalls;
78+
private int maxSizeCalls;
7379

7480
FileChannelReadable(Path path) throws IOException {
7581
this(path.toString(), path);
7682
}
7783

7884
FileChannelReadable(String name, Path path) throws IOException {
85+
this(name, path, DEFAULT_COALESCE_DISTANCE, DEFAULT_COALESCE_MAX_SIZE);
86+
}
87+
88+
FileChannelReadable(String name, Path path, int coalesceDistance, int coalesceMaxSize) throws IOException {
89+
this.coalesceDistance = coalesceDistance;
90+
this.coalesceMaxSize = coalesceMaxSize;
7991
this.name = name;
8092
this.channel = FileChannel.open(path, StandardOpenOption.READ);
8193
this.length = channel.size();
8294
}
8395

96+
@Override
97+
public int coalesceDistance() {
98+
distanceCalls++;
99+
return coalesceDistance;
100+
}
101+
102+
@Override
103+
public int coalesceMaxSize() {
104+
maxSizeCalls++;
105+
return coalesceMaxSize;
106+
}
107+
84108
@Override
85109
public String name() {
86110
return name;
@@ -388,6 +412,52 @@ public void testLargeRoundTrip() throws IOException {
388412
}
389413
}
390414

415+
@ParameterizedTest
416+
@CsvSource({"false, 1048576, 33554432", "true, 1048576, 33554432", "false, 0, 1", "true, 0, 1"})
417+
public void testCoalescingSettingsCapturedOnce(boolean metadata, int distance, int maxSize) throws IOException {
418+
Path path = tempDir.resolve("coalescing.vortex");
419+
Session session = Session.create();
420+
writePeopleFile(session, ArrowAllocation.rootAllocator(), path);
421+
422+
try (FileChannelReadable readable = new FileChannelReadable(path.toString(), path, distance, maxSize)) {
423+
registerReadable(metadata, session, readable);
424+
assertEquals(1, readable.distanceCalls);
425+
assertEquals(1, readable.maxSizeCalls);
426+
}
427+
}
428+
429+
@ParameterizedTest
430+
@CsvSource({
431+
"false, -1, 1024, coalesceDistance",
432+
"true, -1, 1024, coalesceDistance",
433+
"false, 0, 0, coalesceMaxSize",
434+
"true, 0, 0, coalesceMaxSize",
435+
"false, 0, -1, coalesceMaxSize",
436+
"true, 0, -1, coalesceMaxSize"
437+
})
438+
public void testInvalidCoalescingSettingsRejected(boolean metadata, int distance, int maxSize, String setting)
439+
throws IOException {
440+
Path path = tempDir.resolve("invalid_coalescing.vortex");
441+
Files.write(path, new byte[] {0});
442+
Session session = Session.create();
443+
try (FileChannelReadable readable = new FileChannelReadable(path.toString(), path, distance, maxSize)) {
444+
RuntimeException thrown =
445+
assertThrows(RuntimeException.class, () -> registerReadable(metadata, session, readable));
446+
assertTrue(thrown.getMessage().contains(setting), thrown.getMessage());
447+
}
448+
}
449+
450+
/** Registers {@code readable} through the file-metadata or data-source JNI entry point. */
451+
private static void registerReadable(boolean metadata, Session session, NativeReadable readable) {
452+
if (metadata) {
453+
assertEquals(2, NativeFiles.readStatistics(session, readable).size());
454+
} else {
455+
assertEquals(
456+
3L,
457+
DataSource.open(session, readable).rowCount().asOptional().orElseThrow());
458+
}
459+
}
460+
391461
@Test
392462
public void testReadableExceptionPropagates() {
393463
Session session = Session.create();

‎java/vortex-spark/README.md‎

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -65,6 +65,31 @@ Catalog tables read and write through the V1 file format;
6565
`spark.sql.sources.useV1SourceList=vortex` puts every path on it, which is what the `_metadata`
6666
column and dynamic partition pruning need.
6767

68+
## Read coalescing
69+
70+
Nearby storage reads can be merged to reduce requests. Set the maximum gap and merged span in bytes:
71+
72+
```java
73+
Dataset<Row> result = spark.read()
74+
.format("vortex")
75+
.option("vortex.read.coalesce.distance", 262144)
76+
.option("vortex.read.coalesce.maxSize", 16777216)
77+
.load("s3a://bucket/path");
78+
```
79+
80+
The defaults remain a 1 MiB gap and a 16 MiB maximum merged span. The gap must be non-negative
81+
(zero merges only adjacent or overlapping reads); the maximum span must be positive. Both accept
82+
integers up to 2147483647 bytes. The native reader caps maximums near that limit to leave room
83+
for alignment padding (up to 64 KiB minus one byte). Individual reads can exceed the maximum
84+
merged span, and merged reads may include leading alignment padding.
85+
A wider gap trades fewer storage requests for more bytes read, including segments already held
86+
in the file's segment cache.
87+
88+
Options are case-insensitive and work with both V1 and V2 reads.
89+
90+
Java callers supplying their own `NativeReadable` can override `coalesceDistance()` and
91+
`coalesceMaxSize()`. Native code captures these limits once when registering each source.
92+
6893
## Benchmarks
6994

7095
JMH benchmarks live in `common/src/jmh/java`. They measure the per-file open cost of the

‎java/vortex-spark/common/src/main/java/dev/vortex/spark/io/HadoopReadable.java‎

Lines changed: 26 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,8 @@
2222
public final class HadoopReadable extends PooledReadable {
2323
private final Path path;
2424
private final FileSystem fs;
25+
private final int coalesceDistance;
26+
private final int coalesceMaxSize;
2527

2628
/** Opens a readable over {@code path}, stating the file for its length. */
2729
public static HadoopReadable open(Configuration conf, String path) {
@@ -33,20 +35,42 @@ public static HadoopReadable open(Configuration conf, String path) {
3335
* stats every file when it plans the scan, so executors need not stat them again.
3436
*/
3537
public static HadoopReadable open(Configuration conf, String path, long length) {
38+
return open(conf, path, length, DEFAULT_COALESCE_DISTANCE, DEFAULT_COALESCE_MAX_SIZE);
39+
}
40+
41+
/**
42+
* Opens a readable with the given coalescing gap and maximum merged size, both in bytes. Native code rejects a
43+
* negative gap or non-positive size when the source is registered.
44+
*/
45+
public static HadoopReadable open(
46+
Configuration conf, String path, long length, int coalesceDistance, int coalesceMaxSize) {
3647
Path file = new Path(path);
3748
try {
3849
FileSystem fs = file.getFileSystem(conf);
3950
long resolved = length >= 0 ? length : fs.getFileStatus(file).getLen();
40-
return new HadoopReadable(path, file, fs, resolved);
51+
return new HadoopReadable(path, file, fs, resolved, coalesceDistance, coalesceMaxSize);
4152
} catch (IOException e) {
4253
throw new UncheckedIOException("Failed to open " + path, e);
4354
}
4455
}
4556

46-
private HadoopReadable(String name, Path path, FileSystem fs, long length) {
57+
private HadoopReadable(
58+
String name, Path path, FileSystem fs, long length, int coalesceDistance, int coalesceMaxSize) {
4759
super(name, length);
4860
this.path = path;
4961
this.fs = fs;
62+
this.coalesceDistance = coalesceDistance;
63+
this.coalesceMaxSize = coalesceMaxSize;
64+
}
65+
66+
@Override
67+
public int coalesceDistance() {
68+
return coalesceDistance;
69+
}
70+
71+
@Override
72+
public int coalesceMaxSize() {
73+
return coalesceMaxSize;
5074
}
5175

5276
@Override

‎java/vortex-spark/common/src/main/java/dev/vortex/spark/io/VortexIo.java‎

Lines changed: 43 additions & 29 deletions
Original file line numberDiff line numberDiff line change
@@ -24,7 +24,7 @@
2424
* <p>Built on the driver and shipped to executors, so the configuration travels with it, or is broadcast to them.
2525
*/
2626
public final class VortexIo implements Serializable {
27-
private static final long serialVersionUID = 1L;
27+
private static final long serialVersionUID = 2L;
2828

2929
/**
3030
* Option bounding how many concurrent read upcalls the native reader issues against one file. Each in-flight upcall
@@ -34,6 +34,16 @@ public final class VortexIo implements Serializable {
3434

3535
private static final int DEFAULT_READ_CONCURRENCY = 0;
3636

37+
/**
38+
* Maximum gap between merged reads, in bytes. Defaults to 1 MiB; zero merges only adjacent or overlapping reads.
39+
*/
40+
public static final String COALESCE_DISTANCE_OPTION = "vortex.read.coalesce.distance";
41+
42+
/**
43+
* Maximum span of a merged read, in bytes. Must be positive; defaults to 16 MiB. Individual reads may be larger.
44+
*/
45+
public static final String COALESCE_MAX_SIZE_OPTION = "vortex.read.coalesce.maxSize";
46+
3747
/**
3848
* Option bounding how many splits of a file the native scan reads at once, in all. Each split in flight holds the
3949
* decoded chunks of every column it overlaps, and every task in an executor runs its own scan, which by default
@@ -51,53 +61,48 @@ public final class VortexIo implements Serializable {
5161
/** The configuration broadcast to every executor once, or null when this instance carries {@link #conf}. */
5262
private final Broadcast<SerializableHadoopConf> broadcastConf;
5363

54-
private final int readConcurrency;
55-
private final int splitConcurrency;
64+
private final ReadSettings settings;
65+
66+
private record ReadSettings(int readConcurrency, int splitConcurrency, int coalesceDistance, int coalesceMaxSize)
67+
implements Serializable {
68+
static ReadSettings parse(VortexOptions options) {
69+
Objects.requireNonNull(options, "options");
70+
return new ReadSettings(
71+
nonNegative(options, READ_CONCURRENCY_OPTION, DEFAULT_READ_CONCURRENCY),
72+
nonNegative(options, SPLIT_CONCURRENCY_OPTION, DEFAULT_SPLIT_CONCURRENCY),
73+
nonNegative(options, COALESCE_DISTANCE_OPTION, NativeReadable.DEFAULT_COALESCE_DISTANCE),
74+
positive(options, COALESCE_MAX_SIZE_OPTION, NativeReadable.DEFAULT_COALESCE_MAX_SIZE));
75+
}
76+
}
5677

5778
/** Captures {@code hadoopConf} and the read settings found in the format options. */
5879
public static VortexIo create(VortexOptions options, Configuration hadoopConf) {
59-
Objects.requireNonNull(options, "options");
80+
ReadSettings settings = ReadSettings.parse(options);
6081
Objects.requireNonNull(hadoopConf, "hadoopConf");
61-
return new VortexIo(
62-
new SerializableHadoopConf(hadoopConf),
63-
null,
64-
nonNegative(options, READ_CONCURRENCY_OPTION, DEFAULT_READ_CONCURRENCY),
65-
nonNegative(options, SPLIT_CONCURRENCY_OPTION, DEFAULT_SPLIT_CONCURRENCY));
82+
return new VortexIo(new SerializableHadoopConf(hadoopConf), null, settings);
6683
}
6784

6885
/**
6986
* Like {@link #create}, but broadcasts {@code hadoopConf} through {@code context}. Each task that deserializes the
7087
* result then shares the executor's copy, instead of reading the whole configuration back from its own bytes.
7188
*/
7289
public static VortexIo broadcast(VortexOptions options, Configuration hadoopConf, SparkContext context) {
73-
Objects.requireNonNull(options, "options");
90+
ReadSettings settings = ReadSettings.parse(options);
7491
Objects.requireNonNull(hadoopConf, "hadoopConf");
7592
ClassTag<SerializableHadoopConf> tag = ClassTag$.MODULE$.apply(SerializableHadoopConf.class);
76-
return new VortexIo(
77-
null,
78-
context.broadcast(new SerializableHadoopConf(hadoopConf), tag),
79-
nonNegative(options, READ_CONCURRENCY_OPTION, DEFAULT_READ_CONCURRENCY),
80-
nonNegative(options, SPLIT_CONCURRENCY_OPTION, DEFAULT_SPLIT_CONCURRENCY));
93+
return new VortexIo(null, context.broadcast(new SerializableHadoopConf(hadoopConf), tag), settings);
8194
}
8295

8396
/** Hadoop I/O over a default configuration, for callers that have no Spark session to draw one from. */
8497
public static VortexIo defaults() {
85-
return new VortexIo(
86-
new SerializableHadoopConf(new Configuration()),
87-
null,
88-
DEFAULT_READ_CONCURRENCY,
89-
DEFAULT_SPLIT_CONCURRENCY);
98+
return create(VortexOptions.empty(), new Configuration());
9099
}
91100

92101
private VortexIo(
93-
SerializableHadoopConf conf,
94-
Broadcast<SerializableHadoopConf> broadcastConf,
95-
int readConcurrency,
96-
int splitConcurrency) {
102+
SerializableHadoopConf conf, Broadcast<SerializableHadoopConf> broadcastConf, ReadSettings settings) {
97103
this.conf = conf;
98104
this.broadcastConf = broadcastConf;
99-
this.readConcurrency = readConcurrency;
100-
this.splitConcurrency = splitConcurrency;
105+
this.settings = settings;
101106
}
102107

103108
private static int nonNegative(VortexOptions options, String key, int defaultValue) {
@@ -108,18 +113,26 @@ private static int nonNegative(VortexOptions options, String key, int defaultVal
108113
return parsed;
109114
}
110115

116+
private static int positive(VortexOptions options, String key, int defaultValue) {
117+
int parsed = options.getInt(key, defaultValue);
118+
if (parsed <= 0) {
119+
throw new IllegalArgumentException(String.format(Locale.ROOT, "%s must be > 0, got %d", key, parsed));
120+
}
121+
return parsed;
122+
}
123+
111124
public Configuration hadoopConf() {
112125
return conf != null ? conf.value() : broadcastConf.value().value();
113126
}
114127

115128
/** Bound on concurrent read upcalls per file; zero keeps the native default. */
116129
public int readConcurrency() {
117-
return readConcurrency;
130+
return settings.readConcurrency();
118131
}
119132

120133
/** The most splits of a file the native scan reads at once; zero keeps the native default. */
121134
public int splitConcurrency() {
122-
return splitConcurrency;
135+
return settings.splitConcurrency();
123136
}
124137

125138
/**
@@ -133,6 +146,7 @@ public int splitConcurrency() {
133146
*/
134147
public NativeReadable openReadable(VortexFile file) {
135148
VortexFile.requireVortexExtension(file.path());
136-
return HadoopReadable.open(hadoopConf(), file.path(), file.length());
149+
return HadoopReadable.open(
150+
hadoopConf(), file.path(), file.length(), settings.coalesceDistance(), settings.coalesceMaxSize());
137151
}
138152
}

‎java/vortex-spark/common/src/test/java/dev/vortex/spark/io/HadoopReadableTest.java‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -52,6 +52,8 @@ void readsEveryRangeOfTheFile() throws IOException {
5252
Path file = write("data.vortex", content);
5353

5454
try (NativeReadable readable = HadoopReadable.open(conf, file.toString())) {
55+
assertEquals(NativeReadable.DEFAULT_COALESCE_DISTANCE, readable.coalesceDistance());
56+
assertEquals(NativeReadable.DEFAULT_COALESCE_MAX_SIZE, readable.coalesceMaxSize());
5557
assertEquals(SIZE, readable.length());
5658
assertEquals(file.toString(), readable.name());
5759

0 commit comments

Comments
 (0)