Skip to content

Commit 31283ee

Browse files
feat(scan): let a scan request bound the splits it decodes at once (#11127)
- Tracking Issue: #11124 A scan decodes `4 × available_parallelism()` splits at once. In Spark, every task in an executor runs its own scan sized to the whole machine, and each split holds the decoded chunks of every column it overlaps. On wide, nested files that is GiBs of native memory per executor (#11123). This adds a per-scan limit on how many splits are decoded at once. | Layer | API | |---|---| | Rust | `ScanRequest::split_concurrency` → `ScanBuilder::with_some_max_splits_in_flight` | | JNI | `ScanOptions.splitConcurrency()` | | Spark | `vortex.splitConcurrency` read option | **Result:** a full scan of a 180 MB list-layout file, peak live native bytes: | Splits at once | Peak live | Time | |---|---|---| | 48 (default, 4 × 12 cores) | 3.65 GiB | 1.0 s | | 12 | 1.18 GiB | 1.0 s | --------- Signed-off-by: Christopher Wirt <[email protected]>
1 parent 19ae853 commit 31283ee

18 files changed

Lines changed: 398 additions & 38 deletions

File tree

‎docs/user-guide/spark.md‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -196,6 +196,7 @@ own storage clients are not used by the Spark connector.
196196
| Option | Default | Description |
197197
|----------------------------|---------|-------------|
198198
| `vortex.readConcurrency` | `0` | Maximum Hadoop read upcalls per file; `0` uses the native default. |
199+
| `vortex.splitConcurrency` | `0` | Most row splits of a file decoded at once by one task; `0` uses the native default of four per core. Lowering this reduces memory held by concurrent split decoding and Arrow conversion. A scan with both a filter and a limit may also retain one trimmed array per split while preparing output. This does not cap total native memory or I/O read-ahead. |
199200
| `vortex.workerThreads` | `4` | Background threads driving Vortex futures. JVM-wide, and set once per JVM by the first read or write. |
200201
| `vortex.session.provider` | — | Fully-qualified name of a `VortexSessionProvider` supplying a custom session on the driver and on every executor. |
201202
| `vortex.aggregatePushdown` | `true` | Answer `COUNT(*)` from file footers. |

‎java/vortex-jni/src/main/java/dev/vortex/api/DataSource.java‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -238,6 +238,7 @@ public Scan scan(ScanOptions options) {
238238
byte[] selectionRoaringBitmap = selectionRoaringBitmap(options);
239239
long limit = options.limit().orElse(0L);
240240
boolean ordered = options.ordered();
241+
int splitConcurrency = options.splitConcurrency().orElse(0);
241242

242243
long scanPtr = dev.vortex.jni.NativeScan.create(
243244
pointer,
@@ -249,7 +250,8 @@ public Scan scan(ScanOptions options) {
249250
selectionRoaringBitmap,
250251
selectionMode.code(),
251252
limit,
252-
ordered);
253+
ordered,
254+
splitConcurrency);
253255
return Scan.fromPointer(session, scanPtr);
254256
}
255257

‎java/vortex-jni/src/main/java/dev/vortex/api/ScanOptions.java‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@
99
import java.io.UncheckedIOException;
1010
import java.util.Objects;
1111
import java.util.Optional;
12+
import java.util.OptionalInt;
1213
import java.util.OptionalLong;
1314
import org.immutables.value.Value;
1415
import org.roaringbitmap.longlong.Roaring64NavigableMap;
@@ -59,6 +60,25 @@ default boolean ordered() {
5960
return false;
6061
}
6162

63+
/**
64+
* The most splits of each partition the native scan decodes at once, in all, and the most of its batches converted
65+
* to Arrow ahead of the caller. Absent keeps the native default, which scales with the machine's cores.
66+
*
67+
* <p>Each split being decoded holds the decoded chunks of every column it overlaps, so a scan running alongside
68+
* others, such as one Spark task of several in an executor, can lower this to reduce memory held by concurrent
69+
* decoding and Arrow conversion. A scan with both a filter and a limit may also retain one trimmed array per split
70+
* while preparing output. This does not cap total native memory or I/O read-ahead.
71+
*/
72+
OptionalInt splitConcurrency();
73+
74+
@Value.Check
75+
default void validateSplitConcurrency() {
76+
if (splitConcurrency().isPresent() && splitConcurrency().getAsInt() <= 0) {
77+
throw new IllegalArgumentException("split concurrency must be positive, got "
78+
+ splitConcurrency().getAsInt());
79+
}
80+
}
81+
6282
@Value.Check
6383
default void validateSelectionPayload() {
6484
boolean hasIndices = selectionIndices().isPresent();

‎java/vortex-jni/src/main/java/dev/vortex/jni/NativeScan.java‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,7 @@ private NativeScan() {}
2727
* {@code selectionRoaringBitmap})
2828
* @param limit max rows to return, or {@code 0} for "no limit"
2929
* @param ordered true to preserve row order across partitions
30+
* @param splitConcurrency the most splits of each partition read at once, or {@code 0} for the native default
3031
*/
3132
public static native long create(
3233
long dataSourcePointer,
@@ -38,7 +39,8 @@ public static native long create(
3839
byte[] selectionRoaringBitmap,
3940
byte selectionInclude,
4041
long limit,
41-
boolean ordered);
42+
boolean ordered,
43+
int splitConcurrency);
4244

4345
/** Free a scan pointer. */
4446
public static native void free(long pointer);

‎java/vortex-jni/src/test/java/dev/vortex/api/ScanOptionsTest.java‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@
1212
import dev.vortex.api.ScanOptions.SelectionMode;
1313
import java.io.ByteArrayInputStream;
1414
import java.io.DataInputStream;
15+
import java.util.OptionalInt;
1516
import java.util.OptionalLong;
1617
import org.junit.jupiter.api.Test;
1718
import org.roaringbitmap.longlong.Roaring64NavigableMap;
@@ -39,6 +40,21 @@ public void defaultsReadEveryRowAndColumn() {
3940
assertTrue(options.selectionRoaringBitmap().isEmpty());
4041
assertEquals(SelectionMode.INCLUDE_ALL, options.selectionMode());
4142
assertFalse(options.ordered());
43+
assertEquals(OptionalInt.empty(), options.splitConcurrency());
44+
}
45+
46+
@Test
47+
public void splitConcurrencyMustBePositive() {
48+
for (int value : new int[] {0, -1}) {
49+
IllegalArgumentException exception = assertThrows(
50+
IllegalArgumentException.class,
51+
() -> ScanOptions.builder().splitConcurrency(value).build(),
52+
() -> "no exception for " + value);
53+
assertEquals("split concurrency must be positive, got " + value, exception.getMessage());
54+
}
55+
assertEquals(
56+
OptionalInt.of(2),
57+
ScanOptions.builder().splitConcurrency(2).build().splitConcurrency());
4258
}
4359

4460
@Test

‎java/vortex-jni/src/test/java/dev/vortex/api/TestMinimal.java‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -184,6 +184,18 @@ public void testProjectedScanWithFilter() throws Exception {
184184
assertEquals(List.of(new Person("John", BigDecimal.valueOf(10_000L, 2), "VA")), people);
185185
}
186186

187+
@Test
188+
public void testScanWithSplitConcurrencyReadsEveryRow() throws Exception {
189+
BufferAllocator allocator = ArrowAllocation.rootAllocator();
190+
Session session = Session.create();
191+
DataSource ds = DataSource.open(session, writePath);
192+
193+
ScanOptions options =
194+
ScanOptions.builder().splitConcurrency(1).ordered(true).build();
195+
List<Person> people = readAll(ds, options, allocator, TestMinimal::readFullBatch);
196+
assertEquals(MINIMAL_DATA, people);
197+
}
198+
187199
@Test
188200
public void testSelectionIncludesRows() throws Exception {
189201
BufferAllocator allocator = ArrowAllocation.rootAllocator();

‎java/vortex-spark/common/src/main/java/dev/vortex/spark/io/VortexIo.java‎

Lines changed: 39 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -34,19 +34,35 @@ public final class VortexIo implements Serializable {
3434

3535
private static final int DEFAULT_READ_CONCURRENCY = 0;
3636

37+
/**
38+
* Option bounding how many splits of a file the native scan reads at once, in all. Each split in flight holds the
39+
* decoded chunks of every column it overlaps, and every task in an executor runs its own scan, which by default
40+
* keeps several splits in flight per core. Lowering this reduces memory held by concurrent split decoding and Arrow
41+
* conversion. A scan with both a filter and a limit may also retain one trimmed array per split while preparing
42+
* output. This does not cap total native memory or I/O read-ahead. Zero keeps the native default.
43+
*/
44+
public static final String SPLIT_CONCURRENCY_OPTION = "vortex.splitConcurrency";
45+
46+
private static final int DEFAULT_SPLIT_CONCURRENCY = 0;
47+
3748
/** The configuration this instance carries itself, or null when it reads {@link #broadcastConf}. */
3849
private final SerializableHadoopConf conf;
3950

4051
/** The configuration broadcast to every executor once, or null when this instance carries {@link #conf}. */
4152
private final Broadcast<SerializableHadoopConf> broadcastConf;
4253

4354
private final int readConcurrency;
55+
private final int splitConcurrency;
4456

4557
/** Captures {@code hadoopConf} and the read settings found in the format options. */
4658
public static VortexIo create(VortexOptions options, Configuration hadoopConf) {
4759
Objects.requireNonNull(options, "options");
4860
Objects.requireNonNull(hadoopConf, "hadoopConf");
49-
return new VortexIo(new SerializableHadoopConf(hadoopConf), null, parseReadConcurrency(options));
61+
return new VortexIo(
62+
new SerializableHadoopConf(hadoopConf),
63+
null,
64+
nonNegative(options, READ_CONCURRENCY_OPTION, DEFAULT_READ_CONCURRENCY),
65+
nonNegative(options, SPLIT_CONCURRENCY_OPTION, DEFAULT_SPLIT_CONCURRENCY));
5066
}
5167

5268
/**
@@ -58,26 +74,36 @@ public static VortexIo broadcast(VortexOptions options, Configuration hadoopConf
5874
Objects.requireNonNull(hadoopConf, "hadoopConf");
5975
ClassTag<SerializableHadoopConf> tag = ClassTag$.MODULE$.apply(SerializableHadoopConf.class);
6076
return new VortexIo(
61-
null, context.broadcast(new SerializableHadoopConf(hadoopConf), tag), parseReadConcurrency(options));
77+
null,
78+
context.broadcast(new SerializableHadoopConf(hadoopConf), tag),
79+
nonNegative(options, READ_CONCURRENCY_OPTION, DEFAULT_READ_CONCURRENCY),
80+
nonNegative(options, SPLIT_CONCURRENCY_OPTION, DEFAULT_SPLIT_CONCURRENCY));
6281
}
6382

6483
/** Hadoop I/O over a default configuration, for callers that have no Spark session to draw one from. */
6584
public static VortexIo defaults() {
66-
return new VortexIo(new SerializableHadoopConf(new Configuration()), null, DEFAULT_READ_CONCURRENCY);
85+
return new VortexIo(
86+
new SerializableHadoopConf(new Configuration()),
87+
null,
88+
DEFAULT_READ_CONCURRENCY,
89+
DEFAULT_SPLIT_CONCURRENCY);
6790
}
6891

6992
private VortexIo(
70-
SerializableHadoopConf conf, Broadcast<SerializableHadoopConf> broadcastConf, int readConcurrency) {
93+
SerializableHadoopConf conf,
94+
Broadcast<SerializableHadoopConf> broadcastConf,
95+
int readConcurrency,
96+
int splitConcurrency) {
7197
this.conf = conf;
7298
this.broadcastConf = broadcastConf;
7399
this.readConcurrency = readConcurrency;
100+
this.splitConcurrency = splitConcurrency;
74101
}
75102

76-
private static int parseReadConcurrency(VortexOptions options) {
77-
int parsed = options.getInt(READ_CONCURRENCY_OPTION, DEFAULT_READ_CONCURRENCY);
103+
private static int nonNegative(VortexOptions options, String key, int defaultValue) {
104+
int parsed = options.getInt(key, defaultValue);
78105
if (parsed < 0) {
79-
throw new IllegalArgumentException(
80-
String.format(Locale.ROOT, "%s must be >= 0, got %d", READ_CONCURRENCY_OPTION, parsed));
106+
throw new IllegalArgumentException(String.format(Locale.ROOT, "%s must be >= 0, got %d", key, parsed));
81107
}
82108
return parsed;
83109
}
@@ -91,6 +117,11 @@ public int readConcurrency() {
91117
return readConcurrency;
92118
}
93119

120+
/** The most splits of a file the native scan reads at once; zero keeps the native default. */
121+
public int splitConcurrency() {
122+
return splitConcurrency;
123+
}
124+
94125
/**
95126
* Opens a byte source for {@code file}, stating it only if the listing that produced it did not report a size. The
96127
* caller owns the result and must close it once the scan built on it is done.

‎java/vortex-spark/common/src/main/java/dev/vortex/spark/read/VortexPartitionReader.java‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -81,6 +81,9 @@ public VortexPartitionReader(
8181

8282
List<Field> fileColumns = dataSource.arrowSchema(allocator).getFields();
8383
var options = ScanOptions.builder();
84+
if (io.splitConcurrency() > 0) {
85+
options.splitConcurrency(io.splitConcurrency());
86+
}
8487
// Always project, an empty read schema included: a query that needs only partition columns or only a
8588
// row count must not pull every data column off storage. Struct columns that Spark pruned read only the
8689
// nested fields the query uses.

‎java/vortex-spark/common/src/test/java/dev/vortex/spark/VortexV1FallbackTest.java‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -70,6 +70,23 @@ void readsAndWritesThroughTheV1FileFormat() {
7070
.toList());
7171
}
7272

73+
@Test
74+
void aSplitConcurrencyReadsEveryRowThroughV1AndV2() {
75+
Path output = write("split_concurrency", false);
76+
try {
77+
for (String useV1 : new String[] {"", "vortex"}) {
78+
spark.conf().set("spark.sql.sources.useV1SourceList", useV1);
79+
Dataset<Row> read = spark.read()
80+
.format("vortex")
81+
.option("vortex.splitConcurrency", "1")
82+
.load(output.toString());
83+
assertEquals(780L, read.selectExpr("sum(id)").first().getLong(0), "useV1SourceList=" + useV1);
84+
}
85+
} finally {
86+
spark.conf().set("spark.sql.sources.useV1SourceList", "vortex");
87+
}
88+
}
89+
7390
@Test
7491
void readsMissingColumnsWithCodegen() {
7592
Path output = tempDir.resolve("v1_merged");

‎java/vortex-spark/common/src/test/java/dev/vortex/spark/io/VortexIoTest.java‎

Lines changed: 25 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -18,10 +18,30 @@
1818

1919
final class VortexIoTest {
2020
@Test
21-
void readConcurrencyDefaultsToNativeDefault() {
21+
void concurrenciesDefaultToNativeDefaults() {
2222
VortexIo io = VortexIo.create(VortexOptions.empty(), new Configuration());
2323

2424
assertEquals(0, io.readConcurrency());
25+
assertEquals(0, io.splitConcurrency());
26+
}
27+
28+
@Test
29+
void splitConcurrencyIsParsedAndValidated() {
30+
Configuration conf = new Configuration();
31+
32+
assertEquals(
33+
2,
34+
VortexIo.create(options(VortexIo.SPLIT_CONCURRENCY_OPTION, "2"), conf)
35+
.splitConcurrency());
36+
assertEquals(
37+
2,
38+
VortexIo.create(options("VORTEX.SPLITCONCURRENCY", "2"), conf).splitConcurrency());
39+
assertThrows(
40+
IllegalArgumentException.class,
41+
() -> VortexIo.create(options(VortexIo.SPLIT_CONCURRENCY_OPTION, "-1"), conf));
42+
assertThrows(
43+
IllegalArgumentException.class,
44+
() -> VortexIo.create(options(VortexIo.SPLIT_CONCURRENCY_OPTION, "many"), conf));
2545
}
2646

2747
@Test
@@ -54,11 +74,14 @@ void readConcurrencyIsFoundUnderAnyKeyCase() {
5474
void settingsAndConfigurationSurviveSerialization() throws IOException, ClassNotFoundException {
5575
Configuration conf = new Configuration();
5676
conf.set("fs.s3a.endpoint", "https://storage.example");
57-
VortexIo io = VortexIo.create(options(VortexIo.READ_CONCURRENCY_OPTION, "4"), conf);
77+
VortexIo io = VortexIo.create(
78+
VortexOptions.of(Map.of(VortexIo.READ_CONCURRENCY_OPTION, "4", VortexIo.SPLIT_CONCURRENCY_OPTION, "3")),
79+
conf);
5880

5981
VortexIo shipped = roundTrip(io);
6082

6183
assertEquals(4, shipped.readConcurrency());
84+
assertEquals(3, shipped.splitConcurrency());
6285
assertEquals("https://storage.example", shipped.hadoopConf().get("fs.s3a.endpoint"));
6386
}
6487

0 commit comments

Comments
 (0)