Skip to content

Commit 7c1972b

Browse files
fix(spark): release native read resources deterministically (#11367)
## Summary Fixes #11364. Closing a Spark reader currently leaves its Java-owned native read handles and file caches alive until Cleaner processing. Make that release explicit and idempotent, while preserving the resources still needed by derived scans, returned partitions and Arrow batches. Ports the lifecycle fix from [QC #103](quantcast#103). ## Changes - Make `DataSource` and `Scan` implement `AutoCloseable`, retaining Cleaner fallback. Keep native calls synchronized with explicit close; fence scan options so their expressions remain alive across JNI. - Release partitions prefetched by `hasNext()` but never returned. - Let Spark's owning wrappers close at EOF. Reader and footer cleanup attempts every close in ownership order and preserves the original failure. - Release native partition ownership when Arrow stream allocation or import fails. - Reject reads on closed handles with `IllegalStateException`. An open, exhausted scan still returns `false` from `hasNext()` and throws `NoSuchElementException` from `next()`. --------- Signed-off-by: Christopher Wirt <[email protected]>
1 parent 5dac460 commit 7c1972b

8 files changed

Lines changed: 694 additions & 113 deletions

File tree

‎java/vortex-jni/src/main/java/dev/vortex/api/DataSource.java‎

Lines changed: 60 additions & 33 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,8 @@
77
import dev.vortex.VortexCleaner;
88
import dev.vortex.io.NativeReadable;
99
import dev.vortex.jni.NativeDataSource;
10+
import java.lang.ref.Cleaner;
11+
import java.lang.ref.Reference;
1012
import java.util.Arrays;
1113
import java.util.Collections;
1214
import java.util.List;
@@ -22,17 +24,24 @@
2224
* A set of Vortex files opened through a {@link Session}. Data sources are cheap to open (only the first file is read
2325
* eagerly, to determine the schema) and can be scanned multiple times.
2426
*
25-
* <p>Native resources are released automatically via {@link VortexCleaner} when the data source becomes unreachable.
27+
* <p>Call {@link #close()} to release this handle and its cached file resources promptly. Scans and partitions already
28+
* created from it remain valid and retain the resources they still need. {@link VortexCleaner} releases an unclosed
29+
* handle when it becomes unreachable.
30+
*
31+
* <p>Native calls are synchronized with {@code close()} to prevent another thread from freeing the handle during an
32+
* operation.
2633
*/
27-
public final class DataSource {
34+
public final class DataSource implements AutoCloseable {
2835
private final Session session;
2936
private final long pointer;
37+
private final Cleaner.Cleanable cleanable;
38+
private boolean closed;
3039

3140
private DataSource(Session session, long pointer) {
3241
Preconditions.checkArgument(pointer != 0, "invalid data source pointer");
3342
this.session = Objects.requireNonNull(session, "session");
3443
this.pointer = pointer;
35-
VortexCleaner.register(this, () -> NativeDataSource.free(pointer));
44+
cleanable = VortexCleaner.register(this, () -> NativeDataSource.free(pointer));
3645
}
3746

3847
/** Open a single URI. */
@@ -88,8 +97,8 @@ public static DataSource open(Session session, List<NativeReadable> readables) {
8897
* for external I/O abstractions (for example Iceberg's {@code FileIO}). Each file is identified by
8998
* {@link NativeReadable#name()}, which must be unique within {@code readables}.
9099
*
91-
* <p>The readables must remain open until this data source and all scans created from it are closed; native code
92-
* never closes them.
100+
* <p>The readables must remain open until this data source is closed and every derived scan, partition, and Arrow
101+
* reader has finished reading. Native code never closes them.
93102
*
94103
* @param session open session
95104
* @param readables byte sources
@@ -118,7 +127,8 @@ public static DataSource open(Session session, List<NativeReadable> readables, i
118127
}
119128

120129
/** Arrow schema of the data source (and of scans produced from it). */
121-
public Schema arrowSchema(BufferAllocator allocator) {
130+
public synchronized Schema arrowSchema(BufferAllocator allocator) {
131+
Preconditions.checkState(!closed, "data source is closed");
122132
try (ArrowSchema schema = ArrowSchema.allocateNew(allocator)) {
123133
NativeDataSource.arrowSchema(session.nativePointer(), pointer, schema.memoryAddress());
124134
return Data.importSchema(allocator, schema, null);
@@ -130,7 +140,8 @@ public Schema arrowSchema(BufferAllocator allocator) {
130140
* {@code DataSource::row_count}: {@link RowCount.Unknown} when no estimate is available, {@link RowCount.Estimate}
131141
* for an inexact hint, {@link RowCount.Exact} when the count is authoritative.
132142
*/
133-
public RowCount rowCount() {
143+
public synchronized RowCount rowCount() {
144+
Preconditions.checkState(!closed, "data source is closed");
134145
long[] out = new long[2];
135146
NativeDataSource.rowCount(pointer, out);
136147
return switch ((int) out[1]) {
@@ -181,7 +192,8 @@ public OptionalLong asOptional() {
181192
* sizes), {@link ByteSize.Estimate} for an inexact hint (some files contribute extrapolated sizes), and
182193
* {@link ByteSize.Exact} when every file has a known size.
183194
*/
184-
public ByteSize byteSize() {
195+
public synchronized ByteSize byteSize() {
196+
Preconditions.checkState(!closed, "data source is closed");
185197
long[] out = new long[2];
186198
NativeDataSource.byteSize(pointer, out);
187199
return switch ((int) out[1]) {
@@ -226,33 +238,48 @@ public OptionalLong asOptional() {
226238
}
227239

228240
/** Submit a scan. */
229-
public Scan scan(ScanOptions options) {
230-
Objects.requireNonNull(options, "options");
241+
public synchronized Scan scan(ScanOptions options) {
242+
Preconditions.checkState(!closed, "data source is closed");
243+
try {
244+
Objects.requireNonNull(options, "options");
231245

232-
long projectionPtr = options.projection().map(Expression::nativePointer).orElse(0L);
233-
long filterPtr = options.filter().map(Expression::nativePointer).orElse(0L);
234-
long begin = options.rowRangeBegin().orElse(0L);
235-
long end = options.rowRangeEnd().orElse(0L);
236-
ScanOptions.SelectionMode selectionMode = options.selectionMode();
237-
long[] selectionIndices = selectionIndices(options);
238-
byte[] selectionRoaringBitmap = selectionRoaringBitmap(options);
239-
long limit = options.limit().orElse(0L);
240-
boolean ordered = options.ordered();
241-
int splitConcurrency = options.splitConcurrency().orElse(0);
246+
long projectionPtr =
247+
options.projection().map(Expression::nativePointer).orElse(0L);
248+
long filterPtr = options.filter().map(Expression::nativePointer).orElse(0L);
249+
long begin = options.rowRangeBegin().orElse(0L);
250+
long end = options.rowRangeEnd().orElse(0L);
251+
ScanOptions.SelectionMode selectionMode = options.selectionMode();
252+
long[] selectionIndices = selectionIndices(options);
253+
byte[] selectionRoaringBitmap = selectionRoaringBitmap(options);
254+
long limit = options.limit().orElse(0L);
255+
boolean ordered = options.ordered();
256+
int splitConcurrency = options.splitConcurrency().orElse(0);
242257

243-
long scanPtr = dev.vortex.jni.NativeScan.create(
244-
pointer,
245-
projectionPtr,
246-
filterPtr,
247-
begin,
248-
end,
249-
selectionIndices,
250-
selectionRoaringBitmap,
251-
selectionMode.code(),
252-
limit,
253-
ordered,
254-
splitConcurrency);
255-
return Scan.fromPointer(session, scanPtr);
258+
long scanPtr = dev.vortex.jni.NativeScan.create(
259+
pointer,
260+
projectionPtr,
261+
filterPtr,
262+
begin,
263+
end,
264+
selectionIndices,
265+
selectionRoaringBitmap,
266+
selectionMode.code(),
267+
limit,
268+
ordered,
269+
splitConcurrency);
270+
return Scan.fromPointer(session, scanPtr);
271+
} finally {
272+
Reference.reachabilityFence(options);
273+
}
274+
}
275+
276+
/** Releases this handle once; existing scans, partitions, and batches remain valid. */
277+
@Override
278+
public synchronized void close() {
279+
if (!closed) {
280+
closed = true;
281+
cleanable.clean();
282+
}
256283
}
257284

258285
private static long[] selectionIndices(ScanOptions options) {

‎java/vortex-jni/src/main/java/dev/vortex/api/Partition.java‎

Lines changed: 33 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@
66
import com.google.common.base.Preconditions;
77
import dev.vortex.VortexCleaner;
88
import dev.vortex.jni.NativePartition;
9+
import java.lang.ref.Reference;
910
import java.util.OptionalLong;
1011
import java.util.concurrent.atomic.AtomicBoolean;
1112
import org.apache.arrow.c.ArrowArrayStream;
@@ -43,12 +44,16 @@ static Partition fromPointer(Session session, long pointer) {
4344
/** Estimated row count of the partition. Empty when unknown. */
4445
public OptionalLong rowCount() {
4546
Preconditions.checkState(!consumed.get(), "partition already consumed");
46-
long[] out = new long[2];
47-
NativePartition.rowCount(pointer, out);
48-
if (out[1] == 0) {
49-
return OptionalLong.empty();
47+
try {
48+
long[] out = new long[2];
49+
NativePartition.rowCount(pointer, out);
50+
if (out[1] == 0) {
51+
return OptionalLong.empty();
52+
}
53+
return OptionalLong.of(out[0]);
54+
} finally {
55+
Reference.reachabilityFence(this);
5056
}
51-
return OptionalLong.of(out[0]);
5257
}
5358

5459
/**
@@ -59,16 +64,30 @@ public ArrowReader scanArrow(BufferAllocator allocator) {
5964
if (!consumed.compareAndSet(false, true)) {
6065
throw new IllegalStateException("partition already consumed");
6166
}
62-
// Native side unconditionally takes ownership of the partition, regardless of
63-
// whether the call subsequently throws, so it is correct to flip `consumed`
64-
// before invoking JNI. The cleaner then always skips free for this handle.
65-
ArrowArrayStream stream = ArrowArrayStream.allocateNew(allocator);
6667
try {
67-
NativePartition.scanArrow(session.nativePointer(), pointer, stream.memoryAddress());
68-
} catch (RuntimeException ex) {
69-
stream.close();
70-
throw ex;
68+
ArrowArrayStream stream;
69+
try {
70+
stream = ArrowArrayStream.allocateNew(allocator);
71+
} catch (RuntimeException | Error failure) {
72+
// JNI has not taken the partition yet, and the Cleaner skips consumed partitions.
73+
NativePartition.free(pointer);
74+
throw failure;
75+
}
76+
try {
77+
// JNI consumes the partition even when it throws.
78+
NativePartition.scanArrow(session.nativePointer(), pointer, stream.memoryAddress());
79+
// Closes the stream struct once its reader has taken the stream.
80+
return Data.importArrayStream(allocator, stream);
81+
} catch (RuntimeException | Error failure) {
82+
// Import can fail before moving the stream into its reader.
83+
if (stream.snapshot().release != 0) {
84+
stream.release();
85+
}
86+
stream.close();
87+
throw failure;
88+
}
89+
} finally {
90+
Reference.reachabilityFence(this);
7191
}
72-
return Data.importArrayStream(allocator, stream);
7392
}
7493
}

‎java/vortex-jni/src/main/java/dev/vortex/api/Scan.java‎

Lines changed: 53 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -5,9 +5,12 @@
55

66
import com.google.common.base.Preconditions;
77
import dev.vortex.VortexCleaner;
8+
import dev.vortex.jni.NativePartition;
89
import dev.vortex.jni.NativeScan;
10+
import java.lang.ref.Cleaner;
911
import java.util.Iterator;
1012
import java.util.NoSuchElementException;
13+
import java.util.concurrent.atomic.AtomicLong;
1114
import org.apache.arrow.c.ArrowSchema;
1215
import org.apache.arrow.c.Data;
1316
import org.apache.arrow.memory.BufferAllocator;
@@ -16,21 +19,37 @@
1619
/**
1720
* A lazy handle to a set of {@link Partition partitions}.
1821
*
19-
* <p>Once a scan has produced its last partition it is effectively exhausted; native resources are released
20-
* automatically via {@link VortexCleaner} when the scan becomes unreachable.
22+
* <p>Call {@link #close()} to release the scan and any partition prefetched by {@link #hasNext()} but not yet returned
23+
* by {@link #next()}. Partitions already returned remain valid. {@link VortexCleaner} releases an unclosed scan when it
24+
* becomes unreachable.
25+
*
26+
* <p>Native calls are synchronized with {@code close()} to prevent another thread from freeing the handle during an
27+
* operation.
2128
*/
22-
public final class Scan implements Iterator<Partition> {
29+
public final class Scan implements Iterator<Partition>, AutoCloseable {
2330
private final Session session;
2431
private final long pointer;
2532

26-
private long nextPartitionPointer;
33+
private final AtomicLong nextPartitionPointer = new AtomicLong();
34+
private final Cleaner.Cleanable cleanable;
2735
private boolean primed;
36+
private boolean closed;
2837

2938
private Scan(Session session, long pointer) {
3039
Preconditions.checkArgument(pointer != 0, "invalid scan pointer");
3140
this.session = session;
3241
this.pointer = pointer;
33-
VortexCleaner.register(this, () -> NativeScan.free(pointer));
42+
AtomicLong pending = nextPartitionPointer;
43+
cleanable = VortexCleaner.register(this, () -> {
44+
try {
45+
long partition = pending.getAndSet(0);
46+
if (partition != 0) {
47+
NativePartition.free(partition);
48+
}
49+
} finally {
50+
NativeScan.free(pointer);
51+
}
52+
});
3453
}
3554

3655
static Scan fromPointer(Session session, long pointer) {
@@ -40,31 +59,50 @@ static Scan fromPointer(Session session, long pointer) {
4059
/**
4160
* Arrow schema produced by this scan. Must be called before the first call to {@link #hasNext()}/{@link #next()}.
4261
*/
43-
public Schema arrowSchema(BufferAllocator allocator) {
62+
public synchronized Schema arrowSchema(BufferAllocator allocator) {
63+
Preconditions.checkState(!closed, "scan is closed");
4464
try (ArrowSchema schema = ArrowSchema.allocateNew(allocator)) {
4565
NativeScan.arrowSchema(session.nativePointer(), pointer, schema.memoryAddress());
4666
return Data.importSchema(allocator, schema, null);
4767
}
4868
}
4969

70+
/**
71+
* {@inheritDoc}
72+
*
73+
* @throws IllegalStateException if this scan has been closed
74+
*/
5075
@Override
51-
public boolean hasNext() {
52-
if (primed) {
53-
return nextPartitionPointer != 0;
76+
public synchronized boolean hasNext() {
77+
Preconditions.checkState(!closed, "scan is closed");
78+
if (!primed) {
79+
nextPartitionPointer.set(NativeScan.nextPartition(pointer));
80+
primed = true;
5481
}
55-
nextPartitionPointer = NativeScan.nextPartition(pointer);
56-
primed = true;
57-
return nextPartitionPointer != 0;
82+
return nextPartitionPointer.get() != 0;
5883
}
5984

85+
/**
86+
* {@inheritDoc}
87+
*
88+
* @throws IllegalStateException if this scan has been closed
89+
*/
6090
@Override
61-
public Partition next() {
91+
public synchronized Partition next() {
6292
if (!hasNext()) {
6393
throw new NoSuchElementException();
6494
}
65-
long ptr = nextPartitionPointer;
66-
nextPartitionPointer = 0;
95+
long ptr = nextPartitionPointer.getAndSet(0);
6796
primed = false;
6897
return Partition.fromPointer(session, ptr);
6998
}
99+
100+
/** Releases this scan once, without closing partitions already returned to the caller. */
101+
@Override
102+
public synchronized void close() {
103+
if (!closed) {
104+
closed = true;
105+
cleanable.clean();
106+
}
107+
}
70108
}

‎java/vortex-jni/src/main/java/dev/vortex/io/NativeReadable.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,7 @@
1616
* opening one underlying stream per concurrent call, or delegating to a positional-read API).
1717
*
1818
* <p>Lifecycle: the creator owns this object. Native code never calls {@link #close()}; close it only after every data
19-
* source or scan built on top of it has been closed.
19+
* source built on top of it has been closed and every derived scan, partition, and Arrow reader has finished reading.
2020
*/
2121
public interface NativeReadable extends Closeable {
2222
/** Default maximum gap between coalesced reads, in bytes (1 MiB), matching the native object-storage policy. */

0 commit comments

Comments
 (0)