Skip to content

Commit 19a2635

Browse files
fix(jni): refine read handle lifecycle review follow-up
Signed-off-by: Christopher Wirt <[email protected]>
1 parent 7731fbd commit 19a2635

5 files changed

Lines changed: 86 additions & 48 deletions

File tree

‎java/vortex-jni/src/main/java/dev/vortex/api/DataSource.java‎

Lines changed: 18 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,9 @@
2727
* <p>Call {@link #close()} to release this handle and its cached file resources promptly. Scans and partitions already
2828
* created from it remain valid and retain the resources they still need. {@link VortexCleaner} releases an unclosed
2929
* handle when it becomes unreachable.
30+
*
31+
* <p>Native calls are synchronized with {@code close()} to prevent another thread from freeing the handle during an
32+
* operation.
3033
*/
3134
public final class DataSource implements AutoCloseable {
3235
private final Session session;
@@ -129,8 +132,6 @@ public synchronized Schema arrowSchema(BufferAllocator allocator) {
129132
try (ArrowSchema schema = ArrowSchema.allocateNew(allocator)) {
130133
NativeDataSource.arrowSchema(session.nativePointer(), pointer, schema.memoryAddress());
131134
return Data.importSchema(allocator, schema, null);
132-
} finally {
133-
Reference.reachabilityFence(this);
134135
}
135136
}
136137

@@ -141,17 +142,13 @@ public synchronized Schema arrowSchema(BufferAllocator allocator) {
141142
*/
142143
public synchronized RowCount rowCount() {
143144
Preconditions.checkState(!closed, "data source is closed");
144-
try {
145-
long[] out = new long[2];
146-
NativeDataSource.rowCount(pointer, out);
147-
return switch ((int) out[1]) {
148-
case 1 -> new RowCount.Estimate(out[0]);
149-
case 2 -> new RowCount.Exact(out[0]);
150-
default -> RowCount.Unknown.INSTANCE;
151-
};
152-
} finally {
153-
Reference.reachabilityFence(this);
154-
}
145+
long[] out = new long[2];
146+
NativeDataSource.rowCount(pointer, out);
147+
return switch ((int) out[1]) {
148+
case 1 -> new RowCount.Estimate(out[0]);
149+
case 2 -> new RowCount.Exact(out[0]);
150+
default -> RowCount.Unknown.INSTANCE;
151+
};
155152
}
156153

157154
/** Precision-aware row count. See {@link #rowCount()}. */
@@ -197,17 +194,13 @@ public OptionalLong asOptional() {
197194
*/
198195
public synchronized ByteSize byteSize() {
199196
Preconditions.checkState(!closed, "data source is closed");
200-
try {
201-
long[] out = new long[2];
202-
NativeDataSource.byteSize(pointer, out);
203-
return switch ((int) out[1]) {
204-
case 1 -> new ByteSize.Estimate(out[0]);
205-
case 2 -> new ByteSize.Exact(out[0]);
206-
default -> ByteSize.Unknown.INSTANCE;
207-
};
208-
} finally {
209-
Reference.reachabilityFence(this);
210-
}
197+
long[] out = new long[2];
198+
NativeDataSource.byteSize(pointer, out);
199+
return switch ((int) out[1]) {
200+
case 1 -> new ByteSize.Estimate(out[0]);
201+
case 2 -> new ByteSize.Exact(out[0]);
202+
default -> ByteSize.Unknown.INSTANCE;
203+
};
211204
}
212205

213206
/** Precision-aware byte size. See {@link #byteSize()}. */
@@ -276,7 +269,7 @@ public synchronized Scan scan(ScanOptions options) {
276269
splitConcurrency);
277270
return Scan.fromPointer(session, scanPtr);
278271
} finally {
279-
Reference.reachabilityFence(this);
272+
Reference.reachabilityFence(options);
280273
}
281274
}
282275

‎java/vortex-jni/src/main/java/dev/vortex/api/Scan.java‎

Lines changed: 18 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,6 @@
88
import dev.vortex.jni.NativePartition;
99
import dev.vortex.jni.NativeScan;
1010
import java.lang.ref.Cleaner;
11-
import java.lang.ref.Reference;
1211
import java.util.Iterator;
1312
import java.util.NoSuchElementException;
1413
import java.util.concurrent.atomic.AtomicLong;
@@ -23,6 +22,9 @@
2322
* <p>Call {@link #close()} to release the scan and any partition prefetched by {@link #hasNext()} but not yet returned
2423
* by {@link #next()}. Partitions already returned remain valid. {@link VortexCleaner} releases an unclosed scan when it
2524
* becomes unreachable.
25+
*
26+
* <p>Native calls are synchronized with {@code close()} to prevent another thread from freeing the handle during an
27+
* operation.
2628
*/
2729
public final class Scan implements Iterator<Partition>, AutoCloseable {
2830
private final Session session;
@@ -62,27 +64,29 @@ public synchronized Schema arrowSchema(BufferAllocator allocator) {
6264
try (ArrowSchema schema = ArrowSchema.allocateNew(allocator)) {
6365
NativeScan.arrowSchema(session.nativePointer(), pointer, schema.memoryAddress());
6466
return Data.importSchema(allocator, schema, null);
65-
} finally {
66-
Reference.reachabilityFence(this);
6767
}
6868
}
6969

70+
/**
71+
* {@inheritDoc}
72+
*
73+
* @throws IllegalStateException if this scan has been closed
74+
*/
7075
@Override
7176
public synchronized boolean hasNext() {
72-
if (closed) {
73-
return false;
74-
}
75-
try {
76-
if (!primed) {
77-
nextPartitionPointer.set(NativeScan.nextPartition(pointer));
78-
primed = true;
79-
}
80-
return nextPartitionPointer.get() != 0;
81-
} finally {
82-
Reference.reachabilityFence(this);
77+
Preconditions.checkState(!closed, "scan is closed");
78+
if (!primed) {
79+
nextPartitionPointer.set(NativeScan.nextPartition(pointer));
80+
primed = true;
8381
}
82+
return nextPartitionPointer.get() != 0;
8483
}
8584

85+
/**
86+
* {@inheritDoc}
87+
*
88+
* @throws IllegalStateException if this scan has been closed
89+
*/
8690
@Override
8791
public synchronized Partition next() {
8892
if (!hasNext()) {

‎java/vortex-jni/src/test/java/dev/vortex/api/ReadLifecycleTest.java‎

Lines changed: 30 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,8 @@
1919
import java.nio.file.Path;
2020
import java.util.List;
2121
import java.util.NoSuchElementException;
22+
import java.util.concurrent.atomic.AtomicInteger;
23+
import org.apache.arrow.memory.AllocationListener;
2224
import org.apache.arrow.memory.BufferAllocator;
2325
import org.apache.arrow.memory.OutOfMemoryException;
2426
import org.apache.arrow.memory.RootAllocator;
@@ -109,17 +111,26 @@ void primedScanCloseReleasesThePartitionItHasNotReturned() throws Exception {
109111
}
110112

111113
@ParameterizedTest
112-
@ValueSource(longs = {0, 64})
113-
void failedStreamAllocationOrImportReleasesTheConsumedPartition(long limit) throws Exception {
114+
@ValueSource(ints = {1, 2})
115+
void failedStreamAllocationOrImportReleasesTheConsumedPartition(int failingAllocation) throws Exception {
114116
NativeInput input = openNativeInput();
115117
Scan scan = input.source().scan(ScanOptions.of());
116118
Partition partition = scan.next();
117119
scan.close();
118120
input.source().close();
119121
assertFalse(input.readable().refersTo(null));
120-
// Arrow 19 allocates two 40-byte stream structs, rounded to 64 bytes each.
121-
try (BufferAllocator limited = new RootAllocator(limit)) {
122+
AtomicInteger allocations = new AtomicInteger();
123+
AllocationListener listener = new AllocationListener() {
124+
@Override
125+
public void onPreAllocation(long size) {
126+
if (allocations.incrementAndGet() == failingAllocation) {
127+
throw new OutOfMemoryException("injected allocation failure");
128+
}
129+
}
130+
};
131+
try (BufferAllocator limited = new RootAllocator(listener, Long.MAX_VALUE)) {
122132
assertThrows(OutOfMemoryException.class, () -> partition.scanArrow(limited));
133+
assertEquals(failingAllocation, allocations.get());
123134
assertEquals(0, limited.getAllocatedMemory());
124135
assertThrows(IllegalStateException.class, () -> partition.scanArrow(allocator));
125136
assertReadableReleased(input);
@@ -139,15 +150,28 @@ void closingUnusedOrPrimedScanMakesItTerminal(boolean prime) {
139150
}
140151
scan.close();
141152
scan.close();
142-
assertFalse(scan.hasNext());
143-
assertThrows(NoSuchElementException.class, scan::next);
153+
assertThrows(IllegalStateException.class, scan::hasNext);
154+
assertThrows(IllegalStateException.class, scan::next);
144155
assertThrows(IllegalStateException.class, () -> scan.arrowSchema(allocator));
145156
try (Scan another = source.scan(ScanOptions.of())) {
146157
assertTrue(another.hasNext());
147158
}
148159
}
149160
}
150161

162+
@Test
163+
void exhaustedOpenScanRetainsIteratorSemantics() throws IOException {
164+
try (DataSource source = DataSource.open(session, uri);
165+
Scan scan = source.scan(ScanOptions.of())) {
166+
try (ArrowReader reader = scan.next().scanArrow(allocator)) {
167+
assertValues(reader);
168+
}
169+
assertFalse(scan.hasNext());
170+
assertFalse(scan.hasNext());
171+
assertThrows(NoSuchElementException.class, scan::next);
172+
}
173+
}
174+
151175
@Test
152176
void scanSurvivesItsSourceAndReturnedPartitionSurvivesItsScan() throws IOException {
153177
DataSource source = DataSource.open(session, uri);

‎java/vortex-spark/common/src/main/java/dev/vortex/spark/read/VortexPartitionReader.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -186,7 +186,7 @@ private boolean nextBatch() {
186186
closeCurrentReader();
187187
}
188188
if (!scan.hasNext()) {
189-
close();
189+
exhausted = true;
190190
return false;
191191
}
192192
currentReader = scan.next().scanArrow(allocator);

‎java/vortex-spark/common/src/test/java/dev/vortex/spark/read/VortexPartitionReaderLifecycleTest.java‎

Lines changed: 19 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -87,7 +87,6 @@ void closesNativeHandlesWithoutWaitingForGc(String state) throws Exception {
8787
assertEquals(10, reader.get().column(0).getInt(0));
8888
if (state.equals("exhausted")) {
8989
assertFalse(reader.next());
90-
assertClosed(source, scan, readable);
9190
}
9291
}
9392
reader.close();
@@ -100,6 +99,24 @@ void closesNativeHandlesWithoutWaitingForGc(String state) throws Exception {
10099
assertNull(field(reader, "currentBatch"));
101100
}
102101

102+
@Test
103+
void exhaustionLeavesNativeHandlesForTheOwnerToClose() throws Exception {
104+
try (VortexPartitionReader reader = open()) {
105+
DataSource source = (DataSource) field(reader, "dataSource");
106+
Scan scan = (Scan) field(reader, "scan");
107+
NativeReadable readable = (NativeReadable) field(reader, "readable");
108+
assertTrue(reader.next());
109+
assertEquals(10, reader.get().column(0).getInt(0));
110+
assertFalse(reader.next());
111+
assertFalse(reader.next());
112+
assertEquals(3L, source.rowCount().asOptional().orElseThrow());
113+
assertFalse(scan.hasNext());
114+
readable.readFully(0, ByteBuffer.allocateDirect(1));
115+
reader.close();
116+
assertClosed(source, scan, readable);
117+
}
118+
}
119+
103120
@ParameterizedTest
104121
@ValueSource(booleans = {false, true})
105122
void failingBatchReadStillClosesEveryResource(boolean failClose) throws Exception {
@@ -188,7 +205,7 @@ private VortexPartitionReader open() throws IOException {
188205

189206
private static void assertClosed(DataSource source, Scan scan, NativeReadable readable) {
190207
assertThrows(IllegalStateException.class, source::rowCount);
191-
assertFalse(scan.hasNext());
208+
assertThrows(IllegalStateException.class, scan::hasNext);
192209
assertThrows(IllegalStateException.class, () -> readable.readFully(0, ByteBuffer.allocateDirect(1)));
193210
}
194211

0 commit comments

Comments
 (0)