Skip to content

Commit 0c636f1

Browse files
2018ik2018ik
committed
Hold strong reference to all buffers passed to LmdbJava (fixes #207)
1 parent 9cf97a5 commit 0c636f1

File tree

5 files changed

+187
-0
lines changed

5 files changed

+187
-0
lines changed

‎pom.xml‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -88,6 +88,12 @@
8888
<version>0.9.29-1</version>
8989
<optional>true</optional>
9090
</dependency>
91+
<dependency>
92+
<groupId>org.mockito</groupId>
93+
<artifactId>mockito-inline</artifactId>
94+
<version>4.11.0</version>
95+
<scope>test</scope>
96+
</dependency>
9197
</dependencies>
9298
<build>
9399
<plugins>
@@ -110,6 +116,8 @@
110116
</usedDependencies>
111117
<ignoredDependencies>
112118
<ignoredDependency>com.github.jnr:jffi</ignoredDependency>
119+
<ignoredDependency>org.mockito:mockito-core</ignoredDependency>
120+
<ignoredDependency>org.mockito:mockito-inline</ignoredDependency>
113121
</ignoredDependencies>
114122
</configuration>
115123
</plugin>

‎src/main/java/org/lmdbjava/Cursor.java‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -162,6 +162,7 @@ public boolean get(final T key, final T data, final SeekOp op) {
162162
checkRc(rc);
163163
kv.keyOut();
164164
kv.valOut();
165+
ReferenceUtil.reachabilityFence0(key);
165166
return true;
166167
}
167168

@@ -192,6 +193,7 @@ public boolean get(final T key, final GetOp op) {
192193
checkRc(rc);
193194
kv.keyOut();
194195
kv.valOut();
196+
ReferenceUtil.reachabilityFence0(key);
195197
return true;
196198
}
197199

@@ -266,6 +268,8 @@ public boolean put(final T key, final T val, final PutFlags... op) {
266268
return false;
267269
}
268270
checkRc(rc);
271+
ReferenceUtil.reachabilityFence0(key);
272+
ReferenceUtil.reachabilityFence0(val);
269273
return true;
270274
}
271275

@@ -304,6 +308,8 @@ public void putMultiple(final T key, final T val, final int elements,
304308
final int rc = LIB.mdb_cursor_put(ptrCursor, txn.kv().pointerKey(),
305309
dataPtr, mask);
306310
checkRc(rc);
311+
ReferenceUtil.reachabilityFence0(key);
312+
ReferenceUtil.reachabilityFence0(val);
307313
}
308314

309315
/**
@@ -362,6 +368,7 @@ public T reserve(final T key, final int size, final PutFlags... op) {
362368
checkRc(LIB.mdb_cursor_put(ptrCursor, kv.pointerKey(), kv.pointerVal(),
363369
flags));
364370
kv.valOut();
371+
ReferenceUtil.reachabilityFence0(key);
365372
return val();
366373
}
367374

‎src/main/java/org/lmdbjava/Dbi.java‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -171,6 +171,8 @@ public boolean delete(final Txn<T> txn, final T key, final T val) {
171171
return false;
172172
}
173173
checkRc(rc);
174+
ReferenceUtil.reachabilityFence0(key);
175+
ReferenceUtil.reachabilityFence0(val);
174176
return true;
175177
}
176178

@@ -239,6 +241,7 @@ public T get(final Txn<T> txn, final T key) {
239241
return null;
240242
}
241243
checkRc(rc);
244+
ReferenceUtil.reachabilityFence0(key);
242245
return txn.kv().valOut(); // marked as out in LMDB C docs
243246
}
244247

@@ -386,6 +389,8 @@ public boolean put(final Txn<T> txn, final T key, final T val,
386389
return false;
387390
}
388391
checkRc(rc);
392+
ReferenceUtil.reachabilityFence0(key);
393+
ReferenceUtil.reachabilityFence0(val);
389394
return true;
390395
}
391396

@@ -421,6 +426,7 @@ public T reserve(final Txn<T> txn, final T key, final int size,
421426
checkRc(LIB.mdb_put(txn.pointer(), ptr, txn.kv().pointerKey(), txn.kv()
422427
.pointerVal(), flags));
423428
txn.kv().valOut(); // marked as in,out in LMDB C docs
429+
ReferenceUtil.reachabilityFence0(key);
424430
return txn.val();
425431
}
426432

‎src/main/java/org/lmdbjava/ReferenceUtil.java‎

Lines changed: 55 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,55 @@
1+
/*-
2+
* #%L
3+
* LmdbJava
4+
* %%
5+
* Copyright (C) 2016 - 2023 The LmdbJava Open Source Project
6+
* %%
7+
* Licensed under the Apache License, Version 2.0 (the "License");
8+
* you may not use this file except in compliance with the License.
9+
* You may obtain a copy of the License at
10+
*
11+
* http://www.apache.org/licenses/LICENSE-2.0
12+
*
13+
* Unless required by applicable law or agreed to in writing, software
14+
* distributed under the License is distributed on an "AS IS" BASIS,
15+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16+
* See the License for the specific language governing permissions and
17+
* limitations under the License.
18+
* #L%
19+
*/
20+
21+
package org.lmdbjava;
22+
23+
import edu.umd.cs.findbugs.annotations.SuppressFBWarnings;
24+
25+
public final class ReferenceUtil {
26+
/**
27+
* Ensures that the object referenced by the given reference remains
28+
* <a href="package-summary.html#reachability"><em>strongly reachable</em></a>,
29+
* regardless of any prior actions of the program that might otherwise cause
30+
* the object to become unreachable; thus, the referenced object is not
31+
* reclaimable by garbage collection at least until after the invocation of
32+
* this method.
33+
*
34+
* <p> Recent versions of the JDK have a nasty habit of prematurely deciding objects are unreachable.
35+
* see: https://stackoverflow.com/questions/26642153/finalize-called-on-strongly-reachable-object-in-java-8
36+
* The Java 9 method Reference.reachabilityFence offers a solution to this problem.
37+
*
38+
* <p> This method is always implemented as a synchronization on {@code ref}, not as
39+
* {@code Reference.reachabilityFence} for consistency across platforms and to allow building on JDK 6-8.
40+
* <b>It is the caller's responsibility to ensure that this synchronization will not cause deadlock.</b>
41+
*
42+
* @param ref the reference. If {@code null}, this method has no effect.
43+
* @see https://github.com/netty/netty/pull/8410
44+
*/
45+
@SuppressFBWarnings({"ESync_EMPTY_SYNC", "UC_USELESS_VOID_METHOD"})
46+
public static void reachabilityFence0(Object ref) {
47+
if (ref != null) {
48+
synchronized (ref) {
49+
// Empty synchronized is ok: https://stackoverflow.com/a/31933260/1151521
50+
}
51+
}
52+
}
53+
54+
private ReferenceUtil() {}
55+
}

‎src/test/java/org/lmdbjava/GarbageCollectionTest.java‎

Lines changed: 111 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,111 @@
1+
/*-
2+
* #%L
3+
* LmdbJava
4+
* %%
5+
* Copyright (C) 2016 - 2023 The LmdbJava Open Source Project
6+
* %%
7+
* Licensed under the Apache License, Version 2.0 (the "License");
8+
* you may not use this file except in compliance with the License.
9+
* You may obtain a copy of the License at
10+
*
11+
* http://www.apache.org/licenses/LICENSE-2.0
12+
*
13+
* Unless required by applicable law or agreed to in writing, software
14+
* distributed under the License is distributed on an "AS IS" BASIS,
15+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16+
* See the License for the specific language governing permissions and
17+
* limitations under the License.
18+
* #L%
19+
*/
20+
21+
package org.lmdbjava;
22+
23+
import edu.umd.cs.findbugs.annotations.SuppressFBWarnings;
24+
import org.junit.Rule;
25+
import org.junit.Test;
26+
import org.junit.rules.TemporaryFolder;
27+
import org.mockito.MockedStatic;
28+
import org.mockito.Mockito;
29+
30+
import java.io.File;
31+
import java.io.IOException;
32+
import java.nio.ByteBuffer;
33+
34+
import static java.nio.ByteBuffer.allocateDirect;
35+
import static java.nio.charset.StandardCharsets.UTF_8;
36+
import static org.junit.Assert.fail;
37+
import static org.lmdbjava.DbiFlags.MDB_CREATE;
38+
import static org.lmdbjava.Env.create;
39+
40+
@SuppressFBWarnings({"DM_GC", "RV_RETURN_VALUE_IGNORED_NO_SIDE_EFFECT"})
41+
@SuppressWarnings("PMD.DoNotCallGarbageCollectionExplicitly")
42+
public class GarbageCollectionTest {
43+
44+
private static final String DB_NAME = "my DB";
45+
private static final String KEY_PREFIX = "Uncorruptedkey";
46+
private static final String VAL_PREFIX = "Uncorruptedval";
47+
48+
@Rule
49+
public final TemporaryFolder tmp = new TemporaryFolder();
50+
private void putBuffer(Dbi<ByteBuffer> db, Txn<ByteBuffer> txn, int i) {
51+
ByteBuffer key = allocateDirect(24);
52+
ByteBuffer val = allocateDirect(24);
53+
key.put((KEY_PREFIX+i).getBytes(UTF_8)).flip();
54+
val.put((VAL_PREFIX+i).getBytes(UTF_8)).flip();
55+
db.put(txn, key, val);
56+
}
57+
@Test
58+
public void buffersNotGarbageCollectedTest() throws IOException {
59+
final File path = tmp.newFolder();
60+
61+
final Env<ByteBuffer> env = create()
62+
.setMapSize(2_085_760_999)
63+
.setMaxDbs(1)
64+
.open(path);
65+
final Dbi<ByteBuffer> db = env.openDbi(DB_NAME, MDB_CREATE);
66+
67+
// Trigger compilation and whatnot
68+
try (Txn<ByteBuffer> txn = env.txnWrite()) {
69+
for (int i = 0; i < 5000; i++) {
70+
putBuffer(db, txn, i);
71+
}
72+
txn.commit();
73+
}
74+
// Call gc before writing to lmdb and after last reference to buffer by changing the behavior of mask
75+
try (MockedStatic<MaskedFlag> mockedStatic = Mockito.mockStatic(MaskedFlag.class)) {
76+
mockedStatic.when(MaskedFlag::mask).thenAnswer(invocationOnMock -> {
77+
System.gc();
78+
return 0;
79+
});
80+
try (Txn<ByteBuffer> txn = env.txnWrite()) {
81+
for (int i = 0; i < 1000; i++) {
82+
putBuffer(db, txn, i);
83+
}
84+
txn.commit();
85+
}
86+
}
87+
88+
// Find corrupt keys
89+
try (Txn<ByteBuffer> txn = env.txnRead()) {
90+
try (Cursor<ByteBuffer> c = db.openCursor(txn)) {
91+
if (c.first()) {
92+
do {
93+
byte[] rkey = new byte[c.key().remaining()];
94+
c.key().get(rkey);
95+
byte[] rval = new byte[c.val().remaining()];
96+
c.val().get(rval);
97+
String skey = new String(rkey, UTF_8);
98+
String sval = new String(rval, UTF_8);
99+
if (!skey.startsWith("Uncorruptedkey")) {
100+
fail("Found corrupt key " + skey);
101+
}
102+
if (!sval.startsWith("Uncorruptedval")) {
103+
fail("Found corrupt val " + sval);
104+
}
105+
} while (c.next());
106+
}
107+
}
108+
}
109+
env.close();
110+
}
111+
}

0 commit comments

Comments
 (0)