MVStore OOM as the amount of data increases and use the file based store #3618
Description
OS windows 10, java 8, and h2-2.1.214.jar.
The test case
The standalone test program.
import org.h2.mvstore.*;
import java.nio.charset.*;
import java.util.*;
import java.text.*;
import java.io.*;
public class CrudTest {
static final Charset charset = StandardCharsets.UTF_8;
static final int o = Integer.getInteger("o", 0); // Offset
static final int n = Integer.getInteger("n", 10_000_000); // Limit
static final int p = Integer.getInteger("p", 10_000); // Print and commit
static final String op = System.getProperty("op", "q"); // operation
static final boolean nul = Boolean.getBoolean("null-value");
static final int newDelta = Integer.getInteger("new-value-delta", 0);
static final int oldDelta = Integer.getInteger("old-value-delta", 0);
static final int cacheSize = Integer.getInteger("cache-size", 16);
static final int threads = Integer.getInteger("threads", 10);
public static void main(String[] args) throws InterruptedException {
long s = System.currentTimeMillis();
String mainName = Thread.currentThread().getName();
String fileName = String.format("tmp%sCrudTest.mv", File.separator);
DateFormat tf = new SimpleDateFormat("HH:mm:ss.SSS");
System.out.printf("%s[%s] open store ...%n", tf.format(new Date()), mainName);
MVStore store = new MVStore.Builder()
.fileName(fileName)
.cacheSize(cacheSize/*MB*/)
.open();
System.out.printf("%s[%s] open store OK%n", tf.format(new Date()), mainName);
Thread[] workers = new Thread[threads];
int limitPerWorker = (n - o) / threads;
try {
Map<String, Double> account = store.openMap("account");
for (int wi = 0; wi < threads; ++wi) {
int offset = limitPerWorker * wi;
int limit;
if (wi == threads - 1) limit = n;
else limit = offset + limitPerWorker;
workers[wi] = new Thread(() -> {
long ts = System.currentTimeMillis();
DateFormat df = new SimpleDateFormat("HH:mm:ss.SSS");
String thrName = Thread.currentThread().getName();
int i = offset;
try {
System.out.printf("%s[%s] op %s, range %d -> %d%n",
df.format(new Date()), thrName, op, offset, limit);
switch(op) {
case "i":
case "u":
for (; i < limit; ++i) {
String name = "acc-" + i;
double balance = i % 10000;
// add/update the key-value pair to the store.
Double old = account.put(name, balance + newDelta);
if (i % p == 0) {
store.commit();
System.out.printf("%s[%s] op %s, commit at %d%n",
df.format(new Date()), thrName, op, i);
}
if (nul) {
if (old != null) throw new AssertionError(name + "'s exists: " + old);
} else if (old == null || old != balance + oldDelta) {
throw new AssertionError(name + "'s balance error: old = " + old);
}
}
break;
case "d":
for (; i < limit; ++i) {
String name = "acc-" + i;
double balance = i % 10000;
Double old = account.remove(name);
if (i % p == 0) {
store.commit();
System.out.printf("%s[%s] op %s, commit at %d%n",
df.format(new Date()), thrName, op, i);
}
if (nul) {
if (old != null) throw new AssertionError(name + "'s exists: " + old);
} else if (old == null || old != balance + oldDelta) {
throw new AssertionError(name + "'s balance error: old = " + old);
}
}
break;
default:
for (; i < limit; ++i) {
String name = "acc-" + i;
double balance = i % 10000;
Double old = account.get(name);
if (i % p == 0) {
System.out.printf("%s[%s] op %s, i %d%n",
df.format(new Date()), thrName, op, i);
}
if (nul) {
if (old != null) throw new AssertionError(name + "'s exists: " + old);
} else if (old == null || old != balance + oldDelta) {
throw new AssertionError(name + "'s balance error: old = " + old);
}
}
break;
}
} finally {
long te = System.currentTimeMillis();
System.out.printf("%s[%s] op %s, null-value %s, i %d, items %d, time %dms%n",
df.format(new Date()), thrName, op, nul, i, n, te - ts);
}
}, "worker-"+wi);
workers[wi].start();
} // for-threads
} finally {
for (int i = 0; i < threads; ++i) {
workers[i].join();
}
store.close();
long e = System.currentTimeMillis();
System.out.printf("%s[%s] op %s, threads %s, null-value %s, items %d, time %dms%n",
tf.format(new Date()), mainName, op, threads, nul, n, e - s);
}
}
}These test data and result.
- JVM heap 64MB, and data items 0.1 billion.
>java -Xmx64m -Dn=100000000 -Dop=i -Dnull-value=true CrudTest18:49:54.395: op i, commit at 86960000
18:49:55.066: op i, commit at 86970000
18:49:55.840: op i, commit at 86980000
18:49:56.732: op i, commit at 86990000
18:49:57.685: op i, commit at 87000000
18:49:58.951: op i, commit at 87010000
18:49:59.795: op i, commit at 87020000
Op i, null-value true, i 87030000, items 100000000, time 391480ms
Exception in thread "main" org.h2.mvstore.MVStoreException: java.util.concurrent.ExecutionException: org.h2.mvstore.MVStoreException: java.lang.OutOfMemoryError: Java heap space [2.1.214/3] [2.1.214/3]
at org.h2.mvstore.DataUtils.newMVStoreException(DataUtils.java:1004)
at org.h2.mvstore.MVStore.storeNow(MVStore.java:1529)
at org.h2.mvstore.MVStore.store(MVStore.java:1496)
at org.h2.mvstore.MVStore.commit(MVStore.java:1469)
at org.h2.mvstore.MVStore.commit(MVStore.java:1458)
at CrudTest.main(CrudTest.java:33)
Caused by: java.util.concurrent.ExecutionException: org.h2.mvstore.MVStoreException: java.lang.OutOfMemoryError: Java heap space [2.1.214/3]
at java.util.concurrent.FutureTask.report(FutureTask.java:122)
at java.util.concurrent.FutureTask.get(FutureTask.java:192)
at org.h2.mvstore.MVStore.submitOrRun(MVStore.java:1541)
at org.h2.mvstore.MVStore.storeNow(MVStore.java:1517)
... 4 more- JVM heap 128MB, and the data items 1 billion.
>java -Xmx128m -Do=100000001 -Dn=900000000 -Dop=i -Dnull-value=true CrudTest20:16:35.150: op i, commit at 326590000
20:16:36.228: op i, commit at 326600000
20:16:37.307: op i, commit at 326610000
Op i, null-value true, i 326620000, items 900000000, time 1223004ms
Exception in thread "main" org.h2.mvstore.MVStoreException: java.util.concurrent.ExecutionException: org.h2.mvstore.MVStoreException: java.lang.OutOfMemoryError: Java heap space [2.1.214/3] [2.1.214/3]
at org.h2.mvstore.DataUtils.newMVStoreException(DataUtils.java:1004)
at org.h2.mvstore.MVStore.storeNow(MVStore.java:1529)
at org.h2.mvstore.MVStore.store(MVStore.java:1496)
at org.h2.mvstore.MVStore.commit(MVStore.java:1469)
at org.h2.mvstore.MVStore.commit(MVStore.java:1458)
at CrudTest.main(CrudTest.java:37)
Caused by: java.util.concurrent.ExecutionException: org.h2.mvstore.MVStoreException: java.lang.OutOfMemoryError: Java heap space [2.1.214/3]
at java.util.concurrent.FutureTask.report(FutureTask.java:122)
at java.util.concurrent.FutureTask.get(FutureTask.java:192)
at org.h2.mvstore.MVStore.submitOrRun(MVStore.java:1541)
at org.h2.mvstore.MVStore.storeNow(MVStore.java:1517)
... 4 more
Caused by: org.h2.mvstore.MVStoreException: java.lang.OutOfMemoryError: Java heap space [2.1.214/3]
at org.h2.mvstore.DataUtils.newMVStoreException(DataUtils.java:1004)
at org.h2.mvstore.MVStore.serializeAndStore(MVStore.java:1605)
at org.h2.mvstore.MVStore.lambda$storeNow$4(MVStore.java:1518)
at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:750)- JVM heap 1024MB, and the data items 2 billion.
>java -Xmx1024m -Do=0 -Dn=2000000000 -Dop=i -Dnull-value=true -Dthreads=10 CrudTest23:39:27.858[worker-0] op i, commit at 105310000
23:39:29.267[worker-0] op i, commit at 105320000
23:39:29.267[worker-0] op i, null-value true, i 105320001, items 2000000000, time 3669314ms
23:39:29.267[main] op i, threads 10, null-value true, items 2000000000, time 3669457ms
Exception in thread "worker-0" org.h2.mvstore.MVStoreException: Map account(2) is closed. org.h2.mvstore.MVStoreException: java.lang.OutOfMemoryError: Capacity: 3145728 [2.1.214/3] [2.1.214/4]
at org.h2.mvstore.DataUtils.newMVStoreException(DataUtils.java:1004)
at org.h2.mvstore.MVMap.beforeWrite(MVMap.java:962)
at org.h2.mvstore.MVMap.operate(MVMap.java:1757)
at org.h2.mvstore.MVMap.put(MVMap.java:156)
at CrudTest.lambda$main$0(CrudTest.java:58)
at java.lang.Thread.run(Thread.java:750)
Caused by: org.h2.mvstore.MVStoreException: java.lang.OutOfMemoryError: Capacity: 3145728 [2.1.214/3]
at org.h2.mvstore.DataUtils.newMVStoreException(DataUtils.java:1004)
at org.h2.mvstore.MVStore.serializeAndStore(MVStore.java:1605)
at org.h2.mvstore.MVStore.lambda$storeNow$4(MVStore.java:1518)
at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
... 1 more
Caused by: java.lang.OutOfMemoryError: Capacity: 3145728
at org.h2.mvstore.WriteBuffer.grow(WriteBuffer.java:322)
at org.h2.mvstore.WriteBuffer.ensureCapacity(WriteBuffer.java:302)
at org.h2.mvstore.WriteBuffer.putStringData(WriteBuffer.java:75)
at org.h2.mvstore.type.ObjectDataType$StringType.write(ObjectDataType.java:1160)
at org.h2.mvstore.type.ObjectDataType$StringType.write(ObjectDataType.java:1126)
at org.h2.mvstore.type.ObjectDataType.write(ObjectDataType.java:142)
at org.h2.mvstore.type.BasicDataType.write(BasicDataType.java:67)
at org.h2.mvstore.Page.write(Page.java:720)
at org.h2.mvstore.Page$Leaf.writeUnsavedRecursive(Page.java:1618)
at org.h2.mvstore.Page$NonLeaf.writeChildrenRecursive(Page.java:1344)
at org.h2.mvstore.Page$NonLeaf.writeUnsavedRecursive(Page.java:1330)
at org.h2.mvstore.Page$NonLeaf.writeChildrenRecursive(Page.java:1344)
at org.h2.mvstore.Page$NonLeaf.writeUnsavedRecursive(Page.java:1330)
at org.h2.mvstore.Page$NonLeaf.writeChildrenRecursive(Page.java:1344)
at org.h2.mvstore.Page$NonLeaf.writeUnsavedRecursive(Page.java:1330)
at org.h2.mvstore.Page$NonLeaf.writeChildrenRecursive(Page.java:1344)
at org.h2.mvstore.Page$NonLeaf.writeUnsavedRecursive(Page.java:1330)
at org.h2.mvstore.Page$NonLeaf.writeChildrenRecursive(Page.java:1344)
at org.h2.mvstore.Page$NonLeaf.writeUnsavedRecursive(Page.java:1330)
at org.h2.mvstore.Page$NonLeaf.writeChildrenRecursive(Page.java:1344)
at org.h2.mvstore.Page$NonLeaf.writeUnsavedRecursive(Page.java:1330)
at org.h2.mvstore.MVStore.serializeToBuffer(MVStore.java:1669)
at org.h2.mvstore.MVStore.serializeAndStore(MVStore.java:1598)
... 6 more