Skip to content

Commit 7dcdcdc

Browse files
PengchengLiugszadovszky
authored andcommitted
PARQUET-1485: Fix Snappy direct memory leak (apache#581)
1 parent 6901a20 commit 7dcdcdc

3 files changed

Lines changed: 122 additions & 6 deletions

File tree

Lines changed: 70 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,70 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one
3+
* or more contributor license agreements. See the NOTICE file
4+
* distributed with this work for additional information
5+
* regarding copyright ownership. The ASF licenses this file
6+
* to you under the Apache License, Version 2.0 (the
7+
* "License"); you may not use this file except in compliance
8+
* with the License. You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing,
13+
* software distributed under the License is distributed on an
14+
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15+
* KIND, either express or implied. See the License for the
16+
* specific language governing permissions and limitations
17+
* under the License.
18+
*/
19+
package org.apache.parquet.hadoop.codec;
20+
21+
import java.lang.reflect.Field;
22+
import java.lang.reflect.InvocationTargetException;
23+
import java.lang.reflect.Method;
24+
import java.nio.ByteBuffer;
25+
26+
import org.slf4j.Logger;
27+
import org.slf4j.LoggerFactory;
28+
29+
/**
30+
* A Helper class which use reflections to clean up DirectBuffer. It's implemented for
31+
* better compatibility with both java8 and java9+, because the Cleaner class is moved to
32+
* another place since java9+.
33+
*/
34+
public class CleanUtil {
35+
private static final Logger logger = LoggerFactory.getLogger(CleanUtil.class);
36+
private static final Field CLEANER_FIELD;
37+
private static final Method CLEAN_METHOD;
38+
39+
static {
40+
ByteBuffer buf = null;
41+
Field cleanerField = null;
42+
Method cleanMethod = null;
43+
try {
44+
buf = ByteBuffer.allocateDirect(1);
45+
cleanerField = buf.getClass().getDeclaredField("cleaner");
46+
cleanerField.setAccessible(true);
47+
Object cleaner = cleanerField.get(buf);
48+
cleanMethod = cleaner.getClass().getDeclaredMethod("clean");
49+
} catch (NoSuchFieldException | NoSuchMethodException | IllegalAccessException e) {
50+
logger.warn("Initialization failed for cleanerField or cleanMethod", e);
51+
} finally {
52+
clean(buf);
53+
}
54+
CLEANER_FIELD = cleanerField;
55+
CLEAN_METHOD = cleanMethod;
56+
}
57+
58+
public static void clean(ByteBuffer buffer) {
59+
if (CLEANER_FIELD == null || CLEAN_METHOD == null) {
60+
return;
61+
}
62+
try {
63+
Object cleaner = CLEANER_FIELD.get(buffer);
64+
CLEAN_METHOD.invoke(cleaner);
65+
} catch (IllegalAccessException | InvocationTargetException | NullPointerException e) {
66+
// Ignore clean failure
67+
logger.warn("Clean failed for buffer " + buffer.getClass().getSimpleName(), e);
68+
}
69+
}
70+
}

‎parquet-hadoop/src/main/java/org/apache/parquet/hadoop/codec/SnappyCompressor.java‎

Lines changed: 25 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -32,16 +32,23 @@
3232
* entire input in setInput and compresses it as one compressed block.
3333
*/
3434
public class SnappyCompressor implements Compressor {
35+
private static final int initialBufferSize = 64 * 1024 * 1024;
36+
3537
// Buffer for compressed output. This buffer grows as necessary.
36-
private ByteBuffer outputBuffer = ByteBuffer.allocateDirect(0);
38+
private ByteBuffer outputBuffer = ByteBuffer.allocateDirect(initialBufferSize);
3739

3840
// Buffer for uncompressed input. This buffer grows as necessary.
39-
private ByteBuffer inputBuffer = ByteBuffer.allocateDirect(0);
41+
private ByteBuffer inputBuffer = ByteBuffer.allocateDirect(initialBufferSize);
4042

4143
private long bytesRead = 0L;
4244
private long bytesWritten = 0L;
4345
private boolean finishCalled = false;
4446

47+
public SnappyCompressor() {
48+
inputBuffer.limit(0);
49+
outputBuffer.limit(0);
50+
}
51+
4552
/**
4653
* Fills specified buffer with compressed data. Returns actual number
4754
* of bytes of compressed data. A return value of 0 indicates that
@@ -66,7 +73,9 @@ public synchronized int compress(byte[] buffer, int off, int len) throws IOExcep
6673
// There is uncompressed input, compress it now
6774
int maxOutputSize = Snappy.maxCompressedLength(inputBuffer.position());
6875
if (maxOutputSize > outputBuffer.capacity()) {
76+
ByteBuffer oldBuffer = outputBuffer;
6977
outputBuffer = ByteBuffer.allocateDirect(maxOutputSize);
78+
CleanUtil.clean(oldBuffer);
7079
}
7180
// Reset the previous outputBuffer
7281
outputBuffer.clear();
@@ -97,7 +106,9 @@ public synchronized void setInput(byte[] buffer, int off, int len) {
97106
ByteBuffer tmp = ByteBuffer.allocateDirect(inputBuffer.position() + len);
98107
inputBuffer.rewind();
99108
tmp.put(inputBuffer);
109+
ByteBuffer oldBuffer = inputBuffer;
100110
inputBuffer = tmp;
111+
CleanUtil.clean(oldBuffer);
101112
} else {
102113
inputBuffer.limit(inputBuffer.position() + len);
103114
}
@@ -146,6 +157,18 @@ public void reinit(Configuration c) {
146157

147158
@Override
148159
public synchronized void reset() {
160+
if (inputBuffer.capacity() > initialBufferSize) {
161+
ByteBuffer oldBuffer = inputBuffer;
162+
inputBuffer = ByteBuffer.allocateDirect(initialBufferSize);
163+
CleanUtil.clean(oldBuffer);
164+
}
165+
166+
if (outputBuffer.capacity() > initialBufferSize) {
167+
ByteBuffer oldBuffer = outputBuffer;
168+
outputBuffer = ByteBuffer.allocateDirect(initialBufferSize);
169+
CleanUtil.clean(oldBuffer);
170+
}
171+
149172
finishCalled = false;
150173
bytesRead = bytesWritten = 0;
151174
inputBuffer.rewind();

‎parquet-hadoop/src/main/java/org/apache/parquet/hadoop/codec/SnappyDecompressor.java‎

Lines changed: 27 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -27,14 +27,21 @@
2727
import org.apache.parquet.Preconditions;
2828

2929
public class SnappyDecompressor implements Decompressor {
30+
private static final int initialBufferSize = 64 * 1024 * 1024;
31+
3032
// Buffer for uncompressed output. This buffer grows as necessary.
31-
private ByteBuffer outputBuffer = ByteBuffer.allocateDirect(0);
33+
private ByteBuffer outputBuffer = ByteBuffer.allocateDirect(initialBufferSize);
3234

3335
// Buffer for compressed input. This buffer grows as necessary.
34-
private ByteBuffer inputBuffer = ByteBuffer.allocateDirect(0);
36+
private ByteBuffer inputBuffer = ByteBuffer.allocateDirect(initialBufferSize);
3537

3638
private boolean finished;
37-
39+
40+
public SnappyDecompressor() {
41+
inputBuffer.limit(0);
42+
outputBuffer.limit(0);
43+
}
44+
3845
/**
3946
* Fills specified buffer with uncompressed data. Returns actual number
4047
* of bytes of uncompressed data. A return value of 0 indicates that
@@ -61,7 +68,9 @@ public synchronized int decompress(byte[] buffer, int off, int len) throws IOExc
6168
// There is compressed input, decompress it now.
6269
int decompressedSize = Snappy.uncompressedLength(inputBuffer);
6370
if (decompressedSize > outputBuffer.capacity()) {
71+
ByteBuffer oldBuffer = outputBuffer;
6472
outputBuffer = ByteBuffer.allocateDirect(decompressedSize);
73+
CleanUtil.clean(oldBuffer);
6574
}
6675

6776
// Reset the previous outputBuffer (i.e. set position to 0)
@@ -102,7 +111,9 @@ public synchronized void setInput(byte[] buffer, int off, int len) {
102111
ByteBuffer newBuffer = ByteBuffer.allocateDirect(inputBuffer.position() + len);
103112
inputBuffer.rewind();
104113
newBuffer.put(inputBuffer);
105-
inputBuffer = newBuffer;
114+
ByteBuffer oldBuffer = inputBuffer;
115+
inputBuffer = newBuffer;
116+
CleanUtil.clean(oldBuffer);
106117
} else {
107118
inputBuffer.limit(inputBuffer.position() + len);
108119
}
@@ -131,6 +142,18 @@ public synchronized boolean needsInput() {
131142

132143
@Override
133144
public synchronized void reset() {
145+
if (inputBuffer.capacity() > initialBufferSize) {
146+
ByteBuffer oldBuffer = inputBuffer;
147+
inputBuffer = ByteBuffer.allocateDirect(initialBufferSize);
148+
CleanUtil.clean(oldBuffer);
149+
}
150+
151+
if (outputBuffer.capacity() > initialBufferSize) {
152+
ByteBuffer oldBuffer = outputBuffer;
153+
outputBuffer = ByteBuffer.allocateDirect(initialBufferSize);
154+
CleanUtil.clean(oldBuffer);
155+
}
156+
134157
finished = false;
135158
inputBuffer.rewind();
136159
outputBuffer.rewind();

0 commit comments

Comments
 (0)