Skip to content

Commit 8bc579e

Browse files
committed
apacheGH-3817: Decompress empty pages with the direct codecs
An empty page, such as the empty dictionary page Arrow C++ writes for a column chunk of nulls alone, compresses to no bytes, and the direct Snappy and LZ4_RAW decompressors reject empty input. Skip decompressing when the page is empty.
1 parent 4b37c19 commit 8bc579e

2 files changed

Lines changed: 11 additions & 1 deletion

File tree

‎parquet-hadoop/src/main/java/org/apache/parquet/hadoop/DirectCodecFactory.java‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -207,6 +207,11 @@ private abstract class BaseDecompressor extends BytesDecompressor {
207207

208208
@Override
209209
public BytesInput decompress(BytesInput bytes, int decompressedSize) throws IOException {
210+
// An empty page, such as an empty dictionary's, may compress to no bytes at all, which some
211+
// codecs reject as input.
212+
if (decompressedSize == 0) {
213+
return BytesInput.empty();
214+
}
210215
try (ByteBufferReleaser releaser = inputAllocator.getReleaser()) {
211216
ByteBuffer input = bytes.toByteBuffer(releaser);
212217
ByteBuffer output = outputAllocator.allocate(decompressedSize);
@@ -225,6 +230,10 @@ public BytesInput decompress(BytesInput bytes, int decompressedSize) throws IOEx
225230
@Override
226231
public void decompress(ByteBuffer input, int compressedSize, ByteBuffer output, int decompressedSize)
227232
throws IOException {
233+
if (decompressedSize == 0) {
234+
input.position(input.position() + compressedSize);
235+
return;
236+
}
228237
int origInputLimit = input.limit();
229238
input.limit(input.position() + compressedSize);
230239
int origOutputLimit = output.limit();

‎parquet-hadoop/src/test/java/org/apache/parquet/hadoop/TestDirectCodecFactory.java‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -211,7 +211,8 @@ public void createDirectFactoryWithHeapAllocatorFails() {
211211

212212
@Test
213213
public void compressionCodecs() {
214-
final int[] sizes = {4 * 1024, 1 * 1024 * 1024};
214+
// Zero as for an empty page, such as an empty dictionary's.
215+
final int[] sizes = {0, 4 * 1024, 1 * 1024 * 1024};
215216
final boolean[] comp = {true, false};
216217
Set<CompressionCodecName> codecsToSkip = new HashSet<>();
217218
codecsToSkip.add(LZO); // not distributed because it is GPL

0 commit comments

Comments
 (0)