Skip to content

Commit 4c698fb

Browse files
authored
GH-15203: [Java] Implement writing compressed files (#15223)
* Closes: #15203 Authored-by: David Li <[email protected]> Signed-off-by: David Li <[email protected]>
1 parent e4019ad commit 4c698fb

8 files changed

Lines changed: 255 additions & 52 deletions

File tree

‎java/compression/src/test/java/org/apache/arrow/compression/TestCompressionCodec.java‎

Lines changed: 167 additions & 42 deletions
Original file line numberDiff line numberDiff line change
@@ -19,11 +19,21 @@
1919

2020
import static org.junit.jupiter.api.Assertions.assertArrayEquals;
2121
import static org.junit.jupiter.api.Assertions.assertEquals;
22+
import static org.junit.jupiter.api.Assertions.assertFalse;
23+
import static org.junit.jupiter.api.Assertions.assertNotEquals;
2224
import static org.junit.jupiter.api.Assertions.assertTrue;
2325

26+
import java.io.ByteArrayOutputStream;
27+
import java.io.IOException;
28+
import java.nio.channels.Channels;
29+
import java.nio.charset.StandardCharsets;
2430
import java.util.ArrayList;
31+
import java.util.Arrays;
2532
import java.util.Collection;
33+
import java.util.HashMap;
2634
import java.util.List;
35+
import java.util.function.BiConsumer;
36+
import java.util.stream.Stream;
2737

2838
import org.apache.arrow.memory.ArrowBuf;
2939
import org.apache.arrow.memory.BufferAllocator;
@@ -32,80 +42,80 @@
3242
import org.apache.arrow.vector.IntVector;
3343
import org.apache.arrow.vector.VarBinaryVector;
3444
import org.apache.arrow.vector.VarCharVector;
45+
import org.apache.arrow.vector.VectorSchemaRoot;
46+
import org.apache.arrow.vector.VectorUnloader;
3547
import org.apache.arrow.vector.compression.CompressionCodec;
3648
import org.apache.arrow.vector.compression.CompressionUtil;
3749
import org.apache.arrow.vector.compression.NoCompressionCodec;
50+
import org.apache.arrow.vector.dictionary.DictionaryProvider;
51+
import org.apache.arrow.vector.ipc.ArrowFileReader;
52+
import org.apache.arrow.vector.ipc.ArrowFileWriter;
53+
import org.apache.arrow.vector.ipc.ArrowStreamReader;
54+
import org.apache.arrow.vector.ipc.ArrowStreamWriter;
3855
import org.apache.arrow.vector.ipc.message.ArrowFieldNode;
39-
import org.junit.After;
40-
import org.junit.Before;
41-
import org.junit.Test;
42-
import org.junit.runner.RunWith;
43-
import org.junit.runners.Parameterized;
56+
import org.apache.arrow.vector.ipc.message.IpcOption;
57+
import org.apache.arrow.vector.types.pojo.ArrowType;
58+
import org.apache.arrow.vector.types.pojo.Field;
59+
import org.apache.arrow.vector.types.pojo.Schema;
60+
import org.apache.arrow.vector.util.ByteArrayReadableSeekableByteChannel;
61+
import org.junit.jupiter.api.AfterEach;
62+
import org.junit.jupiter.api.BeforeEach;
63+
import org.junit.jupiter.params.ParameterizedTest;
64+
import org.junit.jupiter.params.provider.Arguments;
65+
import org.junit.jupiter.params.provider.MethodSource;
4466

4567
/**
4668
* Test cases for {@link CompressionCodec}s.
4769
*/
48-
@RunWith(Parameterized.class)
49-
public class TestCompressionCodec {
50-
51-
private final CompressionCodec codec;
52-
70+
class TestCompressionCodec {
5371
private BufferAllocator allocator;
5472

55-
private final int vectorLength;
56-
57-
@Before
58-
public void init() {
73+
@BeforeEach
74+
void init() {
5975
allocator = new RootAllocator(Integer.MAX_VALUE);
6076
}
6177

62-
@After
63-
public void terminate() {
78+
@AfterEach
79+
void terminate() {
6480
allocator.close();
6581
}
6682

67-
public TestCompressionCodec(CompressionUtil.CodecType type, int vectorLength, CompressionCodec codec) {
68-
this.codec = codec;
69-
this.vectorLength = vectorLength;
70-
}
71-
72-
@Parameterized.Parameters(name = "codec = {0}, length = {1}")
73-
public static Collection<Object[]> getCodecs() {
74-
List<Object[]> params = new ArrayList<>();
83+
static Collection<Arguments> codecs() {
84+
List<Arguments> params = new ArrayList<>();
7585

7686
int[] lengths = new int[] {10, 100, 1000};
7787
for (int len : lengths) {
7888
CompressionCodec dumbCodec = NoCompressionCodec.INSTANCE;
79-
params.add(new Object[]{dumbCodec.getCodecType(), len, dumbCodec});
89+
params.add(Arguments.arguments(len, dumbCodec));
8090

8191
CompressionCodec lz4Codec = new Lz4CompressionCodec();
82-
params.add(new Object[]{lz4Codec.getCodecType(), len, lz4Codec});
92+
params.add(Arguments.arguments(len, lz4Codec));
8393

8494
CompressionCodec zstdCodec = new ZstdCompressionCodec();
85-
params.add(new Object[]{zstdCodec.getCodecType(), len, zstdCodec});
86-
95+
params.add(Arguments.arguments(len, zstdCodec));
8796
}
8897
return params;
8998
}
9099

91-
private List<ArrowBuf> compressBuffers(List<ArrowBuf> inputBuffers) {
100+
private List<ArrowBuf> compressBuffers(CompressionCodec codec, List<ArrowBuf> inputBuffers) {
92101
List<ArrowBuf> outputBuffers = new ArrayList<>(inputBuffers.size());
93102
for (ArrowBuf buf : inputBuffers) {
94103
outputBuffers.add(codec.compress(allocator, buf));
95104
}
96105
return outputBuffers;
97106
}
98107

99-
private List<ArrowBuf> deCompressBuffers(List<ArrowBuf> inputBuffers) {
108+
private List<ArrowBuf> deCompressBuffers(CompressionCodec codec, List<ArrowBuf> inputBuffers) {
100109
List<ArrowBuf> outputBuffers = new ArrayList<>(inputBuffers.size());
101110
for (ArrowBuf buf : inputBuffers) {
102111
outputBuffers.add(codec.decompress(allocator, buf));
103112
}
104113
return outputBuffers;
105114
}
106115

107-
@Test
108-
public void testCompressFixedWidthBuffers() throws Exception {
116+
@ParameterizedTest
117+
@MethodSource("codecs")
118+
void testCompressFixedWidthBuffers(int vectorLength, CompressionCodec codec) throws Exception {
109119
// prepare vector to compress
110120
IntVector origVec = new IntVector("vec", allocator);
111121
origVec.allocateNew(vectorLength);
@@ -121,8 +131,8 @@ public void testCompressFixedWidthBuffers() throws Exception {
121131

122132
// compress & decompress
123133
List<ArrowBuf> origBuffers = origVec.getFieldBuffers();
124-
List<ArrowBuf> compressedBuffers = compressBuffers(origBuffers);
125-
List<ArrowBuf> decompressedBuffers = deCompressBuffers(compressedBuffers);
134+
List<ArrowBuf> compressedBuffers = compressBuffers(codec, origBuffers);
135+
List<ArrowBuf> decompressedBuffers = deCompressBuffers(codec, compressedBuffers);
126136

127137
assertEquals(2, decompressedBuffers.size());
128138

@@ -144,8 +154,9 @@ public void testCompressFixedWidthBuffers() throws Exception {
144154
AutoCloseables.close(decompressedBuffers);
145155
}
146156

147-
@Test
148-
public void testCompressVariableWidthBuffers() throws Exception {
157+
@ParameterizedTest
158+
@MethodSource("codecs")
159+
void testCompressVariableWidthBuffers(int vectorLength, CompressionCodec codec) throws Exception {
149160
// prepare vector to compress
150161
VarCharVector origVec = new VarCharVector("vec", allocator);
151162
origVec.allocateNew();
@@ -161,8 +172,8 @@ public void testCompressVariableWidthBuffers() throws Exception {
161172

162173
// compress & decompress
163174
List<ArrowBuf> origBuffers = origVec.getFieldBuffers();
164-
List<ArrowBuf> compressedBuffers = compressBuffers(origBuffers);
165-
List<ArrowBuf> decompressedBuffers = deCompressBuffers(compressedBuffers);
175+
List<ArrowBuf> compressedBuffers = compressBuffers(codec, origBuffers);
176+
List<ArrowBuf> decompressedBuffers = deCompressBuffers(codec, compressedBuffers);
166177

167178
assertEquals(3, decompressedBuffers.size());
168179

@@ -184,8 +195,9 @@ public void testCompressVariableWidthBuffers() throws Exception {
184195
AutoCloseables.close(decompressedBuffers);
185196
}
186197

187-
@Test
188-
public void testEmptyBuffer() throws Exception {
198+
@ParameterizedTest
199+
@MethodSource("codecs")
200+
void testEmptyBuffer(int vectorLength, CompressionCodec codec) throws Exception {
189201
final VarBinaryVector origVec = new VarBinaryVector("vec", allocator);
190202

191203
origVec.allocateNew(vectorLength);
@@ -194,8 +206,8 @@ public void testEmptyBuffer() throws Exception {
194206
origVec.setValueCount(vectorLength);
195207

196208
final List<ArrowBuf> origBuffers = origVec.getFieldBuffers();
197-
final List<ArrowBuf> compressedBuffers = compressBuffers(origBuffers);
198-
final List<ArrowBuf> decompressedBuffers = deCompressBuffers(compressedBuffers);
209+
final List<ArrowBuf> compressedBuffers = compressBuffers(codec, origBuffers);
210+
final List<ArrowBuf> decompressedBuffers = deCompressBuffers(codec, compressedBuffers);
199211

200212
// orchestrate new vector
201213
VarBinaryVector newVec = new VarBinaryVector("new vec", allocator);
@@ -210,4 +222,117 @@ public void testEmptyBuffer() throws Exception {
210222
newVec.close();
211223
AutoCloseables.close(decompressedBuffers);
212224
}
225+
226+
private static Stream<CompressionUtil.CodecType> codecTypes() {
227+
return Arrays.stream(CompressionUtil.CodecType.values());
228+
}
229+
230+
@ParameterizedTest
231+
@MethodSource("codecTypes")
232+
void testReadWriteStream(CompressionUtil.CodecType codec) throws Exception {
233+
withRoot(codec, (factory, root) -> {
234+
ByteArrayOutputStream compressedStream = new ByteArrayOutputStream();
235+
try (final ArrowStreamWriter writer = new ArrowStreamWriter(
236+
root, new DictionaryProvider.MapDictionaryProvider(),
237+
Channels.newChannel(compressedStream),
238+
IpcOption.DEFAULT, factory, codec)) {
239+
writer.start();
240+
writer.writeBatch();
241+
writer.end();
242+
} catch (IOException e) {
243+
throw new RuntimeException(e);
244+
}
245+
246+
try (ArrowStreamReader reader = new ArrowStreamReader(
247+
new ByteArrayReadableSeekableByteChannel(compressedStream.toByteArray()), allocator, factory)) {
248+
assertTrue(reader.loadNextBatch());
249+
assertTrue(root.equals(reader.getVectorSchemaRoot()));
250+
assertFalse(reader.loadNextBatch());
251+
} catch (IOException e) {
252+
throw new RuntimeException(e);
253+
}
254+
});
255+
}
256+
257+
@ParameterizedTest
258+
@MethodSource("codecTypes")
259+
void testReadWriteFile(CompressionUtil.CodecType codec) throws Exception {
260+
withRoot(codec, (factory, root) -> {
261+
ByteArrayOutputStream compressedStream = new ByteArrayOutputStream();
262+
try (final ArrowFileWriter writer = new ArrowFileWriter(
263+
root, new DictionaryProvider.MapDictionaryProvider(),
264+
Channels.newChannel(compressedStream),
265+
new HashMap<>(), IpcOption.DEFAULT, factory, codec)) {
266+
writer.start();
267+
writer.writeBatch();
268+
writer.end();
269+
} catch (IOException e) {
270+
throw new RuntimeException(e);
271+
}
272+
273+
try (ArrowFileReader reader = new ArrowFileReader(
274+
new ByteArrayReadableSeekableByteChannel(compressedStream.toByteArray()), allocator, factory)) {
275+
assertTrue(reader.loadNextBatch());
276+
assertTrue(root.equals(reader.getVectorSchemaRoot()));
277+
assertFalse(reader.loadNextBatch());
278+
} catch (IOException e) {
279+
throw new RuntimeException(e);
280+
}
281+
});
282+
}
283+
284+
/** Unloading a vector should not free source buffers. */
285+
@ParameterizedTest
286+
@MethodSource("codecTypes")
287+
void testUnloadCompressed(CompressionUtil.CodecType codec) {
288+
withRoot(codec, (factory, root) -> {
289+
root.getFieldVectors().forEach((vector) -> {
290+
Arrays.stream(vector.getBuffers(/*clear*/ false)).forEach((buf) -> {
291+
assertNotEquals(0, buf.getReferenceManager().getRefCount());
292+
});
293+
});
294+
295+
final VectorUnloader unloader = new VectorUnloader(
296+
root, /*includeNullCount*/ true, factory.createCodec(codec), /*alignBuffers*/ true);
297+
unloader.getRecordBatch().close();
298+
299+
root.getFieldVectors().forEach((vector) -> {
300+
Arrays.stream(vector.getBuffers(/*clear*/ false)).forEach((buf) -> {
301+
assertNotEquals(0, buf.getReferenceManager().getRefCount());
302+
});
303+
});
304+
});
305+
}
306+
307+
void withRoot(CompressionUtil.CodecType codec, BiConsumer<CompressionCodec.Factory, VectorSchemaRoot> testBody) {
308+
final Schema schema = new Schema(Arrays.asList(
309+
Field.nullable("ints", new ArrowType.Int(32, true)),
310+
Field.nullable("strings", ArrowType.Utf8.INSTANCE)));
311+
CompressionCodec.Factory factory = codec == CompressionUtil.CodecType.NO_COMPRESSION ?
312+
NoCompressionCodec.Factory.INSTANCE : CommonsCompressionFactory.INSTANCE;
313+
try (final VectorSchemaRoot root = VectorSchemaRoot.create(schema, allocator)) {
314+
final IntVector ints = (IntVector) root.getVector(0);
315+
final VarCharVector strings = (VarCharVector) root.getVector(1);
316+
// Doesn't get compresed
317+
ints.setSafe(0, 0x4a3e);
318+
ints.setSafe(1, 0x8aba);
319+
ints.setSafe(2, 0x4362);
320+
ints.setSafe(3, 0x383f);
321+
// Gets compressed
322+
String compressibleString = " "; // 16 bytes
323+
compressibleString = compressibleString + compressibleString;
324+
compressibleString = compressibleString + compressibleString;
325+
compressibleString = compressibleString + compressibleString;
326+
compressibleString = compressibleString + compressibleString;
327+
compressibleString = compressibleString + compressibleString; // 512 bytes
328+
byte[] compressibleData = compressibleString.getBytes(StandardCharsets.UTF_8);
329+
strings.setSafe(0, compressibleData);
330+
strings.setSafe(1, compressibleData);
331+
strings.setSafe(2, compressibleData);
332+
strings.setSafe(3, compressibleData);
333+
root.setRowCount(4);
334+
335+
testBody.accept(factory, root);
336+
}
337+
}
213338
}

‎java/vector/src/main/java/org/apache/arrow/vector/VectorUnloader.java‎

Lines changed: 9 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -69,7 +69,7 @@ public VectorUnloader(
6969
VectorSchemaRoot root, boolean includeNullCount, CompressionCodec codec, boolean alignBuffers) {
7070
this.root = root;
7171
this.includeNullCount = includeNullCount;
72-
this.codec = codec;
72+
this.codec = codec == null ? NoCompressionCodec.INSTANCE : codec;
7373
this.alignBuffers = alignBuffers;
7474
}
7575

@@ -83,8 +83,10 @@ public ArrowRecordBatch getRecordBatch() {
8383
for (FieldVector vector : root.getFieldVectors()) {
8484
appendNodes(vector, nodes, buffers);
8585
}
86+
// Do NOT retain buffers in ArrowRecordBatch constructor since we have already retained them.
8687
return new ArrowRecordBatch(
87-
root.getRowCount(), nodes, buffers, CompressionUtil.createBodyCompression(codec), alignBuffers);
88+
root.getRowCount(), nodes, buffers, CompressionUtil.createBodyCompression(codec), alignBuffers,
89+
/*retainBuffers*/ false);
8890
}
8991

9092
private void appendNodes(FieldVector vector, List<ArrowFieldNode> nodes, List<ArrowBuf> buffers) {
@@ -97,6 +99,11 @@ private void appendNodes(FieldVector vector, List<ArrowFieldNode> nodes, List<Ar
9799
vector.getField(), vector.getClass().getSimpleName(), fieldBuffers));
98100
}
99101
for (ArrowBuf buf : fieldBuffers) {
102+
// If the codec is NoCompressionCodec, then it will return the input buffer unchanged. In that case,
103+
// we need to retain it for ArrowRecordBatch. Otherwise, it will return a new buffer, and also close
104+
// the input buffer. In that case, we need to retain the input buffer still to avoid modifying
105+
// the source VectorSchemaRoot.
106+
buf.getReferenceManager().retain();
100107
buffers.add(codec.compress(vector.getAllocator(), buf));
101108
}
102109
for (FieldVector child : vector.getChildrenFromFields()) {

‎java/vector/src/main/java/org/apache/arrow/vector/compression/AbstractCompressionCodec.java‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -46,6 +46,7 @@ public ArrowBuf compress(BufferAllocator allocator, ArrowBuf uncompressedBuffer)
4646
if (compressedLength > uncompressedLength) {
4747
// compressed buffer is larger, send the raw buffer
4848
compressedBuffer.close();
49+
// XXX: this makes a copy of uncompressedBuffer
4950
compressedBuffer = CompressionUtil.packageRawBuffer(allocator, uncompressedBuffer);
5051
} else {
5152
writeUncompressedLength(compressedBuffer, uncompressedLength);

‎java/vector/src/main/java/org/apache/arrow/vector/ipc/ArrowFileWriter.java‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,8 @@
2525

2626
import org.apache.arrow.util.VisibleForTesting;
2727
import org.apache.arrow.vector.VectorSchemaRoot;
28+
import org.apache.arrow.vector.compression.CompressionCodec;
29+
import org.apache.arrow.vector.compression.CompressionUtil;
2830
import org.apache.arrow.vector.dictionary.DictionaryProvider;
2931
import org.apache.arrow.vector.ipc.message.ArrowBlock;
3032
import org.apache.arrow.vector.ipc.message.ArrowDictionaryBatch;
@@ -69,6 +71,13 @@ public ArrowFileWriter(VectorSchemaRoot root, DictionaryProvider provider, Writa
6971
this.metaData = metaData;
7072
}
7173

74+
public ArrowFileWriter(VectorSchemaRoot root, DictionaryProvider provider, WritableByteChannel out,
75+
Map<String, String> metaData, IpcOption option, CompressionCodec.Factory compressionFactory,
76+
CompressionUtil.CodecType codecType) {
77+
super(root, provider, out, option, compressionFactory, codecType);
78+
this.metaData = metaData;
79+
}
80+
7281
@Override
7382
protected void startInternal(WriteChannel out) throws IOException {
7483
ArrowMagic.writeMagic(out, true);

0 commit comments

Comments
 (0)