1919
2020import static org .junit .jupiter .api .Assertions .assertArrayEquals ;
2121import static org .junit .jupiter .api .Assertions .assertEquals ;
22+ import static org .junit .jupiter .api .Assertions .assertFalse ;
23+ import static org .junit .jupiter .api .Assertions .assertNotEquals ;
2224import static org .junit .jupiter .api .Assertions .assertTrue ;
2325
26+ import java .io .ByteArrayOutputStream ;
27+ import java .io .IOException ;
28+ import java .nio .channels .Channels ;
29+ import java .nio .charset .StandardCharsets ;
2430import java .util .ArrayList ;
31+ import java .util .Arrays ;
2532import java .util .Collection ;
33+ import java .util .HashMap ;
2634import java .util .List ;
35+ import java .util .function .BiConsumer ;
36+ import java .util .stream .Stream ;
2737
2838import org .apache .arrow .memory .ArrowBuf ;
2939import org .apache .arrow .memory .BufferAllocator ;
3242import org .apache .arrow .vector .IntVector ;
3343import org .apache .arrow .vector .VarBinaryVector ;
3444import org .apache .arrow .vector .VarCharVector ;
45+ import org .apache .arrow .vector .VectorSchemaRoot ;
46+ import org .apache .arrow .vector .VectorUnloader ;
3547import org .apache .arrow .vector .compression .CompressionCodec ;
3648import org .apache .arrow .vector .compression .CompressionUtil ;
3749import org .apache .arrow .vector .compression .NoCompressionCodec ;
50+ import org .apache .arrow .vector .dictionary .DictionaryProvider ;
51+ import org .apache .arrow .vector .ipc .ArrowFileReader ;
52+ import org .apache .arrow .vector .ipc .ArrowFileWriter ;
53+ import org .apache .arrow .vector .ipc .ArrowStreamReader ;
54+ import org .apache .arrow .vector .ipc .ArrowStreamWriter ;
3855import org .apache .arrow .vector .ipc .message .ArrowFieldNode ;
39- import org .junit .After ;
40- import org .junit .Before ;
41- import org .junit .Test ;
42- import org .junit .runner .RunWith ;
43- import org .junit .runners .Parameterized ;
56+ import org .apache .arrow .vector .ipc .message .IpcOption ;
57+ import org .apache .arrow .vector .types .pojo .ArrowType ;
58+ import org .apache .arrow .vector .types .pojo .Field ;
59+ import org .apache .arrow .vector .types .pojo .Schema ;
60+ import org .apache .arrow .vector .util .ByteArrayReadableSeekableByteChannel ;
61+ import org .junit .jupiter .api .AfterEach ;
62+ import org .junit .jupiter .api .BeforeEach ;
63+ import org .junit .jupiter .params .ParameterizedTest ;
64+ import org .junit .jupiter .params .provider .Arguments ;
65+ import org .junit .jupiter .params .provider .MethodSource ;
4466
4567/**
4668 * Test cases for {@link CompressionCodec}s.
4769 */
48- @ RunWith (Parameterized .class )
49- public class TestCompressionCodec {
50-
51- private final CompressionCodec codec ;
52-
70+ class TestCompressionCodec {
5371 private BufferAllocator allocator ;
5472
55- private final int vectorLength ;
56-
57- @ Before
58- public void init () {
73+ @ BeforeEach
74+ void init () {
5975 allocator = new RootAllocator (Integer .MAX_VALUE );
6076 }
6177
62- @ After
63- public void terminate () {
78+ @ AfterEach
79+ void terminate () {
6480 allocator .close ();
6581 }
6682
67- public TestCompressionCodec (CompressionUtil .CodecType type , int vectorLength , CompressionCodec codec ) {
68- this .codec = codec ;
69- this .vectorLength = vectorLength ;
70- }
71-
72- @ Parameterized .Parameters (name = "codec = {0}, length = {1}" )
73- public static Collection <Object []> getCodecs () {
74- List <Object []> params = new ArrayList <>();
83+ static Collection <Arguments > codecs () {
84+ List <Arguments > params = new ArrayList <>();
7585
7686 int [] lengths = new int [] {10 , 100 , 1000 };
7787 for (int len : lengths ) {
7888 CompressionCodec dumbCodec = NoCompressionCodec .INSTANCE ;
79- params .add (new Object []{ dumbCodec . getCodecType (), len , dumbCodec } );
89+ params .add (Arguments . arguments ( len , dumbCodec ) );
8090
8191 CompressionCodec lz4Codec = new Lz4CompressionCodec ();
82- params .add (new Object []{ lz4Codec . getCodecType (), len , lz4Codec } );
92+ params .add (Arguments . arguments ( len , lz4Codec ) );
8393
8494 CompressionCodec zstdCodec = new ZstdCompressionCodec ();
85- params .add (new Object []{zstdCodec .getCodecType (), len , zstdCodec });
86-
95+ params .add (Arguments .arguments (len , zstdCodec ));
8796 }
8897 return params ;
8998 }
9099
91- private List <ArrowBuf > compressBuffers (List <ArrowBuf > inputBuffers ) {
100+ private List <ArrowBuf > compressBuffers (CompressionCodec codec , List <ArrowBuf > inputBuffers ) {
92101 List <ArrowBuf > outputBuffers = new ArrayList <>(inputBuffers .size ());
93102 for (ArrowBuf buf : inputBuffers ) {
94103 outputBuffers .add (codec .compress (allocator , buf ));
95104 }
96105 return outputBuffers ;
97106 }
98107
99- private List <ArrowBuf > deCompressBuffers (List <ArrowBuf > inputBuffers ) {
108+ private List <ArrowBuf > deCompressBuffers (CompressionCodec codec , List <ArrowBuf > inputBuffers ) {
100109 List <ArrowBuf > outputBuffers = new ArrayList <>(inputBuffers .size ());
101110 for (ArrowBuf buf : inputBuffers ) {
102111 outputBuffers .add (codec .decompress (allocator , buf ));
103112 }
104113 return outputBuffers ;
105114 }
106115
107- @ Test
108- public void testCompressFixedWidthBuffers () throws Exception {
116+ @ ParameterizedTest
117+ @ MethodSource ("codecs" )
118+ void testCompressFixedWidthBuffers (int vectorLength , CompressionCodec codec ) throws Exception {
109119 // prepare vector to compress
110120 IntVector origVec = new IntVector ("vec" , allocator );
111121 origVec .allocateNew (vectorLength );
@@ -121,8 +131,8 @@ public void testCompressFixedWidthBuffers() throws Exception {
121131
122132 // compress & decompress
123133 List <ArrowBuf > origBuffers = origVec .getFieldBuffers ();
124- List <ArrowBuf > compressedBuffers = compressBuffers (origBuffers );
125- List <ArrowBuf > decompressedBuffers = deCompressBuffers (compressedBuffers );
134+ List <ArrowBuf > compressedBuffers = compressBuffers (codec , origBuffers );
135+ List <ArrowBuf > decompressedBuffers = deCompressBuffers (codec , compressedBuffers );
126136
127137 assertEquals (2 , decompressedBuffers .size ());
128138
@@ -144,8 +154,9 @@ public void testCompressFixedWidthBuffers() throws Exception {
144154 AutoCloseables .close (decompressedBuffers );
145155 }
146156
147- @ Test
148- public void testCompressVariableWidthBuffers () throws Exception {
157+ @ ParameterizedTest
158+ @ MethodSource ("codecs" )
159+ void testCompressVariableWidthBuffers (int vectorLength , CompressionCodec codec ) throws Exception {
149160 // prepare vector to compress
150161 VarCharVector origVec = new VarCharVector ("vec" , allocator );
151162 origVec .allocateNew ();
@@ -161,8 +172,8 @@ public void testCompressVariableWidthBuffers() throws Exception {
161172
162173 // compress & decompress
163174 List <ArrowBuf > origBuffers = origVec .getFieldBuffers ();
164- List <ArrowBuf > compressedBuffers = compressBuffers (origBuffers );
165- List <ArrowBuf > decompressedBuffers = deCompressBuffers (compressedBuffers );
175+ List <ArrowBuf > compressedBuffers = compressBuffers (codec , origBuffers );
176+ List <ArrowBuf > decompressedBuffers = deCompressBuffers (codec , compressedBuffers );
166177
167178 assertEquals (3 , decompressedBuffers .size ());
168179
@@ -184,8 +195,9 @@ public void testCompressVariableWidthBuffers() throws Exception {
184195 AutoCloseables .close (decompressedBuffers );
185196 }
186197
187- @ Test
188- public void testEmptyBuffer () throws Exception {
198+ @ ParameterizedTest
199+ @ MethodSource ("codecs" )
200+ void testEmptyBuffer (int vectorLength , CompressionCodec codec ) throws Exception {
189201 final VarBinaryVector origVec = new VarBinaryVector ("vec" , allocator );
190202
191203 origVec .allocateNew (vectorLength );
@@ -194,8 +206,8 @@ public void testEmptyBuffer() throws Exception {
194206 origVec .setValueCount (vectorLength );
195207
196208 final List <ArrowBuf > origBuffers = origVec .getFieldBuffers ();
197- final List <ArrowBuf > compressedBuffers = compressBuffers (origBuffers );
198- final List <ArrowBuf > decompressedBuffers = deCompressBuffers (compressedBuffers );
209+ final List <ArrowBuf > compressedBuffers = compressBuffers (codec , origBuffers );
210+ final List <ArrowBuf > decompressedBuffers = deCompressBuffers (codec , compressedBuffers );
199211
200212 // orchestrate new vector
201213 VarBinaryVector newVec = new VarBinaryVector ("new vec" , allocator );
@@ -210,4 +222,117 @@ public void testEmptyBuffer() throws Exception {
210222 newVec .close ();
211223 AutoCloseables .close (decompressedBuffers );
212224 }
225+
226+ private static Stream <CompressionUtil .CodecType > codecTypes () {
227+ return Arrays .stream (CompressionUtil .CodecType .values ());
228+ }
229+
230+ @ ParameterizedTest
231+ @ MethodSource ("codecTypes" )
232+ void testReadWriteStream (CompressionUtil .CodecType codec ) throws Exception {
233+ withRoot (codec , (factory , root ) -> {
234+ ByteArrayOutputStream compressedStream = new ByteArrayOutputStream ();
235+ try (final ArrowStreamWriter writer = new ArrowStreamWriter (
236+ root , new DictionaryProvider .MapDictionaryProvider (),
237+ Channels .newChannel (compressedStream ),
238+ IpcOption .DEFAULT , factory , codec )) {
239+ writer .start ();
240+ writer .writeBatch ();
241+ writer .end ();
242+ } catch (IOException e ) {
243+ throw new RuntimeException (e );
244+ }
245+
246+ try (ArrowStreamReader reader = new ArrowStreamReader (
247+ new ByteArrayReadableSeekableByteChannel (compressedStream .toByteArray ()), allocator , factory )) {
248+ assertTrue (reader .loadNextBatch ());
249+ assertTrue (root .equals (reader .getVectorSchemaRoot ()));
250+ assertFalse (reader .loadNextBatch ());
251+ } catch (IOException e ) {
252+ throw new RuntimeException (e );
253+ }
254+ });
255+ }
256+
257+ @ ParameterizedTest
258+ @ MethodSource ("codecTypes" )
259+ void testReadWriteFile (CompressionUtil .CodecType codec ) throws Exception {
260+ withRoot (codec , (factory , root ) -> {
261+ ByteArrayOutputStream compressedStream = new ByteArrayOutputStream ();
262+ try (final ArrowFileWriter writer = new ArrowFileWriter (
263+ root , new DictionaryProvider .MapDictionaryProvider (),
264+ Channels .newChannel (compressedStream ),
265+ new HashMap <>(), IpcOption .DEFAULT , factory , codec )) {
266+ writer .start ();
267+ writer .writeBatch ();
268+ writer .end ();
269+ } catch (IOException e ) {
270+ throw new RuntimeException (e );
271+ }
272+
273+ try (ArrowFileReader reader = new ArrowFileReader (
274+ new ByteArrayReadableSeekableByteChannel (compressedStream .toByteArray ()), allocator , factory )) {
275+ assertTrue (reader .loadNextBatch ());
276+ assertTrue (root .equals (reader .getVectorSchemaRoot ()));
277+ assertFalse (reader .loadNextBatch ());
278+ } catch (IOException e ) {
279+ throw new RuntimeException (e );
280+ }
281+ });
282+ }
283+
284+ /** Unloading a vector should not free source buffers. */
285+ @ ParameterizedTest
286+ @ MethodSource ("codecTypes" )
287+ void testUnloadCompressed (CompressionUtil .CodecType codec ) {
288+ withRoot (codec , (factory , root ) -> {
289+ root .getFieldVectors ().forEach ((vector ) -> {
290+ Arrays .stream (vector .getBuffers (/*clear*/ false )).forEach ((buf ) -> {
291+ assertNotEquals (0 , buf .getReferenceManager ().getRefCount ());
292+ });
293+ });
294+
295+ final VectorUnloader unloader = new VectorUnloader (
296+ root , /*includeNullCount*/ true , factory .createCodec (codec ), /*alignBuffers*/ true );
297+ unloader .getRecordBatch ().close ();
298+
299+ root .getFieldVectors ().forEach ((vector ) -> {
300+ Arrays .stream (vector .getBuffers (/*clear*/ false )).forEach ((buf ) -> {
301+ assertNotEquals (0 , buf .getReferenceManager ().getRefCount ());
302+ });
303+ });
304+ });
305+ }
306+
307+ void withRoot (CompressionUtil .CodecType codec , BiConsumer <CompressionCodec .Factory , VectorSchemaRoot > testBody ) {
308+ final Schema schema = new Schema (Arrays .asList (
309+ Field .nullable ("ints" , new ArrowType .Int (32 , true )),
310+ Field .nullable ("strings" , ArrowType .Utf8 .INSTANCE )));
311+ CompressionCodec .Factory factory = codec == CompressionUtil .CodecType .NO_COMPRESSION ?
312+ NoCompressionCodec .Factory .INSTANCE : CommonsCompressionFactory .INSTANCE ;
313+ try (final VectorSchemaRoot root = VectorSchemaRoot .create (schema , allocator )) {
314+ final IntVector ints = (IntVector ) root .getVector (0 );
315+ final VarCharVector strings = (VarCharVector ) root .getVector (1 );
316+ // Doesn't get compresed
317+ ints .setSafe (0 , 0x4a3e );
318+ ints .setSafe (1 , 0x8aba );
319+ ints .setSafe (2 , 0x4362 );
320+ ints .setSafe (3 , 0x383f );
321+ // Gets compressed
322+ String compressibleString = " " ; // 16 bytes
323+ compressibleString = compressibleString + compressibleString ;
324+ compressibleString = compressibleString + compressibleString ;
325+ compressibleString = compressibleString + compressibleString ;
326+ compressibleString = compressibleString + compressibleString ;
327+ compressibleString = compressibleString + compressibleString ; // 512 bytes
328+ byte [] compressibleData = compressibleString .getBytes (StandardCharsets .UTF_8 );
329+ strings .setSafe (0 , compressibleData );
330+ strings .setSafe (1 , compressibleData );
331+ strings .setSafe (2 , compressibleData );
332+ strings .setSafe (3 , compressibleData );
333+ root .setRowCount (4 );
334+
335+ testBody .accept (factory , root );
336+ }
337+ }
213338}
0 commit comments