|
20 | 20 | import dev.vortex.api.VortexWriteSummary; |
21 | 21 | import dev.vortex.api.VortexWriter; |
22 | 22 | import dev.vortex.arrow.ArrowAllocation; |
| 23 | +import dev.vortex.io.NativeWritable; |
23 | 24 | import java.io.IOException; |
| 25 | +import java.io.OutputStream; |
24 | 26 | import java.nio.file.Files; |
25 | 27 | import java.nio.file.Path; |
26 | 28 | import java.util.ArrayList; |
27 | 29 | import java.util.HashMap; |
28 | 30 | import java.util.List; |
29 | 31 | import java.util.Map; |
| 32 | +import java.util.Random; |
30 | 33 | import org.apache.arrow.c.ArrowArray; |
31 | 34 | import org.apache.arrow.c.ArrowSchema; |
32 | 35 | import org.apache.arrow.c.Data; |
@@ -61,6 +64,7 @@ public final class JNIWriterTest { |
61 | 64 | private static final byte[] VARIANT_METADATA = new byte[] {0x01, 0x00}; |
62 | 65 | private static final byte[] VARIANT_INT8_42 = new byte[] {0x0c, 0x2a}; |
63 | 66 | private static final byte[] VARIANT_TRUE = new byte[] {0x04}; |
| 67 | + private static final long PROSE_ROWS = 20_000; |
64 | 68 |
|
65 | 69 | @TempDir |
66 | 70 | Path tempDir; |
@@ -492,6 +496,79 @@ public void testLegacyEditionWritesLegacyStatistics() throws IOException { |
492 | 496 | assertEquals(new DataSource.RowCount.Exact(3L), ds.rowCount()); |
493 | 497 | } |
494 | 498 |
|
| 499 | + /** The compact strategy trades read throughput for size, and the file it writes reads like any other. */ |
| 500 | + @ParameterizedTest |
| 501 | + @ValueSource(booleans = {false, true}) |
| 502 | + public void testCompactWritesASmallerFile(boolean toStream) throws IOException { |
| 503 | + BufferAllocator allocator = ArrowAllocation.rootAllocator(); |
| 504 | + VortexWriteSummary standard = writeProse(allocator, "standard.vortex", false, toStream); |
| 505 | + VortexWriteSummary compact = writeProse(allocator, "compact.vortex", true, toStream); |
| 506 | + |
| 507 | + assertEquals(PROSE_ROWS, compact.rowCount()); |
| 508 | + assertTrue( |
| 509 | + compact.fileSize() < standard.fileSize(), |
| 510 | + "compact " + compact.fileSize() + " bytes, standard " + standard.fileSize() + " bytes"); |
| 511 | + assertEquals( |
| 512 | + standard.columnStatistics().get(0).lowerBound(), |
| 513 | + compact.columnStatistics().get(0).lowerBound()); |
| 514 | + assertEquals( |
| 515 | + standard.columnStatistics().get(0).upperBound(), |
| 516 | + compact.columnStatistics().get(0).upperBound()); |
| 517 | + |
| 518 | + DataSource ds = DataSource.open( |
| 519 | + Session.create(), tempDir.resolve("compact.vortex").toUri().toString()); |
| 520 | + assertEquals(new DataSource.RowCount.Exact(PROSE_ROWS), ds.rowCount()); |
| 521 | + } |
| 522 | + |
| 523 | + /** Writes rows of random words, text that a general-purpose compressor shrinks further than the default does. */ |
| 524 | + private VortexWriteSummary writeProse(BufferAllocator allocator, String name, boolean compact, boolean toStream) |
| 525 | + throws IOException { |
| 526 | + Schema schema = new Schema(List.of(Field.notNullable("text", new ArrowType.Utf8()))); |
| 527 | + Path path = tempDir.resolve(name); |
| 528 | + Random random = new Random(7); |
| 529 | + String[] words = new String[4096]; |
| 530 | + for (int i = 0; i < words.length; i++) { |
| 531 | + words[i] = Long.toString(random.nextLong() & Long.MAX_VALUE, 36); |
| 532 | + } |
| 533 | + try (VectorSchemaRoot root = VectorSchemaRoot.create(schema, allocator); |
| 534 | + OutputStream out = Files.newOutputStream(path)) { |
| 535 | + VarCharVector text = (VarCharVector) root.getVector("text"); |
| 536 | + text.allocateNew((int) PROSE_ROWS); |
| 537 | + for (int i = 0; i < PROSE_ROWS; i++) { |
| 538 | + StringBuilder row = new StringBuilder(); |
| 539 | + for (int w = 0; w < 12; w++) { |
| 540 | + row.append(words[random.nextInt(words.length)]).append(' '); |
| 541 | + } |
| 542 | + text.setSafe(i, row.toString().getBytes(UTF_8)); |
| 543 | + } |
| 544 | + root.setRowCount((int) PROSE_ROWS); |
| 545 | + |
| 546 | + Session session = Session.create(); |
| 547 | + NativeWritable sink = new NativeWritable() { |
| 548 | + @Override |
| 549 | + public void write(byte[] buffer, int offset, int length) throws IOException { |
| 550 | + out.write(buffer, offset, length); |
| 551 | + } |
| 552 | + |
| 553 | + @Override |
| 554 | + public void flush() throws IOException { |
| 555 | + out.flush(); |
| 556 | + } |
| 557 | + |
| 558 | + @Override |
| 559 | + public void close() {} |
| 560 | + }; |
| 561 | + VortexWriter.Builder builder = toStream |
| 562 | + ? VortexWriter.builder(session, sink, schema, allocator) |
| 563 | + : VortexWriter.builder( |
| 564 | + session, path.toAbsolutePath().toUri().toString(), schema, allocator); |
| 565 | + try (VortexWriter writer = builder.compact(compact).build()) { |
| 566 | + writer.writeBatch(root, allocator); |
| 567 | + return writer.finish(); |
| 568 | + } |
| 569 | + } |
| 570 | + } |
| 571 | + |
495 | 572 | private static void writePeopleBatch(VortexWriter writer, BufferAllocator allocator) throws IOException { |
496 | 573 | try (VectorSchemaRoot root = VectorSchemaRoot.create(personSchema(), allocator)) { |
497 | 574 | VarCharVector nameVec = (VarCharVector) root.getVector("name"); |
|
0 commit comments