|
18 | 18 | */ |
19 | 19 | package org.apache.parquet.column.mem; |
20 | 20 |
|
| 21 | +import static org.apache.parquet.schema.PrimitiveType.PrimitiveTypeName.BINARY; |
| 22 | +import static org.apache.parquet.schema.PrimitiveType.PrimitiveTypeName.INT32; |
21 | 23 | import static org.junit.Assert.assertEquals; |
| 24 | +import static org.junit.Assert.assertTrue; |
22 | 25 |
|
23 | 26 | import org.apache.parquet.column.ColumnDescriptor; |
24 | 27 | import org.apache.parquet.column.ColumnReader; |
| 28 | +import org.apache.parquet.column.ColumnWriteStore; |
25 | 29 | import org.apache.parquet.column.ColumnWriter; |
26 | 30 | import org.apache.parquet.column.ParquetProperties; |
27 | 31 | import org.apache.parquet.column.impl.ColumnReadStoreImpl; |
28 | 32 | import org.apache.parquet.column.impl.ColumnWriteStoreV1; |
| 33 | +import org.apache.parquet.column.impl.ColumnWriteStoreV2; |
| 34 | +import org.apache.parquet.column.page.DataPage; |
| 35 | +import org.apache.parquet.column.page.PageReader; |
29 | 36 | import org.apache.parquet.column.page.mem.MemPageStore; |
30 | 37 | import org.apache.parquet.example.DummyRecordConverter; |
31 | 38 | import org.apache.parquet.io.api.Binary; |
32 | 39 | import org.apache.parquet.schema.MessageType; |
33 | 40 | import org.apache.parquet.schema.MessageTypeParser; |
| 41 | +import org.apache.parquet.schema.PrimitiveType.PrimitiveTypeName; |
| 42 | +import org.apache.parquet.schema.Types; |
34 | 43 | import org.junit.Test; |
35 | 44 | import org.slf4j.Logger; |
36 | 45 | import org.slf4j.LoggerFactory; |
@@ -166,6 +175,68 @@ public void testMemColumnSeveralPagesRepeated() throws Exception { |
166 | 175 | } |
167 | 176 | } |
168 | 177 |
|
| 178 | + @Test |
| 179 | + public void testPageSize() { |
| 180 | + MessageType schema = Types.buildMessage() |
| 181 | + .requiredList().requiredElement(BINARY).named("binary_col") |
| 182 | + .requiredList().requiredElement(INT32).named("int32_col") |
| 183 | + .named("msg"); |
| 184 | + System.out.println(schema); |
| 185 | + MemPageStore memPageStore = new MemPageStore(123); |
| 186 | + |
| 187 | + // Using V2 pages so we have rowCount info |
| 188 | + ColumnWriteStore writeStore = new ColumnWriteStoreV2(schema, memPageStore, ParquetProperties.builder() |
| 189 | + .withPageSize(1024) // Less than 10 records for binary_col |
| 190 | + .withMinRowCountForPageSizeCheck(1) // Enforce having precise page sizing |
| 191 | + .withPageRowCountLimit(10) |
| 192 | + .withDictionaryEncoding(false) // Enforce having large binary_col pages |
| 193 | + .build()); |
| 194 | + ColumnDescriptor binaryCol = schema.getColumnDescription(new String[] { "binary_col", "list", "element" }); |
| 195 | + ColumnWriter binaryColWriter = writeStore.getColumnWriter(binaryCol); |
| 196 | + ColumnDescriptor int32Col = schema.getColumnDescription(new String[] { "int32_col", "list", "element" }); |
| 197 | + ColumnWriter int32ColWriter = writeStore.getColumnWriter(int32Col); |
| 198 | + // Writing 123 records |
| 199 | + for (int i = 0; i < 123; ++i) { |
| 200 | + // Writing 10 values per record |
| 201 | + for (int j = 0; j < 10; ++j) { |
| 202 | + binaryColWriter.write(Binary.fromString("aaaaaaaaaaaa"), j == 0 ? 0 : 2, 2); |
| 203 | + int32ColWriter.write(42, j == 0 ? 0 : 2, 2); |
| 204 | + } |
| 205 | + writeStore.endRecord(); |
| 206 | + } |
| 207 | + writeStore.flush(); |
| 208 | + |
| 209 | + // Check that all the binary_col pages are <= 1024 bytes |
| 210 | + { |
| 211 | + PageReader binaryColPageReader = memPageStore.getPageReader(binaryCol); |
| 212 | + assertEquals(1230, binaryColPageReader.getTotalValueCount()); |
| 213 | + int pageCnt = 0; |
| 214 | + int valueCnt = 0; |
| 215 | + while (valueCnt < binaryColPageReader.getTotalValueCount()) { |
| 216 | + DataPage page = binaryColPageReader.readPage(); |
| 217 | + ++pageCnt; |
| 218 | + valueCnt += page.getValueCount(); |
| 219 | + LOG.info("binary_col page-{}: {} bytes, {} rows", pageCnt, page.getCompressedSize(), page.getIndexRowCount().get()); |
| 220 | + assertTrue("Compressed size should be less than 1024", page.getCompressedSize() <= 1024); |
| 221 | + } |
| 222 | + } |
| 223 | + |
| 224 | + // Check that all the int32_col pages contain <= 10 rows |
| 225 | + { |
| 226 | + PageReader int32ColPageReader = memPageStore.getPageReader(int32Col); |
| 227 | + assertEquals(1230, int32ColPageReader.getTotalValueCount()); |
| 228 | + int pageCnt = 0; |
| 229 | + int valueCnt = 0; |
| 230 | + while (valueCnt < int32ColPageReader.getTotalValueCount()) { |
| 231 | + DataPage page = int32ColPageReader.readPage(); |
| 232 | + ++pageCnt; |
| 233 | + valueCnt += page.getValueCount(); |
| 234 | + LOG.info("int32_col page-{}: {} bytes, {} rows", pageCnt, page.getCompressedSize(), page.getIndexRowCount().get()); |
| 235 | + assertTrue("Row count should be less than 10", page.getIndexRowCount().get() <= 10); |
| 236 | + } |
| 237 | + } |
| 238 | + } |
| 239 | + |
169 | 240 | private ColumnWriteStoreV1 newColumnWriteStoreImpl(MemPageStore memPageStore) { |
170 | 241 | return new ColumnWriteStoreV1(memPageStore, |
171 | 242 | ParquetProperties.builder() |
|
0 commit comments