|
18 | 18 | */ |
19 | 19 | package org.apache.parquet.column.mem; |
20 | 20 |
|
21 | | -import static org.apache.parquet.schema.PrimitiveType.PrimitiveTypeName.BINARY; |
22 | | -import static org.apache.parquet.schema.PrimitiveType.PrimitiveTypeName.INT32; |
23 | 21 | import static org.junit.Assert.assertEquals; |
24 | | -import static org.junit.Assert.assertTrue; |
25 | 22 |
|
26 | 23 | import org.apache.parquet.column.ColumnDescriptor; |
27 | 24 | import org.apache.parquet.column.ColumnReader; |
28 | | -import org.apache.parquet.column.ColumnWriteStore; |
29 | 25 | import org.apache.parquet.column.ColumnWriter; |
30 | 26 | import org.apache.parquet.column.ParquetProperties; |
31 | 27 | import org.apache.parquet.column.impl.ColumnReadStoreImpl; |
32 | 28 | import org.apache.parquet.column.impl.ColumnWriteStoreV1; |
33 | | -import org.apache.parquet.column.impl.ColumnWriteStoreV2; |
34 | | -import org.apache.parquet.column.page.DataPage; |
35 | | -import org.apache.parquet.column.page.PageReader; |
36 | 29 | import org.apache.parquet.column.page.mem.MemPageStore; |
37 | 30 | import org.apache.parquet.example.DummyRecordConverter; |
38 | 31 | import org.apache.parquet.io.api.Binary; |
39 | 32 | import org.apache.parquet.schema.MessageType; |
40 | 33 | import org.apache.parquet.schema.MessageTypeParser; |
41 | | -import org.apache.parquet.schema.PrimitiveType.PrimitiveTypeName; |
42 | | -import org.apache.parquet.schema.Types; |
43 | 34 | import org.junit.Test; |
44 | 35 | import org.slf4j.Logger; |
45 | 36 | import org.slf4j.LoggerFactory; |
@@ -175,68 +166,6 @@ public void testMemColumnSeveralPagesRepeated() throws Exception { |
175 | 166 | } |
176 | 167 | } |
177 | 168 |
|
178 | | - @Test |
179 | | - public void testPageSize() { |
180 | | - MessageType schema = Types.buildMessage() |
181 | | - .requiredList().requiredElement(BINARY).named("binary_col") |
182 | | - .requiredList().requiredElement(INT32).named("int32_col") |
183 | | - .named("msg"); |
184 | | - System.out.println(schema); |
185 | | - MemPageStore memPageStore = new MemPageStore(123); |
186 | | - |
187 | | - // Using V2 pages so we have rowCount info |
188 | | - ColumnWriteStore writeStore = new ColumnWriteStoreV2(schema, memPageStore, ParquetProperties.builder() |
189 | | - .withPageSize(1024) // Less than 10 records for binary_col |
190 | | - .withMinRowCountForPageSizeCheck(1) // Enforce having precise page sizing |
191 | | - .withPageRowCountLimit(10) |
192 | | - .withDictionaryEncoding(false) // Enforce having large binary_col pages |
193 | | - .build()); |
194 | | - ColumnDescriptor binaryCol = schema.getColumnDescription(new String[] { "binary_col", "list", "element" }); |
195 | | - ColumnWriter binaryColWriter = writeStore.getColumnWriter(binaryCol); |
196 | | - ColumnDescriptor int32Col = schema.getColumnDescription(new String[] { "int32_col", "list", "element" }); |
197 | | - ColumnWriter int32ColWriter = writeStore.getColumnWriter(int32Col); |
198 | | - // Writing 123 records |
199 | | - for (int i = 0; i < 123; ++i) { |
200 | | - // Writing 10 values per record |
201 | | - for (int j = 0; j < 10; ++j) { |
202 | | - binaryColWriter.write(Binary.fromString("aaaaaaaaaaaa"), j == 0 ? 0 : 2, 2); |
203 | | - int32ColWriter.write(42, j == 0 ? 0 : 2, 2); |
204 | | - } |
205 | | - writeStore.endRecord(); |
206 | | - } |
207 | | - writeStore.flush(); |
208 | | - |
209 | | - // Check that all the binary_col pages are <= 1024 bytes |
210 | | - { |
211 | | - PageReader binaryColPageReader = memPageStore.getPageReader(binaryCol); |
212 | | - assertEquals(1230, binaryColPageReader.getTotalValueCount()); |
213 | | - int pageCnt = 0; |
214 | | - int valueCnt = 0; |
215 | | - while (valueCnt < binaryColPageReader.getTotalValueCount()) { |
216 | | - DataPage page = binaryColPageReader.readPage(); |
217 | | - ++pageCnt; |
218 | | - valueCnt += page.getValueCount(); |
219 | | - LOG.info("binary_col page-{}: {} bytes, {} rows", pageCnt, page.getCompressedSize(), page.getIndexRowCount().get()); |
220 | | - assertTrue("Compressed size should be less than 1024", page.getCompressedSize() <= 1024); |
221 | | - } |
222 | | - } |
223 | | - |
224 | | - // Check that all the int32_col pages contain <= 10 rows |
225 | | - { |
226 | | - PageReader int32ColPageReader = memPageStore.getPageReader(int32Col); |
227 | | - assertEquals(1230, int32ColPageReader.getTotalValueCount()); |
228 | | - int pageCnt = 0; |
229 | | - int valueCnt = 0; |
230 | | - while (valueCnt < int32ColPageReader.getTotalValueCount()) { |
231 | | - DataPage page = int32ColPageReader.readPage(); |
232 | | - ++pageCnt; |
233 | | - valueCnt += page.getValueCount(); |
234 | | - LOG.info("int32_col page-{}: {} bytes, {} rows", pageCnt, page.getCompressedSize(), page.getIndexRowCount().get()); |
235 | | - assertTrue("Row count should be less than 10", page.getIndexRowCount().get() <= 10); |
236 | | - } |
237 | | - } |
238 | | - } |
239 | | - |
240 | 169 | private ColumnWriteStoreV1 newColumnWriteStoreImpl(MemPageStore memPageStore) { |
241 | 170 | return new ColumnWriteStoreV1(memPageStore, |
242 | 171 | ParquetProperties.builder() |
|
0 commit comments