@@ -180,26 +180,30 @@ public void insertionLeavesOtherRowGroupsDeduplicating() throws IOException {
180180 /** The list vector of TestContentDefinedChunker, through a writer that holds back group nulls. */
181181 @ Test
182182 public void listWrittenByRecordsMatchesArrowCpp () throws IOException {
183- MessageType schema = MessageTypeParser .parseMessageType (
184- "message t { optional group v (LIST) { repeated group list { optional int64 element; } } }" );
185- Path file = path ("list" );
186- try (ParquetWriter <Group > writer = builder (file , schema )
187- .withContentDefinedChunking (options (64 * 1024 , 256 * 1024 , 0 ))
188- .build ()) {
189- SimpleGroupFactory f = new SimpleGroupFactory (schema );
190- for (long x : lcg (200_000 )) {
191- Group row = f .newGroup ();
192- if (Long .remainderUnsigned (x , 11 ) != 0 ) {
193- Group list = row .addGroup ("v" );
194- for (int k = 0 ; k < Long .remainderUnsigned (x , 4 ); k ++) {
195- list .addGroup ("list" ).append ("element" , x >>> (16 * k ));
196- }
197- }
198- writer .write (row );
199- }
200- }
183+ assertThat (pageCounts (writeLists ("list" , UnaryOperator .identity ()), 0 ))
184+ .containsExactlyElementsOf (OPTIONAL_LIST_OF_INT64_PAGES );
185+ }
201186
202- assertThat (pageCounts (file , 0 )).containsExactlyElementsOf (OPTIONAL_LIST_OF_INT64_PAGES );
187+ /**
188+ * The page size check schedule of the column write store, which also decides when the held back group
189+ * nulls reach the column writers, moves no page boundary: each column writer applies the page size
190+ * limit itself, inside chunks and keeping every chunk boundary.
191+ */
192+ @ Test
193+ public void pageSizeCheckScheduleMovesNoPageBoundary () throws IOException {
194+ List <Long > chunks = pageEnds (pageCountsByRowGroup (writeLists ("chunks" , UnaryOperator .identity ()), 0 ));
195+ List <Long > scheduled =
196+ pageEnds (pageCountsByRowGroup (writeLists ("scheduled" , b -> b .withPageSize (16 * 1024 )), 0 ));
197+ List <Long > everyRecord = pageEnds (pageCountsByRowGroup (
198+ writeLists (
199+ "every-record" ,
200+ b -> b .withPageSize (16 * 1024 )
201+ .withMinRowCountForPageSizeCheck (1 )
202+ .withMaxRowCountForPageSizeCheck (1 )),
203+ 0 ));
204+
205+ assertThat (scheduled ).containsAll (chunks ).hasSizeGreaterThan (2 * chunks .size ());
206+ assertThat (everyRecord ).containsExactlyElementsOf (scheduled );
203207 }
204208
205209 @ Test
@@ -301,6 +305,29 @@ private Path write(
301305 return file ;
302306 }
303307
308+ /** The list vector of TestContentDefinedChunker: up to three elements a row, about one row in eleven null. */
309+ private Path writeLists (String label , UnaryOperator <ExampleParquetWriter .Builder > configure ) throws IOException {
310+ MessageType schema = MessageTypeParser .parseMessageType (
311+ "message t { optional group v (LIST) { repeated group list { optional int64 element; } } }" );
312+ Path file = path (label );
313+ try (ParquetWriter <Group > writer = configure
314+ .apply (builder (file , schema ).withContentDefinedChunking (options (64 * 1024 , 256 * 1024 , 0 )))
315+ .build ()) {
316+ SimpleGroupFactory f = new SimpleGroupFactory (schema );
317+ for (long x : lcg (200_000 )) {
318+ Group row = f .newGroup ();
319+ if (Long .remainderUnsigned (x , 11 ) != 0 ) {
320+ Group list = row .addGroup ("v" );
321+ for (int k = 0 ; k < Long .remainderUnsigned (x , 4 ); k ++) {
322+ list .addGroup ("list" ).append ("element" , x >>> (16 * k ));
323+ }
324+ }
325+ writer .write (row );
326+ }
327+ }
328+ return file ;
329+ }
330+
304331 private Configuration outputFormatConf () {
305332 Configuration conf = new Configuration ();
306333 GroupWriteSupport .setSchema (SCHEMA , conf );
0 commit comments