Skip to content

Commit ef414ba

Browse files
committed
apacheGH-3817: Test that the page size check schedule moves no page boundary
Under content defined chunking each column writer applies the page limits at every record start, so the column write store's page size check schedule, which also decides when MessageColumnIO hands its held back nulls to the column writers, must not move a page boundary. Test it through ParquetWriter and document it.
1 parent 81f9c8a commit ef414ba

3 files changed

Lines changed: 52 additions & 21 deletions

File tree

‎parquet-column/src/main/java/org/apache/parquet/column/ParquetProperties.java‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -953,7 +953,8 @@ public Builder withPageRowCountLimit(int rowCount) {
953953
/**
954954
* EXPERIMENTAL: Enable content defined chunking of data pages with the given options, such as
955955
* {@link CdcOptions#DEFAULT}, or disable it with {@code null}. Disabled by default. The page size
956-
* and row count limits still apply. Column write stores must then be made with
956+
* and row count limits still apply, checked by each column writer at every record start instead of
957+
* on the page size check schedule. Column write stores must then be made with
957958
* {@link ParquetProperties#newColumnWriteStore(MessageType, PageWriteStore, BloomFilterWriteStore,
958959
* List)}, passing every row group of a file the chunkers {@link ContentDefinedChunker#forColumns}
959960
* made for it.

‎parquet-hadoop/README.md‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -287,7 +287,10 @@ true if the reader is using a `DirectByteBufferAllocator`
287287
**Property:** `parquet.page.content-defined-chunking.enabled`
288288
**Description:** EXPERIMENTAL: Whether data pages also end at boundaries derived from a rolling hash of the
289289
column's values, so that files sharing a run of values share byte-identical pages a content addressable storage
290-
system can deduplicate, as Arrow C++ and arrow-rs write them. The files need no reader support.
290+
system can deduplicate, as Arrow C++ and arrow-rs write them. The files need no reader support.
291+
The page size and row count limits still end pages inside chunks: each column writer checks them at every record
292+
start rather than on the schedule of `parquet.page.size.row.check.min` and `parquet.page.size.row.check.max`,
293+
which then only paces the row group size check.
291294
**Default value:** `false`
292295

293296
---

‎parquet-hadoop/src/test/java/org/apache/parquet/hadoop/TestContentDefinedChunking.java‎

Lines changed: 46 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -180,26 +180,30 @@ public void insertionLeavesOtherRowGroupsDeduplicating() throws IOException {
180180
/** The list vector of TestContentDefinedChunker, through a writer that holds back group nulls. */
181181
@Test
182182
public void listWrittenByRecordsMatchesArrowCpp() throws IOException {
183-
MessageType schema = MessageTypeParser.parseMessageType(
184-
"message t { optional group v (LIST) { repeated group list { optional int64 element; } } }");
185-
Path file = path("list");
186-
try (ParquetWriter<Group> writer = builder(file, schema)
187-
.withContentDefinedChunking(options(64 * 1024, 256 * 1024, 0))
188-
.build()) {
189-
SimpleGroupFactory f = new SimpleGroupFactory(schema);
190-
for (long x : lcg(200_000)) {
191-
Group row = f.newGroup();
192-
if (Long.remainderUnsigned(x, 11) != 0) {
193-
Group list = row.addGroup("v");
194-
for (int k = 0; k < Long.remainderUnsigned(x, 4); k++) {
195-
list.addGroup("list").append("element", x >>> (16 * k));
196-
}
197-
}
198-
writer.write(row);
199-
}
200-
}
183+
assertThat(pageCounts(writeLists("list", UnaryOperator.identity()), 0))
184+
.containsExactlyElementsOf(OPTIONAL_LIST_OF_INT64_PAGES);
185+
}
201186

202-
assertThat(pageCounts(file, 0)).containsExactlyElementsOf(OPTIONAL_LIST_OF_INT64_PAGES);
187+
/**
188+
* The page size check schedule of the column write store, which also decides when the held back group
189+
* nulls reach the column writers, moves no page boundary: each column writer applies the page size
190+
* limit itself, inside chunks and keeping every chunk boundary.
191+
*/
192+
@Test
193+
public void pageSizeCheckScheduleMovesNoPageBoundary() throws IOException {
194+
List<Long> chunks = pageEnds(pageCountsByRowGroup(writeLists("chunks", UnaryOperator.identity()), 0));
195+
List<Long> scheduled =
196+
pageEnds(pageCountsByRowGroup(writeLists("scheduled", b -> b.withPageSize(16 * 1024)), 0));
197+
List<Long> everyRecord = pageEnds(pageCountsByRowGroup(
198+
writeLists(
199+
"every-record",
200+
b -> b.withPageSize(16 * 1024)
201+
.withMinRowCountForPageSizeCheck(1)
202+
.withMaxRowCountForPageSizeCheck(1)),
203+
0));
204+
205+
assertThat(scheduled).containsAll(chunks).hasSizeGreaterThan(2 * chunks.size());
206+
assertThat(everyRecord).containsExactlyElementsOf(scheduled);
203207
}
204208

205209
@Test
@@ -301,6 +305,29 @@ private Path write(
301305
return file;
302306
}
303307

308+
/** The list vector of TestContentDefinedChunker: up to three elements a row, about one row in eleven null. */
309+
private Path writeLists(String label, UnaryOperator<ExampleParquetWriter.Builder> configure) throws IOException {
310+
MessageType schema = MessageTypeParser.parseMessageType(
311+
"message t { optional group v (LIST) { repeated group list { optional int64 element; } } }");
312+
Path file = path(label);
313+
try (ParquetWriter<Group> writer = configure
314+
.apply(builder(file, schema).withContentDefinedChunking(options(64 * 1024, 256 * 1024, 0)))
315+
.build()) {
316+
SimpleGroupFactory f = new SimpleGroupFactory(schema);
317+
for (long x : lcg(200_000)) {
318+
Group row = f.newGroup();
319+
if (Long.remainderUnsigned(x, 11) != 0) {
320+
Group list = row.addGroup("v");
321+
for (int k = 0; k < Long.remainderUnsigned(x, 4); k++) {
322+
list.addGroup("list").append("element", x >>> (16 * k));
323+
}
324+
}
325+
writer.write(row);
326+
}
327+
}
328+
return file;
329+
}
330+
304331
private Configuration outputFormatConf() {
305332
Configuration conf = new Configuration();
306333
GroupWriteSupport.setSchema(SCHEMA, conf);

0 commit comments

Comments
 (0)