Skip to content

Commit 6e24b12

Browse files
committed
apacheGH-3817: Chunk only, and leave the page limits as they are
The chunker answers only whether a triplet starts a new chunk, as Arrow C++ and arrow-rs decide it: roll the triplet, then ask NeedNewChunk at record starts. The column writer ends its page there unless the page is empty. The column write store applies the page limits on its schedule as without content defined chunking, so the chunker no longer reads the writer's page state.
1 parent 1193df6 commit 6e24b12

8 files changed

Lines changed: 153 additions & 204 deletions

File tree

‎parquet-column/src/main/java/org/apache/parquet/column/ParquetProperties.java‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -953,8 +953,7 @@ public Builder withPageRowCountLimit(int rowCount) {
953953
/**
954954
* EXPERIMENTAL: Enable content defined chunking of data pages with the given options, such as
955955
* {@link CdcOptions#DEFAULT}, or disable it with {@code null}. Disabled by default. The page size
956-
* and row count limits still apply, checked by each column writer at every record start instead of
957-
* on the page size check schedule. Column write stores must then be made with
956+
* and row count limits still apply. Column write stores must then be made with
958957
* {@link ParquetProperties#newColumnWriteStore(MessageType, PageWriteStore, BloomFilterWriteStore,
959958
* List)}, passing every row group of a file the chunkers {@link ContentDefinedChunker#forColumns}
960959
* made for it.

‎parquet-column/src/main/java/org/apache/parquet/column/impl/ColumnWriteStoreBase.java‎

Lines changed: 0 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -263,11 +263,6 @@ public void close() {
263263
public void endRecord() {
264264
++rowCount;
265265
if (rowCount >= rowCountForNextSizeCheck) {
266-
if (props.isContentDefinedChunkingEnabled()) {
267-
// The column writers apply the page limits; this schedule only flushes cached nulls.
268-
rowCountForNextSizeCheck = rowCount + props.getMaxRowCountForPageSizeCheck();
269-
return;
270-
}
271266
sizeCheck();
272267
}
273268
}

‎parquet-column/src/main/java/org/apache/parquet/column/impl/ColumnWriterBase.java‎

Lines changed: 21 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -117,8 +117,8 @@ private void repetitionLevel(int repetitionLevel) {
117117
public void writeNull(int repetitionLevel, int definitionLevel) {
118118
if (DEBUG) log(null, repetitionLevel, definitionLevel);
119119
try {
120-
if (chunker.needNewPage(repetitionLevel, definitionLevel, this)) {
121-
writePage();
120+
if (chunker.startsNewChunk(repetitionLevel, definitionLevel)) {
121+
writePageIfNotEmpty();
122122
}
123123
repetitionLevel(repetitionLevel);
124124
definitionLevel(definitionLevel);
@@ -158,8 +158,8 @@ public long getBufferedSizeInMemory() {
158158
public void write(double value, int repetitionLevel, int definitionLevel) {
159159
if (DEBUG) log(value, repetitionLevel, definitionLevel);
160160
try {
161-
if (chunker.needNewPage(value, repetitionLevel, definitionLevel, this)) {
162-
writePage();
161+
if (chunker.startsNewChunk(value, repetitionLevel, definitionLevel)) {
162+
writePageIfNotEmpty();
163163
}
164164
repetitionLevel(repetitionLevel);
165165
definitionLevel(definitionLevel);
@@ -183,8 +183,8 @@ public void write(double value, int repetitionLevel, int definitionLevel) {
183183
public void write(float value, int repetitionLevel, int definitionLevel) {
184184
if (DEBUG) log(value, repetitionLevel, definitionLevel);
185185
try {
186-
if (chunker.needNewPage(value, repetitionLevel, definitionLevel, this)) {
187-
writePage();
186+
if (chunker.startsNewChunk(value, repetitionLevel, definitionLevel)) {
187+
writePageIfNotEmpty();
188188
}
189189
repetitionLevel(repetitionLevel);
190190
definitionLevel(definitionLevel);
@@ -208,8 +208,8 @@ public void write(float value, int repetitionLevel, int definitionLevel) {
208208
public void write(Binary value, int repetitionLevel, int definitionLevel) {
209209
if (DEBUG) log(value, repetitionLevel, definitionLevel);
210210
try {
211-
if (chunker.needNewPage(value, repetitionLevel, definitionLevel, this)) {
212-
writePage();
211+
if (chunker.startsNewChunk(value, repetitionLevel, definitionLevel)) {
212+
writePageIfNotEmpty();
213213
}
214214
repetitionLevel(repetitionLevel);
215215
definitionLevel(definitionLevel);
@@ -233,8 +233,8 @@ public void write(Binary value, int repetitionLevel, int definitionLevel) {
233233
public void write(boolean value, int repetitionLevel, int definitionLevel) {
234234
if (DEBUG) log(value, repetitionLevel, definitionLevel);
235235
try {
236-
if (chunker.needNewPage(value, repetitionLevel, definitionLevel, this)) {
237-
writePage();
236+
if (chunker.startsNewChunk(value, repetitionLevel, definitionLevel)) {
237+
writePageIfNotEmpty();
238238
}
239239
repetitionLevel(repetitionLevel);
240240
definitionLevel(definitionLevel);
@@ -258,8 +258,8 @@ public void write(boolean value, int repetitionLevel, int definitionLevel) {
258258
public void write(int value, int repetitionLevel, int definitionLevel) {
259259
if (DEBUG) log(value, repetitionLevel, definitionLevel);
260260
try {
261-
if (chunker.needNewPage(value, repetitionLevel, definitionLevel, this)) {
262-
writePage();
261+
if (chunker.startsNewChunk(value, repetitionLevel, definitionLevel)) {
262+
writePageIfNotEmpty();
263263
}
264264
repetitionLevel(repetitionLevel);
265265
definitionLevel(definitionLevel);
@@ -283,8 +283,8 @@ public void write(int value, int repetitionLevel, int definitionLevel) {
283283
public void write(long value, int repetitionLevel, int definitionLevel) {
284284
if (DEBUG) log(value, repetitionLevel, definitionLevel);
285285
try {
286-
if (chunker.needNewPage(value, repetitionLevel, definitionLevel, this)) {
287-
writePage();
286+
if (chunker.startsNewChunk(value, repetitionLevel, definitionLevel)) {
287+
writePageIfNotEmpty();
288288
}
289289
repetitionLevel(repetitionLevel);
290290
definitionLevel(definitionLevel);
@@ -389,8 +389,13 @@ int getValueCount() {
389389
return this.valueCount;
390390
}
391391

392-
int getPageRowCount() {
393-
return this.pageRowCount;
392+
/**
393+
* Writes the current data to a new page in the page store, unless there is none.
394+
*/
395+
private void writePageIfNotEmpty() {
396+
if (valueCount > 0) {
397+
writePage();
398+
}
394399
}
395400

396401
/**

‎parquet-column/src/main/java/org/apache/parquet/column/impl/ContentDefinedChunker.java‎

Lines changed: 44 additions & 91 deletions
Original file line numberDiff line numberDiff line change
@@ -29,10 +29,11 @@
2929
import org.apache.parquet.schema.MessageType;
3030

3131
/**
32-
* Decides the content defined data page boundaries of one leaf column for a whole file. The column
33-
* writer asks {@code needNewPage} before writing each triplet, which this chunker answers by rolling
34-
* the triplet into its hash; a no-op chunker, {@link #noop()}, stands in while content defined
35-
* chunking is disabled.
32+
* Decides the content defined chunk boundaries of one leaf column for a whole file. The column writer
33+
* asks {@code startsNewChunk} with each triplet before writing it, which rolls the triplet into the
34+
* hash and answers whether it starts a new chunk, and ends its page there; the page limits end pages as
35+
* they do without content defined chunking. A no-op chunker, {@link #noop()}, stands in while content defined chunking is
36+
* disabled.
3637
*
3738
* <p>Ported from Arrow C++ {@code parquet/chunker_internal} and arrow-rs {@code column/chunker/cdc.rs},
3839
* whose boundaries it must reproduce. Not thread safe.
@@ -46,9 +47,6 @@ public class ContentDefinedChunker {
4647
private final long rollingHashMask;
4748
private final int maxDef;
4849
private final int maxRep;
49-
private final long pageSizeThreshold;
50-
private final int pageValueCountThreshold;
51-
private final int pageRowCountLimit;
5250

5351
private long rollingHash;
5452
private boolean hasMatched;
@@ -69,7 +67,7 @@ public static List<ContentDefinedChunker> forColumns(MessageType schema, Parquet
6967
.collect(Collectors.toList());
7068
}
7169

72-
/** The chunker of a column without content defined chunking, which never ends a page. */
70+
/** The chunker of a column without content defined chunking, which never ends a chunk. */
7371
static ContentDefinedChunker noop() {
7472
return NOOP;
7573
}
@@ -81,9 +79,6 @@ static ContentDefinedChunker noop() {
8179
this.rollingHashMask = calculateMask(minChunkSize, maxChunkSize, options.getNormLevel());
8280
this.maxDef = path.getMaxDefinitionLevel();
8381
this.maxRep = path.getMaxRepetitionLevel();
84-
this.pageSizeThreshold = props.getPageSizeThreshold();
85-
this.pageValueCountThreshold = props.getPageValueCountThreshold();
86-
this.pageRowCountLimit = props.getPageRowCountLimit();
8782
}
8883

8984
/** Only for {@link NoopChunker}, which uses none of the state. */
@@ -93,9 +88,6 @@ private ContentDefinedChunker() {
9388
this.rollingHashMask = 0;
9489
this.maxDef = 0;
9590
this.maxRep = 0;
96-
this.pageSizeThreshold = 0;
97-
this.pageValueCountThreshold = 0;
98-
this.pageRowCountLimit = 0;
9991
}
10092

10193
/**
@@ -141,44 +133,60 @@ public static long calculateMask(long minChunkSize, long maxChunkSize, int normL
141133
return -1L << (Long.SIZE - effectiveBits);
142134
}
143135

144-
void roll(int value, int repetitionLevel, int definitionLevel) {
136+
/**
137+
* Rolls a triplet into the hash and answers whether it starts a new chunk, which only the first
138+
* triplet of a record can; see {@link #needNewChunk()}.
139+
*
140+
* @param value the value of the triplet
141+
* @param repetitionLevel the repetition level of the triplet
142+
* @param definitionLevel the definition level of the triplet
143+
* @return whether a chunk ends before the triplet
144+
*/
145+
boolean startsNewChunk(int value, int repetitionLevel, int definitionLevel) {
145146
rollLevels(repetitionLevel, definitionLevel);
146147
roll(value, 4);
148+
return repetitionLevel == 0 && needNewChunk();
147149
}
148150

149-
void roll(long value, int repetitionLevel, int definitionLevel) {
151+
boolean startsNewChunk(long value, int repetitionLevel, int definitionLevel) {
150152
rollLevels(repetitionLevel, definitionLevel);
151153
roll(value, 8);
154+
return repetitionLevel == 0 && needNewChunk();
152155
}
153156

154-
void roll(float value, int repetitionLevel, int definitionLevel) {
157+
boolean startsNewChunk(float value, int repetitionLevel, int definitionLevel) {
155158
rollLevels(repetitionLevel, definitionLevel);
156159
// Raw bits: floatToIntBits canonicalizes NaN payloads and would diverge from the references.
157160
roll(Float.floatToRawIntBits(value), 4);
161+
return repetitionLevel == 0 && needNewChunk();
158162
}
159163

160-
void roll(double value, int repetitionLevel, int definitionLevel) {
164+
boolean startsNewChunk(double value, int repetitionLevel, int definitionLevel) {
161165
rollLevels(repetitionLevel, definitionLevel);
162166
roll(Double.doubleToRawLongBits(value), 8);
167+
return repetitionLevel == 0 && needNewChunk();
163168
}
164169

165-
void roll(boolean value, int repetitionLevel, int definitionLevel) {
170+
boolean startsNewChunk(boolean value, int repetitionLevel, int definitionLevel) {
166171
rollLevels(repetitionLevel, definitionLevel);
167172
// One byte per boolean, not one bit, as the references hash it.
168173
roll(value ? 1 : 0, 1);
174+
return repetitionLevel == 0 && needNewChunk();
169175
}
170176

171-
void roll(Binary value, int repetitionLevel, int definitionLevel) {
177+
boolean startsNewChunk(Binary value, int repetitionLevel, int definitionLevel) {
172178
rollLevels(repetitionLevel, definitionLevel);
173179
roll(value);
180+
return repetitionLevel == 0 && needNewChunk();
174181
}
175182

176183
/**
177184
* Rolls the triplet of a null, which has no value to hash even when {@code definitionLevel ==
178-
* maxDef}, as an omitted top-level required field does.
185+
* maxDef}, as an omitted top-level required field does, and answers whether a chunk ends before it.
179186
*/
180-
void rollNull(int repetitionLevel, int definitionLevel) {
187+
boolean startsNewChunk(int repetitionLevel, int definitionLevel) {
181188
rollLevels(repetitionLevel, definitionLevel);
189+
return repetitionLevel == 0 && needNewChunk();
182190
}
183191

184192
/**
@@ -194,63 +202,6 @@ private void rollLevels(int repetitionLevel, int definitionLevel) {
194202
}
195203
}
196204

197-
/**
198-
* Whether the column writer should end its page before this triplet, which it asks before writing
199-
* each one; {@code needNewPage(repetitionLevel, definitionLevel, writer)} asks for a null.
200-
*/
201-
boolean needNewPage(int value, int repetitionLevel, int definitionLevel, ColumnWriterBase writer) {
202-
roll(value, repetitionLevel, definitionLevel);
203-
return pageEndsBefore(repetitionLevel, writer);
204-
}
205-
206-
boolean needNewPage(long value, int repetitionLevel, int definitionLevel, ColumnWriterBase writer) {
207-
roll(value, repetitionLevel, definitionLevel);
208-
return pageEndsBefore(repetitionLevel, writer);
209-
}
210-
211-
boolean needNewPage(float value, int repetitionLevel, int definitionLevel, ColumnWriterBase writer) {
212-
roll(value, repetitionLevel, definitionLevel);
213-
return pageEndsBefore(repetitionLevel, writer);
214-
}
215-
216-
boolean needNewPage(double value, int repetitionLevel, int definitionLevel, ColumnWriterBase writer) {
217-
roll(value, repetitionLevel, definitionLevel);
218-
return pageEndsBefore(repetitionLevel, writer);
219-
}
220-
221-
boolean needNewPage(boolean value, int repetitionLevel, int definitionLevel, ColumnWriterBase writer) {
222-
roll(value, repetitionLevel, definitionLevel);
223-
return pageEndsBefore(repetitionLevel, writer);
224-
}
225-
226-
boolean needNewPage(Binary value, int repetitionLevel, int definitionLevel, ColumnWriterBase writer) {
227-
roll(value, repetitionLevel, definitionLevel);
228-
return pageEndsBefore(repetitionLevel, writer);
229-
}
230-
231-
boolean needNewPage(int repetitionLevel, int definitionLevel, ColumnWriterBase writer) {
232-
rollNull(repetitionLevel, definitionLevel);
233-
return pageEndsBefore(repetitionLevel, writer);
234-
}
235-
236-
/**
237-
* At a record start, if a chunk ends there or a page limit is reached, counted from the page start
238-
* so that it cuts in the same places after an edit. Asking consumes a match, so a match inside a
239-
* record carries over to the next record start. A boundary on the first value of a page has no page
240-
* to end.
241-
*/
242-
private boolean pageEndsBefore(int repetitionLevel, ColumnWriterBase writer) {
243-
if (repetitionLevel != 0) {
244-
return false;
245-
}
246-
boolean chunkBoundary = needNewChunk();
247-
return writer.getValueCount() > 0
248-
&& (chunkBoundary
249-
|| writer.getPageRowCount() >= pageRowCountLimit
250-
|| writer.getValueCount() >= pageValueCountThreshold
251-
|| writer.getCurrentPageBufferedSize() >= pageSizeThreshold);
252-
}
253-
254205
/** Arrow C++ {@code Roll<kByteWidth>}: the {@code byteWidth} low-order bytes, little-endian. */
255206
private void roll(long bits, int byteWidth) {
256207
chunkSize += byteWidth;
@@ -288,11 +239,13 @@ private void roll(Binary value) {
288239
}
289240

290241
/**
291-
* Arrow C++ {@code NeedNewChunk}: a chunk ends after eight matches, each against the next gear
292-
* hash table, which approximates a normal chunk size distribution; or at {@code maxChunkSize}.
293-
* Neither resets the rolling hash, and the maximum size cut leaves the run counter alone.
242+
* Arrow C++ {@code NeedNewChunk} and arrow-rs {@code need_new_chunk}: a chunk ends after eight
243+
* matches, each against the next gear hash table, which approximates a normal chunk size
244+
* distribution, or at {@code maxChunkSize}. Neither resets the rolling hash, and the maximum size cut
245+
* leaves the run counter alone. Asked at record starts only, as they ask it, so a match or the maximum
246+
* size reached inside a record is kept until the next one.
294247
*/
295-
boolean needNewChunk() {
248+
private boolean needNewChunk() {
296249
if (hasMatched) {
297250
hasMatched = false;
298251
if (++nthRun >= GearHashTable.TABLE.length) {
@@ -308,40 +261,40 @@ boolean needNewChunk() {
308261
return false;
309262
}
310263

311-
/** The chunker of a column without content defined chunking: it rolls nothing and ends no page. */
264+
/** The chunker of a column without content defined chunking: it rolls nothing and ends no chunk. */
312265
private static final class NoopChunker extends ContentDefinedChunker {
313266
@Override
314-
boolean needNewPage(int value, int repetitionLevel, int definitionLevel, ColumnWriterBase writer) {
267+
boolean startsNewChunk(int value, int repetitionLevel, int definitionLevel) {
315268
return false;
316269
}
317270

318271
@Override
319-
boolean needNewPage(long value, int repetitionLevel, int definitionLevel, ColumnWriterBase writer) {
272+
boolean startsNewChunk(long value, int repetitionLevel, int definitionLevel) {
320273
return false;
321274
}
322275

323276
@Override
324-
boolean needNewPage(float value, int repetitionLevel, int definitionLevel, ColumnWriterBase writer) {
277+
boolean startsNewChunk(float value, int repetitionLevel, int definitionLevel) {
325278
return false;
326279
}
327280

328281
@Override
329-
boolean needNewPage(double value, int repetitionLevel, int definitionLevel, ColumnWriterBase writer) {
282+
boolean startsNewChunk(double value, int repetitionLevel, int definitionLevel) {
330283
return false;
331284
}
332285

333286
@Override
334-
boolean needNewPage(boolean value, int repetitionLevel, int definitionLevel, ColumnWriterBase writer) {
287+
boolean startsNewChunk(boolean value, int repetitionLevel, int definitionLevel) {
335288
return false;
336289
}
337290

338291
@Override
339-
boolean needNewPage(Binary value, int repetitionLevel, int definitionLevel, ColumnWriterBase writer) {
292+
boolean startsNewChunk(Binary value, int repetitionLevel, int definitionLevel) {
340293
return false;
341294
}
342295

343296
@Override
344-
boolean needNewPage(int repetitionLevel, int definitionLevel, ColumnWriterBase writer) {
297+
boolean startsNewChunk(int repetitionLevel, int definitionLevel) {
345298
return false;
346299
}
347300
}

0 commit comments

Comments
 (0)