Skip to content

Commit 81f9c8a

Browse files
committed
apacheGH-3817: Ask the chunker one question per value
The column writer asks needNewPage(value, repetitionLevel, definitionLevel, writer) before writing each triplet and ends the page when it says so, instead of rolling the value itself and then asking. Rolling is the content defined chunker's own business, so another chunker can answer the same question its own way, and the no-op chunker inlines to a single false.
1 parent 97abe51 commit 81f9c8a

2 files changed

Lines changed: 87 additions & 37 deletions

File tree

‎parquet-column/src/main/java/org/apache/parquet/column/impl/ColumnWriterBase.java‎

Lines changed: 21 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -115,12 +115,6 @@ private void repetitionLevel(int repetitionLevel) {
115115
}
116116
}
117117

118-
private void endPageAtChunkBoundary(int repetitionLevel) {
119-
if (chunker.needNewPage(repetitionLevel, this)) {
120-
writePage();
121-
}
122-
}
123-
124118
/**
125119
* Writes the current null value
126120
*
@@ -131,8 +125,9 @@ private void endPageAtChunkBoundary(int repetitionLevel) {
131125
public void writeNull(int repetitionLevel, int definitionLevel) {
132126
if (DEBUG) log(null, repetitionLevel, definitionLevel);
133127
try {
134-
chunker.rollNull(repetitionLevel, definitionLevel);
135-
endPageAtChunkBoundary(repetitionLevel);
128+
if (chunker.needNewPage(repetitionLevel, definitionLevel, this)) {
129+
writePage();
130+
}
136131
repetitionLevel(repetitionLevel);
137132
definitionLevel(definitionLevel);
138133
collector.writeNull(repetitionLevel, definitionLevel);
@@ -171,8 +166,9 @@ public long getBufferedSizeInMemory() {
171166
public void write(double value, int repetitionLevel, int definitionLevel) {
172167
if (DEBUG) log(value, repetitionLevel, definitionLevel);
173168
try {
174-
chunker.roll(value, repetitionLevel, definitionLevel);
175-
endPageAtChunkBoundary(repetitionLevel);
169+
if (chunker.needNewPage(value, repetitionLevel, definitionLevel, this)) {
170+
writePage();
171+
}
176172
repetitionLevel(repetitionLevel);
177173
definitionLevel(definitionLevel);
178174
dataColumn.writeDouble(value);
@@ -195,8 +191,9 @@ public void write(double value, int repetitionLevel, int definitionLevel) {
195191
public void write(float value, int repetitionLevel, int definitionLevel) {
196192
if (DEBUG) log(value, repetitionLevel, definitionLevel);
197193
try {
198-
chunker.roll(value, repetitionLevel, definitionLevel);
199-
endPageAtChunkBoundary(repetitionLevel);
194+
if (chunker.needNewPage(value, repetitionLevel, definitionLevel, this)) {
195+
writePage();
196+
}
200197
repetitionLevel(repetitionLevel);
201198
definitionLevel(definitionLevel);
202199
dataColumn.writeFloat(value);
@@ -219,8 +216,9 @@ public void write(float value, int repetitionLevel, int definitionLevel) {
219216
public void write(Binary value, int repetitionLevel, int definitionLevel) {
220217
if (DEBUG) log(value, repetitionLevel, definitionLevel);
221218
try {
222-
chunker.roll(value, repetitionLevel, definitionLevel);
223-
endPageAtChunkBoundary(repetitionLevel);
219+
if (chunker.needNewPage(value, repetitionLevel, definitionLevel, this)) {
220+
writePage();
221+
}
224222
repetitionLevel(repetitionLevel);
225223
definitionLevel(definitionLevel);
226224
dataColumn.writeBytes(value);
@@ -243,8 +241,9 @@ public void write(Binary value, int repetitionLevel, int definitionLevel) {
243241
public void write(boolean value, int repetitionLevel, int definitionLevel) {
244242
if (DEBUG) log(value, repetitionLevel, definitionLevel);
245243
try {
246-
chunker.roll(value, repetitionLevel, definitionLevel);
247-
endPageAtChunkBoundary(repetitionLevel);
244+
if (chunker.needNewPage(value, repetitionLevel, definitionLevel, this)) {
245+
writePage();
246+
}
248247
repetitionLevel(repetitionLevel);
249248
definitionLevel(definitionLevel);
250249
dataColumn.writeBoolean(value);
@@ -267,8 +266,9 @@ public void write(boolean value, int repetitionLevel, int definitionLevel) {
267266
public void write(int value, int repetitionLevel, int definitionLevel) {
268267
if (DEBUG) log(value, repetitionLevel, definitionLevel);
269268
try {
270-
chunker.roll(value, repetitionLevel, definitionLevel);
271-
endPageAtChunkBoundary(repetitionLevel);
269+
if (chunker.needNewPage(value, repetitionLevel, definitionLevel, this)) {
270+
writePage();
271+
}
272272
repetitionLevel(repetitionLevel);
273273
definitionLevel(definitionLevel);
274274
dataColumn.writeInteger(value);
@@ -291,8 +291,9 @@ public void write(int value, int repetitionLevel, int definitionLevel) {
291291
public void write(long value, int repetitionLevel, int definitionLevel) {
292292
if (DEBUG) log(value, repetitionLevel, definitionLevel);
293293
try {
294-
chunker.roll(value, repetitionLevel, definitionLevel);
295-
endPageAtChunkBoundary(repetitionLevel);
294+
if (chunker.needNewPage(value, repetitionLevel, definitionLevel, this)) {
295+
writePage();
296+
}
296297
repetitionLevel(repetitionLevel);
297298
definitionLevel(definitionLevel);
298299
dataColumn.writeLong(value);

‎parquet-column/src/main/java/org/apache/parquet/column/impl/ContentDefinedChunker.java‎

Lines changed: 66 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -30,8 +30,9 @@
3030

3131
/**
3232
* Decides the content defined data page boundaries of one leaf column for a whole file. The column
33-
* writer rolls every triplet into the hash and asks {@link #needNewPage} whether a page ends before
34-
* it; a no-op chunker, {@link #noop()}, stands in while content defined chunking is disabled.
33+
* writer asks {@code needNewPage} before writing each triplet, which this chunker answers by rolling
34+
* the triplet into its hash; a no-op chunker, {@link #noop()}, stands in while content defined
35+
* chunking is disabled.
3536
*
3637
* <p>Ported from Arrow C++ {@code parquet/chunker_internal} and arrow-rs {@code column/chunker/cdc.rs},
3738
* whose boundaries it must reproduce. Not thread safe.
@@ -194,12 +195,51 @@ private void rollLevels(int repetitionLevel, int definitionLevel) {
194195
}
195196

196197
/**
197-
* Whether the page ends before this triplet: at a record start, if a chunk ends there or a page
198-
* limit is reached, counted from the page start so that it cuts in the same places after an edit.
199-
* Asking consumes a match, so a match inside a record carries over to the next record start. A
200-
* boundary on the first value of a page has no page to end.
198+
* Whether the column writer should end its page before this triplet, which it asks before writing
199+
* each one; {@code needNewPage(repetitionLevel, definitionLevel, writer)} asks for a null.
201200
*/
202-
boolean needNewPage(int repetitionLevel, ColumnWriterBase writer) {
201+
boolean needNewPage(int value, int repetitionLevel, int definitionLevel, ColumnWriterBase writer) {
202+
roll(value, repetitionLevel, definitionLevel);
203+
return pageEndsBefore(repetitionLevel, writer);
204+
}
205+
206+
boolean needNewPage(long value, int repetitionLevel, int definitionLevel, ColumnWriterBase writer) {
207+
roll(value, repetitionLevel, definitionLevel);
208+
return pageEndsBefore(repetitionLevel, writer);
209+
}
210+
211+
boolean needNewPage(float value, int repetitionLevel, int definitionLevel, ColumnWriterBase writer) {
212+
roll(value, repetitionLevel, definitionLevel);
213+
return pageEndsBefore(repetitionLevel, writer);
214+
}
215+
216+
boolean needNewPage(double value, int repetitionLevel, int definitionLevel, ColumnWriterBase writer) {
217+
roll(value, repetitionLevel, definitionLevel);
218+
return pageEndsBefore(repetitionLevel, writer);
219+
}
220+
221+
boolean needNewPage(boolean value, int repetitionLevel, int definitionLevel, ColumnWriterBase writer) {
222+
roll(value, repetitionLevel, definitionLevel);
223+
return pageEndsBefore(repetitionLevel, writer);
224+
}
225+
226+
boolean needNewPage(Binary value, int repetitionLevel, int definitionLevel, ColumnWriterBase writer) {
227+
roll(value, repetitionLevel, definitionLevel);
228+
return pageEndsBefore(repetitionLevel, writer);
229+
}
230+
231+
boolean needNewPage(int repetitionLevel, int definitionLevel, ColumnWriterBase writer) {
232+
rollNull(repetitionLevel, definitionLevel);
233+
return pageEndsBefore(repetitionLevel, writer);
234+
}
235+
236+
/**
237+
* At a record start, if a chunk ends there or a page limit is reached, counted from the page start
238+
* so that it cuts in the same places after an edit. Asking consumes a match, so a match inside a
239+
* record carries over to the next record start. A boundary on the first value of a page has no page
240+
* to end.
241+
*/
242+
private boolean pageEndsBefore(int repetitionLevel, ColumnWriterBase writer) {
203243
if (repetitionLevel != 0) {
204244
return false;
205245
}
@@ -271,28 +311,37 @@ boolean needNewChunk() {
271311
/** The chunker of a column without content defined chunking: it rolls nothing and ends no page. */
272312
private static final class NoopChunker extends ContentDefinedChunker {
273313
@Override
274-
void roll(int value, int repetitionLevel, int definitionLevel) {}
275-
276-
@Override
277-
void roll(long value, int repetitionLevel, int definitionLevel) {}
314+
boolean needNewPage(int value, int repetitionLevel, int definitionLevel, ColumnWriterBase writer) {
315+
return false;
316+
}
278317

279318
@Override
280-
void roll(float value, int repetitionLevel, int definitionLevel) {}
319+
boolean needNewPage(long value, int repetitionLevel, int definitionLevel, ColumnWriterBase writer) {
320+
return false;
321+
}
281322

282323
@Override
283-
void roll(double value, int repetitionLevel, int definitionLevel) {}
324+
boolean needNewPage(float value, int repetitionLevel, int definitionLevel, ColumnWriterBase writer) {
325+
return false;
326+
}
284327

285328
@Override
286-
void roll(boolean value, int repetitionLevel, int definitionLevel) {}
329+
boolean needNewPage(double value, int repetitionLevel, int definitionLevel, ColumnWriterBase writer) {
330+
return false;
331+
}
287332

288333
@Override
289-
void roll(Binary value, int repetitionLevel, int definitionLevel) {}
334+
boolean needNewPage(boolean value, int repetitionLevel, int definitionLevel, ColumnWriterBase writer) {
335+
return false;
336+
}
290337

291338
@Override
292-
void rollNull(int repetitionLevel, int definitionLevel) {}
339+
boolean needNewPage(Binary value, int repetitionLevel, int definitionLevel, ColumnWriterBase writer) {
340+
return false;
341+
}
293342

294343
@Override
295-
boolean needNewPage(int repetitionLevel, ColumnWriterBase writer) {
344+
boolean needNewPage(int repetitionLevel, int definitionLevel, ColumnWriterBase writer) {
296345
return false;
297346
}
298347
}

0 commit comments

Comments
 (0)