2929import org .apache .parquet .schema .MessageType ;
3030
3131/**
32- * Decides the content defined data page boundaries of one leaf column for a whole file. The column
33- * writer asks {@code needNewPage} before writing each triplet, which this chunker answers by rolling
34- * the triplet into its hash; a no-op chunker, {@link #noop()}, stands in while content defined
35- * chunking is disabled.
32+ * Decides the content defined chunk boundaries of one leaf column for a whole file. The column writer
33+ * asks {@code startsNewChunk} with each triplet before writing it, which rolls the triplet into the
34+ * hash and answers whether it starts a new chunk, and ends its page there; the page limits end pages as
35+ * they do without content defined chunking. A no-op chunker, {@link #noop()}, stands in while content defined chunking is
36+ * disabled.
3637 *
3738 * <p>Ported from Arrow C++ {@code parquet/chunker_internal} and arrow-rs {@code column/chunker/cdc.rs},
3839 * whose boundaries it must reproduce. Not thread safe.
@@ -46,9 +47,6 @@ public class ContentDefinedChunker {
4647 private final long rollingHashMask ;
4748 private final int maxDef ;
4849 private final int maxRep ;
49- private final long pageSizeThreshold ;
50- private final int pageValueCountThreshold ;
51- private final int pageRowCountLimit ;
5250
5351 private long rollingHash ;
5452 private boolean hasMatched ;
@@ -69,7 +67,7 @@ public static List<ContentDefinedChunker> forColumns(MessageType schema, Parquet
6967 .collect (Collectors .toList ());
7068 }
7169
72- /** The chunker of a column without content defined chunking, which never ends a page . */
70+ /** The chunker of a column without content defined chunking, which never ends a chunk . */
7371 static ContentDefinedChunker noop () {
7472 return NOOP ;
7573 }
@@ -81,9 +79,6 @@ static ContentDefinedChunker noop() {
8179 this .rollingHashMask = calculateMask (minChunkSize , maxChunkSize , options .getNormLevel ());
8280 this .maxDef = path .getMaxDefinitionLevel ();
8381 this .maxRep = path .getMaxRepetitionLevel ();
84- this .pageSizeThreshold = props .getPageSizeThreshold ();
85- this .pageValueCountThreshold = props .getPageValueCountThreshold ();
86- this .pageRowCountLimit = props .getPageRowCountLimit ();
8782 }
8883
8984 /** Only for {@link NoopChunker}, which uses none of the state. */
@@ -93,9 +88,6 @@ private ContentDefinedChunker() {
9388 this .rollingHashMask = 0 ;
9489 this .maxDef = 0 ;
9590 this .maxRep = 0 ;
96- this .pageSizeThreshold = 0 ;
97- this .pageValueCountThreshold = 0 ;
98- this .pageRowCountLimit = 0 ;
9991 }
10092
10193 /**
@@ -141,44 +133,60 @@ public static long calculateMask(long minChunkSize, long maxChunkSize, int normL
141133 return -1L << (Long .SIZE - effectiveBits );
142134 }
143135
144- void roll (int value , int repetitionLevel , int definitionLevel ) {
136+ /**
137+ * Rolls a triplet into the hash and answers whether it starts a new chunk, which only the first
138+ * triplet of a record can; see {@link #needNewChunk()}.
139+ *
140+ * @param value the value of the triplet
141+ * @param repetitionLevel the repetition level of the triplet
142+ * @param definitionLevel the definition level of the triplet
143+ * @return whether a chunk ends before the triplet
144+ */
145+ boolean startsNewChunk (int value , int repetitionLevel , int definitionLevel ) {
145146 rollLevels (repetitionLevel , definitionLevel );
146147 roll (value , 4 );
148+ return repetitionLevel == 0 && needNewChunk ();
147149 }
148150
149- void roll (long value , int repetitionLevel , int definitionLevel ) {
151+ boolean startsNewChunk (long value , int repetitionLevel , int definitionLevel ) {
150152 rollLevels (repetitionLevel , definitionLevel );
151153 roll (value , 8 );
154+ return repetitionLevel == 0 && needNewChunk ();
152155 }
153156
154- void roll (float value , int repetitionLevel , int definitionLevel ) {
157+ boolean startsNewChunk (float value , int repetitionLevel , int definitionLevel ) {
155158 rollLevels (repetitionLevel , definitionLevel );
156159 // Raw bits: floatToIntBits canonicalizes NaN payloads and would diverge from the references.
157160 roll (Float .floatToRawIntBits (value ), 4 );
161+ return repetitionLevel == 0 && needNewChunk ();
158162 }
159163
160- void roll (double value , int repetitionLevel , int definitionLevel ) {
164+ boolean startsNewChunk (double value , int repetitionLevel , int definitionLevel ) {
161165 rollLevels (repetitionLevel , definitionLevel );
162166 roll (Double .doubleToRawLongBits (value ), 8 );
167+ return repetitionLevel == 0 && needNewChunk ();
163168 }
164169
165- void roll (boolean value , int repetitionLevel , int definitionLevel ) {
170+ boolean startsNewChunk (boolean value , int repetitionLevel , int definitionLevel ) {
166171 rollLevels (repetitionLevel , definitionLevel );
167172 // One byte per boolean, not one bit, as the references hash it.
168173 roll (value ? 1 : 0 , 1 );
174+ return repetitionLevel == 0 && needNewChunk ();
169175 }
170176
171- void roll (Binary value , int repetitionLevel , int definitionLevel ) {
177+ boolean startsNewChunk (Binary value , int repetitionLevel , int definitionLevel ) {
172178 rollLevels (repetitionLevel , definitionLevel );
173179 roll (value );
180+ return repetitionLevel == 0 && needNewChunk ();
174181 }
175182
176183 /**
177184 * Rolls the triplet of a null, which has no value to hash even when {@code definitionLevel ==
178- * maxDef}, as an omitted top-level required field does.
185+ * maxDef}, as an omitted top-level required field does, and answers whether a chunk ends before it .
179186 */
180- void rollNull (int repetitionLevel , int definitionLevel ) {
187+ boolean startsNewChunk (int repetitionLevel , int definitionLevel ) {
181188 rollLevels (repetitionLevel , definitionLevel );
189+ return repetitionLevel == 0 && needNewChunk ();
182190 }
183191
184192 /**
@@ -194,63 +202,6 @@ private void rollLevels(int repetitionLevel, int definitionLevel) {
194202 }
195203 }
196204
197- /**
198- * Whether the column writer should end its page before this triplet, which it asks before writing
199- * each one; {@code needNewPage(repetitionLevel, definitionLevel, writer)} asks for a null.
200- */
201- boolean needNewPage (int value , int repetitionLevel , int definitionLevel , ColumnWriterBase writer ) {
202- roll (value , repetitionLevel , definitionLevel );
203- return pageEndsBefore (repetitionLevel , writer );
204- }
205-
206- boolean needNewPage (long value , int repetitionLevel , int definitionLevel , ColumnWriterBase writer ) {
207- roll (value , repetitionLevel , definitionLevel );
208- return pageEndsBefore (repetitionLevel , writer );
209- }
210-
211- boolean needNewPage (float value , int repetitionLevel , int definitionLevel , ColumnWriterBase writer ) {
212- roll (value , repetitionLevel , definitionLevel );
213- return pageEndsBefore (repetitionLevel , writer );
214- }
215-
216- boolean needNewPage (double value , int repetitionLevel , int definitionLevel , ColumnWriterBase writer ) {
217- roll (value , repetitionLevel , definitionLevel );
218- return pageEndsBefore (repetitionLevel , writer );
219- }
220-
221- boolean needNewPage (boolean value , int repetitionLevel , int definitionLevel , ColumnWriterBase writer ) {
222- roll (value , repetitionLevel , definitionLevel );
223- return pageEndsBefore (repetitionLevel , writer );
224- }
225-
226- boolean needNewPage (Binary value , int repetitionLevel , int definitionLevel , ColumnWriterBase writer ) {
227- roll (value , repetitionLevel , definitionLevel );
228- return pageEndsBefore (repetitionLevel , writer );
229- }
230-
231- boolean needNewPage (int repetitionLevel , int definitionLevel , ColumnWriterBase writer ) {
232- rollNull (repetitionLevel , definitionLevel );
233- return pageEndsBefore (repetitionLevel , writer );
234- }
235-
236- /**
237- * At a record start, if a chunk ends there or a page limit is reached, counted from the page start
238- * so that it cuts in the same places after an edit. Asking consumes a match, so a match inside a
239- * record carries over to the next record start. A boundary on the first value of a page has no page
240- * to end.
241- */
242- private boolean pageEndsBefore (int repetitionLevel , ColumnWriterBase writer ) {
243- if (repetitionLevel != 0 ) {
244- return false ;
245- }
246- boolean chunkBoundary = needNewChunk ();
247- return writer .getValueCount () > 0
248- && (chunkBoundary
249- || writer .getPageRowCount () >= pageRowCountLimit
250- || writer .getValueCount () >= pageValueCountThreshold
251- || writer .getCurrentPageBufferedSize () >= pageSizeThreshold );
252- }
253-
254205 /** Arrow C++ {@code Roll<kByteWidth>}: the {@code byteWidth} low-order bytes, little-endian. */
255206 private void roll (long bits , int byteWidth ) {
256207 chunkSize += byteWidth ;
@@ -288,11 +239,13 @@ private void roll(Binary value) {
288239 }
289240
290241 /**
291- * Arrow C++ {@code NeedNewChunk}: a chunk ends after eight matches, each against the next gear
292- * hash table, which approximates a normal chunk size distribution; or at {@code maxChunkSize}.
293- * Neither resets the rolling hash, and the maximum size cut leaves the run counter alone.
242+ * Arrow C++ {@code NeedNewChunk} and arrow-rs {@code need_new_chunk}: a chunk ends after eight
243+ * matches, each against the next gear hash table, which approximates a normal chunk size
244+ * distribution, or at {@code maxChunkSize}. Neither resets the rolling hash, and the maximum size cut
245+ * leaves the run counter alone. Asked at record starts only, as they ask it, so a match or the maximum
246+ * size reached inside a record is kept until the next one.
294247 */
295- boolean needNewChunk () {
248+ private boolean needNewChunk () {
296249 if (hasMatched ) {
297250 hasMatched = false ;
298251 if (++nthRun >= GearHashTable .TABLE .length ) {
@@ -308,40 +261,40 @@ boolean needNewChunk() {
308261 return false ;
309262 }
310263
311- /** The chunker of a column without content defined chunking: it rolls nothing and ends no page . */
264+ /** The chunker of a column without content defined chunking: it rolls nothing and ends no chunk . */
312265 private static final class NoopChunker extends ContentDefinedChunker {
313266 @ Override
314- boolean needNewPage (int value , int repetitionLevel , int definitionLevel , ColumnWriterBase writer ) {
267+ boolean startsNewChunk (int value , int repetitionLevel , int definitionLevel ) {
315268 return false ;
316269 }
317270
318271 @ Override
319- boolean needNewPage (long value , int repetitionLevel , int definitionLevel , ColumnWriterBase writer ) {
272+ boolean startsNewChunk (long value , int repetitionLevel , int definitionLevel ) {
320273 return false ;
321274 }
322275
323276 @ Override
324- boolean needNewPage (float value , int repetitionLevel , int definitionLevel , ColumnWriterBase writer ) {
277+ boolean startsNewChunk (float value , int repetitionLevel , int definitionLevel ) {
325278 return false ;
326279 }
327280
328281 @ Override
329- boolean needNewPage (double value , int repetitionLevel , int definitionLevel , ColumnWriterBase writer ) {
282+ boolean startsNewChunk (double value , int repetitionLevel , int definitionLevel ) {
330283 return false ;
331284 }
332285
333286 @ Override
334- boolean needNewPage (boolean value , int repetitionLevel , int definitionLevel , ColumnWriterBase writer ) {
287+ boolean startsNewChunk (boolean value , int repetitionLevel , int definitionLevel ) {
335288 return false ;
336289 }
337290
338291 @ Override
339- boolean needNewPage (Binary value , int repetitionLevel , int definitionLevel , ColumnWriterBase writer ) {
292+ boolean startsNewChunk (Binary value , int repetitionLevel , int definitionLevel ) {
340293 return false ;
341294 }
342295
343296 @ Override
344- boolean needNewPage (int repetitionLevel , int definitionLevel , ColumnWriterBase writer ) {
297+ boolean startsNewChunk (int repetitionLevel , int definitionLevel ) {
345298 return false ;
346299 }
347300 }
0 commit comments