|
30 | 30 |
|
31 | 31 | /** |
32 | 32 | * Decides the content defined data page boundaries of one leaf column for a whole file. The column |
33 | | - * writer rolls every triplet into the hash and asks {@link #needNewPage} whether a page ends before |
34 | | - * it; a no-op chunker, {@link #noop()}, stands in while content defined chunking is disabled. |
| 33 | + * writer asks {@code needNewPage} before writing each triplet, which this chunker answers by rolling |
| 34 | + * the triplet into its hash; a no-op chunker, {@link #noop()}, stands in while content defined |
| 35 | + * chunking is disabled. |
35 | 36 | * |
36 | 37 | * <p>Ported from Arrow C++ {@code parquet/chunker_internal} and arrow-rs {@code column/chunker/cdc.rs}, |
37 | 38 | * whose boundaries it must reproduce. Not thread safe. |
@@ -194,12 +195,51 @@ private void rollLevels(int repetitionLevel, int definitionLevel) { |
194 | 195 | } |
195 | 196 |
|
196 | 197 | /** |
197 | | - * Whether the page ends before this triplet: at a record start, if a chunk ends there or a page |
198 | | - * limit is reached, counted from the page start so that it cuts in the same places after an edit. |
199 | | - * Asking consumes a match, so a match inside a record carries over to the next record start. A |
200 | | - * boundary on the first value of a page has no page to end. |
| 198 | + * Whether the column writer should end its page before this triplet, which it asks before writing |
| 199 | + * each one; {@code needNewPage(repetitionLevel, definitionLevel, writer)} asks for a null. |
201 | 200 | */ |
202 | | - boolean needNewPage(int repetitionLevel, ColumnWriterBase writer) { |
| 201 | + boolean needNewPage(int value, int repetitionLevel, int definitionLevel, ColumnWriterBase writer) { |
| 202 | + roll(value, repetitionLevel, definitionLevel); |
| 203 | + return pageEndsBefore(repetitionLevel, writer); |
| 204 | + } |
| 205 | + |
| 206 | + boolean needNewPage(long value, int repetitionLevel, int definitionLevel, ColumnWriterBase writer) { |
| 207 | + roll(value, repetitionLevel, definitionLevel); |
| 208 | + return pageEndsBefore(repetitionLevel, writer); |
| 209 | + } |
| 210 | + |
| 211 | + boolean needNewPage(float value, int repetitionLevel, int definitionLevel, ColumnWriterBase writer) { |
| 212 | + roll(value, repetitionLevel, definitionLevel); |
| 213 | + return pageEndsBefore(repetitionLevel, writer); |
| 214 | + } |
| 215 | + |
| 216 | + boolean needNewPage(double value, int repetitionLevel, int definitionLevel, ColumnWriterBase writer) { |
| 217 | + roll(value, repetitionLevel, definitionLevel); |
| 218 | + return pageEndsBefore(repetitionLevel, writer); |
| 219 | + } |
| 220 | + |
| 221 | + boolean needNewPage(boolean value, int repetitionLevel, int definitionLevel, ColumnWriterBase writer) { |
| 222 | + roll(value, repetitionLevel, definitionLevel); |
| 223 | + return pageEndsBefore(repetitionLevel, writer); |
| 224 | + } |
| 225 | + |
| 226 | + boolean needNewPage(Binary value, int repetitionLevel, int definitionLevel, ColumnWriterBase writer) { |
| 227 | + roll(value, repetitionLevel, definitionLevel); |
| 228 | + return pageEndsBefore(repetitionLevel, writer); |
| 229 | + } |
| 230 | + |
| 231 | + boolean needNewPage(int repetitionLevel, int definitionLevel, ColumnWriterBase writer) { |
| 232 | + rollNull(repetitionLevel, definitionLevel); |
| 233 | + return pageEndsBefore(repetitionLevel, writer); |
| 234 | + } |
| 235 | + |
| 236 | + /** |
| 237 | + * At a record start, if a chunk ends there or a page limit is reached, counted from the page start |
| 238 | + * so that it cuts in the same places after an edit. Asking consumes a match, so a match inside a |
| 239 | + * record carries over to the next record start. A boundary on the first value of a page has no page |
| 240 | + * to end. |
| 241 | + */ |
| 242 | + private boolean pageEndsBefore(int repetitionLevel, ColumnWriterBase writer) { |
203 | 243 | if (repetitionLevel != 0) { |
204 | 244 | return false; |
205 | 245 | } |
@@ -271,28 +311,37 @@ boolean needNewChunk() { |
271 | 311 | /** The chunker of a column without content defined chunking: it rolls nothing and ends no page. */ |
272 | 312 | private static final class NoopChunker extends ContentDefinedChunker { |
273 | 313 | @Override |
274 | | - void roll(int value, int repetitionLevel, int definitionLevel) {} |
275 | | - |
276 | | - @Override |
277 | | - void roll(long value, int repetitionLevel, int definitionLevel) {} |
| 314 | + boolean needNewPage(int value, int repetitionLevel, int definitionLevel, ColumnWriterBase writer) { |
| 315 | + return false; |
| 316 | + } |
278 | 317 |
|
279 | 318 | @Override |
280 | | - void roll(float value, int repetitionLevel, int definitionLevel) {} |
| 319 | + boolean needNewPage(long value, int repetitionLevel, int definitionLevel, ColumnWriterBase writer) { |
| 320 | + return false; |
| 321 | + } |
281 | 322 |
|
282 | 323 | @Override |
283 | | - void roll(double value, int repetitionLevel, int definitionLevel) {} |
| 324 | + boolean needNewPage(float value, int repetitionLevel, int definitionLevel, ColumnWriterBase writer) { |
| 325 | + return false; |
| 326 | + } |
284 | 327 |
|
285 | 328 | @Override |
286 | | - void roll(boolean value, int repetitionLevel, int definitionLevel) {} |
| 329 | + boolean needNewPage(double value, int repetitionLevel, int definitionLevel, ColumnWriterBase writer) { |
| 330 | + return false; |
| 331 | + } |
287 | 332 |
|
288 | 333 | @Override |
289 | | - void roll(Binary value, int repetitionLevel, int definitionLevel) {} |
| 334 | + boolean needNewPage(boolean value, int repetitionLevel, int definitionLevel, ColumnWriterBase writer) { |
| 335 | + return false; |
| 336 | + } |
290 | 337 |
|
291 | 338 | @Override |
292 | | - void rollNull(int repetitionLevel, int definitionLevel) {} |
| 339 | + boolean needNewPage(Binary value, int repetitionLevel, int definitionLevel, ColumnWriterBase writer) { |
| 340 | + return false; |
| 341 | + } |
293 | 342 |
|
294 | 343 | @Override |
295 | | - boolean needNewPage(int repetitionLevel, ColumnWriterBase writer) { |
| 344 | + boolean needNewPage(int repetitionLevel, int definitionLevel, ColumnWriterBase writer) { |
296 | 345 | return false; |
297 | 346 | } |
298 | 347 | } |
|
0 commit comments