@@ -60,6 +60,7 @@ abstract class ColumnWriterBase implements ColumnWriter {
6060 private StatusManager statusManager = StatusManager .create ();
6161
6262 private final ColumnValueCollector collector ;
63+ private final ContentDefinedChunker chunker ;
6364
6465 ColumnWriterBase (ColumnDescriptor path , PageWriter pageWriter , ParquetProperties props ) {
6566 this (path , pageWriter , null , props );
@@ -70,6 +71,15 @@ abstract class ColumnWriterBase implements ColumnWriter {
7071 PageWriter pageWriter ,
7172 BloomFilterWriter bloomFilterWriter ,
7273 ParquetProperties props ) {
74+ this (path , pageWriter , bloomFilterWriter , ContentDefinedChunker .noop (), props );
75+ }
76+
77+ ColumnWriterBase (
78+ ColumnDescriptor path ,
79+ PageWriter pageWriter ,
80+ BloomFilterWriter bloomFilterWriter ,
81+ ContentDefinedChunker chunker ,
82+ ParquetProperties props ) {
7383 this .path = path ;
7484 this .pageWriter = pageWriter ;
7585
@@ -78,6 +88,7 @@ abstract class ColumnWriterBase implements ColumnWriter {
7888 this .dataColumn = props .newValuesWriter (path );
7989
8090 this .collector = new ColumnValueCollector (path , bloomFilterWriter , props );
91+ this .chunker = chunker ;
8192 }
8293
8394 void initStatusManager (StatusManager statusManager ) {
@@ -104,6 +115,12 @@ private void repetitionLevel(int repetitionLevel) {
104115 }
105116 }
106117
118+ private void endPageAtChunkBoundary (int repetitionLevel ) {
119+ if (chunker .needNewPage (repetitionLevel , this )) {
120+ writePage ();
121+ }
122+ }
123+
107124 /**
108125 * Writes the current null value
109126 *
@@ -114,6 +131,8 @@ private void repetitionLevel(int repetitionLevel) {
114131 public void writeNull (int repetitionLevel , int definitionLevel ) {
115132 if (DEBUG ) log (null , repetitionLevel , definitionLevel );
116133 try {
134+ chunker .rollNull (repetitionLevel , definitionLevel );
135+ endPageAtChunkBoundary (repetitionLevel );
117136 repetitionLevel (repetitionLevel );
118137 definitionLevel (definitionLevel );
119138 collector .writeNull (repetitionLevel , definitionLevel );
@@ -152,6 +171,8 @@ public long getBufferedSizeInMemory() {
152171 public void write (double value , int repetitionLevel , int definitionLevel ) {
153172 if (DEBUG ) log (value , repetitionLevel , definitionLevel );
154173 try {
174+ chunker .roll (value , repetitionLevel , definitionLevel );
175+ endPageAtChunkBoundary (repetitionLevel );
155176 repetitionLevel (repetitionLevel );
156177 definitionLevel (definitionLevel );
157178 dataColumn .writeDouble (value );
@@ -174,6 +195,8 @@ public void write(double value, int repetitionLevel, int definitionLevel) {
174195 public void write (float value , int repetitionLevel , int definitionLevel ) {
175196 if (DEBUG ) log (value , repetitionLevel , definitionLevel );
176197 try {
198+ chunker .roll (value , repetitionLevel , definitionLevel );
199+ endPageAtChunkBoundary (repetitionLevel );
177200 repetitionLevel (repetitionLevel );
178201 definitionLevel (definitionLevel );
179202 dataColumn .writeFloat (value );
@@ -196,6 +219,8 @@ public void write(float value, int repetitionLevel, int definitionLevel) {
196219 public void write (Binary value , int repetitionLevel , int definitionLevel ) {
197220 if (DEBUG ) log (value , repetitionLevel , definitionLevel );
198221 try {
222+ chunker .roll (value , repetitionLevel , definitionLevel );
223+ endPageAtChunkBoundary (repetitionLevel );
199224 repetitionLevel (repetitionLevel );
200225 definitionLevel (definitionLevel );
201226 dataColumn .writeBytes (value );
@@ -218,6 +243,8 @@ public void write(Binary value, int repetitionLevel, int definitionLevel) {
218243 public void write (boolean value , int repetitionLevel , int definitionLevel ) {
219244 if (DEBUG ) log (value , repetitionLevel , definitionLevel );
220245 try {
246+ chunker .roll (value , repetitionLevel , definitionLevel );
247+ endPageAtChunkBoundary (repetitionLevel );
221248 repetitionLevel (repetitionLevel );
222249 definitionLevel (definitionLevel );
223250 dataColumn .writeBoolean (value );
@@ -240,6 +267,8 @@ public void write(boolean value, int repetitionLevel, int definitionLevel) {
240267 public void write (int value , int repetitionLevel , int definitionLevel ) {
241268 if (DEBUG ) log (value , repetitionLevel , definitionLevel );
242269 try {
270+ chunker .roll (value , repetitionLevel , definitionLevel );
271+ endPageAtChunkBoundary (repetitionLevel );
243272 repetitionLevel (repetitionLevel );
244273 definitionLevel (definitionLevel );
245274 dataColumn .writeInteger (value );
@@ -262,6 +291,8 @@ public void write(int value, int repetitionLevel, int definitionLevel) {
262291 public void write (long value , int repetitionLevel , int definitionLevel ) {
263292 if (DEBUG ) log (value , repetitionLevel , definitionLevel );
264293 try {
294+ chunker .roll (value , repetitionLevel , definitionLevel );
295+ endPageAtChunkBoundary (repetitionLevel );
265296 repetitionLevel (repetitionLevel );
266297 definitionLevel (definitionLevel );
267298 dataColumn .writeLong (value );
0 commit comments