Skip to content

Commit 4befe56

Browse files
committed
apacheGH-3817: Chunk in the column writer, as Arrow C++ does
ColumnWriterBase carries its column's ContentDefinedChunker, passed in by the store as the bloom filter writer is, instead of being wrapped by ChunkingColumnWriter. Without content defined chunking it gets a no-op chunker, as it gets no-op statistics, which the JIT inlines away, so the write path has no added branch. ContentDefinedChunker follows Arrow C++: the column writer rolls each triplet and asks needNewPage() at record starts, built on Roll-like primitives and needNewChunk(). ContentDefinedChunker.forColumns takes the writer properties, for the page limits. Written files are unchanged.
1 parent dac60cd commit 4befe56

12 files changed

Lines changed: 215 additions & 211 deletions

File tree

‎parquet-column/src/main/java/org/apache/parquet/column/impl/ChunkingColumnWriter.java‎

Lines changed: 0 additions & 124 deletions
This file was deleted.

‎parquet-column/src/main/java/org/apache/parquet/column/impl/ColumnWriteStoreBase.java‎

Lines changed: 26 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,7 @@
2323
import static java.util.Collections.unmodifiableMap;
2424

2525
import java.util.Arrays;
26+
import java.util.Iterator;
2627
import java.util.List;
2728
import java.util.Map;
2829
import java.util.Map.Entry;
@@ -126,59 +127,63 @@ public ColumnWriter getColumnWriter(ColumnDescriptor path) {
126127
Preconditions.checkState(
127128
chunkers != null || !props.isContentDefinedChunkingEnabled(),
128129
"Content defined chunking needs the chunkers of the file");
130+
Preconditions.checkArgument(
131+
!props.isContentDefinedChunkingEnabled()
132+
|| chunkers.size() == schema.getColumns().size(),
133+
"Expected a chunker per column");
129134
this.props = props;
130135
this.thresholdTolerance = (long) (props.getPageSizeThreshold() * THRESHOLD_TOLERANCE_RATIO);
136+
Iterator<ContentDefinedChunker> columnChunkers =
137+
props.isContentDefinedChunkingEnabled() ? chunkers.iterator() : null;
131138
Map<ColumnDescriptor, ColumnWriterBase> mcolumns = new TreeMap<>();
132139
for (ColumnDescriptor path : schema.getColumns()) {
133140
PageWriter pageWriter = pageWriteStore.getPageWriter(path);
141+
ContentDefinedChunker chunker =
142+
columnChunkers != null ? columnChunkers.next() : ContentDefinedChunker.noop();
134143
if (props.isBloomFilterEnabled(path)) {
135144
BloomFilterWriter bloomFilterWriter = bloomFilterWriteStore.getBloomFilterWriter(path);
136-
mcolumns.put(path, createColumnWriterBase(path, pageWriter, bloomFilterWriter, props));
145+
mcolumns.put(path, createColumnWriterBase(path, pageWriter, bloomFilterWriter, chunker, props));
137146
} else {
138-
mcolumns.put(path, createColumnWriterBase(path, pageWriter, null, props));
147+
mcolumns.put(path, createColumnWriterBase(path, pageWriter, null, chunker, props));
139148
}
140149
}
141150
this.columns = unmodifiableMap(mcolumns);
142151

143152
this.rowCountForNextSizeCheck = props.getMinRowCountForPageSizeCheck();
144153

145-
Map<ColumnDescriptor, ? extends ColumnWriter> writers =
146-
props.isContentDefinedChunkingEnabled() ? chunkingColumns(schema, chunkers, props) : columns;
147-
148154
columnWriterProvider = new ColumnWriterProvider() {
149155
@Override
150156
public ColumnWriter getColumnWriter(ColumnDescriptor path) {
151-
return writers.get(path);
157+
return columns.get(path);
152158
}
153159
};
154160
}
155161

156-
/** Wraps each column writer with its chunker, the file's chunkers being in the schema's column order. */
157-
private Map<ColumnDescriptor, ColumnWriter> chunkingColumns(
158-
MessageType schema, List<ContentDefinedChunker> chunkers, ParquetProperties props) {
159-
List<ColumnDescriptor> paths = schema.getColumns();
160-
Preconditions.checkArgument(
161-
chunkers.size() == paths.size(), "Expected %s chunkers but got %s", paths.size(), chunkers.size());
162-
Map<ColumnDescriptor, ColumnWriter> chunkingColumns = new TreeMap<>();
163-
for (int i = 0; i < paths.size(); i++) {
164-
chunkingColumns.put(
165-
paths.get(i), new ChunkingColumnWriter(columns.get(paths.get(i)), chunkers.get(i), props));
166-
}
167-
return chunkingColumns;
162+
private ColumnWriterBase createColumnWriterBase(
163+
ColumnDescriptor path,
164+
PageWriter pageWriter,
165+
BloomFilterWriter bloomFilterWriter,
166+
ParquetProperties props) {
167+
return createColumnWriterBase(path, pageWriter, bloomFilterWriter, ContentDefinedChunker.noop(), props);
168168
}
169169

170170
private ColumnWriterBase createColumnWriterBase(
171171
ColumnDescriptor path,
172172
PageWriter pageWriter,
173173
BloomFilterWriter bloomFilterWriter,
174+
ContentDefinedChunker chunker,
174175
ParquetProperties props) {
175-
ColumnWriterBase columnWriterBase = createColumnWriter(path, pageWriter, bloomFilterWriter, props);
176+
ColumnWriterBase columnWriterBase = createColumnWriter(path, pageWriter, bloomFilterWriter, chunker, props);
176177
columnWriterBase.initStatusManager(statusManager);
177178
return columnWriterBase;
178179
}
179180

180181
abstract ColumnWriterBase createColumnWriter(
181-
ColumnDescriptor path, PageWriter pageWriter, BloomFilterWriter bloomFilterWriter, ParquetProperties props);
182+
ColumnDescriptor path,
183+
PageWriter pageWriter,
184+
BloomFilterWriter bloomFilterWriter,
185+
ContentDefinedChunker chunker,
186+
ParquetProperties props);
182187

183188
@Override
184189
public ColumnWriter getColumnWriter(ColumnDescriptor path) {
@@ -260,7 +265,7 @@ public void endRecord() {
260265
++rowCount;
261266
if (rowCount >= rowCountForNextSizeCheck) {
262267
if (props.isContentDefinedChunkingEnabled()) {
263-
// ChunkingColumnWriter applies the page limits; this schedule only flushes cached nulls.
268+
// The column writers apply the page limits; this schedule only flushes cached nulls.
264269
rowCountForNextSizeCheck = rowCount + props.getMaxRowCountForPageSizeCheck();
265270
return;
266271
}

‎parquet-column/src/main/java/org/apache/parquet/column/impl/ColumnWriteStoreV1.java‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -61,7 +61,8 @@ ColumnWriterBase createColumnWriter(
6161
ColumnDescriptor path,
6262
PageWriter pageWriter,
6363
BloomFilterWriter bloomFilterWriter,
64+
ContentDefinedChunker chunker,
6465
ParquetProperties props) {
65-
return new ColumnWriterV1(path, pageWriter, bloomFilterWriter, props);
66+
return new ColumnWriterV1(path, pageWriter, bloomFilterWriter, chunker, props);
6667
}
6768
}

‎parquet-column/src/main/java/org/apache/parquet/column/impl/ColumnWriteStoreV2.java‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -56,7 +56,8 @@ ColumnWriterBase createColumnWriter(
5656
ColumnDescriptor path,
5757
PageWriter pageWriter,
5858
BloomFilterWriter bloomFilterWriter,
59+
ContentDefinedChunker chunker,
5960
ParquetProperties props) {
60-
return new ColumnWriterV2(path, pageWriter, bloomFilterWriter, props);
61+
return new ColumnWriterV2(path, pageWriter, bloomFilterWriter, chunker, props);
6162
}
6263
}

‎parquet-column/src/main/java/org/apache/parquet/column/impl/ColumnWriterBase.java‎

Lines changed: 31 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -60,6 +60,7 @@ abstract class ColumnWriterBase implements ColumnWriter {
6060
private StatusManager statusManager = StatusManager.create();
6161

6262
private final ColumnValueCollector collector;
63+
private final ContentDefinedChunker chunker;
6364

6465
ColumnWriterBase(ColumnDescriptor path, PageWriter pageWriter, ParquetProperties props) {
6566
this(path, pageWriter, null, props);
@@ -70,6 +71,15 @@ abstract class ColumnWriterBase implements ColumnWriter {
7071
PageWriter pageWriter,
7172
BloomFilterWriter bloomFilterWriter,
7273
ParquetProperties props) {
74+
this(path, pageWriter, bloomFilterWriter, ContentDefinedChunker.noop(), props);
75+
}
76+
77+
ColumnWriterBase(
78+
ColumnDescriptor path,
79+
PageWriter pageWriter,
80+
BloomFilterWriter bloomFilterWriter,
81+
ContentDefinedChunker chunker,
82+
ParquetProperties props) {
7383
this.path = path;
7484
this.pageWriter = pageWriter;
7585

@@ -78,6 +88,7 @@ abstract class ColumnWriterBase implements ColumnWriter {
7888
this.dataColumn = props.newValuesWriter(path);
7989

8090
this.collector = new ColumnValueCollector(path, bloomFilterWriter, props);
91+
this.chunker = chunker;
8192
}
8293

8394
void initStatusManager(StatusManager statusManager) {
@@ -104,6 +115,12 @@ private void repetitionLevel(int repetitionLevel) {
104115
}
105116
}
106117

118+
private void endPageAtChunkBoundary(int repetitionLevel) {
119+
if (chunker.needNewPage(repetitionLevel, this)) {
120+
writePage();
121+
}
122+
}
123+
107124
/**
108125
* Writes the current null value
109126
*
@@ -114,6 +131,8 @@ private void repetitionLevel(int repetitionLevel) {
114131
public void writeNull(int repetitionLevel, int definitionLevel) {
115132
if (DEBUG) log(null, repetitionLevel, definitionLevel);
116133
try {
134+
chunker.rollNull(repetitionLevel, definitionLevel);
135+
endPageAtChunkBoundary(repetitionLevel);
117136
repetitionLevel(repetitionLevel);
118137
definitionLevel(definitionLevel);
119138
collector.writeNull(repetitionLevel, definitionLevel);
@@ -152,6 +171,8 @@ public long getBufferedSizeInMemory() {
152171
public void write(double value, int repetitionLevel, int definitionLevel) {
153172
if (DEBUG) log(value, repetitionLevel, definitionLevel);
154173
try {
174+
chunker.roll(value, repetitionLevel, definitionLevel);
175+
endPageAtChunkBoundary(repetitionLevel);
155176
repetitionLevel(repetitionLevel);
156177
definitionLevel(definitionLevel);
157178
dataColumn.writeDouble(value);
@@ -174,6 +195,8 @@ public void write(double value, int repetitionLevel, int definitionLevel) {
174195
public void write(float value, int repetitionLevel, int definitionLevel) {
175196
if (DEBUG) log(value, repetitionLevel, definitionLevel);
176197
try {
198+
chunker.roll(value, repetitionLevel, definitionLevel);
199+
endPageAtChunkBoundary(repetitionLevel);
177200
repetitionLevel(repetitionLevel);
178201
definitionLevel(definitionLevel);
179202
dataColumn.writeFloat(value);
@@ -196,6 +219,8 @@ public void write(float value, int repetitionLevel, int definitionLevel) {
196219
public void write(Binary value, int repetitionLevel, int definitionLevel) {
197220
if (DEBUG) log(value, repetitionLevel, definitionLevel);
198221
try {
222+
chunker.roll(value, repetitionLevel, definitionLevel);
223+
endPageAtChunkBoundary(repetitionLevel);
199224
repetitionLevel(repetitionLevel);
200225
definitionLevel(definitionLevel);
201226
dataColumn.writeBytes(value);
@@ -218,6 +243,8 @@ public void write(Binary value, int repetitionLevel, int definitionLevel) {
218243
public void write(boolean value, int repetitionLevel, int definitionLevel) {
219244
if (DEBUG) log(value, repetitionLevel, definitionLevel);
220245
try {
246+
chunker.roll(value, repetitionLevel, definitionLevel);
247+
endPageAtChunkBoundary(repetitionLevel);
221248
repetitionLevel(repetitionLevel);
222249
definitionLevel(definitionLevel);
223250
dataColumn.writeBoolean(value);
@@ -240,6 +267,8 @@ public void write(boolean value, int repetitionLevel, int definitionLevel) {
240267
public void write(int value, int repetitionLevel, int definitionLevel) {
241268
if (DEBUG) log(value, repetitionLevel, definitionLevel);
242269
try {
270+
chunker.roll(value, repetitionLevel, definitionLevel);
271+
endPageAtChunkBoundary(repetitionLevel);
243272
repetitionLevel(repetitionLevel);
244273
definitionLevel(definitionLevel);
245274
dataColumn.writeInteger(value);
@@ -262,6 +291,8 @@ public void write(int value, int repetitionLevel, int definitionLevel) {
262291
public void write(long value, int repetitionLevel, int definitionLevel) {
263292
if (DEBUG) log(value, repetitionLevel, definitionLevel);
264293
try {
294+
chunker.roll(value, repetitionLevel, definitionLevel);
295+
endPageAtChunkBoundary(repetitionLevel);
265296
repetitionLevel(repetitionLevel);
266297
definitionLevel(definitionLevel);
267298
dataColumn.writeLong(value);

‎parquet-column/src/main/java/org/apache/parquet/column/impl/ColumnWriterV1.java‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -47,6 +47,15 @@ public ColumnWriterV1(
4747
super(path, pageWriter, bloomFilterWriter, props);
4848
}
4949

50+
ColumnWriterV1(
51+
ColumnDescriptor path,
52+
PageWriter pageWriter,
53+
BloomFilterWriter bloomFilterWriter,
54+
ContentDefinedChunker chunker,
55+
ParquetProperties props) {
56+
super(path, pageWriter, bloomFilterWriter, chunker, props);
57+
}
58+
5059
@Override
5160
ValuesWriter createRLWriter(ParquetProperties props, ColumnDescriptor path) {
5261
return props.newRepetitionLevelWriter(path);

‎parquet-column/src/main/java/org/apache/parquet/column/impl/ColumnWriterV2.java‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -69,6 +69,15 @@ public BytesInput getBytes() {
6969
super(path, pageWriter, bloomFilterWriter, props);
7070
}
7171

72+
ColumnWriterV2(
73+
ColumnDescriptor path,
74+
PageWriter pageWriter,
75+
BloomFilterWriter bloomFilterWriter,
76+
ContentDefinedChunker chunker,
77+
ParquetProperties props) {
78+
super(path, pageWriter, bloomFilterWriter, chunker, props);
79+
}
80+
7281
@Override
7382
ValuesWriter createRLWriter(ParquetProperties props, ColumnDescriptor path) {
7483
return path.getMaxRepetitionLevel() == 0

0 commit comments

Comments
 (0)