Skip to content
This repository was archived by the owner on Sep 2, 2024. It is now read-only.

Commit 22a26ec

Browse files
mccheahrobert3005
authored andcommitted
Revert "PARQUET-1414: Limit page size based on maximum row count (apache#531)"
* Revert "PARQUET-1414: Simplify next row count check calculation (apache#537)" This reverts commit 7f561b6. * Revert "PARQUET-1414: Limit page size based on maximum row count (apache#531)" This reverts commit 1e0760a.
1 parent 6e8d68e commit 22a26ec

6 files changed

Lines changed: 9 additions & 129 deletions

File tree

‎FORK.md‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,8 @@
11
# Differences to mainline
2-
This repo exists mostly to make releases of parquet-mr more often. The only difference to upstream is solution for [PARQUET-686](https://issues.apache.org/jira/browse/PARQUET-686).
2+
This repo exists mostly to make releases of parquet-mr more often. The only difference to upstream is as follows:
3+
4+
1. Solution for [PARQUET-686](https://issues.apache.org/jira/browse/PARQUET-686).
5+
2. Temporarily revert [PARQUET-1414](https://issues.apache.org/jira/browse/PARQUET-1414) because it causes Spark to write unreadable empty Parquet pages.
36

47
The change that we had made that upstream only made in statistics v2 is to change binary comparison to be unsigned and declare all statistics priori to that change as corrupted. This lets us more quickly take advantage of binary statistics and removes burden on user to know whether they should account for signed binary comparison in their values.
58

‎parquet-column/src/main/java/org/apache/parquet/column/ParquetProperties.java‎

Lines changed: 2 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -48,7 +48,6 @@ public class ParquetProperties {
4848
public static final int DEFAULT_MINIMUM_RECORD_COUNT_FOR_CHECK = 100;
4949
public static final int DEFAULT_MAXIMUM_RECORD_COUNT_FOR_CHECK = 10000;
5050
public static final int DEFAULT_COLUMN_INDEX_TRUNCATE_LENGTH = 64;
51-
public static final int DEFAULT_PAGE_ROW_COUNT_LIMIT = 20_000;
5251

5352
public static final ValuesWriterFactory DEFAULT_VALUES_WRITER_FACTORY = new DefaultValuesWriterFactory();
5453

@@ -86,11 +85,10 @@ public static WriterVersion fromString(String name) {
8685
private final ByteBufferAllocator allocator;
8786
private final ValuesWriterFactory valuesWriterFactory;
8887
private final int columnIndexTruncateLength;
89-
private final int pageRowCountLimit;
9088

9189
private ParquetProperties(WriterVersion writerVersion, int pageSize, int dictPageSize, boolean enableDict, int minRowCountForPageSizeCheck,
9290
int maxRowCountForPageSizeCheck, boolean estimateNextSizeCheck, ByteBufferAllocator allocator,
93-
ValuesWriterFactory writerFactory, int columnIndexMinMaxTruncateLength, int pageRowCountLimit) {
91+
ValuesWriterFactory writerFactory, int columnIndexMinMaxTruncateLength) {
9492
this.pageSizeThreshold = pageSize;
9593
this.initialSlabSize = CapacityByteArrayOutputStream
9694
.initialSlabSizeHeuristic(MIN_SLAB_SIZE, pageSizeThreshold, 10);
@@ -104,7 +102,6 @@ private ParquetProperties(WriterVersion writerVersion, int pageSize, int dictPag
104102

105103
this.valuesWriterFactory = writerFactory;
106104
this.columnIndexTruncateLength = columnIndexMinMaxTruncateLength;
107-
this.pageRowCountLimit = pageRowCountLimit;
108105
}
109106

110107
public ValuesWriter newRepetitionLevelWriter(ColumnDescriptor path) {
@@ -197,10 +194,6 @@ public boolean estimateNextSizeCheck() {
197194
return estimateNextSizeCheck;
198195
}
199196

200-
public int getPageRowCountLimit() {
201-
return pageRowCountLimit;
202-
}
203-
204197
public static Builder builder() {
205198
return new Builder();
206199
}
@@ -220,22 +213,18 @@ public static class Builder {
220213
private ByteBufferAllocator allocator = new HeapByteBufferAllocator();
221214
private ValuesWriterFactory valuesWriterFactory = DEFAULT_VALUES_WRITER_FACTORY;
222215
private int columnIndexTruncateLength = DEFAULT_COLUMN_INDEX_TRUNCATE_LENGTH;
223-
private int pageRowCountLimit = DEFAULT_PAGE_ROW_COUNT_LIMIT;
224216

225217
private Builder() {
226218
}
227219

228220
private Builder(ParquetProperties toCopy) {
229-
this.pageSize = toCopy.pageSizeThreshold;
230221
this.enableDict = toCopy.enableDictionary;
231222
this.dictPageSize = toCopy.dictionaryPageSizeThreshold;
232223
this.writerVersion = toCopy.writerVersion;
233224
this.minRowCountForPageSizeCheck = toCopy.minRowCountForPageSizeCheck;
234225
this.maxRowCountForPageSizeCheck = toCopy.maxRowCountForPageSizeCheck;
235226
this.estimateNextSizeCheck = toCopy.estimateNextSizeCheck;
236-
this.valuesWriterFactory = toCopy.valuesWriterFactory;
237227
this.allocator = toCopy.allocator;
238-
this.pageRowCountLimit = toCopy.pageRowCountLimit;
239228
}
240229

241230
/**
@@ -324,17 +313,11 @@ public Builder withColumnIndexTruncateLength(int length) {
324313
return this;
325314
}
326315

327-
public Builder withPageRowCountLimit(int rowCount) {
328-
Preconditions.checkArgument(rowCount > 0, "Invalid row count limit for pages: " + rowCount);
329-
pageRowCountLimit = rowCount;
330-
return this;
331-
}
332-
333316
public ParquetProperties build() {
334317
ParquetProperties properties =
335318
new ParquetProperties(writerVersion, pageSize, dictPageSize,
336319
enableDict, minRowCountForPageSizeCheck, maxRowCountForPageSizeCheck,
337-
estimateNextSizeCheck, allocator, valuesWriterFactory, columnIndexTruncateLength, pageRowCountLimit);
320+
estimateNextSizeCheck, allocator, valuesWriterFactory, columnIndexTruncateLength);
338321
// we pass a constructed but uninitialized factory to ParquetProperties above as currently
339322
// creation of ValuesWriters is invoked from within ParquetProperties. In the future
340323
// we'd like to decouple that and won't need to pass an object to properties and then pass the

‎parquet-column/src/main/java/org/apache/parquet/column/impl/ColumnWriteStoreBase.java‎

Lines changed: 3 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -67,7 +67,7 @@ private interface ColumnWriterProvider {
6767

6868
this.columns = new TreeMap<>();
6969

70-
this.rowCountForNextSizeCheck = min(props.getMinRowCountForPageSizeCheck(), props.getPageRowCountLimit());
70+
this.rowCountForNextSizeCheck = props.getMinRowCountForPageSizeCheck();
7171

7272
columnWriterProvider = new ColumnWriterProvider() {
7373
@Override
@@ -95,7 +95,7 @@ public ColumnWriter getColumnWriter(ColumnDescriptor path) {
9595
}
9696
this.columns = unmodifiableMap(mcolumns);
9797

98-
this.rowCountForNextSizeCheck = min(props.getMinRowCountForPageSizeCheck(), props.getPageRowCountLimit());
98+
this.rowCountForNextSizeCheck = props.getMinRowCountForPageSizeCheck();
9999

100100
columnWriterProvider = new ColumnWriterProvider() {
101101
@Override
@@ -190,17 +190,13 @@ public void endRecord() {
190190

191191
private void sizeCheck() {
192192
long minRecordToWait = Long.MAX_VALUE;
193-
int pageRowCountLimit = props.getPageRowCountLimit();
194-
long rowCountForNextRowCountCheck = rowCount + pageRowCountLimit;
195193
for (ColumnWriterBase writer : columns.values()) {
196194
long usedMem = writer.getCurrentPageBufferedSize();
197195
long rows = rowCount - writer.getRowsWrittenSoFar();
198196
long remainingMem = props.getPageSizeThreshold() - usedMem;
199-
if (remainingMem <= thresholdTolerance || rows >= pageRowCountLimit) {
197+
if (remainingMem <= thresholdTolerance) {
200198
writer.writePage();
201199
remainingMem = props.getPageSizeThreshold();
202-
} else {
203-
rowCountForNextRowCountCheck = min(rowCountForNextRowCountCheck, writer.getRowsWrittenSoFar() + pageRowCountLimit);
204200
}
205201
long rowsToFillPage =
206202
usedMem == 0 ?
@@ -223,10 +219,5 @@ private void sizeCheck() {
223219
} else {
224220
rowCountForNextSizeCheck = rowCount + props.getMinRowCountForPageSizeCheck();
225221
}
226-
227-
// Do the check earlier if required to keep the row count limit
228-
if (rowCountForNextRowCountCheck < rowCountForNextSizeCheck) {
229-
rowCountForNextSizeCheck = rowCountForNextRowCountCheck;
230-
}
231222
}
232223
}

‎parquet-column/src/test/java/org/apache/parquet/column/mem/TestMemColumn.java‎

Lines changed: 0 additions & 71 deletions
Original file line numberDiff line numberDiff line change
@@ -18,28 +18,19 @@
1818
*/
1919
package org.apache.parquet.column.mem;
2020

21-
import static org.apache.parquet.schema.PrimitiveType.PrimitiveTypeName.BINARY;
22-
import static org.apache.parquet.schema.PrimitiveType.PrimitiveTypeName.INT32;
2321
import static org.junit.Assert.assertEquals;
24-
import static org.junit.Assert.assertTrue;
2522

2623
import org.apache.parquet.column.ColumnDescriptor;
2724
import org.apache.parquet.column.ColumnReader;
28-
import org.apache.parquet.column.ColumnWriteStore;
2925
import org.apache.parquet.column.ColumnWriter;
3026
import org.apache.parquet.column.ParquetProperties;
3127
import org.apache.parquet.column.impl.ColumnReadStoreImpl;
3228
import org.apache.parquet.column.impl.ColumnWriteStoreV1;
33-
import org.apache.parquet.column.impl.ColumnWriteStoreV2;
34-
import org.apache.parquet.column.page.DataPage;
35-
import org.apache.parquet.column.page.PageReader;
3629
import org.apache.parquet.column.page.mem.MemPageStore;
3730
import org.apache.parquet.example.DummyRecordConverter;
3831
import org.apache.parquet.io.api.Binary;
3932
import org.apache.parquet.schema.MessageType;
4033
import org.apache.parquet.schema.MessageTypeParser;
41-
import org.apache.parquet.schema.PrimitiveType.PrimitiveTypeName;
42-
import org.apache.parquet.schema.Types;
4334
import org.junit.Test;
4435
import org.slf4j.Logger;
4536
import org.slf4j.LoggerFactory;
@@ -175,68 +166,6 @@ public void testMemColumnSeveralPagesRepeated() throws Exception {
175166
}
176167
}
177168

178-
@Test
179-
public void testPageSize() {
180-
MessageType schema = Types.buildMessage()
181-
.requiredList().requiredElement(BINARY).named("binary_col")
182-
.requiredList().requiredElement(INT32).named("int32_col")
183-
.named("msg");
184-
System.out.println(schema);
185-
MemPageStore memPageStore = new MemPageStore(123);
186-
187-
// Using V2 pages so we have rowCount info
188-
ColumnWriteStore writeStore = new ColumnWriteStoreV2(schema, memPageStore, ParquetProperties.builder()
189-
.withPageSize(1024) // Less than 10 records for binary_col
190-
.withMinRowCountForPageSizeCheck(1) // Enforce having precise page sizing
191-
.withPageRowCountLimit(10)
192-
.withDictionaryEncoding(false) // Enforce having large binary_col pages
193-
.build());
194-
ColumnDescriptor binaryCol = schema.getColumnDescription(new String[] { "binary_col", "list", "element" });
195-
ColumnWriter binaryColWriter = writeStore.getColumnWriter(binaryCol);
196-
ColumnDescriptor int32Col = schema.getColumnDescription(new String[] { "int32_col", "list", "element" });
197-
ColumnWriter int32ColWriter = writeStore.getColumnWriter(int32Col);
198-
// Writing 123 records
199-
for (int i = 0; i < 123; ++i) {
200-
// Writing 10 values per record
201-
for (int j = 0; j < 10; ++j) {
202-
binaryColWriter.write(Binary.fromString("aaaaaaaaaaaa"), j == 0 ? 0 : 2, 2);
203-
int32ColWriter.write(42, j == 0 ? 0 : 2, 2);
204-
}
205-
writeStore.endRecord();
206-
}
207-
writeStore.flush();
208-
209-
// Check that all the binary_col pages are <= 1024 bytes
210-
{
211-
PageReader binaryColPageReader = memPageStore.getPageReader(binaryCol);
212-
assertEquals(1230, binaryColPageReader.getTotalValueCount());
213-
int pageCnt = 0;
214-
int valueCnt = 0;
215-
while (valueCnt < binaryColPageReader.getTotalValueCount()) {
216-
DataPage page = binaryColPageReader.readPage();
217-
++pageCnt;
218-
valueCnt += page.getValueCount();
219-
LOG.info("binary_col page-{}: {} bytes, {} rows", pageCnt, page.getCompressedSize(), page.getIndexRowCount().get());
220-
assertTrue("Compressed size should be less than 1024", page.getCompressedSize() <= 1024);
221-
}
222-
}
223-
224-
// Check that all the int32_col pages contain <= 10 rows
225-
{
226-
PageReader int32ColPageReader = memPageStore.getPageReader(int32Col);
227-
assertEquals(1230, int32ColPageReader.getTotalValueCount());
228-
int pageCnt = 0;
229-
int valueCnt = 0;
230-
while (valueCnt < int32ColPageReader.getTotalValueCount()) {
231-
DataPage page = int32ColPageReader.readPage();
232-
++pageCnt;
233-
valueCnt += page.getValueCount();
234-
LOG.info("int32_col page-{}: {} bytes, {} rows", pageCnt, page.getCompressedSize(), page.getIndexRowCount().get());
235-
assertTrue("Row count should be less than 10", page.getIndexRowCount().get() <= 10);
236-
}
237-
}
238-
}
239-
240169
private ColumnWriteStoreV1 newColumnWriteStoreImpl(MemPageStore memPageStore) {
241170
return new ColumnWriteStoreV1(memPageStore,
242171
ParquetProperties.builder()

‎parquet-hadoop/src/main/java/org/apache/parquet/hadoop/ParquetOutputFormat.java‎

Lines changed: 0 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -144,7 +144,6 @@ public static enum JobSummaryLevel {
144144
public static final String MAX_ROW_COUNT_FOR_PAGE_SIZE_CHECK = "parquet.page.size.row.check.max";
145145
public static final String ESTIMATE_PAGE_SIZE_CHECK = "parquet.page.size.check.estimate";
146146
public static final String COLUMN_INDEX_TRUNCATE_LENGTH = "parquet.columnindex.truncate.length";
147-
public static final String PAGE_ROW_COUNT_LIMIT = "parquet.page.row.count.limit";
148147

149148
public static JobSummaryLevel getJobSummaryLevel(Configuration conf) {
150149
String level = conf.get(JOB_SUMMARY_LEVEL);
@@ -326,18 +325,6 @@ private static int getColumnIndexTruncateLength(Configuration conf) {
326325
return conf.getInt(COLUMN_INDEX_TRUNCATE_LENGTH, ParquetProperties.DEFAULT_COLUMN_INDEX_TRUNCATE_LENGTH);
327326
}
328327

329-
public static void setPageRowCountLimit(JobContext jobContext, int rowCount) {
330-
setPageRowCountLimit(getConfiguration(jobContext), rowCount);
331-
}
332-
333-
public static void setPageRowCountLimit(Configuration conf, int rowCount) {
334-
conf.setInt(PAGE_ROW_COUNT_LIMIT, rowCount);
335-
}
336-
337-
private static int getPageRowCountLimit(Configuration conf) {
338-
return conf.getInt(PAGE_ROW_COUNT_LIMIT, ParquetProperties.DEFAULT_PAGE_ROW_COUNT_LIMIT);
339-
}
340-
341328
private WriteSupport<T> writeSupport;
342329
private ParquetOutputCommitter committer;
343330

@@ -393,7 +380,6 @@ public RecordWriter<Void, T> getRecordWriter(Configuration conf, Path file, Comp
393380
.withMinRowCountForPageSizeCheck(getMinRowCountForPageSizeCheck(conf))
394381
.withMaxRowCountForPageSizeCheck(getMaxRowCountForPageSizeCheck(conf))
395382
.withColumnIndexTruncateLength(getColumnIndexTruncateLength(conf))
396-
.withPageRowCountLimit(getPageRowCountLimit(conf))
397383
.build();
398384

399385
long blockSize = getLongBlockSize(conf);
@@ -412,7 +398,6 @@ public RecordWriter<Void, T> getRecordWriter(Configuration conf, Path file, Comp
412398
LOG.info("Min row count for page size check is: {}", props.getMinRowCountForPageSizeCheck());
413399
LOG.info("Max row count for page size check is: {}", props.getMaxRowCountForPageSizeCheck());
414400
LOG.info("Truncate length for column indexes is: {}", props.getColumnIndexTruncateLength());
415-
LOG.info("Page row count limit to {}", props.getPageRowCountLimit());
416401
}
417402

418403
WriteContext init = writeSupport.init(conf);

‎parquet-hadoop/src/main/java/org/apache/parquet/hadoop/ParquetWriter.java‎

Lines changed: 0 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -425,17 +425,6 @@ public SELF withPageSize(int pageSize) {
425425
return self();
426426
}
427427

428-
/**
429-
* Sets the Parquet format page row count limit used by the constructed writer.
430-
*
431-
* @param rowCount limit for the number of rows stored in a page
432-
* @return this builder for method chaining
433-
*/
434-
public SELF withPageRowCountLimit(int rowCount) {
435-
encodingPropsBuilder.withPageRowCountLimit(rowCount);
436-
return self();
437-
}
438-
439428
/**
440429
* Set the Parquet format dictionary page size used by the constructed
441430
* writer.

0 commit comments

Comments
 (0)