Skip to content

Commit d5bc4e7

Browse files
committed
apacheGH-3817: Simplify content defined chunking and align it with Arrow
- Chunker state lives per file, in a ContentDefinedChunkers that InternalParquetRecordWriter keeps and passes to every row group's store through a new ParquetProperties.newColumnWriteStore overload. With chunking enabled, a store made without it fails fast rather than silently restarting the chunking. - Content defined chunking is configured through one setter, withContentDefinedChunking(CdcOptions), where null disables it, and named as Arrow names it: CdcOptions, ContentDefinedChunker. - ChunkingColumnWriter wraps each column writer once, when the store is built, and checks the page limits at every record start, counted from the page start; cached nulls are flushed on the unchunked schedule's maximum. - Under chunking a dictionary falls back to plain encoding only when it outgrows its page size limit, as in Arrow C++ and arrow-rs, and column chunks of nulls alone get an empty dictionary page. - Binary values are hashed through one path. - The tests fail when chunking is switched off or broken, cover the cases the Arrow C++ and arrow-rs suites do (V2 pages, dictionary encoding, edits across row groups, a chunker per column, the record writer path), and follow the project's test guidelines. The benchmark writes one mixed dataset.
1 parent 65c3e08 commit d5bc4e7

32 files changed

Lines changed: 1429 additions & 2082 deletions

‎parquet-benchmarks/src/main/java/org/apache/parquet/benchmarks/CdcWriteBenchmarks.java‎

Lines changed: 0 additions & 159 deletions
This file was deleted.
Lines changed: 98 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,98 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one
3+
* or more contributor license agreements. See the NOTICE file
4+
* distributed with this work for additional information
5+
* regarding copyright ownership. The ASF licenses this file
6+
* to you under the Apache License, Version 2.0 (the
7+
* "License"); you may not use this file except in compliance
8+
* with the License. You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing,
13+
* software distributed under the License is distributed on an
14+
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15+
* KIND, either express or implied. See the License for the
16+
* specific language governing permissions and limitations
17+
* under the License.
18+
*/
19+
package org.apache.parquet.benchmarks;
20+
21+
import java.io.IOException;
22+
import java.util.ArrayList;
23+
import java.util.List;
24+
import java.util.Random;
25+
import java.util.concurrent.TimeUnit;
26+
import org.apache.parquet.column.CdcOptions;
27+
import org.apache.parquet.example.data.Group;
28+
import org.apache.parquet.example.data.simple.SimpleGroupFactory;
29+
import org.apache.parquet.hadoop.ParquetFileWriter;
30+
import org.apache.parquet.hadoop.ParquetWriter;
31+
import org.apache.parquet.hadoop.example.ExampleParquetWriter;
32+
import org.apache.parquet.hadoop.metadata.CompressionCodecName;
33+
import org.apache.parquet.schema.MessageType;
34+
import org.apache.parquet.schema.MessageTypeParser;
35+
import org.openjdk.jmh.annotations.Benchmark;
36+
import org.openjdk.jmh.annotations.BenchmarkMode;
37+
import org.openjdk.jmh.annotations.Fork;
38+
import org.openjdk.jmh.annotations.Measurement;
39+
import org.openjdk.jmh.annotations.Mode;
40+
import org.openjdk.jmh.annotations.OutputTimeUnit;
41+
import org.openjdk.jmh.annotations.Param;
42+
import org.openjdk.jmh.annotations.Scope;
43+
import org.openjdk.jmh.annotations.Setup;
44+
import org.openjdk.jmh.annotations.State;
45+
import org.openjdk.jmh.annotations.Warmup;
46+
47+
/** The same rows written with and without content defined chunking, to {@link BlackHoleOutputFile}. */
48+
@BenchmarkMode(Mode.AverageTime)
49+
@Fork(1)
50+
@Warmup(iterations = 3)
51+
@Measurement(iterations = 5)
52+
@OutputTimeUnit(TimeUnit.MILLISECONDS)
53+
@State(Scope.Thread)
54+
public class ContentDefinedChunkingWriteBenchmarks {
55+
56+
private static final MessageType SCHEMA = MessageTypeParser.parseMessageType("message m {"
57+
+ " required int64 l; optional double d; optional binary s (STRING);"
58+
+ " optional group list (LIST) { repeated group list { optional int64 element; } } }");
59+
60+
@Param({"false", "true"})
61+
public boolean chunking;
62+
63+
private final List<Group> rows = new ArrayList<>();
64+
65+
@Setup
66+
public void setup() {
67+
Random random = new Random(TestDataFactory.DEFAULT_SEED);
68+
SimpleGroupFactory factory = new SimpleGroupFactory(SCHEMA);
69+
for (int i = 0; i < 100_000; i++) {
70+
Group row = factory.newGroup().append("l", random.nextLong());
71+
if (random.nextInt(10) > 0) {
72+
row.append("d", random.nextDouble()).append("s", Long.toString(random.nextLong(), 36));
73+
}
74+
Group list = row.addGroup("list");
75+
for (int n = random.nextInt(9); n > 0; n--) {
76+
list.addGroup("list").append("element", random.nextLong());
77+
}
78+
rows.add(row);
79+
}
80+
}
81+
82+
/** Dictionary encoding is off and the page row count limit lifted, so that the hashing dominates. */
83+
@Benchmark
84+
public void write() throws IOException {
85+
try (ParquetWriter<Group> writer = ExampleParquetWriter.builder(BlackHoleOutputFile.INSTANCE)
86+
.withWriteMode(ParquetFileWriter.Mode.OVERWRITE)
87+
.withType(SCHEMA)
88+
.withCompressionCodec(CompressionCodecName.UNCOMPRESSED)
89+
.withDictionaryEncoding(false)
90+
.withPageRowCountLimit(Integer.MAX_VALUE)
91+
.withContentDefinedChunking(chunking ? CdcOptions.DEFAULT : null)
92+
.build()) {
93+
for (Group row : rows) {
94+
writer.write(row);
95+
}
96+
}
97+
}
98+
}

‎parquet-column/src/main/java/org/apache/parquet/column/CdcOptions.java‎

Lines changed: 8 additions & 36 deletions
Original file line numberDiff line numberDiff line change
@@ -18,22 +18,13 @@
1818
*/
1919
package org.apache.parquet.column;
2020

21-
import org.apache.parquet.Preconditions;
22-
import org.apache.parquet.internal.column.chunking.RollingHashMask;
23-
2421
/**
25-
* EXPERIMENTAL: The size envelope and normalization level of content defined chunking (CDC), which
26-
* ends data pages at boundaries derived from the column's values, so that files sharing a run of
27-
* values share byte-identical pages. Sizes are measured on values and levels before encoding.
28-
* Dictionary ids follow the order values first appear in, so an edit that adds new values renumbers
29-
* the later ones within its row group: columns of many distinct values deduplicate best without
30-
* dictionary encoding.
22+
* EXPERIMENTAL: Options of content defined chunking, which ends data pages at boundaries derived from
23+
* the column's values, so that files sharing a run of values share byte-identical pages. Sizes are
24+
* measured on values and levels before encoding.
3125
*
3226
* <p>The settings and defaults match {@code CdcOptions} in Arrow C++ and arrow-rs, and so do the
33-
* chunk boundaries for the same physical values. Arrow types converted before writing, such as
34-
* narrow integers, coerced timestamps and decimals, are hashed differently there. A ParquetWriter
35-
* continues chunk boundaries across row groups, as arrow-rs does; Arrow C++ restarts them in every
36-
* row group, so its chunks match only in the first.
27+
* chunk boundaries for the same physical values.
3728
*
3829
* @see ParquetProperties.Builder#withContentDefinedChunking(CdcOptions)
3930
*/
@@ -50,8 +41,6 @@ private CdcOptions(Builder builder) {
5041
this.minChunkSize = builder.minChunkSize;
5142
this.maxChunkSize = builder.maxChunkSize;
5243
this.normLevel = builder.normLevel;
53-
// Validate now rather than when the first column writer is built.
54-
RollingHashMask.calculate(minChunkSize, maxChunkSize, normLevel);
5544
}
5645

5746
/**
@@ -97,45 +86,30 @@ public static class Builder {
9786
private Builder() {}
9887

9988
/**
100-
* Set the minimum chunk size in bytes, 256 KiB by default. The rolling hash is not updated
101-
* until a chunk reaches this size, so no chunk is shorter but a file's last; pages can be, where
102-
* a row group or a page limit ends one.
89+
* Set the minimum chunk size in bytes, 256 KiB by default.
10390
*
10491
* @param minChunkSize the minimum chunk size in bytes
10592
* @return this builder for method chaining
10693
*/
10794
public Builder withMinChunkSize(long minChunkSize) {
108-
Preconditions.checkArgument(
109-
minChunkSize >= 0,
110-
"Invalid content defined chunking minimum chunk size (negative): %s",
111-
minChunkSize);
11295
this.minChunkSize = minChunkSize;
11396
return this;
11497
}
11598

11699
/**
117-
* Set the maximum chunk size in bytes, 1 MiB by default. A chunk ends when it reaches this size,
118-
* whatever the rolling hash says. {@link ParquetProperties.Builder#withPageSize(int)} separately
119-
* limits the page size; below this it splits chunks into more pages, in the same places after
120-
* an edit, so it does not cost deduplication.
100+
* Set the maximum chunk size in bytes, 1 MiB by default.
121101
*
122102
* @param maxChunkSize the maximum chunk size in bytes
123103
* @return this builder for method chaining
124104
*/
125105
public Builder withMaxChunkSize(long maxChunkSize) {
126-
Preconditions.checkArgument(
127-
maxChunkSize > 0,
128-
"Invalid content defined chunking maximum chunk size (not positive): %s",
129-
maxChunkSize);
130106
this.maxChunkSize = maxChunkSize;
131107
return this;
132108
}
133109

134110
/**
135-
* Set the normalization level of the rolling hash mask, 0 by default. Raising it makes a
136-
* boundary more likely, which tightens the chunk size distribution and improves deduplication
137-
* at the cost of more small pages; lowering it does the reverse. Values outside
138-
* {@code [-3, 3]} are not useful.
111+
* Set the normalization level of the rolling hash mask, 0 by default. Raising it makes chunk
112+
* sizes more uniform at the cost of more small pages.
139113
*
140114
* @param normLevel the normalization level
141115
* @return this builder for method chaining
@@ -147,8 +121,6 @@ public Builder withNormLevel(int normLevel) {
147121

148122
/**
149123
* @return the options
150-
* @throws IllegalArgumentException if the maximum chunk size is not greater than the minimum,
151-
* or the envelope is too narrow for the normalization level
152124
*/
153125
public CdcOptions build() {
154126
return new CdcOptions(this);

0 commit comments

Comments
 (0)