Skip to content

Commit 807f2ad

Browse files
kszucsclaude
andcommitted
apacheGH-3817: Simplify content defined chunking and align it with Arrow
- Chunker state lives per file, in a ContentDefinedChunkers that InternalParquetRecordWriter keeps and passes to every row group's store through a new ParquetProperties.newColumnWriteStore overload. With chunking enabled, a store made without it fails fast rather than silently restarting the chunking. - Content defined chunking is configured through one setter, withContentDefinedChunking(CdcOptions), where null disables it, and named as Arrow names it: CdcOptions, ContentDefinedChunker. - ChunkingColumnWriter wraps each column writer once, when the store is built, and checks the page limits at every record start, counted from the page start; cached nulls are flushed on the unchunked schedule's maximum. - Under chunking a dictionary falls back to plain encoding only when it outgrows its page size limit, as in Arrow C++ and arrow-rs, and column chunks of nulls alone get an empty dictionary page. - Binary values are hashed through one path. - The tests fail when chunking is switched off or broken, cover the cases the Arrow C++ and arrow-rs suites do (V2 pages, dictionary encoding, edits across row groups, a chunker per column, the record writer path), and follow the project's test guidelines. The benchmark writes one mixed dataset. Co-Authored-By: Claude Opus 5.5 <[email protected]>
1 parent 65c3e08 commit 807f2ad

32 files changed

Lines changed: 1347 additions & 1858 deletions

‎parquet-benchmarks/src/main/java/org/apache/parquet/benchmarks/CdcWriteBenchmarks.java‎

Lines changed: 0 additions & 159 deletions
This file was deleted.
Lines changed: 98 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,98 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one
3+
* or more contributor license agreements. See the NOTICE file
4+
* distributed with this work for additional information
5+
* regarding copyright ownership. The ASF licenses this file
6+
* to you under the Apache License, Version 2.0 (the
7+
* "License"); you may not use this file except in compliance
8+
* with the License. You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing,
13+
* software distributed under the License is distributed on an
14+
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15+
* KIND, either express or implied. See the License for the
16+
* specific language governing permissions and limitations
17+
* under the License.
18+
*/
19+
package org.apache.parquet.benchmarks;
20+
21+
import java.io.IOException;
22+
import java.util.ArrayList;
23+
import java.util.List;
24+
import java.util.Random;
25+
import java.util.concurrent.TimeUnit;
26+
import org.apache.parquet.column.CdcOptions;
27+
import org.apache.parquet.example.data.Group;
28+
import org.apache.parquet.example.data.simple.SimpleGroupFactory;
29+
import org.apache.parquet.hadoop.ParquetFileWriter;
30+
import org.apache.parquet.hadoop.ParquetWriter;
31+
import org.apache.parquet.hadoop.example.ExampleParquetWriter;
32+
import org.apache.parquet.hadoop.metadata.CompressionCodecName;
33+
import org.apache.parquet.schema.MessageType;
34+
import org.apache.parquet.schema.MessageTypeParser;
35+
import org.openjdk.jmh.annotations.Benchmark;
36+
import org.openjdk.jmh.annotations.BenchmarkMode;
37+
import org.openjdk.jmh.annotations.Fork;
38+
import org.openjdk.jmh.annotations.Measurement;
39+
import org.openjdk.jmh.annotations.Mode;
40+
import org.openjdk.jmh.annotations.OutputTimeUnit;
41+
import org.openjdk.jmh.annotations.Param;
42+
import org.openjdk.jmh.annotations.Scope;
43+
import org.openjdk.jmh.annotations.Setup;
44+
import org.openjdk.jmh.annotations.State;
45+
import org.openjdk.jmh.annotations.Warmup;
46+
47+
/** The same rows written with and without content defined chunking, to {@link BlackHoleOutputFile}. */
48+
@BenchmarkMode(Mode.AverageTime)
49+
@Fork(1)
50+
@Warmup(iterations = 3)
51+
@Measurement(iterations = 5)
52+
@OutputTimeUnit(TimeUnit.MILLISECONDS)
53+
@State(Scope.Thread)
54+
public class ContentDefinedChunkingWriteBenchmarks {
55+
56+
private static final MessageType SCHEMA = MessageTypeParser.parseMessageType("message m {"
57+
+ " required int64 l; optional double d; optional binary s (STRING);"
58+
+ " optional group list (LIST) { repeated group list { optional int64 element; } } }");
59+
60+
@Param({"false", "true"})
61+
public boolean chunking;
62+
63+
private final List<Group> rows = new ArrayList<>();
64+
65+
@Setup
66+
public void setup() {
67+
Random random = new Random(TestDataFactory.DEFAULT_SEED);
68+
SimpleGroupFactory factory = new SimpleGroupFactory(SCHEMA);
69+
for (int i = 0; i < 100_000; i++) {
70+
Group row = factory.newGroup().append("l", random.nextLong());
71+
if (random.nextInt(10) > 0) {
72+
row.append("d", random.nextDouble()).append("s", Long.toString(random.nextLong(), 36));
73+
}
74+
Group list = row.addGroup("list");
75+
for (int n = random.nextInt(9); n > 0; n--) {
76+
list.addGroup("list").append("element", random.nextLong());
77+
}
78+
rows.add(row);
79+
}
80+
}
81+
82+
/** Dictionary encoding is off and the page row count limit lifted, so that the hashing dominates. */
83+
@Benchmark
84+
public void write() throws IOException {
85+
try (ParquetWriter<Group> writer = ExampleParquetWriter.builder(BlackHoleOutputFile.INSTANCE)
86+
.withWriteMode(ParquetFileWriter.Mode.OVERWRITE)
87+
.withType(SCHEMA)
88+
.withCompressionCodec(CompressionCodecName.UNCOMPRESSED)
89+
.withDictionaryEncoding(false)
90+
.withPageRowCountLimit(Integer.MAX_VALUE)
91+
.withContentDefinedChunking(chunking ? CdcOptions.DEFAULT : null)
92+
.build()) {
93+
for (Group row : rows) {
94+
writer.write(row);
95+
}
96+
}
97+
}
98+
}

‎parquet-column/src/main/java/org/apache/parquet/column/CdcOptions.java‎

Lines changed: 10 additions & 33 deletions
Original file line numberDiff line numberDiff line change
@@ -18,22 +18,15 @@
1818
*/
1919
package org.apache.parquet.column;
2020

21-
import org.apache.parquet.Preconditions;
2221
import org.apache.parquet.internal.column.chunking.RollingHashMask;
2322

2423
/**
25-
* EXPERIMENTAL: The size envelope and normalization level of content defined chunking (CDC), which
26-
* ends data pages at boundaries derived from the column's values, so that files sharing a run of
27-
* values share byte-identical pages. Sizes are measured on values and levels before encoding.
28-
* Dictionary ids follow the order values first appear in, so an edit that adds new values renumbers
29-
* the later ones within its row group: columns of many distinct values deduplicate best without
30-
* dictionary encoding.
24+
* EXPERIMENTAL: Options of content defined chunking, which ends data pages at boundaries derived from
25+
* the column's values, so that files sharing a run of values share byte-identical pages. Sizes are
26+
* measured on values and levels before encoding.
3127
*
3228
* <p>The settings and defaults match {@code CdcOptions} in Arrow C++ and arrow-rs, and so do the
33-
* chunk boundaries for the same physical values. Arrow types converted before writing, such as
34-
* narrow integers, coerced timestamps and decimals, are hashed differently there. A ParquetWriter
35-
* continues chunk boundaries across row groups, as arrow-rs does; Arrow C++ restarts them in every
36-
* row group, so its chunks match only in the first.
29+
* chunk boundaries for the same physical values.
3730
*
3831
* @see ParquetProperties.Builder#withContentDefinedChunking(CdcOptions)
3932
*/
@@ -50,7 +43,6 @@ private CdcOptions(Builder builder) {
5043
this.minChunkSize = builder.minChunkSize;
5144
this.maxChunkSize = builder.maxChunkSize;
5245
this.normLevel = builder.normLevel;
53-
// Validate now rather than when the first column writer is built.
5446
RollingHashMask.calculate(minChunkSize, maxChunkSize, normLevel);
5547
}
5648

@@ -97,45 +89,30 @@ public static class Builder {
9789
private Builder() {}
9890

9991
/**
100-
* Set the minimum chunk size in bytes, 256 KiB by default. The rolling hash is not updated
101-
* until a chunk reaches this size, so no chunk is shorter but a file's last; pages can be, where
102-
* a row group or a page limit ends one.
92+
* Set the minimum chunk size in bytes, 256 KiB by default.
10393
*
10494
* @param minChunkSize the minimum chunk size in bytes
10595
* @return this builder for method chaining
10696
*/
10797
public Builder withMinChunkSize(long minChunkSize) {
108-
Preconditions.checkArgument(
109-
minChunkSize >= 0,
110-
"Invalid content defined chunking minimum chunk size (negative): %s",
111-
minChunkSize);
11298
this.minChunkSize = minChunkSize;
11399
return this;
114100
}
115101

116102
/**
117-
* Set the maximum chunk size in bytes, 1 MiB by default. A chunk ends when it reaches this size,
118-
* whatever the rolling hash says. {@link ParquetProperties.Builder#withPageSize(int)} separately
119-
* limits the page size; below this it splits chunks into more pages, in the same places after
120-
* an edit, so it does not cost deduplication.
103+
* Set the maximum chunk size in bytes, 1 MiB by default.
121104
*
122105
* @param maxChunkSize the maximum chunk size in bytes
123106
* @return this builder for method chaining
124107
*/
125108
public Builder withMaxChunkSize(long maxChunkSize) {
126-
Preconditions.checkArgument(
127-
maxChunkSize > 0,
128-
"Invalid content defined chunking maximum chunk size (not positive): %s",
129-
maxChunkSize);
130109
this.maxChunkSize = maxChunkSize;
131110
return this;
132111
}
133112

134113
/**
135-
* Set the normalization level of the rolling hash mask, 0 by default. Raising it makes a
136-
* boundary more likely, which tightens the chunk size distribution and improves deduplication
137-
* at the cost of more small pages; lowering it does the reverse. Values outside
138-
* {@code [-3, 3]} are not useful.
114+
* Set the normalization level of the rolling hash mask, 0 by default. Raising it makes chunk
115+
* sizes more uniform at the cost of more small pages.
139116
*
140117
* @param normLevel the normalization level
141118
* @return this builder for method chaining
@@ -147,8 +124,8 @@ public Builder withNormLevel(int normLevel) {
147124

148125
/**
149126
* @return the options
150-
* @throws IllegalArgumentException if the maximum chunk size is not greater than the minimum,
151-
* or the envelope is too narrow for the normalization level
127+
* @throws IllegalArgumentException if the minimum chunk size is negative, the maximum is not
128+
* greater than the minimum, or the envelope is too narrow for the normalization level
152129
*/
153130
public CdcOptions build() {
154131
return new CdcOptions(this);

0 commit comments

Comments
 (0)