Skip to content

Commit ecf5dfe

Browse files
committed
apacheGH-3817: Simplify content defined chunking and align it with Arrow
- Chunker state lives per file: InternalParquetRecordWriter creates the chunkers of the file up front, one per leaf column, only when chunking is enabled, as Arrow C++ and arrow-rs do, and passes them to every row group's store through a new ParquetProperties.newColumnWriteStore overload. With chunking enabled, a store made without them fails fast rather than silently restarting the chunking. CdcOptions are validated when built. - Content defined chunking is configured through one setter, withContentDefinedChunking(CdcOptions), where null disables it, and named as Arrow names it: CdcOptions, ContentDefinedChunker. - ChunkingColumnWriter wraps each column writer once, when the store is built. It rolls every triplet into the column's chunker and, at record starts, ends the page where needNewChunk() says, as Arrow C++ does, or where a page limit is reached, counted from the page start; cached nulls are flushed on the unchunked schedule's maximum. - Binary values are hashed through one path. - The tests fail when chunking is switched off or broken, cover the cases the Arrow C++ and arrow-rs suites do (V2 pages, dictionary encoding, edits across row groups, a chunker per column, the record writer path), and follow the project's test guidelines. The benchmark writes one mixed dataset.
1 parent 65c3e08 commit ecf5dfe

33 files changed

Lines changed: 1811 additions & 2528 deletions

‎parquet-benchmarks/src/main/java/org/apache/parquet/benchmarks/CdcWriteBenchmarks.java‎

Lines changed: 0 additions & 159 deletions
This file was deleted.
Lines changed: 98 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,98 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one
3+
* or more contributor license agreements. See the NOTICE file
4+
* distributed with this work for additional information
5+
* regarding copyright ownership. The ASF licenses this file
6+
* to you under the Apache License, Version 2.0 (the
7+
* "License"); you may not use this file except in compliance
8+
* with the License. You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing,
13+
* software distributed under the License is distributed on an
14+
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15+
* KIND, either express or implied. See the License for the
16+
* specific language governing permissions and limitations
17+
* under the License.
18+
*/
19+
package org.apache.parquet.benchmarks;
20+
21+
import java.io.IOException;
22+
import java.util.ArrayList;
23+
import java.util.List;
24+
import java.util.Random;
25+
import java.util.concurrent.TimeUnit;
26+
import org.apache.parquet.column.CdcOptions;
27+
import org.apache.parquet.example.data.Group;
28+
import org.apache.parquet.example.data.simple.SimpleGroupFactory;
29+
import org.apache.parquet.hadoop.ParquetFileWriter;
30+
import org.apache.parquet.hadoop.ParquetWriter;
31+
import org.apache.parquet.hadoop.example.ExampleParquetWriter;
32+
import org.apache.parquet.hadoop.metadata.CompressionCodecName;
33+
import org.apache.parquet.schema.MessageType;
34+
import org.apache.parquet.schema.MessageTypeParser;
35+
import org.openjdk.jmh.annotations.Benchmark;
36+
import org.openjdk.jmh.annotations.BenchmarkMode;
37+
import org.openjdk.jmh.annotations.Fork;
38+
import org.openjdk.jmh.annotations.Measurement;
39+
import org.openjdk.jmh.annotations.Mode;
40+
import org.openjdk.jmh.annotations.OutputTimeUnit;
41+
import org.openjdk.jmh.annotations.Param;
42+
import org.openjdk.jmh.annotations.Scope;
43+
import org.openjdk.jmh.annotations.Setup;
44+
import org.openjdk.jmh.annotations.State;
45+
import org.openjdk.jmh.annotations.Warmup;
46+
47+
/** The same rows written with and without content defined chunking, to {@link BlackHoleOutputFile}. */
48+
@BenchmarkMode(Mode.AverageTime)
49+
@Fork(1)
50+
@Warmup(iterations = 3)
51+
@Measurement(iterations = 5)
52+
@OutputTimeUnit(TimeUnit.MILLISECONDS)
53+
@State(Scope.Thread)
54+
public class ContentDefinedChunkingWriteBenchmarks {
55+
56+
private static final MessageType SCHEMA = MessageTypeParser.parseMessageType("message m {"
57+
+ " required int64 l; optional double d; optional binary s (STRING);"
58+
+ " optional group list (LIST) { repeated group list { optional int64 element; } } }");
59+
60+
@Param({"false", "true"})
61+
public boolean chunking;
62+
63+
private final List<Group> rows = new ArrayList<>();
64+
65+
@Setup
66+
public void setup() {
67+
Random random = new Random(TestDataFactory.DEFAULT_SEED);
68+
SimpleGroupFactory factory = new SimpleGroupFactory(SCHEMA);
69+
for (int i = 0; i < 100_000; i++) {
70+
Group row = factory.newGroup().append("l", random.nextLong());
71+
if (random.nextInt(10) > 0) {
72+
row.append("d", random.nextDouble()).append("s", Long.toString(random.nextLong(), 36));
73+
}
74+
Group list = row.addGroup("list");
75+
for (int n = random.nextInt(9); n > 0; n--) {
76+
list.addGroup("list").append("element", random.nextLong());
77+
}
78+
rows.add(row);
79+
}
80+
}
81+
82+
/** Dictionary encoding is off and the page row count limit lifted, so that the hashing dominates. */
83+
@Benchmark
84+
public void write() throws IOException {
85+
try (ParquetWriter<Group> writer = ExampleParquetWriter.builder(BlackHoleOutputFile.INSTANCE)
86+
.withWriteMode(ParquetFileWriter.Mode.OVERWRITE)
87+
.withType(SCHEMA)
88+
.withCompressionCodec(CompressionCodecName.UNCOMPRESSED)
89+
.withDictionaryEncoding(false)
90+
.withPageRowCountLimit(Integer.MAX_VALUE)
91+
.withContentDefinedChunking(chunking ? CdcOptions.DEFAULT : null)
92+
.build()) {
93+
for (Group row : rows) {
94+
writer.write(row);
95+
}
96+
}
97+
}
98+
}

‎parquet-column/src/main/java/org/apache/parquet/column/CdcOptions.java‎

Lines changed: 12 additions & 35 deletions
Original file line numberDiff line numberDiff line change
@@ -18,22 +18,15 @@
1818
*/
1919
package org.apache.parquet.column;
2020

21-
import org.apache.parquet.Preconditions;
22-
import org.apache.parquet.internal.column.chunking.RollingHashMask;
21+
import org.apache.parquet.column.impl.ContentDefinedChunker;
2322

2423
/**
25-
* EXPERIMENTAL: The size envelope and normalization level of content defined chunking (CDC), which
26-
* ends data pages at boundaries derived from the column's values, so that files sharing a run of
27-
* values share byte-identical pages. Sizes are measured on values and levels before encoding.
28-
* Dictionary ids follow the order values first appear in, so an edit that adds new values renumbers
29-
* the later ones within its row group: columns of many distinct values deduplicate best without
30-
* dictionary encoding.
24+
* EXPERIMENTAL: Options of content defined chunking, which ends data pages at boundaries derived from
25+
* the column's values, so that files sharing a run of values share byte-identical pages. Sizes are
26+
* measured on values and levels before encoding.
3127
*
3228
* <p>The settings and defaults match {@code CdcOptions} in Arrow C++ and arrow-rs, and so do the
33-
* chunk boundaries for the same physical values. Arrow types converted before writing, such as
34-
* narrow integers, coerced timestamps and decimals, are hashed differently there. A ParquetWriter
35-
* continues chunk boundaries across row groups, as arrow-rs does; Arrow C++ restarts them in every
36-
* row group, so its chunks match only in the first.
29+
* chunk boundaries for the same physical values.
3730
*
3831
* @see ParquetProperties.Builder#withContentDefinedChunking(CdcOptions)
3932
*/
@@ -50,8 +43,7 @@ private CdcOptions(Builder builder) {
5043
this.minChunkSize = builder.minChunkSize;
5144
this.maxChunkSize = builder.maxChunkSize;
5245
this.normLevel = builder.normLevel;
53-
// Validate now rather than when the first column writer is built.
54-
RollingHashMask.calculate(minChunkSize, maxChunkSize, normLevel);
46+
ContentDefinedChunker.calculateMask(minChunkSize, maxChunkSize, normLevel);
5547
}
5648

5749
/**
@@ -97,45 +89,30 @@ public static class Builder {
9789
private Builder() {}
9890

9991
/**
100-
* Set the minimum chunk size in bytes, 256 KiB by default. The rolling hash is not updated
101-
* until a chunk reaches this size, so no chunk is shorter but a file's last; pages can be, where
102-
* a row group or a page limit ends one.
92+
* Set the minimum chunk size in bytes, 256 KiB by default.
10393
*
10494
* @param minChunkSize the minimum chunk size in bytes
10595
* @return this builder for method chaining
10696
*/
10797
public Builder withMinChunkSize(long minChunkSize) {
108-
Preconditions.checkArgument(
109-
minChunkSize >= 0,
110-
"Invalid content defined chunking minimum chunk size (negative): %s",
111-
minChunkSize);
11298
this.minChunkSize = minChunkSize;
11399
return this;
114100
}
115101

116102
/**
117-
* Set the maximum chunk size in bytes, 1 MiB by default. A chunk ends when it reaches this size,
118-
* whatever the rolling hash says. {@link ParquetProperties.Builder#withPageSize(int)} separately
119-
* limits the page size; below this it splits chunks into more pages, in the same places after
120-
* an edit, so it does not cost deduplication.
103+
* Set the maximum chunk size in bytes, 1 MiB by default.
121104
*
122105
* @param maxChunkSize the maximum chunk size in bytes
123106
* @return this builder for method chaining
124107
*/
125108
public Builder withMaxChunkSize(long maxChunkSize) {
126-
Preconditions.checkArgument(
127-
maxChunkSize > 0,
128-
"Invalid content defined chunking maximum chunk size (not positive): %s",
129-
maxChunkSize);
130109
this.maxChunkSize = maxChunkSize;
131110
return this;
132111
}
133112

134113
/**
135-
* Set the normalization level of the rolling hash mask, 0 by default. Raising it makes a
136-
* boundary more likely, which tightens the chunk size distribution and improves deduplication
137-
* at the cost of more small pages; lowering it does the reverse. Values outside
138-
* {@code [-3, 3]} are not useful.
114+
* Set the normalization level of the rolling hash mask, 0 by default. Raising it makes chunk
115+
* sizes more uniform at the cost of more small pages.
139116
*
140117
* @param normLevel the normalization level
141118
* @return this builder for method chaining
@@ -147,8 +124,8 @@ public Builder withNormLevel(int normLevel) {
147124

148125
/**
149126
* @return the options
150-
* @throws IllegalArgumentException if the maximum chunk size is not greater than the minimum,
151-
* or the envelope is too narrow for the normalization level
127+
* @throws IllegalArgumentException if the minimum chunk size is negative, the maximum is not
128+
* greater than the minimum, or the size range is too narrow for the normalization level
152129
*/
153130
public CdcOptions build() {
154131
return new CdcOptions(this);

0 commit comments

Comments
 (0)