Skip to content

Commit 97a880c

Browse files
author
Zoltan Ivanfi
committed
Experiment.
1 parent a7b5a4b commit 97a880c

4 files changed

Lines changed: 171 additions & 10 deletions

File tree

‎parquet-benchmarks/run.sh‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,9 @@
2222
SCRIPT_PATH=$( cd "$(dirname "$0")" ; pwd -P )
2323

2424
echo "Starting WRITE benchmarks"
25-
java -jar ${SCRIPT_PATH}/target/parquet-benchmarks.jar p*Write* "$@"
25+
java -XX:+PreserveFramePointer -jar ${SCRIPT_PATH}/target/parquet-benchmarks.jar p*Write* -wi 0 -i 1 -f 3 -tu s -bm ss -rf json
26+
exit 0
27+
2628
echo "Generating test data"
2729
java -cp ${SCRIPT_PATH}/target/parquet-benchmarks.jar org.apache.parquet.benchmarks.DataGenerator generate
2830
echo "Data generated, starting READ benchmarks"

‎parquet-benchmarks/src/main/java/org/apache/parquet/benchmarks/WriteBenchmarks.java‎

Lines changed: 8 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -55,7 +55,7 @@ public void write1MRowsDefaultBlockAndPageSizeUncompressed()
5555
PAGE_SIZE_DEFAULT,
5656
FIXED_LEN_BYTEARRAY_SIZE,
5757
UNCOMPRESSED,
58-
ONE_MILLION);
58+
50 * ONE_MILLION);
5959
}
6060

6161
@Benchmark
@@ -69,7 +69,7 @@ public void write1MRowsBS256MPS4MUncompressed()
6969
PAGE_SIZE_4M,
7070
FIXED_LEN_BYTEARRAY_SIZE,
7171
UNCOMPRESSED,
72-
ONE_MILLION);
72+
50 * ONE_MILLION);
7373
}
7474

7575
@Benchmark
@@ -83,7 +83,7 @@ public void write1MRowsBS256MPS8MUncompressed()
8383
PAGE_SIZE_8M,
8484
FIXED_LEN_BYTEARRAY_SIZE,
8585
UNCOMPRESSED,
86-
ONE_MILLION);
86+
50 * ONE_MILLION);
8787
}
8888

8989
@Benchmark
@@ -97,7 +97,7 @@ public void write1MRowsBS512MPS4MUncompressed()
9797
PAGE_SIZE_4M,
9898
FIXED_LEN_BYTEARRAY_SIZE,
9999
UNCOMPRESSED,
100-
ONE_MILLION);
100+
50 * ONE_MILLION);
101101
}
102102

103103
@Benchmark
@@ -111,7 +111,7 @@ public void write1MRowsBS512MPS8MUncompressed()
111111
PAGE_SIZE_8M,
112112
FIXED_LEN_BYTEARRAY_SIZE,
113113
UNCOMPRESSED,
114-
ONE_MILLION);
114+
50 * ONE_MILLION);
115115
}
116116

117117
//TODO how to handle lzo jar?
@@ -126,7 +126,7 @@ public void write1MRowsBS512MPS8MUncompressed()
126126
// PAGE_SIZE_DEFAULT,
127127
// FIXED_LEN_BYTEARRAY_SIZE,
128128
// LZO,
129-
// ONE_MILLION);
129+
// 50 * ONE_MILLION);
130130
// }
131131

132132
@Benchmark
@@ -140,7 +140,7 @@ public void write1MRowsDefaultBlockAndPageSizeSNAPPY()
140140
PAGE_SIZE_DEFAULT,
141141
FIXED_LEN_BYTEARRAY_SIZE,
142142
SNAPPY,
143-
ONE_MILLION);
143+
50 * ONE_MILLION);
144144
}
145145

146146
@Benchmark
@@ -154,6 +154,6 @@ public void write1MRowsDefaultBlockAndPageSizeGZIP()
154154
PAGE_SIZE_DEFAULT,
155155
FIXED_LEN_BYTEARRAY_SIZE,
156156
GZIP,
157-
ONE_MILLION);
157+
50 * ONE_MILLION);
158158
}
159159
}
Lines changed: 159 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,159 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one
3+
* or more contributor license agreements. See the NOTICE file
4+
* distributed with this work for additional information
5+
* regarding copyright ownership. The ASF licenses this file
6+
* to you under the Apache License, Version 2.0 (the
7+
* "License"); you may not use this file except in compliance
8+
* with the License. You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing,
13+
* software distributed under the License is distributed on an
14+
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15+
* KIND, either express or implied. See the License for the
16+
* specific language governing permissions and limitations
17+
* under the License.
18+
*/
19+
package org.apache.parquet.benchmarks;
20+
21+
import org.openjdk.jmh.annotations.Benchmark;
22+
import org.openjdk.jmh.annotations.Level;
23+
import org.openjdk.jmh.annotations.Setup;
24+
import org.openjdk.jmh.annotations.State;
25+
26+
import static org.openjdk.jmh.annotations.Scope.Thread;
27+
import static org.apache.parquet.benchmarks.BenchmarkConstants.*;
28+
import static org.apache.parquet.benchmarks.BenchmarkFiles.*;
29+
30+
import java.io.IOException;
31+
32+
import static org.apache.parquet.column.ParquetProperties.WriterVersion.PARQUET_1_0;
33+
import static org.apache.parquet.hadoop.metadata.CompressionCodecName.GZIP;
34+
import static org.apache.parquet.hadoop.metadata.CompressionCodecName.SNAPPY;
35+
import static org.apache.parquet.hadoop.metadata.CompressionCodecName.UNCOMPRESSED;
36+
37+
@State(Thread)
38+
public class WriteBenchmarksParquet1 {
39+
private DataGenerator dataGenerator = new DataGenerator();
40+
41+
@Setup(Level.Iteration)
42+
public void cleanup() {
43+
//clean existing test data at the beginning of each iteration
44+
dataGenerator.cleanup();
45+
}
46+
47+
@Benchmark
48+
public void write1MRowsDefaultBlockAndPageSizeUncompressed()
49+
throws IOException
50+
{
51+
dataGenerator.generateData(file_1M,
52+
configuration,
53+
PARQUET_1_0,
54+
BLOCK_SIZE_DEFAULT,
55+
PAGE_SIZE_DEFAULT,
56+
FIXED_LEN_BYTEARRAY_SIZE,
57+
UNCOMPRESSED,
58+
50 * ONE_MILLION);
59+
}
60+
61+
@Benchmark
62+
public void write1MRowsBS256MPS4MUncompressed()
63+
throws IOException
64+
{
65+
dataGenerator.generateData(file_1M_BS256M_PS4M,
66+
configuration,
67+
PARQUET_1_0,
68+
BLOCK_SIZE_256M,
69+
PAGE_SIZE_4M,
70+
FIXED_LEN_BYTEARRAY_SIZE,
71+
UNCOMPRESSED,
72+
50 * ONE_MILLION);
73+
}
74+
75+
@Benchmark
76+
public void write1MRowsBS256MPS8MUncompressed()
77+
throws IOException
78+
{
79+
dataGenerator.generateData(file_1M_BS256M_PS8M,
80+
configuration,
81+
PARQUET_1_0,
82+
BLOCK_SIZE_256M,
83+
PAGE_SIZE_8M,
84+
FIXED_LEN_BYTEARRAY_SIZE,
85+
UNCOMPRESSED,
86+
50 * ONE_MILLION);
87+
}
88+
89+
@Benchmark
90+
public void write1MRowsBS512MPS4MUncompressed()
91+
throws IOException
92+
{
93+
dataGenerator.generateData(file_1M_BS512M_PS4M,
94+
configuration,
95+
PARQUET_1_0,
96+
BLOCK_SIZE_512M,
97+
PAGE_SIZE_4M,
98+
FIXED_LEN_BYTEARRAY_SIZE,
99+
UNCOMPRESSED,
100+
50 * ONE_MILLION);
101+
}
102+
103+
@Benchmark
104+
public void write1MRowsBS512MPS8MUncompressed()
105+
throws IOException
106+
{
107+
dataGenerator.generateData(file_1M_BS512M_PS8M,
108+
configuration,
109+
PARQUET_1_0,
110+
BLOCK_SIZE_512M,
111+
PAGE_SIZE_8M,
112+
FIXED_LEN_BYTEARRAY_SIZE,
113+
UNCOMPRESSED,
114+
50 * ONE_MILLION);
115+
}
116+
117+
//TODO how to handle lzo jar?
118+
// @Benchmark
119+
// public void write1MRowsDefaultBlockAndPageSizeLZO()
120+
// throws IOException
121+
// {
122+
// dataGenerator.generateData(parquetFile_1M_LZO,
123+
// configuration,
124+
// WriterVersion.PARQUET_1_0,
125+
// BLOCK_SIZE_DEFAULT,
126+
// PAGE_SIZE_DEFAULT,
127+
// FIXED_LEN_BYTEARRAY_SIZE,
128+
// LZO,
129+
// 50 * ONE_MILLION);
130+
// }
131+
132+
@Benchmark
133+
public void write1MRowsDefaultBlockAndPageSizeSNAPPY()
134+
throws IOException
135+
{
136+
dataGenerator.generateData(file_1M_SNAPPY,
137+
configuration,
138+
PARQUET_1_0,
139+
BLOCK_SIZE_DEFAULT,
140+
PAGE_SIZE_DEFAULT,
141+
FIXED_LEN_BYTEARRAY_SIZE,
142+
SNAPPY,
143+
50 * ONE_MILLION);
144+
}
145+
146+
@Benchmark
147+
public void write1MRowsDefaultBlockAndPageSizeGZIP()
148+
throws IOException
149+
{
150+
dataGenerator.generateData(file_1M_GZIP,
151+
configuration,
152+
PARQUET_1_0,
153+
BLOCK_SIZE_DEFAULT,
154+
PAGE_SIZE_DEFAULT,
155+
FIXED_LEN_BYTEARRAY_SIZE,
156+
GZIP,
157+
50 * ONE_MILLION);
158+
}
159+
}

‎parquet-hadoop/src/main/java/org/apache/parquet/hadoop/InternalParquetRecordWriter.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -43,7 +43,7 @@ class InternalParquetRecordWriter<T> {
4343
private static final Logger LOG = LoggerFactory.getLogger(InternalParquetRecordWriter.class);
4444

4545
private static final int MINIMUM_RECORD_COUNT_FOR_CHECK = 100;
46-
private static final int MAXIMUM_RECORD_COUNT_FOR_CHECK = 10000;
46+
private static final int MAXIMUM_RECORD_COUNT_FOR_CHECK = 100;
4747

4848
private final ParquetFileWriter parquetFileWriter;
4949
private final WriteSupport<T> writeSupport;

0 commit comments

Comments
 (0)