Skip to content

Commit fa30df3

Browse files
committed
[fix](iceberg) Support GZIP for iceberg/hive Parquet writers. (#68854)
Issue Number: N/A Related PR: #64723 Problem Summary: Iceberg tables with `write.parquet.compression-codec = gzip` (the default before Iceberg 1.4) and Hive tables with `parquet.compression = GZIP` could not be written by Doris: the BE Parquet writers only accepted PLAIN/SNAPPY/ZSTD/LZ4 and failed with `[INTERNAL_ERROR]Unsupported compress type GZ with parquet`, while Spark/Trino wrote the same table fine. - BE: map `TFileCompressType.GZ` to `TParquetCompressionType.GZIP`, and move the whole `TFileCompressType` -> `TParquetCompressionType` mapping into `ParquetBuildHelper` so the Iceberg data-file writer, the Iceberg delete-file writer and the Hive Parquet writer share one copy. - FE: accept `gzip` for Parquet in the Hive `CREATE TABLE` compression allow-list. Tests: the Iceberg Parquet-compression suite now writes and reads gzip data and merge-on-read delete files and checks the codec with `parquet_meta()`; the Hive DDL suite adds gzip to the Parquet compression matrix; FE unit tests cover the gzip allow-list and the codec mapping. None - Test <!-- At least one of them must be included. --> - [x] Regression test - [x] Unit Test - [ ] Manual test (add detailed scripts or steps below) - [ ] No need to test or manual test. Explain why: - [ ] This is a refactor/code format and no logic has been changed. - [ ] Previous test can cover this change. - [ ] No code files have been changed. - [ ] Other reason <!-- Add your reason? --> - Behavior changed: - [ ] No. - [x] Yes. <!-- Explain the behavior change --> Iceberg/Hive Parquet writes with GZIP no longer fail; the file is written with the standard Parquet GZIP codec, the same codec Spark/Trino write for that table property. - Does this need documentation? - [x] No. - [ ] Yes. <!-- Add document PR link here. eg: apache/doris-website#1214 --> - [ ] Confirm the release note - [ ] Confirm test cases - [ ] Confirm document - [ ] Add branch pick label <!-- Add branch pick label that this PR should merge into -->
1 parent 542108d commit fa30df3

9 files changed

Lines changed: 179 additions & 77 deletions

File tree

‎be/src/exec/sink/writer/iceberg/viceberg_delete_file_writer.cpp‎

Lines changed: 3 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,7 @@
2323
#include "format/table/iceberg/types.h"
2424
#include "format/transformer/viceberg_parquet_writer.h"
2525
#include "format/transformer/vorc_transformer.h"
26+
#include "format/transformer/vparquet_writer.h"
2627
#include "io/file_factory.h"
2728
#include "runtime/runtime_state.h"
2829

@@ -82,25 +83,8 @@ Status VIcebergDeleteFileWriter::open(RuntimeState* state, RuntimeProfile* profi
8283
// Create file format transformer based on format type
8384
switch (_file_format) {
8485
case TFileFormatType::FORMAT_PARQUET: {
85-
TParquetCompressionType::type parquet_compression_type;
86-
switch (_compress_type) {
87-
case TFileCompressType::PLAIN:
88-
parquet_compression_type = TParquetCompressionType::UNCOMPRESSED;
89-
break;
90-
case TFileCompressType::SNAPPYBLOCK:
91-
parquet_compression_type = TParquetCompressionType::SNAPPY;
92-
break;
93-
case TFileCompressType::ZSTD:
94-
parquet_compression_type = TParquetCompressionType::ZSTD;
95-
break;
96-
case TFileCompressType::LZ4BLOCK:
97-
// Hadoop-framed Parquet LZ4 (not LZ4_RAW) for cross-engine compatibility.
98-
parquet_compression_type = TParquetCompressionType::LZ4_HADOOP;
99-
break;
100-
default:
101-
return Status::InternalError("Unsupported compress type {} with parquet",
102-
to_string(_compress_type));
103-
}
86+
TParquetCompressionType::type parquet_compression_type =
87+
DORIS_TRY(ParquetBuildHelper::to_parquet_compression_type(_compress_type));
10488

10589
ParquetFileOptions parquet_options = {parquet_compression_type,
10690
TParquetVersion::PARQUET_1_0, false, false};

‎be/src/exec/sink/writer/iceberg/viceberg_partition_writer.cpp‎

Lines changed: 8 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,7 @@
2525
#include "format/table/iceberg/schema.h"
2626
#include "format/transformer/viceberg_parquet_writer.h"
2727
#include "format/transformer/vorc_transformer.h"
28+
#include "format/transformer/vparquet_writer.h"
2829
#include "io/file_factory.h"
2930
#include "runtime/runtime_state.h"
3031

@@ -97,34 +98,17 @@ Status VIcebergPartitionWriter::open(RuntimeState* state, RuntimeProfile* profil
9798

9899
switch (_file_format_type) {
99100
case TFileFormatType::FORMAT_PARQUET: {
100-
TParquetCompressionType::type parquet_compression_type;
101-
switch (_compress_type) {
102-
case TFileCompressType::PLAIN: {
103-
parquet_compression_type = TParquetCompressionType::UNCOMPRESSED;
101+
auto parquet_compression_type =
102+
ParquetBuildHelper::to_parquet_compression_type(_compress_type);
103+
if (!parquet_compression_type.has_value()) {
104+
// The target file is already created, so let the shared cleanup below remove it
105+
// instead of returning early.
106+
open_status = parquet_compression_type.error();
104107
break;
105108
}
106-
case TFileCompressType::SNAPPYBLOCK: {
107-
parquet_compression_type = TParquetCompressionType::SNAPPY;
108-
break;
109-
}
110-
case TFileCompressType::ZSTD: {
111-
parquet_compression_type = TParquetCompressionType::ZSTD;
112-
break;
113-
}
114-
case TFileCompressType::LZ4BLOCK: {
115-
// Map Doris LZ4 to the Hadoop-framed Parquet LZ4 codec (not LZ4_RAW) so the file
116-
// stays readable across Spark/Iceberg/Trino. See ParquetBuildHelper.
117-
parquet_compression_type = TParquetCompressionType::LZ4_HADOOP;
118-
break;
119-
}
120-
default: {
121-
return Status::InternalError("Unsupported compress type {} with parquet",
122-
to_string(_compress_type));
123-
}
124-
}
125109
// Parquet 1.0 coerces Arrow nanoseconds to microseconds, including nested fields.
126110
ParquetFileOptions parquet_options = {
127-
.compression_type = parquet_compression_type,
111+
.compression_type = parquet_compression_type.value(),
128112
.parquet_version = contains_nanosecond_timestamp(_schema.root_struct())
129113
? TParquetVersion::PARQUET_2_LATEST
130114
: TParquetVersion::PARQUET_1_0,

‎be/src/exec/sink/writer/vhive_partition_writer.cpp‎

Lines changed: 3 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,7 @@
2424
#include "format/transformer/vcsv_transformer.h"
2525
#include "format/transformer/vhive_parquet_writer.h"
2626
#include "format/transformer/vorc_transformer.h"
27+
#include "format/transformer/vparquet_writer.h"
2728
#include "io/file_factory.h"
2829
#include "io/fs/s3_file_writer.h"
2930
#include "runtime/runtime_state.h"
@@ -73,30 +74,8 @@ Status VHivePartitionWriter::open(RuntimeState* state, RuntimeProfile* operator_
7374

7475
switch (_file_format_type) {
7576
case TFileFormatType::FORMAT_PARQUET: {
76-
TParquetCompressionType::type parquet_compression_type;
77-
switch (_hive_compress_type) {
78-
case TFileCompressType::PLAIN: {
79-
parquet_compression_type = TParquetCompressionType::UNCOMPRESSED;
80-
break;
81-
}
82-
case TFileCompressType::SNAPPYBLOCK: {
83-
parquet_compression_type = TParquetCompressionType::SNAPPY;
84-
break;
85-
}
86-
case TFileCompressType::ZSTD: {
87-
parquet_compression_type = TParquetCompressionType::ZSTD;
88-
break;
89-
}
90-
case TFileCompressType::LZ4BLOCK: {
91-
// Hadoop-framed Parquet LZ4 (not LZ4_RAW) for cross-engine compatibility.
92-
parquet_compression_type = TParquetCompressionType::LZ4_HADOOP;
93-
break;
94-
}
95-
default: {
96-
return Status::InternalError("Unsupported hive compress type {} with parquet",
97-
to_string(_hive_compress_type));
98-
}
99-
}
77+
TParquetCompressionType::type parquet_compression_type =
78+
DORIS_TRY(ParquetBuildHelper::to_parquet_compression_type(_hive_compress_type));
10079
// TODO: INT96 is kept for Hive 2/3 compatibility. Add an explicit option before
10180
// changing the default Hive parquet timestamp encoding to standard logical types.
10281
ParquetFileOptions parquet_options = {parquet_compression_type,

‎be/src/format/transformer/vparquet_writer.cpp‎

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -108,6 +108,25 @@ void ParquetOutputStream::set_written_len(int64_t written_len) {
108108
_written_len = written_len;
109109
}
110110

111+
Result<TParquetCompressionType::type> ParquetBuildHelper::to_parquet_compression_type(
112+
TFileCompressType::type compression_type) {
113+
switch (compression_type) {
114+
case TFileCompressType::PLAIN:
115+
return TParquetCompressionType::UNCOMPRESSED;
116+
case TFileCompressType::SNAPPYBLOCK:
117+
return TParquetCompressionType::SNAPPY;
118+
case TFileCompressType::ZSTD:
119+
return TParquetCompressionType::ZSTD;
120+
case TFileCompressType::GZ:
121+
return TParquetCompressionType::GZIP;
122+
case TFileCompressType::LZ4BLOCK:
123+
return TParquetCompressionType::LZ4_HADOOP;
124+
default:
125+
return ResultError(Status::InternalError("Unsupported compress type {} with parquet",
126+
to_string(compression_type)));
127+
}
128+
}
129+
111130
void ParquetBuildHelper::build_compression_type(
112131
::parquet::WriterProperties::Builder& builder,
113132
const TParquetCompressionType::type& compression_type) {

‎be/src/format/transformer/vparquet_writer.h‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -73,6 +73,17 @@ class ParquetOutputStream : public arrow::io::OutputStream {
7373

7474
class ParquetBuildHelper {
7575
public:
76+
// Maps the Doris file-level compression codec onto the Parquet codec to write with, so the
77+
// Iceberg/Hive writers share one definition instead of repeating the switch:
78+
// PLAIN -> UNCOMPRESSED, SNAPPYBLOCK -> SNAPPY, ZSTD -> ZSTD,
79+
// GZ -> GZIP, LZ4BLOCK -> LZ4_HADOOP.
80+
// GZ is what Iceberg's `write.parquet.compression-codec=gzip` (the table default before
81+
// Iceberg 1.4) and Hive's `parquet.compression=GZIP` resolve to, and GZIP is the codec
82+
// Spark/Trino write for those properties. LZ4_HADOOP is the Hadoop-framed LZ4 codec, which
83+
// keeps the file readable by engines that do not support LZ4_RAW.
84+
static Result<TParquetCompressionType::type> to_parquet_compression_type(
85+
TFileCompressType::type compression_type);
86+
7687
static void build_compression_type(::parquet::WriterProperties::Builder& builder,
7788
const TParquetCompressionType::type& compression_type);
7889

‎fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveUtil.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -68,7 +68,7 @@ public final class HiveUtil {
6868
public static final Set<String> SUPPORTED_ORC_COMPRESSIONS =
6969
ImmutableSet.of("plain", "zlib", "snappy", "zstd", "lz4");
7070
public static final Set<String> SUPPORTED_PARQUET_COMPRESSIONS =
71-
ImmutableSet.of("plain", "snappy", "zstd", "lz4");
71+
ImmutableSet.of("plain", "snappy", "gzip", "zstd", "lz4");
7272
public static final Set<String> SUPPORTED_TEXT_COMPRESSIONS =
7373
ImmutableSet.of("plain", "gzip", "zstd", "bzip2", "lz4", "snappy");
7474

Lines changed: 47 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,47 @@
1+
// Licensed to the Apache Software Foundation (ASF) under one
2+
// or more contributor license agreements. See the NOTICE file
3+
// distributed with this work for additional information
4+
// regarding copyright ownership. The ASF licenses this file
5+
// to you under the Apache License, Version 2.0 (the
6+
// "License"); you may not use this file except in compliance
7+
// with the License. You may obtain a copy of the License at
8+
//
9+
// http://www.apache.org/licenses/LICENSE-2.0
10+
//
11+
// Unless required by applicable law or agreed to in writing,
12+
// software distributed under the License is distributed on an
13+
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
// KIND, either express or implied. See the License for the
15+
// specific language governing permissions and limitations
16+
// under the License.
17+
18+
package org.apache.doris.datasource.hive;
19+
20+
import org.apache.doris.catalog.Column;
21+
import org.apache.doris.catalog.PrimitiveType;
22+
23+
import com.google.common.collect.ImmutableList;
24+
import com.google.common.collect.Maps;
25+
import org.apache.hadoop.hive.metastore.api.Table;
26+
import org.junit.jupiter.api.Assertions;
27+
import org.junit.jupiter.api.Test;
28+
29+
import java.util.Map;
30+
import java.util.Optional;
31+
32+
public class HiveUtilTest {
33+
@Test
34+
public void testExplicitParquetGzipCompression() {
35+
Map<String, String> properties = Maps.newHashMap();
36+
properties.put(HiveUtil.COMPRESSION_KEY, "gzip");
37+
HiveTableMetadata metadata = new HiveTableMetadata(
38+
"db", "table", Optional.empty(),
39+
ImmutableList.of(new Column("id", PrimitiveType.INT)),
40+
ImmutableList.of(), properties, "parquet", "");
41+
42+
Table table = HiveUtil.toHiveTable(metadata);
43+
44+
Assertions.assertEquals("gzip", table.getParameters().get("parquet.compression"));
45+
Assertions.assertFalse(table.getParameters().containsKey(HiveUtil.COMPRESSION_KEY));
46+
}
47+
}

‎regression-test/suites/external_table_p0/hive/ddl/test_hive_ddl.groovy‎

Lines changed: 26 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -419,6 +419,27 @@ suite("test_hive_ddl", "p0,external,hive,external_docker,external_docker_hive")
419419
}
420420
}
421421

422+
// GZIP used to be rejected by the Doris Parquet writer with
423+
// "Unsupported compress type GZ with parquet"; the write must succeed and land on the
424+
// GZIP Parquet codec. Asserted instead of `qt_` because the expectation is one codec name.
425+
if (file_format.equals("parquet") && compression.equals("gzip")) {
426+
sql """ INSERT INTO tbl_${file_format}_${compression} VALUES ('doris_gzip') """
427+
def gzipRows = sql """ SELECT * FROM tbl_${file_format}_${compression} ORDER BY col """
428+
assertEquals(1, gzipRows.size())
429+
assertEquals("doris_gzip", gzipRows[0][0].toString())
430+
String hdfsPort = context.config.otherConfigs.get("hive2HdfsPort")
431+
String externalEnvIp = context.config.otherConfigs.get("externalEnvIp")
432+
def gzipCodec = sql """
433+
SELECT DISTINCT compression
434+
FROM parquet_meta(
435+
"uri" = "hdfs://${externalEnvIp}:${hdfsPort}/user/hive/warehouse/test_hive_compress.db/tbl_parquet_gzip/*",
436+
"hadoop.username" = "doris",
437+
"mode" = "parquet_metadata"
438+
)
439+
"""
440+
assertEquals("GZIP", gzipCodec[0][0].toString().toUpperCase())
441+
}
442+
422443
sql """DROP TABLE `tbl_${file_format}_${compression}`"""
423444
sql """ drop database if exists `test_hive_compress` """;
424445
}
@@ -774,7 +795,7 @@ suite("test_hive_ddl", "p0,external,hive,external_docker,external_docker_hive")
774795
sql """set enable_fallback_to_original_planner=false;"""
775796
test_db(catalog_name)
776797
test_loc_db(externalEnvIp, hdfs_port, catalog_name)
777-
def compressions = ["snappy", "zlib", "zstd", "lz4"]
798+
def compressions = ["snappy", "zlib", "zstd", "lz4", "gzip"]
778799
for (String file_format in file_formats) {
779800
logger.info("Process file format " + file_format)
780801
test_loc_tbl(file_format, externalEnvIp, hdfs_port, catalog_name)
@@ -785,6 +806,10 @@ suite("test_hive_ddl", "p0,external,hive,external_docker,external_docker_hive")
785806
if (file_format.equals("parquet") && compression.equals("zlib")) {
786807
continue
787808
}
809+
// ORC has no GZIP codec (it spells the same codec `zlib`), so only test it for Parquet.
810+
if (file_format.equals("orc") && compression.equals("gzip")) {
811+
continue
812+
}
788813
logger.info("Process file compression " + compression)
789814
test_tbl_compress(compression, file_format, catalog_name)
790815
}

‎regression-test/suites/external_table_p0/iceberg/write/test_iceberg_write_parquet_compression.groovy‎

Lines changed: 61 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,8 @@
1616
// under the License.
1717

1818
// Verifies that Doris writes Iceberg Parquet LZ4 as Hadoop-framed Parquet "LZ4"
19-
// (not LZ4_RAW) and that Iceberg ORC/delete-file LZ4 paths are reachable.
19+
// (not LZ4_RAW), that Iceberg ORC/delete-file LZ4 paths are reachable, and that
20+
// `write.parquet.compression-codec=gzip` is honoured by the Parquet data/delete writers.
2021
suite("test_iceberg_write_parquet_compression", "p0,external") {
2122
String enabled = context.config.otherConfigs.get("enableIcebergTest")
2223
if (enabled == null || !enabled.equalsIgnoreCase("true")) {
@@ -137,9 +138,10 @@ suite("test_iceberg_write_parquet_compression", "p0,external") {
137138
ORDER BY compression
138139
"""
139140

140-
// GZIP is intentionally not yet supported by the Doris Parquet writer; the INSERT must
141-
// fail explicitly rather than silently fall back. Remove this block when GZIP support is
142-
// added (and update the cross-engine product test expectation in tandem).
141+
// GZIP is a standard Iceberg Parquet codec (the table default before Iceberg 1.4) and used to
142+
// be rejected by the Doris Parquet writer with "Unsupported compress type GZ with parquet".
143+
// The assertions below are intentionally not `qt_` cases: the expected result is a single
144+
// codec name, so an explicit check keeps the intent obvious.
143145
sql """ drop table if exists ${db}.tbl_gzip """
144146
sql """
145147
CREATE TABLE ${db}.tbl_gzip (a STRING, b BIGINT) PROPERTIES (
@@ -148,8 +150,59 @@ suite("test_iceberg_write_parquet_compression", "p0,external") {
148150
'format-version' = '2',
149151
'write.parquet.compression-codec' = 'gzip'
150152
)"""
151-
test {
152-
sql """ INSERT INTO ${db}.tbl_gzip VALUES ('doris0', 0) """
153-
exception "Unsupported compress type GZ with parquet"
154-
}
153+
sql """ INSERT INTO ${db}.tbl_gzip VALUES ('doris0', 0), ('doris1', 1) """
154+
def gzipRows = sql """ SELECT a, b FROM ${db}.tbl_gzip ORDER BY b """
155+
assertEquals(2, gzipRows.size())
156+
assertEquals("doris1", gzipRows[1][0].toString())
157+
def gzipDataFiles = sql """ SELECT file_path FROM ${db}.tbl_gzip\$files ORDER BY file_path """
158+
String gzipDataFile = gzipDataFiles[0][0].toString()
159+
def gzipDataCodec = sql """
160+
SELECT DISTINCT compression
161+
FROM parquet_meta(
162+
"uri" = "${gzipDataFile}",
163+
"s3.access_key" = "admin",
164+
"s3.secret_key" = "password",
165+
"s3.endpoint" = "http://${externalEnvIp}:${minio_port}",
166+
"s3.region" = "us-east-1",
167+
"mode" = "parquet_metadata"
168+
)
169+
"""
170+
assertEquals("GZIP", gzipDataCodec[0][0].toString().toUpperCase())
171+
172+
// The delete-file writer must honour the same table property, otherwise DELETE/merge-on-read
173+
// on a gzip table fails where INSERT succeeds.
174+
sql """ drop table if exists ${db}.tbl_gzip_delete """
175+
sql """
176+
CREATE TABLE ${db}.tbl_gzip_delete (id INT, name STRING) PROPERTIES (
177+
'write-format' = 'parquet',
178+
'write.format.default' = 'parquet',
179+
'format-version' = '2',
180+
'write.parquet.compression-codec' = 'gzip',
181+
'write.delete.mode' = 'merge-on-read',
182+
'write.update.mode' = 'merge-on-read',
183+
'write.merge.mode' = 'merge-on-read'
184+
)"""
185+
sql """ INSERT INTO ${db}.tbl_gzip_delete VALUES (1, 'a'), (2, 'b'), (3, 'c') """
186+
sql """ DELETE FROM ${db}.tbl_gzip_delete WHERE id = 2 """
187+
def gzipDeleteRows = sql """ SELECT id, name FROM ${db}.tbl_gzip_delete ORDER BY id """
188+
assertEquals(2, gzipDeleteRows.size())
189+
assertEquals("3", gzipDeleteRows[1][0].toString())
190+
def gzipDeleteFiles = sql """
191+
SELECT file_path
192+
FROM ${db}.tbl_gzip_delete\$delete_files
193+
ORDER BY file_path
194+
"""
195+
String gzipDeleteFile = gzipDeleteFiles[0][0].toString()
196+
def gzipDeleteCodec = sql """
197+
SELECT DISTINCT compression
198+
FROM parquet_meta(
199+
"uri" = "${gzipDeleteFile}",
200+
"s3.access_key" = "admin",
201+
"s3.secret_key" = "password",
202+
"s3.endpoint" = "http://${externalEnvIp}:${minio_port}",
203+
"s3.region" = "us-east-1",
204+
"mode" = "parquet_metadata"
205+
)
206+
"""
207+
assertEquals("GZIP", gzipDeleteCodec[0][0].toString().toUpperCase())
155208
}

0 commit comments

Comments
 (0)