Skip to content

Commit 500cad6

Browse files
committed
[fix](iceberg) Support GZIP for iceberg/hive Parquet writers.
The BE Parquet writers only accepted PLAIN/SNAPPY/ZSTD/LZ4 and rejected the GZ codec that Iceberg's write.parquet.compression-codec=gzip (and Hive's parquet.compression=GZIP) map to, so INSERT/DELETE failed with 'Unsupported compress type GZ with parquet' while Spark/Trino wrote the same table fine. Factor the whole TFileCompressType -> TParquetCompressionType mapping into ParquetBuildHelper (GZ -> GZIP included) so the Iceberg data-file writer, the Iceberg delete-file writer and the Hive Parquet writer share one definition instead of three copies of the switch, and accept gzip in the Hive CREATE TABLE parquet compression allow-list.
1 parent e9165fa commit 500cad6

10 files changed

Lines changed: 147 additions & 81 deletions

File tree

‎be/src/exec/sink/writer/iceberg/viceberg_delete_file_writer.cpp‎

Lines changed: 3 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,7 @@
2323
#include "format/table/iceberg/types.h"
2424
#include "format/transformer/viceberg_parquet_writer.h"
2525
#include "format/transformer/vorc_transformer.h"
26+
#include "format/transformer/vparquet_writer.h"
2627
#include "io/file_factory.h"
2728
#include "runtime/runtime_state.h"
2829

@@ -82,25 +83,8 @@ Status VIcebergDeleteFileWriter::open(RuntimeState* state, RuntimeProfile* profi
8283
// Create file format transformer based on format type
8384
switch (_file_format) {
8485
case TFileFormatType::FORMAT_PARQUET: {
85-
TParquetCompressionType::type parquet_compression_type;
86-
switch (_compress_type) {
87-
case TFileCompressType::PLAIN:
88-
parquet_compression_type = TParquetCompressionType::UNCOMPRESSED;
89-
break;
90-
case TFileCompressType::SNAPPYBLOCK:
91-
parquet_compression_type = TParquetCompressionType::SNAPPY;
92-
break;
93-
case TFileCompressType::ZSTD:
94-
parquet_compression_type = TParquetCompressionType::ZSTD;
95-
break;
96-
case TFileCompressType::LZ4BLOCK:
97-
// Hadoop-framed Parquet LZ4 (not LZ4_RAW) for cross-engine compatibility.
98-
parquet_compression_type = TParquetCompressionType::LZ4_HADOOP;
99-
break;
100-
default:
101-
return Status::InternalError("Unsupported compress type {} with parquet",
102-
to_string(_compress_type));
103-
}
86+
TParquetCompressionType::type parquet_compression_type =
87+
DORIS_TRY(ParquetBuildHelper::to_parquet_compression_type(_compress_type));
10488

10589
ParquetFileOptions parquet_options = {parquet_compression_type,
10690
TParquetVersion::PARQUET_1_0, false, false};

‎be/src/exec/sink/writer/iceberg/viceberg_partition_writer.cpp‎

Lines changed: 8 additions & 28 deletions
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,7 @@
2525
#include "format/table/iceberg/schema.h"
2626
#include "format/transformer/viceberg_parquet_writer.h"
2727
#include "format/transformer/vorc_transformer.h"
28+
#include "format/transformer/vparquet_writer.h"
2829
#include "io/file_factory.h"
2930
#include "runtime/runtime_state.h"
3031

@@ -75,36 +76,15 @@ Status VIcebergPartitionWriter::open(RuntimeState* state, RuntimeProfile* profil
7576
Status open_status;
7677
switch (_file_format_type) {
7778
case TFileFormatType::FORMAT_PARQUET: {
78-
TParquetCompressionType::type parquet_compression_type;
79-
switch (_compress_type) {
80-
case TFileCompressType::PLAIN: {
81-
parquet_compression_type = TParquetCompressionType::UNCOMPRESSED;
79+
auto parquet_compression_type =
80+
ParquetBuildHelper::to_parquet_compression_type(_compress_type);
81+
if (!parquet_compression_type.has_value()) {
82+
// The target file is already created, so let the shared cleanup below remove it
83+
// instead of returning early.
84+
open_status = parquet_compression_type.error();
8285
break;
8386
}
84-
case TFileCompressType::SNAPPYBLOCK: {
85-
parquet_compression_type = TParquetCompressionType::SNAPPY;
86-
break;
87-
}
88-
case TFileCompressType::ZSTD: {
89-
parquet_compression_type = TParquetCompressionType::ZSTD;
90-
break;
91-
}
92-
case TFileCompressType::LZ4BLOCK: {
93-
// Map Doris LZ4 to the Hadoop-framed Parquet LZ4 codec (not LZ4_RAW) so the file
94-
// stays readable across Spark/Iceberg/Trino. See ParquetBuildHelper.
95-
parquet_compression_type = TParquetCompressionType::LZ4_HADOOP;
96-
break;
97-
}
98-
default: {
99-
open_status = Status::InternalError("Unsupported compress type {} with parquet",
100-
to_string(_compress_type));
101-
break;
102-
}
103-
}
104-
if (!open_status.ok()) {
105-
break;
106-
}
107-
ParquetFileOptions parquet_options = {.compression_type = parquet_compression_type,
87+
ParquetFileOptions parquet_options = {.compression_type = parquet_compression_type.value(),
10888
.parquet_version = TParquetVersion::PARQUET_1_0,
10989
.parquet_disable_dictionary = false,
11090
.enable_int96_timestamps = false};

‎be/src/exec/sink/writer/vhive_partition_writer.cpp‎

Lines changed: 3 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,7 @@
2525
#include "format/transformer/vcsv_transformer.h"
2626
#include "format/transformer/vhive_parquet_writer.h"
2727
#include "format/transformer/vorc_transformer.h"
28+
#include "format/transformer/vparquet_writer.h"
2829
#include "io/file_factory.h"
2930
#include "io/fs/s3_file_system.h"
3031
#include "io/fs/s3_file_writer.h"
@@ -88,30 +89,8 @@ Status VHivePartitionWriter::open(RuntimeState* state, RuntimeProfile* operator_
8889

8990
switch (_file_format_type) {
9091
case TFileFormatType::FORMAT_PARQUET: {
91-
TParquetCompressionType::type parquet_compression_type;
92-
switch (_hive_compress_type) {
93-
case TFileCompressType::PLAIN: {
94-
parquet_compression_type = TParquetCompressionType::UNCOMPRESSED;
95-
break;
96-
}
97-
case TFileCompressType::SNAPPYBLOCK: {
98-
parquet_compression_type = TParquetCompressionType::SNAPPY;
99-
break;
100-
}
101-
case TFileCompressType::ZSTD: {
102-
parquet_compression_type = TParquetCompressionType::ZSTD;
103-
break;
104-
}
105-
case TFileCompressType::LZ4BLOCK: {
106-
// Hadoop-framed Parquet LZ4 (not LZ4_RAW) for cross-engine compatibility.
107-
parquet_compression_type = TParquetCompressionType::LZ4_HADOOP;
108-
break;
109-
}
110-
default: {
111-
return Status::InternalError("Unsupported hive compress type {} with parquet",
112-
to_string(_hive_compress_type));
113-
}
114-
}
92+
TParquetCompressionType::type parquet_compression_type =
93+
DORIS_TRY(ParquetBuildHelper::to_parquet_compression_type(_hive_compress_type));
11594
// TODO: INT96 is kept for Hive 2/3 compatibility. Add an explicit option before
11695
// changing the default Hive parquet timestamp encoding to standard logical types.
11796
ParquetFileOptions parquet_options = {parquet_compression_type,

‎be/src/format/transformer/vparquet_writer.cpp‎

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -97,6 +97,25 @@ void ParquetOutputStream::set_written_len(int64_t written_len) {
9797
_written_len = written_len;
9898
}
9999

100+
Result<TParquetCompressionType::type> ParquetBuildHelper::to_parquet_compression_type(
101+
TFileCompressType::type compression_type) {
102+
switch (compression_type) {
103+
case TFileCompressType::PLAIN:
104+
return TParquetCompressionType::UNCOMPRESSED;
105+
case TFileCompressType::SNAPPYBLOCK:
106+
return TParquetCompressionType::SNAPPY;
107+
case TFileCompressType::ZSTD:
108+
return TParquetCompressionType::ZSTD;
109+
case TFileCompressType::GZ:
110+
return TParquetCompressionType::GZIP;
111+
case TFileCompressType::LZ4BLOCK:
112+
return TParquetCompressionType::LZ4_HADOOP;
113+
default:
114+
return ResultError(Status::InternalError("Unsupported compress type {} with parquet",
115+
to_string(compression_type)));
116+
}
117+
}
118+
100119
void ParquetBuildHelper::build_compression_type(
101120
::parquet::WriterProperties::Builder& builder,
102121
const TParquetCompressionType::type& compression_type) {

‎be/src/format/transformer/vparquet_writer.h‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -72,6 +72,17 @@ class ParquetOutputStream : public arrow::io::OutputStream {
7272

7373
class ParquetBuildHelper {
7474
public:
75+
// Maps the Doris file-level compression codec onto the Parquet codec to write with, so the
76+
// Iceberg/Hive writers share one definition instead of repeating the switch:
77+
// PLAIN -> UNCOMPRESSED, SNAPPYBLOCK -> SNAPPY, ZSTD -> ZSTD,
78+
// GZ -> GZIP, LZ4BLOCK -> LZ4_HADOOP.
79+
// GZ is what Iceberg's `write.parquet.compression-codec=gzip` (the table default before
80+
// Iceberg 1.4) and Hive's `parquet.compression=GZIP` resolve to, and GZIP is the codec
81+
// Spark/Trino write for those properties. LZ4_HADOOP is the Hadoop-framed LZ4 codec, which
82+
// keeps the file readable by engines that do not support LZ4_RAW.
83+
static Result<TParquetCompressionType::type> to_parquet_compression_type(
84+
TFileCompressType::type compression_type);
85+
7586
static void build_compression_type(::parquet::WriterProperties::Builder& builder,
7687
const TParquetCompressionType::type& compression_type);
7788

‎fe/fe-connector/fe-connector-hive/src/test/java/org/apache/doris/connector/hive/HiveWritePlanProviderTest.java‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -388,6 +388,10 @@ public void planWriteCompressionPerFormat() {
388388
Assertions.assertEquals(TFileCompressType.SNAPPYBLOCK,
389389
compressionFor(PARQUET_INPUT_FORMAT, Collections.singletonMap("parquet.compression", "snappy"),
390390
Collections.emptyMap()));
391+
// GZIP is a valid parquet.compression value; BE maps GZ to the parquet GZIP codec.
392+
Assertions.assertEquals(TFileCompressType.GZ,
393+
compressionFor(PARQUET_INPUT_FORMAT, Collections.singletonMap("parquet.compression", "gzip"),
394+
Collections.emptyMap()));
391395
Assertions.assertEquals(TFileCompressType.GZ,
392396
compressionFor(TEXT_INPUT_FORMAT, Collections.singletonMap("text.compression", "gzip"),
393397
Collections.emptyMap()));

‎fe/fe-connector/fe-connector-hms/src/main/java/org/apache/doris/connector/hms/HmsWriteConverter.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -62,7 +62,7 @@ public final class HmsWriteConverter {
6262
private static final Set<String> SUPPORTED_ORC_COMPRESSIONS =
6363
unmodifiableSet("plain", "zlib", "snappy", "zstd", "lz4");
6464
private static final Set<String> SUPPORTED_PARQUET_COMPRESSIONS =
65-
unmodifiableSet("plain", "snappy", "zstd", "lz4");
65+
unmodifiableSet("plain", "snappy", "gzip", "zstd", "lz4");
6666
private static final Set<String> SUPPORTED_TEXT_COMPRESSIONS =
6767
unmodifiableSet("plain", "gzip", "zstd", "bzip2", "lz4", "snappy");
6868

‎fe/fe-connector/fe-connector-hms/src/test/java/org/apache/doris/connector/hms/HmsWriteConverterTest.java‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -144,6 +144,17 @@ public void testExplicitCompressionHonoredAndKeyRemoved() {
144144
Assertions.assertFalse(table.getParameters().containsKey("compression"));
145145
}
146146

147+
@Test
148+
public void testExplicitParquetGzipCompression() {
149+
// GZIP is a valid Hive `parquet.compression` value, and the Doris Parquet writer now
150+
// honours it, so the CREATE TABLE allow-list must accept it instead of rejecting it.
151+
Table table = HmsWriteConverter.toHiveTable(
152+
baseTable("parquet", Collections.singletonList(col("id", "INT")),
153+
Collections.emptyList()).properties(mutableMap("compression", "gzip")).build());
154+
Assertions.assertEquals("gzip", table.getParameters().get("parquet.compression"));
155+
Assertions.assertFalse(table.getParameters().containsKey("compression"));
156+
}
157+
147158
@Test
148159
public void testUnsupportedCompressionAndFormatThrow() {
149160
Assertions.assertThrows(IllegalArgumentException.class, () ->

‎regression-test/suites/external_table_p0/hive/ddl/test_hive_ddl.groovy‎

Lines changed: 26 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -419,6 +419,27 @@ suite("test_hive_ddl", "p0,external") {
419419
}
420420
}
421421

422+
// GZIP used to be rejected by the Doris Parquet writer with
423+
// "Unsupported compress type GZ with parquet"; the write must succeed and land on the
424+
// GZIP Parquet codec. Asserted instead of `qt_` because the expectation is one codec name.
425+
if (file_format.equals("parquet") && compression.equals("gzip")) {
426+
sql """ INSERT INTO tbl_${file_format}_${compression} VALUES ('doris_gzip') """
427+
def gzipRows = sql """ SELECT * FROM tbl_${file_format}_${compression} ORDER BY col """
428+
assertEquals(1, gzipRows.size())
429+
assertEquals("doris_gzip", gzipRows[0][0].toString())
430+
String hdfsPort = context.config.otherConfigs.get("hive2HdfsPort")
431+
String externalEnvIp = context.config.otherConfigs.get("externalEnvIp")
432+
def gzipCodec = sql """
433+
SELECT DISTINCT compression
434+
FROM parquet_meta(
435+
"uri" = "hdfs://${externalEnvIp}:${hdfsPort}/user/hive/warehouse/test_hive_compress.db/tbl_parquet_gzip/*",
436+
"hadoop.username" = "doris",
437+
"mode" = "parquet_metadata"
438+
)
439+
"""
440+
assertEquals("GZIP", gzipCodec[0][0].toString().toUpperCase())
441+
}
442+
422443
sql """DROP TABLE `tbl_${file_format}_${compression}`"""
423444
sql """ drop database if exists `test_hive_compress` """;
424445
}
@@ -754,7 +775,7 @@ suite("test_hive_ddl", "p0,external") {
754775
sql """set enable_fallback_to_original_planner=false;"""
755776
test_db(catalog_name)
756777
test_loc_db(externalEnvIp, hdfs_port, catalog_name)
757-
def compressions = ["snappy", "zlib", "zstd", "lz4"]
778+
def compressions = ["snappy", "zlib", "zstd", "lz4", "gzip"]
758779
for (String file_format in file_formats) {
759780
logger.info("Process file format " + file_format)
760781
test_loc_tbl(file_format, externalEnvIp, hdfs_port, catalog_name)
@@ -765,6 +786,10 @@ suite("test_hive_ddl", "p0,external") {
765786
if (file_format.equals("parquet") && compression.equals("zlib")) {
766787
continue
767788
}
789+
// ORC has no GZIP codec (it spells the same codec `zlib`), so only test it for Parquet.
790+
if (file_format.equals("orc") && compression.equals("gzip")) {
791+
continue
792+
}
768793
logger.info("Process file compression " + compression)
769794
test_tbl_compress(compression, file_format, catalog_name)
770795
}

‎regression-test/suites/external_table_p0/iceberg/write/test_iceberg_write_parquet_compression.groovy‎

Lines changed: 61 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,8 @@
1616
// under the License.
1717

1818
// Verifies that Doris writes Iceberg Parquet LZ4 as Hadoop-framed Parquet "LZ4"
19-
// (not LZ4_RAW) and that Iceberg ORC/delete-file LZ4 paths are reachable.
19+
// (not LZ4_RAW), that Iceberg ORC/delete-file LZ4 paths are reachable, and that
20+
// `write.parquet.compression-codec=gzip` is honoured by the Parquet data/delete writers.
2021
suite("test_iceberg_write_parquet_compression", "p0,external") {
2122
String enabled = context.config.otherConfigs.get("enableIcebergTest")
2223
if (enabled == null || !enabled.equalsIgnoreCase("true")) {
@@ -137,9 +138,10 @@ suite("test_iceberg_write_parquet_compression", "p0,external") {
137138
ORDER BY compression
138139
"""
139140

140-
// GZIP is intentionally not yet supported by the Doris Parquet writer; the INSERT must
141-
// fail explicitly rather than silently fall back. Remove this block when GZIP support is
142-
// added (and update the cross-engine product test expectation in tandem).
141+
// GZIP is a standard Iceberg Parquet codec (the table default before Iceberg 1.4) and used to
142+
// be rejected by the Doris Parquet writer with "Unsupported compress type GZ with parquet".
143+
// The assertions below are intentionally not `qt_` cases: the expected result is a single
144+
// codec name, so an explicit check keeps the intent obvious.
143145
sql """ drop table if exists ${db}.tbl_gzip """
144146
sql """
145147
CREATE TABLE ${db}.tbl_gzip (a STRING, b BIGINT) PROPERTIES (
@@ -148,8 +150,59 @@ suite("test_iceberg_write_parquet_compression", "p0,external") {
148150
'format-version' = '2',
149151
'write.parquet.compression-codec' = 'gzip'
150152
)"""
151-
test {
152-
sql """ INSERT INTO ${db}.tbl_gzip VALUES ('doris0', 0) """
153-
exception "Unsupported compress type GZ with parquet"
154-
}
153+
sql """ INSERT INTO ${db}.tbl_gzip VALUES ('doris0', 0), ('doris1', 1) """
154+
def gzipRows = sql """ SELECT a, b FROM ${db}.tbl_gzip ORDER BY b """
155+
assertEquals(2, gzipRows.size())
156+
assertEquals("doris1", gzipRows[1][0].toString())
157+
def gzipDataFiles = sql """ SELECT file_path FROM ${db}.tbl_gzip\$files ORDER BY file_path """
158+
String gzipDataFile = gzipDataFiles[0][0].toString()
159+
def gzipDataCodec = sql """
160+
SELECT DISTINCT compression
161+
FROM parquet_meta(
162+
"uri" = "${gzipDataFile}",
163+
"s3.access_key" = "admin",
164+
"s3.secret_key" = "password",
165+
"s3.endpoint" = "http://${externalEnvIp}:${minio_port}",
166+
"s3.region" = "us-east-1",
167+
"mode" = "parquet_metadata"
168+
)
169+
"""
170+
assertEquals("GZIP", gzipDataCodec[0][0].toString().toUpperCase())
171+
172+
// The delete-file writer must honour the same table property, otherwise DELETE/merge-on-read
173+
// on a gzip table fails where INSERT succeeds.
174+
sql """ drop table if exists ${db}.tbl_gzip_delete """
175+
sql """
176+
CREATE TABLE ${db}.tbl_gzip_delete (id INT, name STRING) PROPERTIES (
177+
'write-format' = 'parquet',
178+
'write.format.default' = 'parquet',
179+
'format-version' = '2',
180+
'write.parquet.compression-codec' = 'gzip',
181+
'write.delete.mode' = 'merge-on-read',
182+
'write.update.mode' = 'merge-on-read',
183+
'write.merge.mode' = 'merge-on-read'
184+
)"""
185+
sql """ INSERT INTO ${db}.tbl_gzip_delete VALUES (1, 'a'), (2, 'b'), (3, 'c') """
186+
sql """ DELETE FROM ${db}.tbl_gzip_delete WHERE id = 2 """
187+
def gzipDeleteRows = sql """ SELECT id, name FROM ${db}.tbl_gzip_delete ORDER BY id """
188+
assertEquals(2, gzipDeleteRows.size())
189+
assertEquals("3", gzipDeleteRows[1][0].toString())
190+
def gzipDeleteFiles = sql """
191+
SELECT file_path
192+
FROM ${db}.tbl_gzip_delete\$delete_files
193+
ORDER BY file_path
194+
"""
195+
String gzipDeleteFile = gzipDeleteFiles[0][0].toString()
196+
def gzipDeleteCodec = sql """
197+
SELECT DISTINCT compression
198+
FROM parquet_meta(
199+
"uri" = "${gzipDeleteFile}",
200+
"s3.access_key" = "admin",
201+
"s3.secret_key" = "password",
202+
"s3.endpoint" = "http://${externalEnvIp}:${minio_port}",
203+
"s3.region" = "us-east-1",
204+
"mode" = "parquet_metadata"
205+
)
206+
"""
207+
assertEquals("GZIP", gzipDeleteCodec[0][0].toString().toUpperCase())
155208
}

0 commit comments

Comments
 (0)