Skip to content

Commit a2a1cf5

Browse files
committed
[fix](iceberg) Publish Parquet split offsets in data file metadata
1 parent 7a5d64e commit a2a1cf5

9 files changed

Lines changed: 241 additions & 6 deletions

File tree

‎be/src/exec/sink/writer/iceberg/viceberg_partition_writer.cpp‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -217,6 +217,16 @@ Status VIcebergPartitionWriter::_build_iceberg_commit_data(TIcebergCommitData* c
217217
commit_data->__set_file_size(_file_format_transformer->written_len());
218218
commit_data->__set_file_content(TFileContent::DATA);
219219
commit_data->__set_partition_values(_partition_values);
220+
// Split offsets describe the physical file, even when the table disables all column metrics.
221+
if (_file_format_type == TFileFormatType::FORMAT_PARQUET) {
222+
std::vector<int64_t> offsets;
223+
RETURN_IF_ERROR(static_cast<VIcebergParquetWriter*>(_file_format_transformer.get())
224+
->collect_split_offsets_after_close(&offsets));
225+
// Empty files have no splits; leave the optional field absent, as with an older BE.
226+
if (!offsets.empty()) {
227+
commit_data->__set_split_offsets(offsets);
228+
}
229+
}
220230
// ORC collection reopens the file, so honor the FE policy before any footer work.
221231
if (!_collect_column_stats) {
222232
return Status::OK();

‎be/src/format/transformer/viceberg_parquet_writer.cpp‎

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -115,6 +115,32 @@ std::unique_ptr<ArrowBlockConvertor> VIcebergParquetWriter::_create_arrow_block_
115115
_iceberg_schema, &_iceberg_schema_json, timezone_name, timezone);
116116
}
117117

118+
Status VIcebergParquetWriter::collect_split_offsets_after_close(std::vector<int64_t>* offsets) {
119+
auto metadata = _file_metadata();
120+
// Arrow publishes file metadata only after Close(), so offsets cannot be collected earlier.
121+
if (metadata == nullptr) {
122+
return Status::InternalError("File metadata is not available");
123+
}
124+
offsets->clear();
125+
offsets->reserve(metadata->num_row_groups());
126+
for (int i = 0; i < metadata->num_row_groups(); ++i) {
127+
auto row_group = metadata->RowGroup(i);
128+
// Empty files/groups have no page boundary to advertise as a scan split.
129+
if (row_group->num_rows() == 0 || row_group->num_columns() == 0) {
130+
continue;
131+
}
132+
auto first_column = row_group->ColumnChunk(0);
133+
int64_t offset = first_column->data_page_offset();
134+
// A split must include the dictionary page, not just the first data page that references it.
135+
if (first_column->has_dictionary_page()) {
136+
offset = std::min(offset, first_column->dictionary_page_offset());
137+
}
138+
offsets->push_back(offset);
139+
}
140+
std::sort(offsets->begin(), offsets->end());
141+
return Status::OK();
142+
}
143+
118144
Status VIcebergParquetWriter::collect_file_statistics_after_close(TIcebergColumnStats* stats) {
119145
std::shared_ptr<::parquet::FileMetaData> file_metadata = _file_metadata();
120146
if (file_metadata == nullptr) {

‎be/src/format/transformer/viceberg_parquet_writer.h‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -44,6 +44,8 @@ class VIcebergParquetWriter final : public VParquetWriter {
4444

4545
Status collect_file_statistics_after_close(TIcebergColumnStats* stats);
4646

47+
Status collect_split_offsets_after_close(std::vector<int64_t>* offsets);
48+
4749
protected:
4850
std::unique_ptr<ArrowBlockConvertor> _create_arrow_block_convertor(
4951
DataTypes types, std::vector<std::string> names, const std::string& timezone_name,

‎be/test/exec/sink/writer/iceberg/iceberg_partition_writer_test.cpp‎

Lines changed: 62 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -15,16 +15,24 @@
1515
// specific language governing permissions and limitations
1616
// under the License.
1717

18+
#include <arrow/memory_pool.h>
1819
#include <gtest/gtest.h>
1920

2021
#include <optional>
2122

23+
#include "core/data_type/data_type_number.h"
2224
#include "exec/sink/writer/iceberg/viceberg_partition_writer.h"
2325
#include "exec/sink/writer/iceberg/viceberg_sort_writer.h"
26+
#include "format/table/iceberg/schema_parser.h"
27+
#include "format/transformer/viceberg_parquet_writer.h"
2428
#include "io/fs/local_file_system.h"
29+
#include "runtime/exec_env.h"
2530
#include "testutil/mock/mock_descriptors.h"
2631
#include "testutil/mock/mock_runtime_state.h"
2732
#include "testutil/mock/mock_slot_ref.h"
33+
#include "util/defer_op.h"
34+
#include "util/timezone_utils.h"
35+
#include "util/uid_util.h"
2836

2937
namespace doris {
3038

@@ -73,13 +81,14 @@ class VIcebergPartitionWriterTest : public testing::Test {
7381
static std::unique_ptr<VIcebergPartitionWriter> make_writer(
7482
const TDataSink& sink, const VExprContextSPtrs& output_exprs,
7583
const iceberg::Schema& schema, const std::string* schema_json,
76-
const std::map<std::string, std::string>& hadoop_conf) {
84+
const std::map<std::string, std::string>& hadoop_conf,
85+
TFileFormatType::type format = TFileFormatType::FORMAT_ORC) {
7786
IPartitionWriterBase::WriteInfo write_info;
7887
write_info.file_type = TFileType::FILE_LOCAL;
7988
return std::make_unique<VIcebergPartitionWriter>(
8089
sink, std::vector<std::string> {}, output_exprs, schema, schema_json,
81-
std::vector<std::string> {}, std::move(write_info), "data", 0,
82-
TFileFormatType::FORMAT_ORC, TFileCompressType::ZLIB, hadoop_conf);
90+
std::vector<std::string> {}, std::move(write_info), "data", 0, format,
91+
TFileCompressType::ZLIB, hadoop_conf);
8392
}
8493

8594
static void install_fake_transformer(VIcebergPartitionWriter* writer,
@@ -90,6 +99,11 @@ class VIcebergPartitionWriterTest : public testing::Test {
9099
output_exprs, std::move(close_status), destroyed);
91100
}
92101

102+
static void install_parquet_transformer(VIcebergPartitionWriter* writer,
103+
std::unique_ptr<VIcebergParquetWriter> transformer) {
104+
writer->_file_format_transformer = std::move(transformer);
105+
}
106+
93107
static Status build_commit_data(VIcebergPartitionWriter* writer,
94108
TIcebergCommitData* commit_data) {
95109
return writer->_build_iceberg_commit_data(commit_data);
@@ -114,6 +128,51 @@ TEST_F(VIcebergPartitionWriterTest, OrcSkipsFooterCollectionWhenMetricsAreDisabl
114128
EXPECT_FALSE(commit_data.__isset.column_stats);
115129
}
116130

131+
TEST_F(VIcebergPartitionWriterTest, ParquetKeepsSplitOffsetsWhenColumnMetricsAreDisabled) {
132+
TimezoneUtils::load_timezones_to_cache();
133+
auto* env = ExecEnv::GetInstance();
134+
auto* previous_pool = env->_arrow_memory_pool;
135+
env->_arrow_memory_pool = arrow::default_memory_pool();
136+
const auto path = "./iceberg_split_offsets_" + UniqueId::gen_uid().to_string() + ".parquet";
137+
auto fs = io::global_local_filesystem();
138+
Defer cleanup([&] {
139+
env->_arrow_memory_pool = previous_pool;
140+
static_cast<void>(fs->delete_file(path));
141+
});
142+
auto type = std::make_shared<DataTypeInt32>();
143+
auto output_exprs = MockSlotRef::create_mock_contexts(DataTypes {type});
144+
const std::string json =
145+
R"({"type":"struct","fields":[{"id":1,"name":"value","required":true,"type":"int"}]})";
146+
auto schema = iceberg::SchemaParser::from_json(json);
147+
io::FileWriterPtr file_writer;
148+
ASSERT_TRUE(fs->create_file(path, &file_writer).ok());
149+
RuntimeState state;
150+
state.set_timezone("UTC");
151+
ParquetFileOptions options {.compression_type = TParquetCompressionType::SNAPPY,
152+
.parquet_version = TParquetVersion::PARQUET_1_0,
153+
.parquet_disable_dictionary = false,
154+
.enable_int96_timestamps = false};
155+
auto transformer = std::make_unique<VIcebergParquetWriter>(
156+
&state, file_writer.get(), output_exprs, std::vector<std::string> {"value"}, false,
157+
options, &json, *schema);
158+
ASSERT_TRUE(transformer->open().ok());
159+
auto column = type->create_column();
160+
column->insert(Field::create_field<TYPE_INT>(7));
161+
Block block;
162+
block.insert({std::move(column), type, "value"});
163+
ASSERT_TRUE(transformer->write(block).ok());
164+
ASSERT_TRUE(transformer->close().ok());
165+
auto writer = make_writer(make_table_sink(false), output_exprs, *schema, &json, {},
166+
TFileFormatType::FORMAT_PARQUET);
167+
install_parquet_transformer(writer.get(), std::move(transformer));
168+
TIcebergCommitData commit;
169+
ASSERT_TRUE(build_commit_data(writer.get(), &commit).ok());
170+
EXPECT_FALSE(commit.__isset.column_stats);
171+
ASSERT_TRUE(commit.__isset.split_offsets);
172+
ASSERT_EQ(1, commit.split_offsets.size());
173+
EXPECT_EQ(4, commit.split_offsets[0]);
174+
}
175+
117176
TEST_F(VIcebergPartitionWriterTest, MissingPolicyKeepsCollectionEnabledForRollingUpgrade) {
118177
VExprContextSPtrs output_exprs;
119178
iceberg::Schema schema(std::vector<iceberg::NestedField> {});

‎be/test/format/transformer/vparquet_writer_test.cpp‎

Lines changed: 66 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,7 @@
2828
#include <optional>
2929
#include <string_view>
3030

31+
#include "common/config.h"
3132
#include "core/block/block.h"
3233
#include "core/column/column_array.h"
3334
#include "core/column/column_decimal.h"
@@ -50,6 +51,7 @@
5051
#include "runtime/exec_env.h"
5152
#include "runtime/runtime_state.h"
5253
#include "testutil/mock/mock_slot_ref.h"
54+
#include "util/defer_op.h"
5355
#include "util/timezone_utils.h"
5456
#include "util/uid_util.h"
5557

@@ -128,11 +130,75 @@ class VParquetWriterTest : public testing::Test {
128130
}
129131
}
130132

133+
void check_split_offsets(int row_groups, bool dictionary) {
134+
const auto previous_row_group_size = config::min_row_group_size;
135+
config::min_row_group_size = 0;
136+
Defer restore_row_group_size([&] { config::min_row_group_size = previous_row_group_size; });
137+
auto type = std::make_shared<DataTypeInt32>();
138+
auto output_exprs = MockSlotRef::create_mock_contexts(DataTypes {type});
139+
const std::string json =
140+
R"({"type":"struct","fields":[{"id":1,"name":"value","required":true,"type":"int"}]})";
141+
auto schema = iceberg::SchemaParser::from_json(json);
142+
io::FileWriterPtr file_writer;
143+
ASSERT_TRUE(_fs->create_file(_file_path, &file_writer).ok());
144+
RuntimeState state;
145+
state.set_timezone("UTC");
146+
ParquetFileOptions options {.compression_type = TParquetCompressionType::SNAPPY,
147+
.parquet_version = TParquetVersion::PARQUET_1_0,
148+
.parquet_disable_dictionary = !dictionary,
149+
.enable_int96_timestamps = false};
150+
VIcebergParquetWriter writer(&state, file_writer.get(), output_exprs, {"value"}, false,
151+
options, &json, *schema);
152+
ASSERT_TRUE(writer.open().ok());
153+
for (int group = 0; group < row_groups; ++group) {
154+
auto column = type->create_column();
155+
for (int row = 0; row < 100; ++row) {
156+
column->insert(Field::create_field<TYPE_INT>(row % 3));
157+
}
158+
Block block;
159+
block.insert({std::move(column), type, "value"});
160+
ASSERT_TRUE(writer.write(block).ok());
161+
}
162+
ASSERT_TRUE(writer.close().ok());
163+
std::vector<int64_t> offsets {999999};
164+
ASSERT_TRUE(writer.collect_split_offsets_after_close(&offsets).ok());
165+
ASSERT_EQ(row_groups, offsets.size());
166+
auto reader = ::parquet::ParquetFileReader::OpenFile(_file_path, false);
167+
auto metadata = reader->metadata();
168+
for (int i = 0; i < row_groups; ++i) {
169+
auto row_group = metadata->RowGroup(i);
170+
auto column = row_group->ColumnChunk(0);
171+
ASSERT_EQ(dictionary, column->has_dictionary_page());
172+
EXPECT_EQ(dictionary ? column->dictionary_page_offset() : column->data_page_offset(),
173+
offsets[i]);
174+
EXPECT_GT(offsets[i], i == 0 ? 0 : offsets[i - 1]);
175+
}
176+
std::vector<int64_t> repeated;
177+
ASSERT_TRUE(writer.collect_split_offsets_after_close(&repeated).ok());
178+
EXPECT_EQ(offsets, repeated);
179+
}
180+
131181
arrow::MemoryPool* _previous_pool = nullptr;
132182
std::string _file_path;
133183
std::shared_ptr<io::FileSystem> _fs;
134184
};
135185

186+
TEST_F(VParquetWriterTest, IcebergSplitOffsetsIncludeDictionaryPages) {
187+
check_split_offsets(3, true);
188+
}
189+
190+
TEST_F(VParquetWriterTest, IcebergSplitOffsetsWithoutDictionary) {
191+
check_split_offsets(3, false);
192+
}
193+
194+
TEST_F(VParquetWriterTest, IcebergSplitOffsetsSingleRowGroup) {
195+
check_split_offsets(1, true);
196+
}
197+
198+
TEST_F(VParquetWriterTest, IcebergSplitOffsetsEmptyFile) {
199+
check_split_offsets(0, true);
200+
}
201+
136202
TEST_F(VParquetWriterTest, WritesIcebergPrimitiveAndCollectsMetrics) {
137203
auto type = make_nullable(std::make_shared<DataTypeInt32>());
138204
auto output_exprs = MockSlotRef::create_mock_contexts(DataTypes {type});

‎fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergWriterHelper.java‎

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -184,7 +184,7 @@ private static WriteResult convertToWriterResult(
184184
partitionData = Optional.of(convertToPartitionData(partitionValues, spec, zone));
185185
}
186186
DataFile dataFile = genDataFile(fileFormat, location, spec, partitionData, recordCount, fileSize,
187-
metrics, sortOrder);
187+
metrics, sortOrder, commitData.getSplitOffsets());
188188
dataFiles.add(dataFile);
189189
}
190190
return WriteResult.builder()
@@ -194,14 +194,18 @@ private static WriteResult convertToWriterResult(
194194

195195
private static DataFile genDataFile(FileFormat format, String location, PartitionSpec spec,
196196
Optional<PartitionData> partitionData, long recordCount, long fileSize, Metrics metrics,
197-
SortOrder sortOrder) {
197+
SortOrder sortOrder, List<Long> splitOffsets) {
198198
DataFiles.Builder builder = DataFiles.builder(spec)
199199
.withPath(location)
200200
.withFileSizeInBytes(fileSize)
201201
.withRecordCount(recordCount)
202202
.withMetrics(metrics)
203203
.withSortOrder(sortOrder)
204204
.withFormat(format);
205+
// Older BE fragments omit this optional field; only Parquet data writers report these boundaries.
206+
if (format == FileFormat.PARQUET && splitOffsets != null && !splitOffsets.isEmpty()) {
207+
builder.withSplitOffsets(splitOffsets);
208+
}
205209
partitionData.ifPresent(builder::withPartition);
206210
return builder.build();
207211
}

‎fe/fe-connector/fe-connector-iceberg/src/test/java/org/apache/doris/connector/iceberg/IcebergWriterHelperTest.java‎

Lines changed: 54 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -26,20 +26,25 @@
2626
import org.apache.iceberg.DataFiles;
2727
import org.apache.iceberg.DeleteFile;
2828
import org.apache.iceberg.FileFormat;
29+
import org.apache.iceberg.FileScanTask;
2930
import org.apache.iceberg.MetadataColumns;
3031
import org.apache.iceberg.PartitionSpec;
3132
import org.apache.iceberg.Schema;
3233
import org.apache.iceberg.Table;
3334
import org.apache.iceberg.catalog.Namespace;
3435
import org.apache.iceberg.catalog.TableIdentifier;
3536
import org.apache.iceberg.inmemory.InMemoryCatalog;
37+
import org.apache.iceberg.io.CloseableIterable;
3638
import org.apache.iceberg.io.WriteResult;
3739
import org.apache.iceberg.types.Conversions;
3840
import org.apache.iceberg.types.Types;
41+
import org.apache.thrift.TDeserializer;
42+
import org.apache.thrift.protocol.TJSONProtocol;
3943
import org.junit.jupiter.api.Assertions;
4044
import org.junit.jupiter.api.Test;
4145

4246
import java.nio.ByteBuffer;
47+
import java.nio.charset.StandardCharsets;
4348
import java.time.ZoneOffset;
4449
import java.util.ArrayList;
4550
import java.util.Arrays;
@@ -212,6 +217,55 @@ public void convertToWriterResultBuildsDataFiles() {
212217
Assertions.assertEquals(FileFormat.PARQUET, df.format());
213218
}
214219

220+
@Test
221+
public void convertToWriterResultCarriesSplitOffsetsFromCommitProtocol() throws Exception {
222+
Table table = tableWith("write.format.default", "parquet", "write.metadata.metrics.default", "none");
223+
TIcebergCommitData commit = new TIcebergCommitData();
224+
// Decode the optional field on the wire so this also exercises compatibility with older BE fragments.
225+
String fragment = "{\"1\":{\"str\":\"data.parquet\"},\"2\":{\"i64\":100},"
226+
+ "\"3\":{\"i64\":4096},\"15\":{\"lst\":[\"i64\",2,4,2048]}}";
227+
new TDeserializer(new TJSONProtocol.Factory()).deserialize(commit, fragment.getBytes(StandardCharsets.UTF_8));
228+
DataFile file = IcebergWriterHelper.convertToWriterResult(
229+
table, Collections.singletonList(commit), ZoneOffset.UTC).dataFiles()[0];
230+
Assertions.assertEquals(Arrays.asList(4L, 2048L), file.splitOffsets());
231+
table.newAppend().appendFile(file).commit();
232+
try (CloseableIterable<FileScanTask> tasks = table.newScan().planFiles()) {
233+
Assertions.assertEquals(Arrays.asList(4L, 2048L), tasks.iterator().next().file().splitOffsets());
234+
}
235+
}
236+
237+
@Test
238+
public void convertToWriterResultKeepsOffsetsPerFileAndSupportsOldFragments() {
239+
Table table = tableWith("write.format.default", "parquet");
240+
List<TIcebergCommitData> commits = new ArrayList<>();
241+
for (int i = 0; i < 3; i++) {
242+
TIcebergCommitData commit = new TIcebergCommitData();
243+
commit.setFilePath("data-" + i + ".parquet");
244+
commit.setRowCount(10);
245+
commit.setFileSize(4096);
246+
if (i < 2) {
247+
commit.setSplitOffsets(i == 0 ? Arrays.asList(4L, 1024L) : Collections.singletonList(4L));
248+
}
249+
commits.add(commit);
250+
}
251+
DataFile[] files = IcebergWriterHelper.convertToWriterResult(table, commits, ZoneOffset.UTC).dataFiles();
252+
Assertions.assertEquals(Arrays.asList(4L, 1024L), files[0].splitOffsets());
253+
Assertions.assertEquals(Collections.singletonList(4L), files[1].splitOffsets());
254+
Assertions.assertNull(files[2].splitOffsets());
255+
}
256+
257+
@Test
258+
public void convertToWriterResultDoesNotApplyParquetOffsetsToOrc() {
259+
TIcebergCommitData commit = new TIcebergCommitData();
260+
commit.setFilePath("data.orc");
261+
commit.setRowCount(10);
262+
commit.setFileSize(4096);
263+
commit.setSplitOffsets(Collections.singletonList(4L));
264+
DataFile file = IcebergWriterHelper.convertToWriterResult(tableWith("write.format.default", "orc"),
265+
Collections.singletonList(commit), ZoneOffset.UTC).dataFiles()[0];
266+
Assertions.assertNull(file.splitOffsets());
267+
}
268+
215269
@Test
216270
public void convertToWriterResultUsesPinnedMetadataAfterTableEvolution() {
217271
// Files are labeled with the spec, sort order, format and metrics policy the writer was pinned to, even after

‎gensrc/thrift/DataSinks.thrift‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -461,6 +461,8 @@ struct TIcebergCommitData {
461461
13: optional i64 content_size_in_bytes
462462
// Rows newly affected by the current statement for this commit item.
463463
14: optional i64 affected_rows
464+
// Parquet data-file row-group starts, independent of column metrics policy.
465+
15: optional list<i64> split_offsets
464466
}
465467

466468
struct TSortField {

0 commit comments

Comments
 (0)