Skip to content

Commit 1052fc3

Browse files
authored
[fix](iceberg) Validate required NULLs before file creation (#69327)
### What problem does this PR solve? Issue Number: N/A Related PR: N/A Problem Summary: Iceberg required fields could be written as NULL for ORC and Parquet, and failed writes could leave data files behind. This PR adds a shared BE Block validator for required nested fields and MAP keys, runs validation before writers/files are created, and cleans up failed files without publishing a snapshot. FE rollback now also removes files already reported by completed BE fragments. ### Release note Fix Iceberg ORC/Parquet writes to reject required NULL values and clean up failed files without publishing a snapshot. ### Check List (For Author) - Test - [ ] Regression test - [x] Unit Test - BE validator, TableWriter, and cleanup tests pass. - FE rollback tests pass. - [x] Manual test - Verified ORC/Parquet failures leave 0 snapshots, 0 physical objects, and 0 rows; optional parent NULL writes successfully. - [ ] No need to test or manual test. Explain why: - [ ] This is a refactor/code format and no logic has been changed. - [ ] Previous test can cover this change. - [ ] No code files have been changed. - [ ] Other reason - Behavior changed: - [ ] No. - [x] Yes. - Does this need documentation? - [x] No. - [ ] Yes. ### Check List (For Reviewer who merge this PR) - [ ] Confirm the release note - [ ] Confirm test cases - [ ] Confirm document - [ ] Add branch pick label
1 parent 960967b commit 1052fc3

13 files changed

Lines changed: 1062 additions & 89 deletions

File tree

Lines changed: 355 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,355 @@
1+
// Licensed to the Apache Software Foundation (ASF) under one
2+
// or more contributor license agreements. See the NOTICE file
3+
// distributed with this work for additional information
4+
// regarding copyright ownership. The ASF licenses this file
5+
// to you under the Apache License, Version 2.0 (the
6+
// "License"); you may not use this file except in compliance
7+
// with the License. You may obtain a copy of the License at
8+
//
9+
// http://www.apache.org/licenses/LICENSE-2.0
10+
//
11+
// Unless required by applicable law or agreed to in writing,
12+
// software distributed under the License is distributed on an
13+
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
// KIND, either express or implied. See the License for the
15+
// specific language governing permissions and limitations
16+
// under the License.
17+
18+
#include "exec/sink/writer/iceberg/iceberg_write_validator.h"
19+
20+
#include <fmt/format.h>
21+
22+
#include <algorithm>
23+
#include <optional>
24+
#include <string>
25+
#include <string_view>
26+
#include <vector>
27+
28+
#include "core/assert_cast.h"
29+
#include "core/block/block.h"
30+
#include "core/column/column_array.h"
31+
#include "core/column/column_map.h"
32+
#include "core/column/column_nullable.h"
33+
#include "core/column/column_struct.h"
34+
#include "core/data_type/data_type_array.h"
35+
#include "core/data_type/data_type_map.h"
36+
#include "core/data_type/data_type_nullable.h"
37+
#include "core/data_type/data_type_struct.h"
38+
#include "format/table/iceberg/schema.h"
39+
#include "format/table/iceberg/types.h"
40+
41+
namespace doris::iceberg {
42+
namespace {
43+
44+
using Path = std::vector<std::string_view>;
45+
46+
struct ValidationLocation {
47+
size_t row = 0;
48+
std::optional<size_t> ordinal;
49+
std::string_view ordinal_kind;
50+
};
51+
52+
enum class NonNullReason {
53+
SCHEMA_REQUIRED,
54+
MAP_KEY,
55+
};
56+
57+
std::string path_to_string(const Path& path) {
58+
std::string result;
59+
for (size_t i = 0; i < path.size(); ++i) {
60+
if (i > 0) {
61+
result.push_back('.');
62+
}
63+
result.append(path[i]);
64+
}
65+
return result;
66+
}
67+
68+
Status null_violation(const Path& path, const ValidationLocation& location, NonNullReason reason) {
69+
std::string position = fmt::format("row {}", location.row);
70+
if (location.ordinal.has_value()) {
71+
position += fmt::format(", {} {}", location.ordinal_kind, *location.ordinal);
72+
}
73+
return Status::DataQualityError("Iceberg {} '{}' contains NULL at {}",
74+
reason == NonNullReason::MAP_KEY ? "map key" : "required field",
75+
path_to_string(path), position);
76+
}
77+
78+
bool requires_validation(const NestedField& field) {
79+
if (field.is_required()) {
80+
return true;
81+
}
82+
switch (field.field_type()->type_id()) {
83+
case TypeID::STRUCT: {
84+
const auto& fields = field.field_type()->as_struct_type()->fields();
85+
return std::any_of(fields.begin(), fields.end(),
86+
[](const NestedField& child) { return requires_validation(child); });
87+
}
88+
case TypeID::LIST:
89+
return requires_validation(field.field_type()->as_list_type()->element_field());
90+
case TypeID::MAP:
91+
// Iceberg map keys are required independently of the map field's nullability.
92+
return true;
93+
default:
94+
return false;
95+
}
96+
}
97+
98+
template <typename Offsets>
99+
Status validate_collection_offsets(const Offsets& offsets, size_t position, size_t row,
100+
std::string_view container_name, const Path& path, size_t* start,
101+
size_t* end) {
102+
if (position >= offsets.size()) {
103+
return Status::InternalError(
104+
"Iceberg {} field '{}' has {} offsets, but position {} is required", container_name,
105+
path_to_string(path), offsets.size(), position);
106+
}
107+
*start = position == 0 ? 0 : offsets[position - 1];
108+
*end = offsets[position];
109+
if (*start > *end) {
110+
return Status::InternalError(
111+
"Iceberg {} field '{}' has invalid offsets at row {}: [{}, {}]", container_name,
112+
path_to_string(path), row, *start, *end);
113+
}
114+
return Status::OK();
115+
}
116+
117+
Status validate_value(const NestedField& field, const IColumn& column, const DataTypePtr& type,
118+
size_t position, const ValidationLocation& location, Path& path,
119+
NonNullReason reason) {
120+
if (position >= column.size()) {
121+
return Status::InternalError(
122+
"Iceberg write column '{}' has {} values, but position {} "
123+
"is required",
124+
path_to_string(path), column.size(), position);
125+
}
126+
127+
if (column.is_null_at(position)) {
128+
if (reason == NonNullReason::MAP_KEY || field.is_required()) {
129+
return null_violation(path, location, reason);
130+
}
131+
// An optional STRUCT/LIST/MAP is a NULL value in its own right. Its children and
132+
// elements are masked by the parent NULL and must not be inspected.
133+
return Status::OK();
134+
}
135+
136+
const IColumn* data_column = &column;
137+
if (const auto* nullable = check_and_get_column<ColumnNullable>(column)) {
138+
data_column = &nullable->get_nested_column();
139+
}
140+
141+
const auto primitive_type = remove_nullable(type)->get_primitive_type();
142+
switch (field.field_type()->type_id()) {
143+
case TypeID::STRUCT: {
144+
if (primitive_type != TYPE_STRUCT) {
145+
return Status::InternalError("Iceberg STRUCT field '{}' does not match Doris type {}",
146+
path_to_string(path), type->get_name());
147+
}
148+
const auto* struct_column = check_and_get_column<ColumnStruct>(*data_column);
149+
const auto* struct_type = assert_cast<const DataTypeStruct*>(remove_nullable(type).get());
150+
if (struct_column == nullptr) {
151+
return Status::InternalError("Iceberg STRUCT field '{}' is not stored as a STRUCT",
152+
path_to_string(path));
153+
}
154+
const auto& fields = field.field_type()->as_struct_type()->fields();
155+
if (struct_column->tuple_size() != fields.size() ||
156+
struct_type->get_elements().size() != fields.size()) {
157+
return Status::InternalError(
158+
"Iceberg STRUCT field '{}' has {} schema fields, {} Doris type fields, and "
159+
"{} column fields",
160+
path_to_string(path), fields.size(), struct_type->get_elements().size(),
161+
struct_column->tuple_size());
162+
}
163+
for (size_t i = 0; i < fields.size(); ++i) {
164+
if (!requires_validation(fields[i])) {
165+
continue;
166+
}
167+
path.emplace_back(fields[i].field_name());
168+
Status status = validate_value(fields[i], struct_column->get_column(i),
169+
struct_type->get_element(i), position, location, path,
170+
NonNullReason::SCHEMA_REQUIRED);
171+
path.pop_back();
172+
RETURN_IF_ERROR(status);
173+
}
174+
return Status::OK();
175+
}
176+
case TypeID::LIST: {
177+
if (primitive_type != TYPE_ARRAY) {
178+
return Status::InternalError("Iceberg LIST field '{}' does not match Doris type {}",
179+
path_to_string(path), type->get_name());
180+
}
181+
const auto* array_column = check_and_get_column<ColumnArray>(*data_column);
182+
const auto* array_type = assert_cast<const DataTypeArray*>(remove_nullable(type).get());
183+
if (array_column == nullptr) {
184+
return Status::InternalError("Iceberg LIST field '{}' is not stored as an ARRAY",
185+
path_to_string(path));
186+
}
187+
const auto& offsets = array_column->get_offsets();
188+
size_t start = 0;
189+
size_t end = 0;
190+
RETURN_IF_ERROR(validate_collection_offsets(offsets, position, location.row, "LIST", path,
191+
&start, &end));
192+
if (end > array_column->get_data().size()) {
193+
return Status::InternalError(
194+
"Iceberg LIST field '{}' has invalid offsets at row {}: [{}, {}], "
195+
"element count {}",
196+
path_to_string(path), location.row, start, end,
197+
array_column->get_data().size());
198+
}
199+
const auto& element = field.field_type()->as_list_type()->element_field();
200+
if (!requires_validation(element)) {
201+
return Status::OK();
202+
}
203+
path.emplace_back(element.field_name());
204+
for (size_t i = start; i < end; ++i) {
205+
ValidationLocation element_location {
206+
.row = location.row,
207+
.ordinal = i - start,
208+
.ordinal_kind = "element",
209+
};
210+
Status status =
211+
validate_value(element, array_column->get_data(), array_type->get_nested_type(),
212+
i, element_location, path, NonNullReason::SCHEMA_REQUIRED);
213+
if (!status.ok()) {
214+
path.pop_back();
215+
return status;
216+
}
217+
}
218+
path.pop_back();
219+
return Status::OK();
220+
}
221+
case TypeID::MAP: {
222+
if (primitive_type != TYPE_MAP) {
223+
return Status::InternalError("Iceberg MAP field '{}' does not match Doris type {}",
224+
path_to_string(path), type->get_name());
225+
}
226+
const auto* map_column = check_and_get_column<ColumnMap>(*data_column);
227+
const auto* map_type = assert_cast<const DataTypeMap*>(remove_nullable(type).get());
228+
if (map_column == nullptr) {
229+
return Status::InternalError("Iceberg MAP field '{}' is not stored as a MAP",
230+
path_to_string(path));
231+
}
232+
const auto& offsets = map_column->get_offsets();
233+
size_t start = 0;
234+
size_t end = 0;
235+
RETURN_IF_ERROR(validate_collection_offsets(offsets, position, location.row, "MAP", path,
236+
&start, &end));
237+
if (end > map_column->get_keys().size() || end > map_column->get_values().size()) {
238+
return Status::InternalError(
239+
"Iceberg MAP field '{}' has invalid offsets at row {}: [{}, {}], "
240+
"key count {}, value count {}",
241+
path_to_string(path), location.row, start, end, map_column->get_keys().size(),
242+
map_column->get_values().size());
243+
}
244+
245+
const auto& map_type_desc = field.field_type()->as_map_type();
246+
path.emplace_back(map_type_desc->key_field().field_name());
247+
for (size_t i = start; i < end; ++i) {
248+
ValidationLocation entry_location {
249+
.row = location.row,
250+
.ordinal = i - start,
251+
.ordinal_kind = "map entry",
252+
};
253+
Status status = validate_value(map_type_desc->key_field(), map_column->get_keys(),
254+
map_type->get_key_type(), i, entry_location, path,
255+
NonNullReason::MAP_KEY);
256+
if (!status.ok()) {
257+
path.pop_back();
258+
return status;
259+
}
260+
}
261+
path.pop_back();
262+
263+
const auto& value = map_type_desc->value_field();
264+
if (!requires_validation(value)) {
265+
return Status::OK();
266+
}
267+
path.emplace_back(value.field_name());
268+
for (size_t i = start; i < end; ++i) {
269+
ValidationLocation entry_location {
270+
.row = location.row,
271+
.ordinal = i - start,
272+
.ordinal_kind = "map entry",
273+
};
274+
Status status =
275+
validate_value(value, map_column->get_values(), map_type->get_value_type(), i,
276+
entry_location, path, NonNullReason::SCHEMA_REQUIRED);
277+
if (!status.ok()) {
278+
path.pop_back();
279+
return status;
280+
}
281+
}
282+
path.pop_back();
283+
return Status::OK();
284+
}
285+
default:
286+
return Status::OK();
287+
}
288+
}
289+
290+
} // namespace
291+
292+
Status validate_write_block(const Schema& schema, const Block& block) {
293+
if (block.columns() != schema.columns().size()) {
294+
return Status::InternalError(
295+
"Iceberg write block has {} columns, but pinned schema has {} fields",
296+
block.columns(), schema.columns().size());
297+
}
298+
299+
Path path;
300+
path.reserve(8);
301+
for (size_t field_index = 0; field_index < schema.columns().size(); ++field_index) {
302+
const auto& field = schema.columns()[field_index];
303+
if (!requires_validation(field)) {
304+
continue;
305+
}
306+
const auto& column_with_name = block.get_by_position(field_index);
307+
ColumnPtr column = column_with_name.column->convert_to_full_column_if_const();
308+
if (!column) {
309+
return Status::InternalError("Iceberg write block column {} is null", field_index);
310+
}
311+
if (column->size() < block.rows()) {
312+
return Status::InternalError(
313+
"Iceberg write block column '{}' has {} values, but block has {} rows",
314+
field.field_name(), column->size(), block.rows());
315+
}
316+
317+
path.emplace_back(field.field_name());
318+
if (field.field_type()->is_primitive_type() && field.is_required()) {
319+
if (column->has_null()) {
320+
for (size_t row = 0; row < block.rows(); ++row) {
321+
if (!column->is_null_at(row)) {
322+
continue;
323+
}
324+
ValidationLocation location {
325+
.row = row,
326+
.ordinal = std::nullopt,
327+
.ordinal_kind = {},
328+
};
329+
Status status = null_violation(path, location, NonNullReason::SCHEMA_REQUIRED);
330+
path.pop_back();
331+
return status;
332+
}
333+
}
334+
path.pop_back();
335+
continue;
336+
}
337+
for (size_t row = 0; row < block.rows(); ++row) {
338+
ValidationLocation location {
339+
.row = row,
340+
.ordinal = std::nullopt,
341+
.ordinal_kind = {},
342+
};
343+
Status status = validate_value(field, *column, column_with_name.type, row, location,
344+
path, NonNullReason::SCHEMA_REQUIRED);
345+
if (!status.ok()) {
346+
path.pop_back();
347+
return status;
348+
}
349+
}
350+
path.pop_back();
351+
}
352+
return Status::OK();
353+
}
354+
355+
} // namespace doris::iceberg
Lines changed: 35 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,35 @@
1+
// Licensed to the Apache Software Foundation (ASF) under one
2+
// or more contributor license agreements. See the NOTICE file
3+
// distributed with this work for additional information
4+
// regarding copyright ownership. The ASF licenses this file
5+
// to you under the Apache License, Version 2.0 (the
6+
// "License"); you may not use this file except in compliance
7+
// with the License. You may obtain a copy of the License at
8+
//
9+
// http://www.apache.org/licenses/LICENSE-2.0
10+
//
11+
// Unless required by applicable law or agreed to in writing,
12+
// software distributed under the License is distributed on an
13+
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
// KIND, either express or implied. See the License for the
15+
// specific language governing permissions and limitations
16+
// under the License.
17+
18+
#pragma once
19+
20+
#include "common/status.h"
21+
22+
namespace doris {
23+
24+
class Block;
25+
26+
namespace iceberg {
27+
28+
class Schema;
29+
30+
// Validates the data-dependent Iceberg write contract for one materialized Block.
31+
// Schema identity, field IDs, partition specs, and sort orders are pinned and checked in FE.
32+
Status validate_write_block(const Schema& schema, const Block& block);
33+
34+
} // namespace iceberg
35+
} // namespace doris

0 commit comments

Comments
 (0)