Skip to content

Commit 8b8880d

Browse files
committed
[fix](iceberg) Validate required NULL writes on branch-4.2
### What problem does this PR solve? Issue Number: N/A Related PR: #69327 Problem Summary: Iceberg ORC and Parquet writes could accept NULL required values, including nested STRUCT/LIST/MAP fields and MAP keys. Failed writes could also leave physical files produced by writers or completed BE fragments behind. Validate every prepared block before creating or dispatching partition writers, while preserving optional-parent NULL masking. Destroy failed writer streams before deferring file deletion to the existing outer cleanup lifecycle. Adapt FE cleanup to branch-4.2's legacy IcebergTransaction and retained catalog lease. Clean failed-commit files before releasing the lease, and retain files when Iceberg reports an unknown commit state. Carry over the necessary sorted writer close error propagation that the original PR's baseline already has. Split container validation into focused helpers for the target lint thresholds. ### Release note Iceberg ORC/Parquet writes reject required NULL values and clean up failed files without publishing a snapshot. ### Check List (For Author) - Test: Modified Java files pass Checkstyle; repository clang-format 16 and whitespace checks pass; regression framework compilation and 47 framework unit tests pass. FE build and focused unit-test attempts were interrupted by confirmed host OOM kills. BE ASAN product/UT configuration succeeds, but full compilation and BE unit execution remain incomplete. Both focused regression suites were attempted and failed connecting to the isolated FE. Validator clang-tidy diagnostics were fixed; the check remains blocked by an existing unmatched NOLINTEND in core/types.h. - Behavior changed: Yes, invalid required NULL writes are rejected and failed external files are cleaned up. - Does this need documentation: No
1 parent 91f3bc7 commit 8b8880d

15 files changed

Lines changed: 1236 additions & 153 deletions

File tree

Lines changed: 374 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,374 @@
1+
// Licensed to the Apache Software Foundation (ASF) under one
2+
// or more contributor license agreements. See the NOTICE file
3+
// distributed with this work for additional information
4+
// regarding copyright ownership. The ASF licenses this file
5+
// to you under the Apache License, Version 2.0 (the
6+
// "License"); you may not use this file except in compliance
7+
// with the License. You may obtain a copy of the License at
8+
//
9+
// http://www.apache.org/licenses/LICENSE-2.0
10+
//
11+
// Unless required by applicable law or agreed to in writing,
12+
// software distributed under the License is distributed on an
13+
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
// KIND, either express or implied. See the License for the
15+
// specific language governing permissions and limitations
16+
// under the License.
17+
18+
#include "exec/sink/writer/iceberg/iceberg_write_validator.h"
19+
20+
#include <fmt/format.h>
21+
22+
#include <algorithm>
23+
#include <optional>
24+
#include <string>
25+
#include <string_view>
26+
#include <vector>
27+
28+
#include "core/assert_cast.h"
29+
#include "core/block/block.h"
30+
#include "core/column/column_array.h"
31+
#include "core/column/column_map.h"
32+
#include "core/column/column_nullable.h"
33+
#include "core/column/column_struct.h"
34+
#include "core/data_type/data_type_array.h"
35+
#include "core/data_type/data_type_map.h"
36+
#include "core/data_type/data_type_nullable.h"
37+
#include "core/data_type/data_type_struct.h"
38+
#include "format/table/iceberg/schema.h"
39+
#include "format/table/iceberg/types.h"
40+
41+
namespace doris::iceberg {
42+
namespace {
43+
44+
using Path = std::vector<std::string_view>;
45+
46+
struct ValidationLocation {
47+
size_t row = 0;
48+
std::optional<size_t> ordinal;
49+
std::string_view ordinal_kind;
50+
};
51+
52+
enum class NonNullReason {
53+
SCHEMA_REQUIRED,
54+
MAP_KEY,
55+
};
56+
57+
std::string path_to_string(const Path& path) {
58+
std::string result;
59+
for (size_t i = 0; i < path.size(); ++i) {
60+
if (i > 0) {
61+
result.push_back('.');
62+
}
63+
result.append(path[i]);
64+
}
65+
return result;
66+
}
67+
68+
Status null_violation(const Path& path, const ValidationLocation& location, NonNullReason reason) {
69+
std::string position = fmt::format("row {}", location.row);
70+
if (location.ordinal.has_value()) {
71+
position += fmt::format(", {} {}", location.ordinal_kind, *location.ordinal);
72+
}
73+
return Status::DataQualityError("Iceberg {} '{}' contains NULL at {}",
74+
reason == NonNullReason::MAP_KEY ? "map key" : "required field",
75+
path_to_string(path), position);
76+
}
77+
78+
bool requires_validation(const NestedField& field) {
79+
if (field.is_required()) {
80+
return true;
81+
}
82+
switch (field.field_type()->type_id()) {
83+
case TypeID::STRUCT: {
84+
const auto& fields = field.field_type()->as_struct_type()->fields();
85+
return std::ranges::any_of(
86+
fields, [](const NestedField& child) { return requires_validation(child); });
87+
}
88+
case TypeID::LIST:
89+
return requires_validation(field.field_type()->as_list_type()->element_field());
90+
case TypeID::MAP:
91+
// Iceberg map keys are required independently of the map field's nullability.
92+
return true;
93+
default:
94+
return false;
95+
}
96+
}
97+
98+
template <typename Offsets>
99+
Status validate_collection_offsets(const Offsets& offsets, size_t position, size_t row,
100+
std::string_view container_name, const Path& path, size_t* start,
101+
size_t* end) {
102+
if (position >= offsets.size()) {
103+
return Status::InternalError(
104+
"Iceberg {} field '{}' has {} offsets, but position {} is required", container_name,
105+
path_to_string(path), offsets.size(), position);
106+
}
107+
*start = offsets[position - 1];
108+
*end = offsets[position];
109+
if (*start > *end) {
110+
return Status::InternalError(
111+
"Iceberg {} field '{}' has invalid offsets at row {}: [{}, {}]", container_name,
112+
path_to_string(path), row, *start, *end);
113+
}
114+
return Status::OK();
115+
}
116+
117+
Status validate_value(const NestedField& field, const IColumn& column, const DataTypePtr& type,
118+
size_t position, const ValidationLocation& location, Path& path,
119+
NonNullReason reason);
120+
121+
Status validate_struct(const NestedField& field, const IColumn& column, const DataTypePtr& type,
122+
size_t position, const ValidationLocation& location, Path& path) {
123+
const IColumn* data_column = &column;
124+
const auto primitive_type = remove_nullable(type)->get_primitive_type();
125+
if (primitive_type != TYPE_STRUCT) {
126+
return Status::InternalError("Iceberg STRUCT field '{}' does not match Doris type {}",
127+
path_to_string(path), type->get_name());
128+
}
129+
const auto* struct_column = check_and_get_column<ColumnStruct>(*data_column);
130+
const auto* struct_type = assert_cast<const DataTypeStruct*>(remove_nullable(type).get());
131+
if (struct_column == nullptr) {
132+
return Status::InternalError("Iceberg STRUCT field '{}' is not stored as a STRUCT",
133+
path_to_string(path));
134+
}
135+
const auto& fields = field.field_type()->as_struct_type()->fields();
136+
if (struct_column->tuple_size() != fields.size() ||
137+
struct_type->get_elements().size() != fields.size()) {
138+
return Status::InternalError(
139+
"Iceberg STRUCT field '{}' has {} schema fields, {} Doris type fields, and "
140+
"{} column fields",
141+
path_to_string(path), fields.size(), struct_type->get_elements().size(),
142+
struct_column->tuple_size());
143+
}
144+
for (size_t i = 0; i < fields.size(); ++i) {
145+
if (!requires_validation(fields[i])) {
146+
continue;
147+
}
148+
path.emplace_back(fields[i].field_name());
149+
Status status =
150+
validate_value(fields[i], struct_column->get_column(i), struct_type->get_element(i),
151+
position, location, path, NonNullReason::SCHEMA_REQUIRED);
152+
path.pop_back();
153+
RETURN_IF_ERROR(status);
154+
}
155+
return Status::OK();
156+
}
157+
158+
Status validate_list(const NestedField& field, const IColumn& column, const DataTypePtr& type,
159+
size_t position, const ValidationLocation& location, Path& path) {
160+
const IColumn* data_column = &column;
161+
const auto primitive_type = remove_nullable(type)->get_primitive_type();
162+
if (primitive_type != TYPE_ARRAY) {
163+
return Status::InternalError("Iceberg LIST field '{}' does not match Doris type {}",
164+
path_to_string(path), type->get_name());
165+
}
166+
const auto* array_column = check_and_get_column<ColumnArray>(*data_column);
167+
const auto* array_type = assert_cast<const DataTypeArray*>(remove_nullable(type).get());
168+
if (array_column == nullptr) {
169+
return Status::InternalError("Iceberg LIST field '{}' is not stored as an ARRAY",
170+
path_to_string(path));
171+
}
172+
const auto& offsets = array_column->get_offsets();
173+
size_t start = 0;
174+
size_t end = 0;
175+
RETURN_IF_ERROR(validate_collection_offsets(offsets, position, location.row, "LIST", path,
176+
&start, &end));
177+
if (end > array_column->get_data().size()) {
178+
return Status::InternalError(
179+
"Iceberg LIST field '{}' has invalid offsets at row {}: [{}, {}], "
180+
"element count {}",
181+
path_to_string(path), location.row, start, end, array_column->get_data().size());
182+
}
183+
const auto& element = field.field_type()->as_list_type()->element_field();
184+
if (!requires_validation(element)) {
185+
return Status::OK();
186+
}
187+
path.emplace_back(element.field_name());
188+
for (size_t i = start; i < end; ++i) {
189+
ValidationLocation element_location {
190+
.row = location.row,
191+
.ordinal = i - start,
192+
.ordinal_kind = "element",
193+
};
194+
Status status =
195+
validate_value(element, array_column->get_data(), array_type->get_nested_type(), i,
196+
element_location, path, NonNullReason::SCHEMA_REQUIRED);
197+
if (!status.ok()) {
198+
path.pop_back();
199+
return status;
200+
}
201+
}
202+
path.pop_back();
203+
return Status::OK();
204+
}
205+
206+
Status validate_map(const NestedField& field, const IColumn& column, const DataTypePtr& type,
207+
size_t position, const ValidationLocation& location, Path& path) {
208+
const IColumn* data_column = &column;
209+
const auto primitive_type = remove_nullable(type)->get_primitive_type();
210+
if (primitive_type != TYPE_MAP) {
211+
return Status::InternalError("Iceberg MAP field '{}' does not match Doris type {}",
212+
path_to_string(path), type->get_name());
213+
}
214+
const auto* map_column = check_and_get_column<ColumnMap>(*data_column);
215+
const auto* map_type = assert_cast<const DataTypeMap*>(remove_nullable(type).get());
216+
if (map_column == nullptr) {
217+
return Status::InternalError("Iceberg MAP field '{}' is not stored as a MAP",
218+
path_to_string(path));
219+
}
220+
const auto& offsets = map_column->get_offsets();
221+
size_t start = 0;
222+
size_t end = 0;
223+
RETURN_IF_ERROR(validate_collection_offsets(offsets, position, location.row, "MAP", path,
224+
&start, &end));
225+
if (end > map_column->get_keys().size() || end > map_column->get_values().size()) {
226+
return Status::InternalError(
227+
"Iceberg MAP field '{}' has invalid offsets at row {}: [{}, {}], "
228+
"key count {}, value count {}",
229+
path_to_string(path), location.row, start, end, map_column->get_keys().size(),
230+
map_column->get_values().size());
231+
}
232+
233+
const auto& map_type_desc = field.field_type()->as_map_type();
234+
path.emplace_back(map_type_desc->key_field().field_name());
235+
for (size_t i = start; i < end; ++i) {
236+
ValidationLocation entry_location {
237+
.row = location.row,
238+
.ordinal = i - start,
239+
.ordinal_kind = "map entry",
240+
};
241+
Status status = validate_value(map_type_desc->key_field(), map_column->get_keys(),
242+
map_type->get_key_type(), i, entry_location, path,
243+
NonNullReason::MAP_KEY);
244+
if (!status.ok()) {
245+
path.pop_back();
246+
return status;
247+
}
248+
}
249+
path.pop_back();
250+
251+
const auto& value = map_type_desc->value_field();
252+
if (!requires_validation(value)) {
253+
return Status::OK();
254+
}
255+
path.emplace_back(value.field_name());
256+
for (size_t i = start; i < end; ++i) {
257+
ValidationLocation entry_location {
258+
.row = location.row,
259+
.ordinal = i - start,
260+
.ordinal_kind = "map entry",
261+
};
262+
Status status = validate_value(value, map_column->get_values(), map_type->get_value_type(),
263+
i, entry_location, path, NonNullReason::SCHEMA_REQUIRED);
264+
if (!status.ok()) {
265+
path.pop_back();
266+
return status;
267+
}
268+
}
269+
path.pop_back();
270+
return Status::OK();
271+
}
272+
273+
Status validate_value(const NestedField& field, const IColumn& column, const DataTypePtr& type,
274+
size_t position, const ValidationLocation& location, Path& path,
275+
NonNullReason reason) {
276+
if (position >= column.size()) {
277+
return Status::InternalError(
278+
"Iceberg write column '{}' has {} values, but position {} "
279+
"is required",
280+
path_to_string(path), column.size(), position);
281+
}
282+
283+
if (column.is_null_at(position)) {
284+
if (reason == NonNullReason::MAP_KEY || field.is_required()) {
285+
return null_violation(path, location, reason);
286+
}
287+
// An optional STRUCT/LIST/MAP is a NULL value in its own right. Its children and
288+
// elements are masked by the parent NULL and must not be inspected.
289+
return Status::OK();
290+
}
291+
292+
const IColumn* data_column = &column;
293+
if (const auto* nullable = check_and_get_column<ColumnNullable>(column)) {
294+
data_column = &nullable->get_nested_column();
295+
}
296+
297+
switch (field.field_type()->type_id()) {
298+
case TypeID::STRUCT:
299+
return validate_struct(field, *data_column, type, position, location, path);
300+
case TypeID::LIST:
301+
return validate_list(field, *data_column, type, position, location, path);
302+
case TypeID::MAP:
303+
return validate_map(field, *data_column, type, position, location, path);
304+
default:
305+
return Status::OK();
306+
}
307+
}
308+
309+
} // namespace
310+
311+
Status validate_write_block(const Schema& schema, const Block& block) {
312+
if (block.columns() != schema.columns().size()) {
313+
return Status::InternalError(
314+
"Iceberg write block has {} columns, but pinned schema has {} fields",
315+
block.columns(), schema.columns().size());
316+
}
317+
318+
Path path;
319+
path.reserve(8);
320+
for (size_t field_index = 0; field_index < schema.columns().size(); ++field_index) {
321+
const auto& field = schema.columns()[field_index];
322+
if (!requires_validation(field)) {
323+
continue;
324+
}
325+
const auto& column_with_name = block.get_by_position(field_index);
326+
ColumnPtr column = column_with_name.column->convert_to_full_column_if_const();
327+
if (!column) {
328+
return Status::InternalError("Iceberg write block column {} is null", field_index);
329+
}
330+
if (column->size() < block.rows()) {
331+
return Status::InternalError(
332+
"Iceberg write block column '{}' has {} values, but block has {} rows",
333+
field.field_name(), column->size(), block.rows());
334+
}
335+
336+
path.emplace_back(field.field_name());
337+
if (field.field_type()->is_primitive_type() && field.is_required()) {
338+
if (column->has_null()) {
339+
for (size_t row = 0; row < block.rows(); ++row) {
340+
if (!column->is_null_at(row)) {
341+
continue;
342+
}
343+
ValidationLocation location {
344+
.row = row,
345+
.ordinal = std::nullopt,
346+
.ordinal_kind = {},
347+
};
348+
Status status = null_violation(path, location, NonNullReason::SCHEMA_REQUIRED);
349+
path.pop_back();
350+
return status;
351+
}
352+
}
353+
path.pop_back();
354+
continue;
355+
}
356+
for (size_t row = 0; row < block.rows(); ++row) {
357+
ValidationLocation location {
358+
.row = row,
359+
.ordinal = std::nullopt,
360+
.ordinal_kind = {},
361+
};
362+
Status status = validate_value(field, *column, column_with_name.type, row, location,
363+
path, NonNullReason::SCHEMA_REQUIRED);
364+
if (!status.ok()) {
365+
path.pop_back();
366+
return status;
367+
}
368+
}
369+
path.pop_back();
370+
}
371+
return Status::OK();
372+
}
373+
374+
} // namespace doris::iceberg

0 commit comments

Comments
 (0)