Skip to content

Commit d7c653c

Browse files
committed
[refactor](iceberg) Simplify Iceberg write validator
### What problem does this PR solve? Issue Number: N/A Problem Summary: Remove unused validator state, fold one-use location formatting, share LIST/MAP offset validation, and rename the local validation location helper without changing validation behavior. ### Release note None ### Check List (For Author) - Test: Unit Test / Manual test - Behavior changed: No - Does this need documentation: No
1 parent 4fc50e7 commit d7c653c

1 file changed

Lines changed: 57 additions & 56 deletions

File tree

‎be/src/exec/sink/writer/iceberg/iceberg_write_validator.cpp‎

Lines changed: 57 additions & 56 deletions
Original file line numberDiff line numberDiff line change
@@ -43,7 +43,7 @@ namespace {
4343

4444
using Path = std::vector<std::string_view>;
4545

46-
struct PositionInfo {
46+
struct ValidationLocation {
4747
size_t row = 0;
4848
std::optional<size_t> ordinal;
4949
std::string_view ordinal_kind;
@@ -65,22 +65,18 @@ std::string path_to_string(const Path& path) {
6565
return result;
6666
}
6767

68-
std::string position_to_string(const PositionInfo& position) {
69-
std::string result = fmt::format("row {}", position.row);
70-
if (position.ordinal.has_value()) {
71-
result += fmt::format(", {} {}", position.ordinal_kind, *position.ordinal);
68+
Status null_violation(const Path& path, const ValidationLocation& location, NonNullReason reason) {
69+
std::string position = fmt::format("row {}", location.row);
70+
if (location.ordinal.has_value()) {
71+
position += fmt::format(", {} {}", location.ordinal_kind, *location.ordinal);
7272
}
73-
return result;
74-
}
75-
76-
Status null_violation(const Path& path, const PositionInfo& position, NonNullReason reason) {
7773
return Status::DataQualityError("Iceberg {} '{}' contains NULL at {}",
7874
reason == NonNullReason::MAP_KEY ? "map key" : "required field",
79-
path_to_string(path), position_to_string(position));
75+
path_to_string(path), position);
8076
}
8177

82-
bool requires_validation(const NestedField& field, bool force_required = false) {
83-
if (force_required || field.is_required()) {
78+
bool requires_validation(const NestedField& field) {
79+
if (field.is_required()) {
8480
return true;
8581
}
8682
switch (field.field_type()->type_id()) {
@@ -99,12 +95,27 @@ bool requires_validation(const NestedField& field, bool force_required = false)
9995
}
10096
}
10197

102-
Status validate_value(const NestedField& field, const IColumn& column, const DataTypePtr& type,
103-
size_t position, const PositionInfo& position_info, Path& path,
104-
NonNullReason reason);
98+
template <typename Offsets>
99+
Status validate_collection_offsets(const Offsets& offsets, size_t position, size_t row,
100+
std::string_view container_name, const Path& path, size_t* start,
101+
size_t* end) {
102+
if (position >= offsets.size()) {
103+
return Status::InternalError(
104+
"Iceberg {} field '{}' has {} offsets, but position {} is required", container_name,
105+
path_to_string(path), offsets.size(), position);
106+
}
107+
*start = position == 0 ? 0 : offsets[position - 1];
108+
*end = offsets[position];
109+
if (*start > *end) {
110+
return Status::InternalError(
111+
"Iceberg {} field '{}' has invalid offsets at row {}: [{}, {}]", container_name,
112+
path_to_string(path), row, *start, *end);
113+
}
114+
return Status::OK();
115+
}
105116

106117
Status validate_value(const NestedField& field, const IColumn& column, const DataTypePtr& type,
107-
size_t position, const PositionInfo& position_info, Path& path,
118+
size_t position, const ValidationLocation& location, Path& path,
108119
NonNullReason reason) {
109120
if (position >= column.size()) {
110121
return Status::InternalError(
@@ -115,7 +126,7 @@ Status validate_value(const NestedField& field, const IColumn& column, const Dat
115126

116127
if (column.is_null_at(position)) {
117128
if (reason == NonNullReason::MAP_KEY || field.is_required()) {
118-
return null_violation(path, position_info, reason);
129+
return null_violation(path, location, reason);
119130
}
120131
// An optional STRUCT/LIST/MAP is a NULL value in its own right. Its children and
121132
// elements are masked by the parent NULL and must not be inspected.
@@ -155,8 +166,8 @@ Status validate_value(const NestedField& field, const IColumn& column, const Dat
155166
}
156167
path.emplace_back(fields[i].field_name());
157168
Status status = validate_value(fields[i], struct_column->get_column(i),
158-
struct_type->get_element(i), position, position_info,
159-
path, NonNullReason::SCHEMA_REQUIRED);
169+
struct_type->get_element(i), position, location, path,
170+
NonNullReason::SCHEMA_REQUIRED);
160171
path.pop_back();
161172
RETURN_IF_ERROR(status);
162173
}
@@ -174,19 +185,15 @@ Status validate_value(const NestedField& field, const IColumn& column, const Dat
174185
path_to_string(path));
175186
}
176187
const auto& offsets = array_column->get_offsets();
177-
if (position >= offsets.size()) {
178-
return Status::InternalError(
179-
"Iceberg LIST field '{}' has {} offsets, but position {} "
180-
"is required",
181-
path_to_string(path), offsets.size(), position);
182-
}
183-
const size_t start = position == 0 ? 0 : offsets[position - 1];
184-
const size_t end = offsets[position];
185-
if (start > end || end > array_column->get_data().size()) {
188+
size_t start = 0;
189+
size_t end = 0;
190+
RETURN_IF_ERROR(validate_collection_offsets(offsets, position, location.row, "LIST", path,
191+
&start, &end));
192+
if (end > array_column->get_data().size()) {
186193
return Status::InternalError(
187194
"Iceberg LIST field '{}' has invalid offsets at row {}: [{}, {}], "
188195
"element count {}",
189-
path_to_string(path), position_info.row, start, end,
196+
path_to_string(path), location.row, start, end,
190197
array_column->get_data().size());
191198
}
192199
const auto& element = field.field_type()->as_list_type()->element_field();
@@ -195,14 +202,14 @@ Status validate_value(const NestedField& field, const IColumn& column, const Dat
195202
}
196203
path.emplace_back(element.field_name());
197204
for (size_t i = start; i < end; ++i) {
198-
PositionInfo element_position {
199-
.row = position_info.row,
205+
ValidationLocation element_location {
206+
.row = location.row,
200207
.ordinal = i - start,
201208
.ordinal_kind = "element",
202209
};
203210
Status status =
204211
validate_value(element, array_column->get_data(), array_type->get_nested_type(),
205-
i, element_position, path, NonNullReason::SCHEMA_REQUIRED);
212+
i, element_location, path, NonNullReason::SCHEMA_REQUIRED);
206213
if (!status.ok()) {
207214
path.pop_back();
208215
return status;
@@ -223,33 +230,28 @@ Status validate_value(const NestedField& field, const IColumn& column, const Dat
223230
path_to_string(path));
224231
}
225232
const auto& offsets = map_column->get_offsets();
226-
if (position >= offsets.size()) {
227-
return Status::InternalError(
228-
"Iceberg MAP field '{}' has {} offsets, but position {} "
229-
"is required",
230-
path_to_string(path), offsets.size(), position);
231-
}
232-
const size_t start = position == 0 ? 0 : offsets[position - 1];
233-
const size_t end = offsets[position];
234-
if (start > end || end > map_column->get_keys().size() ||
235-
end > map_column->get_values().size()) {
233+
size_t start = 0;
234+
size_t end = 0;
235+
RETURN_IF_ERROR(validate_collection_offsets(offsets, position, location.row, "MAP", path,
236+
&start, &end));
237+
if (end > map_column->get_keys().size() || end > map_column->get_values().size()) {
236238
return Status::InternalError(
237239
"Iceberg MAP field '{}' has invalid offsets at row {}: [{}, {}], "
238240
"key count {}, value count {}",
239-
path_to_string(path), position_info.row, start, end,
240-
map_column->get_keys().size(), map_column->get_values().size());
241+
path_to_string(path), location.row, start, end, map_column->get_keys().size(),
242+
map_column->get_values().size());
241243
}
242244

243245
const auto& map_type_desc = field.field_type()->as_map_type();
244246
path.emplace_back(map_type_desc->key_field().field_name());
245247
for (size_t i = start; i < end; ++i) {
246-
PositionInfo entry_position {
247-
.row = position_info.row,
248+
ValidationLocation entry_location {
249+
.row = location.row,
248250
.ordinal = i - start,
249251
.ordinal_kind = "map entry",
250252
};
251253
Status status = validate_value(map_type_desc->key_field(), map_column->get_keys(),
252-
map_type->get_key_type(), i, entry_position, path,
254+
map_type->get_key_type(), i, entry_location, path,
253255
NonNullReason::MAP_KEY);
254256
if (!status.ok()) {
255257
path.pop_back();
@@ -264,14 +266,14 @@ Status validate_value(const NestedField& field, const IColumn& column, const Dat
264266
}
265267
path.emplace_back(value.field_name());
266268
for (size_t i = start; i < end; ++i) {
267-
PositionInfo entry_position {
268-
.row = position_info.row,
269+
ValidationLocation entry_location {
270+
.row = location.row,
269271
.ordinal = i - start,
270272
.ordinal_kind = "map entry",
271273
};
272274
Status status =
273275
validate_value(value, map_column->get_values(), map_type->get_value_type(), i,
274-
entry_position, path, NonNullReason::SCHEMA_REQUIRED);
276+
entry_location, path, NonNullReason::SCHEMA_REQUIRED);
275277
if (!status.ok()) {
276278
path.pop_back();
277279
return status;
@@ -319,13 +321,12 @@ Status validate_write_block(const Schema& schema, const Block& block) {
319321
if (!column->is_null_at(row)) {
320322
continue;
321323
}
322-
PositionInfo position_info {
324+
ValidationLocation location {
323325
.row = row,
324326
.ordinal = std::nullopt,
325327
.ordinal_kind = {},
326328
};
327-
Status status =
328-
null_violation(path, position_info, NonNullReason::SCHEMA_REQUIRED);
329+
Status status = null_violation(path, location, NonNullReason::SCHEMA_REQUIRED);
329330
path.pop_back();
330331
return status;
331332
}
@@ -334,13 +335,13 @@ Status validate_write_block(const Schema& schema, const Block& block) {
334335
continue;
335336
}
336337
for (size_t row = 0; row < block.rows(); ++row) {
337-
PositionInfo position_info {
338+
ValidationLocation location {
338339
.row = row,
339340
.ordinal = std::nullopt,
340341
.ordinal_kind = {},
341342
};
342-
Status status = validate_value(field, *column, column_with_name.type, row,
343-
position_info, path, NonNullReason::SCHEMA_REQUIRED);
343+
Status status = validate_value(field, *column, column_with_name.type, row, location,
344+
path, NonNullReason::SCHEMA_REQUIRED);
344345
if (!status.ok()) {
345346
path.pop_back();
346347
return status;

0 commit comments

Comments
 (0)