@@ -43,7 +43,7 @@ namespace {
4343
4444using Path = std::vector<std::string_view>;
4545
46- struct PositionInfo {
46+ struct ValidationLocation {
4747 size_t row = 0 ;
4848 std::optional<size_t > ordinal;
4949 std::string_view ordinal_kind;
@@ -65,22 +65,18 @@ std::string path_to_string(const Path& path) {
6565 return result;
6666}
6767
68- std::string position_to_string (const PositionInfo& position ) {
69- std::string result = fmt::format (" row {}" , position .row );
70- if (position .ordinal .has_value ()) {
71- result += fmt::format (" , {} {}" , position .ordinal_kind , *position .ordinal );
68+ Status null_violation (const Path& path, const ValidationLocation& location, NonNullReason reason ) {
69+ std::string position = fmt::format (" row {}" , location .row );
70+ if (location .ordinal .has_value ()) {
71+ position += fmt::format (" , {} {}" , location .ordinal_kind , *location .ordinal );
7272 }
73- return result;
74- }
75-
76- Status null_violation (const Path& path, const PositionInfo& position, NonNullReason reason) {
7773 return Status::DataQualityError (" Iceberg {} '{}' contains NULL at {}" ,
7874 reason == NonNullReason::MAP_KEY ? " map key" : " required field" ,
79- path_to_string (path), position_to_string ( position) );
75+ path_to_string (path), position);
8076}
8177
82- bool requires_validation (const NestedField& field, bool force_required = false ) {
83- if (force_required || field.is_required ()) {
78+ bool requires_validation (const NestedField& field) {
79+ if (field.is_required ()) {
8480 return true ;
8581 }
8682 switch (field.field_type ()->type_id ()) {
@@ -99,12 +95,27 @@ bool requires_validation(const NestedField& field, bool force_required = false)
9995 }
10096}
10197
102- Status validate_value (const NestedField& field, const IColumn& column, const DataTypePtr& type,
103- size_t position, const PositionInfo& position_info, Path& path,
104- NonNullReason reason);
98+ template <typename Offsets>
99+ Status validate_collection_offsets (const Offsets& offsets, size_t position, size_t row,
100+ std::string_view container_name, const Path& path, size_t * start,
101+ size_t * end) {
102+ if (position >= offsets.size ()) {
103+ return Status::InternalError (
104+ " Iceberg {} field '{}' has {} offsets, but position {} is required" , container_name,
105+ path_to_string (path), offsets.size (), position);
106+ }
107+ *start = position == 0 ? 0 : offsets[position - 1 ];
108+ *end = offsets[position];
109+ if (*start > *end) {
110+ return Status::InternalError (
111+ " Iceberg {} field '{}' has invalid offsets at row {}: [{}, {}]" , container_name,
112+ path_to_string (path), row, *start, *end);
113+ }
114+ return Status::OK ();
115+ }
105116
106117Status validate_value (const NestedField& field, const IColumn& column, const DataTypePtr& type,
107- size_t position, const PositionInfo& position_info , Path& path,
118+ size_t position, const ValidationLocation& location , Path& path,
108119 NonNullReason reason) {
109120 if (position >= column.size ()) {
110121 return Status::InternalError (
@@ -115,7 +126,7 @@ Status validate_value(const NestedField& field, const IColumn& column, const Dat
115126
116127 if (column.is_null_at (position)) {
117128 if (reason == NonNullReason::MAP_KEY || field.is_required ()) {
118- return null_violation (path, position_info , reason);
129+ return null_violation (path, location , reason);
119130 }
120131 // An optional STRUCT/LIST/MAP is a NULL value in its own right. Its children and
121132 // elements are masked by the parent NULL and must not be inspected.
@@ -155,8 +166,8 @@ Status validate_value(const NestedField& field, const IColumn& column, const Dat
155166 }
156167 path.emplace_back (fields[i].field_name ());
157168 Status status = validate_value (fields[i], struct_column->get_column (i),
158- struct_type->get_element (i), position, position_info ,
159- path, NonNullReason::SCHEMA_REQUIRED );
169+ struct_type->get_element (i), position, location, path ,
170+ NonNullReason::SCHEMA_REQUIRED );
160171 path.pop_back ();
161172 RETURN_IF_ERROR (status);
162173 }
@@ -174,19 +185,15 @@ Status validate_value(const NestedField& field, const IColumn& column, const Dat
174185 path_to_string (path));
175186 }
176187 const auto & offsets = array_column->get_offsets ();
177- if (position >= offsets.size ()) {
178- return Status::InternalError (
179- " Iceberg LIST field '{}' has {} offsets, but position {} "
180- " is required" ,
181- path_to_string (path), offsets.size (), position);
182- }
183- const size_t start = position == 0 ? 0 : offsets[position - 1 ];
184- const size_t end = offsets[position];
185- if (start > end || end > array_column->get_data ().size ()) {
188+ size_t start = 0 ;
189+ size_t end = 0 ;
190+ RETURN_IF_ERROR (validate_collection_offsets (offsets, position, location.row , " LIST" , path,
191+ &start, &end));
192+ if (end > array_column->get_data ().size ()) {
186193 return Status::InternalError (
187194 " Iceberg LIST field '{}' has invalid offsets at row {}: [{}, {}], "
188195 " element count {}" ,
189- path_to_string (path), position_info .row , start, end,
196+ path_to_string (path), location .row , start, end,
190197 array_column->get_data ().size ());
191198 }
192199 const auto & element = field.field_type ()->as_list_type ()->element_field ();
@@ -195,14 +202,14 @@ Status validate_value(const NestedField& field, const IColumn& column, const Dat
195202 }
196203 path.emplace_back (element.field_name ());
197204 for (size_t i = start; i < end; ++i) {
198- PositionInfo element_position {
199- .row = position_info .row ,
205+ ValidationLocation element_location {
206+ .row = location .row ,
200207 .ordinal = i - start,
201208 .ordinal_kind = " element" ,
202209 };
203210 Status status =
204211 validate_value (element, array_column->get_data (), array_type->get_nested_type (),
205- i, element_position , path, NonNullReason::SCHEMA_REQUIRED );
212+ i, element_location , path, NonNullReason::SCHEMA_REQUIRED );
206213 if (!status.ok ()) {
207214 path.pop_back ();
208215 return status;
@@ -223,33 +230,28 @@ Status validate_value(const NestedField& field, const IColumn& column, const Dat
223230 path_to_string (path));
224231 }
225232 const auto & offsets = map_column->get_offsets ();
226- if (position >= offsets.size ()) {
227- return Status::InternalError (
228- " Iceberg MAP field '{}' has {} offsets, but position {} "
229- " is required" ,
230- path_to_string (path), offsets.size (), position);
231- }
232- const size_t start = position == 0 ? 0 : offsets[position - 1 ];
233- const size_t end = offsets[position];
234- if (start > end || end > map_column->get_keys ().size () ||
235- end > map_column->get_values ().size ()) {
233+ size_t start = 0 ;
234+ size_t end = 0 ;
235+ RETURN_IF_ERROR (validate_collection_offsets (offsets, position, location.row , " MAP" , path,
236+ &start, &end));
237+ if (end > map_column->get_keys ().size () || end > map_column->get_values ().size ()) {
236238 return Status::InternalError (
237239 " Iceberg MAP field '{}' has invalid offsets at row {}: [{}, {}], "
238240 " key count {}, value count {}" ,
239- path_to_string (path), position_info .row , start, end,
240- map_column->get_keys (). size (), map_column-> get_values ().size ());
241+ path_to_string (path), location .row , start, end, map_column-> get_keys (). size () ,
242+ map_column->get_values ().size ());
241243 }
242244
243245 const auto & map_type_desc = field.field_type ()->as_map_type ();
244246 path.emplace_back (map_type_desc->key_field ().field_name ());
245247 for (size_t i = start; i < end; ++i) {
246- PositionInfo entry_position {
247- .row = position_info .row ,
248+ ValidationLocation entry_location {
249+ .row = location .row ,
248250 .ordinal = i - start,
249251 .ordinal_kind = " map entry" ,
250252 };
251253 Status status = validate_value (map_type_desc->key_field (), map_column->get_keys (),
252- map_type->get_key_type (), i, entry_position , path,
254+ map_type->get_key_type (), i, entry_location , path,
253255 NonNullReason::MAP_KEY );
254256 if (!status.ok ()) {
255257 path.pop_back ();
@@ -264,14 +266,14 @@ Status validate_value(const NestedField& field, const IColumn& column, const Dat
264266 }
265267 path.emplace_back (value.field_name ());
266268 for (size_t i = start; i < end; ++i) {
267- PositionInfo entry_position {
268- .row = position_info .row ,
269+ ValidationLocation entry_location {
270+ .row = location .row ,
269271 .ordinal = i - start,
270272 .ordinal_kind = " map entry" ,
271273 };
272274 Status status =
273275 validate_value (value, map_column->get_values (), map_type->get_value_type (), i,
274- entry_position , path, NonNullReason::SCHEMA_REQUIRED );
276+ entry_location , path, NonNullReason::SCHEMA_REQUIRED );
275277 if (!status.ok ()) {
276278 path.pop_back ();
277279 return status;
@@ -319,13 +321,12 @@ Status validate_write_block(const Schema& schema, const Block& block) {
319321 if (!column->is_null_at (row)) {
320322 continue ;
321323 }
322- PositionInfo position_info {
324+ ValidationLocation location {
323325 .row = row,
324326 .ordinal = std::nullopt ,
325327 .ordinal_kind = {},
326328 };
327- Status status =
328- null_violation (path, position_info, NonNullReason::SCHEMA_REQUIRED );
329+ Status status = null_violation (path, location, NonNullReason::SCHEMA_REQUIRED );
329330 path.pop_back ();
330331 return status;
331332 }
@@ -334,13 +335,13 @@ Status validate_write_block(const Schema& schema, const Block& block) {
334335 continue ;
335336 }
336337 for (size_t row = 0 ; row < block.rows (); ++row) {
337- PositionInfo position_info {
338+ ValidationLocation location {
338339 .row = row,
339340 .ordinal = std::nullopt ,
340341 .ordinal_kind = {},
341342 };
342- Status status = validate_value (field, *column, column_with_name.type , row,
343- position_info, path, NonNullReason::SCHEMA_REQUIRED );
343+ Status status = validate_value (field, *column, column_with_name.type , row, location,
344+ path, NonNullReason::SCHEMA_REQUIRED );
344345 if (!status.ok ()) {
345346 path.pop_back ();
346347 return status;
0 commit comments