|
24 | 24 | #include <string> |
25 | 25 | #include <utility> |
26 | 26 |
|
| 27 | +#include "common/consts.h" |
| 28 | +#include "core/assert_cast.h" |
| 29 | +#include "core/column/column_const.h" |
| 30 | +#include "core/column/column_nullable.h" |
| 31 | +#include "core/column/column_string.h" |
| 32 | +#include "core/column/column_vector.h" |
27 | 33 | #include "core/data_type/data_type_array.h" |
28 | 34 | #include "core/data_type/data_type_factory.hpp" |
29 | 35 | #include "core/data_type/data_type_map.h" |
30 | 36 | #include "core/data_type/data_type_nullable.h" |
| 37 | +#include "core/data_type/data_type_number.h" |
31 | 38 | #include "core/data_type/data_type_struct.h" |
32 | 39 | #include "core/data_type/data_type_variant_v2.h" |
33 | 40 | #include "exprs/vexpr_context.h" |
|
37 | 44 | #include "format_v2/jni/paimon_jni_reader.h" |
38 | 45 | #include "format_v2/table/schema_history_util.h" |
39 | 46 | #include "gen_cpp/PlanNodes_types.h" |
| 47 | +#include "util/string_util.h" |
40 | 48 |
|
41 | 49 | namespace doris::format::paimon { |
42 | 50 | namespace { |
@@ -229,10 +237,14 @@ Status PaimonReader::prepare_split(const format::SplitReadOptions& options) { |
229 | 237 | SCOPED_TIMER(_profile.total_timer); |
230 | 238 | SCOPED_TIMER(_profile.prepare_split_timer); |
231 | 239 | _split_schema_id = -1; |
| 240 | + _original_file_path.clear(); |
232 | 241 | const auto& paimon_params = options.current_range.table_format_params.paimon_params; |
233 | 242 | if (paimon_params.__isset.schema_id) { |
234 | 243 | _split_schema_id = paimon_params.schema_id; |
235 | 244 | } |
| 245 | + if (paimon_params.__isset.original_file_path) { |
| 246 | + _original_file_path = paimon_params.original_file_path; |
| 247 | + } |
236 | 248 | } |
237 | 249 | RETURN_IF_ERROR(format::TableReader::prepare_split(options)); |
238 | 250 | SCOPED_TIMER(_profile.total_timer); |
@@ -280,10 +292,105 @@ Status PaimonReader::annotate_file_schema(std::vector<format::ColumnDefinition>* |
280 | 292 | Status PaimonReader::customize_file_scan_request(format::FileScanRequest* file_request) { |
281 | 293 | DORIS_CHECK(file_request != nullptr); |
282 | 294 | RETURN_IF_ERROR(format::TableReader::customize_file_scan_request(file_request)); |
| 295 | + if (_need_metadata_columns()) { |
| 296 | + if (_original_file_path.empty()) { |
| 297 | + return Status::InvalidArgument( |
| 298 | + "Paimon metadata columns require a FileScannerV2 native Parquet/ORC split with " |
| 299 | + "the original RawFile path"); |
| 300 | + } |
| 301 | + RETURN_IF_ERROR(_append_row_position_output_column(file_request)); |
| 302 | + } |
283 | 303 | file_request->variant_schema_overrides = _variant_schema_overrides; |
284 | 304 | return Status::OK(); |
285 | 305 | } |
286 | 306 |
|
| 307 | +Status PaimonReader::materialize_virtual_columns(Block* table_block) { |
| 308 | + DORIS_CHECK(table_block != nullptr); |
| 309 | + for (size_t column_idx = 0; column_idx < _data_reader.column_mapper->mappings().size(); |
| 310 | + ++column_idx) { |
| 311 | + const auto& mapping = _data_reader.column_mapper->mappings()[column_idx]; |
| 312 | + switch (mapping.virtual_column_type) { |
| 313 | + case format::TableVirtualColumnType::PAIMON_FILE_PATH: |
| 314 | + RETURN_IF_ERROR(_materialize_file_path(table_block, column_idx)); |
| 315 | + break; |
| 316 | + case format::TableVirtualColumnType::PAIMON_ROW_POSITION: |
| 317 | + RETURN_IF_ERROR(_materialize_row_position(table_block, column_idx)); |
| 318 | + break; |
| 319 | + default: |
| 320 | + break; |
| 321 | + } |
| 322 | + } |
| 323 | + return Status::OK(); |
| 324 | +} |
| 325 | + |
| 326 | +std::string PaimonReader::_data_file_path() const { |
| 327 | + DORIS_CHECK(!_original_file_path.empty()); |
| 328 | + return _original_file_path; |
| 329 | +} |
| 330 | + |
| 331 | +Status PaimonReader::_append_row_position_output_column(format::FileScanRequest* request) { |
| 332 | + const auto row_position_column_id = format::LocalColumnId(format::ROW_POSITION_COLUMN_ID); |
| 333 | + _append_file_scan_column(request, row_position_column_id, &request->non_predicate_columns); |
| 334 | + _row_position_block_position = request->local_positions.at(row_position_column_id).value(); |
| 335 | + return Status::OK(); |
| 336 | +} |
| 337 | + |
| 338 | +Status PaimonReader::_materialize_file_path(Block* table_block, size_t column_idx) { |
| 339 | + DORIS_CHECK(_row_position_block_position < _data_reader.block_template.columns()); |
| 340 | + const auto& row_position_column = assert_cast<const ColumnInt64&>( |
| 341 | + *_data_reader.block_template.get_by_position(_row_position_block_position).column); |
| 342 | + auto column = table_block->get_by_position(column_idx).type->create_column(); |
| 343 | + auto* nullable_column = check_and_get_column<ColumnNullable>(*column); |
| 344 | + auto* string_column = nullable_column != nullptr |
| 345 | + ? check_and_get_column<ColumnString>( |
| 346 | + nullable_column->get_nested_column_ptr().get()) |
| 347 | + : check_and_get_column<ColumnString>(column.get()); |
| 348 | + DORIS_CHECK(string_column != nullptr); |
| 349 | + const auto file_path = _data_file_path(); |
| 350 | + string_column->insert_data(file_path.data(), file_path.size()); |
| 351 | + if (nullable_column != nullptr) { |
| 352 | + nullable_column->get_null_map_data().resize_fill(1, 0); |
| 353 | + } |
| 354 | + table_block->replace_by_position( |
| 355 | + column_idx, ColumnConst::create(std::move(column), row_position_column.size())); |
| 356 | + return Status::OK(); |
| 357 | +} |
| 358 | + |
| 359 | +Status PaimonReader::_materialize_row_position(Block* table_block, size_t column_idx) { |
| 360 | + DORIS_CHECK(_row_position_block_position < _data_reader.block_template.columns()); |
| 361 | + const auto& row_position_column = assert_cast<const ColumnInt64&>( |
| 362 | + *_data_reader.block_template.get_by_position(_row_position_block_position).column); |
| 363 | + auto column = table_block->get_by_position(column_idx).type->create_column(); |
| 364 | + auto* nullable_column = check_and_get_column<ColumnNullable>(*column); |
| 365 | + auto* int_column = nullable_column != nullptr |
| 366 | + ? check_and_get_column<ColumnInt64>( |
| 367 | + nullable_column->get_nested_column_ptr().get()) |
| 368 | + : check_and_get_column<ColumnInt64>(column.get()); |
| 369 | + DORIS_CHECK(int_column != nullptr); |
| 370 | + int_column->get_data().assign(row_position_column.get_data().begin(), |
| 371 | + row_position_column.get_data().end()); |
| 372 | + if (nullable_column != nullptr) { |
| 373 | + nullable_column->get_null_map_data().resize_fill(row_position_column.size(), 0); |
| 374 | + } |
| 375 | + table_block->replace_by_position(column_idx, std::move(column)); |
| 376 | + return Status::OK(); |
| 377 | +} |
| 378 | + |
| 379 | +bool PaimonReader::_need_metadata_columns() const { |
| 380 | + if (_data_reader.column_mapper != nullptr) { |
| 381 | + return std::ranges::any_of(_data_reader.column_mapper->mappings(), [](const auto& mapping) { |
| 382 | + return mapping.virtual_column_type == |
| 383 | + format::TableVirtualColumnType::PAIMON_FILE_PATH || |
| 384 | + mapping.virtual_column_type == |
| 385 | + format::TableVirtualColumnType::PAIMON_ROW_POSITION; |
| 386 | + }); |
| 387 | + } |
| 388 | + return std::ranges::any_of(_projected_columns, [](const auto& column) { |
| 389 | + return iequal(column.name, BeConsts::PAIMON_FILE_PATH_COL) || |
| 390 | + iequal(column.name, BeConsts::PAIMON_ROW_POSITION_COL); |
| 391 | + }); |
| 392 | +} |
| 393 | + |
287 | 394 | Status PaimonReader::_parse_deletion_vector_file(const TTableFormatFileDesc& t_desc, |
288 | 395 | DeleteFileDesc* desc, bool* has_delete_file) { |
289 | 396 | DORIS_CHECK(desc != nullptr); |
|
0 commit comments