|
23 | 23 | #include <arrow/ipc/api.h> |
24 | 24 | #include <gtest/gtest.h> |
25 | 25 |
|
| 26 | +#include "core/column/column_nullable.h" |
26 | 27 | #include "core/column/column_vector.h" |
27 | 28 | #include "core/data_type/data_type_array.h" |
28 | 29 | #include "core/data_type/data_type_factory.hpp" |
@@ -289,4 +290,136 @@ TEST_F(ArrowBlockConvertorTest, TableConvertersRejectMismatchedNestedSchemas) { |
289 | 290 | } |
290 | 291 | } |
291 | 292 |
|
| 293 | +TEST_F(ArrowBlockConvertorTest, FlightRejectsInvalidUtf8BeforeReturningBatch) { |
| 294 | + auto type = DataTypeFactory::instance().create_data_type(TYPE_STRING, false); |
| 295 | + for (const std::string& value : |
| 296 | + {std::string("\x84"), std::string("\xc0\xaf"), std::string("\xed\xa0\x80"), |
| 297 | + std::string("\xf4\x90\x80\x80")}) { |
| 298 | + auto column = type->create_column(); |
| 299 | + column->insert(Field::create_field<TYPE_STRING>(value)); |
| 300 | + Block block {{std::move(column), type, "payload"}}; |
| 301 | + ArrowFlightArrowBlockConvertor flight(block, "UTC", cctz::utc_time_zone()); |
| 302 | + ASSERT_TRUE(flight.init().ok()); |
| 303 | + const ArrowBlockConvertor& converter = flight; |
| 304 | + std::shared_ptr<arrow::RecordBatch> batch; |
| 305 | + const auto status = converter.convert_to_arrow(block, arrow::default_memory_pool(), &batch); |
| 306 | + EXPECT_EQ(ErrorCode::INVALID_ARGUMENT, status.code()) << status; |
| 307 | + EXPECT_NE(std::string::npos, status.to_string().find("payload")); |
| 308 | + EXPECT_NE(std::string::npos, status.to_string().find("UTF8")); |
| 309 | + EXPECT_EQ(nullptr, batch); |
| 310 | + |
| 311 | + DorisArrowBlockConvertor ordinary(block, "UTC", cctz::utc_time_zone()); |
| 312 | + ASSERT_TRUE(ordinary.init().ok()); |
| 313 | + EXPECT_TRUE(ordinary.convert_to_arrow(block, arrow::default_memory_pool(), &batch).ok()); |
| 314 | + } |
| 315 | +} |
| 316 | + |
| 317 | +TEST_F(ArrowBlockConvertorTest, FlightChecksEachStringAndEveryBatch) { |
| 318 | + auto type = DataTypeFactory::instance().create_data_type(TYPE_STRING, false); |
| 319 | + auto column = type->create_column(); |
| 320 | + for (const std::string& value : |
| 321 | + {std::string("valid"), std::string("\xc2"), std::string("\xa2")}) { |
| 322 | + column->insert(Field::create_field<TYPE_STRING>(value)); |
| 323 | + } |
| 324 | + Block block {{std::move(column), type, "payload"}}; |
| 325 | + ArrowFlightArrowBlockConvertor converter(block, "UTC", cctz::utc_time_zone()); |
| 326 | + ASSERT_TRUE(converter.init().ok()); |
| 327 | + std::shared_ptr<arrow::RecordBatch> batch; |
| 328 | + ASSERT_TRUE(converter.convert_to_arrow(block, arrow::default_memory_pool(), &batch, 0, 1).ok()); |
| 329 | + ASSERT_EQ(1, batch->num_rows()); |
| 330 | + batch.reset(); |
| 331 | + // Adjacent invalid strings form valid UTF-8 when concatenated; row boundaries matter. |
| 332 | + EXPECT_FALSE( |
| 333 | + converter.convert_to_arrow(block, arrow::default_memory_pool(), &batch, 1, 3).ok()); |
| 334 | + EXPECT_EQ(nullptr, batch); |
| 335 | +} |
| 336 | + |
| 337 | +TEST_F(ArrowBlockConvertorTest, FlightPreservesValidTextNullsAndBinary) { |
| 338 | + auto text = make_nullable(DataTypeFactory::instance().create_data_type(TYPE_STRING, false)); |
| 339 | + auto strings = text->create_column(); |
| 340 | + strings->insert(Field::create_field<TYPE_STRING>(std::string("\xe4\xb8\xad\xf0\x9f\x98\x80"))); |
| 341 | + strings->insert(Field::create_field<TYPE_STRING>(std::string())); |
| 342 | + strings->insert(Field::create_field<TYPE_STRING>(std::string("a\0b", 3))); |
| 343 | + strings->insert_default(); |
| 344 | + auto binary = DataTypeFactory::instance().create_data_type(TYPE_VARBINARY, false); |
| 345 | + auto bytes = binary->create_column(); |
| 346 | + for (int i = 0; i < 4; ++i) { |
| 347 | + bytes->insert_data("\x84\0\xff", 3); |
| 348 | + } |
| 349 | + Block block {{std::move(strings), text, "text"}, {std::move(bytes), binary, "binary"}}; |
| 350 | + ArrowFlightArrowBlockConvertor converter(block, "UTC", cctz::utc_time_zone()); |
| 351 | + ASSERT_TRUE(converter.init().ok()); |
| 352 | + std::shared_ptr<arrow::RecordBatch> batch; |
| 353 | + ASSERT_TRUE(converter.convert_to_arrow(block, arrow::default_memory_pool(), &batch).ok()); |
| 354 | + ASSERT_TRUE(batch->ValidateFull().ok()); |
| 355 | + const auto& values = static_cast<const arrow::StringArray&>(*batch->column(0)); |
| 356 | + EXPECT_EQ("\xe4\xb8\xad\xf0\x9f\x98\x80", values.GetString(0)); |
| 357 | + EXPECT_EQ("", values.GetString(1)); |
| 358 | + EXPECT_EQ(std::string("a\0b", 3), values.GetString(2)); |
| 359 | + EXPECT_TRUE(values.IsNull(3)); |
| 360 | + ASSERT_EQ(arrow::Type::BINARY, batch->column(1)->type_id()); |
| 361 | + const auto& binary_values = static_cast<const arrow::BinaryArray&>(*batch->column(1)); |
| 362 | + EXPECT_EQ(std::string("\x84\0\xff", 3), binary_values.GetString(0)); |
| 363 | +} |
| 364 | + |
| 365 | +TEST_F(ArrowBlockConvertorTest, FlightRejectsNestedInvalidUtf8) { |
| 366 | + auto text = make_nullable(DataTypeFactory::instance().create_data_type(TYPE_STRING, false)); |
| 367 | + const auto invalid = Field::create_field<TYPE_STRING>(std::string("\x84")); |
| 368 | + const auto valid = Field::create_field<TYPE_STRING>(std::string("key")); |
| 369 | + DataTypes types {std::make_shared<DataTypeArray>(text), |
| 370 | + std::make_shared<DataTypeStruct>(DataTypes {text}, Strings {"child"}), |
| 371 | + std::make_shared<DataTypeMap>(text, text), |
| 372 | + std::make_shared<DataTypeMap>(text, text)}; |
| 373 | + FieldVector fields { |
| 374 | + Field::create_field<TYPE_ARRAY>(Array {invalid}), |
| 375 | + Field::create_field<TYPE_STRUCT>(Struct {invalid}), |
| 376 | + Field::create_field<TYPE_MAP>(Map {Field::create_field<TYPE_ARRAY>(Array {valid}), |
| 377 | + Field::create_field<TYPE_ARRAY>(Array {invalid})}), |
| 378 | + Field::create_field<TYPE_MAP>(Map {Field::create_field<TYPE_ARRAY>(Array {invalid}), |
| 379 | + Field::create_field<TYPE_ARRAY>(Array {valid})})}; |
| 380 | + for (size_t i = 0; i < types.size(); ++i) { |
| 381 | + SCOPED_TRACE(types[i]->get_name()); |
| 382 | + auto column = types[i]->create_column(); |
| 383 | + column->insert(fields[i]); |
| 384 | + Block block {{std::move(column), types[i], "nested"}}; |
| 385 | + ArrowFlightArrowBlockConvertor converter(block, "UTC", cctz::utc_time_zone()); |
| 386 | + ASSERT_TRUE(converter.init().ok()); |
| 387 | + std::shared_ptr<arrow::RecordBatch> batch; |
| 388 | + const auto status = converter.convert_to_arrow(block, arrow::default_memory_pool(), &batch); |
| 389 | + EXPECT_EQ(ErrorCode::INVALID_ARGUMENT, status.code()) << status; |
| 390 | + EXPECT_NE(std::string::npos, status.to_string().find("nested")); |
| 391 | + EXPECT_EQ(nullptr, batch); |
| 392 | + } |
| 393 | +} |
| 394 | + |
| 395 | +TEST_F(ArrowBlockConvertorTest, FlightRejectsInvalidLargeString) { |
| 396 | + auto type = DataTypeFactory::instance().create_data_type(TYPE_STRING, false); |
| 397 | + auto column = type->create_column(); |
| 398 | + column->insert(Field::create_field<TYPE_STRING>(std::string("\x84"))); |
| 399 | + Block block {{std::move(column), type, "large_text"}}; |
| 400 | + auto schema = arrow::schema({arrow::field("large_text", arrow::large_utf8(), false)}); |
| 401 | + ArrowFlightArrowBlockConvertor converter(schema, cctz::utc_time_zone()); |
| 402 | + std::shared_ptr<arrow::RecordBatch> batch; |
| 403 | + const auto status = converter.convert_to_arrow(block, arrow::default_memory_pool(), &batch); |
| 404 | + EXPECT_EQ(ErrorCode::INVALID_ARGUMENT, status.code()) << status; |
| 405 | + EXPECT_NE(std::string::npos, status.to_string().find("large_text")); |
| 406 | + EXPECT_EQ(nullptr, batch); |
| 407 | +} |
| 408 | + |
| 409 | +TEST_F(ArrowBlockConvertorTest, FlightIgnoresBytesMaskedByNull) { |
| 410 | + auto text = DataTypeFactory::instance().create_data_type(TYPE_STRING, false); |
| 411 | + auto values = text->create_column(); |
| 412 | + values->insert_data("\x84", 1); |
| 413 | + auto nulls = ColumnUInt8::create(); |
| 414 | + nulls->insert_value(1); |
| 415 | + Block block {{ColumnNullable::create(std::move(values), std::move(nulls)), make_nullable(text), |
| 416 | + "nullable_text"}}; |
| 417 | + ArrowFlightArrowBlockConvertor converter(block, "UTC", cctz::utc_time_zone()); |
| 418 | + ASSERT_TRUE(converter.init().ok()); |
| 419 | + std::shared_ptr<arrow::RecordBatch> batch; |
| 420 | + ASSERT_TRUE(converter.convert_to_arrow(block, arrow::default_memory_pool(), &batch).ok()); |
| 421 | + ASSERT_TRUE(batch->ValidateFull().ok()); |
| 422 | + EXPECT_TRUE(batch->column(0)->IsNull(0)); |
| 423 | +} |
| 424 | + |
292 | 425 | } // namespace doris |
0 commit comments