Skip to content

Commit dbd1700

Browse files
committed
[fix](arrow) Validate Flight result arrays before publishing batches
Reject malformed UTF-8 with a column-specific server-side error. Validate completed Flight arrays recursively before returning a batch, preserving binary values and NULL handling. Add unit and SQL regression coverage. Validation: four tests reproduced the defect before the fix; all 46 related ASAN converter and Arrow serde tests passed afterward. All changed C++ files passed clang-format 16. The Groovy suite compiled; live Flight SQL regression execution is pending CI.
1 parent dee10ac commit dbd1700

4 files changed

Lines changed: 235 additions & 3 deletions

File tree

‎be/src/format/arrow/arrow_block_convertor.cpp‎

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -341,6 +341,25 @@ Status ArrowBlockConvertor::init() {
341341
return Status::OK();
342342
}
343343

344+
Status ArrowFlightArrowBlockConvertor::convert_to_arrow(const Block& block, arrow::MemoryPool* pool,
345+
std::shared_ptr<arrow::RecordBatch>* result,
346+
size_t start_row, size_t end_row) const {
347+
std::shared_ptr<arrow::RecordBatch> batch;
348+
RETURN_IF_ERROR(ArrowBlockConvertor::convert_to_arrow(block, pool, &batch, start_row, end_row));
349+
// String builders accept arbitrary bytes, but Flight UTF-8 values must be valid per row,
350+
// including nested children. Validate before publishing the batch to the reader.
351+
for (int i = 0; i < batch->num_columns(); ++i) {
352+
auto status = batch->column(i)->ValidateFull();
353+
if (!status.ok()) {
354+
return Status::InvalidArgument("Invalid Arrow Flight result in column {} ('{}'): {}",
355+
i + 1, batch->schema()->field(i)->name(),
356+
status.ToString());
357+
}
358+
}
359+
*result = std::move(batch);
360+
return Status::OK();
361+
}
362+
344363
Status DorisArrowBlockConvertor::init() {
345364
if (_arrow_schema == nullptr) {
346365
// cctz names fixed offsets as "Fixed/UTC+HH:MM:SS", which is not the Arrow

‎be/src/format/arrow/arrow_block_convertor.h‎

Lines changed: 7 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -58,9 +58,9 @@ class ArrowBlockConvertor {
5858
virtual Status init();
5959
const std::shared_ptr<arrow::Schema>& arrow_schema() const { return _arrow_schema; }
6060

61-
Status convert_to_arrow(const Block& block, arrow::MemoryPool* pool,
62-
std::shared_ptr<arrow::RecordBatch>* result, size_t start_row = 0,
63-
size_t end_row = 0) const;
61+
virtual Status convert_to_arrow(const Block& block, arrow::MemoryPool* pool,
62+
std::shared_ptr<arrow::RecordBatch>* result,
63+
size_t start_row = 0, size_t end_row = 0) const;
6464

6565
virtual Status convert_from_arrow(const std::shared_ptr<arrow::RecordBatch>& batch,
6666
const DataTypes& types, Block* block) const;
@@ -115,6 +115,10 @@ class DorisArrowBlockConvertor : public ArrowBlockConvertor {
115115
class ArrowFlightArrowBlockConvertor final : public DorisArrowBlockConvertor {
116116
public:
117117
using DorisArrowBlockConvertor::DorisArrowBlockConvertor;
118+
119+
Status convert_to_arrow(const Block& block, arrow::MemoryPool* pool,
120+
std::shared_ptr<arrow::RecordBatch>* result, size_t start_row = 0,
121+
size_t end_row = 0) const override;
118122
};
119123

120124
class PythonArrowBlockConvertor final : public DorisArrowBlockConvertor {

‎be/test/format/arrow/arrow_block_convertor_test.cpp‎

Lines changed: 133 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,7 @@
2323
#include <arrow/ipc/api.h>
2424
#include <gtest/gtest.h>
2525

26+
#include "core/column/column_nullable.h"
2627
#include "core/column/column_vector.h"
2728
#include "core/data_type/data_type_array.h"
2829
#include "core/data_type/data_type_factory.hpp"
@@ -289,4 +290,136 @@ TEST_F(ArrowBlockConvertorTest, TableConvertersRejectMismatchedNestedSchemas) {
289290
}
290291
}
291292

293+
TEST_F(ArrowBlockConvertorTest, FlightRejectsInvalidUtf8BeforeReturningBatch) {
294+
auto type = DataTypeFactory::instance().create_data_type(TYPE_STRING, false);
295+
for (const std::string& value :
296+
{std::string("\x84"), std::string("\xc0\xaf"), std::string("\xed\xa0\x80"),
297+
std::string("\xf4\x90\x80\x80")}) {
298+
auto column = type->create_column();
299+
column->insert(Field::create_field<TYPE_STRING>(value));
300+
Block block {{std::move(column), type, "payload"}};
301+
ArrowFlightArrowBlockConvertor flight(block, "UTC", cctz::utc_time_zone());
302+
ASSERT_TRUE(flight.init().ok());
303+
const ArrowBlockConvertor& converter = flight;
304+
std::shared_ptr<arrow::RecordBatch> batch;
305+
const auto status = converter.convert_to_arrow(block, arrow::default_memory_pool(), &batch);
306+
EXPECT_EQ(ErrorCode::INVALID_ARGUMENT, status.code()) << status;
307+
EXPECT_NE(std::string::npos, status.to_string().find("payload"));
308+
EXPECT_NE(std::string::npos, status.to_string().find("UTF8"));
309+
EXPECT_EQ(nullptr, batch);
310+
311+
DorisArrowBlockConvertor ordinary(block, "UTC", cctz::utc_time_zone());
312+
ASSERT_TRUE(ordinary.init().ok());
313+
EXPECT_TRUE(ordinary.convert_to_arrow(block, arrow::default_memory_pool(), &batch).ok());
314+
}
315+
}
316+
317+
TEST_F(ArrowBlockConvertorTest, FlightChecksEachStringAndEveryBatch) {
318+
auto type = DataTypeFactory::instance().create_data_type(TYPE_STRING, false);
319+
auto column = type->create_column();
320+
for (const std::string& value :
321+
{std::string("valid"), std::string("\xc2"), std::string("\xa2")}) {
322+
column->insert(Field::create_field<TYPE_STRING>(value));
323+
}
324+
Block block {{std::move(column), type, "payload"}};
325+
ArrowFlightArrowBlockConvertor converter(block, "UTC", cctz::utc_time_zone());
326+
ASSERT_TRUE(converter.init().ok());
327+
std::shared_ptr<arrow::RecordBatch> batch;
328+
ASSERT_TRUE(converter.convert_to_arrow(block, arrow::default_memory_pool(), &batch, 0, 1).ok());
329+
ASSERT_EQ(1, batch->num_rows());
330+
batch.reset();
331+
// Adjacent invalid strings form valid UTF-8 when concatenated; row boundaries matter.
332+
EXPECT_FALSE(
333+
converter.convert_to_arrow(block, arrow::default_memory_pool(), &batch, 1, 3).ok());
334+
EXPECT_EQ(nullptr, batch);
335+
}
336+
337+
TEST_F(ArrowBlockConvertorTest, FlightPreservesValidTextNullsAndBinary) {
338+
auto text = make_nullable(DataTypeFactory::instance().create_data_type(TYPE_STRING, false));
339+
auto strings = text->create_column();
340+
strings->insert(Field::create_field<TYPE_STRING>(std::string("\xe4\xb8\xad\xf0\x9f\x98\x80")));
341+
strings->insert(Field::create_field<TYPE_STRING>(std::string()));
342+
strings->insert(Field::create_field<TYPE_STRING>(std::string("a\0b", 3)));
343+
strings->insert_default();
344+
auto binary = DataTypeFactory::instance().create_data_type(TYPE_VARBINARY, false);
345+
auto bytes = binary->create_column();
346+
for (int i = 0; i < 4; ++i) {
347+
bytes->insert_data("\x84\0\xff", 3);
348+
}
349+
Block block {{std::move(strings), text, "text"}, {std::move(bytes), binary, "binary"}};
350+
ArrowFlightArrowBlockConvertor converter(block, "UTC", cctz::utc_time_zone());
351+
ASSERT_TRUE(converter.init().ok());
352+
std::shared_ptr<arrow::RecordBatch> batch;
353+
ASSERT_TRUE(converter.convert_to_arrow(block, arrow::default_memory_pool(), &batch).ok());
354+
ASSERT_TRUE(batch->ValidateFull().ok());
355+
const auto& values = static_cast<const arrow::StringArray&>(*batch->column(0));
356+
EXPECT_EQ("\xe4\xb8\xad\xf0\x9f\x98\x80", values.GetString(0));
357+
EXPECT_EQ("", values.GetString(1));
358+
EXPECT_EQ(std::string("a\0b", 3), values.GetString(2));
359+
EXPECT_TRUE(values.IsNull(3));
360+
ASSERT_EQ(arrow::Type::BINARY, batch->column(1)->type_id());
361+
const auto& binary_values = static_cast<const arrow::BinaryArray&>(*batch->column(1));
362+
EXPECT_EQ(std::string("\x84\0\xff", 3), binary_values.GetString(0));
363+
}
364+
365+
TEST_F(ArrowBlockConvertorTest, FlightRejectsNestedInvalidUtf8) {
366+
auto text = make_nullable(DataTypeFactory::instance().create_data_type(TYPE_STRING, false));
367+
const auto invalid = Field::create_field<TYPE_STRING>(std::string("\x84"));
368+
const auto valid = Field::create_field<TYPE_STRING>(std::string("key"));
369+
DataTypes types {std::make_shared<DataTypeArray>(text),
370+
std::make_shared<DataTypeStruct>(DataTypes {text}, Strings {"child"}),
371+
std::make_shared<DataTypeMap>(text, text),
372+
std::make_shared<DataTypeMap>(text, text)};
373+
FieldVector fields {
374+
Field::create_field<TYPE_ARRAY>(Array {invalid}),
375+
Field::create_field<TYPE_STRUCT>(Struct {invalid}),
376+
Field::create_field<TYPE_MAP>(Map {Field::create_field<TYPE_ARRAY>(Array {valid}),
377+
Field::create_field<TYPE_ARRAY>(Array {invalid})}),
378+
Field::create_field<TYPE_MAP>(Map {Field::create_field<TYPE_ARRAY>(Array {invalid}),
379+
Field::create_field<TYPE_ARRAY>(Array {valid})})};
380+
for (size_t i = 0; i < types.size(); ++i) {
381+
SCOPED_TRACE(types[i]->get_name());
382+
auto column = types[i]->create_column();
383+
column->insert(fields[i]);
384+
Block block {{std::move(column), types[i], "nested"}};
385+
ArrowFlightArrowBlockConvertor converter(block, "UTC", cctz::utc_time_zone());
386+
ASSERT_TRUE(converter.init().ok());
387+
std::shared_ptr<arrow::RecordBatch> batch;
388+
const auto status = converter.convert_to_arrow(block, arrow::default_memory_pool(), &batch);
389+
EXPECT_EQ(ErrorCode::INVALID_ARGUMENT, status.code()) << status;
390+
EXPECT_NE(std::string::npos, status.to_string().find("nested"));
391+
EXPECT_EQ(nullptr, batch);
392+
}
393+
}
394+
395+
TEST_F(ArrowBlockConvertorTest, FlightRejectsInvalidLargeString) {
396+
auto type = DataTypeFactory::instance().create_data_type(TYPE_STRING, false);
397+
auto column = type->create_column();
398+
column->insert(Field::create_field<TYPE_STRING>(std::string("\x84")));
399+
Block block {{std::move(column), type, "large_text"}};
400+
auto schema = arrow::schema({arrow::field("large_text", arrow::large_utf8(), false)});
401+
ArrowFlightArrowBlockConvertor converter(schema, cctz::utc_time_zone());
402+
std::shared_ptr<arrow::RecordBatch> batch;
403+
const auto status = converter.convert_to_arrow(block, arrow::default_memory_pool(), &batch);
404+
EXPECT_EQ(ErrorCode::INVALID_ARGUMENT, status.code()) << status;
405+
EXPECT_NE(std::string::npos, status.to_string().find("large_text"));
406+
EXPECT_EQ(nullptr, batch);
407+
}
408+
409+
TEST_F(ArrowBlockConvertorTest, FlightIgnoresBytesMaskedByNull) {
410+
auto text = DataTypeFactory::instance().create_data_type(TYPE_STRING, false);
411+
auto values = text->create_column();
412+
values->insert_data("\x84", 1);
413+
auto nulls = ColumnUInt8::create();
414+
nulls->insert_value(1);
415+
Block block {{ColumnNullable::create(std::move(values), std::move(nulls)), make_nullable(text),
416+
"nullable_text"}};
417+
ArrowFlightArrowBlockConvertor converter(block, "UTC", cctz::utc_time_zone());
418+
ASSERT_TRUE(converter.init().ok());
419+
std::shared_ptr<arrow::RecordBatch> batch;
420+
ASSERT_TRUE(converter.convert_to_arrow(block, arrow::default_memory_pool(), &batch).ok());
421+
ASSERT_TRUE(batch->ValidateFull().ok());
422+
EXPECT_TRUE(batch->column(0)->IsNull(0));
423+
}
424+
292425
} // namespace doris
Lines changed: 76 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,76 @@
1+
// Licensed to the Apache Software Foundation (ASF) under one
2+
// or more contributor license agreements. See the NOTICE file
3+
// distributed with this work for additional information
4+
// regarding copyright ownership. The ASF licenses this file
5+
// to you under the Apache License, Version 2.0 (the
6+
// "License"); you may not use this file except in compliance
7+
// with the License. You may obtain a copy of the License at
8+
//
9+
// http://www.apache.org/licenses/LICENSE-2.0
10+
//
11+
// Unless required by applicable law or agreed to in writing,
12+
// software distributed under the License is distributed on an
13+
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
// KIND, either express or implied. See the License for the
15+
// specific language governing permissions and limitations
16+
// under the License.
17+
18+
suite("test_flight_utf8_validation", "arrow_flight_sql") {
19+
sql "DROP TABLE IF EXISTS flight_utf8_input"
20+
sql """CREATE TABLE flight_utf8_input (id INT, encoded STRING)
21+
DUPLICATE KEY(id) DISTRIBUTED BY HASH(id) BUCKETS 1
22+
PROPERTIES ("replication_num" = "1")"""
23+
sql """INSERT INTO flight_utf8_input VALUES
24+
(1, '616263'), (2, 'E4B8ADF09F9880'), (3, ''), (4, NULL),
25+
(5, '84'), (6, 'C0AF'), (7, 'EDA080'), (8, 'F4908080'),
26+
(9, 'C2'), (10, 'A2')"""
27+
28+
// TestAction uses the current JDBC connection, so explicitly bind it to Flight.
29+
def flightUrl = context.getArrowFlightSqlConnection().getMetaData().getURL()
30+
connect(context.config.jdbcUser, context.config.jdbcPassword, flightUrl) {
31+
def input = "${context.dbName}.flight_utf8_input"
32+
for (def id : [5, 6, 7, 8]) {
33+
test {
34+
sql "SELECT unhex(encoded) AS payload FROM ${input} WHERE id = ${id}"
35+
exception "Invalid UTF8"
36+
}
37+
}
38+
test {
39+
sql "SELECT unhex('84') AS payload"
40+
exception "Invalid UTF8"
41+
}
42+
// These bytes are valid only when concatenated; each row must be valid independently.
43+
test {
44+
sql "SELECT unhex(encoded) AS payload FROM ${input} WHERE id IN (9, 10) ORDER BY id"
45+
exception "Invalid UTF8"
46+
}
47+
for (def expression : ["array(unhex(encoded))",
48+
"named_struct('child', unhex(encoded))",
49+
"map('key', unhex(encoded))",
50+
"map(unhex(encoded), 'value')"]) {
51+
test {
52+
sql "SELECT ${expression} AS nested FROM ${input} WHERE id = 5"
53+
exception "Invalid UTF8"
54+
}
55+
}
56+
// Reaching an invalid value after valid rows must fail rather than publish malformed text.
57+
test {
58+
sql "SELECT unhex(encoded) AS payload FROM ${input} ORDER BY id"
59+
exception "Invalid UTF8"
60+
}
61+
test {
62+
sql "SELECT unhex(encoded) FROM ${input} WHERE id <= 4 ORDER BY id"
63+
result [['abc'], ['中😀'], [''], [null]]
64+
}
65+
// Binary payloads may contain arbitrary bytes and must keep their binary Arrow type.
66+
context.getConnection().createStatement().withCloseable { statement ->
67+
statement.executeQuery("SELECT to_binary(encoded) AS payload FROM ${input} WHERE id = 5")
68+
.withCloseable { rows ->
69+
assertTrue(rows.next())
70+
assertTrue(rows.getObject(1) instanceof byte[])
71+
assertEquals([0x84], rows.getBytes(1).collect { it & 0xff })
72+
assertFalse(rows.next())
73+
}
74+
}
75+
}
76+
}

0 commit comments

Comments
 (0)