Skip to content

Commit 46625fe

Browse files
committed
[fix](flight) Handle pending Variant subcolumns and verify parallel endpoints
1 parent 0aa34e5 commit 46625fe

4 files changed

Lines changed: 141 additions & 16 deletions

File tree

‎be/src/core/data_type_serde/data_type_variant_serde.cpp‎

Lines changed: 12 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -207,9 +207,18 @@ Status append_legacy_arrow_document(const ColumnVariant& column, size_t index,
207207
subcolumn->data.is_empty_nested(index)) {
208208
continue;
209209
}
210-
fields.push_back({subcolumn->path.get_path(),
211-
subcolumn->data.get_finalized_column_ptr(),
212-
subcolumn->data.get_least_common_type(), index});
210+
if (subcolumn->data.is_finalized()) {
211+
fields.push_back({subcolumn->path.get_path(),
212+
subcolumn->data.get_finalized_column_ptr(),
213+
subcolumn->data.get_least_common_type(), index});
214+
} else {
215+
// Scans may leave lazy defaults or multiple parts. Materialize only this row,
216+
// without mutating shared input or repeatedly copying an entire batch.
217+
auto value = subcolumn->data.cut(index, 1);
218+
value.finalize();
219+
fields.push_back({subcolumn->path.get_path(), value.get_finalized_column_ptr(),
220+
value.get_least_common_type(), 0});
221+
}
213222
}
214223
append_serialized(assert_cast<const ColumnMap&>(*column.get_sparse_column()));
215224
}

‎be/test/format/arrow/arrow_flight_variant_test.cpp‎

Lines changed: 74 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -246,6 +246,80 @@ TEST(ArrowFlightVariantTest, LegacyDocumentsPreserveTypedPaths) {
246246
}
247247
}
248248

249+
TEST(ArrowFlightVariantTest, LegacyPendingDocumentDefaultsPreserveValuesAndSlices) {
250+
auto legacy = ColumnVariant::create(0);
251+
auto root_type = make_nullable(std::make_shared<DataTypeString>());
252+
auto roots = root_type->create_column();
253+
roots->insert_many_defaults(3);
254+
legacy->create_root(root_type, std::move(roots));
255+
auto decimal_type = std::make_shared<DataTypeDecimal128>(20, 2);
256+
auto decimals = ColumnDecimal128V3::create(0, 2);
257+
const Int128 exact = 900719925474099301LL;
258+
decimals->insert_value(Decimal128V3(exact));
259+
ASSERT_TRUE(legacy->add_sub_column(PathInData("amount"), 3));
260+
auto* amount = legacy->get_subcolumn(PathInData("amount"));
261+
*amount = ColumnVariant::Subcolumn(1, true);
262+
amount->insert(decimal_type->get_field_with_data_type(*decimals, 0));
263+
amount->insert_default();
264+
// Scans can return lazy prefix/suffix defaults; output must not finalize the shared input.
265+
ASSERT_FALSE(amount->is_finalized());
266+
auto nulls = ColumnUInt8::create();
267+
nulls->get_data().assign({0, 0, 1});
268+
Block block {{ColumnNullable::create(std::move(legacy), std::move(nulls)),
269+
make_nullable(std::make_shared<DataTypeVariant>()), "v"}};
270+
ArrowFlightArrowBlockConvertor converter(arrow::schema({arrow::field("v", native_variant())}),
271+
cctz::utc_time_zone());
272+
std::shared_ptr<arrow::RecordBatch> batch;
273+
auto status = converter.convert_to_arrow(block, arrow::default_memory_pool(), &batch);
274+
ASSERT_TRUE(status.ok()) << status;
275+
EXPECT_EQ(value_at(*batch->column(0), 0).num_elements(), 0);
276+
VariantRef value;
277+
ASSERT_TRUE(value_at(*batch->column(0), 1).object_find({"amount", 6}, &value));
278+
EXPECT_EQ(value.get_decimal().unscaled, exact);
279+
EXPECT_EQ(value.get_decimal().scale, 2);
280+
EXPECT_TRUE(batch->column(0)->IsNull(2));
281+
ASSERT_TRUE(converter.convert_to_arrow(block, arrow::default_memory_pool(), &batch, 1, 3).ok());
282+
ASSERT_TRUE(value_at(*batch->column(0), 0).object_find({"amount", 6}, &value));
283+
EXPECT_EQ(value.get_decimal().unscaled, exact);
284+
EXPECT_TRUE(batch->column(0)->IsNull(1));
285+
EXPECT_FALSE(amount->is_finalized());
286+
}
287+
288+
TEST(ArrowFlightVariantTest, LegacyScanDocumentWithPendingArrayDefaults) {
289+
auto type = std::make_shared<DataTypeVariant>(9);
290+
auto column = type->create_column();
291+
DataTypeSerDe::FormatOptions options;
292+
for (std::string json : {"42", R"("text")", R"({"a":[1,null,"x"]})", "null"}) {
293+
Slice slice(json.data(), json.size());
294+
ASSERT_TRUE(
295+
type->get_serde()->deserialize_one_cell_from_json(*column, slice, options).ok());
296+
}
297+
auto& legacy = assert_cast<ColumnVariant&>(*column);
298+
legacy.get_subcolumns().get_mutable_root()->data.finalize();
299+
auto* array = legacy.get_subcolumn(PathInData("a"));
300+
ASSERT_NE(array, nullptr);
301+
ASSERT_FALSE(array->is_finalized());
302+
auto nulls = ColumnUInt8::create();
303+
nulls->get_data().assign({0, 0, 0, 1});
304+
Block block {{ColumnNullable::create(std::move(column), std::move(nulls)), make_nullable(type),
305+
"v"}};
306+
ArrowFlightArrowBlockConvertor converter(arrow::schema({arrow::field("v", native_variant())}),
307+
cctz::utc_time_zone());
308+
std::shared_ptr<arrow::RecordBatch> batch;
309+
auto status = converter.convert_to_arrow(block, arrow::default_memory_pool(), &batch);
310+
ASSERT_TRUE(status.ok()) << status;
311+
EXPECT_EQ(value_at(*batch->column(0), 0).get_int(), 42);
312+
EXPECT_EQ(value_at(*batch->column(0), 1).get_string().to_string(), "text");
313+
VariantRef value;
314+
ASSERT_TRUE(value_at(*batch->column(0), 2).object_find({"a", 1}, &value));
315+
ASSERT_EQ(value.num_elements(), 3);
316+
EXPECT_EQ(value.array_at(0).get_int(), 1);
317+
EXPECT_TRUE(value.array_at(1).is_null());
318+
EXPECT_EQ(value.array_at(2).get_string().to_string(), "x");
319+
EXPECT_TRUE(batch->column(0)->IsNull(3));
320+
EXPECT_FALSE(array->is_finalized());
321+
}
322+
249323
TEST(ArrowFlightVariantTest, NativeResultPreservesValuesAndSqlNulls) {
250324
TimezoneUtils::load_timezones_to_cache();
251325
ASSERT_TRUE(register_arrow_variant_extension().ok());

‎fe/fe-core/src/test/java/org/apache/doris/service/arrowflight/FlightSqlNativeVariantTest.java‎

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,7 @@
2828
import org.apache.doris.system.Backend;
2929
import org.apache.doris.system.BackendHbResponse;
3030
import org.apache.doris.system.SystemInfoService;
31+
import org.apache.doris.thrift.TBackendInfo;
3132
import org.apache.doris.thrift.TColumnDesc;
3233
import org.apache.doris.thrift.TDataSink;
3334
import org.apache.doris.thrift.TPrimitiveType;
@@ -162,6 +163,24 @@ public void failedHeartbeatClearsCapabilityBeforeBackendIsMarkedDead() throws Ex
162163
}
163164
}
164165

166+
@Test
167+
public void heartbeatCapabilitiesRemainIndependent() {
168+
// Field 11 already belongs to Paimon; sharing its wire ID would enable the wrong reader.
169+
Assert.assertEquals(11, TBackendInfo._Fields.SUPPORTS_PAIMON_RUST_READER.getThriftFieldId());
170+
Assert.assertEquals(12, TBackendInfo._Fields.ARROW_FLIGHT_NATIVE_VARIANT_SUPPORTED.getThriftFieldId());
171+
Backend backend = new Backend(12349, "127.0.0.1", 9050);
172+
for (boolean paimon : new boolean[] {false, true}) {
173+
for (boolean variant : new boolean[] {false, true}) {
174+
BackendHbResponse response = heartbeat(backend.getId(), variant);
175+
response.setPaimonRustReaderSupported(paimon);
176+
backend.handleHbResponse(GsonUtils.GSON.fromJson(
177+
GsonUtils.GSON.toJson(response), BackendHbResponse.class), true);
178+
Assert.assertEquals(paimon, backend.isPaimonRustReaderSupported());
179+
Assert.assertEquals(variant, backend.isArrowFlightNativeVariantSupported());
180+
}
181+
}
182+
}
183+
165184
private static BackendHbResponse heartbeat(long id, boolean supported) {
166185
BackendHbResponse response = new BackendHbResponse(id, 9060, 8040, 8060,
167186
1, 1, "test", "mix", 0, 0, false, 8815);

‎regression-test/suites/arrow_flight_sql_p0/test_flight_native_variant.groovy‎

Lines changed: 36 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -15,10 +15,12 @@
1515
// specific language governing permissions and limitations
1616
// under the License.
1717

18+
import org.apache.arrow.driver.jdbc.shaded.com.google.protobuf.Any
1819
import org.apache.arrow.driver.jdbc.shaded.org.apache.arrow.flight.CallOptions
1920
import org.apache.arrow.driver.jdbc.shaded.org.apache.arrow.flight.FlightClient
2021
import org.apache.arrow.driver.jdbc.shaded.org.apache.arrow.flight.Location
2122
import org.apache.arrow.driver.jdbc.shaded.org.apache.arrow.flight.sql.FlightSqlClient
23+
import org.apache.arrow.driver.jdbc.shaded.org.apache.arrow.flight.sql.impl.FlightSql
2224
import org.apache.arrow.driver.jdbc.shaded.org.apache.arrow.memory.RootAllocator
2325

2426
import java.util.concurrent.TimeUnit
@@ -39,11 +41,26 @@ suite("test_flight_native_variant", "arrow_flight_sql") {
3941
Location.forGrpcInsecure(frontend.Host.toString(), frontend.ArrowFlightSqlPort.toString().toInteger())).build()
4042
def client = new FlightSqlClient(feClient)
4143
def auth
42-
def read = { String query, Closure inspect ->
44+
def read = { String query, Closure inspect, boolean parallel = false, int resultBackendCount = 1 ->
4345
int count = 0
44-
client.execute(query, auth).endpoints.each { endpoint ->
46+
def info = client.execute(query, auth)
47+
assertFalse(info.endpoints.isEmpty())
48+
// Multiple buckets alone do not prove coverage of native output from different result BEs.
49+
if (parallel) {
50+
def resultAddresses = info.endpoints.collect { endpoint ->
51+
def fields = Any.parseFrom(endpoint.ticket.bytes).unpack(FlightSql.TicketStatementQuery.class)
52+
.statementHandle.toStringUtf8().split("&")
53+
"${fields[1]}:${fields[2]}".toString()
54+
}
55+
assertEquals(info.endpoints.size(), resultAddresses.toSet().size(), "Duplicate result backends")
56+
if (resultBackendCount > 1) {
57+
assertTrue(info.endpoints.size() > 1, "Expected multiple native Variant result backends")
58+
}
59+
}
60+
info.endpoints.each { endpoint ->
4561
FlightClient.builder(allocator, endpoint.locations[0]).build().withCloseable { beClient ->
4662
beClient.getStream(endpoint.ticket, auth, CallOptions.timeout(30, TimeUnit.SECONDS)).withCloseable { stream ->
63+
assertEquals(info.schema, stream.schema, "Result endpoints must share the published schema")
4764
while (stream.next()) {
4865
inspect(stream.root)
4966
count += stream.root.rowCount
@@ -58,19 +75,25 @@ suite("test_flight_native_variant", "arrow_flight_sql") {
5875
auth = feClient.authenticateBasicToken(context.config.otherConfigs.get("extArrowFlightSqlUser"),
5976
context.config.otherConfigs.get("extArrowFlightSqlPassword")).get()
6077
executeSetting("SET enable_sql_cache=false")
78+
executeSetting("SET enable_nereids_distribute_planner=true")
79+
executeSetting("SET parallel_pipeline_task_num=8")
6180
jdbc_sql("DROP TABLE IF EXISTS ${table}")
6281
jdbc_sql("""CREATE TABLE ${table} (id INT, v VARIANT)
63-
DUPLICATE KEY(id) DISTRIBUTED BY HASH(id) BUCKETS 3
82+
DUPLICATE KEY(id) DISTRIBUTED BY HASH(id) BUCKETS 60
6483
PROPERTIES("replication_num"="1")""")
65-
jdbc_sql("""INSERT INTO ${table} VALUES
66-
(1, ${variantV2Function}('42')), (2, ${variantV2Function}('"text"')),
67-
(3, ${variantV2Function}('{"a":[1,null,"x"]}')), (4, NULL)""")
84+
jdbc_sql("""INSERT INTO ${table}
85+
SELECT number + 1, ${variantV2Function}(CASE number % 4
86+
WHEN 0 THEN '42' WHEN 1 THEN '"text"'
87+
WHEN 2 THEN '{"a":[1,null,"x"]}' ELSE NULL END)
88+
FROM numbers("number"="60")""")
89+
def resultBackendCount = jdbc_sql_return_maparray("SHOW TABLETS FROM ${table}")
90+
.collect { it.BackendId }.unique().size()
6891
[false, true].each { parallel ->
6992
executeSetting("SET enable_parallel_result_sink=${parallel}")
7093
[false, true, false].each { nativeVariant ->
7194
executeSetting("SET enable_arrow_flight_sql_native_variant=${nativeVariant}")
7295
def seen = []
73-
assertEquals(4, read("SELECT id, v FROM ${table}", { root ->
96+
assertEquals(60, read("SELECT id, v FROM ${table}", { root ->
7497
def vector = root.getVector(1)
7598
def field = vector.field
7699
if (nativeVariant) {
@@ -87,19 +110,19 @@ suite("test_flight_native_variant", "arrow_flight_sql") {
87110
for (int i = 0; i < root.rowCount; i++) {
88111
int id = root.getVector(0).get(i)
89112
seen.add(id)
90-
assertEquals(id == 4, vector.isNull(i))
91-
if (nativeVariant && id != 4) {
113+
assertEquals(id % 4 == 0, vector.isNull(i))
114+
if (nativeVariant && id % 4 != 0) {
92115
assertTrue(vector.getChild("metadata").get(i).length > 0)
93116
assertTrue(vector.getChild("value").get(i).length > 0)
94-
if (id == 1) {
117+
if (id % 4 == 1) {
95118
assertEquals([12, 42], vector.getChild("value").get(i).collect { it & 0xff })
96119
}
97-
} else if (!nativeVariant && id == 1) {
120+
} else if (!nativeVariant && id % 4 == 1) {
98121
assertEquals("42", vector.getObject(i).toString())
99122
}
100123
}
101-
}))
102-
assertEquals([1, 2, 3, 4], seen.sort())
124+
}, parallel, resultBackendCount))
125+
assertEquals((1..60).toList(), seen.sort())
103126
}
104127
}
105128
executeSetting("SET enable_arrow_flight_sql_native_variant=true")

0 commit comments

Comments
 (0)