Skip to content

Commit 3b3f2a6

Browse files
morningmanclaude
andcommitted
[fix](be) Report failed _query_plan scans to the external scanner
### What problem does this PR solve? Issue Number: None Related PR: #69771 Problem Summary: The Spark and Flink Doris connectors read a table through FE's _query_plan API and the BE Thrift scanner (open_scanner / get_next), which drains the queue that MemoryScratchSink fills with Arrow batches. When the fragment failed, the sink's close() still put the nullptr sentinel and nothing recorded the failure: no code has called ResultQueueMgr::update_queue_status since the non-pipeline engine was removed (#35803), so RecordBatchQueue::status() stayed OK and get_next reported eos. A scan error, a cancellation or, with this PR, a batch the Arrow Flight convertor rejects (invalid UTF-8, a DATETIME before year 1) ended the read normally with the rows fetched so far: a table with one invalid UTF-8 row read zero rows and no error. MemoryScratchSinkLocalState::close now records a failed exec_status in the queue before the sentinel (update_status ignores OK), and fetch_result checks the queue status again when it takes the sentinel, because a fetch already waiting for data passed the first check before the failure. get_next then returns the error, for example "[INVALID_ARGUMENT]Invalid Arrow Flight result in column 2 (''): Invalid: Invalid UTF8 sequence at string index 1". ### Release note A failed read through the _query_plan scan API (Spark / Flink Doris connectors) now returns the error instead of ending as a complete result. ### Check List (For Author) - Test: Unit Test / Regression test - BE UT ResultQueueMgrTest: a fetch waiting for data reports a failure recorded before the sentinel; an OK status keeps a normal end. - Regression arrow_flight_sql_p0/test_query_plan_scan_validation reads _query_plan scans over the BE Thrift scanner: before this change the invalid UTF-8 scan read 0 rows and no error; with it, the UTF-8 and timestamp scans get the errors and a valid scan reads all rows. - Behavior changed: Yes. Such scans fail instead of returning partial data. - Does this need documentation: No Co-Authored-By: Claude Opus 5.5 <[email protected]>
1 parent 5e729b1 commit 3b3f2a6

4 files changed

Lines changed: 183 additions & 1 deletion

File tree

‎be/src/exec/operator/memory_scratch_sink_operator.cpp‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -57,6 +57,10 @@ Status MemoryScratchSinkLocalState::close(RuntimeState* state, Status exec_statu
5757
SCOPED_TIMER(exec_time_counter());
5858
SCOPED_TIMER(_close_timer);
5959
if (_queue != nullptr) {
60+
// The nullptr sentinel alone reads as a complete result. Record a failed execution (a scan
61+
// error, a rejected Arrow batch, a cancellation) first, so the external scanner reports it
62+
// instead of ending with the batches queued so far. update_status() ignores OK.
63+
_queue->update_status(exec_status);
6064
_queue->blocking_put(nullptr);
6165
}
6266
RETURN_IF_ERROR(Base::close(state, exec_status));

‎be/src/runtime/result_queue_mgr.cpp‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -63,9 +63,12 @@ Status ResultQueueMgr::fetch_result(const TUniqueId& fragment_instance_id,
6363
if (success) {
6464
// sentinel nullptr indicates scan end
6565
if (*result == nullptr) {
66-
*eos = true;
6766
// put sentinel for consistency, avoid repeated invoking fetch result when have no rowbatch
6867
queue->blocking_put(nullptr);
68+
// The sink records a failed execution before it puts the sentinel. Check again here:
69+
// a fetch that was already waiting for data passed the check above before the failure.
70+
RETURN_IF_ERROR(queue->status());
71+
*eos = true;
6972
} else {
7073
*eos = false;
7174
}

‎be/test/runtime/result_queue_mgr_test.cpp‎

Lines changed: 55 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -26,11 +26,14 @@
2626
#include <gtest/gtest-message.h>
2727
#include <gtest/gtest-test-part.h>
2828

29+
#include <chrono>
2930
#include <memory>
3031
#include <ostream>
32+
#include <thread>
3133
#include <utility>
3234
#include <vector>
3335

36+
#include "common/status.h"
3437
#include "gtest/gtest_pred_impl.h"
3538
#include "runtime/record_batch_queue.h"
3639

@@ -134,6 +137,58 @@ TEST_F(ResultQueueMgrTest, fetch_result_end) {
134137
EXPECT_TRUE(result == nullptr);
135138
}
136139

140+
// MemoryScratchSink puts the sentinel when its task closes, also after a failure. A successful
141+
// close records OK, which must leave the sentinel a normal end of the scan.
142+
TEST_F(ResultQueueMgrTest, fetch_result_end_after_ok_status) {
143+
ResultQueueMgr queue_mgr;
144+
TUniqueId query_id;
145+
query_id.lo = 10;
146+
query_id.hi = 100;
147+
148+
BlockQueueSharedPtr block_queue_t;
149+
queue_mgr.create_queue(query_id, &block_queue_t);
150+
block_queue_t->update_status(Status::OK());
151+
block_queue_t->blocking_put(nullptr);
152+
153+
std::shared_ptr<arrow::RecordBatch> result;
154+
bool eos = false;
155+
EXPECT_TRUE(queue_mgr.fetch_result(query_id, &result, &eos).ok());
156+
EXPECT_TRUE(eos);
157+
}
158+
159+
// A failed scan (for example a batch the Arrow Flight convertor rejects) also ends with the
160+
// sentinel. The failure is recorded before it, and must reach the external scanner instead of a
161+
// successful end with the rows fetched so far, also when its fetch already waits for data: that
162+
// fetch passed the status check before the failure happened.
163+
TEST_F(ResultQueueMgrTest, fetch_result_reports_failure_recorded_before_sentinel) {
164+
ResultQueueMgr queue_mgr;
165+
TUniqueId query_id;
166+
query_id.lo = 10;
167+
query_id.hi = 100;
168+
169+
BlockQueueSharedPtr block_queue_t;
170+
queue_mgr.create_queue(query_id, &block_queue_t);
171+
172+
std::shared_ptr<arrow::RecordBatch> waiting_result;
173+
bool waiting_eos = false;
174+
Status waiting_status;
175+
std::thread waiting_fetch([&]() {
176+
waiting_status = queue_mgr.fetch_result(query_id, &waiting_result, &waiting_eos);
177+
});
178+
// Let the fetch pass its first status check and block on the empty queue.
179+
std::this_thread::sleep_for(std::chrono::milliseconds(200));
180+
block_queue_t->update_status(Status::InvalidArgument("rejected Arrow batch"));
181+
block_queue_t->blocking_put(nullptr);
182+
waiting_fetch.join();
183+
EXPECT_TRUE(waiting_status.is<ErrorCode::INVALID_ARGUMENT>()) << waiting_status;
184+
185+
// Later fetches keep reporting the failure instead of the end of the scan.
186+
std::shared_ptr<arrow::RecordBatch> result;
187+
bool eos = false;
188+
const auto status = queue_mgr.fetch_result(query_id, &result, &eos);
189+
EXPECT_TRUE(status.is<ErrorCode::INVALID_ARGUMENT>()) << status;
190+
}
191+
137192
TEST_F(ResultQueueMgrTest, normal_cancel) {
138193
TUniqueId query_id;
139194
query_id.lo = 10;
Lines changed: 120 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,120 @@
1+
// Licensed to the Apache Software Foundation (ASF) under one
2+
// or more contributor license agreements. See the NOTICE file
3+
// distributed with this work for additional information
4+
// regarding copyright ownership. The ASF licenses this file
5+
// to you under the Apache License, Version 2.0 (the
6+
// "License"); you may not use this file except in compliance
7+
// with the License. You may obtain a copy of the License at
8+
//
9+
// http://www.apache.org/licenses/LICENSE-2.0
10+
//
11+
// Unless required by applicable law or agreed to in writing,
12+
// software distributed under the License is distributed on an
13+
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
// KIND, either express or implied. See the License for the
15+
// specific language governing permissions and limitations
16+
// under the License.
17+
18+
import org.apache.arrow.driver.jdbc.shaded.org.apache.arrow.memory.RootAllocator
19+
import org.apache.arrow.driver.jdbc.shaded.org.apache.arrow.vector.ipc.ArrowStreamReader
20+
import org.apache.doris.regression.util.Http
21+
import org.apache.doris.thrift.BackendService
22+
import org.apache.doris.thrift.TScanCloseParams
23+
import org.apache.doris.thrift.TScanNextBatchParams
24+
import org.apache.doris.thrift.TScanOpenParams
25+
import org.apache.doris.thrift.TStatusCode
26+
import org.apache.thrift.protocol.TBinaryProtocol
27+
import org.apache.thrift.transport.TSocket
28+
29+
// The Spark and Flink Doris connectors read a table through FE's _query_plan API and the BE
30+
// Thrift scanner (open_scanner / get_next / close_scanner), and the BE builds the batches with
31+
// the Arrow Flight convertor. A scan that fails on the BE, here on values that convertor
32+
// rejects, must reach the reader as an error instead of a normal end after the rows fetched so
33+
// far (none here).
34+
suite("test_query_plan_scan_validation", "arrow_flight_sql") {
35+
def db = context.dbName
36+
sql "DROP TABLE IF EXISTS query_plan_scan_input"
37+
sql """CREATE TABLE query_plan_scan_input (id INT, encoded STRING, event_time STRING)
38+
DUPLICATE KEY(id) DISTRIBUTED BY HASH(id) BUCKETS 1
39+
PROPERTIES ("replication_num" = "1")"""
40+
sql """INSERT INTO query_plan_scan_input VALUES
41+
(1, '616263', '2024-01-01 00:00:00'), (2, '84', '0000-12-31 00:00:00')"""
42+
43+
// Reads a _query_plan scan the way the connectors do. Returns the number of rows read and
44+
// the error get_next reported, or null when it reported the end of the scan.
45+
def scan = { String query ->
46+
def plan = Http.POST(
47+
"http://${context.config.feHttpAddress}/api/${db}/query_plan_scan_input/_query_plan",
48+
[sql: query], true, context.config.feHttpUser, context.config.feHttpPassword)
49+
assertEquals(0, plan.code, plan.toString())
50+
assertEquals(200, plan.data.status, plan.data.toString())
51+
// One bucket and one replica: the plan has a single tablet on a single BE.
52+
def tablets = plan.data.partitions.keySet().collect { it as long }
53+
String routing = plan.data.partitions.values().first().routings.first()
54+
String host = routing.substring(0, routing.lastIndexOf(':')).replace('[', '').replace(']', '')
55+
int port = routing.substring(routing.lastIndexOf(':') + 1) as int
56+
57+
def socket = new TSocket(host, port, 60000)
58+
socket.open()
59+
def allocator = new RootAllocator()
60+
try {
61+
def client = new BackendService.Client(new TBinaryProtocol(socket))
62+
def params = new TScanOpenParams()
63+
params.setCluster("default_cluster")
64+
params.setDatabase(db)
65+
params.setTable("query_plan_scan_input")
66+
params.setTabletIds(tablets)
67+
params.setOpaquedQueryPlan(plan.data.opaqued_query_plan)
68+
params.setUser(context.config.feHttpUser)
69+
params.setPasswd(context.config.feHttpPassword ?: "")
70+
def opened = client.openScanner(params)
71+
assertEquals(TStatusCode.OK, opened.getStatus().getStatusCode(), opened.toString())
72+
long rows = 0
73+
String error = null
74+
try {
75+
while (true) {
76+
def batch = client.getNext(new TScanNextBatchParams()
77+
.setContextId(opened.getContextId()).setOffset(rows))
78+
if (batch.getStatus().getStatusCode() != TStatusCode.OK) {
79+
error = batch.getStatus().getErrorMsgs().join("; ")
80+
break
81+
}
82+
if (batch.isEos()) {
83+
break
84+
}
85+
def reader = new ArrowStreamReader(new ByteArrayInputStream(batch.getRows()), allocator)
86+
try {
87+
while (reader.loadNextBatch()) {
88+
rows += reader.getVectorSchemaRoot().getRowCount()
89+
}
90+
} finally {
91+
reader.close()
92+
}
93+
}
94+
} finally {
95+
client.closeScanner(new TScanCloseParams().setContextId(opened.getContextId()))
96+
}
97+
return [rows, error]
98+
} finally {
99+
allocator.close()
100+
socket.close()
101+
}
102+
}
103+
104+
// Values the convertor accepts are read in full and the scan ends normally.
105+
def (validRows, validError) = scan("""SELECT id, unhex(encoded), CAST(event_time AS DATETIME(6))
106+
FROM ${db}.query_plan_scan_input WHERE id = 1""")
107+
assertEquals(1L, validRows)
108+
assertNull(validError)
109+
110+
// Invalid UTF-8 in a STRING result.
111+
def (utf8Rows, utf8Error) = scan("SELECT id, unhex(encoded) FROM ${db}.query_plan_scan_input")
112+
assertTrue(utf8Error != null && utf8Error.contains("Invalid UTF8"),
113+
"Expected an invalid UTF-8 scan error, read ${utf8Rows} rows, error: ${utf8Error}")
114+
115+
// A DATETIME before year 1.
116+
def (timeRows, timeError) = scan("""SELECT id, CAST(event_time AS DATETIME(6))
117+
FROM ${db}.query_plan_scan_input""")
118+
assertTrue(timeError != null && timeError.contains("outside the supported 0001-9999 range"),
119+
"Expected an out-of-range timestamp scan error, read ${timeRows} rows, error: ${timeError}")
120+
}

0 commit comments

Comments
 (0)