Skip to content

Commit b096062

Browse files
authored
[feature-wip](arrow-flight)(step6) Support regression test (#27847)
Design Documentation Linked to #25514 Regression test add a new group: arrow_flight_sql, ./run-regression-test.sh -g arrow_flight_sql to run regression-test, can use jdbc:arrow-flight-sql to run all Suites whose group contains arrow_flight_sql. ./run-regression-test.sh -g p0,arrow_flight_sql to run regression-test, can use jdbc:arrow-flight-sql to run all Suites whose group contains arrow_flight_sql, and use jdbc:mysql to run other Suites whose group contains p0 but does not contain arrow_flight_sql. Requires attention, the formats of jdbc:arrow-flight-sql and jdbc:mysql and mysql client query results are different, for example: Datatime field type: jdbc:mysql returns 2010-01-02T05:09:06, mysql client returns 2010-01-02 05:09:06, jdbc:arrow-flight-sql also returns 2010-01-02 05:09 :06. Array and Map field types: jdbc:mysql returns ["ab", "efg", null], {"f1": 1, "f2": "a"}, jdbc:arrow-flight-sql returns ["ab ","efg",null], {"f1":1,"f2":"a"}, which is missing spaces. Float field type: jdbc:mysql and mysql client returns 6.333, jdbc:arrow-flight-sql returns 6.333000183105469, in query_p0/subquery/test_subquery.groovy. If the query result is empty, jdbc:arrow-flight-sql returns empty and jdbc:mysql returns \N. use database; and query should be divided into two SQL executions as much as possible. otherwise the results may not be as expected. For example: USE information_schema; select cast ("0.0101031417" as datetime) The result is 2000-01-01 03:14:1 (constant fold), select cast ("0.0101031417" as datetime) The result is null (no constant fold), In addition, doris jdbc:arrow-flight-sql still has unfinished parts, such as: Unsupported data type: Decimal256. INVALID_ARGUMENT: [INTERNAL_ERROR]Fail to convert block data to arrow data, error: [E3] write_column_to_arrow with type Decimal256 Unsupported null value of map key. INVALID_ARGUMENT: [INTERNAL_ERROR]Fail to convert block data to arrow data, error: [E33] Can not write null value of map key to arrow. Unsupported data type: ARRAY<MAP<TEXT,TEXT>> jdbc:arrow-flight-sql not support connecting to specify DB name, such asjdbc:arrow-flight-sql://127.0.0.1:9090/{db_name}", In order to be compatible with regression-test, use db_nameis added before all SQLs whenjdbc:arrow-flight-sql` runs regression test. select timediff("2010-01-01 01:00:00", "2010-01-02 01:00:00");, error java.lang.NumberFormatException: For input string: "-24:00:00"
1 parent e427ac9 commit b096062

61 files changed

Lines changed: 298 additions & 88 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎be/src/common/status.h‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -447,6 +447,7 @@ class [[nodiscard]] Status {
447447
void to_protobuf(PStatus* status) const;
448448

449449
std::string to_string() const;
450+
std::string to_string_no_stack() const;
450451

451452
/// @return A json representation of this status.
452453
std::string to_json() const;
@@ -519,6 +520,13 @@ inline std::string Status::to_string() const {
519520
return ss.str();
520521
}
521522

523+
inline std::string Status::to_string_no_stack() const {
524+
std::stringstream ss;
525+
ss << '[' << code_as_string() << ']';
526+
ss << msg();
527+
return ss.str();
528+
}
529+
522530
// some generally useful macros
523531
#define RETURN_IF_ERROR(stmt) \
524532
do { \

‎be/src/util/arrow/row_batch.cpp‎

Lines changed: 9 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -48,6 +48,9 @@ using strings::Substitute;
4848

4949
Status convert_to_arrow_type(const TypeDescriptor& type, std::shared_ptr<arrow::DataType>* result) {
5050
switch (type.type) {
51+
case TYPE_NULL:
52+
*result = arrow::null();
53+
break;
5154
case TYPE_TINYINT:
5255
*result = arrow::int8();
5356
break;
@@ -79,6 +82,7 @@ Status convert_to_arrow_type(const TypeDescriptor& type, std::shared_ptr<arrow::
7982
case TYPE_DATETIMEV2:
8083
case TYPE_STRING:
8184
case TYPE_JSONB:
85+
case TYPE_OBJECT:
8286
*result = arrow::utf8();
8387
break;
8488
case TYPE_DECIMALV2:
@@ -95,6 +99,9 @@ Status convert_to_arrow_type(const TypeDescriptor& type, std::shared_ptr<arrow::
9599
case TYPE_IPV6:
96100
*result = arrow::utf8();
97101
break;
102+
case TYPE_DECIMAL256:
103+
*result = std::make_shared<arrow::Decimal256Type>(type.precision, type.scale);
104+
break;
98105
case TYPE_BOOLEAN:
99106
*result = arrow::boolean();
100107
break;
@@ -131,7 +138,8 @@ Status convert_to_arrow_type(const TypeDescriptor& type, std::shared_ptr<arrow::
131138
break;
132139
}
133140
default:
134-
return Status::InvalidArgument("Unknown primitive type({})", type.type);
141+
return Status::InvalidArgument("Unknown primitive type({}) convert to Arrow type",
142+
type.type);
135143
}
136144
return Status::OK();
137145
}

‎be/src/util/arrow/utils.cpp‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -36,7 +36,10 @@ arrow::Status to_arrow_status(const Status& status) {
3636
if (status.ok()) {
3737
return arrow::Status::OK();
3838
} else {
39-
return arrow::Status::Invalid(status.to_string());
39+
// The length of exception msg returned to the ADBC Client cannot larger than 8192,
40+
// otherwise ADBC Client will receive:
41+
// `INTERNAL: http2 exception Header size exceeded max allowed size (8192)`.
42+
return arrow::Status::Invalid(status.to_string_no_stack());
4043
}
4144
}
4245

‎fe/fe-core/src/main/java/org/apache/doris/qe/StmtExecutor.java‎

Lines changed: 8 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1417,11 +1417,14 @@ private void handleQueryStmt() throws Exception {
14171417
}
14181418

14191419
// handle selects that fe can do without be, so we can make sql tools happy, especially the setup step.
1420-
Optional<ResultSet> resultSet = planner.handleQueryInFe(parsedStmt);
1421-
if (resultSet.isPresent()) {
1422-
sendResultSet(resultSet.get());
1423-
LOG.info("Query {} finished", DebugUtil.printId(context.queryId));
1424-
return;
1420+
// TODO FE not support doris field type conversion to arrow field type.
1421+
if (!context.getConnectType().equals(ConnectType.ARROW_FLIGHT_SQL)) {
1422+
Optional<ResultSet> resultSet = planner.handleQueryInFe(parsedStmt);
1423+
if (resultSet.isPresent()) {
1424+
sendResultSet(resultSet.get());
1425+
LOG.info("Query {} finished", DebugUtil.printId(context.queryId));
1426+
return;
1427+
}
14251428
}
14261429

14271430
MysqlChannel channel = null;

‎fe/fe-core/src/main/java/org/apache/doris/service/arrowflight/DorisFlightSqlProducer.java‎

Lines changed: 30 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -88,6 +88,12 @@
8888
import java.util.concurrent.ExecutorService;
8989
import java.util.concurrent.Executors;
9090

91+
/**
92+
* Implementation of Arrow Flight SQL service
93+
*
94+
* All methods must catch all possible Exceptions, print and throw CallStatus,
95+
* otherwise error message will be discarded.
96+
*/
9197
public class DorisFlightSqlProducer implements FlightSqlProducer, AutoCloseable {
9298
private static final Logger LOG = LogManager.getLogger(DorisFlightSqlProducer.class);
9399
private final Location location;
@@ -175,9 +181,9 @@ public void closePreparedStatement(final ActionClosePreparedStatementRequest req
175181

176182
private FlightInfo executeQueryStatement(String peerIdentity, ConnectContext connectContext, String query,
177183
final FlightDescriptor descriptor) {
178-
Preconditions.checkState(null != connectContext);
179-
Preconditions.checkState(!query.isEmpty());
180184
try {
185+
Preconditions.checkState(null != connectContext);
186+
Preconditions.checkState(!query.isEmpty());
181187
// After the previous query was executed, there was no getStreamStatement to take away the result.
182188
connectContext.getFlightSqlChannel().reset();
183189
final FlightSqlConnectProcessor flightSQLConnectProcessor = new FlightSqlConnectProcessor(connectContext);
@@ -294,6 +300,9 @@ private ActionCreatePreparedStatementResult buildCreatePreparedStatementResult(B
294300
public void createPreparedStatement(final ActionCreatePreparedStatementRequest request, final CallContext context,
295301
final StreamListener<Result> listener) {
296302
// TODO can only execute complete SQL, not support SQL parameters.
303+
// For Python: the Python code will try to create a prepared statement (this is to fit DBAPI, IIRC) and
304+
// if the server raises any error except for NotImplemented it will fail. (If it gets NotImplemented,
305+
// it will ignore and execute without a prepared statement.) see: https://github.com/apache/arrow/issues/38786
297306
executorService.submit(() -> {
298307
ConnectContext connectContext = flightSessionsManager.getConnectContext(context.peerIdentity());
299308
try {
@@ -344,20 +353,27 @@ public Runnable acceptPutStatement(CommandStatementUpdate command, CallContext c
344353
public Runnable acceptPutPreparedStatementUpdate(CommandPreparedStatementUpdate command, CallContext context,
345354
FlightStream flightStream, StreamListener<PutResult> ackStream) {
346355
return () -> {
347-
while (flightStream.next()) {
348-
final VectorSchemaRoot root = flightStream.getRoot();
349-
final int rowCount = root.getRowCount();
350-
// TODO support update
351-
Preconditions.checkState(rowCount == 0);
352-
353-
final int recordCount = -1;
354-
final DoPutUpdateResult build = DoPutUpdateResult.newBuilder().setRecordCount(recordCount).build();
355-
try (final ArrowBuf buffer = rootAllocator.buffer(build.getSerializedSize())) {
356-
buffer.writeBytes(build.toByteArray());
357-
ackStream.onNext(PutResult.metadata(buffer));
356+
try {
357+
while (flightStream.next()) {
358+
final VectorSchemaRoot root = flightStream.getRoot();
359+
final int rowCount = root.getRowCount();
360+
// TODO support update
361+
Preconditions.checkState(rowCount == 0);
362+
363+
final int recordCount = -1;
364+
final DoPutUpdateResult build = DoPutUpdateResult.newBuilder().setRecordCount(recordCount).build();
365+
try (final ArrowBuf buffer = rootAllocator.buffer(build.getSerializedSize())) {
366+
buffer.writeBytes(build.toByteArray());
367+
ackStream.onNext(PutResult.metadata(buffer));
368+
}
358369
}
370+
ackStream.onCompleted();
371+
} catch (Exception e) {
372+
String errMsg = "acceptPutPreparedStatementUpdate failed, " + e.getMessage() + ", "
373+
+ Util.getRootCauseMessage(e);
374+
LOG.warn(errMsg, e);
375+
throw CallStatus.INTERNAL.withDescription(errMsg).withCause(e).toRuntimeException();
359376
}
360-
ackStream.onCompleted();
361377
};
362378
}
363379

‎fe/fe-core/src/main/java/org/apache/doris/service/arrowflight/DorisFlightSqlService.java‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@
1818
package org.apache.doris.service.arrowflight;
1919

2020
import org.apache.doris.common.Config;
21+
import org.apache.doris.service.FrontendOptions;
2122
import org.apache.doris.service.arrowflight.auth2.FlightBearerTokenAuthenticator;
2223
import org.apache.doris.service.arrowflight.sessions.FlightSessionsManager;
2324
import org.apache.doris.service.arrowflight.sessions.FlightSessionsWithTokenManager;
@@ -39,13 +40,13 @@
3940
public class DorisFlightSqlService {
4041
private static final Logger LOG = LogManager.getLogger(DorisFlightSqlService.class);
4142
private final FlightServer flightServer;
42-
private volatile boolean running;
4343
private final FlightTokenManager flightTokenManager;
4444
private final FlightSessionsManager flightSessionsManager;
45+
private volatile boolean running;
4546

4647
public DorisFlightSqlService(int port) {
4748
BufferAllocator allocator = new RootAllocator();
48-
Location location = Location.forGrpcInsecure("0.0.0.0", port);
49+
Location location = Location.forGrpcInsecure(FrontendOptions.getLocalHostAddress(), port);
4950
this.flightTokenManager = new FlightTokenManagerImpl(Config.arrow_flight_token_cache_size,
5051
Config.arrow_flight_token_alive_time);
5152
this.flightSessionsManager = new FlightSessionsWithTokenManager(flightTokenManager);

‎fe/fe-core/src/main/java/org/apache/doris/service/arrowflight/results/FlightSqlChannel.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -140,7 +140,7 @@ public VectorSchemaRoot createOneOneSchemaRoot(String colName, String colValue)
140140

141141
public void addOKResult(String queryId, String query) {
142142
final FlightSqlResultCacheEntry flightSqlResultCacheEntry = new FlightSqlResultCacheEntry(
143-
createOneOneSchemaRoot("StatusResult", "OK"), query);
143+
createOneOneSchemaRoot("StatusResult", "0"), query);
144144
resultCache.put(queryId, flightSqlResultCacheEntry);
145145
}
146146

‎fe/fe-core/src/main/java/org/apache/doris/service/arrowflight/sessions/FlightSessionsManager.java‎

Lines changed: 0 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -20,13 +20,10 @@
2020

2121
import org.apache.doris.analysis.UserIdentity;
2222
import org.apache.doris.catalog.Env;
23-
import org.apache.doris.common.ErrorCode;
2423
import org.apache.doris.qe.ConnectContext;
2524
import org.apache.doris.service.ExecuteEnv;
2625
import org.apache.doris.system.SystemInfoService;
2726

28-
import org.apache.arrow.flight.CallStatus;
29-
3027
/**
3128
* Manages Flight User Session ConnectContext.
3229
*/
@@ -65,10 +62,6 @@ static ConnectContext buildConnectContext(String peerIdentity, UserIdentity user
6562
connectContext.getEnv().getAuth().getInsertTimeout(connectContext.getQualifiedUser()));
6663

6764
connectContext.setConnectScheduler(ExecuteEnv.getInstance().getScheduler());
68-
if (!ExecuteEnv.getInstance().getScheduler().registerConnection(connectContext)) {
69-
connectContext.getState().setError(ErrorCode.ERR_TOO_MANY_USER_CONNECTIONS, "Reach limit of connections");
70-
throw CallStatus.UNAUTHENTICATED.withDescription("Reach limit of connections").toRuntimeException();
71-
}
7265
return connectContext;
7366
}
7467
}

‎fe/fe-core/src/main/java/org/apache/doris/service/arrowflight/sessions/FlightSessionsWithTokenManager.java‎

Lines changed: 15 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@
1717

1818
package org.apache.doris.service.arrowflight.sessions;
1919

20+
import org.apache.doris.common.ErrorCode;
2021
import org.apache.doris.common.util.Util;
2122
import org.apache.doris.qe.ConnectContext;
2223
import org.apache.doris.service.ExecuteEnv;
@@ -27,13 +28,17 @@
2728
import org.apache.logging.log4j.LogManager;
2829
import org.apache.logging.log4j.Logger;
2930

31+
import java.util.concurrent.atomic.AtomicInteger;
32+
3033
public class FlightSessionsWithTokenManager implements FlightSessionsManager {
3134
private static final Logger LOG = LogManager.getLogger(FlightSessionsWithTokenManager.class);
3235

3336
private final FlightTokenManager flightTokenManager;
37+
private final AtomicInteger nextConnectionId;
3438

3539
public FlightSessionsWithTokenManager(FlightTokenManager flightTokenManager) {
3640
this.flightTokenManager = flightTokenManager;
41+
this.nextConnectionId = new AtomicInteger(0);
3742
}
3843

3944
@Override
@@ -65,8 +70,16 @@ public ConnectContext createConnectContext(String peerIdentity) {
6570
return null;
6671
}
6772
flightTokenDetails.setCreatedSession(true);
68-
return FlightSessionsManager.buildConnectContext(peerIdentity, flightTokenDetails.getUserIdentity(),
69-
flightTokenDetails.getRemoteIp());
73+
ConnectContext connectContext = FlightSessionsManager.buildConnectContext(peerIdentity,
74+
flightTokenDetails.getUserIdentity(), flightTokenDetails.getRemoteIp());
75+
connectContext.setConnectionId(nextConnectionId.getAndAdd(1));
76+
connectContext.resetLoginTime();
77+
if (!ExecuteEnv.getInstance().getScheduler().registerConnection(connectContext)) {
78+
connectContext.getState()
79+
.setError(ErrorCode.ERR_TOO_MANY_USER_CONNECTIONS, "Reach limit of connections");
80+
throw CallStatus.UNAUTHENTICATED.withDescription("Reach limit of connections").toRuntimeException();
81+
}
82+
return connectContext;
7083
} catch (IllegalArgumentException e) {
7184
LOG.error("Bearer token validation failed.", e);
7285
throw CallStatus.UNAUTHENTICATED.toRuntimeException();

‎regression-test/conf/regression-conf.groovy‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -182,6 +182,12 @@ s3Endpoint = "cos.ap-hongkong.myqcloud.com"
182182
s3BucketName = "doris-build-hk-1308700295"
183183
s3Region = "ap-hongkong"
184184

185+
//arrow flight sql test config
186+
extArrowFlightSqlHost = "127.0.0.1"
187+
extArrowFlightSqlPort = 9090
188+
extArrowFlightSqlUser = "root"
189+
extArrowFlightSqlPassword= ""
190+
185191
// iceberg rest catalog config
186192
iceberg_rest_uri_port=18181
187193

0 commit comments

Comments
 (0)