|
| 1 | +// Licensed to the Apache Software Foundation (ASF) under one |
| 2 | +// or more contributor license agreements. See the NOTICE file |
| 3 | +// distributed with this work for additional information |
| 4 | +// regarding copyright ownership. The ASF licenses this file |
| 5 | +// to you under the Apache License, Version 2.0 (the |
| 6 | +// "License"); you may not use this file except in compliance |
| 7 | +// with the License. You may obtain a copy of the License at |
| 8 | +// |
| 9 | +// http://www.apache.org/licenses/LICENSE-2.0 |
| 10 | +// |
| 11 | +// Unless required by applicable law or agreed to in writing, |
| 12 | +// software distributed under the License is distributed on an |
| 13 | +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
| 14 | +// KIND, either express or implied. See the License for the |
| 15 | +// specific language governing permissions and limitations |
| 16 | +// under the License. |
| 17 | + |
| 18 | +import org.apache.arrow.driver.jdbc.shaded.org.apache.arrow.memory.RootAllocator |
| 19 | +import org.apache.arrow.driver.jdbc.shaded.org.apache.arrow.vector.ipc.ArrowStreamReader |
| 20 | +import org.apache.doris.regression.util.Http |
| 21 | +import org.apache.doris.thrift.BackendService |
| 22 | +import org.apache.doris.thrift.TScanCloseParams |
| 23 | +import org.apache.doris.thrift.TScanNextBatchParams |
| 24 | +import org.apache.doris.thrift.TScanOpenParams |
| 25 | +import org.apache.doris.thrift.TStatusCode |
| 26 | +import org.apache.thrift.protocol.TBinaryProtocol |
| 27 | +import org.apache.thrift.transport.TSocket |
| 28 | + |
| 29 | +// The Spark and Flink Doris connectors read a table through FE's _query_plan API and the BE |
| 30 | +// Thrift scanner (open_scanner / get_next / close_scanner), and the BE builds the batches with |
| 31 | +// the Arrow Flight convertor. A scan that fails on the BE, here on values that convertor |
| 32 | +// rejects, must reach the reader as an error instead of a normal end after the rows fetched so |
| 33 | +// far (none here). |
| 34 | +suite("test_query_plan_scan_validation", "arrow_flight_sql") { |
| 35 | + def db = context.dbName |
| 36 | + sql "DROP TABLE IF EXISTS query_plan_scan_input" |
| 37 | + sql """CREATE TABLE query_plan_scan_input (id INT, encoded STRING, event_time STRING) |
| 38 | + DUPLICATE KEY(id) DISTRIBUTED BY HASH(id) BUCKETS 1 |
| 39 | + PROPERTIES ("replication_num" = "1")""" |
| 40 | + sql """INSERT INTO query_plan_scan_input VALUES |
| 41 | + (1, '616263', '2024-01-01 00:00:00'), (2, '84', '0000-12-31 00:00:00')""" |
| 42 | + |
| 43 | + // Reads a _query_plan scan the way the connectors do. Returns the number of rows read and |
| 44 | + // the error get_next reported, or null when it reported the end of the scan. |
| 45 | + def scan = { String query -> |
| 46 | + def plan = Http.POST( |
| 47 | + "http://${context.config.feHttpAddress}/api/${db}/query_plan_scan_input/_query_plan", |
| 48 | + [sql: query], true, context.config.feHttpUser, context.config.feHttpPassword) |
| 49 | + assertEquals(0, plan.code, plan.toString()) |
| 50 | + assertEquals(200, plan.data.status, plan.data.toString()) |
| 51 | + // One bucket and one replica: the plan has a single tablet on a single BE. |
| 52 | + def tablets = plan.data.partitions.keySet().collect { it as long } |
| 53 | + String routing = plan.data.partitions.values().first().routings.first() |
| 54 | + String host = routing.substring(0, routing.lastIndexOf(':')).replace('[', '').replace(']', '') |
| 55 | + int port = routing.substring(routing.lastIndexOf(':') + 1) as int |
| 56 | + |
| 57 | + def socket = new TSocket(host, port, 60000) |
| 58 | + socket.open() |
| 59 | + def allocator = new RootAllocator() |
| 60 | + try { |
| 61 | + def client = new BackendService.Client(new TBinaryProtocol(socket)) |
| 62 | + def params = new TScanOpenParams() |
| 63 | + params.setCluster("default_cluster") |
| 64 | + params.setDatabase(db) |
| 65 | + params.setTable("query_plan_scan_input") |
| 66 | + params.setTabletIds(tablets) |
| 67 | + params.setOpaquedQueryPlan(plan.data.opaqued_query_plan) |
| 68 | + params.setUser(context.config.feHttpUser) |
| 69 | + params.setPasswd(context.config.feHttpPassword ?: "") |
| 70 | + def opened = client.openScanner(params) |
| 71 | + assertEquals(TStatusCode.OK, opened.getStatus().getStatusCode(), opened.toString()) |
| 72 | + long rows = 0 |
| 73 | + String error = null |
| 74 | + try { |
| 75 | + while (true) { |
| 76 | + def batch = client.getNext(new TScanNextBatchParams() |
| 77 | + .setContextId(opened.getContextId()).setOffset(rows)) |
| 78 | + if (batch.getStatus().getStatusCode() != TStatusCode.OK) { |
| 79 | + error = batch.getStatus().getErrorMsgs().join("; ") |
| 80 | + break |
| 81 | + } |
| 82 | + if (batch.isEos()) { |
| 83 | + break |
| 84 | + } |
| 85 | + def reader = new ArrowStreamReader(new ByteArrayInputStream(batch.getRows()), allocator) |
| 86 | + try { |
| 87 | + while (reader.loadNextBatch()) { |
| 88 | + rows += reader.getVectorSchemaRoot().getRowCount() |
| 89 | + } |
| 90 | + } finally { |
| 91 | + reader.close() |
| 92 | + } |
| 93 | + } |
| 94 | + } finally { |
| 95 | + client.closeScanner(new TScanCloseParams().setContextId(opened.getContextId())) |
| 96 | + } |
| 97 | + return [rows, error] |
| 98 | + } finally { |
| 99 | + allocator.close() |
| 100 | + socket.close() |
| 101 | + } |
| 102 | + } |
| 103 | + |
| 104 | + // Values the convertor accepts are read in full and the scan ends normally. |
| 105 | + def (validRows, validError) = scan("""SELECT id, unhex(encoded), CAST(event_time AS DATETIME(6)) |
| 106 | + FROM ${db}.query_plan_scan_input WHERE id = 1""") |
| 107 | + assertEquals(1L, validRows) |
| 108 | + assertNull(validError) |
| 109 | + |
| 110 | + // Invalid UTF-8 in a STRING result. |
| 111 | + def (utf8Rows, utf8Error) = scan("SELECT id, unhex(encoded) FROM ${db}.query_plan_scan_input") |
| 112 | + assertTrue(utf8Error != null && utf8Error.contains("Invalid UTF8"), |
| 113 | + "Expected an invalid UTF-8 scan error, read ${utf8Rows} rows, error: ${utf8Error}") |
| 114 | + |
| 115 | + // A DATETIME before year 1. |
| 116 | + def (timeRows, timeError) = scan("""SELECT id, CAST(event_time AS DATETIME(6)) |
| 117 | + FROM ${db}.query_plan_scan_input""") |
| 118 | + assertTrue(timeError != null && timeError.contains("outside the supported 0001-9999 range"), |
| 119 | + "Expected an out-of-range timestamp scan error, read ${timeRows} rows, error: ${timeError}") |
| 120 | +} |
0 commit comments