|
| 1 | +// Licensed to the Apache Software Foundation (ASF) under one |
| 2 | +// or more contributor license agreements. See the NOTICE file |
| 3 | +// distributed with this work for additional information |
| 4 | +// regarding copyright ownership. The ASF licenses this file |
| 5 | +// to you under the Apache License, Version 2.0 (the |
| 6 | +// "License"); you may not use this file except in compliance |
| 7 | +// with the License. You may obtain a copy of the License at |
| 8 | +// |
| 9 | +// http://www.apache.org/licenses/LICENSE-2.0 |
| 10 | +// |
| 11 | +// Unless required by applicable law or agreed to in writing, |
| 12 | +// software distributed under the License is distributed on an |
| 13 | +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
| 14 | +// KIND, either express or implied. See the License for the |
| 15 | +// specific language governing permissions and limitations |
| 16 | +// under the License. |
| 17 | + |
| 18 | +import org.apache.doris.regression.util.JdbcUtils |
| 19 | + |
| 20 | +// Regression for https://github.com/apache/doris/issues/67364 |
| 21 | +// |
| 22 | +// The FE sql cache is shared by every protocol, but its rows are MySQL wire protocol packets that |
| 23 | +// StmtExecutor.sendCachedValues replays through a MysqlChannel, and an Arrow Flight SQL connection |
| 24 | +// has none. Replaying an entry created by an identical MySQL query used to fail |
| 25 | +// Preconditions.checkState(connectType == MYSQL) in StmtExecutor.sendFields() and reach the client |
| 26 | +// as "INTERNAL ... IllegalStateException, msg: null", for any result type. The issue was reported |
| 27 | +// on raw HLL / QUANTILE_STATE columns only because that sql text happened to be the one primed |
| 28 | +// through the MySQL control session. |
| 29 | +// |
| 30 | +// Two setup details decide whether this test can reproduce the bug at all -- get either wrong and |
| 31 | +// it stays green on a broken FE: |
| 32 | +// |
| 33 | +// 1. The flight statements are sent on the raw flight connection. Suite.arrow_flight_sql() |
| 34 | +// prepends "USE <db>;" to the statement, which changes the sql text and therefore the cache |
| 35 | +// key (NereidsSqlCacheManager.generateCacheKey is "<catalog>.<db>:<user>:<sql text>"). |
| 36 | +// 2. Both sessions must agree on every session variable the cache compares |
| 37 | +// (NereidsSqlCacheManager.usedVariablesChanged compares the whole affectQueryResult* set). |
| 38 | +// The MySQL JDBC driver adds STRICT_TRANS_TABLES to sql_mode at connect time while the Arrow |
| 39 | +// Flight JDBC driver does not, and sql_mode is affectQueryResultInPlan, so an unaligned |
| 40 | +// sql_mode alone makes every flight lookup miss. |
| 41 | +suite("test_sql_cache_over_arrow_flight") { |
| 42 | + def mysqlConn = context.getConn() |
| 43 | + def flightConn = context.getArrowFlightSqlConnection() |
| 44 | + |
| 45 | + def runOnMysql = { String stmt -> |
| 46 | + def (result, meta) = JdbcUtils.executeToList(mysqlConn, stmt) |
| 47 | + return result |
| 48 | + } |
| 49 | + def runOnFlight = { String stmt -> |
| 50 | + def (result, meta) = JdbcUtils.executeToList(flightConn, stmt) |
| 51 | + return result |
| 52 | + } |
| 53 | + |
| 54 | + def hasSqlCache = { String stmt -> |
| 55 | + def (explainRows, meta) = JdbcUtils.executeToList(mysqlConn, "explain physical plan " + stmt) |
| 56 | + return explainRows.collect { row -> row.get(0).toString() }.join("\n").contains("PhysicalSqlCache") |
| 57 | + } |
| 58 | + |
| 59 | + // Create the cache entry on the MySQL connection, and wait until an identical statement is |
| 60 | + // actually served from it, so the flight query below really runs against a populated cache. |
| 61 | + def primeSqlCacheOnMysql = { String stmt -> |
| 62 | + for (int i = 0; i < 60; ++i) { |
| 63 | + runOnMysql(stmt) |
| 64 | + if (hasSqlCache(stmt)) { |
| 65 | + return |
| 66 | + } |
| 67 | + sleep(1000) |
| 68 | + } |
| 69 | + throw new IllegalStateException("failed to create sql cache for: " + stmt) |
| 70 | + } |
| 71 | + |
| 72 | + // JdbcUtils renders a binary column as an "0x.." hex string, but falls back to the raw object |
| 73 | + // when the driver does not implement getBytes(). |
| 74 | + def isNonEmptyBinary = { value -> |
| 75 | + if (value == null) { |
| 76 | + return false |
| 77 | + } |
| 78 | + if (value instanceof byte[]) { |
| 79 | + return ((byte[]) value).length > 0 |
| 80 | + } |
| 81 | + return value.toString().length() > "0x".length() |
| 82 | + } |
| 83 | + |
| 84 | + withGlobalLock("cache_last_version_interval_second") { |
| 85 | + runOnMysql "ADMIN SET ALL FRONTENDS CONFIG ('cache_last_version_interval_second' = '0')" |
| 86 | + |
| 87 | + def dbName = context.dbName |
| 88 | + runOnMysql "USE `${dbName}`" |
| 89 | + runOnFlight "USE `${dbName}`" |
| 90 | + runOnMysql "set enable_sql_cache=true" |
| 91 | + runOnFlight "set enable_sql_cache=true" |
| 92 | + // See note 2 above: without this the flight lookup always misses and the test is toothless. |
| 93 | + runOnMysql "set sql_mode='ONLY_FULL_GROUP_BY'" |
| 94 | + runOnFlight "set sql_mode='ONLY_FULL_GROUP_BY'" |
| 95 | + |
| 96 | + // The cache key is the catalog, the database, the user and the sql text, and the lookup |
| 97 | + // additionally compares the session variables that affect the result. The statements below |
| 98 | + // are byte identical on both connections, so assert the rest of the inputs match too. |
| 99 | + assertEquals(runOnMysql("select database()")[0][0], runOnFlight("select database()")[0][0]) |
| 100 | + assertEquals(runOnMysql("select current_user()")[0][0], runOnFlight("select current_user()")[0][0]) |
| 101 | + assertEquals(runOnMysql("select @@sql_mode")[0][0], runOnFlight("select @@sql_mode")[0][0]) |
| 102 | + |
| 103 | + // 1. A constant result, cached in the FE itself (PhysicalOneRowRelation.computeResultInFe |
| 104 | + // -> tryAddFeSqlCache). This replays through the resultSet branch of sendCachedValues, |
| 105 | + // needs no table and no quiet window, and is the cheapest way to hit the bug. |
| 106 | + def constantSql = "select 1 as c, 'x' as s" |
| 107 | + primeSqlCacheOnMysql(constantSql) |
| 108 | + def constantOnFlight = runOnFlight(constantSql) |
| 109 | + assertEquals(1, constantOnFlight.size()) |
| 110 | + assertEquals(1, constantOnFlight[0][0] as int) |
| 111 | + assertEquals("x", constantOnFlight[0][1].toString()) |
| 112 | + |
| 113 | + def tblName = "test_sql_cache_over_arrow_flight_tbl" |
| 114 | + runOnMysql "DROP TABLE IF EXISTS ${tblName}" |
| 115 | + runOnMysql """ |
| 116 | + CREATE TABLE ${tblName} ( |
| 117 | + k INT, |
| 118 | + h HLL HLL_UNION, |
| 119 | + q QUANTILE_STATE QUANTILE_UNION |
| 120 | + ) AGGREGATE KEY(k) |
| 121 | + DISTRIBUTED BY HASH(k) BUCKETS 1 |
| 122 | + PROPERTIES("replication_num"="1") |
| 123 | + """ |
| 124 | + runOnMysql "INSERT INTO ${tblName} SELECT 1, HLL_HASH('x'), TO_QUANTILE_STATE(1, 2048)" |
| 125 | + |
| 126 | + // 2. A plain scalar result read from a table, cached on the BE. The failure was protocol |
| 127 | + // specific, not type specific. |
| 128 | + def scalarSql = "select k from ${tblName} order by k" |
| 129 | + primeSqlCacheOnMysql(scalarSql) |
| 130 | + def scalarOnFlight = runOnFlight(scalarSql) |
| 131 | + assertEquals(1, scalarOnFlight.size()) |
| 132 | + assertEquals(1, scalarOnFlight[0][0] as int) |
| 133 | + |
| 134 | + // 3. The raw aggregate state columns from the issue. HLL and QUANTILE_STATE are carried as |
| 135 | + // arrow binary (be/src/format/arrow/arrow_row_batch.cpp), so flight returns the serialized |
| 136 | + // state, while the MySQL protocol keeps showing NULL under |
| 137 | + // return_object_data_as_binary=false. Asserting both at once also proves the flight result |
| 138 | + // is produced by the BE rather than replayed from the MySQL rows sitting in the cache. |
| 139 | + def rawStateSql = "select h, q from ${tblName}" |
| 140 | + primeSqlCacheOnMysql(rawStateSql) |
| 141 | + def rawStateOnMysql = runOnMysql(rawStateSql) |
| 142 | + assertEquals(1, rawStateOnMysql.size()) |
| 143 | + assertNull(rawStateOnMysql[0][0]) |
| 144 | + assertNull(rawStateOnMysql[0][1]) |
| 145 | + def rawStateOnFlight = runOnFlight(rawStateSql) |
| 146 | + assertEquals(1, rawStateOnFlight.size()) |
| 147 | + assertTrue(isNonEmptyBinary(rawStateOnFlight[0][0]), |
| 148 | + "expect a non empty HLL state over arrow flight, but got: " + rawStateOnFlight[0][0]) |
| 149 | + assertTrue(isNonEmptyBinary(rawStateOnFlight[0][1]), |
| 150 | + "expect a non empty QUANTILE_STATE over arrow flight, but got: " + rawStateOnFlight[0][1]) |
| 151 | + |
| 152 | + // 4. The server side conversions the issue used as a workaround. |
| 153 | + def convertedSql = "select hll_cardinality(h) as c, quantile_percent(q, 0.5) as p from ${tblName}" |
| 154 | + primeSqlCacheOnMysql(convertedSql) |
| 155 | + def convertedOnFlight = runOnFlight(convertedSql) |
| 156 | + assertEquals(1, convertedOnFlight.size()) |
| 157 | + assertEquals(1L, convertedOnFlight[0][0] as long) |
| 158 | + assertEquals(1.0d, convertedOnFlight[0][1] as double, 1e-9) |
| 159 | + |
| 160 | + // The flight queries must not have consumed the cache: a cached plan reaching a non MySQL |
| 161 | + // connection is exactly the crash this test guards against, and the entries must still be |
| 162 | + // there for the MySQL session afterwards. |
| 163 | + assertTrue(hasSqlCache(constantSql)) |
| 164 | + assertTrue(hasSqlCache(scalarSql)) |
| 165 | + assertTrue(hasSqlCache(rawStateSql)) |
| 166 | + assertTrue(hasSqlCache(convertedSql)) |
| 167 | + } |
| 168 | +} |
0 commit comments