|
| 1 | +// Licensed to the Apache Software Foundation (ASF) under one |
| 2 | +// or more contributor license agreements. See the NOTICE file |
| 3 | +// distributed with this work for additional information |
| 4 | +// regarding copyright ownership. The ASF licenses this file |
| 5 | +// to you under the Apache License, Version 2.0 (the |
| 6 | +// "License"); you may not use this file except in compliance |
| 7 | +// with the License. You may obtain a copy of the License at |
| 8 | +// |
| 9 | +// http://www.apache.org/licenses/LICENSE-2.0 |
| 10 | +// |
| 11 | +// Unless required by applicable law or agreed to in writing, |
| 12 | +// software distributed under the License is distributed on an |
| 13 | +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
| 14 | +// KIND, either express or implied. See the License for the |
| 15 | +// specific language governing permissions and limitations |
| 16 | +// under the License. |
| 17 | + |
| 18 | +package org.apache.doris.nereids.stats; |
| 19 | + |
| 20 | +import org.apache.doris.common.FeConstants; |
| 21 | +import org.apache.doris.statistics.ResultRow; |
| 22 | +import org.apache.doris.statistics.util.StatisticsUtil; |
| 23 | + |
| 24 | +import org.apache.logging.log4j.LogManager; |
| 25 | +import org.apache.logging.log4j.Logger; |
| 26 | + |
| 27 | +import java.util.ArrayList; |
| 28 | +import java.util.List; |
| 29 | + |
| 30 | +/** |
| 31 | + * Persistence of manually injected (pinned) hbo statistics into the internal database |
| 32 | + * ({@code __internal_schema.hbo_statistics}), controlled by |
| 33 | + * {@code Config.hbo_persist_pinned_to_internal_db}. |
| 34 | + * |
| 35 | + * <p>The in-memory {@link HboPlanStatisticsManager} stays authoritative on the read path; |
| 36 | + * SET/DELETE are written through synchronously (best effort, failures only logged) and a FE |
| 37 | + * loads the whole table into memory lazily on first use, so pinned entries survive a FE |
| 38 | + * restart. All FEs share the table through the backend storage. |
| 39 | + */ |
| 40 | +public class HboStatisticsStore { |
| 41 | + private static final Logger LOG = LogManager.getLogger(HboStatisticsStore.class); |
| 42 | + |
| 43 | + private static final String INTERNAL_DB = FeConstants.INTERNAL_DB_NAME; |
| 44 | + private static final String TABLE = "hbo_statistics"; |
| 45 | + private static final String FULL_QUALIFIED = "internal." + INTERNAL_DB + "." + TABLE; |
| 46 | + private static final int STRUCT_MAX_LEN = 65533; |
| 47 | + |
| 48 | + private static final String DDL = "CREATE TABLE IF NOT EXISTS `internal`.`" + INTERNAL_DB + "`.`" + TABLE |
| 49 | + + "` (\n" |
| 50 | + + " `fingerprint` varchar(64) NOT NULL COMMENT \"\",\n" |
| 51 | + + " `row_count` bigint NOT NULL COMMENT \"\",\n" |
| 52 | + + " `node_type` varchar(1024) NULL COMMENT \"\",\n" |
| 53 | + + " `struct_info` varchar(" + STRUCT_MAX_LEN + ") NULL COMMENT \"\",\n" |
| 54 | + + " `create_time_ms` bigint NOT NULL COMMENT \"\"\n" |
| 55 | + + ") ENGINE = olap\n" |
| 56 | + + "UNIQUE KEY(`fingerprint`)\n" |
| 57 | + + "COMMENT \"Doris internal hbo pinned statistics table, DO NOT MODIFY IT\"\n" |
| 58 | + + "DISTRIBUTED BY HASH(`fingerprint`)\n" |
| 59 | + + "BUCKETS 1\n" |
| 60 | + + "PROPERTIES (\"replication_num\" = \"1\")"; |
| 61 | + |
| 62 | + private HboStatisticsStore() { |
| 63 | + } |
| 64 | + |
| 65 | + /** Create the table if it does not exist. */ |
| 66 | + public static void ensureTable() throws Exception { |
| 67 | + StatisticsUtil.execUpdate(DDL); |
| 68 | + } |
| 69 | + |
| 70 | + /** Upsert one pinned entry (UNIQUE KEY fingerprint replaces on conflict). */ |
| 71 | + public static void persist(String fingerprint, long rows, String nodeType, String structCanonical, |
| 72 | + long createTimeMs) { |
| 73 | + try { |
| 74 | + ensureTable(); |
| 75 | + String struct = structCanonical == null ? "" : structCanonical; |
| 76 | + if (struct.length() > STRUCT_MAX_LEN) { |
| 77 | + struct = struct.substring(0, STRUCT_MAX_LEN); |
| 78 | + } |
| 79 | + String sql = "INSERT INTO " + FULL_QUALIFIED |
| 80 | + + " (`fingerprint`, `row_count`, `node_type`, `struct_info`, `create_time_ms`) VALUES ('" |
| 81 | + + escape(fingerprint) + "', " + rows + ", '" + escape(nodeType) + "', '" |
| 82 | + + escape(struct) + "', " + createTimeMs + ")"; |
| 83 | + StatisticsUtil.execUpdate(sql); |
| 84 | + } catch (Throwable t) { |
| 85 | + LOG.warn("failed to persist hbo pinned statistics for fingerprint {}", fingerprint, t); |
| 86 | + } |
| 87 | + } |
| 88 | + |
| 89 | + /** Remove a pinned entry. */ |
| 90 | + public static void delete(String fingerprint) { |
| 91 | + try { |
| 92 | + ensureTable(); |
| 93 | + String sql = "DELETE FROM " + FULL_QUALIFIED + " WHERE `fingerprint` = '" + escape(fingerprint) + "'"; |
| 94 | + StatisticsUtil.execUpdate(sql); |
| 95 | + } catch (Throwable t) { |
| 96 | + LOG.warn("failed to delete hbo pinned statistics for fingerprint {}", fingerprint, t); |
| 97 | + } |
| 98 | + } |
| 99 | + |
| 100 | + /** Load all pinned entries into memory. Returns empty list when the table is not present. */ |
| 101 | + public static List<HboPlanStatisticsManager.PinnedHboStatistics> loadAll() { |
| 102 | + List<HboPlanStatisticsManager.PinnedHboStatistics> result = new ArrayList<>(); |
| 103 | + try { |
| 104 | + ensureTable(); |
| 105 | + List<ResultRow> rows = StatisticsUtil.execStatisticQuery( |
| 106 | + "SELECT `fingerprint`, `row_count`, `node_type`, `struct_info`, `create_time_ms` FROM " |
| 107 | + + FULL_QUALIFIED); |
| 108 | + for (ResultRow row : rows) { |
| 109 | + try { |
| 110 | + result.add(new HboPlanStatisticsManager.PinnedHboStatistics( |
| 111 | + row.get(0), |
| 112 | + Long.parseLong(row.get(1)), |
| 113 | + row.get(2) == null ? "" : row.get(2), |
| 114 | + row.get(3) == null ? "" : row.get(3), |
| 115 | + Long.parseLong(row.get(4)))); |
| 116 | + } catch (NumberFormatException e) { |
| 117 | + LOG.warn("skip malformed hbo pinned statistics row {}", row, e); |
| 118 | + } |
| 119 | + } |
| 120 | + } catch (Throwable t) { |
| 121 | + LOG.warn("failed to load hbo pinned statistics from internal table", t); |
| 122 | + } |
| 123 | + return result; |
| 124 | + } |
| 125 | + |
| 126 | + private static String escape(String value) { |
| 127 | + return value == null ? "" : value.replace("'", "''"); |
| 128 | + } |
| 129 | +} |
0 commit comments