|
| 1 | +// Licensed to the Apache Software Foundation (ASF) under one |
| 2 | +// or more contributor license agreements. See the NOTICE file |
| 3 | +// distributed with this work for additional information |
| 4 | +// regarding copyright ownership. The ASF licenses this file |
| 5 | +// to you under the Apache License, Version 2.0 (the |
| 6 | +// "License"); you may not use this file except in compliance |
| 7 | +// with the License. You may obtain a copy of the License at |
| 8 | +// |
| 9 | +// http://www.apache.org/licenses/LICENSE-2.0 |
| 10 | +// |
| 11 | +// Unless required by applicable law or agreed to in writing, |
| 12 | +// software distributed under the License is distributed on an |
| 13 | +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
| 14 | +// KIND, either express or implied. See the License for the |
| 15 | +// specific language governing permissions and limitations |
| 16 | +// under the License. |
| 17 | + |
| 18 | +package org.apache.doris.connector.iceberg; |
| 19 | + |
| 20 | +import org.apache.doris.connector.cache.CacheSpec; |
| 21 | +import org.apache.doris.connector.cache.MetaCacheEntry; |
| 22 | + |
| 23 | +import org.apache.iceberg.catalog.Namespace; |
| 24 | +import org.apache.iceberg.catalog.TableIdentifier; |
| 25 | + |
| 26 | +import java.util.Objects; |
| 27 | +import java.util.concurrent.ForkJoinPool; |
| 28 | +import java.util.function.Supplier; |
| 29 | + |
| 30 | +/** |
| 31 | + * Per-catalog cache of an iceberg table's inferred write file-format name (PERF-03), keyed by |
| 32 | + * {@code (TableIdentifier, snapshotId)}. Kills the per-query revival of the #64134 heavy fallback: when a table |
| 33 | + * carries neither the {@code write-format} nickname nor {@code write.format.default} (migrated tables, and any |
| 34 | + * table whose writer never set the property — parquet is an implicit default, not stored), |
| 35 | + * {@link IcebergScanPlanProvider#getScanNodeProperties} resolves the scan-level {@code file_format_type} through |
| 36 | + * {@link IcebergWriterHelper#getFileFormat}, which falls back to an unfiltered {@code table.newScan().planFiles()} |
| 37 | + * — a whole-table manifest scan (remote FileIO) — just to read the FIRST data file's format. That fallback was |
| 38 | + * re-run on every query/EXPLAIN with no cross-query reuse; this cache collapses it to one scan per |
| 39 | + * {@code (table, snapshot)}. |
| 40 | + * |
| 41 | + * <p><b>Snapshot-keyed, so always correct.</b> A snapshot is immutable, so its first data file's format is a |
| 42 | + * pure function of the key; a new commit yields a new snapshot id (a new key -> a live scan). Within the TTL |
| 43 | + * the snapshot id itself is held stable by {@link IcebergLatestSnapshotCache}, which is what makes the key stable |
| 44 | + * across queries. The key snapshot is the table's {@code currentSnapshot().snapshotId()} — exactly what the |
| 45 | + * inference reads (it scans {@code table.newScan()}, never the handle's time-travel pin), matching legacy |
| 46 | + * {@code IcebergUtils.getFileFormat} and PERF-02's partition cache verbatim. |
| 47 | + * |
| 48 | + * <p><b>Only the inference fallback is cached.</b> The two cheap property probes ({@code write-format}, |
| 49 | + * {@code write.format.default}) stay outside this cache — the caller consults it only when both miss. |
| 50 | + * |
| 51 | + * <p><b>No credential gate</b> (unlike {@link IcebergTableCache}, like {@link IcebergPartitionCache}): the cached |
| 52 | + * value is a bare format-name {@link String} with no {@code FileIO} / credential, so it is safe to share across |
| 53 | + * users and is built unconditionally (only the TTL knob disables it). |
| 54 | + * |
| 55 | + * <p>Backed identically to {@link IcebergPartitionCache}: a contextual, access-TTL {@link MetaCacheEntry} with |
| 56 | + * manual miss-load, so the inference runs OUTSIDE Caffeine's compute lock and a failed scan's exception |
| 57 | + * propagates verbatim and is NOT cached (the next query retries — legacy parity). TTL is |
| 58 | + * {@code meta.cache.iceberg.table.ttl-second}; {@code <= 0} disables (read live). Lives on the long-lived |
| 59 | + * per-catalog {@link IcebergConnector}; a REFRESH CATALOG rebuilds the connector and thus the cache. |
| 60 | + */ |
| 61 | +final class IcebergFormatCache { |
| 62 | + |
| 63 | + /** Immutable composite key: a table's inferred format is distinct per pinned snapshot id. */ |
| 64 | + static final class Key { |
| 65 | + final TableIdentifier id; |
| 66 | + final long snapshotId; |
| 67 | + |
| 68 | + Key(TableIdentifier id, long snapshotId) { |
| 69 | + this.id = id; |
| 70 | + this.snapshotId = snapshotId; |
| 71 | + } |
| 72 | + |
| 73 | + @Override |
| 74 | + public boolean equals(Object o) { |
| 75 | + if (this == o) { |
| 76 | + return true; |
| 77 | + } |
| 78 | + if (!(o instanceof Key)) { |
| 79 | + return false; |
| 80 | + } |
| 81 | + Key that = (Key) o; |
| 82 | + return snapshotId == that.snapshotId && Objects.equals(id, that.id); |
| 83 | + } |
| 84 | + |
| 85 | + @Override |
| 86 | + public int hashCode() { |
| 87 | + return Objects.hash(id, snapshotId); |
| 88 | + } |
| 89 | + } |
| 90 | + |
| 91 | + private final MetaCacheEntry<Key, String> entry; |
| 92 | + |
| 93 | + IcebergFormatCache(long ttlSeconds, int maxSize) { |
| 94 | + // Mirror IcebergPartitionCache: translate the connector's "<= 0 disables" contract to CacheSpec's ttl == 0 |
| 95 | + // (disabled) rather than passing a negative value through (which CacheSpec reads as "no expiration"). |
| 96 | + CacheSpec spec = ttlSeconds > 0 |
| 97 | + ? CacheSpec.of(true, ttlSeconds, maxSize) |
| 98 | + : CacheSpec.of(true, CacheSpec.CACHE_TTL_DISABLE_CACHE, maxSize); |
| 99 | + this.entry = new MetaCacheEntry<>("iceberg-format", null, spec, |
| 100 | + ForkJoinPool.commonPool(), false, true, 0L, true); |
| 101 | + } |
| 102 | + |
| 103 | + /** Caching is on only when the TTL is positive; ttl-second <= 0 means "always infer live". */ |
| 104 | + boolean isEnabled() { |
| 105 | + return entry.stats().isEffectiveEnabled(); |
| 106 | + } |
| 107 | + |
| 108 | + /** |
| 109 | + * Returns the cached format name for {@code key} if present and unexpired, else runs {@code loader} (the live |
| 110 | + * whole-table format inference), caches and returns it. Disabled cache -> {@code loader} every call. The |
| 111 | + * loader runs OUTSIDE Caffeine's compute lock (single-flight per key) and its exception propagates unwrapped |
| 112 | + * and is NOT cached. |
| 113 | + */ |
| 114 | + String getOrLoad(Key key, Supplier<String> loader) { |
| 115 | + return entry.get(key, ignored -> loader.get()); |
| 116 | + } |
| 117 | + |
| 118 | + /** Drops every cached snapshot entry for one table so the next read infers live (REFRESH TABLE). */ |
| 119 | + void invalidate(TableIdentifier id) { |
| 120 | + entry.invalidateIf(key -> key.id.equals(id)); |
| 121 | + } |
| 122 | + |
| 123 | + /** Drops every cached entry for one database (REFRESH DATABASE / DROP DATABASE); db match = namespace equality. */ |
| 124 | + void invalidateDb(String dbName) { |
| 125 | + Namespace ns = Namespace.of(dbName); |
| 126 | + entry.invalidateIf(key -> key.id.namespace().equals(ns)); |
| 127 | + } |
| 128 | + |
| 129 | + /** Drops all cached entries (REFRESH CATALOG). */ |
| 130 | + void invalidateAll() { |
| 131 | + entry.invalidateAll(); |
| 132 | + } |
| 133 | + |
| 134 | + /** Test-only: current number of cached entries (accurate map membership, not Caffeine's estimate). */ |
| 135 | + int size() { |
| 136 | + int[] count = {0}; |
| 137 | + entry.forEach((key, value) -> count[0]++); |
| 138 | + return count[0]; |
| 139 | + } |
| 140 | + |
| 141 | + /** Test-only: how many times the live loader (the whole-table format inference) actually ran — the metric gate. */ |
| 142 | + long loadCountForTest() { |
| 143 | + return entry.stats().getLoadSuccessCount(); |
| 144 | + } |
| 145 | +} |
0 commit comments