Skip to content

Commit 46d4b02

Browse files
committed
[fix](lance) Correct access cache refresh and credential handling
1 parent 75d34a2 commit 46d4b02

7 files changed

Lines changed: 184 additions & 29 deletions

File tree

‎fe/fe-core/src/main/java/org/apache/doris/catalog/RefreshManager.java‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,7 @@
3030
import org.apache.doris.datasource.hive.HMSExternalTable;
3131
import org.apache.doris.datasource.hive.HiveExternalMetaCache;
3232
import org.apache.doris.datasource.iceberg.IcebergExternalTable;
33+
import org.apache.doris.datasource.lance.LanceExternalCatalog;
3334
import org.apache.doris.persist.OperationType;
3435

3536
import com.google.common.base.Strings;
@@ -90,6 +91,8 @@ public void handleRefreshDb(String catalogName, String dbName) throws DdlExcepti
9091
throw new DdlException("Only support refresh database in external catalog");
9192
}
9293
DatabaseIf db = catalog.getDbOrDdlException(dbName);
94+
// Local DB-object eviction also resets metadata; only an explicit refresh retires access.
95+
invalidateLanceTableAccess(catalog);
9396
refreshDbInternal((ExternalDatabase) db);
9497

9598
ExternalObjectLog log = ExternalObjectLog.createForRefreshDb(catalog.getId(), db.getFullName());
@@ -100,7 +103,9 @@ public void replayRefreshDb(ExternalObjectLog log) {
100103
ExternalCatalog catalog = (ExternalCatalog) Env.getCurrentEnv().getCatalogMgr().getCatalog(log.getCatalogId());
101104
if (catalog == null) {
102105
LOG.warn("failed to find catalog when replaying refresh db: {}", log.debugForRefreshDb());
106+
return;
103107
}
108+
invalidateLanceTableAccess(catalog);
104109
Optional<ExternalDatabase<? extends ExternalTable>> db;
105110
if (!Strings.isNullOrEmpty(log.getDbName())) {
106111
db = catalog.getDbForReplay(log.getDbName());
@@ -115,6 +120,14 @@ public void replayRefreshDb(ExternalObjectLog log) {
115120
}
116121
}
117122

123+
private void invalidateLanceTableAccess(CatalogIf catalog) {
124+
// Access entries outlive the bounded DB/table object caches. Replay must invalidate by
125+
// catalog identity before its cache-only object lookup can return early, including ID logs.
126+
if (catalog instanceof LanceExternalCatalog) {
127+
((LanceExternalCatalog) catalog).invalidateTableAccessCache();
128+
}
129+
}
130+
118131
private void refreshDbInternal(ExternalDatabase db) {
119132
db.resetMetaToUninitialized();
120133
LOG.info("refresh database {} in catalog {}", db.getFullName(), db.getCatalog().getName());
@@ -160,6 +173,7 @@ public void replayRefreshTable(ExternalObjectLog log) {
160173
LOG.warn("failed to find catalog when replaying refresh table: {}", log.debugForRefreshTable());
161174
return;
162175
}
176+
invalidateLanceTableAccess(catalog);
163177
Optional<ExternalDatabase<? extends ExternalTable>> db;
164178
if (!Strings.isNullOrEmpty(log.getDbName())) {
165179
db = catalog.getDbForReplay(log.getDbName());

‎fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalMetaCacheMgr.java‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -486,7 +486,6 @@ public void removeCatalogByEngine(long catalogId, String engine) {
486486
}
487487

488488
public void invalidateDb(long catalogId, String dbName) {
489-
invalidateLanceTableAccess(catalogId);
490489
routeCatalogEngines(catalogId, cache -> safeInvalidate(
491490
cache, catalogId, "invalidateDb", () -> cache.invalidateDb(catalogId, dbName)));
492491
}

‎fe/fe-core/src/main/java/org/apache/doris/datasource/lance/LanceCatalogClient.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -120,7 +120,7 @@ static LanceCatalogClient create(AbstractLanceProperties properties,
120120
this.session = session;
121121
this.namespaceClient = new LanceNamespaceClient(
122122
namespace, catalogType, rootDatabase, parentNamespace, storageProperties,
123-
tableAccessCacheTtlSeconds, Ticker.systemTicker(), System::currentTimeMillis);
123+
tableAccessCacheTtlSeconds, Ticker.systemTicker());
124124
this.namespaceStorageOptions = Collections.unmodifiableMap(new HashMap<>(namespaceStorageOptions));
125125
}
126126

‎fe/fe-core/src/main/java/org/apache/doris/datasource/lance/LanceExternalCatalog.java‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -277,6 +277,13 @@ synchronized LanceCatalogClient.Lease acquireClient() {
277277
return client.acquire();
278278
}
279279

280+
@Override
281+
public void unregisterDatabase(String dbName) {
282+
// Dropping a namespace is a semantic change, unlike routine local DB-object eviction.
283+
invalidateTableAccessCache();
284+
super.unregisterDatabase(dbName);
285+
}
286+
280287
public synchronized void invalidateTableAccessCache() {
281288
// Invalidation must not initialize JNI resources or wait for an in-flight namespace call.
282289
if (client != null) {

‎fe/fe-core/src/main/java/org/apache/doris/datasource/lance/LanceNamespaceClient.java‎

Lines changed: 18 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -39,6 +39,8 @@
3939
import org.lance.namespace.model.ListTablesResponse;
4040
import org.lance.namespace.model.TableExistsRequest;
4141

42+
import java.net.URI;
43+
import java.net.URISyntaxException;
4244
import java.util.ArrayDeque;
4345
import java.util.ArrayList;
4446
import java.util.Collections;
@@ -49,7 +51,6 @@
4951
import java.util.Queue;
5052
import java.util.Set;
5153
import java.util.concurrent.TimeUnit;
52-
import java.util.function.LongSupplier;
5354

5455
/**
5556
* Namespace requests and table access resolution. The catalog client owns the native resources
@@ -68,27 +69,25 @@ final class LanceNamespaceClient {
6869
private final Object namespaceLock = new Object();
6970
private final long tableAccessTtlNanos;
7071
private final Ticker ticker;
71-
private final LongSupplier currentTimeMillis;
7272
private volatile Cache<List<String>, CachedTableAccess> tableAccessCache;
7373

7474
LanceNamespaceClient(LanceNamespace namespace, String catalogType, String rootDatabase,
7575
List<String> parentNamespace, List<StorageProperties> storageProperties) {
7676
this(namespace, catalogType, rootDatabase, parentNamespace, storageProperties,
7777
AbstractLanceProperties.DEFAULT_TABLE_ACCESS_CACHE_TTL_SECONDS,
78-
Ticker.systemTicker(), System::currentTimeMillis);
78+
Ticker.systemTicker());
7979
}
8080

8181
LanceNamespaceClient(LanceNamespace namespace, String catalogType, String rootDatabase,
8282
List<String> parentNamespace, List<StorageProperties> storageProperties,
83-
long tableAccessTtlSeconds, Ticker ticker, LongSupplier currentTimeMillis) {
83+
long tableAccessTtlSeconds, Ticker ticker) {
8484
this.namespace = namespace;
8585
this.catalogType = catalogType;
8686
this.rootDatabase = rootDatabase;
8787
this.parentNamespace = Collections.unmodifiableList(new ArrayList<>(parentNamespace));
8888
this.storageProperties = Collections.unmodifiableList(new ArrayList<>(storageProperties));
8989
this.tableAccessTtlNanos = TimeUnit.SECONDS.toNanos(tableAccessTtlSeconds);
9090
this.ticker = ticker;
91-
this.currentTimeMillis = currentTimeMillis;
9291
this.tableAccessCache = newTableAccessCache();
9392
}
9493

@@ -243,28 +242,27 @@ private CachedTableAccess loadTableAccess(List<String> tableId) {
243242
Map<String, String> storageOptions = LanceStorageOptions.fromDorisAndVendedStorageOptions(datasetUri,
244243
storageProperties, table.getStorageOptions());
245244
return new CachedTableAccess(new LanceTableAccess(datasetUri, storageOptions),
246-
tableAccessTtlNanos(table.getStorageOptions()));
245+
tableAccessTtlNanos(datasetUri, table.getStorageOptions()));
247246
}
248247

249-
private long tableAccessTtlNanos(Map<String, String> vendedOptions) {
250-
if (vendedOptions == null || vendedOptions.isEmpty()) {
251-
return tableAccessTtlNanos;
252-
}
253-
// Vended options can contain temporary credentials. Never assume they are permanent
254-
// when expiry is absent, and reserve time for planning and dispatch to the BE.
255-
String expiry = vendedOptions.get("expires_at_millis");
256-
if (expiry == null) {
248+
private long tableAccessTtlNanos(String datasetUri, Map<String, String> vendedOptions) {
249+
// The BE cannot renew credentials during a scan. A fixed expiry margin cannot cover
250+
// arbitrary query durations, so preserve per-read vending even with a reported deadline.
251+
if (vendedOptions != null && !vendedOptions.isEmpty()) {
257252
return 0;
258253
}
259254
try {
260-
long expiresAtMillis = Long.parseLong(expiry);
261-
long now = currentTimeMillis.getAsLong();
262-
if (expiresAtMillis <= now) {
255+
URI uri = new URI(datasetUri.trim());
256+
// Presigned/SAS credentials may live in the URI even when storage_options is empty.
257+
// Also check registry-based authorities, for which URI.getRawUserInfo() returns null.
258+
if (uri.isOpaque() || uri.getRawUserInfo() != null || uri.getRawQuery() != null
259+
|| uri.getRawFragment() != null
260+
|| (uri.getRawAuthority() != null && uri.getRawAuthority().contains("@"))) {
263261
return 0;
264262
}
265-
long remainingMillis = Math.max(0, expiresAtMillis - now - TimeUnit.SECONDS.toMillis(30));
266-
return Math.min(tableAccessTtlNanos, TimeUnit.MILLISECONDS.toNanos(remainingMillis));
267-
} catch (NumberFormatException e) {
263+
return tableAccessTtlNanos;
264+
} catch (URISyntaxException e) {
265+
// Unclassified locators remain usable but must not be assumed credential-free.
268266
return 0;
269267
}
270268
}

‎fe/fe-core/src/test/java/org/apache/doris/datasource/lance/LanceCatalogLifecycleTest.java‎

Lines changed: 122 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -18,10 +18,13 @@
1818
package org.apache.doris.datasource.lance;
1919

2020
import org.apache.doris.catalog.Env;
21+
import org.apache.doris.catalog.RefreshManager;
2122
import org.apache.doris.datasource.CatalogMgr;
2223
import org.apache.doris.datasource.ExternalCatalog;
2324
import org.apache.doris.datasource.ExternalMetaCacheMgr;
25+
import org.apache.doris.datasource.ExternalObjectLog;
2426
import org.apache.doris.datasource.lance.job.LanceIndexDatasetLocator;
27+
import org.apache.doris.persist.EditLog;
2528

2629
import org.apache.arrow.memory.BufferAllocator;
2730
import org.junit.jupiter.api.Assertions;
@@ -37,6 +40,7 @@
3740
import java.util.Collections;
3841
import java.util.HashMap;
3942
import java.util.Map;
43+
import java.util.Optional;
4044
import java.util.concurrent.CountDownLatch;
4145
import java.util.concurrent.ExecutorService;
4246
import java.util.concurrent.Executors;
@@ -208,7 +212,7 @@ public void testMetadataRefreshInvalidatesAccessWithoutClosingSession() throws E
208212
caches.invalidateTable(catalog.getId(), "mapped_db", "mapped_table");
209213
caches.invalidateDb(catalog.getId(), "mapped_db");
210214
caches.invalidateCatalog(catalog.getId());
211-
Mockito.verify(client, Mockito.times(3)).invalidateTableAccessCache();
215+
Mockito.verify(client, Mockito.times(2)).invalidateTableAccessCache();
212216
Mockito.verify(session, Mockito.never()).close();
213217
Mockito.verify(catalog, Mockito.never()).createClient();
214218
} finally {
@@ -235,6 +239,123 @@ public void testIndexJobLocatorBypassesQueryAccessCache() throws Exception {
235239
}
236240
}
237241

242+
@Test
243+
public void testRoutineDatabaseObjectCleanupPreservesHotAccess() throws Exception {
244+
try (AccessFixture fixture = new AccessFixture()) {
245+
LanceNamespaceClient access = fixture.access();
246+
access.resolveTableAccess("default", "items");
247+
new LanceExternalDatabase(fixture.catalog, 1, "cold_db", "cold_db").resetMetaToUninitialized();
248+
access.resolveTableAccess("default", "items");
249+
Mockito.verify(fixture.namespace).describeTable(Mockito.any());
250+
}
251+
}
252+
253+
@Test
254+
public void testRefreshReplayInvalidatesAccessWithMissingObjects() throws Exception {
255+
for (boolean missingDatabase : new boolean[] {false, true}) {
256+
for (boolean legacyIds : new boolean[] {false, true}) {
257+
try (AccessFixture fixture = new AccessFixture()) {
258+
LanceNamespaceClient access = fixture.access();
259+
access.resolveTableAccess("default", "items");
260+
LanceExternalDatabase database = Mockito.mock(LanceExternalDatabase.class);
261+
Mockito.doReturn(missingDatabase ? Optional.empty() : Optional.of(database))
262+
.when(fixture.catalog).getDbForReplay("mapped_db");
263+
Mockito.doReturn(missingDatabase ? Optional.empty() : Optional.of(database))
264+
.when(fixture.catalog).getDbForReplay(1L);
265+
ExternalObjectLog log = ExternalObjectLog.createForRefreshTable(
266+
fixture.catalog.getId(), "mapped_db", "mapped_table", 0);
267+
if (legacyIds) {
268+
log.setDbName(null);
269+
log.setTableName(null);
270+
log.setDbId(1L);
271+
log.setTableId(2L);
272+
}
273+
// Access entries can survive eviction of the smaller database/table object caches.
274+
new RefreshManager().replayRefreshTable(log);
275+
access.resolveTableAccess("default", "items");
276+
Mockito.verify(fixture.namespace, Mockito.times(2)).describeTable(Mockito.any());
277+
Mockito.verify(fixture.catalog, Mockito.never()).createClient();
278+
}
279+
}
280+
}
281+
}
282+
283+
@Test
284+
public void testDatabaseRefreshReplayWithoutDatabaseObject() throws Exception {
285+
try (AccessFixture fixture = new AccessFixture()) {
286+
LanceNamespaceClient access = fixture.access();
287+
access.resolveTableAccess("default", "items");
288+
Mockito.doReturn(Optional.empty()).when(fixture.catalog).getDbForReplay("mapped_db");
289+
new RefreshManager().replayRefreshDb(
290+
ExternalObjectLog.createForRefreshDb(fixture.catalog.getId(), "mapped_db"));
291+
access.resolveTableAccess("default", "items");
292+
Mockito.verify(fixture.namespace, Mockito.times(2)).describeTable(Mockito.any());
293+
}
294+
}
295+
296+
@Test
297+
public void testExplicitDatabaseRefreshInvalidatesAccess() throws Exception {
298+
try (AccessFixture fixture = new AccessFixture()) {
299+
LanceNamespaceClient access = fixture.access();
300+
access.resolveTableAccess("default", "items");
301+
LanceExternalDatabase database = new LanceExternalDatabase(fixture.catalog, 1, "mapped_db", "default");
302+
Mockito.doReturn(database).when(fixture.catalog).getDbOrDdlException("mapped_db");
303+
new RefreshManager().handleRefreshDb("lifecycle", "mapped_db");
304+
access.resolveTableAccess("default", "items");
305+
Mockito.verify(fixture.namespace, Mockito.times(2)).describeTable(Mockito.any());
306+
}
307+
}
308+
309+
@Test
310+
public void testNamespaceRemovalInvalidatesAccessWithoutDatabaseObjects() throws Exception {
311+
try (AccessFixture fixture = new AccessFixture()) {
312+
LanceNamespaceClient access = fixture.access();
313+
access.resolveTableAccess("default", "items");
314+
setField(ExternalCatalog.class, fixture.catalog, "initialized", false);
315+
fixture.catalog.unregisterDatabase("mapped_db");
316+
access.resolveTableAccess("default", "items");
317+
Mockito.verify(fixture.namespace, Mockito.times(2)).describeTable(Mockito.any());
318+
Mockito.verify(fixture.catalog, Mockito.never()).createClient();
319+
}
320+
}
321+
322+
private static final class AccessFixture implements AutoCloseable {
323+
private final LanceNamespace namespace = Mockito.mock(LanceNamespace.class);
324+
private final LanceCatalogClient client = new LanceCatalogClient(namespace,
325+
Mockito.mock(BufferAllocator.class), Mockito.mock(Session.class), "filesystem", "default",
326+
Collections.emptyList(), Collections.emptyList(), Collections.emptyMap(), Collections.emptyList());
327+
private final LanceExternalCatalog catalog = catalog(client);
328+
private final MockedStatic<Env> currentEnv;
329+
330+
private AccessFixture() throws Exception {
331+
Mockito.when(namespace.describeTable(Mockito.any())).thenReturn(
332+
new DescribeTableResponse().tableUri("file:///warehouse/items.lance"));
333+
ExternalMetaCacheMgr caches = Env.getCurrentEnv().getExtMetaCacheMgr();
334+
Env env = Mockito.mock(Env.class);
335+
CatalogMgr catalogs = Mockito.mock(CatalogMgr.class);
336+
Mockito.when(env.getCatalogMgr()).thenReturn(catalogs);
337+
Mockito.when(env.getExtMetaCacheMgr()).thenReturn(caches);
338+
Mockito.when(env.getEditLog()).thenReturn(Mockito.mock(EditLog.class));
339+
Mockito.doReturn(catalog).when(catalogs).getCatalog("lifecycle");
340+
long catalogId = catalog.getId();
341+
Mockito.doReturn(catalog).when(catalogs).getCatalog(catalogId);
342+
currentEnv = Mockito.mockStatic(Env.class);
343+
currentEnv.when(Env::getCurrentEnv).thenReturn(env);
344+
}
345+
346+
private LanceNamespaceClient access() throws Exception {
347+
Field field = LanceCatalogClient.class.getDeclaredField("namespaceClient");
348+
field.setAccessible(true);
349+
return (LanceNamespaceClient) field.get(client);
350+
}
351+
352+
@Override
353+
public void close() {
354+
currentEnv.close();
355+
catalog.onClose();
356+
}
357+
}
358+
238359
private static LanceCatalogClient client(Session session) {
239360
return new LanceCatalogClient(Mockito.mock(LanceNamespace.class), Mockito.mock(BufferAllocator.class),
240361
session, "filesystem", "default", Collections.emptyList(), Collections.emptyList(),

‎fe/fe-core/src/test/java/org/apache/doris/datasource/lance/LanceTableAccessCacheTest.java‎

Lines changed: 22 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -87,7 +87,7 @@ public void testZeroTtlDisablesCaching() {
8787
}
8888

8989
@Test
90-
public void testVendedCredentialsExpireBeforeTheirDeadline() {
90+
public void testVendedCredentialsAreResolvedForEveryRead() {
9191
LanceNamespace namespace = namespace();
9292
AtomicLong millis = new AtomicLong(1_000_000);
9393
Map<String, String> options = new HashMap<>();
@@ -100,13 +100,29 @@ public void testVendedCredentialsExpireBeforeTheirDeadline() {
100100
Assertions.assertThrows(UnsupportedOperationException.class,
101101
() -> first.getStorageOptions().put("aws_session_token", "modified"));
102102
millis.addAndGet(9_000);
103-
Assertions.assertSame(first, client.resolveTableAccess("default", "items"));
104-
Mockito.verify(namespace).describeTable(Mockito.any());
105-
millis.addAndGet(1_000);
106-
client.resolveTableAccess("default", "items");
103+
// A new scan may outlive the remaining credential lifetime, regardless of the cache TTL.
104+
Assertions.assertNotSame(first, client.resolveTableAccess("default", "items"));
107105
Mockito.verify(namespace, Mockito.times(2)).describeTable(Mockito.any());
108106
}
109107

108+
@Test
109+
public void testSignedUrisWithoutStorageOptionsAreNotCached() {
110+
for (String uri : new String[] {
111+
"s3://example-bucket/items.lance?X-Amz-Signature=example",
112+
"az://container/items.lance?sig=example&se=example",
113+
"s3://example:password@example-bucket/items.lance",
114+
"s3://example@example_bucket/items.lance",
115+
"s3://example-bucket/items.lance#example"}) {
116+
LanceNamespace namespace = namespace();
117+
Mockito.when(namespace.describeTable(Mockito.any())).thenReturn(
118+
new DescribeTableResponse().tableUri(uri));
119+
LanceNamespaceClient client = client(namespace, "rest", 60, new AtomicLong(1_000_000));
120+
client.resolveTableAccess("default", "items");
121+
client.resolveTableAccess("default", "items");
122+
Mockito.verify(namespace, Mockito.times(2)).describeTable(Mockito.any());
123+
}
124+
}
125+
110126
@Test
111127
public void testUnsafeVendedExpiryIsNotCached() {
112128
for (String expiry : new String[] {null, "invalid", "-9223372036854775808", "999999", "1029999"}) {
@@ -258,6 +274,6 @@ private static LanceNamespace namespace() {
258274

259275
private static LanceNamespaceClient client(LanceNamespace namespace, String type, int ttl, AtomicLong millis) {
260276
return new LanceNamespaceClient(namespace, type, "default", Collections.emptyList(), Collections.emptyList(),
261-
ttl, () -> TimeUnit.MILLISECONDS.toNanos(millis.get()), millis::get);
277+
ttl, () -> TimeUnit.MILLISECONDS.toNanos(millis.get()));
262278
}
263279
}

0 commit comments

Comments
 (0)