Skip to content

Commit 930e477

Browse files
morningmanclaude
andcommitted
[feat](catalog) P6.6-C6 #63068 FE: capability-gated credential injection + per-user cache bypass
FE bridge for the Iceberg REST per-user session feature (builds on the SPI DTO): - ConnectorSessionBuilder.from(ctx): copy the retained per-connection DelegatedCredential + sessionId onto the ConnectorSession ONLY when the target connector declares SUPPORTS_USER_SESSION (least-privilege); map fe-core DelegatedCredential -> neutral ConnectorDelegatedCredential by enum name so an added-but-unmapped type fails loud rather than being silently dropped. - PluginDrivenExternalCatalog.shouldBypassTableNameCache / shouldBypassDbNameCache = supportsUserSession() && ctx.hasDelegatedCredential(). Under session=user the REST server returns PER-USER metadata, so the shared (catalog+name-keyed, NOT user-keyed) db/table-name caches are bypassed to prevent cross-user leakage; a session with no credential keeps the shared cache (the fail-closed rejection then happens connector-side on the actual read, never by serving/poisoning a shared entry). - ExternalCatalog/ExternalDatabase: db-level live-path bypass (getFilteredDatabaseNames(false)/getDbNullableWithoutCache/matchesLocalDbName), still re-appending information_schema+mysql and never mutating the shared lowerCaseTo*Name lookup when bypassing (updateLookup=false). Tests: ConnectorSessionImplTest (capability-gated inject / non-capable skip / no-cred), PluginDrivenExternalCatalogSessionBypassTest (the bypass decision across capability + credential + null-context). Co-Authored-By: Claude Opus 4.8 (1M context) <[email protected]> Claude-Session: https://claude.ai/code/session_013KKXx3btbQc6w3MbZbEoRf
1 parent 55c991d commit 930e477

6 files changed

Lines changed: 367 additions & 3 deletions

File tree

‎fe/fe-core/src/main/java/org/apache/doris/connector/ConnectorSessionBuilder.java‎

Lines changed: 67 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,10 @@
1919

2020
import org.apache.doris.common.Config;
2121
import org.apache.doris.common.util.DebugUtil;
22+
import org.apache.doris.connector.api.ConnectorDelegatedCredential;
2223
import org.apache.doris.connector.api.ConnectorSession;
24+
import org.apache.doris.datasource.DelegatedCredential;
25+
import org.apache.doris.datasource.SessionContext;
2326
import org.apache.doris.qe.ConnectContext;
2427
import org.apache.doris.qe.GlobalVariable;
2528
import org.apache.doris.qe.VariableMgr;
@@ -43,6 +46,16 @@ public final class ConnectorSessionBuilder {
4346
private String catalogName;
4447
private Map<String, String> catalogProperties = Collections.emptyMap();
4548
private Map<String, String> sessionProperties = Collections.emptyMap();
49+
// The originating ConnectContext (from #from), read at build() time to pull the retained per-connection
50+
// delegated credential + session id off SessionContext — but ONLY when the target connector declares
51+
// SUPPORTS_USER_SESSION (userSessionCapable). Kept as a reference (not eagerly extracted) so a non-opt-in
52+
// connector never even touches the credential (least-privilege).
53+
private ConnectContext connectContext;
54+
private boolean userSessionCapable;
55+
// Explicit overrides for tests / callers without a live ConnectContext; when set (and capable) they win
56+
// over the ConnectContext extraction.
57+
private String sessionId;
58+
private ConnectorDelegatedCredential delegatedCredential;
4659

4760
private ConnectorSessionBuilder() {}
4861

@@ -59,6 +72,7 @@ public static ConnectorSessionBuilder from(ConnectContext ctx) {
5972
b.timeZone = ctx.getSessionVariable().getTimeZone();
6073
b.locale = "en_US"; // Doris doesn't have per-session locale yet
6174
b.sessionProperties = extractSessionProperties(ctx);
75+
b.connectContext = ctx; // read for the delegated credential at build() time, gated by capability
6276
return b;
6377
}
6478

@@ -102,10 +116,62 @@ public ConnectorSessionBuilder withTimeZone(String timeZone) {
102116
return this;
103117
}
104118

119+
/**
120+
* Declares whether the target connector consumes the user's delegated credential
121+
* ({@link org.apache.doris.connector.api.ConnectorCapability#SUPPORTS_USER_SESSION}). When {@code false}
122+
* (the default), {@link #build()} carries neither the session id nor the credential onto the session, so a
123+
* connector that would never use the OIDC token never receives it (least-privilege).
124+
*/
125+
public ConnectorSessionBuilder withUserSessionCapability(boolean capable) {
126+
this.userSessionCapable = capable;
127+
return this;
128+
}
129+
130+
/** Sets the session id explicitly (for callers without a live {@link ConnectContext}, e.g. tests). */
131+
public ConnectorSessionBuilder withSessionId(String sessionId) {
132+
this.sessionId = sessionId;
133+
return this;
134+
}
135+
136+
/** Sets the delegated credential explicitly (for callers without a live {@link ConnectContext}, e.g. tests). */
137+
public ConnectorSessionBuilder withDelegatedCredential(ConnectorDelegatedCredential credential) {
138+
this.delegatedCredential = credential;
139+
return this;
140+
}
141+
105142
/** Builds an immutable {@link ConnectorSession} instance. */
106143
public ConnectorSession build() {
144+
String sid = null;
145+
ConnectorDelegatedCredential cred = null;
146+
// Only a SUPPORTS_USER_SESSION connector receives the credential. An explicit override (tests) wins;
147+
// otherwise pull the retained per-connection SessionContext off the originating ConnectContext (the
148+
// #63068 generic base re-materializes it on the executing FE, incl. after observer->master forwarding).
149+
if (userSessionCapable) {
150+
if (delegatedCredential != null) {
151+
sid = sessionId;
152+
cred = delegatedCredential;
153+
} else if (connectContext != null) {
154+
SessionContext sc = connectContext.getSessionContext();
155+
if (sc != null && sc.hasDelegatedCredential()) {
156+
sid = sc.getSessionId();
157+
cred = toConnectorCredential(sc.getDelegatedCredential().get());
158+
}
159+
}
160+
}
107161
return new ConnectorSessionImpl(queryId, user, timeZone, locale,
108-
catalogId, catalogName, catalogProperties, sessionProperties);
162+
catalogId, catalogName, catalogProperties, sessionProperties, sid, cred);
163+
}
164+
165+
/**
166+
* Maps the fe-core {@link DelegatedCredential} to the neutral SPI {@link ConnectorDelegatedCredential} so
167+
* the connector never imports a fe-core type. The {@code Type} is bridged by enum name — the two enums are
168+
* kept constant-for-constant identical ({@code ACCESS_TOKEN/ID_TOKEN/JWT/SAML}), so an added-but-unmapped
169+
* type fails loud here rather than being silently dropped.
170+
*/
171+
private static ConnectorDelegatedCredential toConnectorCredential(DelegatedCredential credential) {
172+
return new ConnectorDelegatedCredential(
173+
ConnectorDelegatedCredential.Type.valueOf(credential.getType().name()),
174+
credential.getToken(), credential.getExpiresAtMillis());
109175
}
110176

111177
/**

‎fe/fe-core/src/main/java/org/apache/doris/connector/ConnectorSessionImpl.java‎

Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@
1818
package org.apache.doris.connector;
1919

2020
import org.apache.doris.catalog.Env;
21+
import org.apache.doris.connector.api.ConnectorDelegatedCredential;
2122
import org.apache.doris.connector.api.ConnectorSession;
2223
import org.apache.doris.connector.api.handle.ConnectorTransaction;
2324

@@ -41,6 +42,12 @@ public class ConnectorSessionImpl implements ConnectorSession {
4142
private final String catalogName;
4243
private final Map<String, String> catalogProperties;
4344
private final Map<String, String> sessionProperties;
45+
// Per-connection session id (preserved across FE observer->master forwarding) and the user's delegated
46+
// credential, populated by ConnectorSessionBuilder ONLY for a SUPPORTS_USER_SESSION connector (both null
47+
// otherwise). The credential is connection-scoped, in-memory only, and never persisted (see
48+
// ConnectorDelegatedCredential); getSessionId() falls back to the queryId when no session id was captured.
49+
private final String sessionId;
50+
private final ConnectorDelegatedCredential delegatedCredential;
4451
// Otherwise-immutable session; this is bound once by the insert executor at write time
4552
// for connectors using the SPI transaction model (e.g. maxcompute), and read back by the
4653
// connector's planWrite via getCurrentTransaction(). volatile for cross-thread visibility.
@@ -49,6 +56,14 @@ public class ConnectorSessionImpl implements ConnectorSession {
4956
ConnectorSessionImpl(String queryId, String user, String timeZone, String locale,
5057
long catalogId, String catalogName, Map<String, String> catalogProperties,
5158
Map<String, String> sessionProperties) {
59+
this(queryId, user, timeZone, locale, catalogId, catalogName, catalogProperties, sessionProperties,
60+
null, null);
61+
}
62+
63+
ConnectorSessionImpl(String queryId, String user, String timeZone, String locale,
64+
long catalogId, String catalogName, Map<String, String> catalogProperties,
65+
Map<String, String> sessionProperties, String sessionId,
66+
ConnectorDelegatedCredential delegatedCredential) {
5267
this.queryId = queryId != null ? queryId : "";
5368
this.user = user != null ? user : "";
5469
this.timeZone = timeZone != null ? timeZone : "UTC";
@@ -59,13 +74,27 @@ public class ConnectorSessionImpl implements ConnectorSession {
5974
? Collections.unmodifiableMap(catalogProperties) : Collections.emptyMap();
6075
this.sessionProperties = sessionProperties != null
6176
? Collections.unmodifiableMap(sessionProperties) : Collections.emptyMap();
77+
this.sessionId = sessionId;
78+
this.delegatedCredential = delegatedCredential;
6279
}
6380

6481
@Override
6582
public String getQueryId() {
6683
return queryId;
6784
}
6885

86+
@Override
87+
public String getSessionId() {
88+
// Fall back to the queryId (the SPI default) when no per-connection session id was captured, so a
89+
// non-user-session catalog still returns a non-null id.
90+
return sessionId != null ? sessionId : queryId;
91+
}
92+
93+
@Override
94+
public Optional<ConnectorDelegatedCredential> getDelegatedCredential() {
95+
return Optional.ofNullable(delegatedCredential);
96+
}
97+
6998
@Override
7099
public String getUser() {
71100
return user;

‎fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java‎

Lines changed: 90 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -373,6 +373,22 @@ protected boolean shouldBypassTableNameCache(SessionContext ctx) {
373373
return false;
374374
}
375375

376+
/**
377+
* Returns whether the shared database-name cache should be skipped for the current session (the db-level
378+
* analog of {@link #shouldBypassTableNameCache}).
379+
*
380+
* <p>Catalogs whose remote {@code listDatabaseNames} result depends on session credentials (Iceberg REST
381+
* {@code session=user}) must bypass the shared (catalog-wide, NOT user-keyed) db-name cache, so one user's
382+
* visible database set is never served to another. Default {@code false} — every other catalog keeps the
383+
* cache.</p>
384+
*
385+
* @param ctx session context for the current request
386+
* @return true if database names must be fetched live from the remote source for this session
387+
*/
388+
protected boolean shouldBypassDbNameCache(SessionContext ctx) {
389+
return false;
390+
}
391+
376392
/**
377393
* check if the specified table exist.
378394
*
@@ -568,6 +584,17 @@ public void initAccessController(boolean isDryRun) {
568584
*/
569585
@NotNull
570586
private List<Pair<String, String>> getFilteredDatabaseNames() {
587+
return getFilteredDatabaseNames(true);
588+
}
589+
590+
/**
591+
* @param updateDbNameLookup when {@code false}, the shared {@code lowerCaseToDatabaseName} lookup is NOT
592+
* mutated. The db-name cache-bypass path ({@link #shouldBypassDbNameCache}) passes {@code false} so a
593+
* per-user database listing never overwrites the shared (catalog-wide) case-insensitive lookup with one
594+
* user's visible set — mirrors {@code ExternalDatabase.loadTableNamePairs}'s {@code updateTableNameLookup}.
595+
*/
596+
@NotNull
597+
private List<Pair<String, String>> getFilteredDatabaseNames(boolean updateDbNameLookup) {
571598
List<String> allDatabases = Lists.newArrayList(listDatabaseNames());
572599
allDatabases.remove(InfoSchemaDb.DATABASE_NAME);
573600
allDatabases.add(InfoSchemaDb.DATABASE_NAME);
@@ -577,7 +604,9 @@ private List<Pair<String, String>> getFilteredDatabaseNames() {
577604
Map<String, Boolean> includeDatabaseMap = getIncludeDatabaseMap();
578605
Map<String, Boolean> excludeDatabaseMap = getExcludeDatabaseMap();
579606

580-
lowerCaseToDatabaseName.clear();
607+
if (updateDbNameLookup) {
608+
lowerCaseToDatabaseName.clear();
609+
}
581610
List<Pair<String, String>> remoteToLocalPairs = Lists.newArrayList();
582611

583612
allDatabases = allDatabases.stream().filter(dbName -> {
@@ -596,7 +625,9 @@ private List<Pair<String, String>> getFilteredDatabaseNames() {
596625
for (String remoteDbName : allDatabases) {
597626
String localDbName = fromRemoteDatabaseName(remoteDbName);
598627
// Populate lowercase mapping for case-insensitive lookups
599-
lowerCaseToDatabaseName.put(remoteDbName.toLowerCase(), remoteDbName);
628+
if (updateDbNameLookup) {
629+
lowerCaseToDatabaseName.put(remoteDbName.toLowerCase(), remoteDbName);
630+
}
600631
// Apply lower_case_database_names mode to local name
601632
int dbNameMode = getLowerCaseDatabaseNames();
602633
if (dbNameMode == 1) {
@@ -740,6 +771,14 @@ public String getErrorMsg() {
740771
@Override
741772
public List<String> getDbNames() {
742773
makeSureInitialized();
774+
SessionContext sessionContext = SessionContext.current();
775+
if (shouldBypassDbNameCache(sessionContext)) {
776+
// Per-user listing: read live (the loader's listDatabaseNames already runs under the current
777+
// session) and DO NOT touch the shared lowerCaseToDatabaseName lookup (updateDbNameLookup=false).
778+
return getFilteredDatabaseNames(false).stream()
779+
.map(Pair::value)
780+
.collect(Collectors.toList());
781+
}
743782
return metaCache.listNames();
744783
}
745784

@@ -776,6 +815,11 @@ public ExternalDatabase<? extends ExternalTable> getDbNullable(String dbName) {
776815
return null;
777816
}
778817

818+
SessionContext sessionContext = SessionContext.current();
819+
if (shouldBypassDbNameCache(sessionContext)) {
820+
return getDbNullableWithoutCache(dbName);
821+
}
822+
779823
// information_schema db name is case-insensitive.
780824
// So, we first convert it to standard database name.
781825
if (dbName.equalsIgnoreCase(InfoSchemaDb.DATABASE_NAME)) {
@@ -795,6 +839,50 @@ public ExternalDatabase<? extends ExternalTable> getDbNullable(String dbName) {
795839
return metaCache.getMetaObj(dbName, Util.genIdByName(name, dbName)).orElse(null);
796840
}
797841

842+
/**
843+
* Live (no-cache) counterpart of {@link #getDbNullable(String)} for the db-name cache-bypass path
844+
* ({@link #shouldBypassDbNameCache}). Resolves the requested db against the per-user live listing (never
845+
* populating the shared caches) and builds the {@link ExternalDatabase} object directly. Mirrors
846+
* {@code ExternalDatabase.getTableNullableWithoutCache} one level up.
847+
*/
848+
private ExternalDatabase<? extends ExternalTable> getDbNullableWithoutCache(String dbName) {
849+
// Normalize the case-insensitive system db names, mirroring the cached getDbNullable path.
850+
String requestedDbName = dbName;
851+
if (dbName.equalsIgnoreCase(InfoSchemaDb.DATABASE_NAME)) {
852+
requestedDbName = InfoSchemaDb.DATABASE_NAME;
853+
} else if (dbName.equalsIgnoreCase(MysqlDb.DATABASE_NAME)) {
854+
requestedDbName = MysqlDb.DATABASE_NAME;
855+
}
856+
final String target = requestedDbName;
857+
Optional<Pair<String, String>> dbNamePair = getFilteredDatabaseNames(false).stream()
858+
.filter(pair -> matchesLocalDbName(pair.value(), target))
859+
.findFirst();
860+
if (!dbNamePair.isPresent()) {
861+
return null;
862+
}
863+
String remoteDbName = dbNamePair.get().key();
864+
String localDbName = dbNamePair.get().value();
865+
// checkExists=false: the pair came from the live listing, so re-listing (getDbNames) is both
866+
// unnecessary and would recurse back through this bypass — build the db object directly.
867+
return buildDbForInit(remoteDbName, localDbName, Util.genIdByName(name, localDbName), logType, false);
868+
}
869+
870+
/** Matches a live listing's LOCAL db name against a requested name, honoring the db-name case modes. */
871+
private boolean matchesLocalDbName(String localDbName, String dbName) {
872+
// System dbs (already normalized to canonical form by the caller) match case-insensitively.
873+
if (dbName.equals(InfoSchemaDb.DATABASE_NAME) || dbName.equals(MysqlDb.DATABASE_NAME)) {
874+
return localDbName.equalsIgnoreCase(dbName);
875+
}
876+
int mode = getLowerCaseDatabaseNames();
877+
if (mode == 1) {
878+
return localDbName.equals(dbName.toLowerCase());
879+
}
880+
if (mode == 2 || Boolean.parseBoolean(getLowerCaseMetaNames())) {
881+
return localDbName.equalsIgnoreCase(dbName);
882+
}
883+
return localDbName.equals(dbName);
884+
}
885+
798886
@Nullable
799887
@Override
800888
public ExternalDatabase<? extends ExternalTable> getDbNullable(long dbId) {

‎fe/fe-core/src/main/java/org/apache/doris/datasource/PluginDrivenExternalCatalog.java‎

Lines changed: 37 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -938,19 +938,56 @@ public String fromRemoteTableName(String remoteDatabaseName, String remoteTableN
938938
public ConnectorSession buildConnectorSession() {
939939
ConnectContext ctx = ConnectContext.get();
940940
if (ctx != null) {
941+
// Interactive path: inject the user's delegated credential when the connector opts in
942+
// (SUPPORTS_USER_SESSION). The credential rides the session and is consumed connector-side.
941943
return ConnectorSessionBuilder.from(ctx)
942944
.withCatalogId(getId())
943945
.withCatalogName(getName())
944946
.withCatalogProperties(catalogProperty.getProperties())
947+
.withUserSessionCapability(supportsUserSession())
945948
.build();
946949
}
950+
// Background/internal path (no ConnectContext): never carries a delegated credential — a
951+
// session=user connector then fails closed on interactive callers and gets no borrowed identity here.
947952
return ConnectorSessionBuilder.create()
948953
.withCatalogId(getId())
949954
.withCatalogName(getName())
950955
.withCatalogProperties(catalogProperty.getProperties())
951956
.build();
952957
}
953958

959+
/**
960+
* Whether the backing connector projects the querying user's delegated credential onto the remote
961+
* metadata source ({@link ConnectorCapability#SUPPORTS_USER_SESSION}), gating both the FE credential
962+
* injection above and the shared-cache bypass ({@link #shouldBypassTableNameCache}).
963+
*/
964+
private boolean supportsUserSession() {
965+
return connector != null
966+
&& connector.getCapabilities().contains(ConnectorCapability.SUPPORTS_USER_SESSION);
967+
}
968+
969+
/**
970+
* Under a {@link ConnectorCapability#SUPPORTS_USER_SESSION} connector carrying a per-request delegated
971+
* credential, the remote source returns PER-USER table metadata, so the shared (catalog+name-keyed, NOT
972+
* user-keyed) table-name cache must be bypassed — otherwise one user's REST-authorized/vended table set
973+
* would be served to another (cross-user leakage). A session with no credential keeps the shared cache;
974+
* the fail-closed rejection then happens connector-side on the actual metadata read, never here.
975+
*/
976+
@Override
977+
protected boolean shouldBypassTableNameCache(SessionContext ctx) {
978+
return supportsUserSession() && ctx != null && ctx.hasDelegatedCredential();
979+
}
980+
981+
/**
982+
* Db-level analog of {@link #shouldBypassTableNameCache}: under a session=user connector with a per-request
983+
* credential the remote source returns PER-USER databases, so the shared db-name cache is bypassed to avoid
984+
* leaking one user's visible database set to another (O2). Same capability + credential gate.
985+
*/
986+
@Override
987+
protected boolean shouldBypassDbNameCache(SessionContext ctx) {
988+
return supportsUserSession() && ctx != null && ctx.hasDelegatedCredential();
989+
}
990+
954991
@Override
955992
protected ExternalDatabase<? extends ExternalTable> buildDbForInit(String remoteDbName, String localDbName,
956993
long dbId, InitCatalogLog.Type logType, boolean checkExists) {

0 commit comments

Comments
 (0)