Skip to content

Commit a07216e

Browse files
morningmanclaude
andcommitted
[feat](catalog) P6.6-C6 #63068 iceberg: per-user REST session routing (metadata + scan/write/procedure)
Iceberg connector consumer for the per-user REST session feature: - IcebergConnectorProperties + IcebergRestMetaStoreProperties: iceberg.rest.session (none|user) + iceberg.rest.oauth2.delegated-token-mode (access_token|token_exchange) + optional session-timeout, with validation (session=user requires security.type=oauth2; relax the "oauth2 requires static token/credential" rule for user-session, whose identity is the per-request user token). - IcebergDelegatedCredentialUtils + IcebergSessionCatalogAdapter: bridge the neutral ConnectorDelegatedCredential to an iceberg SessionCatalog.SessionContext (access_token = verbatim OAuth2 bearer; token_exchange = the typed token-type key) over a SINGLE shared RESTSessionCatalog. delegatedCatalog/delegatedViewCatalog fail closed on a tokenless session=user request (never borrow a shared identity). - IcebergConnector: build a RESTSessionCatalog for a session=user REST catalog; newCatalogBackedOps(session) routes per-request metadata through the querying user's delegated catalog; declare SUPPORTS_USER_SESSION only for that config. - scan/write/procedure providers take a Function<ConnectorSession, IcebergCatalogOps> resolver (connector passes this::newCatalogBackedOps); the legacy ops constructors bind a constant s->ops so existing tests/behaviour stay byte-identical. This makes SELECT / INSERT-DELETE-MERGE / ALTER TABLE ... EXECUTE run per-user. The write COMMIT was already per-user (IcebergConnectorTransaction is opened by beginTransaction over the session-aware metadata ops). Tests: IcebergProviderSessionRoutingTest (each provider resolves ops with the call session + fail-closed), IcebergSessionCatalogAdapterTest (credential-key mapping per mode + fail-closed + fromString), IcebergConnectorValidatePropertiesTest (+session validation). IcebergConnectorTest/IcebergProcedureOpsTest adjusted for the resolver constructor (reflection reads catalogOpsResolver; a bare-null ctor arg is cast). Co-Authored-By: Claude Opus 4.8 (1M context) <[email protected]> Claude-Session: https://claude.ai/code/session_013KKXx3btbQc6w3MbZbEoRf
1 parent 930e477 commit a07216e

14 files changed

Lines changed: 952 additions & 45 deletions

File tree

‎fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergCatalogOps.java‎

Lines changed: 30 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -197,6 +197,11 @@ class CatalogBackedIcebergCatalogOps implements IcebergCatalogOps {
197197
private static final String NAMESPACE_LOCATION_PROP = "location";
198198

199199
private final Catalog catalog;
200+
// Explicit view catalog for the session-aware (iceberg.rest.session=user) path: a per-request
201+
// RESTSessionCatalog.asCatalog(ctx) is a Catalog + SupportsNamespaces but NOT a ViewCatalog, so its view
202+
// facet (asViewCatalog(ctx)) is injected here separately. null on the shared path, where the catalog
203+
// itself is cast to ViewCatalog when it implements it (RESTCatalog does).
204+
private final ViewCatalog viewCatalog;
200205
// Listing-parity gating mirrored from legacy IcebergMetadataOps (threaded from IcebergConnector):
201206
private final boolean restFlavor;
202207
private final boolean nestedNamespaceEnabled;
@@ -209,7 +214,13 @@ public CatalogBackedIcebergCatalogOps(Catalog catalog) {
209214

210215
public CatalogBackedIcebergCatalogOps(Catalog catalog, boolean restFlavor,
211216
boolean nestedNamespaceEnabled, boolean viewEnabled, Optional<String> externalCatalogName) {
217+
this(catalog, null, restFlavor, nestedNamespaceEnabled, viewEnabled, externalCatalogName);
218+
}
219+
220+
public CatalogBackedIcebergCatalogOps(Catalog catalog, ViewCatalog viewCatalog, boolean restFlavor,
221+
boolean nestedNamespaceEnabled, boolean viewEnabled, Optional<String> externalCatalogName) {
212222
this.catalog = catalog;
223+
this.viewCatalog = viewCatalog;
213224
this.restFlavor = restFlavor;
214225
this.nestedNamespaceEnabled = nestedNamespaceEnabled;
215226
this.viewEnabled = viewEnabled;
@@ -264,7 +275,7 @@ public List<String> listTableNames(String dbName) {
264275
if (!isViewCatalogEnabled()) {
265276
return tableNames;
266277
}
267-
List<String> views = ((ViewCatalog) catalog).listViews(ns).stream()
278+
List<String> views = resolveViewCatalog().listViews(ns).stream()
268279
.map(TableIdentifier::name)
269280
.collect(Collectors.toList());
270281
if (views.isEmpty()) {
@@ -283,7 +294,7 @@ public List<String> listViewNames(String dbName) {
283294
if (!isViewCatalogEnabled()) {
284295
return Collections.emptyList();
285296
}
286-
return ((ViewCatalog) catalog).listViews(toNamespace(dbName)).stream()
297+
return resolveViewCatalog().listViews(toNamespace(dbName)).stream()
287298
.map(TableIdentifier::name)
288299
.collect(Collectors.toList());
289300
}
@@ -300,7 +311,7 @@ public boolean viewExists(String dbName, String viewName) {
300311
if (!isViewCatalogEnabled()) {
301312
return false;
302313
}
303-
return ((ViewCatalog) catalog).viewExists(toTableIdentifier(dbName, viewName));
314+
return resolveViewCatalog().viewExists(toTableIdentifier(dbName, viewName));
304315
}
305316

306317
@Override
@@ -312,7 +323,7 @@ public View loadView(String dbName, String viewName) {
312323
if (!isViewCatalogEnabled()) {
313324
throw new DorisConnectorException("View is not supported with not view catalog.");
314325
}
315-
return ((ViewCatalog) catalog).loadView(toTableIdentifier(dbName, viewName));
326+
return resolveViewCatalog().loadView(toTableIdentifier(dbName, viewName));
316327
}
317328

318329
@Override
@@ -322,7 +333,7 @@ public void dropView(String dbName, String viewName) {
322333
if (!isViewCatalogEnabled()) {
323334
throw new DorisConnectorException("Drop Iceberg view is not supported with not view catalog.");
324335
}
325-
((ViewCatalog) catalog).dropView(toTableIdentifier(dbName, viewName));
336+
resolveViewCatalog().dropView(toTableIdentifier(dbName, viewName));
326337
}
327338

328339
@Override
@@ -674,14 +685,26 @@ private SupportsNamespaces requireNamespaces() {
674685
return (SupportsNamespaces) catalog;
675686
}
676687

677-
/** View filtering is on iff the catalog is a {@link ViewCatalog} and (for REST) views are enabled. */
688+
/** View filtering is on iff a view catalog is resolvable and (for REST) views are enabled. */
678689
private boolean isViewCatalogEnabled() {
679-
if (!(catalog instanceof ViewCatalog)) {
690+
if (resolveViewCatalog() == null) {
680691
return false;
681692
}
682693
return !restFlavor || viewEnabled;
683694
}
684695

696+
/**
697+
* The view catalog to route view ops through: the explicitly-injected one (the session=user per-request
698+
* {@code asViewCatalog(ctx)}), else the backing catalog itself when it implements {@link ViewCatalog}
699+
* (the shared RESTCatalog path). {@code null} when the catalog has no view facet.
700+
*/
701+
private ViewCatalog resolveViewCatalog() {
702+
if (viewCatalog != null) {
703+
return viewCatalog;
704+
}
705+
return catalog instanceof ViewCatalog ? (ViewCatalog) catalog : null;
706+
}
707+
685708
/** The root namespace to start database listing from: the external-catalog level, else empty. */
686709
private Namespace rootNamespace() {
687710
return externalCatalogName.map(Namespace::of).orElseGet(Namespace::empty);

‎fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergConnector.java‎

Lines changed: 112 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -44,9 +44,12 @@
4444
import org.apache.iceberg.CatalogProperties;
4545
import org.apache.iceberg.CatalogUtil;
4646
import org.apache.iceberg.catalog.Catalog;
47+
import org.apache.iceberg.catalog.SessionCatalog;
4748
import org.apache.iceberg.catalog.TableIdentifier;
49+
import org.apache.iceberg.catalog.ViewCatalog;
4850
import org.apache.iceberg.io.FileIO;
4951
import org.apache.iceberg.rest.RESTCatalog;
52+
import org.apache.iceberg.rest.RESTSessionCatalog;
5053
import org.apache.iceberg.util.ThreadPools;
5154
import org.apache.logging.log4j.LogManager;
5255
import org.apache.logging.log4j.Logger;
@@ -141,6 +144,11 @@ public class IcebergConnector implements Connector {
141144
private final Map<String, String> properties;
142145
private final ConnectorContext context;
143146
private volatile Catalog icebergCatalog;
147+
// Session-aware REST catalog + adapter, built ONLY for a REST catalog configured iceberg.rest.session=user
148+
// (#63068). The RESTSessionCatalog is a SINGLE shared instance; per-request asCatalog(ctx)/asViewCatalog(ctx)
149+
// (through the adapter) attach the querying user's delegated credential. Both null for every other catalog.
150+
private volatile RESTSessionCatalog restSessionCatalog;
151+
private volatile IcebergSessionCatalogAdapter sessionCatalogAdapter;
144152
// T08 connector-internal caches (D6, 0 SPI). Final per-catalog fields: a REFRESH CATALOG rebuilds the
145153
// connector (PluginDrivenExternalCatalog.onClose nulls + recreates it) and thus drops both caches. The
146154
// manifest cache is path-keyed, no-TTL, capacity-bounded; it is consumed only when
@@ -198,7 +206,7 @@ static long resolveTableCacheTtlSecond(Map<String, String> properties) {
198206
@Override
199207
public ConnectorMetadata getMetadata(ConnectorSession session) {
200208
return new IcebergConnectorMetadata(
201-
newCatalogBackedOps(), properties, context, latestSnapshotCache);
209+
newCatalogBackedOps(session), properties, context, latestSnapshotCache);
202210
}
203211

204212
/**
@@ -389,6 +397,33 @@ private IcebergCatalogOps newCatalogBackedOps() {
389397
restFlavor, nestedNamespaceEnabled, viewEnabled, externalCatalogName);
390398
}
391399

400+
/**
401+
* Session-aware variant used by {@link #getMetadata}: for a {@code iceberg.rest.session=user} catalog it
402+
* routes through the per-request delegated catalog + view catalog (FAIL-CLOSED — a session that carries no
403+
* delegated credential is rejected by {@code IcebergSessionCatalogAdapter.delegatedCatalog}, never served a
404+
* shared identity), so metadata reads are authorized as the querying user. For every other catalog it is
405+
* identical to {@link #newCatalogBackedOps()} (the shared catalog). getMetadata is invoked per operation with
406+
* the current session, so resolving the per-user catalog here covers each metadata call (#63068 parity).
407+
*/
408+
private IcebergCatalogOps newCatalogBackedOps(ConnectorSession session) {
409+
if (!isUserSessionEnabled()) {
410+
return newCatalogBackedOps();
411+
}
412+
getOrCreateCatalog(); // ensure the shared RESTSessionCatalog + adapter are built
413+
IcebergSessionCatalogAdapter adapter = sessionCatalogAdapter;
414+
Catalog perUserCatalog = adapter.delegatedCatalog(session);
415+
ViewCatalog perUserViewCatalog = adapter.delegatedViewCatalog(session);
416+
boolean nestedNamespaceEnabled = Boolean.parseBoolean(properties.getOrDefault(
417+
IcebergConnectorProperties.REST_NESTED_NAMESPACE_ENABLED, "false"));
418+
boolean viewEnabled = Boolean.parseBoolean(properties.getOrDefault(
419+
IcebergConnectorProperties.REST_VIEW_ENABLED, "true"));
420+
Optional<String> externalCatalogName =
421+
Optional.ofNullable(properties.get(IcebergConnectorProperties.EXTERNAL_CATALOG_NAME));
422+
// restFlavor is unconditionally true here (isUserSessionEnabled() ⇒ a REST catalog).
423+
return new IcebergCatalogOps.CatalogBackedIcebergCatalogOps(perUserCatalog, perUserViewCatalog,
424+
true, nestedNamespaceEnabled, viewEnabled, externalCatalogName);
425+
}
426+
392427
/**
393428
* REFRESH TABLE hook: drop the cached latest snapshot for one table so the next query re-pins live
394429
* (mirrors PaimonConnector). The names are the REMOTE db/table (RefreshManager passes remote names, the
@@ -448,7 +483,7 @@ public ConnectorScanPlanProvider getScanPlanProvider() {
448483
// (newCatalogBackedOps) — the listing-only flags (nested-namespace / view) are inert on this path but
449484
// threaded for parity with the legacy single per-catalog IcebergMetadataOps.
450485
return new IcebergScanPlanProvider(properties,
451-
newCatalogBackedOps(), context, manifestCache,
486+
this::newCatalogBackedOps, context, manifestCache,
452487
rewritableDeleteStash);
453488
}
454489

@@ -459,7 +494,7 @@ public ConnectorWritePlanProvider getWritePlanProvider() {
459494
// IcebergConnectorTransaction. It resolves the target via catalogOps.loadTable, so it shares the
460495
// fully-threaded ops (newCatalogBackedOps) — external_catalog.name must apply to INSERT/DELETE/MERGE.
461496
return new IcebergWritePlanProvider(properties,
462-
newCatalogBackedOps(), context,
497+
this::newCatalogBackedOps, context,
463498
rewritableDeleteStash);
464499
}
465500

@@ -470,7 +505,7 @@ public ConnectorProcedureOps getProcedureOps() {
470505
// via catalogOps.loadTable, so it shares the fully-threaded ops (newCatalogBackedOps) —
471506
// external_catalog.name must apply to ALTER TABLE ... EXECUTE on REST 3-level catalogs.
472507
return new IcebergProcedureOps(properties,
473-
newCatalogBackedOps(), context);
508+
this::newCatalogBackedOps, context);
474509
}
475510

476511
/**
@@ -510,13 +545,33 @@ public Set<ConnectorCapability> getCapabilities() {
510545
// path reproduces this ONLY under this capability (PluginDrivenExternalTable.supportsExternalMetadataPreload),
511546
// so post-cutover iceberg keeps async pre-load instead of degrading to synchronous bind-time load. Pure
512547
// lock-latency optimization, opt-in via enable_preload_external_metadata. Inert pre-cutover (P6.6).
513-
return EnumSet.of(ConnectorCapability.SUPPORTS_MVCC_SNAPSHOT,
548+
EnumSet<ConnectorCapability> capabilities = EnumSet.of(ConnectorCapability.SUPPORTS_MVCC_SNAPSHOT,
514549
ConnectorCapability.SUPPORTS_COLUMN_AUTO_ANALYZE,
515550
ConnectorCapability.SUPPORTS_TOPN_LAZY_MATERIALIZE,
516551
ConnectorCapability.SUPPORTS_SHOW_CREATE_DDL,
517552
ConnectorCapability.SUPPORTS_VIEW,
518553
ConnectorCapability.SUPPORTS_NESTED_COLUMN_PRUNE,
519554
ConnectorCapability.SUPPORTS_METADATA_PRELOAD);
555+
// SUPPORTS_USER_SESSION: only a REST catalog configured iceberg.rest.session=user projects the querying
556+
// user's delegated credential onto a per-request Iceberg REST SessionCatalog (#63068 re-migration). This
557+
// gates FE credential injection + shared-cache bypass; every other flavor/config authenticates with a
558+
// single static catalog identity and must NOT declare it (least-privilege).
559+
if (isUserSessionEnabled()) {
560+
capabilities.add(ConnectorCapability.SUPPORTS_USER_SESSION);
561+
}
562+
return capabilities;
563+
}
564+
565+
/**
566+
* Whether this catalog is a REST catalog configured {@code iceberg.rest.session=user} — the single gate for
567+
* the per-user session machinery (capability declaration, the shared {@code RESTSessionCatalog} build, and
568+
* the session-aware catalog routing). {@code IcebergRestMetaStoreProperties.validate} has already enforced
569+
* that {@code session=user} implies {@code security.type=oauth2}.
570+
*/
571+
boolean isUserSessionEnabled() {
572+
return IcebergConnectorProperties.SESSION_USER.equalsIgnoreCase(
573+
properties.get(IcebergConnectorProperties.REST_SESSION))
574+
&& IcebergConnectorProperties.TYPE_REST.equals(IcebergCatalogFactory.resolveFlavor(properties));
520575
}
521576

522577
private Catalog getOrCreateCatalog() {
@@ -589,12 +644,56 @@ private Catalog createCatalog() {
589644
break;
590645
}
591646

647+
// REST + iceberg.rest.session=user: build a session-aware RESTSessionCatalog (not the all-in-one
648+
// RESTCatalog) so per-request asCatalog(ctx)/asViewCatalog(ctx) can attach the querying user's delegated
649+
// credential (#63068). The default (non-delegated) catalog is asCatalog(empty), identical to what a
650+
// RESTCatalog exposes; the shared session catalog + adapter are memoized for the per-user routing.
651+
if (isUserSessionEnabled()) {
652+
return buildRestSessionCatalogDefault(catalogName, catalogProps, conf);
653+
}
654+
592655
LOG.info("Creating Iceberg catalog '{}' flavor='{}' impl='{}'",
593656
catalogName, flavor, catalogProps.get(CatalogProperties.CATALOG_IMPL));
594657
return buildCatalogAuthenticated(flavor,
595658
() -> CatalogUtil.buildIcebergCatalog(catalogName, catalogProps, conf));
596659
}
597660

661+
/**
662+
* Builds the default catalog for a {@code iceberg.rest.session=user} REST catalog: a SINGLE shared
663+
* {@link RESTSessionCatalog} (built directly — NOT via {@code CatalogUtil.buildIcebergCatalog}, which returns
664+
* the all-in-one {@code RESTCatalog} and hides the session catalog behind a private field). Its
665+
* {@code asCatalog(empty)} is the default (non-delegated) catalog; its {@code asCatalog(ctx)} /
666+
* {@code asViewCatalog(ctx)} are used per request by {@link #sessionCatalogAdapter}. Memoizes
667+
* {@link #restSessionCatalog} + {@link #sessionCatalogAdapter} as a side effect. The optional
668+
* {@code iceberg.rest.session-timeout} maps to the iceberg AuthSession timeout.
669+
*/
670+
private Catalog buildRestSessionCatalogDefault(String catalogName, Map<String, String> catalogProps,
671+
Configuration conf) {
672+
Map<String, String> sessionProps = new HashMap<>(catalogProps);
673+
// Built directly via new RESTSessionCatalog(), so the CatalogUtil catalog-impl key is neither needed nor
674+
// valid here; drop it.
675+
sessionProps.remove(CatalogProperties.CATALOG_IMPL);
676+
String sessionTimeout = properties.get(IcebergConnectorProperties.REST_SESSION_TIMEOUT);
677+
if (StringUtils.isNotBlank(sessionTimeout)) {
678+
sessionProps.put(CatalogProperties.AUTH_SESSION_TIMEOUT_MS, sessionTimeout);
679+
}
680+
IcebergSessionCatalogAdapter.DelegatedTokenMode tokenMode =
681+
IcebergSessionCatalogAdapter.DelegatedTokenMode.fromString(properties.getOrDefault(
682+
IcebergConnectorProperties.REST_DELEGATED_TOKEN_MODE,
683+
IcebergConnectorProperties.DELEGATED_TOKEN_MODE_ACCESS_TOKEN));
684+
LOG.info("Creating Iceberg REST user-session catalog '{}' (delegated-token-mode={})", catalogName, tokenMode);
685+
return buildCatalogAuthenticated(IcebergConnectorProperties.TYPE_REST, () -> {
686+
RESTSessionCatalog sessionCatalog = new RESTSessionCatalog();
687+
CatalogUtil.configureHadoopConf(sessionCatalog, conf);
688+
sessionCatalog.initialize(catalogName, sessionProps);
689+
Catalog defaultCatalog = sessionCatalog.asCatalog(SessionCatalog.SessionContext.createEmpty());
690+
this.restSessionCatalog = sessionCatalog;
691+
this.sessionCatalogAdapter =
692+
new IcebergSessionCatalogAdapter(defaultCatalog, sessionCatalog, tokenMode);
693+
return defaultCatalog;
694+
});
695+
}
696+
598697
/**
599698
* Creates the bespoke {@code s3tables} catalog, mirroring legacy {@code IcebergS3TablesMetaStoreProperties}: a
600699
* hand-built {@link S3TablesClient} (region + credentials + optional {@code s3tables.endpoint} override + the
@@ -1069,5 +1168,13 @@ public void close() throws IOException {
10691168
}
10701169
icebergCatalog = null;
10711170
}
1171+
// The session=user default catalog (asCatalog(empty)) is a lightweight view and NOT Closeable, so close
1172+
// the shared underlying RESTSessionCatalog (its REST client + OAuth2 auth resources) explicitly here.
1173+
RESTSessionCatalog sc = restSessionCatalog;
1174+
if (sc != null) {
1175+
sc.close();
1176+
restSessionCatalog = null;
1177+
sessionCatalogAdapter = null;
1178+
}
10721179
}
10731180
}

‎fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergConnectorProperties.java‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -53,6 +53,20 @@ private IcebergConnectorProperties() {
5353
public static final String REST_NESTED_NAMESPACE_ENABLED = "iceberg.rest.nested-namespace-enabled";
5454
public static final String REST_VIEW_ENABLED = "iceberg.rest.view-enabled";
5555

56+
// -- REST per-user session (OIDC delegated credential; #63068 re-migration) --
57+
// iceberg.rest.session = none (default, one shared catalog identity) | user (project the querying user's
58+
// delegated credential onto a per-request Iceberg REST SessionCatalog; requires security.type=oauth2 and
59+
// gates the SUPPORTS_USER_SESSION capability). delegated-token-mode picks how the token is attached:
60+
// access_token = verbatim OAuth2 bearer; token_exchange = typed token key so the REST server exchanges it.
61+
public static final String REST_SESSION = "iceberg.rest.session";
62+
public static final String SESSION_NONE = "none";
63+
public static final String SESSION_USER = "user";
64+
public static final String REST_DELEGATED_TOKEN_MODE = "iceberg.rest.oauth2.delegated-token-mode";
65+
public static final String DELEGATED_TOKEN_MODE_ACCESS_TOKEN = "access_token";
66+
public static final String DELEGATED_TOKEN_MODE_TOKEN_EXCHANGE = "token_exchange";
67+
// Optional per-session OAuth2 AuthSession timeout (maps to CatalogProperties.AUTH_SESSION_TIMEOUT_MS).
68+
public static final String REST_SESSION_TIMEOUT = "iceberg.rest.session-timeout";
69+
5670
// -- Namespace hierarchy (REST 3-level <catalog>.<db>.<table>) --
5771
// Mirrors legacy IcebergExternalCatalog.EXTERNAL_CATALOG_NAME: when present, this catalog level is
5872
// appended to every namespace and roots database listing.

0 commit comments

Comments
 (0)