Skip to content

Commit d3b3311

Browse files
committed
[fix](fe) Close catalog driver and endpoint validation gaps
### What problem does this PR solve? Issue Number: None Related PR: #69588 Problem Summary: ALTER CATALOG could persist an unchecked driver_url while its connector provider was unavailable. Restoring the plugin then consumed the URL without statement-time validation. Reject interactive property updates without a provider, while keeping degraded catalog replay unchanged. Also parse schemed endpoints with URI host/port semantics so query, fragment, and userinfo cannot hide an HMS destination from the configured SSRF probe. Exercise production provider lookup and the real degraded replay path in the gate tests. ### Release note ALTER CATALOG property updates require an available connector provider. With an SSRF checker configured, HMS URI query, fragment, and userinfo components no longer bypass destination checks. Replay and lazy rebuild remain unchanged. ### Check List (For Author) - Test: Unit Test - 88 selected tests passed via run-fe-ut.sh; Maven validate with Checkstyle and architecture gates; git diff --check - Behavior changed: Yes - reject property ALTER without a provider and probe the actual destination of schemed HMS URIs - Does this need documentation: No
1 parent 181767d commit d3b3311

5 files changed

Lines changed: 221 additions & 59 deletions

File tree

‎fe/fe-core/src/main/java/org/apache/doris/datasource/plugin/PluginDrivenExternalCatalog.java‎

Lines changed: 25 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -91,6 +91,8 @@
9191
import org.apache.logging.log4j.Logger;
9292

9393
import java.io.IOException;
94+
import java.net.URI;
95+
import java.net.URISyntaxException;
9496
import java.util.ArrayList;
9597
import java.util.Arrays;
9698
import java.util.HashMap;
@@ -352,16 +354,17 @@ private void checkDriverUrlsAgainstOperatorGate(Map<String, String> candidate,
352354

353355
/**
354356
* The connector's declared security-sensitive property surface, looked up by catalog type WITHOUT
355-
* initializing a connector (so neither gate forces a catalog to come to life). {@link
356-
* ConnectorSecuritySurface#EMPTY} when no provider backs this type — exactly the state in which the
357-
* connector can neither load a driver jar nor open a connection, so there is nothing to police.
357+
* initializing a connector (so neither gate forces a catalog to come to life). A missing provider
358+
* cannot validate newly submitted properties: accepting them would bypass the gates when its plugin
359+
* is restored, since replay and lazy reconstruction deliberately skip statement-time validation.
360+
* Only an available provider may declare {@link ConnectorSecuritySurface#EMPTY}.
358361
*/
359-
// Package-private, not private: a unit test overrides it to inject a surface without registering a
360-
// connector plugin, exactly as it exercises the gate logic in isolation.
361-
ConnectorSecuritySurface securitySurface(Map<String, String> properties) {
362+
private ConnectorSecuritySurface securitySurface(Map<String, String> properties) throws DdlException {
362363
return ConnectorFactory.findProvider(getType(), properties)
363364
.map(ConnectorProvider::securitySurface)
364-
.orElse(ConnectorSecuritySurface.EMPTY);
365+
.orElseThrow(() -> new DdlException("No ConnectorProvider found for catalog '" + name
366+
+ "', type '" + getType() + "'. Install the connector plugin "
367+
+ "before altering catalog properties."));
365368
}
366369

367370
/**
@@ -422,23 +425,27 @@ static HostAndPort parseEndpoint(String value) {
422425
if (value == null || value.isEmpty()) {
423426
return null;
424427
}
425-
// Drop a leading scheme (thrift://, http://, ...) and any trailing path, leaving the authority.
426-
String authority = value;
427-
int schemeIdx = authority.indexOf("://");
428-
if (schemeIdx >= 0) {
429-
authority = authority.substring(schemeIdx + 3);
430-
}
431-
int slashIdx = authority.indexOf('/');
432-
if (slashIdx >= 0) {
433-
authority = authority.substring(0, slashIdx);
434-
}
435428
try {
429+
if (value.contains("://")) {
430+
// Match the HMS client's URI semantics: query, fragment and userinfo do not name
431+
// the socket destination. HostAndPort.fromParts also strips IPv6 brackets.
432+
URI uri = new URI(value);
433+
if (uri.getHost() == null || uri.getPort() < 0) {
434+
return null;
435+
}
436+
return HostAndPort.fromParts(uri.getHost(), uri.getPort());
437+
}
438+
String authority = value;
439+
int slashIdx = authority.indexOf('/');
440+
if (slashIdx >= 0) {
441+
authority = authority.substring(0, slashIdx);
442+
}
436443
HostAndPort hostAndPort = HostAndPort.fromString(authority);
437444
if (!hostAndPort.hasPort() || hostAndPort.getHost().isEmpty()) {
438445
return null;
439446
}
440447
return hostAndPort;
441-
} catch (IllegalArgumentException e) {
448+
} catch (URISyntaxException | IllegalArgumentException e) {
442449
return null;
443450
}
444451
}

‎fe/fe-core/src/test/java/org/apache/doris/datasource/hive/HMSExternalCatalogParquetTimeZoneTest.java‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -20,15 +20,32 @@
2020
import org.apache.doris.common.DdlException;
2121
import org.apache.doris.common.util.FileFormatConstants;
2222
import org.apache.doris.common.util.FileFormatUtils;
23+
import org.apache.doris.connector.ConnectorFactory;
24+
import org.apache.doris.connector.ConnectorPluginManager;
25+
import org.apache.doris.connector.hive.HiveConnectorProvider;
2326
import org.apache.doris.datasource.plugin.PluginDrivenExternalCatalog;
2427

28+
import org.junit.jupiter.api.AfterEach;
2529
import org.junit.jupiter.api.Assertions;
30+
import org.junit.jupiter.api.BeforeEach;
2631
import org.junit.jupiter.api.Test;
2732

2833
import java.util.HashMap;
2934
import java.util.Map;
3035

3136
public class HMSExternalCatalogParquetTimeZoneTest {
37+
@BeforeEach
38+
public void setUp() {
39+
ConnectorPluginManager manager = new ConnectorPluginManager();
40+
manager.registerProvider(new HiveConnectorProvider());
41+
ConnectorFactory.initPluginManager(manager);
42+
}
43+
44+
@AfterEach
45+
public void tearDown() {
46+
ConnectorFactory.initPluginManager(new ConnectorPluginManager());
47+
}
48+
3249
private static Map<String, String> baseProperties() {
3350
Map<String, String> properties = new HashMap<>();
3451
properties.put("type", "hms");

‎fe/fe-core/src/test/java/org/apache/doris/datasource/plugin/PluginDrivenExternalCatalogConcurrencyTest.java‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -20,13 +20,16 @@
2020
import org.apache.doris.catalog.Env;
2121
import org.apache.doris.common.DdlException;
2222
import org.apache.doris.common.util.FileFormatConstants;
23+
import org.apache.doris.connector.ConnectorFactory;
24+
import org.apache.doris.connector.ConnectorPluginManager;
2325
import org.apache.doris.connector.cache.CacheSpec;
2426
import org.apache.doris.connector.cache.CatalogMetaCache;
2527
import org.apache.doris.connector.cache.MetaCache;
2628
import org.apache.doris.connector.cache.MetaCacheBudgetManager;
2729
import org.apache.doris.connector.cache.MetaCacheDefinition;
2830
import org.apache.doris.connector.cache.MetaCacheSizeEstimate;
2931
import org.apache.doris.connector.cache.ScopePath;
32+
import org.apache.doris.connector.hive.HiveConnectorProvider;
3033
import org.apache.doris.connector.spi.Connector;
3134
import org.apache.doris.connector.spi.ConnectorMetadata;
3235
import org.apache.doris.connector.spi.ConnectorSession;
@@ -70,6 +73,9 @@ public class PluginDrivenExternalCatalogConcurrencyTest {
7073

7174
@BeforeEach
7275
public void setUp() {
76+
ConnectorPluginManager manager = new ConnectorPluginManager();
77+
manager.registerProvider(new HiveConnectorProvider());
78+
ConnectorFactory.initPluginManager(manager);
7379
Env env = Mockito.mock(Env.class);
7480
cacheMgr = Mockito.mock(ExternalMetaCacheMgr.class);
7581
AccessControllerManager accessManager = Mockito.mock(AccessControllerManager.class);
@@ -84,6 +90,7 @@ public void setUp() {
8490
@AfterEach
8591
public void tearDown() {
8692
mockedEnv.close();
93+
ConnectorFactory.initPluginManager(new ConnectorPluginManager());
8794
}
8895

8996
@Test

‎fe/fe-core/src/test/java/org/apache/doris/datasource/plugin/PluginDrivenExternalCatalogDriverUrlGateTest.java‎

Lines changed: 102 additions & 29 deletions
Original file line numberDiff line numberDiff line change
@@ -22,13 +22,19 @@
2222
import org.apache.doris.connector.ConnectorFactory;
2323
import org.apache.doris.connector.ConnectorPluginManager;
2424
import org.apache.doris.connector.spi.Connector;
25+
import org.apache.doris.connector.spi.ConnectorContext;
26+
import org.apache.doris.connector.spi.ConnectorProvider;
2527
import org.apache.doris.connector.spi.ConnectorSecuritySurface;
2628
import org.apache.doris.connector.spi.ConnectorSession;
29+
import org.apache.doris.datasource.CatalogFactory;
30+
import org.apache.doris.datasource.log.CatalogLog;
2731

2832
import org.junit.jupiter.api.AfterEach;
2933
import org.junit.jupiter.api.Assertions;
3034
import org.junit.jupiter.api.BeforeEach;
3135
import org.junit.jupiter.api.Test;
36+
import org.junit.jupiter.params.ParameterizedTest;
37+
import org.junit.jupiter.params.provider.CsvSource;
3238
import org.mockito.Mockito;
3339

3440
import java.util.Collections;
@@ -44,8 +50,7 @@
4450
* {@code resetToUninitialized} then makes the new driver_url effective on the next metadata access. So
4551
* without this gate an operator who restricts {@code jdbc_driver_secure_path} gets the restriction enforced
4652
* at CREATE and silently bypassed by a follow-up ALTER — i.e. the config would not actually protect
47-
* anything. The checked keys come from the key table in
48-
* {@code PluginDrivenExternalCatalog#driverUrlKeysOf}.
53+
* anything. The checked keys come from the registered provider's {@code securitySurface()}.
4954
*/
5055
public class PluginDrivenExternalCatalogDriverUrlGateTest {
5156

@@ -58,8 +63,14 @@ public void setUp() {
5863
savedSecurePath = Config.jdbc_driver_secure_path;
5964
// The operator has locked driver jars down to one directory — the posture this gate exists to keep.
6065
Config.jdbc_driver_secure_path = ALLOWED_DIR;
61-
// A fresh empty manager so provider-side validation is a no-op and the gate is tested in isolation.
62-
ConnectorFactory.initPluginManager(new ConnectorPluginManager());
66+
ConnectorPluginManager manager = new ConnectorPluginManager();
67+
registerSurface(manager, "jdbc",
68+
ConnectorSecuritySurface.ofJdbcDriverUrls(null, "driver_url", "jdbc.driver_url"));
69+
registerSurface(manager, "iceberg",
70+
ConnectorSecuritySurface.ofJdbcDriverUrls("iceberg.catalog.type", "iceberg.jdbc.driver_url"));
71+
registerSurface(manager, "paimon", ConnectorSecuritySurface.ofJdbcDriverUrls(
72+
"paimon.catalog.type", "paimon.jdbc.driver_url", "jdbc.driver_url"));
73+
ConnectorFactory.initPluginManager(manager);
6374
}
6475

6576
@AfterEach
@@ -140,7 +151,7 @@ public void alterRejectsJdbcPrefixedAliasDriverUrl() {
140151
// "driver_url" also stored this update is dead config that never loads. The gate still checks
141152
// every spelling in the candidate, deliberately fail-closed: which alias wins is the holder's
142153
// business, and a rejected dead value is repairable in the same ALTER.
143-
// MUTATION: dropping "jdbc.driver_url" from the key table -> red.
154+
// MUTATION: dropping "jdbc.driver_url" from the security surface -> red.
144155
TestCatalog catalog = new TestCatalog(
145156
props("jdbc", "driver_url", "file://" + ALLOWED_DIR + "/mysql.jar"));
146157

@@ -152,7 +163,7 @@ public void alterRejectsJdbcPrefixedAliasDriverUrl() {
152163

153164
@Test
154165
public void alterRejectsPaimonJdbcAliasDriverUrl() {
155-
// MUTATION: dropping "jdbc.driver_url" from the paimon row of the key table -> red.
166+
// MUTATION: dropping "jdbc.driver_url" from the paimon security surface -> red.
156167
Map<String, String> stored = props("paimon", "jdbc.driver_url",
157168
"file://" + ALLOWED_DIR + "/mysql.jar");
158169
stored.put("paimon.catalog.type", "jdbc");
@@ -183,7 +194,7 @@ public void alterNotTouchingDriverUrlSkipsTheGate() throws Exception {
183194
public void alterFlippingFlavorToJdbcGatesTheStoredDriverUrl() {
184195
// Flipping iceberg.catalog.type to jdbc brings a previously-dead driver_url to life, so the
185196
// flavor key must count as a gate trigger even though no driver-url key changed.
186-
// MUTATION: nulling the flavorKey in the iceberg row of driverUrlKeysOf -> red.
197+
// MUTATION: nulling the iceberg security surface's flavorKey -> red.
187198
Map<String, String> stored = props("iceberg", "iceberg.jdbc.driver_url",
188199
"http://elsewhere.test/dead-config.jar");
189200
stored.put("iceberg.catalog.type", "rest");
@@ -197,9 +208,8 @@ public void alterFlippingFlavorToJdbcGatesTheStoredDriverUrl() {
197208
@Test
198209
public void alterRejectsTraversalEvenWhenSecurePathIsWildcard() {
199210
// The mandatory rule is non-configurable: with jdbc_driver_secure_path=* the allow-list gate
200-
// accepts everything, and on a degraded catalog whose plugin is absent the provider-side rule
201-
// silently no-ops (this test's empty plugin manager models exactly that), so the engine-side
202-
// check in the gate is the last line.
211+
// accepts everything. These recording providers declare keys without implementing property
212+
// validation, so this verifies the engine-side rule through the real provider lookup.
203213
// MUTATION: drop the JdbcDriverUrlSecurity.check call from the gate loop -> red.
204214
Config.jdbc_driver_secure_path = "*";
205215
TestCatalog catalog = new TestCatalog(
@@ -213,6 +223,88 @@ public void alterRejectsTraversalEvenWhenSecurePathIsWildcard() {
213223
Assertions.assertTrue(e.getMessage().contains("path traversal"), e.getMessage());
214224
}
215225

226+
@ParameterizedTest
227+
@CsvSource({"jdbc,driver_url", "jdbc,jdbc.driver_url", "iceberg,iceberg.jdbc.driver_url",
228+
"paimon,paimon.jdbc.driver_url", "paimon,jdbc.driver_url"})
229+
public void alterRejectsDriverUrlWhenProviderIsUnavailable(String type, String key) throws Exception {
230+
ConnectorFactory.initPluginManager(new ConnectorPluginManager());
231+
Map<String, String> stored = props(type, key, "file://" + ALLOWED_DIR + "/mysql.jar");
232+
stored.put(type + ".catalog.type", "jdbc");
233+
// Exercise a real replayed, degraded catalog and the production security-surface lookup.
234+
PluginDrivenExternalCatalog catalog = replayCatalog(stored);
235+
Map<String, String> current = new HashMap<>(catalog.getProperties());
236+
237+
DdlException e = Assertions.assertThrows(DdlException.class,
238+
() -> catalog.validatePropertiesBeforeUpdate(current,
239+
Collections.singletonMap(key, "http://attacker.test/evil.jar")));
240+
Assertions.assertTrue(e.getMessage().contains("No ConnectorProvider found"), e.getMessage());
241+
Assertions.assertEquals(current, catalog.getProperties(), "rejected ALTER must not publish properties");
242+
}
243+
244+
@Test
245+
public void alterRejectsTraversalWithoutProviderEvenWhenSecurePathIsWildcard() throws Exception {
246+
Config.jdbc_driver_secure_path = "*";
247+
ConnectorFactory.initPluginManager(new ConnectorPluginManager());
248+
PluginDrivenExternalCatalog catalog = replayCatalog(
249+
props("jdbc", "driver_url", "file://" + ALLOWED_DIR + "/mysql.jar"));
250+
251+
Assertions.assertThrows(DdlException.class,
252+
() -> catalog.validatePropertiesBeforeUpdate(catalog.getProperties(),
253+
Collections.singletonMap("driver_url", "file:///opt/drivers/../../../etc/evil.jar")));
254+
}
255+
256+
@Test
257+
public void alterRequiresProviderEvenForUnrelatedProperties() throws Exception {
258+
ConnectorFactory.initPluginManager(new ConnectorPluginManager());
259+
PluginDrivenExternalCatalog catalog = replayCatalog(
260+
props("jdbc", "driver_url", "file://" + ALLOWED_DIR + "/mysql.jar"));
261+
262+
Assertions.assertThrows(DdlException.class,
263+
() -> catalog.validatePropertiesBeforeUpdate(catalog.getProperties(),
264+
Collections.singletonMap("only_specified_database", "true")));
265+
}
266+
267+
@Test
268+
public void alterAcceptsProviderDeclaringNoSecuritySensitiveProperties() throws Exception {
269+
ConnectorPluginManager manager = new ConnectorPluginManager();
270+
registerSurface(manager, "endpoint_free", ConnectorSecuritySurface.EMPTY);
271+
ConnectorFactory.initPluginManager(manager);
272+
Map<String, String> stored = props("endpoint_free", "custom_property", "old");
273+
274+
new TestCatalog(stored).validatePropertiesBeforeUpdate(stored,
275+
Collections.singletonMap("custom_property", "new"));
276+
}
277+
278+
private static PluginDrivenExternalCatalog replayCatalog(Map<String, String> properties) throws Exception {
279+
CatalogLog log = new CatalogLog();
280+
log.setCatalogId(1L);
281+
log.setCatalogName("degraded-driver-gate-catalog");
282+
log.setResource("");
283+
log.setComment("");
284+
log.setProps(properties);
285+
return (PluginDrivenExternalCatalog) CatalogFactory.createFromLog(log);
286+
}
287+
288+
private static void registerSurface(ConnectorPluginManager manager, String type,
289+
ConnectorSecuritySurface surface) {
290+
manager.registerProvider(new ConnectorProvider() {
291+
@Override
292+
public String getType() {
293+
return type;
294+
}
295+
296+
@Override
297+
public Connector create(Map<String, String> properties, ConnectorContext context) {
298+
throw new AssertionError("Security-surface lookup must not initialize a connector");
299+
}
300+
301+
@Override
302+
public ConnectorSecuritySurface securitySurface() {
303+
return surface;
304+
}
305+
});
306+
}
307+
216308
private static Map<String, String> props(String type, String key, String value) {
217309
Map<String, String> props = new HashMap<>();
218310
props.put("type", type);
@@ -240,24 +332,5 @@ protected void initLocalObjectsImpl() {
240332
public ConnectorSession buildConnectorSession() {
241333
return Mockito.mock(ConnectorSession.class);
242334
}
243-
244-
@Override
245-
ConnectorSecuritySurface securitySurface(Map<String, String> properties) {
246-
// Mirror the real providers' declarations so the gate sees the same driver-url keys it would in
247-
// production, without registering a connector plugin. The real keys are pinned by each
248-
// connector's own provider test.
249-
switch (getType()) {
250-
case "jdbc":
251-
return ConnectorSecuritySurface.ofJdbcDriverUrls(null, "driver_url", "jdbc.driver_url");
252-
case "iceberg":
253-
return ConnectorSecuritySurface.ofJdbcDriverUrls(
254-
"iceberg.catalog.type", "iceberg.jdbc.driver_url");
255-
case "paimon":
256-
return ConnectorSecuritySurface.ofJdbcDriverUrls(
257-
"paimon.catalog.type", "paimon.jdbc.driver_url", "jdbc.driver_url");
258-
default:
259-
return ConnectorSecuritySurface.EMPTY;
260-
}
261-
}
262335
}
263336
}

0 commit comments

Comments
 (0)