Skip to content

Commit 7d23148

Browse files
authored
[fix](fe) Validate Hive HDFS HA properties during catalog DDL (#68585)
### What problem does this PR solve? Issue Number: DORIS-29461 Related PR: None Problem Summary: Plugin-driven Hive catalogs accepted incomplete HDFS HA properties during CREATE and ALTER. For example, a catalog with `dfs.nameservices=ns1` but no `dfs.ha.namenodes.ns1`, or with an empty nameservice list, could be persisted successfully and only fail when the first external table accessed storage. The DDL validation path now binds Hive storage properties before persisting the catalog, and HDFS HA validation rejects empty nameservice entries and validates the required NameNode, RPC address, and failover provider properties. The behavior is covered by FE unit tests and an external regression suite. ### Release note Hive Catalog CREATE and ALTER now reject incomplete HDFS HA configurations during DDL validation. ### Check List (For Author) - Test: Unit Test / Regression test - FE unit tests: `PluginDrivenExternalCatalogConcurrencyTest`, `HdfsPropertiesTest` - External regression: `external_table_p0/hive/test_hive_ha_catalog_validation` - FE build and Checkstyle passed - Behavior changed: Yes (invalid Hive HDFS HA configurations fail during CREATE/ALTER instead of being deferred until storage access) - Does this need documentation: No
1 parent 3e3bacd commit 7d23148

9 files changed

Lines changed: 254 additions & 40 deletions

File tree

‎fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogMgr.java‎

Lines changed: 66 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -288,6 +288,14 @@ public void createCatalog(CreateCatalogCommand cmd) throws UserException {
288288
}
289289
long id = Env.getCurrentEnv().getNextId();
290290
CatalogIf catalog = CatalogFactory.createFromCommand(id, cmd);
291+
if (catalog instanceof ExternalCatalog) {
292+
try {
293+
((ExternalCatalog) catalog).checkProperties();
294+
} catch (DdlException | RuntimeException e) {
295+
catalog.onCreateFailure();
296+
throw e;
297+
}
298+
}
291299
createCatalogImpl(catalog, cmd.getCatalogName(), cmd.isSetIfNotExists());
292300
}
293301

@@ -410,26 +418,60 @@ public void alterCatalogComment(String catalogName, String comment) throws UserE
410418
* Modify the catalog property and write the meta log.
411419
*/
412420
public void alterCatalogProps(String catalogName, Map<String, String> newProperties) throws UserException {
413-
Runnable accessControllerCleanup = () -> { };
414-
writeLock();
415-
try {
416-
CatalogIf catalog = nameToCatalog.get(catalogName);
417-
if (catalog == null) {
418-
throw new DdlException("No catalog found with name: " + catalogName);
421+
Map<String, String> updates = Maps.newHashMap(newProperties);
422+
while (true) {
423+
CatalogIf catalog;
424+
Map<String, String> oldProperties;
425+
readLock();
426+
try {
427+
catalog = nameToCatalog.get(catalogName);
428+
if (catalog == null) {
429+
throw new DdlException("No catalog found with name: " + catalogName);
430+
}
431+
oldProperties = Maps.newHashMap(catalog.getProperties());
432+
if (updates.containsKey("type") && !catalog.getType()
433+
.equalsIgnoreCase(updates.get("type"))) {
434+
throw new DdlException("Can't modify the type of catalog property with name: " + catalogName);
435+
}
436+
} finally {
437+
readUnlock();
419438
}
420-
Map<String, String> oldProperties = catalog.getProperties();
421-
if (newProperties.containsKey("type") && !catalog.getType()
422-
.equalsIgnoreCase(newProperties.get("type"))) {
423-
throw new DdlException("Can't modify the type of catalog property with name: " + catalogName);
439+
440+
// Filesystem binding may read Hadoop XML files. Validate the detached snapshot before
441+
// taking the global catalog write lock, then retry if another ALTER changed the snapshot.
442+
boolean validatedWithoutMutation = catalog instanceof ExternalCatalog
443+
&& validateCatalogPropsBeforeUpdate((ExternalCatalog) catalog, oldProperties, updates);
444+
Runnable accessControllerCleanup = () -> { };
445+
writeLock();
446+
try {
447+
if (nameToCatalog.get(catalogName) != catalog
448+
|| !oldProperties.equals(catalog.getProperties())) {
449+
continue;
450+
}
451+
CatalogLog log = new CatalogLog();
452+
log.setCatalogId(catalog.getId());
453+
log.setNewProps(updates);
454+
accessControllerCleanup = applyAlterCatalogProps(
455+
log, oldProperties, false, true, validatedWithoutMutation);
456+
Env.getCurrentEnv().getEditLog().logCatalogLog(OperationType.OP_ALTER_CATALOG_PROPS, log);
457+
return;
458+
} finally {
459+
writeUnlock();
460+
accessControllerCleanup.run();
424461
}
425-
CatalogLog log = new CatalogLog();
426-
log.setCatalogId(catalog.getId());
427-
log.setNewProps(newProperties);
428-
accessControllerCleanup = applyAlterCatalogProps(log, oldProperties, false, true);
429-
Env.getCurrentEnv().getEditLog().logCatalogLog(OperationType.OP_ALTER_CATALOG_PROPS, log);
430-
} finally {
431-
writeUnlock();
432-
accessControllerCleanup.run();
462+
}
463+
}
464+
465+
private boolean validateCatalogPropsBeforeUpdate(ExternalCatalog catalog, Map<String, String> oldProperties,
466+
Map<String, String> newProperties) throws DdlException {
467+
try {
468+
return catalog.validatePropertiesBeforeUpdate(oldProperties, newProperties);
469+
} catch (Exception validationException) {
470+
if (validationException instanceof DdlException) {
471+
throw (DdlException) validationException;
472+
}
473+
throw new DdlException("Invalid catalog properties: "
474+
+ validationException.getMessage(), validationException);
433475
}
434476
}
435477

@@ -553,9 +595,6 @@ public void replayCreateCatalog(CatalogLog log) throws DdlException {
553595
private void createCatalogInternal(CatalogIf catalog, boolean isReplay) throws DdlException {
554596
writeLock();
555597
try {
556-
if (!isReplay && catalog instanceof ExternalCatalog) {
557-
((ExternalCatalog) catalog).checkProperties();
558-
}
559598
Map<String, String> props = catalog.getProperties();
560599
if (props.containsKey(METADATA_REFRESH_INTERVAL_SEC)) {
561600
// need refresh
@@ -647,24 +686,25 @@ public void replayAlterCatalogProps(CatalogLog log, Map<String, String> oldPrope
647686
Runnable accessControllerCleanup = () -> { };
648687
writeLock();
649688
try {
650-
accessControllerCleanup = applyAlterCatalogProps(log, oldProperties, isReplay, true);
689+
accessControllerCleanup = applyAlterCatalogProps(log, oldProperties, isReplay, true, false);
651690
} finally {
652691
writeUnlock();
653692
accessControllerCleanup.run();
654693
}
655694
}
656695

657696
private Runnable applyAlterCatalogProps(CatalogLog log, Map<String, String> oldProperties,
658-
boolean isReplay, boolean deferAccessControllerCleanup) throws DdlException {
697+
boolean isReplay, boolean deferAccessControllerCleanup, boolean validatedWithoutMutation)
698+
throws DdlException {
659699
CatalogIf catalog = idToCatalog.get(log.getCatalogId());
660700
if (catalog instanceof ExternalCatalog) {
661701
Map<String, String> newProps = log.getNewProps();
662-
if (!isReplay) {
702+
if (!isReplay && !validatedWithoutMutation) {
663703
ExternalCatalog externalCatalog = (ExternalCatalog) catalog;
664704
try {
665-
boolean validatedWithoutMutation = externalCatalog.validatePropertiesBeforeUpdate(
705+
boolean validatedWithoutMutationInLock = externalCatalog.validatePropertiesBeforeUpdate(
666706
oldProperties, newProps);
667-
if (!validatedWithoutMutation) {
707+
if (!validatedWithoutMutationInLock) {
668708
synchronized (externalCatalog) {
669709
Map<String, String> currentProperties = Maps.newHashMap(externalCatalog.getProperties());
670710
try {

‎fe/fe-core/src/main/java/org/apache/doris/datasource/plugin/PluginDrivenExternalCatalog.java‎

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -69,7 +69,9 @@
6969
import org.apache.doris.datasource.connector.converter.ConnectorPartitionFieldConverter;
7070
import org.apache.doris.datasource.log.ExternalObjectLog;
7171
import org.apache.doris.datasource.log.InitCatalogLog;
72+
import org.apache.doris.foundation.property.StoragePropertiesException;
7273
import org.apache.doris.foundation.security.JdbcDriverUrlSecurity;
74+
import org.apache.doris.fs.FileSystemFactory;
7375
import org.apache.doris.nereids.trees.plans.commands.info.AddPartitionFieldOp;
7476
import org.apache.doris.nereids.trees.plans.commands.info.CreateTableInfo;
7577
import org.apache.doris.nereids.trees.plans.commands.info.DropPartitionFieldOp;
@@ -230,6 +232,7 @@ public void checkProperties() throws DdlException {
230232
} catch (IllegalArgumentException e) {
231233
throw new DdlException(e.getMessage());
232234
}
235+
validateStorageProperties(catalogProperty);
233236
// Validate function_rules JSON if present (shared across all connector types).
234237
String functionRules = catalogProperty.getOrDefault("function_rules", null);
235238
ExternalFunctionRules.check(functionRules);
@@ -253,11 +256,33 @@ public boolean validatePropertiesBeforeUpdate(
253256
} catch (IllegalArgumentException e) {
254257
throw new DdlException(e.getMessage(), e);
255258
}
259+
validateStorageProperties(candidateProperty);
256260
checkDriverUrlsAgainstOperatorGate(candidate, updatedProperties);
257261
ExternalFunctionRules.check(candidateProperty.getOrDefault("function_rules", null));
258262
return true;
259263
}
260264

265+
/**
266+
* Bind Hive catalog storage properties during DDL validation so HDFS configuration errors are
267+
* reported by CREATE/ALTER instead of being deferred until the first table access.
268+
*/
269+
private void validateStorageProperties(CatalogProperty property) throws DdlException {
270+
if (!"hms".equalsIgnoreCase(getType())) {
271+
return;
272+
}
273+
String nameservices = property.getProperties().get("dfs.nameservices");
274+
if (nameservices != null
275+
&& java.util.Arrays.stream(nameservices.split(","))
276+
.map(String::trim).noneMatch(value -> !value.isEmpty())) {
277+
throw new DdlException("Property dfs.nameservices must contain a nameservice");
278+
}
279+
try {
280+
FileSystemFactory.bindAllStorageProperties(property.getProperties());
281+
} catch (IllegalArgumentException | StoragePropertiesException e) {
282+
throw new DdlException(e.getMessage(), e);
283+
}
284+
}
285+
261286
/**
262287
* Applies the operator's driver-jar gate ({@code jdbc_driver_secure_path} /
263288
* {@code jdbc_driver_url_white_list}) to every driver_url these properties would make the connector

‎fe/fe-core/src/main/java/org/apache/doris/datasource/storage/StorageAdapter.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -206,7 +206,7 @@ public static boolean matchesProviderGuess(String providerName, Map<String, Stri
206206
* Broker/Local/Http verbatim backend maps stay clean — and only when a
207207
* {@code *config.resources} key is present (the only consumer of the marker).
208208
*/
209-
private static Map<String, String> withHadoopConfigDir(Map<String, String> props) {
209+
public static Map<String, String> withHadoopConfigDir(Map<String, String> props) {
210210
if (props == null || props.containsKey("_HADOOP_CONFIG_DIR_")
211211
|| StringUtils.isBlank(Config.hadoop_config_dir)) {
212212
return props;

‎fe/fe-core/src/main/java/org/apache/doris/fs/FileSystemFactory.java‎

Lines changed: 8 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -97,18 +97,19 @@ public static FileSystemPluginManager getPluginManager() {
9797
*/
9898
public static org.apache.doris.filesystem.FileSystem getFileSystem(Map<String, String> properties)
9999
throws IOException {
100+
Map<String, String> normalizedProperties = StorageAdapter.withHadoopConfigDir(properties);
100101
FileSystemPluginManager mgr = pluginManager;
101102
if (mgr != null) {
102-
return mgr.createFileSystem(properties);
103+
return mgr.createFileSystem(normalizedProperties);
103104
}
104105
// Fallback: ServiceLoader discovery (unit-test / migration path)
105106
List<FileSystemProvider> providers = getProviders();
106107
List<String> tried = new ArrayList<>();
107108
for (FileSystemProvider provider : providers) {
108-
if (provider.supports(properties)) {
109+
if (provider.supports(normalizedProperties)) {
109110
LOG.debug("FileSystemFactory: selected SPI provider '{}' for keys={}",
110111
provider.name(), properties.keySet());
111-
return provider.create(properties);
112+
return provider.create(normalizedProperties);
112113
}
113114
tried.add(provider.name());
114115
}
@@ -135,16 +136,17 @@ public static List<org.apache.doris.filesystem.properties.StorageProperties> bin
135136
// fe-core Config, so the HDFS plugin's config-resource loader reads this system property instead. Keep
136137
// the key in sync with HdfsConfigFileLoader.CONFIG_DIR_PROPERTY ("doris.hadoop.config.dir").
137138
System.setProperty("doris.hadoop.config.dir", Config.hadoop_config_dir);
139+
Map<String, String> normalizedProperties = StorageAdapter.withHadoopConfigDir(properties);
138140
FileSystemPluginManager mgr = pluginManager;
139141
if (mgr != null) {
140-
return new ArrayList<>(mgr.bindAll(properties));
142+
return new ArrayList<>(mgr.bindAll(normalizedProperties));
141143
}
142144
// Fallback: ServiceLoader discovery (unit-test / migration path), mirroring getFileSystem(Map).
143145
List<org.apache.doris.filesystem.properties.StorageProperties> result = new ArrayList<>();
144146
for (FileSystemProvider provider : getProviders()) {
145-
if (provider.supports(properties)) {
147+
if (provider.supports(normalizedProperties)) {
146148
try {
147-
result.add(provider.bind(properties));
149+
result.add(provider.bind(normalizedProperties));
148150
} catch (UnsupportedOperationException e) {
149151
LOG.debug("FileSystemProvider {} has no typed binding; skipping in "
150152
+ "bindAllStorageProperties", provider.name());

‎fe/fe-core/src/test/java/org/apache/doris/datasource/plugin/PluginDrivenExternalCatalogConcurrencyTest.java‎

Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -100,6 +100,36 @@ public void testAlterRejectsInvalidHiveParquetTimezone() {
100100
Assertions.assertTrue(exception.getMessage().contains("short timezone aliases are not supported"));
101101
}
102102

103+
@Test
104+
public void testCreateRejectsIncompleteHiveHaConfiguration() {
105+
Map<String, String> properties = hiveCatalogProperties();
106+
properties.put("dfs.nameservices", "ns1");
107+
PluginDrivenExternalCatalog catalog = new PluginDrivenExternalCatalog(
108+
1L, "test-catalog", null, properties, "", null);
109+
110+
DdlException exception = Assertions.assertThrows(DdlException.class, catalog::checkProperties);
111+
Assertions.assertTrue(exception.getMessage().contains("dfs.ha.namenodes.ns1"), exception.getMessage());
112+
}
113+
114+
@Test
115+
public void testAlterRejectsIncompleteHiveHaConfiguration() {
116+
Map<String, String> properties = hiveCatalogProperties();
117+
PluginDrivenExternalCatalog catalog = new PluginDrivenExternalCatalog(
118+
1L, "test-catalog", null, properties, "", null);
119+
120+
DdlException exception = Assertions.assertThrows(DdlException.class,
121+
() -> catalog.validatePropertiesBeforeUpdate(properties,
122+
Collections.singletonMap("dfs.nameservices", "ns1")));
123+
Assertions.assertTrue(exception.getMessage().contains("dfs.ha.namenodes.ns1"), exception.getMessage());
124+
}
125+
126+
private static Map<String, String> hiveCatalogProperties() {
127+
Map<String, String> properties = new HashMap<>();
128+
properties.put("type", "hms");
129+
properties.put("hive.metastore.uris", "thrift://localhost:9083");
130+
return properties;
131+
}
132+
103133
@Test
104134
public void testPropertyUpdateInvalidatesCatalogCaches() throws Exception {
105135
TestablePluginCatalog catalog = new TestablePluginCatalog(

‎fe/fe-filesystem/fe-filesystem-hdfs-base/src/main/java/org/apache/doris/filesystem/hdfs/properties/HdfsConfigFileLoader.java‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -46,7 +46,8 @@ public static Map<String, String> load(String resourcesPath, String configDir) {
4646
String dir = configDir == null ? "" : configDir;
4747
Configuration conf = new Configuration();
4848
for (String resource : resourcesPath.split(",")) {
49-
String path = dir + resource.trim();
49+
String resourceName = resource.trim();
50+
String path = new File(resourceName).isAbsolute() ? resourceName : dir + resourceName;
5051
File file = new File(path);
5152
if (file.exists() && file.isFile()) {
5253
conf.addResource(new Path(file.toURI()));

‎fe/fe-filesystem/fe-filesystem-hdfs-base/src/main/java/org/apache/doris/filesystem/hdfs/properties/HdfsPropertiesUtils.java‎

Lines changed: 8 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -218,10 +218,14 @@ public static void checkHaConfig(Map<String, String> hdfsProperties) {
218218
// No nameservice configured => HA is not enabled, nothing to validate
219219
return;
220220
}
221-
for (String dfsservice : splitAndTrim(dfsNameservices)) {
222-
if (dfsservice.isEmpty()) {
223-
continue;
224-
}
221+
// Keep runtime binding compatible with catalogs written by older versions. Hadoop also
222+
// ignores empty tokens produced by a trailing comma in dfs.nameservices.
223+
List<String> services = splitAndTrim(dfsNameservices);
224+
if (services.isEmpty()) {
225+
// Keep runtime binding compatible with catalogs persisted by older versions.
226+
return;
227+
}
228+
for (String dfsservice : services) {
225229
// 2. Check dfs.ha.namenodes.<nameservice>
226230
String haNnKey = HdfsClientConfigKeys.DFS_HA_NAMENODES_KEY_PREFIX + "." + dfsservice;
227231
String namenodes = hdfsProperties.getOrDefault(haNnKey, "");

‎fe/fe-filesystem/fe-filesystem-hdfs/src/test/java/org/apache/doris/filesystem/hdfs/properties/HdfsPropertiesTest.java‎

Lines changed: 23 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -63,6 +63,27 @@ void simpleAuthInjectsFallbackDefault() {
6363
Assertions.assertFalse(HdfsConfigBuilder.isKerberosEnabled(resolved));
6464
}
6565

66+
@Test
67+
void emptyNameserviceListRemainsCompatibleAtRuntime() {
68+
Map<String, String> raw = new HashMap<>();
69+
raw.put("dfs.nameservices", ",");
70+
71+
Assertions.assertDoesNotThrow(() -> resolve(raw));
72+
}
73+
74+
@Test
75+
void nameserviceListWithTrailingEmptyEntryRemainsCompatible() {
76+
Map<String, String> raw = new HashMap<>();
77+
raw.put("dfs.nameservices", "ns1,");
78+
raw.put("dfs.ha.namenodes.ns1", "nn1,nn2");
79+
raw.put("dfs.namenode.rpc-address.ns1.nn1", "host1:8020");
80+
raw.put("dfs.namenode.rpc-address.ns1.nn2", "host2:8020");
81+
raw.put("dfs.client.failover.proxy.provider.ns1",
82+
"org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider");
83+
84+
Assertions.assertDoesNotThrow(() -> resolve(raw));
85+
}
86+
6687
@Test
6788
void userOverriddenHadoopKeysArePreserved() {
6889
Map<String, String> raw = new HashMap<>();
@@ -102,8 +123,8 @@ void xmlResourcesAreLoadedFromAbsolutePathWhenNoConfigDirInjected(@TempDir Path
102123

103124
Map<String, String> raw = new HashMap<>();
104125
raw.put("fs.defaultFS", "hdfs://ns");
105-
// No _HADOOP_CONFIG_DIR_ injected: the resource is given as a full absolute path
106-
// and must load as-is (configDir falls back to "").
126+
// An absolute resource must remain absolute even when the configured Hadoop directory is set.
127+
raw.put("_HADOOP_CONFIG_DIR_", tmp.resolve("configured").toString() + "/");
107128
raw.put("hadoop.config.resources", site.toString());
108129

109130
Map<String, String> resolved = resolve(raw);

0 commit comments

Comments
 (0)