Skip to content

Commit d83289d

Browse files
authored
[feat](fs)Refactor Catalog FS Access to Use Unified StorageProperties (apache#52084)
### What problem does this PR solve? Refactor Catalog FS Access to Use Unified StorageProperties apache#50238 #### Description: This PR refactors the file system access logic within the Catalog framework to adopt the new StorageProperties abstraction. Many internal catalogs—such as HMS—rely on the internal file system for file access from the Frontend (FE). This change ensures that all file system operations initiated within the Catalog layer now use the unified StorageProperties mechanism, which provides consistent configuration handling and supports pluggable backends like HDFS, S3, OSS, COS, etc. #### Changes: Integrated StorageProperties into Catalog-related file system logic. Replaced legacy FS property resolution with StorageProperties throughout Catalog modules. Ensured backward compatibility with existing Catalogs using internal FS (e.g., HMS). Refactored test cases to assert FS normalization behavior line-by-line instead of using a static map.
1 parent d58e068 commit d83289d

39 files changed

Lines changed: 946 additions & 965 deletions

‎fe/fe-common/src/main/java/org/apache/doris/common/security/authentication/AuthenticationConfig.java‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -60,6 +60,12 @@ public static AuthenticationConfig getKerberosConfig(Configuration conf) {
6060
return AuthenticationConfig.getKerberosConfig(conf, HADOOP_KERBEROS_PRINCIPAL, HADOOP_KERBEROS_KEYTAB);
6161
}
6262

63+
public static AuthenticationConfig getKerberosConfig(Map<String, String> params) {
64+
Configuration conf = new Configuration();
65+
params.forEach(conf::set);
66+
return AuthenticationConfig.getKerberosConfig(conf, HADOOP_KERBEROS_PRINCIPAL, HADOOP_KERBEROS_KEYTAB);
67+
}
68+
6369
public static AuthenticationConfig getSimpleAuthenticationConfig(Configuration conf) {
6470
return AuthenticationConfig.createSimpleAuthenticationConfig(conf);
6571
}

‎fe/fe-core/src/main/java/org/apache/doris/common/util/LocationPath.java‎

Lines changed: 222 additions & 373 deletions
Large diffs are not rendered by default.

‎fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogProperty.java‎

Lines changed: 35 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,9 @@
1919

2020
import org.apache.doris.catalog.Env;
2121
import org.apache.doris.catalog.Resource;
22+
import org.apache.doris.common.UserException;
2223
import org.apache.doris.datasource.property.PropertyConverter;
24+
import org.apache.doris.datasource.property.storage.StorageProperties;
2325

2426
import com.google.common.base.Strings;
2527
import com.google.common.collect.Maps;
@@ -29,7 +31,9 @@
2931
import org.apache.logging.log4j.Logger;
3032

3133
import java.util.HashMap;
34+
import java.util.List;
3235
import java.util.Map;
36+
import java.util.function.Function;
3337

3438
/**
3539
* CatalogProperty to store the properties for catalog.
@@ -44,6 +48,8 @@ public class CatalogProperty {
4448
@SerializedName(value = "properties")
4549
private Map<String, String> properties;
4650

51+
private volatile Map<StorageProperties.Type, StorageProperties> storagePropertiesMap;
52+
4753
private volatile Resource catalogResource = null;
4854

4955
public CatalogProperty(String resource, Map<String, String> properties) {
@@ -92,13 +98,29 @@ public Map<String, String> getProperties() {
9298

9399
public void modifyCatalogProps(Map<String, String> props) {
94100
properties.putAll(PropertyConverter.convertToMetaProperties(props));
101+
this.storagePropertiesMap = null;
102+
}
103+
104+
private void reInitCatalogStorageProperties() {
105+
this.storagePropertiesMap = new HashMap<>();
106+
List<StorageProperties> storageProperties;
107+
try {
108+
storageProperties = StorageProperties.createAll(this.properties);
109+
this.storagePropertiesMap.putAll(storageProperties.stream()
110+
.collect(java.util.stream.Collectors.toMap(StorageProperties::getType, Function.identity())));
111+
} catch (UserException e) {
112+
throw new RuntimeException(e);
113+
}
114+
95115
}
96116

97117
public void rollBackCatalogProps(Map<String, String> props) {
98118
properties.clear();
99119
properties = new HashMap<>(props);
120+
this.storagePropertiesMap = null;
100121
}
101122

123+
102124
public Map<String, String> getHadoopProperties() {
103125
Map<String, String> hadoopProperties = getProperties();
104126
hadoopProperties.putAll(PropertyConverter.convertToHadoopFSProperties(getProperties()));
@@ -107,9 +129,22 @@ public Map<String, String> getHadoopProperties() {
107129

108130
public void addProperty(String key, String val) {
109131
this.properties.put(key, val);
132+
this.storagePropertiesMap = null; // reset storage properties map
110133
}
111134

112135
public void deleteProperty(String key) {
113136
this.properties.remove(key);
137+
this.storagePropertiesMap = null;
138+
}
139+
140+
public Map<StorageProperties.Type, StorageProperties> getStoragePropertiesMap() {
141+
if (storagePropertiesMap == null) {
142+
synchronized (this) {
143+
if (storagePropertiesMap == null) {
144+
reInitCatalogStorageProperties();
145+
}
146+
}
147+
}
148+
return storagePropertiesMap;
114149
}
115150
}

‎fe/fe-core/src/main/java/org/apache/doris/datasource/FileSplit.java‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -60,7 +60,7 @@ public FileSplit(LocationPath path, long start, long length, long fileLength,
6060
this.modificationTime = modificationTime < 0 ? 0 : modificationTime;
6161
this.hosts = hosts == null ? new String[0] : hosts;
6262
this.partitionValues = partitionValues;
63-
this.locationType = path.isBindBroker() ? TFileType.FILE_BROKER : path.getTFileTypeForBE();
63+
this.locationType = path.getTFileTypeForBE();
6464
}
6565

6666
public String[] getHosts() {
@@ -78,7 +78,7 @@ public Object getInfo() {
7878

7979
@Override
8080
public String getPathString() {
81-
return path.toString();
81+
return path.getNormalizedLocation();
8282
}
8383

8484
public static class FileSplitCreator implements SplitCreator {

‎fe/fe-core/src/main/java/org/apache/doris/datasource/FileSplitter.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -54,7 +54,7 @@ public static List<Split> splitFile(
5454
blockLocations = new BlockLocation[0];
5555
}
5656
List<Split> result = Lists.newArrayList();
57-
TFileCompressType compressType = Util.inferFileCompressTypeByPath(path.get());
57+
TFileCompressType compressType = Util.inferFileCompressTypeByPath(path.getNormalizedLocation());
5858
if (!splittable || compressType != TFileCompressType.PLAIN) {
5959
if (LOG.isDebugEnabled()) {
6060
LOG.debug("Path {} is not splittable.", path);

‎fe/fe-core/src/main/java/org/apache/doris/datasource/hive/AcidUtil.java‎

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,7 @@
2121
import org.apache.doris.common.util.LocationPath;
2222
import org.apache.doris.datasource.hive.AcidInfo.DeleteDeltaInfo;
2323
import org.apache.doris.datasource.hive.HiveMetaStoreCache.FileCacheValue;
24+
import org.apache.doris.datasource.property.storage.StorageProperties;
2425
import org.apache.doris.fs.FileSystem;
2526
import org.apache.doris.fs.remote.RemoteFile;
2627

@@ -239,7 +240,8 @@ public boolean accept(String fileName) {
239240
// when using the Hive 4 library directly, this method is implemented.
240241
//Ref: hive/ql/src/java/org/apache/hadoop/hive/ql/io/AcidUtils.java#getAcidState
241242
public static FileCacheValue getAcidState(FileSystem fileSystem, HivePartition partition,
242-
Map<String, String> txnValidIds, Map<String, String> catalogProps, boolean isFullAcid) throws Exception {
243+
Map<String, String> txnValidIds, Map<StorageProperties.Type, StorageProperties> storagePropertiesMap,
244+
boolean isFullAcid) throws Exception {
243245

244246
// Ref: https://issues.apache.org/jira/browse/HIVE-18192
245247
// Readers should use the combination of ValidTxnList and ValidWriteIdList(Table) for snapshot isolation.
@@ -420,7 +422,7 @@ && isValidBase(fileSystem, dirPath, base, validWriteIdList)) {
420422
continue;
421423
}
422424
remoteFiles.stream().filter(f -> fileFilter.accept(f.getName())).forEach(file -> {
423-
LocationPath path = new LocationPath(file.getPath().toString(), catalogProps);
425+
LocationPath path = LocationPath.of(file.getPath().toString(), storagePropertiesMap);
424426
fileCacheValue.addFile(file, path);
425427
});
426428
} else {
@@ -435,7 +437,7 @@ && isValidBase(fileSystem, dirPath, base, validWriteIdList)) {
435437
if (status.ok()) {
436438
remoteFiles.stream().filter(f -> fileFilter.accept(f.getName()))
437439
.forEach(file -> {
438-
LocationPath path = new LocationPath(file.getPath().toString(), catalogProps);
440+
LocationPath path = LocationPath.of(file.getPath().toString(), storagePropertiesMap);
439441
fileCacheValue.addFile(file, path);
440442
});
441443
} else {

‎fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalCatalog.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -174,7 +174,7 @@ protected void initLocalObjectsImpl() {
174174
true,
175175
preExecutionAuthenticator);
176176
FileSystemProvider fileSystemProvider = new FileSystemProviderImpl(Env.getCurrentEnv().getExtMetaCacheMgr(),
177-
this.bindBrokerName(), this.catalogProperty.getHadoopProperties());
177+
this.catalogProperty.getStoragePropertiesMap());
178178
this.fileSystemExecutor = ThreadPoolManager.newDaemonFixedThreadPool(FILE_SYSTEM_EXECUTOR_THREAD_NUM,
179179
Integer.MAX_VALUE, String.format("hms_committer_%s_file_system_executor_pool", name), true);
180180
transactionManager = TransactionManagerFactory.createHiveTransactionManager(hiveOps, fileSystemProvider,

‎fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalTable.java‎

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -47,6 +47,7 @@
4747
import org.apache.doris.datasource.mvcc.MvccSnapshot;
4848
import org.apache.doris.datasource.mvcc.MvccTable;
4949
import org.apache.doris.datasource.mvcc.MvccUtil;
50+
import org.apache.doris.datasource.property.storage.StorageProperties;
5051
import org.apache.doris.datasource.systable.SupportedSysTables;
5152
import org.apache.doris.datasource.systable.SysTable;
5253
import org.apache.doris.fs.FileSystemDirectoryLister;
@@ -600,6 +601,10 @@ public Map<String, String> getCatalogProperties() {
600601
return catalog.getProperties();
601602
}
602603

604+
public Map<StorageProperties.Type, StorageProperties> getStoragePropertiesMap() {
605+
return catalog.getCatalogProperty().getStoragePropertiesMap();
606+
}
607+
603608
public Map<String, String> getHadoopProperties() {
604609
return catalog.getCatalogProperty().getHadoopProperties();
605610
}
@@ -1119,13 +1124,12 @@ private List<HiveMetaStoreCache.FileCacheValue> getFilesForPartitions(
11191124
getRemoteTable().getSd().getLocation(), null, Maps.newHashMap()));
11201125
}
11211126
// Get files for all partitions.
1122-
String bindBrokerName = catalog.bindBrokerName();
11231127
if (LOG.isDebugEnabled()) {
11241128
for (HivePartition partition : hivePartitions) {
11251129
LOG.debug("Chosen partition for table {}. [{}]", name, partition.toString());
11261130
}
11271131
}
1128-
return cache.getFilesByPartitionsWithoutCache(hivePartitions, bindBrokerName,
1132+
return cache.getFilesByPartitionsWithoutCache(hivePartitions,
11291133
new FileSystemDirectoryLister(), null);
11301134
}
11311135

0 commit comments

Comments
 (0)