Skip to content

Commit efe8e4c

Browse files
committed
[fix](split) FileSystemCacheKey are always different in overload equals
1 parent e73c33e commit efe8e4c

4 files changed

Lines changed: 48 additions & 70 deletions

File tree

‎fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveMetaStoreCache.java‎

Lines changed: 12 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -350,11 +350,11 @@ private FileCacheValue getFileCache(String location, String inputFormat,
350350
List<String> partitionValues,
351351
String bindBrokerName) throws UserException {
352352
FileCacheValue result = new FileCacheValue();
353+
Map<String, String> properties = new HashMap<>();
354+
jobConf.iterator().forEachRemaining(e -> properties.put(e.getKey(), e.getValue()));
353355
RemoteFileSystem fs = Env.getCurrentEnv().getExtMetaCacheMgr().getFsCache().getRemoteFileSystem(
354356
new FileSystemCache.FileSystemCacheKey(LocationPath.getFSIdentity(
355-
location, bindBrokerName),
356-
catalog.getCatalogProperty().getProperties(),
357-
bindBrokerName, jobConf));
357+
location, bindBrokerName), properties, bindBrokerName));
358358
result.setSplittable(HiveUtil.isSplittable(fs, inputFormat, location));
359359
// For Tez engine, it may generate subdirectoies for "union" query.
360360
// So there may be files and directories in the table directory at the same time. eg:
@@ -782,12 +782,12 @@ public List<FileCacheValue> getFilesByTransaction(List<HivePartition> partitions
782782
return Collections.emptyList();
783783
}
784784
String acidVersionPath = new Path(baseOrDeltaPath, "_orc_acid_version").toUri().toString();
785+
Map<String, String> properties = new HashMap<>();
786+
jobConf.iterator().forEachRemaining(e -> properties.put(e.getKey(), e.getValue()));
785787
RemoteFileSystem fs = Env.getCurrentEnv().getExtMetaCacheMgr().getFsCache().getRemoteFileSystem(
786788
new FileSystemCache.FileSystemCacheKey(
787789
LocationPath.getFSIdentity(baseOrDeltaPath.toUri().toString(),
788-
bindBrokerName),
789-
catalog.getCatalogProperty().getProperties(),
790-
bindBrokerName, jobConf));
790+
bindBrokerName), properties, bindBrokerName));
791791
Status status = fs.exists(acidVersionPath);
792792
if (status != Status.OK) {
793793
if (status.getErrCode() == ErrCode.NOT_FOUND) {
@@ -807,10 +807,12 @@ public List<FileCacheValue> getFilesByTransaction(List<HivePartition> partitions
807807
List<DeleteDeltaInfo> deleteDeltas = new ArrayList<>();
808808
for (AcidUtils.ParsedDelta delta : directory.getCurrentDirectories()) {
809809
String location = delta.getPath().toString();
810+
Map<String, String> properties = new HashMap<>();
811+
jobConf.iterator().forEachRemaining(e -> properties.put(e.getKey(), e.getValue()));
810812
RemoteFileSystem fs = Env.getCurrentEnv().getExtMetaCacheMgr().getFsCache().getRemoteFileSystem(
811813
new FileSystemCache.FileSystemCacheKey(
812814
LocationPath.getFSIdentity(location, bindBrokerName),
813-
catalog.getCatalogProperty().getProperties(), bindBrokerName, jobConf));
815+
properties, bindBrokerName));
814816
List<RemoteFile> remoteFiles = new ArrayList<>();
815817
Status status = fs.listFiles(location, false, remoteFiles);
816818
if (status.ok()) {
@@ -832,10 +834,12 @@ public List<FileCacheValue> getFilesByTransaction(List<HivePartition> partitions
832834
// base
833835
if (directory.getBaseDirectory() != null) {
834836
String location = directory.getBaseDirectory().toString();
837+
Map<String, String> properties = new HashMap<>();
838+
jobConf.iterator().forEachRemaining(e -> properties.put(e.getKey(), e.getValue()));
835839
RemoteFileSystem fs = Env.getCurrentEnv().getExtMetaCacheMgr().getFsCache().getRemoteFileSystem(
836840
new FileSystemCache.FileSystemCacheKey(
837841
LocationPath.getFSIdentity(location, bindBrokerName),
838-
catalog.getCatalogProperty().getProperties(), bindBrokerName, jobConf));
842+
properties, bindBrokerName));
839843
List<RemoteFile> remoteFiles = new ArrayList<>();
840844
Status status = fs.listFiles(location, false, remoteFiles);
841845
if (status.ok()) {

‎fe/fe-core/src/main/java/org/apache/doris/fs/FileSystemCache.java‎

Lines changed: 9 additions & 31 deletions
Original file line numberDiff line numberDiff line change
@@ -23,16 +23,14 @@
2323
import org.apache.doris.fs.remote.RemoteFileSystem;
2424

2525
import com.github.benmanes.caffeine.cache.LoadingCache;
26-
import org.apache.hadoop.conf.Configuration;
2726

28-
import java.util.HashMap;
2927
import java.util.Map;
3028
import java.util.Objects;
3129
import java.util.OptionalLong;
3230

3331
public class FileSystemCache {
3432

35-
private final LoadingCache<FileSystemCacheKey, RemoteFileSystem> fileSystemCache;
33+
private LoadingCache<FileSystemCacheKey, RemoteFileSystem> fileSystemCache;
3634

3735
public FileSystemCache() {
3836
// no need to set refreshAfterWrite, because the FileSystem is created once and never changed
@@ -42,11 +40,11 @@ public FileSystemCache() {
4240
Config.max_remote_file_system_cache_num,
4341
false,
4442
null);
45-
fileSystemCache = fsCacheFactory.buildCache(this::loadFileSystem);
43+
fileSystemCache = fsCacheFactory.buildCache(key -> loadFileSystem(key));
4644
}
4745

4846
private RemoteFileSystem loadFileSystem(FileSystemCacheKey key) {
49-
return FileSystemFactory.getRemoteFileSystem(key.type, key.getFsProperties(), key.bindBrokerName);
47+
return FileSystemFactory.getRemoteFileSystem(key.type, key.properties, key.bindBrokerName);
5048
}
5149

5250
public RemoteFileSystem getRemoteFileSystem(FileSystemCacheKey key) {
@@ -59,32 +57,13 @@ public static class FileSystemCacheKey {
5957
private final String fsIdent;
6058
private final Map<String, String> properties;
6159
private final String bindBrokerName;
62-
// only for creating new file system
63-
private final Configuration conf;
6460

6561
public FileSystemCacheKey(Pair<FileSystemType, String> fs,
66-
Map<String, String> properties,
67-
String bindBrokerName,
68-
Configuration conf) {
62+
Map<String, String> properties, String bindBrokerName) {
6963
this.type = fs.first;
7064
this.fsIdent = fs.second;
7165
this.properties = properties;
7266
this.bindBrokerName = bindBrokerName;
73-
this.conf = conf;
74-
}
75-
76-
public FileSystemCacheKey(Pair<FileSystemType, String> fs,
77-
Map<String, String> properties, String bindBrokerName) {
78-
this(fs, properties, bindBrokerName, null);
79-
}
80-
81-
public Map<String, String> getFsProperties() {
82-
if (conf == null) {
83-
return properties;
84-
}
85-
Map<String, String> result = new HashMap<>();
86-
conf.iterator().forEachRemaining(e -> result.put(e.getKey(), e.getValue()));
87-
return result;
8867
}
8968

9069
@Override
@@ -95,14 +74,13 @@ public boolean equals(Object obj) {
9574
if (!(obj instanceof FileSystemCacheKey)) {
9675
return false;
9776
}
98-
FileSystemCacheKey o = (FileSystemCacheKey) obj;
99-
boolean equalsWithoutBroker = type.equals(o.type)
100-
&& fsIdent.equals(o.fsIdent)
101-
&& properties.equals(o.properties);
77+
boolean equalsWithoutBroker = type.equals(((FileSystemCacheKey) obj).type)
78+
&& fsIdent.equals(((FileSystemCacheKey) obj).fsIdent)
79+
&& properties == ((FileSystemCacheKey) obj).properties;
10280
if (bindBrokerName == null) {
103-
return equalsWithoutBroker && o.bindBrokerName == null;
81+
return equalsWithoutBroker;
10482
}
105-
return equalsWithoutBroker && bindBrokerName.equals(o.bindBrokerName);
83+
return equalsWithoutBroker && bindBrokerName.equals(((FileSystemCacheKey) obj).bindBrokerName);
10684
}
10785

10886
@Override

‎fe/fe-core/src/main/java/org/apache/doris/fs/remote/S3FileSystem.java‎

Lines changed: 10 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -58,20 +58,16 @@ private void initFsProperties() {
5858
@Override
5959
protected FileSystem nativeFileSystem(String remotePath) throws UserException {
6060
if (dfsFileSystem == null) {
61-
synchronized (this) {
62-
if (dfsFileSystem == null) {
63-
Configuration conf = new Configuration();
64-
System.setProperty("com.amazonaws.services.s3.enableV4", "true");
65-
// the entry value in properties may be null, and
66-
PropertyConverter.convertToHadoopFSProperties(properties).entrySet().stream()
67-
.filter(entry -> entry.getKey() != null && entry.getValue() != null)
68-
.forEach(entry -> conf.set(entry.getKey(), entry.getValue()));
69-
try {
70-
dfsFileSystem = FileSystem.get(new Path(remotePath).toUri(), conf);
71-
} catch (Exception e) {
72-
throw new UserException("Failed to get S3 FileSystem for " + e.getMessage(), e);
73-
}
74-
}
61+
Configuration conf = new Configuration();
62+
System.setProperty("com.amazonaws.services.s3.enableV4", "true");
63+
// the entry value in properties may be null, and
64+
PropertyConverter.convertToHadoopFSProperties(properties).entrySet().stream()
65+
.filter(entry -> entry.getKey() != null && entry.getValue() != null)
66+
.forEach(entry -> conf.set(entry.getKey(), entry.getValue()));
67+
try {
68+
dfsFileSystem = FileSystem.get(new Path(remotePath).toUri(), conf);
69+
} catch (Exception e) {
70+
throw new UserException("Failed to get S3 FileSystem for " + e.getMessage(), e);
7571
}
7672
}
7773
return dfsFileSystem;

‎fe/fe-core/src/main/java/org/apache/doris/fs/remote/dfs/DFSFileSystem.java‎

Lines changed: 17 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -73,25 +73,25 @@ public DFSFileSystem(StorageBackend.StorageType type, Map<String, String> proper
7373
@VisibleForTesting
7474
@Override
7575
public FileSystem nativeFileSystem(String remotePath) throws UserException {
76-
if (dfsFileSystem == null) {
77-
synchronized (this) {
78-
if (dfsFileSystem == null) {
79-
Configuration conf = new HdfsConfiguration();
80-
for (Map.Entry<String, String> propEntry : properties.entrySet()) {
81-
conf.set(propEntry.getKey(), propEntry.getValue());
82-
}
76+
if (dfsFileSystem != null) {
77+
return dfsFileSystem;
78+
}
8379

84-
dfsFileSystem = HadoopUGI.ugiDoAs(AuthenticationConfig.getKerberosConfig(conf), () -> {
85-
try {
86-
return FileSystem.get(new Path(remotePath).toUri(), conf);
87-
} catch (IOException e) {
88-
throw new RuntimeException(e);
89-
}
90-
});
91-
operations = new HDFSFileOperations(dfsFileSystem);
92-
}
93-
}
80+
Configuration conf = new HdfsConfiguration();
81+
for (Map.Entry<String, String> propEntry : properties.entrySet()) {
82+
conf.set(propEntry.getKey(), propEntry.getValue());
9483
}
84+
85+
dfsFileSystem = HadoopUGI.ugiDoAs(AuthenticationConfig.getKerberosConfig(conf), () -> {
86+
try {
87+
return FileSystem.get(new Path(remotePath).toUri(), conf);
88+
} catch (IOException e) {
89+
throw new RuntimeException(e);
90+
}
91+
});
92+
93+
Preconditions.checkNotNull(dfsFileSystem);
94+
operations = new HDFSFileOperations(dfsFileSystem);
9595
return dfsFileSystem;
9696
}
9797

0 commit comments

Comments
 (0)