Skip to content

Commit fb62467

Browse files
weixiaoxing3caiconghui1
authored andcommitted
[fix][JD] modify hdfs fileSystem cache key (apache#317)
1 parent 01aec10 commit fb62467

3 files changed

Lines changed: 50 additions & 26 deletions

File tree

‎fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveMetaStoreCache.java‎

Lines changed: 36 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -432,7 +432,8 @@ private Map<PartitionCacheKey, HivePartition> loadPartitions(Iterable<? extends
432432
Preconditions.checkNotNull(bdpAuthContext, "bdp auth info cannot be null");
433433
for (Partition partition : partitions) {
434434
StorageDescriptor sd = partition.getSd();
435-
ret.put(new PartitionCacheKey(bdpAuthContext.getHadoopUserName(), dbName, tblName, partition.getValues()),
435+
ret.put(new PartitionCacheKey(bdpAuthContext.getHadoopUserName(), bdpAuthContext.getUserToken(), dbName,
436+
tblName, partition.getValues()),
436437
new HivePartition(dbName, tblName, false,
437438
sd.getInputFormat(), sd.getLocation(), partition.getValues(), partition.getParameters()));
438439
}
@@ -449,6 +450,7 @@ private FileCacheValue getFileCache(String location, String inputFormat,
449450
Preconditions.checkNotNull(bdpAuthContext, "bdp auth info cannot be null");
450451
RemoteFileSystem fs = Env.getCurrentEnv().getExtMetaCacheMgr().getFsCache().getRemoteFileSystem(
451452
new FileSystemCache.FileSystemCacheKey(bdpAuthContext.getHadoopUserName(),
453+
bdpAuthContext.getUserToken(),
452454
LocationPath.getFSIdentity(location, bindBrokerName),
453455
catalog.getCatalogProperty().getProperties(),
454456
bindBrokerName, jobConf));
@@ -610,9 +612,10 @@ public List<FileCacheValue> getFilesByPartitions(List<HivePartition> partitions,
610612
Preconditions.checkNotNull(bdpAuthContext, "bdp auth info cannot be null");
611613
List<FileCacheKey> keys = partitions.stream().map(p -> p.isDummyPartition()
612614
? FileCacheKey.createDummyCacheKey(bdpAuthContext.getHadoopUserName(),
613-
p.getDbName(), p.getTblName(), p.getPath(), p.getInputFormat(), bindBrokerName)
614-
: new FileCacheKey(bdpAuthContext.getHadoopUserName(), p.getPath(), p.getInputFormat(),
615-
p.getPartitionValues(), bindBrokerName))
615+
bdpAuthContext.getUserToken(), p.getDbName(), p.getTblName(), p.getPath(),
616+
p.getInputFormat(), bindBrokerName)
617+
: new FileCacheKey(bdpAuthContext.getHadoopUserName(), bdpAuthContext.getUserToken(),
618+
p.getPath(), p.getInputFormat(), p.getPartitionValues(), bindBrokerName))
616619
.collect(Collectors.toList());
617620
if (withCache) {
618621
fileLists = new ArrayList<>(fileCacheRef.get().getAll(keys).values());
@@ -646,8 +649,8 @@ public List<FileCacheValue> getFilesByPartitions(List<HivePartition> partitions,
646649

647650
public HivePartition getHivePartition(String dbName, String name, List<String> partitionValues) {
648651
Preconditions.checkNotNull(BDPAuthContext.get(), "bdp auth info cannot be null");
649-
return partitionCache.get(new PartitionCacheKey(BDPAuthContext.get().getHadoopUserName(), dbName, name,
650-
partitionValues));
652+
return partitionCache.get(new PartitionCacheKey(BDPAuthContext.get().getHadoopUserName(),
653+
BDPAuthContext.get().getUserToken(), dbName, name, partitionValues));
651654
}
652655

653656
public List<HivePartition> getAllPartitionsWithCache(String dbName, String name,
@@ -666,7 +669,8 @@ private List<HivePartition> getAllPartitions(String dbName, String name, List<Li
666669
BDPAuthContext bdpAuthContext = BDPAuthContext.get();
667670
Preconditions.checkNotNull(bdpAuthContext, "bdp auth info cannot be null");
668671
List<PartitionCacheKey> keys = partitionValuesList.stream()
669-
.map(p -> new PartitionCacheKey(bdpAuthContext.getHadoopUserName(), dbName, name, p))
672+
.map(p -> new PartitionCacheKey(bdpAuthContext.getHadoopUserName(), bdpAuthContext.getUserToken(),
673+
dbName, name, p))
670674
.collect(Collectors.toList());
671675

672676
List<HivePartition> partitions;
@@ -954,6 +958,7 @@ public List<FileCacheValue> getFilesByTransaction(List<HivePartition> partitions
954958
String acidVersionPath = new Path(baseOrDeltaPath, "_orc_acid_version").toUri().toString();
955959
RemoteFileSystem fs = Env.getCurrentEnv().getExtMetaCacheMgr().getFsCache().getRemoteFileSystem(
956960
new FileSystemCache.FileSystemCacheKey(bdpAuthContext.getHadoopUserName(),
961+
bdpAuthContext.getUserToken(),
957962
LocationPath.getFSIdentity(baseOrDeltaPath.toUri().toString(),
958963
bindBrokerName),
959964
catalog.getCatalogProperty().getProperties(),
@@ -979,6 +984,7 @@ public List<FileCacheValue> getFilesByTransaction(List<HivePartition> partitions
979984
String location = delta.getPath().toString();
980985
RemoteFileSystem fs = Env.getCurrentEnv().getExtMetaCacheMgr().getFsCache().getRemoteFileSystem(
981986
new FileSystemCache.FileSystemCacheKey(bdpAuthContext.getHadoopUserName(),
987+
bdpAuthContext.getUserToken(),
982988
LocationPath.getFSIdentity(location, bindBrokerName),
983989
catalog.getCatalogProperty().getProperties(), bindBrokerName, jobConf));
984990
List<RemoteFile> remoteFiles = new ArrayList<>();
@@ -1007,6 +1013,7 @@ public List<FileCacheValue> getFilesByTransaction(List<HivePartition> partitions
10071013
String location = directory.getBaseDirectory().toString();
10081014
RemoteFileSystem fs = Env.getCurrentEnv().getExtMetaCacheMgr().getFsCache().getRemoteFileSystem(
10091015
new FileSystemCache.FileSystemCacheKey(bdpAuthContext.getHadoopUserName(),
1016+
bdpAuthContext.getUserToken(),
10101017
LocationPath.getFSIdentity(location, bindBrokerName),
10111018
catalog.getCatalogProperty().getProperties(), bindBrokerName, jobConf));
10121019
List<RemoteFile> remoteFiles = new ArrayList<>();
@@ -1169,12 +1176,15 @@ public String toString() {
11691176
@Data
11701177
public static class PartitionCacheKey {
11711178
private String hadoopUserName;
1179+
private String userToken;
11721180
private String dbName;
11731181
private String tblName;
11741182
private List<String> values;
11751183

1176-
public PartitionCacheKey(String hadoopUserName, String dbName, String tblName, List<String> values) {
1184+
public PartitionCacheKey(String hadoopUserName, String userToken, String dbName, String tblName,
1185+
List<String> values) {
11771186
this.hadoopUserName = hadoopUserName;
1187+
this.userToken = userToken;
11781188
this.dbName = dbName;
11791189
this.tblName = tblName;
11801190
this.values = values;
@@ -1189,19 +1199,20 @@ public boolean equals(Object obj) {
11891199
return false;
11901200
}
11911201
return hadoopUserName.equals(((PartitionCacheKey) obj).hadoopUserName)
1202+
&& userToken.equals(((PartitionCacheKey) obj).userToken)
11921203
&& dbName.equals(((PartitionCacheKey) obj).dbName)
11931204
&& tblName.equals(((PartitionCacheKey) obj).tblName)
11941205
&& Objects.equals(values, ((PartitionCacheKey) obj).values);
11951206
}
11961207

11971208
@Override
11981209
public int hashCode() {
1199-
return Objects.hash(hadoopUserName, dbName, tblName, values);
1210+
return Objects.hash(hadoopUserName, userToken, dbName, tblName, values);
12001211
}
12011212

12021213
@Override
12031214
public String toString() {
1204-
return "PartitionCacheKey{" + "hadoopUserName='" + hadoopUserName + '\''
1215+
return "PartitionCacheKey{" + "hadoopUserName='" + hadoopUserName + '\'' + "userToken='" + userToken + '\''
12051216
+ "dbName='" + dbName + '\'' + ", tblName='" + tblName + '\'' + ", values="
12061217
+ values + '}';
12071218
}
@@ -1210,6 +1221,7 @@ public String toString() {
12101221
@Data
12111222
public static class FileCacheKey {
12121223
private String hadoopUserName;
1224+
private String userToken;
12131225
private String dummyKey;
12141226
private String location;
12151227
// not in key
@@ -1221,19 +1233,22 @@ public static class FileCacheKey {
12211233
// partitionValues would be ["part1", "part2"]
12221234
protected List<String> partitionValues;
12231235

1224-
public FileCacheKey(String hadoopUserName, String location, String inputFormat, List<String> partitionValues,
1236+
public FileCacheKey(String hadoopUserName, String userToken, String location, String inputFormat,
1237+
List<String> partitionValues,
12251238
String bindBrokerName) {
12261239
this.hadoopUserName = hadoopUserName;
1240+
this.userToken = userToken;
12271241
this.location = location;
12281242
this.inputFormat = inputFormat;
12291243
this.partitionValues = partitionValues == null ? Lists.newArrayList() : partitionValues;
12301244
this.bindBrokerName = bindBrokerName;
12311245
}
12321246

1233-
public static FileCacheKey createDummyCacheKey(String hadoopUserName, String dbName, String tblName,
1234-
String location, String inputFormat, String bindBrokerName) {
1235-
FileCacheKey fileCacheKey = new FileCacheKey(hadoopUserName, location, inputFormat, null,
1236-
bindBrokerName);
1247+
public static FileCacheKey createDummyCacheKey(String hadoopUserName, String userToken, String dbName,
1248+
String tblName, String location, String inputFormat,
1249+
String bindBrokerName) {
1250+
FileCacheKey fileCacheKey = new FileCacheKey(hadoopUserName, userToken, location, inputFormat,
1251+
null, bindBrokerName);
12371252
fileCacheKey.dummyKey = dbName + "." + tblName;
12381253
return fileCacheKey;
12391254
}
@@ -1248,25 +1263,27 @@ public boolean equals(Object obj) {
12481263
}
12491264
if (dummyKey != null) {
12501265
return hadoopUserName.equals(((FileCacheKey) obj).hadoopUserName)
1266+
&& userToken.equals(((FileCacheKey) obj).userToken)
12511267
&& dummyKey.equals(((FileCacheKey) obj).dummyKey);
12521268
}
12531269
return hadoopUserName.equals(((FileCacheKey) obj).hadoopUserName)
1270+
&& userToken.equals(((FileCacheKey) obj).userToken)
12541271
&& location.equals(((FileCacheKey) obj).location)
12551272
&& Objects.equals(partitionValues, ((FileCacheKey) obj).partitionValues);
12561273
}
12571274

12581275
@Override
12591276
public int hashCode() {
12601277
if (dummyKey != null) {
1261-
return Objects.hash(hadoopUserName, dummyKey);
1278+
return Objects.hash(hadoopUserName, userToken, dummyKey);
12621279
}
1263-
return Objects.hash(hadoopUserName, location, partitionValues);
1280+
return Objects.hash(hadoopUserName, userToken, location, partitionValues);
12641281
}
12651282

12661283
@Override
12671284
public String toString() {
1268-
return "FileCacheKey{" + "hadoopUserName='" + hadoopUserName + '\'' + ", location='" + location
1269-
+ '\'' + ", inputFormat='" + inputFormat + '\'' + '}';
1285+
return "FileCacheKey{" + "hadoopUserName='" + hadoopUserName + '\'' + "userToken='" + userToken + '\'' +
1286+
", location='" + location + '\'' + ", inputFormat='" + inputFormat + '\'' + '}';
12701287
}
12711288
}
12721289

‎fe/fe-core/src/main/java/org/apache/doris/fs/FileSystemCache.java‎

Lines changed: 13 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -56,6 +56,7 @@ public RemoteFileSystem getRemoteFileSystem(FileSystemCacheKey key) {
5656
public static class FileSystemCacheKey {
5757

5858
private final String hadoopUserName;
59+
private final String userToken;
5960
private final FileSystemType type;
6061
// eg: hdfs://nameservices1
6162
private final String fsIdent;
@@ -65,21 +66,25 @@ public static class FileSystemCacheKey {
6566
private final Configuration conf;
6667

6768
public FileSystemCacheKey(String hadoopUserName,
69+
String userToken,
6870
Pair<FileSystemType, String> fs,
6971
Map<String, String> properties,
7072
String bindBrokerName,
7173
Configuration conf) {
7274
this.hadoopUserName = hadoopUserName;
75+
this.userToken = userToken;
7376
this.type = fs.first;
7477
this.fsIdent = fs.second;
7578
this.properties = properties;
7679
this.bindBrokerName = bindBrokerName;
7780
this.conf = conf;
7881
}
7982

80-
public FileSystemCacheKey(String hadoopUserName, Pair<FileSystemType, String> fs,
81-
Map<String, String> properties, String bindBrokerName) {
82-
this(hadoopUserName, fs, properties, bindBrokerName, null);
83+
public FileSystemCacheKey(String hadoopUserName, String userToken,
84+
Pair<FileSystemType, String> fs,
85+
Map<String, String> properties,
86+
String bindBrokerName) {
87+
this(hadoopUserName, userToken, fs, properties, bindBrokerName, null);
8388
}
8489

8590
public Map<String, String> getFsProperties() {
@@ -100,7 +105,9 @@ public boolean equals(Object obj) {
100105
return false;
101106
}
102107
FileSystemCacheKey o = (FileSystemCacheKey) obj;
103-
boolean equalsWithoutBroker = hadoopUserName.equals(o.hadoopUserName) && type.equals(o.type)
108+
boolean equalsWithoutBroker = hadoopUserName.equals(o.hadoopUserName)
109+
&& userToken.equals(o.userToken)
110+
&& type.equals(o.type)
104111
&& fsIdent.equals(o.fsIdent)
105112
&& properties.equals(o.properties);
106113
if (bindBrokerName == null) {
@@ -112,9 +119,9 @@ public boolean equals(Object obj) {
112119
@Override
113120
public int hashCode() {
114121
if (bindBrokerName == null) {
115-
return Objects.hash(hadoopUserName, properties, fsIdent, type);
122+
return Objects.hash(hadoopUserName, userToken, properties, fsIdent, type);
116123
}
117-
return Objects.hash(hadoopUserName, properties, fsIdent, type, bindBrokerName);
124+
return Objects.hash(hadoopUserName, userToken, properties, fsIdent, type, bindBrokerName);
118125
}
119126
}
120127
}

‎fe/fe-core/src/main/java/org/apache/doris/fs/remote/SwitchingFileSystem.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -124,7 +124,7 @@ public Status listDirectories(String remotePath, Set<String> result) {
124124

125125
public FileSystem fileSystem(String location) {
126126
return extMetaCacheMgr.getFsCache().getRemoteFileSystem(
127-
new FileSystemCache.FileSystemCacheKey("",
127+
new FileSystemCache.FileSystemCacheKey("", "",
128128
LocationPath.getFSIdentity(location,
129129
bindBrokerName), properties, bindBrokerName));
130130
}

0 commit comments

Comments
 (0)