@@ -432,7 +432,8 @@ private Map<PartitionCacheKey, HivePartition> loadPartitions(Iterable<? extends
432432 Preconditions .checkNotNull (bdpAuthContext , "bdp auth info cannot be null" );
433433 for (Partition partition : partitions ) {
434434 StorageDescriptor sd = partition .getSd ();
435- ret .put (new PartitionCacheKey (bdpAuthContext .getHadoopUserName (), dbName , tblName , partition .getValues ()),
435+ ret .put (new PartitionCacheKey (bdpAuthContext .getHadoopUserName (), bdpAuthContext .getUserToken (), dbName ,
436+ tblName , partition .getValues ()),
436437 new HivePartition (dbName , tblName , false ,
437438 sd .getInputFormat (), sd .getLocation (), partition .getValues (), partition .getParameters ()));
438439 }
@@ -449,6 +450,7 @@ private FileCacheValue getFileCache(String location, String inputFormat,
449450 Preconditions .checkNotNull (bdpAuthContext , "bdp auth info cannot be null" );
450451 RemoteFileSystem fs = Env .getCurrentEnv ().getExtMetaCacheMgr ().getFsCache ().getRemoteFileSystem (
451452 new FileSystemCache .FileSystemCacheKey (bdpAuthContext .getHadoopUserName (),
453+ bdpAuthContext .getUserToken (),
452454 LocationPath .getFSIdentity (location , bindBrokerName ),
453455 catalog .getCatalogProperty ().getProperties (),
454456 bindBrokerName , jobConf ));
@@ -610,9 +612,10 @@ public List<FileCacheValue> getFilesByPartitions(List<HivePartition> partitions,
610612 Preconditions .checkNotNull (bdpAuthContext , "bdp auth info cannot be null" );
611613 List <FileCacheKey > keys = partitions .stream ().map (p -> p .isDummyPartition ()
612614 ? FileCacheKey .createDummyCacheKey (bdpAuthContext .getHadoopUserName (),
613- p .getDbName (), p .getTblName (), p .getPath (), p .getInputFormat (), bindBrokerName )
614- : new FileCacheKey (bdpAuthContext .getHadoopUserName (), p .getPath (), p .getInputFormat (),
615- p .getPartitionValues (), bindBrokerName ))
615+ bdpAuthContext .getUserToken (), p .getDbName (), p .getTblName (), p .getPath (),
616+ p .getInputFormat (), bindBrokerName )
617+ : new FileCacheKey (bdpAuthContext .getHadoopUserName (), bdpAuthContext .getUserToken (),
618+ p .getPath (), p .getInputFormat (), p .getPartitionValues (), bindBrokerName ))
616619 .collect (Collectors .toList ());
617620 if (withCache ) {
618621 fileLists = new ArrayList <>(fileCacheRef .get ().getAll (keys ).values ());
@@ -646,8 +649,8 @@ public List<FileCacheValue> getFilesByPartitions(List<HivePartition> partitions,
646649
647650 public HivePartition getHivePartition (String dbName , String name , List <String > partitionValues ) {
648651 Preconditions .checkNotNull (BDPAuthContext .get (), "bdp auth info cannot be null" );
649- return partitionCache .get (new PartitionCacheKey (BDPAuthContext .get ().getHadoopUserName (), dbName , name ,
650- partitionValues ));
652+ return partitionCache .get (new PartitionCacheKey (BDPAuthContext .get ().getHadoopUserName (),
653+ BDPAuthContext . get (). getUserToken (), dbName , name , partitionValues ));
651654 }
652655
653656 public List <HivePartition > getAllPartitionsWithCache (String dbName , String name ,
@@ -666,7 +669,8 @@ private List<HivePartition> getAllPartitions(String dbName, String name, List<Li
666669 BDPAuthContext bdpAuthContext = BDPAuthContext .get ();
667670 Preconditions .checkNotNull (bdpAuthContext , "bdp auth info cannot be null" );
668671 List <PartitionCacheKey > keys = partitionValuesList .stream ()
669- .map (p -> new PartitionCacheKey (bdpAuthContext .getHadoopUserName (), dbName , name , p ))
672+ .map (p -> new PartitionCacheKey (bdpAuthContext .getHadoopUserName (), bdpAuthContext .getUserToken (),
673+ dbName , name , p ))
670674 .collect (Collectors .toList ());
671675
672676 List <HivePartition > partitions ;
@@ -954,6 +958,7 @@ public List<FileCacheValue> getFilesByTransaction(List<HivePartition> partitions
954958 String acidVersionPath = new Path (baseOrDeltaPath , "_orc_acid_version" ).toUri ().toString ();
955959 RemoteFileSystem fs = Env .getCurrentEnv ().getExtMetaCacheMgr ().getFsCache ().getRemoteFileSystem (
956960 new FileSystemCache .FileSystemCacheKey (bdpAuthContext .getHadoopUserName (),
961+ bdpAuthContext .getUserToken (),
957962 LocationPath .getFSIdentity (baseOrDeltaPath .toUri ().toString (),
958963 bindBrokerName ),
959964 catalog .getCatalogProperty ().getProperties (),
@@ -979,6 +984,7 @@ public List<FileCacheValue> getFilesByTransaction(List<HivePartition> partitions
979984 String location = delta .getPath ().toString ();
980985 RemoteFileSystem fs = Env .getCurrentEnv ().getExtMetaCacheMgr ().getFsCache ().getRemoteFileSystem (
981986 new FileSystemCache .FileSystemCacheKey (bdpAuthContext .getHadoopUserName (),
987+ bdpAuthContext .getUserToken (),
982988 LocationPath .getFSIdentity (location , bindBrokerName ),
983989 catalog .getCatalogProperty ().getProperties (), bindBrokerName , jobConf ));
984990 List <RemoteFile > remoteFiles = new ArrayList <>();
@@ -1007,6 +1013,7 @@ public List<FileCacheValue> getFilesByTransaction(List<HivePartition> partitions
10071013 String location = directory .getBaseDirectory ().toString ();
10081014 RemoteFileSystem fs = Env .getCurrentEnv ().getExtMetaCacheMgr ().getFsCache ().getRemoteFileSystem (
10091015 new FileSystemCache .FileSystemCacheKey (bdpAuthContext .getHadoopUserName (),
1016+ bdpAuthContext .getUserToken (),
10101017 LocationPath .getFSIdentity (location , bindBrokerName ),
10111018 catalog .getCatalogProperty ().getProperties (), bindBrokerName , jobConf ));
10121019 List <RemoteFile > remoteFiles = new ArrayList <>();
@@ -1169,12 +1176,15 @@ public String toString() {
11691176 @ Data
11701177 public static class PartitionCacheKey {
11711178 private String hadoopUserName ;
1179+ private String userToken ;
11721180 private String dbName ;
11731181 private String tblName ;
11741182 private List <String > values ;
11751183
1176- public PartitionCacheKey (String hadoopUserName , String dbName , String tblName , List <String > values ) {
1184+ public PartitionCacheKey (String hadoopUserName , String userToken , String dbName , String tblName ,
1185+ List <String > values ) {
11771186 this .hadoopUserName = hadoopUserName ;
1187+ this .userToken = userToken ;
11781188 this .dbName = dbName ;
11791189 this .tblName = tblName ;
11801190 this .values = values ;
@@ -1189,19 +1199,20 @@ public boolean equals(Object obj) {
11891199 return false ;
11901200 }
11911201 return hadoopUserName .equals (((PartitionCacheKey ) obj ).hadoopUserName )
1202+ && userToken .equals (((PartitionCacheKey ) obj ).userToken )
11921203 && dbName .equals (((PartitionCacheKey ) obj ).dbName )
11931204 && tblName .equals (((PartitionCacheKey ) obj ).tblName )
11941205 && Objects .equals (values , ((PartitionCacheKey ) obj ).values );
11951206 }
11961207
11971208 @ Override
11981209 public int hashCode () {
1199- return Objects .hash (hadoopUserName , dbName , tblName , values );
1210+ return Objects .hash (hadoopUserName , userToken , dbName , tblName , values );
12001211 }
12011212
12021213 @ Override
12031214 public String toString () {
1204- return "PartitionCacheKey{" + "hadoopUserName='" + hadoopUserName + '\''
1215+ return "PartitionCacheKey{" + "hadoopUserName='" + hadoopUserName + '\'' + "userToken='" + userToken + '\''
12051216 + "dbName='" + dbName + '\'' + ", tblName='" + tblName + '\'' + ", values="
12061217 + values + '}' ;
12071218 }
@@ -1210,6 +1221,7 @@ public String toString() {
12101221 @ Data
12111222 public static class FileCacheKey {
12121223 private String hadoopUserName ;
1224+ private String userToken ;
12131225 private String dummyKey ;
12141226 private String location ;
12151227 // not in key
@@ -1221,19 +1233,22 @@ public static class FileCacheKey {
12211233 // partitionValues would be ["part1", "part2"]
12221234 protected List <String > partitionValues ;
12231235
1224- public FileCacheKey (String hadoopUserName , String location , String inputFormat , List <String > partitionValues ,
1236+ public FileCacheKey (String hadoopUserName , String userToken , String location , String inputFormat ,
1237+ List <String > partitionValues ,
12251238 String bindBrokerName ) {
12261239 this .hadoopUserName = hadoopUserName ;
1240+ this .userToken = userToken ;
12271241 this .location = location ;
12281242 this .inputFormat = inputFormat ;
12291243 this .partitionValues = partitionValues == null ? Lists .newArrayList () : partitionValues ;
12301244 this .bindBrokerName = bindBrokerName ;
12311245 }
12321246
1233- public static FileCacheKey createDummyCacheKey (String hadoopUserName , String dbName , String tblName ,
1234- String location , String inputFormat , String bindBrokerName ) {
1235- FileCacheKey fileCacheKey = new FileCacheKey (hadoopUserName , location , inputFormat , null ,
1236- bindBrokerName );
1247+ public static FileCacheKey createDummyCacheKey (String hadoopUserName , String userToken , String dbName ,
1248+ String tblName , String location , String inputFormat ,
1249+ String bindBrokerName ) {
1250+ FileCacheKey fileCacheKey = new FileCacheKey (hadoopUserName , userToken , location , inputFormat ,
1251+ null , bindBrokerName );
12371252 fileCacheKey .dummyKey = dbName + "." + tblName ;
12381253 return fileCacheKey ;
12391254 }
@@ -1248,25 +1263,27 @@ public boolean equals(Object obj) {
12481263 }
12491264 if (dummyKey != null ) {
12501265 return hadoopUserName .equals (((FileCacheKey ) obj ).hadoopUserName )
1266+ && userToken .equals (((FileCacheKey ) obj ).userToken )
12511267 && dummyKey .equals (((FileCacheKey ) obj ).dummyKey );
12521268 }
12531269 return hadoopUserName .equals (((FileCacheKey ) obj ).hadoopUserName )
1270+ && userToken .equals (((FileCacheKey ) obj ).userToken )
12541271 && location .equals (((FileCacheKey ) obj ).location )
12551272 && Objects .equals (partitionValues , ((FileCacheKey ) obj ).partitionValues );
12561273 }
12571274
12581275 @ Override
12591276 public int hashCode () {
12601277 if (dummyKey != null ) {
1261- return Objects .hash (hadoopUserName , dummyKey );
1278+ return Objects .hash (hadoopUserName , userToken , dummyKey );
12621279 }
1263- return Objects .hash (hadoopUserName , location , partitionValues );
1280+ return Objects .hash (hadoopUserName , userToken , location , partitionValues );
12641281 }
12651282
12661283 @ Override
12671284 public String toString () {
1268- return "FileCacheKey{" + "hadoopUserName='" + hadoopUserName + '\'' + ", location ='" + location
1269- + '\'' + ", inputFormat='" + inputFormat + '\'' + '}' ;
1285+ return "FileCacheKey{" + "hadoopUserName='" + hadoopUserName + '\'' + "userToken ='" + userToken + '\'' +
1286+ ", location='" + location + '\'' + ", inputFormat='" + inputFormat + '\'' + '}' ;
12701287 }
12711288 }
12721289
0 commit comments