@@ -420,6 +420,7 @@ def __init__(
420420 self .version_aware = version_aware
421421 self .client_kwargs = client_kwargs or {}
422422 self .config_kwargs = config_kwargs or {}
423+ self .requester_pays = requester_pays
423424 self .req_kw = {"RequestPayer" : "requester" } if requester_pays else {}
424425 self .s3_additional_kwargs = s3_additional_kwargs or {}
425426 self .use_ssl = use_ssl
@@ -469,10 +470,18 @@ async def _call_s3(self, method, *akwarglist, **kwargs):
469470
470471 def _get_s3_method_kwargs (self , method , * akwarglist , ** kwargs ):
471472 additional_kwargs = self .s3_additional_kwargs .copy ()
473+ # Apply request-wide defaults; unsupported parameters are filtered below.
474+ additional_kwargs .update (self .req_kw )
472475 for akwargs in akwarglist :
473476 additional_kwargs .update (akwargs )
474477 # Add the normal kwargs in
475478 additional_kwargs .update (kwargs )
479+ # A narrower scope can explicitly disable the request-wide default.
480+ if (
481+ "RequestPayer" in additional_kwargs
482+ and additional_kwargs ["RequestPayer" ] is None
483+ ):
484+ additional_kwargs .pop ("RequestPayer" )
476485 # filter all kwargs
477486 return self ._filter_kwargs (method , additional_kwargs )
478487
@@ -808,7 +817,7 @@ def _open(
808817 if fill_cache is None :
809818 fill_cache = self .default_fill_cache
810819 if requester_pays is None :
811- requester_pays = bool ( self .req_kw )
820+ requester_pays = self .requester_pays
812821
813822 acl = (
814823 acl
@@ -1258,16 +1267,16 @@ async def _exists(self, path):
12581267 except FileNotFoundError :
12591268 # might still be a bucket we can access but don't own
12601269 pass
1261- try :
1262- await self ._call_s3 ( "head_bucket" , Bucket = bucket , ** self . req_kw )
1263- return True
1264- except Exception :
1265- pass
1266- try :
1267- await self . _call_s3 ( "get_bucket_location" , Bucket = bucket , ** self . req_kw )
1268- return True
1269- except Exception :
1270- return False
1270+ probes = [( "head_bucket" , {}), ( "get_bucket_location" , {})]
1271+ if self .requester_pays :
1272+ probes . append (( "list_objects_v2" , { "MaxKeys" : 1 }))
1273+ for method , extra in probes :
1274+ try :
1275+ await self . _call_s3 ( method , Bucket = bucket , ** extra )
1276+ return True
1277+ except Exception as e :
1278+ logger . debug ( "bucket probe %s failed for %s: %s" , method , bucket , e )
1279+ return False
12711280
12721281 exists = sync_wrapper (_exists )
12731282
@@ -1311,7 +1320,6 @@ async def _call_and_read():
13111320 Key = key ,
13121321 ** version_id_kw (version_id or vers ),
13131322 ** head ,
1314- ** self .req_kw ,
13151323 )
13161324 try :
13171325 return await resp ["Body" ].read ()
@@ -1743,7 +1751,6 @@ async def _info(self, path, bucket=None, key=None, refresh=False, version_id=Non
17431751 Bucket = bucket ,
17441752 Key = key ,
17451753 ** version_id_kw (version_id ),
1746- ** self .req_kw ,
17471754 )
17481755 return {
17491756 "ETag" : out .get ("ETag" , "" ),
@@ -1761,7 +1768,7 @@ async def _info(self, path, bucket=None, key=None, refresh=False, version_id=Non
17611768 raise translate_boto_error (e , set_cause = False )
17621769 else :
17631770 try :
1764- out = await self ._call_s3 ("head_bucket" , Bucket = bucket , ** self . req_kw )
1771+ out = await self ._call_s3 ("head_bucket" , Bucket = bucket )
17651772 return {
17661773 "name" : bucket ,
17671774 "type" : "directory" ,
@@ -1782,7 +1789,6 @@ async def _info(self, path, bucket=None, key=None, refresh=False, version_id=Non
17821789 Prefix = key .rstrip ("/" ) + "/" if key else "" ,
17831790 Delimiter = "/" ,
17841791 MaxKeys = 1 ,
1785- ** self .req_kw ,
17861792 )
17871793 if (
17881794 out .get ("KeyCount" , 0 ) > 0
@@ -1880,7 +1886,6 @@ async def _object_version_info(self, path, **kwargs):
18801886 kwargs ,
18811887 Bucket = bucket ,
18821888 Prefix = key ,
1883- ** self .req_kw ,
18841889 )
18851890 versions .extend (out ["Versions" ])
18861891 kwargs .update (
@@ -1912,7 +1917,6 @@ async def _metadata(self, path, refresh=False, **kwargs):
19121917 Bucket = bucket ,
19131918 Key = key ,
19141919 ** version_id_kw (version_id ),
1915- ** self .req_kw ,
19161920 )
19171921 meta = {k .replace ("_" , "-" ): v for k , v in response ["Metadata" ].items ()}
19181922 return meta
@@ -2413,7 +2417,7 @@ async def _rm_versioned_bucket_contents(self, bucket):
24132417 await self .set_session ()
24142418 s3 = await self .get_s3 (bucket )
24152419 pag = s3 .get_paginator ("list_object_versions" )
2416- async for plist in pag .paginate (Bucket = bucket ):
2420+ async for plist in pag .paginate (Bucket = bucket , ** self . req_kw ):
24172421 obs = plist .get ("Versions" , []) + plist .get ("DeleteMarkers" , [])
24182422 delete_keys = {
24192423 "Objects" : [
@@ -2546,6 +2550,8 @@ def __init__(
25462550 self .fill_cache = fill_cache
25472551 self .s3_additional_kwargs = s3_additional_kwargs or {}
25482552 self .req_kw = {"RequestPayer" : "requester" } if requester_pays else {}
2553+ self ._request_payer_kw = {"RequestPayer" : self .req_kw .get ("RequestPayer" )}
2554+ self .read_kw = self ._request_payer_kw
25492555 if "r" not in mode :
25502556 if block_size < 5 * 2 ** 20 :
25512557 raise ValueError ("Block size must be >=5MB" )
@@ -2589,7 +2595,6 @@ def __init__(
25892595 Bucket = bucket ,
25902596 Key = key ,
25912597 ** version_id_kw (version_id ),
2592- ** self .req_kw ,
25932598 )
25942599
25952600 head = {
@@ -2615,10 +2620,19 @@ def __init__(
26152620 and "ETag" in self .details
26162621 and not s3 .local_expiry_check
26172622 ):
2618- self .req_kw ["IfMatch" ] = self .details ["ETag" ]
2623+ self .read_kw = {
2624+ ** self ._request_payer_kw ,
2625+ "IfMatch" : self .details ["ETag" ],
2626+ }
26192627
26202628 def _call_s3 (self , method , * kwarglist , ** kwargs ):
2621- return self .fs .call_s3 (method , self .s3_additional_kwargs , * kwarglist , ** kwargs )
2629+ return self .fs .call_s3 (
2630+ method ,
2631+ self .s3_additional_kwargs ,
2632+ self ._request_payer_kw ,
2633+ * kwarglist ,
2634+ ** kwargs ,
2635+ )
26222636
26232637 def _initiate_upload (self ):
26242638 if self .autocommit and not self .append_block and self .tell () < self .blocksize :
@@ -2694,7 +2708,7 @@ def _fetch_range(self, start, end):
26942708 self .version_id ,
26952709 start ,
26962710 end ,
2697- req_kw = self .req_kw ,
2711+ req_kw = self .read_kw ,
26982712 details = self .details ,
26992713 )
27002714
0 commit comments