Skip to content

Commit d3dd9b7

Browse files
authored
Fix requester pays propogation and bug with combining head_bucket and requester_pays=True (#1039)
1 parent 609950a commit d3dd9b7

2 files changed

Lines changed: 100 additions & 22 deletions

File tree

‎s3fs/core.py‎

Lines changed: 36 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -420,6 +420,7 @@ def __init__(
420420
self.version_aware = version_aware
421421
self.client_kwargs = client_kwargs or {}
422422
self.config_kwargs = config_kwargs or {}
423+
self.requester_pays = requester_pays
423424
self.req_kw = {"RequestPayer": "requester"} if requester_pays else {}
424425
self.s3_additional_kwargs = s3_additional_kwargs or {}
425426
self.use_ssl = use_ssl
@@ -469,10 +470,18 @@ async def _call_s3(self, method, *akwarglist, **kwargs):
469470

470471
def _get_s3_method_kwargs(self, method, *akwarglist, **kwargs):
471472
additional_kwargs = self.s3_additional_kwargs.copy()
473+
# Apply request-wide defaults; unsupported parameters are filtered below.
474+
additional_kwargs.update(self.req_kw)
472475
for akwargs in akwarglist:
473476
additional_kwargs.update(akwargs)
474477
# Add the normal kwargs in
475478
additional_kwargs.update(kwargs)
479+
# A narrower scope can explicitly disable the request-wide default.
480+
if (
481+
"RequestPayer" in additional_kwargs
482+
and additional_kwargs["RequestPayer"] is None
483+
):
484+
additional_kwargs.pop("RequestPayer")
476485
# filter all kwargs
477486
return self._filter_kwargs(method, additional_kwargs)
478487

@@ -808,7 +817,7 @@ def _open(
808817
if fill_cache is None:
809818
fill_cache = self.default_fill_cache
810819
if requester_pays is None:
811-
requester_pays = bool(self.req_kw)
820+
requester_pays = self.requester_pays
812821

813822
acl = (
814823
acl
@@ -1258,16 +1267,16 @@ async def _exists(self, path):
12581267
except FileNotFoundError:
12591268
# might still be a bucket we can access but don't own
12601269
pass
1261-
try:
1262-
await self._call_s3("head_bucket", Bucket=bucket, **self.req_kw)
1263-
return True
1264-
except Exception:
1265-
pass
1266-
try:
1267-
await self._call_s3("get_bucket_location", Bucket=bucket, **self.req_kw)
1268-
return True
1269-
except Exception:
1270-
return False
1270+
probes = [("head_bucket", {}), ("get_bucket_location", {})]
1271+
if self.requester_pays:
1272+
probes.append(("list_objects_v2", {"MaxKeys": 1}))
1273+
for method, extra in probes:
1274+
try:
1275+
await self._call_s3(method, Bucket=bucket, **extra)
1276+
return True
1277+
except Exception as e:
1278+
logger.debug("bucket probe %s failed for %s: %s", method, bucket, e)
1279+
return False
12711280

12721281
exists = sync_wrapper(_exists)
12731282

@@ -1311,7 +1320,6 @@ async def _call_and_read():
13111320
Key=key,
13121321
**version_id_kw(version_id or vers),
13131322
**head,
1314-
**self.req_kw,
13151323
)
13161324
try:
13171325
return await resp["Body"].read()
@@ -1743,7 +1751,6 @@ async def _info(self, path, bucket=None, key=None, refresh=False, version_id=Non
17431751
Bucket=bucket,
17441752
Key=key,
17451753
**version_id_kw(version_id),
1746-
**self.req_kw,
17471754
)
17481755
return {
17491756
"ETag": out.get("ETag", ""),
@@ -1761,7 +1768,7 @@ async def _info(self, path, bucket=None, key=None, refresh=False, version_id=Non
17611768
raise translate_boto_error(e, set_cause=False)
17621769
else:
17631770
try:
1764-
out = await self._call_s3("head_bucket", Bucket=bucket, **self.req_kw)
1771+
out = await self._call_s3("head_bucket", Bucket=bucket)
17651772
return {
17661773
"name": bucket,
17671774
"type": "directory",
@@ -1782,7 +1789,6 @@ async def _info(self, path, bucket=None, key=None, refresh=False, version_id=Non
17821789
Prefix=key.rstrip("/") + "/" if key else "",
17831790
Delimiter="/",
17841791
MaxKeys=1,
1785-
**self.req_kw,
17861792
)
17871793
if (
17881794
out.get("KeyCount", 0) > 0
@@ -1880,7 +1886,6 @@ async def _object_version_info(self, path, **kwargs):
18801886
kwargs,
18811887
Bucket=bucket,
18821888
Prefix=key,
1883-
**self.req_kw,
18841889
)
18851890
versions.extend(out["Versions"])
18861891
kwargs.update(
@@ -1912,7 +1917,6 @@ async def _metadata(self, path, refresh=False, **kwargs):
19121917
Bucket=bucket,
19131918
Key=key,
19141919
**version_id_kw(version_id),
1915-
**self.req_kw,
19161920
)
19171921
meta = {k.replace("_", "-"): v for k, v in response["Metadata"].items()}
19181922
return meta
@@ -2413,7 +2417,7 @@ async def _rm_versioned_bucket_contents(self, bucket):
24132417
await self.set_session()
24142418
s3 = await self.get_s3(bucket)
24152419
pag = s3.get_paginator("list_object_versions")
2416-
async for plist in pag.paginate(Bucket=bucket):
2420+
async for plist in pag.paginate(Bucket=bucket, **self.req_kw):
24172421
obs = plist.get("Versions", []) + plist.get("DeleteMarkers", [])
24182422
delete_keys = {
24192423
"Objects": [
@@ -2546,6 +2550,8 @@ def __init__(
25462550
self.fill_cache = fill_cache
25472551
self.s3_additional_kwargs = s3_additional_kwargs or {}
25482552
self.req_kw = {"RequestPayer": "requester"} if requester_pays else {}
2553+
self._request_payer_kw = {"RequestPayer": self.req_kw.get("RequestPayer")}
2554+
self.read_kw = self._request_payer_kw
25492555
if "r" not in mode:
25502556
if block_size < 5 * 2**20:
25512557
raise ValueError("Block size must be >=5MB")
@@ -2589,7 +2595,6 @@ def __init__(
25892595
Bucket=bucket,
25902596
Key=key,
25912597
**version_id_kw(version_id),
2592-
**self.req_kw,
25932598
)
25942599

25952600
head = {
@@ -2615,10 +2620,19 @@ def __init__(
26152620
and "ETag" in self.details
26162621
and not s3.local_expiry_check
26172622
):
2618-
self.req_kw["IfMatch"] = self.details["ETag"]
2623+
self.read_kw = {
2624+
**self._request_payer_kw,
2625+
"IfMatch": self.details["ETag"],
2626+
}
26192627

26202628
def _call_s3(self, method, *kwarglist, **kwargs):
2621-
return self.fs.call_s3(method, self.s3_additional_kwargs, *kwarglist, **kwargs)
2629+
return self.fs.call_s3(
2630+
method,
2631+
self.s3_additional_kwargs,
2632+
self._request_payer_kw,
2633+
*kwarglist,
2634+
**kwargs,
2635+
)
26222636

26232637
def _initiate_upload(self):
26242638
if self.autocommit and not self.append_block and self.tell() < self.blocksize:
@@ -2694,7 +2708,7 @@ def _fetch_range(self, start, end):
26942708
self.version_id,
26952709
start,
26962710
end,
2697-
req_kw=self.req_kw,
2711+
req_kw=self.read_kw,
26982712
details=self.details,
26992713
)
27002714

‎s3fs/tests/test_s3fs.py‎

Lines changed: 64 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2074,11 +2074,75 @@ def test_requester_pays(s3):
20742074
fn = test_bucket_name + "/myfile"
20752075
s3 = S3FileSystem(requester_pays=True, client_kwargs={"endpoint_url": endpoint_uri})
20762076
assert s3.req_kw["RequestPayer"] == "requester"
2077+
put_kwargs = s3._get_s3_method_kwargs(
2078+
s3.s3.put_object, Bucket=test_bucket_name, Key="myfile"
2079+
)
2080+
assert put_kwargs["RequestPayer"] == "requester"
2081+
list_kwargs = s3._get_s3_method_kwargs(
2082+
s3.s3.list_objects_v2, Bucket=test_bucket_name, MaxKeys=1
2083+
)
2084+
assert list_kwargs["RequestPayer"] == "requester"
2085+
assert "RequestPayer" not in s3._get_s3_method_kwargs(s3.s3.list_buckets)
20772086
s3.touch(fn)
20782087
with s3.open(fn, "rb") as f:
20792088
assert f.req_kw["RequestPayer"] == "requester"
20802089

20812090

2091+
@pytest.mark.parametrize(
2092+
"filesystem_requester_pays,file_requester_pays,expected",
2093+
[(False, True, True), (True, False, False)],
2094+
)
2095+
@pytest.mark.parametrize("method", ["get_object", "put_object"])
2096+
def test_requester_pays_per_file_override(
2097+
filesystem_requester_pays,
2098+
file_requester_pays,
2099+
expected,
2100+
method,
2101+
):
2102+
fn = test_bucket_name + "/myfile"
2103+
s3 = S3FileSystem(
2104+
requester_pays=filesystem_requester_pays,
2105+
client_kwargs={"endpoint_url": endpoint_uri},
2106+
skip_instance_cache=True,
2107+
)
2108+
f = s3fs.core.S3File(s3, fn, mode="rb", requester_pays=file_requester_pays, size=0)
2109+
request_kwargs = s3._get_s3_method_kwargs(
2110+
getattr(s3.s3, method),
2111+
f.s3_additional_kwargs,
2112+
f._request_payer_kw,
2113+
Bucket=test_bucket_name,
2114+
Key="myfile",
2115+
)
2116+
f.close()
2117+
2118+
assert ("RequestPayer" in request_kwargs) is expected
2119+
if expected:
2120+
assert request_kwargs["RequestPayer"] == "requester"
2121+
2122+
2123+
def test_requester_pays_makedirs_uses_object_listing():
2124+
class MetadataRestrictedS3FileSystem(S3FileSystem):
2125+
def __init__(self, *args, **kwargs):
2126+
super().__init__(*args, **kwargs)
2127+
self.calls = []
2128+
2129+
async def _call_s3(self, method, *akwarglist, **kwargs):
2130+
self.calls.append(method)
2131+
if method == "list_objects_v2":
2132+
return {"Contents": []}
2133+
raise PermissionError("Bucket metadata is not accessible")
2134+
2135+
s3 = MetadataRestrictedS3FileSystem(
2136+
requester_pays=True,
2137+
client_kwargs={"endpoint_url": endpoint_uri},
2138+
skip_instance_cache=True,
2139+
)
2140+
s3.makedirs("requester-pays-bucket/checkpoints/run", exist_ok=True)
2141+
2142+
assert s3.calls == ["head_bucket", "get_bucket_location", "list_objects_v2"]
2143+
assert "create_bucket" not in s3.calls
2144+
2145+
20822146
def test_credentials():
20832147
s3 = S3FileSystem(
20842148
key="foo", secret="foo", client_kwargs={"endpoint_url": endpoint_uri}

0 commit comments

Comments
 (0)