Skip to content

Commit 85a6ec5

Browse files
Retry transient errors on each listing page (#1042)
Co-authored-by: Martin Durant <[email protected]>
1 parent 359cf57 commit 85a6ec5

3 files changed

Lines changed: 50 additions & 18 deletions

File tree

‎docs/source/changelog.rst‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@ Changelog
44
Dev
55
---
66

7+
- retry transient errors on each page of a listing instead of failing or truncating it (#982)
78
- Preserve the original coroutine-related error when its traceback has no response (#1011)
89

910
2026.7.0

‎s3fs/core.py‎

Lines changed: 29 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -906,26 +906,17 @@ async def _iterdir(
906906
raise ValueError(
907907
"versions cannot be specified if the filesystem is not version aware"
908908
)
909-
await self.set_session()
910-
s3 = await self.get_s3(bucket)
911909
if self.version_aware:
912910
method = "list_object_versions"
913911
contents_key = "Versions"
914912
else:
915913
method = "list_objects_v2"
916914
contents_key = "Contents"
917-
pag = s3.get_paginator(method)
918-
config = {}
915+
kwargs = dict(Bucket=bucket, Prefix=prefix, Delimiter=delimiter, **self.req_kw)
919916
if max_items is not None:
920-
config.update(MaxItems=max_items, PageSize=2 * max_items)
921-
it = pag.paginate(
922-
Bucket=bucket,
923-
Prefix=prefix,
924-
Delimiter=delimiter,
925-
PaginationConfig=config,
926-
**self.req_kw,
927-
)
928-
async for i in it:
917+
kwargs["MaxKeys"] = 2 * max_items
918+
remaining = max_items
919+
async for i in self._list_pages(method, **kwargs):
929920
for l in i.get("CommonPrefixes", []):
930921
c = {
931922
"Key": l["Prefix"][:-1],
@@ -935,12 +926,33 @@ async def _iterdir(
935926
}
936927
self._fill_info(c, bucket, versions=False)
937928
yield c
938-
for c in i.get(contents_key, []):
929+
contents = i.get(contents_key, [])
930+
if remaining is not None:
931+
contents = contents[:remaining]
932+
remaining -= len(contents)
933+
for c in contents:
939934
if not self.version_aware or c.get("IsLatest") or versions:
940935
c["type"] = "file"
941936
c["size"] = c["Size"]
942937
self._fill_info(c, bucket, versions=versions)
943938
yield c
939+
if remaining is not None and remaining <= 0:
940+
return
941+
942+
async def _list_pages(self, method, **kwargs):
943+
# Each page goes through _call_s3, so a transient error on one page is
944+
# retried like any other request instead of failing (or silently
945+
# truncating) the whole listing, as the paginator did (#982).
946+
while True:
947+
out = await self._call_s3(method, **kwargs)
948+
yield out
949+
if not out.get("IsTruncated"):
950+
return
951+
if method == "list_object_versions":
952+
kwargs["KeyMarker"] = out.get("NextKeyMarker", "")
953+
kwargs["VersionIdMarker"] = out.get("NextVersionIdMarker", "")
954+
else:
955+
kwargs["ContinuationToken"] = out["NextContinuationToken"]
944956

945957
@staticmethod
946958
def _fill_info(f, bucket, versions=False):
@@ -2415,10 +2427,9 @@ async def _make_bucket_versioned(self, bucket, versioned: bool = True):
24152427

24162428
async def _rm_versioned_bucket_contents(self, bucket):
24172429
"""Remove a versioned bucket and all contents"""
2418-
await self.set_session()
2419-
s3 = await self.get_s3(bucket)
2420-
pag = s3.get_paginator("list_object_versions")
2421-
async for plist in pag.paginate(Bucket=bucket, **self.req_kw):
2430+
async for plist in self._list_pages(
2431+
"list_object_versions", Bucket=bucket, **self.req_kw
2432+
):
24222433
obs = plist.get("Versions", []) + plist.get("DeleteMarkers", [])
24232434
delete_keys = {
24242435
"Objects": [

‎s3fs/tests/test_s3fs.py‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@
77
import io
88
import os
99
import random
10+
import socket
1011
import requests
1112
import time
1213
import sys
@@ -1149,6 +1150,25 @@ async def head_object(*args, **kwargs):
11491150
assert exc.value.__cause__ is None
11501151

11511152

1153+
def test_listing_retries_transient_errors(monkeypatch, s3):
1154+
# a transient error on a listing page is retried like any other call (#982)
1155+
expected = s3.find(test_bucket_name)
1156+
original = type(s3.s3).list_objects_v2
1157+
calls = []
1158+
1159+
async def list_objects_v2(self, *args, **kwargs):
1160+
calls.append(kwargs)
1161+
if len(calls) == 1:
1162+
raise socket.timeout
1163+
return await original(self, *args, **kwargs)
1164+
1165+
monkeypatch.setattr(type(s3.s3), "list_objects_v2", list_objects_v2)
1166+
s3.invalidate_cache()
1167+
1168+
assert s3.find(test_bucket_name) == expected
1169+
assert len(calls) == 2
1170+
1171+
11521172
def test_local_expiry_check(s3):
11531173
s3 = S3FileSystem(
11541174
local_expiry_check=True,

0 commit comments

Comments
 (0)