@@ -906,26 +906,17 @@ async def _iterdir(
906906 raise ValueError (
907907 "versions cannot be specified if the filesystem is not version aware"
908908 )
909- await self .set_session ()
910- s3 = await self .get_s3 (bucket )
911909 if self .version_aware :
912910 method = "list_object_versions"
913911 contents_key = "Versions"
914912 else :
915913 method = "list_objects_v2"
916914 contents_key = "Contents"
917- pag = s3 .get_paginator (method )
918- config = {}
915+ kwargs = dict (Bucket = bucket , Prefix = prefix , Delimiter = delimiter , ** self .req_kw )
919916 if max_items is not None :
920- config .update (MaxItems = max_items , PageSize = 2 * max_items )
921- it = pag .paginate (
922- Bucket = bucket ,
923- Prefix = prefix ,
924- Delimiter = delimiter ,
925- PaginationConfig = config ,
926- ** self .req_kw ,
927- )
928- async for i in it :
917+ kwargs ["MaxKeys" ] = 2 * max_items
918+ remaining = max_items
919+ async for i in self ._list_pages (method , ** kwargs ):
929920 for l in i .get ("CommonPrefixes" , []):
930921 c = {
931922 "Key" : l ["Prefix" ][:- 1 ],
@@ -935,12 +926,33 @@ async def _iterdir(
935926 }
936927 self ._fill_info (c , bucket , versions = False )
937928 yield c
938- for c in i .get (contents_key , []):
929+ contents = i .get (contents_key , [])
930+ if remaining is not None :
931+ contents = contents [:remaining ]
932+ remaining -= len (contents )
933+ for c in contents :
939934 if not self .version_aware or c .get ("IsLatest" ) or versions :
940935 c ["type" ] = "file"
941936 c ["size" ] = c ["Size" ]
942937 self ._fill_info (c , bucket , versions = versions )
943938 yield c
939+ if remaining is not None and remaining <= 0 :
940+ return
941+
942+ async def _list_pages (self , method , ** kwargs ):
943+ # Each page goes through _call_s3, so a transient error on one page is
944+ # retried like any other request instead of failing (or silently
945+ # truncating) the whole listing, as the paginator did (#982).
946+ while True :
947+ out = await self ._call_s3 (method , ** kwargs )
948+ yield out
949+ if not out .get ("IsTruncated" ):
950+ return
951+ if method == "list_object_versions" :
952+ kwargs ["KeyMarker" ] = out .get ("NextKeyMarker" , "" )
953+ kwargs ["VersionIdMarker" ] = out .get ("NextVersionIdMarker" , "" )
954+ else :
955+ kwargs ["ContinuationToken" ] = out ["NextContinuationToken" ]
944956
945957 @staticmethod
946958 def _fill_info (f , bucket , versions = False ):
@@ -2415,10 +2427,9 @@ async def _make_bucket_versioned(self, bucket, versioned: bool = True):
24152427
24162428 async def _rm_versioned_bucket_contents (self , bucket ):
24172429 """Remove a versioned bucket and all contents"""
2418- await self .set_session ()
2419- s3 = await self .get_s3 (bucket )
2420- pag = s3 .get_paginator ("list_object_versions" )
2421- async for plist in pag .paginate (Bucket = bucket , ** self .req_kw ):
2430+ async for plist in self ._list_pages (
2431+ "list_object_versions" , Bucket = bucket , ** self .req_kw
2432+ ):
24222433 obs = plist .get ("Versions" , []) + plist .get ("DeleteMarkers" , [])
24232434 delete_keys = {
24242435 "Objects" : [
0 commit comments