@@ -661,11 +661,6 @@ async def _lsdir(
661661 prefix = "" ,
662662 versions = False ,
663663 ):
664- if versions and not self .version_aware :
665- raise ValueError (
666- "versions cannot be specified if the filesystem is not version aware"
667- )
668-
669664 bucket , key , _ = self .split_path (path )
670665 if not prefix :
671666 prefix = ""
@@ -674,53 +669,20 @@ async def _lsdir(
674669 if path not in self .dircache or refresh or not delimiter or versions :
675670 try :
676671 logger .debug ("Get directory listing page for %s" % path )
677- await self .set_session ()
678- s3 = await self .get_s3 (bucket )
679- if self .version_aware :
680- method = "list_object_versions"
681- contents_key = "Versions"
682- else :
683- method = "list_objects_v2"
684- contents_key = "Contents"
685- pag = s3 .get_paginator (method )
686- config = {}
687- if max_items is not None :
688- config .update (MaxItems = max_items , PageSize = 2 * max_items )
689- it = pag .paginate (
690- Bucket = bucket ,
691- Prefix = prefix ,
692- Delimiter = delimiter ,
693- PaginationConfig = config ,
694- ** self .req_kw ,
695- )
672+ dirs = []
696673 files = []
697- dircache = []
698- async for i in it :
699- dircache .extend (i .get ("CommonPrefixes" , []))
700- for c in i .get (contents_key , []):
701- if not self .version_aware or c .get ("IsLatest" ) or versions :
702- c ["type" ] = "file"
703- c ["size" ] = c ["Size" ]
704- files .append (c )
705- if dircache :
706- files .extend (
707- [
708- {
709- "Key" : l ["Prefix" ][:- 1 ],
710- "Size" : 0 ,
711- "StorageClass" : "DIRECTORY" ,
712- "type" : "directory" ,
713- "size" : 0 ,
714- }
715- for l in dircache
716- ]
717- )
718- for f in files :
719- f ["Key" ] = "/" .join ([bucket , f ["Key" ]])
720- f ["name" ] = f ["Key" ]
721- version_id = f .get ("VersionId" )
722- if versions and version_id and version_id != "null" :
723- f ["name" ] += f"?versionId={ version_id } "
674+ async for c in self ._iterdir (
675+ bucket ,
676+ max_items = max_items ,
677+ delimiter = delimiter ,
678+ prefix = prefix ,
679+ versions = versions ,
680+ ):
681+ if c ["type" ] == "directory" :
682+ dirs .append (c )
683+ else :
684+ files .append (c )
685+ files += dirs
724686 except ClientError as e :
725687 raise translate_boto_error (e )
726688
@@ -729,6 +691,62 @@ async def _lsdir(
729691 return files
730692 return self .dircache [path ]
731693
694+ async def _iterdir (
695+ self , bucket , max_items = None , delimiter = "/" , prefix = "" , versions = False
696+ ):
697+ """Iterate asynchronously over files and directories under `prefix`.
698+
699+ The contents are yielded in arbitrary order as info dicts.
700+ """
701+ if versions and not self .version_aware :
702+ raise ValueError (
703+ "versions cannot be specified if the filesystem is not version aware"
704+ )
705+ await self .set_session ()
706+ s3 = await self .get_s3 (bucket )
707+ if self .version_aware :
708+ method = "list_object_versions"
709+ contents_key = "Versions"
710+ else :
711+ method = "list_objects_v2"
712+ contents_key = "Contents"
713+ pag = s3 .get_paginator (method )
714+ config = {}
715+ if max_items is not None :
716+ config .update (MaxItems = max_items , PageSize = 2 * max_items )
717+ it = pag .paginate (
718+ Bucket = bucket ,
719+ Prefix = prefix ,
720+ Delimiter = delimiter ,
721+ PaginationConfig = config ,
722+ ** self .req_kw ,
723+ )
724+ async for i in it :
725+ for l in i .get ("CommonPrefixes" , []):
726+ c = {
727+ "Key" : l ["Prefix" ][:- 1 ],
728+ "Size" : 0 ,
729+ "StorageClass" : "DIRECTORY" ,
730+ "type" : "directory" ,
731+ }
732+ self ._fill_info (c , bucket , versions = False )
733+ yield c
734+ for c in i .get (contents_key , []):
735+ if not self .version_aware or c .get ("IsLatest" ) or versions :
736+ c ["type" ] = "file"
737+ c ["size" ] = c ["Size" ]
738+ self ._fill_info (c , bucket , versions = versions )
739+ yield c
740+
741+ @staticmethod
742+ def _fill_info (f , bucket , versions = False ):
743+ f ["size" ] = f ["Size" ]
744+ f ["Key" ] = "/" .join ([bucket , f ["Key" ]])
745+ f ["name" ] = f ["Key" ]
746+ version_id = f .get ("VersionId" )
747+ if versions and version_id and version_id != "null" :
748+ f ["name" ] += f"?versionId={ version_id } "
749+
732750 async def _glob (self , path , ** kwargs ):
733751 if path .startswith ("*" ):
734752 raise ValueError ("Cannot traverse all of S3" )
0 commit comments