Skip to content

Commit 5917684

Browse files
authored
Create async iterator version of _lsdir() (#670)
* Extract _do_lsdir() from _lsdir() * Refactor _do_lsdir() to do most of the work inside the loop * Replace _do_lsdir() with async generator _iterdir() * Add docstring for _iterdir() * Move 'versions' consistency check from _lsdir() to _iterdir()
1 parent f26f379 commit 5917684

1 file changed

Lines changed: 69 additions & 51 deletions

File tree

‎s3fs/core.py‎

Lines changed: 69 additions & 51 deletions
Original file line numberDiff line numberDiff line change
@@ -661,11 +661,6 @@ async def _lsdir(
661661
prefix="",
662662
versions=False,
663663
):
664-
if versions and not self.version_aware:
665-
raise ValueError(
666-
"versions cannot be specified if the filesystem is not version aware"
667-
)
668-
669664
bucket, key, _ = self.split_path(path)
670665
if not prefix:
671666
prefix = ""
@@ -674,53 +669,20 @@ async def _lsdir(
674669
if path not in self.dircache or refresh or not delimiter or versions:
675670
try:
676671
logger.debug("Get directory listing page for %s" % path)
677-
await self.set_session()
678-
s3 = await self.get_s3(bucket)
679-
if self.version_aware:
680-
method = "list_object_versions"
681-
contents_key = "Versions"
682-
else:
683-
method = "list_objects_v2"
684-
contents_key = "Contents"
685-
pag = s3.get_paginator(method)
686-
config = {}
687-
if max_items is not None:
688-
config.update(MaxItems=max_items, PageSize=2 * max_items)
689-
it = pag.paginate(
690-
Bucket=bucket,
691-
Prefix=prefix,
692-
Delimiter=delimiter,
693-
PaginationConfig=config,
694-
**self.req_kw,
695-
)
672+
dirs = []
696673
files = []
697-
dircache = []
698-
async for i in it:
699-
dircache.extend(i.get("CommonPrefixes", []))
700-
for c in i.get(contents_key, []):
701-
if not self.version_aware or c.get("IsLatest") or versions:
702-
c["type"] = "file"
703-
c["size"] = c["Size"]
704-
files.append(c)
705-
if dircache:
706-
files.extend(
707-
[
708-
{
709-
"Key": l["Prefix"][:-1],
710-
"Size": 0,
711-
"StorageClass": "DIRECTORY",
712-
"type": "directory",
713-
"size": 0,
714-
}
715-
for l in dircache
716-
]
717-
)
718-
for f in files:
719-
f["Key"] = "/".join([bucket, f["Key"]])
720-
f["name"] = f["Key"]
721-
version_id = f.get("VersionId")
722-
if versions and version_id and version_id != "null":
723-
f["name"] += f"?versionId={version_id}"
674+
async for c in self._iterdir(
675+
bucket,
676+
max_items=max_items,
677+
delimiter=delimiter,
678+
prefix=prefix,
679+
versions=versions,
680+
):
681+
if c["type"] == "directory":
682+
dirs.append(c)
683+
else:
684+
files.append(c)
685+
files += dirs
724686
except ClientError as e:
725687
raise translate_boto_error(e)
726688

@@ -729,6 +691,62 @@ async def _lsdir(
729691
return files
730692
return self.dircache[path]
731693

694+
async def _iterdir(
695+
self, bucket, max_items=None, delimiter="/", prefix="", versions=False
696+
):
697+
"""Iterate asynchronously over files and directories under `prefix`.
698+
699+
The contents are yielded in arbitrary order as info dicts.
700+
"""
701+
if versions and not self.version_aware:
702+
raise ValueError(
703+
"versions cannot be specified if the filesystem is not version aware"
704+
)
705+
await self.set_session()
706+
s3 = await self.get_s3(bucket)
707+
if self.version_aware:
708+
method = "list_object_versions"
709+
contents_key = "Versions"
710+
else:
711+
method = "list_objects_v2"
712+
contents_key = "Contents"
713+
pag = s3.get_paginator(method)
714+
config = {}
715+
if max_items is not None:
716+
config.update(MaxItems=max_items, PageSize=2 * max_items)
717+
it = pag.paginate(
718+
Bucket=bucket,
719+
Prefix=prefix,
720+
Delimiter=delimiter,
721+
PaginationConfig=config,
722+
**self.req_kw,
723+
)
724+
async for i in it:
725+
for l in i.get("CommonPrefixes", []):
726+
c = {
727+
"Key": l["Prefix"][:-1],
728+
"Size": 0,
729+
"StorageClass": "DIRECTORY",
730+
"type": "directory",
731+
}
732+
self._fill_info(c, bucket, versions=False)
733+
yield c
734+
for c in i.get(contents_key, []):
735+
if not self.version_aware or c.get("IsLatest") or versions:
736+
c["type"] = "file"
737+
c["size"] = c["Size"]
738+
self._fill_info(c, bucket, versions=versions)
739+
yield c
740+
741+
@staticmethod
742+
def _fill_info(f, bucket, versions=False):
743+
f["size"] = f["Size"]
744+
f["Key"] = "/".join([bucket, f["Key"]])
745+
f["name"] = f["Key"]
746+
version_id = f.get("VersionId")
747+
if versions and version_id and version_id != "null":
748+
f["name"] += f"?versionId={version_id}"
749+
732750
async def _glob(self, path, **kwargs):
733751
if path.startswith("*"):
734752
raise ValueError("Cannot traverse all of S3")

0 commit comments

Comments
 (0)