Skip to content

Commit 14708b8

Browse files
authored
Storage staging to ABFSS (feast-dev#52)
Cherry-picked upstream PR feast-dev#769 Changed WASB implementation of Azure file staging to ABFSS to support directory file enumeration, for retrieving Spark job output (collection of files) in feast-dev#37
1 parent 5b697f6 commit 14708b8

8 files changed

Lines changed: 543 additions & 446 deletions

File tree

‎sdk/python/feast/job.py‎

Lines changed: 18 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@
55

66
import fastavro
77
import pandas as pd
8+
import glob
89
from google.protobuf.json_format import MessageToJson
910

1011
from feast.core.CoreService_pb2 import ListIngestionJobsRequest
@@ -21,7 +22,7 @@
2122
from feast.serving.ServingService_pb2 import Job as JobProto
2223
from feast.serving.ServingService_pb2_grpc import ServingServiceStub
2324
from feast.source import Source
24-
from feast.staging.staging_strategy import StagingStrategy
25+
from feast.staging.storage_client import get_staging_client
2526

2627
# Maximum no of seconds to wait until the retrieval jobs status is DONE in Feast
2728
# Currently set to the maximum query execution time limit in BigQuery
@@ -46,7 +47,6 @@ def __init__(
4647
"""
4748
self.job_proto = job_proto
4849
self.serving_stub = serving_stub
49-
self.staging_strategy = StagingStrategy()
5050

5151
@property
5252
def id(self):
@@ -124,12 +124,23 @@ def result(self, timeout_sec: int = DEFAULT_TIMEOUT_SEC):
124124
"""
125125
uris = self.get_avro_files(timeout_sec)
126126
for file_uri in uris:
127-
file_obj = self.staging_strategy.execute_file_download(file_uri)
128-
file_obj.seek(0)
129-
avro_reader = fastavro.reader(file_obj)
127+
(file_obj, dir_obj) = get_staging_client(file_uri.scheme).download_file(
128+
file_uri
129+
)
130+
if file_obj:
131+
file_obj.seek(0)
132+
avro_reader = fastavro.reader(file_obj)
133+
134+
for record in avro_reader:
135+
yield record
136+
137+
if dir_obj:
138+
for file_name in glob.glob(f"{dir_obj.name}/*.avro"):
139+
with open(file_name, "rb") as file_obj:
140+
avro_reader = fastavro.reader(file_obj)
130141

131-
for record in avro_reader:
132-
yield record
142+
for record in avro_reader:
143+
yield record
133144

134145
def to_dataframe(self, timeout_sec: int = DEFAULT_TIMEOUT_SEC) -> pd.DataFrame:
135146
"""

‎sdk/python/feast/loaders/file.py‎

Lines changed: 14 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -23,7 +23,7 @@
2323
import pandas as pd
2424
from pandavro import to_avro
2525

26-
from feast.staging.staging_strategy import StagingStrategy
26+
from feast.staging.storage_client import get_staging_client
2727

2828

2929
def export_source_to_staging_location(
@@ -60,7 +60,6 @@ def export_source_to_staging_location(
6060
remote staging location.
6161
"""
6262

63-
staging_strategy = StagingStrategy()
6463
uri = urlparse(staging_location_uri)
6564

6665
# Prepare Avro file to be exported to staging location
@@ -74,32 +73,35 @@ def export_source_to_staging_location(
7473
df=source, dir_path=uri_path
7574
)
7675
elif isinstance(source, str):
77-
if urlparse(source).scheme in ["", "file"]:
76+
source_uri = urlparse(source)
77+
if source_uri.scheme in ["", "file"]:
7878
# Local file provided as a source
79-
dir_path = None
79+
dir_path = ""
8080
file_name = os.path.basename(source)
8181
source_path = os.path.abspath(
82-
os.path.join(urlparse(source).netloc, urlparse(source).path)
82+
os.path.join(source_uri.netloc, source_uri.path)
8383
)
8484
else:
8585
# gs, s3 file provided as a source.
86-
return staging_strategy.execute_get_source_files(source)
86+
return get_staging_client(source_uri.scheme).list_files(source_uri)
8787
else:
8888
raise Exception(
8989
f"Only string and DataFrame types are allowed as a "
9090
f"source, {type(source)} was provided."
9191
)
9292

9393
# Push data to required staging location
94-
staging_strategy.execute_file_upload(
95-
uri.scheme,
96-
source_path,
97-
uri.hostname,
98-
str(uri.path).strip("/") + "/" + file_name,
94+
get_staging_client(uri.scheme).upload_file(
95+
source_path, uri, str(uri.path).strip("/") + "/" + file_name,
9996
)
10097

10198
# Clean up, remove local staging file
102-
if dir_path and isinstance(source, pd.DataFrame) and len(str(dir_path)) > 4:
99+
if (
100+
dir_path
101+
and isinstance(source, pd.DataFrame)
102+
and uri.scheme != "file"
103+
and len(str(dir_path)) > 4
104+
):
103105
shutil.rmtree(dir_path)
104106

105107
return [staging_location_uri.rstrip("/") + "/" + file_name]

0 commit comments

Comments
 (0)