Skip to content

Commit 575f145

Browse files
authored
Merge pull request feast-dev#2 from leonid133/feature/registry_s3
Feature/registry s3
2 parents 4ef14a9 + b012a07 commit 575f145

5 files changed

Lines changed: 99 additions & 10 deletions

File tree

‎sdk/python/feast/cli.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -236,7 +236,7 @@ def materialize_incremental_command(end_ts: str, views: List[str]):
236236
@click.option(
237237
"--template",
238238
"-t",
239-
type=click.Choice(["local", "gcp"], case_sensitive=False),
239+
type=click.Choice(["local", "gcp", "aws_dynamo"], case_sensitive=False),
240240
help="Specify a template for the created project",
241241
default="local",
242242
)

‎sdk/python/feast/infra/aws_dynamo_provider.py‎

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -27,9 +27,9 @@
2727
class AwsDynamoProvider(Provider):
2828
_aws_project_id: Optional[str]
2929

30-
def __init__(self, config: Optional[DatastoreOnlineStoreConfig]):
31-
if config:
32-
self._aws_project_id = config.project_id
30+
def __init__(self, config: Optional[RepoConfig]):
31+
if config and config.online_store and config.online_store.project_id:
32+
self._aws_project_id = config.online_store.project_id
3333
else:
3434
self._aws_project_id = None
3535

@@ -48,6 +48,7 @@ def update_infra(
4848
client = self._initialize_client()
4949

5050
for table_name in tables_to_keep:
51+
# TODO: add table creation to dynamo.
5152
table = client.Table(table_name.name)
5253
table.update_item(
5354
Key={

‎sdk/python/feast/registry.py‎

Lines changed: 77 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -56,6 +56,8 @@ def __init__(self, registry_path: str, cache_ttl: timedelta):
5656
uri = urlparse(registry_path)
5757
if uri.scheme == "gs":
5858
self._registry_store: RegistryStore = GCSRegistryStore(registry_path)
59+
elif uri.scheme == "s3":
60+
self._registry_store: RegistryStore = AwsS3RegistryStore(registry_path)
5961
elif uri.scheme == "file" or uri.scheme == "":
6062
self._registry_store = LocalRegistryStore(registry_path)
6163
else:
@@ -477,3 +479,78 @@ def _write_registry(self, registry_proto: RegistryProto):
477479
file_obj.seek(0)
478480
blob.upload_from_file(file_obj)
479481
return
482+
483+
484+
class AwsS3RegistryStore(RegistryStore):
485+
def __init__(self, uri: str):
486+
try:
487+
import boto3
488+
except ImportError:
489+
raise ImportError(
490+
"Install package boto3==1.17.* for gcs support"
491+
"run ```pip install boto3==1.17.*```"
492+
)
493+
self._uri = urlparse(uri)
494+
self._bucket = self._uri.hostname
495+
self._key = self._uri.path.lstrip("/")
496+
return
497+
498+
def get_registry_proto(self):
499+
import boto3
500+
import botocore
501+
502+
file_obj = TemporaryFile()
503+
registry_proto = RegistryProto()
504+
s3 = boto3.resource('s3')
505+
try:
506+
bucket = s3.Bucket(self._bucket)
507+
s3.meta.client.head_bucket(Bucket=bucket.name)
508+
except botocore.client.ClientError as e:
509+
# If a client error is thrown, then check that it was a 404 error.
510+
# If it was a 404 error, then the bucket does not exist.
511+
error_code = int(e.response['Error']['Code'])
512+
if error_code == 404:
513+
raise Exception(
514+
f"No bucket named {self._bucket} exists; please create it first."
515+
)
516+
else:
517+
raise Exception(f'Private Registry Bucket {self._bucket}. Forbidden Access!')
518+
519+
try:
520+
obj = bucket.Object(self._key)
521+
obj.download_fileobj(file_obj)
522+
file_obj.seek(0)
523+
registry_proto.ParseFromString(file_obj.read())
524+
return registry_proto
525+
except botocore.exceptions.ClientError as e:
526+
if e.response['Error']['Code'] == "404":
527+
raise FileNotFoundError(
528+
f'Registry not found at path "{self._uri.geturl()}". Have you run "feast apply"?'
529+
)
530+
else:
531+
raise FileNotFoundError(
532+
f'Registry is not able to locate data under path "{self._uri.geturl()}" with [original error]: {e.response}'
533+
)
534+
535+
def update_registry_proto(self, updater: Callable[[RegistryProto], RegistryProto]):
536+
try:
537+
registry_proto = self.get_registry_proto()
538+
except FileNotFoundError:
539+
registry_proto = RegistryProto()
540+
registry_proto.registry_schema_version = REGISTRY_SCHEMA_VERSION
541+
registry_proto = updater(registry_proto)
542+
self._write_registry(registry_proto)
543+
return
544+
545+
def _write_registry(self, registry_proto: RegistryProto):
546+
import boto3
547+
registry_proto.version_id = str(uuid.uuid4())
548+
registry_proto.last_updated.FromDatetime(datetime.utcnow())
549+
# we have already checked the bucket exists so no need to do it again
550+
registry_bucket = boto3.resource('s3').Bucket(self._bucket)
551+
registry_db = registry_bucket.Object(self._key)
552+
file_obj = TemporaryFile()
553+
file_obj.write(registry_proto.SerializeToString())
554+
file_obj.seek(0)
555+
registry_db.upload_fileobj(file_obj)
556+
return

‎sdk/python/feast/repo_config.py‎

Lines changed: 13 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -33,8 +33,13 @@ class DatastoreOnlineStoreConfig(FeastBaseModel):
3333
project_id: Optional[StrictStr] = None
3434
""" (optional) GCP Project Id """
3535

36+
class DynamoOnlineStoreConfig(FeastBaseModel):
37+
"""Online store config for DynamoDB store"""
38+
type: Literal["dynamo"] = "dynamo"
39+
"""Online store type selector"""
40+
project_id: Optional[StrictStr] = None
3641

37-
OnlineStoreConfig = Union[DatastoreOnlineStoreConfig, SqliteOnlineStoreConfig]
42+
OnlineStoreConfig = Union[DatastoreOnlineStoreConfig, SqliteOnlineStoreConfig, DynamoOnlineStoreConfig]
3843

3944

4045
class RegistryConfig(FeastBaseModel):
@@ -63,7 +68,7 @@ class RepoConfig(FeastBaseModel):
6368
"""
6469

6570
provider: StrictStr
66-
""" str: local or gcp """
71+
""" str: local or gcp or aws_dynamo """
6772

6873
online_store: OnlineStoreConfig = SqliteOnlineStoreConfig()
6974
""" OnlineStoreConfig: Online store configuration (optional depending on provider) """
@@ -102,18 +107,20 @@ def _validate_online_store_config(cls, values):
102107
values["online_store"]["type"] = "datastore"
103108
elif values["provider"] == "aws_dynamo":
104109
values["online_store"]["type"] = "datastore"
105-
110+
elif values["provider"] == "aws_dynamo":
111+
values["online_store"]["type"] = "dynamo"
106112
online_store_type = values["online_store"]["type"]
107-
108113
# Make sure the user hasn't provided the wrong type
109-
assert online_store_type in ["datastore", "sqlite"]
114+
assert online_store_type in ["datastore", "sqlite", "dynamo"]
110115

111116
# Validate the dict to ensure one of the union types match
112117
try:
113118
if online_store_type == "sqlite":
114119
SqliteOnlineStoreConfig(**values["online_store"])
115-
elif values["online_store"]["type"] == "datastore":
120+
elif online_store_type == "datastore":
116121
DatastoreOnlineStoreConfig(**values["online_store"])
122+
elif online_store_type == "dynamo":
123+
DynamoOnlineStoreConfig(**values["online_store"])
117124
else:
118125
raise ValidationError(
119126
f"Invalid online store type {online_store_type}"
Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
project: my_project
2+
registry: s3://feast-provectus-demo/data/registry.db
3+
provider: aws_dynamo
4+
online_store:

0 commit comments

Comments
 (0)