@@ -56,6 +56,8 @@ def __init__(self, registry_path: str, cache_ttl: timedelta):
5656 uri = urlparse (registry_path )
5757 if uri .scheme == "gs" :
5858 self ._registry_store : RegistryStore = GCSRegistryStore (registry_path )
59+ elif uri .scheme == "s3" :
60+ self ._registry_store : RegistryStore = AwsS3RegistryStore (registry_path )
5961 elif uri .scheme == "file" or uri .scheme == "" :
6062 self ._registry_store = LocalRegistryStore (registry_path )
6163 else :
@@ -477,3 +479,78 @@ def _write_registry(self, registry_proto: RegistryProto):
477479 file_obj .seek (0 )
478480 blob .upload_from_file (file_obj )
479481 return
482+
483+
484+ class AwsS3RegistryStore (RegistryStore ):
485+ def __init__ (self , uri : str ):
486+ try :
487+ import boto3
488+ except ImportError :
489+ raise ImportError (
490+ "Install package boto3==1.17.* for gcs support"
491+ "run ```pip install boto3==1.17.*```"
492+ )
493+ self ._uri = urlparse (uri )
494+ self ._bucket = self ._uri .hostname
495+ self ._key = self ._uri .path .lstrip ("/" )
496+ return
497+
498+ def get_registry_proto (self ):
499+ import boto3
500+ import botocore
501+
502+ file_obj = TemporaryFile ()
503+ registry_proto = RegistryProto ()
504+ s3 = boto3 .resource ('s3' )
505+ try :
506+ bucket = s3 .Bucket (self ._bucket )
507+ s3 .meta .client .head_bucket (Bucket = bucket .name )
508+ except botocore .client .ClientError as e :
509+ # If a client error is thrown, then check that it was a 404 error.
510+ # If it was a 404 error, then the bucket does not exist.
511+ error_code = int (e .response ['Error' ]['Code' ])
512+ if error_code == 404 :
513+ raise Exception (
514+ f"No bucket named { self ._bucket } exists; please create it first."
515+ )
516+ else :
517+ raise Exception (f'Private Registry Bucket { self ._bucket } . Forbidden Access!' )
518+
519+ try :
520+ obj = bucket .Object (self ._key )
521+ obj .download_fileobj (file_obj )
522+ file_obj .seek (0 )
523+ registry_proto .ParseFromString (file_obj .read ())
524+ return registry_proto
525+ except botocore .exceptions .ClientError as e :
526+ if e .response ['Error' ]['Code' ] == "404" :
527+ raise FileNotFoundError (
528+ f'Registry not found at path "{ self ._uri .geturl ()} ". Have you run "feast apply"?'
529+ )
530+ else :
531+ raise FileNotFoundError (
532+ f'Registry is not able to locate data under path "{ self ._uri .geturl ()} " with [original error]: { e .response } '
533+ )
534+
535+ def update_registry_proto (self , updater : Callable [[RegistryProto ], RegistryProto ]):
536+ try :
537+ registry_proto = self .get_registry_proto ()
538+ except FileNotFoundError :
539+ registry_proto = RegistryProto ()
540+ registry_proto .registry_schema_version = REGISTRY_SCHEMA_VERSION
541+ registry_proto = updater (registry_proto )
542+ self ._write_registry (registry_proto )
543+ return
544+
545+ def _write_registry (self , registry_proto : RegistryProto ):
546+ import boto3
547+ registry_proto .version_id = str (uuid .uuid4 ())
548+ registry_proto .last_updated .FromDatetime (datetime .utcnow ())
549+ # we have already checked the bucket exists so no need to do it again
550+ registry_bucket = boto3 .resource ('s3' ).Bucket (self ._bucket )
551+ registry_db = registry_bucket .Object (self ._key )
552+ file_obj = TemporaryFile ()
553+ file_obj .write (registry_proto .SerializeToString ())
554+ file_obj .seek (0 )
555+ registry_db .upload_fileobj (file_obj )
556+ return
0 commit comments