Skip to content

Commit f7911b7

Browse files
achalsfelixwang9817
andcommitted
fix: Add spark to lambda dockerfile (#2480)
* add spark to lambda dockerfile Signed-off-by: Achal Shah <[email protected]> * add *args Signed-off-by: Achal Shah <[email protected]> * Add *args Signed-off-by: Felix Wang <[email protected]> * Pin protobuf==3.19.4 Signed-off-by: Felix Wang <[email protected]> * Remove *args Signed-off-by: Felix Wang <[email protected]> * Add a range Signed-off-by: Achal Shah <[email protected]> * Add a todo Signed-off-by: Achal Shah <[email protected]> * cleanup prints Signed-off-by: Achal Shah <[email protected]> * lock deps Signed-off-by: Achal Shah <[email protected]> * lock deps correctly Signed-off-by: Achal Shah <[email protected]> * fix lint Signed-off-by: Achal Shah <[email protected]> * fix lint take 2 Signed-off-by: Achal Shah <[email protected]> * Undo general updates Signed-off-by: Achal Shah <[email protected]> Co-authored-by: Felix Wang <[email protected]>
1 parent 9baba23 commit f7911b7

5 files changed

Lines changed: 24 additions & 13 deletions

File tree

‎sdk/python/feast/infra/feature_servers/aws_lambda/Dockerfile‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,8 @@ COPY protos protos
99
COPY README.md README.md
1010

1111
# Install Feast for AWS with Lambda dependencies
12-
RUN pip3 install -e 'sdk/python[aws,redis]'
12+
# TODO(achals): The additional spark deps should be removed. Details at https://github.com/feast-dev/feast/pull/2480.
13+
RUN pip3 install -e 'sdk/python[aws,redis,spark]'
1314
RUN pip3 install -r sdk/python/feast/infra/feature_servers/aws_lambda/requirements.txt --target "${LAMBDA_TASK_ROOT}"
1415

1516
# Set the CMD to your handler (could also be done as a parameter override outside of the Dockerfile)

‎sdk/python/feast/infra/offline_stores/bigquery.py‎

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -338,12 +338,14 @@ def to_bigquery(
338338

339339
def _to_arrow_internal(self) -> pyarrow.Table:
340340
with self._query_generator() as query:
341-
return self._execute_query(query).to_arrow()
341+
q = self._execute_query(query=query)
342+
assert q
343+
return q.to_arrow()
342344

343345
@log_exceptions_and_usage
344346
def _execute_query(
345347
self, query, job_config=None, timeout: int = 1800
346-
) -> bigquery.job.query.QueryJob:
348+
) -> Optional[bigquery.job.query.QueryJob]:
347349
bq_job = self.client.query(query, job_config=job_config)
348350

349351
if job_config and job_config.dry_run:
@@ -426,7 +428,7 @@ def _get_table_reference_for_new_entity(
426428
dataset.location = dataset_location if dataset_location else "US"
427429

428430
try:
429-
client.get_dataset(dataset)
431+
client.get_dataset(dataset.reference)
430432
except NotFound:
431433
# Only create the dataset if it does not exist
432434
client.create_dataset(dataset, exists_ok=True)

‎sdk/python/feast/proto_json.py‎

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,8 @@
1515
JsonObject = Any
1616

1717

18+
# TODO: These methods need to be updated when bumping the version of protobuf.
19+
# https://github.com/feast-dev/feast/issues/2484
1820
def _patch_proto_json_encoding(
1921
proto_type: Type[ProtoMessage],
2022
to_json_object: Callable[[_Printer, ProtoMessage], JsonObject],
@@ -68,7 +70,7 @@ def to_json_object(printer: _Printer, message: ProtoMessage) -> JsonObject:
6870
return value
6971

7072
def from_json_object(
71-
parser: _Parser, value: JsonObject, message: ProtoMessage
73+
parser: _Parser, value: JsonObject, message: ProtoMessage,
7274
) -> None:
7375
if value is None:
7476
message.null_val = 0
@@ -140,7 +142,7 @@ def to_json_object(printer: _Printer, message: ProtoMessage) -> JsonObject:
140142
return [printer._MessageToJsonObject(item) for item in message.val]
141143

142144
def from_json_object(
143-
parser: _Parser, value: JsonObject, message: ProtoMessage
145+
parser: _Parser, value: JsonObject, message: ProtoMessage,
144146
) -> None:
145147
array = value if isinstance(value, list) else value["val"]
146148
for item in array:
@@ -181,7 +183,7 @@ def to_json_object(printer: _Printer, message: ProtoMessage) -> JsonObject:
181183
return list(message.val)
182184

183185
def from_json_object(
184-
parser: _Parser, value: JsonObject, message: ProtoMessage
186+
parser: _Parser, value: JsonObject, message: ProtoMessage,
185187
) -> None:
186188
array = value if isinstance(value, list) else value["val"]
187189
message.val.extend(array)

‎sdk/python/requirements/py3.7-ci-requirements.txt‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -400,7 +400,7 @@ mypy==0.931
400400
# via feast (setup.py)
401401
mypy-extensions==0.4.3
402402
# via mypy
403-
mypy-protobuf==3.1.0
403+
mypy-protobuf==3.1
404404
# via feast (setup.py)
405405
nbclient==0.5.11
406406
# via nbconvert
@@ -850,4 +850,4 @@ zipp==3.7.0
850850

851851
# The following packages are considered to be unsafe in a requirements file:
852852
# pip
853-
# setuptools
853+
# setuptools

‎sdk/python/setup.py‎

Lines changed: 10 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -53,7 +53,7 @@
5353
"mmh3",
5454
"pandas>=1.0.0",
5555
"pandavro==1.5.*",
56-
"protobuf>=3.10",
56+
"protobuf>=3.10,<3.20",
5757
"proto-plus<1.19.7",
5858
"pyarrow>=4.0.0",
5959
"pydantic>=1.0.0",
@@ -112,7 +112,7 @@
112112
"mock==2.0.0",
113113
"moto",
114114
"mypy==0.931",
115-
"mypy-protobuf==3.1.0",
115+
"mypy-protobuf==3.1",
116116
"avro==1.10.0",
117117
"gcsfs",
118118
"urllib3>=1.25.4",
@@ -149,7 +149,7 @@
149149
+ GE_REQUIRED
150150
)
151151

152-
DEV_REQUIRED = ["mypy-protobuf>=3.1.0", "grpcio-testing==1.*"] + CI_REQUIRED
152+
DEV_REQUIRED = ["mypy-protobuf==3.1", "grpcio-testing==1.*"] + CI_REQUIRED
153153

154154
# Get git repo root directory
155155
repo_root = str(pathlib.Path(__file__).resolve().parent.parent.parent)
@@ -264,7 +264,13 @@ def run(self):
264264
],
265265
entry_points={"console_scripts": ["feast=feast.cli:cli"]},
266266
use_scm_version=use_scm_version,
267-
setup_requires=["setuptools_scm", "grpcio", "grpcio-tools==1.34.0", "mypy-protobuf==3.1.0", "sphinx!=4.0.0"],
267+
setup_requires=[
268+
"setuptools_scm",
269+
"grpcio",
270+
"grpcio-tools==1.34.0",
271+
"mypy-protobuf==3.1",
272+
"sphinx!=4.0.0",
273+
],
268274
package_data={
269275
"": [
270276
"protos/feast/**/*.proto",

0 commit comments

Comments
 (0)