Skip to content

Commit 395b46a

Browse files
adchiaadchia
authored
Merge branch 'master' into update_fields_optional
2 parents 9eccab4 + 71d7ae2 commit 395b46a

8 files changed

Lines changed: 52 additions & 36 deletions

File tree

‎sdk/python/feast/infra/aws.py‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -119,6 +119,8 @@ def _deploy_feature_server(self, project: str, image_uri: str):
119119
lambda_client = boto3.client("lambda")
120120
api_gateway_client = boto3.client("apigatewayv2")
121121
function = aws_utils.get_lambda_function(lambda_client, resource_name)
122+
_logger.debug("Using function name: %s", resource_name)
123+
_logger.debug("Found function: %s", function)
122124

123125
if function is None:
124126
# If the Lambda function does not exist, create it.
@@ -309,7 +311,7 @@ def _create_or_get_repository_uri(self, ecr_client):
309311

310312
def _get_lambda_name(project: str):
311313
lambda_prefix = AWS_LAMBDA_FEATURE_SERVER_REPOSITORY
312-
lambda_suffix = f"{project}-{_get_docker_image_version()}"
314+
lambda_suffix = f"{project}-{_get_docker_image_version().replace('.', '_')}"
313315
# AWS Lambda name can't have the length greater than 64 bytes.
314316
# This usually occurs during integration tests where feast version is long
315317
if len(lambda_prefix) + len(lambda_suffix) >= 63:
@@ -338,7 +340,7 @@ def _get_docker_image_version() -> str:
338340
else:
339341
version = get_version()
340342
if "dev" in version:
341-
version = version[: version.find("dev") - 1].replace(".", "_")
343+
version = version[: version.find("dev") - 1]
342344
_logger.warning(
343345
"You are trying to use AWS Lambda feature server while Feast is in a development mode. "
344346
f"Feast will use a docker image version {version} derived from Feast SDK "
@@ -347,8 +349,6 @@ def _get_docker_image_version() -> str:
347349
"> git fetch --all --tags\n"
348350
"> pip install -e sdk/python"
349351
)
350-
else:
351-
version = version.replace(".", "_")
352352
return version
353353

354354

‎sdk/python/feast/infra/offline_stores/bigquery_source.py‎

Lines changed: 10 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@
33

44
from feast import type_map
55
from feast.data_source import DataSource
6-
from feast.errors import DataSourceNoNameException, DataSourceNotFoundException
6+
from feast.errors import DataSourceNotFoundException
77
from feast.protos.feast.core.DataSource_pb2 import DataSource as DataSourceProto
88
from feast.protos.feast.core.SavedDataset_pb2 import (
99
SavedDatasetStorage as SavedDatasetStorageProto,
@@ -16,19 +16,18 @@
1616
class BigQuerySource(DataSource):
1717
def __init__(
1818
self,
19-
name: Optional[str] = None,
2019
event_timestamp_column: Optional[str] = "",
2120
table: Optional[str] = None,
2221
table_ref: Optional[str] = None,
2322
created_timestamp_column: Optional[str] = "",
2423
field_mapping: Optional[Dict[str, str]] = None,
2524
date_partition_column: Optional[str] = "",
2625
query: Optional[str] = None,
26+
name: Optional[str] = None,
2727
):
2828
"""Create a BigQuerySource from an existing table or query.
2929
3030
Args:
31-
name (optional): Name for the source. Defaults to the table_ref if not specified.
3231
table (optional): The BigQuery table where features can be found.
3332
table_ref (optional): (Deprecated) The BigQuery table where features can be found.
3433
event_timestamp_column: Event timestamp column used for point in time joins of feature values.
@@ -37,13 +36,13 @@ def __init__(
3736
or view. Only used for feature columns, not entities or timestamp columns.
3837
date_partition_column (optional): Timestamp column used for partitioning.
3938
query (optional): SQL query to execute to generate data for this data source.
40-
39+
name (optional): Name for the source. Defaults to the table_ref if not specified.
4140
Example:
4241
>>> from feast import BigQuerySource
4342
>>> my_bigquery_source = BigQuerySource(table="gcp_project:bq_dataset.bq_table")
4443
"""
4544
if table is None and table_ref is None and query is None:
46-
raise ValueError('No "table" argument provided.')
45+
raise ValueError('No "table" or "query" argument provided.')
4746
if not table and table_ref:
4847
warnings.warn(
4948
(
@@ -63,7 +62,12 @@ def __init__(
6362
elif table_ref:
6463
_name = table_ref
6564
else:
66-
raise DataSourceNoNameException()
65+
warnings.warn(
66+
(
67+
"Starting in Feast 0.21, Feast will require either a name for a data source (if using query) or `table`."
68+
),
69+
DeprecationWarning,
70+
)
6771

6872
super().__init__(
6973
_name if _name else "",

‎sdk/python/feast/infra/offline_stores/file_source.py‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -20,19 +20,18 @@ class FileSource(DataSource):
2020
def __init__(
2121
self,
2222
path: str,
23-
name: Optional[str] = "",
2423
event_timestamp_column: Optional[str] = "",
2524
file_format: Optional[FileFormat] = None,
2625
created_timestamp_column: Optional[str] = "",
2726
field_mapping: Optional[Dict[str, str]] = None,
2827
date_partition_column: Optional[str] = "",
2928
s3_endpoint_override: Optional[str] = None,
29+
name: Optional[str] = "",
3030
):
3131
"""Create a FileSource from a file containing feature data. Only Parquet format supported.
3232
3333
Args:
3434
35-
name (optional): Name for the file source. Defaults to the path.
3635
path: File path to file containing feature data. Must contain an event_timestamp column, entity columns and
3736
feature columns.
3837
event_timestamp_column: Event timestamp column used for point in time joins of feature values.
@@ -42,6 +41,7 @@ def __init__(
4241
or view. Only used for feature columns, not entities or timestamp columns.
4342
date_partition_column (optional): Timestamp column used for partitioning.
4443
s3_endpoint_override (optional): Overrides AWS S3 enpoint with custom S3 storage
44+
name (optional): Name for the file source. Defaults to the path.
4545
4646
Examples:
4747
>>> from feast import FileSource

‎sdk/python/feast/infra/offline_stores/redshift_source.py‎

Lines changed: 11 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -1,12 +1,9 @@
1+
import warnings
12
from typing import Callable, Dict, Iterable, Optional, Tuple
23

34
from feast import type_map
45
from feast.data_source import DataSource
5-
from feast.errors import (
6-
DataSourceNoNameException,
7-
DataSourceNotFoundException,
8-
RedshiftCredentialsError,
9-
)
6+
from feast.errors import DataSourceNotFoundException, RedshiftCredentialsError
107
from feast.protos.feast.core.DataSource_pb2 import DataSource as DataSourceProto
118
from feast.protos.feast.core.SavedDataset_pb2 import (
129
SavedDatasetStorage as SavedDatasetStorageProto,
@@ -19,20 +16,19 @@
1916
class RedshiftSource(DataSource):
2017
def __init__(
2118
self,
22-
name: Optional[str] = None,
2319
event_timestamp_column: Optional[str] = "",
2420
table: Optional[str] = None,
2521
schema: Optional[str] = None,
2622
created_timestamp_column: Optional[str] = "",
2723
field_mapping: Optional[Dict[str, str]] = None,
2824
date_partition_column: Optional[str] = "",
2925
query: Optional[str] = None,
26+
name: Optional[str] = None,
3027
):
3128
"""
3229
Creates a RedshiftSource object.
3330
3431
Args:
35-
name (optional): Name for the source. Defaults to the table_ref if not specified.
3632
event_timestamp_column (optional): Event timestamp column used for point in
3733
time joins of feature values.
3834
table (optional): Redshift table where the features are stored.
@@ -43,6 +39,7 @@ def __init__(
4339
source to column names in a feature table or view.
4440
date_partition_column (optional): Timestamp column used for partitioning.
4541
query (optional): The query to be executed to obtain the features.
42+
name (optional): Name for the source. Defaults to the table_ref if not specified.
4643
"""
4744
if table is None and query is None:
4845
raise ValueError('No "table" argument provided.')
@@ -51,11 +48,15 @@ def __init__(
5148
if table:
5249
_name = table
5350
else:
54-
raise DataSourceNoNameException()
51+
warnings.warn(
52+
(
53+
"Starting in Feast 0.21, Feast will require either a name for a data source (if using query) or `table`."
54+
),
55+
DeprecationWarning,
56+
)
5557

56-
# TODO(adchia): figure out what to do if user uses the query to start
5758
super().__init__(
58-
_name,
59+
_name if _name else "",
5960
event_timestamp_column,
6061
created_timestamp_column,
6162
field_mapping,

‎sdk/python/feast/infra/offline_stores/snowflake_source.py‎

Lines changed: 10 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,8 @@
1+
import warnings
12
from typing import Callable, Dict, Iterable, Optional, Tuple
23

34
from feast import type_map
45
from feast.data_source import DataSource
5-
from feast.errors import DataSourceNoNameException
66
from feast.protos.feast.core.DataSource_pb2 import DataSource as DataSourceProto
77
from feast.protos.feast.core.SavedDataset_pb2 import (
88
SavedDatasetStorage as SavedDatasetStorageProto,
@@ -15,7 +15,6 @@
1515
class SnowflakeSource(DataSource):
1616
def __init__(
1717
self,
18-
name: Optional[str] = None,
1918
database: Optional[str] = None,
2019
schema: Optional[str] = None,
2120
table: Optional[str] = None,
@@ -24,12 +23,12 @@ def __init__(
2423
created_timestamp_column: Optional[str] = "",
2524
field_mapping: Optional[Dict[str, str]] = None,
2625
date_partition_column: Optional[str] = "",
26+
name: Optional[str] = None,
2727
):
2828
"""
2929
Creates a SnowflakeSource object.
3030
3131
Args:
32-
name (optional): Name for the source. Defaults to the table if not specified.
3332
database (optional): Snowflake database where the features are stored.
3433
schema (optional): Snowflake schema in which the table is located.
3534
table (optional): Snowflake table where the features are stored.
@@ -41,7 +40,7 @@ def __init__(
4140
field_mapping (optional): A dictionary mapping of column names in this data
4241
source to column names in a feature table or view.
4342
date_partition_column (optional): Timestamp column used for partitioning.
44-
43+
name (optional): Name for the source. Defaults to the table if not specified.
4544
"""
4645
if table is None and query is None:
4746
raise ValueError('No "table" argument provided.')
@@ -52,10 +51,15 @@ def __init__(
5251
if table:
5352
_name = table
5453
else:
55-
raise DataSourceNoNameException()
54+
warnings.warn(
55+
(
56+
"Starting in Feast 0.21, Feast will require either a name for a data source (if using query) or `table`."
57+
),
58+
DeprecationWarning,
59+
)
5660

5761
super().__init__(
58-
_name,
62+
_name if _name else "",
5963
event_timestamp_column,
6064
created_timestamp_column,
6165
field_mapping,

‎sdk/python/tests/integration/feature_repos/repo_configuration.py‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -70,7 +70,6 @@
7070
if os.getenv("FEAST_IS_LOCAL_TEST", "False") != "True":
7171
DEFAULT_FULL_REPO_CONFIGS.extend(
7272
[
73-
# Redis configurations
7473
IntegrationTestRepoConfig(online_store=REDIS_CONFIG),
7574
IntegrationTestRepoConfig(online_store=REDIS_CLUSTER_CONFIG),
7675
# GCP configurations
@@ -263,7 +262,7 @@ def values(self):
263262

264263

265264
def construct_universal_feature_views(
266-
data_sources: UniversalDataSources,
265+
data_sources: UniversalDataSources, with_odfv: bool = True,
267266
) -> UniversalFeatureViews:
268267
driver_hourly_stats = create_driver_hourly_stats_feature_view(data_sources.driver)
269268
return UniversalFeatureViews(
@@ -275,7 +274,9 @@ def construct_universal_feature_views(
275274
"driver": driver_hourly_stats,
276275
"input_request": create_conv_rate_request_data_source(),
277276
}
278-
),
277+
)
278+
if with_odfv
279+
else None,
279280
driver_age_request_fv=create_driver_age_request_feature_view(),
280281
order=create_order_feature_view(data_sources.orders),
281282
location=create_location_stats_feature_view(data_sources.location),
@@ -358,7 +359,6 @@ def construct_test_environment(
358359
registry = RegistryConfig(
359360
path=str(Path(repo_dir_name) / "registry.db"), cache_ttl_seconds=1,
360361
)
361-
362362
config = RepoConfig(
363363
registry=registry,
364364
project=project,

‎sdk/python/tests/integration/online_store/test_universal_online.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -271,7 +271,7 @@ def _get_online_features_dict_remotely(
271271
if response.get("message") != "Internal Server Error":
272272
break
273273
# Sleep between retries to give the server some time to start
274-
time.sleep(1)
274+
time.sleep(15)
275275
else:
276276
raise Exception("Failed to get online features from remote feature server")
277277
if "metadata" not in response:

‎sdk/python/tests/integration/registration/test_inference.py‎

Lines changed: 10 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,9 @@
2222
update_data_sources_with_inferred_event_timestamp_col,
2323
update_entities_with_inferred_types_from_feature_views,
2424
)
25+
from feast.infra.offline_stores.contrib.spark_offline_store.spark_source import (
26+
SparkSource,
27+
)
2528
from feast.on_demand_feature_view import on_demand_feature_view
2629
from tests.utils.data_source_utils import (
2730
prep_file_source,
@@ -83,7 +86,7 @@ def test_infer_datasource_names_file():
8386

8487
def test_infer_datasource_names_dwh():
8588
table = "project.table"
86-
dwh_classes = [BigQuerySource, RedshiftSource, SnowflakeSource]
89+
dwh_classes = [BigQuerySource, RedshiftSource, SnowflakeSource, SparkSource]
8790

8891
for dwh_class in dwh_classes:
8992
data_source = dwh_class(table=table)
@@ -98,9 +101,13 @@ def test_infer_datasource_names_dwh():
98101
assert data_source_with_query.name == source_name
99102

100103
# If we have a query and no name, throw an error
101-
with pytest.raises(DataSourceNoNameException):
102-
print(f"Testing dwh {dwh_class}")
104+
if dwh_class == SparkSource:
105+
with pytest.raises(DataSourceNoNameException):
106+
print(f"Testing dwh {dwh_class}")
107+
data_source = dwh_class(query="test_query")
108+
else:
103109
data_source = dwh_class(query="test_query")
110+
assert data_source.name == ""
104111

105112

106113
@pytest.mark.integration

0 commit comments

Comments
 (0)