Skip to content

Commit df51b94

Browse files
authored
ci: Use testcontainers to spin up redis and datastore instances (#2493)
* ci: Use testcontainers to spin up redis instances for tests Signed-off-by: Achal Shah <[email protected]> * fix test Signed-off-by: Achal Shah <[email protected]> * fix test Signed-off-by: Achal Shah <[email protected]> * fix type Signed-off-by: Achal Shah <[email protected]> * fix type Signed-off-by: Achal Shah <[email protected]> * fix typo and repr Signed-off-by: Achal Shah <[email protected]> * datastore baby Signed-off-by: Achal Shah <[email protected]> * Make containers optional Signed-off-by: Achal Shah <[email protected]> * more optionality Signed-off-by: Achal Shah <[email protected]> * makefile target and docs Signed-off-by: Achal Shah <[email protected]> * makefile target and docs Signed-off-by: Achal Shah <[email protected]> * cr Signed-off-by: Achal Shah <[email protected]>
1 parent 34f2b59 commit df51b94

16 files changed

Lines changed: 160 additions & 17 deletions

‎CONTRIBUTING.md‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -161,6 +161,21 @@ To test across clouds, on top of setting up Redis, you also need GCP / AWS / Sno
161161

162162
Then run `make test-python-integration`. Note that for Snowflake / GCP / AWS, this will create new temporary tables / datasets.
163163

164+
#### (Experimental) Run full integration tests against containerized services
165+
Test across clouds requires existing accounts on GCP / AWS / Snowflake, and may incur costs when using these services.
166+
167+
For this approach of running tests, you'll need to have docker set up locally: [Get Docker](https://docs.docker.com/get-docker/)
168+
169+
It's possible to run some integration tests against emulated local versions of these services, using ephemeral containers.
170+
These tests create new temporary tables / datasets locally only, and they are cleaned up. when the containers are torn down.
171+
172+
The services with containerized replacements currently implemented are:
173+
- Datastore
174+
- Redis
175+
176+
You can run `make test-python-integration-container` to run tests against the containerized versions of dependencies.
177+
178+
164179
## Feast Java Serving
165180
See [Java contributing guide](java/CONTRIBUTING.md)
166181

‎Makefile‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -67,6 +67,9 @@ test-python:
6767
test-python-integration:
6868
FEAST_USAGE=False IS_TEST=True python -m pytest -n 8 --integration sdk/python/tests
6969

70+
test-python-integration-container:
71+
FEAST_USAGE=False IS_TEST=True FEAST_LOCAL_ONLINE_CONTAINER=True python -m pytest -n 8 --integration sdk/python/tests
72+
7073
test-python-universal-contrib:
7174
PYTHONPATH='.' FULL_REPO_CONFIGS_MODULE=sdk.python.feast.infra.offline_stores.contrib.contrib_repo_configuration FEAST_USAGE=False IS_TEST=True python -m pytest -n 8 --integration --universal sdk/python/tests
7275

‎sdk/python/setup.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -126,7 +126,7 @@
126126
"pytest-mock==1.10.4",
127127
"Sphinx!=4.0.0,<4.4.0",
128128
"sphinx-rtd-theme",
129-
"testcontainers==3.4.2",
129+
"testcontainers>=3.5",
130130
"adlfs==0.5.9",
131131
"firebase-admin==4.5.2",
132132
"pre-commit",

‎sdk/python/tests/conftest.py‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -184,6 +184,8 @@ def cleanup():
184184
e.feature_store.teardown()
185185
if proc.is_alive():
186186
proc.kill()
187+
if e.online_store_creator:
188+
e.online_store_creator.teardown()
187189

188190
request.addfinalizer(cleanup)
189191

@@ -245,6 +247,8 @@ def go_data_sources(request, go_environment):
245247
def cleanup():
246248
# logger.info("Running cleanup in %s, Request: %s", worker_id, request.param)
247249
go_environment.data_source_creator.teardown()
250+
if go_environment.online_store_creator:
251+
go_environment.online_store_creator.teardown()
248252

249253
request.addfinalizer(cleanup)
250254
return construct_universal_test_data(go_environment)
@@ -259,6 +263,8 @@ def e2e_data_sources(environment: Environment, request):
259263

260264
def cleanup():
261265
environment.data_source_creator.teardown()
266+
if environment.online_store_creator:
267+
environment.online_store_creator.teardown()
262268

263269
request.addfinalizer(cleanup)
264270

‎sdk/python/tests/integration/feature_repos/integration_test_repo_config.py‎

Lines changed: 17 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -1,15 +1,18 @@
11
from dataclasses import dataclass
2-
from typing import Dict, Type, Union
2+
from typing import Dict, Optional, Type, Union
33

44
from tests.integration.feature_repos.universal.data_source_creator import (
55
DataSourceCreator,
66
)
77
from tests.integration.feature_repos.universal.data_sources.file import (
88
FileDataSourceCreator,
99
)
10+
from tests.integration.feature_repos.universal.online_store_creator import (
11+
OnlineStoreCreator,
12+
)
1013

1114

12-
@dataclass(frozen=True)
15+
@dataclass(frozen=False)
1316
class IntegrationTestRepoConfig:
1417
"""
1518
This class should hold all possible parameters that may need to be varied by individual tests.
@@ -19,19 +22,26 @@ class IntegrationTestRepoConfig:
1922
online_store: Union[str, Dict] = "sqlite"
2023

2124
offline_store_creator: Type[DataSourceCreator] = FileDataSourceCreator
25+
online_store_creator: Optional[Type[OnlineStoreCreator]] = None
2226

2327
full_feature_names: bool = True
2428
infer_features: bool = False
2529
python_feature_server: bool = False
2630
go_feature_server: bool = False
2731

2832
def __repr__(self) -> str:
29-
if isinstance(self.online_store, str):
30-
online_store_type = self.online_store
31-
elif self.online_store["type"] == "redis":
32-
online_store_type = self.online_store.get("redis_type", "redis")
33+
if not self.online_store_creator:
34+
if isinstance(self.online_store, str):
35+
online_store_type = self.online_store
36+
elif isinstance(self.online_store, dict):
37+
if self.online_store["type"] == "redis":
38+
online_store_type = self.online_store.get("redis_type", "redis")
39+
else:
40+
online_store_type = self.online_store["type"]
41+
else:
42+
online_store_type = self.online_store.__name__
3343
else:
34-
online_store_type = self.online_store["type"]
44+
online_store_type = self.online_store_creator.__name__
3545

3646
return ":".join(
3747
[

‎sdk/python/tests/integration/feature_repos/repo_configuration.py‎

Lines changed: 32 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -44,6 +44,15 @@
4444
create_order_feature_view,
4545
create_pushable_feature_view,
4646
)
47+
from tests.integration.feature_repos.universal.online_store.datastore import (
48+
DatastoreOnlineStoreCreator,
49+
)
50+
from tests.integration.feature_repos.universal.online_store.redis import (
51+
RedisOnlineStoreCreator,
52+
)
53+
from tests.integration.feature_repos.universal.online_store_creator import (
54+
OnlineStoreCreator,
55+
)
4756

4857
DYNAMO_CONFIG = {"type": "dynamodb", "region": "us-west-2"}
4958
# Port 12345 will chosen as default for redis node configuration because Redis Cluster is started off of nodes
@@ -115,6 +124,18 @@
115124
else:
116125
FULL_REPO_CONFIGS = DEFAULT_FULL_REPO_CONFIGS
117126

127+
if os.getenv("FEAST_LOCAL_ONLINE_CONTAINER", "False").lower() == "true":
128+
replacements = {"datastore": DatastoreOnlineStoreCreator}
129+
replacement_dicts = [(REDIS_CONFIG, RedisOnlineStoreCreator)]
130+
for c in FULL_REPO_CONFIGS:
131+
if isinstance(c.online_store, dict):
132+
for _replacement in replacement_dicts:
133+
if c.online_store == _replacement[0]:
134+
c.online_store_creator = _replacement[1]
135+
elif c.online_store in replacements:
136+
c.online_store_creator = replacements[c.online_store]
137+
138+
118139
GO_REPO_CONFIGS = [
119140
IntegrationTestRepoConfig(online_store=REDIS_CONFIG, go_feature_server=True,),
120141
]
@@ -299,6 +320,7 @@ class Environment:
299320
data_source_creator: DataSourceCreator
300321
python_feature_server: bool
301322
worker_id: str
323+
online_store_creator: Optional[OnlineStoreCreator] = None
302324

303325
def __post_init__(self):
304326
self.end_date = datetime.utcnow().replace(microsecond=0, second=0, minute=0)
@@ -341,9 +363,16 @@ def construct_test_environment(
341363
project = f"{test_suite_name}_{run_id}_{run_num}"
342364

343365
offline_creator: DataSourceCreator = test_repo_config.offline_store_creator(project)
344-
345366
offline_store_config = offline_creator.create_offline_store_config()
346-
online_store = test_repo_config.online_store
367+
368+
if test_repo_config.online_store_creator:
369+
online_creator = test_repo_config.online_store_creator(project)
370+
online_store = (
371+
test_repo_config.online_store
372+
) = online_creator.create_online_store()
373+
else:
374+
online_creator = None
375+
online_store = test_repo_config.online_store
347376

348377
repo_dir_name = tempfile.mkdtemp()
349378

@@ -392,6 +421,7 @@ def construct_test_environment(
392421
data_source_creator=offline_creator,
393422
python_feature_server=test_repo_config.python_feature_server,
394423
worker_id=worker_id,
424+
online_store_creator=online_creator,
395425
)
396426

397427
return environment

‎sdk/python/tests/integration/feature_repos/universal/data_source_creator.py‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,9 @@
99

1010

1111
class DataSourceCreator(ABC):
12+
def __init__(self, project_name: str):
13+
self.project_name = project_name
14+
1215
@abstractmethod
1316
def create_data_source(
1417
self,

‎sdk/python/tests/integration/feature_repos/universal/data_sources/bigquery.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -18,8 +18,8 @@ class BigQueryDataSourceCreator(DataSourceCreator):
1818
dataset: Optional[Dataset] = None
1919

2020
def __init__(self, project_name: str):
21+
super().__init__(project_name)
2122
self.client = bigquery.Client()
22-
self.project_name = project_name
2323
self.gcp_project = self.client.project
2424
self.dataset_id = f"{self.gcp_project}.{project_name}"
2525

‎sdk/python/tests/integration/feature_repos/universal/data_sources/file.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,7 @@ class FileDataSourceCreator(DataSourceCreator):
2222
files: List[Any]
2323

2424
def __init__(self, project_name: str):
25-
self.project_name = project_name
25+
super().__init__(project_name)
2626
self.files = []
2727

2828
def create_data_source(

‎sdk/python/tests/integration/feature_repos/universal/data_sources/redshift.py‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -19,8 +19,7 @@ class RedshiftDataSourceCreator(DataSourceCreator):
1919
tables: List[str] = []
2020

2121
def __init__(self, project_name: str):
22-
super().__init__()
23-
self.project_name = project_name
22+
super().__init__(project_name)
2423
self.client = aws_utils.get_redshift_data_client("us-west-2")
2524
self.s3 = aws_utils.get_s3_resource("us-west-2")
2625

0 commit comments

Comments
 (0)