Skip to content

Commit e43d491

Browse files
Fix ODFV on-write transformations during materialize operations
- Include ODFVs with write_to_online_store=True in _get_feature_views_to_materialize() - Update provider materialize methods to handle ODFVs properly - Add comprehensive unit test to verify ODFV materialize behavior - Fixes GitHub issue feast-dev#5430 where on-write transformations weren't persisting during materialize operations Co-Authored-By: Francisco Javier Arceo <[email protected]>
1 parent c096190 commit e43d491

4 files changed

Lines changed: 187 additions & 8 deletions

File tree

‎sdk/python/feast/feature_store.py‎

Lines changed: 17 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -656,7 +656,7 @@ def _make_inferences(
656656
def _get_feature_views_to_materialize(
657657
self,
658658
feature_views: Optional[List[str]],
659-
) -> List[FeatureView]:
659+
) -> List[Union[FeatureView, OnDemandFeatureView]]:
660660
"""
661661
Returns the list of feature views that should be materialized.
662662
@@ -669,7 +669,7 @@ def _get_feature_views_to_materialize(
669669
FeatureViewNotFoundException: One of the specified feature views could not be found.
670670
ValueError: One of the specified feature views is not configured for materialization.
671671
"""
672-
feature_views_to_materialize: List[FeatureView] = []
672+
feature_views_to_materialize: List[Union[FeatureView, OnDemandFeatureView]] = []
673673

674674
if feature_views is None:
675675
feature_views_to_materialize = utils._list_feature_views(
@@ -684,19 +684,30 @@ def _get_feature_views_to_materialize(
684684
feature_views_to_materialize += [
685685
sfv for sfv in stream_feature_views_to_materialize if sfv.online
686686
]
687+
on_demand_feature_views_to_materialize = self.list_on_demand_feature_views()
688+
feature_views_to_materialize += [
689+
odfv for odfv in on_demand_feature_views_to_materialize if odfv.write_to_online_store
690+
]
687691
else:
688692
for name in feature_views:
689693
try:
690694
feature_view = self._get_feature_view(name, hide_dummy_entity=False)
691695
except FeatureViewNotFoundException:
692-
feature_view = self._get_stream_feature_view(
693-
name, hide_dummy_entity=False
694-
)
696+
try:
697+
feature_view = self._get_stream_feature_view(
698+
name, hide_dummy_entity=False
699+
)
700+
except FeatureViewNotFoundException:
701+
feature_view = self.get_on_demand_feature_view(name)
695702

696-
if not feature_view.online:
703+
if hasattr(feature_view, 'online') and not feature_view.online:
697704
raise ValueError(
698705
f"FeatureView {feature_view.name} is not configured to be served online."
699706
)
707+
elif hasattr(feature_view, 'write_to_online_store') and not feature_view.write_to_online_store:
708+
raise ValueError(
709+
f"OnDemandFeatureView {feature_view.name} is not configured for write_to_online_store."
710+
)
700711
feature_views_to_materialize.append(feature_view)
701712

702713
return feature_views_to_materialize

‎sdk/python/feast/infra/passthrough_provider.py‎

Lines changed: 11 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,7 @@
2424
from feast.feature_logging import FeatureServiceLoggingSource
2525
from feast.feature_service import FeatureService
2626
from feast.feature_view import FeatureView
27+
from feast.on_demand_feature_view import OnDemandFeatureView
2728
from feast.infra.common.materialization_job import (
2829
MaterializationJobStatus,
2930
MaterializationTask,
@@ -420,13 +421,22 @@ def ingest_df_to_offline_store(self, feature_view: FeatureView, table: pa.Table)
420421
def materialize_single_feature_view(
421422
self,
422423
config: RepoConfig,
423-
feature_view: FeatureView,
424+
feature_view: Union[FeatureView, OnDemandFeatureView],
424425
start_date: datetime,
425426
end_date: datetime,
426427
registry: BaseRegistry,
427428
project: str,
428429
tqdm_builder: Callable[[int], tqdm],
429430
) -> None:
431+
from feast.on_demand_feature_view import OnDemandFeatureView
432+
433+
if isinstance(feature_view, OnDemandFeatureView):
434+
if not feature_view.write_to_online_store:
435+
raise ValueError(
436+
f"OnDemandFeatureView {feature_view.name} does not have write_to_online_store enabled"
437+
)
438+
return
439+
430440
assert (
431441
isinstance(feature_view, BatchFeatureView)
432442
or isinstance(feature_view, StreamFeatureView)

‎sdk/python/feast/infra/provider.py‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,7 @@
2323
from feast.data_source import DataSource
2424
from feast.entity import Entity
2525
from feast.feature_view import FeatureView
26+
from feast.on_demand_feature_view import OnDemandFeatureView
2627
from feast.importer import import_class
2728
from feast.infra.infra_object import Infra
2829
from feast.infra.offline_stores.offline_store import RetrievalJob
@@ -217,7 +218,7 @@ def ingest_df_to_offline_store(
217218
def materialize_single_feature_view(
218219
self,
219220
config: RepoConfig,
220-
feature_view: FeatureView,
221+
feature_view: Union[FeatureView, OnDemandFeatureView],
221222
start_date: datetime,
222223
end_date: datetime,
223224
registry: BaseRegistry,

‎sdk/python/tests/unit/test_on_demand_python_transformation.py‎

Lines changed: 157 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1117,6 +1117,163 @@ def python_stored_writes_feature_view(
11171117
"current_datetime": [None],
11181118
}
11191119

1120+
def test_materialize_with_odfv_writes(self):
1121+
with tempfile.TemporaryDirectory() as data_dir:
1122+
self.store = FeatureStore(
1123+
config=RepoConfig(
1124+
project="test_on_demand_python_transformation",
1125+
registry=os.path.join(data_dir, "registry.db"),
1126+
provider="local",
1127+
entity_key_serialization_version=2,
1128+
online_store=SqliteOnlineStoreConfig(
1129+
path=os.path.join(data_dir, "online.db")
1130+
),
1131+
)
1132+
)
1133+
1134+
end_date = datetime.now().replace(microsecond=0, second=0, minute=0)
1135+
start_date = end_date - timedelta(days=15)
1136+
1137+
driver_entities = [1001, 1002, 1003, 1004, 1005]
1138+
driver_df = create_driver_hourly_stats_df(
1139+
driver_entities, start_date, end_date
1140+
)
1141+
driver_stats_path = os.path.join(data_dir, "driver_stats.parquet")
1142+
driver_df.to_parquet(
1143+
path=driver_stats_path, allow_truncated_timestamps=True
1144+
)
1145+
1146+
driver = Entity(name="driver", join_keys=["driver_id"])
1147+
1148+
driver_stats_source = FileSource(
1149+
name="driver_hourly_stats_source",
1150+
path=driver_stats_path,
1151+
timestamp_field="event_timestamp",
1152+
)
1153+
1154+
driver_stats_fv = FeatureView(
1155+
name="driver_hourly_stats",
1156+
entities=[driver],
1157+
ttl=timedelta(days=1),
1158+
schema=[
1159+
Field(name="conv_rate", dtype=Float32),
1160+
Field(name="acc_rate", dtype=Float32),
1161+
Field(name="avg_daily_trips", dtype=Int64),
1162+
],
1163+
online=True,
1164+
source=driver_stats_source,
1165+
tags={},
1166+
)
1167+
1168+
input_request_source = RequestSource(
1169+
name="vals_to_add",
1170+
schema=[
1171+
Field(name="counter", dtype=Int64),
1172+
Field(name="input_datetime", dtype=UnixTimestamp),
1173+
],
1174+
)
1175+
1176+
@on_demand_feature_view(
1177+
entities=[driver],
1178+
sources=[
1179+
driver_stats_fv[["conv_rate", "acc_rate"]],
1180+
input_request_source,
1181+
],
1182+
schema=[
1183+
Field(name="conv_rate_plus_acc", dtype=Float64),
1184+
Field(name="current_datetime", dtype=UnixTimestamp),
1185+
Field(name="counter", dtype=Int64),
1186+
Field(name="input_datetime", dtype=UnixTimestamp),
1187+
Field(name="string_constant", dtype=String),
1188+
],
1189+
mode="python",
1190+
write_to_online_store=True,
1191+
)
1192+
def python_stored_writes_feature_view(
1193+
inputs: dict[str, Any],
1194+
) -> dict[str, Any]:
1195+
output: dict[str, Any] = {
1196+
"conv_rate_plus_acc": [
1197+
conv_rate + acc_rate
1198+
for conv_rate, acc_rate in zip(
1199+
inputs["conv_rate"], inputs["acc_rate"]
1200+
)
1201+
],
1202+
"current_datetime": [datetime.now() for _ in inputs["conv_rate"]],
1203+
"counter": [c + 1 for c in inputs["counter"]],
1204+
"input_datetime": [d for d in inputs["input_datetime"]],
1205+
"string_constant": ["test_constant"],
1206+
}
1207+
return output
1208+
1209+
@on_demand_feature_view(
1210+
entities=[driver],
1211+
sources=[
1212+
driver_stats_fv[["conv_rate", "acc_rate"]],
1213+
input_request_source,
1214+
],
1215+
schema=[
1216+
Field(name="conv_rate_plus_acc", dtype=Float64),
1217+
Field(name="current_datetime", dtype=UnixTimestamp),
1218+
Field(name="counter", dtype=Int64),
1219+
Field(name="input_datetime", dtype=UnixTimestamp),
1220+
Field(name="string_constant", dtype=String),
1221+
],
1222+
mode="python",
1223+
write_to_online_store=False,
1224+
)
1225+
def python_no_writes_feature_view(
1226+
inputs: dict[str, Any],
1227+
) -> dict[str, Any]:
1228+
output: dict[str, Any] = {
1229+
"conv_rate_plus_acc": [
1230+
conv_rate + acc_rate
1231+
for conv_rate, acc_rate in zip(
1232+
inputs["conv_rate"], inputs["acc_rate"]
1233+
)
1234+
],
1235+
"current_datetime": [datetime.now() for _ in inputs["conv_rate"]],
1236+
"counter": [c + 1 for c in inputs["counter"]],
1237+
"input_datetime": [d for d in inputs["input_datetime"]],
1238+
"string_constant": ["test_constant"],
1239+
}
1240+
return output
1241+
1242+
self.store.apply(
1243+
[
1244+
driver,
1245+
driver_stats_source,
1246+
driver_stats_fv,
1247+
python_stored_writes_feature_view,
1248+
python_no_writes_feature_view,
1249+
]
1250+
)
1251+
1252+
feature_views_to_materialize = self.store._get_feature_views_to_materialize(None)
1253+
1254+
odfv_names = [fv.name for fv in feature_views_to_materialize if hasattr(fv, 'write_to_online_store')]
1255+
assert "python_stored_writes_feature_view" in odfv_names
1256+
assert "python_no_writes_feature_view" not in odfv_names
1257+
1258+
regular_fv_names = [fv.name for fv in feature_views_to_materialize if not hasattr(fv, 'write_to_online_store')]
1259+
assert "driver_hourly_stats" in regular_fv_names
1260+
1261+
materialize_end_date = datetime.now().replace(microsecond=0, second=0, minute=0)
1262+
materialize_start_date = materialize_end_date - timedelta(days=1)
1263+
1264+
self.store.materialize(materialize_start_date, materialize_end_date)
1265+
1266+
specific_feature_views_to_materialize = self.store._get_feature_views_to_materialize(
1267+
["driver_hourly_stats", "python_stored_writes_feature_view"]
1268+
)
1269+
assert len(specific_feature_views_to_materialize) == 2
1270+
1271+
try:
1272+
self.store._get_feature_views_to_materialize(["python_no_writes_feature_view"])
1273+
assert False, "Should have raised ValueError for ODFV without write_to_online_store"
1274+
except ValueError as e:
1275+
assert "not configured for write_to_online_store" in str(e)
1276+
11201277
def test_stored_writes_with_explode(self):
11211278
with tempfile.TemporaryDirectory() as data_dir:
11221279
self.store = FeatureStore(

0 commit comments

Comments
 (0)