Skip to content

Commit 711a9d4

Browse files
committed
fix: Create sqlite tables for on demand feature views with write_to_online_store
feast apply on the local provider with the sqlite online store goes through FeatureStore.plan() / _apply_diffs() rather than FeatureStore.apply(). SqliteOnlineStore.plan() only produced tables for feature views, stream feature views and label views, so an on demand feature view with write_to_online_store=True was registered without a backing table and the subsequent materialize / write_to_online_store failed with 'sqlite3.OperationalError: no such table'. Include such on demand feature views in plan(), mirroring what update() already does for the non-plan apply path, and add unit tests. Part of feast-dev#5478 Signed-off-by: ryota717 <[email protected]>
1 parent 808aa05 commit 711a9d4

2 files changed

Lines changed: 77 additions & 1 deletion

File tree

‎sdk/python/feast/infra/online_stores/sqlite.py‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -54,6 +54,7 @@
5454
from feast.infra.online_stores.online_store import OnlineStore
5555
from feast.infra.online_stores.vector_store import VectorStoreConfig
5656
from feast.labeling.label_view import LabelView
57+
from feast.on_demand_feature_view import OnDemandFeatureView
5758
from feast.protos.feast.core.InfraObject_pb2 import InfraObject as InfraObjectProto
5859
from feast.protos.feast.core.Registry_pb2 import Registry as RegistryProto
5960
from feast.protos.feast.core.SqliteTable_pb2 import SqliteTable as SqliteTableProto
@@ -495,6 +496,19 @@ def plan(
495496
)
496497
)
497498

499+
# On demand feature views with write_to_online_store=True persist their
500+
# transformed features, so they need a table just like regular feature views.
501+
for odfv_proto in desired_registry_proto.on_demand_feature_views:
502+
if odfv_proto.spec.write_to_online_store:
503+
odfv = OnDemandFeatureView.from_proto(odfv_proto)
504+
infra_objects.append(
505+
SqliteTable(
506+
path=self._get_db_path(config),
507+
name=_table_id(project, odfv, versioning),
508+
include_value_num=include_value_num,
509+
)
510+
)
511+
498512
return infra_objects
499513

500514
def teardown(

‎sdk/python/tests/unit/infra/online_store/test_sqlite_plan.py‎

Lines changed: 63 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,16 +1,21 @@
11
from datetime import timedelta
22

3+
import pandas as pd
4+
35
from feast.data_format import AvroFormat
46
from feast.data_source import KafkaSource
7+
from feast.entity import Entity
58
from feast.feature_view import FeatureView
69
from feast.field import Field
710
from feast.infra.offline_stores.dask import DaskOfflineStoreConfig
811
from feast.infra.offline_stores.file_source import FileSource
912
from feast.infra.online_stores.sqlite import SqliteOnlineStore, SqliteOnlineStoreConfig
13+
from feast.on_demand_feature_view import OnDemandFeatureView
1014
from feast.protos.feast.core.Registry_pb2 import Registry as RegistryProto
1115
from feast.repo_config import RepoConfig
1216
from feast.stream_feature_view import StreamFeatureView
13-
from feast.types import String
17+
from feast.transformation.pandas_transformation import PandasTransformation
18+
from feast.types import Int64, String
1419

1520

1621
def _repo_config() -> RepoConfig:
@@ -86,3 +91,60 @@ def test_plan_succeeds_with_batch_and_stream_feature_views_together(self):
8691
"test_project_batch_view",
8792
"test_project_driver_dropoffs_stream",
8893
]
94+
95+
96+
def _on_demand_feature_view(
97+
name: str, source: FeatureView, write_to_online_store: bool
98+
) -> OnDemandFeatureView:
99+
def transform(inputs: pd.DataFrame) -> pd.DataFrame:
100+
return pd.DataFrame({"derived": inputs["value"]})
101+
102+
return OnDemandFeatureView(
103+
name=name,
104+
entities=[Entity(name="driver", join_keys=["driver_id"])],
105+
sources=[source],
106+
schema=[Field(name="derived", dtype=Int64)],
107+
feature_transformation=PandasTransformation(
108+
udf=transform, udf_string="transform"
109+
),
110+
write_to_online_store=write_to_online_store,
111+
)
112+
113+
114+
class TestSqliteOnlineStorePlanWithOnDemandFeatureViews:
115+
"""feast apply on the local provider goes through plan(), so on demand
116+
feature views with write_to_online_store=True must get a table here too,
117+
otherwise materialize / write_to_online_store fails with 'no such table'."""
118+
119+
def test_plan_includes_on_demand_feature_views_with_writes(self):
120+
config = _repo_config()
121+
source = _feature_view("source_view")
122+
registry_proto = RegistryProto()
123+
registry_proto.feature_views.append(source.to_proto())
124+
registry_proto.on_demand_feature_views.append(
125+
_on_demand_feature_view(
126+
"odfv_on_write", source, write_to_online_store=True
127+
).to_proto()
128+
)
129+
130+
infra_objects = SqliteOnlineStore().plan(config, registry_proto)
131+
132+
assert sorted(o.name for o in infra_objects) == [
133+
"test_project_odfv_on_write",
134+
"test_project_source_view",
135+
]
136+
137+
def test_plan_skips_on_demand_feature_views_without_writes(self):
138+
config = _repo_config()
139+
source = _feature_view("source_view")
140+
registry_proto = RegistryProto()
141+
registry_proto.feature_views.append(source.to_proto())
142+
registry_proto.on_demand_feature_views.append(
143+
_on_demand_feature_view(
144+
"odfv_on_read", source, write_to_online_store=False
145+
).to_proto()
146+
)
147+
148+
infra_objects = SqliteOnlineStore().plan(config, registry_proto)
149+
150+
assert [o.name for o in infra_objects] == ["test_project_source_view"]

0 commit comments

Comments
 (0)