Skip to content

Commit 6c3aaa7

Browse files
committed
feat(server): Add async materialization via ?async=true query param
When ?async=true is passed to /materialize and /materialize-incremental, the endpoint fires off materialization in a background thread and returns 202 Accepted immediately. Concurrent requests for an already-MATERIALIZING FV are rejected with 409. Without ?async=true, behavior is unchanged. Client-side additions: - remote=True param on store.materialize() / materialize_incremental() to delegate to the feature server (URL/TLS from online_store config) - wait=False support with store.poll_materialization() for status polling - FeatureView state set to MATERIALIZING before 202, reset on failure Server-side additions: - ?async=true on existing /materialize and /materialize-incremental - ?force=true to override stuck MATERIALIZING state - Four module-level helpers for testability: _authorize_materialize_views, _check_already_materializing, _update_fv_state, _parse_materialize_timestamps Registry fixes: - SQL and Snowflake registries now set FV state to AVAILABLE_ONLINE in apply_materialization() (parity with file-based registry) Addresses feast-dev#4526 Signed-off-by: Aniket Paluskar <[email protected]>
1 parent 104ad10 commit 6c3aaa7

5 files changed

Lines changed: 206 additions & 53 deletions

File tree

‎sdk/python/feast/feature_server.py‎

Lines changed: 171 additions & 45 deletions
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,7 @@
3131
from fastapi import (
3232
Depends,
3333
FastAPI,
34+
Query,
3435
Request,
3536
Response,
3637
WebSocket,
@@ -54,6 +55,7 @@
5455
)
5556
from feast.feast_object import FeastObject
5657
from feast.feature_server_utils import convert_response_to_dict
58+
from feast.feature_view import FeatureViewState
5759
from feast.feature_view_utils import get_feature_view_from_feature_store
5860
from feast.filter_models import ComparisonFilter, CompoundFilter
5961
from feast.permissions.action import WRITE, AuthzedAction
@@ -94,12 +96,14 @@ class MaterializeRequest(BaseModel):
9496
feature_views: Optional[List[str]] = None
9597
disable_event_timestamp: bool = False
9698
full_feature_names: bool = False
99+
version: Optional[str] = None
97100

98101

99102
class MaterializeIncrementalRequest(BaseModel):
100103
end_ts: str
101104
feature_views: Optional[List[str]] = None
102105
full_feature_names: bool = False
106+
version: Optional[str] = None
103107

104108

105109
class GetOnlineFeaturesRequest(BaseModel):
@@ -333,6 +337,96 @@ async def load_static_artifacts(app: FastAPI, store):
333337
logger.warning(f"Failed to load static artifacts: {e}")
334338

335339

340+
def _authorize_materialize_views(
341+
store: "feast.FeatureStore",
342+
feature_view_names: Optional[List[str]],
343+
) -> List[str]:
344+
"""Resolve + authorize feature views for materialization.
345+
346+
Returns the resolved list of FV names (all eligible FVs when
347+
feature_view_names is None).
348+
"""
349+
feature_views_to_materialize = store._get_feature_views_to_materialize(
350+
feature_view_names
351+
)
352+
for fv in feature_views_to_materialize:
353+
assert_permissions(
354+
resource=fv,
355+
actions=[AuthzedAction.WRITE_ONLINE],
356+
)
357+
return [fv.name for fv in feature_views_to_materialize]
358+
359+
360+
def _check_already_materializing(
361+
store: "feast.FeatureStore",
362+
fv_names: List[str],
363+
) -> Optional[JSONResponse]:
364+
"""Return a 409 JSONResponse if any requested FV is already MATERIALIZING."""
365+
conflicting: List[str] = []
366+
for fv_name in fv_names:
367+
try:
368+
fv = store.registry.get_feature_view(
369+
fv_name, store.project, allow_cache=False
370+
)
371+
if getattr(fv, "state", None) == FeatureViewState.MATERIALIZING:
372+
conflicting.append(fv_name)
373+
except Exception:
374+
pass
375+
if conflicting:
376+
return JSONResponse(
377+
status_code=409,
378+
content={
379+
"error": (
380+
f"Cannot start async materialization — the following feature "
381+
f"views are already in MATERIALIZING state: {conflicting}. "
382+
f"Use ?force=true to override."
383+
),
384+
"feature_views": conflicting,
385+
},
386+
)
387+
return None
388+
389+
390+
def _update_fv_state(
391+
store: "feast.FeatureStore",
392+
fv_names: List[str],
393+
state: FeatureViewState,
394+
) -> None:
395+
"""Set FV state in the registry for each named feature view."""
396+
for fv_name in fv_names:
397+
try:
398+
fv = store.registry.get_feature_view(
399+
fv_name, store.project, allow_cache=False
400+
)
401+
fv.state = state
402+
store.registry.apply_feature_view(fv, store.project)
403+
except Exception:
404+
logger.warning(f"Failed to set state={state} for {fv_name}")
405+
406+
407+
def _parse_materialize_timestamps(
408+
request: "MaterializeRequest",
409+
) -> tuple:
410+
"""Parse and validate start/end timestamps from a MaterializeRequest."""
411+
if request.disable_event_timestamp:
412+
now = datetime.now()
413+
return datetime(1970, 1, 1), now
414+
415+
if not request.start_ts or not request.end_ts:
416+
raise ValueError(
417+
"start_ts and end_ts are required when disable_event_timestamp is False"
418+
)
419+
try:
420+
start_date = utils.make_tzaware(parser.parse(request.start_ts))
421+
end_date = utils.make_tzaware(parser.parse(request.end_ts))
422+
except (ValueError, TypeError) as e:
423+
raise ValueError(f"Invalid timestamp format: {e}") from e
424+
425+
if start_date >= end_date:
426+
raise ValueError(f"start_ts ({start_date}) must be before end_ts ({end_date})")
427+
return start_date, end_date
428+
429+
336430
def get_app(
337431
store: "feast.FeatureStore",
338432
registry_ttl_sec: int = DEFAULT_FEATURE_SERVER_REGISTRY_TTL,
@@ -798,36 +892,47 @@ async def chat_ui():
798892
return Response(content=content, media_type="text/html")
799893

800894
@app.post("/materialize", dependencies=[Depends(inject_user_details)])
801-
async def materialize(request: MaterializeRequest) -> None:
895+
async def materialize(
896+
request: MaterializeRequest,
897+
async_mode: bool = Query(False, alias="async"),
898+
force: bool = Query(False),
899+
):
802900
with feast_metrics.track_request_latency("/materialize"):
803-
if request.feature_views:
804-
for feature_view in request.feature_views:
805-
resource = await _get_feast_object(feature_view, True)
806-
assert_permissions(
807-
resource=resource,
808-
actions=[AuthzedAction.WRITE_ONLINE],
809-
)
810-
else:
811-
feature_views_to_materialize = store._get_feature_views_to_materialize(
812-
None
813-
)
814-
for fv in feature_views_to_materialize:
815-
assert_permissions(
816-
resource=fv,
817-
actions=[AuthzedAction.WRITE_ONLINE],
818-
)
901+
fv_names = _authorize_materialize_views(store, request.feature_views)
902+
start_date, end_date = _parse_materialize_timestamps(request)
903+
904+
if async_mode:
905+
if not force:
906+
conflict = _check_already_materializing(store, fv_names)
907+
if conflict:
908+
return conflict
909+
910+
_update_fv_state(store, fv_names, FeatureViewState.MATERIALIZING)
911+
912+
def _run_materialize():
913+
try:
914+
store.materialize(
915+
start_date,
916+
end_date,
917+
fv_names,
918+
disable_event_timestamp=request.disable_event_timestamp,
919+
full_feature_names=request.full_feature_names,
920+
version=request.version,
921+
)
922+
except Exception as e:
923+
logger.error(
924+
f"Async materialization failed for {fv_names}: {e}",
925+
exc_info=True,
926+
)
927+
_update_fv_state(store, fv_names, FeatureViewState.GENERATED)
819928

820-
if request.disable_event_timestamp:
821-
now = datetime.now()
822-
start_date = datetime(1970, 1, 1)
823-
end_date = now
824-
else:
825-
if not request.start_ts or not request.end_ts:
826-
raise ValueError(
827-
"start_ts and end_ts are required when disable_event_timestamp is False"
828-
)
829-
start_date = utils.make_tzaware(parser.parse(request.start_ts))
830-
end_date = utils.make_tzaware(parser.parse(request.end_ts))
929+
loop = asyncio.get_running_loop()
930+
loop.run_in_executor(None, _run_materialize)
931+
932+
return JSONResponse(
933+
status_code=202,
934+
content={"status": "accepted", "feature_views": fv_names},
935+
)
831936

832937
await run_in_threadpool(
833938
store.materialize,
@@ -839,27 +944,48 @@ async def materialize(request: MaterializeRequest) -> None:
839944
)
840945

841946
@app.post("/materialize-incremental", dependencies=[Depends(inject_user_details)])
842-
async def materialize_incremental(request: MaterializeIncrementalRequest) -> None:
947+
async def materialize_incremental(
948+
request: MaterializeIncrementalRequest,
949+
async_mode: bool = Query(False, alias="async"),
950+
force: bool = Query(False),
951+
):
843952
with feast_metrics.track_request_latency("/materialize-incremental"):
844-
if request.feature_views:
845-
for feature_view in request.feature_views:
846-
resource = await _get_feast_object(feature_view, True)
847-
assert_permissions(
848-
resource=resource,
849-
actions=[AuthzedAction.WRITE_ONLINE],
850-
)
851-
else:
852-
feature_views_to_materialize = store._get_feature_views_to_materialize(
853-
None
953+
fv_names = _authorize_materialize_views(store, request.feature_views)
954+
end_date = utils.make_tzaware(parser.parse(request.end_ts))
955+
956+
if async_mode:
957+
if not force:
958+
conflict = _check_already_materializing(store, fv_names)
959+
if conflict:
960+
return conflict
961+
962+
_update_fv_state(store, fv_names, FeatureViewState.MATERIALIZING)
963+
964+
def _run_materialize_incremental():
965+
try:
966+
store.materialize_incremental(
967+
end_date,
968+
fv_names,
969+
full_feature_names=request.full_feature_names,
970+
)
971+
except Exception as e:
972+
logger.error(
973+
f"Async materialize-incremental failed for {fv_names}: {e}",
974+
exc_info=True,
975+
)
976+
_update_fv_state(store, fv_names, FeatureViewState.GENERATED)
977+
978+
loop = asyncio.get_running_loop()
979+
loop.run_in_executor(None, _run_materialize_incremental)
980+
981+
return JSONResponse(
982+
status_code=202,
983+
content={"status": "accepted", "feature_views": fv_names},
854984
)
855-
for fv in feature_views_to_materialize:
856-
assert_permissions(
857-
resource=fv,
858-
actions=[AuthzedAction.WRITE_ONLINE],
859-
)
985+
860986
await run_in_threadpool(
861987
store.materialize_incremental,
862-
utils.make_tzaware(parser.parse(request.end_ts)),
988+
end_date,
863989
request.feature_views,
864990
full_feature_names=request.full_feature_names,
865991
)

‎sdk/python/feast/feature_store.py‎

Lines changed: 26 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -510,7 +510,7 @@ def _transition_fv_to_materializing(
510510
511511
Rolls back all already-transitioned FVs if this one can't transition.
512512
"""
513-
previous_state = getattr(feature_view, "state", None)
513+
previous_states[feature_view.name] = getattr(feature_view, "state", None)
514514
if (
515515
hasattr(feature_view, "state")
516516
and feature_view.state != FeatureViewState.STATE_UNSPECIFIED
@@ -523,7 +523,6 @@ def _transition_fv_to_materializing(
523523
)
524524
feature_view.state = FeatureViewState.MATERIALIZING
525525
self.registry.apply_feature_view(feature_view, self.project, commit=True)
526-
previous_states[feature_view.name] = previous_state
527526

528527
def _submit_and_process_materialization_jobs(
529528
self,
@@ -2427,6 +2426,31 @@ def _materialize_odfv(
24272426
)
24282427
self.write_to_online_store(feature_view.name, df=transformed_df)
24292428

2429+
def _get_remote_materialize_url(self) -> str:
2430+
"""Get the feature server URL from online_store.path for remote materialization."""
2431+
online_cfg = self.config.online_store
2432+
url = getattr(online_cfg, "path", None)
2433+
if not url:
2434+
raise ValueError(
2435+
"online_store.path must be set to use remote materialization. "
2436+
"Configure online_store with type: remote and a valid path."
2437+
)
2438+
return url.rstrip("/")
2439+
2440+
def _get_remote_http_session(self):
2441+
"""Get an HTTP session with auth configured for the feature server."""
2442+
import requests
2443+
2444+
auth_config = getattr(self.config, "auth_config", None)
2445+
if auth_config and getattr(auth_config, "type", "no_auth") != "no_auth":
2446+
from feast.permissions.client.http_auth_requests_wrapper import (
2447+
get_http_auth_requests_session,
2448+
)
2449+
2450+
return get_http_auth_requests_session(auth_config)
2451+
2452+
return requests.Session()
2453+
24302454
def materialize_incremental(
24312455
self,
24322456
end_date: datetime,
@@ -2560,8 +2584,6 @@ def tqdm_builder(length):
25602584
)
25612585
else:
25622586
for feature_view, start_date in regular_fvs_with_dates:
2563-
# Transition state to MATERIALIZING before starting.
2564-
# Only enforce when the state machine is active (not STATE_UNSPECIFIED).
25652587
previous_state = getattr(feature_view, "state", None)
25662588
if (
25672589
hasattr(feature_view, "state")
@@ -2593,7 +2615,6 @@ def tqdm_builder(length):
25932615
)
25942616
except Exception:
25952617
fv_success = False
2596-
# Roll back state to previous value on failure.
25972618
if (
25982619
hasattr(feature_view, "state")
25992620
and previous_state is not None
@@ -2752,8 +2773,6 @@ def tqdm_builder(length):
27522773
)
27532774
else:
27542775
for feature_view, fv_start in regular_fvs_with_dates:
2755-
# Transition state to MATERIALIZING before starting.
2756-
# Only enforce when the state machine is active (not STATE_UNSPECIFIED).
27572776
previous_state = getattr(feature_view, "state", None)
27582777
if (
27592778
hasattr(feature_view, "state")
@@ -2786,7 +2805,6 @@ def tqdm_builder(length):
27862805
)
27872806
except Exception:
27882807
fv_success = False
2789-
# Roll back state to previous value on failure.
27902808
if (
27912809
hasattr(feature_view, "state")
27922810
and previous_state is not None

‎sdk/python/feast/feature_view.py‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -416,6 +416,7 @@ def __eq__(self, other):
416416
or normalize_version_string(self.version)
417417
!= normalize_version_string(other.version)
418418
or self.org != other.org
419+
or self.state != other.state
419420
):
420421
return False
421422

‎sdk/python/feast/infra/registry/snowflake.py‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1147,6 +1147,10 @@ def apply_materialization(
11471147
FeatureViewNotFoundException,
11481148
)
11491149
fv.materialization_intervals.append((start_date, end_date))
1150+
if hasattr(fv, "state"):
1151+
from feast.feature_view import FeatureViewState
1152+
1153+
fv.state = FeatureViewState.AVAILABLE_ONLINE
11501154
self._apply_object(
11511155
fv_table_str,
11521156
project,

‎sdk/python/feast/infra/registry/sql.py‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1228,6 +1228,10 @@ def apply_materialization(
12281228
FeatureViewNotFoundException,
12291229
)
12301230
fv.materialization_intervals.append((start_date, end_date))
1231+
if hasattr(fv, "state"):
1232+
from feast.feature_view import FeatureViewState
1233+
1234+
fv.state = FeatureViewState.AVAILABLE_ONLINE
12311235
self._apply_object(
12321236
table, project, "feature_view_name", fv, "feature_view_proto"
12331237
)

0 commit comments

Comments
 (0)