Skip to content

Commit 5dd869c

Browse files
nickquinn408ntkathole
authored andcommitted
feat: Add registry list feature views by updated since (feast-dev#6092)
Signed-off-by: Nick Quinn <[email protected]>
1 parent effa9ca commit 5dd869c

16 files changed

Lines changed: 580 additions & 207 deletions

File tree

‎docs/reference/feature-servers/registry-server.md‎

Lines changed: 11 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -214,6 +214,7 @@ Most endpoints support these common query parameters:
214214
- `feature` (optional): Filter feature views by feature name
215215
- `feature_service` (optional): Filter feature views by feature service name
216216
- `data_source` (optional): Filter feature views by data source name
217+
- `updated_since` (optional): Only return feature views updated at or after this ISO-8601 UTC timestamp (e.g. `2024-01-01T00:00:00Z`)
217218
- `page` (optional): Page number for pagination
218219
- `limit` (optional): Number of items per page
219220
- `sort_by` (optional): Field to sort by
@@ -223,27 +224,31 @@ Most endpoints support these common query parameters:
223224
# Basic list
224225
curl -H "Authorization: Bearer <token>" \
225226
"http://localhost:6572/api/v1/feature_views?project=my_project"
226-
227+
227228
# With pagination and relationships
228229
curl -H "Authorization: Bearer <token>" \
229230
"http://localhost:6572/api/v1/feature_views?project=my_project&include_relationships=true&page=1&limit=5&sort_by=name"
230-
231+
231232
# Filter by entity
232233
curl -H "Authorization: Bearer <token>" \
233234
"http://localhost:6572/api/v1/feature_views?project=my_project&entity=user"
234-
235+
235236
# Filter by feature
236237
curl -H "Authorization: Bearer <token>" \
237238
"http://localhost:6572/api/v1/feature_views?project=my_project&feature=age"
238-
239+
239240
# Filter by data source
240241
curl -H "Authorization: Bearer <token>" \
241242
"http://localhost:6572/api/v1/feature_views?project=my_project&data_source=user_profile_source"
242-
243+
243244
# Filter by feature service
244245
curl -H "Authorization: Bearer <token>" \
245246
"http://localhost:6572/api/v1/feature_views?project=my_project&feature_service=user_service"
246-
247+
248+
# Filter by last-updated timestamp
249+
curl -H "Authorization: Bearer <token>" \
250+
"http://localhost:6572/api/v1/feature_views?project=my_project&updated_since=2024-06-01T00:00:00Z"
251+
247252
# Multiple filters combined
248253
curl -H "Authorization: Bearer <token>" \
249254
"http://localhost:6572/api/v1/feature_views?project=my_project&entity=user&feature=age"

‎protos/feast/registry/RegistryServer.proto‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -278,6 +278,7 @@ message ListAllFeatureViewsRequest {
278278
string data_source = 7;
279279
PaginationParams pagination = 8;
280280
SortingParams sorting = 9;
281+
google.protobuf.Timestamp updated_since = 10;
281282
}
282283

283284
message ListAllFeatureViewsResponse {

‎sdk/python/feast/api/registry/rest/feature_views.py‎

Lines changed: 17 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,8 @@
1-
from typing import Dict
1+
from datetime import timezone
2+
from typing import Dict, Optional
23

34
from fastapi import APIRouter, Depends, Query
5+
from google.protobuf import timestamp_pb2
46

57
from feast.api.registry.rest.codegen_utils import render_feature_view_code
68
from feast.api.registry.rest.rest_utils import (
@@ -237,10 +239,23 @@ def list_all_feature_views(
237239
data_source: str = Query(
238240
None, description="Filter feature views by data source name"
239241
),
242+
updated_since: Optional[str] = Query(
243+
None,
244+
description="Only return feature views updated at or after this ISO-8601 UTC timestamp (e.g. 2024-01-01T00:00:00Z)",
245+
),
240246
tags: Dict[str, str] = Depends(parse_tags),
241247
pagination_params: dict = Depends(get_pagination_params),
242248
sorting_params: dict = Depends(get_sorting_params),
243249
):
250+
updated_since_proto = None
251+
if updated_since is not None:
252+
from datetime import datetime
253+
254+
dt = datetime.fromisoformat(updated_since.replace("Z", "+00:00"))
255+
ts = timestamp_pb2.Timestamp()
256+
ts.FromDatetime(dt.astimezone(timezone.utc))
257+
updated_since_proto = ts
258+
244259
req = RegistryServer_pb2.ListAllFeatureViewsRequest(
245260
project=project,
246261
allow_cache=allow_cache,
@@ -251,6 +266,7 @@ def list_all_feature_views(
251266
data_source=data_source,
252267
pagination=create_grpc_pagination_params(pagination_params),
253268
sorting=create_grpc_sorting_params(sorting_params),
269+
updated_since=updated_since_proto,
254270
)
255271
response = grpc_call(grpc_handler.ListAllFeatureViews, req)
256272
any_feature_views = response.get("featureViews", [])

‎sdk/python/feast/infra/registry/base_registry.py‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -484,6 +484,7 @@ def list_all_feature_views(
484484
project: str,
485485
allow_cache: bool = False,
486486
tags: Optional[dict[str, str]] = None,
487+
updated_since: Optional[datetime] = None,
487488
) -> List[BaseFeatureView]:
488489
"""
489490
Retrieve a list of feature views of all types from the registry
@@ -492,6 +493,7 @@ def list_all_feature_views(
492493
allow_cache: Allow returning feature views from the cached registry
493494
project: Filter feature views based on project name
494495
tags: Filter by tags
496+
updated_since: Only return feature views updated at or after this timestamp
495497

496498
Returns:
497499
List of feature views

‎sdk/python/feast/infra/registry/caching_registry.py‎

Lines changed: 19 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@
33
import threading
44
import warnings
55
from abc import abstractmethod
6-
from datetime import timedelta
6+
from datetime import datetime, timedelta
77
from threading import Lock
88
from typing import Any, Dict, List, Optional
99

@@ -121,7 +121,10 @@ def get_any_feature_view(
121121

122122
@abstractmethod
123123
def _list_all_feature_views(
124-
self, project: str, tags: Optional[dict[str, str]]
124+
self,
125+
project: str,
126+
tags: Optional[dict[str, str]],
127+
updated_since: Optional[datetime] = None,
125128
) -> List[BaseFeatureView]:
126129
pass
127130

@@ -130,13 +133,25 @@ def list_all_feature_views(
130133
project: str,
131134
allow_cache: bool = False,
132135
tags: Optional[dict[str, str]] = None,
136+
updated_since: Optional[datetime] = None,
133137
) -> List[BaseFeatureView]:
134138
if allow_cache:
135139
self._refresh_cached_registry_if_necessary()
136-
return proto_registry_utils.list_all_feature_views(
140+
feature_views = proto_registry_utils.list_all_feature_views(
137141
self.cached_registry_proto, project, tags
138142
)
139-
return self._list_all_feature_views(project, tags)
143+
if updated_since is not None:
144+
# last_updated_timestamp from proto is offset-naive UTC; normalise for comparison
145+
cutoff = updated_since.replace(tzinfo=None) if updated_since.tzinfo else updated_since
146+
feature_views = [
147+
fv
148+
for fv in feature_views
149+
if fv.last_updated_timestamp is not None
150+
and fv.last_updated_timestamp >= cutoff
151+
]
152+
else:
153+
feature_views = self._list_all_feature_views(project, tags, updated_since)
154+
return feature_views
140155

141156
@abstractmethod
142157
def _get_feature_view(self, name: str, project: str) -> FeatureView:

‎sdk/python/feast/infra/registry/registry.py‎

Lines changed: 12 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -969,13 +969,24 @@ def list_all_feature_views(
969969
project: str,
970970
allow_cache: bool = False,
971971
tags: Optional[dict[str, str]] = None,
972+
updated_since: Optional[datetime] = None,
972973
) -> List[BaseFeatureView]:
973974
registry_proto = self._get_registry_proto(
974975
project=project, allow_cache=allow_cache
975976
)
976-
return proto_registry_utils.list_all_feature_views(
977+
feature_views = proto_registry_utils.list_all_feature_views(
977978
registry_proto, project, tags
978979
)
980+
if updated_since is not None:
981+
# last_updated_timestamp from proto is offset-naive UTC; normalise for comparison
982+
cutoff = updated_since.replace(tzinfo=None) if updated_since.tzinfo else updated_since
983+
feature_views = [
984+
fv
985+
for fv in feature_views
986+
if fv.last_updated_timestamp is not None
987+
and fv.last_updated_timestamp >= cutoff
988+
]
989+
return feature_views
979990

980991
def get_any_feature_view(
981992
self, name: str, project: str, allow_cache: bool = False

‎sdk/python/feast/infra/registry/remote.py‎

Lines changed: 14 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -320,9 +320,22 @@ def list_all_feature_views(
320320
project: str,
321321
allow_cache: bool = False,
322322
tags: Optional[dict[str, str]] = None,
323+
updated_since: Optional[datetime] = None,
323324
) -> List[BaseFeatureView]:
325+
updated_since_proto = None
326+
if updated_since is not None:
327+
ts = Timestamp()
328+
ts.FromDatetime(
329+
updated_since.replace(tzinfo=None)
330+
if updated_since.tzinfo
331+
else updated_since
332+
)
333+
updated_since_proto = ts
324334
request = RegistryServer_pb2.ListAllFeatureViewsRequest(
325-
project=project, allow_cache=allow_cache, tags=tags
335+
project=project,
336+
allow_cache=allow_cache,
337+
tags=tags,
338+
updated_since=updated_since_proto,
326339
)
327340

328341
response: RegistryServer_pb2.ListAllFeatureViewsResponse = (

‎sdk/python/feast/infra/registry/snowflake.py‎

Lines changed: 32 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -636,28 +636,45 @@ def list_all_feature_views(
636636
project: str,
637637
allow_cache: bool = False,
638638
tags: Optional[dict[str, str]] = None,
639+
updated_since: Optional[datetime] = None,
639640
) -> List[BaseFeatureView]:
640641
if allow_cache:
641642
self._refresh_cached_registry_if_necessary()
642-
return proto_registry_utils.list_all_feature_views(
643+
feature_views = proto_registry_utils.list_all_feature_views(
643644
self.cached_registry_proto, project, tags
644645
)
645-
646-
return (
647-
cast(
648-
list[BaseFeatureView],
649-
self.list_feature_views(project, allow_cache, tags),
650-
)
651-
+ cast(
652-
list[BaseFeatureView],
653-
self.list_stream_feature_views(project, allow_cache, tags),
654-
)
655-
+ cast(
656-
list[BaseFeatureView],
657-
self.list_on_demand_feature_views(project, allow_cache, tags),
658-
)
646+
if updated_since is not None:
647+
cutoff = updated_since.replace(tzinfo=None) if updated_since.tzinfo else updated_since
648+
feature_views = [
649+
fv
650+
for fv in feature_views
651+
if fv.last_updated_timestamp is not None
652+
and fv.last_updated_timestamp >= cutoff
653+
]
654+
return feature_views
655+
656+
feature_views = cast(
657+
list[BaseFeatureView],
658+
self.list_feature_views(project, allow_cache, tags),
659+
) + cast(
660+
list[BaseFeatureView],
661+
self.list_stream_feature_views(project, allow_cache, tags),
662+
) + cast(
663+
list[BaseFeatureView],
664+
self.list_on_demand_feature_views(project, allow_cache, tags),
659665
)
660666

667+
if updated_since is not None:
668+
cutoff = updated_since.replace(tzinfo=None) if updated_since.tzinfo else updated_since
669+
feature_views = [
670+
fv
671+
for fv in feature_views
672+
if fv.last_updated_timestamp is not None
673+
and fv.last_updated_timestamp >= cutoff
674+
]
675+
676+
return feature_views
677+
661678
def get_infra(self, project: str, allow_cache: bool = False) -> Infra:
662679
infra_object = self._get_object(
663680
"MANAGED_INFRA",

‎sdk/python/feast/infra/registry/sql.py‎

Lines changed: 28 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -387,7 +387,10 @@ def _get_stream_feature_view(self, name: str, project: str):
387387
)
388388

389389
def _list_stream_feature_views(
390-
self, project: str, tags: Optional[dict[str, str]]
390+
self,
391+
project: str,
392+
tags: Optional[dict[str, str]],
393+
updated_since: Optional[datetime] = None,
391394
) -> List[StreamFeatureView]:
392395
return self._list_objects(
393396
stream_feature_views,
@@ -396,6 +399,7 @@ def _list_stream_feature_views(
396399
StreamFeatureView,
397400
"feature_view_proto",
398401
tags=tags,
402+
updated_since=updated_since,
399403
)
400404

401405
def apply_entity(self, entity: Entity, project: str, commit: bool = True):
@@ -457,20 +461,23 @@ def _get_any_feature_view(self, name: str, project: str) -> BaseFeatureView:
457461
return fv
458462

459463
def _list_all_feature_views(
460-
self, project: str, tags: Optional[dict[str, str]]
464+
self,
465+
project: str,
466+
tags: Optional[dict[str, str]],
467+
updated_since: Optional[datetime] = None,
461468
) -> List[BaseFeatureView]:
462469
return (
463470
cast(
464471
list[BaseFeatureView],
465-
self._list_feature_views(project=project, tags=tags),
472+
self._list_feature_views(project=project, tags=tags, updated_since=updated_since),
466473
)
467474
+ cast(
468475
list[BaseFeatureView],
469-
self._list_stream_feature_views(project=project, tags=tags),
476+
self._list_stream_feature_views(project=project, tags=tags, updated_since=updated_since),
470477
)
471478
+ cast(
472479
list[BaseFeatureView],
473-
self._list_on_demand_feature_views(project=project, tags=tags),
480+
self._list_on_demand_feature_views(project=project, tags=tags, updated_since=updated_since),
474481
)
475482
)
476483

@@ -890,7 +897,10 @@ def _list_feature_services(
890897
)
891898

892899
def _list_feature_views(
893-
self, project: str, tags: Optional[dict[str, str]]
900+
self,
901+
project: str,
902+
tags: Optional[dict[str, str]],
903+
updated_since: Optional[datetime] = None,
894904
) -> List[FeatureView]:
895905
return self._list_objects(
896906
feature_views,
@@ -899,6 +909,7 @@ def _list_feature_views(
899909
FeatureView,
900910
"feature_view_proto",
901911
tags=tags,
912+
updated_since=updated_since,
902913
)
903914

904915
def _list_saved_datasets(
@@ -914,7 +925,10 @@ def _list_saved_datasets(
914925
)
915926

916927
def _list_on_demand_feature_views(
917-
self, project: str, tags: Optional[dict[str, str]]
928+
self,
929+
project: str,
930+
tags: Optional[dict[str, str]],
931+
updated_since: Optional[datetime] = None,
918932
) -> List[OnDemandFeatureView]:
919933
return self._list_objects(
920934
on_demand_feature_views,
@@ -923,6 +937,7 @@ def _list_on_demand_feature_views(
923937
OnDemandFeatureView,
924938
"feature_view_proto",
925939
tags=tags,
940+
updated_since=updated_since,
926941
)
927942

928943
def _list_project_metadata(self, project: str) -> List[ProjectMetadata]:
@@ -1486,9 +1501,15 @@ def _list_objects(
14861501
python_class: Any,
14871502
proto_field_name: str,
14881503
tags: Optional[dict[str, str]] = None,
1504+
updated_since: Optional[datetime] = None,
14891505
):
14901506
with self.read_engine.begin() as conn:
14911507
stmt = select(table).where(table.c.project_id == project)
1508+
if updated_since is not None:
1509+
stmt = stmt.where(
1510+
table.c.last_updated_timestamp
1511+
>= int(updated_since.timestamp())
1512+
)
14921513
rows = conn.execute(stmt).all()
14931514
if rows:
14941515
objects = []

‎sdk/python/feast/protos/feast/registry/RegistryServer_pb2.py‎

Lines changed: 166 additions & 166 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

0 commit comments

Comments
 (0)