Skip to content

Commit c0bc4c0

Browse files
committed
feat: Add registry list feature views by updated since (feast-dev#6092)
Signed-off-by: Nick Quinn <[email protected]>
1 parent cb9cbd0 commit c0bc4c0

16 files changed

Lines changed: 580 additions & 207 deletions

File tree

‎docs/reference/feature-servers/registry-server.md‎

Lines changed: 11 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -214,6 +214,7 @@ Most endpoints support these common query parameters:
214214
- `feature` (optional): Filter feature views by feature name
215215
- `feature_service` (optional): Filter feature views by feature service name
216216
- `data_source` (optional): Filter feature views by data source name
217+
- `updated_since` (optional): Only return feature views updated at or after this ISO-8601 UTC timestamp (e.g. `2024-01-01T00:00:00Z`)
217218
- `page` (optional): Page number for pagination
218219
- `limit` (optional): Number of items per page
219220
- `sort_by` (optional): Field to sort by
@@ -223,27 +224,31 @@ Most endpoints support these common query parameters:
223224
# Basic list
224225
curl -H "Authorization: Bearer <token>" \
225226
"http://localhost:6572/api/v1/feature_views?project=my_project"
226-
227+
227228
# With pagination and relationships
228229
curl -H "Authorization: Bearer <token>" \
229230
"http://localhost:6572/api/v1/feature_views?project=my_project&include_relationships=true&page=1&limit=5&sort_by=name"
230-
231+
231232
# Filter by entity
232233
curl -H "Authorization: Bearer <token>" \
233234
"http://localhost:6572/api/v1/feature_views?project=my_project&entity=user"
234-
235+
235236
# Filter by feature
236237
curl -H "Authorization: Bearer <token>" \
237238
"http://localhost:6572/api/v1/feature_views?project=my_project&feature=age"
238-
239+
239240
# Filter by data source
240241
curl -H "Authorization: Bearer <token>" \
241242
"http://localhost:6572/api/v1/feature_views?project=my_project&data_source=user_profile_source"
242-
243+
243244
# Filter by feature service
244245
curl -H "Authorization: Bearer <token>" \
245246
"http://localhost:6572/api/v1/feature_views?project=my_project&feature_service=user_service"
246-
247+
248+
# Filter by last-updated timestamp
249+
curl -H "Authorization: Bearer <token>" \
250+
"http://localhost:6572/api/v1/feature_views?project=my_project&updated_since=2024-06-01T00:00:00Z"
251+
247252
# Multiple filters combined
248253
curl -H "Authorization: Bearer <token>" \
249254
"http://localhost:6572/api/v1/feature_views?project=my_project&entity=user&feature=age"

‎protos/feast/registry/RegistryServer.proto‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -278,6 +278,7 @@ message ListAllFeatureViewsRequest {
278278
string data_source = 7;
279279
PaginationParams pagination = 8;
280280
SortingParams sorting = 9;
281+
google.protobuf.Timestamp updated_since = 10;
281282
}
282283

283284
message ListAllFeatureViewsResponse {

‎sdk/python/feast/api/registry/rest/feature_views.py‎

Lines changed: 17 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,8 @@
1-
from typing import Dict
1+
from datetime import timezone
2+
from typing import Dict, Optional
23

34
from fastapi import APIRouter, Depends, Query
5+
from google.protobuf import timestamp_pb2
46

57
from feast.api.registry.rest.codegen_utils import render_feature_view_code
68
from feast.api.registry.rest.rest_utils import (
@@ -237,10 +239,23 @@ def list_all_feature_views(
237239
data_source: str = Query(
238240
None, description="Filter feature views by data source name"
239241
),
242+
updated_since: Optional[str] = Query(
243+
None,
244+
description="Only return feature views updated at or after this ISO-8601 UTC timestamp (e.g. 2024-01-01T00:00:00Z)",
245+
),
240246
tags: Dict[str, str] = Depends(parse_tags),
241247
pagination_params: dict = Depends(get_pagination_params),
242248
sorting_params: dict = Depends(get_sorting_params),
243249
):
250+
updated_since_proto = None
251+
if updated_since is not None:
252+
from datetime import datetime
253+
254+
dt = datetime.fromisoformat(updated_since.replace("Z", "+00:00"))
255+
ts = timestamp_pb2.Timestamp()
256+
ts.FromDatetime(dt.astimezone(timezone.utc))
257+
updated_since_proto = ts
258+
244259
req = RegistryServer_pb2.ListAllFeatureViewsRequest(
245260
project=project,
246261
allow_cache=allow_cache,
@@ -251,6 +266,7 @@ def list_all_feature_views(
251266
data_source=data_source,
252267
pagination=create_grpc_pagination_params(pagination_params),
253268
sorting=create_grpc_sorting_params(sorting_params),
269+
updated_since=updated_since_proto,
254270
)
255271
response = grpc_call(grpc_handler.ListAllFeatureViews, req)
256272
any_feature_views = response.get("featureViews", [])

‎sdk/python/feast/infra/registry/base_registry.py‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -477,6 +477,7 @@ def list_all_feature_views(
477477
project: str,
478478
allow_cache: bool = False,
479479
tags: Optional[dict[str, str]] = None,
480+
updated_since: Optional[datetime] = None,
480481
) -> List[BaseFeatureView]:
481482
"""
482483
Retrieve a list of feature views of all types from the registry
@@ -485,6 +486,7 @@ def list_all_feature_views(
485486
allow_cache: Allow returning feature views from the cached registry
486487
project: Filter feature views based on project name
487488
tags: Filter by tags
489+
updated_since: Only return feature views updated at or after this timestamp
488490
489491
Returns:
490492
List of feature views

‎sdk/python/feast/infra/registry/caching_registry.py‎

Lines changed: 19 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@
33
import threading
44
import warnings
55
from abc import abstractmethod
6-
from datetime import timedelta
6+
from datetime import datetime, timedelta
77
from threading import Lock
88
from typing import List, Optional
99

@@ -121,7 +121,10 @@ def get_any_feature_view(
121121

122122
@abstractmethod
123123
def _list_all_feature_views(
124-
self, project: str, tags: Optional[dict[str, str]]
124+
self,
125+
project: str,
126+
tags: Optional[dict[str, str]],
127+
updated_since: Optional[datetime] = None,
125128
) -> List[BaseFeatureView]:
126129
pass
127130

@@ -130,13 +133,25 @@ def list_all_feature_views(
130133
project: str,
131134
allow_cache: bool = False,
132135
tags: Optional[dict[str, str]] = None,
136+
updated_since: Optional[datetime] = None,
133137
) -> List[BaseFeatureView]:
134138
if allow_cache:
135139
self._refresh_cached_registry_if_necessary()
136-
return proto_registry_utils.list_all_feature_views(
140+
feature_views = proto_registry_utils.list_all_feature_views(
137141
self.cached_registry_proto, project, tags
138142
)
139-
return self._list_all_feature_views(project, tags)
143+
if updated_since is not None:
144+
# last_updated_timestamp from proto is offset-naive UTC; normalise for comparison
145+
cutoff = updated_since.replace(tzinfo=None) if updated_since.tzinfo else updated_since
146+
feature_views = [
147+
fv
148+
for fv in feature_views
149+
if fv.last_updated_timestamp is not None
150+
and fv.last_updated_timestamp >= cutoff
151+
]
152+
else:
153+
feature_views = self._list_all_feature_views(project, tags, updated_since)
154+
return feature_views
140155

141156
@abstractmethod
142157
def _get_feature_view(self, name: str, project: str) -> FeatureView:

‎sdk/python/feast/infra/registry/registry.py‎

Lines changed: 12 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -640,13 +640,24 @@ def list_all_feature_views(
640640
project: str,
641641
allow_cache: bool = False,
642642
tags: Optional[dict[str, str]] = None,
643+
updated_since: Optional[datetime] = None,
643644
) -> List[BaseFeatureView]:
644645
registry_proto = self._get_registry_proto(
645646
project=project, allow_cache=allow_cache
646647
)
647-
return proto_registry_utils.list_all_feature_views(
648+
feature_views = proto_registry_utils.list_all_feature_views(
648649
registry_proto, project, tags
649650
)
651+
if updated_since is not None:
652+
# last_updated_timestamp from proto is offset-naive UTC; normalise for comparison
653+
cutoff = updated_since.replace(tzinfo=None) if updated_since.tzinfo else updated_since
654+
feature_views = [
655+
fv
656+
for fv in feature_views
657+
if fv.last_updated_timestamp is not None
658+
and fv.last_updated_timestamp >= cutoff
659+
]
660+
return feature_views
650661

651662
def get_any_feature_view(
652663
self, name: str, project: str, allow_cache: bool = False

‎sdk/python/feast/infra/registry/remote.py‎

Lines changed: 14 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -316,9 +316,22 @@ def list_all_feature_views(
316316
project: str,
317317
allow_cache: bool = False,
318318
tags: Optional[dict[str, str]] = None,
319+
updated_since: Optional[datetime] = None,
319320
) -> List[BaseFeatureView]:
321+
updated_since_proto = None
322+
if updated_since is not None:
323+
ts = Timestamp()
324+
ts.FromDatetime(
325+
updated_since.replace(tzinfo=None)
326+
if updated_since.tzinfo
327+
else updated_since
328+
)
329+
updated_since_proto = ts
320330
request = RegistryServer_pb2.ListAllFeatureViewsRequest(
321-
project=project, allow_cache=allow_cache, tags=tags
331+
project=project,
332+
allow_cache=allow_cache,
333+
tags=tags,
334+
updated_since=updated_since_proto,
322335
)
323336

324337
response: RegistryServer_pb2.ListAllFeatureViewsResponse = (

‎sdk/python/feast/infra/registry/snowflake.py‎

Lines changed: 32 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -626,28 +626,45 @@ def list_all_feature_views(
626626
project: str,
627627
allow_cache: bool = False,
628628
tags: Optional[dict[str, str]] = None,
629+
updated_since: Optional[datetime] = None,
629630
) -> List[BaseFeatureView]:
630631
if allow_cache:
631632
self._refresh_cached_registry_if_necessary()
632-
return proto_registry_utils.list_all_feature_views(
633+
feature_views = proto_registry_utils.list_all_feature_views(
633634
self.cached_registry_proto, project, tags
634635
)
635-
636-
return (
637-
cast(
638-
list[BaseFeatureView],
639-
self.list_feature_views(project, allow_cache, tags),
640-
)
641-
+ cast(
642-
list[BaseFeatureView],
643-
self.list_stream_feature_views(project, allow_cache, tags),
644-
)
645-
+ cast(
646-
list[BaseFeatureView],
647-
self.list_on_demand_feature_views(project, allow_cache, tags),
648-
)
636+
if updated_since is not None:
637+
cutoff = updated_since.replace(tzinfo=None) if updated_since.tzinfo else updated_since
638+
feature_views = [
639+
fv
640+
for fv in feature_views
641+
if fv.last_updated_timestamp is not None
642+
and fv.last_updated_timestamp >= cutoff
643+
]
644+
return feature_views
645+
646+
feature_views = cast(
647+
list[BaseFeatureView],
648+
self.list_feature_views(project, allow_cache, tags),
649+
) + cast(
650+
list[BaseFeatureView],
651+
self.list_stream_feature_views(project, allow_cache, tags),
652+
) + cast(
653+
list[BaseFeatureView],
654+
self.list_on_demand_feature_views(project, allow_cache, tags),
649655
)
650656

657+
if updated_since is not None:
658+
cutoff = updated_since.replace(tzinfo=None) if updated_since.tzinfo else updated_since
659+
feature_views = [
660+
fv
661+
for fv in feature_views
662+
if fv.last_updated_timestamp is not None
663+
and fv.last_updated_timestamp >= cutoff
664+
]
665+
666+
return feature_views
667+
651668
def get_infra(self, project: str, allow_cache: bool = False) -> Infra:
652669
infra_object = self._get_object(
653670
"MANAGED_INFRA",

‎sdk/python/feast/infra/registry/sql.py‎

Lines changed: 28 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -349,7 +349,10 @@ def _get_stream_feature_view(self, name: str, project: str):
349349
)
350350

351351
def _list_stream_feature_views(
352-
self, project: str, tags: Optional[dict[str, str]]
352+
self,
353+
project: str,
354+
tags: Optional[dict[str, str]],
355+
updated_since: Optional[datetime] = None,
353356
) -> List[StreamFeatureView]:
354357
return self._list_objects(
355358
stream_feature_views,
@@ -358,6 +361,7 @@ def _list_stream_feature_views(
358361
StreamFeatureView,
359362
"feature_view_proto",
360363
tags=tags,
364+
updated_since=updated_since,
361365
)
362366

363367
def apply_entity(self, entity: Entity, project: str, commit: bool = True):
@@ -419,20 +423,23 @@ def _get_any_feature_view(self, name: str, project: str) -> BaseFeatureView:
419423
return fv
420424

421425
def _list_all_feature_views(
422-
self, project: str, tags: Optional[dict[str, str]]
426+
self,
427+
project: str,
428+
tags: Optional[dict[str, str]],
429+
updated_since: Optional[datetime] = None,
423430
) -> List[BaseFeatureView]:
424431
return (
425432
cast(
426433
list[BaseFeatureView],
427-
self._list_feature_views(project=project, tags=tags),
434+
self._list_feature_views(project=project, tags=tags, updated_since=updated_since),
428435
)
429436
+ cast(
430437
list[BaseFeatureView],
431-
self._list_stream_feature_views(project=project, tags=tags),
438+
self._list_stream_feature_views(project=project, tags=tags, updated_since=updated_since),
432439
)
433440
+ cast(
434441
list[BaseFeatureView],
435-
self._list_on_demand_feature_views(project=project, tags=tags),
442+
self._list_on_demand_feature_views(project=project, tags=tags, updated_since=updated_since),
436443
)
437444
)
438445

@@ -619,7 +626,10 @@ def _list_feature_services(
619626
)
620627

621628
def _list_feature_views(
622-
self, project: str, tags: Optional[dict[str, str]]
629+
self,
630+
project: str,
631+
tags: Optional[dict[str, str]],
632+
updated_since: Optional[datetime] = None,
623633
) -> List[FeatureView]:
624634
return self._list_objects(
625635
feature_views,
@@ -628,6 +638,7 @@ def _list_feature_views(
628638
FeatureView,
629639
"feature_view_proto",
630640
tags=tags,
641+
updated_since=updated_since,
631642
)
632643

633644
def _list_saved_datasets(
@@ -643,7 +654,10 @@ def _list_saved_datasets(
643654
)
644655

645656
def _list_on_demand_feature_views(
646-
self, project: str, tags: Optional[dict[str, str]]
657+
self,
658+
project: str,
659+
tags: Optional[dict[str, str]],
660+
updated_since: Optional[datetime] = None,
647661
) -> List[OnDemandFeatureView]:
648662
return self._list_objects(
649663
on_demand_feature_views,
@@ -652,6 +666,7 @@ def _list_on_demand_feature_views(
652666
OnDemandFeatureView,
653667
"feature_view_proto",
654668
tags=tags,
669+
updated_since=updated_since,
655670
)
656671

657672
def _list_project_metadata(self, project: str) -> List[ProjectMetadata]:
@@ -1097,9 +1112,15 @@ def _list_objects(
10971112
python_class: Any,
10981113
proto_field_name: str,
10991114
tags: Optional[dict[str, str]] = None,
1115+
updated_since: Optional[datetime] = None,
11001116
):
11011117
with self.read_engine.begin() as conn:
11021118
stmt = select(table).where(table.c.project_id == project)
1119+
if updated_since is not None:
1120+
stmt = stmt.where(
1121+
table.c.last_updated_timestamp
1122+
>= int(updated_since.timestamp())
1123+
)
11031124
rows = conn.execute(stmt).all()
11041125
if rows:
11051126
objects = []

‎sdk/python/feast/protos/feast/registry/RegistryServer_pb2.py‎

Lines changed: 166 additions & 166 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

0 commit comments

Comments
 (0)