Skip to content

Commit 627a9f3

Browse files
committed
feat: Add registry list feature views by updated since (feast-dev#6092)
Signed-off-by: Nick Quinn <[email protected]>
1 parent 0d02614 commit 627a9f3

16 files changed

Lines changed: 497 additions & 132 deletions

File tree

‎docs/reference/feature-servers/registry-server.md‎

Lines changed: 11 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -214,6 +214,7 @@ Most endpoints support these common query parameters:
214214
- `feature` (optional): Filter feature views by feature name
215215
- `feature_service` (optional): Filter feature views by feature service name
216216
- `data_source` (optional): Filter feature views by data source name
217+
- `updated_since` (optional): Only return feature views updated at or after this ISO-8601 UTC timestamp (e.g. `2024-01-01T00:00:00Z`)
217218
- `page` (optional): Page number for pagination
218219
- `limit` (optional): Number of items per page
219220
- `sort_by` (optional): Field to sort by
@@ -223,27 +224,31 @@ Most endpoints support these common query parameters:
223224
# Basic list
224225
curl -H "Authorization: Bearer <token>" \
225226
"http://localhost:6572/api/v1/feature_views?project=my_project"
226-
227+
227228
# With pagination and relationships
228229
curl -H "Authorization: Bearer <token>" \
229230
"http://localhost:6572/api/v1/feature_views?project=my_project&include_relationships=true&page=1&limit=5&sort_by=name"
230-
231+
231232
# Filter by entity
232233
curl -H "Authorization: Bearer <token>" \
233234
"http://localhost:6572/api/v1/feature_views?project=my_project&entity=user"
234-
235+
235236
# Filter by feature
236237
curl -H "Authorization: Bearer <token>" \
237238
"http://localhost:6572/api/v1/feature_views?project=my_project&feature=age"
238-
239+
239240
# Filter by data source
240241
curl -H "Authorization: Bearer <token>" \
241242
"http://localhost:6572/api/v1/feature_views?project=my_project&data_source=user_profile_source"
242-
243+
243244
# Filter by feature service
244245
curl -H "Authorization: Bearer <token>" \
245246
"http://localhost:6572/api/v1/feature_views?project=my_project&feature_service=user_service"
246-
247+
248+
# Filter by last-updated timestamp
249+
curl -H "Authorization: Bearer <token>" \
250+
"http://localhost:6572/api/v1/feature_views?project=my_project&updated_since=2024-06-01T00:00:00Z"
251+
247252
# Multiple filters combined
248253
curl -H "Authorization: Bearer <token>" \
249254
"http://localhost:6572/api/v1/feature_views?project=my_project&entity=user&feature=age"

‎protos/feast/registry/RegistryServer.proto‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -291,6 +291,7 @@ message ListAllFeatureViewsRequest {
291291
string data_source = 7;
292292
PaginationParams pagination = 8;
293293
SortingParams sorting = 9;
294+
google.protobuf.Timestamp updated_since = 10;
294295
}
295296

296297
message ListAllFeatureViewsResponse {

‎sdk/python/feast/api/registry/rest/feature_views.py‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,10 @@
11
import logging
2+
from datetime import timezone
23
from typing import Dict, List, Optional
34

45
from fastapi import APIRouter, Depends, Query
56
from fastapi.responses import JSONResponse
7+
from google.protobuf import timestamp_pb2
68
from google.protobuf.duration_pb2 import Duration
79
from pydantic import BaseModel
810

@@ -266,10 +268,23 @@ def list_all_feature_views(
266268
data_source: str = Query(
267269
None, description="Filter feature views by data source name"
268270
),
271+
updated_since: Optional[str] = Query(
272+
None,
273+
description="Only return feature views updated at or after this ISO-8601 UTC timestamp (e.g. 2024-01-01T00:00:00Z)",
274+
),
269275
tags: Dict[str, str] = Depends(parse_tags),
270276
pagination_params: dict = Depends(get_pagination_params),
271277
sorting_params: dict = Depends(get_sorting_params),
272278
):
279+
updated_since_proto = None
280+
if updated_since is not None:
281+
from datetime import datetime
282+
283+
dt = datetime.fromisoformat(updated_since.replace("Z", "+00:00"))
284+
ts = timestamp_pb2.Timestamp()
285+
ts.FromDatetime(dt.astimezone(timezone.utc))
286+
updated_since_proto = ts
287+
273288
req = RegistryServer_pb2.ListAllFeatureViewsRequest(
274289
project=project,
275290
allow_cache=allow_cache,
@@ -280,6 +295,7 @@ def list_all_feature_views(
280295
data_source=data_source,
281296
pagination=create_grpc_pagination_params(pagination_params),
282297
sorting=create_grpc_sorting_params(sorting_params),
298+
updated_since=updated_since_proto,
283299
)
284300
response = grpc_call(grpc_handler.ListAllFeatureViews, req)
285301
any_feature_views = response.get("featureViews", [])

‎sdk/python/feast/infra/registry/base_registry.py‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -567,6 +567,7 @@ def list_all_feature_views(
567567
allow_cache: bool = False,
568568
tags: Optional[dict[str, str]] = None,
569569
skip_udf: bool = False,
570+
updated_since: Optional[datetime] = None,
570571
) -> List[BaseFeatureView]:
571572
"""
572573
Retrieve a list of feature views of all types from the registry
@@ -576,6 +577,7 @@ def list_all_feature_views(
576577
project: Filter feature views based on project name
577578
tags: Filter by tags
578579
skip_udf: Skip deserializing UDFs (for metadata-only operations)
580+
updated_since: Only return feature views updated at or after this timestamp
579581
580582
Returns:
581583
List of feature views

‎sdk/python/feast/infra/registry/caching_registry.py‎

Lines changed: 26 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@
33
import threading
44
import warnings
55
from abc import abstractmethod
6-
from datetime import timedelta
6+
from datetime import datetime, timedelta
77
from threading import Lock
88
from typing import Any, Dict, List, Optional
99

@@ -122,7 +122,11 @@ def get_any_feature_view(
122122

123123
@abstractmethod
124124
def _list_all_feature_views(
125-
self, project: str, tags: Optional[dict[str, str]], **kwargs: Any
125+
self,
126+
project: str,
127+
tags: Optional[dict[str, str]],
128+
updated_since: Optional[datetime] = None,
129+
**kwargs: Any,
126130
) -> List[BaseFeatureView]:
127131
pass
128132

@@ -132,13 +136,31 @@ def list_all_feature_views(
132136
allow_cache: bool = False,
133137
tags: Optional[dict[str, str]] = None,
134138
skip_udf: bool = False,
139+
updated_since: Optional[datetime] = None,
135140
) -> List[BaseFeatureView]:
136141
if allow_cache:
137142
self._refresh_cached_registry_if_necessary()
138-
return proto_registry_utils.list_all_feature_views(
143+
feature_views = proto_registry_utils.list_all_feature_views(
139144
self.cached_registry_proto, project, tags, skip_udf=skip_udf
140145
)
141-
return self._list_all_feature_views(project, tags, skip_udf=skip_udf)
146+
if updated_since is not None:
147+
# last_updated_timestamp from proto is offset-naive UTC; normalise for comparison
148+
cutoff = (
149+
updated_since.replace(tzinfo=None)
150+
if updated_since.tzinfo
151+
else updated_since
152+
)
153+
feature_views = [
154+
fv
155+
for fv in feature_views
156+
if fv.last_updated_timestamp is not None
157+
and fv.last_updated_timestamp >= cutoff
158+
]
159+
else:
160+
feature_views = self._list_all_feature_views(
161+
project, tags, updated_since=updated_since, skip_udf=skip_udf
162+
)
163+
return feature_views
142164

143165
@abstractmethod
144166
def _get_feature_view(self, name: str, project: str) -> FeatureView:

‎sdk/python/feast/infra/registry/registry.py‎

Lines changed: 16 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1083,13 +1083,28 @@ def list_all_feature_views(
10831083
allow_cache: bool = False,
10841084
tags: Optional[dict[str, str]] = None,
10851085
skip_udf: bool = False,
1086+
updated_since: Optional[datetime] = None,
10861087
) -> List[BaseFeatureView]:
10871088
registry_proto = self._get_registry_proto(
10881089
project=project, allow_cache=allow_cache
10891090
)
1090-
return proto_registry_utils.list_all_feature_views(
1091+
feature_views = proto_registry_utils.list_all_feature_views(
10911092
registry_proto, project, tags, skip_udf=skip_udf
10921093
)
1094+
if updated_since is not None:
1095+
# last_updated_timestamp from proto is offset-naive UTC; normalise for comparison
1096+
cutoff = (
1097+
updated_since.replace(tzinfo=None)
1098+
if updated_since.tzinfo
1099+
else updated_since
1100+
)
1101+
feature_views = [
1102+
fv
1103+
for fv in feature_views
1104+
if fv.last_updated_timestamp is not None
1105+
and fv.last_updated_timestamp >= cutoff
1106+
]
1107+
return feature_views
10931108

10941109
def get_any_feature_view(
10951110
self, name: str, project: str, allow_cache: bool = False

‎sdk/python/feast/infra/registry/remote.py‎

Lines changed: 14 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -403,9 +403,22 @@ def list_all_feature_views(
403403
allow_cache: bool = False,
404404
tags: Optional[dict[str, str]] = None,
405405
skip_udf: bool = False,
406+
updated_since: Optional[datetime] = None,
406407
) -> List[BaseFeatureView]:
408+
updated_since_proto = None
409+
if updated_since is not None:
410+
ts = Timestamp()
411+
ts.FromDatetime(
412+
updated_since.replace(tzinfo=None)
413+
if updated_since.tzinfo
414+
else updated_since
415+
)
416+
updated_since_proto = ts
407417
request = RegistryServer_pb2.ListAllFeatureViewsRequest(
408-
project=project, allow_cache=allow_cache, tags=tags
418+
project=project,
419+
allow_cache=allow_cache,
420+
tags=tags,
421+
updated_since=updated_since_proto,
409422
)
410423

411424
response: RegistryServer_pb2.ListAllFeatureViewsResponse = (

‎sdk/python/feast/infra/registry/snowflake.py‎

Lines changed: 31 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -656,14 +656,28 @@ def list_all_feature_views(
656656
allow_cache: bool = False,
657657
tags: Optional[dict[str, str]] = None,
658658
skip_udf: bool = False,
659+
updated_since: Optional[datetime] = None,
659660
) -> List[BaseFeatureView]:
660661
if allow_cache:
661662
registry_proto = self._refresh_cached_registry_if_necessary()
662-
return proto_registry_utils.list_all_feature_views(
663+
feature_views = proto_registry_utils.list_all_feature_views(
663664
registry_proto, project, tags, skip_udf=skip_udf
664665
)
666+
if updated_since is not None:
667+
cutoff = (
668+
updated_since.replace(tzinfo=None)
669+
if updated_since.tzinfo
670+
else updated_since
671+
)
672+
feature_views = [
673+
fv
674+
for fv in feature_views
675+
if fv.last_updated_timestamp is not None
676+
and fv.last_updated_timestamp >= cutoff
677+
]
678+
return feature_views
665679

666-
return (
680+
feature_views = (
667681
cast(
668682
list[BaseFeatureView],
669683
self.list_feature_views(project, allow_cache, tags, skip_udf=skip_udf),
@@ -686,6 +700,21 @@ def list_all_feature_views(
686700
)
687701
)
688702

703+
if updated_since is not None:
704+
cutoff = (
705+
updated_since.replace(tzinfo=None)
706+
if updated_since.tzinfo
707+
else updated_since
708+
)
709+
feature_views = [
710+
fv
711+
for fv in feature_views
712+
if fv.last_updated_timestamp is not None
713+
and fv.last_updated_timestamp >= cutoff
714+
]
715+
716+
return feature_views
717+
689718
def get_infra(self, project: str, allow_cache: bool = False) -> Infra:
690719
infra_object = self._get_object(
691720
"MANAGED_INFRA",

‎sdk/python/feast/infra/registry/sql.py‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1597,6 +1597,7 @@ def _list_objects(
15971597
tags: Optional[dict[str, str]] = None,
15981598
proto_only: bool = False,
15991599
skip_udf: bool = False,
1600+
updated_since: Optional[datetime] = None,
16001601
):
16011602
"""
16021603
Args:
@@ -1618,6 +1619,10 @@ def _list_objects(
16181619

16191620
with self.read_engine.begin() as conn:
16201621
stmt = select(table).where(table.c.project_id == project)
1622+
if updated_since is not None:
1623+
stmt = stmt.where(
1624+
table.c.last_updated_timestamp >= int(updated_since.timestamp())
1625+
)
16211626
rows = conn.execute(stmt).all()
16221627
if rows:
16231628
objects = []

‎sdk/python/feast/protos/feast/registry/RegistryServer_pb2.py‎

Lines changed: 112 additions & 112 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

0 commit comments

Comments
 (0)