Skip to content

Commit aca9589

Browse files
committed
feat: add DynamoDB in-place list update support for array-based features
This PR adds support for in-place list updates (append/prepend) for array-based features in DynamoDB online store, addressing issue feast-dev#5687. Key changes: - Add update_online_store() and update_online_store_async() methods to FeatureStore for list operations on existing feature values - Implement read-modify-write pattern in DynamoDB store to handle list operations while maintaining compatibility with existing protobuf serialization format - Add comprehensive tests for list append, prepend, mixed operations, and new entity handling The implementation uses a read-modify-write approach because existing data is stored as serialized protobuf bytes, not native DynamoDB lists. This maintains backward compatibility with existing online_read methods. Signed-off-by: Anshi Shrivastava <[email protected]>
1 parent b37b7d0 commit aca9589

4 files changed

Lines changed: 933 additions & 1 deletion

File tree

‎sdk/python/feast/feature_store.py‎

Lines changed: 142 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2253,6 +2253,148 @@ async def write_to_online_store_async(
22532253
provider = self._get_provider()
22542254
await provider.ingest_df_async(feature_view, df)
22552255

2256+
def update_online_store(
2257+
self,
2258+
feature_view_name: str,
2259+
df: pd.DataFrame,
2260+
update_expressions: Dict[str, str],
2261+
allow_registry_cache: bool = True,
2262+
) -> None:
2263+
"""
2264+
Update features using DynamoDB-specific list operations.
2265+
2266+
This method provides efficient in-place list updates using DynamoDB's native
2267+
UpdateItem operations with list_append and other expressions. This is more
2268+
efficient than the read-modify-write pattern for array-based features.
2269+
2270+
Args:
2271+
feature_view_name: The feature view to update.
2272+
df: DataFrame with new values to append/prepend to existing lists.
2273+
update_expressions: Dict mapping feature names to DynamoDB update expressions.
2274+
Examples:
2275+
- {"transactions": "list_append(transactions, :new_val)"} # append
2276+
- {"recent_items": "list_append(:new_val, recent_items)"} # prepend
2277+
allow_registry_cache: Whether to allow cached registry.
2278+
2279+
Raises:
2280+
NotImplementedError: If online store doesn't support update expressions.
2281+
ValueError: If the feature view or update expressions are invalid.
2282+
2283+
Example:
2284+
# Append new transactions to existing transaction history
2285+
store.update_online_store(
2286+
feature_view_name="user_transactions",
2287+
df=new_transactions_df,
2288+
update_expressions={
2289+
"transaction_history": "list_append(transaction_history, :new_val)",
2290+
"recent_amounts": "list_append(:new_val, recent_amounts)" # prepend
2291+
}
2292+
)
2293+
"""
2294+
# Check if online store supports update expressions
2295+
provider = self._get_provider()
2296+
if not hasattr(provider.online_store, "update_online_store"):
2297+
raise NotImplementedError(
2298+
f"Online store {type(provider.online_store).__name__} "
2299+
"does not support update expressions. This feature is only available "
2300+
"with DynamoDB online store."
2301+
)
2302+
2303+
feature_view, df = self._get_feature_view_and_df_for_online_write(
2304+
feature_view_name=feature_view_name,
2305+
df=df,
2306+
allow_registry_cache=allow_registry_cache,
2307+
transform_on_write=False, # Don't transform for updates
2308+
)
2309+
2310+
# Validate that the dataframe has meaningful feature data
2311+
if df is not None:
2312+
if df.empty:
2313+
warnings.warn("Cannot update with empty dataframe")
2314+
return
2315+
2316+
# Check if feature columns are empty
2317+
feature_column_names = [f.name for f in feature_view.features]
2318+
if feature_column_names:
2319+
feature_df = df[feature_column_names]
2320+
if feature_df.empty or feature_df.isnull().all().all():
2321+
warnings.warn("Cannot update with empty feature columns")
2322+
return
2323+
2324+
# Prepare data for online store
2325+
from feast.infra.passthrough_provider import PassthroughProvider
2326+
2327+
rows_to_write = PassthroughProvider._prep_rows_to_write_for_ingestion(
2328+
feature_view=feature_view,
2329+
df=df,
2330+
)
2331+
2332+
# Call DynamoDB-specific method
2333+
provider.online_store.update_online_store(
2334+
config=self.config,
2335+
table=feature_view,
2336+
data=rows_to_write,
2337+
update_expressions=update_expressions,
2338+
progress=None,
2339+
)
2340+
2341+
async def update_online_store_async(
2342+
self,
2343+
feature_view_name: str,
2344+
df: pd.DataFrame,
2345+
update_expressions: Dict[str, str],
2346+
allow_registry_cache: bool = True,
2347+
) -> None:
2348+
"""
2349+
Async version of update_online_store.
2350+
"""
2351+
# Check if online store supports update expressions
2352+
provider = self._get_provider()
2353+
if not hasattr(provider.online_store, "update_online_store_async"):
2354+
raise NotImplementedError(
2355+
f"Online store {type(provider.online_store).__name__} "
2356+
"does not support async update expressions. This feature is only available "
2357+
"with DynamoDB online store."
2358+
)
2359+
2360+
feature_view, df = self._get_feature_view_and_df_for_online_write(
2361+
feature_view_name=feature_view_name,
2362+
df=df,
2363+
allow_registry_cache=allow_registry_cache,
2364+
transform_on_write=False, # Don't transform for updates
2365+
)
2366+
2367+
# Validate that the dataframe has meaningful feature data
2368+
if df is not None:
2369+
if df.empty:
2370+
warnings.warn("Cannot update with empty dataframe")
2371+
return
2372+
2373+
# Check if feature columns are empty
2374+
feature_column_names = [f.name for f in feature_view.features]
2375+
if feature_column_names:
2376+
feature_df = df[feature_column_names]
2377+
if feature_df.empty or feature_df.isnull().all().all():
2378+
warnings.warn("Cannot update with empty feature columns")
2379+
return
2380+
2381+
# Prepare data for online store
2382+
from feast.infra.passthrough_provider import PassthroughProvider
2383+
2384+
rows_to_write = PassthroughProvider._prep_rows_to_write_for_ingestion(
2385+
feature_view=feature_view,
2386+
df=df,
2387+
)
2388+
2389+
# Call DynamoDB-specific async method
2390+
await provider.online_store.update_online_store_async(
2391+
config=self.config,
2392+
table=feature_view,
2393+
data=rows_to_write,
2394+
update_expressions=update_expressions,
2395+
progress=None,
2396+
)
2397+
22562398
def write_to_offline_store(
22572399
self,
22582400
feature_view_name: str,

0 commit comments

Comments
 (0)