Skip to content

Commit c3d81ed

Browse files
committed
feat: add DynamoDB in-place list update support for array-based features
This PR adds support for in-place list updates (append/prepend) for array-based features in DynamoDB online store, addressing issue feast-dev#5687. Key changes: - Add update_online_store() and update_online_store_async() methods to FeatureStore for list operations on existing feature values - Implement read-modify-write pattern in DynamoDB store to handle list operations while maintaining compatibility with existing protobuf serialization format - Add comprehensive tests for list append, prepend, mixed operations, and new entity handling The implementation uses a read-modify-write approach because existing data is stored as serialized protobuf bytes, not native DynamoDB lists. This maintains backward compatibility with existing online_read methods.
1 parent d66358f commit c3d81ed

4 files changed

Lines changed: 935 additions & 1 deletion

File tree

‎sdk/python/feast/feature_store.py‎

Lines changed: 143 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2213,6 +2213,149 @@ async def write_to_online_store_async(
22132213
provider = self._get_provider()
22142214
await provider.ingest_df_async(feature_view, df)
22152215

2216+
def update_online_store(
2217+
self,
2218+
feature_view_name: str,
2219+
df: pd.DataFrame,
2220+
update_expressions: Dict[str, str],
2221+
allow_registry_cache: bool = True,
2222+
) -> None:
2223+
"""
2224+
Update features using DynamoDB-specific list operations.
2225+
2226+
This method provides efficient in-place list updates using DynamoDB's native
2227+
UpdateItem operations with list_append and other expressions. This is more
2228+
efficient than the read-modify-write pattern for array-based features.
2229+
2230+
Args:
2231+
feature_view_name: The feature view to update.
2232+
df: DataFrame with new values to append/prepend to existing lists.
2233+
update_expressions: Dict mapping feature names to DynamoDB update expressions.
2234+
Examples:
2235+
- {"transactions": "list_append(transactions, :new_val)"} # append
2236+
- {"recent_items": "list_append(:new_val, recent_items)"} # prepend
2237+
- {"sliding_window": "list_append(if_not_exists(sliding_window, :empty_list), :new_val)[:10]"} # sliding window
2238+
allow_registry_cache: Whether to allow cached registry.
2239+
2240+
Raises:
2241+
NotImplementedError: If online store doesn't support update expressions.
2242+
ValueError: If the feature view or update expressions are invalid.
2243+
2244+
Example:
2245+
# Append new transactions to existing transaction history
2246+
store.update_online_store(
2247+
feature_view_name="user_transactions",
2248+
df=new_transactions_df,
2249+
update_expressions={
2250+
"transaction_history": "list_append(transaction_history, :new_val)",
2251+
"recent_amounts": "list_append(:new_val, recent_amounts)[:10]"
2252+
}
2253+
)
2254+
"""
2255+
# Check if online store supports update expressions
2256+
provider = self._get_provider()
2257+
if not hasattr(provider.online_store, "update_online_store"):
2258+
raise NotImplementedError(
2259+
f"Online store {type(provider.online_store).__name__} "
2260+
"does not support update expressions. This feature is only available "
2261+
"with DynamoDB online store."
2262+
)
2263+
2264+
feature_view, df = self._get_feature_view_and_df_for_online_write(
2265+
feature_view_name=feature_view_name,
2266+
df=df,
2267+
allow_registry_cache=allow_registry_cache,
2268+
transform_on_write=False, # Don't transform for updates
2269+
)
2270+
2271+
# Validate that the dataframe has meaningful feature data
2272+
if df is not None:
2273+
if df.empty:
2274+
warnings.warn("Cannot update with empty dataframe")
2275+
return
2276+
2277+
# Check if feature columns are empty
2278+
feature_column_names = [f.name for f in feature_view.features]
2279+
if feature_column_names:
2280+
feature_df = df[feature_column_names]
2281+
if feature_df.empty or feature_df.isnull().all().all():
2282+
warnings.warn("Cannot update with empty feature columns")
2283+
return
2284+
2285+
# Prepare data for online store
2286+
from feast.infra.passthrough_provider import PassthroughProvider
2287+
2288+
rows_to_write = PassthroughProvider._prep_rows_to_write_for_ingestion(
2289+
feature_view=feature_view,
2290+
df=df,
2291+
)
2292+
2293+
# Call DynamoDB-specific method
2294+
provider.online_store.update_online_store(
2295+
config=self.config,
2296+
table=feature_view,
2297+
data=rows_to_write,
2298+
update_expressions=update_expressions,
2299+
progress=None,
2300+
)
2301+
2302+
async def update_online_store_async(
2303+
self,
2304+
feature_view_name: str,
2305+
df: pd.DataFrame,
2306+
update_expressions: Dict[str, str],
2307+
allow_registry_cache: bool = True,
2308+
) -> None:
2309+
"""
2310+
Async version of update_online_store.
2311+
"""
2312+
# Check if online store supports update expressions
2313+
provider = self._get_provider()
2314+
if not hasattr(provider.online_store, "update_online_store_async"):
2315+
raise NotImplementedError(
2316+
f"Online store {type(provider.online_store).__name__} "
2317+
"does not support async update expressions. This feature is only available "
2318+
"with DynamoDB online store."
2319+
)
2320+
2321+
feature_view, df = self._get_feature_view_and_df_for_online_write(
2322+
feature_view_name=feature_view_name,
2323+
df=df,
2324+
allow_registry_cache=allow_registry_cache,
2325+
transform_on_write=False, # Don't transform for updates
2326+
)
2327+
2328+
# Validate that the dataframe has meaningful feature data
2329+
if df is not None:
2330+
if df.empty:
2331+
warnings.warn("Cannot update with empty dataframe")
2332+
return
2333+
2334+
# Check if feature columns are empty
2335+
feature_column_names = [f.name for f in feature_view.features]
2336+
if feature_column_names:
2337+
feature_df = df[feature_column_names]
2338+
if feature_df.empty or feature_df.isnull().all().all():
2339+
warnings.warn("Cannot update with empty feature columns")
2340+
return
2341+
2342+
# Prepare data for online store
2343+
from feast.infra.passthrough_provider import PassthroughProvider
2344+
2345+
rows_to_write = PassthroughProvider._prep_rows_to_write_for_ingestion(
2346+
feature_view=feature_view,
2347+
df=df,
2348+
)
2349+
2350+
# Call DynamoDB-specific async method
2351+
await provider.online_store.update_online_store_async(
2352+
config=self.config,
2353+
table=feature_view,
2354+
data=rows_to_write,
2355+
update_expressions=update_expressions,
2356+
progress=None,
2357+
)
2358+
22162359
def write_to_offline_store(
22172360
self,
22182361
feature_view_name: str,

0 commit comments

Comments
 (0)