Skip to content

Commit e0e51f7

Browse files
patelchaitanyntkathole
authored andcommitted
fix: Fail Ray materialization when the online store write fails
When the online store refused a write, the Ray engine only logged the error and still reported the job as SUCCEEDED. Users kept getting old feature values and had no sign that anything went wrong. Now the error is raised again after it is logged, and the write step is no longer wrapped in safe_batch_processor, which hid every error. A failed write now fails the job, the same as with the Local engine. Fixes feast-dev#7024 Signed-off-by: Chaitany Patel <[email protected]>
1 parent 68c5d54 commit e0e51f7

4 files changed

Lines changed: 47 additions & 2 deletions

File tree

‎sdk/python/feast/infra/compute_engines/ray/nodes.py‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@
2020
from feast.infra.compute_engines.dag.value import DAGValue
2121
from feast.infra.compute_engines.ray.config import RayComputeEngineConfig
2222
from feast.infra.compute_engines.ray.utils import (
23+
_is_empty_batch,
2324
safe_batch_processor,
2425
write_to_online_store,
2526
)
@@ -861,9 +862,12 @@ def execute(self, context: ExecutionContext) -> DAGValue:
861862
feature_view=self.feature_view, repo_config=context.repo_config
862863
)
863864

864-
@safe_batch_processor
865+
# Not wrapped in safe_batch_processor: a failed write must fail the job.
865866
def write_batch_with_serialized_artifacts(batch: pd.DataFrame) -> pd.DataFrame:
866867
"""Write each batch using pre-serialized artifacts."""
868+
if _is_empty_batch(batch):
869+
return batch
870+
867871
(
868872
feature_view,
869873
online_store,

‎sdk/python/feast/infra/compute_engines/ray/utils.py‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -81,6 +81,7 @@ def write_to_online_store(
8181

8282
except Exception as e:
8383
logger.error(f"Failed to write to online store for {feature_view.name}: {e}")
84+
raise
8485

8586

8687
# Ray Data batch type: pandas DataFrame, numpy dict, or pyarrow Table
@@ -178,6 +179,7 @@ def _write_batch(batch: pa.Table) -> pa.Table:
178179
logger.error(
179180
f"Distributed write to online store failed for {feature_view.name}: {e}"
180181
)
182+
raise
181183

182184

183185
def safe_batch_processor(

‎sdk/python/tests/component/ray/test_compute.py‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -151,7 +151,10 @@ def tqdm_builder(length):
151151
ray_materialize_jobs = engine.materialize(registry, task)
152152

153153
assert len(ray_materialize_jobs) == 1
154-
assert ray_materialize_jobs[0].status() == MaterializationJobStatus.SUCCEEDED
154+
# The Ray engine cannot yet write an aggregated BatchFeatureView to the
155+
# online store (see the TODO in feast.utils._convert_arrow_to_proto), and
156+
# that write error must fail the job instead of being swallowed.
157+
assert ray_materialize_jobs[0].status() == MaterializationJobStatus.ERROR
155158

156159

157160
@pytest.mark.integration

‎sdk/python/tests/component/ray/test_nodes.py‎

Lines changed: 36 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@
66
import pytest
77
import ray
88

9+
from feast import Entity, FeatureView, Field, FileSource, RepoConfig
910
from feast.aggregation import Aggregation
1011
from feast.infra.compute_engines.dag.context import ColumnInfo
1112
from feast.infra.compute_engines.dag.model import DAGFormat
@@ -19,13 +20,16 @@
1920
RayJoinNode,
2021
RayReadNode,
2122
RayTransformationNode,
23+
RayWriteNode,
2224
)
25+
from feast.infra.online_stores.sqlite import SqliteOnlineStoreConfig
2326
from feast.infra.ray_initializer import (
2427
RayConfigManager,
2528
RayExecutionMode,
2629
ensure_ray_initialized,
2730
get_ray_wrapper,
2831
)
32+
from feast.types import Float32
2933

3034

3135
class DummyInputNode(DAGNode):
@@ -439,6 +443,38 @@ def test_ray_dedup_node_materialization_no_join_keys(
439443
)
440444

441445

446+
def test_ray_write_node_raises_when_online_write_fails(
447+
ray_session, mock_context, sample_data, tmp_path
448+
):
449+
"""A failed online store write must fail the write node, not be swallowed."""
450+
driver = Entity(name="driver", join_keys=["driver_id"])
451+
feature_view = FeatureView(
452+
name="driver_stats",
453+
entities=[driver],
454+
schema=[Field(name="conv_rate", dtype=Float32)],
455+
source=FileSource(
456+
path=str(tmp_path / "source.parquet"), timestamp_field="event_timestamp"
457+
),
458+
online=True,
459+
offline=False,
460+
)
461+
# The online table was never created, so every write raises "no such table".
462+
mock_context.repo_config = RepoConfig(
463+
project="test_project",
464+
registry=str(tmp_path / "registry.db"),
465+
provider="local",
466+
entity_key_serialization_version=3,
467+
online_store=SqliteOnlineStoreConfig(path=str(tmp_path / "online.db")),
468+
)
469+
input_value = DAGValue(data=ray.data.from_pandas(sample_data), format=DAGFormat.RAY)
470+
node = RayWriteNode(name="write", feature_view=feature_view)
471+
node.add_input(DummyInputNode("input_node", input_value))
472+
mock_context.node_outputs = {"input_node": input_value}
473+
474+
with pytest.raises(Exception, match="no such table"):
475+
node.execute(mock_context)
476+
477+
442478
def test_ray_config_validation():
443479
"""Test Ray configuration validation."""
444480
# Test valid configuration

0 commit comments

Comments
 (0)