Skip to content

Commit b4d932a

Browse files
committed
fix: Address review feedback on vector length validation
Three findings from review of #6909. Tolerate genuine nulls on the DataFrame path (@ntkathole). lengths.isna() was true both for values that were not sequences and for rows that were genuinely null, so a null vector was reported as "not a sequence". Nulls are now skipped, which also makes this path consistent with the Arrow path, which already tolerated null rows. Report the first offending row across both failure modes (Copilot). Checking the not-a-sequence mask fully before the length mask meant a non-sequence in a later row was reported ahead of a wrong length in an earlier one. The masks are now unioned and the first offending position selected before choosing the message, which restores the row ordering the original iterrows loop had. Value lookup is positional so a DataFrame with a duplicate index reports a scalar type rather than a Series. Validate the on-demand branch too (Copilot). _convert_arrow_to_proto dispatches to _convert_arrow_odfv_to_proto or _convert_arrow_fv_to_proto, and the check sat in the regular branch only, so a stored ODFV declaring vector_length had its transformed output unchecked. The call moved up into the dispatcher, ahead of the branch, so both paths are covered. Each finding has a test that fails against the previous commit and passes here. Signed-off-by: hao-xu5 <[email protected]>
1 parent 5e66d6e commit b4d932a

3 files changed

Lines changed: 113 additions & 18 deletions

File tree

‎sdk/python/feast/feature_store.py‎

Lines changed: 30 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -3640,25 +3640,41 @@ def _validate_vector_features(self, feature_view, df: pd.DataFrame) -> None:
36403640
expected = vector_field.vector_length
36413641
column = df[name]
36423642

3643-
# Single pass over the column. Sequences report their length, anything else
3644-
# reports None so the two failure modes stay distinguishable.
3645-
lengths = column.map(lambda v: len(v) if hasattr(v, "__len__") else None)
3643+
# Null vectors carry no length to compare. Skipping them keeps this
3644+
# consistent with the Arrow path, which also tolerates null rows, and
3645+
# stops a genuine null being reported as "not a sequence".
3646+
is_null = column.isna()
3647+
3648+
# Single pass over the column. na_action leaves nulls as NaN, so a NaN
3649+
# length that is not null means the value was not a sequence at all.
3650+
lengths = column.map(
3651+
lambda v: len(v) if hasattr(v, "__len__") else None, na_action="ignore"
3652+
)
36463653

3647-
not_a_sequence = lengths.isna()
3648-
if not_a_sequence.any():
3649-
i = not_a_sequence.idxmax()
3650-
raise ValueError(
3651-
f"Row {i}: Vector feature '{name}' is not a sequence. Got: {type(column[i])}"
3652-
)
3654+
not_a_sequence = lengths.isna() & ~is_null
3655+
mismatched = lengths.notna() & (lengths != expected)
3656+
offending = not_a_sequence | mismatched
3657+
if not offending.any():
3658+
return
36533659

3654-
mismatched = lengths != expected
3655-
if mismatched.any():
3656-
i = mismatched.idxmax()
3660+
# Report the first offending row across both failure modes, by position,
3661+
# so the message is right regardless of which failure comes first and
3662+
# correct even when the index has duplicates.
3663+
position = int(offending.to_numpy().argmax())
3664+
label = column.index[position]
3665+
3666+
if bool(not_a_sequence.to_numpy()[position]):
36573667
raise ValueError(
3658-
f"Row {i}: Vector length {lengths[i]} does not match expected {expected} "
3659-
f"for feature '{name}' in feature view '{feature_view.name}'."
3668+
f"Row {label}: Vector feature '{name}' is not a sequence. "
3669+
f"Got: {type(column.iloc[position])}"
36603670
)
36613671

3672+
raise ValueError(
3673+
f"Row {label}: Vector length {int(lengths.iloc[position])} does not match "
3674+
f"expected {expected} for feature '{name}' in feature view "
3675+
f"'{feature_view.name}'."
3676+
)
3677+
36623678
def _get_feature_view_and_df_for_online_write(
36633679
self,
36643680
feature_view_name: str,

‎sdk/python/feast/utils.py‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -502,6 +502,9 @@ def _convert_arrow_to_proto(
502502
) -> List[Tuple[EntityKeyProto, Dict[str, ValueProto], datetime, Optional[datetime]]]:
503503
# This is a workaround for isinstance(feature_view, OnDemandFeatureView), which triggers a circular import
504504
# Check for source_request_sources or source_feature_view_projections attributes to identify ODFVs
505+
# Validate before branching so both regular and on-demand feature views are covered.
506+
_validate_vector_field_lengths(table, feature_view)
507+
505508
if (
506509
getattr(feature_view, "source_request_sources", None) is not None
507510
or getattr(feature_view, "source_feature_view_projections", None) is not None
@@ -525,8 +528,6 @@ def _convert_arrow_fv_to_proto(
525528
f"Feature view '{feature_view.name}' has no batch_source and cannot be converted to proto."
526529
)
527530

528-
_validate_vector_field_lengths(table, feature_view)
529-
530531
# TODO: This will break if the feature view has aggregations or transformations
531532
columns = [
532533
(field.name, field.dtype.to_value_type()) for field in feature_view.features

‎sdk/python/tests/unit/test_vector_length_validation.py‎

Lines changed: 80 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -11,10 +11,10 @@
1111
import pyarrow
1212
import pytest
1313

14-
from feast import Entity, FeatureView, Field, FileSource
14+
from feast import Entity, FeatureView, Field, FileSource, RequestSource
1515
from feast.feature_store import FeatureStore
1616
from feast.types import Array, Float32, String
17-
from feast.utils import _validate_vector_field_lengths
17+
from feast.utils import _convert_arrow_to_proto, _validate_vector_field_lengths
1818
from feast.value_type import ValueType
1919

2020
VECTOR_LENGTH = 4
@@ -172,3 +172,81 @@ def test_numpy_vectors_are_accepted(self):
172172
}
173173
)
174174
_validate_df(_feature_view(), df)
175+
176+
177+
class TestReviewFeedback:
178+
"""Cases raised in review of #6909."""
179+
180+
def test_null_vectors_are_tolerated_on_the_dataframe_path(self):
181+
"""A genuine null has no length, and must not be called 'not a sequence'.
182+
183+
Matches the Arrow path, which already tolerates null rows.
184+
"""
185+
df = pd.DataFrame(
186+
{
187+
"embedding": [[0.0] * VECTOR_LENGTH, None, np.nan],
188+
"label": list("abc"),
189+
}
190+
)
191+
_validate_df(_feature_view(), df)
192+
193+
def test_first_offending_row_wins_across_both_failure_modes(self):
194+
"""A bad length in row 0 outranks a non-sequence in row 1."""
195+
df = pd.DataFrame(
196+
{"embedding": [[0.0] * 9, 3.14, [0.0] * 4], "label": list("abc")}
197+
)
198+
with pytest.raises(ValueError, match="Row 0: Vector length 9"):
199+
_validate_df(_feature_view(), df)
200+
201+
def test_non_sequence_wins_when_it_comes_first(self):
202+
df = pd.DataFrame(
203+
{"embedding": [3.14, [0.0] * 9, [0.0] * 4], "label": list("abc")}
204+
)
205+
with pytest.raises(
206+
ValueError, match="Row 0: Vector feature .* is not a sequence"
207+
):
208+
_validate_df(_feature_view(), df)
209+
210+
def test_duplicate_index_reports_a_scalar_type_not_a_series(self):
211+
"""Positional access keeps a duplicate index from yielding a Series."""
212+
df = pd.DataFrame(
213+
{"embedding": [[0.0] * 4, 3.14, [0.0] * 4], "label": list("abc")},
214+
index=[7, 7, 7],
215+
)
216+
with pytest.raises(ValueError, match=r"Got: <class 'float'>"):
217+
_validate_df(_feature_view(), df)
218+
219+
def test_on_demand_feature_view_output_is_validated(self):
220+
"""_convert_arrow_to_proto must validate before it branches to the ODFV path."""
221+
from feast.on_demand_feature_view import on_demand_feature_view
222+
223+
request_source = RequestSource(
224+
name="req",
225+
schema=[Field(name="raw", dtype=String)],
226+
)
227+
228+
@on_demand_feature_view(
229+
sources=[request_source],
230+
schema=[
231+
Field(
232+
name="embedding",
233+
dtype=Array(Float32),
234+
vector_index=True,
235+
vector_length=VECTOR_LENGTH,
236+
)
237+
],
238+
mode="python",
239+
write_to_online_store=True,
240+
)
241+
def odfv(inputs):
242+
return {"embedding": inputs["raw"]}
243+
244+
# Width 8 against a declared 4 must be rejected on the ODFV branch too.
245+
table = pyarrow.table(
246+
{
247+
"embedding": _fixed_size_list(8, num_rows=2),
248+
"req": pyarrow.array(["a", "b"]),
249+
}
250+
)
251+
with pytest.raises(ValueError, match="does not match expected 4"):
252+
_convert_arrow_to_proto(table, odfv, {})

0 commit comments

Comments
 (0)