Skip to content

Commit d0c8984

Browse files
alan-gauthier-jtalan-gauthier-jt
authored
fix: Ambiguous truth value of array during materialization (#6259)
* fix: ambiguous truth value of array during materialization Signed-off-by: Alan Gauthier <[email protected]> * chore: refacto from pr feedback Signed-off-by: Alan Gauthier <[email protected]> --------- Signed-off-by: Alan Gauthier <[email protected]>
1 parent 9b185ee commit d0c8984

2 files changed

Lines changed: 171 additions & 14 deletions

File tree

‎sdk/python/feast/type_map.py‎

Lines changed: 63 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -909,6 +909,16 @@ def _convert_list_values_to_proto(
909909
]
910910

911911

912+
def _is_array_like(value: Any) -> bool:
913+
"""Return True if *value* is array-like (numpy array or any sized,
914+
non-string, non-bytes container). Array-like values in a scalar
915+
feature column cannot be mapped to a protobuf scalar field and are
916+
therefore always treated as null."""
917+
return isinstance(value, np.ndarray) or (
918+
hasattr(value, "__len__") and not isinstance(value, (str, bytes))
919+
)
920+
921+
912922
def _convert_scalar_values_to_proto(
913923
feast_value_type: ValueType,
914924
values: List[Any],
@@ -929,16 +939,34 @@ def _convert_scalar_values_to_proto(
929939
return [ProtoValue()] * len(values)
930940

931941
if feast_value_type == ValueType.UNIX_TIMESTAMP:
932-
int_timestamps = _python_datetime_to_int_timestamp(values)
933-
return [ProtoValue(unix_timestamp_val=ts) for ts in int_timestamps] # type: ignore
942+
out: List[Any] = [None] * len(values)
943+
clean_indices: List[int] = []
944+
clean_values: List[Any] = []
945+
for i, value in enumerate(values):
946+
if _is_array_like(value) or value is None:
947+
out[i] = ProtoValue()
948+
else:
949+
clean_indices.append(i)
950+
clean_values.append(value)
951+
if clean_values:
952+
timestamps = _python_datetime_to_int_timestamp(clean_values)
953+
for i, ts in zip(clean_indices, timestamps):
954+
out[i] = ProtoValue(unix_timestamp_val=ts) # type: ignore
955+
return out
934956

935957
field_name, func, valid_scalar_types = PYTHON_SCALAR_VALUE_TYPE_TO_PROTO_VALUE[
936958
feast_value_type
937959
]
938960

939-
# Validate scalar types
961+
# Validate scalar types. The caller guarantees that *sample* is not
962+
# array-like (array-like values are filtered out when picking the sample
963+
# for scalar columns in python_values_to_proto_values).
940964
if valid_scalar_types:
941-
if (sample == 0 or sample == 0.0) and feast_value_type != ValueType.BOOL:
965+
try:
966+
is_zero = sample == 0 or sample == 0.0
967+
except (ValueError, TypeError):
968+
is_zero = False
969+
if is_zero and feast_value_type != ValueType.BOOL:
942970
# Numpy converts 0 to int, but column type may be float
943971
allowed_types = {np.int64, int, np.float64, float, decimal.Decimal}
944972
assert type(sample) in allowed_types, (
@@ -951,20 +979,35 @@ def _convert_scalar_values_to_proto(
951979

952980
# Handle BOOL specially due to np.bool_ conversion requirement
953981
if feast_value_type == ValueType.BOOL:
954-
return [
955-
ProtoValue(
956-
**{field_name: func(bool(value) if type(value) is np.bool_ else value)}
957-
) # type: ignore
958-
if not pd.isnull(value)
959-
else ProtoValue()
960-
for value in values
961-
]
982+
out = []
983+
for value in values:
984+
if _is_array_like(value):
985+
# Array-like value in a scalar BOOL column: treat as null.
986+
out.append(ProtoValue())
987+
elif not pd.isnull(value):
988+
out.append(
989+
ProtoValue(
990+
**{
991+
field_name: func(
992+
bool(value) if type(value) is np.bool_ else value
993+
)
994+
}
995+
) # type: ignore
996+
)
997+
else:
998+
out.append(ProtoValue())
999+
return out
9621000

9631001
# Generic scalar conversion
9641002
out = []
9651003
for value in values:
9661004
if isinstance(value, ProtoValue):
9671005
out.append(value)
1006+
elif _is_array_like(value):
1007+
# Array-like value in a scalar column: always treat as null.
1008+
# pd.isnull() is vectorised and would return an ndarray here,
1009+
# making `not pd.isnull(value)` raise ValueError.
1010+
out.append(ProtoValue())
9681011
elif not pd.isnull(value):
9691012
out.append(ProtoValue(**{field_name: func(value)}))
9701013
else:
@@ -1107,12 +1150,18 @@ def _python_value_to_proto_value(
11071150
if "set" in type_name_lower:
11081151
return _python_set_to_proto_values(feast_value_type, values)
11091152

1110-
# Scalar types
1153+
# Scalar types — pick a sample that is not array-like so that the type
1154+
# validation in _convert_scalar_values_to_proto always receives a plain
1155+
# scalar (array-like values in a scalar column are treated as null).
11111156
if (
11121157
feast_value_type in PYTHON_SCALAR_VALUE_TYPE_TO_PROTO_VALUE
11131158
or feast_value_type == ValueType.UNIX_TIMESTAMP
11141159
):
1115-
return _convert_scalar_values_to_proto(feast_value_type, values, sample)
1160+
scalar_sample = next(
1161+
(v for v in values if _non_empty_value(v) and not _is_array_like(v)),
1162+
None,
1163+
)
1164+
return _convert_scalar_values_to_proto(feast_value_type, values, scalar_sample)
11161165

11171166
raise Exception(f"Unsupported data type: {feast_value_type}")
11181167

‎sdk/python/tests/unit/test_type_map.py‎

Lines changed: 108 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1845,3 +1845,111 @@ def test_pa_to_feast_value_type_nested(self):
18451845
pa_to_feast_value_type("list<item: list<item: double>>")
18461846
== ValueType.VALUE_LIST
18471847
)
1848+
1849+
1850+
class TestEmptyArrayAsNull:
1851+
"""Regression tests for https://github.com/feast-dev/feast/issues/6255
1852+
Ensure that an empty numpy array in a scalar feature column is treated as
1853+
null rather than raising ``ValueError: The truth value of an empty array is
1854+
ambiguous``.
1855+
"""
1856+
1857+
def test_empty_numpy_array_treated_as_null_double(self):
1858+
from feast.protos.feast.types.Value_pb2 import Value as ProtoValue
1859+
1860+
result = python_values_to_proto_values(
1861+
[np.array([]), 1.0, None], ValueType.DOUBLE
1862+
)
1863+
assert result[0] == ProtoValue(), (
1864+
"empty array should produce an empty ProtoValue"
1865+
)
1866+
assert result[1].double_val == 1.0
1867+
assert result[2] == ProtoValue(), (
1868+
"None should still produce an empty ProtoValue"
1869+
)
1870+
1871+
def test_empty_numpy_array_treated_as_null_int64(self):
1872+
from feast.protos.feast.types.Value_pb2 import Value as ProtoValue
1873+
1874+
result = python_values_to_proto_values(
1875+
[np.array([]), 42, None], ValueType.INT64
1876+
)
1877+
assert result[0] == ProtoValue(), (
1878+
"empty array should produce an empty ProtoValue"
1879+
)
1880+
assert result[1].int64_val == 42
1881+
assert result[2] == ProtoValue()
1882+
1883+
def test_empty_numpy_array_treated_as_null_bool(self):
1884+
from feast.protos.feast.types.Value_pb2 import Value as ProtoValue
1885+
1886+
result = python_values_to_proto_values(
1887+
[np.array([]), True, None], ValueType.BOOL
1888+
)
1889+
assert result[0] == ProtoValue(), (
1890+
"empty array should produce an empty ProtoValue"
1891+
)
1892+
assert result[1].bool_val is True
1893+
assert result[2] == ProtoValue()
1894+
1895+
def test_array_with_null_element_treated_as_null(self):
1896+
"""A non-empty array containing any null element in a scalar column is treated as null."""
1897+
from feast.protos.feast.types.Value_pb2 import Value as ProtoValue
1898+
1899+
result = python_values_to_proto_values(
1900+
[np.array([np.nan, 1.0]), 3.0], ValueType.DOUBLE
1901+
)
1902+
assert result[0] == ProtoValue(), (
1903+
"array with null element should produce an empty ProtoValue"
1904+
)
1905+
assert result[1].double_val == 3.0
1906+
1907+
def test_non_empty_array_without_nulls_is_treated_as_null(self):
1908+
"""A non-empty numpy array in a scalar column is always treated as null.
1909+
1910+
A scalar feature column cannot hold an ndarray value (protobuf would
1911+
reject it), so any array-like value – empty or not – is mapped to an
1912+
empty ProtoValue() rather than crashing with ValueError.
1913+
"""
1914+
from feast.protos.feast.types.Value_pb2 import Value as ProtoValue
1915+
1916+
result = python_values_to_proto_values(
1917+
[np.array([1.0, 2.0]), 3.0, None], ValueType.DOUBLE
1918+
)
1919+
# array-like value in a scalar column → null, not a crash
1920+
assert result[0] == ProtoValue(), (
1921+
"non-empty array in scalar column should be null"
1922+
)
1923+
assert result[1].double_val == 3.0
1924+
assert result[2] == ProtoValue()
1925+
1926+
def test_empty_numpy_array_treated_as_null_unix_timestamp(self):
1927+
"""Array-like values in a scalar UNIX_TIMESTAMP column must not crash."""
1928+
from datetime import datetime, timezone
1929+
1930+
from feast.protos.feast.types.Value_pb2 import Value as ProtoValue
1931+
1932+
ts = datetime(2024, 1, 1, tzinfo=timezone.utc)
1933+
result = python_values_to_proto_values(
1934+
[np.array([]), ts, None], ValueType.UNIX_TIMESTAMP
1935+
)
1936+
assert result[0] == ProtoValue(), (
1937+
"empty array in UNIX_TIMESTAMP scalar column should produce null"
1938+
)
1939+
assert result[1].unix_timestamp_val == int(ts.timestamp())
1940+
assert result[2] == ProtoValue()
1941+
1942+
def test_non_empty_array_treated_as_null_unix_timestamp(self):
1943+
"""Non-empty array in a UNIX_TIMESTAMP scalar column should produce null, not crash."""
1944+
from datetime import datetime, timezone
1945+
1946+
from feast.protos.feast.types.Value_pb2 import Value as ProtoValue
1947+
1948+
ts = datetime(2024, 6, 15, tzinfo=timezone.utc)
1949+
result = python_values_to_proto_values(
1950+
[np.array([1, 2, 3]), ts], ValueType.UNIX_TIMESTAMP
1951+
)
1952+
assert result[0] == ProtoValue(), (
1953+
"non-empty array in UNIX_TIMESTAMP scalar column should produce null"
1954+
)
1955+
assert result[1].unix_timestamp_val == int(ts.timestamp())

0 commit comments

Comments
 (0)