Skip to content

Commit f2829bd

Browse files
Mrun25ntkathole
Mrun25
authored and
ntkathole
committed
fix: infer Feast types for additional Arrow types and report column name on inference failure (feast-dev#7040)
Signed-off-by: Mrun25 <[email protected]>
1 parent ce02644 commit f2829bd

4 files changed

Lines changed: 72 additions & 8 deletions

File tree

‎sdk/python/feast/inference.py‎

Lines changed: 19 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,19 @@
2121
from feast.value_type import ValueType
2222

2323

24+
def _infer_column_value_type(
25+
source: DataSource, col_name: str, col_datatype: str
26+
) -> ValueType:
27+
"""Map a source column type to a Feast ValueType, naming the column on failure."""
28+
try:
29+
return source.source_datatype_to_feast_value_type()(col_datatype)
30+
except ValueError as e:
31+
raise RegistryInferenceFailure(
32+
"feature view",
33+
f"column '{col_name}' has an unsupported type: {e}",
34+
) from e
35+
36+
2437
def update_data_sources_with_inferred_event_timestamp_col(
2538
data_sources: List[DataSource], config: RepoConfig
2639
) -> None:
@@ -247,7 +260,7 @@ def _infer_features_and_entities(
247260
field = Field(
248261
name=col_name,
249262
dtype=from_value_type(
250-
fv.batch_source.source_datatype_to_feast_value_type()(col_datatype)
263+
_infer_column_value_type(fv.batch_source, col_name, col_datatype)
251264
),
252265
)
253266
if field.name not in [
@@ -266,8 +279,8 @@ def _infer_features_and_entities(
266279
field = Field(
267280
name=feature_name,
268281
dtype=from_value_type(
269-
fv.batch_source.source_datatype_to_feast_value_type()(
270-
col_datatype
282+
_infer_column_value_type(
283+
fv.batch_source, col_name, col_datatype
271284
)
272285
),
273286
)
@@ -322,7 +335,7 @@ def _infer_on_demand_features_and_entities(
322335
field = Field(
323336
name=col_name,
324337
dtype=from_value_type(
325-
batch_source.source_datatype_to_feast_value_type()(col_datatype)
338+
_infer_column_value_type(batch_source, col_name, col_datatype)
326339
),
327340
)
328341
if field.name not in [
@@ -343,8 +356,8 @@ def _infer_on_demand_features_and_entities(
343356
field = Field(
344357
name=feature_name,
345358
dtype=from_value_type(
346-
batch_source.source_datatype_to_feast_value_type()(
347-
col_datatype
359+
_infer_column_value_type(
360+
batch_source, col_name, col_datatype
348361
)
349362
),
350363
)

‎sdk/python/feast/infra/offline_stores/offline_utils.py‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -297,7 +297,10 @@ def get_pyarrow_schema_from_batch_source(
297297
pa_schema = []
298298
column_names = []
299299
for column_name, column_type in column_names_and_types:
300-
value_type = batch_source.source_datatype_to_feast_value_type()(column_type)
300+
try:
301+
value_type = batch_source.source_datatype_to_feast_value_type()(column_type)
302+
except ValueError as e:
303+
raise ValueError(f"column '{column_name}': {e}") from e
301304
if value_type in (ValueType.VALUE_LIST, ValueType.VALUE_SET):
302305
pa_type = _parse_pa_type_str(column_type)
303306
else:

‎sdk/python/feast/type_map.py‎

Lines changed: 22 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1653,10 +1653,26 @@ def pa_to_feast_value_type(pa_type_as_str: str) -> ValueType:
16531653

16541654
if pa_type_as_str.startswith("timestamp"):
16551655
value_type = ValueType.UNIX_TIMESTAMP
1656+
elif pa_type_as_str.startswith("date32") or pa_type_as_str.startswith("date64"):
1657+
# Mirror the DataFrame inference path, which maps "date" to UNIX_TIMESTAMP.
1658+
value_type = ValueType.UNIX_TIMESTAMP
16561659
elif pa_type_as_str.startswith("map<"):
16571660
value_type = ValueType.MAP
16581661
elif pa_type_as_str == "large_string":
16591662
value_type = ValueType.JSON
1663+
elif pa_type_as_str == "large_binary":
1664+
value_type = ValueType.BYTES
1665+
elif pa_type_as_str == "halffloat":
1666+
value_type = ValueType.FLOAT
1667+
elif pa_type_as_str.startswith("decimal128") or pa_type_as_str.startswith(
1668+
"decimal256"
1669+
):
1670+
# Mirror the DataFrame inference path, which maps "decimal" to DECIMAL.
1671+
value_type = ValueType.DECIMAL
1672+
elif pa_type_as_str.startswith("dictionary<"):
1673+
# pandas writes "category" columns as Arrow dictionaries; the DataFrame
1674+
# inference path maps "category" to STRING.
1675+
value_type = ValueType.STRING
16601676
elif pa_type_as_str.startswith("struct<") or pa_type_as_str.startswith("struct{"):
16611677
value_type = ValueType.STRUCT
16621678
else:
@@ -1683,7 +1699,12 @@ def pa_to_feast_value_type(pa_type_as_str: str) -> ValueType:
16831699
"list<element: bytes>": ValueType.BYTES_LIST,
16841700
"list<element: float>": ValueType.FLOAT_LIST,
16851701
}
1686-
value_type = type_map[pa_type_as_str]
1702+
value_type = type_map.get(pa_type_as_str)
1703+
if value_type is None:
1704+
raise ValueError(
1705+
f"Feast cannot infer a value type for the Arrow type '{pa_type_as_str}'. "
1706+
"Declare the feature view schema explicitly, or extend the Arrow type mapping."
1707+
)
16871708

16881709
if is_list:
16891710
value_type = ValueType[value_type.name + "_LIST"]

‎sdk/python/tests/unit/test_type_map.py‎

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2742,3 +2742,30 @@ def test_pandas_and_arrow_int_inference_agree(dtype):
27422742
)
27432743
from_arrow = pa_to_feast_value_type(str(pyarrow.array(values).type))
27442744
assert from_pandas == from_arrow
2745+
2746+
2747+
@pytest.mark.parametrize(
2748+
"pa_type, expected",
2749+
[
2750+
("date32[day]", ValueType.UNIX_TIMESTAMP),
2751+
("date64[ms]", ValueType.UNIX_TIMESTAMP),
2752+
("decimal128(3, 2)", ValueType.DECIMAL),
2753+
("decimal256(10, 4)", ValueType.DECIMAL),
2754+
(
2755+
"dictionary<values=string, indices=int8, ordered=0>",
2756+
ValueType.STRING,
2757+
),
2758+
("halffloat", ValueType.FLOAT),
2759+
("large_binary", ValueType.BYTES),
2760+
],
2761+
)
2762+
def test_pa_to_feast_value_type_pandas_written_parquet_types(pa_type, expected):
2763+
"""Arrow types written by pandas to Parquet infer the same Feast type the
2764+
DataFrame inference path produces (issue #7040)."""
2765+
assert pa_to_feast_value_type(pa_type) == expected
2766+
2767+
2768+
def test_pa_to_feast_value_type_unsupported_type_raises_descriptive_error():
2769+
"""An unmapped Arrow type raises a descriptive error, not a bare KeyError."""
2770+
with pytest.raises(ValueError, match=r"time32\[s\]"):
2771+
pa_to_feast_value_type("time32[s]")

0 commit comments

Comments
 (0)