Skip to content

Commit 2f38847

Browse files
committed
fix: infer Feast types for additional Arrow types and report column name on inference failure (feast-dev#7040)
Signed-off-by: Mrun25 <[email protected]>
1 parent 7903e2e commit 2f38847

4 files changed

Lines changed: 72 additions & 8 deletions

File tree

‎sdk/python/feast/inference.py‎

Lines changed: 19 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,19 @@
2121
from feast.value_type import ValueType
2222

2323

24+
def _infer_column_value_type(
25+
source: DataSource, col_name: str, col_datatype: str
26+
) -> ValueType:
27+
"""Map a source column type to a Feast ValueType, naming the column on failure."""
28+
try:
29+
return source.source_datatype_to_feast_value_type()(col_datatype)
30+
except ValueError as e:
31+
raise RegistryInferenceFailure(
32+
"feature view",
33+
f"column '{col_name}' has an unsupported type: {e}",
34+
) from e
35+
36+
2437
def update_data_sources_with_inferred_event_timestamp_col(
2538
data_sources: List[DataSource], config: RepoConfig
2639
) -> None:
@@ -247,7 +260,7 @@ def _infer_features_and_entities(
247260
field = Field(
248261
name=col_name,
249262
dtype=from_value_type(
250-
fv.batch_source.source_datatype_to_feast_value_type()(col_datatype)
263+
_infer_column_value_type(fv.batch_source, col_name, col_datatype)
251264
),
252265
)
253266
if field.name not in [
@@ -266,8 +279,8 @@ def _infer_features_and_entities(
266279
field = Field(
267280
name=feature_name,
268281
dtype=from_value_type(
269-
fv.batch_source.source_datatype_to_feast_value_type()(
270-
col_datatype
282+
_infer_column_value_type(
283+
fv.batch_source, col_name, col_datatype
271284
)
272285
),
273286
)
@@ -322,7 +335,7 @@ def _infer_on_demand_features_and_entities(
322335
field = Field(
323336
name=col_name,
324337
dtype=from_value_type(
325-
batch_source.source_datatype_to_feast_value_type()(col_datatype)
338+
_infer_column_value_type(batch_source, col_name, col_datatype)
326339
),
327340
)
328341
if field.name not in [
@@ -343,8 +356,8 @@ def _infer_on_demand_features_and_entities(
343356
field = Field(
344357
name=feature_name,
345358
dtype=from_value_type(
346-
batch_source.source_datatype_to_feast_value_type()(
347-
col_datatype
359+
_infer_column_value_type(
360+
batch_source, col_name, col_datatype
348361
)
349362
),
350363
)

‎sdk/python/feast/infra/offline_stores/offline_utils.py‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -297,7 +297,10 @@ def get_pyarrow_schema_from_batch_source(
297297
pa_schema = []
298298
column_names = []
299299
for column_name, column_type in column_names_and_types:
300-
value_type = batch_source.source_datatype_to_feast_value_type()(column_type)
300+
try:
301+
value_type = batch_source.source_datatype_to_feast_value_type()(column_type)
302+
except ValueError as e:
303+
raise ValueError(f"column '{column_name}': {e}") from e
301304
if value_type in (ValueType.VALUE_LIST, ValueType.VALUE_SET):
302305
pa_type = _parse_pa_type_str(column_type)
303306
else:

‎sdk/python/feast/type_map.py‎

Lines changed: 22 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1628,10 +1628,26 @@ def pa_to_feast_value_type(pa_type_as_str: str) -> ValueType:
16281628

16291629
if pa_type_as_str.startswith("timestamp"):
16301630
value_type = ValueType.UNIX_TIMESTAMP
1631+
elif pa_type_as_str.startswith("date32") or pa_type_as_str.startswith("date64"):
1632+
# Mirror the DataFrame inference path, which maps "date" to UNIX_TIMESTAMP.
1633+
value_type = ValueType.UNIX_TIMESTAMP
16311634
elif pa_type_as_str.startswith("map<"):
16321635
value_type = ValueType.MAP
16331636
elif pa_type_as_str == "large_string":
16341637
value_type = ValueType.JSON
1638+
elif pa_type_as_str == "large_binary":
1639+
value_type = ValueType.BYTES
1640+
elif pa_type_as_str == "halffloat":
1641+
value_type = ValueType.FLOAT
1642+
elif pa_type_as_str.startswith("decimal128") or pa_type_as_str.startswith(
1643+
"decimal256"
1644+
):
1645+
# Mirror the DataFrame inference path, which maps "decimal" to DECIMAL.
1646+
value_type = ValueType.DECIMAL
1647+
elif pa_type_as_str.startswith("dictionary<"):
1648+
# pandas writes "category" columns as Arrow dictionaries; the DataFrame
1649+
# inference path maps "category" to STRING.
1650+
value_type = ValueType.STRING
16351651
elif pa_type_as_str.startswith("struct<") or pa_type_as_str.startswith("struct{"):
16361652
value_type = ValueType.STRUCT
16371653
else:
@@ -1658,7 +1674,12 @@ def pa_to_feast_value_type(pa_type_as_str: str) -> ValueType:
16581674
"list<element: bytes>": ValueType.BYTES_LIST,
16591675
"list<element: float>": ValueType.FLOAT_LIST,
16601676
}
1661-
value_type = type_map[pa_type_as_str]
1677+
value_type = type_map.get(pa_type_as_str)
1678+
if value_type is None:
1679+
raise ValueError(
1680+
f"Feast cannot infer a value type for the Arrow type '{pa_type_as_str}'. "
1681+
"Declare the feature view schema explicitly, or extend the Arrow type mapping."
1682+
)
16621683

16631684
if is_list:
16641685
value_type = ValueType[value_type.name + "_LIST"]

‎sdk/python/tests/unit/test_type_map.py‎

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2568,3 +2568,30 @@ def test_pandas_and_arrow_int_inference_agree(dtype):
25682568
)
25692569
from_arrow = pa_to_feast_value_type(str(pyarrow.array(values).type))
25702570
assert from_pandas == from_arrow
2571+
2572+
2573+
@pytest.mark.parametrize(
2574+
"pa_type, expected",
2575+
[
2576+
("date32[day]", ValueType.UNIX_TIMESTAMP),
2577+
("date64[ms]", ValueType.UNIX_TIMESTAMP),
2578+
("decimal128(3, 2)", ValueType.DECIMAL),
2579+
("decimal256(10, 4)", ValueType.DECIMAL),
2580+
(
2581+
"dictionary<values=string, indices=int8, ordered=0>",
2582+
ValueType.STRING,
2583+
),
2584+
("halffloat", ValueType.FLOAT),
2585+
("large_binary", ValueType.BYTES),
2586+
],
2587+
)
2588+
def test_pa_to_feast_value_type_pandas_written_parquet_types(pa_type, expected):
2589+
"""Arrow types written by pandas to Parquet infer the same Feast type the
2590+
DataFrame inference path produces (issue #7040)."""
2591+
assert pa_to_feast_value_type(pa_type) == expected
2592+
2593+
2594+
def test_pa_to_feast_value_type_unsupported_type_raises_descriptive_error():
2595+
"""An unmapped Arrow type raises a descriptive error, not a bare KeyError."""
2596+
with pytest.raises(ValueError, match=r"time32\[s\]"):
2597+
pa_to_feast_value_type("time32[s]")

0 commit comments

Comments
 (0)