Skip to content

Commit 3afa78e

Browse files
fix: Fix type mapping spark (#4071)
1 parent 9ed0a09 commit 3afa78e

1 file changed

Lines changed: 7 additions & 1 deletion

File tree

‎sdk/python/feast/type_map.py‎

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -752,7 +752,7 @@ def _non_empty_value(value: Any) -> bool:
752752

753753
def spark_to_feast_value_type(spark_type_as_str: str) -> ValueType:
754754
# TODO not all spark types are convertible
755-
# Current non-convertible types: interval, map, struct, structfield, decimal, binary
755+
# Current non-convertible types: interval, map, struct, structfield, binary
756756
type_map: Dict[str, ValueType] = {
757757
"null": ValueType.UNKNOWN,
758758
"byte": ValueType.BYTES,
@@ -762,6 +762,7 @@ def spark_to_feast_value_type(spark_type_as_str: str) -> ValueType:
762762
"bigint": ValueType.INT64,
763763
"long": ValueType.INT64,
764764
"double": ValueType.DOUBLE,
765+
"decimal": ValueType.DOUBLE,
765766
"float": ValueType.FLOAT,
766767
"boolean": ValueType.BOOL,
767768
"timestamp": ValueType.UNIX_TIMESTAMP,
@@ -770,10 +771,15 @@ def spark_to_feast_value_type(spark_type_as_str: str) -> ValueType:
770771
"array<int>": ValueType.INT32_LIST,
771772
"array<bigint>": ValueType.INT64_LIST,
772773
"array<double>": ValueType.DOUBLE_LIST,
774+
"array<decimal>": ValueType.DOUBLE_LIST,
773775
"array<float>": ValueType.FLOAT_LIST,
774776
"array<boolean>": ValueType.BOOL_LIST,
775777
"array<timestamp>": ValueType.UNIX_TIMESTAMP_LIST,
776778
}
779+
if spark_type_as_str.startswith("decimal"):
780+
spark_type_as_str = "decimal"
781+
if spark_type_as_str.startswith("array<decimal"):
782+
spark_type_as_str = "array<decimal>"
777783
# TODO: Find better way of doing this.
778784
if not isinstance(spark_type_as_str, str) or spark_type_as_str not in type_map:
779785
return ValueType.NULL

0 commit comments

Comments
 (0)