Skip to content

Commit 0fe7ff6

Browse files
soooojinleeclaude
andcommitted
feat: Support nested collection types (Array/Set of Array/Set) (feast-dev#5947)
Add support for 2-level nested collection types: Array(Array(T)), Array(Set(T)), Set(Array(T)), and Set(Set(T)). - Add 4 generic ValueType enums (LIST_LIST, LIST_SET, SET_LIST, SET_SET) backed by RepeatedValue proto messages - Persist inner type info in Field tags (feast:nested_inner_type), following the existing Struct schema tag pattern - Handle edge cases: empty inner collections, Set dedup at inner level, depth limit enforcement (2 levels max) - Add proto/JSON/remote transport serialization support - Add 25 unit tests covering all combinations and edge cases Signed-off-by: Soojin Lee <[email protected]> Co-Authored-By: Claude Opus 4.6 <[email protected]> Signed-off-by: soojin <[email protected]>
1 parent 18ede88 commit 0fe7ff6

11 files changed

Lines changed: 503 additions & 64 deletions

File tree

‎protos/feast/types/Value.proto‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -57,6 +57,10 @@ message ValueType {
5757
JSON_LIST = 33;
5858
STRUCT = 34;
5959
STRUCT_LIST = 35;
60+
LIST_LIST = 36;
61+
LIST_SET = 37;
62+
SET_LIST = 38;
63+
SET_SET = 39;
6064
}
6165
}
6266

@@ -96,6 +100,10 @@ message Value {
96100
StringList json_list_val = 33;
97101
Map struct_val = 34;
98102
MapList struct_list_val = 35;
103+
RepeatedValue list_list_val = 36;
104+
RepeatedValue list_set_val = 37;
105+
RepeatedValue set_list_val = 38;
106+
RepeatedValue set_set_val = 39;
99107
}
100108
}
101109

‎sdk/python/feast/field.py‎

Lines changed: 33 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,7 @@
2323
from feast.value_type import ValueType
2424

2525
STRUCT_SCHEMA_TAG = "feast:struct_schema"
26+
NESTED_COLLECTION_INNER_TYPE_TAG = "feast:nested_inner_type"
2627

2728

2829
@typechecked
@@ -118,7 +119,7 @@ def __str__(self):
118119

119120
def to_proto(self) -> FieldProto:
120121
"""Converts a Field object to its protobuf representation."""
121-
from feast.types import Array
122+
from feast.types import Array, Set
122123

123124
value_type = self.dtype.to_value_type()
124125
vector_search_metric = self.vector_search_metric or ""
@@ -128,6 +129,11 @@ def to_proto(self) -> FieldProto:
128129
tags[STRUCT_SCHEMA_TAG] = _serialize_struct_schema(self.dtype)
129130
elif isinstance(self.dtype, Array) and isinstance(self.dtype.base_type, Struct):
130131
tags[STRUCT_SCHEMA_TAG] = _serialize_struct_schema(self.dtype.base_type)
132+
# Persist nested collection type info in tags
133+
if isinstance(self.dtype, (Array, Set)) and isinstance(
134+
self.dtype.base_type, (Array, Set)
135+
):
136+
tags[NESTED_COLLECTION_INNER_TYPE_TAG] = _feast_type_to_str(self.dtype)
131137
return FieldProto(
132138
name=self.name,
133139
value_type=value_type.value,
@@ -155,17 +161,30 @@ def from_proto(cls, field_proto: FieldProto):
155161
# Reconstruct Struct type from persisted schema in tags
156162
from feast.types import Array
157163

164+
internal_tags = {STRUCT_SCHEMA_TAG, NESTED_COLLECTION_INNER_TYPE_TAG}
158165
dtype: FeastType
159166
if value_type == ValueType.STRUCT and STRUCT_SCHEMA_TAG in tags:
160167
dtype = _deserialize_struct_schema(tags[STRUCT_SCHEMA_TAG])
161-
user_tags = {k: v for k, v in tags.items() if k != STRUCT_SCHEMA_TAG}
168+
user_tags = {k: v for k, v in tags.items() if k not in internal_tags}
162169
elif value_type == ValueType.STRUCT_LIST and STRUCT_SCHEMA_TAG in tags:
163170
inner_struct = _deserialize_struct_schema(tags[STRUCT_SCHEMA_TAG])
164171
dtype = Array(inner_struct)
165-
user_tags = {k: v for k, v in tags.items() if k != STRUCT_SCHEMA_TAG}
172+
user_tags = {k: v for k, v in tags.items() if k not in internal_tags}
173+
elif (
174+
value_type
175+
in (
176+
ValueType.LIST_LIST,
177+
ValueType.LIST_SET,
178+
ValueType.SET_LIST,
179+
ValueType.SET_SET,
180+
)
181+
and NESTED_COLLECTION_INNER_TYPE_TAG in tags
182+
):
183+
dtype = _str_to_feast_type(tags[NESTED_COLLECTION_INNER_TYPE_TAG])
184+
user_tags = {k: v for k, v in tags.items() if k not in internal_tags}
166185
else:
167186
dtype = from_value_type(value_type=value_type)
168-
user_tags = tags
187+
user_tags = {k: v for k, v in tags.items() if k not in internal_tags}
169188

170189
return cls(
171190
name=field_proto.name,
@@ -198,6 +217,7 @@ def _feast_type_to_str(feast_type: FeastType) -> str:
198217
from feast.types import (
199218
Array,
200219
PrimitiveFeastType,
220+
Set,
201221
)
202222

203223
if isinstance(feast_type, PrimitiveFeastType):
@@ -209,6 +229,8 @@ def _feast_type_to_str(feast_type: FeastType) -> str:
209229
return json.dumps({"__struct__": nested})
210230
elif isinstance(feast_type, Array):
211231
return f"Array({_feast_type_to_str(feast_type.base_type)})"
232+
elif isinstance(feast_type, Set):
233+
return f"Set({_feast_type_to_str(feast_type.base_type)})"
212234
else:
213235
return str(feast_type)
214236

@@ -218,6 +240,7 @@ def _str_to_feast_type(type_str: str) -> FeastType:
218240
from feast.types import (
219241
Array,
220242
PrimitiveFeastType,
243+
Set,
221244
)
222245

223246
# Check if it's an Array type
@@ -226,6 +249,12 @@ def _str_to_feast_type(type_str: str) -> FeastType:
226249
base_type = _str_to_feast_type(inner)
227250
return Array(base_type)
228251

252+
# Check if it's a Set type
253+
if type_str.startswith("Set(") and type_str.endswith(")"):
254+
inner = type_str[4:-1]
255+
base_type = _str_to_feast_type(inner)
256+
return Set(base_type)
257+
229258
# Check if it's a nested Struct (JSON encoded)
230259
if type_str.startswith("{"):
231260
try:

‎sdk/python/feast/infra/online_stores/remote.py‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -94,6 +94,11 @@ def _proto_value_to_transport_value(proto_value: ValueProto) -> Any:
9494
if val_attr == "json_list_val":
9595
return list(getattr(proto_value, val_attr).val)
9696

97+
# Nested collection types use feast_value_type_to_python_type
98+
# which handles recursive conversion of RepeatedValue protos.
99+
if val_attr in ("list_list_val", "list_set_val", "set_list_val", "set_set_val"):
100+
return feast_value_type_to_python_type(proto_value)
101+
97102
# Map/Struct types are converted to Python dicts by
98103
# feast_value_type_to_python_type. Serialise them to JSON strings
99104
# so the server-side DataFrame gets VARCHAR columns instead of

‎sdk/python/feast/proto_json.py‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -63,6 +63,12 @@ def to_json_object(printer: _Printer, message: ProtoMessage) -> JsonObject:
6363
# to JSON. The parse back result will be different from original message.
6464
if which is None or which == "null_val":
6565
return None
66+
elif which in ("list_list_val", "list_set_val", "set_list_val", "set_set_val"):
67+
# Nested collection: RepeatedValue containing Values
68+
repeated = getattr(message, which)
69+
value = [
70+
printer._MessageToJsonObject(inner_val) for inner_val in repeated.val
71+
]
6672
elif "_list_" in which:
6773
value = list(getattr(message, which).val)
6874
else:

‎sdk/python/feast/protos/feast/types/Value_pb2.py‎

Lines changed: 43 additions & 43 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

0 commit comments

Comments
 (0)