Skip to content

Commit c79a160

Browse files
authored
Python: Fix in-memory upserts with key storage names (#14548)
### Motivation and Context Fixes #14547. Dictionary and dataclass records with a key `storage_name` fail during in-memory upsert because their serialized records no longer contain the Python key name. ### Description Track the key field name when converting each record to a dictionary, before its model type is lost. Default dictionary/dataclass serialization uses storage names; default Pydantic serialization uses logical names or configured serialization aliases. Index the serialized value without changing record fields or values. Preserve logical-name-first lookup for opaque `to_dict`, `model_dump`, and `model_serializer` callbacks, with a storage-name fallback when the logical key is absent. A custom `serialize` callback produces a store-specific record and uses storage names. A callback returning `None` still falls back to the normal dictionary conversion. In-memory string filters use the fields produced by the record's serializer. Default dict/dataclass serialization uses `storage_name` (for example, `x.stored_content`), while default Pydantic `model_dump()` keeps logical names (`x.content`); configured aliases or custom serialization may change those names. This PR preserves those existing field mappings and corrects the key used for indexing and upsert's returned IDs. It does not normalize the filter view across model types. Regressions cover single/batch upsert, get, search and delete, both directions of field-name collision, aliases, and custom serialization. Six additional cases catch transparent custom callbacks that were incorrectly treated as storage-named output in the previous revision. Validation: 155 in-memory/data tests pass; package mypy (555 files), scoped native pre-commit hooks, UTF-8/LF checks and wheel build pass. The wheel contains the exact connector source tested. Independent checks also cover serialized key transformations, derived record types, callbacks returning multiple dictionaries, vector generation, and partial serialization fallback. The build tool reports its existing pre-release-mode deprecation warning. Full unit-suite collection previously stopped with 41 optional-provider/backend import errors; the full suite and cloud integration tests were not run. ### Contribution Checklist - [ ] The code builds clean without any errors or warnings - [x] The PR follows the SK Contribution Guidelines and scoped pre-submission checks raise no violations - [ ] All unit tests pass, and I have added new tests where possible - [ ] I didn't break anyone 😄
1 parent 555c46b commit c79a160

2 files changed

Lines changed: 244 additions & 4 deletions

File tree

‎python/semantic_kernel/connectors/in_memory.py‎

Lines changed: 40 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,7 @@
66
from typing import Any, ClassVar, Final, Generic, TypeVar, cast
77

88
from numpy import dot
9-
from pydantic import Field
9+
from pydantic import BaseModel, ConfigDict, Field
1010
from scipy.spatial.distance import cityblock, cosine, euclidean, hamming, sqeuclidean
1111
from typing_extensions import override
1212

@@ -27,7 +27,7 @@
2727
)
2828
from semantic_kernel.exceptions import VectorSearchExecutionException, VectorStoreModelValidationError
2929
from semantic_kernel.exceptions.vector_store_exceptions import VectorStoreModelException, VectorStoreOperationException
30-
from semantic_kernel.kernel_types import OneOrMany
30+
from semantic_kernel.kernel_types import OneOrList, OneOrMany
3131
from semantic_kernel.utils.feature_stage_decorator import release_candidate
3232
from semantic_kernel.utils.list_handler import empty_generator
3333

@@ -81,6 +81,15 @@ def __delattr__(self, name) -> None:
8181
raise AttributeError(name)
8282

8383

84+
class _InMemoryRecord(dict[str, Any]):
85+
"""A serialized record with the name of its key field."""
86+
87+
def __init__(self, record: dict[str, Any], key_field_name: str):
88+
"""Keep the dumped fields and the name of their key field."""
89+
super().__init__(record)
90+
self.key_field_name = key_field_name
91+
92+
8493
class ReadOnlyAttributeDict(Mapping[TAKey, TAValue], Generic[TAKey, TAValue]):
8594
"""A read-only mapping that allows attribute access to keys."""
8695

@@ -632,14 +641,41 @@ async def _inner_get(
632641
async def _inner_upsert(self, records: Sequence[Any], **kwargs: Any) -> Sequence[TKey]:
633642
updated_keys = []
634643
for record in records:
644+
key_field_name = (
645+
record.key_field_name if isinstance(record, _InMemoryRecord) else self._key_field_storage_name
646+
)
635647
record = AttributeDict(record)
636-
self.inner_storage[record[self._key_field_name]] = record
637-
updated_keys.append(record[self._key_field_name])
648+
if key_field_name not in record:
649+
key_field_name = (
650+
self._key_field_storage_name if self._key_field_storage_name in record else self._key_field_name
651+
)
652+
key = record[key_field_name]
653+
self.inner_storage[key] = record
654+
updated_keys.append(key)
638655
return updated_keys
639656

640657
def _deserialize_store_models_to_dicts(self, records: Sequence[Any], **kwargs: Any) -> Sequence[dict[str, Any]]:
641658
return records
642659

660+
def _serialize_data_model_to_dict(self, record: TModel, **kwargs: Any) -> OneOrList[dict[str, Any]]:
661+
"""Keep track of key names before records lose their model type."""
662+
serialized = super()._serialize_data_model_to_dict(record, **kwargs)
663+
key_field_name = self._key_field_name
664+
if not self.definition.to_dict:
665+
if not isinstance(record, BaseModel):
666+
key_field_name = self._key_field_storage_name
667+
elif (
668+
type(record).model_dump is BaseModel.model_dump
669+
and not type(record).__pydantic_decorators__.model_serializers
670+
and "serialize_by_alias" in ConfigDict.__annotations__
671+
and type(record).model_config.get("serialize_by_alias")
672+
and (key_field := type(record).model_fields.get(key_field_name))
673+
):
674+
key_field_name = key_field.serialization_alias or key_field_name
675+
if isinstance(serialized, list):
676+
return [_InMemoryRecord(item, key_field_name) for item in serialized]
677+
return _InMemoryRecord(serialized, key_field_name)
678+
643679
def _serialize_dicts_to_store_models(self, records: Sequence[dict[str, Any]], **kwargs: Any) -> Sequence[Any]:
644680
return records
645681

‎python/tests/unit/connectors/memory/test_in_memory.py‎

Lines changed: 204 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@
22

33
import ast
44

5+
from pydantic import ConfigDict, Field, model_serializer
56
from pytest import fixture, mark, raises
67

78
from semantic_kernel.connectors.in_memory import InMemoryCollection, InMemoryStore
@@ -35,6 +36,209 @@ async def test_upsert(collection):
3536
assert collection.inner_storage == {"testid": record}
3637

3738

39+
@mark.parametrize("batch", [False, True])
40+
@mark.parametrize("model_kind", ["dict", "dataclass", "pydantic"])
41+
async def test_upsert_with_key_storage_name(definition, dataclass_vector_data_model, record_type, batch, model_kind):
42+
definition.key_field.storage_name = "stored_id"
43+
model_type = {"dict": dict, "dataclass": dataclass_vector_data_model, "pydantic": record_type}[model_kind]
44+
collection = InMemoryCollection(collection_name="test", record_type=model_type, definition=definition)
45+
records = [
46+
model_type(id="first", content="first content", vector=[1.0, 0.0, 0.0, 0.0, 0.0]),
47+
model_type(id="second", content="second content", vector=[0.0, 1.0, 0.0, 0.0, 0.0]),
48+
]
49+
50+
keys = await collection.upsert(records if batch else records[0])
51+
assert keys == (["first", "second"] if batch else "first")
52+
assert await collection.get("first", include_vectors=True) == records[0]
53+
if batch:
54+
assert await collection.get(["first", "second"], include_vectors=True) == records
55+
56+
results = await collection.search(vector=[1.0, 0.0, 0.0, 0.0, 0.0], include_vectors=True)
57+
found = [result.record async for result in results.results]
58+
assert found == (records if batch else records[:1])
59+
60+
await collection.delete("first")
61+
assert await collection.get("first") is None
62+
if batch:
63+
assert await collection.get("second", include_vectors=True) == records[1]
64+
65+
66+
@mark.parametrize("batch", [False, True])
67+
async def test_upsert_when_storage_name_matches_logical_field(definition, record_type, batch):
68+
definition.key_field.storage_name = "content"
69+
collection = InMemoryCollection(collection_name="test", record_type=record_type, definition=definition)
70+
record = record_type(id="first", content="unrelated", vector=[1.0, 0.0, 0.0, 0.0, 0.0])
71+
72+
keys = await collection.upsert([record] if batch else record)
73+
assert keys == (["first"] if batch else "first")
74+
assert "first" in collection.inner_storage
75+
assert "unrelated" not in collection.inner_storage
76+
77+
await collection.delete("first")
78+
assert collection.inner_storage == {}
79+
80+
81+
@mark.parametrize("batch", [False, True])
82+
@mark.parametrize("serializer", ["default", "serialize_none", "model_serialize_none"])
83+
async def test_upsert_when_key_storage_name_matches_another_logical_field(definition, record_type, batch, serializer):
84+
definition.key_field.storage_name = "content"
85+
if serializer == "serialize_none":
86+
definition.serialize = lambda record, **kwargs: None
87+
elif serializer == "model_serialize_none":
88+
89+
class CustomRecord(record_type):
90+
def serialize(self, **kwargs):
91+
return None
92+
93+
record_type = CustomRecord
94+
collection = InMemoryCollection(collection_name="test", record_type=record_type, definition=definition)
95+
records = [
96+
record_type(id="first", content="unrelated first", vector=[1.0, 0.0, 0.0, 0.0, 0.0]),
97+
record_type(id="second", content="unrelated second", vector=[0.0, 1.0, 0.0, 0.0, 0.0]),
98+
]
99+
100+
keys = await collection.upsert(records if batch else records[0])
101+
assert keys == (["first", "second"] if batch else "first")
102+
assert set(collection.inner_storage) == ({"first", "second"} if batch else {"first"})
103+
assert collection.inner_storage["first"] == records[0].model_dump()
104+
105+
await collection.delete(["first", "second"] if batch else "first")
106+
assert collection.inner_storage == {}
107+
108+
109+
@mark.parametrize("batch", [False, True])
110+
@mark.parametrize("model_kind", ["dict", "dataclass"])
111+
async def test_upsert_when_another_storage_name_matches_key(definition, dataclass_vector_data_model, batch, model_kind):
112+
definition.key_field.storage_name = "stored_id"
113+
definition.fields[1].storage_name = "id"
114+
model_type = dict if model_kind == "dict" else dataclass_vector_data_model
115+
collection = InMemoryCollection(collection_name="test", record_type=model_type, definition=definition)
116+
records = [
117+
model_type(id="first", content="unrelated first", vector=[1.0, 0.0, 0.0, 0.0, 0.0]),
118+
model_type(id="second", content="unrelated second", vector=[0.0, 1.0, 0.0, 0.0, 0.0]),
119+
]
120+
121+
keys = await collection.upsert(records if batch else records[0])
122+
assert keys == (["first", "second"] if batch else "first")
123+
assert await collection.get("first", include_vectors=True) == records[0]
124+
if batch:
125+
assert await collection.get(["first", "second"], include_vectors=True) == records
126+
127+
await collection.delete(["first", "second"] if batch else "first")
128+
assert collection.inner_storage == {}
129+
130+
131+
@mark.parametrize("batch", [False, True])
132+
@mark.parametrize("serializer", ["to_dict", "serialize", "model_serialize", "model_dump", "model_serializer"])
133+
async def test_upsert_with_pydantic_custom_storage_serialization(definition, record_type, batch, serializer):
134+
definition.key_field.storage_name = "stored_id"
135+
content_name = "id" if serializer in {"serialize", "model_serialize"} else "stored_content"
136+
definition.fields[1].storage_name = content_name
137+
if serializer == "model_serialize":
138+
139+
class CustomRecord(record_type):
140+
def serialize(self, **kwargs):
141+
return {"stored_id": self.id, content_name: self.content, "vector": self.vector}
142+
143+
model_type = CustomRecord
144+
elif serializer == "model_dump":
145+
146+
class CustomDumpRecord(record_type):
147+
def model_dump(self, **kwargs):
148+
return {"stored_id": self.id, content_name: self.content, "vector": self.vector}
149+
150+
model_type = CustomDumpRecord
151+
elif serializer == "model_serializer":
152+
153+
class CustomSerializedRecord(record_type):
154+
@model_serializer
155+
def serialize_model(self):
156+
return {"stored_id": self.id, content_name: self.content, "vector": self.vector}
157+
158+
model_type = CustomSerializedRecord
159+
else:
160+
model_type = record_type
161+
setattr(
162+
definition,
163+
serializer,
164+
lambda record, **kwargs: {"stored_id": record.id, content_name: record.content, "vector": record.vector},
165+
)
166+
collection = InMemoryCollection(collection_name="test", record_type=model_type, definition=definition)
167+
records = [
168+
model_type(id="first", content="unrelated first", vector=[1.0, 0.0, 0.0, 0.0, 0.0]),
169+
model_type(id="second", content="unrelated second", vector=[0.0, 1.0, 0.0, 0.0, 0.0]),
170+
]
171+
172+
keys = await collection.upsert(records if batch else records[0])
173+
assert keys == (["first", "second"] if batch else "first")
174+
assert set(collection.inner_storage) == ({"first", "second"} if batch else {"first"})
175+
176+
await collection.delete(["first", "second"] if batch else "first")
177+
assert collection.inner_storage == {}
178+
179+
180+
@mark.parametrize("batch", [False, True])
181+
async def test_upsert_with_pydantic_serialization_aliases(definition, record_type, batch):
182+
definition.key_field.storage_name = "stored_id"
183+
definition.fields[1].storage_name = "id"
184+
185+
class AliasedRecord(record_type):
186+
model_config = ConfigDict(serialize_by_alias=True)
187+
id: str = Field(serialization_alias="stored_id")
188+
content: str = Field(serialization_alias="id")
189+
190+
collection = InMemoryCollection(collection_name="test", record_type=AliasedRecord, definition=definition)
191+
records = [
192+
AliasedRecord(id="first", content="unrelated first", vector=[1.0, 0.0, 0.0, 0.0, 0.0]),
193+
AliasedRecord(id="second", content="unrelated second", vector=[0.0, 1.0, 0.0, 0.0, 0.0]),
194+
]
195+
196+
keys = await collection.upsert(records if batch else records[0])
197+
assert keys == (["first", "second"] if batch else "first")
198+
assert set(collection.inner_storage) == ({"first", "second"} if batch else {"first"})
199+
200+
await collection.delete(["first", "second"] if batch else "first")
201+
assert collection.inner_storage == {}
202+
203+
204+
@mark.parametrize("batch", [False, True])
205+
@mark.parametrize("serializer", ["model_dump", "model_serializer", "to_dict"])
206+
async def test_upsert_with_custom_logical_serialization(definition, record_type, batch, serializer):
207+
definition.key_field.storage_name = "content"
208+
definition.fields[1].storage_name = "id"
209+
if serializer == "model_dump":
210+
211+
class CustomRecord(record_type):
212+
def model_dump(self, **kwargs):
213+
return super().model_dump(**kwargs)
214+
215+
model_type = CustomRecord
216+
elif serializer == "model_serializer":
217+
218+
class CustomRecord(record_type):
219+
@model_serializer(mode="wrap")
220+
def serialize_model(self, handler):
221+
return handler(self)
222+
223+
model_type = CustomRecord
224+
else:
225+
model_type = record_type
226+
definition.to_dict = lambda record, **kwargs: record.model_dump()
227+
228+
collection = InMemoryCollection(collection_name="test", record_type=model_type, definition=definition)
229+
records = [
230+
model_type(id="first", content="unrelated first", vector=[1.0, 0.0, 0.0, 0.0, 0.0]),
231+
model_type(id="second", content="unrelated second", vector=[0.0, 1.0, 0.0, 0.0, 0.0]),
232+
]
233+
234+
keys = await collection.upsert(records if batch else records[0])
235+
assert keys == (["first", "second"] if batch else "first")
236+
assert set(collection.inner_storage) == ({"first", "second"} if batch else {"first"})
237+
238+
await collection.delete(["first", "second"] if batch else "first")
239+
assert collection.inner_storage == {}
240+
241+
38242
async def test_get(collection):
39243
record = {"id": "testid", "content": "test content", "vector": [0.1, 0.2, 0.3, 0.4, 0.5]}
40244
await collection.upsert(record)

0 commit comments

Comments
 (0)