Skip to content

Commit b871957

Browse files
authored
Java: allow customising the reading of memory records. Fixes #3060 (#3241)
Fixes #3060
1 parent 11eba8f commit b871957

4 files changed

Lines changed: 172 additions & 26 deletions

File tree

‎java/connectors/semantickernel-connectors-memory-azurecognitivesearch/pom.xml‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -49,6 +49,16 @@
4949
<artifactId>log4j-slf4j2-impl</artifactId>
5050
<scope>test</scope>
5151
</dependency>
52+
<dependency>
53+
<groupId>com.microsoft.semantic-kernel</groupId>
54+
<artifactId>semantickernel-core</artifactId>
55+
<scope>test</scope>
56+
</dependency>
57+
<dependency>
58+
<groupId>com.microsoft.semantic-kernel</groupId>
59+
<artifactId>semantickernel-connectors-ai-openai</artifactId>
60+
<scope>test</scope>
61+
</dependency>
5262
</dependencies>
5363

5464
</project>

‎java/connectors/semantickernel-connectors-memory-azurecognitivesearch/src/main/java/com/microsoft/semantickernel/connectors/memory/azurecognitivesearch/AzureCognitiveSearchMemoryStore.java‎

Lines changed: 53 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -35,8 +35,10 @@
3535
import java.util.Locale;
3636
import java.util.Map;
3737
import java.util.concurrent.ConcurrentHashMap;
38+
import java.util.function.Function;
3839
import java.util.stream.Collectors;
3940
import javax.annotation.Nonnull;
41+
import javax.annotation.Nullable;
4042
import reactor.core.publisher.Flux;
4143
import reactor.core.publisher.Mono;
4244
import reactor.util.function.Tuple2;
@@ -54,6 +56,7 @@ public class AzureCognitiveSearchMemoryStore implements MemoryStore {
5456
private final SearchIndexAsyncClient _adminClient;
5557

5658
private final Map<String, SearchAsyncClient> _clientsByIndex = new ConcurrentHashMap<>();
59+
private final Function<SearchDocument, MemoryRecord> memoryRecordMapper;
5760

5861
/// <summary>
5962
/// Create a new instance of memory storage using Azure Cognitive Search.
@@ -74,7 +77,8 @@ public AzureCognitiveSearchMemoryStore(@Nonnull String endpoint, @Nonnull String
7477
.endpoint(endpoint)
7578
.credential(new AzureKeyCredential(apiKey))
7679
.clientOptions(clientOptions())
77-
.buildAsyncClient());
80+
.buildAsyncClient(),
81+
null);
7882
}
7983

8084
/**
@@ -90,16 +94,20 @@ public AzureCognitiveSearchMemoryStore(
9094
.endpoint(endpoint)
9195
.credential(credentials)
9296
.clientOptions(clientOptions())
93-
.buildAsyncClient());
97+
.buildAsyncClient(),
98+
null);
9499
}
95100

96101
/**
97102
* Create a new instance of memory storage using Azure Cognitive Search.
98103
*
99104
* @param searchIndexAsyncClient the Azure search documents index client to use
100105
*/
101-
public AzureCognitiveSearchMemoryStore(@Nonnull SearchIndexAsyncClient searchIndexAsyncClient) {
106+
public AzureCognitiveSearchMemoryStore(
107+
@Nonnull SearchIndexAsyncClient searchIndexAsyncClient,
108+
@Nullable Function<SearchDocument, MemoryRecord> memoryRecordMapper) {
102109
this._adminClient = searchIndexAsyncClient;
110+
this.memoryRecordMapper = memoryRecordMapper;
103111
}
104112

105113
@Override
@@ -203,6 +211,39 @@ public Mono<Collection<Tuple2<MemoryRecord, Float>>> getNearestMatchesAsync(
203211
int limit,
204212
float minRelevanceScore,
205213
boolean withEmbedding) {
214+
return getNearestMatchesAsync(
215+
collectionName,
216+
embedding,
217+
limit,
218+
minRelevanceScore,
219+
withEmbedding,
220+
memoryRecordMapper);
221+
}
222+
223+
/**
224+
* Gets the nearest matches to the {@link Embedding} of type {@code Float}. Does not guarantee
225+
* that the collection exists.
226+
*
227+
* <p>If a memoryRecordMapper is provided this overrides any mapper provided when constructing
228+
* this memory store.
229+
*
230+
* @param collectionName The name associated with a collection of embeddings.
231+
* @param embedding The {@link Embedding} to compare the collection's embeddings with.
232+
* @param limit The maximum number of similarity results to return.
233+
* @param minRelevanceScore The minimum relevance threshold for returned results.
234+
* @param withEmbedding If true, the embeddings will be returned in the memory records.
235+
* @param memoryRecordMapper a mapper that controls how to map the search document to a memory
236+
* record
237+
* @return A collection of tuples where item1 is a {@link MemoryRecord} and item2 is its
238+
* similarity score as a {@code Float}.
239+
*/
240+
public Mono<Collection<Tuple2<MemoryRecord, Float>>> getNearestMatchesAsync(
241+
@Nonnull String collectionName,
242+
@Nonnull Embedding embedding,
243+
int limit,
244+
float minRelevanceScore,
245+
boolean withEmbedding,
246+
@Nullable Function<SearchDocument, MemoryRecord> memoryRecordMapper) {
206247
collectionName = normalizeIndexName(collectionName);
207248
SearchAsyncClient client = getSearchClient(collectionName);
208249

@@ -218,9 +259,15 @@ public Mono<Collection<Tuple2<MemoryRecord, Float>>> getNearestMatchesAsync(
218259
.filter(result -> (double) minRelevanceScore <= result.getScore())
219260
.map(
220261
result -> {
221-
MemoryRecord memoryRecord =
222-
result.getDocument(AzureCognitiveSearchMemoryRecord.class)
223-
.toMemoryRecord(withEmbedding);
262+
MemoryRecord memoryRecord;
263+
if (memoryRecordMapper != null) {
264+
SearchDocument doc = result.getDocument(SearchDocument.class);
265+
memoryRecord = memoryRecordMapper.apply(doc);
266+
} else {
267+
AzureCognitiveSearchMemoryRecord doc =
268+
result.getDocument(AzureCognitiveSearchMemoryRecord.class);
269+
memoryRecord = doc.toMemoryRecord(withEmbedding);
270+
}
224271
float score = (float) result.getScore();
225272
return Tuples.of(memoryRecord, score);
226273
})

‎java/connectors/semantickernel-connectors-memory-azurecognitivesearch/src/test/java/com/microsoft/semantickernel/connectors/memory/azurecognitivesearch/AzureCognitiveSearchMemoryTests.java‎

Lines changed: 108 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -4,36 +4,40 @@
44
import static org.junit.jupiter.api.Assertions.assertEquals;
55
import static org.junit.jupiter.api.Assertions.assertTrue;
66

7-
import com.azure.core.credential.AzureKeyCredential;
87
import com.azure.core.http.HttpHeaders;
98
import com.azure.core.http.HttpPipeline;
109
import com.azure.core.http.HttpRequest;
1110
import com.azure.core.http.HttpResponse;
12-
import com.azure.core.http.policy.HttpLogDetailLevel;
13-
import com.azure.core.http.policy.HttpLogOptions;
11+
import com.azure.core.http.rest.Response;
1412
import com.azure.core.util.Context;
1513
import com.azure.core.util.tracing.Tracer;
14+
import com.azure.search.documents.SearchAsyncClient;
15+
import com.azure.search.documents.SearchDocument;
1616
import com.azure.search.documents.indexes.SearchIndexAsyncClient;
17-
import com.azure.search.documents.indexes.SearchIndexClientBuilder;
17+
import com.azure.search.documents.models.SearchResult;
18+
import com.azure.search.documents.util.SearchPagedFlux;
1819
import com.microsoft.semantickernel.Kernel;
1920
import com.microsoft.semantickernel.SKBuilders;
2021
import com.microsoft.semantickernel.ai.embeddings.Embedding;
2122
import com.microsoft.semantickernel.ai.embeddings.EmbeddingGeneration;
2223
import com.microsoft.semantickernel.memory.MemoryQueryResult;
24+
import com.microsoft.semantickernel.memory.MemoryRecord;
25+
import com.microsoft.semantickernel.memory.MemoryRecordMetadata;
2326
import java.nio.charset.StandardCharsets;
2427
import java.util.ArrayList;
2528
import java.util.Arrays;
2629
import java.util.Base64;
2730
import java.util.List;
2831
import java.util.function.Function;
32+
import javax.annotation.Nullable;
2933
import org.junit.jupiter.api.BeforeEach;
3034
import org.junit.jupiter.api.Disabled;
3135
import org.junit.jupiter.api.Test;
3236
import org.mockito.Mockito;
3337
import org.mockito.invocation.InvocationOnMock;
38+
import reactor.core.publisher.Flux;
3439
import reactor.core.publisher.Mono;
3540

36-
@Disabled("Pending https://github.com/microsoft/semantic-kernel/issues/1797")
3741
class AzureCognitiveSearchMemoryTests {
3842

3943
// com.azure.core.implementation.http.rest.RestProxyBase.RestProxyBase
@@ -85,22 +89,51 @@ public boolean isEnabled() {
8589

8690
@BeforeEach
8791
void setUp() {
92+
kernel = buildKernel(null);
93+
}
8894

89-
String apiKey = "fake-key";
90-
HttpPipeline pipeline = setUpHttpPipeline();
95+
private Kernel buildKernel(
96+
@Nullable Function<SearchDocument, MemoryRecord> memoryRecordMapper) {
97+
SearchAsyncClient searchAsyncClient = Mockito.mock(SearchAsyncClient.class);
98+
SearchIndexAsyncClient searchIndexAsyncClient = Mockito.mock(SearchIndexAsyncClient.class);
99+
Response response = Mockito.mock(Response.class);
100+
AzureCognitiveSearchMemoryRecord azureCognitiveSearchMemoryRecord =
101+
Mockito.mock(AzureCognitiveSearchMemoryRecord.class);
102+
MemoryRecord memoryRecord = Mockito.mock(MemoryRecord.class);
103+
SearchPagedFlux searchPagedFlux = Mockito.mock(SearchPagedFlux.class);
104+
SearchResult searchResult = Mockito.mock(SearchResult.class);
105+
106+
Mockito.when(searchResult.getDocument(Mockito.any())).thenReturn(new SearchDocument());
107+
Mockito.when(searchIndexAsyncClient.getSearchAsyncClient(Mockito.any()))
108+
.thenReturn(searchAsyncClient);
109+
110+
Mockito.when(searchPagedFlux.filter(Mockito.any())).thenReturn(Flux.just(searchResult));
111+
112+
Mockito.when(
113+
searchAsyncClient.getDocumentWithResponse(
114+
Mockito.any(), Mockito.any(), Mockito.any()))
115+
.thenReturn(Mono.just(response));
116+
117+
Mockito.when(searchAsyncClient.search(Mockito.any(), Mockito.any()))
118+
.thenReturn(searchPagedFlux);
119+
Mockito.when(response.getValue()).thenReturn(azureCognitiveSearchMemoryRecord);
120+
Mockito.when(azureCognitiveSearchMemoryRecord.toMemoryRecord(Mockito.anyBoolean()))
121+
.thenReturn(memoryRecord);
91122

92-
SearchIndexAsyncClient searchIndexAsyncClient =
93-
new SearchIndexClientBuilder()
123+
EmbeddingGeneration<String> embeddingGeneration =
124+
new EmbeddingGeneration<String>() {
125+
/*
126+
new SearchIndexClientBuilder()
94127
.endpoint("https://fake-random-test-host/fake-path")
95128
.credential(new AzureKeyCredential(apiKey))
96129
.pipeline(pipeline)
97130
.httpLogOptions(
98-
new HttpLogOptions()
99-
.setLogLevel(HttpLogDetailLevel.BODY_AND_HEADERS))
131+
new HttpLogOptions()
132+
.setLogLevel(HttpLogDetailLevel.BODY_AND_HEADERS))
100133
.buildAsyncClient();
101134
102-
EmbeddingGeneration<String> embeddingGeneration =
103-
new EmbeddingGeneration<String>() {
135+
*/
136+
104137
List<Embedding> embeddings = new ArrayList<>();
105138

106139
{
@@ -113,12 +146,12 @@ public Mono<List<Embedding>> generateEmbeddingsAsync(List<String> data) {
113146
}
114147
};
115148

116-
kernel =
117-
SKBuilders.kernel()
118-
.withMemoryStorage(
119-
new AzureCognitiveSearchMemoryStore(searchIndexAsyncClient))
120-
.withDefaultAIService(embeddingGeneration)
121-
.build();
149+
return SKBuilders.kernel()
150+
.withMemoryStorage(
151+
new AzureCognitiveSearchMemoryStore(
152+
searchIndexAsyncClient, memoryRecordMapper))
153+
.withDefaultAIService(embeddingGeneration)
154+
.build();
122155
}
123156

124157
private HttpPipeline setUpHttpPipeline() {
@@ -158,6 +191,7 @@ private Mono<HttpResponse> answerHttpRequest(InvocationOnMock invocation) {
158191
return Mono.just(httpResponse);
159192
}
160193

194+
@Disabled("Pending https://github.com/microsoft/semantic-kernel/issues/1797")
161195
@Test
162196
void azureCognitiveSearchMemoryStoreShouldBeProperlyInitialized() {
163197
// Arrange
@@ -172,6 +206,7 @@ void azureCognitiveSearchMemoryStoreShouldBeProperlyInitialized() {
172206
assertTrue(collections.contains("fake-index"));
173207
}
174208

209+
@Disabled("Pending https://github.com/microsoft/semantic-kernel/issues/1797")
175210
@Test
176211
void azureCognitiveSearchMemoryStoreCanSaveReferenceAsync() {
177212
// Arrange
@@ -205,6 +240,7 @@ void azureCognitiveSearchMemoryStoreCanSaveReferenceAsync() {
205240
assertEquals("fake-key", key);
206241
}
207242

243+
@Disabled("Pending https://github.com/microsoft/semantic-kernel/issues/1797")
208244
@Test
209245
void azureCognitiveSearchMemoryStoreCanSaveInformationAsync() {
210246
// Arrange
@@ -237,6 +273,7 @@ void azureCognitiveSearchMemoryStoreCanSaveInformationAsync() {
237273
assertEquals("fake-key", key);
238274
}
239275

276+
@Disabled("Pending https://github.com/microsoft/semantic-kernel/issues/1797")
240277
@Test
241278
void azureCognitiveSearchMemoryStoreGetAsync() {
242279
// Arrange
@@ -267,6 +304,7 @@ void azureCognitiveSearchMemoryStoreGetAsync() {
267304
assertEquals(key, result.getMetadata().getId());
268305
}
269306

307+
@Disabled("Pending https://github.com/microsoft/semantic-kernel/issues/1797")
270308
@Test
271309
void azureCognitiveSearchMemoryStoreRemoveAsync() {
272310
// Arrange
@@ -298,6 +336,7 @@ void azureCognitiveSearchMemoryStoreRemoveAsync() {
298336
// Assert
299337
}
300338

339+
@Disabled("Pending https://github.com/microsoft/semantic-kernel/issues/1797")
301340
@Test
302341
void azureCognitiveSearchMemoryStoreSearchAsync() {
303342
// Arrange
@@ -342,4 +381,54 @@ void azureCognitiveSearchMemoryStoreSearchAsync() {
342381
assertEquals(1, results.size());
343382
assertEquals(key, results.get(0).getMetadata().getId());
344383
}
384+
385+
@Test
386+
void azureCognitiveSearchMemoryStoreGetAsyncWithCustomMapping() {
387+
// Do not convert to lambda, it will break mockito
388+
Function<SearchDocument, MemoryRecord> transform =
389+
Mockito.spy(
390+
new Function<SearchDocument, MemoryRecord>() {
391+
@Override
392+
public MemoryRecord apply(SearchDocument searchDocument) {
393+
return new MemoryRecord(
394+
new MemoryRecordMetadata(
395+
false,
396+
"an-id",
397+
"fake-text",
398+
"fake-description",
399+
"fake-additionalmetadata",
400+
"fake-externalsourcename"),
401+
new Embedding(Arrays.asList(1.0f, 2.0f, 3.0f)),
402+
"an-id",
403+
null);
404+
}
405+
});
406+
407+
kernel = buildKernel(transform);
408+
// Arrange
409+
String key = "fake-key";
410+
String base64EncodedKey =
411+
new String(
412+
Base64.getUrlEncoder().encode("fake-key".getBytes(StandardCharsets.UTF_8)));
413+
414+
responseBody =
415+
httpRequest ->
416+
"{"
417+
+ " \"Id\": \""
418+
+ base64EncodedKey
419+
+ "\","
420+
+ " \"Text\": \"fake-text\","
421+
+ " \"Description\": \"fake-description\","
422+
+ " \"AdditionalMetadata\": \"fake-additionalmetadata\","
423+
+ " \"ExternalSourceName\": \"fake-externalsourcename\","
424+
+ " \"Reference\": false"
425+
+ "}";
426+
427+
responseStatusCode = httpRequest -> 200;
428+
429+
// Act
430+
kernel.getMemory().searchAsync("fake-index", "a-query", 1, 0.0f, false).block();
431+
432+
Mockito.verify(transform, Mockito.atLeastOnce()).apply(Mockito.any());
433+
}
345434
}

‎java/semantickernel-api/src/main/java/com/microsoft/semantickernel/memory/MemoryStore.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -122,7 +122,7 @@ Mono<Collection<MemoryRecord>> getBatchAsync(
122122
* @param minRelevanceScore The minimum relevance threshold for returned results.
123123
* @param withEmbeddings If true, the embeddings will be returned in the memory records.
124124
* @return A collection of tuples where item1 is a {@link MemoryRecord} and item2 is its
125-
* similarity score as a {@code double}.
125+
* similarity score as a {@code Float}.
126126
*/
127127
Mono<Collection<Tuple2<MemoryRecord, Float>>> getNearestMatchesAsync(
128128
@Nonnull String collectionName,

0 commit comments

Comments
 (0)