Skip to content

Commit 4368b5e

Browse files
authored
.Net: Add mongodb hybrid keyword search implementation (#10525)
### Motivation and Context #9399 ### Description - Add a MongoDB hybrid search implementation using keywords - Add integration tests and update unit tests. ### Contribution Checklist <!-- Before submitting this PR, please make sure: --> - [x] The code builds clean without any errors or warnings - [x] The PR follows the [SK Contribution Guidelines](https://github.com/microsoft/semantic-kernel/blob/main/CONTRIBUTING.md) and the [pre-submission formatting script](https://github.com/microsoft/semantic-kernel/blob/main/CONTRIBUTING.md#development-scripts) raises no violations - [x] All unit tests pass, and I have added new tests where possible - [x] I didn't break anyone 😄
1 parent d9f0ee0 commit 4368b5e

6 files changed

Lines changed: 443 additions & 73 deletions

File tree

‎dotnet/src/Connectors/Connectors.Memory.MongoDB/MongoDBVectorStoreCollectionCreateMapping.cs‎

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -75,6 +75,34 @@ public static BsonArray GetFilterableDataIndexFields(
7575
return indexArray;
7676
}
7777

78+
/// <summary>
79+
/// Returns a list of of fields to index for full text search data properties.
80+
/// </summary>
81+
/// <param name="dataProperties">Collection of data properties for index creation.</param>
82+
/// <param name="storagePropertyNames">A dictionary that maps from a property name to the storage name.</param>
83+
public static List<BsonElement> GetFullTextSearchableDataIndexFields(
84+
IReadOnlyList<VectorStoreRecordDataProperty> dataProperties,
85+
Dictionary<string, string> storagePropertyNames)
86+
{
87+
var fieldElements = new List<BsonElement>();
88+
89+
// Create separate index for each data property
90+
foreach (var property in dataProperties)
91+
{
92+
if (property.IsFullTextSearchable)
93+
{
94+
var dataPropertyName = storagePropertyNames[property.DataModelPropertyName];
95+
96+
fieldElements.Add(new BsonElement(dataPropertyName, new BsonArray()
97+
{
98+
new BsonDocument() { { "type", "string" }, }
99+
}));
100+
}
101+
}
102+
103+
return fieldElements;
104+
}
105+
78106
/// <summary>
79107
/// More information about MongoDB distance functions here: <see href="https://www.mongodb.com/docs/atlas/atlas-vector-search/vector-search-type/#atlas-vector-search-index-fields" />.
80108
/// </summary>

‎dotnet/src/Connectors/Connectors.Memory.MongoDB/MongoDBVectorStoreCollectionSearchMapping.cs‎

Lines changed: 254 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -13,8 +13,8 @@ namespace Microsoft.SemanticKernel.Connectors.MongoDB;
1313
/// </summary>
1414
internal static class MongoDBVectorStoreCollectionSearchMapping
1515
{
16-
/// <summary>Returns distance function specified on vector property or default <see cref="MongoDBConstants.DefaultDistanceFunction"/>.</summary>
17-
public static string GetVectorPropertyDistanceFunction(string? distanceFunction) => !string.IsNullOrWhiteSpace(distanceFunction) ? distanceFunction! : MongoDBConstants.DefaultDistanceFunction;
16+
/// <summary>Returns distance function specified on vector property or default.</summary>
17+
public static string GetVectorPropertyDistanceFunction(string? distanceFunction) => !string.IsNullOrWhiteSpace(distanceFunction) ? distanceFunction! : DistanceFunction.CosineSimilarity;
1818

1919
/// <summary>
2020
/// Build MongoDB filter from the provided <see cref="VectorSearchFilter"/>.
@@ -117,13 +117,263 @@ public static BsonDocument GetProjectionQuery(string scorePropertyName, string d
117117
{
118118
return new BsonDocument
119119
{
120-
{ "$project",
121-
new BsonDocument
120+
{
121+
"$project", new BsonDocument
122122
{
123123
{ scorePropertyName, new BsonDocument { { "$meta", "vectorSearchScore" } } },
124124
{ documentPropertyName, "$$ROOT" }
125125
}
126126
}
127127
};
128128
}
129+
130+
/// <summary>Returns a pipeline for hybrid search using vector search and full text search.</summary>
131+
public static BsonDocument[] GetHybridSearchPipeline<TVector>(
132+
TVector vector,
133+
ICollection<string> keywords,
134+
string collectionName,
135+
string vectorIndexName,
136+
string fullTextSearchIndexName,
137+
string vectorPropertyName,
138+
string textPropertyName,
139+
string scorePropertyName,
140+
string documentPropertyName,
141+
int limit,
142+
int numCandidates,
143+
BsonDocument? filter)
144+
{
145+
// Create the FullTextSearch pipeline first.
146+
var ftsPipeline = new List<BsonDocument>
147+
{
148+
// The full text search stage.
149+
GetFullTextSearchQuery(keywords, fullTextSearchIndexName, textPropertyName, filter),
150+
// Limit the results to the maximum that we may require.
151+
new()
152+
{
153+
{
154+
"$limit", limit
155+
}
156+
},
157+
// Converts the list of documents to a single document with an array property containing all the source documents.
158+
GroupDocsSection(),
159+
// Creates separate documents again where each has a new rank property based on the index of the document.
160+
UnwindDocsArraySection(),
161+
// Add a weighted score based on the rank of the document.
162+
AddScore("fts_score", 0.9),
163+
// Project the score, the id and the original document as properties, so that we can join with the vector search results on id.
164+
ProjectWithScore("fts_score"),
165+
};
166+
167+
// Add filtering to the FullTextSearch pipeline if filter is provided.
168+
if (filter is not null)
169+
{
170+
// Insert filter at the second position, since
171+
// MongoDB requires search to be the first stage.
172+
ftsPipeline.Insert(1, new BsonDocument
173+
{
174+
{
175+
"$match", filter
176+
}
177+
});
178+
}
179+
180+
// Create combined pipeline with the vector search part first.
181+
var pipeline = new BsonDocument[]
182+
{
183+
// The vector search stage.
184+
GetSearchQuery(vector, vectorIndexName, vectorPropertyName, limit, numCandidates, filter),
185+
// Converts the list of documents to a single document with an array property containing all the source documents.
186+
GroupDocsSection(),
187+
// Creates separate documents again where each has a new rank property based on the index of the document.
188+
UnwindDocsArraySection(),
189+
// Add a weighted score based on the rank of the document.
190+
AddScore("vs_score", 0.1),
191+
// Project the score, the id and the original document as properties, so that we can join with the vector search results on id.
192+
ProjectWithScore("vs_score"),
193+
// Union the vector search results with the results from the full text search pipeilne.
194+
new()
195+
{
196+
{
197+
"$unionWith", new BsonDocument
198+
{
199+
{ "coll", collectionName },
200+
{ "pipeline", new BsonArray(ftsPipeline) }
201+
}
202+
}
203+
},
204+
// Group by id and store scores from each pipeline, so that we don't have duplicate documents.
205+
new()
206+
{
207+
{
208+
"$group", new BsonDocument
209+
{
210+
{ "_id", "$_id" },
211+
{ "docs", new BsonDocument { { "$first", "$docs" } } },
212+
{ "vs_score", new BsonDocument { { "$max", "$vs_score" } } },
213+
{ "fts_score", new BsonDocument { { "$max", "$fts_score" } } }
214+
}
215+
}
216+
},
217+
// If a score is missing (i.e. the document was only found in the other pipeline), default the missing score to 0.
218+
new()
219+
{
220+
{
221+
"$project", new BsonDocument
222+
{
223+
{ "_id", 1 },
224+
{ "docs", 1 },
225+
{ "vs_score", new BsonDocument { { "$ifNull", new BsonArray { "$vs_score", 0 } } } },
226+
{ "fts_score", new BsonDocument { { "$ifNull", new BsonArray { "$fts_score", 0 } } } }
227+
}
228+
}
229+
},
230+
// Calculate a combined score based on the vector search and full text search scores.
231+
new()
232+
{
233+
{
234+
"$project", new BsonDocument
235+
{
236+
{ scorePropertyName, new BsonDocument { { "$add", new BsonArray { "$fts_score", "$vs_score" } } } },
237+
{ "vs_score", 1 },
238+
{ "fts_score", 1 },
239+
{ documentPropertyName, "$docs" }
240+
}
241+
}
242+
},
243+
// Sort by score desc.
244+
new()
245+
{
246+
{
247+
"$sort", new BsonDocument
248+
{
249+
{ scorePropertyName, -1 }
250+
}
251+
}
252+
},
253+
// Take the required N results.
254+
new()
255+
{
256+
{
257+
"$limit", limit
258+
}
259+
},
260+
};
261+
262+
return pipeline;
263+
}
264+
265+
/// <summary>Builds the full text search query stage.</summary>
266+
private static BsonDocument GetFullTextSearchQuery(
267+
ICollection<string> keywords,
268+
string fullTextSearchIndexName,
269+
string textPropertyName,
270+
BsonDocument? filter)
271+
{
272+
var fullTextSearchQuery = new BsonDocument
273+
{
274+
{
275+
"$search", new BsonDocument
276+
{
277+
{ "index", fullTextSearchIndexName },
278+
{ "text",
279+
new BsonDocument
280+
{
281+
{ "query", new BsonArray(keywords) },
282+
{ "path", textPropertyName },
283+
{ "matchCriteria", "any" }
284+
}
285+
}
286+
}
287+
}
288+
};
289+
290+
return fullTextSearchQuery;
291+
}
292+
293+
/// <summary>Create a stage that groups all documents into a single document with an array property containing all the source documents.</summary>
294+
private static BsonDocument GroupDocsSection()
295+
{
296+
return new BsonDocument
297+
{
298+
{
299+
"$group", new BsonDocument
300+
{
301+
{ "_id", BsonNull.Value },
302+
{ "docs", new BsonDocument { { "$push", "$$ROOT" } } }
303+
}
304+
}
305+
};
306+
}
307+
308+
/// <summary>Creates a stage that splits an array of documents from a single document into separate documents and adds an index property for each document based on its index in the array.</summary>
309+
private static BsonDocument UnwindDocsArraySection()
310+
{
311+
return new BsonDocument
312+
{
313+
{
314+
"$unwind", new BsonDocument
315+
{
316+
{ "path", "$docs" },
317+
{ "includeArrayIndex", "rank" }
318+
}
319+
}
320+
};
321+
}
322+
323+
/// <summary>Adds a weighted score to each document based on the rank property on the document.</summary>
324+
private static BsonDocument AddScore(string scoreName, double weight)
325+
{
326+
return new()
327+
{
328+
{
329+
"$addFields", new BsonDocument
330+
{
331+
{
332+
scoreName, new BsonDocument
333+
{
334+
{
335+
"$multiply", new BsonArray()
336+
{
337+
weight,
338+
new BsonDocument
339+
{
340+
{ "$divide", new BsonArray()
341+
{
342+
1.0,
343+
new BsonDocument
344+
{
345+
{ "$add", new BsonArray()
346+
{
347+
"$rank",
348+
60
349+
}
350+
}
351+
}
352+
}
353+
}
354+
}
355+
}
356+
}
357+
}
358+
},
359+
}
360+
}
361+
};
362+
}
363+
364+
/// <summary>Projects the score, the id and the original document as properties.</summary>
365+
private static BsonDocument ProjectWithScore(string scoreName)
366+
{
367+
return new()
368+
{
369+
{
370+
"$project", new BsonDocument
371+
{
372+
{ scoreName, 1 },
373+
{ "_id", "$docs._id" },
374+
{ "docs", "$docs" }
375+
}
376+
}
377+
};
378+
}
129379
}

0 commit comments

Comments
 (0)