Skip to content

Commit d9f0ee0

Browse files
authored
.Net: Add Azure CosmosDB NoSQL Hybrid Keyword Search (#10524)
### Motivation and Context #9399 ### Description - Add an azure ComsosDB NoSQL hybrid search implementation using keywords - Add integration tests and update unit tests. - The required features to implement this is only available in preview versions of the SDK right now. ### Contribution Checklist <!-- Before submitting this PR, please make sure: --> - [x] The code builds clean without any errors or warnings - [x] The PR follows the [SK Contribution Guidelines](https://github.com/microsoft/semantic-kernel/blob/main/CONTRIBUTING.md) and the [pre-submission formatting script](https://github.com/microsoft/semantic-kernel/blob/main/CONTRIBUTING.md#development-scripts) raises no violations - [x] All unit tests pass, and I have added new tests where possible - [x] I didn't break anyone 😄
1 parent 66ad52c commit d9f0ee0

7 files changed

Lines changed: 256 additions & 122 deletions

File tree

‎dotnet/Directory.Packages.props‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -120,7 +120,7 @@
120120
<PackageVersion Include="YamlDotNet" Version="15.3.0" />
121121
<PackageVersion Include="Fluid.Core" Version="2.11.1" />
122122
<!-- Memory stores -->
123-
<PackageVersion Include="Microsoft.Azure.Cosmos" Version="3.45.2" />
123+
<PackageVersion Include="Microsoft.Azure.Cosmos" Version="3.48.0-preview.0" />
124124
<PackageVersion Include="Pgvector" Version="0.2.0" />
125125
<PackageVersion Include="NRedisStack" Version="0.12.0" />
126126
<PackageVersion Include="Milvus.Client" Version="2.3.0-preview.1" />

‎dotnet/src/Connectors/Connectors.AzureCosmosDBNoSQL.UnitTests/AzureCosmosDBNoSQLVectorStoreCollectionQueryBuilderTests.cs‎

Lines changed: 79 additions & 38 deletions
Original file line numberDiff line numberDiff line change
@@ -35,16 +35,18 @@ public void BuildSearchQueryByDefaultReturnsValidQueryDefinition()
3535
.EqualTo("TestProperty2", "test-value-2")
3636
.AnyTagEqualTo("TestProperty3", "test-value-3");
3737

38-
var searchOptions = new VectorSearchOptions { Filter = filter, Skip = 5, Top = 10 };
39-
4038
// Act
4139
var queryDefinition = AzureCosmosDBNoSQLVectorStoreCollectionQueryBuilder.BuildSearchQuery(
4240
vector,
41+
null,
4342
fields,
4443
this._storagePropertyNames,
4544
vectorPropertyName,
45+
null,
4646
ScorePropertyName,
47-
searchOptions);
47+
filter,
48+
10,
49+
5);
4850

4951
var queryText = queryDefinition.QueryText;
5052
var queryParameters = queryDefinition.GetQueryParameters();
@@ -54,22 +56,16 @@ public void BuildSearchQueryByDefaultReturnsValidQueryDefinition()
5456
Assert.Contains("FROM x", queryText);
5557
Assert.Contains("WHERE x.test_property_2 = @cv0 AND ARRAY_CONTAINS(x.test_property_3, @cv1)", queryText);
5658
Assert.Contains("ORDER BY VectorDistance(x.test_property_1, @vector)", queryText);
57-
Assert.Contains("OFFSET @offset LIMIT @limit", queryText);
59+
Assert.Contains("OFFSET 5 LIMIT 10", queryText);
5860

5961
Assert.Equal("@vector", queryParameters[0].Name);
6062
Assert.Equal(vector, queryParameters[0].Value);
6163

62-
Assert.Equal("@offset", queryParameters[1].Name);
63-
Assert.Equal(5, queryParameters[1].Value);
64-
65-
Assert.Equal("@limit", queryParameters[2].Name);
66-
Assert.Equal(10, queryParameters[2].Value);
67-
68-
Assert.Equal("@cv0", queryParameters[3].Name);
69-
Assert.Equal("test-value-2", queryParameters[3].Value);
64+
Assert.Equal("@cv0", queryParameters[1].Name);
65+
Assert.Equal("test-value-2", queryParameters[1].Value);
7066

71-
Assert.Equal("@cv1", queryParameters[4].Name);
72-
Assert.Equal("test-value-3", queryParameters[4].Value);
67+
Assert.Equal("@cv1", queryParameters[2].Name);
68+
Assert.Equal("test-value-3", queryParameters[2].Value);
7369
}
7470

7571
[Fact]
@@ -84,39 +80,38 @@ public void BuildSearchQueryWithoutOffsetReturnsQueryDefinitionWithTopParameter(
8480
.EqualTo("TestProperty2", "test-value-2")
8581
.AnyTagEqualTo("TestProperty3", "test-value-3");
8682

87-
var searchOptions = new VectorSearchOptions { Filter = filter, Top = 10 };
88-
8983
// Act
9084
var queryDefinition = AzureCosmosDBNoSQLVectorStoreCollectionQueryBuilder.BuildSearchQuery(
9185
vector,
86+
null,
9287
fields,
9388
this._storagePropertyNames,
9489
vectorPropertyName,
90+
null,
9591
ScorePropertyName,
96-
searchOptions);
92+
filter,
93+
10,
94+
0);
9795

9896
var queryText = queryDefinition.QueryText;
9997
var queryParameters = queryDefinition.GetQueryParameters();
10098

10199
// Assert
102-
Assert.Contains("SELECT TOP @top x.test_property_1,x.test_property_2,x.test_property_3,VectorDistance(x.test_property_1, @vector) AS TestScore", queryText);
100+
Assert.Contains("SELECT TOP 10 x.test_property_1,x.test_property_2,x.test_property_3,VectorDistance(x.test_property_1, @vector) AS TestScore", queryText);
103101
Assert.Contains("FROM x", queryText);
104102
Assert.Contains("WHERE x.test_property_2 = @cv0 AND ARRAY_CONTAINS(x.test_property_3, @cv1)", queryText);
105103
Assert.Contains("ORDER BY VectorDistance(x.test_property_1, @vector)", queryText);
106104

107-
Assert.DoesNotContain("OFFSET @offset LIMIT @limit", queryText);
105+
Assert.DoesNotContain("OFFSET 0 LIMIT 10", queryText);
108106

109107
Assert.Equal("@vector", queryParameters[0].Name);
110108
Assert.Equal(vector, queryParameters[0].Value);
111109

112-
Assert.Equal("@top", queryParameters[1].Name);
113-
Assert.Equal(10, queryParameters[1].Value);
114-
115-
Assert.Equal("@cv0", queryParameters[2].Name);
116-
Assert.Equal("test-value-2", queryParameters[2].Value);
110+
Assert.Equal("@cv0", queryParameters[1].Name);
111+
Assert.Equal("test-value-2", queryParameters[1].Value);
117112

118-
Assert.Equal("@cv1", queryParameters[3].Name);
119-
Assert.Equal("test-value-3", queryParameters[3].Value);
113+
Assert.Equal("@cv1", queryParameters[2].Name);
114+
Assert.Equal("test-value-3", queryParameters[2].Value);
120115
}
121116

122117
[Fact]
@@ -129,17 +124,19 @@ public void BuildSearchQueryWithInvalidFilterThrowsException()
129124

130125
var filter = new VectorSearchFilter().EqualTo("non-existent-property", "test-value-2");
131126

132-
var searchOptions = new VectorSearchOptions { Filter = filter, Skip = 5, Top = 10 };
133-
134127
// Act & Assert
135128
Assert.Throws<InvalidOperationException>(() =>
136129
AzureCosmosDBNoSQLVectorStoreCollectionQueryBuilder.BuildSearchQuery(
137130
vector,
131+
null,
138132
fields,
139133
this._storagePropertyNames,
140134
vectorPropertyName,
135+
null,
141136
ScorePropertyName,
142-
searchOptions));
137+
filter,
138+
10,
139+
5));
143140
}
144141

145142
[Fact]
@@ -150,31 +147,28 @@ public void BuildSearchQueryWithoutFilterDoesNotContainWhereClause()
150147
var vectorPropertyName = "test_property_1";
151148
var fields = this._storagePropertyNames.Values.ToList();
152149

153-
var searchOptions = new VectorSearchOptions { Skip = 5, Top = 10 };
154-
155150
// Act
156151
var queryDefinition = AzureCosmosDBNoSQLVectorStoreCollectionQueryBuilder.BuildSearchQuery(
157152
vector,
153+
null,
158154
fields,
159155
this._storagePropertyNames,
160156
vectorPropertyName,
157+
null,
161158
ScorePropertyName,
162-
searchOptions);
159+
null,
160+
10,
161+
5);
163162

164163
var queryText = queryDefinition.QueryText;
165164
var queryParameters = queryDefinition.GetQueryParameters();
166165

167166
// Assert
168167
Assert.DoesNotContain("WHERE", queryText);
168+
Assert.Contains("OFFSET 5 LIMIT 10", queryText);
169169

170170
Assert.Equal("@vector", queryParameters[0].Name);
171171
Assert.Equal(vector, queryParameters[0].Value);
172-
173-
Assert.Equal("@offset", queryParameters[1].Name);
174-
Assert.Equal(5, queryParameters[1].Value);
175-
176-
Assert.Equal("@limit", queryParameters[2].Name);
177-
Assert.Equal(10, queryParameters[2].Value);
178172
}
179173

180174
[Fact]
@@ -211,4 +205,51 @@ public void BuildSelectQueryByDefaultReturnsValidQueryDefinition()
211205
Assert.Equal("@pk0", queryParameters[1].Name);
212206
Assert.Equal("partition_key", queryParameters[1].Value);
213207
}
208+
209+
[Fact]
210+
public void BuildSearchQueryWithHybridFieldsReturnsValidHybridQueryDefinition()
211+
{
212+
// Arrange
213+
var vector = new ReadOnlyMemory<float>([1f, 2f, 3f]);
214+
var keywordText = "hybrid";
215+
var vectorPropertyName = "test_property_1";
216+
var textPropertyName = "test_property_2";
217+
var fields = this._storagePropertyNames.Values.ToList();
218+
219+
var filter = new VectorSearchFilter()
220+
.EqualTo("TestProperty2", "test-value-2")
221+
.AnyTagEqualTo("TestProperty3", "test-value-3");
222+
223+
// Act
224+
var queryDefinition = AzureCosmosDBNoSQLVectorStoreCollectionQueryBuilder.BuildSearchQuery(
225+
vector,
226+
[keywordText],
227+
fields,
228+
this._storagePropertyNames,
229+
vectorPropertyName,
230+
textPropertyName,
231+
ScorePropertyName,
232+
filter,
233+
10,
234+
5);
235+
236+
var queryText = queryDefinition.QueryText;
237+
var queryParameters = queryDefinition.GetQueryParameters();
238+
239+
// Assert
240+
Assert.Contains("SELECT x.test_property_1,x.test_property_2,x.test_property_3,VectorDistance(x.test_property_1, @vector) AS TestScore", queryText);
241+
Assert.Contains("FROM x", queryText);
242+
Assert.Contains("WHERE x.test_property_2 = @cv0 AND ARRAY_CONTAINS(x.test_property_3, @cv1)", queryText);
243+
Assert.Contains("ORDER BY RANK RRF(VectorDistance(x.test_property_1, @vector), FullTextScore(x.test_property_2, [\"hybrid\"]))", queryText);
244+
Assert.Contains("OFFSET 5 LIMIT 10", queryText);
245+
246+
Assert.Equal("@vector", queryParameters[0].Name);
247+
Assert.Equal(vector, queryParameters[0].Value);
248+
249+
Assert.Equal("@cv0", queryParameters[1].Name);
250+
Assert.Equal("test-value-2", queryParameters[1].Value);
251+
252+
Assert.Equal("@cv1", queryParameters[2].Name);
253+
Assert.Equal("test-value-3", queryParameters[2].Value);
254+
}
214255
}

‎dotnet/src/Connectors/Connectors.Memory.AzureCosmosDBNoSQL/AzureCosmosDBNoSQLVectorStoreCollectionQueryBuilder.cs‎

Lines changed: 29 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -23,28 +23,38 @@ internal static class AzureCosmosDBNoSQLVectorStoreCollectionQueryBuilder
2323
/// </summary>
2424
public static QueryDefinition BuildSearchQuery<TVector>(
2525
TVector vector,
26+
ICollection<string>? keywords,
2627
List<string> fields,
2728
Dictionary<string, string> storagePropertyNames,
2829
string vectorPropertyName,
30+
string? textPropertyName,
2931
string scorePropertyName,
30-
VectorSearchOptions searchOptions)
32+
VectorSearchFilter? vectorSearchFilter,
33+
int top,
34+
int skip)
3135
{
3236
Verify.NotNull(vector);
3337

3438
const string VectorVariableName = "@vector";
35-
const string OffsetVariableName = "@offset";
36-
const string LimitVariableName = "@limit";
37-
const string TopVariableName = "@top";
39+
// TODO: Use parameterized query for keywords when FullTextScore with parameters is supported.
40+
//const string KeywordsVariableName = "@keywords";
3841

3942
var tableVariableName = AzureCosmosDBNoSQLConstants.TableQueryVariableName;
4043

4144
var fieldsArgument = fields.Select(field => $"{tableVariableName}.{field}");
4245
var vectorDistanceArgument = $"VectorDistance({tableVariableName}.{vectorPropertyName}, {VectorVariableName})";
4346
var vectorDistanceArgumentWithAlias = $"{vectorDistanceArgument} AS {scorePropertyName}";
4447

48+
// Passing keywords using a parameter is not yet supported for FullTextScore so doing some crude string sanitization in the mean time to frustrate script injection.
49+
var sanitizedKeywords = keywords is not null ? keywords.Select(x => x.Replace("\"", "")) : null;
50+
var formattedKeywords = sanitizedKeywords is not null ? $"[\"{string.Join("\", \"", sanitizedKeywords)}\"]" : null;
51+
var fullTextScoreArgument = textPropertyName is not null && keywords is not null ? $"FullTextScore({tableVariableName}.{textPropertyName}, {formattedKeywords})" : null;
52+
53+
var rankingArgument = fullTextScoreArgument is null ? vectorDistanceArgument : $"RANK RRF({vectorDistanceArgument}, {fullTextScoreArgument})";
54+
4555
var selectClauseArguments = string.Join(SelectClauseDelimiter, [.. fieldsArgument, vectorDistanceArgumentWithAlias]);
4656

47-
var filter = BuildSearchFilter(searchOptions.Filter, storagePropertyNames);
57+
var filter = BuildSearchFilter(vectorSearchFilter, storagePropertyNames);
4858

4959
var filterQueryParameters = filter?.QueryParameters;
5060
var filterWhereClauseArguments = filter?.WhereClauseArguments;
@@ -58,8 +68,9 @@ public static QueryDefinition BuildSearchQuery<TVector>(
5868
string.Empty;
5969

6070
// If Offset is not configured, use Top parameter instead of Limit/Offset
61-
// since it's more optimized.
62-
var topArgument = searchOptions.Skip == 0 ? $"TOP {TopVariableName} " : string.Empty;
71+
// since it's more optimized. Hybrid search doesn't allow top to be passed as a parameter
72+
// so directly add it to the query here.
73+
var topArgument = skip == 0 ? $"TOP {top} " : string.Empty;
6374

6475
var builder = new StringBuilder();
6576

@@ -71,19 +82,21 @@ public static QueryDefinition BuildSearchQuery<TVector>(
7182
builder.AppendLine($"WHERE {string.Join(AndConditionDelimiter, filterWhereClauseArguments)}");
7283
}
7384

74-
builder.AppendLine($"ORDER BY {vectorDistanceArgument}");
85+
builder.AppendLine($"ORDER BY {rankingArgument}");
7586

76-
if (!string.IsNullOrEmpty(topArgument))
87+
if (string.IsNullOrEmpty(topArgument))
7788
{
78-
queryParameters.Add(TopVariableName, searchOptions.Top);
79-
}
80-
else
81-
{
82-
builder.AppendLine($"OFFSET {OffsetVariableName} LIMIT {LimitVariableName}");
83-
queryParameters.Add(OffsetVariableName, searchOptions.Skip);
84-
queryParameters.Add(LimitVariableName, searchOptions.Top);
89+
// Hybrid search doesn't allow offset and limit to be passed as parameters
90+
// so directly add it to the query here.
91+
builder.AppendLine($"OFFSET {skip} LIMIT {top}");
8592
}
8693

94+
// TODO: Use parameterized query for keywords when FullTextScore with parameters is supported.
95+
//if (fullTextScoreArgument is not null)
96+
//{
97+
// queryParameters.Add(KeywordsVariableName, keywords!.ToArray());
98+
//}
99+
87100
var queryDefinition = new QueryDefinition(builder.ToString());
88101

89102
if (filterQueryParameters is { Count: > 0 })

0 commit comments

Comments
 (0)