Skip to content

Commit fc68934

Browse files
committed
Merging SK main upstream with qdrant connector branch for PR push to SK public repo-TEW
1 parent d31554f commit fc68934

7 files changed

Lines changed: 400 additions & 0 deletions

File tree

Lines changed: 103 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,103 @@
1+
// Copyright (c) Microsoft. All rights reserved.
2+
3+
using System;
4+
using System.Collections.Generic;
5+
using System.Threading.Tasks;
6+
using IntegrationTests.TestSettings;
7+
using Microsoft.Extensions.Configuration;
8+
using Microsoft.SemanticKernel.AI.Embeddings;
9+
using Microsoft.SemanticKernel.AI.OpenAI.Services;
10+
using Xunit;
11+
using Xunit.Abstractions;
12+
13+
namespace IntegrationTests.AI;
14+
15+
public sealed class OpenAIEmbeddingTests : IDisposable
16+
{
17+
private const int AdaVectorLength = 1536;
18+
private readonly IConfigurationRoot _configuration;
19+
20+
public OpenAIEmbeddingTests(ITestOutputHelper output)
21+
{
22+
this._testOutputHelper = new RedirectOutput(output);
23+
Console.SetOut(this._testOutputHelper);
24+
25+
// Load configuration
26+
this._configuration = new ConfigurationBuilder()
27+
.AddJsonFile(path: "testsettings.json", optional: false, reloadOnChange: true)
28+
.AddJsonFile(path: "testsettings.development.json", optional: true, reloadOnChange: true)
29+
.AddEnvironmentVariables()
30+
.AddUserSecrets<OpenAIEmbeddingTests>()
31+
.Build();
32+
}
33+
34+
[Theory(Skip = "OpenAI will often throttle requests. This test is for manual verification.")]
35+
[InlineData("test sentence")]
36+
public async Task OpenAITestAsync(string testInputString)
37+
{
38+
// Arrange
39+
OpenAIConfiguration? openAIConfiguration = this._configuration.GetSection("OpenAIEmbeddings").Get<OpenAIConfiguration>();
40+
Assert.NotNull(openAIConfiguration);
41+
42+
OpenAITextEmbeddings embeddingGenerator = new OpenAITextEmbeddings(openAIConfiguration.ModelId, openAIConfiguration.ApiKey);
43+
44+
// Act
45+
var singleResult = await embeddingGenerator.GenerateEmbeddingAsync(testInputString);
46+
var batchResult = await embeddingGenerator.GenerateEmbeddingsAsync(new List<string> { testInputString, testInputString, testInputString });
47+
48+
// Assert
49+
Assert.Equal(AdaVectorLength, singleResult.Count);
50+
Assert.Equal(3, batchResult.Count);
51+
52+
embeddingGenerator.Dispose();
53+
}
54+
55+
[Theory]
56+
[InlineData("test sentence")]
57+
public async Task AzureOpenAITestAsync(string testInputString)
58+
{
59+
// Arrange
60+
AzureOpenAIConfiguration? azureOpenAIConfiguration = this._configuration.GetSection("AzureOpenAIEmbeddings").Get<AzureOpenAIConfiguration>();
61+
Assert.NotNull(azureOpenAIConfiguration);
62+
63+
AzureTextEmbeddings embeddingGenerator = new AzureTextEmbeddings(azureOpenAIConfiguration.DeploymentName,
64+
azureOpenAIConfiguration.Endpoint,
65+
azureOpenAIConfiguration.ApiKey,
66+
"2022-12-01");
67+
68+
// Act
69+
var singleResult = await embeddingGenerator.GenerateEmbeddingAsync(testInputString);
70+
var batchResult = await embeddingGenerator.GenerateEmbeddingsAsync(new List<string> { testInputString, testInputString, testInputString });
71+
72+
// Assert
73+
Assert.Equal(AdaVectorLength, singleResult.Count);
74+
Assert.Equal(3, batchResult.Count);
75+
76+
embeddingGenerator.Dispose();
77+
}
78+
79+
#region internals
80+
81+
private readonly RedirectOutput _testOutputHelper;
82+
83+
public void Dispose()
84+
{
85+
this.Dispose(true);
86+
GC.SuppressFinalize(this);
87+
}
88+
89+
~OpenAIEmbeddingTests()
90+
{
91+
this.Dispose(false);
92+
}
93+
94+
private void Dispose(bool disposing)
95+
{
96+
if (disposing)
97+
{
98+
this._testOutputHelper.Dispose();
99+
}
100+
}
101+
102+
#endregion
103+
}
Lines changed: 45 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,45 @@
1+
// Copyright (c) Microsoft. All rights reserved.
2+
3+
using System;
4+
using System.Collections.Generic;
5+
using System.Text.Json.Serialization;
6+
using Microsoft.SemanticKernel.AI.Embeddings;
7+
8+
namespace Qdrant.DotNet;
9+
10+
public class VectorRecordMetadata
11+
{
12+
/// <summary>
13+
/// An ID that allows to map the vector to some external resource
14+
/// </summary>
15+
// [JsonPropertyName("id")]
16+
// public string Id { get; set; } = string.Empty;
17+
18+
19+
20+
/// <summary>
21+
/// Optional payload to include
22+
/// </summary>
23+
[JsonPropertyName("payload")]
24+
public Dictionary<string, object> Payload { get; set; } = new Dictionary<string, object>();
25+
26+
/// <summary>
27+
/// Optional tags to allow filtering, e.g. "email", "message", "photo"
28+
/// </summary>
29+
[JsonPropertyName("tags")]
30+
public List<string> Tags { get; set; } = new();
31+
}
32+
33+
public class VectorRecord : VectorRecordMetadata, DataEntry<IEmbeddingWithMetadata<TEmbedding>>
34+
{
35+
/// <summary>
36+
/// Vector data
37+
/// </summary>
38+
[JsonPropertyName("vector")]
39+
public Embedding<TEmbedding> Vector; //= Array.Empty<float>();
40+
41+
42+
//public Embedding<float> Embedding => new(this.Vector);
43+
44+
45+
}
Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,28 @@
1+
<Project Sdk="Microsoft.NET.Sdk">
2+
3+
<PropertyGroup>
4+
<TargetFramework>netstandard2.1</TargetFramework>
5+
<Nullable>enable</Nullable>
6+
<AssemblyName>Qdrant.DotNet</AssemblyName>
7+
<RootNamespace>Qdrant.DotNet</RootNamespace>
8+
<LangVersion>10</LangVersion>
9+
<Version>0.1</Version>
10+
<Title>Qdrant DotNet</Title>
11+
<Authors>Microsoft</Authors>
12+
<PackageProjectUrl>https://github.com/microsoft/qdrant-dotnet</PackageProjectUrl>
13+
<RepositoryUrl>https://github.com/microsoft/qdrant-dotnet</RepositoryUrl>
14+
<Company>Microsoft</Company>
15+
<Product>Qdrant DotNet</Product>
16+
<PackageTags>SDK</PackageTags>
17+
</PropertyGroup>
18+
19+
<ItemGroup>
20+
<PackageReference Include="Microsoft.CSharp" VersionOverride="4.7.0" />
21+
<PackageReference Include="Microsoft.Extensions.Configuration.Abstractions" VersionOverride="5.0.0" />
22+
<PackageReference Include="Microsoft.Extensions.Logging.Console" VersionOverride="5.0.0" />
23+
<PackageReference Include="Newtonsoft.Json" VersionOverride="13.0.2" />
24+
<PackageReference Include="System.Linq.Async" VersionOverride="6.0.1" />
25+
<PackageReference Include="System.Text.Json" VersionOverride="6.0.7" />
26+
</ItemGroup>
27+
28+
</Project>
Lines changed: 130 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,130 @@
1+
// Copyright (c) Microsoft. All rights reserved.
2+
3+
using System;
4+
using System.Collections.Generic;
5+
using System.Collections.Concurrent;
6+
using System.Diagnostics.CodeAnalysis;
7+
using System.Globalization;
8+
using System.Runtime.CompilerServices;
9+
using System.Text.Json;
10+
using System.Threading;
11+
using System.Threading.Tasks;
12+
using Microsoft.SemanticKernel.Memory;
13+
using Microsoft.SemanticKernel.Memory.Storage;
14+
using Qdrant.DotNet;
15+
using Qdrant.DotNet.Internal.Diagnostics;
16+
17+
namespace Microsoft.SemanticKernel.Skills.Memory.VectorDB;
18+
19+
/// <summary>
20+
/// /// An implementation of <see cref="IDataStore{TValue}"/> backed by a Qdrant Vector database.
21+
/// </summary>
22+
/// <remarks>The vector data is saved to Qdrant Vector Database run locally or in cloud, specified in the constructor.
23+
/// The data persists between subsequent instances. </remarks>
24+
/// <typeparam name="TValue">The type of data to be stored in this data store.</typeparam>
25+
public class QdrantDataStore<TValue> : IDataStore<TValue>
26+
{
27+
/// <summary>
28+
/// Connect a Qdrant Vector database
29+
/// </summary>
30+
/// <param name="endpoint">IP Address or URL endpoint to Qdrant Vectore Database.</param>
31+
/// <param name="port">Port to be used for Qdrant Vector DB</param>
32+
public static QdrantDataStore<TValue> ConnectVectorDB(string endpoint, int port)
33+
{
34+
QdrantDb vectorDBConnection = QdrantVectorDB.CreateVectorDBConnection(endpoint, port);
35+
return new QdrantDataStore<QdrantDb>(vectorDBConnection);
36+
}
37+
38+
/// <inheritdoc/>
39+
public IAsyncEnumerable<string> GetContainersAsync()
40+
{
41+
//Go get all collection in DB of them everytime
42+
return this._qdrantstore.Keys.ToAsyncEnumerable();
43+
}
44+
45+
/// <inheritdoc/>
46+
public IAsyncEnumerable<DataEntry<TValue>> GetAllAsync(string container, CancellationToken cancel = default)
47+
{
48+
49+
IVectorDbCollection vdbCollection = this._qdrantstore.GetOrNull(container);
50+
if (vdbCollection != null) {return vdbCollection; }
51+
52+
try
53+
{
54+
vdbCollection = await this._qdrantdbClient.GetCollection(container);
55+
this._qdrantstore.AddOrUpdate(container, vdbCollection);
56+
}
57+
catch(VectorDBException ex)
58+
{
59+
throw new VectorDbException($"Error while getting collection {container} from Qdrant Vector DB", ex);
60+
}
61+
62+
}
63+
64+
/// <inheritdoc/>
65+
public Task<DataEntry<TValue>?> GetAsync(string container, string key, CancellationToken cancel = default)
66+
{
67+
IVectorDbCollection vdbCollection = this._qdrantstore.GetOrNull(container);
68+
69+
return Task.FromResult<DataEntry<TValue>?>(null);
70+
}
71+
72+
/// <inheritdoc/>
73+
Task<DataEntry<TValue>> PutAsync(string container, DataEntry<TValue> data, CancellationToken cancel = default){}
74+
75+
/// <inheritdoc/>
76+
Task RemoveAsync(string container, string key, CancellationToken cancel = default);
77+
78+
#region private ================================================================================
79+
80+
private readonly QdrantDb _qdrantdbClient;
81+
private readonly ConcurrentDictionary<string, IVectorDbCollection> _qdrantstore = new();
82+
83+
/// <summary>
84+
/// Constructor
85+
/// </summary>
86+
/// <param name="qdrantdbClient">Qdrant VectorDB Client</param>
87+
private QdrantDataStore(QdrantDb qdrantdbClient)
88+
{
89+
this._qdrantdbClient = qdrantdbClient;
90+
}
91+
92+
private static string? ValueToString(TValue? value)
93+
{
94+
if (value != null)
95+
{
96+
if (typeof(TValue) == typeof(string))
97+
{
98+
return value.ToString();
99+
}
100+
101+
return JsonSerializer.Serialize(value);
102+
}
103+
104+
return null;
105+
}
106+
107+
private static string? ToTimestampString(DateTimeOffset? timestamp)
108+
{
109+
return timestamp?.ToString("u", CultureInfo.InvariantCulture);
110+
}
111+
112+
private static DateTimeOffset? ParseTimestamp(string? str)
113+
{
114+
if (!string.IsNullOrEmpty(str)
115+
&& DateTimeOffset.TryParse(str, CultureInfo.InvariantCulture, DateTimeStyles.AssumeUniversal, out DateTimeOffset timestamp))
116+
{
117+
return timestamp;
118+
}
119+
120+
return null;
121+
}
122+
123+
#endregion
124+
125+
}
126+
127+
128+
129+
130+
Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,28 @@
1+
// Copyright (c) Microsoft. All rights reserved.
2+
3+
using System;
4+
using System.Collections.Generic;
5+
using System.Collections.Concurrent;
6+
using System.Diagnostics.CodeAnalysis;
7+
using System.Globalization;
8+
using System.Runtime.CompilerServices;
9+
using System.Text.Json;
10+
using System.Threading;
11+
using System.Threading.Tasks;
12+
using Microsoft.SemanticKernel.Memory.Storage;
13+
14+
namespace Microsoft.SemanticKernel.Skills.Memory.VectorDB;
15+
16+
public class QdrantMemoryStore<TEmbedding> : QdrantDataStore<IEmbeddingWithMetadata<TEmbedding>>, IMemoryStore<TEmbedding>
17+
where TEmbedding : unmanaged
18+
{
19+
20+
21+
}
22+
23+
#region private ================================================================================
24+
private readonly ConcurrentDictionary<string, IVectorDbCollection> m_containers = new();
25+
private readonly int m_dimension;
26+
private readonly IVectorDBClient _qdrantdb;
27+
28+
#endregion

0 commit comments

Comments
 (0)