Skip to content

Commit 4a594e3

Browse files
committed
Updating full Memory file that was to be copied to forked repo-TEW
1 parent 962c437 commit 4a594e3

1 file changed

Lines changed: 172 additions & 0 deletions

File tree

Lines changed: 172 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,172 @@
1+
// Copyright (c) Microsoft. All rights reserved.
2+
3+
using System;
4+
using System.Collections.Generic;
5+
using System.Collections.Concurrent;
6+
using System.Threading;
7+
using System.Threading.Tasks;
8+
using Microsoft.SemanticKernel.AI.Embeddings;
9+
using Microsoft.SemanticKernel.Memory;
10+
using Microsoft.SemanticKernel.Memory.Storage;
11+
using Qdrant.DotNet;
12+
using Qdrant.DotNet.Internal.Diagnostics;
13+
using System.Linq;
14+
using Qdrant.DotNet.Internal;
15+
16+
17+
namespace Microsoft.SemanticKernel.Connectors.VectorDB;
18+
19+
/// <summary>
20+
/// An implementation of <see cref="ILongTermMemoryStore{TEmbedding}"/> which is an embedding of type float
21+
/// backed by a Qdrant Vector database. The type of Embedding (float) to be stored in this data store which has the interface of
22+
/// <see cref="IDataStore{VectorRecordData}"/>
23+
/// </summary>
24+
/// <remarks>The Embedding data is saved to a Qdrant Vector database instance specified in the constructor by url and port.
25+
/// The embedding data persists between subsequent instances and has similarity search capability.
26+
/// </remarks>
27+
public class QdrantMemoryStore : ILongTermMemoryStore<float>
28+
{
29+
public QdrantMemoryStore(string host, int port)
30+
{
31+
this._qdrantclient = new QdrantVectorDb(host, port);
32+
}
33+
34+
public async IAsyncEnumerable<string> GetCollectionsAsync(CancellationToken cancel = default)
35+
{
36+
yield return null;
37+
}
38+
39+
40+
41+
public async Task<DataEntry<VectorRecordData<float>>?> GetAsync(string collection, string key, CancellationToken cancel = default)
42+
{
43+
DataEntry<VectorRecordData<float>>? vectorresult = null;
44+
45+
if (this._qdrantdata.ContainsKey(collection) && this._qdrantdata[collection].ContainsKey(key) )
46+
{
47+
vectorresult = this._qdrantdata[collection][key];
48+
}
49+
else
50+
{
51+
try
52+
{
53+
var vectordata = await this._qdrantclient.GetVectorDataAsync(collection, key);
54+
if (vectordata != null)
55+
{
56+
if (!this._qdrantdata.ContainsKey(collection))
57+
{
58+
this._qdrantdata.TryAdd(collection, new ConcurrentDictionary<string, DataEntry<VectorRecordData<float>>>());
59+
this._qdrantdata[collection].TryAdd(key,vectordata);
60+
}
61+
else
62+
{
63+
this._qdrantdata[collection].TryAdd(key,vectordata);
64+
}
65+
66+
vectorresult = vectordata;
67+
}
68+
}
69+
catch(Exception ex)
70+
{
71+
throw new VectorDbException($"Failed to get vector data from Qdrant {ex.Message}");
72+
}
73+
}
74+
75+
return vectorresult;
76+
}
77+
78+
public async Task<DataEntry<VectorRecordData<float>>> PutAsync(string collection, DataEntry<VectorRecordData<float>> data, CancellationToken cancel = default)
79+
{
80+
DataEntry<VectorRecordData<float>> vectordata; //= new DataEntry<VectorRecordData<float>>(data.Key, data.Value);
81+
bool result = await this._qdrantclient.PutVectorDataAsync<DataEntry<VectorRecordData<float>>> (collection, data.Key, data, data.Value.Payload);
82+
if (!result)
83+
{
84+
throw new VectorDbException("Failed to put vector data into Qdrant");
85+
}
86+
87+
vectordata = await this._qdrantclient.GetVectorDataAsync(collection, data.Key);
88+
89+
if (vectordata == null)
90+
{
91+
throw new VectorDbException("Failed to put and retrieve vector data from Qdrant");
92+
}
93+
return vectordata;
94+
95+
}
96+
97+
public Task RemoveAsync(string collection, string key, CancellationToken cancel = default)
98+
{
99+
bool result;
100+
try
101+
{
102+
result = this._qdrantclient.DeleteVectorDataAsync(collection, key).Result;
103+
if (result)
104+
{
105+
if (this._qdrantdata.ContainsKey(collection) && this._qdrantdata[collection].ContainsKey(key) )
106+
{
107+
this._qdrantdata[collection].TryRemove(key, out _);
108+
}
109+
}
110+
}
111+
catch(Exception ex)
112+
{
113+
throw new VectorDbException($"Failed to remove vector data from Qdrant {ex.Message}");
114+
}
115+
return Task.CompletedTask;
116+
}
117+
118+
public async IAsyncEnumerable<(VectorRecordData<float>, double)> GetNearestMatchesAsync(string collection, Embedding<float> embedding, int limit = 1, double minRelevanceScore = 0)
119+
{
120+
121+
DataEntry<VectorRecordData<float>> vectordata = new DataEntry<VectorRecordData<float>>()
122+
{
123+
Value = new VectorRecordData<float>(embedding, new Dictionary<string, object>(), new List<string>()),
124+
Timestamp = DateTime.Now,
125+
};
126+
127+
128+
var result = await this._qdrantclient.SearchVectorBySimilarityAsync(collection, vectordata,minRelevanceScore);
129+
await foreach (var rd in result.ToAsyncEnumerable())
130+
{
131+
yield return (rd.Key.Value!, rd.Value);
132+
}
133+
}
134+
135+
async IAsyncEnumerable<(IEmbeddingWithMetadata<float>, double)> IEmbeddingIndex<float>.GetNearestMatchesAsync(string collection, Embedding<float> embedding, int limit, double minRelevanceScore)
136+
{
137+
DataEntry<VectorRecordData<float>> vectordata = new DataEntry<VectorRecordData<float>>()
138+
{
139+
Value = new VectorRecordData<float>(embedding, new Dictionary<string, object>(), new List<string>()),
140+
Timestamp = DateTime.Now,
141+
};
142+
var result = await this._qdrantclient.SearchVectorBySimilarityAsync(collection, vectordata, minRelevanceScore);
143+
await foreach (var match in result.ToAsyncEnumerable())
144+
{
145+
yield return (match.Key.Value!, match.Value);
146+
}
147+
148+
}
149+
150+
public IAsyncEnumerable<DataEntry<VectorRecordData<float>>> GetAllAsync(string collection, CancellationToken cancel = default)
151+
{
152+
153+
IAsyncEnumerable<DataEntry<VectorRecordData<float>>> vectorresult;
154+
IVectorDbCollection vectorCollection = this._qdrantclient.GetCollectionDataAsync(collection).Result;
155+
156+
vectorresult = vectorCollection.GetAllVectorsAsync().Result!;
157+
158+
return vectorresult!;
159+
160+
}
161+
162+
163+
164+
#region private ================================================================================
165+
private readonly ConcurrentDictionary<string, IVectorDbCollection> _qdrantcollection = new();
166+
private readonly ConcurrentDictionary<string, ConcurrentDictionary<string, DataEntry<VectorRecordData<float>>>> _qdrantdata = new();
167+
private QdrantVectorDb _qdrantclient;
168+
private readonly int _defaultvectorsize = 1536;
169+
private readonly VectorDistanceType _defaultdistancetype = VectorDistanceType.Cosine;
170+
#endregion
171+
172+
}

0 commit comments

Comments
 (0)