Skip to content

Commit 429e180

Browse files
xzfcxzfc
committed
feat: add CollectionInfo::configuration_status field
1 parent cd406f4 commit 429e180

13 files changed

Lines changed: 228 additions & 7 deletions

File tree

‎docs/grpc/docs.md‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,7 @@
2424
- [CollectionOperationResponse](#qdrant-CollectionOperationResponse)
2525
- [CollectionParams](#qdrant-CollectionParams)
2626
- [CollectionParamsDiff](#qdrant-CollectionParamsDiff)
27+
- [ConfigurationStatus](#qdrant-ConfigurationStatus)
2728
- [CreateAlias](#qdrant-CreateAlias)
2829
- [CreateCollection](#qdrant-CreateCollection)
2930
- [CreateCollection.MetadataEntry](#qdrant-CreateCollection-MetadataEntry)
@@ -608,6 +609,7 @@
608609
| payload_schema | [CollectionInfo.PayloadSchemaEntry](#qdrant-CollectionInfo-PayloadSchemaEntry) | repeated | Collection data types |
609610
| points_count | [uint64](#uint64) | optional | Approximate number of points in the collection |
610611
| indexed_vectors_count | [uint64](#uint64) | optional | Approximate number of indexed vectors in the collection. |
612+
| configuration_status | [ConfigurationStatus](#qdrant-ConfigurationStatus) | | status of configuration inconsistencies |
611613

612614

613615

@@ -686,6 +688,22 @@
686688

687689

688690

691+
<a name="qdrant-ConfigurationStatus"></a>
692+
693+
### ConfigurationStatus
694+
695+
696+
697+
| Field | Type | Label | Description |
698+
| ----- | ---- | ----- | ----------- |
699+
| ok | [bool](#bool) | | |
700+
| warning | [string](#string) | | |
701+
702+
703+
704+
705+
706+
689707
<a name="qdrant-CreateAlias"></a>
690708

691709
### CreateAlias

‎docs/redoc/master/openapi.json‎

Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6595,6 +6595,7 @@
65956595
"type": "object",
65966596
"required": [
65976597
"config",
6598+
"configuration_status",
65986599
"optimizer_status",
65996600
"payload_schema",
66006601
"segments_count",
@@ -6607,6 +6608,9 @@
66076608
"optimizer_status": {
66086609
"$ref": "#/components/schemas/OptimizersStatus"
66096610
},
6611+
"configuration_status": {
6612+
"$ref": "#/components/schemas/ConfigurationStatus"
6613+
},
66106614
"indexed_vectors_count": {
66116615
"description": "Approximate number of indexed vectors in the collection. Indexed vectors in large segments are faster to query, as it is stored in a specialized vector index.",
66126616
"type": "integer",
@@ -6674,6 +6678,31 @@
66746678
}
66756679
]
66766680
},
6681+
"ConfigurationStatus": {
6682+
"description": "Current state of the collection",
6683+
"oneOf": [
6684+
{
6685+
"description": "Configuration is valid",
6686+
"type": "string",
6687+
"enum": [
6688+
"ok"
6689+
]
6690+
},
6691+
{
6692+
"description": "Configuration has warnings",
6693+
"type": "object",
6694+
"required": [
6695+
"warning"
6696+
],
6697+
"properties": {
6698+
"warning": {
6699+
"type": "string"
6700+
}
6701+
},
6702+
"additionalProperties": false
6703+
}
6704+
]
6705+
},
66776706
"CollectionConfig": {
66786707
"description": "Information about the collection configuration",
66796708
"type": "object",

‎lib/api/src/grpc/proto/collections.proto‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -162,6 +162,11 @@ message OptimizerStatus {
162162
string error = 2;
163163
}
164164

165+
message ConfigurationStatus {
166+
bool ok = 1;
167+
string warning = 2;
168+
}
169+
165170
message HnswConfigDiff {
166171
/*
167172
Number of edges per node in the index graph. Larger the value - more accurate the search, more space required.
@@ -572,6 +577,7 @@ message CollectionInfo {
572577
map<string, PayloadSchemaInfo> payload_schema = 8; // Collection data types
573578
optional uint64 points_count = 9; // Approximate number of points in the collection
574579
optional uint64 indexed_vectors_count = 10; // Approximate number of indexed vectors in the collection.
580+
ConfigurationStatus configuration_status = 11; // status of configuration inconsistencies
575581
}
576582

577583
message ChangeAliases {

‎lib/api/src/grpc/qdrant.rs‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -375,6 +375,15 @@ pub struct OptimizerStatus {
375375
#[prost(string, tag = "2")]
376376
pub error: ::prost::alloc::string::String,
377377
}
378+
#[derive(serde::Serialize)]
379+
#[allow(clippy::derive_partial_eq_without_eq)]
380+
#[derive(Clone, PartialEq, ::prost::Message)]
381+
pub struct ConfigurationStatus {
382+
#[prost(bool, tag = "1")]
383+
pub ok: bool,
384+
#[prost(string, tag = "2")]
385+
pub warning: ::prost::alloc::string::String,
386+
}
378387
#[derive(validator::Validate)]
379388
#[derive(serde::Serialize)]
380389
#[allow(clippy::derive_partial_eq_without_eq)]
@@ -1233,6 +1242,9 @@ pub struct CollectionInfo {
12331242
/// Approximate number of indexed vectors in the collection.
12341243
#[prost(uint64, optional, tag = "10")]
12351244
pub indexed_vectors_count: ::core::option::Option<u64>,
1245+
/// status of configuration inconsistencies
1246+
#[prost(message, optional, tag = "11")]
1247+
pub configuration_status: ::core::option::Option<ConfigurationStatus>,
12361248
}
12371249
#[derive(validator::Validate)]
12381250
#[derive(serde::Serialize)]

‎lib/collection/src/collection/collection_ops.rs‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -405,4 +405,14 @@ impl Collection {
405405
};
406406
Ok(info)
407407
}
408+
409+
pub async fn validate_and_print_warnings(&self) -> CollectionResult<()> {
410+
let status = self.collection_config.read().await.validate_configuration();
411+
if let ConfigurationStatus::Warning(warnings) = status {
412+
for warning in warnings.lines() {
413+
log::warn!("Collection {}: {warning}", self.name());
414+
}
415+
}
416+
Ok(())
417+
}
408418
}

‎lib/collection/src/collection/state_management.rs‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -173,6 +173,8 @@ impl Collection {
173173

174174
self.collection_config.read().await.save(&self.path)?;
175175

176+
self.validate_and_print_warnings().await?;
177+
176178
if recreate_optimizers {
177179
self.recreate_optimizers_blocking().await?;
178180
}

‎lib/collection/src/config.rs‎

Lines changed: 42 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
use std::collections::{BTreeMap, HashMap, HashSet};
2+
use std::fmt::Write as _;
23
use std::fs::File;
3-
use std::io::{Read, Write};
4+
use std::io::{Read, Write as _};
45
use std::num::{NonZeroU32, NonZeroUsize};
56
use std::path::Path;
67

@@ -22,8 +23,8 @@ use wal::WalOptions;
2223

2324
use crate::operations::config_diff::{DiffConfig, QuantizationConfigDiff};
2425
use crate::operations::types::{
25-
CollectionError, CollectionResult, SparseVectorParams, SparseVectorsConfig, VectorParams,
26-
VectorParamsDiff, VectorsConfig, VectorsConfigDiff,
26+
CollectionError, CollectionResult, ConfigurationStatus, SparseVectorParams,
27+
SparseVectorsConfig, VectorParams, VectorParamsDiff, VectorsConfig, VectorsConfigDiff,
2728
};
2829
use crate::operations::validation;
2930
use crate::optimizers_builder::OptimizersConfig;
@@ -276,6 +277,44 @@ impl CollectionConfigInternal {
276277
}
277278
}
278279

280+
/// Check for configuration inconsistencies.
281+
pub fn validate_configuration(&self) -> ConfigurationStatus {
282+
let mut messages = String::new();
283+
284+
for (vector_name, vector_config) in self.params.vectors.params_iter() {
285+
let vector_hnsw = self.hnsw_config.update_opt(vector_config.hnsw_config);
286+
287+
let vector_quantization =
288+
vector_config.quantization_config.is_some() || self.quantization_config.is_some();
289+
290+
if vector_hnsw.copy_vectors.unwrap_or_default() {
291+
if !vector_quantization {
292+
writeln!(
293+
&mut messages,
294+
"The `hnsw_config.copy_vectors` option for vector '{vector_name}' \
295+
requires quantization to be enabled. This option will be ignored."
296+
)
297+
.unwrap();
298+
}
299+
if vector_config.multivector_config.is_some() {
300+
writeln!(
301+
&mut messages,
302+
"The `hnsw_config.copy_vectors` option for vector '{vector_name}' \
303+
is not compatible with multivectors. This option will be ignored."
304+
)
305+
.unwrap();
306+
}
307+
}
308+
}
309+
310+
if !messages.is_empty() {
311+
messages.pop(); // trim the last newline
312+
ConfigurationStatus::Warning(messages)
313+
} else {
314+
ConfigurationStatus::Ok
315+
}
316+
}
317+
279318
pub fn to_base_segment_config(&self) -> CollectionResult<SegmentConfig> {
280319
self.params
281320
.to_base_segment_config(self.quantization_config.as_ref())

‎lib/collection/src/operations/conversions.rs‎

Lines changed: 29 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -25,10 +25,10 @@ use tonic::Status;
2525
use super::cluster_ops::ReshardingDirection;
2626
use super::consistency_params::ReadConsistency;
2727
use super::types::{
28-
CollectionConfig, ContextExamplePair, CoreSearchRequest, Datatype, DiscoverRequestInternal,
29-
GroupsResult, Modifier, PointGroup, RecommendExample, RecommendGroupsRequestInternal,
30-
ReshardingInfo, SparseIndexParams, SparseVectorParams, SparseVectorsConfig, VectorParamsDiff,
31-
VectorsConfigDiff,
28+
CollectionConfig, ConfigurationStatus, ContextExamplePair, CoreSearchRequest, Datatype,
29+
DiscoverRequestInternal, GroupsResult, Modifier, PointGroup, RecommendExample,
30+
RecommendGroupsRequestInternal, ReshardingInfo, SparseIndexParams, SparseVectorParams,
31+
SparseVectorsConfig, VectorParamsDiff, VectorsConfigDiff,
3232
};
3333
use crate::config::{
3434
CollectionParams, ShardingMethod, WalConfig, default_replication_factor,
@@ -380,6 +380,7 @@ impl From<CollectionInfo> for api::grpc::qdrant::CollectionInfo {
380380
let CollectionInfo {
381381
status,
382382
optimizer_status,
383+
configuration_status,
383384
indexed_vectors_count,
384385
points_count,
385386
segments_count,
@@ -447,6 +448,15 @@ impl From<CollectionInfo> for api::grpc::qdrant::CollectionInfo {
447448
api::grpc::qdrant::OptimizerStatus { ok: false, error }
448449
}
449450
}),
451+
configuration_status: Some(match configuration_status {
452+
ConfigurationStatus::Ok => api::grpc::qdrant::ConfigurationStatus {
453+
ok: true,
454+
warning: "".to_string(),
455+
},
456+
ConfigurationStatus::Warning(warning) => {
457+
api::grpc::qdrant::ConfigurationStatus { ok: false, warning }
458+
}
459+
}),
450460
indexed_vectors_count: indexed_vectors_count.map(|count| count as u64),
451461
points_count: points_count.map(|count| count as u64),
452462
segments_count: segments_count as u64,
@@ -841,6 +851,7 @@ impl TryFrom<api::grpc::qdrant::GetCollectionInfoResponse> for CollectionInfo {
841851
let api::grpc::qdrant::CollectionInfo {
842852
status,
843853
optimizer_status,
854+
configuration_status,
844855
indexed_vectors_count,
845856
points_count,
846857
segments_count,
@@ -861,6 +872,20 @@ impl TryFrom<api::grpc::qdrant::GetCollectionInfoResponse> for CollectionInfo {
861872
}
862873
}
863874
},
875+
configuration_status: match configuration_status {
876+
None => {
877+
return Err(Status::invalid_argument(
878+
"Malformed ConfigurationStatus type",
879+
));
880+
}
881+
Some(api::grpc::qdrant::ConfigurationStatus { ok, warning }) => {
882+
if ok {
883+
ConfigurationStatus::Ok
884+
} else {
885+
ConfigurationStatus::Warning(warning)
886+
}
887+
}
888+
},
864889
indexed_vectors_count: indexed_vectors_count.map(|count| count as usize),
865890
points_count: points_count.map(|count| count as usize),
866891
segments_count: segments_count as usize,

‎lib/collection/src/operations/types.rs‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -126,6 +126,20 @@ pub enum OptimizersStatus {
126126
Error(String),
127127
}
128128

129+
/// Current state of the collection
130+
#[derive(
131+
Debug, Default, Serialize, JsonSchema, Anonymize, PartialEq, Eq, PartialOrd, Ord, Clone,
132+
)]
133+
#[serde(rename_all = "snake_case")]
134+
pub enum ConfigurationStatus {
135+
/// Configuration is valid
136+
#[default]
137+
Ok,
138+
/// Configuration has warnings
139+
#[anonymize(false)]
140+
Warning(String),
141+
}
142+
129143
/// Point data
130144
#[derive(Clone, Debug, PartialEq)]
131145
pub struct RecordInternal {
@@ -218,6 +232,8 @@ pub struct CollectionInfo {
218232
pub status: CollectionStatus,
219233
/// Status of optimizers
220234
pub optimizer_status: OptimizersStatus,
235+
/// Status of configuration validation
236+
pub configuration_status: ConfigurationStatus,
221237
/// Approximate number of indexed vectors in the collection.
222238
/// Indexed vectors in large segments are faster to query,
223239
/// as it is stored in a specialized vector index.
@@ -239,6 +255,7 @@ impl CollectionInfo {
239255
Self {
240256
status: CollectionStatus::Green,
241257
optimizer_status: OptimizersStatus::Ok,
258+
configuration_status: collection_config.validate_configuration(),
242259
indexed_vectors_count: Some(0),
243260
points_count: Some(0),
244261
segments_count: 0,
@@ -262,6 +279,7 @@ impl From<ShardInfoInternal> for CollectionInfo {
262279
Self {
263280
status: status.into(),
264281
optimizer_status,
282+
configuration_status: config.validate_configuration(),
265283
indexed_vectors_count: Some(indexed_vectors_count),
266284
points_count: Some(points_count),
267285
segments_count,

‎lib/segment/src/index/hnsw_index/hnsw.rs‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -617,6 +617,7 @@ impl HNSWIndex {
617617
.unwrap_or_default()
618618
.then(|| {
619619
// NOTE: the configuration is silently ignored if try_new fails.
620+
// TODO: add comment
620621
StorageGraphLinksVectors::try_new(
621622
&vector_storage_ref,
622623
quantized_vectors_ref.as_ref(),

0 commit comments

Comments
 (0)