Skip to content

Commit 780d406

Browse files
xzfcxzfc
authored
Add CollectionInfo::warnings field (#7293)
* refactor: swap DiffConfig parmeters was: diff.update(&config) now: config.update(diff) * refactor: introduce DiffConfig::update_opt * feat: add CollectionInfo::configuration_status field * rename ConfigurationStatus to CollectionWarning
1 parent a32810c commit 780d406

16 files changed

Lines changed: 265 additions & 108 deletions

File tree

‎docs/grpc/docs.md‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,7 @@
2424
- [CollectionOperationResponse](#qdrant-CollectionOperationResponse)
2525
- [CollectionParams](#qdrant-CollectionParams)
2626
- [CollectionParamsDiff](#qdrant-CollectionParamsDiff)
27+
- [CollectionWarning](#qdrant-CollectionWarning)
2728
- [CreateAlias](#qdrant-CreateAlias)
2829
- [CreateCollection](#qdrant-CreateCollection)
2930
- [CreateCollection.MetadataEntry](#qdrant-CreateCollection-MetadataEntry)
@@ -608,6 +609,7 @@
608609
| payload_schema | [CollectionInfo.PayloadSchemaEntry](#qdrant-CollectionInfo-PayloadSchemaEntry) | repeated | Collection data types |
609610
| points_count | [uint64](#uint64) | optional | Approximate number of points in the collection |
610611
| indexed_vectors_count | [uint64](#uint64) | optional | Approximate number of indexed vectors in the collection. |
612+
| warnings | [CollectionWarning](#qdrant-CollectionWarning) | repeated | Warnings related to the collection |
611613

612614

613615

@@ -686,6 +688,21 @@
686688

687689

688690

691+
<a name="qdrant-CollectionWarning"></a>
692+
693+
### CollectionWarning
694+
695+
696+
697+
| Field | Type | Label | Description |
698+
| ----- | ---- | ----- | ----------- |
699+
| message | [string](#string) | | |
700+
701+
702+
703+
704+
705+
689706
<a name="qdrant-CreateAlias"></a>
690707

691708
### CreateAlias

‎docs/redoc/master/openapi.json‎

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6607,6 +6607,13 @@
66076607
"optimizer_status": {
66086608
"$ref": "#/components/schemas/OptimizersStatus"
66096609
},
6610+
"warnings": {
6611+
"description": "Warnings related to the collection",
6612+
"type": "array",
6613+
"items": {
6614+
"$ref": "#/components/schemas/CollectionWarning"
6615+
}
6616+
},
66106617
"indexed_vectors_count": {
66116618
"description": "Approximate number of indexed vectors in the collection. Indexed vectors in large segments are faster to query, as it is stored in a specialized vector index.",
66126619
"type": "integer",
@@ -6674,6 +6681,18 @@
66746681
}
66756682
]
66766683
},
6684+
"CollectionWarning": {
6685+
"type": "object",
6686+
"required": [
6687+
"message"
6688+
],
6689+
"properties": {
6690+
"message": {
6691+
"description": "Warning message",
6692+
"type": "string"
6693+
}
6694+
}
6695+
},
66776696
"CollectionConfig": {
66786697
"description": "Information about the collection configuration",
66796698
"type": "object",

‎lib/api/src/grpc/proto/collections.proto‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -162,6 +162,10 @@ message OptimizerStatus {
162162
string error = 2;
163163
}
164164

165+
message CollectionWarning {
166+
string message = 1;
167+
}
168+
165169
message HnswConfigDiff {
166170
/*
167171
Number of edges per node in the index graph. Larger the value - more accurate the search, more space required.
@@ -572,6 +576,7 @@ message CollectionInfo {
572576
map<string, PayloadSchemaInfo> payload_schema = 8; // Collection data types
573577
optional uint64 points_count = 9; // Approximate number of points in the collection
574578
optional uint64 indexed_vectors_count = 10; // Approximate number of indexed vectors in the collection.
579+
repeated CollectionWarning warnings = 11; // Warnings related to the collection
575580
}
576581

577582
message ChangeAliases {

‎lib/api/src/grpc/qdrant.rs‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -375,6 +375,13 @@ pub struct OptimizerStatus {
375375
#[prost(string, tag = "2")]
376376
pub error: ::prost::alloc::string::String,
377377
}
378+
#[derive(serde::Serialize)]
379+
#[allow(clippy::derive_partial_eq_without_eq)]
380+
#[derive(Clone, PartialEq, ::prost::Message)]
381+
pub struct CollectionWarning {
382+
#[prost(string, tag = "1")]
383+
pub message: ::prost::alloc::string::String,
384+
}
378385
#[derive(validator::Validate)]
379386
#[derive(serde::Serialize)]
380387
#[allow(clippy::derive_partial_eq_without_eq)]
@@ -1233,6 +1240,9 @@ pub struct CollectionInfo {
12331240
/// Approximate number of indexed vectors in the collection.
12341241
#[prost(uint64, optional, tag = "10")]
12351242
pub indexed_vectors_count: ::core::option::Option<u64>,
1243+
/// Warnings related to the collection
1244+
#[prost(message, repeated, tag = "11")]
1245+
pub warnings: ::prost::alloc::vec::Vec<CollectionWarning>,
12361246
}
12371247
#[derive(validator::Validate)]
12381248
#[derive(serde::Serialize)]

‎lib/collection/src/collection/collection_ops.rs‎

Lines changed: 11 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -34,7 +34,7 @@ impl Collection {
3434
) -> CollectionResult<()> {
3535
{
3636
let mut config = self.collection_config.write().await;
37-
config.params = params_diff.update(&config.params)?;
37+
config.params = config.params.update(params_diff)?;
3838
}
3939
self.collection_config.read().await.save(&self.path)?;
4040
Ok(())
@@ -51,7 +51,7 @@ impl Collection {
5151
) -> CollectionResult<()> {
5252
{
5353
let mut config = self.collection_config.write().await;
54-
config.hnsw_config = hnsw_config_diff.update(&config.hnsw_config)?;
54+
config.hnsw_config = config.hnsw_config.update(hnsw_config_diff)?;
5555
}
5656
self.collection_config.read().await.save(&self.path)?;
5757
Ok(())
@@ -104,8 +104,7 @@ impl Collection {
104104
) -> CollectionResult<()> {
105105
{
106106
let mut config = self.collection_config.write().await;
107-
config.optimizer_config =
108-
DiffConfig::update(optimizer_config_diff, &config.optimizer_config)?;
107+
config.optimizer_config = config.optimizer_config.update(optimizer_config_diff)?;
109108
}
110109
self.collection_config.read().await.save(&self.path)?;
111110
Ok(())
@@ -187,7 +186,7 @@ impl Collection {
187186
{
188187
let mut config = self.collection_config.write().await;
189188
if let Some(current_config) = config.strict_mode_config.as_mut() {
190-
*current_config = strict_mode_diff.update(current_config)?;
189+
*current_config = current_config.update(strict_mode_diff)?;
191190
} else {
192191
config.strict_mode_config = Some(strict_mode_diff);
193192
}
@@ -406,4 +405,11 @@ impl Collection {
406405
};
407406
Ok(info)
408407
}
408+
409+
pub async fn print_warnings(&self) {
410+
let warnings = self.collection_config.read().await.get_warnings();
411+
for warning in warnings {
412+
log::warn!("Collection {}: {}", self.name(), warning.message);
413+
}
414+
}
409415
}

‎lib/collection/src/collection/mod.rs‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -132,7 +132,7 @@ impl Collection {
132132
let mut effective_optimizers_config = collection_config.optimizer_config.clone();
133133
if let Some(optimizers_overwrite) = optimizers_overwrite.clone() {
134134
effective_optimizers_config =
135-
optimizers_overwrite.update(&effective_optimizers_config)?;
135+
effective_optimizers_config.update(optimizers_overwrite)?;
136136
}
137137

138138
let shard_key = shard_key_mapping
@@ -253,8 +253,8 @@ impl Collection {
253253
let mut effective_optimizers_config = collection_config.optimizer_config.clone();
254254

255255
if let Some(optimizers_overwrite) = optimizers_overwrite.clone() {
256-
effective_optimizers_config = optimizers_overwrite
257-
.update(&effective_optimizers_config)
256+
effective_optimizers_config = effective_optimizers_config
257+
.update(optimizers_overwrite)
258258
.expect("Can not apply optimizer overwrite");
259259
}
260260

@@ -847,7 +847,7 @@ impl Collection {
847847
let config = self.collection_config.read().await;
848848

849849
if let Some(optimizers_overwrite) = self.optimizers_overwrite.clone() {
850-
Ok(optimizers_overwrite.update(&config.optimizer_config)?)
850+
Ok(config.optimizer_config.update(optimizers_overwrite)?)
851851
} else {
852852
Ok(config.optimizer_config.clone())
853853
}

‎lib/collection/src/collection/state_management.rs‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -173,6 +173,8 @@ impl Collection {
173173

174174
self.collection_config.read().await.save(&self.path)?;
175175

176+
self.print_warnings().await;
177+
176178
if recreate_optimizers {
177179
self.recreate_optimizers_blocking().await?;
178180
}

‎lib/collection/src/collection_manager/optimizers/config_mismatch_optimizer.rs‎

Lines changed: 8 additions & 32 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,3 @@
1-
use std::borrow::Cow;
21
use std::collections::HashSet;
32
use std::path::{Path, PathBuf};
43
use std::sync::Arc;
@@ -73,34 +72,6 @@ impl ConfigMismatchOptimizer {
7372
.and_then(|index| index.on_disk)
7473
}
7574

76-
/// Calculates and HNSW config that should be used for a given vector
77-
/// with current configuration.
78-
///
79-
/// Takes vector-specific HNSW config (if any) and merges it with the collection-wide config.
80-
fn get_required_hnsw_config(&self, vector_name: &VectorName) -> Cow<'_, HnswConfig> {
81-
let target_hnsw_collection = &self.hnsw_config;
82-
// Select vector specific target HNSW config
83-
let target_hnsw_vector = self
84-
.collection_params
85-
.vectors
86-
.get_params(vector_name)
87-
.and_then(|vector_params| vector_params.hnsw_config)
88-
.map(|vector_hnsw| vector_hnsw.update(target_hnsw_collection))
89-
.and_then(|hnsw| match hnsw {
90-
Ok(hnsw) => Some(hnsw),
91-
Err(err) => {
92-
log::warn!(
93-
"Failed to merge collection and vector HNSW config, ignoring: {err}"
94-
);
95-
None
96-
}
97-
});
98-
match target_hnsw_vector {
99-
Some(target_hnsw) => Cow::Owned(target_hnsw),
100-
None => Cow::Borrowed(target_hnsw_collection),
101-
}
102-
}
103-
10475
fn worst_segment(
10576
&self,
10677
segments: LockedSegmentHolder,
@@ -141,7 +112,12 @@ impl ConfigMismatchOptimizer {
141112
Indexes::Plain {} => {}
142113
Indexes::Hnsw(effective_hnsw) => {
143114
// Select segment if we have an HNSW mismatch that requires rebuild
144-
let target_hnsw = self.get_required_hnsw_config(vector_name);
115+
let target_hnsw = self.hnsw_config.update_opt(
116+
self.collection_params
117+
.vectors
118+
.get_params(vector_name)
119+
.and_then(|vector_params| vector_params.hnsw_config),
120+
);
145121
if effective_hnsw.mismatch_requires_rebuild(&target_hnsw) {
146122
return true;
147123
}
@@ -601,12 +577,12 @@ mod tests {
601577
.for_each(|segment| {
602578
assert_eq!(
603579
segment.config().vector_data[VECTOR1_NAME].index,
604-
Indexes::Hnsw(hnsw_config_vector1.update(&hnsw_config_collection).unwrap()),
580+
Indexes::Hnsw(hnsw_config_collection.update(hnsw_config_vector1).unwrap()),
605581
"HNSW config of vector1 is not what we expect",
606582
);
607583
assert_eq!(
608584
segment.config().vector_data[VECTOR2_NAME].index,
609-
Indexes::Hnsw(hnsw_config_vector2.update(&hnsw_config_collection).unwrap()),
585+
Indexes::Hnsw(hnsw_config_collection.update(hnsw_config_vector2).unwrap()),
610586
"HNSW config of vector2 is not what we expect",
611587
);
612588
});

‎lib/collection/src/collection_manager/optimizers/segment_optimizer.rs‎

Lines changed: 1 addition & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -254,9 +254,7 @@ pub trait SegmentOptimizer {
254254
.vectors
255255
.get_params(vector_name)
256256
.and_then(|params| params.hnsw_config);
257-
let vector_hnsw = param_hnsw
258-
.and_then(|c| c.update(collection_hnsw).ok())
259-
.unwrap_or_else(|| collection_hnsw.clone());
257+
let vector_hnsw = collection_hnsw.update_opt(param_hnsw);
260258
config.index = Indexes::Hnsw(vector_hnsw);
261259

262260
// Assign quantization config

‎lib/collection/src/config.rs‎

Lines changed: 37 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
use std::collections::{BTreeMap, HashMap, HashSet};
2-
use std::io::{Read, Write};
2+
use std::io::{Read, Write as _};
33
use std::num::{NonZeroU32, NonZeroUsize};
44
use std::path::Path;
55

@@ -22,8 +22,8 @@ use wal::WalOptions;
2222

2323
use crate::operations::config_diff::{DiffConfig, QuantizationConfigDiff};
2424
use crate::operations::types::{
25-
CollectionError, CollectionResult, SparseVectorParams, SparseVectorsConfig, VectorParams,
26-
VectorParamsDiff, VectorsConfig, VectorsConfigDiff,
25+
CollectionError, CollectionResult, CollectionWarning, SparseVectorParams, SparseVectorsConfig,
26+
VectorParams, VectorParamsDiff, VectorsConfig, VectorsConfigDiff,
2727
};
2828
use crate::operations::validation;
2929
use crate::optimizers_builder::OptimizersConfig;
@@ -276,6 +276,39 @@ impl CollectionConfigInternal {
276276
}
277277
}
278278

279+
/// Get warnings related to this configuration
280+
pub fn get_warnings(&self) -> Vec<CollectionWarning> {
281+
let mut warnings = Vec::new();
282+
283+
for (vector_name, vector_config) in self.params.vectors.params_iter() {
284+
let vector_hnsw = self.hnsw_config.update_opt(vector_config.hnsw_config);
285+
286+
let vector_quantization =
287+
vector_config.quantization_config.is_some() || self.quantization_config.is_some();
288+
289+
if vector_hnsw.copy_vectors.unwrap_or_default() {
290+
if !vector_quantization {
291+
warnings.push(CollectionWarning {
292+
message: format!(
293+
"The `hnsw_config.copy_vectors` option for vector '{vector_name}' \
294+
requires quantization to be enabled. This option will be ignored."
295+
),
296+
});
297+
}
298+
if vector_config.multivector_config.is_some() {
299+
warnings.push(CollectionWarning {
300+
message: format!(
301+
"The `hnsw_config.copy_vectors` option for vector '{vector_name}' \
302+
is not compatible with multivectors. This option will be ignored."
303+
),
304+
});
305+
}
306+
}
307+
}
308+
309+
warnings
310+
}
311+
279312
pub fn to_base_segment_config(&self) -> CollectionResult<SegmentConfig> {
280313
self.params
281314
.to_base_segment_config(self.quantization_config.as_ref())
@@ -416,7 +449,7 @@ impl CollectionParams {
416449

417450
if let Some(hnsw_diff) = hnsw_config {
418451
if let Some(existing_hnsw) = &vector_params.hnsw_config {
419-
vector_params.hnsw_config = Some(hnsw_diff.update(existing_hnsw)?);
452+
vector_params.hnsw_config = Some(existing_hnsw.update(hnsw_diff)?);
420453
} else {
421454
vector_params.hnsw_config = Some(hnsw_diff);
422455
}

0 commit comments

Comments
 (0)