Skip to content

Commit 821aa0a

Browse files
JojiiOfficialJojiiOfficial
authored
Only measure HW if on disk for bool index (#6296)
* Only measure HW if on disk for bool index * Remove missed measurement
1 parent 43dc550 commit 821aa0a

4 files changed

Lines changed: 63 additions & 67 deletions

File tree

‎lib/common/common/src/counter/iterator_hw_measurement.rs‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -47,6 +47,23 @@ pub trait HwMeasurementIteratorExt: Iterator {
4747
})
4848
}
4949

50+
/// Measures the hardware usage of an iterator with the size of a single value being represented as a fraction.
51+
fn measure_hw_with_acc_and_fraction<R>(
52+
self,
53+
hw_acc: HwMeasurementAcc,
54+
fraction: usize,
55+
mut f: R,
56+
) -> OnFinalCount<Self, impl FnMut(usize)>
57+
where
58+
Self: Sized,
59+
R: FnMut(&HardwareCounterCell) -> &CounterCell,
60+
{
61+
OnFinalCount::new(self, move |total_count| {
62+
let hw_counter = hw_acc.get_counter_cell();
63+
f(&hw_counter).incr_delta(total_count / fraction);
64+
})
65+
}
66+
5067
/// Measures the hardware usage of an iterator with the size of a single value being represented as a fraction.
5168
fn measure_hw_with_cell_and_fraction<R>(
5269
self,

‎lib/segment/src/index/field_index/bool_index/mmap_bool_index.rs‎

Lines changed: 37 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
use std::path::{Path, PathBuf};
22

33
use bitvec::slice::BitSlice;
4+
use common::counter::conditioned_counter::ConditionedCounter;
45
use common::counter::hardware_counter::HardwareCounterCell;
56
use common::counter::iterator_hw_measurement::HwMeasurementIteratorExt;
67
use common::types::PointOffsetType;
@@ -29,6 +30,7 @@ pub struct MmapBoolIndex {
2930
falses_count: usize,
3031
trues_slice: DynamicMmapFlags,
3132
falses_slice: DynamicMmapFlags,
33+
populated: bool,
3234
}
3335

3436
impl MmapBoolIndex {
@@ -79,21 +81,32 @@ impl MmapBoolIndex {
7981
indexed_count: 0,
8082
trues_count: 0,
8183
falses_count: 0,
84+
populated: populate,
8285
})
8386
}
8487

88+
fn make_conditioned_counter<'a>(
89+
&self,
90+
hw_counter: &'a HardwareCounterCell,
91+
) -> ConditionedCounter<'a> {
92+
let on_disk = !self.populated; // Measure if on disk.
93+
ConditionedCounter::new(on_disk, hw_counter)
94+
}
95+
8596
fn set_or_insert(
8697
&mut self,
8798
id: u32,
8899
has_true: bool,
89100
has_false: bool,
90101
hw_counter: &HardwareCounterCell,
91102
) -> OperationResult<()> {
103+
let hw_counter = self.make_conditioned_counter(hw_counter);
104+
92105
// Set or insert the flags
93106
let prev_true =
94-
set_or_insert_flag(&mut self.trues_slice, id as usize, has_true, hw_counter)?;
107+
set_or_insert_flag(&mut self.trues_slice, id as usize, has_true, &hw_counter)?;
95108
let prev_false =
96-
set_or_insert_flag(&mut self.falses_slice, id as usize, has_false, hw_counter)?;
109+
set_or_insert_flag(&mut self.falses_slice, id as usize, has_false, &hw_counter)?;
97110

98111
let was_indexed = prev_true || prev_false;
99112
let is_indexed = has_true || has_false;
@@ -209,6 +222,8 @@ impl MmapBoolIndex {
209222
is_true: bool,
210223
hw_counter: &HardwareCounterCell,
211224
) -> bool {
225+
let hw_counter = self.make_conditioned_counter(hw_counter);
226+
212227
hw_counter
213228
.payload_index_io_read_counter()
214229
.incr_delta(size_of::<bool>());
@@ -227,12 +242,14 @@ impl MmapBoolIndex {
227242
&'a self,
228243
hw_counter: &'a HardwareCounterCell,
229244
) -> impl Iterator<Item = (bool, IdIter<'a>)> + 'a {
245+
let hw_counter = self.make_conditioned_counter(hw_counter);
246+
230247
[
231248
(false, Box::new(self.falses_slice.iter_trues()) as IdIter),
232249
(true, Box::new(self.trues_slice.iter_trues()) as IdIter),
233250
]
234251
.into_iter()
235-
.measure_hw_with_cell_and_fraction(hw_counter, u8::BITS as usize, |i| {
252+
.measure_hw_with_acc(hw_counter.new_accumulator(), u8::BITS as usize, |i| {
236253
i.payload_index_io_read_counter()
237254
})
238255
}
@@ -272,7 +289,7 @@ fn set_or_insert_flag(
272289
flags: &mut DynamicMmapFlags,
273290
key: usize,
274291
value: bool,
275-
hw_counter: &HardwareCounterCell,
292+
hw_counter: &ConditionedCounter,
276293
) -> OperationResult<bool> {
277294
let counter = hw_counter.payload_index_io_write_counter();
278295

@@ -314,7 +331,8 @@ impl FieldIndexBuilderTrait for MmapBoolIndexBuilder {
314331
payload: &[&serde_json::Value],
315332
hw_counter: &HardwareCounterCell,
316333
) -> OperationResult<()> {
317-
self.0.add_point(id, payload, hw_counter)
334+
let hw_counter = self.0.make_conditioned_counter(hw_counter);
335+
self.0.add_point(id, payload, &hw_counter)
318336
}
319337

320338
fn finalize(self) -> OperationResult<Self::FieldIndexType> {
@@ -335,10 +353,12 @@ impl ValueIndexer for MmapBoolIndex {
335353
return Ok(());
336354
}
337355

356+
let hw_counter = self.make_conditioned_counter(hw_counter);
357+
338358
let has_true = values.iter().any(|v| *v);
339359
let has_false = values.iter().any(|v| !*v);
340360

341-
self.set_or_insert(id, has_true, has_false, hw_counter)?;
361+
self.set_or_insert(id, has_true, has_false, &hw_counter)?;
342362

343363
Ok(())
344364
}
@@ -369,6 +389,7 @@ impl PayloadFieldIndex for MmapBoolIndex {
369389
falses_count,
370390
trues_slice,
371391
falses_slice,
392+
populated: _,
372393
} = self;
373394

374395
*indexed_count = calculated_indexed_count as usize;
@@ -391,6 +412,7 @@ impl PayloadFieldIndex for MmapBoolIndex {
391412
falses_count: _,
392413
trues_slice,
393414
falses_slice,
415+
populated: _,
394416
} = self;
395417

396418
let trues_flusher = trues_slice.flusher();
@@ -415,6 +437,8 @@ impl PayloadFieldIndex for MmapBoolIndex {
415437
condition: &'a FieldCondition,
416438
hw_counter: &'a HardwareCounterCell,
417439
) -> Option<Box<dyn Iterator<Item = PointOffsetType> + 'a>> {
440+
let hw_counter = self.make_conditioned_counter(hw_counter);
441+
418442
match &condition.r#match {
419443
Some(Match::Value(MatchValue {
420444
value: ValueVariants::Bool(value),
@@ -423,9 +447,11 @@ impl PayloadFieldIndex for MmapBoolIndex {
423447
.get_slice_for(*value)
424448
.iter_ones()
425449
.map(|x| x as PointOffsetType)
426-
.measure_hw_with_cell_and_fraction(hw_counter, u8::BITS as usize, |i| {
427-
i.payload_index_io_read_counter()
428-
});
450+
.measure_hw_with_acc_and_fraction(
451+
hw_counter.new_accumulator(),
452+
u8::BITS as usize,
453+
|i| i.payload_index_io_read_counter(),
454+
);
429455
Some(Box::new(iter))
430456
}
431457
_ => None,
@@ -437,6 +463,8 @@ impl PayloadFieldIndex for MmapBoolIndex {
437463
condition: &FieldCondition,
438464
hw_counter: &HardwareCounterCell,
439465
) -> Option<CardinalityEstimation> {
466+
let hw_counter = self.make_conditioned_counter(hw_counter);
467+
440468
match &condition.r#match {
441469
Some(Match::Value(MatchValue {
442470
value: ValueVariants::Bool(value),

‎lib/segment/src/index/field_index/bool_index/mod.rs‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -30,7 +30,7 @@ impl BoolIndex {
3030
hw_acc: &'a HardwareCounterCell,
3131
) -> Box<dyn Iterator<Item = (bool, IdIter<'a>)> + 'a> {
3232
match self {
33-
BoolIndex::Simple(index) => Box::new(index.iter_values_map(hw_acc)),
33+
BoolIndex::Simple(index) => Box::new(index.iter_values_map()),
3434
BoolIndex::Mmap(index) => Box::new(index.iter_values_map(hw_acc)),
3535
}
3636
}
@@ -70,7 +70,7 @@ impl BoolIndex {
7070
hw_counter: &HardwareCounterCell,
7171
) -> bool {
7272
match self {
73-
BoolIndex::Simple(index) => index.check_values_any(point_id, is_true, hw_counter),
73+
BoolIndex::Simple(index) => index.check_values_any(point_id, is_true),
7474
BoolIndex::Mmap(index) => index.check_values_any(point_id, is_true, hw_counter),
7575
}
7676
}

‎lib/segment/src/index/field_index/bool_index/simple_bool_index.rs‎

Lines changed: 7 additions & 56 deletions
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,6 @@ use std::path::PathBuf;
22
use std::sync::Arc;
33

44
use common::counter::hardware_counter::HardwareCounterCell;
5-
use common::counter::iterator_hw_measurement::HwMeasurementIteratorExt;
65
use common::types::PointOffsetType;
76
use parking_lot::RwLock;
87
use rocksdb::DB;
@@ -23,8 +22,6 @@ use crate::types::{FieldCondition, Match, MatchValue, PayloadKeyType, ValueVaria
2322

2423
mod memory {
2524
use bitvec::vec::BitVec;
26-
use common::counter::hardware_counter::HardwareCounterCell;
27-
use common::counter::iterator_hw_measurement::HwMeasurementIteratorExt;
2825
use common::ext::BitSliceExt as _;
2926
use common::types::PointOffsetType;
3027

@@ -173,34 +170,10 @@ mod memory {
173170
self.indexed_count
174171
}
175172

176-
pub fn iter_has_true_measured<'a>(
177-
&'a self,
178-
hw_counter: &'a HardwareCounterCell,
179-
) -> impl Iterator<Item = PointOffsetType> + 'a {
180-
self.trues
181-
.iter_ones()
182-
.map(|v| v as PointOffsetType)
183-
.measure_hw_with_cell_and_fraction(hw_counter, u8::BITS as usize, |i| {
184-
i.payload_index_io_read_counter()
185-
})
186-
}
187-
188173
pub fn iter_has_true(&self) -> impl Iterator<Item = PointOffsetType> + '_ {
189174
self.trues.iter_ones().map(|v| v as PointOffsetType)
190175
}
191176

192-
pub fn iter_has_false_measured<'a>(
193-
&'a self,
194-
hw_counter: &'a HardwareCounterCell,
195-
) -> impl Iterator<Item = PointOffsetType> + 'a {
196-
self.falses
197-
.iter_ones()
198-
.map(|v| v as PointOffsetType)
199-
.measure_hw_with_cell_and_fraction(hw_counter, u8::BITS as usize, |i| {
200-
i.payload_index_io_read_counter()
201-
})
202-
}
203-
204177
pub fn iter_has_false(&self) -> impl Iterator<Item = PointOffsetType> + '_ {
205178
self.falses.iter_ones().map(|v| v as PointOffsetType)
206179
}
@@ -243,16 +216,7 @@ impl SimpleBoolIndex {
243216
}
244217
}
245218

246-
pub fn check_values_any(
247-
&self,
248-
point_id: PointOffsetType,
249-
is_true: bool,
250-
hw_counter: &HardwareCounterCell,
251-
) -> bool {
252-
hw_counter
253-
.payload_index_io_read_counter()
254-
.incr_delta(size_of::<bool>());
255-
219+
pub fn check_values_any(&self, point_id: PointOffsetType, is_true: bool) -> bool {
256220
if is_true {
257221
self.values_has_true(point_id)
258222
} else {
@@ -279,18 +243,12 @@ impl SimpleBoolIndex {
279243
self.memory.get(point_id).has_false()
280244
}
281245

282-
pub fn iter_values_map<'a>(
283-
&'a self,
284-
hw_counter: &'a HardwareCounterCell,
285-
) -> impl Iterator<Item = (bool, IdIter<'a>)> + 'a {
246+
pub fn iter_values_map(&self) -> impl Iterator<Item = (bool, IdIter)> {
286247
[
287248
(false, Box::new(self.memory.iter_has_false()) as IdIter),
288249
(true, Box::new(self.memory.iter_has_true()) as IdIter),
289250
]
290251
.into_iter()
291-
.measure_hw_with_cell(hw_counter, size_of::<usize>(), |i| {
292-
i.payload_index_io_read_counter()
293-
})
294252
}
295253

296254
pub fn iter_values(&self) -> impl Iterator<Item = bool> + '_ {
@@ -377,16 +335,16 @@ impl PayloadFieldIndex for SimpleBoolIndex {
377335
fn filter<'a>(
378336
&'a self,
379337
condition: &'a crate::types::FieldCondition,
380-
hw_counter: &'a HardwareCounterCell,
338+
_: &'a HardwareCounterCell,
381339
) -> Option<Box<dyn Iterator<Item = PointOffsetType> + 'a>> {
382340
match &condition.r#match {
383341
Some(Match::Value(MatchValue {
384342
value: ValueVariants::Bool(value),
385343
})) => {
386344
if *value {
387-
Some(Box::new(self.memory.iter_has_true_measured(hw_counter)))
345+
Some(Box::new(self.memory.iter_has_true()))
388346
} else {
389-
Some(Box::new(self.memory.iter_has_false_measured(hw_counter)))
347+
Some(Box::new(self.memory.iter_has_false()))
390348
}
391349
}
392350
_ => None,
@@ -396,7 +354,7 @@ impl PayloadFieldIndex for SimpleBoolIndex {
396354
fn estimate_cardinality(
397355
&self,
398356
condition: &FieldCondition,
399-
hw_counter: &HardwareCounterCell,
357+
_: &HardwareCounterCell,
400358
) -> Option<CardinalityEstimation> {
401359
match &condition.r#match {
402360
Some(Match::Value(MatchValue {
@@ -408,9 +366,6 @@ impl PayloadFieldIndex for SimpleBoolIndex {
408366
self.memory.falses_count()
409367
};
410368

411-
// In mmap index we also measure this amount.
412-
hw_counter.payload_index_io_read_counter().incr_delta(count);
413-
414369
let estimation = CardinalityEstimation::exact(count)
415370
.with_primary_clause(PrimaryCondition::Condition(Box::new(condition.clone())));
416371

@@ -464,7 +419,7 @@ impl ValueIndexer for SimpleBoolIndex {
464419
&mut self,
465420
id: PointOffsetType,
466421
values: Vec<bool>,
467-
hw_counter: &HardwareCounterCell,
422+
_: &HardwareCounterCell,
468423
) -> OperationResult<()> {
469424
if values.is_empty() {
470425
return Ok(());
@@ -479,10 +434,6 @@ impl ValueIndexer for SimpleBoolIndex {
479434

480435
let item_bytes = item.as_bytes();
481436

482-
hw_counter
483-
.payload_index_io_write_counter()
484-
.incr_delta(size_of_val(&id) + size_of_val(&item_bytes));
485-
486437
self.db_wrapper.put(id.to_be_bytes(), item_bytes)?;
487438

488439
Ok(())

0 commit comments

Comments
 (0)