Skip to content

Commit e2fb02c

Browse files
square-tombstudio
andcommitted
perf(mobile): merge only enough small verified-name batches to fill one
At the four-partial cap, retire the smallest partial batches only until the new subjects fill a batch (at least one when they would leave a partial of their own), instead of re-reading every partial. Replace a batch whose subject was evicted and tracked again before the sync, so its check mark and any pruned cached head are restored. Co-authored-by: studio <dc2fc481343672df31a689840c09299972e704dac592e0da6f18486646a7ee3e@localhost> Signed-off-by: Tom Brow <[email protected]>
1 parent 249020b commit e2fb02c

2 files changed

Lines changed: 103 additions & 23 deletions

File tree

‎mobile/lib/shared/verified_profiles/verified_profile_directory.dart‎

Lines changed: 39 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -21,13 +21,18 @@ const _discoveryRefresh = Duration(minutes: 5);
2121
/// tolerated, a flood before EOSE fails the batch instead of buffering it.
2222
const _maxEarlyPerSubject = 4;
2323

24-
/// Batches smaller than [verifiedProfileBatchSize] allowed before they are
25-
/// merged into full ones.
24+
/// Batches smaller than [verifiedProfileBatchSize] allowed before the smallest
25+
/// are merged into the next new one until it fills a batch.
2626
const _maxPartialBatches = 4;
2727

2828
class _Batch {
29-
_Batch(this.subjects);
29+
_Batch(this.subjects, this.interest);
3030
final List<String> subjects;
31+
32+
/// The newest interest when the batch opened. A subject whose interest is
33+
/// newer was evicted and tracked again since, which dropped its `_checked`
34+
/// mark and maybe its cached head, so the batch must be read again.
35+
final int interest;
3136
final List<NostrEvent> early = [];
3237

3338
/// Live events awaiting proof, newest per subject, so a fast stream cannot
@@ -284,14 +289,19 @@ class VerifiedProfilesNotifier extends Notifier<Map<String, VerifiedProfile>> {
284289
_publish();
285290
return;
286291
}
287-
// Keep every open batch whose subjects are all still tracked, so newly
288-
// tracked people get a batch of their own instead of re-reading everyone
289-
// already checked. A retried batch is replaced so a fresh snapshot can
290-
// prove absence, and a batch that lost a subject to eviction is replaced
291-
// too.
292+
// Keep every open batch whose subjects are all still tracked since it
293+
// opened, so newly tracked people get a batch of their own instead of
294+
// re-reading everyone already checked. A retried batch is replaced so a
295+
// fresh snapshot can prove absence, and a batch that lost a subject to
296+
// eviction is replaced too, even if the subject was tracked again.
292297
for (final id in _batches.keys.toList()) {
293298
final batch = _batches[id]!;
294-
if (!batch.stale && batch.subjects.every(_wanted.containsKey)) continue;
299+
if (!batch.stale &&
300+
batch.subjects.every(
301+
(subject) => (_wanted[subject] ?? 1 << 62) <= batch.interest,
302+
)) {
303+
continue;
304+
}
295305
_close(id);
296306
}
297307
var uncovered = _uncoveredSubjects();
@@ -300,14 +310,28 @@ class VerifiedProfilesNotifier extends Notifier<Map<String, VerifiedProfile>> {
300310
_publish();
301311
return;
302312
}
303-
// Small additions each open a small batch. Past a few of those, merge
304-
// them into full ones, so the subscription count stays near the minimum.
313+
// Small additions each open a small batch. Past a few of those, merge the
314+
// smallest into the new subjects until they fill a batch, so the
315+
// subscription count stays near the minimum while re-reading as few
316+
// checked subjects as possible.
305317
final partial = [
306-
for (final MapEntry(:key, :value) in _batches.entries)
307-
if (value.subjects.length < verifiedProfileBatchSize) key,
318+
for (final batch in _batches.entries)
319+
if (batch.value.subjects.length < verifiedProfileBatchSize) batch,
308320
];
309321
if (partial.length >= _maxPartialBatches) {
310-
partial.forEach(_close);
322+
partial.sort(
323+
(a, b) => a.value.subjects.length.compareTo(b.value.subjects.length),
324+
);
325+
// Retire at least one unless the new subjects split evenly, since
326+
// they can add a partial batch of their own.
327+
var size = uncovered.length;
328+
var retired = size % verifiedProfileBatchSize == 0;
329+
for (final MapEntry(:key, :value) in partial) {
330+
if (retired && size >= verifiedProfileBatchSize) break;
331+
size += value.subjects.length;
332+
retired = true;
333+
_close(key);
334+
}
311335
uncovered = _uncoveredSubjects();
312336
}
313337
for (var i = 0; i < uncovered.length; i += verifiedProfileBatchSize) {
@@ -342,7 +366,7 @@ class VerifiedProfilesNotifier extends Notifier<Map<String, VerifiedProfile>> {
342366
VerifiedProfileAuthority authority,
343367
int generation,
344368
) async {
345-
final batch = _Batch(chunk);
369+
final batch = _Batch(chunk, _interest);
346370
_batches[id] = batch;
347371
void fail() {
348372
if (!batch.active) return;

‎mobile/test/shared/verified_profiles/verified_profile_directory_test.dart‎

Lines changed: 64 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -226,23 +226,41 @@ void main() {
226226
expect(h.profiles[subjectB]?.checked, isTrue);
227227
});
228228

229-
journey('small batches merge into one past four', (h) {
229+
journey('past four small batches, the smallest merge until one is full', (h) {
230+
String key(int i) => i.toRadixString(16).padLeft(64, '0');
231+
final notifier = h.container.read(verifiedProfilesProvider.notifier);
232+
h.start([for (var i = 0; i < 64; i++) key(i)]);
233+
for (var group = 1; group < 4; group++) {
234+
notifier.track([for (var i = 0; i < 64; i++) key(group * 64 + i)]);
235+
h.settle();
236+
}
237+
final before = h.dFilters.length;
238+
expect(before, 4, reason: 'four partial batches of 64');
239+
notifier.track([key(256)]);
240+
h.settle();
241+
// Two of the 64s fill one batch with the newcomer; the other two stay.
242+
final reads = [
243+
for (final f in h.dFilters.skip(before)) (f['#d'] as List).cast<String>(),
244+
];
245+
expect([for (final r in reads) r.length], [128, 1]);
246+
expect(reads.expand((r) => r), contains(key(256)));
247+
});
248+
249+
journey('a large addition past four small batches still retires one', (h) {
230250
String key(int i) => i.toRadixString(16).padLeft(64, '0');
231-
h.start([key(0)]);
232251
final notifier = h.container.read(verifiedProfilesProvider.notifier);
252+
h.start([key(0)]);
233253
for (var i = 1; i < 4; i++) {
234254
notifier.track([key(i)]);
235255
h.settle();
236256
}
237257
final before = h.dFilters.length;
238-
expect(before, 4);
239-
notifier.track([key(4)]);
258+
notifier.track([for (var i = 4; i < 133; i++) key(i)]);
240259
h.settle();
260+
// 129 newcomers would leave a fifth partial; one single folds in.
241261
expect(
242-
[for (final f in h.dFilters.skip(before)) f['#d']],
243-
[
244-
[for (var i = 0; i < 5; i++) key(i)],
245-
],
262+
[for (final f in h.dFilters.skip(before)) (f['#d'] as List).length],
263+
[128, 2],
246264
);
247265
});
248266

@@ -261,6 +279,44 @@ void main() {
261279
);
262280
});
263281

282+
journey('a subject evicted and tracked again before the sync is re-read', (
283+
h,
284+
) {
285+
String key(int i) => i.toRadixString(16).padLeft(64, '0');
286+
h.relay.seed([signedProjection(profileTags(subjectA))]);
287+
h.start([subjectA]);
288+
final notifier = h.container.read(verifiedProfilesProvider.notifier);
289+
notifier.track([for (var i = 0; i < 1023; i++) key(i)]);
290+
h.settle();
291+
expect(h.profiles[subjectA]?.checked, isTrue);
292+
// A is evicted, then tracked again, which evicts key(0) in another batch.
293+
notifier
294+
..track([key(1023)])
295+
..track([subjectA]);
296+
h.settle();
297+
expect(h.profiles[subjectA]?.checked, isTrue);
298+
});
299+
300+
journey('a re-tracked subject keeps its name when the head cache trims', (h) {
301+
String key(int i) => i.toRadixString(16).padLeft(64, '0');
302+
h.relay.seed([signedProjection(profileTags(subjectA))]);
303+
h.start([subjectA]);
304+
final notifier = h.container.read(verifiedProfilesProvider.notifier);
305+
// Each round evicts A, may trim its cached head, then tracks it again
306+
// before the sync; past 4,096 cached heads only a re-read restores it.
307+
for (var round = 0; round < 6; round++) {
308+
final subjects = [for (var i = 0; i < 1023; i++) key(round * 1023 + i)];
309+
h.relay.seed([
310+
for (final s in subjects) signedProjection(profileTags(s)),
311+
]);
312+
notifier
313+
..track(subjects)
314+
..track([subjectA]);
315+
h.settle();
316+
expect(h.profiles[subjectA]?.displayName, 'Ada Lovelace');
317+
}
318+
});
319+
264320
journey('a large roster keeps the 1,024 most recently tracked subjects', (h) {
265321
String key(int i) => i.toRadixString(16).padLeft(64, '0');
266322
h.start([for (var i = 0; i < 10; i++) key(i)]);

0 commit comments

Comments
 (0)