Skip to content

Commit e838b02

Browse files
TheSentinel454Elrondclaude
committed
fix(desktop): bound liveness resume and replay waits
Address the second review round on the reconnect resume cursor: - The liveUntil term never resumes later than one hour ago, matching the relay's silent-drop repair window. The relay can drop a fan-out frame under backpressure without closing the socket, so pings after the drop do not prove a subscription missed nothing. - A frame processed after a longer silence than the stall watchdog allows may have been queued through a sleep, so it no longer extends liveness credit; credit also resets per connection. - The stall watchdog runs during replay, each live EOSE deadline runs from its REQ's send, and one timeout falls back to bounded history instead of three fresh waits on the same REQ. - When the original live EOSE was never seen, no EOSE ends the history fallback's dedupe; only its deadline or teardown does. Co-authored-by: Elrond <28d6302a099e5225b02c4155ac4236e4912603df2ab08dbfc2f4fef08ce598c8@buzz.block.builderlab.xyz> Co-Authored-By: Claude Opus 5.5 <[email protected]> Signed-off-by: Luke Tornquist <[email protected]>
1 parent 211c6e8 commit e838b02

8 files changed

Lines changed: 419 additions & 97 deletions

‎desktop/src/shared/api/relayClientResumeCursor.test.mjs‎

Lines changed: 114 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -156,7 +156,8 @@ test("a close after EOSE stamps liveUntil at the last frame and the replay resum
156156
window.clearTimeout(client.reconnectTimeout);
157157
client.reconnectTimeout = null;
158158

159-
now = 2_100_000;
159+
// Reconnect late enough that liveUntil, not the silent-drop window, binds.
160+
now = 5_000_000;
160161
client.wsId = 8;
161162
writes.length = 0;
162163
await client.replayLiveSubscriptions();
@@ -166,6 +167,42 @@ test("a close after EOSE stamps liveUntil at the last frame and the replay resum
166167
);
167168
});
168169

170+
test("a quiet subscription whose fan-out was dropped still replays the drop on a prompt reconnect", async () => {
171+
// The relay can drop a fan-out frame under backpressure and keep the socket
172+
// open, so pings after the drop do not prove the subscription missed nothing.
173+
now = 10_000_000;
174+
const { client, ids } = await liveClient([quiet]);
175+
client.stallWatchdog.start();
176+
const droppedAt = 10_100; // stored, but its fan-out frame never arrived
177+
for (now = 10_030_000; now <= 12_000_000; now += 30_000) {
178+
await client.handleWsMessage(
179+
{ type: "Ping", data: [] },
180+
client.connectionGeneration,
181+
);
182+
}
183+
now -= 30_000;
184+
await close(client, 1006);
185+
const subscription = client.subscriptions.get(ids[0]);
186+
assert.equal(subscription.liveUntil, Math.floor(now / 1_000));
187+
assert.ok(
188+
subscription.liveUntil - 960 > droppedAt,
189+
"precondition: the lookback alone would skip the drop",
190+
);
191+
window.clearTimeout(client.reconnectTimeout);
192+
client.reconnectTimeout = null;
193+
194+
now += 2_000;
195+
client.wsId = 8;
196+
writes.length = 0;
197+
await client.replayLiveSubscriptions();
198+
const { since } = writes[0].frame[2];
199+
assert.equal(since, Math.floor(now / 1_000) - 3_600);
200+
assert.ok(
201+
droppedAt >= since,
202+
`drop at ${droppedAt} must satisfy since ${since}`,
203+
);
204+
});
205+
169206
test("a subscription that never reached EOSE keeps replaying its original filter", async () => {
170207
now = 2_000_000;
171208
const second = { ...quiet, "#h": ["channel-1"] };
@@ -177,6 +214,7 @@ test("a subscription that never reached EOSE keeps replaying its original filter
177214
window.clearTimeout(client.reconnectTimeout);
178215
client.reconnectTimeout = null;
179216

217+
now = 5_000_000;
180218
client.wsId = 8;
181219
writes.length = 0;
182220
await client.replayLiveSubscriptions();
@@ -284,6 +322,49 @@ test("a peer Ping is liveness; the close after it is not", async () => {
284322
assert.equal(client.subscriptions.get(ids[0]).liveUntil, 2_060);
285323
});
286324

325+
test("a frame queued through a sleep does not count as liveness", async () => {
326+
now = 2_000_000;
327+
const { client, ids } = await liveClient([quiet]);
328+
client.stallWatchdog.start();
329+
now = 2_030_000;
330+
await deliver(client, ["NOTICE", "hello"]); // last frame before the sleep
331+
// A ping and a NOTICE were queued just before the sleep; the renderer
332+
// processes them on wake, ahead of the overdue idle tick and the close.
333+
now = 2_030_000 + 3_600_000;
334+
await client.handleWsMessage(
335+
{ type: "Ping", data: [] },
336+
client.connectionGeneration,
337+
);
338+
now += 1_000;
339+
await deliver(client, ["NOTICE", "queued"]);
340+
await close(client, 1006);
341+
assert.equal(
342+
client.subscriptions.get(ids[0]).liveUntil,
343+
2_030,
344+
"stamped at the last frame before the gap, not at wake",
345+
);
346+
});
347+
348+
test("a new connection's liveness credit does not inherit the previous socket's", async () => {
349+
now = 2_000_000;
350+
const { client, ids } = await liveClient([quiet]);
351+
client.relayUrl = "ws://relay.test";
352+
await close(client, 1006);
353+
// Reconnect well past the gap limit: the AUTH OK starts fresh credit
354+
// instead of reading as a frame after a gap on the old socket's clock.
355+
timers.get(client.reconnectTimeout).at += 300_000;
356+
await redial(client);
357+
const authAt = now;
358+
now += 30_000;
359+
await deliver(client, ["EOSE", ids[0]]);
360+
now += 1_000;
361+
await close(client, 1006);
362+
assert.equal(
363+
client.subscriptions.get(ids[0]).liveUntil,
364+
Math.floor((authAt + 30_000) / 1_000),
365+
);
366+
});
367+
287368
for (const code of [1012, 1001]) {
288369
test(`close ${code} spreads the first reconnect over [0, 10s] and keeps the backoff escalated`, async () => {
289370
const { client } = await liveClient([quiet]);
@@ -369,18 +450,19 @@ test("a close delivered late during replay stamps the last frame, not the wake",
369450
await close(client, 1006);
370451
await redial(client);
371452
// The quiet channel reaches EOSE while the paged repair is still pending.
372-
now = 2_100_000;
453+
now += 30_000;
454+
const eoseAt = Math.floor(now / 1_000);
373455
await deliver(client, ["EOSE", ids[0]]);
374456
await deliver(client, ["EOSE", ids[1]]);
375457
await flush();
376458
assert.equal(repairRequests.length, 1, "precondition: repair in flight");
377-
assert.equal(intervals.size, 0, "precondition: watchdog not started yet");
459+
assert.equal(intervals.size, 1, "the watchdog runs during replay");
378460

379461
// An hour of sleep, then the close is delivered.
380-
now = 2_100_000 + 3_600_000;
462+
now += 3_600_000;
381463
await close(client, 1006);
382464
const subscription = client.subscriptions.get(ids[0]);
383-
assert.equal(subscription.liveUntil, 2_100, "stamped at the EOSE frame");
465+
assert.equal(subscription.liveUntil, eoseAt, "stamped at the EOSE frame");
384466

385467
// The in-flight HTTP page settles; the stale attempt then stands down.
386468
for (const { resolve } of repairRequests.splice(0)) resolve([]);
@@ -389,7 +471,33 @@ test("a close delivered late during replay stamps the last frame, not the wake",
389471
writes.length = 0;
390472
await redial(client);
391473
const quietReq = writes.find(({ frame }) => frame[1] === ids[0]);
392-
assert.equal(quietReq.frame[2].since, 2_100 - 960);
474+
assert.equal(quietReq.frame[2].since, eoseAt - 960);
475+
});
476+
477+
test("a half-open socket that never acknowledges replay is reset by the stall watchdog", async () => {
478+
now = 2_000_000;
479+
const { client, ids } = await liveClient([paged]);
480+
client.relayUrl = "ws://relay.test";
481+
await close(client, 1006);
482+
await redial(client);
483+
assert.deepEqual(writes.at(-1).frame, [
484+
"REQ",
485+
ids[0],
486+
{ ...paged, limit: 0 },
487+
]);
488+
assert.notEqual(client.connectPromise, null, "precondition: parked on EOSE");
489+
assert.equal(client.connectionStateEmitter.get(), "connected");
490+
assert.equal(intervals.size, 1, "the watchdog is armed while parked");
491+
492+
// No EOSE, no close: the idle tick is what notices.
493+
now += 70_000;
494+
const [tick] = intervals.values();
495+
tick();
496+
await flush();
497+
assert.equal(client.connectionStateEmitter.get(), "stalled");
498+
assert.equal(client.connectPromise, null, "connect() released by the reset");
499+
assert.equal(intervals.size, 0);
500+
assert.equal(repairRequests.length, 0, "no blind repair");
393501
});
394502

395503
test("disposing a subscription parked on its paged live EOSE releases connect()", async () => {

‎desktop/src/shared/api/relayClientSession.ts‎

Lines changed: 10 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -41,6 +41,7 @@ import {
4141
replayLiveSubscriptions,
4242
retireConnectionSubscriptions,
4343
} from "@/shared/api/relayReconnectReplay";
44+
import { creditInboundLiveness } from "@/shared/api/relayResumeCursor";
4445
import { RelayLiveReqDrain } from "./relayLiveReqDrain";
4546
import { RelayChannelAccessRevocations } from "./relayChannelAccessRevocations";
4647
import { publishSessionEvent } from "@/shared/api/relayEventPublisher";
@@ -98,7 +99,7 @@ export class RelayClient {
9899
private notifyReconnectListeners = false;
99100
private onMessageChannel: Channel<unknown> | null = null;
100101
private connectionGeneration = 0;
101-
private lastInboundAtMs = 0; // Last peer frame (any connection); 0 = none.
102+
private lastInboundAtMs = 0; // Liveness credit on this connection; 0 = none.
102103
private sessionEpoch = 0;
103104
private liveSessionAbort = new AbortController();
104105
private visibleChannelId: string | null = null;
@@ -545,6 +546,7 @@ export class RelayClient {
545546
this.hasConnectedOnce ? "reconnecting" : "connecting",
546547
);
547548
const generation = ++this.connectionGeneration;
549+
this.lastInboundAtMs = 0;
548550
const inbound = createRelayInboundBuffer(
549551
async (delivery) => {
550552
for (const message of toRelayFrames(delivery))
@@ -591,10 +593,12 @@ export class RelayClient {
591593
this.reconnectBackoff.resetWhenStable();
592594

593595
this.connectionStateEmitter.set("connected");
596+
// Watch the socket during replay too: a half-open socket that never
597+
// sends EOSE or a close must not hold recovery behind the EOSE waits.
598+
this.stallWatchdog.start();
594599
await this.replayLiveSubscriptions();
595600
if (generation !== this.connectionGeneration)
596601
throw new Error("Relay connection attempt was superseded.");
597-
this.stallWatchdog.start();
598602
this.emitReconnectIfNeeded();
599603
return generation;
600604
} catch (error) {
@@ -874,7 +878,10 @@ export class RelayClient {
874878
}
875879
// Only peer frames prove liveness; Close/Error say when the loss was seen.
876880
this.stallWatchdog.recordInbound();
877-
this.lastInboundAtMs = Date.now();
881+
this.lastInboundAtMs = creditInboundLiveness(
882+
this.lastInboundAtMs,
883+
Date.now(),
884+
);
878885

879886
const payload = getTextPayload(message);
880887
if (!payload) {

‎desktop/src/shared/api/relayClientShared.ts‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -105,6 +105,14 @@ type LiveSubscription = {
105105
seenEventIds: Set<string>;
106106
liveEose: boolean;
107107
repairDone: boolean;
108+
/** When the restored live REQ went out; its EOSE deadline runs from here. */
109+
liveReqSentAtMs?: number;
110+
/**
111+
* Set when the history fallback follows an unacknowledged live REQ: a late
112+
* original EOSE is then indistinguishable from the fallback's, so no EOSE
113+
* ends dedupe; only the fallback's deadline or teardown does.
114+
*/
115+
ignoreEose?: boolean;
108116
/**
109117
* Wakes the repair worker waiting for the restored live REQ's EOSE: on
110118
* EOSE, or when a newer replay supersedes this one.

‎desktop/src/shared/api/relayClosedRecovery.ts‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -260,7 +260,7 @@ export function markReconnectLiveEose(
260260
generation: number,
261261
) {
262262
const replay = subscription.reconnectReplay;
263-
if (!replay || replay.generation !== generation) return;
263+
if (!replay || replay.generation !== generation || replay.ignoreEose) return;
264264
replay.liveEose = true;
265265
replay.liveEoseWaiter?.();
266266
replay.liveEoseWaiter = undefined;

0 commit comments

Comments
 (0)