Skip to content

Commit fc22719

Browse files
committed
fix(broker): serialize thread unsubscribe against reacquisition
A thread resumed while its automatic thread/unsubscribe was still in flight was never released again: the later release reused the stale pending request, so the shared app-server connection stayed subscribed after every client had closed. If the app-server had processed the unsubscribe after the resume, the new owner would also have lost its subscription. Never reuse a pending unsubscribe. A release chains a fresh request behind any in-flight one and re-checks ownership before sending. A resume or review that provisionally claims a thread waits, bounded to five seconds, for an in-flight unsubscribe of that thread before it is dispatched, so a late unsubscribe cannot overtake the new subscription and a hung cleanup cannot wedge the broker. Reply to a failed request and clear the busy state before releasing provisional owners for the same reason. Add the unsubscribe-delayed and resume-fails-unsubscribe-hangs fixture behaviors, four regression tests, and assert the retry bound in the existing failure test.
1 parent 1203f5d commit fc22719

3 files changed

Lines changed: 200 additions & 13 deletions

File tree

‎plugins/codex/scripts/app-server-broker.mjs‎

Lines changed: 54 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,23 @@ import { parseBrokerEndpoint } from "./lib/broker-endpoint.mjs";
1212
const STREAMING_METHODS = new Set(["turn/start", "review/start", "thread/compact/start"]);
1313
const SUBSCRIBING_METHODS = new Set(["thread/start", "thread/resume", "thread/fork"]);
1414
const UNSUBSCRIBE_RETRY_DELAYS_MS = [100, 500, 2000];
15+
// Upper bound on how long a request waits for an in-flight thread/unsubscribe of
16+
// the same thread. A hung cleanup request must not wedge the shared broker.
17+
const UNSUBSCRIBE_WAIT_TIMEOUT_MS = 5000;
18+
19+
function settleWithin(promise, timeoutMs) {
20+
let timer;
21+
return Promise.race([
22+
promise.then(
23+
() => {},
24+
() => {}
25+
),
26+
new Promise((resolve) => {
27+
timer = setTimeout(resolve, timeoutMs);
28+
timer.unref?.();
29+
})
30+
]).finally(() => clearTimeout(timer));
31+
}
1532

1633
function buildSubscriptionThreadIds(method, result) {
1734
const threadIds = new Set();
@@ -167,25 +184,39 @@ async function main() {
167184
}
168185

169186
function requestThreadUnsubscribe(threadId) {
170-
const pending = pendingUnsubscribes.get(threadId);
171-
if (pending) {
172-
return pending;
173-
}
174-
const request = appClient.request("thread/unsubscribe", { threadId }).then(
175-
(result) => ({ result, error: null }),
187+
// Never reuse an earlier request: the thread may have been reacquired and
188+
// released again while that request was in flight. Chain behind it so the
189+
// app-server sees one unsubscribe at a time, then re-check ownership.
190+
const previous = pendingUnsubscribes.get(threadId);
191+
const execute = async () => {
192+
if (previous) {
193+
await settleWithin(previous, UNSUBSCRIBE_WAIT_TIMEOUT_MS);
194+
}
195+
if (threadSockets.has(threadId)) {
196+
return { result: null, error: null, skipped: true };
197+
}
198+
const result = await appClient.request("thread/unsubscribe", { threadId });
199+
return { result, error: null };
200+
};
201+
let request;
202+
request = execute().then(
203+
(outcome) => {
204+
if (pendingUnsubscribes.get(threadId) === request) {
205+
pendingUnsubscribes.delete(threadId);
206+
}
207+
return outcome;
208+
},
176209
(error) => {
210+
if (pendingUnsubscribes.get(threadId) === request) {
211+
pendingUnsubscribes.delete(threadId);
212+
}
177213
process.stderr.write(
178214
`Failed to unsubscribe Codex thread ${threadId}: ${error instanceof Error ? error.message : String(error)}\n`
179215
);
180216
return { result: null, error };
181217
}
182218
);
183219
pendingUnsubscribes.set(threadId, request);
184-
void request.finally(() => {
185-
if (pendingUnsubscribes.get(threadId) === request) {
186-
pendingUnsubscribes.delete(threadId);
187-
}
188-
});
189220
return request;
190221
}
191222

@@ -426,6 +457,15 @@ async function main() {
426457
}
427458
}
428459
activeRequestSocket = socket;
460+
// Let an in-flight unsubscribe for the same thread settle first so it cannot
461+
// overtake the new subscription. The wait is bounded: a hung cleanup request
462+
// must not block this client or keep the broker busy for everyone else.
463+
await Promise.all(
464+
[...provisionalThreadIds].map((threadId) => {
465+
const pending = pendingUnsubscribes.get(threadId);
466+
return pending ? settleWithin(pending, UNSUBSCRIBE_WAIT_TIMEOUT_MS) : null;
467+
})
468+
);
429469

430470
try {
431471
const result =
@@ -442,7 +482,6 @@ async function main() {
442482
activeRequestSocket = null;
443483
}
444484
} catch (error) {
445-
await releaseThreadOwners(socket, addedProvisionalThreadIds);
446485
send(socket, {
447486
id: message.id,
448487
error: buildJsonRpcError(error.rpcCode ?? -32000, error.message)
@@ -453,6 +492,9 @@ async function main() {
453492
if (activeStreamSocket === socket && !isStreaming) {
454493
activeStreamSocket = null;
455494
}
495+
// Release after replying: a hung upstream unsubscribe must not withhold
496+
// the error or leave the broker busy for other clients.
497+
void releaseThreadOwners(socket, addedProvisionalThreadIds);
456498
}
457499
}
458500

‎tests/broker-subscriptions.test.mjs‎

Lines changed: 118 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -250,6 +250,53 @@ test("broker keeps a resumed thread subscribed until its final client closes", a
250250
await waitForUnsubscribes(broker.statePath, [threadId]);
251251
});
252252

253+
test("broker serializes a resume behind an in-flight unsubscribe", async (t) => {
254+
const broker = startBroker("unsubscribe-delayed");
255+
t.after(() => broker.stop());
256+
assert.equal(await broker.listening(), true, `broker never listened: ${broker.stderr()}`);
257+
258+
const firstClient = await connectClient(broker.socketPath);
259+
const threadId = (await firstClient.request("thread/start", { cwd: process.cwd(), ephemeral: false })).thread.id;
260+
await firstClient.end();
261+
const unsubscribeStarted = await waitFor(() =>
262+
readState(broker.statePath)?.unsubscribeRequests?.includes(threadId)
263+
);
264+
assert.equal(unsubscribeStarted, true, "unsubscribe request was not observed");
265+
266+
const secondClient = await connectClient(broker.socketPath);
267+
await secondClient.request("thread/resume", { threadId });
268+
assert.deepEqual(readState(broker.statePath).requestOrder, ["unsubscribe:response", "thread/resume"]);
269+
assert.deepEqual(readState(broker.statePath).subscriptions, [threadId]);
270+
271+
await secondClient.end();
272+
await waitForUnsubscribes(broker.statePath, [threadId, threadId]);
273+
assert.deepEqual(readState(broker.statePath).subscriptions, []);
274+
});
275+
276+
test("broker cancels a retry when a client reacquires the thread", async (t) => {
277+
const broker = startBroker("unsubscribe-fails-once");
278+
t.after(() => broker.stop());
279+
assert.equal(await broker.listening(), true, `broker never listened: ${broker.stderr()}`);
280+
281+
const firstClient = await connectClient(broker.socketPath);
282+
const threadId = (await firstClient.request("thread/start", { cwd: process.cwd(), ephemeral: false })).thread.id;
283+
await firstClient.end();
284+
const firstAttemptObserved = await waitFor(
285+
() => readState(broker.statePath)?.unsubscribeRequests?.length === 1
286+
);
287+
assert.equal(firstAttemptObserved, true, "initial unsubscribe request was not observed");
288+
289+
const secondClient = await connectClient(broker.socketPath);
290+
await secondClient.request("thread/resume", { threadId });
291+
await delay(3000);
292+
assert.deepEqual(readState(broker.statePath).subscriptions, [threadId]);
293+
294+
await secondClient.end();
295+
const unsubscribed = await waitFor(() => readState(broker.statePath)?.subscriptions?.length === 0);
296+
assert.equal(unsubscribed, true, "reacquired thread was not unsubscribed after its final owner closed");
297+
assert.equal(readState(broker.statePath).unsubscribeRequests.at(-1), threadId);
298+
});
299+
253300
test("broker unsubscribes source and detached review threads", async (t) => {
254301
const broker = startBroker();
255302
t.after(() => broker.stop());
@@ -448,6 +495,40 @@ test("broker serializes subscription requests from one downstream client", async
448495
await waitForUnsubscribes(broker.statePath, [threadId]);
449496
});
450497

498+
test("broker replies to a failed resume even when the upstream unsubscribe hangs", async (t) => {
499+
const broker = startBroker("resume-fails-unsubscribe-hangs");
500+
t.after(() => broker.stop());
501+
assert.equal(await broker.listening(), true, `broker never listened: ${broker.stderr()}`);
502+
503+
// Nobody owns this thread, so the failed provisional claim releases it upstream.
504+
const threadId = "thr_unowned";
505+
const firstClient = await connectClient(broker.socketPath);
506+
const failedResume = await waitWithTimeout(
507+
firstClient.request("thread/resume", { threadId, persistFullHistory: true }).then(
508+
() => ({ status: "fulfilled" }),
509+
(error) => ({ status: "rejected", error })
510+
),
511+
2000
512+
);
513+
assert.notEqual(failedResume, null, "failed resume did not receive a response");
514+
assert.equal(failedResume.status, "rejected");
515+
assert.match(failedResume.error.message, /forced resume failure/);
516+
517+
const secondClient = await connectClient(broker.socketPath);
518+
const secondStarted = await waitWithTimeout(
519+
secondClient.request("thread/start", { cwd: process.cwd(), ephemeral: false }),
520+
2000
521+
);
522+
assert.notEqual(secondStarted, null, "broker remained busy after the failed resume");
523+
524+
const unsubscribeStarted = await waitFor(() =>
525+
readState(broker.statePath)?.unsubscribeRequests?.includes(threadId)
526+
);
527+
assert.equal(unsubscribeStarted, true, "the released thread was not unsubscribed upstream");
528+
await secondClient.end();
529+
await firstClient.end();
530+
});
531+
451532
test("broker keeps shared upstream subscriptions when one client explicitly unsubscribes", async (t) => {
452533
const broker = startBroker();
453534
t.after(() => broker.stop());
@@ -503,6 +584,13 @@ test("broker logs upstream unsubscribe failures", async (t) => {
503584
() => broker.stderr().includes(`Failed to unsubscribe Codex thread ${threadId}: thread unsubscribe failed`)
504585
);
505586
assert.equal(warningObserved, true, "unsubscribe failure was not logged");
587+
const retryBoundReached = await waitFor(
588+
() => readState(broker.statePath)?.unsubscribeRequests?.length === 4,
589+
{ timeoutMs: 6000 }
590+
);
591+
assert.equal(retryBoundReached, true, "unsubscribe retries did not reach the expected bound");
592+
await delay(500);
593+
assert.equal(readState(broker.statePath).unsubscribeRequests.length, 4);
506594
assert.deepEqual(readState(broker.statePath).subscriptions, [threadId]);
507595
});
508596

@@ -523,3 +611,33 @@ test("broker retries a transient upstream unsubscribe failure", async (t) => {
523611
assert.deepEqual(readState(broker.statePath).unsubscribeRequests, [threadId, threadId]);
524612
assert.match(broker.stderr(), new RegExp(`Failed to unsubscribe Codex thread ${threadId}`));
525613
});
614+
615+
test("broker bounds the wait for a hung unsubscribe before a resume proceeds", async (t) => {
616+
const broker = startBroker("resume-fails-unsubscribe-hangs");
617+
t.after(() => broker.stop());
618+
assert.equal(await broker.listening(), true, `broker never listened: ${broker.stderr()}`);
619+
620+
const firstClient = await connectClient(broker.socketPath);
621+
const threadId = (await firstClient.request("thread/start", { cwd: process.cwd(), ephemeral: false })).thread.id;
622+
await firstClient.end();
623+
const unsubscribeStarted = await waitFor(() =>
624+
readState(broker.statePath)?.unsubscribeRequests?.includes(threadId)
625+
);
626+
assert.equal(unsubscribeStarted, true, "unsubscribe request was not observed");
627+
628+
const secondClient = await connectClient(broker.socketPath);
629+
const startedAt = Date.now();
630+
const resumed = await waitWithTimeout(secondClient.request("thread/resume", { threadId }), 8000);
631+
assert.notEqual(resumed, null, "resume never completed while the upstream unsubscribe hung");
632+
assert.ok(Date.now() - startedAt >= 4000, "resume did not wait for the in-flight unsubscribe");
633+
assert.deepEqual(readState(broker.statePath).subscriptions, [threadId]);
634+
635+
const thirdClient = await connectClient(broker.socketPath);
636+
const thirdStarted = await waitWithTimeout(
637+
thirdClient.request("thread/start", { cwd: process.cwd(), ephemeral: false }),
638+
2000
639+
);
640+
assert.notEqual(thirdStarted, null, "broker remained busy after the bounded wait");
641+
await thirdClient.end();
642+
await secondClient.end();
643+
});

‎tests/fake-codex-fixture.mjs‎

Lines changed: 28 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,7 @@ const readline = require("node:readline");
1919
2020
function loadState() {
2121
if (!fs.existsSync(STATE_PATH)) {
22-
return { nextThreadId: 1, nextTurnId: 1, appServerStarts: 0, threads: [], subscriptions: [], unsubscribeRequests: [], capabilities: null, lastInterrupt: null };
22+
return { nextThreadId: 1, nextTurnId: 1, appServerStarts: 0, threads: [], subscriptions: [], unsubscribeRequests: [], requestOrder: [], capabilities: null, lastInterrupt: null };
2323
}
2424
return JSON.parse(fs.readFileSync(STATE_PATH, "utf8"));
2525
}
@@ -347,6 +347,12 @@ rl.on("line", (line) => {
347347
}
348348
349349
case "thread/resume": {
350+
if (BEHAVIOR === "resume-fails-unsubscribe-hangs" && message.params.persistFullHistory === true) {
351+
setTimeout(() => {
352+
send({ id: message.id, error: { code: -32000, message: "forced resume failure" } });
353+
}, 50);
354+
break;
355+
}
350356
if (BEHAVIOR === "overlapping-resume" && message.params.persistFullHistory === true) {
351357
setTimeout(() => {
352358
send({ id: message.id, error: { code: -32000, message: "forced delayed resume failure" } });
@@ -358,6 +364,9 @@ rl.on("line", (line) => {
358364
}
359365
const thread = ensureThread(state, message.params.threadId);
360366
thread.updatedAt = now();
367+
if (BEHAVIOR === "unsubscribe-delayed") {
368+
state.requestOrder = [...(state.requestOrder || []), "thread/resume"];
369+
}
361370
state.subscriptions = [...new Set([...(state.subscriptions || []), thread.id])];
362371
saveState(state);
363372
send({ id: message.id, result: { thread: buildThread(thread), model: message.params.model || "gpt-5.4", modelProvider: "openai", serviceTier: null, cwd: thread.cwd, approvalPolicy: "never", sandbox: { type: "readOnly", access: { type: "fullAccess" }, networkAccess: false }, reasoningEffort: null } });
@@ -379,6 +388,10 @@ rl.on("line", (line) => {
379388
const wasSubscribed = subscriptions.includes(message.params.threadId);
380389
const wasLoaded = state.threads.some((thread) => thread.id === message.params.threadId);
381390
state.unsubscribeRequests = [...(state.unsubscribeRequests || []), message.params.threadId];
391+
if (BEHAVIOR === "resume-fails-unsubscribe-hangs") {
392+
saveState(state);
393+
break;
394+
}
382395
if (
383396
BEHAVIOR === "unsubscribe-fails" ||
384397
(BEHAVIOR === "unsubscribe-fails-once" && state.unsubscribeRequests.length === 1)
@@ -387,6 +400,20 @@ rl.on("line", (line) => {
387400
send({ id: message.id, error: { code: -32000, message: "thread unsubscribe failed" } });
388401
break;
389402
}
403+
if (BEHAVIOR === "unsubscribe-delayed") {
404+
state.subscriptions = subscriptions.filter((threadId) => threadId !== message.params.threadId);
405+
saveState(state);
406+
setTimeout(() => {
407+
const delayedState = loadState();
408+
delayedState.requestOrder = [...(delayedState.requestOrder || []), "unsubscribe:response"];
409+
saveState(delayedState);
410+
send({
411+
id: message.id,
412+
result: { status: wasSubscribed ? "unsubscribed" : wasLoaded ? "notSubscribed" : "notLoaded" }
413+
});
414+
}, 300);
415+
break;
416+
}
390417
if (BEHAVIOR === "unsubscribe-notifies") {
391418
send({
392419
method: "thread/status/changed",

0 commit comments

Comments
 (0)