Skip to content

Commit 883c7d9

Browse files
feat(server): pause running threads before an application update (#215)
An update used to cancel in-flight turns when the process restarted, so there was nothing to resume. Stop those threads before the handoff, and if one is still running when the process exits, mark it interrupted. Model: Grok 4.7 via Cursor Co-authored-by: Cursor <[email protected]>
1 parent e5ef716 commit 883c7d9

20 files changed

Lines changed: 528 additions & 36 deletions

‎apps/desktop/src/updates/DesktopUpdates.ts‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -635,12 +635,12 @@ export const make = Effect.gen(function* () {
635635
// means quitAndInstall's app.quit() exits before the pool's
636636
// scope cascade has a chance to run its stop finalizer, so the
637637
// WSL child gets hard-killed by the OS instead of receiving
638-
// SIGTERM + grace. Stops run concurrently with the same 5s
639-
// budget the primary had on its own.
638+
// SIGTERM + grace. The budget covers pausing running threads
639+
// before the process exits, so they can be resumed after relaunch.
640640
const instances = yield* pool.list;
641641
yield* Effect.forEach(
642642
instances,
643-
(instance) => instance.stop({ timeout: Duration.seconds(5) }),
643+
(instance) => instance.stop({ timeout: Duration.seconds(30) }),
644644
{ concurrency: "unbounded" },
645645
);
646646
yield* electronUpdater.quitAndInstall({

‎apps/mobile/src/features/settings/SettingsEnvironmentDetailRouteScreen.tsx‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -104,7 +104,7 @@ function EnvironmentDetail({ environmentId }: { readonly environmentId: Environm
104104
return;
105105
Alert.alert(
106106
`Update ${environment?.environmentLabel ?? "environment"}?`,
107-
`Install T3 Code ${targetVersion}. ${capabilities.serverSelfUpdate === "desktop-managed" ? "The desktop app will close and relaunch." : "The server will restart and reconnect."} Running threads may be interrupted.`,
107+
`Install T3 Code ${targetVersion}. ${capabilities.serverSelfUpdate === "desktop-managed" ? "The desktop app will close and relaunch." : "The server will restart and reconnect."} Running threads will be paused so you can resume them.`,
108108
[
109109
{ text: "Cancel", style: "cancel" },
110110
{

‎apps/mobile/src/features/settings/SettingsServerControlsRouteScreen.tsx‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -465,7 +465,7 @@ function ServerSettingsDetail(props: { readonly page: SettingsPage }) {
465465
label="Continue after restart"
466466
subtitle={
467467
supportsContinuation
468-
? "Resume interrupted threads after an update or restart."
468+
? "Resume interrupted threads after a crash or restart. Updates pause them so you can resume."
469469
: "Update older servers to control restart continuation."
470470
}
471471
value={uniform("continueThreadsAfterServerUpdate")}

‎apps/server/src/cloud/selfUpdate.test.ts‎

Lines changed: 81 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -161,6 +161,52 @@ it.layer(NodeServices.layer)("server self update", (it) => {
161161
}),
162162
);
163163

164+
it.effect("pauses running threads before the boot-service handoff", () =>
165+
Effect.gen(function* () {
166+
const events: string[] = [];
167+
const selfUpdate = yield* ServerSelfUpdate.withRunningThreadContinuation({
168+
mode: "web",
169+
selfUpdate: {
170+
update: (_input, reportProgress = () => Effect.void) =>
171+
reportProgress("downloading").pipe(
172+
Effect.andThen(reportProgress("installing")),
173+
Effect.as({
174+
targetVersion: "1.1.0",
175+
method: "boot-service" as const,
176+
updateId: "update-id",
177+
}),
178+
),
179+
commitDesktopUpdate: () => Effect.never,
180+
},
181+
pause: Effect.sync(() => {
182+
events.push("pause");
183+
}),
184+
prepare: Effect.sync(() => {
185+
events.push("prepare");
186+
return [ThreadId.make("thread-running")];
187+
}),
188+
clear: () => Effect.void,
189+
});
190+
191+
yield* selfUpdate.update({ targetVersion: "1.1.0", continueRunningThreads: true }, (stage) =>
192+
Effect.sync(() => void events.push(stage)),
193+
);
194+
yield* selfUpdate.update({ targetVersion: "1.1.0" }, (stage) =>
195+
Effect.sync(() => void events.push(stage)),
196+
);
197+
198+
expect(events).toEqual([
199+
"downloading",
200+
"pause",
201+
"prepare",
202+
"installing",
203+
"downloading",
204+
"pause",
205+
"installing",
206+
]);
207+
}),
208+
);
209+
164210
it.effect("marks desktop threads only when the prepared update commits", () =>
165211
Effect.gen(function* () {
166212
const threadId = ThreadId.make("thread-running-desktop");
@@ -210,6 +256,41 @@ it.layer(NodeServices.layer)("server self update", (it) => {
210256
}),
211257
);
212258

259+
it.effect("pauses desktop threads before the prepared update commits", () =>
260+
Effect.gen(function* () {
261+
const events: string[] = [];
262+
const commitError = new ServerSelfUpdateError({ reason: "install failed" });
263+
const selfUpdate = yield* ServerSelfUpdate.withRunningThreadContinuation({
264+
mode: "desktop",
265+
selfUpdate: {
266+
update: () =>
267+
Effect.succeed({
268+
targetVersion: "1.2.0",
269+
method: "desktop-app" as const,
270+
desktopUpdateToken: "desktop-token",
271+
}),
272+
commitDesktopUpdate: () =>
273+
Effect.sync(() => events.push("commit")).pipe(Effect.andThen(Effect.fail(commitError))),
274+
},
275+
pause: Effect.sync(() => {
276+
events.push("pause");
277+
}),
278+
prepare: Effect.sync(() => {
279+
events.push("prepare");
280+
return [ThreadId.make("thread-running-desktop")];
281+
}),
282+
clear: () => Effect.void,
283+
});
284+
285+
yield* selfUpdate.update({ targetVersion: "1.2.0", continueRunningThreads: true });
286+
expect(events).toEqual([]);
287+
expect(yield* selfUpdate.commitDesktopUpdate("desktop-token").pipe(Effect.flip)).toBe(
288+
commitError,
289+
);
290+
expect(events).toEqual(["pause", "prepare", "commit"]);
291+
}),
292+
);
293+
213294
it.effect("reports a failed continuation-marker cleanup", () =>
214295
Effect.gen(function* () {
215296
const updateError = new ServerSelfUpdateError({ reason: "update failed" });

‎apps/server/src/cloud/selfUpdate.ts‎

Lines changed: 33 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -67,6 +67,8 @@ export const withRunningThreadContinuation = Effect.fn(
6767
)(function* (input: {
6868
readonly mode: ServerConfig.RuntimeMode;
6969
readonly selfUpdate: ServerSelfUpdate["Service"];
70+
/** Stops running threads before the process is replaced, so the user can resume them. */
71+
readonly pause?: Effect.Effect<void, ServerSelfUpdateError>;
7072
readonly prepare: Effect.Effect<ReadonlyArray<ThreadId>, ServerSelfUpdateError>;
7173
readonly clear: (
7274
threadIds: ReadonlyArray<ThreadId>,
@@ -91,29 +93,44 @@ export const withRunningThreadContinuation = Effect.fn(
9193
request,
9294
reportProgress = () => Effect.void,
9395
) => {
96+
let paused = false;
9497
let prepared = false;
9598
let handoffAccepted = false;
9699
let continuationThreadIds: ReadonlyArray<ThreadId> = [];
100+
const pauseBeforeHandoff = (stage: ServerSelfUpdateProgressStage) =>
101+
input.pause !== undefined && input.mode !== "desktop" && stage === "installing" && !paused
102+
? input.pause.pipe(
103+
Effect.tap(() =>
104+
Effect.sync(() => {
105+
paused = true;
106+
}),
107+
),
108+
)
109+
: Effect.void;
97110
return clearOnError(
98111
input.selfUpdate
99112
.update(
100113
request,
101114
(stage) =>
102-
(request.continueRunningThreads === true &&
103-
input.mode !== "desktop" &&
104-
stage === "installing" &&
105-
!prepared
106-
? input.prepare.pipe(
107-
Effect.tap((threadIds) =>
108-
Effect.sync(() => {
109-
prepared = true;
110-
continuationThreadIds = threadIds;
111-
}),
112-
),
113-
Effect.asVoid,
114-
)
115-
: Effect.void
116-
).pipe(Effect.andThen(reportProgress(stage))),
115+
pauseBeforeHandoff(stage).pipe(
116+
Effect.andThen(
117+
request.continueRunningThreads === true &&
118+
input.mode !== "desktop" &&
119+
stage === "installing" &&
120+
!prepared
121+
? input.prepare.pipe(
122+
Effect.tap((threadIds) =>
123+
Effect.sync(() => {
124+
prepared = true;
125+
continuationThreadIds = threadIds;
126+
}),
127+
),
128+
Effect.asVoid,
129+
)
130+
: Effect.void,
131+
),
132+
Effect.andThen(reportProgress(stage)),
133+
),
117134
() =>
118135
Effect.sync(() => {
119136
handoffAccepted = true;
@@ -148,6 +165,7 @@ export const withRunningThreadContinuation = Effect.fn(
148165
let continuationThreadIds: ReadonlyArray<ThreadId> = [];
149166
return yield* clearOnError(
150167
Effect.gen(function* () {
168+
yield* input.pause ?? Effect.void;
151169
continuationThreadIds = shouldContinue ? yield* input.prepare : [];
152170
return yield* input.selfUpdate.commitDesktopUpdate(requestId, () =>
153171
Effect.sync(() => {
Lines changed: 44 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,44 @@
1+
// @effect-diagnostics nodeBuiltinImport:off globalDate:off - the helper under test is synchronous.
2+
import * as NodeFS from "node:fs";
3+
import * as NodeOS from "node:os";
4+
import * as NodePath from "node:path";
5+
import { expect, it } from "@effect/vitest";
6+
7+
import { DESKTOP_UPDATE_RESTART_MARKER_FILE } from "@t3tools/contracts";
8+
9+
import { SERVICE_STATE_FILE, SERVICE_STOP_MARKER_FILE } from "./serviceProtocol.ts";
10+
import { updateRestartPendingSync } from "./updateHandoff.ts";
11+
12+
function runtimeDir(): string {
13+
const dir = NodeFS.mkdtempSync(NodePath.join(NodeOS.tmpdir(), "t3-update-restart-"));
14+
const runtime = NodePath.join(dir, "runtime");
15+
NodeFS.mkdirSync(runtime);
16+
return runtime;
17+
}
18+
19+
it("sees a fresh desktop update marker", () => {
20+
const runtime = runtimeDir();
21+
NodeFS.writeFileSync(NodePath.join(runtime, DESKTOP_UPDATE_RESTART_MARKER_FILE), "");
22+
expect(updateRestartPendingSync(NodePath.dirname(runtime))).toBe(true);
23+
});
24+
25+
it("ignores a stale desktop update marker", () => {
26+
const runtime = runtimeDir();
27+
const marker = NodePath.join(runtime, DESKTOP_UPDATE_RESTART_MARKER_FILE);
28+
NodeFS.writeFileSync(marker, "");
29+
const stale = new Date(Date.now() - 5 * 60_000);
30+
NodeFS.utimesSync(marker, stale, stale);
31+
expect(updateRestartPendingSync(NodePath.dirname(runtime))).toBe(false);
32+
});
33+
34+
it("sees a pending service update and ignores an explicit stop", () => {
35+
const runtime = runtimeDir();
36+
const baseDir = NodePath.dirname(runtime);
37+
NodeFS.writeFileSync(
38+
NodePath.join(runtime, SERVICE_STATE_FILE),
39+
JSON.stringify({ update: { status: "pending" } }),
40+
);
41+
expect(updateRestartPendingSync(baseDir)).toBe(true);
42+
NodeFS.writeFileSync(NodePath.join(runtime, SERVICE_STOP_MARKER_FILE), "");
43+
expect(updateRestartPendingSync(baseDir)).toBe(false);
44+
});

‎apps/server/src/cloud/updateHandoff.ts‎

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,11 @@
1+
// @effect-diagnostics nodeBuiltinImport:off globalDate:off - a signal handler reads the update marker before Effect finalizers run.
12
/**
23
* Whether this server is going down only to be replaced by an update. Shutdown
34
* keeps the managed tunnel across those restarts instead of releasing it.
45
*/
56
import { DESKTOP_UPDATE_RESTART_MARKER_FILE } from "@t3tools/contracts";
7+
import * as NodeFS from "node:fs";
8+
import * as NodePath from "node:path";
69
import * as Clock from "effect/Clock";
710
import * as Duration from "effect/Duration";
811
import * as Effect from "effect/Effect";
@@ -52,6 +55,31 @@ export const pendingUpdateHandoffExists = Effect.gen(function* () {
5255
return !stopping;
5356
});
5457

58+
/**
59+
* True when this process is about to be replaced by an update. Safe from a
60+
* signal handler: it does not remove the desktop marker, so a later reader can
61+
* still consume it.
62+
*/
63+
export function updateRestartPendingSync(baseDir: string): boolean {
64+
const runtimeDir = NodePath.join(baseDir, "runtime");
65+
try {
66+
const stat = NodeFS.statSync(NodePath.join(runtimeDir, DESKTOP_UPDATE_RESTART_MARKER_FILE));
67+
if (Date.now() - stat.mtimeMs < Duration.toMillis(DESKTOP_UPDATE_RESTART_MARKER_TTL)) {
68+
return true;
69+
}
70+
} catch {
71+
// No desktop update marker.
72+
}
73+
try {
74+
if (NodeFS.existsSync(NodePath.join(runtimeDir, SERVICE_STOP_MARKER_FILE))) return false;
75+
return serviceStateHasPendingUpdate(
76+
NodeFS.readFileSync(NodePath.join(runtimeDir, SERVICE_STATE_FILE), "utf8"),
77+
);
78+
} catch {
79+
return false;
80+
}
81+
}
82+
5583
// The desktop app writes its marker right before it stops this server to
5684
// install an update, whether a remote client or the local app started it.
5785
// Reading consumes it, so shutdown checks it first. Only a fresh marker counts,

‎apps/server/src/orchestration-v2/ProviderRuntimeRecoveryService.test.ts‎

Lines changed: 59 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -578,6 +578,65 @@ it.effect("cancels a stale waiting run when no checkpoint capture can finish it"
578578
}).pipe(Effect.provide(layer));
579579
});
580580

581+
it.effect("interrupts a running turn when shutdown is an application update", () => {
582+
const threadId = ThreadId.make("thread_update_pause");
583+
const runId = RunId.make("run_update_pause");
584+
let committedInput: Parameters<EventSink.EventSinkV2["Service"]["commitCommand"]>[0] | null =
585+
null;
586+
const projection = {
587+
thread: { id: threadId },
588+
runtimeRequests: [],
589+
providerSessions: [],
590+
providerThreads: [],
591+
providerTurns: [],
592+
runs: [
593+
{
594+
id: runId,
595+
status: "running",
596+
providerInstanceId: ProviderInstanceId.make("codex"),
597+
},
598+
],
599+
attempts: [],
600+
nodes: [],
601+
subagents: [],
602+
messages: [],
603+
turnItems: [],
604+
} as unknown as OrchestrationV2ThreadProjection;
605+
const layer = ProviderRuntimeRecovery.layer.pipe(
606+
Layer.provide(ServerSettings.layerTest()),
607+
Layer.provide(
608+
Layer.mergeAll(
609+
Layer.mock(ProjectionStore.ProjectionStoreV2)({
610+
getRecoveryThreadIds: () => Effect.succeed([threadId]),
611+
getRuntimeRecoveryProjection: () => Effect.succeed(projection),
612+
}),
613+
Layer.mock(EventSink.EventSinkV2)({
614+
commitCommand: (input) => {
615+
committedInput = input;
616+
return Effect.succeed({ committed: true, cancelledEffectCount: 0 } as never);
617+
},
618+
}),
619+
IdAllocator.layer,
620+
Layer.mock(EffectWorker.OrchestrationEffectWorkerV2)({
621+
runRecoveryOnce: Effect.succeed(false),
622+
}),
623+
Layer.mock(EffectOutbox.EffectOutboxV2)({
624+
listByCommandId: () => Effect.succeed([]),
625+
reconcileAfterProcessLoss: Effect.succeed({ requeued: 0, cancelled: 0 }),
626+
}),
627+
),
628+
),
629+
);
630+
631+
return Effect.gen(function* () {
632+
yield* (yield* ProviderRuntimeRecovery.ProviderRuntimeRecoveryService).reconcile("shutdown", {
633+
pausedForUpdate: true,
634+
});
635+
const runEvent = committedInput?.events.find((event) => event.type === "run.updated");
636+
assert.equal(runEvent?.type === "run.updated" ? runEvent.payload.status : null, "interrupted");
637+
}).pipe(Effect.provide(layer));
638+
});
639+
581640
it.effect("closes a secret request card's form when its run is recovered", () => {
582641
const threadId = ThreadId.make("thread_secret_recovery");
583642
const runId = RunId.make("run_secret_recovery");

0 commit comments

Comments
 (0)