Skip to content

Commit d9e39e1

Browse files
fix(daemon): isolate disconnect cancellation and resource teardown
Cancel HTTP requests that lose their client before response headers, and scope disconnect cancellation to the affected request/device/session instead of a global Apple runner abort. Make session resource teardown failure-isolated so one rejected step no longer skips later cleanup, while preserving lease release and session deletion. Closes #1220 Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
1 parent ae21067 commit d9e39e1

8 files changed

Lines changed: 473 additions & 148 deletions

File tree

‎src/daemon/handlers/__tests__/session-close-shutdown.test.ts‎

Lines changed: 103 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -730,3 +730,106 @@ test('close --shutdown returns success and failure payload when shutdownSimulato
730730
expect(shutdown?.error?.message).toBe('simctl shutdown failed');
731731
}
732732
});
733+
734+
test('daemon session teardown attempts every resource after an earlier cleanup rejects', async () => {
735+
const sessionName = 'android-teardown-isolation-session';
736+
const activeCapture = {
737+
type: 'trace',
738+
kind: 'perfetto',
739+
packageName: 'com.example.app',
740+
appPid: '1234',
741+
profilerPid: '5678',
742+
remotePath: '/data/misc/perfetto-traces/app.perfetto-trace',
743+
outPath: '/tmp/app.perfetto-trace',
744+
startedAt: Date.now(),
745+
state: 'running',
746+
};
747+
const session = {
748+
...makeSession(sessionName, {
749+
platform: 'android',
750+
id: 'emulator-5554',
751+
name: 'Pixel',
752+
kind: 'emulator',
753+
booted: true,
754+
}),
755+
appBundleId: 'com.example.app',
756+
nativePerf: {
757+
android: activeCapture,
758+
},
759+
} as unknown as SessionState;
760+
761+
// The native-perf cleanup runs before the snapshot-helper cleanup.
762+
mockCleanupAndroidNativePerfSession.mockRejectedValueOnce(
763+
new AppError('COMMAND_FAILED', 'perfetto stop failed'),
764+
);
765+
766+
await expect(teardownSessionResources(session, sessionName)).rejects.toMatchObject({
767+
code: 'COMMAND_FAILED',
768+
details: expect.objectContaining({
769+
reason: 'session_cleanup_incomplete',
770+
failedSteps: ['android_native_perf'],
771+
}),
772+
});
773+
774+
// The later resource still runs despite the earlier rejection.
775+
expect(mockStopAndroidSnapshotHelperSessionForDevice).toHaveBeenCalledWith(session.device);
776+
});
777+
778+
test('close still runs later cleanup and deletes the session after an earlier cleanup rejects', async () => {
779+
const sessionStore = makeSessionStore();
780+
const sessionName = 'android-close-isolation-session';
781+
const activeCapture = {
782+
type: 'trace',
783+
kind: 'perfetto',
784+
packageName: 'com.example.app',
785+
appPid: '1234',
786+
profilerPid: '5678',
787+
remotePath: '/data/misc/perfetto-traces/app.perfetto-trace',
788+
outPath: '/tmp/app.perfetto-trace',
789+
startedAt: Date.now(),
790+
state: 'running',
791+
};
792+
const session = {
793+
...makeSession(sessionName, {
794+
platform: 'android',
795+
id: 'emulator-5554',
796+
name: 'Pixel',
797+
kind: 'emulator',
798+
booted: true,
799+
}),
800+
appBundleId: 'com.example.app',
801+
nativePerf: {
802+
android: activeCapture,
803+
},
804+
} as unknown as SessionState;
805+
sessionStore.set(sessionName, session);
806+
807+
mockCleanupAndroidNativePerfSession.mockRejectedValueOnce(
808+
new AppError('COMMAND_FAILED', 'perfetto stop failed'),
809+
);
810+
811+
await expect(
812+
handleSessionCommands({
813+
req: {
814+
token: 't',
815+
session: sessionName,
816+
command: 'close',
817+
positionals: [],
818+
flags: {},
819+
},
820+
sessionName,
821+
logPath: path.join(os.tmpdir(), 'daemon.log'),
822+
sessionStore,
823+
invoke: noopInvoke,
824+
}),
825+
).rejects.toMatchObject({
826+
details: expect.objectContaining({
827+
reason: 'session_cleanup_incomplete',
828+
failedSteps: ['android_native_perf'],
829+
}),
830+
});
831+
832+
// A later cleanup step still ran, and the session was still deleted.
833+
expect(mockStopAndroidSnapshotHelperSessionForDevice).toHaveBeenCalledWith(session.device);
834+
expect(sessionStore.get(sessionName)).toBeUndefined();
835+
});

‎src/daemon/handlers/session-close.ts‎

Lines changed: 42 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -25,13 +25,15 @@ import type { LeaseRegistry } from '../lease-registry.ts';
2525
import { releaseSessionLease } from '../lease-lifecycle.ts';
2626
import type { LeaseLifecycleProvider } from './lease.ts';
2727
import {
28+
reportSessionCleanupFailures,
2829
restoreSessionAndroidIme,
2930
stopAppleRunnerForClose,
3031
stopSessionAndroidNativePerfCapture,
3132
stopSessionAndroidSnapshotHelper,
3233
stopSessionAppLog,
3334
stopSessionApplePerfCapture,
3435
stopSessionAudioProbe,
36+
type SessionCleanupFailure,
3537
} from '../session-teardown.ts';
3638

3739
async function maybeShutdownSessionTarget(params: {
@@ -72,29 +74,49 @@ export async function handleCloseCommand(params: {
7274
return await closeWithoutSession(req, logPath);
7375
}
7476
let providerData: Record<string, unknown> | undefined;
77+
// Resource teardown is failure-isolated: a rejected step is collected instead of
78+
// short-circuiting the rest, so every subsequent resource (and the runner stop)
79+
// is still attempted. Lease release and session deletion below run regardless,
80+
// and any collected failures are surfaced as an aggregate after cleanup.
81+
const cleanupFailures: SessionCleanupFailure[] = [];
82+
const attemptCleanup = async (step: string, run: () => Promise<void>): Promise<void> => {
83+
try {
84+
await run();
85+
} catch (error) {
86+
cleanupFailures.push({ step, error });
87+
}
88+
};
7589
try {
76-
await stopSessionAppLog(session);
77-
await stopSessionAudioProbe(session, 'session-close');
78-
await stopSessionApplePerfCapture(session);
79-
await stopSessionAndroidNativePerfCapture(session);
80-
await stopSessionAndroidSnapshotHelper(session);
81-
await restoreSessionAndroidIme(session, sessionStore.resolveDaemonStateDir());
90+
await attemptCleanup('app_log', () => stopSessionAppLog(session));
91+
await attemptCleanup('audio_probe', async () => {
92+
await stopSessionAudioProbe(session, 'session-close');
93+
});
94+
await attemptCleanup('apple_perf', () => stopSessionApplePerfCapture(session));
95+
await attemptCleanup('android_native_perf', () => stopSessionAndroidNativePerfCapture(session));
96+
await attemptCleanup('android_snapshot_helper', () =>
97+
stopSessionAndroidSnapshotHelper(session),
98+
);
99+
await attemptCleanup('android_ime', () =>
100+
restoreSessionAndroidIme(session, sessionStore.resolveDaemonStateDir()),
101+
);
82102
if (shouldDispatchPlatformClose(req, session)) {
83103
if (shouldStopAppleRunnerBeforeTargetedClose(session)) {
84-
await stopAppleRunnerForClose(session);
104+
await attemptCleanup('apple_runner_pre_close', () => stopAppleRunnerForClose(session));
85105
}
86-
await dispatchCommand(session.device, 'close', req.positionals ?? [], req.flags?.out, {
87-
...contextFromFlags(logPath, req.flags, session.appBundleId, session.trace?.outPath),
106+
await attemptCleanup('platform_close', async () => {
107+
await dispatchCommand(session.device, 'close', req.positionals ?? [], req.flags?.out, {
108+
...contextFromFlags(logPath, req.flags, session.appBundleId, session.trace?.outPath),
109+
});
110+
await settleIosSimulator(session.device, IOS_SIMULATOR_POST_CLOSE_SETTLE_MS);
88111
});
89-
await settleIosSimulator(session.device, IOS_SIMULATOR_POST_CLOSE_SETTLE_MS);
90112
}
91113
if (
92114
isApplePlatform(session.device.platform) &&
93115
!shouldRetainAppleRunnerAfterClose(req, session)
94116
) {
95117
// The targeted close path stops before dispatch to avoid runner/app races.
96118
// Stop again here for idempotent cleanup, and keep cleanup-sensitive closes explicit.
97-
await stopAppleRunnerForClose(session);
119+
await attemptCleanup('apple_runner', () => stopAppleRunnerForClose(session));
98120
} else if (isApplePlatform(session.device.platform)) {
99121
emitDiagnostic({
100122
level: 'debug',
@@ -123,7 +145,9 @@ export async function handleCloseCommand(params: {
123145
session.recordSession = true;
124146
}
125147
sessionStore.writeSessionLog(session);
126-
await cleanupRetainedMaterializedPathsForSession(sessionName).catch(() => {});
148+
await attemptCleanup('materialized_paths', () =>
149+
cleanupRetainedMaterializedPathsForSession(sessionName),
150+
);
127151
} finally {
128152
// Always drop the local session, even if provider-side release fails:
129153
// a failed close must not strand device ownership until inactivity expiry.
@@ -133,6 +157,12 @@ export async function handleCloseCommand(params: {
133157
sessionStore.delete(sessionName);
134158
}
135159
}
160+
const cleanupAggregate = reportSessionCleanupFailures({
161+
sessionName,
162+
phase: 'session_close_cleanup_failed',
163+
failures: cleanupFailures,
164+
});
165+
if (cleanupAggregate) throw cleanupAggregate;
136166
const shutdownResult = await maybeShutdownSessionTarget({
137167
device: session.device,
138168
shutdownRequested: req.flags?.shutdown,

‎src/daemon/server/http-server.ts‎

Lines changed: 11 additions & 43 deletions
Original file line numberDiff line numberDiff line change
@@ -13,14 +13,12 @@ import type { DaemonInstallSource, DaemonInvokeFn, DaemonRequest } from '../type
1313
import { normalizeTenantId } from '../config.ts';
1414
import {
1515
clearRequestCanceled,
16-
isRequestCanceled,
1716
markRequestCanceled,
1817
registerRequestAbort,
1918
resolveRequestTrackingId,
2019
} from '../../request/cancel.ts';
2120
import path from 'node:path';
2221
import { pathToFileURL } from 'node:url';
23-
import { sleep } from '../../utils/timeouts.ts';
2422
import { type RequestProgressEvent, withRequestProgressSink } from '../../request/progress.ts';
2523
import {
2624
serializeDaemonProgressEnvelope,
@@ -71,9 +69,6 @@ type HttpAuthDecision =
7169
| { ok: false; statusCode: number; response: JsonRpcResponse };
7270

7371
const MAX_HTTP_RPC_BODY_BYTES = 1024 * 1024;
74-
const CLIENT_DISCONNECT_ABORT_POLL_INTERVAL_MS = 200;
75-
const CLIENT_DISCONNECT_ABORT_MAX_WINDOW_MS = 15_000;
76-
const IOS_RUNNER_ABORT_REPLAY_COMMANDS = new Set(['replay', 'test']);
7772
const COMMAND_RPC_METHODS = new Set(['agent_device.command', 'agent-device.command']);
7873
const INSTALL_FROM_SOURCE_RPC_METHODS = new Set([
7974
'agent_device.install_from_source',
@@ -655,8 +650,14 @@ export async function createDaemonHttpServer(options: {
655650
};
656651
}
657652

658-
const abortIosRunnerOnDisconnect = shouldAbortIosRunnerSessionsOnDisconnect(daemonRequest);
659653
let canceledInFlight = false;
654+
// Request-scoped cancellation: mark this request canceled whenever its client
655+
// vanishes before the response finishes, regardless of whether headers were
656+
// already sent. `markRequestCanceled` aborts only this request's AbortSignal,
657+
// so in-flight runner work owned by the request is canceled without touching
658+
// other requests, other devices, or non-Apple work. The guard below keys off
659+
// the response's own completion state, so a normal end is never misclassified
660+
// as a disconnect.
660661
const markCanceledIfResponseIncomplete = () => {
661662
if (handlerCompleted || res.writableFinished || canceledInFlight) return;
662663
canceledInFlight = true;
@@ -666,18 +667,14 @@ export async function createDaemonHttpServer(options: {
666667
phase: 'request_client_disconnected',
667668
data: {
668669
requestId: requestIdForCleanup,
669-
abortIosRunnerSessions: abortIosRunnerOnDisconnect,
670670
},
671671
});
672-
if (abortIosRunnerOnDisconnect) {
673-
void abortInFlightIosRunnerSessionsWhileDisconnected(requestIdForCleanup);
674-
}
675672
};
676673
req.on('aborted', markCanceledIfResponseIncomplete);
677-
res.on('close', () => {
678-
if (res.headersSent) markCanceledIfResponseIncomplete();
679-
});
680-
if (req.aborted || (res.destroyed && res.headersSent)) {
674+
// `res` close fires for both pre-header and post-header disconnects; the
675+
// completion guard distinguishes a real disconnect from a finished response.
676+
res.on('close', markCanceledIfResponseIncomplete);
677+
if (req.aborted || res.destroyed) {
681678
markCanceledIfResponseIncomplete();
682679
}
683680

@@ -745,35 +742,6 @@ export async function createDaemonHttpServer(options: {
745742
});
746743
}
747744

748-
async function abortInFlightIosRunnerSessionsWhileDisconnected(
749-
requestId: string | undefined,
750-
): Promise<void> {
751-
try {
752-
const deadline = Date.now() + CLIENT_DISCONNECT_ABORT_MAX_WINDOW_MS;
753-
while (isRequestCanceled(requestId) && Date.now() < deadline) {
754-
const { abortAllIosRunnerSessions } =
755-
await import('../../platforms/apple/core/runner/runner-client.ts');
756-
await abortAllIosRunnerSessions();
757-
if (!isRequestCanceled(requestId)) break;
758-
await sleep(CLIENT_DISCONNECT_ABORT_POLL_INTERVAL_MS);
759-
}
760-
} catch (error) {
761-
emitDiagnostic({
762-
level: 'error',
763-
phase: 'request_client_disconnect_abort_failed',
764-
data: {
765-
message: error instanceof Error ? error.message : String(error),
766-
},
767-
});
768-
}
769-
}
770-
771-
function shouldAbortIosRunnerSessionsOnDisconnect(req: DaemonRequest): boolean {
772-
if (req.flags?.platform === 'android') return false;
773-
if (req.flags?.platform === 'ios' || req.flags?.platform === 'macos') return true;
774-
return IOS_RUNNER_ABORT_REPLAY_COMMANDS.has(req.command);
775-
}
776-
777745
async function authorizeAuxiliaryHttpRequest(params: {
778746
req: http.IncomingMessage;
779747
res: http.ServerResponse;

‎src/daemon/server/transport.ts‎

Lines changed: 7 additions & 28 deletions
Original file line numberDiff line numberDiff line change
@@ -12,17 +12,13 @@ import {
1212
} from '../../request/cancel.ts';
1313
import { emitDiagnostic } from '../../utils/diagnostics.ts';
1414
import { consumeTextLines } from '../../utils/line-stream.ts';
15-
import { sleep } from '../../utils/timeouts.ts';
1615
import { withRequestProgressSink } from '../../request/progress.ts';
1716
import {
1817
serializeDaemonProgressEnvelope,
1918
serializeDaemonResponseEnvelope,
2019
shouldStreamRequestProgress,
2120
} from '../request-progress-protocol.ts';
2221

23-
const disconnectAbortPollIntervalMs = 200;
24-
const disconnectAbortMaxWindowMs = 15_000;
25-
2622
export type DaemonServer = (net.Server | HttpServer) & {
2723
destroyConnections?: () => void;
2824
};
@@ -36,7 +32,11 @@ export function createSocketServer(handleRequest: DaemonInvokeFn): DaemonServer
3632
let inFlightRequests = 0;
3733
const activeRequestIds = new Set<string>();
3834
let canceledInFlight = false;
39-
const cancelInFlightRunnerSessions = () => {
35+
// On disconnect, cancel only this connection's in-flight requests via their
36+
// request-scoped AbortSignals. That aborts runner work owned by those requests
37+
// without a global Apple runner abort, so an unrelated request's runner work
38+
// (or non-Apple work) is never torn down by another client's disconnect.
39+
const cancelInFlightRequests = () => {
4040
if (canceledInFlight || inFlightRequests === 0) return;
4141
canceledInFlight = true;
4242
for (const requestId of activeRequestIds) {
@@ -49,31 +49,10 @@ export function createSocketServer(handleRequest: DaemonInvokeFn): DaemonServer
4949
inFlightRequests,
5050
},
5151
});
52-
void (async () => {
53-
try {
54-
const deadline = Date.now() + disconnectAbortMaxWindowMs;
55-
while (inFlightRequests > 0 && Date.now() < deadline) {
56-
const { abortAllIosRunnerSessions } =
57-
await import('../../platforms/apple/core/runner/runner-client.ts');
58-
await abortAllIosRunnerSessions();
59-
if (inFlightRequests <= 0) break;
60-
await sleep(disconnectAbortPollIntervalMs);
61-
}
62-
} catch (err) {
63-
emitDiagnostic({
64-
level: 'error',
65-
phase: 'request_client_disconnect_abort_failed',
66-
data: {
67-
message: err instanceof Error ? err.message : String(err),
68-
inFlightRequests,
69-
},
70-
});
71-
}
72-
})();
7352
};
7453
socket.setEncoding('utf8');
75-
socket.on('close', cancelInFlightRunnerSessions);
76-
socket.on('error', cancelInFlightRunnerSessions);
54+
socket.on('close', cancelInFlightRequests);
55+
socket.on('error', cancelInFlightRequests);
7756
socket.on('data', async (chunk) => {
7857
const parsed = consumeTextLines(buffer, chunk);
7958
buffer = parsed.buffer;

0 commit comments

Comments
 (0)