Skip to content

Commit e5d5886

Browse files
lidge-junIngwannu
andauthored
fix(windows): drain response spill publications before shutdown (#3044)
* fix(windows): move response spill ACL work off event loop * fix(windows): drain response spills before shutdown snapshot * fix(windows): cancel abandoned spill publications * fix(shutdown): persist state before reporting spill cleanup failure * fix(shutdown): terminalize spills when fallback budget expires * fix(shutdown): guard and isolate spill terminalization * test(shutdown): make ACL reserve proof deterministic * fix(windows): bound async spill ACL attempts * fix(windows): abandon wedged ACL subprocesses --------- Co-authored-by: Ingwannu <[email protected]>
1 parent 293a2b8 commit e5d5886

17 files changed

Lines changed: 1828 additions & 99 deletions

‎src/cli/index.ts‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -363,11 +363,12 @@ async function handleStart(options: { block?: boolean } = {}) {
363363
shutdownStartedAt = now;
364364
console.log("\n🛑 Shutting down opencodex proxy...");
365365
void (async () => {
366+
let shutdownSucceeded = false;
366367
try {
367-
await drainAndShutdown(server, config.shutdownTimeoutMs ?? 5000);
368+
shutdownSucceeded = await drainAndShutdown(server, config.shutdownTimeoutMs ?? 5000);
368369
} finally {
369370
const restored = syncCleanup(); // idempotent (cleaned-guard); also re-run by process.on("exit")
370-
process.exit(restored ? 0 : 1);
371+
process.exit(restored && shutdownSucceeded ? 0 : 1);
371372
}
372373
})();
373374
};

‎src/config/paths.ts‎

Lines changed: 18 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
import { chmodSync, existsSync } from "node:fs";
22
import { homedir } from "node:os";
33
import { join, resolve } from "node:path";
4-
import { hardenSecretDir } from "../lib/windows-secret-acl";
4+
import { hardenSecretDirAsync, windowsSecretAclApplies } from "../lib/windows-secret-acl";
55
import { assertNotRealHomeUnderTest } from "../lib/test-home-guard";
66

77
/**
@@ -14,6 +14,7 @@ export function expandUserPath(raw: string): string {
1414
return raw;
1515
}
1616
let resolvedConfigDirCache: { raw: string | undefined; path: string } | null = null;
17+
const configDirHardeningFlights = new Map<string, Promise<void>>();
1718

1819
export function getConfigDir(): string {
1920
const raw = process.env["OPENCODEX_HOME"]?.trim() || undefined;
@@ -34,7 +35,21 @@ export function hardenConfigDir(): void {
3435
assertNotRealHomeUnderTest(dir);
3536
if (!existsSync(dir)) return;
3637
try { chmodSync(dir, 0o700); } catch { /* best-effort */ }
37-
if (process.platform === "win32") {
38-
hardenSecretDir(dir, { required: false });
38+
if (windowsSecretAclApplies() && !configDirHardeningFlights.has(dir)) {
39+
// This is an optional read-path harden. Waiting synchronously here used to stop the Bun
40+
// event loop (including /healthz) for the full icacls timeout. Required mutation paths keep
41+
// their own awaited/fail-closed hardening; ordinary config reads only start one soft flight.
42+
const flight = hardenSecretDirAsync(dir, { required: false })
43+
.then(() => undefined)
44+
.catch(() => undefined)
45+
.finally(() => {
46+
if (configDirHardeningFlights.get(dir) === flight) configDirHardeningFlights.delete(dir);
47+
});
48+
configDirHardeningFlights.set(dir, flight);
3949
}
4050
}
51+
52+
/** Test-only: settle optional config-directory hardening without exposing it to production callers. */
53+
export async function flushConfigDirHardeningForTests(): Promise<void> {
54+
await Promise.all([...configDirHardeningFlights.values()]);
55+
}

‎src/lib/bounded-subprocess.ts‎

Lines changed: 36 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,36 @@
1+
export interface KillableSubprocess {
2+
exited: Promise<number>;
3+
kill(): unknown;
4+
unref?(): unknown;
5+
}
6+
7+
export interface BoundedSubprocessExit {
8+
exitCode: number | null;
9+
timedOut: boolean;
10+
}
11+
12+
/** Kill at the deadline and abandon immediately; late exit/rejection remains observed. */
13+
export function waitForSubprocessExit(
14+
proc: KillableSubprocess,
15+
timeoutMs: number,
16+
): Promise<BoundedSubprocessExit> {
17+
return new Promise(resolve => {
18+
let settled = false;
19+
let timer: ReturnType<typeof setTimeout> | undefined;
20+
const finish = (result: BoundedSubprocessExit): void => {
21+
if (settled) return;
22+
settled = true;
23+
if (timer !== undefined) clearTimeout(timer);
24+
resolve(result);
25+
};
26+
timer = setTimeout(() => {
27+
try { proc.kill(); } catch { /* already exited */ }
28+
try { proc.unref?.(); } catch { /* abandonment is still authoritative */ }
29+
finish({ exitCode: null, timedOut: true });
30+
}, Math.max(1, timeoutMs));
31+
void proc.exited.then(
32+
exitCode => finish({ exitCode, timedOut: false }),
33+
() => finish({ exitCode: null, timedOut: false }),
34+
);
35+
});
36+
}

‎src/lib/windows-secret-acl.ts‎

Lines changed: 47 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,7 @@
3131

3232
import { existsSync, statSync } from "node:fs";
3333
import { env, platform } from "node:process";
34+
import { waitForSubprocessExit } from "./bounded-subprocess";
3435
import { resolveTrustedWindowsIcaclsExe } from "./windows-elevation";
3536
import {
3637
cachedCurrentWindowsIdentity,
@@ -216,6 +217,11 @@ export interface HardenResult {
216217

217218
export interface HardenOptions {
218219
required: boolean;
220+
/**
221+
* Explicit total budget for this harden call. Shutdown recovery uses a reduced
222+
* caller-owned slice instead of opening the normal 30-second window.
223+
*/
224+
deadlineMs?: number;
219225
/**
220226
* Optional timeout-memo key distinct from `targetPath` (issue #612).
221227
* Atomic writers mint a fresh `.tmp` path per write; keying the timeout cache by the
@@ -257,7 +263,11 @@ const HARDEN_DEADLINE_MIN_MS = 1_000;
257263
const HARDEN_DEADLINE_MAX_MS = 60_000;
258264

259265
/** Resolve the total harden budget once per call (env mutation cannot change it midway). */
260-
function resolveHardenDeadlineMs(): number {
266+
function resolveHardenDeadlineMs(overrideMs?: number): number {
267+
if (overrideMs !== undefined) {
268+
if (!Number.isSafeInteger(overrideMs) || overrideMs <= 0) return 1;
269+
return Math.min(HARDEN_DEADLINE_MAX_MS, overrideMs);
270+
}
261271
const raw = env["OPENCODEX_ACL_TIMEOUT_MS"]?.trim();
262272
if (!raw) return HARDEN_DEADLINE_DEFAULT_MS;
263273
const parsed = Number(raw);
@@ -328,27 +338,16 @@ function defaultIcaclsRunner(args: string[], timeoutMs: number): IcaclsResult {
328338

329339
/**
330340
* Async icacls runner (#612): yields the event loop while waiting for the child.
331-
* Timeout provenance is recorded by our timer (async Subprocess has no exitedDueToTimeout);
332-
* we still await process exit before classifying so settlement is confirmed.
341+
* Async Subprocess has no exitedDueToTimeout, so the shared settlement helper
342+
* classifies the deadline and abandons a child that does not settle after kill.
333343
*/
334344
async function defaultAsyncIcaclsRunner(args: string[], timeoutMs: number): Promise<IcaclsResult> {
335345
const proc = trySpawnIcacls(args);
336346
if (!proc) return spawnFailedResult();
337-
let timedOutByUs = false;
338-
const timer = setTimeout(() => {
339-
timedOutByUs = true;
340-
try { proc.kill(); } catch { /* already exited */ }
341-
}, Math.max(1, timeoutMs));
342-
let exitCode: number | null = null;
343-
try {
344-
exitCode = await proc.exited;
345-
} finally {
346-
clearTimeout(timer);
347-
}
348-
const stdout = proc.stdout
347+
const { exitCode, timedOut } = await waitForSubprocessExit(proc, timeoutMs);
348+
const stdout = !timedOut && proc.stdout
349349
? await new Response(proc.stdout).text().catch(() => "")
350350
: "";
351-
const timedOut = timedOutByUs;
352351
return {
353352
success: !timedOut && exitCode === 0,
354353
exitCode: timedOut ? null : exitCode,
@@ -357,6 +356,24 @@ async function defaultAsyncIcaclsRunner(args: string[], timeoutMs: number): Prom
357356
};
358357
}
359358

359+
function awaitAsyncIcaclsRunner(args: string[], timeoutMs: number): Promise<IcaclsResult> {
360+
return new Promise(resolve => {
361+
let settled = false;
362+
let timer: ReturnType<typeof setTimeout> | undefined;
363+
const finish = (result: IcaclsResult): void => {
364+
if (settled) return;
365+
settled = true;
366+
if (timer !== undefined) clearTimeout(timer);
367+
resolve(result);
368+
};
369+
timer = setTimeout(
370+
() => finish({ success: false, exitCode: null, timedOut: true, stdout: "" }),
371+
Math.max(1, timeoutMs),
372+
);
373+
void asyncIcaclsRunner(args, timeoutMs).then(finish, () => finish(spawnFailedResult()));
374+
});
375+
}
376+
360377
let icaclsRunner: IcaclsRunner = defaultIcaclsRunner;
361378
let asyncIcaclsRunner: AsyncIcaclsRunner = defaultAsyncIcaclsRunner;
362379
let platformOverride: string | null = null;
@@ -537,12 +554,14 @@ function shouldVerifyExistingAcl(): boolean {
537554
return env["OPENCODEX_ACL_VERIFY_EXISTING"] === "1";
538555
}
539556

540-
function existingAclAlreadyCompliant(targetPath: string, directory: boolean): boolean {
557+
function existingAclAlreadyCompliant(targetPath: string, directory: boolean, deadline: number): boolean {
541558
if (!shouldVerifyExistingAcl()) return false;
542559
const identity = cachedCurrentWindowsIdentity();
543560
if (!identity) return false;
544561
try {
545-
const result = icaclsRunner([targetPath], resolveHardenDeadlineMs());
562+
const remaining = deadline - nowFn();
563+
if (remaining <= 0) return false;
564+
const result = icaclsRunner([targetPath], remaining);
546565
return result.success && existingAclIsCompliant(targetPath, directory, result.stdout, identity.name);
547566
} catch {
548567
return false;
@@ -552,12 +571,15 @@ function existingAclAlreadyCompliant(targetPath: string, directory: boolean): bo
552571
async function existingAclAlreadyCompliantAsync(
553572
targetPath: string,
554573
directory: boolean,
574+
deadline: number,
555575
): Promise<boolean> {
556576
if (!shouldVerifyExistingAcl()) return false;
557577
const identity = cachedCurrentWindowsIdentity();
558578
if (!identity) return false;
559579
try {
560-
const result = await asyncIcaclsRunner([targetPath], resolveHardenDeadlineMs());
580+
const remaining = deadline - nowFn();
581+
if (remaining <= 0) return false;
582+
const result = await awaitAsyncIcaclsRunner([targetPath], remaining);
561583
return result.success && existingAclIsCompliant(targetPath, directory, result.stdout, identity.name);
562584
} catch {
563585
return false;
@@ -618,7 +640,7 @@ async function runIcaclsAsync(targetPath: string, directory: boolean, deadline:
618640
if (remaining <= 0) {
619641
throw icaclsError(step, { success: false, exitCode: null, timedOut: true, stdout: "" });
620642
}
621-
return asyncIcaclsRunner(args, remaining);
643+
return awaitAsyncIcaclsRunner(args, remaining);
622644
};
623645
const runOrThrow = async (step: string, args: string[]): Promise<void> => {
624646
const result = await run(step, args);
@@ -751,7 +773,7 @@ async function describeAclStateAfterTimeoutAsync(targetPath: string, deadline: n
751773
for (const sid of BROAD_SIDS) {
752774
const remaining = deadline - nowFn();
753775
if (remaining <= 0) return "ACL state unverified (budget exhausted)";
754-
const found = await asyncIcaclsRunner([targetPath, "/findsid", sid], remaining);
776+
const found = await awaitAsyncIcaclsRunner([targetPath, "/findsid", sid], remaining);
755777
if (!found.success) return "ACL state unverified (probe failed)";
756778
if (found.stdout.includes(targetPath)) return "broad ACL grants still present";
757779
}
@@ -788,15 +810,15 @@ function hardenEntry(
788810
if (!existsSync(targetPath)) { cache.delete(targetPath); return { ok: true }; }
789811
if (effectivePlatform() !== "win32") return { ok: true };
790812
if (memoSatisfied(cache, targetPath)) return { ok: true };
791-
if (existingAclAlreadyCompliant(targetPath, directory)) return { ok: true };
813+
const deadline = nowFn() + resolveHardenDeadlineMs(opts.deadlineMs);
814+
if (existingAclAlreadyCompliant(targetPath, directory, deadline)) return { ok: true };
792815
const memoKey = timeoutMemoKey(targetPath, opts);
793816
const timeoutMemoError = timeoutMemoErrorIfBlocked(memoKey, opts);
794817
if (timeoutMemoError) {
795818
if (opts.required) throw timeoutMemoError;
796819
return { ok: false, diagnostics: timeoutMemoError.message };
797820
}
798821

799-
const deadline = nowFn() + resolveHardenDeadlineMs();
800822
let lastErr: unknown;
801823
for (let attempt = 0; attempt < 2; attempt++) {
802824
if (attempt > 0 && deadline - nowFn() <= 0) break; // retry only while budget remains
@@ -841,15 +863,15 @@ async function hardenEntryAsync(
841863
if (!existsSync(targetPath)) { cache.delete(targetPath); return { ok: true }; }
842864
if (effectivePlatform() !== "win32") return { ok: true };
843865
if (memoSatisfied(cache, targetPath)) return { ok: true };
844-
if (await existingAclAlreadyCompliantAsync(targetPath, directory)) return { ok: true };
866+
const deadline = nowFn() + resolveHardenDeadlineMs(opts.deadlineMs);
867+
if (await existingAclAlreadyCompliantAsync(targetPath, directory, deadline)) return { ok: true };
845868
const memoKey = timeoutMemoKey(targetPath, opts);
846869
const timeoutMemoError = timeoutMemoErrorIfBlocked(memoKey, opts);
847870
if (timeoutMemoError) {
848871
if (opts.required) throw timeoutMemoError;
849872
return { ok: false, diagnostics: timeoutMemoError.message };
850873
}
851874

852-
const deadline = nowFn() + resolveHardenDeadlineMs();
853875
let lastErr: unknown;
854876
for (let attempt = 0; attempt < 2; attempt++) {
855877
if (attempt > 0 && deadline - nowFn() <= 0) break;

‎src/lib/windows-user-principal.ts‎

Lines changed: 8 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,7 @@
2222

2323
import { existsSync } from "node:fs";
2424
import { win32 as windowsPath } from "node:path";
25+
import { waitForSubprocessExit } from "./bounded-subprocess";
2526

2627
import {
2728
resolveTrustedWindowsPowerShellExe,
@@ -143,18 +144,8 @@ async function defaultAsyncWindowsPrincipalRunner(
143144
stderr: "ignore",
144145
windowsHide: true,
145146
});
146-
let timedOut = false;
147-
const timer = setTimeout(() => {
148-
timedOut = true;
149-
try { proc.kill(); } catch { /* already exited */ }
150-
}, Math.max(1, timeoutMs));
151-
let exitCode: number | null = null;
152-
try {
153-
exitCode = await proc.exited;
154-
} finally {
155-
clearTimeout(timer);
156-
}
157-
const stdout = proc.stdout
147+
const { exitCode, timedOut } = await waitForSubprocessExit(proc, timeoutMs);
148+
const stdout = !timedOut && proc.stdout
158149
? await new Response(proc.stdout).text().catch(() => "")
159150
: "";
160151
return {
@@ -314,11 +305,11 @@ export async function resolveCurrentWindowsPrincipalAsync(timeoutMs: number): Pr
314305
return `*${cachedIdentity.sid}`;
315306
})();
316307
asyncLookupInFlight = lookup;
317-
try {
318-
return await lookup;
319-
} finally {
320-
if (asyncLookupInFlight === lookup) asyncLookupInFlight = null;
321-
}
308+
void lookup.then(
309+
() => { if (asyncLookupInFlight === lookup) asyncLookupInFlight = null; },
310+
() => { if (asyncLookupInFlight === lookup) asyncLookupInFlight = null; },
311+
);
312+
return waitForExistingLookup(lookup, timeoutMs);
322313
}
323314

324315
/** Test seam: replace the sync resolver process and clear its successful cache. */

0 commit comments

Comments
 (0)