Skip to content

Commit 359859d

Browse files
steipetesteipete
authored
fix(gateway): keep first outbound TLS off the main event loop on macOS (system-CA/trustd) (#111473)
1 parent 87fbc19 commit 359859d

6 files changed

Lines changed: 363 additions & 39 deletions

File tree

‎src/daemon/gateway-heap.ts‎

Lines changed: 1 addition & 39 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
/** Adaptive Node heap policy for the managed Gateway service. */
22
import os from "node:os";
3+
import { parseNodeOptionsTokens } from "../infra/node-options.js";
34

45
const MEBIBYTE_BYTES = 1024 * 1024;
56
const GATEWAY_HEAP_FLOOR_MIB = 2048;
@@ -65,45 +66,6 @@ function resolveGatewayHeapLimit(params: GatewayHeapMemoryInputs = {}): GatewayH
6566
};
6667
}
6768

68-
function parseNodeOptionsTokens(nodeOptions: string): string[] | null {
69-
// Match Node's NODE_OPTIONS splitter: space delimiters, double quotes, and
70-
// backslash escapes only inside quotes. Other shell quoting does not apply.
71-
const tokens: string[] = [];
72-
let token = "";
73-
let inQuotes = false;
74-
let tokenStarted = false;
75-
for (let index = 0; index < nodeOptions.length; index += 1) {
76-
let char = nodeOptions[index];
77-
if (char === "\\" && inQuotes) {
78-
index += 1;
79-
if (index >= nodeOptions.length) {
80-
return null;
81-
}
82-
char = nodeOptions[index];
83-
} else if (char === " " && !inQuotes) {
84-
if (tokenStarted) {
85-
tokens.push(token);
86-
token = "";
87-
tokenStarted = false;
88-
}
89-
continue;
90-
} else if (char === '"') {
91-
inQuotes = !inQuotes;
92-
tokenStarted = true;
93-
continue;
94-
}
95-
token += char;
96-
tokenStarted = true;
97-
}
98-
if (inQuotes) {
99-
return null;
100-
}
101-
if (tokenStarted) {
102-
tokens.push(token);
103-
}
104-
return tokens;
105-
}
106-
10769
function parseMaxOldSpaceSizeMiB(nodeOptions: string | undefined): number | null {
10870
if (!nodeOptions) {
10971
return null;

‎src/gateway/server-startup-post-attach.test.ts‎

Lines changed: 36 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1501,6 +1501,42 @@ describe("startGatewayPostAttachRuntime", () => {
15011501
);
15021502
});
15031503

1504+
it("warms the macOS system CA cache before channel startup", async () => {
1505+
let finishWarmup: (() => void) | undefined;
1506+
const warmSystemCa = vi.fn(
1507+
() =>
1508+
new Promise<void>((resolve) => {
1509+
finishWarmup = resolve;
1510+
}),
1511+
);
1512+
const startChannels = vi.fn(async () => {});
1513+
const sidecars = startGatewaySidecars({
1514+
cfg: { hooks: { internal: { enabled: false } } } as never,
1515+
pluginRegistry: createPostAttachParams().pluginRegistry,
1516+
defaultWorkspaceDir: "/tmp/openclaw-workspace",
1517+
deps: {} as never,
1518+
startChannels,
1519+
warmSystemCa,
1520+
log: { warn: vi.fn() },
1521+
logHooks: {
1522+
info: vi.fn(),
1523+
warn: vi.fn(),
1524+
error: vi.fn(),
1525+
},
1526+
logChannels: {
1527+
info: vi.fn(),
1528+
error: vi.fn(),
1529+
},
1530+
});
1531+
1532+
await waitForGatewayTestState(() => expect(warmSystemCa).toHaveBeenCalledTimes(1));
1533+
expect(startChannels).not.toHaveBeenCalled();
1534+
1535+
finishWarmup?.();
1536+
await sidecars;
1537+
expect(startChannels).toHaveBeenCalledTimes(1);
1538+
});
1539+
15041540
it("starts and reports plugin services after channel startup completes", async () => {
15051541
await withEnvAsync(
15061542
{ OPENCLAW_SKIP_CHANNELS: undefined, OPENCLAW_SKIP_PROVIDERS: undefined },

‎src/gateway/server-startup-post-attach.ts‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -34,6 +34,7 @@ import {
3434
type GatewayStartupOutcomeRecorder,
3535
} from "./server-startup-outcomes.js";
3636
import type { startGatewayTailscaleExposure } from "./server-tailscale.js";
37+
import { warmMacOSSystemCaOffMainThread } from "./system-ca-warmup.js";
3738
const ACP_BACKEND_READY_TIMEOUT_MS = 5_000;
3839
const ACP_BACKEND_READY_POLL_MS = 50;
3940
const PROVIDER_AUTH_PREWARM_START_DELAY_MS = 5_000;
@@ -611,9 +612,16 @@ export async function startGatewaySidecars(params: {
611612
logChannels: { info: (msg: string) => void; error: (msg: string) => void };
612613
startupTrace?: GatewayStartupTrace;
613614
startupOutcomes?: GatewayStartupOutcomeRecorder;
615+
warmSystemCa?: typeof warmMacOSSystemCaOffMainThread;
614616
}) {
615617
const postReadySidecars: GatewayPostReadySidecarHandle[] = [];
616618

619+
// Node initializes the process-wide macOS Keychain CA cache synchronously. Warm it in a
620+
// worker before any provider TLS so a slow trustd lookup cannot freeze gateway HTTP/WS.
621+
await measureStartup(params.startupTrace, "sidecars.system-ca", () =>
622+
(params.warmSystemCa ?? warmMacOSSystemCaOffMainThread)({ log: params.log }),
623+
);
624+
617625
const internalHooksConfigured = hasConfiguredInternalHooks(params.cfg);
618626
await measureStartup(params.startupTrace, "sidecars.internal-hooks", async () => {
619627
try {

‎src/gateway/system-ca-warmup.test.ts‎

Lines changed: 140 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,140 @@
1+
import { EventEmitter } from "node:events";
2+
import { afterEach, describe, expect, it, vi } from "vitest";
3+
import { warmMacOSSystemCaOffMainThread } from "./system-ca-warmup.js";
4+
5+
class FakeWorker extends EventEmitter {
6+
unref = vi.fn();
7+
}
8+
9+
describe("warmMacOSSystemCaOffMainThread", () => {
10+
afterEach(() => {
11+
vi.useRealTimers();
12+
});
13+
14+
it.each([
15+
["env only", "darwin", { NODE_USE_SYSTEM_CA: "1" }, [], true],
16+
["dash flag", "darwin", {}, ["--use-system-ca"], true],
17+
["underscore flag", "darwin", {}, ["--use_system_ca"], true],
18+
["equals-form flag", "darwin", {}, ["--use_system_ca=false"], true],
19+
["OpenSSL CA alone", "darwin", {}, ["--use-openssl-ca"], false],
20+
[
21+
"bundled CA does not suppress env system CA",
22+
"darwin",
23+
{ NODE_USE_SYSTEM_CA: "1" },
24+
["--use-bundled-ca"],
25+
true,
26+
],
27+
[
28+
"dash negation overrides env",
29+
"darwin",
30+
{ NODE_USE_SYSTEM_CA: "1" },
31+
["--no-use-system-ca"],
32+
false,
33+
],
34+
[
35+
"equals-form negation overrides env",
36+
"darwin",
37+
{ NODE_USE_SYSTEM_CA: "1" },
38+
["--no-use-system-ca=false"],
39+
false,
40+
],
41+
[
42+
"underscore negation overrides env",
43+
"darwin",
44+
{ NODE_USE_SYSTEM_CA: "1" },
45+
["--no_use_system_ca"],
46+
false,
47+
],
48+
[
49+
"last conflicting flag enables",
50+
"darwin",
51+
{},
52+
["--no-use-system-ca", "--use_system_ca"],
53+
true,
54+
],
55+
[
56+
"last conflicting flag disables",
57+
"darwin",
58+
{},
59+
["--use-system-ca", "--no_use_system_ca"],
60+
false,
61+
],
62+
["NODE_OPTIONS flag", "darwin", { NODE_OPTIONS: '"--use_system_ca"' }, [], true],
63+
[
64+
"NODE_OPTIONS negation overrides env",
65+
"darwin",
66+
{ NODE_USE_SYSTEM_CA: "1", NODE_OPTIONS: "--no-use-system-ca" },
67+
[],
68+
false,
69+
],
70+
[
71+
"execArgv overrides NODE_OPTIONS",
72+
"darwin",
73+
{ NODE_OPTIONS: "--use-system-ca" },
74+
["--no-use-system-ca"],
75+
false,
76+
],
77+
["system CA disabled", "darwin", { NODE_USE_SYSTEM_CA: "0" }, [], false],
78+
["non-macOS", "linux", { NODE_USE_SYSTEM_CA: "1" }, [], false],
79+
] as const)(
80+
"%s: warmup runs iff system CA is effectively enabled on macOS",
81+
async (_name, platform, env, execArgv, shouldWarm) => {
82+
const worker = new FakeWorker();
83+
const createWorker = vi.fn(() => worker);
84+
const warmup = warmMacOSSystemCaOffMainThread({
85+
platform,
86+
env: { ...env },
87+
execArgv: [...execArgv],
88+
createWorker,
89+
});
90+
91+
if (shouldWarm) {
92+
worker.emit("message", { ok: true, certificateCount: 42 });
93+
}
94+
await warmup;
95+
expect(createWorker).toHaveBeenCalledTimes(shouldWarm ? 1 : 0);
96+
expect(worker.unref).toHaveBeenCalledTimes(shouldWarm ? 1 : 0);
97+
},
98+
);
99+
100+
it("waits for the worker while leaving the main event loop available", async () => {
101+
vi.useFakeTimers();
102+
const worker = new FakeWorker();
103+
const log = { warn: vi.fn() };
104+
const warmup = warmMacOSSystemCaOffMainThread({
105+
platform: "darwin",
106+
env: { NODE_USE_SYSTEM_CA: "1" },
107+
warningMs: 10,
108+
log,
109+
createWorker: vi.fn(() => worker),
110+
});
111+
112+
let mainTurnRan = false;
113+
setImmediate(() => {
114+
mainTurnRan = true;
115+
});
116+
await vi.advanceTimersByTimeAsync(10);
117+
118+
expect(mainTurnRan).toBe(true);
119+
expect(log.warn).toHaveBeenCalledWith(
120+
"macOS system CA warmup is still waiting for Keychain trust settings; channel startup remains deferred",
121+
);
122+
expect(worker.unref).toHaveBeenCalledOnce();
123+
124+
worker.emit("message", { ok: true, certificateCount: 42 });
125+
await warmup;
126+
});
127+
128+
it("fails closed when the worker cannot populate the cache", async () => {
129+
const worker = new FakeWorker();
130+
const warmup = warmMacOSSystemCaOffMainThread({
131+
platform: "darwin",
132+
env: { NODE_USE_SYSTEM_CA: "1" },
133+
createWorker: vi.fn(() => worker),
134+
});
135+
136+
worker.emit("message", { ok: false, error: "trust store unavailable" });
137+
138+
await expect(warmup).rejects.toThrow("trust store unavailable");
139+
});
140+
});

0 commit comments

Comments
 (0)