Skip to content

Commit b28fda9

Browse files
vincentkocvincentkoc
committed
fix(tooling): bound RPC RTT readiness bodies
1 parent 09427aa commit b28fda9

2 files changed

Lines changed: 74 additions & 34 deletions

File tree

‎scripts/measure-rpc-rtt.mjs‎

Lines changed: 64 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -9,13 +9,15 @@ import net from "node:net";
99
import path from "node:path";
1010
import { performance } from "node:perf_hooks";
1111
import { fileURLToPath, pathToFileURL } from "node:url";
12+
import { readBoundedResponseText } from "./lib/bounded-response.mjs";
1213

1314
const DEFAULT_METHODS = ["health", "config.get"];
1415
const DEFAULT_ITERATIONS = 10;
1516
/** Maximum time to wait for a spawned gateway to become reachable. */
1617
export const READY_TIMEOUT_MS = 120_000;
1718
/** Per-probe timeout used while polling gateway readiness endpoints. */
1819
export const READY_PROBE_TIMEOUT_MS = 1_000;
20+
const READY_PROBE_RESPONSE_BODY_MAX_BYTES = 64 * 1024;
1921
const GATEWAY_FORCE_KILL_GRACE_MS = 250;
2022
const PARENT_TERMINATION_SIGNALS = ["SIGHUP", "SIGINT", "SIGTERM"];
2123
const IS_DIRECT_RUN =
@@ -126,21 +128,56 @@ function formatErrorMessage(error) {
126128
return String(error);
127129
}
128130

129-
async function readyzReportsReady(response) {
131+
async function readyzReportsReady(response, options = {}) {
130132
if (!response.ok) {
131133
return false;
132134
}
133-
if (typeof response.json !== "function") {
134-
return false;
135-
}
136135
try {
137-
const body = await response.json();
136+
const text = await readBoundedResponseText(
137+
response,
138+
"RPC RTT /readyz",
139+
READY_PROBE_RESPONSE_BODY_MAX_BYTES,
140+
options,
141+
);
142+
const body = JSON.parse(text);
138143
return body && typeof body === "object" && body.ready === true;
139144
} catch {
140145
return false;
141146
}
142147
}
143148

149+
async function fetchReadinessProbe(fetchImpl, url, timeoutMs) {
150+
const controller = new AbortController();
151+
const timeoutError = Object.assign(new Error(`${url} timed out after ${timeoutMs}ms`), {
152+
code: "ETIMEDOUT",
153+
});
154+
let timeout;
155+
const timeoutPromise = new Promise((_, reject) => {
156+
timeout = setTimeout(() => {
157+
controller.abort(timeoutError);
158+
reject(timeoutError);
159+
}, timeoutMs);
160+
timeout.unref?.();
161+
});
162+
try {
163+
const response = await Promise.race([
164+
fetchImpl(url, {
165+
signal: controller.signal,
166+
}),
167+
timeoutPromise,
168+
]);
169+
return {
170+
clearTimeout: () => clearTimeout(timeout),
171+
response,
172+
signal: controller.signal,
173+
timeoutPromise,
174+
};
175+
} catch (error) {
176+
clearTimeout(timeout);
177+
throw error;
178+
}
179+
}
180+
144181
/**
145182
* Polls readiness endpoints while also failing fast if the child exits.
146183
*/
@@ -172,19 +209,33 @@ export async function waitForGatewayReady({
172209
);
173210
}
174211
try {
175-
const response = await fetchImpl(`http://127.0.0.1:${port}/readyz`, {
176-
signal: AbortSignal.timeout(probeTimeoutMs),
177-
});
178-
if (await readyzReportsReady(response)) {
179-
return;
212+
const probe = await fetchReadinessProbe(
213+
fetchImpl,
214+
`http://127.0.0.1:${port}/readyz`,
215+
probeTimeoutMs,
216+
);
217+
try {
218+
if (
219+
await readyzReportsReady(probe.response, {
220+
signal: probe.signal,
221+
timeoutPromise: probe.timeoutPromise,
222+
})
223+
) {
224+
return;
225+
}
226+
} finally {
227+
probe.clearTimeout();
180228
}
181229
} catch {
182230
// The gateway may not have bound the port yet.
183231
}
184232
try {
185-
await fetchImpl(`http://127.0.0.1:${port}/healthz`, {
186-
signal: AbortSignal.timeout(probeTimeoutMs),
187-
});
233+
const probe = await fetchReadinessProbe(
234+
fetchImpl,
235+
`http://127.0.0.1:${port}/healthz`,
236+
probeTimeoutMs,
237+
);
238+
probe.clearTimeout();
188239
} catch {
189240
// Liveness is diagnostic only; /readyz is the usable RPC readiness contract.
190241
}

‎test/scripts/measure-rpc-rtt.test.ts‎

Lines changed: 10 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -43,6 +43,10 @@ class FakeWebSocket extends EventEmitter {
4343
}
4444
}
4545

46+
function jsonResponse(body: unknown, init: ResponseInit = {}): Response {
47+
return new Response(JSON.stringify(body), init);
48+
}
49+
4650
describe("scripts/measure-rpc-rtt.mjs", () => {
4751
it("closes websocket clients that time out before opening", async () => {
4852
FakeWebSocket.instances = [];
@@ -475,12 +479,9 @@ describe("scripts/measure-rpc-rtt.mjs", () => {
475479
const child = new EventEmitter();
476480
const fetchImpl = vi
477481
.fn()
478-
.mockRejectedValueOnce(new DOMException("request timed out", "TimeoutError"))
479-
.mockResolvedValueOnce({ ok: true })
480-
.mockResolvedValueOnce({
481-
json: vi.fn().mockResolvedValue({ ready: true }),
482-
ok: true,
483-
});
482+
.mockResolvedValueOnce(new Response(new ReadableStream<Uint8Array>({ start() {} })))
483+
.mockResolvedValueOnce(jsonResponse({ ok: true, status: "live" }))
484+
.mockResolvedValueOnce(jsonResponse({ failing: [], ready: true }));
484485

485486
await waitForGatewayReady({
486487
child,
@@ -520,21 +521,9 @@ describe("scripts/measure-rpc-rtt.mjs", () => {
520521
const child = new EventEmitter();
521522
const fetchImpl = vi
522523
.fn()
523-
.mockResolvedValueOnce({
524-
json: vi.fn().mockResolvedValue({ failing: ["gateway"], ready: false }),
525-
ok: false,
526-
status: 503,
527-
})
528-
.mockResolvedValueOnce({
529-
json: vi.fn().mockResolvedValue({ ok: true, status: "live" }),
530-
ok: true,
531-
status: 200,
532-
})
533-
.mockResolvedValueOnce({
534-
json: vi.fn().mockResolvedValue({ failing: [], ready: true }),
535-
ok: true,
536-
status: 200,
537-
});
524+
.mockResolvedValueOnce(jsonResponse({ failing: ["gateway"], ready: false }, { status: 503 }))
525+
.mockResolvedValueOnce(jsonResponse({ ok: true, status: "live" }))
526+
.mockResolvedValueOnce(jsonResponse({ failing: [], ready: true }));
538527

539528
await waitForGatewayReady({
540529
child,

0 commit comments

Comments
 (0)