Skip to content

Commit 87f9e42

Browse files
committed
console: show how a replica's workers share its CPU on the redesigned cluster page
Adds a Workers button to the redesigned cluster page's CPU chart. It opens a drawer that reads the replica's per-worker scheduling counters over one WebSocket session while open, and leads with a finding in the CPU troubleshooting guide's terms: a saturated or uneven worker, every worker busy, high CPU inside or outside running dataflows, idle, or balanced. Below it are the replica's CPU against its time running dataflows, the busiest and average worker over time, the busiest workers, and a table of objects with one bar per worker against an even share.
1 parent eaa34e3 commit 87f9e42

27 files changed

Lines changed: 5169 additions & 115 deletions
Lines changed: 180 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,180 @@
1+
// Copyright Materialize, Inc. and contributors. All rights reserved.
2+
//
3+
// Use of this software is governed by the Business Source License
4+
// included in the LICENSE file.
5+
//
6+
// As of the Change Date specified in that file, in accordance with
7+
// the Business Source License, use of this software will be governed
8+
// by the Apache License, Version 2.0.
9+
10+
import { CompiledQuery } from "kysely";
11+
import { ws } from "msw";
12+
13+
import { Column, MzDataType } from "~/api/materialize/types";
14+
import { buildColumn } from "~/api/mocks/buildSqlQueryHandler";
15+
import server from "~/api/mocks/server";
16+
17+
import {
18+
createWebsocketQueryClient,
19+
WebsocketQueryClient,
20+
} from "./WebsocketQueryClient";
21+
22+
interface SentQuery {
23+
query: string;
24+
params: unknown[];
25+
}
26+
27+
type Answer = { columns: Column[]; rows: unknown[][] } | { error: string };
28+
29+
const READY = { type: "ReadyForQuery", payload: "I" };
30+
31+
const WORKER_COLUMNS = [
32+
buildColumn({ name: "workerId", type_oid: MzDataType.int4 }),
33+
buildColumn({ name: "elapsedNs", type_oid: MzDataType.float8 }),
34+
];
35+
36+
/**
37+
* Serves the WebSocket SQL protocol. Each query is answered on a later task,
38+
* so a query sent without waiting for the previous one's `ReadyForQuery`
39+
* arrives while that one is still open and counts as overlapping. Returning
40+
* "close" from `answer` closes the socket instead of answering.
41+
*/
42+
function serveQueries(answer: (query: SentQuery) => Answer | "close") {
43+
const received: SentQuery[] = [];
44+
const socket = { received, overlapped: false };
45+
let isAnswering = false;
46+
server.use(
47+
ws.link(/^wss?:\/\//).addEventListener("connection", ({ client }) => {
48+
let hasOptions = false;
49+
const send = (message: unknown) => client.send(JSON.stringify(message));
50+
client.addEventListener("message", (event) => {
51+
// The first message carries the session options.
52+
if (!hasOptions) {
53+
hasOptions = true;
54+
send(READY);
55+
return;
56+
}
57+
const {
58+
queries: [query],
59+
} = JSON.parse(String(event.data)) as { queries: SentQuery[] };
60+
received.push(query);
61+
if (isAnswering) socket.overlapped = true;
62+
isAnswering = true;
63+
const reply = answer(query);
64+
setTimeout(() => {
65+
isAnswering = false;
66+
if (reply === "close") {
67+
client.close();
68+
return;
69+
}
70+
if ("error" in reply) {
71+
send({
72+
type: "Error",
73+
payload: { message: reply.error, code: "XX000" },
74+
});
75+
} else {
76+
send({ type: "Rows", payload: { columns: reply.columns } });
77+
for (const row of reply.rows) send({ type: "Row", payload: row });
78+
send({ type: "CommandComplete", payload: "SELECT" });
79+
}
80+
send(READY);
81+
}, 10);
82+
});
83+
}),
84+
);
85+
return socket;
86+
}
87+
88+
const workerRows = (rows: unknown[][]): Answer => ({
89+
columns: WORKER_COLUMNS,
90+
rows,
91+
});
92+
93+
const clients: WebsocketQueryClient[] = [];
94+
95+
const createClient = () => {
96+
const client = createWebsocketQueryClient({
97+
httpAddress: "localhost:6876",
98+
});
99+
clients.push(client);
100+
return client;
101+
};
102+
103+
const query = (sql: string, parameters: unknown[] = []) =>
104+
CompiledQuery.raw(sql, parameters) as CompiledQuery<Record<string, unknown>>;
105+
106+
describe("createWebsocketQueryClient", () => {
107+
afterEach(() => {
108+
for (const client of clients.splice(0)) client.close();
109+
});
110+
111+
it("runs queued queries one at a time and resolves each with its own rows keyed by column", async () => {
112+
const socket = serveQueries(({ query: sql }) =>
113+
sql === "SELECT 1"
114+
? workerRows([["0", "1.5"]])
115+
: workerRows([
116+
["0", "2"],
117+
["1", "3"],
118+
]),
119+
);
120+
const client = createClient();
121+
122+
const results = await Promise.all([
123+
client.query(query("SELECT 1")),
124+
client.query(query("SELECT 2")),
125+
]);
126+
127+
expect(results).toEqual([
128+
[{ workerId: 0, elapsedNs: 1.5 }],
129+
[
130+
{ workerId: 0, elapsedNs: 2 },
131+
{ workerId: 1, elapsedNs: 3 },
132+
],
133+
]);
134+
expect(socket.received.map(({ query: sql }) => sql)).toEqual([
135+
"SELECT 1",
136+
"SELECT 2",
137+
]);
138+
expect(socket.overlapped).toBe(false);
139+
});
140+
141+
it("rejects only the query that errored and answers the next one", async () => {
142+
serveQueries(({ query: sql }) =>
143+
sql === "SELECT 1/0"
144+
? { error: "division by zero" }
145+
: workerRows([["0", "1"]]),
146+
);
147+
const client = createClient();
148+
149+
const failed = client.query(query("SELECT 1/0"));
150+
const next = client.query(query("SELECT 1"));
151+
152+
await expect(failed).rejects.toThrow("division by zero");
153+
await expect(next).resolves.toEqual([{ workerId: 0, elapsedNs: 1 }]);
154+
});
155+
156+
it("rejects the pending query and every later one once the socket closes", async () => {
157+
serveQueries(() => "close");
158+
const client = createClient();
159+
160+
await expect(client.query(query("SELECT 1"))).rejects.toThrow(
161+
"Connection closed",
162+
);
163+
await expect(client.query(query("SELECT 2"))).rejects.toThrow(
164+
"Connection closed",
165+
);
166+
});
167+
168+
it("sends numeric parameters as strings", async () => {
169+
const socket = serveQueries(() => workerRows([]));
170+
const client = createClient();
171+
172+
await client.query(
173+
query("SELECT $1::int4, $2::float8, $3", [42, 1.5, "a"]),
174+
);
175+
176+
expect(socket.received).toEqual([
177+
{ query: "SELECT $1::int4, $2::float8, $3", params: ["42", "1.5", "a"] },
178+
]);
179+
});
180+
});
Lines changed: 137 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,137 @@
1+
// Copyright Materialize, Inc. and contributors. All rights reserved.
2+
//
3+
// Use of this software is governed by the Business Source License
4+
// included in the LICENSE file.
5+
//
6+
// As of the Change Date specified in that file, in accordance with
7+
// the Business Source License, use of this software will be governed
8+
// by the Apache License, Version 2.0.
9+
10+
import { CompiledQuery } from "kysely";
11+
12+
import {
13+
buildSessionVariables,
14+
mapColumnToColumnMetadata,
15+
mapRowToObject,
16+
} from ".";
17+
import { buildExtendedRequestQuery } from "./executeSqlV2";
18+
import { MaterializeWebsocket, WebSocketResult } from "./MaterializeWebsocket";
19+
import { ColumnMetadata, SessionVariables } from "./types";
20+
21+
class WebsocketQueryError extends Error {}
22+
23+
/**
24+
* Runs queries one at a time over a single WebSocket session.
25+
*
26+
* Every HTTP request opens a new session, and each session start costs
27+
* `mz_catalog_server` about 35 ms of CPU (measured on an M.1-nano), so a
28+
* caller that repeats a query every few seconds should hold one of these
29+
* instead. Rows come back keyed and converted as `executeSqlV2` returns them.
30+
*/
31+
export interface WebsocketQueryClient {
32+
query: <T>(query: CompiledQuery<T>) => Promise<T[]>;
33+
close: () => void;
34+
}
35+
36+
interface PendingQuery {
37+
resolve: (rows: unknown[]) => void;
38+
reject: (error: Error) => void;
39+
columns: ColumnMetadata[];
40+
rows: unknown[];
41+
error?: Error;
42+
}
43+
44+
export function createWebsocketQueryClient({
45+
httpAddress,
46+
sessionVariables,
47+
}: {
48+
httpAddress: string;
49+
sessionVariables?: SessionVariables;
50+
}): WebsocketQueryClient {
51+
let closedError: Error | undefined;
52+
let pending: PendingQuery | undefined;
53+
let markReady: () => void = () => {};
54+
let failReady: (error: Error) => void = () => {};
55+
const ready = new Promise<void>((resolve, reject) => {
56+
markReady = resolve;
57+
failReady = reject;
58+
});
59+
// Swallows the rejection for callers that never query.
60+
ready.catch(() => {});
61+
// Chains queries so only one is in flight on the session.
62+
let queue: Promise<unknown> = Promise.resolve();
63+
64+
const fail = (error: Error) => {
65+
closedError ??= error;
66+
failReady(closedError);
67+
pending?.reject(closedError);
68+
pending = undefined;
69+
};
70+
71+
const handleMessage = (message: WebSocketResult) => {
72+
switch (message.type) {
73+
case "ReadyForQuery": {
74+
markReady();
75+
if (!pending) return;
76+
const { resolve, reject, rows, error } = pending;
77+
pending = undefined;
78+
if (error) reject(error);
79+
else resolve(rows);
80+
return;
81+
}
82+
case "Rows":
83+
if (pending) {
84+
pending.columns = message.payload.columns.map(
85+
mapColumnToColumnMetadata,
86+
);
87+
}
88+
return;
89+
case "Row":
90+
pending?.rows.push(mapRowToObject(message.payload, pending.columns));
91+
return;
92+
case "Error":
93+
if (pending) {
94+
pending.error = new WebsocketQueryError(message.payload.message);
95+
} else {
96+
fail(new WebsocketQueryError(message.payload.message));
97+
}
98+
return;
99+
default:
100+
return;
101+
}
102+
};
103+
104+
const socket = new MaterializeWebsocket({
105+
httpAddress,
106+
sessionVariables: buildSessionVariables(sessionVariables),
107+
onMessage: handleMessage,
108+
onClose: () => fail(new WebsocketQueryError("Connection closed")),
109+
});
110+
socket.connect();
111+
112+
const run = async <T>(query: CompiledQuery<T>): Promise<T[]> => {
113+
await ready;
114+
if (closedError) throw closedError;
115+
return new Promise<T[]>((resolve, reject) => {
116+
pending = {
117+
resolve: resolve as (rows: unknown[]) => void,
118+
reject,
119+
columns: [],
120+
rows: [],
121+
};
122+
socket.send({ queries: [buildExtendedRequestQuery(query)] });
123+
});
124+
};
125+
126+
return {
127+
query: <T>(query: CompiledQuery<T>) => {
128+
const result = queue.then(() => run(query));
129+
queue = result.catch(() => {});
130+
return result;
131+
},
132+
close: () => {
133+
fail(new WebsocketQueryError("Client closed"));
134+
socket.disconnect();
135+
},
136+
};
137+
}

‎console/src/api/materialize/cluster/replicaProcessMetrics.ts‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,8 @@ export interface ProcessMetricsSample {
3232
/** A size's process count and each process's CPU, memory and disk allocation. */
3333
export interface SizeAllocation {
3434
processes: number;
35+
/** Timely workers per process. */
36+
workers: number;
3537
cpuNanoCores: number;
3638
memoryBytes: number;
3739
diskBytes: number;
@@ -96,6 +98,7 @@ export function buildReplicaSizeAllocationsQuery({
9698
.select([
9799
"size",
98100
sql<number>`processes::float8`.as("processes"),
101+
sql<number>`workers::float8`.as("workers"),
99102
sql<number>`cpu_nano_cores::float8`.as("cpuNanoCores"),
100103
sql<number>`memory_bytes::float8`.as("memoryBytes"),
101104
sql<number>`disk_bytes::float8`.as("diskBytes"),

0 commit comments

Comments
 (0)