Skip to content

Commit 2d1e942

Browse files
committed
console: show how a replica's workers share its CPU on the redesigned cluster page
Adds a Workers tab to the redesigned cluster page. It reads the replica's per-worker scheduling counters over one WebSocket session while open, and reports one measured minute until Measure again: a finding that names the overloaded worker and the object behind it, or says why it isn't skew, one bar per worker, the objects ranked by their time on the worker picked from the bars with their spread and freshness, and the replica's CPU against the workers' time.
1 parent 4d0010e commit 2d1e942

48 files changed

Lines changed: 11558 additions & 206 deletions

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.
Lines changed: 86 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,86 @@
1+
// Copyright Materialize, Inc. and contributors. All rights reserved.
2+
//
3+
// Use of this software is governed by the Business Source License
4+
// included in the LICENSE file.
5+
//
6+
// As of the Change Date specified in that file, in accordance with
7+
// the Business Source License, use of this software will be governed
8+
// by the Apache License, Version 2.0.
9+
10+
import { buildColumns } from "~/api/mocks/buildSqlQueryHandler";
11+
import server from "~/api/mocks/server";
12+
import {
13+
MockSubscribeConnection,
14+
mockSubscribeSocket,
15+
} from "~/test/mockSubscribeSocket";
16+
17+
import { SubscribeManager } from "./SubscribeManager";
18+
19+
const REQUEST = {
20+
queries: [{ query: "SUBSCRIBE t", params: [] }],
21+
};
22+
23+
/** Serves one row per subscribe, and returns each query's connection. */
24+
const serve = () => {
25+
const queries: MockSubscribeConnection[] = [];
26+
server.use(
27+
mockSubscribeSocket((_sql, _params, connection) => {
28+
queries.push(connection);
29+
return { columns: buildColumns(["id"]), rows: [["a"]], timestamp: 10 };
30+
}),
31+
);
32+
return queries;
33+
};
34+
35+
const managers: Array<SubscribeManager<{ id: string }, { id: string }>> = [];
36+
37+
const connect = (closeSocketOnComplete = false) => {
38+
const manager = new SubscribeManager<{ id: string }, { id: string }>({
39+
httpAddress: "subscribe.test",
40+
request: REQUEST,
41+
upsert: { key: (row) => row.data.id },
42+
select: (row) => row.data,
43+
closeSocketOnComplete,
44+
});
45+
managers.push(manager);
46+
manager.connect();
47+
return manager;
48+
};
49+
50+
describe("SubscribeManager", () => {
51+
afterEach(() => {
52+
for (const manager of managers.splice(0)) manager.disconnect();
53+
});
54+
55+
it("applies the waiting updates once when a listener reads the upper", async () => {
56+
const queries = serve();
57+
const manager = connect();
58+
const uppers: Array<number | undefined> = [];
59+
manager.onChange(() => uppers.push(manager.getUpper()));
60+
await vi.waitFor(() =>
61+
expect(manager.getSnapshot().snapshotComplete).toBe(true),
62+
);
63+
64+
queries[0].advance(20, [["b"]]);
65+
queries[0].advance(30);
66+
67+
await vi.waitFor(() => expect(manager.getUpper()).toBe(30));
68+
expect(manager.getSnapshot().data).toEqual([{ id: "a" }, { id: "b" }]);
69+
expect(uppers.at(-1)).toBe(30);
70+
});
71+
72+
it("reports a statement that ends on its own and closes its socket, without sending it again", async () => {
73+
const queries = serve();
74+
const manager = connect(true);
75+
await vi.waitFor(() =>
76+
expect(manager.getSnapshot().snapshotComplete).toBe(true),
77+
);
78+
79+
queries[0].complete();
80+
81+
await vi.waitFor(() => expect(manager.getSnapshot().isComplete).toBe(true));
82+
await vi.waitFor(() => expect(queries[0].isClosed).toBe(true));
83+
expect(manager.getSnapshot().error).toBeUndefined();
84+
expect(queries).toHaveLength(1);
85+
});
86+
});

‎console/src/api/materialize/SubscribeManager.ts‎

Lines changed: 35 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -43,6 +43,11 @@ export interface SubscribeState<T> {
4343
snapshotComplete: boolean;
4444
/** True while a re-subscribe holds the previous snapshot as `data`. */
4545
resubscribing?: boolean;
46+
/**
47+
* The statement ended, as a SUBSCRIBE does at its `UP TO`. Set only with
48+
* `closeSocketOnComplete`, which then closes the socket.
49+
*/
50+
isComplete?: boolean;
4651
error: SubscribeError | undefined;
4752
}
4853

@@ -99,6 +104,7 @@ export class SubscribeManager<T extends object, R> implements Connectable {
99104
/** True when the socket signalled ready before any request was set. */
100105
private readyAwaitingRequest: boolean = false;
101106
private currentTimestamp: number | undefined;
107+
private connectionId: number | undefined;
102108
/** Holds completed timestamp messages, waiting for the flush interval */
103109
private closedTimestampBuffer: SubscribeRow<T>[] = [];
104110
/** Holds messages from the socket until the timestamp is closed */
@@ -234,6 +240,7 @@ export class SubscribeManager<T extends object, R> implements Connectable {
234240
// than flashing the loading state on every reconnect. `resubscribing`
235241
// marks the held data. Any following setState clears it.
236242
this.resetForNewConnection();
243+
this.connectionId = undefined;
237244
if (this.snapshotState.snapshotComplete || this.snapshotState.error) {
238245
this.snapshotState = {
239246
...this.snapshotState,
@@ -261,6 +268,21 @@ export class SubscribeManager<T extends object, R> implements Connectable {
261268
return this.snapshotState;
262269
};
263270

271+
/**
272+
* The subscribe's upper: `getSnapshot().data` holds every update before it.
273+
* Undefined until the snapshot completes. It applies the updates waiting for
274+
* the flush interval, so read the snapshot after calling it.
275+
*/
276+
getUpper = () => {
277+
this.flushSocketBuffer();
278+
return this.currentState.snapshotComplete
279+
? this.currentTimestamp
280+
: undefined;
281+
};
282+
283+
/** The session's `pg_backend_pid()`, once the connection reports it. */
284+
getConnectionId = () => this.connectionId;
285+
264286
private setState(update: Partial<SubscribeState<SubscribeRow<T>>>) {
265287
this.currentState = {
266288
...this.currentState,
@@ -282,6 +304,7 @@ export class SubscribeManager<T extends object, R> implements Connectable {
282304
private onReadyForQuery = () => {
283305
if (this.querySent && this.closeSocketOnComplete) {
284306
this.socket.disconnect();
307+
this.setState({ isComplete: true });
285308
return;
286309
}
287310
if (this.sqlRequest) {
@@ -313,6 +336,9 @@ export class SubscribeManager<T extends object, R> implements Connectable {
313336
},
314337
});
315338
}
339+
if (message.type === "BackendKeyData") {
340+
this.connectionId = message.payload.conn_id;
341+
}
316342
if (message.type === "Rows") {
317343
this.columns = message.payload.columns.map(mapColumnToColumnMetadata);
318344
}
@@ -366,10 +392,10 @@ export class SubscribeManager<T extends object, R> implements Connectable {
366392

367393
private flushSocketBuffer = () => {
368394
if (this.closedTimestampBuffer.length === 0) return;
369-
370-
const clearBuffer = () => {
371-
this.closedTimestampBuffer = [];
372-
};
395+
// NOTE: taken before `setState`. Its listeners may call `getUpper`, which
396+
// flushes again, and a buffer still set then recurses without end.
397+
const closed = this.closedTimestampBuffer;
398+
this.closedTimestampBuffer = [];
373399

374400
const newState = {
375401
...this.currentState,
@@ -384,16 +410,13 @@ export class SubscribeManager<T extends object, R> implements Connectable {
384410
);
385411

386412
// We want to filter out progress messages which have mzState === null
387-
const filteredChanges = this.closedTimestampBuffer.filter(
413+
const filteredChanges = closed.filter(
388414
(change) => change.mzState !== null,
389415
);
390416

391-
// If there are no real changes, we can clear the buffer and return early to avoid
392-
// unnecessary notifications to listeners.
393-
if (filteredChanges.length === 0) {
394-
clearBuffer();
395-
return;
396-
}
417+
// If there are no real changes, return early to avoid unnecessary
418+
// notifications to listeners.
419+
if (filteredChanges.length === 0) return;
397420

398421
for (const change of filteredChanges) {
399422
switch (change.mzState) {
@@ -425,14 +448,10 @@ export class SubscribeManager<T extends object, R> implements Connectable {
425448
// Convert the upsert map back into an array once upsert operation is complete.
426449
newState.data = Array.from(upsertMap.values());
427450
} else {
428-
newState.data = [
429-
...newState.data,
430-
...this.closedTimestampBuffer.values(),
431-
];
451+
newState.data = [...newState.data, ...closed];
432452
}
433453

434454
this.setState(newState);
435-
clearBuffer();
436455
};
437456
}
438457

‎console/src/api/materialize/buildSubscribeQuery.test.ts‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -41,4 +41,14 @@ describe("buildSubscribeQuery", () => {
4141
`SUBSCRIBE ((select "id" from "mz_objects")) WITH (PROGRESS) AS OF AT LEAST TIMESTAMP '${asOfAtLeast.toISOString()}' ENVELOPE UPSERT (KEY ("id"));`,
4242
);
4343
});
44+
45+
it("with up to", () => {
46+
const result = buildSubscribeQuery(selectQuery, {
47+
upsertKey: "id",
48+
upToMs: 1792639991868,
49+
}).compile(queryBuilder);
50+
expect(result.sql).toEqual(
51+
`SUBSCRIBE ((select "id" from "mz_objects")) WITH (PROGRESS) UP TO 1792639991868 ENVELOPE UPSERT (KEY ("id"));`,
52+
);
53+
});
4454
});

‎console/src/api/materialize/buildSubscribeQuery.ts‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,11 @@ import { Compilable, RawBuilder, sql } from "kysely";
1212
export interface SubscribeQueryOptions {
1313
upsertKey: string | string[];
1414
asOfAtLeast?: Date;
15+
/**
16+
* Ends the subscribe once its frontier passes this timestamp, in
17+
* milliseconds since the epoch, whether or not the client is still reading.
18+
*/
19+
upToMs?: number;
1520
}
1621

1722
/**
@@ -30,6 +35,9 @@ export function buildSubscribeQuery<T>(
3035
sql`AS OF AT LEAST TIMESTAMP ${sql.lit(options.asOfAtLeast.toISOString())}`,
3136
);
3237
}
38+
if (options?.upToMs !== undefined) {
39+
subscribeOptions.push(sql`UP TO ${sql.lit(options.upToMs)}`);
40+
}
3341
if (options?.upsertKey) {
3442
const key =
3543
typeof options.upsertKey === "string"

‎console/src/api/materialize/cluster/replicaProcessMetrics.test.sql.ts‎

Lines changed: 7 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -177,13 +177,13 @@ describe("buildReplicaProcessMetricsSubscribe", () => {
177177
});
178178

179179
describe("buildReplicaSizeAllocationsQuery", () => {
180-
it("reads each listed size's process count and per-process allocation", async () => {
180+
it("reads each listed size's process and worker counts and per-process allocation", async () => {
181181
// Reset Materialize state.
182182
await testdrive(`> SET cluster to quickstart;`);
183183
const client = await getMaterializeClient();
184184
const sizes = ["scale=2,workers=4", "scale=1,workers=1"];
185185
const { rows: expected } = await client.query(
186-
`SELECT size, processes, cpu_nano_cores, memory_bytes, disk_bytes
186+
`SELECT size, processes, workers, cpu_nano_cores, memory_bytes, disk_bytes
187187
FROM mz_cluster_replica_sizes WHERE size IN ('${sizes.join("', '")}')`,
188188
);
189189

@@ -193,11 +193,15 @@ describe("buildReplicaSizeAllocationsQuery", () => {
193193

194194
const bySize = new Map(rows.map((row) => [row.size, row]));
195195
expect([...bySize.keys()].sort()).toEqual([...sizes].sort());
196-
expect(Number(bySize.get("scale=2,workers=4")?.processes)).toEqual(2);
196+
expect(bySize.get("scale=2,workers=4")).toMatchObject({
197+
processes: 2,
198+
workers: 4,
199+
});
197200
for (const size of expected) {
198201
expect(bySize.get(size.size)).toEqual({
199202
size: size.size,
200203
processes: Number(size.processes),
204+
workers: Number(size.workers),
201205
cpuNanoCores: Number(size.cpu_nano_cores),
202206
memoryBytes: Number(size.memory_bytes),
203207
diskBytes: size.disk_bytes === null ? null : Number(size.disk_bytes),

‎console/src/api/materialize/cluster/replicaProcessMetrics.ts‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,8 @@ export interface ProcessMetricsSample {
3131
/** A size's process count and each process's CPU, memory and disk allocation. */
3232
export interface SizeAllocation {
3333
processes: number;
34+
/** Timely workers per process. */
35+
workers: number;
3436
cpuNanoCores: number;
3537
memoryBytes: number;
3638
diskBytes: number | null;
@@ -89,6 +91,7 @@ export function buildReplicaSizeAllocationsQuery({
8991
.select([
9092
"size",
9193
sql<number>`processes::float8`.as("processes"),
94+
sql<number>`workers::float8`.as("workers"),
9295
sql<number>`cpu_nano_cores::float8`.as("cpuNanoCores"),
9396
sql<number>`memory_bytes::float8`.as("memoryBytes"),
9497
sql<number | null>`disk_bytes::float8`.as("diskBytes"),

0 commit comments

Comments
 (0)