Skip to content

Commit 5deb050

Browse files
committed
console: count dropped objects toward a hydration a restart cut short
A restart is unfinished when the replica was hydrating an object created during the attempt. The creation times came only from the cluster's current objects, so once the object that caused an out-of-memory restart was dropped, the restart read as finished. The tab now also reads the cluster's indexes and materialized views created over the window and dropped since, from `mz_object_history`. That relation is indexed only by id, so the read scans every object the environment has created: 8.5 ms at 32,000 objects on v26.47, plus about 0.23 ms per 1,000. It runs only while a restart in the window reads as finished with the current objects, and again only when those restarts change. `findRestarts` is split out of `findUnfinishedHydrations` so the hook can tell which restarts read as finished.
1 parent 79e79a4 commit 5deb050

7 files changed

Lines changed: 427 additions & 46 deletions

File tree

Lines changed: 73 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,73 @@
1+
// Copyright Materialize, Inc. and contributors. All rights reserved.
2+
//
3+
// Use of this software is governed by the Business Source License
4+
// included in the LICENSE file.
5+
//
6+
// As of the Change Date specified in that file, in accordance with
7+
// the Business Source License, use of this software will be governed
8+
// by the Apache License, Version 2.0.
9+
10+
import {
11+
executeSqlHttp,
12+
getMaterializeClient,
13+
QUICKSTART_CLUSTER,
14+
} from "~/test/sql/materializeSqlClient";
15+
import { testdrive } from "~/test/sql/mzcompose";
16+
17+
import { buildDroppedObjectCreationTimesQuery } from "./objectCreationTimes";
18+
19+
describe("buildDroppedObjectCreationTimesQuery", () => {
20+
it("returns the cluster's indexes and materialized views created since the start and dropped since", async () => {
21+
// Reset Materialize state.
22+
await testdrive(`
23+
> CREATE TABLE materialize.public.t (a int);
24+
> CREATE INDEX before_start_idx IN CLUSTER quickstart ON materialize.public.t (a);
25+
`);
26+
const client = await getMaterializeClient();
27+
const {
28+
rows: [{ start_date: startDate, cluster_id: clusterId }],
29+
} = await client.query(
30+
"SELECT now() AS start_date, id AS cluster_id FROM mz_clusters WHERE name = 'quickstart'",
31+
);
32+
for (const statement of [
33+
"CREATE INDEX dropped_idx IN CLUSTER quickstart ON materialize.public.t (a)",
34+
"CREATE MATERIALIZED VIEW materialize.public.dropped_mv IN CLUSTER quickstart AS SELECT a FROM materialize.public.t",
35+
"CREATE INDEX kept_idx IN CLUSTER quickstart ON materialize.public.t (a)",
36+
"CREATE TABLE materialize.public.dropped_t (a int)",
37+
]) {
38+
await client.query(statement);
39+
}
40+
const { rows: objects } = await client.query(
41+
"SELECT id, name FROM mz_objects WHERE name IN ('dropped_idx', 'dropped_mv')",
42+
);
43+
for (const statement of [
44+
"DROP INDEX materialize.public.before_start_idx",
45+
"DROP INDEX materialize.public.dropped_idx",
46+
"DROP MATERIALIZED VIEW materialize.public.dropped_mv",
47+
"DROP TABLE materialize.public.dropped_t",
48+
]) {
49+
await client.query(statement);
50+
}
51+
52+
const { rows } = await executeSqlHttp(
53+
buildDroppedObjectCreationTimesQuery({
54+
clusterId,
55+
objectTypes: ["index", "materialized-view"],
56+
startDate: (startDate as Date).toISOString(),
57+
}).compile(),
58+
{ sessionVariables: { cluster: QUICKSTART_CLUSTER } },
59+
);
60+
61+
expect(rows.map(({ id, objectType }) => [id, objectType]).sort()).toEqual(
62+
objects
63+
.map(({ id, name }) => [
64+
id,
65+
name === "dropped_idx" ? "index" : "materialized-view",
66+
])
67+
.sort(),
68+
);
69+
expect(rows.every(({ occurredAt }) => occurredAt instanceof Date)).toBe(
70+
true,
71+
);
72+
});
73+
});

‎console/src/api/materialize/cluster/objectCreationTimes.ts‎

Lines changed: 51 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -47,6 +47,57 @@ export function buildObjectCreationTimesQuery({
4747
return qb;
4848
}
4949

50+
/**
51+
* When each object of `objectTypes` on `clusterId` that was created at or
52+
* after `startDate`, and dropped since, was created.
53+
*
54+
* NOTE: `mz_object_history` is indexed only by id, so filtering by cluster
55+
* scans every object the environment has created. Measured on v26.47: 8.5 ms
56+
* at 32,000 objects, plus about 0.23 ms per 1,000. Looking up ids instead
57+
* costs about 60 µs an id, so it pays off only in much larger environments.
58+
*/
59+
export function buildDroppedObjectCreationTimesQuery({
60+
clusterId,
61+
objectTypes,
62+
startDate,
63+
}: {
64+
clusterId: string;
65+
objectTypes: string[];
66+
startDate: string;
67+
}) {
68+
return queryBuilder
69+
.selectFrom("mz_object_history")
70+
.select([
71+
"id",
72+
"object_type as objectType",
73+
// Never null here: the filter on it leaves out built-in objects, which
74+
// have no creation time.
75+
sql<Date>`created_at`.as("occurredAt"),
76+
])
77+
.where("cluster_id", "=", clusterId)
78+
.where("object_type", "in", objectTypes)
79+
.where("dropped_at", "is not", null)
80+
.where("created_at", ">=", sql<Date>`${sql.lit(startDate)}::timestamptz`);
81+
}
82+
83+
export async function fetchDroppedObjectCreationTimes({
84+
params,
85+
queryKey,
86+
requestOptions,
87+
}: {
88+
params: Parameters<typeof buildDroppedObjectCreationTimesQuery>[0];
89+
queryKey: QueryKey;
90+
requestOptions?: RequestInit;
91+
}) {
92+
const res = await executeSqlV2({
93+
queries: buildDroppedObjectCreationTimesQuery(params).compile(),
94+
queryKey,
95+
requestOptions,
96+
sessionVariables: { transaction_isolation: "serializable" },
97+
});
98+
return res.rows;
99+
}
100+
50101
export async function fetchObjectCreationTimes({
51102
params,
52103
queryKey,

‎console/src/platform/clusters/ClusterPage/Hydration/hydrationModel.test.ts‎

Lines changed: 55 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@
88
// by the Apache License, Version 2.0.
99

1010
import {
11+
findRestarts,
1112
findUnfinishedHydrations,
1213
heapLimitBytes,
1314
Hydration,
@@ -16,6 +17,7 @@ import {
1617
mergeHydrationReplicas,
1718
replicaBootTimes,
1819
ReplicaTransition,
20+
restartsReadingAsFinished,
1921
transformHydrations,
2022
transformObjectHydrationSpans,
2123
} from "./hydrationModel";
@@ -239,20 +241,26 @@ describe("findUnfinishedHydrations", () => {
239241
replicaList?: HydrationReplica[];
240242
objectCreatedAtMs?: number[];
241243
startMs?: number;
242-
}) =>
243-
findUnfinishedHydrations({
244-
transitions,
245-
oomKills,
244+
}) => {
245+
const bootTimes = replicaBootTimes(transitions, replicaList);
246+
return findUnfinishedHydrations({
247+
restarts: findRestarts({
248+
transitions,
249+
oomKills,
250+
replicas: replicaList,
251+
bootTimes,
252+
}),
246253
hydrations,
247254
replicas: replicaList,
248-
bootTimes: replicaBootTimes(transitions, replicaList),
255+
bootTimes,
249256
objectCreatedAtMs,
250257
startMs,
251258
}).map(({ startedAt, endedAt, reason }) => [
252259
startedAt?.toISOString(),
253260
endedAt.toISOString(),
254261
reason,
255262
]);
263+
};
256264

257265
it("reports each out-of-memory restart of a replica that never finished hydrating", () => {
258266
const transitions = [
@@ -430,6 +438,48 @@ describe("findUnfinishedHydrations", () => {
430438
]);
431439
});
432440

441+
it("names the restarts an object missing from the creation times could have cut short", () => {
442+
const transitions = [
443+
transition("u1", "10:00:00", "online"),
444+
transition("u1", "10:12:00", "offline", "oom-killed"),
445+
transition("u1", "10:13:00", "online"),
446+
transition("u1", "10:20:00", "offline", "oom-killed"),
447+
];
448+
const bootTimes = replicaBootTimes(transitions, replicas);
449+
const hydrations = transformHydrations(
450+
[episode("u1", "10:00:01", "10:02:00")],
451+
bootTimes,
452+
);
453+
const restarts = findRestarts({
454+
transitions,
455+
oomKills: [],
456+
replicas,
457+
bootTimes,
458+
});
459+
const readingAsFinished = (startMs: number) =>
460+
restartsReadingAsFinished(
461+
restarts,
462+
findUnfinishedHydrations({
463+
restarts,
464+
hydrations,
465+
replicas,
466+
bootTimes,
467+
objectCreatedAtMs: [],
468+
startMs,
469+
}),
470+
startMs,
471+
).map(({ occurredAt }) => occurredAt.toISOString());
472+
473+
// The first came after the replica finished. The second never finished.
474+
expect(readingAsFinished(at("00:00:00").getTime())).toEqual([
475+
"2026-10-20T10:12:00.000Z",
476+
]);
477+
// With the window from 10:15, the second's attempt began before it.
478+
expect(readingAsFinished(at("10:15:00").getTime())).toEqual([
479+
"2026-10-20T10:20:00.000Z",
480+
]);
481+
});
482+
433483
it("skips a replica going offline when it's dropped, and restarts before the window", () => {
434484
const transitions = [
435485
transition("u1", "08:00:00", "online"),

‎console/src/platform/clusters/ClusterPage/Hydration/hydrationModel.ts‎

Lines changed: 75 additions & 34 deletions
Original file line numberDiff line numberDiff line change
@@ -181,41 +181,22 @@ export function transformHydrations(
181181

182182
/**
183183
* Times a replica went offline, or a process was killed for running out of
184-
* memory, before the replica finished hydrating, from `startMs` on. A replica
185-
* going offline when it's dropped doesn't count.
186-
*
187-
* An attempt runs from the replica's previous restart, or its creation, to the
188-
* restart. Online rows don't bound it: the one a zero-downtime upgrade writes
189-
* comes after the replica already hydrated. An object created during the
190-
* attempt starts a hydration of its own, so the attempt finished only if a
191-
* hydration finished after the last such object. The hydrations before
192-
* `startMs` aren't read, so an attempt that began before it is skipped unless
193-
* such an object was created after it.
194-
*
195-
* NOTE: the history can miss an episode that finished, so a restart after
196-
* one can read as unfinished.
184+
* memory, oldest first. A replica going offline when it's dropped doesn't
185+
* count.
197186
*/
198-
export function findUnfinishedHydrations({
187+
export function findRestarts({
199188
transitions,
200189
oomKills,
201-
hydrations,
202190
replicas,
203191
bootTimes,
204-
objectCreatedAtMs,
205-
startMs,
206192
}: {
207193
/** Process 0's transitions, which stand for the replica's. */
208194
transitions: ReplicaTransition[];
209195
/** OOM kills of any process. */
210196
oomKills: ReplicaTransition[];
211-
/** Every finished hydration, including ones with no objects. */
212-
hydrations: Hydration[];
213197
replicas: HydrationReplica[];
214198
bootTimes: Map<string, number[]>;
215-
/** When the cluster's objects were created, for those created in the window. */
216-
objectCreatedAtMs: number[];
217-
startMs: number;
218-
}): UnfinishedHydration[] {
199+
}): ReplicaTransition[] {
219200
const replicasById = new Map(
220201
replicas.map((replica) => [replica.id, replica]),
221202
);
@@ -265,22 +246,64 @@ export function findUnfinishedHydrations({
265246
}
266247
restarts.push({ ...end });
267248
}
249+
return restarts.filter(({ replicaId, occurredAt, reason }) => {
250+
const droppedMs = replicasById.get(replicaId)?.droppedAt?.getTime();
251+
return (
252+
reason === OOM_REASON ||
253+
droppedMs === undefined ||
254+
occurredAt.getTime() < droppedMs - RESTART_MERGE_MS
255+
);
256+
});
257+
}
258+
259+
/** Identifies a restart, and the unfinished hydration it ended if any. */
260+
const restartId = ({ replicaId, occurredAt }: ReplicaTransition) =>
261+
`${replicaId}|${occurredAt.getTime()}`;
268262

269-
return restarts.flatMap(({ replicaId, occurredAt, reason }, index) => {
263+
/**
264+
* The restarts from `startMs` on that came before the replica finished
265+
* hydrating.
266+
*
267+
* An attempt runs from the replica's previous restart, or its creation, to the
268+
* restart. Online rows don't bound it: the one a zero-downtime upgrade writes
269+
* comes after the replica already hydrated. An object created during the
270+
* attempt starts a hydration of its own, so the attempt finished only if a
271+
* hydration finished after the last such object. The hydrations before
272+
* `startMs` aren't read, so an attempt that began before it is skipped unless
273+
* such an object was created after it.
274+
*
275+
* NOTE: the history can miss an episode that finished, so a restart after
276+
* one can read as unfinished.
277+
*/
278+
export function findUnfinishedHydrations({
279+
restarts,
280+
hydrations,
281+
replicas,
282+
bootTimes,
283+
objectCreatedAtMs,
284+
startMs,
285+
}: {
286+
/** Every restart of the replicas, as `findRestarts` returns them. */
287+
restarts: ReplicaTransition[];
288+
/** Every finished hydration, including ones with no objects. */
289+
hydrations: Hydration[];
290+
replicas: HydrationReplica[];
291+
bootTimes: Map<string, number[]>;
292+
/** When the cluster's objects were created, for those created in the window. */
293+
objectCreatedAtMs: number[];
294+
startMs: number;
295+
}): UnfinishedHydration[] {
296+
const replicasById = new Map(
297+
replicas.map((replica) => [replica.id, replica]),
298+
);
299+
return restarts.flatMap((restart, index) => {
300+
const { replicaId, occurredAt, reason } = restart;
270301
const endMs = occurredAt.getTime();
271302
if (endMs < startMs) return [];
272303
const replica = replicasById.get(replicaId);
273-
const droppedMs = replica?.droppedAt?.getTime();
274-
if (
275-
reason !== OOM_REASON &&
276-
droppedMs !== undefined &&
277-
endMs >= droppedMs - RESTART_MERGE_MS
278-
) {
279-
return [];
280-
}
281304
const previousRestartMs = restarts
282305
.slice(0, index)
283-
.findLast((restart) => restart.replicaId === replicaId)
306+
.findLast((previous) => previous.replicaId === replicaId)
284307
?.occurredAt.getTime();
285308
const attemptStartMs =
286309
previousRestartMs ?? replica?.createdAt?.getTime() ?? -Infinity;
@@ -309,7 +332,7 @@ export function findUnfinishedHydrations({
309332
) ?? attemptStartMs);
310333
return [
311334
{
312-
id: `${replicaId}|${endMs}`,
335+
id: restartId(restart),
313336
replicaId,
314337
startedAt: new Date(hydratingSinceMs),
315338
endedAt: occurredAt,
@@ -319,6 +342,24 @@ export function findUnfinishedHydrations({
319342
});
320343
}
321344

345+
/**
346+
* The restarts from `startMs` on that read as finished. One can still have
347+
* cut a hydration short if the replica was hydrating an object missing from
348+
* the creation times, such as one dropped since.
349+
*/
350+
export function restartsReadingAsFinished(
351+
restarts: ReplicaTransition[],
352+
unfinished: UnfinishedHydration[],
353+
startMs: number,
354+
) {
355+
const unfinishedIds = new Set(unfinished.map(({ id }) => id));
356+
return restarts.filter(
357+
(restart) =>
358+
restart.occurredAt.getTime() >= startMs &&
359+
!unfinishedIds.has(restartId(restart)),
360+
);
361+
}
362+
322363
/** When a replica last booted: its creation or the last time it came online. */
323364
export const latestBoot = (
324365
bootTimes: Map<string, number[]>,

0 commit comments

Comments
 (0)