Skip to content

Commit d69a1ea

Browse files
jdonelsonclaude
andcommitted
console: Move freshness hydration counts out of the shared lag query
buildLagHistoryQuery is shared with Environment Overview, Cluster Overview and the object panel's poll. None of them read a hydration column, but the join added a full scan and reduce of mz_hydration_statuses to all of them. The Freshness page now fetches hydration separately, keyed by the objects it is already showing. Scoping by object ID does not make that read a lookup: mz_hydration_statuses_ind is keyed on (object_id, replica_id), and a lookup needs every key column. Verified on v26.45.0-dev with an IN list and with an unnested array, at 2 and at 1000 IDs. The win is that only this page pays the scan. Co-Authored-By: Claude Opus 5 (1M context) <[email protected]>
1 parent 51eaea3 commit d69a1ea

7 files changed

Lines changed: 177 additions & 41 deletions

File tree

Lines changed: 57 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,57 @@
1+
// Copyright Materialize, Inc. and contributors. All rights reserved.
2+
//
3+
// Use of this software is governed by the Business Source License
4+
// included in the LICENSE file.
5+
//
6+
// As of the Change Date specified in that file, in accordance with
7+
// the Business Source License, use of this software will be governed
8+
// by the Apache License, Version 2.0.
9+
10+
import { QueryKey } from "@tanstack/react-query";
11+
import { sql } from "kysely";
12+
13+
import { executeSqlV2, queryBuilder } from "~/api/materialize";
14+
15+
/**
16+
* Replica hydration counts for a known set of objects.
17+
*
18+
* Kept out of `buildLagHistoryQuery` because that builder is shared with
19+
* Environment Overview, Cluster Overview and the object panel's poll, none of
20+
* which read a hydration column. Joined there, it cost all of them a full scan
21+
* and reduce of `mz_hydration_statuses`.
22+
*
23+
* NOTE: scoping by object ID does not turn the read into an index lookup.
24+
* `mz_hydration_statuses_ind` is keyed on `(object_id, replica_id)`, and a
25+
* lookup needs every key column, so constraining `object_id` alone still plans
26+
* as a full scan. Verified on v26.45.0-dev with both an `IN` list and an
27+
* `unnest`ed array, at 2 and at 1000 IDs. The scoping is still worth keeping
28+
* for the rows it discards before the reduce, but the win here is that only
29+
* this page pays the scan.
30+
*/
31+
export function buildHydrationCountsQuery(objectIds: string[]) {
32+
return queryBuilder
33+
.selectFrom("mz_hydration_statuses as hs")
34+
.where("hs.object_id", "in", objectIds)
35+
.select(({ fn }) => [
36+
"hs.object_id as objectId",
37+
fn.countAll<bigint>().as("totalReplicas"),
38+
sql<bigint>`count(*) FILTER (WHERE hs.hydrated)`.as("hydratedReplicas"),
39+
])
40+
.groupBy("hs.object_id");
41+
}
42+
43+
export async function fetchHydrationCounts({
44+
objectIds,
45+
queryKey,
46+
requestOptions,
47+
}: {
48+
objectIds: string[];
49+
queryKey: QueryKey;
50+
requestOptions?: RequestInit;
51+
}) {
52+
return executeSqlV2({
53+
queries: buildHydrationCountsQuery(objectIds).compile(),
54+
queryKey,
55+
requestOptions,
56+
});
57+
}

‎console/src/api/materialize/freshness/lagHistory.ts‎

Lines changed: 0 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -150,24 +150,6 @@ export function buildLagHistoryQuery(
150150
"mz_object_fully_qualified_names as object_names",
151151
"lag_history.object_id",
152152
"object_names.id",
153-
)
154-
// Aggregated before the join on purpose. `mz_hydration_statuses` has a
155-
// row per (object, replica), so joining it directly would multiply every
156-
// bucket of the time series by the replica count.
157-
.leftJoin(
158-
cte
159-
.selectFrom("mz_hydration_statuses")
160-
.select(({ fn }) => [
161-
"object_id",
162-
fn.countAll<bigint>().as("total_replicas"),
163-
sql<bigint>`count(*) FILTER (WHERE hydrated)`.as(
164-
"hydrated_replicas",
165-
),
166-
])
167-
.groupBy("object_id")
168-
.as("hs"),
169-
"hs.object_id",
170-
"lag_history.object_id",
171153
);
172154

173155
// orderBy and distinctOn inherently gives us the latest max lag per ID since it takes the first row by group
@@ -212,8 +194,6 @@ export function buildLagHistoryQuery(
212194
"object_names.schema_name as schemaName",
213195
"object_names.name as objectName",
214196
"objects.type as objectType",
215-
"hs.hydrated_replicas as hydratedReplicas",
216-
"hs.total_replicas as totalReplicas",
217197
]);
218198
})
219199
.selectFrom("lag_history");

‎console/src/platform/clusters/queries.ts‎

Lines changed: 1 addition & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -896,15 +896,13 @@ export function useReplicaUtilizationHistory(
896896

897897
export const LINE_MAX_COUNT = 10;
898898

899-
/** An object's identity and hydration, invariant across the window. */
899+
/** An object's identity, invariant across the window. */
900900
export interface FreshnessObject {
901901
objectId: string;
902902
objectName: string | null;
903903
schemaName: string | null;
904904
databaseName: string | null;
905905
objectType: string;
906-
hydratedReplicas: number;
907-
totalReplicas: number;
908906
}
909907

910908
export function useClusterFreshness({
@@ -1016,8 +1014,6 @@ export function useClusterFreshness({
10161014
schemaName: row.schemaName,
10171015
databaseName: row.databaseName,
10181016
objectType: row.objectType,
1019-
hydratedReplicas: Number(row.hydratedReplicas ?? 0),
1020-
totalReplicas: Number(row.totalReplicas ?? 0),
10211017
});
10221018
});
10231019

‎console/src/platform/freshness/FreshnessPage.tsx‎

Lines changed: 21 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -56,6 +56,7 @@ import {
5656
PREDICATE_LABELS,
5757
} from "./freshnessRows";
5858
import { FreshnessTable } from "./FreshnessTable";
59+
import { useFreshnessHydration } from "./queries";
5960
import { useFreshnessParams } from "./useFreshnessParams";
6061
import { useSettledThreshold } from "./useSettledThreshold";
6162

@@ -110,6 +111,14 @@ const FreshnessContent = ({
110111
data: { historicalData, startTime, endTime, lines, objectsById },
111112
} = useClusterFreshness({ lookbackMs, clusterId });
112113

114+
// Hydration is its own query: `buildLagHistoryQuery` is shared with pages
115+
// that never show it, and joined there it cost all of them a scan.
116+
const objectIds = React.useMemo(
117+
() => Array.from(objectsById.keys()),
118+
[objectsById],
119+
);
120+
const { data: hydrationByObjectId } = useFreshnessHydration(objectIds);
121+
113122
// Picked by hand in the All objects table, on top of whatever breaches.
114123
const [selectedKeys, setSelectedKeys] = React.useState<ReadonlySet<string>>(
115124
new Set(),
@@ -147,14 +156,22 @@ const FreshnessContent = ({
147156
// object on every pointer move.
148157
const rows = React.useMemo(
149158
() =>
150-
buildFreshnessRows(
159+
buildFreshnessRows({
151160
judged,
152161
statsByKey,
153162
objectsById,
154-
settledThreshold,
163+
hydrationByObjectId,
164+
threshold: settledThreshold,
155165
selectedKeys,
156-
),
157-
[judged, statsByKey, objectsById, settledThreshold, selectedKeys],
166+
}),
167+
[
168+
judged,
169+
statsByKey,
170+
objectsById,
171+
hydrationByObjectId,
172+
settledThreshold,
173+
selectedKeys,
174+
],
158175
);
159176

160177
const breaching = rows.filter((row) => row.breaching);

‎console/src/platform/freshness/freshnessRows.test.ts‎

Lines changed: 16 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,7 @@ import {
2121
Predicate,
2222
sortRows,
2323
} from "./freshnessRows";
24+
import { HydrationCounts } from "./queries";
2425

2526
/**
2627
* A reading as the page receives it: a number is a measured lag in
@@ -86,20 +87,32 @@ const objectsById = new Map<string, FreshnessObject>(
8687
schemaName: "public",
8788
databaseName: "materialize",
8889
objectType: key === "spiky" ? "index" : "materialized-view",
89-
hydratedReplicas: key === "absent" ? 0 : 2,
90-
totalReplicas: 2,
9190
},
9291
]),
9392
);
9493

94+
const hydrationByObjectId = new Map<string, HydrationCounts>(
95+
Object.keys(SERIES).map((key) => [
96+
key,
97+
{ hydratedReplicas: key === "absent" ? 0 : 2, totalReplicas: 2 },
98+
]),
99+
);
100+
95101
const rowsFor = (
96102
predicate: Predicate,
97103
threshold: number,
98104
selected: ReadonlySet<string> = new Set(),
99105
) => {
100106
const stats = buildStats(lines, data);
101107
const judged = judgeLines(lines, data, predicate, stats);
102-
return buildFreshnessRows(judged, stats, objectsById, threshold, selected);
108+
return buildFreshnessRows({
109+
judged,
110+
statsByKey: stats,
111+
objectsById,
112+
hydrationByObjectId,
113+
threshold,
114+
selectedKeys: selected,
115+
});
103116
};
104117

105118
describe("computeStats", () => {

‎console/src/platform/freshness/freshnessRows.ts‎

Lines changed: 20 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,8 @@ import { assignLineColors } from "~/components/ThresholdLineGraph/thresholdLineG
1212
import { ThresholdLineSeries } from "~/components/ThresholdLineGraph/types";
1313
import { FreshnessObject } from "~/platform/clusters/queries";
1414

15+
import { HydrationCounts } from "./queries";
16+
1517
/** Which statistic the threshold judges an object by. */
1618
export type Predicate = "current" | "peak" | "p90";
1719

@@ -153,13 +155,21 @@ export function buildStats(
153155
* threshold chose. Replacement is what would make the threshold and the
154156
* selection fight over the same channel.
155157
*/
156-
export function buildFreshnessRows(
157-
judged: ThresholdLineSeries<DataPoint>[],
158-
statsByKey: Map<string, ObjectStats>,
159-
objectsById: Map<string, FreshnessObject>,
160-
threshold: number,
161-
selectedKeys: ReadonlySet<string>,
162-
): FreshnessRow[] {
158+
export function buildFreshnessRows({
159+
judged,
160+
statsByKey,
161+
objectsById,
162+
hydrationByObjectId,
163+
threshold,
164+
selectedKeys,
165+
}: {
166+
judged: ThresholdLineSeries<DataPoint>[];
167+
statsByKey: Map<string, ObjectStats>;
168+
objectsById: Map<string, FreshnessObject>;
169+
hydrationByObjectId: Map<string, HydrationCounts>;
170+
threshold: number;
171+
selectedKeys: ReadonlySet<string>;
172+
}): FreshnessRow[] {
163173
const colors = assignLineColors(judged);
164174

165175
return judged
@@ -171,6 +181,7 @@ export function buildFreshnessRows(
171181
notQueryable: false,
172182
};
173183
const object = objectsById.get(line.key);
184+
const hydration = hydrationByObjectId.get(line.key);
174185
const breaching =
175186
line.breachValue !== null && line.breachValue > threshold;
176187
const drawn = breaching || selectedKeys.has(line.key);
@@ -182,8 +193,8 @@ export function buildFreshnessRows(
182193
.filter(Boolean)
183194
.join("."),
184195
objectType: object?.objectType ?? "",
185-
hydratedReplicas: object?.hydratedReplicas ?? 0,
186-
totalReplicas: object?.totalReplicas ?? 0,
196+
hydratedReplicas: hydration?.hydratedReplicas ?? 0,
197+
totalReplicas: hydration?.totalReplicas ?? 0,
187198
current: stats.current,
188199
peak: stats.peak,
189200
p90: stats.p90,
Lines changed: 62 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,62 @@
1+
// Copyright Materialize, Inc. and contributors. All rights reserved.
2+
//
3+
// Use of this software is governed by the Business Source License
4+
// included in the LICENSE file.
5+
//
6+
// As of the Change Date specified in that file, in accordance with
7+
// the Business Source License, use of this software will be governed
8+
// by the Apache License, Version 2.0.
9+
10+
import { useSuspenseQuery } from "@tanstack/react-query";
11+
12+
import { buildQueryKeyPart } from "~/api/buildQueryKeySchema";
13+
import { fetchHydrationCounts } from "~/api/materialize/freshness/hydrationCounts";
14+
15+
export interface HydrationCounts {
16+
hydratedReplicas: number;
17+
totalReplicas: number;
18+
}
19+
20+
export const freshnessQueryKeys = {
21+
all: () => ["freshness"] as const,
22+
hydrationCounts: (objectIds: string[]) =>
23+
[
24+
...freshnessQueryKeys.all(),
25+
// Sorted so the key tracks set membership rather than the order the
26+
// objects happened to arrive in.
27+
buildQueryKeyPart("hydrationCounts", {
28+
objectIds: [...objectIds].sort().join(","),
29+
}),
30+
] as const,
31+
};
32+
33+
/**
34+
* Replica hydration for the objects on screen, keyed by object ID.
35+
*
36+
* Its own query rather than a column on the lag history, because that builder
37+
* is shared with pages that never show hydration.
38+
*/
39+
export function useFreshnessHydration(objectIds: string[]) {
40+
return useSuspenseQuery({
41+
queryKey: freshnessQueryKeys.hydrationCounts(objectIds),
42+
queryFn: async ({ queryKey, signal }) => {
43+
if (objectIds.length === 0) return new Map<string, HydrationCounts>();
44+
45+
const { rows } = await fetchHydrationCounts({
46+
objectIds,
47+
queryKey,
48+
requestOptions: { signal },
49+
});
50+
51+
return new Map<string, HydrationCounts>(
52+
rows.map((row) => [
53+
row.objectId,
54+
{
55+
hydratedReplicas: Number(row.hydratedReplicas),
56+
totalReplicas: Number(row.totalReplicas),
57+
},
58+
]),
59+
);
60+
},
61+
});
62+
}

0 commit comments

Comments
 (0)