Skip to content

Commit 003db7b

Browse files
jdonelsonclaude
andcommitted
console: Fetch each freshness query at its own rate
The binned series refetched every 60 seconds. A bin cannot change faster than its own width, which at a 24 hour range is 24 minutes, so that re-read the whole window to change at most one point. The series and the latest readings are separate queries now, each on its own clock. At 24 hours the series refetches 24 times less often. At 1 hour nothing changes. Hydration took its object IDs from the lag result, so it waited for it: a suspending query stops the component before that line is reached. It takes them from the objects it was given, and the requests go together. The hydration query key is region-scoped now. Object IDs repeat across regions, so a bare key could serve the previous region's counts for a cluster whose IDs happened to match. Also removes a doc block that an earlier edit left duplicated. Co-Authored-By: Claude Opus 5 (1M context) <[email protected]>
1 parent 3e67a5c commit 003db7b

4 files changed

Lines changed: 110 additions & 46 deletions

File tree

‎console/src/api/materialize/freshness/lagHistory.ts‎

Lines changed: 23 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -130,12 +130,9 @@ export function buildLatestLagQuery(objectIds: string[]) {
130130
.orderBy(["object_id", "occurred_at desc"]);
131131
}
132132

133-
/**
134-
* The binned series and the latest readings, in one request.
135-
*
136-
* Sent together because they are one question asked at two resolutions, and
137-
* `executeSqlV2` puts an array of queries in a single round trip.
138-
*/
133+
/** How often a wallclock lag reading lands, which paces the latest-reading read. */
134+
export const LATEST_READING_INTERVAL_MS = 60_000;
135+
139136
export async function fetchObjectLagHistory({
140137
objectIds,
141138
lookbackMs,
@@ -148,10 +145,26 @@ export async function fetchObjectLagHistory({
148145
requestOptions?: RequestInit;
149146
}) {
150147
return executeSqlV2({
151-
queries: [
152-
buildObjectLagHistoryQuery({ objectIds, lookbackMs }).compile(),
153-
buildLatestLagQuery(objectIds).compile(),
154-
] as const,
148+
queries: buildObjectLagHistoryQuery({ objectIds, lookbackMs }).compile(),
149+
queryKey,
150+
requestOptions,
151+
sessionVariables: {
152+
transaction_isolation: "serializable",
153+
},
154+
});
155+
}
156+
157+
export async function fetchLatestLag({
158+
objectIds,
159+
queryKey,
160+
requestOptions,
161+
}: {
162+
objectIds: string[];
163+
queryKey: QueryKey;
164+
requestOptions?: RequestInit;
165+
}) {
166+
return executeSqlV2({
167+
queries: buildLatestLagQuery(objectIds).compile(),
155168
queryKey,
156169
requestOptions,
157170
sessionVariables: {

‎console/src/platform/clusters/queries.ts‎

Lines changed: 69 additions & 30 deletions
Original file line numberDiff line numberDiff line change
@@ -88,7 +88,12 @@ import {
8888
fetchReplicaUtilizationHistory,
8989
ReplicaUtilizationHistoryParameters,
9090
} from "~/api/materialize/cluster/replicaUtilizationHistory";
91-
import { fetchObjectLagHistory } from "~/api/materialize/freshness/lagHistory";
91+
import {
92+
calculateBucketSizeFromLookback,
93+
fetchLatestLag,
94+
fetchObjectLagHistory,
95+
LATEST_READING_INTERVAL_MS,
96+
} from "~/api/materialize/freshness/lagHistory";
9297
import { assertNoMoreThanOneRow } from "~/api/materialize/MoreThanOneRowError";
9398
import { fetchOwners } from "~/api/materialize/owners";
9499
import { useSubscribe } from "~/api/materialize/useSubscribe";
@@ -185,16 +190,28 @@ export const clusterQueryKeys = {
185190
...clusterQueryKeys.all(),
186191
buildQueryKeyPart("replicaOfflineEvents", params),
187192
] as const,
188-
clusterFreshness: (params: { lookbackMs: number; objectIds: string[] }) =>
193+
// Sorted so these keys track set membership, not the order the objects
194+
// arrived in.
195+
clusterFreshnessSeries: (params: {
196+
lookbackMs: number;
197+
objectIds: string[];
198+
}) =>
189199
[
190200
...clusterQueryKeys.all(),
191-
// Sorted so the key tracks set membership, not the order the objects
192-
// arrived in.
193-
buildQueryKeyPart("clusterFreshness", {
201+
buildQueryKeyPart("clusterFreshnessSeries", {
194202
lookbackMs: params.lookbackMs,
195203
objectIds: [...params.objectIds].sort().join(","),
196204
}),
197205
] as const,
206+
// No lookback: the newest reading is the newest reading whatever range is
207+
// being graphed, so changing the range must not refetch it.
208+
clusterFreshnessLatest: (params: { objectIds: string[] }) =>
209+
[
210+
...clusterQueryKeys.all(),
211+
buildQueryKeyPart("clusterFreshnessLatest", {
212+
objectIds: [...params.objectIds].sort().join(","),
213+
}),
214+
] as const,
198215
replicaUtilization: () =>
199216
[
200217
...clusterQueryKeys.all(),
@@ -980,9 +997,10 @@ export interface FreshnessObject {
980997
objectType: string;
981998
}
982999

983-
type LagQueryResults = Awaited<ReturnType<typeof fetchObjectLagHistory>>;
984-
type LagReading = LagQueryResults[0]["rows"][number];
985-
type LatestReading = LagQueryResults[1]["rows"][number];
1000+
type LagReading = Awaited<
1001+
ReturnType<typeof fetchObjectLagHistory>
1002+
>["rows"][number];
1003+
type LatestReading = Awaited<ReturnType<typeof fetchLatestLag>>["rows"][number];
9861004

9871005
export interface LagQueryRows {
9881006
/** One binned point per object per bin, each the worst reading in its span. */
@@ -1094,14 +1112,6 @@ export function buildFreshnessData(
10941112
};
10951113
}
10961114

1097-
/**
1098-
* Lag readings for a known set of objects, shaped for the freshness graph.
1099-
*
1100-
* The caller supplies the objects rather than naming a cluster, because the
1101-
* `useAllObjects` subscribe already holds every name, schema, database and
1102-
* type. Resolving those in SQL cost three joins and three full scans per
1103-
* request; here they are a map lookup.
1104-
*/
11051115
/**
11061116
* Lag readings for a known set of objects, shaped for the freshness graph.
11071117
*
@@ -1119,32 +1129,61 @@ export function useClusterFreshness({
11191129
[objects],
11201130
);
11211131

1122-
const query = useSuspenseQuery({
1123-
queryKey: clusterQueryKeys.clusterFreshness({ lookbackMs, objectIds }),
1124-
queryFn: async ({ queryKey, signal }): Promise<LagQueryRows> => {
1125-
if (objectIds.length === 0) return { readings: [], latest: [] };
1132+
// The binned series and the latest readings go in separate requests because
1133+
// they go stale at different rates. A bin cannot change faster than its own
1134+
// width, which at a 24 hour range is 24 minutes, so refetching the series on
1135+
// the one minute clock would re-read the whole window to change at most one
1136+
// point.
1137+
const binSizeMs = calculateBucketSizeFromLookback(lookbackMs);
1138+
1139+
const series = useSuspenseQuery({
1140+
queryKey: clusterQueryKeys.clusterFreshnessSeries({
1141+
lookbackMs,
1142+
objectIds,
1143+
}),
1144+
queryFn: async ({ queryKey, signal }): Promise<LagReading[]> => {
1145+
if (objectIds.length === 0) return [];
11261146

1127-
const [series, latest] = await fetchObjectLagHistory({
1147+
const { rows } = await fetchObjectLagHistory({
11281148
objectIds,
11291149
lookbackMs,
11301150
requestOptions: { signal },
11311151
queryKey,
11321152
});
1133-
return { readings: series.rows, latest: latest.rows };
1153+
return rows;
11341154
},
1135-
// A reading lands once a minute, so a refetch inside that window re-asks a
1136-
// question whose answer cannot have changed, and an interval any longer
1137-
// than it leaves an open page behind the data it is describing.
1138-
staleTime: 60_000,
1139-
refetchInterval: 60_000,
1155+
staleTime: binSizeMs,
1156+
refetchInterval: binSizeMs,
1157+
});
1158+
1159+
const latest = useSuspenseQuery({
1160+
queryKey: clusterQueryKeys.clusterFreshnessLatest({ objectIds }),
1161+
queryFn: async ({ queryKey, signal }): Promise<LatestReading[]> => {
1162+
if (objectIds.length === 0) return [];
1163+
1164+
const { rows } = await fetchLatestLag({
1165+
objectIds,
1166+
requestOptions: { signal },
1167+
queryKey,
1168+
});
1169+
return rows;
1170+
},
1171+
// A reading lands once a minute, so anything shorter re-asks a question
1172+
// whose answer cannot have changed.
1173+
staleTime: LATEST_READING_INTERVAL_MS,
1174+
refetchInterval: LATEST_READING_INTERVAL_MS,
11401175
});
11411176

1142-
// Only `data` is returned. Spreading the query object would make every
1177+
// Only `data` is returned. Spreading a query object would make every
11431178
// consumer observe all of its state, and suspense already covers loading and
11441179
// errors for these callers.
11451180
const data = useMemo(
1146-
() => buildFreshnessData(query.data, objects),
1147-
[query.data, objects],
1181+
() =>
1182+
buildFreshnessData(
1183+
{ readings: series.data, latest: latest.data },
1184+
objects,
1185+
),
1186+
[series.data, latest.data, objects],
11481187
);
11491188

11501189
return { data };

‎console/src/platform/freshness/FreshnessPage.tsx‎

Lines changed: 6 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -119,11 +119,13 @@ const FreshnessContent = ({
119119

120120
// Hydration is its own query: `buildLagHistoryQuery` is shared with pages
121121
// that never show it, and joined there it cost all of them a scan.
122-
const objectIds = React.useMemo(
123-
() => Array.from(objectsById.keys()),
124-
[objectsById],
122+
//
123+
// Its IDs come from `objects` rather than from the lag result, so the two
124+
// requests go together. Taking them from the result would make this one wait,
125+
// since a suspending query stops the component before this line is reached.
126+
const { data: hydrationByObjectId } = useFreshnessHydration(
127+
objects.map((object) => object.objectId),
125128
);
126-
const { data: hydrationByObjectId } = useFreshnessHydration(objectIds);
127129

128130
// Picked by hand in the All objects table, on top of whatever breaches.
129131
const [selectedKeys, setSelectedKeys] = React.useState<ReadonlySet<string>>(

‎console/src/platform/freshness/queries.ts‎

Lines changed: 12 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,10 @@
99

1010
import { useSuspenseQuery } from "@tanstack/react-query";
1111

12-
import { buildQueryKeyPart } from "~/api/buildQueryKeySchema";
12+
import {
13+
buildQueryKeyPart,
14+
buildRegionQueryKey,
15+
} from "~/api/buildQueryKeySchema";
1316
import { fetchHydrationCounts } from "~/api/materialize/freshness/hydrationCounts";
1417

1518
export interface HydrationCounts {
@@ -18,7 +21,10 @@ export interface HydrationCounts {
1821
}
1922

2023
export const freshnessQueryKeys = {
21-
all: () => ["freshness"] as const,
24+
// Region-scoped, because object IDs repeat across regions: `u1` exists in
25+
// every one. A bare key would let a region switch serve the previous
26+
// region's counts for a cluster whose IDs happen to match.
27+
all: () => buildRegionQueryKey("freshness"),
2228
hydrationCounts: (objectIds: string[]) =>
2329
[
2430
...freshnessQueryKeys.all(),
@@ -35,6 +41,10 @@ export const freshnessQueryKeys = {
3541
*
3642
* Its own query rather than a column on the lag history, because that builder
3743
* is shared with pages that never show hydration.
44+
*
45+
* The caller passes the objects it means to show, not the ones the lag query
46+
* returned. Reading them from that result would make this request wait for it,
47+
* since a suspending query stops the component before this line is reached.
3848
*/
3949
export function useFreshnessHydration(objectIds: string[]) {
4050
return useSuspenseQuery({

0 commit comments

Comments
 (0)