Skip to content

Commit 319b940

Browse files
jdonelsonclaude
andcommitted
console: Fetch the latest freshness readings on the page only
Co-Authored-By: Claude Opus 5.5 <[email protected]>
1 parent 39fb494 commit 319b940

5 files changed

Lines changed: 105 additions & 86 deletions

File tree

‎console/src/platform/clusters/queries.test.ts‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -258,7 +258,7 @@ describe("buildFreshnessData", () => {
258258
];
259259

260260
it("runs time forwards whatever order the rows arrive in", () => {
261-
const data = buildFreshnessData({ readings, latest: [] }, objects);
261+
const data = buildFreshnessData(readings, objects);
262262

263263
expect(data.startTime).toBe(60_000);
264264
expect(data.endTime).toBe(180_000);
@@ -268,7 +268,7 @@ describe("buildFreshnessData", () => {
268268
});
269269

270270
it("reports each object's newest bin as its current lag", () => {
271-
const data = buildFreshnessData({ readings, latest: [] }, objects);
271+
const data = buildFreshnessData(readings, objects);
272272

273273
expect(data.currentData[0].lag).toEqual(parse("00:00:01"));
274274
});

‎console/src/platform/clusters/queries.ts‎

Lines changed: 7 additions & 60 deletions
Original file line numberDiff line numberDiff line change
@@ -90,9 +90,7 @@ import {
9090
} from "~/api/materialize/cluster/replicaUtilizationHistory";
9191
import {
9292
calculateBucketSizeFromLookback,
93-
fetchLatestLag,
9493
fetchObjectLagHistory,
95-
LATEST_READING_INTERVAL_MS,
9694
} from "~/api/materialize/freshness/lagHistory";
9795
import { assertNoMoreThanOneRow } from "~/api/materialize/MoreThanOneRowError";
9896
import { fetchOwners } from "~/api/materialize/owners";
@@ -203,15 +201,6 @@ export const clusterQueryKeys = {
203201
objectIds: [...params.objectIds].sort().join(","),
204202
}),
205203
] as const,
206-
// No lookback: the newest reading is the newest reading whatever range is
207-
// being graphed, so changing the range must not refetch it.
208-
clusterFreshnessLatest: (params: { objectIds: string[] }) =>
209-
[
210-
...clusterQueryKeys.all(),
211-
buildQueryKeyPart("clusterFreshnessLatest", {
212-
objectIds: [...params.objectIds].sort().join(","),
213-
}),
214-
] as const,
215204
replicaUtilization: () =>
216205
[
217206
...clusterQueryKeys.all(),
@@ -980,25 +969,19 @@ export interface FreshnessObject {
980969
type LagReading = Awaited<
981970
ReturnType<typeof fetchObjectLagHistory>
982971
>["rows"][number];
983-
type LatestReading = Awaited<ReturnType<typeof fetchLatestLag>>["rows"][number];
984-
985-
export interface LagQueryRows {
986-
/** One binned point per object per bin, each the worst reading in its span. */
987-
readings: LagReading[];
988-
/** One row per object, its most recent reading. */
989-
latest: LatestReading[];
990-
}
991-
992972
/**
993973
* Shapes lag readings for the freshness graph, naming each object from the
994974
* caller's list.
995975
*
996976
* Separate from the query so names are not cached with the readings: the query
997977
* key tracks which objects were asked for, not what they are called, so a
998978
* rename would otherwise survive in the cache until the set changed.
979+
*
980+
* @param readings One binned point per object per bin, each the worst reading
981+
* in its span.
999982
*/
1000983
export function buildFreshnessData(
1001-
{ readings, latest }: LagQueryRows,
984+
readings: LagReading[],
1002985
objects: FreshnessObject[],
1003986
) {
1004987
// Everything below reads row order as time order: the graph's start and end,
@@ -1011,15 +994,6 @@ export function buildFreshnessData(
1011994
objects.map((object) => [object.objectId, object]),
1012995
);
1013996

1014-
// Null where the newest reading reports the object as unreadable, which is
1015-
// not the same as the object having no reading at all.
1016-
const latestByObjectId = new Map<string, number | null>(
1017-
latest.map((row) => [
1018-
row.objectId,
1019-
row.lag === null ? null : sumPostgresIntervalMs(row.lag),
1020-
]),
1021-
);
1022-
1023997
const currentData = flatGroup(rows, (d) => d.objectId)
1024998
.map(([, rowsByObjectId]) => rowsByObjectId.at(-1))
1025999
.filter(notNullOrUndefined)
@@ -1093,7 +1067,6 @@ export function buildFreshnessData(
10931067
historicalData,
10941068
currentData,
10951069
objectsById,
1096-
latestByObjectId,
10971070
lines: Array.from(lines.values()),
10981071
startTime: historicalData.at(0)?.timestamp ?? 0,
10991072
endTime: historicalData.at(-1)?.timestamp ?? 0,
@@ -1143,40 +1116,14 @@ export function useClusterFreshness({
11431116
refetchInterval: binSizeMs,
11441117
});
11451118

1146-
const latest = useQuery({
1147-
queryKey: clusterQueryKeys.clusterFreshnessLatest({ objectIds }),
1148-
queryFn: async ({ queryKey, signal }): Promise<LatestReading[]> => {
1149-
if (objectIds.length === 0) return [];
1150-
1151-
const { rows } = await fetchLatestLag({
1152-
objectIds,
1153-
requestOptions: { signal },
1154-
queryKey,
1155-
});
1156-
return rows;
1157-
},
1158-
// A reading lands once a minute, so anything shorter re-asks a question
1159-
// whose answer cannot have changed.
1160-
staleTime: LATEST_READING_INTERVAL_MS,
1161-
refetchInterval: LATEST_READING_INTERVAL_MS,
1162-
});
1163-
1164-
// Undefined until both have answered. Building from one alone would show a
1165-
// graph whose "Now" column disagrees with it.
11661119
const data = useMemo(
1167-
() =>
1168-
series.data && latest.data
1169-
? buildFreshnessData(
1170-
{ readings: series.data, latest: latest.data },
1171-
objects,
1172-
)
1173-
: undefined,
1174-
[series.data, latest.data, objects],
1120+
() => (series.data ? buildFreshnessData(series.data, objects) : undefined),
1121+
[series.data, objects],
11751122
);
11761123

11771124
// Only these fields are returned. Spreading a query object would make every
11781125
// consumer observe all of its state.
1179-
return { data, isError: series.isError || latest.isError };
1126+
return { data, isError: series.isError };
11801127
}
11811128

11821129
export type FreshnessData = ReturnType<typeof buildFreshnessData>;

‎console/src/platform/freshness/FreshnessPage.tsx‎

Lines changed: 18 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -63,7 +63,11 @@ import {
6363
PREDICATE_LABELS,
6464
} from "./freshnessRows";
6565
import { FreshnessTable } from "./FreshnessTable";
66-
import { HydrationCounts, useFreshnessHydration } from "./queries";
66+
import {
67+
HydrationCounts,
68+
useFreshnessHydration,
69+
useLatestLag,
70+
} from "./queries";
6771
import { parsePredicate, useFreshnessParams } from "./useFreshnessParams";
6872
import { useFreshnessRows } from "./useFreshnessRows";
6973

@@ -118,10 +122,10 @@ const FreshnessContent = ({
118122
// Distinguishes a cluster that has nothing on it from one whose objects have
119123
// not arrived yet. Both are an empty list, and they are not the same state.
120124
const { snapshotComplete } = useAllObjects();
125+
const objectIds = objects.map((object) => object.objectId);
121126
const freshness = useClusterFreshness({ lookbackMs, objects });
122-
const hydration = useFreshnessHydration(
123-
objects.map((object) => object.objectId),
124-
);
127+
const latest = useLatestLag(objectIds);
128+
const hydration = useFreshnessHydration(objectIds);
125129

126130
// Before the headline, the graph and the tables, because each of them would
127131
// otherwise render its own "0" and the page would read as a passing health
@@ -134,10 +138,12 @@ const FreshnessContent = ({
134138
);
135139
}
136140

137-
// An error only replaces the page when there is nothing to show. A failed
138-
// refetch keeps the last good readings on screen.
139-
if (!freshness.data || objects.length === 0) {
140-
return freshness.isError ? (
141+
// Waits for both, because drawing the graph before the latest readings would
142+
// show a "Now" column that disagrees with it. An error only replaces the page
143+
// when there is nothing to show, so a failed refetch keeps the last good
144+
// readings on screen.
145+
if (!freshness.data || !latest.data || objects.length === 0) {
146+
return freshness.isError || latest.isError ? (
141147
<ErrorBox message="An error occurred fetching freshness data." />
142148
) : (
143149
<Box height="320px" width="100%">
@@ -151,28 +157,24 @@ const FreshnessContent = ({
151157
return (
152158
<FreshnessView
153159
data={freshness.data}
160+
latestByObjectId={latest.data}
154161
hydrationByObjectId={hydration.data ?? NO_HYDRATION}
155162
{...viewOptions}
156163
/>
157164
);
158165
};
159166

160167
const FreshnessView = ({
161-
data: {
162-
historicalData,
163-
startTime,
164-
endTime,
165-
lines,
166-
objectsById,
167-
latestByObjectId,
168-
},
168+
data: { historicalData, startTime, endTime, lines, objectsById },
169+
latestByObjectId,
169170
hydrationByObjectId,
170171
rangeLabel,
171172
predicate,
172173
thresholdControl,
173174
typeFilters,
174175
}: FreshnessViewOptions & {
175176
data: FreshnessData;
177+
latestByObjectId: Map<string, number | null>;
176178
hydrationByObjectId: Map<string, HydrationCounts>;
177179
}) => {
178180
const { colors } = useTheme<MaterializeTheme>();

‎console/src/platform/freshness/queries.test.ts‎

Lines changed: 31 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,7 @@ import {
1818
} from "~/platform/clusters/queries";
1919
import { createProviderWrapper } from "~/test/utils";
2020

21-
import { useFreshnessHydration } from "./queries";
21+
import { useFreshnessHydration, useLatestLag } from "./queries";
2222

2323
const objects: FreshnessObject[] = [
2424
{
@@ -30,26 +30,49 @@ const objects: FreshnessObject[] = [
3030
},
3131
];
3232

33+
// Records each SQL request and never answers, so a test sees which requests
34+
// start before any has returned.
35+
const recordRequests = () => {
36+
const pending: string[] = [];
37+
server.use(
38+
http.post("*/api/sql", async ({ request }) => {
39+
pending.push(new URL(request.url).searchParams.get("query_key") ?? "");
40+
await delay("infinite");
41+
}),
42+
);
43+
return pending;
44+
};
45+
3346
describe("freshness queries", () => {
3447
it("sends every request before any has answered", async () => {
35-
const pending: string[] = [];
36-
server.use(
37-
http.post("*/api/sql", async ({ request }) => {
38-
pending.push(new URL(request.url).searchParams.get("query_key") ?? "");
39-
await delay("infinite");
40-
}),
41-
);
48+
const pending = recordRequests();
4249

4350
// The page's hooks, in the page's order.
4451
const ProviderWrapper = await createProviderWrapper();
4552
renderHook(
4653
() => {
4754
useClusterFreshness({ lookbackMs: 3_600_000, objects });
55+
useLatestLag(["u1"]);
4856
useFreshnessHydration(["u1"]);
4957
},
5058
{ wrapper: ProviderWrapper },
5159
);
5260

5361
await waitFor(() => expect(pending).toHaveLength(3));
5462
});
63+
64+
it("asks only for the series when only the graph is shown", async () => {
65+
// Cluster Overview's card reads the series alone.
66+
const pending = recordRequests();
67+
68+
const ProviderWrapper = await createProviderWrapper();
69+
renderHook(() => useClusterFreshness({ lookbackMs: 3_600_000, objects }), {
70+
wrapper: ProviderWrapper,
71+
});
72+
73+
await waitFor(() => expect(pending).toHaveLength(1));
74+
// Long enough for a second request started in the same render to land.
75+
await new Promise((resolve) => setTimeout(resolve, 50));
76+
expect(pending).toHaveLength(1);
77+
});
5578
});

‎console/src/platform/freshness/queries.ts‎

Lines changed: 47 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,11 @@ import {
1414
buildRegionQueryKey,
1515
} from "~/api/buildQueryKeySchema";
1616
import { fetchHydrationCounts } from "~/api/materialize/freshness/hydrationCounts";
17+
import {
18+
fetchLatestLag,
19+
LATEST_READING_INTERVAL_MS,
20+
} from "~/api/materialize/freshness/lagHistory";
21+
import { sumPostgresIntervalMs } from "~/util";
1722

1823
export interface HydrationCounts {
1924
hydratedReplicas: number;
@@ -34,8 +39,50 @@ export const freshnessQueryKeys = {
3439
objectIds: [...objectIds].sort().join(","),
3540
}),
3641
] as const,
42+
// No lookback: the newest reading is the newest reading whatever range is
43+
// being graphed, so changing the range must not refetch it.
44+
latestLag: (objectIds: string[]) =>
45+
[
46+
...freshnessQueryKeys.all(),
47+
buildQueryKeyPart("latestLag", {
48+
objectIds: [...objectIds].sort().join(","),
49+
}),
50+
] as const,
3751
};
3852

53+
/**
54+
* Each object's newest lag in milliseconds, keyed by object ID.
55+
*
56+
* Null where the newest reading reports the object as unreadable, which is not
57+
* the same as the object being absent from the map: that means no reading at
58+
* all.
59+
*/
60+
export function useLatestLag(objectIds: string[]) {
61+
return useQuery({
62+
queryKey: freshnessQueryKeys.latestLag(objectIds),
63+
queryFn: async ({ queryKey, signal }) => {
64+
if (objectIds.length === 0) return new Map<string, number | null>();
65+
66+
const { rows } = await fetchLatestLag({
67+
objectIds,
68+
queryKey,
69+
requestOptions: { signal },
70+
});
71+
72+
return new Map<string, number | null>(
73+
rows.map((row) => [
74+
row.objectId,
75+
row.lag === null ? null : sumPostgresIntervalMs(row.lag),
76+
]),
77+
);
78+
},
79+
// A reading lands once a minute, so anything shorter re-asks a question
80+
// whose answer cannot have changed.
81+
staleTime: LATEST_READING_INTERVAL_MS,
82+
refetchInterval: LATEST_READING_INTERVAL_MS,
83+
});
84+
}
85+
3986
/**
4087
* Replica hydration for the objects on screen, keyed by object ID.
4188
*

0 commit comments

Comments
 (0)