Skip to content

Commit 852be00

Browse files
jdonelsonclaude
andcommitted
console: Take the freshness "Now" column from the latest reading
Readings are binned to 60 points per range, and each point is the worst reading in its span. "Now" read the last point, so at a 24 hour range it answered "the worst of the last 24 minutes" to the question "is this bad right now". An object that spiked twenty minutes ago and recovered still read high, and under the "right now" predicate it was counted as exceeding the threshold and coloured on the graph. A second query returns each object's newest reading, batched into the same request. Its two minute filter keeps it cheap whatever range is being graphed: measured on a 300 object, 24 hour fleet it adds 100ms to a 499ms request, against 2,594ms for computing every statistic from unbinned readings. The graph still plots bin maxima, since that is all 60 points can show, so a point can sit above its row's "Now" value. The footer says so, and drops the mz_internal relation names, which are not a stable interface. Co-Authored-By: Claude Opus 5 (1M context) <[email protected]>
1 parent dfe1420 commit 852be00

6 files changed

Lines changed: 143 additions & 43 deletions

File tree

‎console/src/api/materialize/freshness/lagHistory.ts‎

Lines changed: 36 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -104,6 +104,38 @@ export function buildObjectLagHistoryQuery({
104104
);
105105
}
106106

107+
/**
108+
* Each object's most recent reading.
109+
*
110+
* Separate from the binned series because a bin reports the worst reading in
111+
* its span, which at a 24 hour range is the worst of 24 minutes. That is the
112+
* wrong answer to "is this bad right now", and the only range where the last
113+
* bin happens to be the latest reading is one hour.
114+
*
115+
* The two minute filter is what keeps this cheap: a reading lands every
116+
* minute, so the newest one is always inside that window however wide the
117+
* range being graphed.
118+
*/
119+
export function buildLatestLagQuery(objectIds: string[]) {
120+
return queryBuilder
121+
.selectFrom("mz_wallclock_global_lag_recent_history")
122+
.distinctOn("object_id")
123+
.select(["object_id as objectId", "lag"])
124+
.where("object_id", "in", objectIds)
125+
.where(
126+
(eb) => sql`${eb.ref("occurred_at")} + INTERVAL '2 MINUTES'`,
127+
">=",
128+
sql<Date>`mz_now()`,
129+
)
130+
.orderBy(["object_id", "occurred_at desc"]);
131+
}
132+
133+
/**
134+
* The binned series and the latest readings, in one request.
135+
*
136+
* Sent together because they are one question asked at two resolutions, and
137+
* `executeSqlV2` puts an array of queries in a single round trip.
138+
*/
107139
export async function fetchObjectLagHistory({
108140
objectIds,
109141
lookbackMs,
@@ -116,7 +148,10 @@ export async function fetchObjectLagHistory({
116148
requestOptions?: RequestInit;
117149
}) {
118150
return executeSqlV2({
119-
queries: buildObjectLagHistoryQuery({ objectIds, lookbackMs }).compile(),
151+
queries: [
152+
buildObjectLagHistoryQuery({ objectIds, lookbackMs }).compile(),
153+
buildLatestLagQuery(objectIds).compile(),
154+
] as const,
120155
queryKey,
121156
requestOptions,
122157
sessionVariables: {

‎console/src/platform/clusters/queries.ts‎

Lines changed: 33 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -946,17 +946,16 @@ export interface FreshnessObject {
946946
objectType: string;
947947
}
948948

949-
/**
950-
* Lag readings for a known set of objects, shaped for the freshness graph.
951-
*
952-
* The caller supplies the objects rather than naming a cluster, because the
953-
* `useAllObjects` subscribe already holds every name, schema, database and
954-
* type. Resolving those in SQL cost three joins and three full scans per
955-
* request; here they are a map lookup.
956-
*/
957-
type LagReading = Awaited<
958-
ReturnType<typeof fetchObjectLagHistory>
959-
>["rows"][number];
949+
type LagQueryResults = Awaited<ReturnType<typeof fetchObjectLagHistory>>;
950+
type LagReading = LagQueryResults[0]["rows"][number];
951+
type LatestReading = LagQueryResults[1]["rows"][number];
952+
953+
export interface LagQueryRows {
954+
/** One binned point per object per bin, each the worst reading in its span. */
955+
readings: LagReading[];
956+
/** One row per object, its most recent reading. */
957+
latest: LatestReading[];
958+
}
960959

961960
/**
962961
* Shapes lag readings for the freshness graph, naming each object from the
@@ -967,13 +966,22 @@ type LagReading = Awaited<
967966
* rename would otherwise survive in the cache until the set changed.
968967
*/
969968
export function buildFreshnessData(
970-
rows: LagReading[],
969+
{ readings: rows, latest }: LagQueryRows,
971970
objects: FreshnessObject[],
972971
) {
973972
const objectsById = new Map(
974973
objects.map((object) => [object.objectId, object]),
975974
);
976975

976+
// Null where the newest reading reports the object as unreadable, which is
977+
// not the same as the object having no reading at all.
978+
const latestByObjectId = new Map<string, number | null>(
979+
latest.map((row) => [
980+
row.objectId,
981+
row.lag === null ? null : sumPostgresIntervalMs(row.lag),
982+
]),
983+
);
984+
977985
const currentData = flatGroup(rows, (d) => d.objectId)
978986
// The last row per object is the most current: the query sorts by bucket.
979987
.map(([, rowsByObjectId]) => rowsByObjectId.at(-1))
@@ -1045,12 +1053,21 @@ export function buildFreshnessData(
10451053
historicalData,
10461054
currentData,
10471055
objectsById,
1056+
latestByObjectId,
10481057
lines: Array.from(lines.values()),
10491058
startTime: historicalData.at(0)?.timestamp ?? 0,
10501059
endTime: historicalData.at(-1)?.timestamp ?? 0,
10511060
};
10521061
}
10531062

1063+
/**
1064+
* Lag readings for a known set of objects, shaped for the freshness graph.
1065+
*
1066+
* The caller supplies the objects rather than naming a cluster, because the
1067+
* `useAllObjects` subscribe already holds every name, schema, database and
1068+
* type. Resolving those in SQL cost three joins and three full scans per
1069+
* request; here they are a map lookup.
1070+
*/
10541071
/**
10551072
* Lag readings for a known set of objects, shaped for the freshness graph.
10561073
*
@@ -1070,16 +1087,16 @@ export function useClusterFreshness({
10701087

10711088
const query = useSuspenseQuery({
10721089
queryKey: clusterQueryKeys.clusterFreshness({ lookbackMs, objectIds }),
1073-
queryFn: async ({ queryKey, signal }): Promise<LagReading[]> => {
1074-
if (objectIds.length === 0) return [];
1090+
queryFn: async ({ queryKey, signal }): Promise<LagQueryRows> => {
1091+
if (objectIds.length === 0) return { readings: [], latest: [] };
10751092

1076-
const { rows } = await fetchObjectLagHistory({
1093+
const [series, latest] = await fetchObjectLagHistory({
10771094
objectIds,
10781095
lookbackMs,
10791096
requestOptions: { signal },
10801097
queryKey,
10811098
});
1082-
return rows;
1099+
return { readings: series.rows, latest: latest.rows };
10831100
},
10841101
});
10851102

‎console/src/platform/freshness/FreshnessPage.tsx‎

Lines changed: 16 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -112,7 +112,14 @@ const FreshnessContent = ({
112112
const { colors } = useTheme<MaterializeTheme>();
113113
const objects = useFreshnessObjects(clusterId);
114114
const {
115-
data: { historicalData, startTime, endTime, lines, objectsById },
115+
data: {
116+
historicalData,
117+
startTime,
118+
endTime,
119+
lines,
120+
objectsById,
121+
latestByObjectId,
122+
},
116123
} = useClusterFreshness({ lookbackMs, objects });
117124

118125
// Hydration is its own query: `buildLagHistoryQuery` is shared with pages
@@ -148,8 +155,8 @@ const FreshnessContent = ({
148155
);
149156

150157
const statsByKey = React.useMemo(
151-
() => buildStats(visibleLines, historicalData),
152-
[visibleLines, historicalData],
158+
() => buildStats(visibleLines, historicalData, latestByObjectId),
159+
[visibleLines, historicalData, latestByObjectId],
153160
);
154161
const judged = React.useMemo(
155162
() => judgeLines(visibleLines, historicalData, predicate, statsByKey),
@@ -432,16 +439,12 @@ const FreshnessPage = () => {
432439

433440
<Text textStyle="text-small" color={colors.foreground.secondary}>
434441
Freshness is wallclock lag: how far behind wall clock time an
435-
object&rsquo;s results are. Series from{" "}
436-
<Text as="span" fontFamily="mono">
437-
mz_wallclock_global_lag_recent_history
438-
</Text>
439-
, hydration from{" "}
440-
<Text as="span" fontFamily="mono">
441-
mz_hydration_statuses
442-
</Text>
443-
. Readings are binned to 60 points per range and each point is the
444-
worst lag in its span, so p90 is a percentile of those maxima.
442+
object&rsquo;s results are. Readings are binned to 60 points per
443+
range, and each point is the worst lag in its span, so p90 is a
444+
percentile of those maxima. The &ldquo;Now&rdquo; column is the most
445+
recent single reading rather than a binned point, so at wider ranges a
446+
point on the graph can sit above the &ldquo;Now&rdquo; value in its
447+
row.
445448
</Text>
446449
</VStack>
447450
</MainContentContainer>

‎console/src/platform/freshness/FreshnessTable.tsx‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -106,7 +106,7 @@ export const FreshnessTable = ({
106106
);
107107

108108
/**
109-
* The Now cell, which is where an unreadable object is called out. Its
109+
* The "Now" cell, which is where an unreadable object is called out. Its
110110
* `breachValue` is `Infinity` so that it sorts and highlights as the worst
111111
* row, and naming the state here is what keeps that number off the screen.
112112
*/

‎console/src/platform/freshness/freshnessRows.test.ts‎

Lines changed: 38 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -91,6 +91,17 @@ const objectsById = new Map<string, FreshnessObject>(
9191
]),
9292
);
9393

94+
/**
95+
* What the latest-reading query returns: each object's newest reading, which
96+
* the binned series cannot supply once a bin spans more than a minute.
97+
*/
98+
const latestByObjectId = new Map<string, number | null>(
99+
Object.entries(SERIES).map(([key, readings]) => {
100+
const last = readings.at(-1);
101+
return [key, typeof last === "number" ? last : null];
102+
}),
103+
);
104+
94105
const hydrationByObjectId = new Map<string, HydrationCounts>(
95106
Object.keys(SERIES).map((key) => [
96107
key,
@@ -103,7 +114,7 @@ const rowsFor = (
103114
threshold: number,
104115
selected: ReadonlySet<string> = new Set(),
105116
) => {
106-
const stats = buildStats(lines, data);
117+
const stats = buildStats(lines, data, latestByObjectId);
107118
const judged = judgeLines(lines, data, predicate, stats);
108119
return buildFreshnessRows({
109120
judged,
@@ -117,14 +128,18 @@ const rowsFor = (
117128

118129
describe("computeStats", () => {
119130
it("separates the latest reading from the worst", () => {
120-
expect(computeStats("spiky", data)).toMatchObject({
131+
expect(
132+
computeStats("spiky", data, latestByObjectId.get("spiky")),
133+
).toMatchObject({
121134
current: 430,
122135
peak: 9_000,
123136
});
124137
});
125138

126139
it("is all null for a line that never reported", () => {
127-
expect(computeStats("absent", data)).toEqual({
140+
expect(
141+
computeStats("absent", data, latestByObjectId.get("absent")),
142+
).toEqual({
128143
current: null,
129144
peak: null,
130145
p90: null,
@@ -133,7 +148,11 @@ describe("computeStats", () => {
133148
});
134149

135150
it("does not let an unreadable reading score as zero lag", () => {
136-
const stats = computeStats("unreadable", data);
151+
const stats = computeStats(
152+
"unreadable",
153+
data,
154+
latestByObjectId.get("unreadable"),
155+
);
137156
// The graph's accessor returns 0 for these readings so the line draws at
138157
// the bottom. Inheriting that would score the worst state as the best.
139158
expect(accessorFor("unreadable")(data[4]!)).toBe(0);
@@ -142,6 +161,19 @@ describe("computeStats", () => {
142161
expect(stats.current).toBeNull();
143162
});
144163

164+
it('takes "Now" from the latest reading, not the worst in the last bin', () => {
165+
// The reviewer's case: at a wide range the final bin is a maximum over
166+
// many minutes, so it reads high for an object that has already recovered.
167+
const lastBinMax = Math.max(
168+
...(SERIES.spiky as number[]).map((r) => r as number),
169+
);
170+
expect(lastBinMax).toBe(9_000);
171+
172+
const stats = computeStats("spiky", data, 430);
173+
expect(stats.current).toBe(430);
174+
expect(stats.peak).toBe(9_000);
175+
});
176+
145177
it("discards a short spike that the peak keeps", () => {
146178
// The query hands us 60 readings whatever range is selected, so this is the
147179
// shape p90 actually sees. Six bad readings sit above the 90th percentile;
@@ -150,7 +182,7 @@ describe("computeStats", () => {
150182
{ spike: Array.from({ length: 60 }, (_, i) => (i < 3 ? 9_000 : 400)) },
151183
60,
152184
);
153-
const stats = computeStats("spike", sixty);
185+
const stats = computeStats("spike", sixty, undefined);
154186
expect(stats.peak).toBe(9_000);
155187
expect(stats.p90).toBe(400);
156188
});
@@ -161,7 +193,7 @@ describe("computeStats", () => {
161193
{ spike: Array.from({ length: 60 }, (_, i) => (i < 7 ? 9_000 : 400)) },
162194
60,
163195
);
164-
expect(computeStats("spike", sixty).p90).toBe(9_000);
196+
expect(computeStats("spike", sixty, undefined).p90).toBe(9_000);
165197
});
166198
});
167199

‎console/src/platform/freshness/freshnessRows.ts‎

Lines changed: 19 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -67,7 +67,11 @@ export interface ObjectStats {
6767
* Each reading is also already a maximum over its bin, so these are
6868
* percentiles of maxima rather than of the underlying lag.
6969
*/
70-
export function computeStats(key: string, data: DataPoint[]): ObjectStats {
70+
export function computeStats(
71+
key: string,
72+
data: DataPoint[],
73+
latest: number | null | undefined,
74+
): ObjectStats {
7175
const values: number[] = [];
7276
let notQueryable = false;
7377

@@ -86,15 +90,17 @@ export function computeStats(key: string, data: DataPoint[]): ObjectStats {
8690
}
8791

8892
if (values.length === 0) {
89-
return { current: null, peak: null, p90: null, notQueryable };
93+
return { current: latest ?? null, peak: null, p90: null, notQueryable };
9094
}
9195

9296
const sorted = [...values].sort((a, b) => a - b);
9397
const rank = Math.max(0, Math.ceil(0.9 * sorted.length) - 1);
94-
const last = data.at(-1)?.lag[key];
9598

9699
return {
97-
current: last?.queryable ? last.totalMs : null,
100+
// The newest reading, not the newest bin. A bin reports the worst reading
101+
// in its span, so at a 24 hour range the last one answers "the worst of
102+
// the last 24 minutes" when the question asked was "right now".
103+
current: latest ?? null,
98104
peak: sorted[sorted.length - 1],
99105
p90: sorted[rank],
100106
notQueryable,
@@ -127,7 +133,8 @@ export function judgeLines(
127133
statsByKey: Map<string, ObjectStats>,
128134
): ThresholdLineSeries<DataPoint>[] {
129135
return lines.map((line) => {
130-
const stats = statsByKey.get(line.key) ?? computeStats(line.key, data);
136+
const stats =
137+
statsByKey.get(line.key) ?? computeStats(line.key, data, undefined);
131138
return {
132139
key: line.key,
133140
label: line.label,
@@ -143,8 +150,14 @@ export function judgeLines(
143150
export function buildStats(
144151
lines: { key: string }[],
145152
data: DataPoint[],
153+
latestByObjectId: Map<string, number | null>,
146154
): Map<string, ObjectStats> {
147-
return new Map(lines.map((line) => [line.key, computeStats(line.key, data)]));
155+
return new Map(
156+
lines.map((line) => [
157+
line.key,
158+
computeStats(line.key, data, latestByObjectId.get(line.key)),
159+
]),
160+
);
148161
}
149162

150163
/**

0 commit comments

Comments
 (0)