Skip to content

Commit 2b016dd

Browse files
t3dotggclaude
andcommitted
perf(server): cap latest failures and warning logs while aggregating traces
The aggregator kept every failure and every warning or error log, then sorted and sliced them to 20 at the end. On a ring with many warnings that held tens of thousands of objects. Now all three top lists use one bounded insert, so each holds at most its limit. Output is unchanged. Co-Authored-By: Claude Opus 5.5 (1M context) <[email protected]>
1 parent a18604d commit 2b016dd

2 files changed

Lines changed: 65 additions & 42 deletions

File tree

‎apps/server/src/diagnostics/TraceDiagnostics.test.ts‎

Lines changed: 23 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -310,25 +310,43 @@ describe("TraceDiagnostics", () => {
310310
}),
311311
);
312312

313-
it.effect("keeps only the slowest span occurrences while aggregating large inputs", () =>
313+
it.effect("keeps only the top spans, failures, and warning logs from large inputs", () =>
314314
Effect.sync(() => {
315+
// Shuffled, so some older records arrive after the lists are full.
316+
const indexes = Array.from({ length: 30 }, (_, step) => (step * 7) % 30);
315317
const diagnostics = aggregateLines(
316-
Array.from({ length: 25 }, (_, index) =>
318+
indexes.map((index) =>
317319
record({
318320
name: `span-${index}`,
319321
traceId: `trace-${index}`,
320322
spanId: `span-${index}`,
321323
startMs: index * 1_000,
322324
durationMs: index,
325+
exit: { _tag: "Failure", cause: "Provider crashed" },
326+
events: [
327+
{
328+
name: `warning ${index}`,
329+
timeUnixNano: ns(index * 1_000),
330+
attributes: { "effect.logLevel": "Warning" },
331+
},
332+
],
323333
}),
324334
),
325335
);
336+
const newestTwenty = Array.from({ length: 20 }, (_, rank) => `trace-${29 - rank}`);
326337

327-
assert.equal(diagnostics.recordCount, 25);
328-
assert.equal(diagnostics.slowestSpans.length, 10);
338+
assert.equal(diagnostics.recordCount, 30);
329339
assert.deepStrictEqual(
330340
diagnostics.slowestSpans.map((span) => span.durationMs),
331-
[24, 23, 22, 21, 20, 19, 18, 17, 16, 15],
341+
[29, 28, 27, 26, 25, 24, 23, 22, 21, 20],
342+
);
343+
assert.deepStrictEqual(
344+
diagnostics.latestFailures.map((failure) => failure.traceId),
345+
newestTwenty,
346+
);
347+
assert.deepStrictEqual(
348+
diagnostics.latestWarningAndErrorLogs.map((log) => log.traceId),
349+
newestTwenty,
332350
);
333351
}),
334352
);

‎apps/server/src/diagnostics/TraceDiagnostics.ts‎

Lines changed: 42 additions & 37 deletions
Original file line numberDiff line numberDiff line change
@@ -164,24 +164,43 @@ function isNotFoundError(error: PlatformError.PlatformError): boolean {
164164
return error.reason._tag === "NotFound";
165165
}
166166

167-
function insertBoundedSlowestSpan(
168-
slowestSpans: ServerTraceDiagnosticsSpanOccurrence[],
169-
span: ServerTraceDiagnosticsSpanOccurrence,
167+
/**
168+
* Adds `item` to `items`, which stays sorted by `order` and holds at most
169+
* `limit` entries. Same result as a stable sort and slice over every item, but
170+
* memory stays bounded however many items stream in.
171+
*/
172+
function insertBounded<A>(
173+
items: A[],
174+
item: A,
175+
limit: number,
176+
order: (left: A, right: A) => number,
170177
): void {
171-
if (
172-
slowestSpans.length >= TOP_LIMIT &&
173-
span.durationMs <= slowestSpans[slowestSpans.length - 1]!.durationMs
174-
) {
178+
if (items.length >= limit && order(item, items[items.length - 1]!) >= 0) {
175179
return;
176180
}
177181

178-
slowestSpans.push(span);
179-
slowestSpans.sort((left, right) => right.durationMs - left.durationMs);
180-
if (slowestSpans.length > TOP_LIMIT) {
181-
slowestSpans.length = TOP_LIMIT;
182+
items.push(item);
183+
items.sort(order);
184+
if (items.length > limit) {
185+
items.length = limit;
182186
}
183187
}
184188

189+
const slowestFirst = (
190+
left: ServerTraceDiagnosticsSpanOccurrence,
191+
right: ServerTraceDiagnosticsSpanOccurrence,
192+
) => right.durationMs - left.durationMs;
193+
194+
const latestEndedFirst = (
195+
left: ServerTraceDiagnosticsRecentFailure,
196+
right: ServerTraceDiagnosticsRecentFailure,
197+
) => DateTime.toEpochMillis(right.endedAt) - DateTime.toEpochMillis(left.endedAt);
198+
199+
const latestSeenFirst = (
200+
left: ServerTraceDiagnosticsLogEvent,
201+
right: ServerTraceDiagnosticsLogEvent,
202+
) => DateTime.toEpochMillis(right.seenAt) - DateTime.toEpochMillis(left.seenAt);
203+
185204
/**
186205
* Folds trace NDJSON into diagnostics. Call `addLine` once per line as the
187206
* rotated files stream in, then `finish` for the result.
@@ -265,11 +284,11 @@ export function makeTraceDiagnosticsAggregator(
265284
if (durationMs >= slowSpanThresholdMs) {
266285
slowSpanCount += 1;
267286
}
268-
insertBoundedSlowestSpan(slowestSpans, spanItem);
287+
insertBounded(slowestSpans, spanItem, TOP_LIMIT, slowestFirst);
269288

270289
if (isFailure) {
271290
const cause = readExitCause(parsed.exit);
272-
latestFailures.push({ ...spanItem, cause });
291+
insertBounded(latestFailures, { ...spanItem, cause }, RECENT_LIMIT, latestEndedFirst);
273292

274293
const failureKey = `${name}\0${cause}`;
275294
const existing = failuresByKey.get(failureKey);
@@ -304,14 +323,12 @@ export function makeTraceDiagnosticsAggregator(
304323

305324
const seenAt = unixNanoToDateTime(rawEvent.timeUnixNano) ?? endedAt;
306325
const message = toStringValue(rawEvent.name)?.trim() ?? "Log event";
307-
latestWarningAndErrorLogs.push({
308-
spanName: name,
309-
level,
310-
message,
311-
seenAt,
312-
traceId,
313-
spanId,
314-
});
326+
insertBounded(
327+
latestWarningAndErrorLogs,
328+
{ spanName: name, level, message, seenAt, traceId, spanId },
329+
RECENT_LIMIT,
330+
latestSeenFirst,
331+
);
315332
}
316333
}
317334
};
@@ -359,18 +376,8 @@ export function makeTraceDiagnosticsAggregator(
359376
DateTime.toEpochMillis(right.lastSeenAt) - DateTime.toEpochMillis(left.lastSeenAt),
360377
)
361378
.slice(0, TOP_LIMIT),
362-
latestFailures: latestFailures
363-
.toSorted(
364-
(left, right) =>
365-
DateTime.toEpochMillis(right.endedAt) - DateTime.toEpochMillis(left.endedAt),
366-
)
367-
.slice(0, RECENT_LIMIT),
368-
latestWarningAndErrorLogs: latestWarningAndErrorLogs
369-
.toSorted(
370-
(left, right) =>
371-
DateTime.toEpochMillis(right.seenAt) - DateTime.toEpochMillis(left.seenAt),
372-
)
373-
.slice(0, RECENT_LIMIT),
379+
latestFailures,
380+
latestWarningAndErrorLogs,
374381
partialFailure: input.partialFailure ? Option.some(true) : Option.none(),
375382
error: Option.fromNullishOr(input.error),
376383
};
@@ -418,7 +425,6 @@ export const make = Effect.gen(function* () {
418425
const slowSpanThresholdMs = options.slowSpanThresholdMs ?? DEFAULT_SLOW_SPAN_THRESHOLD_MS;
419426
const paths = toRotatedTracePaths(options.traceFilePath, options.maxFiles);
420427
const aggregator = makeTraceDiagnosticsAggregator(slowSpanThresholdMs);
421-
// One aggregator reads every file, so keep them in order, oldest first.
422428
const results = yield* Effect.forEach(
423429
paths,
424430
(path) =>
@@ -434,9 +440,8 @@ export const make = Effect.gen(function* () {
434440
),
435441
Effect.result,
436442
),
437-
{
438-
concurrency: 1,
439-
},
443+
// Every file feeds one aggregator, so read them one at a time, oldest first.
444+
{ concurrency: 1 },
440445
);
441446
const foundFile = results.some((result) => Result.isSuccess(result) && result.success);
442447
const readFailure = results.find(Result.isFailure);

0 commit comments

Comments
 (0)