Skip to content

Commit daa182b

Browse files
authored
fix(server): surface why process group metrics are not exported (#23969)
Signed-off-by: Yordis Prieto <[email protected]>
1 parent c5332bf commit daa182b

2 files changed

Lines changed: 86 additions & 5 deletions

File tree

‎apps/server/src/resourceTelemetry/ProcessGroupMetrics.test.ts‎

Lines changed: 63 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,17 @@
11
import { assert, describe, it } from "@effect/vitest";
2-
import type { ResourceTelemetryProcess } from "@t3tools/contracts";
2+
import type { ResourceTelemetryProcess, ResourceTelemetrySnapshot } from "@t3tools/contracts";
3+
import * as Effect from "effect/Effect";
4+
import * as Layer from "effect/Layer";
5+
import * as Logger from "effect/Logger";
6+
import * as References from "effect/References";
7+
import * as Ref from "effect/Ref";
8+
import * as TestClock from "effect/testing/TestClock";
39

4-
import { groupProcesses } from "./ProcessGroupMetrics.ts";
10+
import * as ServerConfig from "../config.ts";
11+
import * as ProcessGroupMetrics from "./ProcessGroupMetrics.ts";
12+
import * as ResourceTelemetry from "./ResourceTelemetry.ts";
13+
14+
const { groupProcesses } = ProcessGroupMetrics;
515

616
const process = (
717
pid: number,
@@ -57,3 +67,54 @@ describe("groupProcesses", () => {
5767
});
5868
});
5969
});
70+
71+
describe("ProcessGroupMetrics.layer", () => {
72+
it.effect("logs the first failed sample and the recovery, not every repeat", () => {
73+
const logs: Array<string> = [];
74+
const logger = Logger.make(({ logLevel, message }) => {
75+
logs.push(`${logLevel.toUpperCase()} ${String([message].flat()[0])}`);
76+
});
77+
return Effect.gen(function* () {
78+
const outcomes = yield* Ref.make<ReadonlyArray<"fail" | "ok">>(["fail", "fail", "ok"]);
79+
const refresh = Ref.modify(outcomes, ([next = "ok", ...rest]) => [next, rest] as const).pipe(
80+
Effect.flatMap((next) =>
81+
next === "fail"
82+
? Effect.fail(
83+
new ResourceTelemetry.ResourceTelemetryRefreshFailed({
84+
operation: "refresh",
85+
cause: new Error("Resource monitor binary was not found for darwin/arm64."),
86+
}),
87+
)
88+
: Effect.succeed({ processes: [] } as unknown as ResourceTelemetrySnapshot),
89+
),
90+
);
91+
const telemetry = { refresh } as unknown as ResourceTelemetry.ResourceTelemetry["Service"];
92+
93+
yield* Layer.build(
94+
ProcessGroupMetrics.layer.pipe(
95+
Layer.provide(Layer.succeed(ResourceTelemetry.ResourceTelemetry, telemetry)),
96+
Layer.provide(
97+
ServerConfig.layer({
98+
otlpMetricsUrl: "http://localhost:4318/v1/metrics",
99+
} as ServerConfig.ServerConfig["Service"]),
100+
),
101+
),
102+
);
103+
yield* TestClock.adjust("3 minutes");
104+
105+
assert.deepStrictEqual(logs, [
106+
"WARN process group metrics are not exported: sample failed",
107+
"DEBUG process group metrics sample failed",
108+
"INFO process group metrics sample recovered",
109+
]);
110+
}).pipe(
111+
Effect.scoped,
112+
Effect.provide(
113+
Layer.mergeAll(
114+
Logger.layer([logger], { mergeWithExisting: false }),
115+
Layer.succeed(References.MinimumLogLevel, "Debug"),
116+
),
117+
),
118+
);
119+
});
120+
});

‎apps/server/src/resourceTelemetry/ProcessGroupMetrics.ts‎

Lines changed: 23 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,9 @@
11
import type { ResourceTelemetryProcess } from "@t3tools/contracts";
2+
import * as Cause from "effect/Cause";
23
import * as Effect from "effect/Effect";
34
import * as Layer from "effect/Layer";
45
import * as Metric from "effect/Metric";
6+
import * as Ref from "effect/Ref";
57

68
import * as ServerConfig from "../config.ts";
79
import { metricAttributes } from "../observability/Metrics.ts";
@@ -113,10 +115,28 @@ export const layer = Layer.effectDiscard(
113115
yield* Metric.update(Metric.withAttributes(processCount, attributes), total.count);
114116
}
115117
});
116-
yield* sample().pipe(
117-
Effect.catch((error) =>
118-
Effect.logDebug("process group metrics sample failed", { error: error.message }),
118+
// A failing collector would otherwise leave these series silently absent, so the first
119+
// failure and the recovery are logged; the repeats stay at debug.
120+
const failing = yield* Ref.make(false);
121+
const recordSuccess = Ref.getAndSet(failing, false).pipe(
122+
Effect.flatMap((wasFailing) =>
123+
wasFailing ? Effect.logInfo("process group metrics sample recovered") : Effect.void,
119124
),
125+
);
126+
const recordFailure = (error: ResourceTelemetry.ResourceTelemetryRefreshFailed) => {
127+
const cause = Cause.fail(error);
128+
return Ref.getAndSet(failing, true).pipe(
129+
Effect.flatMap((wasFailing) =>
130+
wasFailing
131+
? Effect.logDebug("process group metrics sample failed", cause)
132+
: Effect.logWarning("process group metrics are not exported: sample failed", cause),
133+
),
134+
Effect.annotateLogs({ operation: error.operation }),
135+
);
136+
};
137+
yield* sample().pipe(
138+
Effect.andThen(recordSuccess),
139+
Effect.catch(recordFailure),
120140
Effect.andThen(Effect.sleep(SAMPLE_INTERVAL)),
121141
Effect.forever,
122142
Effect.forkScoped,

0 commit comments

Comments
 (0)