Skip to content

Commit 4036c40

Browse files
committed
fix(server): surface why process group metrics are not exported
Signed-off-by: Yordis Prieto <[email protected]>
1 parent a68e6e9 commit 4036c40

2 files changed

Lines changed: 86 additions & 5 deletions

File tree

‎apps/server/src/resourceTelemetry/ProcessGroupMetrics.test.ts‎

Lines changed: 63 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,17 @@
11
import { assert, describe, it } from "@effect/vitest";
2-
import type { ResourceTelemetryProcess } from "@t3tools/contracts";
2+
import type { ResourceTelemetryProcess, ResourceTelemetrySnapshot } from "@t3tools/contracts";
3+
import * as Effect from "effect/Effect";
4+
import * as Layer from "effect/Layer";
5+
import * as Logger from "effect/Logger";
6+
import * as References from "effect/References";
7+
import * as Ref from "effect/Ref";
8+
import * as TestClock from "effect/testing/TestClock";
39

4-
import { groupProcesses } from "./ProcessGroupMetrics.ts";
10+
import * as ServerConfig from "../config.ts";
11+
import * as ProcessGroupMetrics from "./ProcessGroupMetrics.ts";
12+
import * as ResourceTelemetry from "./ResourceTelemetry.ts";
13+
14+
const { groupProcesses } = ProcessGroupMetrics;
515

616
const process = (
717
pid: number,
@@ -57,3 +67,54 @@ describe("groupProcesses", () => {
5767
});
5868
});
5969
});
70+
71+
describe("ProcessGroupMetrics.layer", () => {
72+
it.effect("logs the first failed sample and the recovery, not every repeat", () => {
73+
const logs: Array<string> = [];
74+
const logger = Logger.make(({ logLevel, message }) => {
75+
logs.push(`${logLevel.toUpperCase()} ${String([message].flat()[0])}`);
76+
});
77+
return Effect.gen(function* () {
78+
const outcomes = yield* Ref.make<ReadonlyArray<"fail" | "ok">>(["fail", "fail", "ok"]);
79+
const refresh = Ref.modify(outcomes, ([next = "ok", ...rest]) => [next, rest] as const).pipe(
80+
Effect.flatMap((next) =>
81+
next === "fail"
82+
? Effect.fail(
83+
new ResourceTelemetry.ResourceTelemetryRefreshFailed({
84+
operation: "refresh",
85+
cause: new Error("Resource monitor binary was not found for darwin/arm64."),
86+
}),
87+
)
88+
: Effect.succeed({ processes: [] } as unknown as ResourceTelemetrySnapshot),
89+
),
90+
);
91+
const telemetry = { refresh } as unknown as ResourceTelemetry.ResourceTelemetry["Service"];
92+
93+
yield* Layer.build(
94+
ProcessGroupMetrics.layer.pipe(
95+
Layer.provide(Layer.succeed(ResourceTelemetry.ResourceTelemetry, telemetry)),
96+
Layer.provide(
97+
ServerConfig.layer({
98+
otlpMetricsUrl: "http://localhost:4318/v1/metrics",
99+
} as ServerConfig.ServerConfig["Service"]),
100+
),
101+
),
102+
);
103+
yield* TestClock.adjust("3 minutes");
104+
105+
assert.deepStrictEqual(logs, [
106+
"WARN process group metrics are not exported: sample failed",
107+
"DEBUG process group metrics sample failed",
108+
"INFO process group metrics sample recovered",
109+
]);
110+
}).pipe(
111+
Effect.scoped,
112+
Effect.provide(
113+
Layer.mergeAll(
114+
Logger.layer([logger], { mergeWithExisting: false }),
115+
Layer.succeed(References.MinimumLogLevel, "Debug"),
116+
),
117+
),
118+
);
119+
});
120+
});

‎apps/server/src/resourceTelemetry/ProcessGroupMetrics.ts‎

Lines changed: 23 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@ import type { ResourceTelemetryProcess } from "@t3tools/contracts";
22
import * as Effect from "effect/Effect";
33
import * as Layer from "effect/Layer";
44
import * as Metric from "effect/Metric";
5+
import * as Ref from "effect/Ref";
56

67
import * as ServerConfig from "../config.ts";
78
import { metricAttributes } from "../observability/Metrics.ts";
@@ -113,10 +114,29 @@ export const layer = Layer.effectDiscard(
113114
yield* Metric.update(Metric.withAttributes(processCount, attributes), total.count);
114115
}
115116
});
116-
yield* sample().pipe(
117-
Effect.catch((error) =>
118-
Effect.logDebug("process group metrics sample failed", { error: error.message }),
117+
// A failing collector would otherwise leave these series silently absent, so the first
118+
// failure and the recovery are logged; the repeats stay at debug.
119+
const failing = yield* Ref.make(false);
120+
const recordSuccess = Ref.getAndSet(failing, false).pipe(
121+
Effect.flatMap((wasFailing) =>
122+
wasFailing ? Effect.logInfo("process group metrics sample recovered") : Effect.void,
119123
),
124+
);
125+
const recordFailure = (error: ResourceTelemetry.ResourceTelemetryRefreshFailed) => {
126+
const reason = error.cause instanceof Error ? error.cause.message : error.message;
127+
return Ref.getAndSet(failing, true).pipe(
128+
Effect.flatMap((wasFailing) =>
129+
wasFailing
130+
? Effect.logDebug("process group metrics sample failed", { reason })
131+
: Effect.logWarning("process group metrics are not exported: sample failed", {
132+
reason,
133+
}),
134+
),
135+
);
136+
};
137+
yield* sample().pipe(
138+
Effect.andThen(recordSuccess),
139+
Effect.catch(recordFailure),
120140
Effect.andThen(Effect.sleep(SAMPLE_INTERVAL)),
121141
Effect.forever,
122142
Effect.forkScoped,

0 commit comments

Comments
 (0)