Skip to content

Commit bb7eacc

Browse files
author
tsouth89
committed
fix(server): keep provider status after probe timeouts
1 parent d81afa0 commit bb7eacc

10 files changed

Lines changed: 415 additions & 59 deletions

File tree

‎apps/server/src/provider/ClaudeProvider.ts‎

Lines changed: 7 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -33,6 +33,7 @@ import {
3333
providerModelsFromSettings,
3434
spawnAndCollect,
3535
type ServerProviderDraft,
36+
ProviderProbeTimeoutError,
3637
} from "@t3tools/provider-core/server/snapshotProbe";
3738
import { resolveClaudeSdkExecutablePath } from "./Drivers/ClaudeExecutable.ts";
3839
import { makeClaudeEnvironment } from "./Drivers/ClaudeHome.ts";
@@ -461,7 +462,7 @@ export const checkClaudeProviderStatus = Effect.fn("checkClaudeProviderStatus")(
461462
resolveResetCredits?: (version: string) => Effect.Effect<ServerProviderResetCredits | undefined>,
462463
): Effect.fn.Return<
463464
ServerProviderDraft,
464-
never,
465+
ProviderProbeTimeoutError,
465466
ChildProcessSpawner.ChildProcessSpawner | FileSystem.FileSystem | Path.Path
466467
> {
467468
const resolvedEnvironment = environment ?? process.env;
@@ -517,19 +518,11 @@ export const checkClaudeProviderStatus = Effect.fn("checkClaudeProviderStatus")(
517518
}
518519

519520
if (Option.isNone(versionProbe.success)) {
520-
return buildServerProvider({
521-
presentation: CLAUDE_PRESENTATION,
522-
enabled: claudeSettings.enabled,
523-
checkedAt,
524-
models: allModels,
525-
probe: {
526-
installed: true,
527-
version: null,
528-
status: "error",
529-
auth: { status: "unknown" },
530-
message:
531-
"Claude Agent CLI is installed but failed to run. Timed out while running command.",
532-
},
521+
return yield* new ProviderProbeTimeoutError({
522+
provider: "Claude Agent",
523+
probe: "version",
524+
timeoutMs: DEFAULT_TIMEOUT_MS,
525+
installed: true,
533526
});
534527
}
535528

‎apps/server/src/provider/CodexProvider.ts‎

Lines changed: 7 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -38,6 +38,7 @@ import {
3838
buildServerProvider,
3939
COMPACT_SLASH_COMMAND,
4040
type ServerProviderDraft,
41+
ProviderProbeTimeoutError,
4142
} from "@t3tools/provider-core/server/snapshotProbe";
4243
import { expandHomePath } from "@t3tools/provider-core/server/pathExpansion";
4344
import { makeUnavailableUsageLimits } from "@t3tools/provider-core/server/usageLimits";
@@ -584,7 +585,7 @@ export const checkCodexProviderStatus = Effect.fn("checkCodexProviderStatus")(fu
584585
managedAuth?: ServerProvider["auth"],
585586
): Effect.fn.Return<
586587
ServerProviderDraft,
587-
ServerSettingsError,
588+
ServerSettingsError | ProviderProbeTimeoutError,
588589
ChildProcessSpawner.ChildProcessSpawner
589590
> {
590591
const resolvedEnvironment = environment ?? process.env;
@@ -647,19 +648,11 @@ export const checkCodexProviderStatus = Effect.fn("checkCodexProviderStatus")(fu
647648
}
648649

649650
if (Option.isNone(probeResult.success)) {
650-
return buildServerProvider({
651-
presentation: CODEX_PRESENTATION,
652-
enabled: codexSettings.enabled,
653-
checkedAt,
654-
models: emptyModels,
655-
skills: [],
656-
probe: {
657-
installed: true,
658-
version: null,
659-
status: "error",
660-
auth: { status: "unknown" },
661-
message: "Timed out while checking Codex app-server provider status.",
662-
},
651+
return yield* new ProviderProbeTimeoutError({
652+
provider: "Codex",
653+
probe: "app-server status",
654+
timeoutMs: AUTH_PROBE_TIMEOUT_MS,
655+
installed: true,
663656
});
664657
}
665658

‎apps/server/src/provider/CursorProvider.test.ts‎

Lines changed: 36 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,9 @@
11
import type { SDKModel } from "@cursor/sdk";
22
import { describe, expect, it } from "@effect/vitest";
33
import * as Effect from "effect/Effect";
4+
import * as Deferred from "effect/Deferred";
5+
import * as Fiber from "effect/Fiber";
6+
import { TestClock } from "effect/testing";
47
import * as Schema from "effect/Schema";
58
import type { CursorSettings } from "@t3tools/contracts";
69
import { CursorSettings as CursorSettingsSchema } from "@t3tools/contracts";
@@ -206,6 +209,39 @@ describe("checkCursorProviderStatus", () => {
206209
}),
207210
);
208211

212+
it.effect("reports a catalog deadline without declaring the provider broken", () =>
213+
Effect.gen(function* () {
214+
const started = yield* Deferred.make<void>();
215+
const stopped = yield* Deferred.make<void>();
216+
const statusFiber = yield* checkCursorProviderStatus(baseCursorSettings, {
217+
CURSOR_API_KEY: "test-cursor-key",
218+
}).pipe(
219+
Effect.provide(
220+
CursorSdkCatalog.layerTest(() =>
221+
Deferred.succeed(started, undefined).pipe(
222+
Effect.andThen(Effect.never),
223+
Effect.ensuring(Deferred.succeed(stopped, undefined)),
224+
),
225+
),
226+
),
227+
Effect.flip,
228+
Effect.forkChild,
229+
);
230+
231+
yield* Deferred.await(started);
232+
yield* TestClock.adjust("15 seconds");
233+
const timeout = yield* Fiber.join(statusFiber);
234+
expect(timeout).toMatchObject({
235+
_tag: "ProviderProbeTimeoutError",
236+
provider: "Cursor",
237+
probe: "SDK catalog",
238+
timeoutMs: 15_000,
239+
installed: true,
240+
});
241+
yield* Deferred.await(stopped);
242+
}),
243+
);
244+
209245
it.effect("surfaces SDK authentication failures", () =>
210246
Effect.gen(function* () {
211247
const provider = yield* checkCursorProviderStatus(baseCursorSettings, {

‎apps/server/src/provider/CursorProvider.ts‎

Lines changed: 11 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@ import {
2020
buildServerProvider,
2121
providerModelsFromSettings,
2222
type ServerProviderDraft,
23+
ProviderProbeTimeoutError,
2324
} from "@t3tools/provider-core/server/snapshotProbe";
2425
import * as CursorSdkCatalog from "./CursorSdkCatalog.ts";
2526

@@ -250,7 +251,11 @@ export const checkCursorProviderStatus = Effect.fn("checkCursorProviderStatus")(
250251
cursorSettings: CursorSettings,
251252
environment?: NodeJS.ProcessEnv,
252253
authenticationType: "api-key" | "browser" = "api-key",
253-
): Effect.fn.Return<ServerProviderDraft, never, CursorSdkCatalog.CursorSdkCatalog> {
254+
): Effect.fn.Return<
255+
ServerProviderDraft,
256+
ProviderProbeTimeoutError,
257+
CursorSdkCatalog.CursorSdkCatalog
258+
> {
254259
const checkedAt = DateTime.formatIso(yield* DateTime.now);
255260
const fallbackModels = getCursorFallbackModels(cursorSettings);
256261

@@ -317,18 +322,11 @@ export const checkCursorProviderStatus = Effect.fn("checkCursorProviderStatus")(
317322
}
318323

319324
if (Option.isNone(catalogResult.success)) {
320-
return buildServerProvider({
321-
presentation: CURSOR_PRESENTATION,
322-
enabled: cursorSettings.enabled,
323-
checkedAt,
324-
models: fallbackModels,
325-
probe: {
326-
installed: true,
327-
version: null,
328-
status: "error",
329-
auth: { status: "unknown" },
330-
message: `Cursor SDK catalog request timed out after ${CURSOR_SDK_CATALOG_TIMEOUT_MS}ms.`,
331-
},
325+
return yield* new ProviderProbeTimeoutError({
326+
provider: "Cursor",
327+
probe: "SDK catalog",
328+
timeoutMs: CURSOR_SDK_CATALOG_TIMEOUT_MS,
329+
installed: true,
332330
});
333331
}
334332

‎apps/server/src/provider/GrokProvider.ts‎

Lines changed: 7 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,7 @@ import {
3030
providerModelsFromSettings,
3131
spawnAndCollect,
3232
type ServerProviderDraft,
33+
ProviderProbeTimeoutError,
3334
} from "@t3tools/provider-core/server/snapshotProbe";
3435
import {
3536
enrichProviderSnapshotWithVersionAdvisory,
@@ -376,7 +377,7 @@ export const checkGrokProviderStatus = Effect.fn("checkGrokProviderStatus")(func
376377
cwd?: string,
377378
): Effect.fn.Return<
378379
ServerProviderDraft,
379-
never,
380+
ProviderProbeTimeoutError,
380381
ChildProcessSpawner.ChildProcessSpawner | Crypto.Crypto
381382
> {
382383
const checkedAt = DateTime.formatIso(yield* DateTime.now);
@@ -426,18 +427,11 @@ export const checkGrokProviderStatus = Effect.fn("checkGrokProviderStatus")(func
426427
}
427428

428429
if (Option.isNone(versionResult.success)) {
429-
return buildServerProvider({
430-
presentation: GROK_PRESENTATION,
431-
enabled: grokSettings.enabled,
432-
checkedAt,
433-
models: fallbackModels,
434-
probe: {
435-
installed: true,
436-
version: null,
437-
status: "error",
438-
auth: { status: "unknown" },
439-
message: "Grok CLI is installed but timed out while running `grok --version`.",
440-
},
430+
return yield* new ProviderProbeTimeoutError({
431+
provider: "Grok",
432+
probe: "grok --version",
433+
timeoutMs: VERSION_PROBE_TIMEOUT_MS,
434+
installed: true,
441435
});
442436
}
443437

‎apps/server/src/provider/ProviderRegistry.test.ts‎

Lines changed: 36 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@ import * as Fiber from "effect/Fiber";
1212
import * as Layer from "effect/Layer";
1313
import * as PubSub from "effect/PubSub";
1414
import * as Ref from "effect/Ref";
15+
import * as Result from "effect/Result";
1516
import * as Schema from "effect/Schema";
1617
import * as Scope from "effect/Scope";
1718
import * as Sink from "effect/Sink";
@@ -22,6 +23,7 @@ import {
2223
EnvironmentId,
2324
ClaudeSettings,
2425
CodexSettings,
26+
GrokSettings,
2527
DEFAULT_SERVER_SETTINGS,
2628
PROVIDER_WORKSPACE_SNAPSHOT_TTL_MS,
2729
ProviderDriverKind,
@@ -40,6 +42,7 @@ import { applyServerSettingsPatch } from "@t3tools/shared/serverSettings";
4042

4143
import { checkCodexProviderStatus, type CodexAppServerProviderSnapshot } from "./CodexProvider.ts";
4244
import { checkClaudeProviderStatus } from "./ClaudeProvider.ts";
45+
import { checkGrokProviderStatus } from "./GrokProvider.ts";
4346
import * as BackgroundPolicy from "../background/BackgroundPolicy.ts";
4447
import * as AntigravityInstallation from "./AntigravityInstallation.ts";
4548
import * as ModelManifest from "./ModelManifest.ts";
@@ -80,6 +83,7 @@ const disabledDefaultSlots = (
8083
);
8184

8285
const defaultClaudeSettings: ClaudeSettings = Schema.decodeSync(ClaudeSettings)({});
86+
const defaultGrokSettings: GrokSettings = Schema.decodeSync(GrokSettings)({ enabled: true });
8387
const defaultCodexSettings: CodexSettings = Schema.decodeSync(CodexSettings)({});
8488
const decodeCodexSettings = Schema.decodeSync(CodexSettings);
8589
const disabledCodexSettings: CodexSettings = Schema.decodeSync(CodexSettings)({
@@ -597,6 +601,7 @@ it.layer(
597601
const killCalls = yield* Ref.make(0);
598602
const statusFiber = yield* checkCodexProviderStatus(defaultCodexSettings).pipe(
599603
Effect.provide(layerHangingScopedSpawner(killCalls)),
604+
Effect.result,
600605
Effect.forkChild,
601606
);
602607

@@ -605,16 +610,44 @@ it.layer(
605610
yield* Effect.yieldNow;
606611

607612
const status = yield* Fiber.join(statusFiber);
608-
assert.strictEqual(status.status, "error");
613+
assert.strictEqual(Result.isFailure(status), true);
609614
assert.strictEqual(
610-
status.message,
611-
"Timed out while checking Codex app-server provider status.",
615+
Result.isFailure(status) ? status.failure._tag : null,
616+
"ProviderProbeTimeoutError",
612617
);
613618
assert.strictEqual(yield* Ref.get(killCalls), 1);
614619
}),
615620
);
616621
});
617622

623+
it.effect.each(["Claude Agent", "Grok"] as const)(
624+
"closes the %s version probe scope on timeout",
625+
(provider) =>
626+
Effect.gen(function* () {
627+
const killCalls = yield* Ref.make(0);
628+
const check = Effect.gen(function* () {
629+
if (provider === "Grok") {
630+
return yield* checkGrokProviderStatus(defaultGrokSettings);
631+
}
632+
return yield* checkClaudeProviderStatus(defaultClaudeSettings);
633+
});
634+
const statusFiber = yield* check.pipe(
635+
Effect.provide(layerHangingScopedSpawner(killCalls)),
636+
Effect.flip,
637+
Effect.forkChild,
638+
);
639+
640+
yield* Effect.yieldNow;
641+
yield* TestClock.adjust("4 seconds");
642+
const timeout = yield* Fiber.join(statusFiber);
643+
assert.strictEqual(timeout._tag, "ProviderProbeTimeoutError");
644+
assert.strictEqual(timeout.provider, provider);
645+
assert.strictEqual(timeout.timeoutMs, 4_000);
646+
assert.strictEqual(timeout.installed, true);
647+
assert.strictEqual(yield* Ref.get(killCalls), 1);
648+
}),
649+
);
650+
618651
describe("ProviderRegistry.layer", () => {
619652
it("stores workspace skills and commands without changing machine metadata", () => {
620653
const provider = {

‎docs/internals/providers.md‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -49,6 +49,18 @@ client connections and provider-instance rebuilds. Releases are immutable, with
4949
selecting the version for new processes. Running processes hold leases on their version. Updates
5050
and removal must respect those leases instead of replacing executables under a running agent.
5151

52+
## Status probe deadlines
53+
54+
A missed status deadline does not establish that a provider is broken. Checks that would
55+
otherwise publish a failure verdict should raise
56+
[`ProviderProbeTimeoutError`](../../packages/provider-core/src/server/snapshotProbe.ts).
57+
The [managed provider](../../packages/provider-core/src/server/managedProvider.ts) carries
58+
forward the last usable status and marks the check stale. Publishing an error instead would
59+
persist that guess in the per-instance cache and hide a working provider after a restart.
60+
Before any usable check, report the timeout plainly; definite failures such as a missing
61+
executable or rejected credentials still replace the previous status. Metadata-only probes
62+
that already degrade without marking the provider broken can keep that behavior.
63+
5264
## Setup must not happen as a health-check side effect
5365

5466
Opening a provider session can start MCP servers, run hooks, or launch a login browser.

0 commit comments

Comments
 (0)