Skip to content

Commit 78c8f2f

Browse files
committed
fix(web): retry automatic machine selection after resource failures
1 parent c14f601 commit 78c8f2f

5 files changed

Lines changed: 247 additions & 23 deletions

File tree

‎apps/web/src/components/ChatView.tsx‎

Lines changed: 27 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -3892,7 +3892,7 @@ export default function ChatView(props: ChatViewProps) {
38923892
? "Auto balance"
38933893
: loadBalancing.pending
38943894
? "Checking machines…"
3895-
: loadBalancing.failed
3895+
: loadBalancing.status === "unavailable" || loadBalancing.status === "no-candidates"
38963896
? "Auto balance unavailable"
38973897
: "Auto balance"
38983898
: undefined;
@@ -7325,12 +7325,32 @@ export default function ChatView(props: ChatViewProps) {
73257325
if (needsLoadBalancing) {
73267326
toastManager.add({
73277327
type: "warning",
7328-
title: loadBalancing.pending
7329-
? "Checking machine resources"
7330-
: "Choose a machine to continue",
7331-
description: loadBalancing.pending
7332-
? "Resource checks are still running. You can choose a machine in the composer."
7333-
: "No eligible machine has available resources. Choose a machine in the composer to override.",
7328+
...{
7329+
checking: {
7330+
title: "Checking machine resources",
7331+
description:
7332+
"Resource checks are still running. You can choose a machine in the composer.",
7333+
},
7334+
selected: {
7335+
title: "Selecting a machine",
7336+
description: "A machine is ready. Try sending again in a moment.",
7337+
},
7338+
"no-candidates": {
7339+
title: "No eligible machines",
7340+
description:
7341+
"No connected machine for this project has the selected provider available and auto balancing enabled. Check Connections and auto-balance preferences, or choose a machine in the composer.",
7342+
},
7343+
unavailable: {
7344+
title: "Machine resources unavailable",
7345+
description:
7346+
"Some machine resource readings are missing, outdated, or failed. Retrying automatically. You can choose a machine in the composer.",
7347+
},
7348+
"at-capacity": {
7349+
title: "Machines are busy",
7350+
description:
7351+
"All eligible machines have at least 95% CPU usage or at most 5% available memory. Retrying automatically. Choose a machine in the composer to override.",
7352+
},
7353+
}[loadBalancing.status],
73347354
});
73357355
return;
73367356
}
Lines changed: 135 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,135 @@
1+
import { RegistryContext } from "@effect/atom-react";
2+
import { EnvironmentId, type HostResourcesSnapshot } from "@t3tools/contracts";
3+
import { Effect } from "effect";
4+
import { Atom, AtomRegistry } from "effect/unstable/reactivity";
5+
import { act, useLayoutEffect } from "react";
6+
import { create, type ReactTestRenderer } from "react-test-renderer";
7+
import { afterEach, beforeEach, expect, it, vi } from "vite-plus/test";
8+
9+
import { serverEnvironment } from "../state/server";
10+
import { useLoadBalancedEnvironment } from "./useLoadBalancedEnvironment";
11+
12+
vi.mock("../state/server", () => ({ serverEnvironment: { hostResources: vi.fn() } }));
13+
14+
const environmentId = EnvironmentId.make("remote");
15+
const candidates = [environmentId];
16+
const weights = {};
17+
const healthy: HostResourcesSnapshot = {
18+
sampledAt: 0,
19+
cpuUtilization: 0.3,
20+
cpuCount: 12,
21+
availableMemoryBytes: 20_000,
22+
totalMemoryBytes: 32_000,
23+
};
24+
let response: HostResourcesSnapshot | Error;
25+
let reads: number;
26+
let registry: AtomRegistry.AtomRegistry;
27+
let renderer: ReactTestRenderer;
28+
let result: ReturnType<typeof useLoadBalancedEnvironment>;
29+
30+
function Probe({ ids = candidates }: { ids?: readonly EnvironmentId[] }) {
31+
const value = useLoadBalancedEnvironment(ids, weights);
32+
useLayoutEffect(() => {
33+
result = value;
34+
});
35+
return null;
36+
}
37+
38+
beforeEach(() => {
39+
vi.useFakeTimers();
40+
vi.stubGlobal("IS_REACT_ACT_ENVIRONMENT", true);
41+
reads = 0;
42+
registry = AtomRegistry.make();
43+
const resources = Atom.make(
44+
Effect.suspend(() => {
45+
reads++;
46+
return response instanceof Error ? Effect.fail(response) : Effect.succeed(response);
47+
}),
48+
).pipe(Atom.setIdleTTL(0));
49+
vi.mocked(serverEnvironment.hostResources).mockReturnValue(resources);
50+
});
51+
52+
afterEach(async () => {
53+
await act(() => renderer?.unmount());
54+
registry.dispose();
55+
vi.useRealTimers();
56+
vi.unstubAllGlobals();
57+
vi.clearAllMocks();
58+
});
59+
60+
async function mount(ids = candidates) {
61+
await act(() => {
62+
renderer = create(
63+
<RegistryContext.Provider value={registry}>
64+
<Probe ids={ids} />
65+
</RegistryContext.Provider>,
66+
);
67+
});
68+
}
69+
70+
it.each([
71+
["CPU saturation", { ...healthy, cpuUtilization: 0.98 }],
72+
["memory pressure", { ...healthy, availableMemoryBytes: 1_000 }],
73+
["a failed resource request", new Error("Resource check timed out")],
74+
] as const)(
75+
"selects the recovered host after %s without reopening the draft",
76+
async (_, initial) => {
77+
response = initial;
78+
await mount();
79+
expect(result.environmentId).toBeNull();
80+
expect(result.pending).toBe(false);
81+
expect(reads).toBe(1);
82+
83+
response = healthy;
84+
await act(() => vi.advanceTimersByTimeAsync(5_000));
85+
expect(result.environmentId).toBe(environmentId);
86+
expect(reads).toBe(2);
87+
88+
await act(() => vi.advanceTimersByTimeAsync(30_000));
89+
expect(reads).toBe(2);
90+
},
91+
);
92+
93+
it("does not request resources with no eligible machines", async () => {
94+
response = healthy;
95+
await mount([]);
96+
await act(() => vi.advanceTimersByTimeAsync(30_000));
97+
expect(reads).toBe(0);
98+
expect(result.status).toBe("no-candidates");
99+
});
100+
101+
it("keeps retrying failed readings until the host recovers", async () => {
102+
response = new Error("Resource check timed out");
103+
await mount();
104+
expect(result.status).toBe("unavailable");
105+
await act(() => vi.advanceTimersByTimeAsync(5_000));
106+
expect(reads).toBe(2);
107+
expect(result.status).toBe("unavailable");
108+
response = healthy;
109+
await act(() => vi.advanceTimersByTimeAsync(5_000));
110+
expect(result.environmentId).toBe(environmentId);
111+
expect(reads).toBe(3);
112+
});
113+
114+
it("cancels retries when automatic selection is disabled", async () => {
115+
response = { ...healthy, cpuUtilization: 1 };
116+
await mount();
117+
expect(result.status).toBe("at-capacity");
118+
await act(() => {
119+
renderer.update(
120+
<RegistryContext.Provider value={registry}>
121+
<Probe ids={[]} />
122+
</RegistryContext.Provider>,
123+
);
124+
});
125+
await act(() => vi.advanceTimersByTimeAsync(30_000));
126+
expect(reads).toBe(1);
127+
});
128+
129+
it("stops retrying when the draft closes", async () => {
130+
response = new Error("Resource check timed out");
131+
await mount();
132+
await act(() => renderer.unmount());
133+
await act(() => vi.advanceTimersByTimeAsync(30_000));
134+
expect(reads).toBe(1);
135+
});

‎apps/web/src/hooks/useLoadBalancedEnvironment.ts‎

Lines changed: 14 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@ import { RegistryContext, useAtomValue } from "@effect/atom-react";
22
import { chooseLoadBalancedEnvironment } from "@t3tools/client-runtime/load-balancing";
33
import type { EnvironmentId } from "@t3tools/contracts";
44
import { Atom } from "effect/unstable/reactivity";
5-
import { useCallback, useContext, useMemo } from "react";
5+
import { useCallback, useContext, useEffect, useMemo } from "react";
66

77
import { serverEnvironment } from "../state/server";
88

@@ -30,25 +30,34 @@ export function useLoadBalancedEnvironment(
3030
resources: result._tag === "Success" ? result.value : null,
3131
receivedAt: result._tag === "Success" ? result.timestamp : 0,
3232
pending: result._tag === "Initial" || result.waiting,
33-
failed: result._tag === "Failure",
3433
};
3534
}),
3635
),
3736
[environmentIds],
3837
);
3938
const resources = useAtomValue(resourcesAtom);
4039
const pending = resources.some((resource) => resource.pending);
41-
const environmentId = chooseLoadBalancedEnvironment(
40+
const selection = chooseLoadBalancedEnvironment(
4241
resources.map((resource) => ({
4342
...resource,
4443
weight: weights[resource.environmentId] ?? 50,
4544
})),
4645
Date.now(),
47-
) as EnvironmentId | null;
46+
);
47+
const environmentId = selection.environmentId as EnvironmentId | null;
48+
useEffect(() => {
49+
if (resources.length === 0 || pending || environmentId !== null) return;
50+
// A rejected sample must not strand the draft after the host recovers.
51+
const timer = setTimeout(
52+
() => refresh(resources.map((resource) => resource.environmentId)),
53+
5_000,
54+
);
55+
return () => clearTimeout(timer);
56+
}, [resources, pending, environmentId, refresh]);
4857
return {
4958
refresh,
5059
pending,
5160
environmentId,
52-
failed: !pending && environmentId === null && resources.some((resource) => resource.failed),
61+
status: pending ? ("checking" as const) : selection.status,
5362
};
5463
}

‎packages/client-runtime/src/load-balancing.ts‎

Lines changed: 21 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -10,31 +10,46 @@ export function chooseLoadBalancedEnvironment(
1010
weight: number;
1111
}>,
1212
now: number,
13-
): string | null {
13+
): {
14+
environmentId: string | null;
15+
status: "selected" | "no-candidates" | "unavailable" | "at-capacity";
16+
} {
1417
let selected: string | null = null;
1518
let bestScore = 0;
19+
let eligible = 0;
20+
let unavailable = false;
1621
for (const { environmentId, resources, receivedAt, weight } of candidates) {
22+
if (!Number.isFinite(weight) || weight <= 0) continue;
23+
eligible++;
1724
const sampledAt = receivedAt ?? resources?.sampledAt ?? 0;
1825
if (
1926
!resources ||
20-
!Number.isFinite(weight) ||
21-
weight <= 0 ||
2227
now - sampledAt > 15_000 ||
2328
sampledAt > now + 5_000 ||
2429
resources.cpuUtilization === null ||
25-
resources.cpuUtilization >= 0.95 ||
2630
resources.totalMemoryBytes <= 0 ||
2731
resources.cpuCount <= 0
2832
) {
33+
unavailable = true;
2934
continue;
3035
}
3136
const memoryAvailable = resources.availableMemoryBytes / resources.totalMemoryBytes;
32-
if (memoryAvailable <= 0.05) continue;
37+
if (resources.cpuUtilization >= 0.95 || memoryAvailable <= 0.05) continue;
3338
const score = weight * resources.cpuCount * (1 - resources.cpuUtilization) * memoryAvailable;
3439
if (score > bestScore) {
3540
selected = environmentId;
3641
bestScore = score;
3742
}
3843
}
39-
return selected;
44+
return {
45+
environmentId: selected,
46+
status:
47+
selected !== null
48+
? "selected"
49+
: eligible === 0
50+
? "no-candidates"
51+
: unavailable
52+
? "unavailable"
53+
: "at-capacity",
54+
};
4055
}

‎packages/client-runtime/src/state/projectGrouping.test.ts‎

Lines changed: 50 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -27,8 +27,8 @@ describe("load balancing shared project machines", () => {
2727
{ environmentId: "idle", resources, weight: 1 },
2828
{ environmentId: "preferred", resources: { ...resources, cpuCount: 4 }, weight: 3 },
2929
];
30-
expect(chooseLoadBalancedEnvironment(candidates, now)).toBe("preferred");
31-
expect(chooseLoadBalancedEnvironment(candidates.slice(0, 2), now)).toBe("idle");
30+
expect(chooseLoadBalancedEnvironment(candidates, now).environmentId).toBe("preferred");
31+
expect(chooseLoadBalancedEnvironment(candidates.slice(0, 2), now).environmentId).toBe("idle");
3232
});
3333

3434
it("rejects stale, unknown, excluded and saturated machines", () => {
@@ -59,7 +59,7 @@ describe("load balancing shared project machines", () => {
5959
},
6060
],
6161
now,
62-
),
62+
).environmentId,
6363
).toBeNull();
6464
});
6565

@@ -70,8 +70,53 @@ describe("load balancing shared project machines", () => {
7070
receivedAt: now,
7171
weight: 1,
7272
};
73-
expect(chooseLoadBalancedEnvironment([candidate], now)).toBe("different-clock");
74-
expect(chooseLoadBalancedEnvironment([candidate], now + 15_001)).toBeNull();
73+
expect(chooseLoadBalancedEnvironment([candidate], now).environmentId).toBe("different-clock");
74+
expect(chooseLoadBalancedEnvironment([candidate], now + 15_001).environmentId).toBeNull();
75+
});
76+
77+
it("distinguishes no eligible machines from missing readings and resource limits", () => {
78+
const candidate = { environmentId: "remote", resources, weight: 50 };
79+
expect(chooseLoadBalancedEnvironment([], now).status).toBe("no-candidates");
80+
expect(chooseLoadBalancedEnvironment([{ ...candidate, weight: 0 }], now).status).toBe(
81+
"no-candidates",
82+
);
83+
for (const snapshot of [
84+
null,
85+
{ ...resources, sampledAt: now - 15_001 },
86+
{ ...resources, sampledAt: now + 5_001 },
87+
{ ...resources, cpuUtilization: null },
88+
{ ...resources, cpuCount: 0 },
89+
{ ...resources, totalMemoryBytes: 0 },
90+
]) {
91+
expect(
92+
chooseLoadBalancedEnvironment([{ ...candidate, resources: snapshot }], now).status,
93+
).toBe("unavailable");
94+
}
95+
for (const snapshot of [
96+
{ ...resources, cpuUtilization: 0.95 },
97+
{ ...resources, availableMemoryBytes: resources.totalMemoryBytes * 0.05 },
98+
]) {
99+
expect(
100+
chooseLoadBalancedEnvironment([{ ...candidate, resources: snapshot }], now).status,
101+
).toBe("at-capacity");
102+
}
103+
expect(chooseLoadBalancedEnvironment([candidate], now).status).toBe("selected");
104+
});
105+
106+
it("does not claim all hosts are busy when a host's reading is unavailable", () => {
107+
const busy = {
108+
environmentId: "busy",
109+
resources: { ...resources, cpuUtilization: 1 },
110+
weight: 50,
111+
};
112+
const unknown = { environmentId: "unknown", resources: null, weight: 50 };
113+
expect(chooseLoadBalancedEnvironment([busy, unknown], now).status).toBe("unavailable");
114+
expect(
115+
chooseLoadBalancedEnvironment(
116+
[busy, unknown, { environmentId: "healthy", resources, weight: 50 }],
117+
now,
118+
),
119+
).toEqual({ environmentId: "healthy", status: "selected" });
75120
});
76121
});
77122
const repositoryIdentity = {

0 commit comments

Comments
 (0)