Skip to content

Commit be5b247

Browse files
committed
test(server): exercise early recovery through the worker sweep
1 parent 649f714 commit be5b247

1 file changed

Lines changed: 75 additions & 3 deletions

File tree

‎apps/server/src/orchestration-v2/runtimeLayer.test.ts‎

Lines changed: 75 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,7 @@
11
import { SourceControlProviderKind } from "@t3tools/contracts";
22
import { limitRecoveryCommand } from "./UsageLimitRecoveryWorker.ts";
3+
import * as UsageLimitRecoveryWorker from "./UsageLimitRecoveryWorker.ts";
4+
import * as ProviderRegistry from "../provider/ProviderRegistry.ts";
35
import * as McpProviderSessions from "@t3tools/provider-core/server/McpProviderSessions";
46
import * as SourceControlProviderRegistry from "../sourceControl/SourceControlProviderRegistry.ts";
57
import * as NodeServices from "@effect/platform-node/NodeServices";
@@ -18,6 +20,7 @@ import {
1820
TurnItemId,
1921
type ModelSelection,
2022
type OrchestrationV2Run,
23+
type ServerProviderUsageLimits,
2124
ProjectId,
2225
type PullRequestDetail,
2326
type PullRequestComment,
@@ -6281,6 +6284,45 @@ it.layer(layerTest)("usage-limit recovery", (it) => {
62816284
checkedAt: DateTime.formatIso(checkedAt),
62826285
windows: [{ id: "five_hour", kind: "session" as const, label: "5h", usedPercent }],
62836286
});
6287+
const sweep = (
6288+
usageLimits: ServerProviderUsageLimits,
6289+
instanceId = shell.providerInstanceId,
6290+
) =>
6291+
Effect.scoped(
6292+
Layer.build(
6293+
UsageLimitRecoveryWorker.layer.pipe(
6294+
Layer.provide([
6295+
Layer.succeed(ProjectionStore.ProjectionStoreV2, projections),
6296+
Layer.succeed(Orchestrator.OrchestratorV2, orchestrator),
6297+
Layer.mock(ThreadManagementService.ThreadManagementService)({
6298+
dispatch: orchestrator.dispatch,
6299+
}),
6300+
ServerSettings.layerTest(),
6301+
Layer.mock(ProviderRegistry.ProviderRegistry)({
6302+
getProviders: Effect.succeed([
6303+
{
6304+
instanceId,
6305+
driver,
6306+
enabled: true,
6307+
installed: true,
6308+
version: null,
6309+
status: "ready",
6310+
auth: { status: "authenticated" },
6311+
checkedAt: usageLimits.checkedAt,
6312+
models: [],
6313+
slashCommands: [],
6314+
skills: [],
6315+
usageLimits,
6316+
},
6317+
]),
6318+
}),
6319+
Layer.mock(Scheduler.Scheduler)({
6320+
register: (_name, work) => Effect.orDie(work),
6321+
}),
6322+
]),
6323+
),
6324+
),
6325+
);
62846326
const armed = (yield* projections.getLimitRecoveryCandidates({
62856327
now: checkedNow,
62866328
autoResume: true,
@@ -6291,9 +6333,34 @@ it.layer(layerTest)("usage-limit recovery", (it) => {
62916333
// Readings from before the failure, or ones that still show the limit, keep waiting.
62926334
assert.isNull(limitRecoveryCommand(armed!, true, nowMs, false, reading(now, 0)));
62936335
assert.isNull(limitRecoveryCommand(armed!, true, nowMs, false, reading(checkedNow, 100)));
6294-
const resume = limitRecoveryCommand(armed!, true, nowMs, false, reading(checkedNow, 0));
6295-
assert(resume?.type === "message.dispatch");
6296-
yield* orchestrator.dispatch(resume);
6336+
yield* sweep(reading(now, 0));
6337+
yield* sweep(reading(checkedNow, 100));
6338+
yield* sweep(reading(checkedNow, 0), alternateInstanceId);
6339+
assert.lengthOf((yield* orchestrator.getThreadProjection(threadId)).runs, 1);
6340+
const staleResume = limitRecoveryCommand(
6341+
armed!,
6342+
true,
6343+
nowMs,
6344+
false,
6345+
reading(checkedNow, 0),
6346+
);
6347+
assert(staleResume?.type === "message.dispatch");
6348+
yield* orchestrator.dispatch({
6349+
type: "thread.metadata.update",
6350+
commandId: CommandId.make("recovery:cancel:early-reset"),
6351+
threadId,
6352+
limitRecovery: { runId: run.id, resetAt, autoResume: false },
6353+
});
6354+
yield* orchestrator.dispatch(staleResume);
6355+
yield* sweep(reading(checkedNow, 0));
6356+
assert.lengthOf((yield* orchestrator.getThreadProjection(threadId)).runs, 1);
6357+
yield* orchestrator.dispatch({
6358+
type: "thread.metadata.update",
6359+
commandId: CommandId.make("recovery:rearm:early-reset"),
6360+
threadId,
6361+
limitRecovery: { runId: run.id, resetAt, autoResume: true },
6362+
});
6363+
yield* sweep(reading(checkedNow, 0));
62976364
const resumed = yield* orchestrator.getThreadProjection(threadId);
62986365
assert.lengthOf(resumed.runs, 2);
62996366
assert.equal(resumed.runs[1]?.status, "starting");
@@ -6360,6 +6427,8 @@ it.layer(layerTest)("usage-limit recovery", (it) => {
63606427
snooze: false,
63616428
})).find((row) => row.id === threadId)!;
63626429
assert.equal(rearmed.limitRecovery?.waitForReset, true);
6430+
yield* sweep(reading(laterNow, 0));
6431+
assert.lengthOf((yield* orchestrator.getThreadProjection(threadId)).runs, 2);
63636432
assert.isNull(
63646433
limitRecoveryCommand(
63656434
rearmed,
@@ -6372,6 +6441,9 @@ it.layer(layerTest)("usage-limit recovery", (it) => {
63726441
const onTime = limitRecoveryCommand(rearmed, true, Date.parse(resetAt));
63736442
assert(onTime?.type === "message.dispatch");
63746443
assert.isUndefined(onTime.usageLimitClearedAt);
6444+
yield* TestClock.setTime(Date.parse(resetAt));
6445+
yield* orchestrator.dispatch(onTime);
6446+
assert.lengthOf((yield* orchestrator.getThreadProjection(threadId)).runs, 3);
63756447
}),
63766448
);
63776449

0 commit comments

Comments
 (0)