Skip to content

Commit 0dfb120

Browse files
committed
fix(delegation): require callers to choose from model pools
Build: pingdotgg#7751
1 parent 47922df commit 0dfb120

19 files changed

Lines changed: 398 additions & 300 deletions

‎apps/server/src/agents/AgentRoleResolver.test.ts‎

Lines changed: 62 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1008,6 +1008,68 @@ it.layer(Resolver.layer)("AgentRoleResolver", (it) => {
10081008
}),
10091009
);
10101010

1011+
it.effect("requires a caller choice for multi-model roles regardless of list order or outages", () =>
1012+
Effect.gen(function* () {
1013+
const resolver = yield* Resolver.AgentRoleResolver;
1014+
const original = snapshot();
1015+
const build = (models: string[], providers: ReadonlyArray<ServerProvider> = [codex, claude]) =>
1016+
input({
1017+
requireExplicitSelection: true,
1018+
snapshot: {
1019+
...original,
1020+
manifest: {
1021+
...original.manifest,
1022+
lists: {
1023+
...original.manifest.lists,
1024+
workhorse: models.map((model) => ({ model, effort: "medium" })),
1025+
},
1026+
},
1027+
},
1028+
providers,
1029+
});
1030+
const omitted = yield* resolver.preview(build(["sonnet", "sol"]));
1031+
const permuted = yield* resolver.preview(build(["sol", "sonnet"]));
1032+
expect(omitted.selection).toBeNull();
1033+
expect(omitted.refusal).toContain("choose a model");
1034+
expect(omitted.candidates.map(({ model }) => model).sort()).toEqual(
1035+
permuted.candidates.map(({ model }) => model).sort(),
1036+
);
1037+
const offline = yield* refusal(
1038+
build(["sonnet", "sol"], [{ ...codex, availability: "unavailable" }, claude]),
1039+
);
1040+
expect(offline.detail).toContain("choose a model");
1041+
expect(
1042+
(yield* resolver.resolve({ ...build(["sonnet", "sol"]), selection: { model: codexModel } }))
1043+
.model,
1044+
).toBe(codexModel);
1045+
}),
1046+
);
1047+
1048+
it.effect("counts one configured model once across aliases and accounts", () =>
1049+
Effect.gen(function* () {
1050+
const original = snapshot();
1051+
const policy = {
1052+
...original,
1053+
manifest: {
1054+
...original.manifest,
1055+
models: { ...original.manifest.models, solCopy: original.manifest.models.sol! },
1056+
lists: {
1057+
...original.manifest.lists,
1058+
workhorse: [
1059+
{ model: "sol", effort: "medium" },
1060+
{ model: "solCopy", effort: "medium" },
1061+
],
1062+
},
1063+
},
1064+
};
1065+
const other = { ...codex, instanceId: ProviderInstanceId.make("codex-copy") };
1066+
const result = yield* resolve(
1067+
input({ snapshot: policy, providers: [codex, other], requireExplicitSelection: true }),
1068+
);
1069+
expect(result.model).toBe(codexModel);
1070+
}),
1071+
);
1072+
10111073
it.effect("delegates to the member whose usage resets soonest", () =>
10121074
Effect.gen(function* () {
10131075
const usage = (usedPercent: number, resetsAt: string) => ({

‎apps/server/src/agents/AgentRoleResolver.ts‎

Lines changed: 30 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -89,6 +89,8 @@ type ResolutionInput = {
8989
*/
9090
readonly caller?: ModelSelection;
9191
readonly selection?: RequestedModel;
92+
/** Ordinary discovery and new launches require an explicit task-fit choice for multi-model roles. */
93+
readonly requireExplicitSelection?: boolean;
9294
/**
9395
* A running trial's assigned model and effort for this task. It replaces the role's list, so a
9496
* trial can compare what Delegation does not choose today;
@@ -447,6 +449,12 @@ const inspect = Effect.fn("AgentRoleResolver.inspect")(function* (input: Resolut
447449
});
448450
}
449451
}
452+
const choices = new Set(
453+
list.map(({ model: alias }) => {
454+
const binding = snapshot.manifest.models[alias]!;
455+
return `${binding.driverKind}:${binding.model}`;
456+
}),
457+
);
450458
const nowMs = DateTime.toEpochMillis(yield* DateTime.now);
451459
// Which account serves each model follows the pool rule a new pooled thread uses: a member its
452460
// pool switches off, or an account out of usage (at its pool's reserve), never serves while
@@ -580,7 +588,7 @@ const inspect = Effect.fn("AgentRoleResolver.inspect")(function* (input: Resolut
580588
}
581589
// The list order, unless the asking model has its own first choice.
582590
const callerFirst =
583-
input.callerModel === undefined
591+
input.requireExplicitSelection || input.callerModel === undefined
584592
? undefined
585593
: Object.entries(snapshot.manifest.models)
586594
.filter(([, binding]) => binding.model === input.callerModel)
@@ -647,6 +655,15 @@ const inspect = Effect.fn("AgentRoleResolver.inspect")(function* (input: Resolut
647655
};
648656
});
649657
}
658+
if (input.requireExplicitSelection && input.selection?.model === undefined && choices.size > 1) {
659+
return yield* new AgentRoleError({
660+
reason: "selection-denied",
661+
candidates: finalDiagnostics(undefined).map((candidate) =>
662+
candidate.reason === "lower-preference" ? { ...candidate, reason: "eligible" } : candidate,
663+
),
664+
detail: `choose a model for ${request.role} based on this task; available models and their notes are listed by agent_roles`,
665+
});
666+
}
650667
const chosen = candidates[0];
651668
if (chosen === undefined)
652669
return yield* excluded.size > 0 && eligible.every((candidate) => excluded.has(candidate))
@@ -682,13 +699,22 @@ export const layer = Layer.succeed(AgentRoleResolver, {
682699
preview: Effect.fn("AgentRoleResolver.preview")((input) =>
683700
inspect(input).pipe(
684701
Effect.catchTags({
685-
AgentRoleError: (error) =>
686-
Effect.succeed({
702+
AgentRoleError: (error) => {
703+
const admission = managedExecution({
704+
readOnly:
705+
input.request.readOnly === true ||
706+
input.snapshot.manifest.roles[input.request.role]?.sourceAccess === "read-only",
707+
requested: input.request.executionRequirement,
708+
});
709+
return Effect.succeed({
687710
selection: null,
688711
refusal: error.detail === undefined ? error.reason : `${error.reason}: ${error.detail}`,
689712
candidates: error.candidates ?? [],
690713
ranked: [],
691-
}),
714+
executionRequirement: admission.executionRequirement,
715+
executionBoundary: admission.executionBoundary,
716+
});
717+
},
692718
}),
693719
),
694720
),

‎apps/server/src/agents/ManagedAgentService.test.ts‎

Lines changed: 15 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -1285,9 +1285,10 @@ it.layer(testLayer)("ManagedAgentService", (it) => {
12851285
Effect.provideService(HostProcess.Environment, f.environment),
12861286
Effect.map((result) => result.roles.find((role) => role.role === "reviewer")!),
12871287
);
1288-
expect((yield* reviewer()).selection?.model).toBe(claudeModel);
1289-
expect((yield* reviewer("change")).selection?.model).toBe(claudeModel);
1290-
expect((yield* reviewer("spec")).selection?.model).toBe("claude-fable-test");
1288+
expect((yield* reviewer()).selection).toBeNull();
1289+
expect((yield* reviewer("change")).selection).toBeNull();
1290+
expect((yield* reviewer("spec")).selection).toBeNull();
1291+
expect((yield* reviewer("spec")).refusal).toContain("choose a model");
12911292
// Discovery names the permission class, and a tool profile only for the narrow web readers.
12921293
expect((yield* reviewer()).permissionClass).toBe("read-only");
12931294
expect((yield* reviewer()).toolProfile).toBeUndefined();
@@ -1322,7 +1323,8 @@ it.layer(testLayer)("ManagedAgentService", (it) => {
13221323
})
13231324
.pipe(Effect.provideService(HostProcess.Environment, f.environment));
13241325
const reviewer = discovered.roles.find((role) => role.role === "reviewer")!;
1325-
expect(reviewer.selection?.model).toBe(codexModel);
1326+
expect(reviewer.selection).toBeNull();
1327+
expect(reviewer.refusal).toContain("choose a model");
13261328
expect(reviewer.usageLimited).toContainEqual(
13271329
expect.objectContaining({ model: claudeModel, providerInstanceId: claude.instanceId }),
13281330
);
@@ -1331,6 +1333,12 @@ it.layer(testLayer)("ManagedAgentService", (it) => {
13311333
{ model: claudeModel, effort: "medium", available: false },
13321334
{ model: codexModel, effort: "medium", note: "Careful coder", available: true },
13331335
]);
1336+
const omitted = yield* f.prepare({
1337+
requireExplicitSelection: true,
1338+
request: request({ role: "implementer", workspacePath: f.workspacePath }),
1339+
}).pipe(Effect.flip);
1340+
expect(omitted.reason).toBe("selection");
1341+
expect(omitted.message).toContain("choose a model");
13341342
// A choice that must start now, such as a Champion replay, skips the exhausted account.
13351343
const ranked = yield* f.service
13361344
.ranked({
@@ -1725,7 +1733,7 @@ it.layer(testLayer)("ManagedAgentService", (it) => {
17251733
}),
17261734
);
17271735

1728-
it.effect("discovers the model a launch for the same caller would use", () =>
1736+
it.effect("does not infer an ordinary choice from the caller's model", () =>
17291737
Effect.gen(function* () {
17301738
const f = yield* fixture;
17311739
const advisorFor = (callerModel: string) =>
@@ -1740,9 +1748,8 @@ it.layer(testLayer)("ManagedAgentService", (it) => {
17401748
Effect.provideService(HostProcess.Environment, f.environment),
17411749
Effect.map((result) => result.roles.find((role) => role.role === "advisor")!),
17421750
);
1743-
// The policy starts an Opus caller's advisor with Astra; others follow the list.
1744-
expect((yield* advisorFor("claude-opus-test")).selection?.model).toBe("gpt-astra-test");
1745-
expect((yield* advisorFor(claudeModel)).selection?.model).toBe("claude-fable-test");
1751+
expect((yield* advisorFor("claude-opus-test")).selection).toBeNull();
1752+
expect((yield* advisorFor(claudeModel)).selection).toBeNull();
17461753
}),
17471754
);
17481755
it.effect("records an earlier review's change against a new base only when the diffs match", () =>

‎apps/server/src/agents/ManagedAgentService.ts‎

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -199,6 +199,7 @@ export class ManagedAgentService extends Context.Service<
199199
readonly callerModel?: string;
200200
readonly caller?: ModelSelection;
201201
readonly selection?: Resolver.RequestedModel;
202+
readonly requireExplicitSelection?: boolean;
202203
},
203204
) => Effect.Effect<{ binding: AgentTaskBinding; brief: string }, ManagedAgentError>;
204205
readonly validate: (
@@ -638,6 +639,7 @@ const make = Effect.gen(function* () {
638639
role: AgentTaskRequest["role"],
639640
policy: ReturnType<typeof configuredRoles>[number]["policy"],
640641
brief: ReturnType<typeof configuredRoles>[number]["brief"],
642+
requireExplicitSelection = true,
641643
) {
642644
const context = yield* execution(
643645
{
@@ -670,6 +672,7 @@ const make = Effect.gen(function* () {
670672
...(browser === undefined ? {} : { browser }),
671673
providers: input.providers,
672674
...(input.selection === undefined ? {} : { selection: input.selection }),
675+
requireExplicitSelection,
673676
...(input.callerModel === undefined ? {} : { callerModel: input.callerModel }),
674677
...(input.caller === undefined ? {} : { caller: input.caller }),
675678
request: {
@@ -855,7 +858,7 @@ const make = Effect.gen(function* () {
855858
const configured = configuredRoles(snapshot).find(({ role }) => role === input.role);
856859
if (configured === undefined) return [];
857860
const { policy, brief } = configured;
858-
return (yield* discover(input, snapshot, input.role, policy, brief)).ranked;
861+
return (yield* discover(input, snapshot, input.role, policy, brief, false)).ranked;
859862
}),
860863
otherModels: Effect.fn("ManagedAgentService.otherModels")(
861864
function* (input) {
@@ -962,6 +965,8 @@ const make = Effect.gen(function* () {
962965
...(input.recordedBinding === undefined
963966
? {}
964967
: { selection: input.recordedBinding.requested }),
968+
requireExplicitSelection:
969+
input.requireExplicitSelection === true && input.recordedBinding === undefined,
965970
})
966971
.pipe(Effect.mapError((cause) => new ManagedAgentError({ reason: "selection", cause })));
967972
yield* store

‎apps/server/src/champions/ChampionService.test.ts‎

Lines changed: 20 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -3849,31 +3849,19 @@ it.effect(
38493849
const classes = yield* decodeClasses({}).pipe(Effect.orDie);
38503850
const preview = yield* service.classChoices({ projectId });
38513851
assert.isNull(preview.refusal);
3852-
assert.equal(
3853-
preview.roles.find((role) => role.role === "advisor")!.selection!.model,
3854-
"gpt-astra-test",
3855-
);
3852+
assert.isNull(preview.roles.find((role) => role.role === "advisor")!.selection);
38563853
const draft = yield* service.upsert({
38573854
...config,
38583855
classes,
38593856
setupModelSelection: { instanceId: codex.instanceId, model: "historical-model" },
38603857
});
3861-
assert.deepEqual(
3862-
draft.champion.setupModelSelection,
3863-
preview.roles.find((role) => role.role === "advisor")!.selection,
3864-
);
3865-
assert.deepEqual(
3866-
draft.champion.routineModelSelection,
3867-
preview.roles.find((role) => role.role === "implementer")!.selection,
3868-
);
3858+
assert.equal(draft.champion.setupModelSelection.model, "gpt-astra-test");
3859+
assert.equal(draft.champion.routineModelSelection.model, codexModel);
38693860
f.failNextLaunch();
38703861
yield* service.setup({ id: config.id });
38713862
const first = (yield* service.runtimeState({ id: config.id })).attempts[0]!;
38723863
assert.equal(first.resolvedSelection.model, "gpt-astra-test");
3873-
assert.deepEqual(
3874-
first.resolvedSelection,
3875-
preview.roles.find((role) => role.role === "advisor")!.selection,
3876-
);
3864+
assert.deepEqual(first.resolvedSelection, draft.champion.setupModelSelection);
38773865
assert.equal(first.policyHash, preview.policyHash);
38783866
yield* refused(
38793867
service.classChoices({
@@ -3883,7 +3871,10 @@ it.effect(
38833871
);
38843872
// The implementer takes UI work like any other.
38853873
const ui = yield* service.classChoices({ projectId, taskClass: "ui" });
3886-
assert.isNull(ui.roles.find((role) => role.role === "implementer")!.refusal);
3874+
assert.include(
3875+
ui.roles.find((role) => role.role === "implementer")!.refusal,
3876+
"selection-denied",
3877+
);
38873878
f.ownerMode("approval-required");
38883879
const narrowed = yield* service.classChoices({
38893880
projectId,
@@ -3978,9 +3969,14 @@ it.effect("starts a metrics lead thread on the lead role's current model and rec
39783969
yield* Effect.gen(function* () {
39793970
const service = yield* Champions.ChampionService;
39803971
yield* service.upsert({ ...config, classes: yield* decodeClasses({}).pipe(Effect.orDie) });
3981-
const lead = (yield* service.classChoices({ projectId })).roles.find(
3982-
(role) => role.role === "lead",
3983-
)!.selection!;
3972+
const [lead] = yield* managed.ranked({
3973+
role: "lead",
3974+
projectId,
3975+
providers: [codex],
3976+
ceiling: "full-access",
3977+
request: { taskClass: "non-ui", runtimeMode: "full-access" },
3978+
});
3979+
assert.isDefined(lead);
39843980
const { threadId } = yield* service.startMetricsLead({ id: config.id });
39853981
assert.equal(f.counts().launches, 1);
39863982
assert.equal(f.launchRequests[0]!.model, lead.model);
@@ -4022,17 +4018,14 @@ it.effect("starts a metrics lead thread on the lead role's current model and rec
40224018
findings: [],
40234019
summary: "Breached",
40244020
})).findings[0]!;
4025-
const setupClass = (yield* service.get({ id: config.id })).champion.classes!.setup;
40264021
const advice = {
40274022
ownerThreadId: f.caller().threadId,
40284023
clientRequestId: "advice",
40294024
request: { ...repair(breach), role: "advisor" as const, readOnly: true },
40304025
};
40314026
assert.deepEqual(
40324027
yield* service.guardDelegation(advice),
4033-
(yield* service.classChoices({ projectId })).roles.find(
4034-
(role) => role.role === setupClass.role,
4035-
)!.selection!,
4028+
(yield* service.get({ id: config.id })).champion.setupModelSelection,
40364029
);
40374030
const launched = f.counts().launches;
40384031

@@ -4415,6 +4408,7 @@ const reviewedFinding = (
44154408
const eligible = new Set(["cheap", "strong"]);
44164409
const asked: Array<string> = [];
44174410
const managed = {
4411+
ranked: () => Effect.succeed(eligible.has("cheap") ? [{ instanceId, model: "cheap" }] : []),
44184412
roles: (input: {
44194413
readonly request: { readonly role?: string };
44204414
readonly selection?: { readonly model?: string };
@@ -6029,6 +6023,7 @@ describe("owner measurement on a live check", () => {
60296023
eligible = new Set(["cheap", "strong"]),
60306024
) => {
60316025
const managed = {
6026+
ranked: () => Effect.succeed(eligible.has("cheap") ? [{ instanceId, model: "cheap" }] : []),
60326027
roles: (input: {
60336028
readonly request: { readonly role?: string };
60346029
readonly selection?: { readonly model?: string };
@@ -6350,6 +6345,7 @@ describe("owner measurement on a live check", () => {
63506345

63516346
it.effect("charges measurements and investigation branches against one aggregate", () => {
63526347
const managed = {
6348+
ranked: () => Effect.succeed([{ instanceId, model: "cheap" }]),
63536349
roles: (input: { readonly request: { readonly role?: string } }) =>
63546350
Effect.succeed({
63556351
policyHash: "a".repeat(64),

0 commit comments

Comments
 (0)