Skip to content

Commit be3263a

Browse files
committed
fix(server): preserve link answers already in progress during cleanup
An older card may already be making a link when a retry completes. Coordinate answer validation with cleanup and protect active cards so their eventual link state and access are still reported, without blocking unrelated network operations.
1 parent badf0d4 commit be3263a

4 files changed

Lines changed: 279 additions & 86 deletions

File tree

‎apps/server/src/orchestration-v2/Orchestrator.ts‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8377,6 +8377,7 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
83778377
type: _type,
83788378
commandId: _commandId,
83798379
turnItemId: _turnItemId,
8380+
answeringTurnItemIds,
83808381
wake,
83818382
...fields
83828383
} = command;
@@ -8426,12 +8427,14 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
84268427
});
84278428
// The link answers the older cards that asked for the same machine and
84288429
// access. They end without waking anyone: the agent hears this link.
8430+
// A card still being answered stays open for its own answer to end.
84298431
if (!unchanged && command.linkStatus === "linked") {
84308432
for (const item of projection.turnItems) {
84318433
if (
84328434
item.type !== "link_request" ||
84338435
item.linkStatus !== "pending" ||
84348436
item.ordinal >= turnItem.ordinal ||
8437+
answeringTurnItemIds?.includes(item.id) === true ||
84358438
!askedForLinkedMachine(item, command)
84368439
) {
84378440
continue;

‎apps/server/src/peer/PeerLinkRequests.test.ts‎

Lines changed: 232 additions & 66 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@ import {
77
MessageId,
88
NodeId,
99
OrchestratorMcpFailure,
10+
type PeerLinkCreateInput,
1011
ProjectId,
1112
ProviderDriverKind,
1213
ProviderInstanceId,
@@ -125,74 +126,89 @@ const scope: McpInvocationContext.McpInvocationScope = {
125126
issuedAt: 1,
126127
};
127128

128-
/** The laptop: a real orchestrator, its orchestrator toolkit, and its links. */
129-
const makeLaptop = Effect.gen(function* () {
130-
const linking = yield* layerLinkingEnvironment(laptop).pipe(Layer.build);
131-
// The orchestrator's own database, apart from the one links are kept in.
132-
const database = yield* SqlitePersistence.layerMemory.pipe(Layer.build);
133-
const layerDatabase = Layer.succeedContext(database);
134-
const layerOrchestrator = Layer.mergeAll(
135-
layerDatabase,
136-
ProjectionStore.layer.pipe(Layer.provide(layerDatabase)),
137-
ProviderReplayHarness.layerWithRegistry(
138-
{ name: "peer-link-requests" },
139-
ProviderAdapterRegistry.layerFromAdapters([adapter]),
140-
{ databaseLayer: layerDatabase, runEffectWorker: false },
141-
),
142-
).pipe(
143-
Layer.provide(
144-
Layer.succeed(ProviderContinuationRequests.ProviderContinuationRequests, {
145-
offer: () => Effect.void,
146-
take: Effect.never,
147-
}),
148-
),
149-
);
150-
const layerThreads = ThreadManagement.layer.pipe(
151-
Layer.provide(CommandReceiptStore.layer.pipe(Layer.provide(layerDatabase))),
152-
Layer.provideMerge(Layer.merge(layerOrchestrator, ServerSettings.layerTest())),
153-
);
154-
const layerHere = McpHttpServer.layerOrchestratorToolkit.pipe(
155-
Layer.provideMerge(McpServer.McpServer.layer),
156-
Layer.provideMerge(PeerLinkRequests.layer),
157-
Layer.provide(Layer.mock(PeerForwarding.PeerForwarding)({})),
158-
Layer.provide(Layer.mock(RemoteDelegation.RemoteDelegation)({})),
159-
Layer.provide(Layer.succeedContext(linking)),
160-
Layer.provide(NodeCrypto.layer),
161-
Layer.provideMerge(layerThreads),
162-
Layer.provide(Layer.mock(ProviderRegistry.ProviderRegistry)({})),
163-
Layer.provide(Layer.mock(ProviderAdapterRegistry.ProviderAdapterRegistryV2)({})),
164-
Layer.provide(Layer.mock(ScheduledTaskService.ScheduledTaskService)({})),
165-
Layer.provide(Layer.mock(ProjectService.ProjectService)({})),
166-
Layer.provide(Layer.mock(SecretRequests.SecretRequests)({})),
167-
Layer.provide(Layer.mock(ThreadLaunch.ThreadLaunchService)({})),
168-
Layer.provide(Layer.mock(GitVcsDriver.GitVcsDriver)({})),
169-
Layer.provide(
170-
ServerConfig.layerTest(process.cwd(), { prefix: "t3-peer-link-requests-" }).pipe(
171-
Layer.provide(NodeServices.layer),
129+
/**
130+
* The laptop: a real orchestrator, its orchestrator toolkit, and its links.
131+
* `wrapLinks` lets a test hold a real link while it is made.
132+
*/
133+
const makeLaptopWith = (
134+
wrapLinks?: (links: PeerLinks.PeerLinks["Service"]) => PeerLinks.PeerLinks["Service"],
135+
) =>
136+
Effect.gen(function* () {
137+
const linking = yield* layerLinkingEnvironment(laptop).pipe(Layer.build);
138+
const answering =
139+
wrapLinks === undefined
140+
? linking
141+
: Context.add(
142+
linking,
143+
PeerLinks.PeerLinks,
144+
wrapLinks(Context.get(linking, PeerLinks.PeerLinks)),
145+
);
146+
// The orchestrator's own database, apart from the one links are kept in.
147+
const database = yield* SqlitePersistence.layerMemory.pipe(Layer.build);
148+
const layerDatabase = Layer.succeedContext(database);
149+
const layerOrchestrator = Layer.mergeAll(
150+
layerDatabase,
151+
ProjectionStore.layer.pipe(Layer.provide(layerDatabase)),
152+
ProviderReplayHarness.layerWithRegistry(
153+
{ name: "peer-link-requests" },
154+
ProviderAdapterRegistry.layerFromAdapters([adapter]),
155+
{ databaseLayer: layerDatabase, runEffectWorker: false },
172156
),
173-
),
174-
Layer.provide(NodeServices.layer),
175-
Layer.fresh,
176-
);
177-
const here = yield* Layer.build(layerHere);
178-
const server = Context.get(here, McpServer.McpServer);
179-
return {
180-
links: Context.get(linking, PeerLinks.PeerLinks),
181-
linksSql: Context.get(linking, SqlClient.SqlClient),
182-
requests: Context.get(here, PeerLinkRequests.PeerLinkRequests),
183-
orchestrator: Context.get(here, Orchestrator.OrchestratorV2),
184-
sink: Context.get(here, EventSink.EventSinkV2),
185-
projections: Context.get(here, ProjectionStore.ProjectionStoreV2),
186-
sql: Context.get(database, SqlClient.SqlClient),
187-
call: (args: Record<string, unknown>, callScope = scope) =>
188-
server
189-
.callTool({ name: "t3_environment_link", arguments: args })
190-
.pipe(
191-
Effect.provideService(McpInvocationContext.McpInvocationContext, callScope),
192-
Effect.provideService(McpSchema.McpServerClient, mcpClient),
157+
).pipe(
158+
Layer.provide(
159+
Layer.succeed(ProviderContinuationRequests.ProviderContinuationRequests, {
160+
offer: () => Effect.void,
161+
take: Effect.never,
162+
}),
163+
),
164+
);
165+
const layerThreads = ThreadManagement.layer.pipe(
166+
Layer.provide(CommandReceiptStore.layer.pipe(Layer.provide(layerDatabase))),
167+
Layer.provideMerge(Layer.merge(layerOrchestrator, ServerSettings.layerTest())),
168+
);
169+
const layerHere = McpHttpServer.layerOrchestratorToolkit.pipe(
170+
Layer.provideMerge(McpServer.McpServer.layer),
171+
Layer.provideMerge(PeerLinkRequests.layer),
172+
Layer.provide(Layer.mock(PeerForwarding.PeerForwarding)({})),
173+
Layer.provide(Layer.mock(RemoteDelegation.RemoteDelegation)({})),
174+
Layer.provide(Layer.succeedContext(answering)),
175+
Layer.provide(NodeCrypto.layer),
176+
Layer.provideMerge(layerThreads),
177+
Layer.provide(Layer.mock(ProviderRegistry.ProviderRegistry)({})),
178+
Layer.provide(Layer.mock(ProviderAdapterRegistry.ProviderAdapterRegistryV2)({})),
179+
Layer.provide(Layer.mock(ScheduledTaskService.ScheduledTaskService)({})),
180+
Layer.provide(Layer.mock(ProjectService.ProjectService)({})),
181+
Layer.provide(Layer.mock(SecretRequests.SecretRequests)({})),
182+
Layer.provide(Layer.mock(ThreadLaunch.ThreadLaunchService)({})),
183+
Layer.provide(Layer.mock(GitVcsDriver.GitVcsDriver)({})),
184+
Layer.provide(
185+
ServerConfig.layerTest(process.cwd(), { prefix: "t3-peer-link-requests-" }).pipe(
186+
Layer.provide(NodeServices.layer),
193187
),
194-
};
195-
});
188+
),
189+
Layer.provide(NodeServices.layer),
190+
Layer.fresh,
191+
);
192+
const here = yield* Layer.build(layerHere);
193+
const server = Context.get(here, McpServer.McpServer);
194+
return {
195+
links: Context.get(linking, PeerLinks.PeerLinks),
196+
linksSql: Context.get(linking, SqlClient.SqlClient),
197+
requests: Context.get(here, PeerLinkRequests.PeerLinkRequests),
198+
orchestrator: Context.get(here, Orchestrator.OrchestratorV2),
199+
sink: Context.get(here, EventSink.EventSinkV2),
200+
projections: Context.get(here, ProjectionStore.ProjectionStoreV2),
201+
sql: Context.get(database, SqlClient.SqlClient),
202+
call: (args: Record<string, unknown>, callScope = scope) =>
203+
server
204+
.callTool({ name: "t3_environment_link", arguments: args })
205+
.pipe(
206+
Effect.provideService(McpInvocationContext.McpInvocationContext, callScope),
207+
Effect.provideService(McpSchema.McpServerClient, mcpClient),
208+
),
209+
};
210+
});
211+
const makeLaptop = makeLaptopWith();
196212
type Laptop = Effect.Success<typeof makeLaptop>;
197213

198214
/** The laptop's thread, mid-turn when its agent asks, in `runtimeMode`. */
@@ -1195,3 +1211,153 @@ it.live("a later card's failed link leaves the older card open", () =>
11951211
}),
11961212
),
11971213
);
1214+
1215+
/**
1216+
* A laptop whose link to the box waits on `hold` for each access it returns
1217+
* a gate for, so a test can stall one answer inside its real link.
1218+
*/
1219+
const withHeldLinks = <A, E, R>(
1220+
hold: (access: PeerLinkCreateInput["access"]) => Effect.Effect<void> | undefined,
1221+
body: (a: Laptop, b: ServedPeer) => Effect.Effect<A, E, R>,
1222+
) =>
1223+
Effect.scoped(
1224+
Effect.gen(function* () {
1225+
const b = yield* serveBox;
1226+
const a = yield* makeLaptopWith((links) => ({
1227+
...links,
1228+
link: (input) =>
1229+
(hold(input.access) ?? Effect.void).pipe(Effect.andThen(links.link(input))),
1230+
}));
1231+
return yield* body(a, b);
1232+
}),
1233+
).pipe(Effect.provide(NodeHttpServer.layerTest));
1234+
1235+
it.live("an older card answered while a newer one links the same machine keeps its own link", () =>
1236+
Effect.gen(function* () {
1237+
const reached = yield* Deferred.make<void>();
1238+
const release = yield* Deferred.make<void>();
1239+
const fullAccessHeld = Deferred.succeed(reached, undefined).pipe(
1240+
Effect.andThen(Deferred.await(release)),
1241+
);
1242+
yield* withHeldLinks(
1243+
(access) => (access === "full-access" ? fullAccessHeld : undefined),
1244+
(a, b) =>
1245+
Effect.gen(function* () {
1246+
yield* seedThread(a);
1247+
// Both cards suggest the same machine and access.
1248+
const older = yield* ask(a, {
1249+
url: b.url,
1250+
requestedAccess: "full-access",
1251+
clientRequestId: "older",
1252+
});
1253+
const newer = yield* ask(a, {
1254+
url: b.url,
1255+
requestedAccess: "full-access",
1256+
clientRequestId: "newer",
1257+
});
1258+
const olderCode = yield* b.auth.issuePairingCredential();
1259+
const newerCode = yield* b.auth.issuePairingCredential();
1260+
1261+
// The user grants full access on the older card, which stalls while linking.
1262+
const olderAnswer = yield* a.requests
1263+
.answer({
1264+
threadId,
1265+
turnItemId: older.id,
1266+
answer: {
1267+
type: "link",
1268+
urls: [b.url],
1269+
access: "full-access",
1270+
pairingCode: olderCode.credential,
1271+
},
1272+
})
1273+
.pipe(Effect.forkChild);
1274+
yield* Deferred.await(reached);
1275+
1276+
// Meanwhile the newer card links with read-only access and finishes.
1277+
yield* a.requests.answer({
1278+
threadId,
1279+
turnItemId: newer.id,
1280+
answer: {
1281+
type: "link",
1282+
urls: [b.url],
1283+
access: "read-only",
1284+
pairingCode: newerCode.credential,
1285+
},
1286+
});
1287+
// The older card is still being answered, so the newer link leaves it open.
1288+
expect((yield* linkCards(a)).find((card) => card.id === older.id)).toMatchObject({
1289+
status: "waiting",
1290+
linkStatus: "pending",
1291+
});
1292+
1293+
yield* Deferred.succeed(release, undefined);
1294+
yield* Fiber.join(olderAnswer);
1295+
1296+
const cards = yield* linkCards(a);
1297+
const stored = Option.getOrThrow(yield* a.links.get(box.environmentId));
1298+
expect(cards.find((card) => card.id === newer.id)).toMatchObject({
1299+
status: "completed",
1300+
linkStatus: "linked",
1301+
linkedAccess: "read-only",
1302+
});
1303+
// The older card says what the link the box now has grants, and the agent hears it.
1304+
expect(stored.access).toBe("full-access");
1305+
expect(cards.find((card) => card.id === older.id)).toMatchObject({
1306+
status: "completed",
1307+
linkStatus: "linked",
1308+
linkedEnvironmentId: stored.environmentId,
1309+
linkedAccess: stored.access,
1310+
});
1311+
const { messages } = yield* outcomeMessages(a);
1312+
expect(messages.map((message) => message.notification?.outcome)).toEqual([
1313+
"completed",
1314+
"completed",
1315+
]);
1316+
}),
1317+
);
1318+
}),
1319+
);
1320+
1321+
it.live("a card a newer link ended links nothing when answered after", () =>
1322+
Effect.gen(function* () {
1323+
let links = 0;
1324+
yield* withHeldLinks(
1325+
() => Effect.sync(() => void links++),
1326+
(a, b) =>
1327+
Effect.gen(function* () {
1328+
yield* seedThread(a);
1329+
const older = yield* ask(a, {
1330+
url: b.url,
1331+
requestedAccess: "full-access",
1332+
clientRequestId: "older",
1333+
});
1334+
const newer = yield* ask(a, {
1335+
url: b.url,
1336+
requestedAccess: "full-access",
1337+
clientRequestId: "newer",
1338+
});
1339+
yield* linkBox(a, b, newer.id);
1340+
expect(links).toBe(1);
1341+
1342+
const pairing = yield* b.auth.issuePairingCredential();
1343+
const late = yield* a.requests
1344+
.answer({
1345+
threadId,
1346+
turnItemId: older.id,
1347+
answer: {
1348+
type: "link",
1349+
urls: [b.url],
1350+
access: "read-only",
1351+
pairingCode: pairing.credential,
1352+
},
1353+
})
1354+
.pipe(Effect.flip);
1355+
expect(late.reason).toBe("already_answered");
1356+
expect(links).toBe(1);
1357+
expect(Option.getOrThrow(yield* a.links.get(box.environmentId)).access).toBe(
1358+
"full-access",
1359+
);
1360+
}),
1361+
);
1362+
}),
1363+
);

0 commit comments

Comments
 (0)