Skip to content

Commit 988189d

Browse files
t3dotggclaude
andcommitted
fix(server): wake the relay catch-up when this server links
The catch-up retry now backs off to 60 s. Links and publishing changes made through this server's HTTP handlers write the secrets in the same process, so the catch-up publish could wait up to a minute. The relay now has requestCatchUp, which ends the current wait at once. The relay config and preferences handlers call it after they write the secrets. Polling stays for `t3 connect`, which writes from another process. Co-Authored-By: Claude Opus 5.5 (1M context) <[email protected]>
1 parent 5f19376 commit 988189d

7 files changed

Lines changed: 131 additions & 76 deletions

File tree

‎apps/server/integration/OrchestrationEngineHarness.integration.ts‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -420,6 +420,7 @@ export const makeOrchestrationIntegrationHarness = (
420420
Layer.provideMerge(
421421
Layer.succeed(AgentAwarenessRelay.AgentAwarenessRelay, {
422422
publishThread: () => Effect.void,
423+
requestCatchUp: () => Effect.void,
423424
start: () => Effect.void,
424425
}),
425426
),

‎apps/server/src/cloud/http.test.ts‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -33,6 +33,7 @@ import {
3333
type ServiceUpdateRecord,
3434
} from "./serviceProtocol.ts";
3535
import * as ServerEnvironment from "../environment/ServerEnvironment.ts";
36+
import * as AgentAwarenessRelay from "../relay/AgentAwarenessRelay.ts";
3637
import { CLOUD_CLI_DESIRED_LINK_SECRET } from "./CliState.ts";
3738
import * as CliTokenManager from "./CliTokenManager.ts";
3839
import {
@@ -81,6 +82,12 @@ const storeFailure = (tag: "AlreadyExists" | "PermissionDenied") =>
8182
});
8283

8384
const unusedSecretStoreOperation = () => Effect.die("unused secret-store operation");
85+
// Linking wakes the awareness relay; these tests do not run it.
86+
const idleAwarenessRelay = AgentAwarenessRelay.AgentAwarenessRelay.of({
87+
publishThread: () => Effect.void,
88+
requestCatchUp: () => Effect.void,
89+
start: () => Effect.void,
90+
});
8491
const decodeManagedTunnelRecoveryRegistration = Schema.decodeUnknownEffect(
8592
Schema.fromJsonString(RelayManagedEndpointRecoveryRegistrationRequest),
8693
);
@@ -262,6 +269,7 @@ describe("reconcileDesiredCloudLink", () => {
262269
HttpClient.HttpClient,
263270
HttpClient.make(() => unusedSecretStoreOperation()),
264271
),
272+
Effect.provideService(AgentAwarenessRelay.AgentAwarenessRelay, idleAwarenessRelay),
265273
Effect.provide(NodeServices.layer),
266274
),
267275
);
@@ -375,6 +383,7 @@ describe("releaseManagedTunnelOnShutdown", () => {
375383
<A, E, R>(effect: Effect.Effect<A, E, R>) =>
376384
effect.pipe(
377385
Effect.provideService(ServerSecretStore.ServerSecretStore, harness.store),
386+
Effect.provideService(AgentAwarenessRelay.AgentAwarenessRelay, idleAwarenessRelay),
378387
Effect.provideService(
379388
ServerEnvironment.ServerEnvironment,
380389
ServerEnvironment.ServerEnvironment.of({

‎apps/server/src/cloud/http.ts‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -69,6 +69,7 @@ import * as ServerSecretStore from "../auth/ServerSecretStore.ts";
6969
import { requireEnvironmentScope } from "../auth/http.ts";
7070
import * as ServerConfig from "../config.ts";
7171
import * as ServerEnvironment from "../environment/ServerEnvironment.ts";
72+
import * as AgentAwarenessRelay from "../relay/AgentAwarenessRelay.ts";
7273
import * as ManagedEndpointRuntime from "./ManagedEndpointRuntime.ts";
7374
import {
7475
SERVICE_STATE_FILE,
@@ -411,6 +412,7 @@ interface CloudHttpDependencies {
411412
readonly environmentAuth: EnvironmentAuth.EnvironmentAuth["Service"];
412413
readonly cliTokenManager: CliTokenManager.CloudCliTokenManager["Service"];
413414
readonly httpClient: HttpClient.HttpClient;
415+
readonly awarenessRelay: AgentAwarenessRelay.AgentAwarenessRelay["Service"];
414416
}
415417

416418
const cloudHttpDependencies = Effect.gen(function* () {
@@ -421,6 +423,7 @@ const cloudHttpDependencies = Effect.gen(function* () {
421423
environmentAuth: yield* EnvironmentAuth.EnvironmentAuth,
422424
cliTokenManager: yield* CliTokenManager.CloudCliTokenManager,
423425
httpClient: yield* HttpClient.HttpClient,
426+
awarenessRelay: yield* AgentAwarenessRelay.AgentAwarenessRelay,
424427
} satisfies CloudHttpDependencies;
425428
});
426429

@@ -668,6 +671,7 @@ const applyCloudRelayConfig = Effect.fn("environment.cloud.applyRelayConfig")(fu
668671
CLOUD_MINT_PUBLIC_KEY,
669672
stringToBytes(payload.cloudMintPublicKey),
670673
);
674+
yield* dependencies.awarenessRelay.requestCatchUp();
671675
if (payload.endpointRuntime) {
672676
const endpointRuntimeJson = yield* encodeEndpointRuntimeConfigJson(payload.endpointRuntime);
673677
yield* dependencies.secrets.set(
@@ -1349,6 +1353,7 @@ const cloudPreferencesHandler = Effect.fn("environment.cloud.preferences")(
13491353
PUBLISH_AGENT_ACTIVITY_SECRET,
13501354
stringToBytes(String(payload.publishAgentActivity)),
13511355
);
1356+
yield* dependencies.awarenessRelay.requestCatchUp();
13521357
return yield* readCloudLinkState(dependencies);
13531358
},
13541359
Effect.catchIf(

‎apps/server/src/orchestration/Layers/OrchestrationReactor.test.ts‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -108,6 +108,7 @@ describe("OrchestrationReactor", () => {
108108
Layer.provideMerge(
109109
Layer.succeed(AgentAwarenessRelay.AgentAwarenessRelay, {
110110
publishThread: () => Effect.void,
111+
requestCatchUp: () => Effect.void,
111112
start: () => {
112113
started.push("agent-awareness-relay");
113114
return Effect.void;

‎apps/server/src/relay/AgentAwarenessRelay.test.ts‎

Lines changed: 92 additions & 66 deletions
Original file line numberDiff line numberDiff line change
@@ -665,72 +665,6 @@ describe.sequential("signRelayAgentActivityPublishProof", () => {
665665
),
666666
);
667667

668-
it.effect("checks an unlinked environment once a minute and still catches up once linked", () =>
669-
Effect.scoped(
670-
Effect.gen(function* () {
671-
const secrets = makeMemorySecretStore();
672-
let publishSettingReads = 0;
673-
let shellSnapshotReads = 0;
674-
const countingStore = {
675-
...secrets.store,
676-
get: (name: string) =>
677-
Effect.suspend(() => {
678-
if (name === PUBLISH_AGENT_ACTIVITY_SECRET) publishSettingReads += 1;
679-
return secrets.store.get(name);
680-
}),
681-
} satisfies ServerSecretStore.ServerSecretStore["Service"];
682-
683-
const layer = Layer.mergeAll(
684-
Layer.succeed(ServerSecretStore.ServerSecretStore, countingStore),
685-
Layer.succeed(ServerEnvironment.ServerEnvironment, {
686-
getEnvironmentId: Effect.succeed("env-1" as EnvironmentId),
687-
getDescriptor: Effect.die("unused descriptor"),
688-
}),
689-
Layer.succeed(OrchestrationEngineService, {
690-
streamDomainEvents: Stream.never,
691-
} as unknown as OrchestrationEngineShape),
692-
Layer.succeed(ProjectionSnapshotQuery, {
693-
getShellSnapshot: () =>
694-
Effect.sync(() => {
695-
shellSnapshotReads += 1;
696-
return {
697-
snapshotSequence: 1,
698-
projects: [],
699-
threads: [],
700-
updatedAt: "2026-05-25T00:00:00.000Z",
701-
} satisfies OrchestrationShellSnapshot;
702-
}),
703-
} as unknown as ProjectionSnapshotQueryShape),
704-
);
705-
706-
yield* Effect.gen(function* () {
707-
const relay = yield* AgentAwarenessRelay.AgentAwarenessRelay;
708-
yield* relay.start();
709-
710-
// Get past the backoff ramp, then count checks in a steady window.
711-
yield* TestClock.adjust("10 minutes");
712-
const readsBeforeWindow = publishSettingReads;
713-
yield* TestClock.adjust("10 minutes");
714-
expect(publishSettingReads - readsBeforeWindow).toBe(10);
715-
expect(shellSnapshotReads).toBe(0);
716-
717-
yield* secrets.setString(RELAY_URL_SECRET, "https://relay.example.test");
718-
yield* secrets.setString(RELAY_ENVIRONMENT_CREDENTIAL_SECRET, "relay-credential");
719-
yield* secrets.setString(PUBLISH_AGENT_ACTIVITY_SECRET, "true");
720-
yield* TestClock.adjust("1 minute");
721-
expect(shellSnapshotReads).toBe(1);
722-
}).pipe(
723-
Effect.provide(
724-
AgentAwarenessRelay.layer.pipe(
725-
Layer.provide(layer),
726-
Layer.provideMerge(NodeServices.layer),
727-
),
728-
),
729-
);
730-
}),
731-
),
732-
);
733-
734668
it.effect("publishes agent activity to the relay transport URL, not the relay issuer", () =>
735669
Effect.scoped(
736670
Effect.gen(function* () {
@@ -1042,3 +976,95 @@ describe.sequential("signRelayAgentActivityPublishProof", () => {
1042976
),
1043977
);
1044978
});
979+
980+
describe.sequential("startup catch-up", () => {
981+
// An unlinked relay. `link` writes the secrets a link and opt-in store.
982+
// Counts link checks (publish setting reads) and catch-up publishes (shell
983+
// snapshot reads).
984+
function makeUnlinkedRelay() {
985+
const secrets = makeMemorySecretStore();
986+
const counts = { linkChecks: 0, catchUpPublishes: 0 };
987+
const countingStore = {
988+
...secrets.store,
989+
get: (name: string) =>
990+
Effect.suspend(() => {
991+
if (name === PUBLISH_AGENT_ACTIVITY_SECRET) counts.linkChecks += 1;
992+
return secrets.store.get(name);
993+
}),
994+
} satisfies ServerSecretStore.ServerSecretStore["Service"];
995+
996+
const layer = AgentAwarenessRelay.layer.pipe(
997+
Layer.provide(
998+
Layer.mergeAll(
999+
Layer.succeed(ServerSecretStore.ServerSecretStore, countingStore),
1000+
Layer.succeed(ServerEnvironment.ServerEnvironment, {
1001+
getEnvironmentId: Effect.succeed("env-1" as EnvironmentId),
1002+
getDescriptor: Effect.die("unused descriptor"),
1003+
}),
1004+
Layer.succeed(OrchestrationEngineService, {
1005+
streamDomainEvents: Stream.never,
1006+
} as unknown as OrchestrationEngineShape),
1007+
Layer.succeed(ProjectionSnapshotQuery, {
1008+
getShellSnapshot: () =>
1009+
Effect.sync(() => {
1010+
counts.catchUpPublishes += 1;
1011+
return {
1012+
snapshotSequence: 1,
1013+
projects: [],
1014+
threads: [],
1015+
updatedAt: "2026-05-25T00:00:00.000Z",
1016+
} satisfies OrchestrationShellSnapshot;
1017+
}),
1018+
} as unknown as ProjectionSnapshotQueryShape),
1019+
),
1020+
),
1021+
Layer.provideMerge(NodeServices.layer),
1022+
);
1023+
const link = Effect.all(
1024+
[
1025+
secrets.setString(RELAY_URL_SECRET, "https://relay.example.test"),
1026+
secrets.setString(RELAY_ENVIRONMENT_CREDENTIAL_SECRET, "relay-credential"),
1027+
secrets.setString(PUBLISH_AGENT_ACTIVITY_SECRET, "true"),
1028+
],
1029+
{ discard: true },
1030+
);
1031+
return { counts, layer, link };
1032+
}
1033+
1034+
it.effect("checks an unlinked environment once a minute and still catches up once linked", () => {
1035+
const { counts, layer, link } = makeUnlinkedRelay();
1036+
return Effect.gen(function* () {
1037+
const relay = yield* AgentAwarenessRelay.AgentAwarenessRelay;
1038+
yield* relay.start();
1039+
1040+
// Get past the backoff ramp, then count checks in a steady window.
1041+
yield* TestClock.adjust("10 minutes");
1042+
const checksBeforeWindow = counts.linkChecks;
1043+
yield* TestClock.adjust("10 minutes");
1044+
expect(counts.linkChecks - checksBeforeWindow).toBe(10);
1045+
expect(counts.catchUpPublishes).toBe(0);
1046+
1047+
yield* link;
1048+
yield* TestClock.adjust("1 minute");
1049+
expect(counts.catchUpPublishes).toBe(1);
1050+
}).pipe(Effect.provide(layer), Effect.scoped);
1051+
});
1052+
1053+
it.effect("publishes at once when this process links while the check is backed off", () => {
1054+
const { counts, layer, link } = makeUnlinkedRelay();
1055+
return Effect.gen(function* () {
1056+
const relay = yield* AgentAwarenessRelay.AgentAwarenessRelay;
1057+
yield* relay.start();
1058+
1059+
// Backed off to 60 s: the next check is still seconds away.
1060+
yield* TestClock.adjust("10 minutes");
1061+
yield* link;
1062+
yield* TestClock.adjust("1 second");
1063+
expect(counts.catchUpPublishes).toBe(0);
1064+
1065+
yield* relay.requestCatchUp();
1066+
yield* TestClock.adjust("1 second");
1067+
expect(counts.catchUpPublishes).toBe(1);
1068+
}).pipe(Effect.provide(layer), Effect.scoped);
1069+
});
1070+
});

‎apps/server/src/relay/AgentAwarenessRelay.ts‎

Lines changed: 9 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,7 @@ import * as DateTime from "effect/DateTime";
2525
import * as Effect from "effect/Effect";
2626
import * as Layer from "effect/Layer";
2727
import * as Option from "effect/Option";
28+
import * as Queue from "effect/Queue";
2829
import * as Ref from "effect/Ref";
2930
import type * as Scope from "effect/Scope";
3031
import * as Stream from "effect/Stream";
@@ -51,6 +52,8 @@ export class AgentAwarenessRelay extends Context.Service<
5152
AgentAwarenessRelay,
5253
{
5354
readonly publishThread: (threadId: ThreadId) => Effect.Effect<void>;
55+
/** Retries a pending catch-up publish now. Call after this process links or enables publishing. */
56+
readonly requestCatchUp: () => Effect.Effect<void>;
5457
readonly start: () => Effect.Effect<void, never, Scope.Scope>;
5558
}
5659
>()("t3/relay/AgentAwarenessRelay") {}
@@ -309,6 +312,8 @@ export const make = Effect.gen(function* () {
309312
const cloudLinkKeyPair = yield* getOrCreateEnvironmentKeyPairFromSecretStore(secrets);
310313
const startedAt = (yield* DateTime.now).epochMilliseconds;
311314
const activeSnapshotPublishedRef = yield* Ref.make(false);
315+
// Holds at most one pending wake, so a burst of requests costs one retry.
316+
const catchUpRequests = yield* Queue.dropping<void>(1);
312317
const publishedStateByThreadRef = yield* Ref.make(new Map<ThreadId, string>());
313318

314319
const readSecretString = (name: string) =>
@@ -569,7 +574,8 @@ export const make = Effect.gen(function* () {
569574
// Publishes the catch-up snapshot of active threads once the environment is
570575
// linked and publishing is enabled. Many environments never link, so the
571576
// retry backs off from 5 s to 60 s. It polls because `t3 connect` can write
572-
// the secrets from another process.
577+
// the secrets from another process. Links and preference changes made by
578+
// this process call `requestCatchUp`, which ends the wait early.
573579
const publishActiveThreadsOnceWhenConfigured = (logEnabledWhenReady: boolean) =>
574580
Effect.gen(function* () {
575581
let retryDelayMs = 5_000;
@@ -585,7 +591,7 @@ export const make = Effect.gen(function* () {
585591
}
586592
return;
587593
}
588-
yield* Effect.sleep(retryDelayMs);
594+
yield* Effect.race(Effect.sleep(retryDelayMs), Queue.take(catchUpRequests));
589595
retryDelayMs = Math.min(retryDelayMs * 2, 60_000);
590596
}
591597
});
@@ -663,6 +669,7 @@ export const make = Effect.gen(function* () {
663669

664670
return AgentAwarenessRelay.of({
665671
publishThread,
672+
requestCatchUp: () => Queue.offer(catchUpRequests, undefined).pipe(Effect.asVoid),
666673
start,
667674
});
668675
});

‎apps/server/src/server.test.ts‎

Lines changed: 14 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -181,6 +181,7 @@ import * as ServerSecretStore from "./auth/ServerSecretStore.ts";
181181
import * as EnvironmentAuth from "./auth/EnvironmentAuth.ts";
182182
import * as PairingGrantStore from "./auth/PairingGrantStore.ts";
183183
import * as CloudManagedEndpointRuntime from "./cloud/ManagedEndpointRuntime.ts";
184+
import * as AgentAwarenessRelay from "./relay/AgentAwarenessRelay.ts";
184185
import * as CloudCliTokenManager from "./cloud/CliTokenManager.ts";
185186
import * as ProcessDiagnostics from "./diagnostics/ProcessDiagnostics.ts";
186187
import * as HostResources from "./resourceTelemetry/HostResources.ts";
@@ -1173,14 +1174,19 @@ const buildAppUnderTest = (options?: {
11731174
}),
11741175
),
11751176
Layer.provide(
1176-
Layer.succeed(
1177-
CloudManagedEndpointRuntime.CloudManagedEndpointRuntime,
1178-
CloudManagedEndpointRuntime.CloudManagedEndpointRuntime.of({
1179-
applyConfig: () => Effect.succeed({ status: "disabled" }),
1180-
recoveryRequests: Stream.empty,
1181-
requestRecovery: () => Effect.void,
1182-
withLinkStateLock: (effect) => effect,
1183-
...options?.layers?.cloudManagedEndpointRuntime,
1177+
Layer.mergeAll(
1178+
Layer.succeed(
1179+
CloudManagedEndpointRuntime.CloudManagedEndpointRuntime,
1180+
CloudManagedEndpointRuntime.CloudManagedEndpointRuntime.of({
1181+
applyConfig: () => Effect.succeed({ status: "disabled" }),
1182+
recoveryRequests: Stream.empty,
1183+
requestRecovery: () => Effect.void,
1184+
withLinkStateLock: (effect) => effect,
1185+
...options?.layers?.cloudManagedEndpointRuntime,
1186+
}),
1187+
),
1188+
Layer.mock(AgentAwarenessRelay.AgentAwarenessRelay)({
1189+
requestCatchUp: () => Effect.void,
11841190
}),
11851191
),
11861192
),

0 commit comments

Comments
 (0)