Skip to content

Commit 25b6b08

Browse files
committed
feat(server): update provider CLIs in the background when nothing is running
Adds an update window that stays closed while threads run or wait for input, for 15 minutes after any client interaction, and while a scheduled task or usage-limit resume is due within 10 minutes. A new Update automatically setting (on by default) installs provider CLI updates inside that window. The provider maintenance runner is now one server-lifetime instance, so RPC and background updates share the per-installer lock. It was previously built per WebSocket connection, so two clients' updates never queued.
1 parent 9d6b5ea commit 25b6b08

15 files changed

Lines changed: 479 additions & 1 deletion

File tree

‎apps/mobile/src/features/settings/SettingsServerControlsRouteScreen.tsx‎

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -230,6 +230,9 @@ function ServerSettingsDetail(props: { readonly page: SettingsPage }) {
230230
(target) =>
231231
target.environment.serverConfig.environment.capabilities.threadRestartContinuation === true,
232232
);
233+
const supportsAutomaticUpdates = targets.every(
234+
(target) => target.environment.serverConfig.environment.capabilities.automaticUpdates === true,
235+
);
233236
const disabledFor = (key: string) =>
234237
disabled ||
235238
(projectSelected &&
@@ -462,6 +465,22 @@ function ServerSettingsDetail(props: { readonly page: SettingsPage }) {
462465
disabled={disabledFor("enableProviderUpdateChecks")}
463466
onValueChange={(value) => write({ enableProviderUpdateChecks: value })}
464467
/>
468+
<View className="border-t border-border-subtle">
469+
<SettingsSwitchRow
470+
icon="arrow.down.circle"
471+
label="Update automatically"
472+
subtitle={
473+
projectSelected
474+
? "Environment-wide setting. Select All projects to change it."
475+
: supportsAutomaticUpdates
476+
? "Install provider CLI updates while nothing is running."
477+
: "Update older servers to control automatic updates."
478+
}
479+
value={uniform("automaticUpdates")}
480+
disabled={disabledFor("automaticUpdates") || !supportsAutomaticUpdates}
481+
onValueChange={(value) => write({ automaticUpdates: value })}
482+
/>
483+
</View>
465484
<View className="border-t border-border-subtle">
466485
<SettingsSwitchRow
467486
icon="arrow.uturn.forward"

‎apps/server/src/background/BackgroundPolicy.ts‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,7 @@ import * as DateTime from "effect/DateTime";
1616
import * as Context from "effect/Context";
1717
import * as Effect from "effect/Effect";
1818
import * as Layer from "effect/Layer";
19+
import * as Option from "effect/Option";
1920
import * as PubSub from "effect/PubSub";
2021
import * as Ref from "effect/Ref";
2122
import * as Scope from "effect/Scope";
@@ -40,6 +41,8 @@ export class BackgroundPolicy extends Context.Service<
4041
) => Effect.Effect<void>;
4142
readonly reportHostPowerState: (snapshot: HostPowerSnapshot) => Effect.Effect<void>;
4243
readonly snapshot: Effect.Effect<BackgroundPolicySnapshot>;
44+
/** When any client last reported recent interaction, so updates can wait for quiet. */
45+
readonly lastClientInteractionAt: Effect.Effect<Option.Option<DateTime.Utc>>;
4346
readonly streamChanges: Stream.Stream<BackgroundPolicySnapshot>;
4447
readonly subscribe: Effect.Effect<
4548
{
@@ -213,6 +216,7 @@ export const make = Effect.fn("background.policy.make")(function* () {
213216
const hostPowerMonitor = yield* HostPowerMonitor.HostPowerMonitor;
214217
const serverSettings = yield* ServerSettings.ServerSettingsService;
215218
const leasesRef = yield* Ref.make(new Map<string, ClientActivityLease>());
219+
const lastClientInteractionAtRef = yield* Ref.make(Option.none<DateTime.Utc>());
216220
const changes = yield* PubSub.sliding<BackgroundPolicySnapshot>(1);
217221
const publishMutex = yield* Semaphore.make(1);
218222

@@ -266,6 +270,9 @@ export const make = Effect.fn("background.policy.make")(function* () {
266270
expiresAt,
267271
};
268272
yield* Ref.update(leasesRef, (leases) => upsertClientActivityLease(leases, lease, now));
273+
if (input.recentlyInteracted) {
274+
yield* Ref.set(lastClientInteractionAtRef, Option.some(now));
275+
}
269276
yield* publishSnapshotUnlocked;
270277
}),
271278
);
@@ -339,6 +346,7 @@ export const make = Effect.fn("background.policy.make")(function* () {
339346
removeRpcClient,
340347
reportHostPowerState: hostPowerMonitor.report,
341348
snapshot,
349+
lastClientInteractionAt: Ref.get(lastClientInteractionAtRef),
342350
streamChanges: Stream.fromPubSub(changes),
343351
subscribe: subscribeBeforeSnapshot(changes, snapshot, publishMutex),
344352
hasDemand,

‎apps/server/src/environment/ServerEnvironment.ts‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -231,6 +231,7 @@ export const make = Effect.gen(function* () {
231231
projectWorktreeCleanup: true,
232232
worktreesDirectory: true,
233233
threadRestartContinuation: true,
234+
automaticUpdates: true,
234235
projectSettingsOverrides: true,
235236
threadSnooze: true,
236237
environmentThemes: true,

‎apps/server/src/mcp/toolkits/environment/handlers.ts‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,7 @@ export function preferences(settings: ServerSettings) {
1313
defaultThreadEnvMode,
1414
newWorktreesStartFromOrigin,
1515
enableProviderUpdateChecks,
16+
automaticUpdates,
1617
backgroundActivity,
1718
sourceControlWritingStyle,
1819
} = settings;
@@ -21,6 +22,7 @@ export function preferences(settings: ServerSettings) {
2122
defaultThreadEnvMode,
2223
newWorktreesStartFromOrigin,
2324
enableProviderUpdateChecks,
25+
automaticUpdates,
2426
backgroundActivity: { profile: backgroundActivity.profile },
2527
sourceControlWritingStyle: {
2628
...sourceControlWritingStyle,

‎apps/server/src/mcp/toolkits/environment/tools.ts‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@ const PreferenceFields = {
1818
defaultThreadEnvMode: ServerSettings.fields.defaultThreadEnvMode,
1919
newWorktreesStartFromOrigin: ServerSettings.fields.newWorktreesStartFromOrigin,
2020
enableProviderUpdateChecks: ServerSettings.fields.enableProviderUpdateChecks,
21+
automaticUpdates: ServerSettings.fields.automaticUpdates,
2122
backgroundActivity: Schema.Struct({ profile: BackgroundActivityProfileSelection }),
2223
sourceControlWritingStyle: Schema.Struct({
2324
mode: Schema.String,
@@ -59,6 +60,7 @@ const EnvironmentPreferencesTool = Tool.make("t3_environment_preferences_update"
5960
defaultThreadEnvMode: ServerSettingsPatch.fields.defaultThreadEnvMode,
6061
newWorktreesStartFromOrigin: ServerSettingsPatch.fields.newWorktreesStartFromOrigin,
6162
enableProviderUpdateChecks: ServerSettingsPatch.fields.enableProviderUpdateChecks,
63+
automaticUpdates: ServerSettingsPatch.fields.automaticUpdates,
6264
backgroundActivity: Schema.optionalKey(Schema.Struct({ profile: BackgroundActivityProfile })),
6365
sourceControlWritingStyle: ServerSettingsPatch.fields.sourceControlWritingStyle,
6466
}),

‎apps/server/src/orchestration-v2/ProjectionStore.ts‎

Lines changed: 36 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -147,6 +147,9 @@ export type ProjectionStoreV2Error = typeof ProjectionStoreV2Error.Type;
147147
export type ProjectionRecoveryKind =
148148
| "queued-runs"
149149
| "runtime"
150+
// Work a restart or CLI replacement would cut off. Idle provider sessions
151+
// do not count: they resume on the next turn.
152+
| "active-work"
150153
| "subagent-results"
151154
| "delegated-completions";
152155

@@ -564,6 +567,19 @@ function needsRecovery(
564567
)
565568
);
566569
}
570+
case "active-work":
571+
return (
572+
projection.runs.some(
573+
(run) =>
574+
["preparing", "starting", "running", "waiting"].includes(run.status) ||
575+
(run.status === "queued" && run.queueHeld !== true),
576+
) ||
577+
projection.runtimeRequests.some((request) => request.status === "pending") ||
578+
projection.providerThreads.some(
579+
(thread) =>
580+
thread.status === "active" || (thread.pendingBackgroundTasks?.length ?? 0) > 0,
581+
)
582+
);
567583
case "runtime":
568584
return (
569585
projection.runs.some(
@@ -3592,6 +3608,26 @@ export const layer: Layer.Layer<ProjectionStoreV2, never, SqlClient.SqlClient> =
35923608
)
35933609
ELSE 0 END
35943610
`;
3611+
case "active-work":
3612+
return sql`
3613+
SELECT thread_id FROM orchestration_v2_projection_runs
3614+
WHERE status IN ('preparing', 'starting', 'running', 'waiting')
3615+
UNION
3616+
SELECT thread_id FROM orchestration_v2_projection_runs
3617+
WHERE status = 'queued'
3618+
AND CASE WHEN json_valid(payload_json)
3619+
THEN json_extract(payload_json, '$.queueHeld') IS NOT 1
3620+
ELSE 1 END
3621+
UNION
3622+
SELECT thread_id FROM orchestration_v2_projection_runtime_requests
3623+
WHERE status = 'pending'
3624+
UNION
3625+
SELECT thread_id FROM orchestration_v2_projection_provider_threads
3626+
WHERE status = 'active'
3627+
OR CASE WHEN json_valid(payload_json)
3628+
THEN json_array_length(payload_json, '$.pendingBackgroundTasks') > 0
3629+
ELSE 0 END
3630+
`;
35953631
case "runtime":
35963632
return sql`
35973633
WITH pending_provider_threads AS MATERIALIZED (

‎apps/server/src/server.ts‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -157,6 +157,9 @@ import * as ResourceTelemetry from "./resourceTelemetry/ResourceTelemetry.ts";
157157
import * as CursorUsageReader from "./usage/cursorUsageReader.ts";
158158
import * as UsageService from "./usage/UsageService.ts";
159159
import * as RuntimeLayer from "./orchestration-v2/runtimeLayer.ts";
160+
import * as ProviderMaintenanceRunner from "./provider/providerMaintenanceRunner.ts";
161+
import * as ProviderAutoUpdater from "./updates/ProviderAutoUpdater.ts";
162+
import * as UpdateWindow from "./updates/UpdateWindow.ts";
160163
import * as ProjectStore from "./orchestration-v2/ProjectStore.ts";
161164
import * as ThreadSearch from "./orchestration-v2/ThreadSearch.ts";
162165
import * as ResourceCleanupService from "./orchestration-v2/ResourceCleanupService.ts";
@@ -528,7 +531,15 @@ const layerProviderInstallationRefresh = Layer.effectDiscard(
528531
}),
529532
);
530533

534+
// One maintenance runner for RPCs and background updates, so both queue
535+
// behind the same per-installer lock.
536+
const layerProviderUpdates = ProviderAutoUpdater.layer.pipe(
537+
Layer.provide(UpdateWindow.layer.pipe(Layer.provide(ProjectionStoreV2.layer))),
538+
Layer.provideMerge(ProviderMaintenanceRunner.layer),
539+
);
540+
531541
const layerRuntimeCoreDependenciesBase = Layer.mergeAll(
542+
layerProviderUpdates,
532543
AgentAwarenessRelay.layer,
533544
// Asks T3 Connect to deliver webhooks it held while this environment was offline.
534545
HeldHooksWaker.layer,
Lines changed: 100 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,100 @@
1+
import type { ServerProvider } from "@t3tools/contracts";
2+
import * as Clock from "effect/Clock";
3+
import * as Duration from "effect/Duration";
4+
import * as Effect from "effect/Effect";
5+
import * as Layer from "effect/Layer";
6+
import * as Option from "effect/Option";
7+
import * as Schedule from "effect/Schedule";
8+
9+
import * as ServerConfig from "../config.ts";
10+
import * as ProviderRegistry from "../provider/ProviderRegistry.ts";
11+
import * as ProviderMaintenanceRunner from "../provider/providerMaintenanceRunner.ts";
12+
import { forkParked } from "../serverActivation.ts";
13+
import * as ServerSettings from "../serverSettings.ts";
14+
import * as UpdateWindow from "./UpdateWindow.ts";
15+
16+
/** A failed install is retried for the same version only after this long. */
17+
const RETRY_AFTER = Duration.hours(6);
18+
19+
function isAutoUpdatable(provider: ServerProvider): boolean {
20+
const advisory = provider.versionAdvisory;
21+
const updateStatus = provider.updateState?.status;
22+
return (
23+
provider.enabled &&
24+
provider.installed &&
25+
advisory?.status === "behind_latest" &&
26+
advisory.canUpdate &&
27+
advisory.latestVersion !== null &&
28+
updateStatus !== "queued" &&
29+
updateStatus !== "running"
30+
);
31+
}
32+
33+
/**
34+
* Installs provider CLI updates in the background when `automaticUpdates` is
35+
* on, one instance at a time, only while the update window is open.
36+
*/
37+
export const layer = Layer.effectDiscard(
38+
Effect.gen(function* () {
39+
// A dev server shares the host's CLIs with the real install; leave them to it.
40+
if ((yield* ServerConfig.ServerConfig).devUrl !== undefined) return;
41+
const providers = yield* ProviderRegistry.ProviderRegistry;
42+
const runner = yield* ProviderMaintenanceRunner.ProviderMaintenanceRunner;
43+
const updateWindow = yield* UpdateWindow.UpdateWindow;
44+
const serverSettings = yield* ServerSettings.ServerSettingsService;
45+
const timings = yield* UpdateWindow.updateTimings;
46+
// Keyed by instance and target version, so a newer release retries at once.
47+
const attempts = new Map<string, number>();
48+
49+
const pass = Effect.gen(function* () {
50+
const settings = yield* serverSettings.getSettings;
51+
if (!settings.automaticUpdates || !settings.enableProviderUpdateChecks) return;
52+
const nowMs = yield* Clock.currentTimeMillis;
53+
const candidates = (yield* providers.getProviders).filter((provider) => {
54+
if (!isAutoUpdatable(provider)) return false;
55+
const attemptedAt = attempts.get(
56+
`${provider.instanceId}@${provider.versionAdvisory?.latestVersion}`,
57+
);
58+
return attemptedAt === undefined || nowMs - attemptedAt >= Duration.toMillis(RETRY_AFTER);
59+
});
60+
for (const provider of candidates) {
61+
const targetVersion = provider.versionAdvisory?.latestVersion;
62+
const install = Effect.gen(function* () {
63+
attempts.set(`${provider.instanceId}@${targetVersion}`, yield* Clock.currentTimeMillis);
64+
yield* Effect.logInfo("Updating provider in the background", {
65+
instanceId: provider.instanceId,
66+
fromVersion: provider.version,
67+
targetVersion,
68+
});
69+
yield* runner
70+
.updateProvider({ provider: provider.driver, instanceId: provider.instanceId })
71+
.pipe(
72+
Effect.catchCause((cause) =>
73+
Effect.logWarning("Background provider update failed", {
74+
instanceId: provider.instanceId,
75+
cause,
76+
}),
77+
),
78+
);
79+
});
80+
// Rechecked per install: the previous one may have run for minutes.
81+
const installed = yield* updateWindow.runIfOpen({ closesWindows: false }, install);
82+
if (Option.isNone(installed)) {
83+
yield* Effect.logDebug("Provider auto-update waiting for the update window");
84+
return;
85+
}
86+
}
87+
}).pipe(
88+
Effect.catchCause((cause) =>
89+
Effect.logWarning("Provider auto-update pass failed", { cause }),
90+
),
91+
Effect.withSpan("updates.ProviderAutoUpdater.pass"),
92+
);
93+
94+
yield* forkParked(
95+
Effect.sleep(timings.firstPassDelay).pipe(
96+
Effect.andThen(pass.pipe(Effect.repeat(Schedule.spaced(timings.passInterval)))),
97+
),
98+
);
99+
}),
100+
);

0 commit comments

Comments
 (0)