Skip to content

Commit 1bcc4c5

Browse files
steipetesteipete
authored
feat(gateway): add cooperative host suspension (#103618)
* feat(gateway): add cooperative suspension preparation * style: satisfy suspension lint checks * test(gateway): reset work admission between shared suites * fix(gateway): reject upgrades during suspension * fix(gateway): preserve admitted work during suspension * test(gateway): isolate suspension and restart state * fix(gateway): close suspension false-ready gaps * refactor(protocol): slim suspension declaration graph * refactor(plugin-sdk): sever protocol registry edges * fix(gateway): preserve admitted restart follow-ups * fix(gateway): make suspension recovery fail closed * fix(protocol): keep validation formatter re-export only * test(gateway): simplify deferred fixture type * style(gateway): clarify suspension entry name * fix(gateway): retain detached work admission
1 parent fece8c9 commit 1bcc4c5

116 files changed

Lines changed: 7950 additions & 1065 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎apps/shared/OpenClawKit/Sources/OpenClawProtocol/GatewayModels.swift‎

Lines changed: 278 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -425,6 +425,222 @@ public struct ErrorShape: Codable, Sendable {
425425
}
426426
}
427427

428+
public struct GatewaySuspendTaskBlocker: Codable, Sendable {
429+
public let taskid: String
430+
public let status: String
431+
public let runtime: AnyCodable
432+
public let runid: String?
433+
public let label: String?
434+
public let title: String?
435+
436+
public init(
437+
taskid: String,
438+
status: String,
439+
runtime: AnyCodable,
440+
runid: String? = nil,
441+
label: String? = nil,
442+
title: String? = nil)
443+
{
444+
self.taskid = taskid
445+
self.status = status
446+
self.runtime = runtime
447+
self.runid = runid
448+
self.label = label
449+
self.title = title
450+
}
451+
452+
private enum CodingKeys: String, CodingKey {
453+
case taskid = "taskId"
454+
case status
455+
case runtime
456+
case runid = "runId"
457+
case label
458+
case title
459+
}
460+
}
461+
462+
public struct GatewaySuspendBlocker: Codable, Sendable {
463+
public let kind: AnyCodable
464+
public let count: Int
465+
public let message: String
466+
public let task: GatewaySuspendTaskBlocker?
467+
468+
public init(
469+
kind: AnyCodable,
470+
count: Int,
471+
message: String,
472+
task: GatewaySuspendTaskBlocker? = nil)
473+
{
474+
self.kind = kind
475+
self.count = count
476+
self.message = message
477+
self.task = task
478+
}
479+
480+
private enum CodingKeys: String, CodingKey {
481+
case kind
482+
case count
483+
case message
484+
case task
485+
}
486+
}
487+
488+
public struct GatewaySuspendPrepareParams: Codable, Sendable {
489+
public let requestid: String
490+
491+
public init(
492+
requestid: String)
493+
{
494+
self.requestid = requestid
495+
}
496+
497+
private enum CodingKeys: String, CodingKey {
498+
case requestid = "requestId"
499+
}
500+
}
501+
502+
public struct GatewaySuspendPrepareBusyResult: Codable, Sendable {
503+
public let status: String
504+
public let reason: AnyCodable
505+
public let retryafterms: Int
506+
public let activecount: Int
507+
public let blockers: [GatewaySuspendBlocker]
508+
509+
public init(
510+
status: String,
511+
reason: AnyCodable,
512+
retryafterms: Int,
513+
activecount: Int,
514+
blockers: [GatewaySuspendBlocker])
515+
{
516+
self.status = status
517+
self.reason = reason
518+
self.retryafterms = retryafterms
519+
self.activecount = activecount
520+
self.blockers = blockers
521+
}
522+
523+
private enum CodingKeys: String, CodingKey {
524+
case status
525+
case reason
526+
case retryafterms = "retryAfterMs"
527+
case activecount = "activeCount"
528+
case blockers
529+
}
530+
}
531+
532+
public struct GatewaySuspendPrepareReadyResult: Codable, Sendable {
533+
public let status: String
534+
public let suspensionid: String
535+
public let expiresatms: Int
536+
public let activecount: Int
537+
public let blockers: [GatewaySuspendBlocker]
538+
539+
public init(
540+
status: String,
541+
suspensionid: String,
542+
expiresatms: Int,
543+
activecount: Int,
544+
blockers: [GatewaySuspendBlocker])
545+
{
546+
self.status = status
547+
self.suspensionid = suspensionid
548+
self.expiresatms = expiresatms
549+
self.activecount = activecount
550+
self.blockers = blockers
551+
}
552+
553+
private enum CodingKeys: String, CodingKey {
554+
case status
555+
case suspensionid = "suspensionId"
556+
case expiresatms = "expiresAtMs"
557+
case activecount = "activeCount"
558+
case blockers
559+
}
560+
}
561+
562+
public struct GatewaySuspendStatusParams: Codable, Sendable {
563+
public let suspensionid: String
564+
565+
public init(
566+
suspensionid: String)
567+
{
568+
self.suspensionid = suspensionid
569+
}
570+
571+
private enum CodingKeys: String, CodingKey {
572+
case suspensionid = "suspensionId"
573+
}
574+
}
575+
576+
public struct GatewaySuspendStatusRunningResult: Codable, Sendable {
577+
public let status: String
578+
579+
public init(
580+
status: String)
581+
{
582+
self.status = status
583+
}
584+
585+
private enum CodingKeys: String, CodingKey {
586+
case status
587+
}
588+
}
589+
590+
public struct GatewaySuspendStatusReadyResult: Codable, Sendable {
591+
public let status: String
592+
public let expiresatms: Int
593+
594+
public init(
595+
status: String,
596+
expiresatms: Int)
597+
{
598+
self.status = status
599+
self.expiresatms = expiresatms
600+
}
601+
602+
private enum CodingKeys: String, CodingKey {
603+
case status
604+
case expiresatms = "expiresAtMs"
605+
}
606+
}
607+
608+
public struct GatewaySuspendResumeParams: Codable, Sendable {
609+
public let suspensionid: String
610+
611+
public init(
612+
suspensionid: String)
613+
{
614+
self.suspensionid = suspensionid
615+
}
616+
617+
private enum CodingKeys: String, CodingKey {
618+
case suspensionid = "suspensionId"
619+
}
620+
}
621+
622+
public struct GatewaySuspendResumeResult: Codable, Sendable {
623+
public let ok: Bool
624+
public let status: String
625+
public let resumed: Bool
626+
627+
public init(
628+
ok: Bool,
629+
status: String,
630+
resumed: Bool)
631+
{
632+
self.ok = ok
633+
self.status = status
634+
self.resumed = resumed
635+
}
636+
637+
private enum CodingKeys: String, CodingKey {
638+
case ok
639+
case status
640+
case resumed
641+
}
642+
}
643+
428644
public struct EnvironmentSummary: Codable, Sendable {
429645
public let id: String
430646
public let type: String
@@ -9874,6 +10090,68 @@ public struct ShutdownEvent: Codable, Sendable {
987410090
}
987510091
}
987610092

10093+
public enum GatewaySuspendPrepareResult: Codable, Sendable {
10094+
case busy(GatewaySuspendPrepareBusyResult)
10095+
case ready(GatewaySuspendPrepareReadyResult)
10096+
10097+
private enum CodingKeys: String, CodingKey {
10098+
case discriminator = "status"
10099+
}
10100+
10101+
public init(from decoder: Decoder) throws {
10102+
let container = try decoder.container(keyedBy: CodingKeys.self)
10103+
let discriminator = try container.decode(String.self, forKey: .discriminator)
10104+
switch discriminator {
10105+
case "busy": self = try .busy(GatewaySuspendPrepareBusyResult(from: decoder))
10106+
case "ready": self = try .ready(GatewaySuspendPrepareReadyResult(from: decoder))
10107+
default:
10108+
throw DecodingError.dataCorruptedError(
10109+
forKey: .discriminator,
10110+
in: container,
10111+
debugDescription: "Unknown GatewaySuspendPrepareResult discriminator value"
10112+
)
10113+
}
10114+
}
10115+
10116+
public func encode(to encoder: Encoder) throws {
10117+
switch self {
10118+
case .busy(let value): try value.encode(to: encoder)
10119+
case .ready(let value): try value.encode(to: encoder)
10120+
}
10121+
}
10122+
}
10123+
10124+
public enum GatewaySuspendStatusResult: Codable, Sendable {
10125+
case running(GatewaySuspendStatusRunningResult)
10126+
case ready(GatewaySuspendStatusReadyResult)
10127+
10128+
private enum CodingKeys: String, CodingKey {
10129+
case discriminator = "status"
10130+
}
10131+
10132+
public init(from decoder: Decoder) throws {
10133+
let container = try decoder.container(keyedBy: CodingKeys.self)
10134+
let discriminator = try container.decode(String.self, forKey: .discriminator)
10135+
switch discriminator {
10136+
case "running": self = try .running(GatewaySuspendStatusRunningResult(from: decoder))
10137+
case "ready": self = try .ready(GatewaySuspendStatusReadyResult(from: decoder))
10138+
default:
10139+
throw DecodingError.dataCorruptedError(
10140+
forKey: .discriminator,
10141+
in: container,
10142+
debugDescription: "Unknown GatewaySuspendStatusResult discriminator value"
10143+
)
10144+
}
10145+
}
10146+
10147+
public func encode(to encoder: Encoder) throws {
10148+
switch self {
10149+
case .running(let value): try value.encode(to: encoder)
10150+
case .ready(let value): try value.encode(to: encoder)
10151+
}
10152+
}
10153+
}
10154+
987710155
public enum PluginCatalogInstallAction: Codable, Sendable {
987810156
case clawhub(PluginCatalogClawHubInstall)
987910157
case official(PluginCatalogOfficialInstall)

‎docs/docs_map.md‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3339,6 +3339,7 @@ Do not edit it by hand; run `pnpm docs:map:gen`.
33393339
- Headings:
33403340
- H2: What is available today
33413341
- H2: Recommended path
3342+
- H2: Cooperative host suspension
33423343
- H2: App code vs plugin code
33433344
- H2: Related
33443345

‎docs/gateway/external-apps.md‎

Lines changed: 88 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -50,6 +50,94 @@ terminal result. For durable conversation state, use the `sessions.*` methods.
5050
For UI integrations, subscribe to Gateway events and render only the event
5151
families your app understands.
5252

53+
## Cooperative host suspension
54+
55+
Hosting controllers that freeze or snapshot a running process can use the
56+
host-neutral suspension handshake:
57+
58+
1. Stop admitting external ingress controlled by the host.
59+
2. Call `gateway.suspend.prepare` with a stable, unique `requestId`.
60+
3. If the response is `busy`, keep the process running and retry later.
61+
4. If it is `ready`, save the returned `suspensionId`, then freeze or snapshot
62+
the process before `expiresAtMs`.
63+
5. After thaw, or if suspension is abandoned, call `gateway.suspend.resume`
64+
with that `suspensionId` over the existing WebSocket or Admin HTTP control
65+
path.
66+
67+
A prepared Gateway rejects new WebSocket handshakes. A WebSocket controller
68+
must keep its authenticated connection open across the host operation. If that
69+
cannot be guaranteed, enable and use the
70+
[Admin HTTP RPC plugin](/plugins/admin-http-rpc) before preparing. If the
71+
control path is lost, wait for the two-minute lease to expire before
72+
reconnecting; expiry reopens admission automatically.
73+
74+
The RPC contract is:
75+
76+
- `gateway.suspend.prepare` — `operator.admin`; params
77+
`{ "requestId": "stable-host-operation-id" }`
78+
- `gateway.suspend.status` — `operator.read`; params
79+
`{ "suspensionId": "id-from-prepare" }`
80+
- `gateway.suspend.resume` — `operator.admin`; params
81+
`{ "suspensionId": "id-from-prepare" }`
82+
83+
IDs are trimmed, must contain a non-whitespace character, and are limited to
84+
128 characters. A busy prepare result has `status: "busy"`, `reason`,
85+
`retryAfterMs`, `activeCount`, and `blockers`. A ready result has this shape:
86+
87+
```json
88+
{
89+
"status": "ready",
90+
"suspensionId": "2c3f...",
91+
"expiresAtMs": 1770000000000,
92+
"activeCount": 0,
93+
"blockers": []
94+
}
95+
```
96+
97+
Status returns `{"status":"running"}` or a ready result with `expiresAtMs`.
98+
Resume returns `{"ok":true,"status":"running","resumed":true}`; repeating it
99+
after a successful resume returns `resumed: false`.
100+
101+
A competing request ID or transient scheduler-resume failure returns retryable
102+
`UNAVAILABLE` with `retryAfterMs`. During scheduler recovery, prepare, status,
103+
and resume all return that error, the Gateway remains not-ready and
104+
fail-closed, and the host must not freeze or snapshot it. OpenClaw retries the
105+
scheduler automatically and reopens admission only after recovery succeeds. A
106+
mismatched resume ID returns `INVALID_REQUEST`. Prepare shares the Gateway's
107+
control-plane write budget of three attempts per minute; honor the returned
108+
retry delay. WebSocket clients are bucketed by device and IP. Admin HTTP
109+
controllers are bucketed by resolved client IP, so controllers behind one
110+
proxy can share a budget.
111+
112+
Preparation is refuse-only: OpenClaw closes new root/session/command admission,
113+
pauses automatic cron ticks, and inspects work synchronously. If anything is
114+
active, it resumes the scheduler and reopens admission before returning
115+
`busy`; it does not interrupt or drain that work. A ready lease lasts two
116+
minutes. Repeating `prepare` with the same `requestId` renews it; expiry resumes
117+
the scheduler before reopening admission.
118+
Restart emission that becomes due during a ready lease waits until the lease
119+
resumes; an in-flight restart makes preparation return `busy`.
120+
121+
While ready, `/healthz` remains live and `/readyz` returns `503`. Local or
122+
authenticated readiness responses include `gateway-draining`; unauthenticated
123+
remote probes receive only `{ "ready": false }`. The HTTP health probe,
124+
suspension methods on existing WebSocket connections, and an already-enabled
125+
Admin HTTP RPC route remain available. Other RPCs return retryable
126+
`UNAVAILABLE`. Built-in HTTP user-work routes, including OpenAI-compatible
127+
APIs, tool/session operations, node watches, and configured hooks, return
128+
`503` with `error.code: "gateway_unavailable"`.
129+
130+
This handshake does not persist incoming messages, stop third-party channel
131+
transports, or control the hosting platform. The host must fence its ingress
132+
before preparation and remains responsible for wake, snapshot/freeze, and
133+
stop. `activeCount` is the aggregate tracked-work count, while `blockers`
134+
contains the non-zero category counts and bounded task details. This is not a
135+
general process-quiescence barrier. Channel health, maintenance, cache refresh,
136+
plugin-owned HTTP routes, and plugin-owned background work can remain active.
137+
The hosting platform must freeze or snapshot the full process tree and its
138+
filesystem consistently; unregistered work cannot be proven idle by this first
139+
contract.
140+
53141
## App code vs plugin code
54142

55143
Use Gateway RPC when code lives outside OpenClaw:

0 commit comments

Comments
 (0)