Skip to content

Commit 0d98109

Browse files
steipetesteipete
authored
refactor(codex): store app-server thread bindings in SQLite plugin state (#101210)
* test(codex): shorten placeholder auth fixture strings * feat(plugin-state): add reject-new overflow policy for keyed namespaces * feat(agents): thread agent identity and session generation through reset and hooks * refactor(codex): add SQLite-backed app-server thread binding store * refactor(codex): move app-server runtime callers onto the binding store * refactor(codex): move conversation, entry, and thread-tool flows onto the binding store * refactor(codex): move command handlers onto the binding store * feat(codex): import legacy binding sidecars via doctor state migration * fix(codex): keep doctor sidecar scan inside stateDir * fix(codex): make binding migration landable * chore(changelog): defer release note * test(plugin-state): stabilize durable-capacity ordering
1 parent a27e3f3 commit 0d98109

68 files changed

Lines changed: 6527 additions & 2815 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎docs/plugins/sdk-runtime.md‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -589,7 +589,7 @@ two-party event loops that do not go through the shared inbound reply runner.
589589
await store.clear();
590590
```
591591

592-
Keyed stores survive restarts and are isolated by the runtime-bound plugin id. Use `registerIfAbsent(...)` for atomic dedupe claims: it returns `true` when the key was missing or expired and registered, or `false` when a live value already exists without overwriting its value, creation time, or TTL. Limits: `maxEntries` per namespace, 50,000 live rows per plugin, JSON values under 64KB, and optional TTL expiry. When a write would exceed the plugin row cap, the runtime sheds the oldest live rows from the namespace being written; sibling namespaces are not evicted for that write, and the write still fails if the namespace cannot free enough rows.
592+
Keyed stores survive restarts and are isolated by the runtime-bound plugin id. Use `registerIfAbsent(...)` for atomic dedupe claims: it returns `true` when the key was missing or expired and registered, or `false` when a live value already exists without overwriting its value, creation time, or TTL. Limits: `maxEntries` per namespace, 50,000 live rows per plugin, JSON values under 64KB, and optional TTL expiry. By default, a write at either row limit sheds the oldest live rows from the namespace being written; sibling namespaces are not evicted for that write, and the write still fails if the namespace cannot free enough rows. Set `overflowPolicy: "reject-new"` for durable ownership records that must never be evicted: new keys fail at either limit, while existing keys remain updateable.
593593

594594
`openSyncKeyedStore<T>(...)` returns the same store shape with synchronous methods (`register`, `registerIfAbsent`, `lookup`, `consume`, `clear` all return values directly instead of promises) for callers that cannot await.
595595

‎extensions/codex/doctor-contract-api.test.ts‎

Lines changed: 175 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,43 @@
11
// Codex tests cover doctor contract api plugin behavior.
2-
import { describe, expect, it } from "vitest";
3-
import { legacyConfigRules, normalizeCompatibilityConfig } from "./doctor-contract-api.js";
2+
import fs from "node:fs/promises";
3+
import os from "node:os";
4+
import path from "node:path";
5+
import {
6+
createPluginStateKeyedStoreForTests,
7+
resetPluginStateStoreForTests,
8+
} from "openclaw/plugin-sdk/plugin-state-test-runtime";
9+
import type {
10+
OpenKeyedStoreOptions,
11+
PluginDoctorStateMigrationContext,
12+
} from "openclaw/plugin-sdk/runtime-doctor";
13+
import { afterEach, describe, expect, it } from "vitest";
14+
import {
15+
legacyConfigRules,
16+
normalizeCompatibilityConfig,
17+
stateMigrations,
18+
} from "./doctor-contract-api.js";
19+
import {
20+
bindingStoreKey,
21+
CODEX_APP_SERVER_BINDING_MAX_ENTRIES,
22+
CODEX_APP_SERVER_BINDING_NAMESPACE,
23+
type StoredCodexAppServerBinding,
24+
} from "./src/app-server/session-binding.js";
25+
import { legacyCodexConversationBindingId } from "./src/conversation-binding-data.js";
26+
27+
function createDoctorContext(env: NodeJS.ProcessEnv): PluginDoctorStateMigrationContext {
28+
return {
29+
openPluginStateKeyedStore<T>(options: OpenKeyedStoreOptions) {
30+
return createPluginStateKeyedStoreForTests<T>("codex", {
31+
...options,
32+
env: options.env ?? env,
33+
});
34+
},
35+
};
36+
}
37+
38+
afterEach(() => {
39+
resetPluginStateStoreForTests();
40+
});
441

542
describe("codex doctor contract", () => {
643
it("reports the retired dynamic tools profile config key", () => {
@@ -84,6 +121,142 @@ describe("codex doctor contract", () => {
84121
expect(original.plugins.entries.codex.config).toHaveProperty("codexDynamicToolsProfile");
85122
});
86123

124+
it("imports and archives shipped binding sidecars", async () => {
125+
const stateDir = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-codex-doctor-"));
126+
const env = { ...process.env, OPENCLAW_STATE_DIR: stateDir };
127+
const sessionsDir = path.join(stateDir, "agents", "main", "sessions");
128+
const transcriptPath = path.join(sessionsDir, "session-current.jsonl");
129+
const sidecarPath = `${transcriptPath}.codex-app-server.json`;
130+
await fs.mkdir(sessionsDir, { recursive: true });
131+
await fs.writeFile(transcriptPath, '{"type":"session","id":"session-current"}\n', "utf8");
132+
await fs.writeFile(
133+
path.join(sessionsDir, "sessions.json"),
134+
JSON.stringify({
135+
"agent:main:session-1": {
136+
sessionId: "session-current",
137+
sessionFile: "session-current.jsonl",
138+
updatedAt: Date.now(),
139+
},
140+
}),
141+
"utf8",
142+
);
143+
await fs.writeFile(
144+
sidecarPath,
145+
JSON.stringify({
146+
schemaVersion: 2,
147+
threadId: "thread-1",
148+
sessionFile: transcriptPath,
149+
updatedAt: "2026-01-01T00:00:00.000Z",
150+
pluginAppPolicyContext: {
151+
fingerprint: "policy-1",
152+
apps: {
153+
app: {
154+
configKey: "app",
155+
marketplaceName: "openai-curated",
156+
pluginName: "plugin",
157+
allowDestructiveActions: true,
158+
destructiveApprovalMode: "ask",
159+
mcpServerNames: [],
160+
},
161+
},
162+
pluginAppIds: {},
163+
},
164+
}),
165+
"utf8",
166+
);
167+
const params = {
168+
config: {},
169+
env,
170+
stateDir,
171+
oauthDir: path.join(stateDir, "oauth"),
172+
context: createDoctorContext(env),
173+
};
174+
const migration = stateMigrations[0];
175+
if (!migration) {
176+
throw new Error("missing Codex binding migration");
177+
}
178+
179+
await expect(migration.detectLegacyState(params)).resolves.toMatchObject({
180+
preview: [expect.stringContaining("legacy sidecar")],
181+
});
182+
await expect(migration.migrateLegacyState(params)).resolves.toMatchObject({
183+
changes: [expect.stringContaining("Migrated 1")],
184+
warnings: [],
185+
});
186+
187+
const store = createDoctorContext(env).openPluginStateKeyedStore<StoredCodexAppServerBinding>({
188+
namespace: CODEX_APP_SERVER_BINDING_NAMESPACE,
189+
maxEntries: CODEX_APP_SERVER_BINDING_MAX_ENTRIES,
190+
overflowPolicy: "reject-new",
191+
});
192+
await expect(
193+
store.lookup(
194+
bindingStoreKey({
195+
kind: "session",
196+
agentId: "main",
197+
sessionId: "session-current",
198+
sessionKey: "agent:main:session-1",
199+
}),
200+
),
201+
).resolves.toMatchObject({
202+
state: "active",
203+
sessionId: "session-current",
204+
binding: {
205+
threadId: "thread-1",
206+
pluginAppPolicyContext: {
207+
apps: { app: { destructiveApprovalMode: "ask" } },
208+
},
209+
},
210+
});
211+
await expect(
212+
store.lookup(
213+
bindingStoreKey({
214+
kind: "conversation",
215+
bindingId: legacyCodexConversationBindingId(transcriptPath),
216+
}),
217+
),
218+
).resolves.toMatchObject({ state: "active", binding: { threadId: "thread-1" } });
219+
await expect(fs.access(`${sidecarPath}.migrated`)).resolves.toBeUndefined();
220+
await expect(
221+
fs.readFile(path.join(sessionsDir, "sessions.json"), "utf8").then(JSON.parse),
222+
).resolves.toMatchObject({
223+
"agent:main:session-1": { sessionId: "session-current", agentHarnessId: "codex" },
224+
});
225+
226+
await fs.rm(stateDir, { recursive: true, force: true });
227+
});
228+
229+
it("does not scan above stateDir when a session store sits at its parent", async () => {
230+
const outerDir = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-codex-doctor-outer-"));
231+
const stateDir = path.join(outerDir, "state");
232+
await fs.mkdir(stateDir, { recursive: true });
233+
const strayDir = path.join(outerDir, "unrelated");
234+
await fs.mkdir(strayDir, { recursive: true });
235+
await fs.writeFile(
236+
path.join(strayDir, "foreign.jsonl.codex-app-server.json"),
237+
JSON.stringify({ schemaVersion: 2, threadId: "thread-foreign" }),
238+
"utf8",
239+
);
240+
const env = { ...process.env, OPENCLAW_STATE_DIR: stateDir };
241+
const params = {
242+
// The store dir is exactly the parent of stateDir; doctor must treat it
243+
// as an external store (indexed reads only), not a scannable state root.
244+
config: { session: { store: path.join(outerDir, "sessions.json") } },
245+
env,
246+
stateDir,
247+
oauthDir: path.join(stateDir, "oauth"),
248+
context: createDoctorContext(env),
249+
};
250+
const migration = stateMigrations[0];
251+
if (!migration) {
252+
throw new Error("missing Codex binding migration");
253+
}
254+
255+
await expect(migration.detectLegacyState(params)).resolves.toBeNull();
256+
257+
await fs.rm(outerDir, { recursive: true, force: true });
258+
});
259+
87260
it("renames old approval-routed destructive plugin policy values", () => {
88261
const original = {
89262
plugins: {

‎extensions/codex/doctor-contract-api.ts‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -121,3 +121,5 @@ export const sessionRouteStateOwners: DoctorSessionRouteStateOwner[] = [
121121
authProfilePrefixes: ["codex:", "codex-cli:", "openai-codex:"],
122122
},
123123
];
124+
125+
export { stateMigrations } from "./src/migration/session-binding-sidecars.js";

‎extensions/codex/harness.test.ts‎

Lines changed: 40 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,16 @@
11
// Codex tests cover harness plugin behavior.
22
import { describe, expect, it } from "vitest";
33
import { createCodexAppServerAgentHarness } from "./harness.js";
4+
import {
5+
createCodexTestBindingStore,
6+
sessionBindingIdentity,
7+
testCodexAppServerBindingStore,
8+
} from "./src/app-server/session-binding.test-helpers.js";
49

510
describe("Codex agent harness supports()", () => {
6-
const harness = createCodexAppServerAgentHarness();
11+
const harness = createCodexAppServerAgentHarness({
12+
bindingStore: testCodexAppServerBindingStore,
13+
});
714

815
it("supports the canonical codex virtual provider", () => {
916
expect(harness.supports({ provider: "codex", requestedRuntime: "codex" })).toEqual({
@@ -40,8 +47,39 @@ describe("Codex agent harness supports()", () => {
4047
});
4148

4249
it("honors explicit provider id overrides", () => {
43-
const narrowHarness = createCodexAppServerAgentHarness({ providerIds: ["codex"] });
50+
const narrowHarness = createCodexAppServerAgentHarness({
51+
providerIds: ["codex"],
52+
bindingStore: testCodexAppServerBindingStore,
53+
});
4454
const result = narrowHarness.supports({ provider: "openai", requestedRuntime: "codex" });
4555
expect(result.supported).toBe(false);
4656
});
4757
});
58+
59+
describe("Codex agent harness reset()", () => {
60+
it("retires the physical session generation", async () => {
61+
const bindingStore = createCodexTestBindingStore();
62+
const identity = sessionBindingIdentity({
63+
agentId: "worker",
64+
sessionId: "session-1",
65+
sessionKey: "agent:worker:main",
66+
});
67+
await bindingStore.mutate(identity, {
68+
kind: "set",
69+
binding: { threadId: "thread-1", cwd: "/repo" },
70+
});
71+
const harness = createCodexAppServerAgentHarness({ bindingStore });
72+
if (!harness.reset) {
73+
throw new Error("expected Codex harness reset hook");
74+
}
75+
76+
await harness.reset({
77+
agentId: "worker",
78+
sessionId: "session-1",
79+
sessionKey: "agent:worker:main",
80+
reason: "reset",
81+
});
82+
83+
await expect(bindingStore.read(identity)).resolves.toBeUndefined();
84+
});
85+
});

‎extensions/codex/harness.ts‎

Lines changed: 33 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -7,11 +7,13 @@ import type {
77
AgentHarnessCompactResult,
88
ContextEngineHostCapability,
99
} from "openclaw/plugin-sdk/agent-harness-runtime";
10+
import type { OpenClawConfig } from "openclaw/plugin-sdk/config-contracts";
1011
import type {
1112
CodexAppServerListModelsOptions,
1213
CodexAppServerModel,
1314
CodexAppServerModelListResult,
1415
} from "./src/app-server/models.js";
16+
import type { CodexAppServerBindingStore } from "./src/app-server/session-binding.js";
1517

1618
const DEFAULT_CODEX_HARNESS_PROVIDER_IDS = new Set(["codex", "openai"]);
1719
const CODEX_APP_SERVER_CONTEXT_ENGINE_HOST_CAPABILITIES = [
@@ -37,12 +39,14 @@ type CodexAppServerAgentHarness = AgentHarness & {
3739
* Creates the Codex app-server harness used for attempts, side questions,
3840
* compaction, reset, and disposal.
3941
*/
40-
export function createCodexAppServerAgentHarness(options?: {
42+
export function createCodexAppServerAgentHarness(options: {
4143
id?: string;
4244
label?: string;
4345
providerIds?: Iterable<string>;
4446
pluginConfig?: unknown;
4547
resolvePluginConfig?: () => unknown;
48+
resolveConfig?: () => OpenClawConfig | undefined;
49+
bindingStore: CodexAppServerBindingStore;
4650
}): AgentHarness {
4751
const providerIds = new Set(
4852
[...(options?.providerIds ?? DEFAULT_CODEX_HARNESS_PROVIDER_IDS)].map((id) =>
@@ -71,34 +75,59 @@ export function createCodexAppServerAgentHarness(options?: {
7175
// cold provider catalog reads do not pull in the whole Codex runtime.
7276
const { runCodexAppServerAttempt } = await import("./src/app-server/run-attempt.js");
7377
return runCodexAppServerAttempt(params, {
78+
bindingStore: options.bindingStore,
7479
pluginConfig: options?.resolvePluginConfig?.() ?? options?.pluginConfig,
7580
nativeHookRelay: { enabled: true },
7681
});
7782
},
7883
runSideQuestion: async (params) => {
7984
const { runCodexAppServerSideQuestion } = await import("./src/app-server/side-question.js");
8085
return runCodexAppServerSideQuestion(params, {
86+
bindingStore: options.bindingStore,
8187
pluginConfig: options?.resolvePluginConfig?.() ?? options?.pluginConfig,
8288
nativeHookRelay: { enabled: true },
8389
});
8490
},
8591
compact: async (params) => {
8692
const { maybeCompactCodexAppServerSession } = await import("./src/app-server/compact.js");
8793
return maybeCompactCodexAppServerSession(params, {
94+
bindingStore: options.bindingStore,
8895
pluginConfig: options?.resolvePluginConfig?.() ?? options?.pluginConfig,
8996
});
9097
},
9198
compactAfterContextEngine: async (params) => {
9299
const { maybeCompactCodexAppServerSession } = await import("./src/app-server/compact.js");
93100
return maybeCompactCodexAppServerSession(params, {
101+
bindingStore: options.bindingStore,
94102
pluginConfig: options?.resolvePluginConfig?.() ?? options?.pluginConfig,
95103
allowNonManualNativeRequest: true,
96104
});
97105
},
98106
reset: async (params) => {
99-
if (params.sessionFile) {
100-
const { clearCodexAppServerBinding } = await import("./src/app-server/session-binding.js");
101-
await clearCodexAppServerBinding(params.sessionFile);
107+
if (params.sessionId) {
108+
const { reclaimCurrentCodexSessionGeneration, sessionBindingIdentity } =
109+
await import("./src/app-server/session-binding.js");
110+
const identity = sessionBindingIdentity({
111+
agentId: params.agentId,
112+
sessionId: params.sessionId,
113+
sessionKey: params.sessionKey,
114+
});
115+
let retired = await options.bindingStore.retireSessionGeneration(identity);
116+
if (retired === "conflict") {
117+
const reclaimed = await reclaimCurrentCodexSessionGeneration({
118+
bindingStore: options.bindingStore,
119+
identity,
120+
config: options.resolveConfig?.(),
121+
});
122+
if (reclaimed) {
123+
retired = await options.bindingStore.retireSessionGeneration(identity);
124+
}
125+
}
126+
if (retired === "conflict") {
127+
throw new Error(
128+
`Codex binding generation changed before session ${params.sessionId} could reset`,
129+
);
130+
}
102131
}
103132
},
104133
dispose: async () => {

0 commit comments

Comments
 (0)