Skip to content

Commit bb05532

Browse files
wings1029wings1029
authored
Merge 517ad37 into 3ce3ed6 
2 parents 3ce3ed6 + 517ad37 commit bb05532

4 files changed

Lines changed: 334 additions & 10 deletions

File tree

‎scripts/repro-92241.ts‎

Lines changed: 188 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,188 @@
1+
/**
2+
* Real behavior proof for PR #92351 (issue #92241).
3+
*
4+
* Demonstrates the dist-rotation guard:
5+
* 1. isDistRotationError correctly identifies stale-dist ERR_MODULE_NOT_FOUND
6+
* 2. guardedLoad wrapper catches, clears cache, logs operator warning, re-throws
7+
* 3. Non-rotation errors pass through untouched
8+
*/
9+
import { createLazyPromiseLoader, isDistRotationError } from "../src/shared/lazy-promise.js";
10+
import { formatErrorMessage } from "../src/infra/errors.js";
11+
12+
const PASS = "✓";
13+
const FAIL = "✗";
14+
15+
let passed = 0;
16+
let failed = 0;
17+
18+
function assert(label: string, condition: boolean) {
19+
if (condition) {
20+
console.log(` ${PASS} ${label}`);
21+
passed += 1;
22+
} else {
23+
console.log(` ${FAIL} ${label}`);
24+
failed += 1;
25+
}
26+
}
27+
28+
console.log("=== isDistRotationError: positive cases ===");
29+
30+
// Case 1: Linux ERR_MODULE_NOT_FOUND in openclaw/dist
31+
const err1 = Object.assign(
32+
new Error("Cannot find module '/usr/lib/node_modules/openclaw/dist/cleanup-DlVQZQex.js' imported from ..."),
33+
{ code: "ERR_MODULE_NOT_FOUND" },
34+
);
35+
assert(
36+
"Linux ERR_MODULE_NOT_FOUND with openclaw/dist path → true",
37+
isDistRotationError(err1),
38+
);
39+
40+
// Case 2: Windows path with backslashes
41+
const err2 = Object.assign(
42+
new Error("Cannot find module 'C:\\Users\\app\\AppData\\Roaming\\npm\\node_modules\\openclaw\\dist\\chunk-abc123.js'"),
43+
{ code: "MODULE_NOT_FOUND" },
44+
);
45+
assert(
46+
"Windows MODULE_NOT_FOUND with openclaw\\dist path → true",
47+
isDistRotationError(err2),
48+
);
49+
50+
// Case 3: ERR_MODULE_NOT_FOUND with hashed chunk (the actual symptom)
51+
const err3 = Object.assign(
52+
new Error("Cannot find module '/usr/lib/node_modules/openclaw/dist/chunks/cleanup-DbGY5-v-.js'"),
53+
{ code: "ERR_MODULE_NOT_FOUND" },
54+
);
55+
assert(
56+
"Hashed chunk ERR_MODULE_NOT_FOUND in dist → true",
57+
isDistRotationError(err3),
58+
);
59+
60+
console.log("\n=== isDistRotationError: negative cases ===");
61+
62+
// Case 4: ERR_MODULE_NOT_FOUND outside openclaw
63+
const err4 = Object.assign(
64+
new Error("Cannot find module '/usr/lib/node_modules/lodash/index.js'"),
65+
{ code: "ERR_MODULE_NOT_FOUND" },
66+
);
67+
assert(
68+
"ERR_MODULE_NOT_FOUND outside openclaw → false",
69+
!isDistRotationError(err4),
70+
);
71+
72+
// Case 5: Third-party package missing, importer under openclaw/dist/ (not rotation)
73+
const err5 = Object.assign(
74+
new Error(
75+
"Cannot find package 'optional-dep'" +
76+
" imported from /usr/lib/node_modules/openclaw/dist/chunks/get-reply.js",
77+
),
78+
{ code: "ERR_MODULE_NOT_FOUND" },
79+
);
80+
assert(
81+
"third-party dep missing (importer in dist) → false",
82+
!isDistRotationError(err5),
83+
);
84+
85+
// Case 6: Other error code
86+
const err6 = Object.assign(new Error("ENOENT: no such file"), { code: "ENOENT" });
87+
assert(
88+
"ENOENT error → false",
89+
!isDistRotationError(err6),
90+
);
91+
92+
// Case 7: No code property
93+
assert(
94+
"Error without code → false",
95+
!isDistRotationError(new Error("plain")),
96+
);
97+
98+
// Case 8: Non-object inputs
99+
assert("null → false", !isDistRotationError(null));
100+
assert("undefined → false", !isDistRotationError(undefined));
101+
assert("string → false", !isDistRotationError("ERR_MODULE_NOT_FOUND"));
102+
103+
console.log("\n=== guardedLoad: behavior verification ===");
104+
105+
// Simulate the guardedLoad pattern from get-reply.ts
106+
async function guardedLoad<T>(
107+
loader: { load(): Promise<T>; clear(): void },
108+
label: string,
109+
): Promise<T> {
110+
try {
111+
return await loader.load();
112+
} catch (err) {
113+
if (isDistRotationError(err)) {
114+
loader.clear();
115+
console.log(
116+
`[ERROR] auto-reply/reply-loader: bundled module changed under running gateway ` +
117+
`after update/rollback — restart required (lazy module "${label}" failed: ${formatErrorMessage(err)})`,
118+
);
119+
console.log(
120+
`[WARN] auto-reply/reply-loader: run "systemctl --user restart openclaw-gateway.service" ` +
121+
`(or equivalent) to reload dist modules`,
122+
);
123+
}
124+
throw err;
125+
}
126+
}
127+
128+
// Test: normal load succeeds
129+
let loadCalls = 0;
130+
const goodLoader = createLazyPromiseLoader(async () => {
131+
loadCalls += 1;
132+
return { key: `value-${loadCalls}` };
133+
});
134+
135+
const result1 = await guardedLoad(goodLoader, "test-module");
136+
assert(
137+
"guardedLoad: normal load returns value",
138+
result1.key === "value-1",
139+
);
140+
assert(
141+
"guardedLoad: normal load dedupes (single call)",
142+
loadCalls === 1,
143+
);
144+
145+
// Test: dist rotation error → clear + re-throw
146+
let clearCalled = false;
147+
const distErrLoader = {
148+
load: async () => {
149+
throw Object.assign(
150+
new Error("Cannot find module '/usr/lib/node_modules/openclaw/dist/chunks/stale-hash.js'"),
151+
{ code: "ERR_MODULE_NOT_FOUND" },
152+
);
153+
},
154+
clear: () => { clearCalled = true; },
155+
};
156+
157+
console.log("\n--- dist rotation guard triggers (expected error output below) ---");
158+
let threw = false;
159+
try {
160+
await guardedLoad(distErrLoader, "stale-module");
161+
} catch {
162+
threw = true;
163+
}
164+
assert("guardedLoad: dist rotation error re-thrown", threw);
165+
assert("guardedLoad: dist rotation clears stale cache", clearCalled);
166+
assert("guardedLoad: dist rotation propagated to caller", threw);
167+
168+
// Test: non-rotation error → no clear, re-throw
169+
let clearCalled2 = false;
170+
const plainErrLoader = {
171+
load: async () => {
172+
throw Object.assign(new Error("ENOENT: /tmp/missing"), { code: "ENOENT" });
173+
},
174+
clear: () => { clearCalled2 = true; },
175+
};
176+
177+
console.log("--- non-rotation error (expected pass-through, no clear) ---");
178+
let threw2 = false;
179+
try {
180+
await guardedLoad(plainErrLoader, "other-module");
181+
} catch {
182+
threw2 = true;
183+
}
184+
assert("guardedLoad: non-rotation error does NOT clear cache", !clearCalled2);
185+
assert("guardedLoad: non-rotation error still re-thrown", threw2);
186+
187+
console.log(`\n=== Results: ${passed} passed, ${failed} failed ===`);
188+
process.exit(failed > 0 ? 1 : 0);

‎src/auto-reply/reply/get-reply.ts‎

Lines changed: 42 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,11 @@ import { formatErrorMessage } from "../../infra/errors.js";
2222
import { createSubsystemLogger } from "../../logging/subsystem.js";
2323
import { buildAgentHookContextChannelFields } from "../../plugins/hook-agent-context.js";
2424
import { defaultRuntime } from "../../runtime.js";
25-
import { createLazyImportLoader } from "../../shared/lazy-promise.js";
25+
import {
26+
createLazyImportLoader,
27+
isDistRotationError,
28+
type LazyPromiseLoader,
29+
} from "../../shared/lazy-promise.js";
2630
import { resolveCommandTurnTargetSessionKey } from "../command-turn-context.js";
2731
import type { GetReplyOptions } from "../get-reply-options.types.js";
2832
import { DEFAULT_HEARTBEAT_ACK_MAX_CHARS, stripHeartbeatToken } from "../heartbeat.js";
@@ -100,28 +104,58 @@ const linkUnderstandingApplyRuntimeLoader = createLazyImportLoader(
100104
);
101105

102106
const replyResolverTimingLog = createSubsystemLogger("auto-reply/reply-resolver-timing");
107+
const replyLoaderLog = createSubsystemLogger("auto-reply/reply-loader");
108+
109+
/**
110+
* Load a lazy runtime module with dist-rotation guard.
111+
*
112+
* After an in-place `npm install -g` update or rollback, dist chunk hashes
113+
* rotate while the gateway process is still running. A dynamic `import()` of
114+
* a lazy module that transitively references an old hash hits
115+
* `ERR_MODULE_NOT_FOUND`. Without a guard this failure is swallowed silently
116+
* and inbound messages disappear. This wrapper catches the dist-rotation
117+
* case, clears the stale loader cache, emits an operator-visible warning, and
118+
* re-throws so the inbound delivery fails visibly instead of silently.
119+
*/
120+
async function guardedLoad<T>(loader: LazyPromiseLoader<T>, label: string): Promise<T> {
121+
try {
122+
return await loader.load();
123+
} catch (err) {
124+
if (isDistRotationError(err)) {
125+
loader.clear();
126+
replyLoaderLog.error(
127+
`bundled module changed under running gateway after update/rollback — ` +
128+
`restart required (lazy module "${label}" failed: ${formatErrorMessage(err)})`,
129+
);
130+
replyLoaderLog.warn(
131+
`run "systemctl --user restart openclaw-gateway.service" (or equivalent) to reload dist modules`,
132+
);
133+
}
134+
throw err;
135+
}
136+
}
103137
const commandsCoreRuntimeLoader = createLazyImportLoader(
104138
() => import("./commands-core.runtime.js"),
105139
);
106140

107141
function loadSessionResetModelRuntime() {
108-
return sessionResetModelRuntimeLoader.load();
142+
return guardedLoad(sessionResetModelRuntimeLoader, "session-reset-model");
109143
}
110144

111145
function loadStageSandboxMediaRuntime() {
112-
return stageSandboxMediaRuntimeLoader.load();
146+
return guardedLoad(stageSandboxMediaRuntimeLoader, "stage-sandbox-media");
113147
}
114148

115149
function loadMediaUnderstandingApplyRuntime() {
116-
return mediaUnderstandingApplyRuntimeLoader.load();
150+
return guardedLoad(mediaUnderstandingApplyRuntimeLoader, "media-understanding-apply");
117151
}
118152

119153
function loadLinkUnderstandingApplyRuntime() {
120-
return linkUnderstandingApplyRuntimeLoader.load();
154+
return guardedLoad(linkUnderstandingApplyRuntimeLoader, "link-understanding-apply");
121155
}
122156

123157
function loadCommandsCoreRuntime() {
124-
return commandsCoreRuntimeLoader.load();
158+
return guardedLoad(commandsCoreRuntimeLoader, "commands-core");
125159
}
126160

127161
const hookRunnerGlobalLoader = createLazyImportLoader(
@@ -130,11 +164,11 @@ const hookRunnerGlobalLoader = createLazyImportLoader(
130164
const originRoutingLoader = createLazyImportLoader(() => import("./origin-routing.js"));
131165

132166
function loadHookRunnerGlobal() {
133-
return hookRunnerGlobalLoader.load();
167+
return guardedLoad(hookRunnerGlobalLoader, "hook-runner-global");
134168
}
135169

136170
function loadOriginRouting() {
137-
return originRoutingLoader.load();
171+
return guardedLoad(originRoutingLoader, "origin-routing");
138172
}
139173

140174
function mergeSkillFilters(channelFilter?: string[], agentFilter?: string[]): string[] | undefined {

‎src/shared/lazy-promise.test.ts‎

Lines changed: 68 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,10 @@
1-
// Lazy promise tests cover single-flight loading and error reuse behavior.
1+
// Lazy promise tests cover single-flight loading, error reuse, and dist-rotation detection.
22
import { describe, expect, it, vi } from "vitest";
3-
import { createLazyImportLoader, createLazyPromiseLoader } from "./lazy-promise.js";
3+
import {
4+
createLazyImportLoader,
5+
createLazyPromiseLoader,
6+
isDistRotationError,
7+
} from "./lazy-promise.js";
48

59
describe("createLazyPromiseLoader", () => {
610
it("dedupes concurrent loads and reuses the resolved value", async () => {
@@ -58,3 +62,65 @@ describe("createLazyImportLoader", () => {
5862
await expect(loader.load()).resolves.toEqual({ value: "module" });
5963
});
6064
});
65+
66+
describe("isDistRotationError", () => {
67+
it("detects ERR_MODULE_NOT_FOUND inside the openclaw dist tree", () => {
68+
const err = Object.assign(
69+
new Error(
70+
"Cannot find module '/usr/lib/node_modules/openclaw/dist/cleanup-DlVQZQex.js'" +
71+
" imported from /usr/lib/node_modules/openclaw/dist/chunks/get-reply.js",
72+
),
73+
{ code: "ERR_MODULE_NOT_FOUND" },
74+
);
75+
expect(isDistRotationError(err)).toBe(true);
76+
});
77+
78+
it("detects MODULE_NOT_FOUND inside the openclaw dist tree (Windows path)", () => {
79+
const err = Object.assign(
80+
new Error(
81+
"Cannot find module 'C:\\Users\\app\\openclaw\\dist\\chunk-abc123.js'" +
82+
" imported from C:\\Users\\app\\openclaw\\dist\\chunks\\index.js",
83+
),
84+
{ code: "MODULE_NOT_FOUND" },
85+
);
86+
expect(isDistRotationError(err)).toBe(true);
87+
});
88+
89+
it("returns false when a third-party package is missing but importer is under openclaw/dist/", () => {
90+
// Regression: the missing target is a third-party dependency, not a
91+
// dist chunk. The importer path contains openclaw/dist/ but that
92+
// alone does not make this a rotation error.
93+
const err = Object.assign(
94+
new Error(
95+
"Cannot find package 'optional-dep'" +
96+
" imported from /usr/lib/node_modules/openclaw/dist/chunks/get-reply.js",
97+
),
98+
{ code: "ERR_MODULE_NOT_FOUND" },
99+
);
100+
expect(isDistRotationError(err)).toBe(false);
101+
});
102+
103+
it("returns false for ERR_MODULE_NOT_FOUND outside the dist tree", () => {
104+
const err = Object.assign(
105+
new Error("Cannot find module '/usr/lib/node_modules/other-pkg/index.js'"),
106+
{ code: "ERR_MODULE_NOT_FOUND" },
107+
);
108+
expect(isDistRotationError(err)).toBe(false);
109+
});
110+
111+
it("returns false for unrelated error codes", () => {
112+
const err = Object.assign(new Error("Something broke"), { code: "ENOENT" });
113+
expect(isDistRotationError(err)).toBe(false);
114+
});
115+
116+
it("returns false for error objects without a code", () => {
117+
expect(isDistRotationError(new Error("plain error"))).toBe(false);
118+
});
119+
120+
it("returns false for null and non-objects", () => {
121+
expect(isDistRotationError(null)).toBe(false);
122+
expect(isDistRotationError(undefined)).toBe(false);
123+
expect(isDistRotationError("string")).toBe(false);
124+
expect(isDistRotationError(42)).toBe(false);
125+
});
126+
});

‎src/shared/lazy-promise.ts‎

Lines changed: 36 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,39 @@
1+
/**
2+
* Returns true when `err` matches a synthetic ESM failure triggered by an
3+
* in-place package upgrade or rollback: the dist tree hash rotated while the
4+
* gateway process is still running. Dynamic `import()` resolves to
5+
* `ERR_MODULE_NOT_FOUND` for old hashed chunk names that no longer exist on
6+
* disk.
7+
*
8+
* Only classifies as dist rotation when the *missing module itself* is under
9+
* the openclaw dist tree — a third-party package whose importer happens to be
10+
* under openclaw/dist/ is not a rotation error.
11+
*/
12+
export function isDistRotationError(err: unknown): boolean {
13+
if (!err || typeof err !== "object") {
14+
return false;
15+
}
16+
const obj = err as Record<string, unknown>;
17+
const code = typeof obj.code === "string" ? obj.code : undefined;
18+
if (code !== "ERR_MODULE_NOT_FOUND" && code !== "MODULE_NOT_FOUND") {
19+
return false;
20+
}
21+
const msg = typeof obj.message === "string" ? obj.message : "";
22+
// Node.js ESM errors use the format:
23+
// Cannot find module 'MISSING_TARGET' imported from IMPORTER_PATH
24+
// We must verify that the *missing target* is under the dist tree, not
25+
// merely that the importer happens to be there (otherwise a missing
26+
// third-party dependency imported from openclaw/dist/ would be mislabeled).
27+
const missingMatch = msg.match(
28+
/cannot find (?:module|package)\s+'([^']+)'/i,
29+
);
30+
if (missingMatch) {
31+
return /openclaw[/\\]dist[/\\]/i.test(missingMatch[1]);
32+
}
33+
// Fallback: if we cannot parse the message format, check the full text.
34+
return /openclaw[/\\]dist[/\\]/i.test(msg);
35+
}
36+
137
/** Manual-control promise cache for lazy runtime resources. */
238
export type LazyPromiseLoader<T> = {
339
/** Resolves the cached value, creating one load promise when needed. */

0 commit comments

Comments
 (0)