fix(memory): bootstrap lancedb runtime on demand (openclaw#53111)

ngutman · BunsDev · web-flow · commit b98f3634c47e · 2026-03-23T15:07:24.000-05:00
Bootstrap LanceDB into plugin runtime state on first use for packaged/global installs, keep @lancedb/lancedb plugin-local, and add regression coverage for bundled, cached, retry, and Nix fail-fast runtime paths.

Co-authored-by: Val Alexander &lt;68980965+BunsDev@users.noreply.github.com&gt;
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -40,6 +40,7 @@ Docs: https://docs.openclaw.ai
 - ClawHub/macOS: read the local ClawHub login from the macOS Application Support path and still honor XDG config on macOS, so skill browsing uses the logged-in token on both default and XDG-style setups. Fixes #52949. Thanks @scoootscooob.
 - Discord/commands: return an explicit unauthorized reply for privileged native slash commands instead of falling through to Discord's misleading generic completion when auth gates reject the sender. Fixes #53041. Thanks @scoootscooob.
 - Models/OpenAI Codex OAuth: bootstrap the env-configured HTTP/HTTPS proxy dispatcher on the stored-credential refresh path before token renewal runs, so expired Codex OAuth profiles can refresh successfully in proxy-required environments instead of locking users out after the first token expiry.
+- Plugins/memory-lancedb: bootstrap LanceDB into plugin runtime state on first use when the bundled npm install does not already have it, so `plugins.slots.memory="memory-lancedb"` works again after global npm installs without moving LanceDB into OpenClaw core dependencies. Fixes #26100.
 
 ## 2026.3.22
 
diff --git a/extensions/memory-lancedb/index.test.ts b/extensions/memory-lancedb/index.test.ts
@@ -148,14 +148,7 @@ describe("memory plugin e2e", () => {
     const toArray = vi.fn(async () => []);
     const limit = vi.fn(() => ({ toArray }));
     const vectorSearch = vi.fn(() => ({ limit }));
-
-    vi.resetModules();
-    vi.doMock("openai", () => ({
-      default: class MockOpenAI {
-        embeddings = { create: embeddingsCreate };
-      },
-    }));
-    vi.doMock("@lancedb/lancedb", () => ({
+    const loadLanceDbModule = vi.fn(async () => ({
       connect: vi.fn(async () => ({
         tableNames: vi.fn(async () => ["memories"]),
         openTable: vi.fn(async () => ({
@@ -167,6 +160,16 @@ describe("memory plugin e2e", () => {
       })),
     }));
 
+    vi.resetModules();
+    vi.doMock("openai", () => ({
+      default: class MockOpenAI {
+        embeddings = { create: embeddingsCreate };
+      },
+    }));
+    vi.doMock("./lancedb-runtime.js", () => ({
+      loadLanceDbModule,
+    }));
+
     try {
       const { default: memoryPlugin } = await import("./index.js");
       // oxlint-disable-next-line typescript/no-explicit-any
@@ -214,14 +217,15 @@ describe("memory plugin e2e", () => {
       }
       await recallTool.execute("test-call-dims", { query: "hello dimensions" });
 
+      expect(loadLanceDbModule).toHaveBeenCalledTimes(1);
       expect(embeddingsCreate).toHaveBeenCalledWith({
         model: "text-embedding-3-small",
         input: "hello dimensions",
         dimensions: 1024,
       });
     } finally {
       vi.doUnmock("openai");
-      vi.doUnmock("@lancedb/lancedb");
+      vi.doUnmock("./lancedb-runtime.js");
       vi.resetModules();
     }
   });
diff --git a/extensions/memory-lancedb/index.ts b/extensions/memory-lancedb/index.ts
@@ -18,24 +18,12 @@ import {
   memoryConfigSchema,
   vectorDimsForModel,
 } from "./config.js";
+import { loadLanceDbModule } from "./lancedb-runtime.js";
 
 // ============================================================================
 // Types
 // ============================================================================
 
-let lancedbImportPromise: Promise<typeof import("@lancedb/lancedb")> | null = null;
-const loadLanceDB = async (): Promise<typeof import("@lancedb/lancedb")> => {
-  if (!lancedbImportPromise) {
-    lancedbImportPromise = import("@lancedb/lancedb");
-  }
-  try {
-    return await lancedbImportPromise;
-  } catch (err) {
-    // Common on macOS today: upstream package may not ship darwin native bindings.
-    throw new Error(`memory-lancedb: failed to load LanceDB. ${String(err)}`, { cause: err });
-  }
-};
-
 type MemoryEntry = {
   id: string;
   text: string;
@@ -79,7 +67,7 @@ class MemoryDB {
   }
 
   private async doInitialize(): Promise<void> {
-    const lancedb = await loadLanceDB();
+    const lancedb = await loadLanceDbModule();
     this.db = await lancedb.connect(this.dbPath);
     const tables = await this.db.tableNames();
 
diff --git a/extensions/memory-lancedb/lancedb-runtime.test.ts b/extensions/memory-lancedb/lancedb-runtime.test.ts
@@ -0,0 +1,176 @@
+import { describe, expect, it, vi } from "vitest";
+import { createLanceDbRuntimeLoader, type LanceDbRuntimeLogger } from "./lancedb-runtime.js";
+
+const TEST_RUNTIME_MANIFEST = {
+  name: "openclaw-memory-lancedb-runtime",
+  private: true as const,
+  type: "module" as const,
+  dependencies: {
+    "@lancedb/lancedb": "^0.27.1",
+  },
+};
+
+type LanceDbModule = typeof import("@lancedb/lancedb");
+type RuntimeManifest = {
+  name: string;
+  private: true;
+  type: "module";
+  dependencies: Record<string, string>;
+};
+
+function createMockModule(): LanceDbModule {
+  return {
+    connect: vi.fn(),
+  } as unknown as LanceDbModule;
+}
+
+function createLoader(
+  overrides: {
+    env?: NodeJS.ProcessEnv;
+    importBundled?: () => Promise<LanceDbModule>;
+    importResolved?: (resolvedPath: string) => Promise<LanceDbModule>;
+    resolveRuntimeEntry?: (params: {
+      runtimeDir: string;
+      manifest: RuntimeManifest;
+    }) => string | null;
+    installRuntime?: (params: {
+      runtimeDir: string;
+      manifest: RuntimeManifest;
+      env: NodeJS.ProcessEnv;
+      logger?: LanceDbRuntimeLogger;
+    }) => Promise<string>;
+  } = {},
+) {
+  return createLanceDbRuntimeLoader({
+    env: overrides.env ?? ({} as NodeJS.ProcessEnv),
+    resolveStateDir: () => "/tmp/openclaw-state",
+    runtimeManifest: TEST_RUNTIME_MANIFEST,
+    importBundled:
+      overrides.importBundled ??
+      (async () => {
+        throw new Error("Cannot find package '@lancedb/lancedb'");
+      }),
+    importResolved: overrides.importResolved ?? (async () => createMockModule()),
+    resolveRuntimeEntry: overrides.resolveRuntimeEntry ?? (() => null),
+    installRuntime:
+      overrides.installRuntime ??
+      (async ({ runtimeDir }: { runtimeDir: string }) =>
+        `${runtimeDir}/node_modules/@lancedb/lancedb/index.js`),
+  });
+}
+
+describe("lancedb runtime loader", () => {
+  it("uses the bundled module when it is already available", async () => {
+    const bundledModule = createMockModule();
+    const importBundled = vi.fn(async () => bundledModule);
+    const importResolved = vi.fn(async () => createMockModule());
+    const resolveRuntimeEntry = vi.fn(() => null);
+    const installRuntime = vi.fn(async () => "/tmp/openclaw-state/plugin-runtimes/lancedb.js");
+    const loader = createLoader({
+      importBundled,
+      importResolved,
+      resolveRuntimeEntry,
+      installRuntime,
+    });
+
+    await expect(loader.load()).resolves.toBe(bundledModule);
+
+    expect(resolveRuntimeEntry).not.toHaveBeenCalled();
+    expect(installRuntime).not.toHaveBeenCalled();
+    expect(importResolved).not.toHaveBeenCalled();
+  });
+
+  it("reuses an existing user runtime install before attempting a reinstall", async () => {
+    const runtimeModule = createMockModule();
+    const importResolved = vi.fn(async () => runtimeModule);
+    const resolveRuntimeEntry = vi.fn(
+      () => "/tmp/openclaw-state/plugin-runtimes/memory-lancedb/runtime-entry.js",
+    );
+    const installRuntime = vi.fn(
+      async () => "/tmp/openclaw-state/plugin-runtimes/memory-lancedb/runtime-entry.js",
+    );
+    const loader = createLoader({
+      importResolved,
+      resolveRuntimeEntry,
+      installRuntime,
+    });
+
+    await expect(loader.load()).resolves.toBe(runtimeModule);
+
+    expect(resolveRuntimeEntry).toHaveBeenCalledWith(
+      expect.objectContaining({
+        runtimeDir: "/tmp/openclaw-state/plugin-runtimes/memory-lancedb/lancedb",
+      }),
+    );
+    expect(installRuntime).not.toHaveBeenCalled();
+  });
+
+  it("installs LanceDB into user state when the bundled runtime is unavailable", async () => {
+    const runtimeModule = createMockModule();
+    const logger: LanceDbRuntimeLogger = {
+      warn: vi.fn(),
+      info: vi.fn(),
+    };
+    const importResolved = vi.fn(async () => runtimeModule);
+    const resolveRuntimeEntry = vi.fn(() => null);
+    const installRuntime = vi.fn(
+      async ({ runtimeDir }: { runtimeDir: string }) =>
+        `${runtimeDir}/node_modules/@lancedb/lancedb/index.js`,
+    );
+    const loader = createLoader({
+      importResolved,
+      resolveRuntimeEntry,
+      installRuntime,
+    });
+
+    await expect(loader.load(logger)).resolves.toBe(runtimeModule);
+
+    expect(installRuntime).toHaveBeenCalledWith(
+      expect.objectContaining({
+        runtimeDir: "/tmp/openclaw-state/plugin-runtimes/memory-lancedb/lancedb",
+        manifest: TEST_RUNTIME_MANIFEST,
+      }),
+    );
+    expect(logger.warn).toHaveBeenCalledWith(
+      expect.stringContaining(
+        "installing runtime deps under /tmp/openclaw-state/plugin-runtimes/memory-lancedb/lancedb",
+      ),
+    );
+  });
+
+  it("fails fast in nix mode instead of attempting auto-install", async () => {
+    const installRuntime = vi.fn(
+      async ({ runtimeDir }: { runtimeDir: string }) =>
+        `${runtimeDir}/node_modules/@lancedb/lancedb/index.js`,
+    );
+    const loader = createLoader({
+      env: { OPENCLAW_NIX_MODE: "1" } as NodeJS.ProcessEnv,
+      installRuntime,
+    });
+
+    await expect(loader.load()).rejects.toThrow(
+      "memory-lancedb: failed to load LanceDB and Nix mode disables auto-install.",
+    );
+    expect(installRuntime).not.toHaveBeenCalled();
+  });
+
+  it("clears the cached failure so later calls can retry the install", async () => {
+    const runtimeModule = createMockModule();
+    const installRuntime = vi
+      .fn()
+      .mockRejectedValueOnce(new Error("network down"))
+      .mockResolvedValueOnce(
+        "/tmp/openclaw-state/plugin-runtimes/memory-lancedb/lancedb/node_modules/@lancedb/lancedb/index.js",
+      );
+    const importResolved = vi.fn(async () => runtimeModule);
+    const loader = createLoader({
+      installRuntime,
+      importResolved,
+    });
+
+    await expect(loader.load()).rejects.toThrow("network down");
+    await expect(loader.load()).resolves.toBe(runtimeModule);
+
+    expect(installRuntime).toHaveBeenCalledTimes(2);
+  });
+});
diff --git a/extensions/memory-lancedb/lancedb-runtime.ts b/extensions/memory-lancedb/lancedb-runtime.ts
diff --git a/src/plugin-sdk/memory-lancedb.ts b/src/plugin-sdk/memory-lancedb.ts
