perf(ci): cache declarations and remove test import tail

steipete · steipete · commit d4c7873e4529 · 2026-07-16T21:38:23.000-07:00
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -1020,9 +1020,9 @@ jobs:
         uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5
         with:
           path: .artifacts/build-all-cache
-          key: ${{ runner.os }}-build-all-v3-${{ hashFiles('package.json', 'pnpm-lock.yaml', 'npm-shrinkwrap.json', 'packages/plugin-sdk/package.json', 'packages/llm-core/package.json', 'packages/model-catalog-core/package.json', 'packages/memory-host-sdk/package.json', 'scripts/build-all.mjs', 'scripts/write-plugin-sdk-entry-dts.ts', 'scripts/lib/plugin-sdk-entries.mjs', 'tsconfig.json', 'tsconfig.plugin-sdk.dts.json', 'src/plugin-sdk/**', 'packages/llm-core/src/**', 'packages/model-catalog-core/src/**', 'packages/memory-host-sdk/src/**', 'src/types/**', 'src/video-generation/dashscope-compatible.ts', 'src/video-generation/types.ts', 'scripts/copy-export-html-templates.ts', 'scripts/lib/copy-assets.ts', 'src/auto-reply/reply/export-html/**') }}
+          key: ${{ runner.os }}-build-all-v4-${{ hashFiles('package.json', 'pnpm-lock.yaml', 'npm-shrinkwrap.json', 'scripts/build-all.mjs', 'scripts/write-plugin-sdk-entry-dts.ts', 'scripts/lib/plugin-sdk-entries.mjs', 'scripts/lib/plugin-sdk-entrypoints.json', 'scripts/lib/plugin-sdk-private-local-only-subpaths.json', 'scripts/lib/plugin-sdk-deprecated-public-subpaths.json', 'scripts/lib/plugin-sdk-deprecated-barrel-subpaths.json', 'scripts/copy-export-html-templates.ts', 'scripts/lib/copy-assets.ts', 'tsconfig.json', 'tsconfig.plugin-sdk.dts.json', 'src/**', 'packages/**', '!src/**/dist/**', '!src/**/node_modules/**', '!packages/**/dist/**', '!packages/**/node_modules/**') }}
           restore-keys: |
-            ${{ runner.os }}-build-all-v3-
+            ${{ runner.os }}-build-all-v4-
 
       - name: Restore dist build cache
         id: dist_build_cache
diff --git a/docs/ci.md b/docs/ci.md
@@ -112,10 +112,11 @@ The slowest Node test families are split or balanced so each job stays small wit
 - Agentic gateway/server (control-plane) configs split across chat, auth, model, HTTP/plugin, runtime, and startup lanes instead of waiting on built artifacts.
 - Normal CI packs only isolated infra include-pattern shards into deterministic bundles of at most 64 test files, reducing the Node matrix without merging non-isolated command/cron, stateful agents-core, or gateway/server suites. Heavy fixed suites stay on 8 vCPU while the bundled and lower-weight lanes use 4 vCPU.
 - Pull requests on the canonical repository reuse the changed-test resolver against the synthetic merged-tree diff. Precise changes run one targeted Node job; each selected test file gets its own process so stateful suite isolation remains intact. The planner combines sibling tests with import-graph dependents and falls back to the existing 14-job compact full-suite plan for workspace package, package/lockfile, shared harness, split-config, renamed, or deleted changes, public extension-contract changes, tests with special shard setup, partially resolved or empty targets, oversized path or target plans, and planner errors. Targeted plans always retain the full built-artifact boundary gate because its repository scanners cannot be derived from imports. `main` pushes, manual dispatches, and release gates retain the full matrix because canceled superseded `main` runs make a single-push diff insufficient as integration proof.
-- The full Node matrix admits the consistently slow serial tooling and auto-reply command shards first. This keeps the 28-job cap while preventing short alphabetical groups from pushing critical-path work into a later wave.
+- The full Node matrix admits the consistently slow serial tooling, auto-reply command shards, and broad core-fast cache writer first. This keeps the 28-job cap while preventing critical-path work and the next run's transform seed from slipping into a later wave.
 - Broad browser, QA, media, and miscellaneous plugin tests use their dedicated Vitest configs instead of the shared plugin catch-all. Include-pattern shards record timing entries using the CI shard name, so `.artifacts/vitest-shard-timings.json` can distinguish a whole config from a filtered shard.
 - Linux Node shard jobs persist Vitest's experimental filesystem module cache. Trusted Blacksmith jobs use PR-scoped writable overlays seeded from the protected snapshot only when their transform-input generations match; GitHub-hosted and fork jobs use an `actions/cache` fallback with coarse restore prefixes. The planner marks the broad `core-unit-fast` graph as the single writer without coupling cache ownership to matrix order, while every other job restores a private read-only clone. Concurrent Vitest workers retain separate live directories. A transform-input fingerprint clears incompatible lockfile, package, tsconfig, and Vitest-config generations inside stable sticky keys. Only the writer scans and prunes the cache to 75% after it exceeds 2 GiB. A non-cancelling daily or default-branch repository-dispatch warmer prevents rapid `main` pushes from starving the protected seed, and closed PR cache archives are deleted.
 - Node shard and build-artifact jobs also restore Node's portable on-disk compile cache. Independent `test` and `build` namespaces prevent their writers from replacing each other's snapshots: the scheduled test warmer owns the protected test seed, while `build-artifacts` publishes the protected build seed only from trusted `main` pushes. PR jobs read protected snapshots without publishing feature-branch bytecode; fallback archives remain PR-scoped. This reuses V8 bytecode for Node-loaded orchestration, build tooling, and external dependencies across different checkout paths, including when only part of the source graph changes. Vitest child processes disable an inherited compile cache because coverage can be enabled inside dynamic configs and V8 coverage can lose source-position precision when scripts are deserialized from bytecode.
+- The build-artifact job also persists content-fingerprinted `build-all` step outputs. CI's self-built plugin SDK declarations hash the complete repository-owned TypeScript/JSON source graph, exclude installed and generated directories, and restore both flat declarations and package bridges after `tsdown` clears `dist`. Documentation, workflow, plugin, and other changes outside that graph can reuse the declaration snapshot; source changes rebuild it before the export gate runs.
 - `check-additional-*` stripes the supplemental boundary guard list (`scripts/run-additional-boundary-checks.mjs`) into one prompt-heavy shard (`check-additional-boundaries-a`, which includes the Codex prompt snapshot drift check) and one combined shard for the remaining stripes (`check-additional-boundaries-bcd`), each running independent guards concurrently and printing per-check timings. Package-boundary compile/canary work stays together, and runtime topology architecture runs separately from the gateway watch coverage embedded in `build-artifacts`.
 - Gateway watch, channel tests, and the core support-boundary shard run concurrently inside `build-artifacts` after `dist/` and `dist-runtime/` are already built.
 
diff --git a/scripts/build-all.mjs b/scripts/build-all.mjs
@@ -13,25 +13,50 @@ import { resolvePnpmRunner } from "./pnpm-runner.mjs";
 
 const nodeBin = process.execPath;
 const FULL_GIT_COMMIT_RE = /^[0-9a-f]{40}$/iu;
-const BUILD_CACHE_VERSION = 3;
+const BUILD_CACHE_VERSION = 4;
 const PLUGIN_SDK_ENTRY_DTS_CACHE_ENV = [
   "OPENCLAW_BUILD_PRIVATE_QA",
   "OPENCLAW_PLUGIN_SDK_CANONICAL_DTS",
 ];
-const PLUGIN_SDK_ENTRY_DTS_CACHE_INPUTS = [
+const PLUGIN_SDK_ENTRY_DTS_SHARED_CACHE_INPUTS = [
   "scripts/write-plugin-sdk-entry-dts.ts",
   "scripts/lib/plugin-sdk-entries.mjs",
   "scripts/lib/plugin-sdk-entrypoints.json",
   "scripts/lib/plugin-sdk-private-local-only-subpaths.json",
   "scripts/lib/plugin-sdk-deprecated-public-subpaths.json",
   "scripts/lib/plugin-sdk-deprecated-barrel-subpaths.json",
+];
+const PLUGIN_SDK_ENTRY_DTS_CACHE_INPUTS = [
+  ...PLUGIN_SDK_ENTRY_DTS_SHARED_CACHE_INPUTS,
   { path: "dist/plugin-sdk", extensions: [".d.ts"], recursive: false },
 ];
+const PLUGIN_SDK_SELF_BUILT_ENTRY_DTS_CACHE_INPUTS = [
+  ...PLUGIN_SDK_ENTRY_DTS_SHARED_CACHE_INPUTS,
+  "package.json",
+  "pnpm-lock.yaml",
+  "npm-shrinkwrap.json",
+  "tsconfig.json",
+  "tsconfig.plugin-sdk.dts.json",
+  {
+    path: "src",
+    extensions: [".ts", ".tsx", ".mts", ".cts", ".json"],
+    excludeDirectories: ["dist", "node_modules"],
+  },
+  {
+    path: "packages",
+    extensions: [".ts", ".tsx", ".mts", ".cts", ".json"],
+    excludeDirectories: ["dist", "node_modules"],
+  },
+];
 const PLUGIN_SDK_ENTRY_DTS_CACHE_OUTPUTS = [
   "dist/plugin-sdk/webhook-path.js",
   "dist/plugin-sdk/.boundary-entry-shims.stamp",
   ...pluginSdkEntrypoints.map((entry) => `packages/plugin-sdk/dist/src/plugin-sdk/${entry}.d.ts`),
 ];
+const PLUGIN_SDK_SELF_BUILT_ENTRY_DTS_CACHE_OUTPUTS = [
+  { path: "dist/plugin-sdk", extensions: [".d.ts"], recursive: false },
+  ...PLUGIN_SDK_ENTRY_DTS_CACHE_OUTPUTS,
+];
 const PNPM_STEP_NODE_FALLBACKS = new Map([
   ["plugins:assets:build", ["scripts/bundled-plugin-assets.mjs", "--phase", "build"]],
   ["plugins:assets:copy", ["scripts/bundled-plugin-assets.mjs", "--phase", "copy"]],
@@ -275,14 +300,18 @@ export function resolveBuildAllSteps(profile = "full") {
     }
     const mergedEnv = Object.assign({}, step.env, env);
     const merged = Object.assign({}, step, { env: mergedEnv });
-    // Self-built plugin-sdk declarations depend on the whole SDK source
-    // graph, which the canonical-mode cache inputs do not cover; a cache hit
-    // here would restore stale declarations after source changes.
+    // Self-built declarations need both the complete repository-owned type
+    // graph and the flat declarations that this step generates after tsdown
+    // clears dist. Canonical mode keeps its narrower generated-dts cache.
     if (
       step.label === "write-plugin-sdk-entry-dts" &&
       mergedEnv.OPENCLAW_PLUGIN_SDK_CANONICAL_DTS !== "1"
     ) {
-      delete merged.cache;
+      merged.cache = {
+        ...step.cache,
+        inputs: PLUGIN_SDK_SELF_BUILT_ENTRY_DTS_CACHE_INPUTS,
+        outputs: PLUGIN_SDK_SELF_BUILT_ENTRY_DTS_CACHE_OUTPUTS,
+      };
     }
     return merged;
   });
@@ -396,6 +425,10 @@ function cacheEntryIncludesFile(entry, filePath) {
   return entry.extensions.some((extension) => filePath.endsWith(extension));
 }
 
+function cacheEntryExcludesDirectory(entry, name) {
+  return entry.excludeDirectories?.includes(name) ?? false;
+}
+
 function listFilesRecursively(rootPath, fsImpl, cacheEntry = { path: rootPath }) {
   let stat;
   try {
@@ -417,6 +450,9 @@ function listFilesRecursively(rootPath, fsImpl, cacheEntry = { path: rootPath })
       continue;
     }
     const entryPath = path.join(rootPath, dirent.name);
+    if (dirent.isDirectory() && cacheEntryExcludesDirectory(cacheEntry, dirent.name)) {
+      continue;
+    }
     if (dirent.isDirectory() && recursive) {
       out.push(...listFilesRecursively(entryPath, fsImpl, cacheEntry));
     } else if (dirent.isFile() && cacheEntryIncludesFile(cacheEntry, entryPath)) {
diff --git a/scripts/lib/ci-node-test-plan.mjs b/scripts/lib/ci-node-test-plan.mjs
@@ -205,6 +205,9 @@ const FULL_NODE_TEST_ADMISSION_PRIORITY = new Map([
   ["auto-reply-reply-commands-1", 1],
   ["auto-reply-reply-commands-2", 1],
   ["auto-reply-reply-commands-3", 1],
+  // Start the broad cache writer in the first admission wave so later jobs
+  // can reuse its protected transform snapshot on the next run.
+  ["core-unit-fast", 1],
 ]);
 // Commands and cron run non-isolated, so keep their split shards as separate
 // processes. Combining their include lists can retain test state across groups.
diff --git a/src/auto-reply/reply/commands-handlers.order.ts b/src/auto-reply/reply/commands-handlers.order.ts
@@ -0,0 +1,48 @@
+// Keep command precedence inspectable without loading every handler module.
+// The runtime map must cover this complete union before it can build the list.
+export const commandHandlerOrder = [
+  // Plugin text commands must win before built-in auth routing handles /login.
+  "plugin",
+  "login",
+  "dock",
+  "btw",
+  "bash",
+  "activation",
+  "send-policy",
+  "fast",
+  "usage",
+  "session",
+  "restart",
+  "tts",
+  "help",
+  "commands-list",
+  // Keep deterministic /skill usage before broader tool/status fallthrough.
+  "skill-usage",
+  "tools",
+  "status",
+  "goal",
+  "learn",
+  "name",
+  "diagnostics",
+  "tasks",
+  "steer",
+  "allowlist",
+  "approve",
+  "context",
+  "export-session",
+  "export-trajectory",
+  "whoami",
+  "system-agent",
+  "subagents",
+  "acp",
+  "mcp",
+  "plugins",
+  "config",
+  "debug",
+  "models",
+  "stop",
+  "compact",
+  "abort-trigger",
+] as const;
+
+export type CommandHandlerId = (typeof commandHandlerOrder)[number];
diff --git a/src/auto-reply/reply/commands-handlers.registration.test.ts b/src/auto-reply/reply/commands-handlers.registration.test.ts
@@ -1,22 +1,16 @@
-// Registration proof for the full command-handler barrel. Keep barrel-loading
-// assertions here so narrow per-command tests do not import every handler.
 import { describe, expect, it } from "vitest";
-import { loadCommandHandlers } from "./commands-handlers.runtime.js";
-import { handleLoginCommand } from "./commands-login.js";
-import { handleNameCommand } from "./commands-name.js";
-import { handlePluginCommand } from "./commands-plugin.js";
+import { commandHandlerOrder } from "./commands-handlers.order.js";
 
 describe("command handler registration", () => {
   it("registers built-in handlers in the runtime handler list", () => {
-    const handlers = loadCommandHandlers();
-    expect(handlers).toContain(handleNameCommand);
-    expect(handlers).toContain(handleLoginCommand);
+    expect(commandHandlerOrder).toContain("name");
+    expect(commandHandlerOrder).toContain("login");
+    expect(new Set(commandHandlerOrder).size).toBe(commandHandlerOrder.length);
   });
 
   it("keeps plugin text commands ahead of built-in /login", () => {
-    const handlers = loadCommandHandlers();
-    expect(handlers.indexOf(handlePluginCommand)).toBeLessThan(
-      handlers.indexOf(handleLoginCommand),
+    expect(commandHandlerOrder.indexOf("plugin")).toBeLessThan(
+      commandHandlerOrder.indexOf("login"),
     );
   });
 });
diff --git a/src/auto-reply/reply/commands-handlers.runtime.ts b/src/auto-reply/reply/commands-handlers.runtime.ts
@@ -10,6 +10,7 @@ import { handleContextCommand } from "./commands-context-command.js";
 import { handleDiagnosticsCommand } from "./commands-diagnostics.js";
 import { handleDockCommand } from "./commands-dock.js";
 import { handleGoalCommand } from "./commands-goal.js";
+import { commandHandlerOrder, type CommandHandlerId } from "./commands-handlers.order.js";
 import {
   handleCommandsListCommand,
   handleExportTrajectoryCommand,
@@ -44,49 +45,49 @@ import { handleTtsCommands } from "./commands-tts.js";
 import type { CommandHandler } from "./commands-types.js";
 import { handleWhoamiCommand } from "./commands-whoami.js";
 
+const commandHandlersById = {
+  acp: handleAcpCommand,
+  activation: handleActivationCommand,
+  allowlist: handleAllowlistCommand,
+  approve: handleApproveCommand,
+  "abort-trigger": handleAbortTrigger,
+  bash: handleBashCommand,
+  btw: handleBtwCommand,
+  "commands-list": handleCommandsListCommand,
+  compact: handleCompactCommand,
+  config: handleConfigCommand,
+  context: handleContextCommand,
+  debug: handleDebugCommand,
+  diagnostics: handleDiagnosticsCommand,
+  dock: handleDockCommand,
+  "export-session": handleExportSessionCommand,
+  "export-trajectory": handleExportTrajectoryCommand,
+  fast: handleFastCommand,
+  goal: handleGoalCommand,
+  help: handleHelpCommand,
+  learn: handleLearnCommand,
+  login: handleLoginCommand,
+  mcp: handleMcpCommand,
+  models: handleModelsCommand,
+  name: handleNameCommand,
+  plugin: handlePluginCommand,
+  plugins: handlePluginsCommand,
+  restart: handleRestartCommand,
+  "send-policy": handleSendPolicyCommand,
+  session: handleSessionCommand,
+  "skill-usage": handleSkillCommandUsage,
+  status: handleStatusCommand,
+  steer: handleSteerCommand,
+  stop: handleStopCommand,
+  subagents: handleSubagentsCommand,
+  "system-agent": handleSystemAgentCommand,
+  tasks: handleTasksCommand,
+  tools: handleToolsCommand,
+  tts: handleTtsCommands,
+  usage: handleUsageCommand,
+  whoami: handleWhoamiCommand,
+} satisfies Record<CommandHandlerId, CommandHandler>;
+
 export function loadCommandHandlers(): CommandHandler[] {
-  return [
-    handlePluginCommand,
-    handleLoginCommand,
-    handleDockCommand,
-    handleBtwCommand,
-    handleBashCommand,
-    handleActivationCommand,
-    handleSendPolicyCommand,
-    handleFastCommand,
-    handleUsageCommand,
-    handleSessionCommand,
-    handleRestartCommand,
-    handleTtsCommands,
-    handleHelpCommand,
-    handleCommandsListCommand,
-    // Keep deterministic /skill usage on the native command path before the
-    // broader tool/status handlers can fall through to an agent run.
-    handleSkillCommandUsage,
-    handleToolsCommand,
-    handleStatusCommand,
-    handleGoalCommand,
-    handleLearnCommand,
-    handleNameCommand,
-    handleDiagnosticsCommand,
-    handleTasksCommand,
-    handleSteerCommand,
-    handleAllowlistCommand,
-    handleApproveCommand,
-    handleContextCommand,
-    handleExportSessionCommand,
-    handleExportTrajectoryCommand,
-    handleWhoamiCommand,
-    handleSystemAgentCommand,
-    handleSubagentsCommand,
-    handleAcpCommand,
-    handleMcpCommand,
-    handlePluginsCommand,
-    handleConfigCommand,
-    handleDebugCommand,
-    handleModelsCommand,
-    handleStopCommand,
-    handleCompactCommand,
-    handleAbortTrigger,
-  ];
+  return commandHandlerOrder.map((id) => commandHandlersById[id]);
 }
diff --git a/test/scripts/build-all.test.ts b/test/scripts/build-all.test.ts
diff --git a/test/scripts/ci-node-test-plan.test.ts b/test/scripts/ci-node-test-plan.test.ts
diff --git a/test/scripts/ci-workflow-guards.test.ts b/test/scripts/ci-workflow-guards.test.ts
