perf(ci): repair and re-key the extension-boundary sticky lane (#109804)

steipete · web-flow · commit dfd457adc069 · 2026-07-17T10:30:09.000+01:00
check-additional-extension-package-boundary is the light-run critical-path pole. Measured on runs 29564442266/29564411446/29564259862 (8 vCPU, sticky mounts 429-failing): Setup Node ~40s cold install, shard step 142-161s = ~101s cold plugin-sdk dts prep (nine parallel tsgo builds) + ~58s compiling 122 plugins at concurrency 6 + ~1s canary; warm actions/cache path on a main push (29551077288) runs the shard in 67s. Levers chosen from that data: - Re-key the ext-boundary sticky disk from per-PR/per-config keys (v1) to one O(1) protected key (v2), following the node_modules snapshot fix in #109752. Blacksmith's installation-wide 1000-disk cap is saturated and every mount 429s; per-PR keys are a direct contributor. The config/scripts/ lockfile hash moves from the key into the in-job .source-trees marker next to the existing tree-OID + node/pnpm gate. Single semantic writer: the boundary lane commits explicitly (true, not if-missing) on protected pushes only; PR mounts and the check-lint consumer stay read-only, and the seed step is now writer-only (saves ~11s per PR cold run). - Bump the lane 8 -> 32 vCPU and raise OPENCLAW_EXTENSION_BOUNDARY_CONCURRENCY 6 -> 16: both the parallel dts prep and the 122 plugin compiles scale with cores at similar billed core-minutes (same precedent as check-lint/deps). Expected pole time: warm PR runs (protected snapshot tracks every main push) drop to roughly checkout + install-skip + restore + incremental shard, well under 2 minutes; cold runs that touch boundary trees shrink via the core bump. Guarded in ci-workflow-guards so the O(1) key, single-writer commit, and fingerprint composition cannot silently regress.
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -1977,21 +1977,28 @@ jobs:
 
       # check-lint's shard runner rebuilds the same plugin-sdk boundary
       # artifacts the boundary lane snapshots (~72s cold); restore them from
-      # the shared sticky. Strictly read-only (commit: false): if this lane
-      # could commit a freshly formatted disk, a cold-key race with the
-      # boundary lane would permanently seed an empty, marker-less snapshot
-      # that if-missing then refuses to repair.
+      # the shared sticky. Strictly read-only (commit: false): only the
+      # protected boundary lane may publish this repository-global snapshot.
       - name: Mount extension boundary sticky disk
         if: matrix.task == 'lint' && github.event_name != 'workflow_dispatch' && github.repository == 'openclaw/openclaw' && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == 'openclaw/openclaw')
         uses: useblacksmith/stickydisk@5b350170ae4ef55b536b548ef5f5896e76a6b54f # v1.4.0
         with:
-          key: ${{ github.repository }}-ext-boundary-v1-${{ github.event_name == 'pull_request' && format('pr-{0}', github.event.pull_request.number) || 'protected' }}-${{ hashFiles('tsconfig.json', 'tsconfig.plugin-sdk.dts.json', 'packages/plugin-sdk/tsconfig.json', 'scripts/check-extension-package-tsc-boundary.mjs', 'scripts/prepare-extension-package-boundary-artifacts.mjs', 'scripts/write-plugin-sdk-entry-dts.ts', 'scripts/lib/plugin-sdk-entrypoints.json', 'scripts/lib/plugin-sdk-entries.mjs', 'package.json', 'pnpm-lock.yaml') }}
+          # One stable disk for the whole repository. The v1 per-PR/per-config
+          # keys minted a new backing disk for every PR and toolchain change
+          # and helped saturate Blacksmith's installation-wide sticky-disk
+          # budget, 429-failing every mount. Snapshot validity lives in the
+          # in-job marker checked below instead of the key.
+          key: ${{ github.repository }}-ext-boundary-v2
           path: /var/tmp/openclaw-ext-boundary
           commit: "false"
 
       - name: Restore extension boundary artifacts from sticky disk
         if: matrix.task == 'lint' && github.event_name != 'workflow_dispatch' && github.repository == 'openclaw/openclaw' && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == 'openclaw/openclaw')
         shell: bash
+        env:
+          # Config/toolchain inputs the tree-OID gate below cannot see; must
+          # stay identical to the boundary lane's fingerprint composition.
+          BOUNDARY_CONFIG_HASH: ${{ hashFiles('tsconfig.json', 'tsconfig.plugin-sdk.dts.json', 'packages/plugin-sdk/tsconfig.json', 'scripts/check-extension-package-tsc-boundary.mjs', 'scripts/prepare-extension-package-boundary-artifacts.mjs', 'scripts/write-plugin-sdk-entry-dts.ts', 'scripts/lib/plugin-sdk-entrypoints.json', 'scripts/lib/plugin-sdk-entries.mjs', 'package.json', 'pnpm-lock.yaml') }}
         run: |
           set -euo pipefail
           sticky_root=/var/tmp/openclaw-ext-boundary
@@ -2002,7 +2009,7 @@ jobs:
           # Same tree-OID gate as the boundary lane: restore only artifacts
           # produced from identical generative sources, so stale snapshots can
           # neither false-skip rebuilds nor leave ghost declarations.
-          current_trees="$({ git rev-parse HEAD:src/plugin-sdk HEAD:packages HEAD:extensions HEAD:src/auto-reply HEAD:src/types HEAD:src/plugins/types.ts HEAD:src/video-generation HEAD:src/channels; node --version; corepack pnpm --version 2>/dev/null || pnpm --version; })"
+          current_trees="$({ git rev-parse HEAD:src/plugin-sdk HEAD:packages HEAD:extensions HEAD:src/auto-reply HEAD:src/types HEAD:src/plugins/types.ts HEAD:src/video-generation HEAD:src/channels; node --version; corepack pnpm --version 2>/dev/null || pnpm --version; echo "$BOUNDARY_CONFIG_HASH"; })"
           if [ ! -f "$sticky_root/.source-trees" ] || [ "$current_trees" != "$(cat "$sticky_root/.source-trees")" ]; then
             echo "boundary source trees changed since snapshot; lint prepares cold"
             exit 0
@@ -2182,7 +2189,11 @@ jobs:
             runner: blacksmith-8vcpu-ubuntu-2404
           - check_name: check-additional-extension-package-boundary
             group: extension-package-boundary
-            runner: blacksmith-8vcpu-ubuntu-2404
+            # Light-run critical-path pole: cold runs spend ~100s in nine
+            # parallel plugin-sdk dts builds plus ~58s compiling 122 plugins
+            # at concurrency 6 on 8 vCPUs. Both phases scale with cores at
+            # similar billed core-minutes.
+            runner: blacksmith-32vcpu-ubuntu-2404
           - check_name: check-additional-runtime-topology-architecture
             group: runtime-topology-architecture
             runner: blacksmith-4vcpu-ubuntu-2404
@@ -2207,19 +2218,30 @@ jobs:
         if: matrix.group == 'extension-package-boundary' && github.event_name != 'workflow_dispatch' && github.repository == 'openclaw/openclaw' && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == 'openclaw/openclaw')
         uses: useblacksmith/stickydisk@5b350170ae4ef55b536b548ef5f5896e76a6b54f # v1.4.0
         with:
-          # Coarse toolchain key only: source changes are meant to hit a stale
-          # snapshot, whose restored artifacts rebuild incrementally. PR scope
-          # keeps feature snapshots separate from protected pushes.
-          key: ${{ github.repository }}-ext-boundary-v1-${{ github.event_name == 'pull_request' && format('pr-{0}', github.event.pull_request.number) || 'protected' }}-${{ hashFiles('tsconfig.json', 'tsconfig.plugin-sdk.dts.json', 'packages/plugin-sdk/tsconfig.json', 'scripts/check-extension-package-tsc-boundary.mjs', 'scripts/prepare-extension-package-boundary-artifacts.mjs', 'scripts/write-plugin-sdk-entry-dts.ts', 'scripts/lib/plugin-sdk-entrypoints.json', 'scripts/lib/plugin-sdk-entries.mjs', 'package.json', 'pnpm-lock.yaml') }}
+          # One stable disk for the whole repository. The v1 per-PR/per-config
+          # keys minted a new backing disk for every PR and toolchain change
+          # and helped saturate Blacksmith's installation-wide sticky-disk
+          # budget, 429-failing every mount. Snapshot validity lives in the
+          # in-job marker checked below instead of the key, so source and
+          # toolchain changes refresh this disk in place.
+          key: ${{ github.repository }}-ext-boundary-v2
           path: /var/tmp/openclaw-ext-boundary
-          # v1.4.0 skips commit after failed/cancelled steps, so an incomplete
-          # first hydration cannot seed this key.
-          commit: if-missing
+          # Single semantic writer: only protected pushes commit, so
+          # pull_request clones stay read-only and the snapshot tracks main.
+          # Explicit true (not on-change/if-missing) because the allocated-byte
+          # heuristic can miss a same-size refresh, permanently stranding
+          # consumers on a stale marker. v1.4.0 skips commit after
+          # failed/cancelled steps, so a broken build cannot poison this key.
+          commit: ${{ github.event_name != 'pull_request' && 'true' || 'false' }}
 
       - name: Restore extension boundary artifacts from sticky disk
         id: boundary-sticky-restore
         if: matrix.group == 'extension-package-boundary' && github.event_name != 'workflow_dispatch' && github.repository == 'openclaw/openclaw' && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == 'openclaw/openclaw')
         shell: bash
+        env:
+          # Config/toolchain inputs the tree-OID gate below cannot see; must
+          # stay identical to the seed step and check-lint's restore gate.
+          BOUNDARY_CONFIG_HASH: ${{ hashFiles('tsconfig.json', 'tsconfig.plugin-sdk.dts.json', 'packages/plugin-sdk/tsconfig.json', 'scripts/check-extension-package-tsc-boundary.mjs', 'scripts/prepare-extension-package-boundary-artifacts.mjs', 'scripts/write-plugin-sdk-entry-dts.ts', 'scripts/lib/plugin-sdk-entrypoints.json', 'scripts/lib/plugin-sdk-entries.mjs', 'package.json', 'pnpm-lock.yaml') }}
         run: |
           set -euo pipefail
           sticky_root=/var/tmp/openclaw-ext-boundary
@@ -2229,12 +2251,12 @@ jobs:
           fi
           # Restore only when the generative source trees AND the runner
           # toolchain match the snapshotting run. Tree OIDs cover source
-          # content exactly; the node/pnpm versions cover toolchain-only
-          # changes (the sticky key already hashes config/scripts/lockfile, so
-          # those force a fresh key). Restored artifacts are valid by
-          # construction: no clock-skew mtime race can false-skip a rebuild,
-          # and deleted/renamed sources or a toolchain bump build cold.
-          current_trees="$({ git rev-parse HEAD:src/plugin-sdk HEAD:packages HEAD:extensions HEAD:src/auto-reply HEAD:src/types HEAD:src/plugins/types.ts HEAD:src/video-generation HEAD:src/channels; node --version; corepack pnpm --version 2>/dev/null || pnpm --version; })"
+          # content exactly; the node/pnpm versions and the config hash cover
+          # toolchain/config/scripts/lockfile changes the OIDs cannot see.
+          # Restored artifacts are valid by construction: no clock-skew mtime
+          # race can false-skip a rebuild, and deleted/renamed sources or a
+          # toolchain bump build cold.
+          current_trees="$({ git rev-parse HEAD:src/plugin-sdk HEAD:packages HEAD:extensions HEAD:src/auto-reply HEAD:src/types HEAD:src/plugins/types.ts HEAD:src/video-generation HEAD:src/channels; node --version; corepack pnpm --version 2>/dev/null || pnpm --version; echo "$BOUNDARY_CONFIG_HASH"; })"
           if [ ! -f "$sticky_root/.source-trees" ] || [ "$current_trees" != "$(cat "$sticky_root/.source-trees")" ]; then
             echo "boundary source trees changed since snapshot; building cold"
             echo "restored=false" >> "$GITHUB_OUTPUT"
@@ -2310,7 +2332,9 @@ jobs:
           RUN_PROMPT_SNAPSHOTS: ${{ needs.preflight.outputs.run_prompt_snapshots }}
           OPENCLAW_ADDITIONAL_BOUNDARY_SHARD: ${{ matrix.boundary_shard || '' }}
           OPENCLAW_ADDITIONAL_BOUNDARY_CONCURRENCY: 4
-          OPENCLAW_EXTENSION_BOUNDARY_CONCURRENCY: 6
+          # Matches the boundary lane's 32 vCPU runner (cores/2); the
+          # script's default caps at 6 to protect laptops.
+          OPENCLAW_EXTENSION_BOUNDARY_CONCURRENCY: 16
         shell: bash
         run: |
           set -euo pipefail
@@ -2401,16 +2425,22 @@ jobs:
 
           exit "$failures"
 
-      # commit: if-missing snapshots only the first run per key; seed the
-      # payload before the sticky post-action flushes. rsync -aR mirrors the
-      # repo-relative layout the restore step replays.
+      # Only the protected writer refreshes the snapshot (pull_request mounts
+      # never commit, so seeding there would burn wall clock on a discarded
+      # clone). Seed the payload before the sticky post-action flushes.
+      # rsync -aR mirrors the repo-relative layout the restore step replays.
       - name: Seed extension boundary sticky disk
-        if: success() && steps.boundary-sticky-restore.outputs.restored == 'false' && matrix.group == 'extension-package-boundary' && github.event_name != 'workflow_dispatch' && github.repository == 'openclaw/openclaw' && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == 'openclaw/openclaw')
+        if: success() && steps.boundary-sticky-restore.outputs.restored == 'false' && matrix.group == 'extension-package-boundary' && github.event_name != 'workflow_dispatch' && github.event_name != 'pull_request' && github.repository == 'openclaw/openclaw'
         shell: bash
+        env:
+          # Must stay identical to the restore gates above.
+          BOUNDARY_CONFIG_HASH: ${{ hashFiles('tsconfig.json', 'tsconfig.plugin-sdk.dts.json', 'packages/plugin-sdk/tsconfig.json', 'scripts/check-extension-package-tsc-boundary.mjs', 'scripts/prepare-extension-package-boundary-artifacts.mjs', 'scripts/write-plugin-sdk-entry-dts.ts', 'scripts/lib/plugin-sdk-entrypoints.json', 'scripts/lib/plugin-sdk-entries.mjs', 'package.json', 'pnpm-lock.yaml') }}
         run: |
           set -euo pipefail
           sticky_root=/var/tmp/openclaw-ext-boundary
-          rm -rf "$sticky_root/dist" "$sticky_root/packages" "$sticky_root/extensions" "$sticky_root/.snapshot-ready"
+          # A stale marker must not survive a mid-seed failure: drop readiness
+          # first so a partial payload can never be restored as valid.
+          rm -rf "$sticky_root/.snapshot-ready" "$sticky_root/.source-trees" "$sticky_root/dist" "$sticky_root/packages" "$sticky_root/extensions"
           if [ -d dist/plugin-sdk ]; then
             rsync -aR dist/plugin-sdk "$sticky_root/"
           fi
@@ -2419,7 +2449,7 @@ jobs:
           fi
           find extensions -maxdepth 3 \( -name '.boundary-tsc.tsbuildinfo' -o -name '.boundary-tsc.stamp' \) \
             -exec rsync -aR {} "$sticky_root/" \;
-          { git rev-parse HEAD:src/plugin-sdk HEAD:packages HEAD:extensions HEAD:src/auto-reply HEAD:src/types HEAD:src/plugins/types.ts HEAD:src/video-generation HEAD:src/channels; node --version; corepack pnpm --version 2>/dev/null || pnpm --version; } > "$sticky_root/.source-trees"
+          { git rev-parse HEAD:src/plugin-sdk HEAD:packages HEAD:extensions HEAD:src/auto-reply HEAD:src/types HEAD:src/plugins/types.ts HEAD:src/video-generation HEAD:src/channels; node --version; corepack pnpm --version 2>/dev/null || pnpm --version; echo "$BOUNDARY_CONFIG_HASH"; } > "$sticky_root/.source-trees"
           touch "$sticky_root/.snapshot-ready"
 
   # Validate docs (format, lint, broken links) only when docs files changed.
diff --git a/test/scripts/ci-workflow-guards.test.ts b/test/scripts/ci-workflow-guards.test.ts
@@ -2383,6 +2383,67 @@ describe("ci workflow guards", () => {
     expect(source).toContain("blacksmith-8vcpu-windows-2025");
   });
 
+  it("keeps the extension boundary sticky disk on one protected key", () => {
+    const workflow = readCiWorkflow();
+    const additionalJob = workflow.jobs["check-additional-shard"];
+    const checkShardJob = workflow.jobs["check-shard"];
+
+    // Light-run pole: cold prep + 122 plugin compiles scale with cores at
+    // similar billed core-minutes.
+    expect(additionalJob.strategy.matrix.include).toContainEqual({
+      check_name: "check-additional-extension-package-boundary",
+      group: "extension-package-boundary",
+      runner: "blacksmith-32vcpu-ubuntu-2404",
+    });
+    const runStep = additionalJob.steps.find(
+      (step: WorkflowStep) => step.name === "Run additional check shard",
+    );
+    expect(runStep.env.OPENCLAW_EXTENSION_BOUNDARY_CONCURRENCY).toBe(16);
+
+    // O(1) disks: Blacksmith caps sticky disks per installation, and the old
+    // per-PR/per-config keys minted new disks until every mount 429-failed
+    // fleet-wide. Snapshot validity lives in the in-job marker, not the key.
+    const boundaryMount = additionalJob.steps.find(
+      (step: WorkflowStep) => step.name === "Mount extension boundary sticky disk",
+    );
+    const lintMount = checkShardJob.steps.find(
+      (step: WorkflowStep) => step.name === "Mount extension boundary sticky disk",
+    );
+    expect(boundaryMount.with.key).toBe("${{ github.repository }}-ext-boundary-v2");
+    expect(lintMount.with.key).toBe(boundaryMount.with.key);
+    // Single semantic writer: protected pushes commit explicitly (not
+    // on-change/if-missing, whose allocated-byte heuristic can strand a stale
+    // marker); PR clones and the lint consumer stay read-only.
+    expect(boundaryMount.with.commit).toBe(
+      "${{ github.event_name != 'pull_request' && 'true' || 'false' }}",
+    );
+    expect(lintMount.with.commit).toBe("false");
+
+    // The key no longer hashes config/scripts/lockfile, so every gate must
+    // compose the identical marker fingerprint or restores silently tear.
+    const restoreStep = additionalJob.steps.find(
+      (step: WorkflowStep) => step.name === "Restore extension boundary artifacts from sticky disk",
+    );
+    const lintRestoreStep = checkShardJob.steps.find(
+      (step: WorkflowStep) => step.name === "Restore extension boundary artifacts from sticky disk",
+    );
+    const seedStep = additionalJob.steps.find(
+      (step: WorkflowStep) => step.name === "Seed extension boundary sticky disk",
+    );
+    const configHash = seedStep.env.BOUNDARY_CONFIG_HASH;
+    expect(configHash).toContain("hashFiles(");
+    expect(configHash).toContain("pnpm-lock.yaml");
+    expect(restoreStep.env.BOUNDARY_CONFIG_HASH).toBe(configHash);
+    expect(lintRestoreStep.env.BOUNDARY_CONFIG_HASH).toBe(configHash);
+    for (const gate of [restoreStep, lintRestoreStep, seedStep]) {
+      expect(gate.run).toContain('echo "$BOUNDARY_CONFIG_HASH"');
+    }
+    // Seeding is writer-only work: PR mounts never commit, so seeding there
+    // would burn wall clock on a discarded clone.
+    expect(seedStep.if).toContain("github.event_name != 'pull_request'");
+    expect(seedStep.if).toContain("steps.boundary-sticky-restore.outputs.restored == 'false'");
+  });
+
   it("runs the session accessor ratchet as a visible additional check", () => {
     const workflow = readCiWorkflow();
     const additionalJob = workflow.jobs["check-additional-shard"];
