Skip to content

Commit ab37d88

Browse files
TakhoffmanTakhoffman
authored
test: introduce planner-backed test runner, stabilize local builds (#54650)
* test: stabilize ci and local vitest workers * test: introduce planner-backed test runner * test: address planner review follow-ups * test: derive planner budgets from host capabilities * test: restore planner filter helper import * test: align planner explain output with execution * test: keep low profile as serial alias * test: restrict explicit planner file targets * test: clean planner exits and pnpm launch * test: tighten wrapper flag validation * ci: gate heavy fanout on check * test: key shard assignments by unit identity * ci(bun): shard vitest lanes further * test: restore ci overlap and stabilize planner tests * test: relax planner output worker assertions * test: reset plugin runtime state in optional tools suite * ci: split macos node and swift jobs * test: honor no-isolate top-level concurrency budgets * ci: fix macos swift format lint * test: cap max-profile top-level concurrency * ci: shard macos node checks * ci: use four macos node shards * test: normalize explain targets before classification
1 parent 764394c commit ab37d88

19 files changed

Lines changed: 3236 additions & 1714 deletions

‎.github/workflows/ci-bun.yml‎

Lines changed: 10 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -45,11 +45,17 @@ jobs:
4545
matrix:
4646
include:
4747
- shard_index: 1
48-
shard_count: 2
49-
command: OPENCLAW_TEST_ISOLATE=1 bunx vitest run --config vitest.unit.config.ts --shard 1/2
48+
shard_count: 4
49+
command: OPENCLAW_TEST_ISOLATE=1 bunx vitest run --config vitest.unit.config.ts --shard 1/4
5050
- shard_index: 2
51-
shard_count: 2
52-
command: OPENCLAW_TEST_ISOLATE=1 bunx vitest run --config vitest.unit.config.ts --shard 2/2
51+
shard_count: 4
52+
command: OPENCLAW_TEST_ISOLATE=1 bunx vitest run --config vitest.unit.config.ts --shard 2/4
53+
- shard_index: 3
54+
shard_count: 4
55+
command: OPENCLAW_TEST_ISOLATE=1 bunx vitest run --config vitest.unit.config.ts --shard 3/4
56+
- shard_index: 4
57+
shard_count: 4
58+
command: OPENCLAW_TEST_ISOLATE=1 bunx vitest run --config vitest.unit.config.ts --shard 4/4
5359
steps:
5460
- name: Checkout
5561
uses: actions/checkout@v6

‎.github/workflows/ci.yml‎

Lines changed: 152 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -201,9 +201,9 @@ jobs:
201201
- name: Audit production dependencies
202202
run: pre-commit run --config "${PRE_COMMIT_CONFIG_PATH:-.pre-commit-config.yaml}" --all-files pnpm-audit-prod
203203

204-
# Fanout: downstream lanes branch from preflight outputs instead of waiting
205-
# on unrelated Linux checks.
206204
# Build dist once for Node-relevant changes and share it with downstream jobs.
205+
# Keep this overlapping with the fast correctness lanes so green PRs get heavy
206+
# test/build feedback sooner instead of waiting behind a full `check` pass.
207207
build-artifacts:
208208
needs: [scope]
209209
if: needs.scope.outputs.docs_only != 'true' && needs.scope.outputs.run_node == 'true'
@@ -280,7 +280,7 @@ jobs:
280280

281281
checks-fast:
282282
needs: [scope]
283-
if: always() && needs.scope.outputs.docs_only != 'true' && needs.scope.outputs.run_node == 'true'
283+
if: needs.scope.outputs.docs_only != 'true' && needs.scope.outputs.run_node == 'true'
284284
runs-on: blacksmith-16vcpu-ubuntu-2404
285285
timeout-minutes: 20
286286
strategy:
@@ -820,13 +820,9 @@ jobs:
820820
- name: Run ${{ matrix.task }} (${{ matrix.runtime }})
821821
run: ${{ matrix.command }}
822822

823-
# Consolidated macOS job: runs TS tests + Swift lint/build/test sequentially
824-
# on a single runner. GitHub limits macOS concurrent jobs to 5 per org;
825-
# running 4 separate jobs per PR (as before) starved the queue. One job
826-
# per PR allows 5 PRs to run macOS checks simultaneously.
827-
macos:
828-
needs: [scope]
829-
if: github.event_name == 'pull_request' && needs.scope.outputs.docs_only != 'true' && needs.scope.outputs.run_macos == 'true'
823+
macos-node-1:
824+
needs: [scope, build-artifacts]
825+
if: always() && github.event_name == 'pull_request' && needs.scope.outputs.docs_only != 'true' && needs.scope.outputs.run_macos == 'true' && needs.build-artifacts.result == 'success'
830826
runs-on: macos-latest
831827
timeout-minutes: 20
832828
steps:
@@ -841,16 +837,157 @@ jobs:
841837
with:
842838
install-bun: "false"
843839

844-
- name: Build dist (macOS)
845-
run: pnpm build
840+
- name: Download dist artifact
841+
uses: actions/download-artifact@v8
842+
with:
843+
name: dist-build
844+
path: dist/
845+
846+
- name: Download A2UI bundle artifact
847+
uses: actions/download-artifact@v8
848+
with:
849+
name: canvas-a2ui-bundle
850+
path: src/canvas-host/a2ui/
851+
852+
- name: Configure test shard (macOS 1/4)
853+
run: |
854+
echo "OPENCLAW_TEST_SHARDS=4" >> "$GITHUB_ENV"
855+
echo "OPENCLAW_TEST_SHARD_INDEX=1" >> "$GITHUB_ENV"
856+
857+
- name: TS tests (macOS 1/4)
858+
env:
859+
NODE_OPTIONS: --max-old-space-size=4096
860+
run: pnpm test
861+
862+
macos-node-2:
863+
needs: [scope, build-artifacts]
864+
if: always() && github.event_name == 'pull_request' && needs.scope.outputs.docs_only != 'true' && needs.scope.outputs.run_macos == 'true' && needs.build-artifacts.result == 'success'
865+
runs-on: macos-latest
866+
timeout-minutes: 20
867+
steps:
868+
- name: Checkout
869+
uses: actions/checkout@v6
870+
with:
871+
persist-credentials: false
872+
submodules: false
873+
874+
- name: Setup Node environment
875+
uses: ./.github/actions/setup-node-env
876+
with:
877+
install-bun: "false"
878+
879+
- name: Download dist artifact
880+
uses: actions/download-artifact@v8
881+
with:
882+
name: dist-build
883+
path: dist/
884+
885+
- name: Download A2UI bundle artifact
886+
uses: actions/download-artifact@v8
887+
with:
888+
name: canvas-a2ui-bundle
889+
path: src/canvas-host/a2ui/
890+
891+
- name: Configure test shard (macOS 2/4)
892+
run: |
893+
echo "OPENCLAW_TEST_SHARDS=4" >> "$GITHUB_ENV"
894+
echo "OPENCLAW_TEST_SHARD_INDEX=2" >> "$GITHUB_ENV"
895+
896+
- name: TS tests (macOS 2/4)
897+
env:
898+
NODE_OPTIONS: --max-old-space-size=4096
899+
run: pnpm test
900+
901+
macos-node-3:
902+
needs: [scope, build-artifacts]
903+
if: always() && github.event_name == 'pull_request' && needs.scope.outputs.docs_only != 'true' && needs.scope.outputs.run_macos == 'true' && needs.build-artifacts.result == 'success'
904+
runs-on: macos-latest
905+
timeout-minutes: 20
906+
steps:
907+
- name: Checkout
908+
uses: actions/checkout@v6
909+
with:
910+
persist-credentials: false
911+
submodules: false
912+
913+
- name: Setup Node environment
914+
uses: ./.github/actions/setup-node-env
915+
with:
916+
install-bun: "false"
917+
918+
- name: Download dist artifact
919+
uses: actions/download-artifact@v8
920+
with:
921+
name: dist-build
922+
path: dist/
923+
924+
- name: Download A2UI bundle artifact
925+
uses: actions/download-artifact@v8
926+
with:
927+
name: canvas-a2ui-bundle
928+
path: src/canvas-host/a2ui/
929+
930+
- name: Configure test shard (macOS 3/4)
931+
run: |
932+
echo "OPENCLAW_TEST_SHARDS=4" >> "$GITHUB_ENV"
933+
echo "OPENCLAW_TEST_SHARD_INDEX=3" >> "$GITHUB_ENV"
934+
935+
- name: TS tests (macOS 3/4)
936+
env:
937+
NODE_OPTIONS: --max-old-space-size=4096
938+
run: pnpm test
939+
940+
macos-node-4:
941+
needs: [scope, build-artifacts]
942+
if: always() && github.event_name == 'pull_request' && needs.scope.outputs.docs_only != 'true' && needs.scope.outputs.run_macos == 'true' && needs.build-artifacts.result == 'success'
943+
runs-on: macos-latest
944+
timeout-minutes: 20
945+
steps:
946+
- name: Checkout
947+
uses: actions/checkout@v6
948+
with:
949+
persist-credentials: false
950+
submodules: false
951+
952+
- name: Setup Node environment
953+
uses: ./.github/actions/setup-node-env
954+
with:
955+
install-bun: "false"
956+
957+
- name: Download dist artifact
958+
uses: actions/download-artifact@v8
959+
with:
960+
name: dist-build
961+
path: dist/
962+
963+
- name: Download A2UI bundle artifact
964+
uses: actions/download-artifact@v8
965+
with:
966+
name: canvas-a2ui-bundle
967+
path: src/canvas-host/a2ui/
968+
969+
- name: Configure test shard (macOS 4/4)
970+
run: |
971+
echo "OPENCLAW_TEST_SHARDS=4" >> "$GITHUB_ENV"
972+
echo "OPENCLAW_TEST_SHARD_INDEX=4" >> "$GITHUB_ENV"
846973
847-
# --- Run all checks sequentially (fast gates first) ---
848-
- name: TS tests (macOS)
974+
- name: TS tests (macOS 4/4)
849975
env:
850976
NODE_OPTIONS: --max-old-space-size=4096
851977
run: pnpm test
852978

853-
# --- Xcode/Swift setup ---
979+
macos-swift:
980+
needs: [scope]
981+
if: github.event_name == 'pull_request' && needs.scope.outputs.docs_only != 'true' && needs.scope.outputs.run_macos == 'true'
982+
runs-on: macos-latest
983+
timeout-minutes: 20
984+
steps:
985+
- name: Checkout
986+
uses: actions/checkout@v6
987+
with:
988+
persist-credentials: false
989+
submodules: false
990+
854991
- name: Select Xcode 26.1
855992
run: |
856993
sudo xcode-select -s /Applications/Xcode_26.1.app

‎AGENTS.md‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -120,7 +120,7 @@
120120
- For targeted/local debugging, keep using the wrapper: `pnpm test -- <path-or-filter> [vitest args...]` (for example `pnpm test -- src/commands/onboard-search.test.ts -t "shows registered plugin providers"`); do not default to raw `pnpm vitest run ...` because it bypasses wrapper config/profile/pool routing.
121121
- Do not set test workers above 16; tried already.
122122
- Keep Vitest on `forks` only. Do not introduce or reintroduce any non-`forks` Vitest pool or alternate execution mode in configs, wrapper scripts, or default test commands without explicit approval in this chat. This includes `threads`, `vmThreads`, `vmForks`, and any future/nonstandard pool variant.
123-
- If local Vitest runs cause memory pressure (common on non-Mac-Studio hosts), use `OPENCLAW_TEST_PROFILE=low OPENCLAW_TEST_SERIAL_GATEWAY=1 pnpm test` for land/gate runs.
123+
- If local Vitest runs cause memory pressure, the wrapper now derives budgets from host capabilities (CPU, memory band, current load). For a conservative explicit override during land/gate runs, use `OPENCLAW_TEST_PROFILE=serial OPENCLAW_TEST_SERIAL_GATEWAY=1 pnpm test`.
124124
- Live tests (real keys): `OPENCLAW_LIVE_TEST=1 pnpm test:live` (OpenClaw-only) or `LIVE=1 pnpm test:live` (includes provider live tests). Docker: `pnpm test:docker:live-models`, `pnpm test:docker:live-gateway`. Onboarding Docker E2E: `pnpm test:docker:onboard`.
125125
- Full kit + what’s covered: `docs/help/testing.md`.
126126
- Changelog: user-facing changes only; no internal/meta notes (version alignment, appcast reminders, release process).

‎apps/macos/Sources/OpenClaw/OpenClawConfigFile.swift‎

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -493,8 +493,12 @@ enum OpenClawConfigFile {
493493
return
494494
}
495495

496-
let backup = self.readConfigFingerprint(at: configURL.deletingLastPathComponent().appendingPathComponent("\(configURL.lastPathComponent).bak"))
497-
let clobberedPath = self.persistClobberedSnapshot(data: data, configURL: configURL, observedAt: observedAt)
496+
let backup = self.readConfigFingerprint(
497+
at: configURL.deletingLastPathComponent().appendingPathComponent("\(configURL.lastPathComponent).bak"))
498+
let clobberedPath = self.persistClobberedSnapshot(
499+
data: data,
500+
configURL: configURL,
501+
observedAt: observedAt)
498502
self.logger.warning("config observe anomaly (\(suspicious.joined(separator: ", "))) at \(configURL.path)")
499503
self.appendConfigObserveAudit([
500504
"phase": "read",

‎apps/macos/Sources/OpenClaw/SkillsSettings.swift‎

Lines changed: 12 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -259,9 +259,12 @@ private struct SkillRow: View {
259259
guard let raw = self.skill.homepage?.trimmingCharacters(in: .whitespacesAndNewlines) else {
260260
return nil
261261
}
262-
guard !raw.isEmpty, let url = URL(string: raw),
263-
let scheme = url.scheme?.lowercased(),
264-
scheme == "http" || scheme == "https" else {
262+
guard
263+
!raw.isEmpty,
264+
let url = URL(string: raw),
265+
let scheme = url.scheme?.lowercased(),
266+
scheme == "http" || scheme == "https"
267+
else {
265268
return nil
266269
}
267270
return url
@@ -481,9 +484,12 @@ private struct EnvEditorView: View {
481484
guard let raw = self.editor.homepage?.trimmingCharacters(in: .whitespacesAndNewlines) else {
482485
return nil
483486
}
484-
guard !raw.isEmpty, let url = URL(string: raw),
485-
let scheme = url.scheme?.lowercased(),
486-
scheme == "http" || scheme == "https" else {
487+
guard
488+
!raw.isEmpty,
489+
let url = URL(string: raw),
490+
let scheme = url.scheme?.lowercased(),
491+
scheme == "http" || scheme == "https"
492+
else {
487493
return nil
488494
}
489495
return url

‎package.json‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -701,10 +701,10 @@
701701
"test:auth:compat": "vitest run --config vitest.gateway.config.ts src/gateway/server.auth.compat-baseline.test.ts src/gateway/client.test.ts src/gateway/reconnect-gating.test.ts src/gateway/protocol/connect-error-details.test.ts",
702702
"test:build:singleton": "node scripts/test-built-plugin-singleton.mjs",
703703
"test:changed": "pnpm test -- --changed origin/main",
704-
"test:channels": "OPENCLAW_TEST_SKIP_DEFAULT=1 OPENCLAW_TEST_INCLUDE_CHANNELS=1 node scripts/test-parallel.mjs",
704+
"test:channels": "node scripts/test-parallel.mjs --surface channels",
705705
"test:contracts": "pnpm test:contracts:channels && pnpm test:contracts:plugins",
706-
"test:contracts:channels": "OPENCLAW_TEST_PROFILE=low pnpm test -- src/channels/plugins/contracts",
707-
"test:contracts:plugins": "OPENCLAW_TEST_PROFILE=low pnpm test -- src/plugins/contracts",
706+
"test:contracts:channels": "OPENCLAW_TEST_PROFILE=serial pnpm test -- src/channels/plugins/contracts",
707+
"test:contracts:plugins": "OPENCLAW_TEST_PROFILE=serial pnpm test -- src/plugins/contracts",
708708
"test:coverage": "vitest run --config vitest.unit.config.ts --coverage",
709709
"test:coverage:changed": "vitest run --config vitest.unit.config.ts --coverage --changed origin/main",
710710
"test:docker:all": "pnpm test:docker:live-models && pnpm test:docker:live-gateway && pnpm test:docker:openwebui && pnpm test:docker:onboard && pnpm test:docker:gateway-network && pnpm test:docker:qr && pnpm test:docker:doctor-switch && pnpm test:docker:plugins && pnpm test:docker:cleanup",
@@ -720,7 +720,7 @@
720720
"test:e2e": "vitest run --config vitest.e2e.config.ts",
721721
"test:e2e:openshell": "OPENCLAW_E2E_OPENSHELL=1 vitest run --config vitest.e2e.config.ts test/openshell-sandbox.e2e.test.ts",
722722
"test:extension": "node scripts/test-extension.mjs",
723-
"test:extensions": "OPENCLAW_TEST_SKIP_DEFAULT=1 OPENCLAW_TEST_INCLUDE_EXTENSIONS=1 node scripts/test-parallel.mjs",
723+
"test:extensions": "node scripts/test-parallel.mjs --surface extensions",
724724
"test:extensions:memory": "node scripts/profile-extension-memory.mjs",
725725
"test:fast": "vitest run --config vitest.unit.config.ts",
726726
"test:force": "node --import tsx scripts/test-force.ts",
@@ -731,7 +731,6 @@
731731
"test:install:e2e:openai": "OPENCLAW_E2E_MODELS=openai bash scripts/test-install-sh-e2e-docker.sh",
732732
"test:install:smoke": "bash scripts/test-install-sh-docker.sh",
733733
"test:live": "OPENCLAW_LIVE_TEST=1 vitest run --config vitest.live.config.ts",
734-
"test:macmini": "OPENCLAW_TEST_PROFILE=macmini node scripts/test-parallel.mjs",
735734
"test:parallels:linux": "bash scripts/e2e/parallels-linux-smoke.sh",
736735
"test:parallels:macos": "bash scripts/e2e/parallels-macos-smoke.sh",
737736
"test:parallels:npm-update": "bash scripts/e2e/parallels-npm-update-smoke.sh",
@@ -745,6 +744,7 @@
745744
"test:perf:update-memory-hotspots": "node scripts/test-update-memory-hotspots.mjs",
746745
"test:perf:update-timings": "node scripts/test-update-timings.mjs",
747746
"test:sectriage": "pnpm exec vitest run --config vitest.gateway.config.ts && vitest run --config vitest.unit.config.ts --exclude src/daemon/launchd.integration.test.ts --exclude src/process/exec.test.ts",
747+
"test:serial": "node scripts/test-parallel.mjs --profile serial",
748748
"test:startup:memory": "node scripts/check-cli-startup-memory.mjs",
749749
"test:ui": "pnpm lint:ui:no-raw-window-open && pnpm --dir ui test",
750750
"test:voicecall:closedloop": "vitest run extensions/voice-call/src/manager.test.ts extensions/voice-call/src/media-stream.test.ts src/plugins/voice-call.plugin.test.ts --maxWorkers=1",

0 commit comments

Comments
 (0)