Skip to content

Commit 9510d2f

Browse files
noreplyclaude
andcommitted
Fork CI: slice large test runs with vitest --shard; lint fixes
- Test and Test Web run on two runners each. A package with 100+ test files (and Test Web's 548) runs a vitest --shard slice on every runner; the rest are placed whole by test file count. - Script header comments fit upstream's 4-line comment rule; the jq program in fork-ci-result passes shellcheck. - The Ubuntu 24.04 pin is about choosing OS upgrades, not git: the 24.04 image now ships git 2.56 as well. Co-Authored-By: Claude Opus 5.5 <[email protected]>
1 parent 65243f1 commit 9510d2f

4 files changed

Lines changed: 123 additions & 77 deletions

File tree

Lines changed: 40 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -1,29 +1,30 @@
1-
// Fork only. Splits the Test job's packages across the shards of its matrix in
2-
// .github/workflows/fork-tests.yml. Upstream runs the job on one large runner;
3-
// a GitHub-hosted runner has 4 CPUs, so the packages go to several runners.
4-
//
5-
// node .github/scripts/fork-test-shards.cjs 1/2 prints shard 1's packages
6-
//
7-
// TEST_PACKAGES (from the `changes` job) limits the set; empty means every
8-
// package the Test job covers. Packages are weighted by their test file count
9-
// and placed, heaviest first, on the least loaded shard, so the split follows
10-
// new packages and tests without a hand-kept list.
1+
// Fork only: prints one Test shard's work (`fork-test-shards.cjs 1/2`): `split
2+
// <dir>` for a large package every shard runs a vitest slice of, `whole <name>`
3+
// for the rest, heaviest first (by test files) onto the lightest shard.
4+
// TEST_PACKAGES from `changes` limits the set; empty means every package.
115
const childProcess = require("node:child_process");
126
const fs = require("node:fs");
7+
const path = require("node:path");
138

149
const { readWorkspace } = require("./ci-changes.cjs");
1510

1611
// Tested by their own jobs (Test Server, Test Web) or not a package to test.
1712
const OWN_JOB = new Set(["t3", "@t3tools/web", "@t3tools/monorepo"]);
1813
const TEST_FILE = /\.(?:test|spec)\.[cm]?[jt]sx?$/;
14+
// One vitest run of this many files outlasts the rest of a shard.
15+
const SPLIT_AT = 100;
1916

2017
function assign(packages, count) {
21-
const shards = Array.from({ length: count }, () => ({ weight: 0, packages: [] }));
22-
const ordered = packages.toSorted((a, b) => b.weight - a.weight || a.name.localeCompare(b.name));
18+
const split = packages.filter((pkg) => pkg.splittable && pkg.weight >= SPLIT_AT);
19+
const share = split.reduce((total, pkg) => total + pkg.weight, 0) / count;
20+
const shards = Array.from({ length: count }, () => ({ weight: share, whole: [], split }));
21+
const ordered = packages
22+
.filter((pkg) => !split.includes(pkg))
23+
.toSorted((a, b) => b.weight - a.weight || a.name.localeCompare(b.name));
2324
for (const pkg of ordered) {
2425
const lightest = shards.reduce((best, shard) => (shard.weight < best.weight ? shard : best));
2526
lightest.weight += pkg.weight;
26-
lightest.packages.push(pkg);
27+
lightest.whole.push(pkg);
2728
}
2829
return shards;
2930
}
@@ -41,29 +42,44 @@ function main() {
4142
throw new Error("usage: fork-test-shards.cjs <index>/<count>");
4243
const [index, count] = [Number(match[1]), Number(match[2])];
4344
const only = (process.env.TEST_PACKAGES ?? "").split(/\s+/).filter(Boolean);
45+
const root = path.resolve(__dirname, "../..");
4446
const packages = readWorkspace()
4547
.filter((pkg) => pkg.hasTest && !OWN_JOB.has(pkg.name))
4648
.filter((pkg) => only.length === 0 || only.includes(pkg.name))
47-
.map((pkg) => ({ name: pkg.name, weight: Math.max(1, testFiles(pkg.dir)) }));
49+
.map((pkg) => {
50+
const manifest = JSON.parse(
51+
fs.readFileSync(path.join(root, pkg.dir, "package.json"), "utf8"),
52+
);
53+
return {
54+
name: pkg.name,
55+
dir: pkg.dir,
56+
weight: Math.max(1, testFiles(pkg.dir)),
57+
// Run in its directory with --shard, so the script must be vitest itself.
58+
splittable: /^vp test run(\s|$)/.test(manifest.scripts.test),
59+
};
60+
});
4861
const unknown = only.filter((name) => !packages.some((pkg) => pkg.name === name));
4962
if (unknown.length)
5063
throw new Error(`TEST_PACKAGES names no tested package: ${unknown.join(" ")}`);
5164

5265
const shard = assign(packages, count)[index - 1];
53-
for (const pkg of shard.packages) console.log(pkg.name);
66+
for (const pkg of shard.split) console.log(`split ${pkg.dir}`);
67+
for (const pkg of shard.whole) console.log(`whole ${pkg.name}`);
5468
const summary = process.env.GITHUB_STEP_SUMMARY;
5569
if (summary) {
56-
const list = shard.packages.map((pkg) => `\`${pkg.name}\` (${pkg.weight})`).join(", ");
57-
fs.appendFileSync(
58-
summary,
59-
`### Test shard ${index}/${count}\n\n` +
60-
(shard.packages.length
61-
? `${shard.packages.length} of ${packages.length} packages, ${shard.weight} test files: ${list}\n`
62-
: `No packages: the change affects ${packages.length} tested package(s), all on other shards.\n`),
63-
);
70+
const list = (pkgs) => pkgs.map((pkg) => `\`${pkg.name}\` (${pkg.weight})`).join(", ");
71+
const lines = [`### Test shard ${index}/${count}`, ""];
72+
if (shard.split.length) lines.push(`Slice ${index}/${count} of ${list(shard.split)}.`, "");
73+
if (shard.whole.length) lines.push(`Whole: ${list(shard.whole)}.`, "");
74+
if (!shard.split.length && !shard.whole.length)
75+
lines.push(
76+
`Nothing to run: the ${packages.length} affected package(s) are on other shards.`,
77+
"",
78+
);
79+
fs.appendFileSync(summary, `${lines.join("\n")}\n`);
6480
}
6581
}
6682

67-
module.exports = { assign };
83+
module.exports = { assign, SPLIT_AT };
6884

6985
if (require.main === module) main();
Lines changed: 22 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -1,34 +1,38 @@
11
const assert = require("node:assert/strict");
22
const test = require("node:test");
33

4-
const { assign } = require("./fork-test-shards.cjs");
4+
const { assign, SPLIT_AT } = require("./fork-test-shards.cjs");
55

6-
const names = (shards) => shards.map((shard) => shard.packages.map((pkg) => pkg.name));
6+
const pkg = (name, weight, splittable = true) => ({
7+
name,
8+
dir: `packages/${name}`,
9+
weight,
10+
splittable,
11+
});
12+
const whole = (shards) => shards.map((shard) => shard.whole.map((p) => p.name));
13+
const split = (shards) => shards.map((shard) => shard.split.map((p) => p.name));
714

815
test("places the heaviest package first on the lightest shard", () => {
9-
const packages = [
10-
{ name: "small", weight: 1 },
11-
{ name: "big", weight: 10 },
12-
{ name: "mid", weight: 6 },
13-
{ name: "mid2", weight: 5 },
14-
];
15-
assert.deepEqual(names(assign(packages, 2)), [
16+
const packages = [pkg("small", 1), pkg("big", 10), pkg("mid", 6), pkg("mid2", 5)];
17+
assert.deepEqual(whole(assign(packages, 2)), [
1618
["big", "small"],
1719
["mid", "mid2"],
1820
]);
1921
});
2022

2123
test("ties break by name, so every shard computes the same split", () => {
22-
const packages = [
23-
{ name: "b", weight: 2 },
24-
{ name: "a", weight: 2 },
25-
{ name: "c", weight: 2 },
26-
];
27-
assert.deepEqual(names(assign(packages, 2)), [["a", "c"], ["b"]]);
28-
assert.deepEqual(names(assign(packages.toReversed(), 2)), [["a", "c"], ["b"]]);
24+
const packages = [pkg("b", 2), pkg("a", 2), pkg("c", 2)];
25+
assert.deepEqual(whole(assign(packages, 2)), [["a", "c"], ["b"]]);
26+
assert.deepEqual(whole(assign(packages.toReversed(), 2)), [["a", "c"], ["b"]]);
27+
});
28+
29+
test("a large vitest package is sliced on every shard, the rest stay whole", () => {
30+
const packages = [pkg("large", SPLIT_AT + 50), pkg("script", SPLIT_AT, false), pkg("a", 3)];
31+
const shards = assign(packages, 2);
32+
assert.deepEqual(split(shards), [["large"], ["large"]]);
33+
assert.deepEqual(whole(shards), [["script"], ["a"]]);
2934
});
3035

3136
test("every package lands on exactly one shard, and spare shards stay empty", () => {
32-
const packages = [{ name: "only", weight: 3 }];
33-
assert.deepEqual(names(assign(packages, 3)), [["only"], [], []]);
37+
assert.deepEqual(whole(assign([pkg("only", 3)], 3)), [["only"], [], []]);
3438
});

‎.github/scripts/fork-tests-drift.cjs‎

Lines changed: 4 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -1,13 +1,7 @@
1-
// Fork only. .github/workflows/fork-tests.yml copies upstream's test jobs onto
2-
// GitHub-hosted runners. This check fails when upstream's jobs change and the
3-
// copy was not updated: it digests every job and step of the tracked upstream
4-
// workflows and compares them with .github/fork-tests-upstream.json.
5-
//
6-
// node .github/scripts/fork-tests-drift.cjs check (CI)
7-
// node .github/scripts/fork-tests-drift.cjs --record after updating fork-tests.yml
8-
//
9-
// Comment-only and blank-line changes are ignored. No dependencies: the
10-
// workflows are parsed by indentation, which is all a job/step list needs.
1+
// Fork only: fails when upstream's ci.yml or release.yml test jobs change and
2+
// .github/workflows/fork-tests.yml did not follow. Compares a digest of every
3+
// job and step with .github/fork-tests-upstream.json; comments are ignored.
4+
// After porting a change: node .github/scripts/fork-tests-drift.cjs --record
115
const crypto = require("node:crypto");
126
const fs = require("node:fs");
137
const path = require("node:path");

‎.github/workflows/fork-tests.yml‎

Lines changed: 57 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -2,10 +2,10 @@ name: Fork tests
22

33
# Fork only. Upstream's CI checks (ci.yml) on GitHub-hosted runners, because the
44
# fork has none of upstream's runners and upstream's workflows are disabled here.
5-
# Commands follow ci.yml step for step; only the runners, time limits, apt
6-
# setup, the task cache (kept with actions/cache instead of upstream's private
7-
# remote cache) and the Test job's split across two runners differ. Upstream's
8-
# composite actions and change detector (.github/actions,
5+
# Commands follow ci.yml step for step. Only the runners, time limits, apt
6+
# setup and the task cache differ (kept with actions/cache instead of upstream's
7+
# private remote cache), and Test and Test Web split over two runners each.
8+
# Upstream's composite actions and change detector (.github/actions,
99
# .github/scripts/ci-changes.cjs) are used as they are, so a rebase brings their
1010
# changes. The `workflows` job lints the fork workflows (actionlint) and fails
1111
# when upstream's ci.yml or release.yml test jobs change and this copy did not
@@ -14,8 +14,8 @@ name: Fork tests
1414
# Require only `fork-ci-result` in a ruleset: it needs every other job, so jobs
1515
# can be added, split or resharded without touching the ruleset.
1616
#
17-
# Runners: GitHub-hosted only, Ubuntu 24.04 as upstream (ubuntu-latest is now
18-
# 26.04, whose newer git fails an apps/server test upstream never sees). Never
17+
# Runners: GitHub-hosted only, Ubuntu 24.04 like upstream's runners, so an OS
18+
# upgrade is a reviewed change here, not a surprise from ubuntu-latest. Never
1919
# add a self-hosted runner to this workflow: this repository is public, so a
2020
# pull request from any fork would run its code on that machine.
2121
#
@@ -403,9 +403,9 @@ jobs:
403403
exit "$(cat "$RUNNER_TEMP/typecheck-status")"
404404
405405
# Everything except `t3` (apps/server) and the web app, which have their own jobs.
406-
# Upstream runs this on one large runner. Here the packages are split across
407-
# two 4-CPU runners by test file count (.github/scripts/fork-test-shards.cjs);
408-
# one runner took 6 minutes, the longest job of the run.
406+
# Upstream runs this on one large runner; one 4-CPU runner took 6 minutes.
407+
# Two runners split it (.github/scripts/fork-test-shards.cjs): each runs a
408+
# vitest slice of every large package and half of the others, whole.
409409
test:
410410
name: Test ${{ matrix.shard }}
411411
needs: changes
@@ -427,20 +427,21 @@ jobs:
427427
!/apps/mobile/modules/t3-terminal/Vendor/**/*.a
428428
sparse-checkout-cone-mode: false
429429

430-
# Shard 1 always gets the heaviest affected package; a shard left with
431-
# none says so in the summary and stops here.
430+
# A shard with nothing to run says so in the summary and stops here.
432431
- name: Pick this shard's packages
433432
id: shard
434433
env:
435434
TEST_PACKAGES: ${{ needs.changes.outputs.test_packages }}
436435
run: |
437-
packages=$(node .github/scripts/fork-test-shards.cjs "${{ matrix.shard }}/${{ strategy.job-total }}" | tr '\n' ' ')
438-
echo "packages=${packages% }" >> "$GITHUB_OUTPUT"
436+
work=$(node .github/scripts/fork-test-shards.cjs "${{ matrix.shard }}/${{ strategy.job-total }}")
437+
whole=$(awk '$1 == "whole" { printf "%s ", $2 }' <<< "$work")
438+
split=$(awk '$1 == "split" { printf "%s ", $2 }' <<< "$work")
439+
{ echo "whole=${whole% }"; echo "split=${split% }"; echo "any=$([[ -n "$work" ]] && echo true || echo false)"; } >> "$GITHUB_OUTPUT"
439440
440441
# The desktop tests compile the browser secret helper. The install runs in
441442
# the background while Vite+ installs; the step before the tests waits for it.
442443
- name: Start installing browser secret helper build libraries
443-
if: steps.shard.outputs.packages != ''
444+
if: steps.shard.outputs.any == 'true'
444445
run: |
445446
(
446447
set +e
@@ -452,21 +453,21 @@ jobs:
452453
) > "$RUNNER_TEMP/apt.log" 2>&1 < /dev/null &
453454
454455
- name: Setup Vite+
455-
if: steps.shard.outputs.packages != ''
456+
if: steps.shard.outputs.any == 'true'
456457
uses: ./.github/actions/setup-ci-deps
457458
with:
458459
# Lint saves the pnpm store; the others only restore it.
459460
save-pnpm-store: false
460461

461462
- name: Cache Electron download
462-
if: steps.shard.outputs.packages != ''
463+
if: steps.shard.outputs.any == 'true'
463464
uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6
464465
with:
465466
path: ~/.cache/electron
466467
key: electron-${{ runner.os }}-${{ runner.arch }}-${{ hashFiles('node_modules/.pnpm/electron@*/node_modules/electron/package.json') }}
467468

468469
- name: Ensure Electron runtime is installed
469-
if: steps.shard.outputs.packages != ''
470+
if: steps.shard.outputs.any == 'true'
470471
run: vp run --filter @t3tools/desktop ensure:electron
471472

472473
- name: Test preview artifact validation
@@ -478,34 +479,58 @@ jobs:
478479
run: node --test .github/scripts/check-nightly-release.test.cjs .github/scripts/vercel-deployment-output.test.cjs .github/scripts/mobile-preview-comment.test.cjs
479480

480481
- name: Finish installing browser secret helper build libraries
481-
if: steps.shard.outputs.packages != ''
482+
if: steps.shard.outputs.any == 'true'
482483
timeout-minutes: 5
483484
run: |
484485
until test -s "$RUNNER_TEMP/apt-status"; do sleep 1; done
485486
cat "$RUNNER_TEMP/apt.log"
486487
exit "$(cat "$RUNNER_TEMP/apt-status")"
487488
489+
# Whole packages as upstream runs them; a large package's slice runs its
490+
# own test script in its directory with vitest's --shard (`vp run` does
491+
# not pass arguments through to vitest).
488492
- name: Test
489-
if: steps.shard.outputs.packages != ''
493+
if: steps.shard.outputs.any == 'true'
490494
env:
491-
PACKAGES: ${{ steps.shard.outputs.packages }}
495+
WHOLE: ${{ steps.shard.outputs.whole }}
496+
SPLIT: ${{ steps.shard.outputs.split }}
497+
SHARD: ${{ matrix.shard }}/${{ strategy.job-total }}
492498
run: |
493499
set -o pipefail
494-
filters=()
495-
for package in $PACKAGES; do filters+=(--filter "$package"); done
496500
# The runner's trace writer can crash on surrogate pairs in test output.
497-
vp run --parallel --concurrency-limit 4 "${filters[@]}" test 2>&1 | python3 -u -c '
501+
ascii() {
502+
python3 -u -c '
498503
import sys
499504
for line in sys.stdin:
500505
sys.stdout.write(line.encode("ascii", "backslashreplace").decode("ascii"))
501506
'
507+
}
508+
status=0
509+
if [[ -n "$WHOLE" ]]; then
510+
filters=()
511+
for package in $WHOLE; do filters+=(--filter "$package"); done
512+
vp run --parallel --concurrency-limit 4 "${filters[@]}" test 2>&1 | ascii || status=1
513+
fi
514+
for dir in $SPLIT; do
515+
echo "::group::$dir slice $SHARD"
516+
script=$(jq -r .scripts.test "$dir/package.json")
517+
(cd "$dir" && eval "$script --shard=$SHARD") 2>&1 | ascii || status=1
518+
echo "::endgroup::"
519+
done
520+
exit "$status"
502521
522+
# Upstream runs this on one large runner; on a 4-CPU runner the 548 files
523+
# took 7.5 minutes, so two runners each take a vitest slice.
503524
test_web:
504-
name: Test Web
525+
name: Test Web ${{ matrix.shard }}
505526
needs: changes
506527
if: ${{ !cancelled() && needs.changes.outputs.test_web != 'false' }}
507528
runs-on: ubuntu-24.04
508529
timeout-minutes: 15
530+
strategy:
531+
fail-fast: false
532+
matrix:
533+
shard: [1, 2]
509534
steps:
510535
- name: Checkout
511536
uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6
@@ -546,8 +571,13 @@ jobs:
546571
node apps/web/node_modules/playwright/cli.js install-deps chromium
547572
fi
548573
574+
# Upstream: `vp run --filter @t3tools/web test`. Same script, run in its
575+
# directory, because `vp run` does not pass --shard through to vitest.
549576
- name: Test
550-
run: vp run --filter @t3tools/web test
577+
env:
578+
SHARD: ${{ matrix.shard }}/${{ strategy.job-total }}
579+
working-directory: apps/web
580+
run: eval "$(jq -r .scripts.test package.json) --shard=$SHARD"
551581

552582
# apps/server runs its test files one at a time; shards spread them over
553583
# separate runners. The shards gate themselves instead of waiting for
@@ -802,6 +832,8 @@ jobs:
802832
RELEASE_RUN: ${{ inputs.ref != '' }}
803833
run: |
804834
echo "$RESULTS" | jq -r 'to_entries[] | "\(.key): \(.value.result)"'
835+
# A jq program: $release is jq's variable, not the shell's.
836+
# shellcheck disable=SC2016
805837
why='{
806838
build: "the change affects neither desktop, server nor web, and no package needs a typecheck",
807839
test: "the change affects no package this job tests",

0 commit comments

Comments
 (0)