|
| 1 | +// Fork only. Splits the Test job's packages across the shards of its matrix in |
| 2 | +// .github/workflows/fork-tests.yml. Upstream runs the job on one large runner; |
| 3 | +// a GitHub-hosted runner has 4 CPUs, so the packages go to several runners. |
| 4 | +// |
| 5 | +// node .github/scripts/fork-test-shards.cjs 1/2 prints shard 1's packages |
| 6 | +// |
| 7 | +// TEST_PACKAGES (from the `changes` job) limits the set; empty means every |
| 8 | +// package the Test job covers. Packages are weighted by their test file count |
| 9 | +// and placed, heaviest first, on the least loaded shard, so the split follows |
| 10 | +// new packages and tests without a hand-kept list. |
| 11 | +const childProcess = require("node:child_process"); |
| 12 | +const fs = require("node:fs"); |
| 13 | + |
| 14 | +const { readWorkspace } = require("./ci-changes.cjs"); |
| 15 | + |
| 16 | +// Tested by their own jobs (Test Server, Test Web) or not a package to test. |
| 17 | +const OWN_JOB = new Set(["t3", "@t3tools/web", "@t3tools/monorepo"]); |
| 18 | +const TEST_FILE = /\.(?:test|spec)\.[cm]?[jt]sx?$/; |
| 19 | + |
| 20 | +function assign(packages, count) { |
| 21 | + const shards = Array.from({ length: count }, () => ({ weight: 0, packages: [] })); |
| 22 | + const ordered = packages.toSorted((a, b) => b.weight - a.weight || a.name.localeCompare(b.name)); |
| 23 | + for (const pkg of ordered) { |
| 24 | + const lightest = shards.reduce((best, shard) => (shard.weight < best.weight ? shard : best)); |
| 25 | + lightest.weight += pkg.weight; |
| 26 | + lightest.packages.push(pkg); |
| 27 | + } |
| 28 | + return shards; |
| 29 | +} |
| 30 | + |
| 31 | +function testFiles(dir) { |
| 32 | + return childProcess |
| 33 | + .execFileSync("git", ["ls-files", "-z", "--", dir], { encoding: "utf8", maxBuffer: 1 << 26 }) |
| 34 | + .split("\0") |
| 35 | + .filter((file) => TEST_FILE.test(file)).length; |
| 36 | +} |
| 37 | + |
| 38 | +function main() { |
| 39 | + const match = (process.argv[2] ?? "").match(/^(\d+)\/(\d+)$/); |
| 40 | + if (!match || Number(match[1]) < 1 || Number(match[1]) > Number(match[2])) |
| 41 | + throw new Error("usage: fork-test-shards.cjs <index>/<count>"); |
| 42 | + const [index, count] = [Number(match[1]), Number(match[2])]; |
| 43 | + const only = (process.env.TEST_PACKAGES ?? "").split(/\s+/).filter(Boolean); |
| 44 | + const packages = readWorkspace() |
| 45 | + .filter((pkg) => pkg.hasTest && !OWN_JOB.has(pkg.name)) |
| 46 | + .filter((pkg) => only.length === 0 || only.includes(pkg.name)) |
| 47 | + .map((pkg) => ({ name: pkg.name, weight: Math.max(1, testFiles(pkg.dir)) })); |
| 48 | + const unknown = only.filter((name) => !packages.some((pkg) => pkg.name === name)); |
| 49 | + if (unknown.length) |
| 50 | + throw new Error(`TEST_PACKAGES names no tested package: ${unknown.join(" ")}`); |
| 51 | + |
| 52 | + const shard = assign(packages, count)[index - 1]; |
| 53 | + for (const pkg of shard.packages) console.log(pkg.name); |
| 54 | + const summary = process.env.GITHUB_STEP_SUMMARY; |
| 55 | + if (summary) { |
| 56 | + const list = shard.packages.map((pkg) => `\`${pkg.name}\` (${pkg.weight})`).join(", "); |
| 57 | + fs.appendFileSync( |
| 58 | + summary, |
| 59 | + `### Test shard ${index}/${count}\n\n` + |
| 60 | + (shard.packages.length |
| 61 | + ? `${shard.packages.length} of ${packages.length} packages, ${shard.weight} test files: ${list}\n` |
| 62 | + : `No packages: the change affects ${packages.length} tested package(s), all on other shards.\n`), |
| 63 | + ); |
| 64 | + } |
| 65 | +} |
| 66 | + |
| 67 | +module.exports = { assign }; |
| 68 | + |
| 69 | +if (require.main === module) main(); |
0 commit comments