Skip to content

Commit df61199

Browse files
authored
chore: add sharding for playwright tests (#7187)
* add sharding * fix dependnancy * fix playwright shard port reuse * full bust * throw more runners into this run * try to throw money at it and see the effect * tune
1 parent 8caa202 commit df61199

10 files changed

Lines changed: 718 additions & 194 deletions

File tree

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
11
distribute-on:
2-
small-changeset: 3 linux-medium-js
3-
medium-changeset: 6 linux-medium-js
4-
large-changeset: 10 linux-medium-js
2+
small-changeset: 4 linux-medium-js
3+
medium-changeset: 8 linux-medium-js
4+
large-changeset: 14 linux-medium-js

‎e2e/react-start/rsc/package.json‎

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,12 @@
99
"build": "vite build && tsc --noEmit",
1010
"preview": "vite preview",
1111
"start": "pnpx srvx --prod -s ../client dist/server/server.js",
12-
"test:e2e": "rm -rf port*.txt; playwright test --project=chromium"
12+
"test:e2e-full": "pnpm build && sh -c 'rm -f \"port-${E2E_PORT_KEY:-$npm_package_name}.txt\" \"port-${E2E_PORT_KEY:-$npm_package_name}-external.txt\"' && playwright test --project=chromium"
13+
},
14+
"nx": {
15+
"metadata": {
16+
"playwrightShards": 6
17+
}
1318
},
1419
"dependencies": {
1520
"@tanstack/react-router": "workspace:^",

‎e2e/react-start/rsc/playwright.config.ts‎

Lines changed: 14 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,20 @@
1+
import fs from 'node:fs'
12
import { defineConfig, devices } from '@playwright/test'
23
import { getTestServerPort } from '@tanstack/router-e2e-utils'
34
import packageJson from './package.json' with { type: 'json' }
45

5-
const PORT = await getTestServerPort(packageJson.name)
6+
const e2ePortKey = process.env.E2E_PORT_KEY ?? packageJson.name
7+
8+
if (process.env.TEST_WORKER_INDEX === undefined) {
9+
for (const portFile of [
10+
`port-${e2ePortKey}.txt`,
11+
`port-${e2ePortKey}-external.txt`,
12+
]) {
13+
fs.rmSync(portFile, { force: true })
14+
}
15+
}
16+
17+
const PORT = await getTestServerPort(e2ePortKey)
618
const baseURL = `http://localhost:${PORT}`
719

820
export default defineConfig({
@@ -18,7 +30,7 @@ export default defineConfig({
1830
},
1931

2032
webServer: {
21-
command: `PORT=${PORT} pnpm build && PORT=${PORT} pnpm start`,
33+
command: `PORT=${PORT} pnpm start`,
2234
url: baseURL,
2335
reuseExistingServer: !process.env.CI,
2436
stdout: 'pipe',

‎e2e/react-start/rsc/tests/setup/global.setup.ts‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,5 +2,5 @@ import { e2eStartDummyServer } from '@tanstack/router-e2e-utils'
22
import packageJson from '../../package.json' with { type: 'json' }
33

44
export default async function setup() {
5-
await e2eStartDummyServer(packageJson.name)
5+
await e2eStartDummyServer(process.env.E2E_PORT_KEY ?? packageJson.name)
66
}

‎e2e/react-start/rsc/tests/setup/global.teardown.ts‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,5 +2,5 @@ import { e2eStopDummyServer } from '@tanstack/router-e2e-utils'
22
import packageJson from '../../package.json' with { type: 'json' }
33

44
export default async function teardown() {
5-
await e2eStopDummyServer(packageJson.name)
5+
await e2eStopDummyServer(process.env.E2E_PORT_KEY ?? packageJson.name)
66
}

‎nx.json‎

Lines changed: 9 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -42,7 +42,7 @@
4242
},
4343
"test:e2e": {
4444
"cache": true,
45-
"dependsOn": ["^build"],
45+
"dependsOn": ["^build", "test:e2e--*"],
4646
"inputs": ["default", "^production"]
4747
},
4848
"test:e2e:nitro": {
@@ -76,5 +76,12 @@
7676
"dependsOn": ["build"],
7777
"inputs": ["production"]
7878
}
79-
}
79+
},
80+
"plugins": [
81+
{
82+
"name": "Playwright Sharding Plugin",
83+
"plugin": "./scripts/nx/playwright-plugin.ts"
84+
}
85+
],
86+
"analytics": true
8087
}

‎package.json‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -48,8 +48,11 @@
4848
"@arethetypeswrong/cli": "^0.18.2",
4949
"@changesets/cli": "^2.30.0",
5050
"@eslint-react/eslint-plugin": "^1.26.2",
51+
"@nx/devkit": "22.6.5",
5152
"@playwright/test": "^1.57.0",
5253
"@svitejs/changesets-changelog-github-compact": "^1.2.0",
54+
"@swc-node/core": "^1.14.1",
55+
"@swc-node/register": "^1.11.1",
5356
"@tanstack/eslint-config": "0.4.0",
5457
"@tanstack/query-core": "^5.90.19",
5558
"@tanstack/react-query": "^5.90.19",
@@ -64,7 +67,7 @@
6467
"eslint-plugin-unused-imports": "^4.1.4",
6568
"jsdom": "^25.0.1",
6669
"markdown-link-extractor": "^4.0.3",
67-
"nx": "22.5.1",
70+
"nx": "22.6.5",
6871
"prettier": "^3.8.0",
6972
"publint": "^0.3.16",
7073
"react": "^19.2.3",

‎pnpm-lock.yaml‎

Lines changed: 399 additions & 183 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎scripts/nx/playwright-plugin.md‎

Lines changed: 151 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,151 @@
1+
# Playwright E2E Sharding
2+
3+
Use `scripts/nx/playwright-plugin.ts` to turn a Playwright e2e project into Nx shard targets.
4+
5+
The plugin looks for `nx.metadata.playwrightShards` in a project's `package.json`. When present, it generates:
6+
7+
- `test:e2e--shard-1-of-N` through `test:e2e--shard-N-of-N`
8+
- a parent `test:e2e` target that depends on all shard targets
9+
10+
Each shard target runs Playwright with:
11+
12+
- `--shard=<index>/<count>`
13+
- a shard-specific `E2E_PORT_KEY`
14+
- a dependency on `build` and `^build`
15+
16+
## 1. Enable sharding in `package.json`
17+
18+
```json
19+
{
20+
"name": "tanstack-react-start-e2e-rsc",
21+
"nx": {
22+
"metadata": {
23+
"playwrightShards": 6
24+
}
25+
}
26+
}
27+
```
28+
29+
The package name is used to build the shard port key:
30+
31+
```txt
32+
<package-name>-shard-<index>-of-<count>
33+
```
34+
35+
## 2. Do not build inside `playwright.config.ts`
36+
37+
The plugin already makes each shard depend on `build`, so the Playwright `webServer` command should only start the built app.
38+
39+
Good:
40+
41+
```ts
42+
webServer: {
43+
command: `PORT=${PORT} pnpm start`,
44+
url: baseURL,
45+
}
46+
```
47+
48+
Avoid:
49+
50+
```ts
51+
webServer: {
52+
command: `PORT=${PORT} pnpm build && PORT=${PORT} pnpm start`,
53+
url: baseURL,
54+
}
55+
```
56+
57+
Running the build inside every shard causes unnecessary work and can create races when multiple shards share the same project directory.
58+
59+
## 3. Use `E2E_PORT_KEY` for every derived port
60+
61+
If the project uses `getTestServerPort`, `getDummyServerPort`, `e2eStartDummyServer`, or `e2eStopDummyServer`, key them off `process.env.E2E_PORT_KEY` first.
62+
63+
Example:
64+
65+
```ts
66+
import { getTestServerPort } from '@tanstack/router-e2e-utils'
67+
import packageJson from './package.json' with { type: 'json' }
68+
69+
const e2ePortKey = process.env.E2E_PORT_KEY ?? packageJson.name
70+
const PORT = await getTestServerPort(e2ePortKey)
71+
```
72+
73+
Dummy server setup/teardown should use the same key:
74+
75+
```ts
76+
await e2eStartDummyServer(process.env.E2E_PORT_KEY ?? packageJson.name)
77+
await e2eStopDummyServer(process.env.E2E_PORT_KEY ?? packageJson.name)
78+
```
79+
80+
Without this, multiple shards on the same runner will reuse the same `port-*.txt` files and tear each other down.
81+
82+
## 4. Clean stale port files once per Playwright run
83+
84+
If the project allocates ports through `@tanstack/router-e2e-utils`, clean stale `port-*.txt` files before resolving the port.
85+
86+
Important: do this only in the main Playwright config load. Playwright loads the config more than once, and unconditional cleanup can remap the port after the web server has already started.
87+
88+
Example:
89+
90+
```ts
91+
import fs from 'node:fs'
92+
import packageJson from './package.json' with { type: 'json' }
93+
94+
const e2ePortKey = process.env.E2E_PORT_KEY ?? packageJson.name
95+
96+
if (process.env.TEST_WORKER_INDEX === undefined) {
97+
for (const portFile of [
98+
`port-${e2ePortKey}.txt`,
99+
`port-${e2ePortKey}-external.txt`,
100+
]) {
101+
fs.rmSync(portFile, { force: true })
102+
}
103+
}
104+
```
105+
106+
Avoid broad cleanup such as `rm -rf port*.txt` in shared shard runs.
107+
108+
## 5. Keep the base URL derived from the shard key
109+
110+
```ts
111+
const PORT = await getTestServerPort(e2ePortKey)
112+
const baseURL = `http://localhost:${PORT}`
113+
114+
export default defineConfig({
115+
use: {
116+
baseURL,
117+
},
118+
webServer: {
119+
command: `PORT=${PORT} pnpm start`,
120+
url: baseURL,
121+
},
122+
})
123+
```
124+
125+
## 6. Run the generated targets
126+
127+
Examples:
128+
129+
```sh
130+
pnpm nx run tanstack-react-start-e2e-rsc:test:e2e--shard-1-of-6
131+
pnpm nx run tanstack-react-start-e2e-rsc:test:e2e
132+
```
133+
134+
## Checklist
135+
136+
- add `nx.metadata.playwrightShards` to `package.json`
137+
- make sure the project has a working `build` target
138+
- remove `build` from `webServer.command`
139+
- read ports from `process.env.E2E_PORT_KEY ?? packageJson.name`
140+
- use the same key for helper servers in setup and teardown
141+
- clean shard-specific `port-*.txt` files once, guarded by `TEST_WORKER_INDEX === undefined`
142+
- avoid wildcard port cleanup in shared shard runs
143+
144+
## Reference
145+
146+
Working example:
147+
148+
- `e2e/react-start/rsc/package.json`
149+
- `e2e/react-start/rsc/playwright.config.ts`
150+
- `e2e/react-start/rsc/tests/setup/global.setup.ts`
151+
- `e2e/react-start/rsc/tests/setup/global.teardown.ts`

‎scripts/nx/playwright-plugin.ts‎

Lines changed: 130 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,130 @@
1+
import { dirname } from 'node:path'
2+
import { createNodesFromFiles, readJsonFile } from '@nx/devkit'
3+
import type {
4+
CreateNodesContextV2,
5+
CreateNodesV2,
6+
TargetConfiguration,
7+
} from '@nx/devkit'
8+
9+
export const createNodesV2: CreateNodesV2 = [
10+
'**/package.json',
11+
async (configFiles, options, context) => {
12+
return await createNodesFromFiles(
13+
(configFile, options, context) =>
14+
createNodesInternal(configFile, context),
15+
configFiles,
16+
options,
17+
context,
18+
)
19+
},
20+
]
21+
22+
async function createNodesInternal(
23+
configFilePath: string,
24+
context: CreateNodesContextV2,
25+
) {
26+
const projectConfiguration = readJsonFile<{
27+
name?: string
28+
nx?: { metadata?: { playwrightShards?: number } }
29+
}>(configFilePath)
30+
const root = dirname(configFilePath)
31+
32+
if (!projectConfiguration.nx?.metadata?.playwrightShards) {
33+
return {
34+
projects: {
35+
[root]: {},
36+
},
37+
}
38+
}
39+
40+
const { targets, targetGroupEntries } = buildShardedTargets(
41+
root,
42+
projectConfiguration.name ?? root,
43+
projectConfiguration.nx.metadata.playwrightShards,
44+
['default', '^production'],
45+
)
46+
// Project configuration to be merged into the rest of the Nx configuration
47+
return {
48+
projects: {
49+
[root]: {
50+
targets,
51+
metadata: {
52+
targetGroups: {
53+
playwright: targetGroupEntries,
54+
},
55+
},
56+
tags: ['playwright:shareded'],
57+
},
58+
},
59+
}
60+
}
61+
62+
const CI_TARGET_NAME = 'test:e2e'
63+
64+
function buildShardedTargets(
65+
projectRoot: string,
66+
packageName: string,
67+
shardCount: number,
68+
testInputs: TargetConfiguration['inputs'],
69+
): {
70+
targets: Record<string, TargetConfiguration>
71+
targetGroupEntries: Array<string>
72+
} {
73+
const targets: Record<string, TargetConfiguration> = {}
74+
const targetGroup: Array<string> = []
75+
76+
// Create individual shard targets
77+
for (let shardIndex = 1; shardIndex <= shardCount; shardIndex++) {
78+
const shardTargetName = `${CI_TARGET_NAME}--shard-${shardIndex}-of-${shardCount}`
79+
const e2ePortKey = `${packageName}-shard-${shardIndex}-of-${shardCount}`
80+
81+
targets[shardTargetName] = {
82+
executor: 'nx:run-commands',
83+
options: {
84+
command: `playwright test --project=chromium --shard=${shardIndex}/${shardCount}`,
85+
cwd: projectRoot,
86+
env: {
87+
E2E_PORT_KEY: e2ePortKey,
88+
},
89+
},
90+
cache: true,
91+
inputs: testInputs,
92+
dependsOn: [`^build`, 'build'],
93+
metadata: {
94+
technologies: ['playwright'],
95+
description: `Run Playwright shard ${shardIndex} of ${shardCount}`,
96+
help: {
97+
command: `npx playwright run --help`,
98+
example: {
99+
options: {
100+
shard: `${shardIndex}/${shardCount}`,
101+
},
102+
},
103+
},
104+
},
105+
}
106+
107+
targetGroup.push(shardTargetName)
108+
}
109+
110+
// Create the parent CI target that depends on merge (which itself depends on all shards)
111+
targets[CI_TARGET_NAME] = {
112+
executor: 'nx:noop',
113+
cache: true,
114+
inputs: testInputs,
115+
dependsOn: targetGroup.map((shardTargetName) => ({
116+
target: shardTargetName,
117+
projects: 'self' as const,
118+
params: 'forward' as const,
119+
})),
120+
metadata: {
121+
technologies: ['playwright'],
122+
description: `Run all Playwright shards (${shardCount} shards)`,
123+
},
124+
}
125+
126+
return {
127+
targets,
128+
targetGroupEntries: [...targetGroup, CI_TARGET_NAME],
129+
}
130+
}

0 commit comments

Comments
 (0)