Skip to content

Commit cac33e9

Browse files
committed
ci: server shards gate themselves instead of waiting for changes
Waiting on the changes job delayed every gated job by about 15s, which made PRs that run the server tests slower. The shards now run the same detection after checkout and stop early when the change cannot affect the server.
1 parent 3e051c1 commit cac33e9

3 files changed

Lines changed: 98 additions & 59 deletions

File tree

Lines changed: 81 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,81 @@
1+
name: CI changes
2+
description: Decide which CI jobs a change needs. See .github/scripts/ci-changes.cjs.
3+
outputs:
4+
typecheck:
5+
value: ${{ steps.detect.outputs.typecheck }}
6+
build:
7+
value: ${{ steps.detect.outputs.build }}
8+
test:
9+
value: ${{ steps.detect.outputs.test }}
10+
test_packages:
11+
value: ${{ steps.detect.outputs.test_packages }}
12+
test_web:
13+
value: ${{ steps.detect.outputs.test_web }}
14+
test_server:
15+
value: ${{ steps.detect.outputs.test_server }}
16+
rust:
17+
value: ${{ steps.detect.outputs.rust }}
18+
release_smoke:
19+
value: ${{ steps.detect.outputs.release_smoke }}
20+
mobile_native:
21+
value: ${{ steps.detect.outputs.mobile_native }}
22+
runs:
23+
using: composite
24+
steps:
25+
- id: detect
26+
shell: bash
27+
env:
28+
GH_TOKEN: ${{ github.token }}
29+
PR_NUMBER: ${{ github.event.pull_request.number }}
30+
BEFORE_SHA: ${{ github.event.before }}
31+
run: |
32+
set -uo pipefail
33+
34+
decide() {
35+
node .github/scripts/ci-changes.cjs "$@" | tee -a "$GITHUB_OUTPUT"
36+
exit "${PIPESTATUS[0]}"
37+
}
38+
39+
fail_open() {
40+
echo "$* Running every job."
41+
decide
42+
}
43+
44+
count_rows() {
45+
printf '%s\n' "$1" | grep -c . || true
46+
}
47+
48+
# One row per changed file, holding the new path and, for a rename,
49+
# the path it replaced: renaming a file out of a package changes that
50+
# package just like editing it.
51+
row='[.filename, (.previous_filename // empty)] | @tsv'
52+
53+
if [[ -n "${PR_NUMBER}" ]]; then
54+
# The PR files endpoint stops at 3000 files and pagination cannot
55+
# extend it, so cross-check against the count the PR itself reports.
56+
expected=$(gh api "repos/${GITHUB_REPOSITORY}/pulls/${PR_NUMBER}" --jq '.changed_files') \
57+
|| fail_open "Could not read the pull request."
58+
rows=$(gh api "repos/${GITHUB_REPOSITORY}/pulls/${PR_NUMBER}/files" --paginate --jq ".[] | ${row}") \
59+
|| fail_open "Could not resolve changed files."
60+
61+
listed=$(count_rows "$rows")
62+
if [[ "$listed" -lt "$expected" ]]; then
63+
fail_open "GitHub listed only ${listed} of ${expected} changed files."
64+
fi
65+
mode=()
66+
else
67+
rows=$(gh api "repos/${GITHUB_REPOSITORY}/compare/${BEFORE_SHA}...${GITHUB_SHA}" --jq ".files[]? | ${row}") \
68+
|| fail_open "Could not resolve changed files."
69+
70+
# The compare endpoint reports at most 300 files and pagination does
71+
# not extend that list, so a full list may be hiding changes.
72+
listed=$(count_rows "$rows")
73+
if [[ "$listed" -ge 300 ]]; then
74+
fail_open "GitHub listed ${listed} changed files, the compare endpoint maximum."
75+
fi
76+
# main keeps a full run as the backstop for anything the gate misses.
77+
mode=(--full)
78+
fi
79+
80+
tr '\t' '\n' <<< "$rows" > "$RUNNER_TEMP/changed-paths"
81+
decide --paths "$RUNNER_TEMP/changed-paths" "${mode[@]}"

‎.github/scripts/ci-changes.cjs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -61,7 +61,7 @@ const BUILD_ONLY_DEPS = { t3: ["@t3tools/web"] };
6161
// configuration that decides how it lints them, plus the root package.json
6262
// that defines the lint:mobile command.
6363
const MOBILE_NATIVE =
64-
/^apps\/mobile\/.*\.(swift|kt|kts)$|^apps\/mobile\/(\.swiftlint\.yml|detekt\.yml|\.editorconfig|Brewfile)$|^scripts\/mobile-native-static-check\.ts$|^package\.json$|^\.github\/workflows\/ci\.yml$|^\.github\/scripts\/ci-changes\.cjs$/;
64+
/^apps\/mobile\/.*\.(swift|kt|kts)$|^apps\/mobile\/(\.swiftlint\.yml|detekt\.yml|\.editorconfig|Brewfile)$|^scripts\/mobile-native-static-check\.ts$|^package\.json$|^\.github\/workflows\/ci\.yml$|^\.github\/scripts\/ci-changes\.cjs$|^\.github\/actions\/ci-changes\//;
6565

6666
// Jobs that test every package except these, which have jobs of their own.
6767
const OWN_TEST_JOB = new Set(["t3", "@t3tools/web", "@t3tools/monorepo"]);

‎.github/workflows/ci.yml‎

Lines changed: 16 additions & 58 deletions
Original file line numberDiff line numberDiff line change
@@ -45,68 +45,15 @@ jobs:
4545
sparse-checkout: |
4646
/pnpm-workspace.yaml
4747
/.github/scripts/ci-changes.cjs
48+
/.github/actions/ci-changes/
4849
/*/package.json
4950
/*/*/package.json
5051
!/.repos/
5152
sparse-checkout-cone-mode: false
5253

5354
- name: Detect affected jobs
5455
id: detect
55-
env:
56-
GH_TOKEN: ${{ github.token }}
57-
PR_NUMBER: ${{ github.event.pull_request.number }}
58-
BEFORE_SHA: ${{ github.event.before }}
59-
run: |
60-
set -uo pipefail
61-
62-
decide() {
63-
node .github/scripts/ci-changes.cjs "$@" | tee -a "$GITHUB_OUTPUT"
64-
exit "${PIPESTATUS[0]}"
65-
}
66-
67-
fail_open() {
68-
echo "$* Running every job."
69-
decide
70-
}
71-
72-
count_rows() {
73-
printf '%s\n' "$1" | grep -c . || true
74-
}
75-
76-
# One row per changed file, holding the new path and, for a rename,
77-
# the path it replaced: renaming a file out of a package changes that
78-
# package just like editing it.
79-
row='[.filename, (.previous_filename // empty)] | @tsv'
80-
81-
if [[ -n "${PR_NUMBER}" ]]; then
82-
# The PR files endpoint stops at 3000 files and pagination cannot
83-
# extend it, so cross-check against the count the PR itself reports.
84-
expected=$(gh api "repos/${GITHUB_REPOSITORY}/pulls/${PR_NUMBER}" --jq '.changed_files') \
85-
|| fail_open "Could not read the pull request."
86-
rows=$(gh api "repos/${GITHUB_REPOSITORY}/pulls/${PR_NUMBER}/files" --paginate --jq ".[] | ${row}") \
87-
|| fail_open "Could not resolve changed files."
88-
89-
listed=$(count_rows "$rows")
90-
if [[ "$listed" -lt "$expected" ]]; then
91-
fail_open "GitHub listed only ${listed} of ${expected} changed files."
92-
fi
93-
mode=()
94-
else
95-
rows=$(gh api "repos/${GITHUB_REPOSITORY}/compare/${BEFORE_SHA}...${GITHUB_SHA}" --jq ".files[]? | ${row}") \
96-
|| fail_open "Could not resolve changed files."
97-
98-
# The compare endpoint reports at most 300 files and pagination does
99-
# not extend that list, so a full list may be hiding changes.
100-
listed=$(count_rows "$rows")
101-
if [[ "$listed" -ge 300 ]]; then
102-
fail_open "GitHub listed ${listed} changed files, the compare endpoint maximum."
103-
fi
104-
# main keeps a full run as the backstop for anything the gate misses.
105-
mode=(--full)
106-
fi
107-
108-
tr '\t' '\n' <<< "$rows" > "$RUNNER_TEMP/changed-paths"
109-
decide --paths "$RUNNER_TEMP/changed-paths" "${mode[@]}"
56+
uses: ./.github/actions/ci-changes
11057

11158
# Lint, typecheck, and the desktop build each get a runner so they run side by
11259
# side instead of one after another. Lint always runs: it formats every file,
@@ -340,12 +287,15 @@ jobs:
340287
# isolation that flag buys is preserved exactly. One file at a time keeps
341288
# about 1.5 cores busy, so 4 vCPU runners lose nothing against 8. The shards
342289
# split files by recorded duration (apps/server/src/testUtils/weightedShardSequencer.ts).
290+
# The shards are the longest jobs, so they gate themselves instead of waiting
291+
# ~15s for `changes`: a shard the change cannot affect stops after checkout.
343292
test_server:
344293
name: Test Server ${{ matrix.shard }}
345-
needs: changes
346-
if: ${{ !cancelled() && needs.changes.outputs.test_server != 'false' }}
347294
runs-on: blacksmith-4vcpu-ubuntu-2404
348295
timeout-minutes: 10
296+
permissions:
297+
contents: read
298+
pull-requests: read
349299
strategy:
350300
fail-fast: false
351301
matrix:
@@ -359,7 +309,12 @@ jobs:
359309
!/.repos/
360310
sparse-checkout-cone-mode: false
361311

312+
- name: Detect affected jobs
313+
id: changes
314+
uses: ./.github/actions/ci-changes
315+
362316
- name: Setup Vite+
317+
if: steps.changes.outputs.test_server != 'false'
363318
uses: voidzero-dev/setup-vp@v1
364319
with:
365320
node-version-file: package.json
@@ -370,9 +325,11 @@ jobs:
370325
# and none of its tests touch the runtime. Only the non-server `test`
371326
# job, which covers @t3tools/desktop, needs the download.
372327
- name: Install Chromium for server browser integration tests
328+
if: steps.changes.outputs.test_server != 'false'
373329
run: node apps/server/node_modules/playwright-core/cli.js install --with-deps --only-shell chromium
374330

375331
- name: Test
332+
if: steps.changes.outputs.test_server != 'false'
376333
env:
377334
T3CODE_TRANSFER_BUDGET_REPORT_PATH: ${{ runner.temp }}/t3code-transfer-budget.md
378335
T3CODE_TRANSFER_BUDGET_RESULT_PATH: ${{ runner.temp }}/thread-transfer-result.json
@@ -412,7 +369,8 @@ jobs:
412369

413370
transfer-report:
414371
name: Transfer report artifact
415-
needs: test_server
372+
needs: [changes, test_server]
373+
if: needs.changes.outputs.test_server != 'false'
416374
# On Blacksmith like every other job here: the GitHub-hosted queue is shared
417375
# with the PR bots and can leave this job waiting until it is abandoned,
418376
# which fails `Check` on an otherwise green run.

0 commit comments

Comments
 (0)