Skip to content

Commit 87b4de8

Browse files
p0dejep0deje
committed
Enable remote Bazel cache through Google Cloud Storage
This commit lands a support for public read-only remote Bazel cache stored on Google Cloud Storage at https://storage.googleapis.com/seleniumhq-bazel-remote-cache. The storage bucket is public so anyone on the internet can use it to speed up building Selenium locally. The cache is written on the trunk CI builds by specifying --remote_upload_local_results=true in CI .bazelrc. Likewise, Selenium committers can write to the cache by getting the JSON key and adding the following to the .bazelrc.local: build --remote_upload_local_results=true --google_credentials=<path-to-json-key> For now, the bucket is stored in @p0deje personal Google cloud and we should eventually move it SeleniumHQ account if/once we have it.
1 parent e8ae58d commit 87b4de8

9 files changed

Lines changed: 90 additions & 121 deletions

File tree

‎.bazelrc‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -38,6 +38,12 @@ build:linux --workspace_status_command=scripts/build-info.sh
3838
build:macos --workspace_status_command=scripts/build-info.sh
3939
build:windows --workspace_status_command="powershell.exe scripts/build-info.ps1"
4040

41+
# Enable read-only public remote cache.
42+
# To enable write mode, add the following to your .bazelrc.local:
43+
# build --remote_upload_local_results=true --google_credentials=<path-to-json-key>
44+
45+
build --remote_cache=https://storage.googleapis.com/seleniumhq-bazel-remote-cache --remote_upload_local_results=false
46+
4147
# Make sure we get something helpful when tests fail
4248

4349
build --verbose_failures

‎.github/.bazelrc.local‎

Lines changed: 0 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -4,10 +4,6 @@ startup --output_base=/tmp/bazel
44
# Set up caching on local disk so incremental builds are faster
55
# See https://bazel.build/designs/2016/09/30/repository-cache.html
66
build --repository_cache=~/.cache/bazel-repo
7-
test --repository_cache=~/.cache/bazel-repo
8-
# See https://docs.bazel.build/versions/master/remote-caching.html#disk-cache
9-
build --disk_cache=~/.cache/bazel-disk
10-
test --disk_cache=~/.cache/bazel-disk
117

128
# Make output easier to read
139
build --curses=no

‎.github/.bazelrc.windows.local‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,6 @@
11
# Use predefined path for building so we could cache `external/`.
22
startup --output_base=D:/_bazel
3-
build --disk_cache=D:/_bazel-disk
3+
4+
# Set up caching on local disk so incremental builds are faster
5+
# See https://bazel.build/designs/2016/09/30/repository-cache.html
46
build --repository_cache=D:/_bazel-repo

‎.github/actions/cache-bazel/action.yml‎

Lines changed: 13 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,9 @@ inputs:
77
key:
88
description: Extra cache key
99
required: true
10+
gcs-key:
11+
description: GCS Key
12+
required: false
1013
runs:
1114
using: "composite"
1215
steps:
@@ -29,7 +32,6 @@ runs:
2932
D:\_bazel\external\bundle
3033
D:\_bazel\external\rules_ruby
3134
D:\_bazel\external\rules_ruby_dist
32-
D:\_bazel-disk
3335
D:\_bazel-repo
3436
key: ${{ runner.os }}-bazel-${{ inputs.workflow }}-${{ inputs.key }}-${{ hashFiles('**/BUILD.bazel') }}
3537
restore-keys: |
@@ -40,9 +42,18 @@ runs:
4042
with:
4143
path: |
4244
/tmp/bazel/external
43-
~/.cache/bazel-disk
4445
~/.cache/bazel-repo
4546
key: ${{ runner.os }}-bazel-${{ inputs.workflow }}-${{ inputs.key }}-${{ hashFiles('**/BUILD.bazel') }}
4647
restore-keys: |
4748
${{ runner.os }}-bazel-${{ inputs.workflow }}-${{ inputs.key }}-
4849
${{ runner.os }}-bazel-${{ inputs.workflow }}-
50+
- run: |
51+
echo '${{ inputs.gcs-key }}' >> gcs.json
52+
echo "build --remote_upload_local_results=true --google_credentials=gcs.json" >> .bazelrc
53+
if: github.ref == 'refs/head/trunk' && runner.os != 'Windows'
54+
shell: bash
55+
- run: |
56+
echo ${{ inputs.gcs-key }}>>gcs.json
57+
echo "build --remote_upload_local_results=true --google_credentials=gcs.json">>.bazelrc
58+
if: github.ref == 'refs/head/trunk' && runner.os == 'Windows'
59+
shell: cmd

‎.github/workflows/ci-dotnet.yml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -45,6 +45,7 @@ jobs:
4545
with:
4646
workflow: dotnet
4747
key: ${{ matrix.driver }}-${{ matrix.framework }}
48+
gcs-key: ${{ secrets.GOOGLE_CLOUD_STORAGE_KEY }}
4849
- name: Setup dotnet tests
4950
uses: ./.github/actions/bazel
5051
with:

‎.github/workflows/ci-java.yml‎

Lines changed: 32 additions & 64 deletions
Original file line numberDiff line numberDiff line change
@@ -31,14 +31,11 @@ jobs:
3131
- name: Checkout source tree
3232
uses: actions/checkout@v3
3333
- name: Cache Bazel artifacts
34-
uses: actions/cache@v3
34+
uses: ./.github/actions/cache-bazel
3535
with:
36-
path: |
37-
~/.cache/bazel-disk
38-
~/.cache/bazel-repo
39-
key: ${{ runner.os }}-bazel-build-${{ hashFiles('**/BUILD.bazel') }}
40-
restore-keys: |
41-
${{ runner.os }}-bazel-build-
36+
workflow: bazel
37+
key: build
38+
gcs-key: ${{ secrets.GOOGLE_CLOUD_STORAGE_KEY }}
4239
- name: Setup Java
4340
uses: actions/setup-java@v3
4441
with:
@@ -59,15 +56,11 @@ jobs:
5956
with:
6057
fetch-depth: 0
6158
- name: Cache Bazel artifacts
62-
uses: actions/cache@v3
59+
uses: ./.github/actions/cache-bazel
6360
with:
64-
path: |
65-
~/.cache/bazel-disk
66-
~/.cache/bazel-repo
67-
key: ${{ runner.os }}-bazel-small-tests-${{ hashFiles('**/BUILD.bazel') }}
68-
restore-keys: |
69-
${{ runner.os }}-bazel-small-tests-
70-
${{ runner.os }}-bazel-build-
61+
workflow: bazel
62+
key: small-tests
63+
gcs-key: ${{ secrets.GOOGLE_CLOUD_STORAGE_KEY }}
7164
- name: Setup Java
7265
uses: actions/setup-java@v3
7366
with:
@@ -93,15 +86,11 @@ jobs:
9386
with:
9487
fetch-depth: 0
9588
- name: Cache Bazel artifacts
96-
uses: actions/cache@v3
89+
uses: ./.github/actions/cache-bazel
9790
with:
98-
path: |
99-
~/.cache/bazel-disk
100-
~/.cache/bazel-repo
101-
key: ${{ runner.os }}-bazel-medium-tests-${{ hashFiles('**/BUILD.bazel') }}
102-
restore-keys: |
103-
${{ runner.os }}-bazel-medium-tests-
104-
${{ runner.os }}-bazel-build-
91+
workflow: bazel
92+
key: medium-tests
93+
gcs-key: ${{ secrets.GOOGLE_CLOUD_STORAGE_KEY }}
10594
- name: Setup Java
10695
uses: actions/setup-java@v3
10796
with:
@@ -121,15 +110,11 @@ jobs:
121110
- name: Checkout source tree
122111
uses: actions/checkout@v3
123112
- name: Cache Bazel artifacts
124-
uses: actions/cache@v3
113+
uses: ./.github/actions/cache-bazel
125114
with:
126-
path: |
127-
~/.cache/bazel-disk
128-
~/.cache/bazel-repo
129-
key: ${{ runner.os }}-bazel-lint-${{ hashFiles('**/BUILD.bazel') }}
130-
restore-keys: |
131-
${{ runner.os }}-bazel-lint-
132-
${{ runner.os }}-bazel-build-
115+
workflow: bazel
116+
key: lint
117+
gcs-key: ${{ secrets.GOOGLE_CLOUD_STORAGE_KEY }}
133118
- name: Setup Java
134119
uses: actions/setup-java@v3
135120
with:
@@ -156,15 +141,11 @@ jobs:
156141
- name: Checkout source tree
157142
uses: actions/checkout@v3
158143
- name: Cache Bazel artifacts
159-
uses: actions/cache@v3
144+
uses: ./.github/actions/cache-bazel
160145
with:
161-
path: |
162-
~/.cache/bazel-disk
163-
~/.cache/bazel-repo
164-
key: ${{ runner.os }}-bazel-${{ matrix.browser }}-tests-${{ hashFiles('**/BUILD.bazel') }}
165-
restore-keys: |
166-
${{ runner.os }}-bazel-${{ matrix.browser }}-tests-
167-
${{ runner.os }}-bazel-build-
146+
workflow: bazel
147+
key: ${{ matrix.browser }}-tests
148+
gcs-key: ${{ secrets.GOOGLE_CLOUD_STORAGE_KEY }}
168149
- name: Setup Fluxbox
169150
run: sudo apt-get -y install fluxbox
170151
- name: Setup Java
@@ -213,16 +194,11 @@ jobs:
213194
- name: Checkout source tree
214195
uses: actions/checkout@v3
215196
- name: Cache Bazel artifacts
216-
uses: actions/cache@v3
197+
uses: ./.github/actions/cache-bazel
217198
with:
218-
path: |
219-
~/.cache/bazel-disk
220-
~/.cache/bazel-repo
221-
key: ${{ runner.os }}-bazel-${{ matrix.browser }}-server-tests-${{ hashFiles('**/BUILD.bazel') }}
222-
restore-keys: |
223-
${{ runner.os }}-bazel-${{ matrix.browser }}-server-tests-
224-
${{ runner.os }}-bazel-${{ matrix.browser }}-tests-
225-
${{ runner.os }}-bazel-build-
199+
workflow: bazel
200+
key: ${{ matrix.browser }}-server-tests
201+
gcs-key: ${{ secrets.GOOGLE_CLOUD_STORAGE_KEY }}
226202
- name: Setup Fluxbox
227203
run: sudo apt-get -y install fluxbox
228204
- name: Setup Java
@@ -263,15 +239,11 @@ jobs:
263239
- name: Checkout source tree
264240
uses: actions/checkout@v3
265241
- name: Cache Bazel artifacts
266-
uses: actions/cache@v3
242+
uses: ./.github/actions/cache-bazel
267243
with:
268-
path: |
269-
~/.cache/bazel-disk
270-
~/.cache/bazel-repo
271-
key: ${{ runner.os }}-bazel-firefox-tests-${{ hashFiles('**/BUILD.bazel') }}
272-
restore-keys: |
273-
${{ runner.os }}-bazel-firefox-tests-
274-
${{ runner.os }}-bazel-build-
244+
workflow: bazel
245+
key: firefox-tests
246+
gcs-key: ${{ secrets.GOOGLE_CLOUD_STORAGE_KEY }}
275247
- name: Setup Fluxbox
276248
run: sudo apt-get -y install fluxbox
277249
- name: Setup Java
@@ -306,15 +278,11 @@ jobs:
306278
- name: Checkout source tree
307279
uses: actions/checkout@v3
308280
- name: Cache Bazel artifacts
309-
uses: actions/cache@v3
281+
uses: ./.github/actions/cache-bazel
310282
with:
311-
path: |
312-
~/.cache/bazel-disk
313-
~/.cache/bazel-repo
314-
key: ${{ runner.os }}-bazel-firefox-tests-${{ hashFiles('**/BUILD.bazel') }}
315-
restore-keys: |
316-
${{ runner.os }}-bazel-firefox-tests-
317-
${{ runner.os }}-bazel-build-
283+
workflow: bazel
284+
key: firefox-tests
285+
gcs-key: ${{ secrets.GOOGLE_CLOUD_STORAGE_KEY }}
318286
- name: Setup Java
319287
uses: actions/setup-java@v3
320288
with:

‎.github/workflows/ci-javascript.yml‎

Lines changed: 8 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -34,15 +34,11 @@ jobs:
3434
- name: Checkout source tree
3535
uses: actions/checkout@v3
3636
- name: Cache Bazel artifacts
37-
uses: actions/cache@v3
37+
uses: ./.github/actions/cache-bazel
3838
with:
39-
path: |
40-
~/.cache/bazel-disk
41-
~/.cache/bazel-repo
42-
key: ${{ runner.os }}-bazel-node-tests-${{ hashFiles('**/BUILD.bazel') }}
43-
restore-keys: |
44-
${{ runner.os }}-bazel-node-tests-
45-
${{ runner.os }}-bazel-build-
39+
workflow: bazel
40+
key: node-tests
41+
gcs-key: ${{ secrets.GOOGLE_CLOUD_STORAGE_KEY }}
4642
- name: Setup Node.js
4743
uses: actions/setup-node@v1
4844
with:
@@ -86,14 +82,11 @@ jobs:
8682
- name: Checkout source tree
8783
uses: actions/checkout@v3
8884
- name: Cache Bazel artifacts
89-
uses: actions/cache@v3
85+
uses: ./.github/actions/cache-bazel
9086
with:
91-
path: |
92-
~/.cache/bazel-disk
93-
~/.cache/bazel-repo
94-
key: ${{ runner.os }}-bazel-atom-firefox-tests-${{ hashFiles('**/BUILD.bazel') }}
95-
restore-keys: |
96-
${{ runner.os }}-bazel-atom-firefox-tests-
87+
workflow: bazel
88+
key: atom-firefox-tests
89+
gcs-key: ${{ secrets.GOOGLE_CLOUD_STORAGE_KEY }}
9790
- name: Setup Java
9891
uses: actions/setup-java@v3
9992
with:

‎.github/workflows/ci-python.yml‎

Lines changed: 20 additions & 35 deletions
Original file line numberDiff line numberDiff line change
@@ -90,14 +90,11 @@ jobs:
9090
- name: Checkout source tree
9191
uses: actions/checkout@v3
9292
- name: Cache Bazel artifacts
93-
uses: actions/cache@v3
94-
with:
95-
path: |
96-
~/.cache/bazel-disk
97-
~/.cache/bazel-repo
98-
key: ${{ runner.os }}-bazel-python-unit-tests-${{ hashFiles('**/BUILD.bazel') }}
99-
restore-keys: |
100-
${{ runner.os }}-bazel-python-unit-tests-
93+
uses: ./.github/actions/cache-bazel
94+
with:
95+
workflow: python
96+
key: unit-tests
97+
gcs-key: ${{ secrets.GOOGLE_CLOUD_STORAGE_KEY }}
10198
- name: Setup Java
10299
uses: actions/setup-java@v3
103100
with:
@@ -116,15 +113,11 @@ jobs:
116113
- name: Checkout source tree
117114
uses: actions/checkout@v3
118115
- name: Cache Bazel artifacts
119-
uses: actions/cache@v3
120-
with:
121-
path: |
122-
~/.cache/bazel-disk
123-
~/.cache/bazel-repo
124-
key: ${{ runner.os }}-bazel-chrome-tests-${{ hashFiles('**/BUILD.bazel') }}
125-
restore-keys: |
126-
${{ runner.os }}-bazel-chrome-tests-
127-
${{ runner.os }}-bazel-python-unit-tests-
116+
uses: ./.github/actions/cache-bazel
117+
with:
118+
workflow: python
119+
key: chrome-tests
120+
gcs-key: ${{ secrets.GOOGLE_CLOUD_STORAGE_KEY }}
128121
- name: Setup Fluxbox
129122
run: sudo apt-get -y install fluxbox
130123
- name: Setup Chrome
@@ -149,15 +142,11 @@ jobs:
149142
- name: Checkout source tree
150143
uses: actions/checkout@v3
151144
- name: Cache Bazel artifacts
152-
uses: actions/cache@v3
153-
with:
154-
path: |
155-
~/.cache/bazel-disk
156-
~/.cache/bazel-repo
157-
key: ${{ runner.os }}-bazel-firefox-tests-${{ hashFiles('**/BUILD.bazel') }}
158-
restore-keys: |
159-
${{ runner.os }}-bazel-firefox-tests-
160-
${{ runner.os }}-bazel-python-unit-tests-
145+
uses: ./.github/actions/cache-bazel
146+
with:
147+
workflow: python
148+
key: firefox-tests
149+
gcs-key: ${{ secrets.GOOGLE_CLOUD_STORAGE_KEY }}
161150
- name: Setup Fluxbox
162151
run: sudo apt-get -y install fluxbox
163152
- name: Setup Java
@@ -189,15 +178,11 @@ jobs:
189178
- name: Checkout source tree
190179
uses: actions/checkout@v3
191180
- name: Cache Bazel artifacts
192-
uses: actions/cache@v3
193-
with:
194-
path: |
195-
~/.cache/bazel-disk
196-
~/.cache/bazel-repo
197-
key: ${{ runner.os }}-bazel-firefox-tests-${{ hashFiles('**/BUILD.bazel') }}
198-
restore-keys: |
199-
${{ runner.os }}-bazel-firefox-tests-
200-
${{ runner.os }}-bazel-python-unit-tests-
181+
uses: ./.github/actions/cache-bazel
182+
with:
183+
workflow: python
184+
key: firefox-tests
185+
gcs-key: ${{ secrets.GOOGLE_CLOUD_STORAGE_KEY }}
201186
- name: Setup Fluxbox
202187
run: sudo apt-get -y install fluxbox
203188
- name: Setup Java

0 commit comments

Comments
 (0)