Skip to content

Commit a576f33

Browse files
myasnikovdaniilmyasnikovdaniil
committed
refactor(build): mode=max registry cache with a main-only warmer
`--cache-to type=inline` only exports the final stage. 25 of 38 Dockerfiles are multistage, so the expensive `builder` stages (Go/Node compiles) were never cached even with a warm tag. Switch every image to a shared mode=max registry cache. - New `cache-args` macro (hack/common-envs.mk) emits `--cache-from` always and `--cache-to … mode=max` only when WRITE_CACHE=1, so PR builds stay read-only and concurrent PRs never race on the cache ref (the 409 class #2711 fixed for tags). oci-mediatypes/image-manifest keep the cache manifest portable across registries. - Cache ref is CACHE_REGISTRY/<img>:buildcache, co-located with the build registry (CACHE_REGISTRY now defaults to $(REGISTRY)). - New build-main.yaml warms :buildcache on every push to main (WRITE_CACHE=1), serialized via concurrency. All 31 package image recipes converted from the inline cache-from/cache-to pair to `$(call cache-args,…)`; ubuntu-container-disk keeps a per-k8s-version cache tag. Verified with `make -n image` across all packages in both PR (read-only) and main (mode=max write) contexts. Part of #2937. Assisted-By: Claude <[email protected]> Signed-off-by: Myasnikov Daniil <[email protected]>
1 parent b825c31 commit a576f33

33 files changed

Lines changed: 130 additions & 81 deletions

File tree

‎.github/workflows/build-main.yaml‎

Lines changed: 67 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,67 @@
1+
name: Build cache (main)
2+
3+
# Warms the shared mode=max build cache so PR builds start hot.
4+
#
5+
# PR builds (pull-requests.yaml) read the cache but never write it
6+
# (WRITE_CACHE unset -> 0). Only this serialized main build writes
7+
# CACHE_REGISTRY/<img>:buildcache, so concurrent PR builds never race on the
8+
# cache manifest -- the 409 collisions PR #2711 fixed for image tags would
9+
# otherwise reappear on the cache ref. Cache is mode=max so all multistage
10+
# `builder` stages are cached, not just the final stage as with inline.
11+
12+
env:
13+
# Same per-CI registry as pull-requests.yaml; the cache is co-located here.
14+
REGISTRY: iad.ocir.io/idyksih5sir9/cozystack
15+
16+
on:
17+
push:
18+
branches: [main]
19+
paths-ignore:
20+
- 'docs/**'
21+
22+
# Only the newest main commit's cache matters; cancel older in-flight warmers.
23+
concurrency:
24+
group: build-main-cache
25+
cancel-in-progress: true
26+
27+
jobs:
28+
warm-cache:
29+
name: Warm build cache
30+
runs-on: [self-hosted]
31+
timeout-minutes: 90
32+
permissions:
33+
contents: read
34+
packages: write
35+
36+
steps:
37+
- name: Checkout code
38+
uses: actions/checkout@v4
39+
with:
40+
fetch-depth: 0
41+
fetch-tags: true
42+
43+
- name: Set up Docker config
44+
run: |
45+
if [ -d ~/.docker ]; then
46+
cp -r ~/.docker "${{ runner.temp }}/.docker"
47+
fi
48+
49+
- name: Login to OCIR
50+
uses: docker/login-action@v3
51+
with:
52+
username: ${{ secrets.OCIR_USER }}
53+
password: ${{ secrets.OCIR_TOKEN }}
54+
registry: iad.ocir.io
55+
env:
56+
DOCKER_CONFIG: ${{ runner.temp }}/.docker
57+
58+
- name: Build all images and write mode=max cache
59+
run: make build
60+
env:
61+
DOCKER_CONFIG: ${{ runner.temp }}/.docker
62+
# WRITE_CACHE=1 turns on --cache-to (mode=max) for every image.
63+
WRITE_CACHE: '1'
64+
# Publish a floating :main handle; do not move :latest (releases do).
65+
IMAGE_TAG: main
66+
PUBLISH_VERSIONED: '0'
67+
PUBLISH_FLOATING: '0'

‎hack/common-envs.mk‎

Lines changed: 26 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -7,13 +7,12 @@ endif
77

88
REGISTRY ?= ghcr.io/cozystack/cozystack
99

10-
# CACHE_REGISTRY: registry that `--cache-from` reads build cache from. It must
11-
# point at a registry where a `:latest` tag is actually published. PR builds
12-
# set REGISTRY to a per-CI registry and push only unique `pr-<N>-<sha>` tags
13-
# (PUBLISH_FLOATING=0), so they never refresh `:latest` there — reading cache
14-
# from $(REGISTRY):latest would always 404. Release builds publish `:latest`
15-
# to ghcr.io, so default cache reads there and stays warm across PR builds.
16-
CACHE_REGISTRY ?= ghcr.io/cozystack/cozystack
10+
# CACHE_REGISTRY: registry holding the `:buildcache` mode=max cache images that
11+
# `--cache-from`/`--cache-to` read and write (see the cache-args macro below).
12+
# Defaults to $(REGISTRY) so the cache is co-located with the build registry —
13+
# in CI that is the per-CI registry the runners are closest to, keeping cache
14+
# transfers fast and egress-free.
15+
CACHE_REGISTRY ?= $(REGISTRY)
1716

1817
# IMAGE_TAG: build-unique tag pushed for every image. Set by CI to a value
1918
# that does not collide between concurrent builds:
@@ -29,6 +28,12 @@ IMAGE_TAG ?= dev
2928
PUBLISH_VERSIONED ?= 0
3029
PUBLISH_FLOATING ?= 0
3130

31+
# WRITE_CACHE=1 enables `--cache-to` (mode=max) to CACHE_REGISTRY/<img>:buildcache.
32+
# Default off: only the serialized main/release cache-warming build writes the
33+
# shared cache ref. PR builds keep it 0 and read-only, so concurrent PR builds
34+
# never race on the cache manifest (the 409 collisions PR #2711 fixed for tags).
35+
WRITE_CACHE ?= 0
36+
3237
PUSH := 1
3338
LOAD := 0
3439
BUILDER ?=
@@ -55,6 +60,20 @@ define image-tags
5560
--tag $(REGISTRY)/$(1):$(IMAGE_TAG)$(if $(filter 1,$(PUBLISH_VERSIONED)),$(if $(filter-out $(IMAGE_TAG),$(strip $(2))), --tag $(REGISTRY)/$(1):$(strip $(2))))$(if $(filter 1,$(PUBLISH_FLOATING)), --tag $(REGISTRY)/$(1):latest)
5661
endef
5762

63+
# cache-args <image-name> [<cache-tag>]
64+
# Expands to buildx cache flags for one image:
65+
# --cache-from is always emitted (a missing cache 404s harmlessly -> cold build)
66+
# --cache-to is emitted only when WRITE_CACHE=1 (main/release), writing mode=max
67+
# so ALL stages are cached -- including the multistage `builder` layers that
68+
# `--cache-to type=inline` could never export. oci-mediatypes + image-manifest
69+
# keep the cache manifest portable across registries (OCIR/ghcr/ECR).
70+
# <cache-tag> defaults to `buildcache`; pass an explicit tag for images that build
71+
# a distinct artifact per loop iteration (e.g. ubuntu-container-disk per k8s ver).
72+
# $(comma) escapes the literal commas in the --cache-to value so make does not
73+
# mis-parse them as $(if ...) argument separators.
74+
comma := ,
75+
cache-args = --cache-from type=registry,ref=$(CACHE_REGISTRY)/$(1):$(if $(2),$(2),buildcache)$(if $(filter 1,$(WRITE_CACHE)), --cache-to type=registry$(comma)ref=$(CACHE_REGISTRY)/$(1):$(if $(2),$(2),buildcache)$(comma)mode=max$(comma)oci-mediatypes=true$(comma)image-manifest=true)
76+
5877
ifeq ($(COZYSTACK_VERSION),)
5978
$(shell git remote add upstream https://github.com/cozystack/cozystack.git || true)
6079
$(shell git fetch upstream --tags)

‎packages/apps/clickhouse/Makefile‎

Lines changed: 2 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -13,17 +13,15 @@ test:
1313
image:
1414
docker buildx build images/clickhouse-backup \
1515
$(call image-tags,clickhouse-backup,$(CLICKHOUSE_BACKUP_TAG)) \
16-
--cache-from type=registry,ref=$(CACHE_REGISTRY)/clickhouse-backup:latest \
17-
--cache-to type=inline \
16+
$(call cache-args,clickhouse-backup) \
1817
--metadata-file images/clickhouse-backup.json \
1918
$(BUILDX_ARGS)
2019
echo "$(REGISTRY)/clickhouse-backup:$(IMAGE_TAG)@$$(yq e '."containerimage.digest"' images/clickhouse-backup.json -o json -r)" \
2120
> images/clickhouse-backup.tag
2221
rm -f images/clickhouse-backup.json
2322
docker buildx build images/altinity-clickhouse-backup \
2423
$(call image-tags,altinity-clickhouse-backup,$(CLICKHOUSE_BACKUP_TAG)) \
25-
--cache-from type=registry,ref=$(CACHE_REGISTRY)/altinity-clickhouse-backup:latest \
26-
--cache-to type=inline \
24+
$(call cache-args,altinity-clickhouse-backup) \
2725
--metadata-file images/altinity-clickhouse-backup.json \
2826
$(BUILDX_ARGS)
2927
echo "$(REGISTRY)/altinity-clickhouse-backup:$(IMAGE_TAG)@$$(yq e '."containerimage.digest"' images/altinity-clickhouse-backup.json -o json -r)" \

‎packages/apps/http-cache/Makefile‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -8,8 +8,7 @@ image: image-nginx
88
image-nginx:
99
docker buildx build images/nginx-cache \
1010
$(call image-tags,nginx-cache,$(NGINX_CACHE_TAG)) \
11-
--cache-from type=registry,ref=$(CACHE_REGISTRY)/nginx-cache:latest \
12-
--cache-to type=inline \
11+
$(call cache-args,nginx-cache) \
1312
--metadata-file images/nginx-cache.json \
1413
$(BUILDX_ARGS)
1514
echo "$(REGISTRY)/nginx-cache:$(IMAGE_TAG)@$$(yq e '."containerimage.digest"' images/nginx-cache.json -o json -r)" \

‎packages/apps/kubernetes/Makefile‎

Lines changed: 4 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -26,8 +26,7 @@ image-ubuntu-container-disk:
2626
--build-arg KUBERNETES_VERSION=$(ver) \
2727
--tag $(REGISTRY)/ubuntu-container-disk:$(ver)-$(IMAGE_TAG) \
2828
$(if $(filter 1,$(PUBLISH_VERSIONED)),--tag $(REGISTRY)/ubuntu-container-disk:$(ver)) \
29-
--cache-from type=registry,ref=$(CACHE_REGISTRY)/ubuntu-container-disk:$(ver) \
30-
--cache-to type=inline \
29+
$(call cache-args,ubuntu-container-disk,$(ver)-buildcache) \
3130
--metadata-file images/ubuntu-container-disk-$(ver).json \
3231
$(BUILDX_ARGS) && \
3332
echo "$(REGISTRY)/ubuntu-container-disk:$(ver)-$(IMAGE_TAG)@$$(yq e '."containerimage.digest"' images/ubuntu-container-disk-$(ver).json -o json -r)" \
@@ -38,8 +37,7 @@ image-ubuntu-container-disk:
3837
image-kubevirt-cloud-provider:
3938
docker buildx build images/kubevirt-cloud-provider \
4039
$(call image-tags,kubevirt-cloud-provider,$(KUBERNETES_PKG_TAG)) \
41-
--cache-from type=registry,ref=$(CACHE_REGISTRY)/kubevirt-cloud-provider:latest \
42-
--cache-to type=inline \
40+
$(call cache-args,kubevirt-cloud-provider) \
4341
--metadata-file images/kubevirt-cloud-provider.json \
4442
$(BUILDX_ARGS)
4543
echo "$(REGISTRY)/kubevirt-cloud-provider:$(IMAGE_TAG)@$$(yq e '."containerimage.digest"' images/kubevirt-cloud-provider.json -o json -r)" \
@@ -49,8 +47,7 @@ image-kubevirt-cloud-provider:
4947
image-kubevirt-csi-driver:
5048
docker buildx build images/kubevirt-csi-driver \
5149
$(call image-tags,kubevirt-csi-driver,$(KUBERNETES_PKG_TAG)) \
52-
--cache-from type=registry,ref=$(CACHE_REGISTRY)/kubevirt-csi-driver:latest \
53-
--cache-to type=inline \
50+
$(call cache-args,kubevirt-csi-driver) \
5451
--metadata-file images/kubevirt-csi-driver.json \
5552
$(BUILDX_ARGS)
5653
echo "$(REGISTRY)/kubevirt-csi-driver:$(IMAGE_TAG)@$$(yq e '."containerimage.digest"' images/kubevirt-csi-driver.json -o json -r)" \
@@ -63,8 +60,7 @@ image-kubevirt-csi-driver:
6360
image-cluster-autoscaler:
6461
docker buildx build images/cluster-autoscaler \
6562
$(call image-tags,cluster-autoscaler,$(KUBERNETES_PKG_TAG)) \
66-
--cache-from type=registry,ref=$(CACHE_REGISTRY)/cluster-autoscaler:latest \
67-
--cache-to type=inline \
63+
$(call cache-args,cluster-autoscaler) \
6864
--metadata-file images/cluster-autoscaler.json \
6965
$(BUILDX_ARGS)
7066
echo "$(REGISTRY)/cluster-autoscaler:$(IMAGE_TAG)@$$(yq e '."containerimage.digest"' images/cluster-autoscaler.json -o json -r)" \

‎packages/apps/mariadb/Makefile‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -14,8 +14,7 @@ update:
1414
image:
1515
docker buildx build images/mariadb-backup \
1616
$(call image-tags,mariadb-backup,$(MARIADB_BACKUP_TAG)) \
17-
--cache-from type=registry,ref=$(CACHE_REGISTRY)/mariadb-backup:latest \
18-
--cache-to type=inline \
17+
$(call cache-args,mariadb-backup) \
1918
--metadata-file images/mariadb-backup.json \
2019
$(BUILDX_ARGS)
2120
echo "$(REGISTRY)/mariadb-backup:$(IMAGE_TAG)@$$(yq e '."containerimage.digest"' images/mariadb-backup.json -o json -r)" \

‎packages/core/installer/Makefile‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -21,8 +21,7 @@ image-operator:
2121
docker buildx build -f images/cozystack-operator/Dockerfile ../../.. \
2222
$(call image-tags,cozystack-operator,v$(COZYSTACK_VERSION)) \
2323
--build-arg VERSION=v$(COZYSTACK_VERSION) \
24-
--cache-from type=registry,ref=$(CACHE_REGISTRY)/cozystack-operator:latest \
25-
--cache-to type=inline \
24+
$(call cache-args,cozystack-operator) \
2625
--metadata-file images/cozystack-operator.json \
2726
$(BUILDX_ARGS)
2827
IMAGE="$(REGISTRY)/cozystack-operator:$(IMAGE_TAG)@$$(yq -e '.["containerimage.digest"]' images/cozystack-operator.json -o json -r)" \

‎packages/core/platform/Makefile‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -30,8 +30,7 @@ image: image-migrations
3030
image-migrations:
3131
docker buildx build images/migrations \
3232
$(call image-tags,platform-migrations,v$(COZYSTACK_VERSION)) \
33-
--cache-from type=registry,ref=$(CACHE_REGISTRY)/platform-migrations:latest \
34-
--cache-to type=inline \
33+
$(call cache-args,platform-migrations) \
3534
--metadata-file images/migrations.json \
3635
$(BUILDX_ARGS)
3736
IMAGE="$(REGISTRY)/platform-migrations:$(IMAGE_TAG)@$$(yq -e -o json -r '.["containerimage.digest"]' images/migrations.json)" \

‎packages/core/talos/Makefile‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -23,8 +23,7 @@ image-matchbox:
2323
test -f ../../../_out/assets/initramfs-metal-amd64.xz || make talos-initramfs
2424
docker buildx build -f images/matchbox/Dockerfile ../../.. \
2525
$(call image-tags,matchbox,$(TALOS_VERSION)) \
26-
--cache-from type=registry,ref=$(CACHE_REGISTRY)/matchbox:latest \
27-
--cache-to type=inline \
26+
$(call cache-args,matchbox) \
2827
--metadata-file images/matchbox.json \
2928
$(BUILDX_ARGS)
3029
echo "$(REGISTRY)/matchbox:$(IMAGE_TAG)@$$(yq e '."containerimage.digest"' images/matchbox.json -o json -r)" \

‎packages/core/testing/Makefile‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -17,8 +17,7 @@ image: image-e2e-sandbox
1717
image-e2e-sandbox:
1818
docker buildx build -f images/e2e-sandbox/Dockerfile images/e2e-sandbox \
1919
$(call image-tags,e2e-sandbox,v$(COZYSTACK_VERSION)) \
20-
--cache-from type=registry,ref=$(CACHE_REGISTRY)/e2e-sandbox:latest \
21-
--cache-to type=inline \
20+
$(call cache-args,e2e-sandbox) \
2221
--metadata-file images/e2e-sandbox.json \
2322
$(BUILDX_ARGS)
2423
IMAGE="$(REGISTRY)/e2e-sandbox:$(IMAGE_TAG)@$$(yq e '."containerimage.digest"' images/e2e-sandbox.json -o json -r)" \

0 commit comments

Comments
 (0)