Skip to content

Commit 0a48fee

Browse files
authored
Add prek hook to keep the Go toolchain version in sync (#69338)
The Go toolchain version is pinned in eight places (go-sdk/go.mod, the go_example module, the setup-go step in both CI workflow copies, the golang alpine builder image used by the e2e conftest and by breeze, and the default_language_version.golang in the top-level and go-sdk prek configs). They have no cross-file include, so a dependency bump that raises the go directive in go-sdk/go.mod leaves the rest behind and the drift only surfaces as an opaque "go.mod requires go >= <new>" failure deep in CI — as happened in PR #69214. Add a prek hook that treats go-sdk/go.mod as the single source of truth and fails when any other site disagrees at major.minor granularity, pointing directly at the drifting file.
1 parent d19a68a commit 0a48fee

3 files changed

Lines changed: 342 additions & 0 deletions

File tree

‎.pre-commit-config.yaml‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -297,6 +297,21 @@ repos:
297297
^Dockerfile\.ci$
298298
pass_filenames: false
299299
require_serial: true
300+
- id: check-go-version-in-sync
301+
name: Check Go toolchain version is consistent across build files
302+
entry: ./scripts/ci/prek/check_go_version_in_sync.py
303+
language: python
304+
files: >
305+
(?x)
306+
^go-sdk/go\.mod$|
307+
^kubernetes-tests/lang_sdk/go_example/go\.mod$|
308+
^\.github/workflows/ci-(arm|amd)\.yml$|
309+
^airflow-e2e-tests/tests/airflow_e2e_tests/constants\.py$|
310+
^dev/breeze/src/airflow_breeze/commands/kubernetes_commands\.py$|
311+
^go-sdk/\.pre-commit-config\.yaml$|
312+
^\.pre-commit-config\.yaml$
313+
pass_filenames: false
314+
require_serial: true
300315
- id: check-partition-mapper-defaults-in-sync
301316
name: Check partition-mapper core/SDK sync (FanOutMapper table + SegmentWindow/FixedKeyMapper)
302317
entry: ./scripts/ci/prek/check_partition_mapper_defaults_in_sync.py
Lines changed: 197 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,197 @@
1+
#!/usr/bin/env python3
2+
# Licensed to the Apache Software Foundation (ASF) under one
3+
# or more contributor license agreements. See the NOTICE file
4+
# distributed with this work for additional information
5+
# regarding copyright ownership. The ASF licenses this file
6+
# to you under the Apache License, Version 2.0 (the
7+
# "License"); you may not use this file except in compliance
8+
# with the License. You may obtain a copy of the License at
9+
#
10+
# http://www.apache.org/licenses/LICENSE-2.0
11+
#
12+
# Unless required by applicable law or agreed to in writing,
13+
# software distributed under the License is distributed on an
14+
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15+
# KIND, either express or implied. See the License for the
16+
# specific language governing permissions and limitations
17+
# under the License.
18+
"""
19+
Keep the Go toolchain version in sync across every file that pins it.
20+
21+
``golang.org/x/net`` (and other ``golang.org/x`` modules) periodically raise
22+
their minimum Go version. When that happens a dependency bump changes the
23+
``go`` directive in ``go-sdk/go.mod`` but leaves the *other* places that pin
24+
the toolchain untouched — and CI then fails deep inside static checks and the
25+
Go SDK e2e bundle build with ``go.mod requires go >= <new>`` (this is exactly
26+
what happened in PR #69214). Those pins have no cross-file include, so nothing
27+
catches the drift until CI does.
28+
29+
The single source of truth is the ``go`` directive in ``go-sdk/go.mod``. Every
30+
other site below must agree with it at ``major.minor`` granularity (the minor
31+
version is what selects the toolchain and the builder image; the go.mod patch
32+
suffix is not meaningful for those):
33+
34+
- ``go-sdk/go.mod`` -> ``go <maj.min[.patch]>`` (SOURCE OF TRUTH)
35+
- ``kubernetes-tests/lang_sdk/go_example/go.mod`` -> ``go <maj.min[.patch]>`` (resolves the SDK via ``replace``)
36+
- ``.github/workflows/ci-amd.yml`` -> Setup Go ``go-version: <maj.min>``
37+
- ``.github/workflows/ci-arm.yml`` -> Setup Go ``go-version: <maj.min>``
38+
- ``airflow-e2e-tests/tests/airflow_e2e_tests/constants.py`` -> ``GO_BUILDER_IMAGE`` default ``golang:<maj.min>-alpine``
39+
- ``dev/breeze/src/airflow_breeze/commands/kubernetes_commands.py`` -> ``LANG_SDK_GO_BUILDER_IMAGE`` default ``golang:<maj.min>-alpine``
40+
- ``go-sdk/.pre-commit-config.yaml`` -> ``default_language_version.golang`` (the toolchain prek uses for the SDK's ``language: golang`` hooks — this is the pin that broke static checks in #69214, not the setup-go one)
41+
- ``.pre-commit-config.yaml`` -> ``default_language_version.golang`` (top-level default for any ``language: golang`` hook)
42+
43+
When they disagree, bump the drifting sites to the source-of-truth minor
44+
version. If you add a new place that pins the Go version, register it here too.
45+
46+
Run from the repo root:
47+
48+
uv run --project scripts python scripts/ci/prek/check_go_version_in_sync.py
49+
50+
Exits 0 if every site agrees with the source of truth, 1 otherwise.
51+
"""
52+
53+
from __future__ import annotations
54+
55+
import dataclasses
56+
import pathlib
57+
import re
58+
import sys
59+
60+
REPO_ROOT = pathlib.Path(__file__).resolve().parents[3]
61+
62+
# The ``go`` directive: ``go 1.25`` or ``go 1.25.0``.
63+
GO_MOD_DIRECTIVE = re.compile(r"^go\s+(\d+\.\d+(?:\.\d+)?)\s*$", re.MULTILINE)
64+
# Setup Go step: ``go-version: 1.25`` (optionally quoted).
65+
WORKFLOW_GO_VERSION = re.compile(r"^\s*go-version:\s*[\"']?(\d+\.\d+(?:\.\d+)?)[\"']?\s*$", re.MULTILINE)
66+
# Builder image default: ``golang:1.25-alpine``.
67+
GOLANG_BUILDER_IMAGE = re.compile(r"golang:(\d+\.\d+(?:\.\d+)?)-alpine")
68+
# prek ``default_language_version`` entry: `` golang: 1.25.0``.
69+
PREK_GOLANG_VERSION = re.compile(r"^\s*golang:\s*(\d+\.\d+(?:\.\d+)?)\s*$", re.MULTILINE)
70+
71+
72+
@dataclasses.dataclass
73+
class VersionSite:
74+
"""A single location that pins the Go toolchain version."""
75+
76+
label: str
77+
path: pathlib.Path
78+
pattern: re.Pattern[str]
79+
is_source_of_truth: bool = False
80+
81+
def extract(self) -> str | None:
82+
"""Return the captured version string, or ``None`` if the pattern is missing."""
83+
if not self.path.exists():
84+
return None
85+
if m := self.pattern.search(self.path.read_text()):
86+
return m.group(1)
87+
return None
88+
89+
90+
def major_minor(version: str) -> str:
91+
"""Reduce ``1.25`` / ``1.25.0`` to its ``major.minor`` (``1.25``)."""
92+
parts = version.split(".")
93+
return ".".join(parts[:2])
94+
95+
96+
def build_sites(repo_root: pathlib.Path) -> list[VersionSite]:
97+
"""Build the list of Go-version pin sites, resolved against ``repo_root``."""
98+
workflows = repo_root / ".github" / "workflows"
99+
return [
100+
VersionSite(
101+
label="go-sdk/go.mod (go directive)",
102+
path=repo_root / "go-sdk" / "go.mod",
103+
pattern=GO_MOD_DIRECTIVE,
104+
is_source_of_truth=True,
105+
),
106+
VersionSite(
107+
label="kubernetes-tests/lang_sdk/go_example/go.mod (go directive)",
108+
path=repo_root / "kubernetes-tests" / "lang_sdk" / "go_example" / "go.mod",
109+
pattern=GO_MOD_DIRECTIVE,
110+
),
111+
VersionSite(
112+
label=".github/workflows/ci-amd.yml (Setup Go go-version)",
113+
path=workflows / "ci-amd.yml",
114+
pattern=WORKFLOW_GO_VERSION,
115+
),
116+
VersionSite(
117+
label=".github/workflows/ci-arm.yml (Setup Go go-version)",
118+
path=workflows / "ci-arm.yml",
119+
pattern=WORKFLOW_GO_VERSION,
120+
),
121+
VersionSite(
122+
label="airflow-e2e-tests/.../constants.py (GO_BUILDER_IMAGE)",
123+
path=repo_root / "airflow-e2e-tests" / "tests" / "airflow_e2e_tests" / "constants.py",
124+
pattern=GOLANG_BUILDER_IMAGE,
125+
),
126+
VersionSite(
127+
label="dev/breeze/.../kubernetes_commands.py (LANG_SDK_GO_BUILDER_IMAGE)",
128+
path=repo_root
129+
/ "dev"
130+
/ "breeze"
131+
/ "src"
132+
/ "airflow_breeze"
133+
/ "commands"
134+
/ "kubernetes_commands.py",
135+
pattern=GOLANG_BUILDER_IMAGE,
136+
),
137+
VersionSite(
138+
label="go-sdk/.pre-commit-config.yaml (default_language_version.golang)",
139+
path=repo_root / "go-sdk" / ".pre-commit-config.yaml",
140+
pattern=PREK_GOLANG_VERSION,
141+
),
142+
VersionSite(
143+
label=".pre-commit-config.yaml (default_language_version.golang)",
144+
path=repo_root / ".pre-commit-config.yaml",
145+
pattern=PREK_GOLANG_VERSION,
146+
),
147+
]
148+
149+
150+
def check_sync(sites: list[VersionSite], repo_root: pathlib.Path) -> tuple[int, str]:
151+
"""Compare every site against the source of truth. Returns ``(exit_code, report)``."""
152+
results = [(site, site.extract()) for site in sites]
153+
154+
if missing := [site for site, version in results if version is None]:
155+
lines = [f"ERROR: Go version pin not found in {site.path.relative_to(repo_root)}" for site in missing]
156+
lines += [f" (expected pattern {site.pattern.pattern!r})" for site in missing]
157+
return 1, "\n".join(lines)
158+
159+
source = next(site for site, _ in results if site.is_source_of_truth)
160+
expected = major_minor(source.extract()) # type: ignore[arg-type]
161+
162+
drifted = [(site, version) for site, version in results if major_minor(version) != expected] # type: ignore[arg-type]
163+
if not drifted:
164+
return 0, f"OK: Go version is consistently {expected} across all {len(sites)} pin sites."
165+
166+
col = max(len(site.label) for site, _ in results)
167+
lines = [
168+
f"ERROR: Go version drifted from the source of truth ({source.label} = {expected}).",
169+
"",
170+
f" {'PIN SITE':<{col}} VERSION",
171+
]
172+
for site, version in results:
173+
marker = (
174+
" <- source of truth"
175+
if site.is_source_of_truth
176+
else (
177+
" <- DRIFT" if major_minor(version) != expected else "" # type: ignore[arg-type]
178+
)
179+
)
180+
lines.append(f" {site.label:<{col}} {version}{marker}")
181+
lines += [
182+
"",
183+
f"Bump the drifting site(s) to {expected} to match {source.label},",
184+
"or, if you intentionally changed the source of truth, update the other pins to match.",
185+
"After changing a go.mod, run its `go mod tidy` so go.sum stays consistent.",
186+
]
187+
return 1, "\n".join(lines)
188+
189+
190+
def main() -> int:
191+
exit_code, report = check_sync(build_sites(REPO_ROOT), REPO_ROOT)
192+
print(report)
193+
return exit_code
194+
195+
196+
if __name__ == "__main__":
197+
sys.exit(main())
Lines changed: 130 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,130 @@
1+
# Licensed to the Apache Software Foundation (ASF) under one
2+
# or more contributor license agreements. See the NOTICE file
3+
# distributed with this work for additional information
4+
# regarding copyright ownership. The ASF licenses this file
5+
# to you under the Apache License, Version 2.0 (the
6+
# "License"); you may not use this file except in compliance
7+
# with the License. You may obtain a copy of the License at
8+
#
9+
# http://www.apache.org/licenses/LICENSE-2.0
10+
#
11+
# Unless required by applicable law or agreed to in writing,
12+
# software distributed under the License is distributed on an
13+
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
# KIND, either express or implied. See the License for the
15+
# specific language governing permissions and limitations
16+
# under the License.
17+
from __future__ import annotations
18+
19+
from pathlib import Path
20+
21+
import pytest
22+
from check_go_version_in_sync import build_sites, check_sync, major_minor
23+
24+
# Default (all-consistent) version at every pin site. Individual tests override
25+
# one entry to simulate drift. go.mod carries the patch suffix; the workflow and
26+
# image sites carry only major.minor — the check must treat these as equal.
27+
DEFAULT_VERSIONS = {
28+
"go_sdk_mod": "1.25.0",
29+
"go_example_mod": "1.25.0",
30+
"ci_amd": "1.25",
31+
"ci_arm": "1.25",
32+
"constants": "1.25",
33+
"kubernetes_commands": "1.25",
34+
"go_sdk_prek": "1.25.0",
35+
"root_prek": "1.25.0",
36+
}
37+
38+
39+
def _write_tree(root: Path, versions: dict[str, str]) -> None:
40+
"""Materialise a minimal repo tree with a Go version pin at each of the eight sites."""
41+
42+
def write(rel: str, content: str) -> None:
43+
path = root / rel
44+
path.parent.mkdir(parents=True, exist_ok=True)
45+
path.write_text(content)
46+
47+
write("go-sdk/go.mod", f"module github.com/apache/airflow/go-sdk\n\ngo {versions['go_sdk_mod']}\n")
48+
write(
49+
"kubernetes-tests/lang_sdk/go_example/go.mod",
50+
f"module github.com/apache/airflow/kubernetes-tests/lang_sdk/go_example\n\n"
51+
f"go {versions['go_example_mod']}\n",
52+
)
53+
for site, rel in (("ci_amd", "ci-amd.yml"), ("ci_arm", "ci-arm.yml")):
54+
write(
55+
f".github/workflows/{rel}",
56+
f" - name: Setup Go\n with:\n go-version: {versions[site]}\n",
57+
)
58+
write(
59+
"airflow-e2e-tests/tests/airflow_e2e_tests/constants.py",
60+
f'GO_BUILDER_IMAGE = os.environ.get("GO_BUILDER_IMAGE", "golang:{versions["constants"]}-alpine")\n',
61+
)
62+
write(
63+
"dev/breeze/src/airflow_breeze/commands/kubernetes_commands.py",
64+
f'LANG_SDK_GO_BUILDER_IMAGE = os.environ.get("GO_BUILDER_IMAGE", '
65+
f'"golang:{versions["kubernetes_commands"]}-alpine")\n',
66+
)
67+
for site, rel in (
68+
("go_sdk_prek", "go-sdk/.pre-commit-config.yaml"),
69+
("root_prek", ".pre-commit-config.yaml"),
70+
):
71+
write(
72+
rel,
73+
f"default_language_version:\n python: python3\n node: 22.19.0\n golang: {versions[site]}\n",
74+
)
75+
76+
77+
@pytest.mark.parametrize(
78+
("version", "expected"),
79+
[("1.25", "1.25"), ("1.25.0", "1.25"), ("1.24.6", "1.24")],
80+
)
81+
def test_major_minor(version: str, expected: str):
82+
assert major_minor(version) == expected
83+
84+
85+
def test_all_sites_consistent(tmp_path: Path):
86+
_write_tree(tmp_path, DEFAULT_VERSIONS)
87+
exit_code, report = check_sync(build_sites(tmp_path), tmp_path)
88+
assert exit_code == 0
89+
assert "consistently 1.25" in report
90+
91+
92+
def test_go_mod_patch_suffix_matches_major_minor_pins(tmp_path: Path):
93+
"""go.mod may pin 1.25.0 while the workflow/image sites pin 1.25 — that is in sync."""
94+
_write_tree(tmp_path, {**DEFAULT_VERSIONS, "go_sdk_mod": "1.25.0", "ci_amd": "1.25"})
95+
exit_code, _ = check_sync(build_sites(tmp_path), tmp_path)
96+
assert exit_code == 0
97+
98+
99+
@pytest.mark.parametrize(
100+
"drifted_site",
101+
["go_example_mod", "ci_amd", "ci_arm", "constants", "kubernetes_commands", "go_sdk_prek", "root_prek"],
102+
)
103+
def test_drift_in_any_derived_site_is_flagged(tmp_path: Path, drifted_site: str):
104+
"""A single derived site left on the old minor version must fail the check."""
105+
_write_tree(
106+
tmp_path, {**DEFAULT_VERSIONS, drifted_site: "1.24" if "mod" not in drifted_site else "1.24.6"}
107+
)
108+
exit_code, report = check_sync(build_sites(tmp_path), tmp_path)
109+
assert exit_code == 1
110+
assert "drifted from the source of truth" in report
111+
assert "<- DRIFT" in report
112+
113+
114+
def test_source_of_truth_bump_flags_all_stale_sites(tmp_path: Path):
115+
"""Bumping only go-sdk/go.mod (the source of truth) flags every other site — the #69214 scenario."""
116+
_write_tree(tmp_path, {**DEFAULT_VERSIONS, "go_sdk_mod": "1.26.0"})
117+
exit_code, report = check_sync(build_sites(tmp_path), tmp_path)
118+
assert exit_code == 1
119+
# All seven non-source sites are still on the old version -> all flagged.
120+
assert report.count("<- DRIFT") == 7
121+
assert "<- source of truth" in report
122+
123+
124+
def test_missing_pin_is_reported(tmp_path: Path):
125+
_write_tree(tmp_path, DEFAULT_VERSIONS)
126+
(tmp_path / "go-sdk" / "go.mod").write_text("module github.com/apache/airflow/go-sdk\n")
127+
exit_code, report = check_sync(build_sites(tmp_path), tmp_path)
128+
assert exit_code == 1
129+
assert "Go version pin not found" in report
130+
assert "go-sdk/go.mod" in report

0 commit comments

Comments
 (0)