Skip to content

Commit 7da73c8

Browse files
authored
Make the constraints check follow the cooldown rules the constraints use (#73316)
After every provider release wave the `Deps *:constraints` jobs doubled for three or four days and then recovered on their own. The constraints are resolved under uv's `exclude-newer` cooldown with the per-package overrides in the root pyproject.toml, where Airflow's own distributions are exempt, so a fresh provider wave lands in the constraints the same day. The check had its own 4-day cooldown applied to every package, so for those days the pin was newer than "latest" and a plain equality check counted it as outdated. With `--explain-why` every such package then cost a full `uv sync` that tried to pin it to the *older* version and reported that the pin did not take effect. On 2026-09-15 that was 37 providers and about 12 extra minutes per job, with nothing to act on. The check now reads `[tool.uv.exclude-newer-package]` and applies the same rules: no cooldown for exempt distributions, a moved cutoff where one is configured. A pin that is still ahead of "latest" counts as up to date, so no explanation runs for it either. Generated-by: Claude Opus 5
1 parent 47d2827 commit 7da73c8

2 files changed

Lines changed: 139 additions & 9 deletions

File tree

‎dev/breeze/src/airflow_breeze/utils/constraints_version_check.py‎

Lines changed: 68 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -38,6 +38,7 @@
3838
from airflow_breeze.utils.console import Output, console_print
3939
from airflow_breeze.utils.docker_command_utils import execute_command_in_shell
4040
from airflow_breeze.utils.github import download_constraints_file
41+
from airflow_breeze.utils.packages import load_pyproject_toml
4142
from airflow_breeze.utils.parallel import get_temp_file_name
4243
from airflow_breeze.utils.path_utils import AIRFLOW_ROOT_PATH
4344
from airflow_breeze.utils.shared_options import get_verbose
@@ -86,6 +87,15 @@ def is_valid_version(version_str: str, latest_version: Version) -> bool:
8687
return False
8788

8889

90+
def is_newer_version(candidate: str, reference: str) -> bool:
91+
from packaging import version
92+
93+
try:
94+
return version.parse(candidate) > version.parse(reference)
95+
except version.InvalidVersion:
96+
return False
97+
98+
8999
def count_versions_between(releases: dict[str, Any], current_version: str, latest_version: str):
90100
from packaging import version
91101

@@ -185,14 +195,13 @@ def should_show_package(releases, latest_version, constraints_date, mode, is_lat
185195
return True
186196

187197

188-
def get_latest_version_with_cooldown(releases: dict[str, Any], cooldown_days: int) -> str | None:
189-
"""Find the latest non-prerelease version whose release date is outside the cooldown period.
198+
def get_latest_version_with_cooldown(releases: dict[str, Any], cutoff: datetime | None) -> str | None:
199+
"""Find the latest non-prerelease version uploaded no later than ``cutoff`` (any time if None).
190200
191201
Returns the version string, or None if no version qualifies.
192202
"""
193203
from packaging import version
194204

195-
cutoff = datetime.now() - timedelta(days=cooldown_days)
196205
candidates: list[tuple[version.Version, str]] = []
197206
for v, release_files in releases.items():
198207
if not release_files:
@@ -211,14 +220,54 @@ def get_latest_version_with_cooldown(releases: dict[str, Any], cooldown_days: in
211220
).replace(tzinfo=None)
212221
except (KeyError, IndexError, ValueError):
213222
continue
214-
if upload_time <= cutoff:
223+
if cutoff is None or upload_time <= cutoff:
215224
candidates.append((parsed_v, v))
216225
if not candidates:
217226
return None
218227
candidates.sort(key=lambda x: x[0], reverse=True)
219228
return candidates[0][1]
220229

221230

231+
def load_cooldown_overrides() -> dict[str, bool | str]:
232+
"""Per-package ``exclude-newer-package`` entries of the root pyproject.toml, by canonical name.
233+
234+
The constraints are resolved under these rules (Airflow's own distributions are exempt from
235+
the cooldown, a few packages have a moved cutoff), so "latest" has to be read under the same
236+
rules or the check flags pins that resolved exactly as configured.
237+
"""
238+
from packaging.utils import canonicalize_name
239+
240+
tool_uv = load_pyproject_toml(AIRFLOW_ROOT_PATH / "pyproject.toml").get("tool", {}).get("uv", {})
241+
return {
242+
canonicalize_name(name): value for name, value in tool_uv.get("exclude-newer-package", {}).items()
243+
}
244+
245+
246+
_DURATION_RE = re.compile(r"^\s*(\d+(?:\.\d+)?)\s*(minute|hour|day)s?\s*$")
247+
248+
249+
def get_release_cutoff(pkg: str, cooldown_days: int, overrides: dict[str, bool | str]) -> datetime | None:
250+
"""Return the newest upload time still eligible as ``latest`` for ``pkg``, None for no limit.
251+
252+
Mirrors the ``exclude-newer-package`` values uv accepts: ``false`` lifts the cooldown, a
253+
timestamp fixes the cutoff, a duration such as ``"12 hours"`` replaces the window.
254+
"""
255+
from packaging.utils import canonicalize_name
256+
257+
override = overrides.get(canonicalize_name(pkg))
258+
if override is False:
259+
return None
260+
if isinstance(override, str):
261+
if match := _DURATION_RE.match(override):
262+
amount, unit = float(match.group(1)), match.group(2)
263+
return datetime.now() - timedelta(**{f"{unit}s": amount})
264+
try:
265+
return datetime.fromisoformat(override.replace("Z", "+00:00")).replace(tzinfo=None)
266+
except ValueError:
267+
console_print(f"[yellow]Ignoring unparsable exclude-newer-package value for {pkg}: {override}[/]")
268+
return datetime.now() - timedelta(days=cooldown_days)
269+
270+
222271
def get_first_newer_release_date_str(releases, current_version):
223272
from packaging import version
224273

@@ -265,7 +314,12 @@ def constraints_version_check(
265314
):
266315
console_print(f"[bold cyan]Python version:[/] [white]{python}[/]")
267316
console_print(f"[bold cyan]Constraints mode:[/] [white]{airflow_constraints_mode}[/]")
268-
console_print(f"[bold cyan]Cooldown period:[/] [white]{cooldown_days} days[/]\n")
317+
cooldown_overrides = load_cooldown_overrides()
318+
exempt_count = sum(1 for value in cooldown_overrides.values() if value is False)
319+
console_print(
320+
f"[bold cyan]Cooldown period:[/] [white]{cooldown_days} days[/] "
321+
f"[white]({exempt_count} distributions exempt via exclude-newer-package)[/]\n"
322+
)
269323
with tempfile.TemporaryDirectory() as temp_dir:
270324
constraints_file = Path(temp_dir) / "constraints.txt"
271325
download_constraints_file(
@@ -302,6 +356,7 @@ def constraints_version_check(
302356
airflow_constraints_mode=airflow_constraints_mode,
303357
github_repository=github_repository,
304358
cooldown_days=cooldown_days,
359+
cooldown_overrides=cooldown_overrides,
305360
)
306361

307362
print_table_footer(
@@ -445,6 +500,7 @@ def process_packages(
445500
airflow_constraints_mode: str,
446501
github_repository: str | None,
447502
cooldown_days: int = 4,
503+
cooldown_overrides: dict[str, bool | str] | None = None,
448504
) -> tuple[int, int, list[str], dict[str, int]]:
449505
def fetch_pypi_data(pkg: str) -> dict:
450506
pypi_url = f"https://pypi.org/pypi/{pkg}/json"
@@ -472,8 +528,13 @@ def get_release_dates(releases: dict, version: str) -> str:
472528
try:
473529
data = fetch_pypi_data(pkg)
474530
releases = data["releases"]
475-
latest_version_with_cooldown = get_latest_version_with_cooldown(releases, cooldown_days)
531+
cutoff = get_release_cutoff(pkg, cooldown_days, cooldown_overrides or {})
532+
latest_version_with_cooldown = get_latest_version_with_cooldown(releases, cutoff)
476533
latest_version = latest_version_with_cooldown or data["info"]["version"]
534+
if is_newer_version(pinned_version, latest_version):
535+
# The pin can still be ahead of `latest` (an override dropped after the constraints
536+
# resolved with it): nothing to upgrade to, so nothing to explain.
537+
latest_version = pinned_version
477538
latest_release_date = get_release_dates(releases, latest_version)
478539
constraint_release_date = get_release_dates(releases, pinned_version)
479540
is_latest_version = pinned_version == latest_version
@@ -491,7 +552,7 @@ def get_release_dates(releases: dict, version: str) -> str:
491552
format_str=format_str,
492553
is_latest_version=is_latest_version,
493554
versions_behind_str=versions_behind_str,
494-
cooldown_days=cooldown_days,
555+
cooldown_days=0 if cutoff is None else cooldown_days,
495556
)
496557
status_counts[status_category] += 1
497558
if not is_latest_version:

‎dev/breeze/tests/test_constraints_version_check.py‎

Lines changed: 71 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -18,13 +18,15 @@
1818

1919
import contextlib
2020
import json
21+
from datetime import datetime
2122
from pathlib import Path
2223
from unittest import mock
2324

2425
import pytest
2526

2627
from airflow_breeze.utils.constraints_version_check import (
2728
explain_package_upgrade,
29+
get_release_cutoff,
2830
get_table_format,
2931
process_packages,
3032
)
@@ -33,10 +35,15 @@
3335

3436
# Old enough that the default 4-day cooldown never filters these releases out.
3537
OLD_UPLOAD_TIME = "2020-01-01T00:00:00.000000Z"
38+
# Far enough ahead to sit inside any cooldown window, whenever the test runs.
39+
FRESH_UPLOAD_TIME = "2099-01-01T00:00:00.000000Z"
3640

3741

38-
def _pypi_payload(latest: str, *versions: str) -> bytes:
39-
releases = {v: [{"upload_time_iso_8601": OLD_UPLOAD_TIME, "yanked": False}] for v in versions}
42+
def _pypi_payload(latest: str, *versions: str, fresh: str | None = None) -> bytes:
43+
releases = {
44+
v: [{"upload_time_iso_8601": FRESH_UPLOAD_TIME if v == fresh else OLD_UPLOAD_TIME, "yanked": False}]
45+
for v in versions
46+
}
4047
return json.dumps({"info": {"version": latest}, "releases": releases}).encode()
4148

4249

@@ -105,6 +112,68 @@ def test_baseline_is_not_resolved_without_explain_why(mock_baseline, mock_explai
105112
mock_baseline.assert_not_called()
106113

107114

115+
@mock.patch(f"{MODULE}.explain_package_upgrade")
116+
@mock.patch(f"{MODULE}.resolve_baseline_versions")
117+
@mock.patch(f"{MODULE}.get_latest_version_with_cooldown", return_value="1.0.0")
118+
def test_pin_newer_than_cooldown_latest_is_up_to_date(mock_cooldown, mock_baseline, mock_explain, pypi):
119+
# Constraints already moved to 2.0.0 while the cooldown still reports 1.0.0 as latest.
120+
outdated_count, _, explanations, status_counts = _run_process_packages([("pkg-a", "2.0.0")])
121+
122+
assert outdated_count == 0
123+
assert explanations == []
124+
assert status_counts["ok"] == 1
125+
mock_explain.assert_not_called()
126+
mock_baseline.assert_not_called()
127+
128+
129+
@pytest.mark.parametrize(
130+
("overrides", "expected_outdated"),
131+
[
132+
pytest.param({}, 0, id="third-party release inside the cooldown is not latest yet"),
133+
pytest.param({"pkg-a": False}, 1, id="exempt distribution counts its fresh release right away"),
134+
],
135+
)
136+
@mock.patch(f"{MODULE}.explain_package_upgrade", return_value="explanation")
137+
@mock.patch(f"{MODULE}.resolve_baseline_versions", return_value=("baseline log", {}))
138+
def test_cooldown_follows_exclude_newer_package_overrides(
139+
mock_baseline, mock_explain, monkeypatch, overrides, expected_outdated
140+
):
141+
def fake_urlopen(_url):
142+
response = mock.MagicMock()
143+
response.read.return_value = _pypi_payload("2.0.0", "1.0.0", "2.0.0", fresh="2.0.0")
144+
return contextlib.nullcontext(response)
145+
146+
monkeypatch.setattr(f"{MODULE}.urllib.request.urlopen", fake_urlopen)
147+
col_widths, format_str, _, _ = get_table_format([("pkg-a", "1.0.0")])
148+
149+
outdated_count, _, _, _ = process_packages(
150+
packages=[("pkg-a", "1.0.0")],
151+
constraints_date=None,
152+
mode="full",
153+
explain_why=True,
154+
col_widths=col_widths,
155+
format_str=format_str,
156+
python_version="3.11",
157+
airflow_constraints_mode="constraints",
158+
github_repository="apache/airflow",
159+
cooldown_overrides=overrides,
160+
)
161+
162+
assert outdated_count == expected_outdated
163+
assert mock_explain.call_count == expected_outdated
164+
165+
166+
@pytest.mark.parametrize(
167+
("override", "expected"),
168+
[
169+
pytest.param(False, None, id="false lifts the cooldown"),
170+
pytest.param("2026-08-13T00:00:00Z", datetime(2026, 8, 13), id="timestamp fixes the cutoff"),
171+
],
172+
)
173+
def test_release_cutoff_honours_override_shapes(override, expected):
174+
assert get_release_cutoff("Pkg_A", 4, {"pkg-a": override}) == expected
175+
176+
108177
@mock.patch(f"{MODULE}.update_pyproject_dependency")
109178
@mock.patch(f"{MODULE}.preserve_files")
110179
@mock.patch(f"{MODULE}.sync_and_freeze")

0 commit comments

Comments
 (0)