diff --git a/dev/breeze/src/airflow_breeze/utils/constraints_version_check.py b/dev/breeze/src/airflow_breeze/utils/constraints_version_check.py index d5ee2f6cee8e5..10c485fdd618e 100755 --- a/dev/breeze/src/airflow_breeze/utils/constraints_version_check.py +++ b/dev/breeze/src/airflow_breeze/utils/constraints_version_check.py @@ -38,6 +38,7 @@ from airflow_breeze.utils.console import Output, console_print from airflow_breeze.utils.docker_command_utils import execute_command_in_shell from airflow_breeze.utils.github import download_constraints_file +from airflow_breeze.utils.packages import load_pyproject_toml from airflow_breeze.utils.parallel import get_temp_file_name from airflow_breeze.utils.path_utils import AIRFLOW_ROOT_PATH from airflow_breeze.utils.shared_options import get_verbose @@ -86,6 +87,15 @@ def is_valid_version(version_str: str, latest_version: Version) -> bool: return False +def is_newer_version(candidate: str, reference: str) -> bool: + from packaging import version + + try: + return version.parse(candidate) > version.parse(reference) + except version.InvalidVersion: + return False + + def count_versions_between(releases: dict[str, Any], current_version: str, latest_version: str): from packaging import version @@ -185,14 +195,13 @@ def should_show_package(releases, latest_version, constraints_date, mode, is_lat return True -def get_latest_version_with_cooldown(releases: dict[str, Any], cooldown_days: int) -> str | None: - """Find the latest non-prerelease version whose release date is outside the cooldown period. +def get_latest_version_with_cooldown(releases: dict[str, Any], cutoff: datetime | None) -> str | None: + """Find the latest non-prerelease version uploaded no later than ``cutoff`` (any time if None). Returns the version string, or None if no version qualifies. """ from packaging import version - cutoff = datetime.now() - timedelta(days=cooldown_days) candidates: list[tuple[version.Version, str]] = [] for v, release_files in releases.items(): if not release_files: @@ -211,7 +220,7 @@ def get_latest_version_with_cooldown(releases: dict[str, Any], cooldown_days: in ).replace(tzinfo=None) except (KeyError, IndexError, ValueError): continue - if upload_time <= cutoff: + if cutoff is None or upload_time <= cutoff: candidates.append((parsed_v, v)) if not candidates: return None @@ -219,6 +228,46 @@ def get_latest_version_with_cooldown(releases: dict[str, Any], cooldown_days: in return candidates[0][1] +def load_cooldown_overrides() -> dict[str, bool | str]: + """Per-package ``exclude-newer-package`` entries of the root pyproject.toml, by canonical name. + + The constraints are resolved under these rules (Airflow's own distributions are exempt from + the cooldown, a few packages have a moved cutoff), so "latest" has to be read under the same + rules or the check flags pins that resolved exactly as configured. + """ + from packaging.utils import canonicalize_name + + tool_uv = load_pyproject_toml(AIRFLOW_ROOT_PATH / "pyproject.toml").get("tool", {}).get("uv", {}) + return { + canonicalize_name(name): value for name, value in tool_uv.get("exclude-newer-package", {}).items() + } + + +_DURATION_RE = re.compile(r"^\s*(\d+(?:\.\d+)?)\s*(minute|hour|day)s?\s*$") + + +def get_release_cutoff(pkg: str, cooldown_days: int, overrides: dict[str, bool | str]) -> datetime | None: + """Return the newest upload time still eligible as ``latest`` for ``pkg``, None for no limit. + + Mirrors the ``exclude-newer-package`` values uv accepts: ``false`` lifts the cooldown, a + timestamp fixes the cutoff, a duration such as ``"12 hours"`` replaces the window. + """ + from packaging.utils import canonicalize_name + + override = overrides.get(canonicalize_name(pkg)) + if override is False: + return None + if isinstance(override, str): + if match := _DURATION_RE.match(override): + amount, unit = float(match.group(1)), match.group(2) + return datetime.now() - timedelta(**{f"{unit}s": amount}) + try: + return datetime.fromisoformat(override.replace("Z", "+00:00")).replace(tzinfo=None) + except ValueError: + console_print(f"[yellow]Ignoring unparsable exclude-newer-package value for {pkg}: {override}[/]") + return datetime.now() - timedelta(days=cooldown_days) + + def get_first_newer_release_date_str(releases, current_version): from packaging import version @@ -265,7 +314,12 @@ def constraints_version_check( ): console_print(f"[bold cyan]Python version:[/] [white]{python}[/]") console_print(f"[bold cyan]Constraints mode:[/] [white]{airflow_constraints_mode}[/]") - console_print(f"[bold cyan]Cooldown period:[/] [white]{cooldown_days} days[/]\n") + cooldown_overrides = load_cooldown_overrides() + exempt_count = sum(1 for value in cooldown_overrides.values() if value is False) + console_print( + f"[bold cyan]Cooldown period:[/] [white]{cooldown_days} days[/] " + f"[white]({exempt_count} distributions exempt via exclude-newer-package)[/]\n" + ) with tempfile.TemporaryDirectory() as temp_dir: constraints_file = Path(temp_dir) / "constraints.txt" download_constraints_file( @@ -302,6 +356,7 @@ def constraints_version_check( airflow_constraints_mode=airflow_constraints_mode, github_repository=github_repository, cooldown_days=cooldown_days, + cooldown_overrides=cooldown_overrides, ) print_table_footer( @@ -445,6 +500,7 @@ def process_packages( airflow_constraints_mode: str, github_repository: str | None, cooldown_days: int = 4, + cooldown_overrides: dict[str, bool | str] | None = None, ) -> tuple[int, int, list[str], dict[str, int]]: def fetch_pypi_data(pkg: str) -> dict: pypi_url = f"https://pypi.org/pypi/{pkg}/json" @@ -472,8 +528,13 @@ def get_release_dates(releases: dict, version: str) -> str: try: data = fetch_pypi_data(pkg) releases = data["releases"] - latest_version_with_cooldown = get_latest_version_with_cooldown(releases, cooldown_days) + cutoff = get_release_cutoff(pkg, cooldown_days, cooldown_overrides or {}) + latest_version_with_cooldown = get_latest_version_with_cooldown(releases, cutoff) latest_version = latest_version_with_cooldown or data["info"]["version"] + if is_newer_version(pinned_version, latest_version): + # The pin can still be ahead of `latest` (an override dropped after the constraints + # resolved with it): nothing to upgrade to, so nothing to explain. + latest_version = pinned_version latest_release_date = get_release_dates(releases, latest_version) constraint_release_date = get_release_dates(releases, pinned_version) is_latest_version = pinned_version == latest_version @@ -491,7 +552,7 @@ def get_release_dates(releases: dict, version: str) -> str: format_str=format_str, is_latest_version=is_latest_version, versions_behind_str=versions_behind_str, - cooldown_days=cooldown_days, + cooldown_days=0 if cutoff is None else cooldown_days, ) status_counts[status_category] += 1 if not is_latest_version: diff --git a/dev/breeze/tests/test_constraints_version_check.py b/dev/breeze/tests/test_constraints_version_check.py index c61f0d3f1ce9f..85856eb01fa45 100644 --- a/dev/breeze/tests/test_constraints_version_check.py +++ b/dev/breeze/tests/test_constraints_version_check.py @@ -18,6 +18,7 @@ import contextlib import json +from datetime import datetime from pathlib import Path from unittest import mock @@ -25,6 +26,7 @@ from airflow_breeze.utils.constraints_version_check import ( explain_package_upgrade, + get_release_cutoff, get_table_format, process_packages, ) @@ -33,10 +35,15 @@ # Old enough that the default 4-day cooldown never filters these releases out. OLD_UPLOAD_TIME = "2020-01-01T00:00:00.000000Z" +# Far enough ahead to sit inside any cooldown window, whenever the test runs. +FRESH_UPLOAD_TIME = "2099-01-01T00:00:00.000000Z" -def _pypi_payload(latest: str, *versions: str) -> bytes: - releases = {v: [{"upload_time_iso_8601": OLD_UPLOAD_TIME, "yanked": False}] for v in versions} +def _pypi_payload(latest: str, *versions: str, fresh: str | None = None) -> bytes: + releases = { + v: [{"upload_time_iso_8601": FRESH_UPLOAD_TIME if v == fresh else OLD_UPLOAD_TIME, "yanked": False}] + for v in versions + } return json.dumps({"info": {"version": latest}, "releases": releases}).encode() @@ -105,6 +112,68 @@ def test_baseline_is_not_resolved_without_explain_why(mock_baseline, mock_explai mock_baseline.assert_not_called() +@mock.patch(f"{MODULE}.explain_package_upgrade") +@mock.patch(f"{MODULE}.resolve_baseline_versions") +@mock.patch(f"{MODULE}.get_latest_version_with_cooldown", return_value="1.0.0") +def test_pin_newer_than_cooldown_latest_is_up_to_date(mock_cooldown, mock_baseline, mock_explain, pypi): + # Constraints already moved to 2.0.0 while the cooldown still reports 1.0.0 as latest. + outdated_count, _, explanations, status_counts = _run_process_packages([("pkg-a", "2.0.0")]) + + assert outdated_count == 0 + assert explanations == [] + assert status_counts["ok"] == 1 + mock_explain.assert_not_called() + mock_baseline.assert_not_called() + + +@pytest.mark.parametrize( + ("overrides", "expected_outdated"), + [ + pytest.param({}, 0, id="third-party release inside the cooldown is not latest yet"), + pytest.param({"pkg-a": False}, 1, id="exempt distribution counts its fresh release right away"), + ], +) +@mock.patch(f"{MODULE}.explain_package_upgrade", return_value="explanation") +@mock.patch(f"{MODULE}.resolve_baseline_versions", return_value=("baseline log", {})) +def test_cooldown_follows_exclude_newer_package_overrides( + mock_baseline, mock_explain, monkeypatch, overrides, expected_outdated +): + def fake_urlopen(_url): + response = mock.MagicMock() + response.read.return_value = _pypi_payload("2.0.0", "1.0.0", "2.0.0", fresh="2.0.0") + return contextlib.nullcontext(response) + + monkeypatch.setattr(f"{MODULE}.urllib.request.urlopen", fake_urlopen) + col_widths, format_str, _, _ = get_table_format([("pkg-a", "1.0.0")]) + + outdated_count, _, _, _ = process_packages( + packages=[("pkg-a", "1.0.0")], + constraints_date=None, + mode="full", + explain_why=True, + col_widths=col_widths, + format_str=format_str, + python_version="3.11", + airflow_constraints_mode="constraints", + github_repository="apache/airflow", + cooldown_overrides=overrides, + ) + + assert outdated_count == expected_outdated + assert mock_explain.call_count == expected_outdated + + +@pytest.mark.parametrize( + ("override", "expected"), + [ + pytest.param(False, None, id="false lifts the cooldown"), + pytest.param("2026-08-13T00:00:00Z", datetime(2026, 8, 13), id="timestamp fixes the cutoff"), + ], +) +def test_release_cutoff_honours_override_shapes(override, expected): + assert get_release_cutoff("Pkg_A", 4, {"pkg-a": override}) == expected + + @mock.patch(f"{MODULE}.update_pyproject_dependency") @mock.patch(f"{MODULE}.preserve_files") @mock.patch(f"{MODULE}.sync_and_freeze")