diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 322bdac488f4b..95bcd282d1a0d 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -279,6 +279,25 @@ repos: (?x) ^java-sdk/gradle\.properties$| ^java-sdk/sdk/schema/schema\.json$ + - id: regenerate-java-sdk-verification-metadata + name: Regenerate the Java SDK dependency verification metadata + description: "Drop trusted checksums that no Java SDK build task resolves any more" + entry: ./scripts/ci/prek/regenerate_java_sdk_verification_metadata.py + language: python + pass_filenames: false + # example/ and scala_spark_example/ are separate builds this metadata does + # not cover, so their build files are excluded. + files: > + (?x) + ^java-sdk/build\.gradle\.kts$| + ^java-sdk/buildSrc/.*$| + ^java-sdk/gradle\.properties$| + ^java-sdk/gradle/libs\.versions\.toml$| + ^java-sdk/gradle/verification-metadata\.xml$| + ^java-sdk/gradle/wrapper/gradle-wrapper\.properties$| + ^java-sdk/settings\.gradle\.kts$| + ^scripts/ci/prek/regenerate_java_sdk_verification_metadata\.py$| + ^java-sdk/(?!example/|scala_spark_example/)[^/]+/(?:build\.gradle\.kts|gradle\.properties)$ - id: update-java-sdk-readme-matrix name: Update the Java SDK compatibility matrix in java-sdk/README.md and Dokka module doc entry: ./scripts/ci/prek/update_java_sdk_readme_matrix.py diff --git a/dev/breeze/doc/ci/04_selective_checks.md b/dev/breeze/doc/ci/04_selective_checks.md index 27e6039996fa9..489d60a8f7347 100644 --- a/dev/breeze/doc/ci/04_selective_checks.md +++ b/dev/breeze/doc/ci/04_selective_checks.md @@ -519,9 +519,11 @@ when some files are not changed. Those are the rules implemented: type errors (see #68919) * if no `All Python files` changed - `flynt` check is skipped * if no `Helm files` changed - `lint-helm-chart` check is skipped - * if no `Java SDK files` changed - `ktlint` check is skipped (it runs the java-sdk Gradle - wrapper, which downloads the Gradle distribution, so we avoid that download on PRs that do - not touch `java-sdk/`) + * if no `Java SDK files` changed - `ktlint` and + `regenerate-java-sdk-verification-metadata` checks are skipped (both run the java-sdk + Gradle wrapper, which downloads the Gradle distribution, and the latter additionally + resolves the whole Java SDK dependency graph from Maven Central, so we avoid those + downloads on PRs that do not touch `java-sdk/`) * if no `TS SDK files` (`ts-sdk/`) changed - `check-ts-sdk-supervisor-schema` check is skipped (it regenerates and diffs the generated ts-sdk file; a change to the supervisor wire schema alone deliberately does not trigger it - regenerating the ts-sdk types is diff --git a/dev/breeze/src/airflow_breeze/utils/selective_checks.py b/dev/breeze/src/airflow_breeze/utils/selective_checks.py index d831e569227a7..f278d044bc7d6 100644 --- a/dev/breeze/src/airflow_breeze/utils/selective_checks.py +++ b/dev/breeze/src/airflow_breeze/utils/selective_checks.py @@ -1729,6 +1729,10 @@ def skip_prek_hooks(self) -> str: # on a cold cache. Skip it when no java-sdk files changed so unrelated PRs do not # depend on that (intermittently failing) download. prek_hooks_to_skip.add("ktlint") + # Rewriting the verification metadata resolves the entire Java SDK dependency graph + # from Maven Central. Skip it when no java-sdk files changed so unrelated PRs do not + # depend on that resolution. + prek_hooks_to_skip.add("regenerate-java-sdk-verification-metadata") if not self._matching_files(FileGroupForCi.TS_SDK_FILES, CI_FILE_GROUP_MATCHES): # This hook regenerates ts-sdk/src/generated/supervisor.ts from the wire schema and # diffs it. Schema-only changes deliberately do not trigger it: regenerating the diff --git a/dev/breeze/tests/test_selective_checks.py b/dev/breeze/tests/test_selective_checks.py index 0e45e016b48a0..42ac80cd7eebf 100644 --- a/dev/breeze/tests/test_selective_checks.py +++ b/dev/breeze/tests/test_selective_checks.py @@ -114,7 +114,7 @@ "mypy-shared-module_loading,mypy-shared-observability,mypy-shared-plugins_manager," "mypy-shared-providers_discovery,mypy-shared-secrets_backend,mypy-shared-secrets_masker," "mypy-shared-serialization,mypy-shared-state,mypy-shared-template_rendering,mypy-shared-timezones,mypy-task-sdk,mypy-task-sdk-integration-tests," - "ts-compile-lint-simple-auth-manager-ui,ts-compile-lint-ui,update-uv-lock" + "regenerate-java-sdk-verification-metadata,ts-compile-lint-simple-auth-manager-ui,ts-compile-lint-ui,update-uv-lock" ) ALL_SKIPPED_COMMITS_BY_DEFAULT_ON_ALL_TESTS_NEEDED = "identity,update-uv-lock" @@ -128,7 +128,7 @@ "mypy-shared-observability,mypy-shared-plugins_manager,mypy-shared-providers_discovery," "mypy-shared-secrets_backend,mypy-shared-secrets_masker,mypy-shared-serialization," "mypy-shared-state,mypy-shared-template_rendering,mypy-shared-timezones,mypy-task-sdk," - "mypy-task-sdk-integration-tests,update-uv-lock" + "mypy-task-sdk-integration-tests,regenerate-java-sdk-verification-metadata,update-uv-lock" ) ALL_SKIPPED_COMMITS_IF_NO_UI = ( @@ -139,7 +139,7 @@ "mypy-shared-module_loading,mypy-shared-observability,mypy-shared-plugins_manager," "mypy-shared-providers_discovery,mypy-shared-secrets_backend,mypy-shared-secrets_masker," "mypy-shared-serialization,mypy-shared-state,mypy-shared-template_rendering,mypy-shared-timezones,mypy-task-sdk,mypy-task-sdk-integration-tests," - "ts-compile-lint-simple-auth-manager-ui,ts-compile-lint-ui,update-uv-lock" + "regenerate-java-sdk-verification-metadata,ts-compile-lint-simple-auth-manager-ui,ts-compile-lint-ui,update-uv-lock" ) ALL_SKIPPED_COMMITS_IF_NO_HELM_TESTS = ( "check-ts-sdk-supervisor-schema,identity,ktlint,lint-helm-chart," @@ -149,7 +149,7 @@ "mypy-shared-configuration,mypy-shared-dagnode,mypy-shared-listeners,mypy-shared-logging," "mypy-shared-module_loading,mypy-shared-observability,mypy-shared-plugins_manager," "mypy-shared-providers_discovery,mypy-shared-secrets_backend,mypy-shared-secrets_masker," - "mypy-shared-serialization,mypy-shared-state,mypy-shared-template_rendering,mypy-shared-timezones,mypy-task-sdk,mypy-task-sdk-integration-tests,update-uv-lock" + "mypy-shared-serialization,mypy-shared-state,mypy-shared-template_rendering,mypy-shared-timezones,mypy-task-sdk,mypy-task-sdk-integration-tests,regenerate-java-sdk-verification-metadata,update-uv-lock" ) ALL_SKIPPED_COMMITS_IF_NO_UI_AND_HELM_TESTS = ( @@ -161,7 +161,7 @@ "mypy-shared-module_loading,mypy-shared-observability,mypy-shared-plugins_manager," "mypy-shared-providers_discovery,mypy-shared-secrets_backend,mypy-shared-secrets_masker," "mypy-shared-serialization,mypy-shared-state,mypy-shared-template_rendering,mypy-shared-timezones,mypy-task-sdk,mypy-task-sdk-integration-tests," - "ts-compile-lint-simple-auth-manager-ui,ts-compile-lint-ui,update-uv-lock" + "regenerate-java-sdk-verification-metadata,ts-compile-lint-simple-auth-manager-ui,ts-compile-lint-ui,update-uv-lock" ) # API source/test change with NO OpenAPI spec change: the full matrix is no longer @@ -177,7 +177,7 @@ "mypy-shared-providers_discovery,mypy-shared-secrets_backend,mypy-shared-secrets_masker," "mypy-shared-serialization,mypy-shared-state,mypy-shared-template_rendering,mypy-shared-timezones," "mypy-task-sdk,mypy-task-sdk-integration-tests," - "ts-compile-lint-simple-auth-manager-ui,ts-compile-lint-ui,update-uv-lock" + "regenerate-java-sdk-verification-metadata,ts-compile-lint-simple-auth-manager-ui,ts-compile-lint-ui,update-uv-lock" ) ALL_SKIPPED_COMMITS_IF_NO_PROVIDERS_AND_UI = ( @@ -189,7 +189,7 @@ "mypy-shared-module_loading,mypy-shared-observability,mypy-shared-plugins_manager," "mypy-shared-providers_discovery,mypy-shared-secrets_backend,mypy-shared-secrets_masker," "mypy-shared-serialization,mypy-shared-state,mypy-shared-template_rendering,mypy-shared-timezones,mypy-task-sdk,mypy-task-sdk-integration-tests," - "ts-compile-lint-simple-auth-manager-ui,ts-compile-lint-ui,update-uv-lock" + "regenerate-java-sdk-verification-metadata,ts-compile-lint-simple-auth-manager-ui,ts-compile-lint-ui,update-uv-lock" ) ALL_SKIPPED_COMMITS_IF_NO_PROVIDERS = ( @@ -201,7 +201,7 @@ "mypy-shared-module_loading,mypy-shared-observability,mypy-shared-plugins_manager," "mypy-shared-providers_discovery,mypy-shared-secrets_backend,mypy-shared-secrets_masker," "mypy-shared-serialization,mypy-shared-state,mypy-shared-template_rendering,mypy-shared-timezones,mypy-task-sdk,mypy-task-sdk-integration-tests," - "ts-compile-lint-simple-auth-manager-ui,ts-compile-lint-ui,update-uv-lock" + "regenerate-java-sdk-verification-metadata,ts-compile-lint-simple-auth-manager-ui,ts-compile-lint-ui,update-uv-lock" ) @@ -214,7 +214,7 @@ "mypy-shared-module_loading,mypy-shared-observability,mypy-shared-plugins_manager," "mypy-shared-providers_discovery,mypy-shared-secrets_backend,mypy-shared-secrets_masker," "mypy-shared-serialization,mypy-shared-state,mypy-shared-template_rendering,mypy-shared-timezones,mypy-task-sdk,mypy-task-sdk-integration-tests," - "ts-compile-lint-simple-auth-manager-ui,ts-compile-lint-ui,update-uv-lock" + "regenerate-java-sdk-verification-metadata,ts-compile-lint-simple-auth-manager-ui,ts-compile-lint-ui,update-uv-lock" ) ALL_SKIPPED_COMMITS_IF_NO_CODE_PROVIDERS_AND_HELM_TESTS = ( @@ -225,7 +225,7 @@ "mypy-shared-configuration,mypy-shared-dagnode,mypy-shared-listeners,mypy-shared-logging," "mypy-shared-module_loading,mypy-shared-observability,mypy-shared-plugins_manager," "mypy-shared-providers_discovery,mypy-shared-secrets_backend,mypy-shared-secrets_masker," - "mypy-shared-serialization,mypy-shared-state,mypy-shared-template_rendering,mypy-shared-timezones,mypy-task-sdk,mypy-task-sdk-integration-tests,update-uv-lock" + "mypy-shared-serialization,mypy-shared-state,mypy-shared-template_rendering,mypy-shared-timezones,mypy-task-sdk,mypy-task-sdk-integration-tests,regenerate-java-sdk-verification-metadata,update-uv-lock" ) ALL_SKIPPED_COMMITS_IF_NOT_IMPORTANT_FILES_CHANGED = ( @@ -237,7 +237,7 @@ "mypy-shared-module_loading,mypy-shared-observability,mypy-shared-plugins_manager," "mypy-shared-providers_discovery,mypy-shared-secrets_backend,mypy-shared-secrets_masker," "mypy-shared-serialization,mypy-shared-state,mypy-shared-template_rendering,mypy-shared-timezones,mypy-task-sdk,mypy-task-sdk-integration-tests," - "ts-compile-lint-simple-auth-manager-ui,ts-compile-lint-ui,update-uv-lock" + "regenerate-java-sdk-verification-metadata,ts-compile-lint-simple-auth-manager-ui,ts-compile-lint-ui,update-uv-lock" ) @@ -482,7 +482,7 @@ def assert_outputs_are_printed(expected_outputs: dict[str, str], stderr: str): "mypy-shared-module_loading,mypy-shared-observability,mypy-shared-plugins_manager," "mypy-shared-providers_discovery,mypy-shared-secrets_backend,mypy-shared-secrets_masker," "mypy-shared-serialization,mypy-shared-state,mypy-shared-template_rendering,mypy-shared-timezones,mypy-task-sdk,mypy-task-sdk-integration-tests," - "ts-compile-lint-simple-auth-manager-ui,ts-compile-lint-ui,update-uv-lock" + "regenerate-java-sdk-verification-metadata,ts-compile-lint-simple-auth-manager-ui,ts-compile-lint-ui,update-uv-lock" ), "upgrade-to-newer-dependencies": "false", "core-test-types-list-as-strings-in-json": json.dumps( @@ -529,7 +529,7 @@ def assert_outputs_are_printed(expected_outputs: dict[str, str], stderr: str): "mypy-shared-providers_discovery,mypy-shared-secrets_backend,mypy-shared-secrets_masker," "mypy-shared-serialization,mypy-shared-state,mypy-shared-template_rendering," "mypy-shared-timezones,mypy-task-sdk,mypy-task-sdk-integration-tests," - "ts-compile-lint-simple-auth-manager-ui,ts-compile-lint-ui,update-uv-lock" + "regenerate-java-sdk-verification-metadata,ts-compile-lint-simple-auth-manager-ui,ts-compile-lint-ui,update-uv-lock" ), "upgrade-to-newer-dependencies": "false", "core-test-types-list-as-strings-in-json": json.dumps( @@ -779,7 +779,7 @@ def assert_outputs_are_printed(expected_outputs: dict[str, str], stderr: str): "mypy-shared-module_loading,mypy-shared-observability,mypy-shared-plugins_manager," "mypy-shared-providers_discovery,mypy-shared-secrets_backend,mypy-shared-secrets_masker," "mypy-shared-serialization,mypy-shared-state,mypy-shared-template_rendering,mypy-shared-timezones,mypy-task-sdk-integration-tests," - "ts-compile-lint-simple-auth-manager-ui,ts-compile-lint-ui,update-uv-lock" + "regenerate-java-sdk-verification-metadata,ts-compile-lint-simple-auth-manager-ui,ts-compile-lint-ui,update-uv-lock" ), "skip-providers-tests": "false", "upgrade-to-newer-dependencies": "false", @@ -817,7 +817,7 @@ def assert_outputs_are_printed(expected_outputs: dict[str, str], stderr: str): "mypy-shared-module_loading,mypy-shared-observability,mypy-shared-plugins_manager," "mypy-shared-providers_discovery,mypy-shared-secrets_backend,mypy-shared-secrets_masker," "mypy-shared-serialization,mypy-shared-state,mypy-shared-template_rendering,mypy-shared-timezones,mypy-task-sdk," - "ts-compile-lint-simple-auth-manager-ui,ts-compile-lint-ui,update-uv-lock" + "regenerate-java-sdk-verification-metadata,ts-compile-lint-simple-auth-manager-ui,ts-compile-lint-ui,update-uv-lock" ), "skip-providers-tests": "true", "upgrade-to-newer-dependencies": "false", @@ -880,7 +880,7 @@ def assert_outputs_are_printed(expected_outputs: dict[str, str], stderr: str): "mypy-shared-module_loading,mypy-shared-observability,mypy-shared-plugins_manager," "mypy-shared-providers_discovery,mypy-shared-secrets_backend,mypy-shared-secrets_masker," "mypy-shared-serialization,mypy-shared-state,mypy-shared-template_rendering,mypy-shared-timezones,mypy-task-sdk,mypy-task-sdk-integration-tests," - "ts-compile-lint-simple-auth-manager-ui,ts-compile-lint-ui,update-uv-lock" + "regenerate-java-sdk-verification-metadata,ts-compile-lint-simple-auth-manager-ui,ts-compile-lint-ui,update-uv-lock" ), "skip-providers-tests": "true", "upgrade-to-newer-dependencies": "false", @@ -916,7 +916,7 @@ def assert_outputs_are_printed(expected_outputs: dict[str, str], stderr: str): "mypy-shared-module_loading,mypy-shared-observability,mypy-shared-plugins_manager," "mypy-shared-providers_discovery,mypy-shared-secrets_backend,mypy-shared-secrets_masker," "mypy-shared-serialization,mypy-shared-state,mypy-shared-template_rendering,mypy-shared-timezones,mypy-task-sdk,mypy-task-sdk-integration-tests," - "ts-compile-lint-simple-auth-manager-ui,ts-compile-lint-ui,update-uv-lock" + "regenerate-java-sdk-verification-metadata,ts-compile-lint-simple-auth-manager-ui,ts-compile-lint-ui,update-uv-lock" ), "skip-providers-tests": "true", "upgrade-to-newer-dependencies": "false", @@ -1263,7 +1263,7 @@ def assert_outputs_are_printed(expected_outputs: dict[str, str], stderr: str): "mypy-shared-module_loading,mypy-shared-observability,mypy-shared-plugins_manager," "mypy-shared-providers_discovery,mypy-shared-secrets_backend,mypy-shared-secrets_masker," "mypy-shared-serialization,mypy-shared-state,mypy-shared-template_rendering,mypy-shared-timezones,mypy-task-sdk,mypy-task-sdk-integration-tests," - "ts-compile-lint-simple-auth-manager-ui,ts-compile-lint-ui,update-uv-lock" + "regenerate-java-sdk-verification-metadata,ts-compile-lint-simple-auth-manager-ui,ts-compile-lint-ui,update-uv-lock" ), "upgrade-to-newer-dependencies": "false", "core-test-types-list-as-strings-in-json": json.dumps( @@ -1427,7 +1427,7 @@ def assert_outputs_are_printed(expected_outputs: dict[str, str], stderr: str): "mypy-shared-module_loading,mypy-shared-observability,mypy-shared-plugins_manager," "mypy-shared-providers_discovery,mypy-shared-secrets_backend,mypy-shared-secrets_masker," "mypy-shared-serialization,mypy-shared-state,mypy-shared-template_rendering,mypy-shared-timezones,mypy-task-sdk,mypy-task-sdk-integration-tests," - "ts-compile-lint-simple-auth-manager-ui,ts-compile-lint-ui,update-uv-lock" + "regenerate-java-sdk-verification-metadata,ts-compile-lint-simple-auth-manager-ui,ts-compile-lint-ui,update-uv-lock" ), "upgrade-to-newer-dependencies": "false", "core-test-types-list-as-strings-in-json": None, @@ -1798,7 +1798,7 @@ def assert_outputs_are_printed(expected_outputs: dict[str, str], stderr: str): "mypy-shared-secrets_backend,mypy-shared-secrets_masker," "mypy-shared-serialization,mypy-shared-state,mypy-shared-template_rendering," "mypy-shared-timezones,mypy-task-sdk,mypy-task-sdk-integration-tests," - "ts-compile-lint-simple-auth-manager-ui,ts-compile-lint-ui,update-uv-lock" + "regenerate-java-sdk-verification-metadata,ts-compile-lint-simple-auth-manager-ui,ts-compile-lint-ui,update-uv-lock" ), }, id=("Shared logging change keeps only mypy-shared-logging among the mypy-shared-* hooks"), @@ -1820,28 +1820,32 @@ def test_expected_output_pull_request_main( assert_outputs_are_printed(expected_outputs, str(stderr)) +@pytest.mark.parametrize("hook", ["ktlint", "regenerate-java-sdk-verification-metadata"]) @pytest.mark.parametrize( - ("files", "ktlint_skipped"), + ("files", "hook_skipped"), [ pytest.param( ("java-sdk/sdk/build.gradle.kts",), False, - id="ktlint runs when java-sdk files change", + id="runs when java-sdk files change", ), pytest.param( ("SECURITY.md",), True, - id="ktlint skipped when no java-sdk files change", + id="skipped when no java-sdk files change", ), pytest.param( ("java-sdk/README.md",), True, - id="ktlint skipped when only java-sdk docs change", + id="skipped when only java-sdk docs change", ), ], ) -def test_ktlint_hook_only_runs_for_java_sdk_changes(files: tuple[str, ...], ktlint_skipped: bool): - # ktlint downloads the Gradle distribution, so it must be skipped unless java-sdk changed. +def test_java_sdk_gradle_hooks_only_run_for_java_sdk_changes( + files: tuple[str, ...], hook_skipped: bool, hook: str +): + # Both hooks run the java-sdk Gradle wrapper and download from it, so they must be skipped + # unless java-sdk changed. stderr = SelectiveChecks( files=files, commit_ref=NEUTRAL_COMMIT, @@ -1850,7 +1854,7 @@ def test_ktlint_hook_only_runs_for_java_sdk_changes(files: tuple[str, ...], ktli default_branch="main", ) skipped_hooks = get_outputs_from_stderr(str(stderr))["skip-prek-hooks"].split(",") - assert ("ktlint" in skipped_hooks) is ktlint_skipped + assert (hook in skipped_hooks) is hook_skipped @pytest.mark.parametrize( @@ -2729,7 +2733,7 @@ def test_expected_output_push( "mypy-shared-module_loading,mypy-shared-observability,mypy-shared-plugins_manager," "mypy-shared-providers_discovery,mypy-shared-secrets_backend,mypy-shared-secrets_masker," "mypy-shared-serialization,mypy-shared-state,mypy-shared-template_rendering,mypy-shared-timezones,mypy-task-sdk,mypy-task-sdk-integration-tests," - "ts-compile-lint-simple-auth-manager-ui,ts-compile-lint-ui,update-uv-lock" + "regenerate-java-sdk-verification-metadata,ts-compile-lint-simple-auth-manager-ui,ts-compile-lint-ui,update-uv-lock" ), "upgrade-to-newer-dependencies": "false", "core-test-types-list-as-strings-in-json": json.dumps( @@ -2770,7 +2774,7 @@ def test_expected_output_push( "mypy-shared-module_loading,mypy-shared-observability,mypy-shared-plugins_manager," "mypy-shared-providers_discovery,mypy-shared-secrets_backend,mypy-shared-secrets_masker," "mypy-shared-serialization,mypy-shared-state,mypy-shared-template_rendering,mypy-shared-timezones,mypy-task-sdk,mypy-task-sdk-integration-tests," - "ts-compile-lint-simple-auth-manager-ui,ts-compile-lint-ui,update-uv-lock" + "regenerate-java-sdk-verification-metadata,ts-compile-lint-simple-auth-manager-ui,ts-compile-lint-ui,update-uv-lock" ), "run-kubernetes-tests": "true", "upgrade-to-newer-dependencies": "false", @@ -2816,7 +2820,7 @@ def test_expected_output_push( "mypy-shared-module_loading,mypy-shared-observability,mypy-shared-plugins_manager," "mypy-shared-providers_discovery,mypy-shared-secrets_backend,mypy-shared-secrets_masker," "mypy-shared-serialization,mypy-shared-state,mypy-shared-template_rendering,mypy-shared-timezones,mypy-task-sdk,mypy-task-sdk-integration-tests," - "ts-compile-lint-simple-auth-manager-ui,ts-compile-lint-ui,update-uv-lock" + "regenerate-java-sdk-verification-metadata,ts-compile-lint-simple-auth-manager-ui,ts-compile-lint-ui,update-uv-lock" ), "run-kubernetes-tests": "false", "upgrade-to-newer-dependencies": "false", diff --git a/java-sdk/README.md b/java-sdk/README.md index ef99848712760..2a4bd82da6489 100644 --- a/java-sdk/README.md +++ b/java-sdk/README.md @@ -44,17 +44,19 @@ development tools may have further requirements (see the toolchain in Repositories are centralized in `settings.gradle.kts`, and dynamic and changing versions are rejected for project dependency configurations (not for plugin markers or detached configurations — pin those by hand). `buildSrc/` declares its own repositories, but its dependencies *are* covered by this metadata. -To update a dependency or plugin, regenerate from a trusted network. The task list must cover everything CI runs, since only what the invoked tasks resolve gets recorded: +To update a dependency or plugin, regenerate the file from a trusted network. Run the command from the repository root: ```bash -./gradlew --write-verification-metadata sha256 --refresh-dependencies \ - build \ - :sdk:dokkaGeneratePublicationHtml :sdk:dokkaGeneratePublicationJavadoc \ - sourceTarball checksumSourceTarball \ - publishToMavenLocal -PskipSigning=true +prek run regenerate-java-sdk-verification-metadata --all-files ``` -Without `-PskipSigning=true` the signing tasks fail and Gradle still writes metadata from the partial run. Regeneration only appends, so delete superseded entries by hand after a version bump. +The hook runs on its own whenever a change moves the resolved dependency set, and it keeps failing until you stage the rewritten file too. It is a plain script, so you can also run it directly: + +```bash +uv run scripts/ci/prek/regenerate_java_sdk_verification_metadata.py +``` + +Gradle only appends to the file, so the script empties the component list before regenerating. Otherwise every version bump leaves its superseded entries behind, and they stay trusted. The script also owns the task list, which has to cover everything CI builds, because Gradle records only what the invoked tasks resolve. Review every entry in the diff. Generating the file records what the repositories served at that moment; it does not make those bytes trustworthy. Cross-check new coordinates and checksums against the dependency's official release information, and never bypass a failure with lenient or disabled verification. diff --git a/scripts/ci/prek/regenerate_java_sdk_verification_metadata.py b/scripts/ci/prek/regenerate_java_sdk_verification_metadata.py new file mode 100755 index 0000000000000..d2148b7fb41ff --- /dev/null +++ b/scripts/ci/prek/regenerate_java_sdk_verification_metadata.py @@ -0,0 +1,165 @@ +#!/usr/bin/env python3 +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +""" +Rewrite java-sdk/gradle/verification-metadata.xml from an empty component list. + +Gradle only ever appends to that file, so a version bump leaves the superseded +entries behind and they stay trusted for good. Starting from an empty list drops +them, which is what turns a stale checksum into a visible diff. +""" + +from __future__ import annotations + +import pathlib +import subprocess +import sys +import time +from collections.abc import Callable + +REPO_ROOT = pathlib.Path(__file__).resolve().parents[3] +JAVA_SDK = REPO_ROOT / "java-sdk" +METADATA = JAVA_SDK / "gradle" / "verification-metadata.xml" + +GRADLE_TASKS = [ + "build", + ":sdk:dokkaGeneratePublicationHtml", + ":sdk:dokkaGeneratePublicationJavadoc", + "sourceTarball", + "checksumSourceTarball", + "publishToMavenLocal", +] + +MAX_ATTEMPTS = 3 +RETRY_DELAY_SECONDS = 30 + +LICENSE_HEADER = """""" + +ADVISORY = """ +The trust list changed. Entries that disappeared are checksums nothing resolves +any more - usually versions superseded by a dependency bump. They stayed +trusted, so a future direct or transitive dependency could have pulled that +exact version back in with nobody reviewing its checksum. + +Review every entry in the diff before staging it: generating the file records +what the repositories served, it does not make those bytes trustworthy. +""" + + +class RegenerationFailedError(RuntimeError): + """Gradle could not regenerate the metadata within the attempt budget.""" + + +def build_empty_metadata(committed: str) -> str: + """Return the committed metadata with its component list emptied.""" + kept: list[str] = [] + for line in committed.splitlines(): + if "" in line: + kept += [" ", ""] + break + kept.append(line) + return "\n".join(kept) + "\n" + + +def insert_license_header(metadata: str) -> str: + """Put the ASF header back after the XML declaration, unless Gradle kept one.""" + if "Licensed to the Apache Software Foundation" in metadata: + return metadata + declaration, *rest = metadata.splitlines() + return "\n".join([declaration, LICENSE_HEADER, *rest]) + "\n" + + +def run_gradle() -> bool: + """Both properties are needed for the run to reach the end: signing has no key + here, and sourceTarball has no default ref.""" + result = subprocess.run( + [ + "./gradlew", + "--no-daemon", + "--write-verification-metadata", + "sha256", + "--refresh-dependencies", + *GRADLE_TASKS, + "-PskipSigning=true", + "-PgitRef=HEAD", + ], + cwd=JAVA_SDK, + check=False, + ) + return result.returncode == 0 + + +def regenerate( + metadata: pathlib.Path, + gradle: Callable[[], bool], + sleep: Callable[[float], None] = time.sleep, +) -> None: + """Rewrite the metadata in place, putting the committed file back if the run never succeeds.""" + committed = metadata.read_text() + restore = True + try: + for attempt in range(1, MAX_ATTEMPTS + 1): + print(f"==> Regenerating verification metadata (attempt {attempt}/{MAX_ATTEMPTS})", flush=True) + metadata.write_text(build_empty_metadata(committed)) + if gradle(): + metadata.write_text(insert_license_header(metadata.read_text())) + restore = False + return + if attempt < MAX_ATTEMPTS: + print(f"Regeneration failed, retrying in {RETRY_DELAY_SECONDS}s", file=sys.stderr, flush=True) + sleep(RETRY_DELAY_SECONDS) + raise RegenerationFailedError(f"Regeneration failed after {MAX_ATTEMPTS} attempts") + finally: + if restore: + metadata.write_text(committed) + + +def metadata_changed(metadata: pathlib.Path) -> bool: + result = subprocess.run(["git", "diff", "--quiet", "--", str(metadata)], cwd=REPO_ROOT, check=False) + return result.returncode != 0 + + +def main() -> int: + try: + regenerate(METADATA, run_gradle) + except RegenerationFailedError as error: + print(f"ERROR: {error}", file=sys.stderr) + return 1 + if metadata_changed(METADATA): + print(ADVISORY, file=sys.stderr) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/tests/ci/prek/test_regenerate_java_sdk_verification_metadata.py b/scripts/tests/ci/prek/test_regenerate_java_sdk_verification_metadata.py new file mode 100644 index 0000000000000..f4cc9a9c9b788 --- /dev/null +++ b/scripts/tests/ci/prek/test_regenerate_java_sdk_verification_metadata.py @@ -0,0 +1,175 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +from __future__ import annotations + +import pathlib + +import pytest +import regenerate_java_sdk_verification_metadata as regenerator +from regenerate_java_sdk_verification_metadata import ( + LICENSE_HEADER, + METADATA, + RegenerationFailedError, + build_empty_metadata, + insert_license_header, + regenerate, +) + +COMMITTED = f""" +{LICENSE_HEADER} + + + true + + + + + + +""" + + +def component_names(metadata: str) -> list[str]: + return [line.split('name="')[1].split('"')[0] for line in metadata.splitlines() if " bool: + self.calls += 1 + if self.calls <= self.failures: + return False + text = self.metadata.read_text() + kept = [line for line in text.splitlines() if "') + header = [LICENSE_HEADER] if self.keeps_header else [] + self.metadata.write_text( + "\n".join( + [ + '', + *header, + '', + " ", + *kept, + " ", + "", + ] + ) + + "\n" + ) + return True + + +@pytest.fixture +def metadata(tmp_path) -> pathlib.Path: + path = tmp_path / "verification-metadata.xml" + path.write_text(COMMITTED) + return path + + +class TestBuildEmptyMetadata: + def test_empties_the_component_list(self): + assert component_names(build_empty_metadata(COMMITTED)) == [] + + def test_keeps_the_header_and_the_configuration(self): + emptied = build_empty_metadata(COMMITTED) + assert LICENSE_HEADER in emptied + assert "true" in emptied + + +class TestInsertLicenseHeader: + def test_inserts_the_header_after_the_xml_declaration(self): + stripped = '\n\n' + assert insert_license_header(stripped).splitlines()[1] == LICENSE_HEADER.splitlines()[0] + + def test_leaves_an_existing_header_alone(self): + assert insert_license_header(COMMITTED) == COMMITTED + + +class TestRegenerate: + def test_drops_superseded_entries_and_restores_the_header(self, metadata): + regenerate(metadata, FakeGradle(metadata)) + + assert component_names(metadata.read_text()) == ["current"] + assert metadata.read_text().splitlines()[1] == LICENSE_HEADER.splitlines()[0] + + def test_leaves_a_single_header_when_gradle_keeps_the_one_it_was_given(self, metadata): + regenerate(metadata, FakeGradle(metadata, keeps_header=True)) + + assert metadata.read_text().count("Licensed to the Apache Software Foundation") == 1 + + def test_retries_a_failing_run_and_succeeds(self, metadata): + gradle = FakeGradle(metadata, failures=2) + + regenerate(metadata, gradle, sleep=lambda _: None) + + assert gradle.calls == 3 + assert component_names(metadata.read_text()) == ["current"] + + def test_restores_the_committed_file_when_every_attempt_fails(self, metadata): + gradle = FakeGradle(metadata, failures=99) + + with pytest.raises(RegenerationFailedError): + regenerate(metadata, gradle, sleep=lambda _: None) + + assert metadata.read_text() == COMMITTED + + +def test_license_header_matches_the_committed_metadata(): + committed = METADATA.read_text().splitlines() + start = committed.index("") + assert "\n".join(committed[start : end + 1]) == LICENSE_HEADER + + +class TestMain: + def test_prints_the_advisory_when_the_metadata_changed(self, capsys, monkeypatch): + monkeypatch.setattr(regenerator, "regenerate", lambda *_: None) + monkeypatch.setattr(regenerator, "metadata_changed", lambda _: True) + + assert regenerator.main() == 0 + assert "The trust list changed" in capsys.readouterr().err + + def test_stays_quiet_when_the_metadata_is_unchanged(self, capsys, monkeypatch): + monkeypatch.setattr(regenerator, "regenerate", lambda *_: None) + monkeypatch.setattr(regenerator, "metadata_changed", lambda _: False) + + assert regenerator.main() == 0 + assert capsys.readouterr().err == "" + + def test_reports_failure_when_no_attempt_succeeds(self, capsys, monkeypatch): + def never_succeeds(*_): + raise RegenerationFailedError("Regeneration failed after 3 attempts") + + monkeypatch.setattr(regenerator, "regenerate", never_succeeds) + + assert regenerator.main() == 1 + assert "Regeneration failed after 3 attempts" in capsys.readouterr().err