Catch stale entries in the Java SDK dependency trust list

.pre-commit-config.yaml

@@ -279,6 +279,25 @@ repos:
           (?x)
           ^java-sdk/gradle\.properties$|
           ^java-sdk/sdk/schema/schema\.json$
+      - id: regenerate-java-sdk-verification-metadata
+        name: Regenerate the Java SDK dependency verification metadata
+        description: "Drop trusted checksums that no Java SDK build task resolves any more"
+        entry: ./scripts/ci/prek/regenerate_java_sdk_verification_metadata.py
+        language: python
+        pass_filenames: false
+        # example/ and scala_spark_example/ are separate builds this metadata does
+        # not cover, so their build files are excluded.
+        files: >
+          (?x)
+          ^java-sdk/build\.gradle\.kts$|
+          ^java-sdk/buildSrc/.*$|
+          ^java-sdk/gradle\.properties$|
+          ^java-sdk/gradle/libs\.versions\.toml$|
+          ^java-sdk/gradle/verification-metadata\.xml$|
+          ^java-sdk/gradle/wrapper/gradle-wrapper\.properties$|
+          ^java-sdk/settings\.gradle\.kts$|
+          ^scripts/ci/prek/regenerate_java_sdk_verification_metadata\.py$|
+          ^java-sdk/(?!example/|scala_spark_example/)[^/]+/(?:build\.gradle\.kts|gradle\.properties)$
       - id: update-java-sdk-readme-matrix
         name: Update the Java SDK compatibility matrix in java-sdk/README.md and Dokka module doc
         entry: ./scripts/ci/prek/update_java_sdk_readme_matrix.py

dev/breeze/doc/ci/04_selective_checks.md

@@ -519,9 +519,11 @@ when some files are not changed. Those are the rules implemented:
     type errors (see #68919)
   * if no `All Python files` changed - `flynt` check is skipped
   * if no `Helm files` changed - `lint-helm-chart` check is skipped
-  * if no `Java SDK files` changed - `ktlint` check is skipped (it runs the java-sdk Gradle
-    wrapper, which downloads the Gradle distribution, so we avoid that download on PRs that do
-    not touch `java-sdk/`)
+  * if no `Java SDK files` changed - `ktlint` and
+    `regenerate-java-sdk-verification-metadata` checks are skipped (both run the java-sdk
+    Gradle wrapper, which downloads the Gradle distribution, and the latter additionally
+    resolves the whole Java SDK dependency graph from Maven Central, so we avoid those
+    downloads on PRs that do not touch `java-sdk/`)
   * if no `TS SDK files` (`ts-sdk/`) changed - `check-ts-sdk-supervisor-schema` check is
     skipped (it regenerates and diffs the generated ts-sdk file; a change to the supervisor
     wire schema alone deliberately does not trigger it - regenerating the ts-sdk types is

dev/breeze/src/airflow_breeze/utils/selective_checks.py

@@ -1729,6 +1729,10 @@ def skip_prek_hooks(self) -> str:
             # on a cold cache. Skip it when no java-sdk files changed so unrelated PRs do not
             # depend on that (intermittently failing) download.
             prek_hooks_to_skip.add("ktlint")
+            # Rewriting the verification metadata resolves the entire Java SDK dependency graph
+            # from Maven Central. Skip it when no java-sdk files changed so unrelated PRs do not
+            # depend on that resolution.
+            prek_hooks_to_skip.add("regenerate-java-sdk-verification-metadata")
         if not self._matching_files(FileGroupForCi.TS_SDK_FILES, CI_FILE_GROUP_MATCHES):
             # This hook regenerates ts-sdk/src/generated/supervisor.ts from the wire schema and
             # diffs it. Schema-only changes deliberately do not trigger it: regenerating the

java-sdk/README.md

@@ -44,17 +44,19 @@ development tools may have further requirements (see the toolchain in
 
 Repositories are centralized in `settings.gradle.kts`, and dynamic and changing versions are rejected for project dependency configurations (not for plugin markers or detached configurations — pin those by hand). `buildSrc/` declares its own repositories, but its dependencies *are* covered by this metadata.
 
-To update a dependency or plugin, regenerate from a trusted network. The task list must cover everything CI runs, since only what the invoked tasks resolve gets recorded:
+To update a dependency or plugin, regenerate the file from a trusted network. Run the command from the repository root:
 
 ```bash
-./gradlew --write-verification-metadata sha256 --refresh-dependencies \
-  build \
-  :sdk:dokkaGeneratePublicationHtml :sdk:dokkaGeneratePublicationJavadoc \
-  sourceTarball checksumSourceTarball \
-  publishToMavenLocal -PskipSigning=true
+prek run regenerate-java-sdk-verification-metadata --all-files
 ```
 
-Without `-PskipSigning=true` the signing tasks fail and Gradle still writes metadata from the partial run. Regeneration only appends, so delete superseded entries by hand after a version bump.
+The hook runs on its own whenever a change moves the resolved dependency set, and it keeps failing until you stage the rewritten file too. It is a plain script, so you can also run it directly:
+
+```bash
+uv run scripts/ci/prek/regenerate_java_sdk_verification_metadata.py
+```
+
+Gradle only appends to the file, so the script empties the component list before regenerating. Otherwise every version bump leaves its superseded entries behind, and they stay trusted. The script also owns the task list, which has to cover everything CI builds, because Gradle records only what the invoked tasks resolve.
 
 Review every entry in the diff. Generating the file records what the repositories served at that moment; it does not make those bytes trustworthy. Cross-check new coordinates and checksums against the dependency's official release information, and never bypass a failure with lenient or disabled verification.
 

scripts/ci/prek/regenerate_java_sdk_verification_metadata.py

@@ -0,0 +1,165 @@
+#!/usr/bin/env python3
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+"""
+Rewrite java-sdk/gradle/verification-metadata.xml from an empty component list.
+
+Gradle only ever appends to that file, so a version bump leaves the superseded
+entries behind and they stay trusted for good. Starting from an empty list drops
+them, which is what turns a stale checksum into a visible diff.
+"""
+
+from __future__ import annotations
+
+import pathlib
+import subprocess
+import sys
+import time
+from collections.abc import Callable
+
+REPO_ROOT = pathlib.Path(__file__).resolve().parents[3]
+JAVA_SDK = REPO_ROOT / "java-sdk"
+METADATA = JAVA_SDK / "gradle" / "verification-metadata.xml"
+
+GRADLE_TASKS = [
+    "build",
+    ":sdk:dokkaGeneratePublicationHtml",
+    ":sdk:dokkaGeneratePublicationJavadoc",
+    "sourceTarball",
+    "checksumSourceTarball",
+    "publishToMavenLocal",
+]
+
+MAX_ATTEMPTS = 3
+RETRY_DELAY_SECONDS = 30
+
+LICENSE_HEADER = """<!--
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements.  See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership.  The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License.  You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied.  See the License for the
+ specific language governing permissions and limitations
+ under the License.
+-->"""
+
+ADVISORY = """
+The trust list changed. Entries that disappeared are checksums nothing resolves
+any more - usually versions superseded by a dependency bump. They stayed
+trusted, so a future direct or transitive dependency could have pulled that
+exact version back in with nobody reviewing its checksum.
+
+Review every entry in the diff before staging it: generating the file records
+what the repositories served, it does not make those bytes trustworthy.
+"""
+
+
+class RegenerationFailedError(RuntimeError):
+    """Gradle could not regenerate the metadata within the attempt budget."""
+
+
+def build_empty_metadata(committed: str) -> str:
+    """Return the committed metadata with its component list emptied."""
+    kept: list[str] = []
+    for line in committed.splitlines():
+        if "<components>" in line:
+            kept += ["   <components/>", "</verification-metadata>"]
+            break
+        kept.append(line)
+    return "\n".join(kept) + "\n"
+
+
+def insert_license_header(metadata: str) -> str:
+    """Put the ASF header back after the XML declaration, unless Gradle kept one."""
+    if "Licensed to the Apache Software Foundation" in metadata:
+        return metadata
+    declaration, *rest = metadata.splitlines()
+    return "\n".join([declaration, LICENSE_HEADER, *rest]) + "\n"
+
+
+def run_gradle() -> bool:
+    """Both properties are needed for the run to reach the end: signing has no key
+    here, and sourceTarball has no default ref."""
+    result = subprocess.run(
+        [
+            "./gradlew",
+            "--no-daemon",
+            "--write-verification-metadata",
+            "sha256",
+            "--refresh-dependencies",
+            *GRADLE_TASKS,
+            "-PskipSigning=true",
+            "-PgitRef=HEAD",
+        ],
+        cwd=JAVA_SDK,
+        check=False,
+    )
+    return result.returncode == 0
+
+
+def regenerate(
+    metadata: pathlib.Path,
+    gradle: Callable[[], bool],
+    sleep: Callable[[float], None] = time.sleep,
+) -> None:
+    """Rewrite the metadata in place, putting the committed file back if the run never succeeds."""
+    committed = metadata.read_text()
+    restore = True
+    try:
+        for attempt in range(1, MAX_ATTEMPTS + 1):
+            print(f"==> Regenerating verification metadata (attempt {attempt}/{MAX_ATTEMPTS})", flush=True)
+            metadata.write_text(build_empty_metadata(committed))
+            if gradle():
+                metadata.write_text(insert_license_header(metadata.read_text()))
+                restore = False
+                return
+            if attempt < MAX_ATTEMPTS:
+                print(f"Regeneration failed, retrying in {RETRY_DELAY_SECONDS}s", file=sys.stderr, flush=True)
+                sleep(RETRY_DELAY_SECONDS)
+        raise RegenerationFailedError(f"Regeneration failed after {MAX_ATTEMPTS} attempts")
+    finally:
+        if restore:
+            metadata.write_text(committed)
+
+
+def metadata_changed(metadata: pathlib.Path) -> bool:
+    result = subprocess.run(["git", "diff", "--quiet", "--", str(metadata)], cwd=REPO_ROOT, check=False)
+    return result.returncode != 0
+
+
+def main() -> int:
+    try:
+        regenerate(METADATA, run_gradle)
+    except RegenerationFailedError as error:
+        print(f"ERROR: {error}", file=sys.stderr)
+        return 1
+    if metadata_changed(METADATA):
+        print(ADVISORY, file=sys.stderr)
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())