From 4512acc50296542b2e4ce36f1ed5e3dc62b7aeb2 Mon Sep 17 00:00:00 2001 From: Roman Sirokov Date: Mon, 5 Oct 2026 15:48:04 +0300 Subject: [PATCH 01/16] fix(release): make --require-signatures verify, not just find a .sig release_readiness.py is the only thing in CI that ever verifies a release artifact's signature, but --require-signatures asserted only that a non-empty .sig sidecar existed. Cryptographic verification was gated by a separate condition, true only when --public-key was passed (no caller does) or ROCM_CLI_SIGNING_PUBLIC_KEY_PEM was non-empty. GitHub expands an unset secret to the empty string and env_text maps empty to None, so removing or rotating that secret would silently downgrade the gate from "every artifact verifies against the release key" to "a .sig file is present" -- no failure, no warning, the only trace being which message appears in the log. A sidecar produced by the wrong key, truncated, or corrupted would pass. Requiring signatures now means requiring they verify. The presence check and the verification branch collapse into one `verify` flag, the key is resolved once (--public-key, else the inline PEM), and a key that cannot be resolved is a hard error rather than a fallback to the weaker check. Each run prints `signature verification key:` naming the trust root used, so the mode is recorded rather than inferred. Verification is deliberately not defaulted to the in-repo pinned key: AGENTS.md section 9 reserves production trust roots as owner-controlled inputs, and `cargo xtask verify-pinned-keys` already asserts a configured CI key equals the canonical one. That cross-check had the same fail-open shape -- it returned silently when no CI key was configured, so an unset secret read as a clean run. It now reports the skip. It stays non-fatal because ci.yml never exposes the signing secret, so demanding one would fail every PR. No workflow changes: the secret is configured today, so release and nightly already take the verifying path and this is a no-op for them. Signed-off-by: Roman Sirokov --- docs/release-trust.md | 23 +++-- scripts/release_readiness.py | 147 +++++++++++++++++++++++--------- xtask/src/verify_pinned_keys.rs | 29 ++++--- 3 files changed, 143 insertions(+), 56 deletions(-) diff --git a/docs/release-trust.md b/docs/release-trust.md index 3fa23afce4..d61ccd2c3d 100644 --- a/docs/release-trust.md +++ b/docs/release-trust.md @@ -47,12 +47,23 @@ python scripts/release_readiness.py \ ``` The verifier confirms each named archive has the required rocm-cli bundle -files, confirms the `.sha256` sidecar matches the archive bytes and names the -archive being published, and confirms a non-empty `.sig` sidecar exists when -signatures are required. If a public key is provided with `--public-key`, it -also verifies the detached signature with `openssl`. Release CI also enables -`--require-rocm-asset-names`, which rejects branch-like, path-like, or -otherwise unsupported archive names before upload. +files, and confirms the `.sha256` sidecar matches the archive bytes and names +the archive being published. + +`--require-signatures` requires signatures that *verify*, not merely exist: each +archive must have a non-empty `.sig` sidecar, and that signature must check out +against the release signing public key via `cargo xtask verify`. The key is +taken from `--public-key` when given, otherwise from the inline +`ROCM_CLI_SIGNING_PUBLIC_KEY_PEM` that release and nightly CI wire from the +signing-key secret. If neither resolves, the gate fails. It deliberately does +not fall back to checking that a `.sig` file is present: a sidecar produced by +the wrong key, truncated, or corrupted would pass such a check, and because an +unset GitHub secret expands to the empty string, the fallback would be reached +silently whenever the secret was removed or rotated away. The log line +`signature verification key:` names the key the run actually used. + +Release CI also enables `--require-rocm-asset-names`, which rejects branch-like, +path-like, or otherwise unsupported archive names before upload. Release CI also enables `--require-exact-assets`. That rejects stale publishable files in `dist`, including old `.tar.gz`/`.zip` archives and orphan diff --git a/scripts/release_readiness.py b/scripts/release_readiness.py index 1692639aca..1de747c270 100644 --- a/scripts/release_readiness.py +++ b/scripts/release_readiness.py @@ -47,9 +47,10 @@ "bin/rocmd", "install.sh", ) +SIGNING_PUBLIC_KEY_ENV = "ROCM_CLI_SIGNING_PUBLIC_KEY_PEM" PRODUCTION_TRUST_ENV_NAMES = ( "ROCM_CLI_SIGNING_PUBLIC_KEY_PATH", - "ROCM_CLI_SIGNING_PUBLIC_KEY_PEM", + SIGNING_PUBLIC_KEY_ENV, "ROCM_CLI_METADATA_PUBLIC_KEY_PATH", "ROCM_CLI_METADATA_PUBLIC_KEY_PEM", "ROCM_CLI_MODEL_RECIPE_INDEX_PATH", @@ -381,8 +382,8 @@ def validate_archive( archive: Path, *, require_signatures: bool, + verify: bool, public_key: Path | None, - verify_with_env_key: bool = False, require_rocm_asset_names: bool, ) -> list[str]: messages: list[str] = [] @@ -409,7 +410,6 @@ def validate_archive( ) messages.append(f"checksum ok: {archive.name}") - verify = public_key is not None or verify_with_env_key signature = Path(f"{archive}.sig") if require_signatures or verify: if not signature.is_file(): @@ -458,6 +458,30 @@ def env_text(name: str) -> str | None: return value +def resolve_signing_key(explicit: Path | None) -> tuple[Path | None, str]: + """Resolve the public key release signatures are verified against. + + Returns the key path and a label naming where it came from. A ``None`` path + means `cargo xtask verify` reads the inline PEM from + ``ROCM_CLI_SIGNING_PUBLIC_KEY_PEM`` itself, so no temporary key file has to + be materialized here. + + Raises when no key resolves. That is the point of this function: GitHub + expands an unset secret to the empty string, so a missing key is + indistinguishable from a deliberately absent one. Verification has to fail + closed, or rotating the secret away would silently downgrade the release + gate to "a .sig file exists". + """ + if explicit is not None: + return explicit, f"--public-key {explicit}" + if env_text(SIGNING_PUBLIC_KEY_ENV) is not None: + return None, f"${SIGNING_PUBLIC_KEY_ENV}" + raise ReadinessError( + "signature verification requires a release signing public key: pass " + f"--public-key or set {SIGNING_PUBLIC_KEY_ENV}" + ) + + def require_any(label: str, names: list[str]) -> None: if any(env_text(name) for name in names): return @@ -521,14 +545,13 @@ def validate_release( *, assets: list[str], require_signatures: bool, + verify: bool, public_key: Path | None, - verify_with_env_key: bool = False, require_production_trust: bool, require_rocm_asset_names: bool, require_exact_assets: bool, ) -> list[str]: archives = discover_archives(dist, assets) - verify = public_key is not None or verify_with_env_key messages: list[str] = [] if require_exact_assets: if not assets: @@ -545,8 +568,8 @@ def validate_release( validate_archive( archive, require_signatures=require_signatures, + verify=verify, public_key=public_key, - verify_with_env_key=verify_with_env_key, require_rocm_asset_names=require_rocm_asset_names, ) ) @@ -635,6 +658,7 @@ def run_self_test(root: Path) -> None: dist, assets=[], require_signatures=True, + verify=False, public_key=None, require_production_trust=False, require_rocm_asset_names=False, @@ -655,6 +679,7 @@ def run_self_test(root: Path) -> None: exact_dist, assets=[exact_linux_archive.name, exact_windows_archive.name], require_signatures=True, + verify=False, public_key=None, require_production_trust=False, require_rocm_asset_names=True, @@ -672,6 +697,7 @@ def run_self_test(root: Path) -> None: exact_dist, assets=[exact_linux_archive.name, exact_windows_archive.name], require_signatures=True, + verify=False, public_key=None, require_production_trust=False, require_rocm_asset_names=True, @@ -692,6 +718,7 @@ def run_self_test(root: Path) -> None: exact_dist, assets=[exact_linux_archive.name, exact_windows_archive.name], require_signatures=True, + verify=False, public_key=None, require_production_trust=False, require_rocm_asset_names=True, @@ -729,6 +756,7 @@ def run_self_test(root: Path) -> None: strict_nightly_alias.name, ], require_signatures=True, + verify=False, public_key=None, require_production_trust=False, require_rocm_asset_names=True, @@ -743,6 +771,7 @@ def run_self_test(root: Path) -> None: dist, assets=[], require_signatures=True, + verify=False, public_key=None, require_production_trust=False, require_rocm_asset_names=False, @@ -761,6 +790,7 @@ def run_self_test(root: Path) -> None: dist, assets=[], require_signatures=True, + verify=False, public_key=None, require_production_trust=False, require_rocm_asset_names=False, @@ -775,6 +805,7 @@ def run_self_test(root: Path) -> None: dist, assets=[], require_signatures=True, + verify=False, public_key=None, require_production_trust=False, require_rocm_asset_names=False, @@ -792,6 +823,7 @@ def run_self_test(root: Path) -> None: dist, assets=[], require_signatures=True, + verify=False, public_key=None, require_production_trust=False, require_rocm_asset_names=False, @@ -809,6 +841,7 @@ def run_self_test(root: Path) -> None: dist, assets=[], require_signatures=True, + verify=False, public_key=None, require_production_trust=True, require_rocm_asset_names=False, @@ -847,6 +880,7 @@ def run_self_test(root: Path) -> None: dist, assets=[], require_signatures=True, + verify=False, public_key=None, require_production_trust=True, require_rocm_asset_names=False, @@ -864,6 +898,7 @@ def run_self_test(root: Path) -> None: "missing-installer-alias.tar.gz", ], require_signatures=True, + verify=False, public_key=None, require_production_trust=False, require_rocm_asset_names=False, @@ -877,6 +912,7 @@ def run_self_test(root: Path) -> None: dist, assets=["../rocm-cli-linux-amd64.tar.gz"], require_signatures=True, + verify=False, public_key=None, require_production_trust=False, require_rocm_asset_names=False, @@ -892,12 +928,51 @@ def run_self_test(root: Path) -> None: dist, assets=["rocm-cli-test-linux-amd64.tar.gz"], require_signatures=True, + verify=False, public_key=None, require_production_trust=False, require_rocm_asset_names=True, require_exact_assets=False, ), ) + + expect_failure( + "signature verification with no resolvable key", + lambda: run_with_env( + {SIGNING_PUBLIC_KEY_ENV: None}, + lambda: resolve_signing_key(None), + ), + ) + # GitHub expands an unset secret to the empty string, so this is what an + # unconfigured ROCM_CLI_SIGNING_PUBLIC_KEY_PEM actually looks like in CI. + # It must fail exactly like an absent one rather than resolving to a key. + expect_failure( + "signature verification with an empty key", + lambda: run_with_env( + {SIGNING_PUBLIC_KEY_ENV: ""}, + lambda: resolve_signing_key(None), + ), + ) + + env_key, env_source = run_with_env( + {SIGNING_PUBLIC_KEY_ENV: "-----BEGIN PUBLIC KEY-----\nself-test\n"}, + lambda: resolve_signing_key(None), + ) + if env_key is not None or SIGNING_PUBLIC_KEY_ENV not in env_source: + raise ReadinessError( + f"expected the environment key to resolve, got {env_source}" + ) + + explicit = root / "explicit-public-key.pem" + explicit_key, explicit_source = run_with_env( + {SIGNING_PUBLIC_KEY_ENV: "-----BEGIN PUBLIC KEY-----\nself-test\n"}, + lambda: resolve_signing_key(explicit), + ) + if explicit_key != explicit or str(explicit) not in explicit_source: + raise ReadinessError( + f"expected --public-key to win over the environment, got {explicit_source}" + ) + print("release readiness self-test: signing key resolution ok") finally: shutil.rmtree(root, ignore_errors=True) print("release readiness self-test: ok") @@ -917,12 +992,14 @@ def parse_args() -> argparse.Namespace: parser.add_argument( "--require-signatures", action="store_true", - help="Require every archive to have a non-empty .sig sidecar.", + help="Require every archive to carry a detached signature that verifies " + "against the release signing public key.", ) parser.add_argument( "--public-key", type=Path, - help="Verify detached signatures with this public key.", + help="Verify detached signatures with this public key instead of the one " + f"in {SIGNING_PUBLIC_KEY_ENV}.", ) parser.add_argument( "--require-production-trust", @@ -968,37 +1045,31 @@ def main() -> None: require_production_trust = args.require_production_trust or truthy( os.environ.get("ROCM_CLI_REQUIRE_PRODUCTION_TRUST") ) - # Verify archives against the release public key whenever one is configured, - # not just check that a key input exists — so a private/public key mismatch - # fails here instead of at users' installers. Mandatory under production - # trust; otherwise opportunistic: if a signing public key is present in the - # environment (release/nightly wire it from the secret), verify against it. - # An explicit --public-key still wins; with neither, behavior is unchanged. - # - # When relying on the environment key, `cargo xtask verify` reads it directly - # from ROCM_CLI_SIGNING_PUBLIC_KEY_PEM, so no temp key file is materialized here. - public_key = args.public_key - verify_with_env_key = False + # Requiring signatures means requiring they verify. Checking only that a + # .sig exists would pass an artifact signed by the wrong key, truncated, or + # corrupted — so whenever signatures are required, so is cryptographic + # verification, and a key that cannot be resolved is a hard failure rather + # than a quiet downgrade to the presence check. + verify = ( + require_signatures or require_production_trust or args.public_key is not None + ) + public_key: Path | None = None + messages: list[str] = [] try: - if public_key is None and ( - require_production_trust - or env_text("ROCM_CLI_SIGNING_PUBLIC_KEY_PEM") is not None - ): - if env_text("ROCM_CLI_SIGNING_PUBLIC_KEY_PEM") is None: - raise ReadinessError( - "production trust requires the release signing public key: set " - "ROCM_CLI_SIGNING_PUBLIC_KEY_PEM (or pass --public-key)" - ) - verify_with_env_key = True - messages = validate_release( - Path(args.dist), - assets=args.asset, - require_signatures=require_signatures, - public_key=public_key, - verify_with_env_key=verify_with_env_key, - require_production_trust=require_production_trust, - require_rocm_asset_names=args.require_rocm_asset_names, - require_exact_assets=args.require_exact_assets, + if verify: + public_key, key_source = resolve_signing_key(args.public_key) + messages.append(f"signature verification key: {key_source}") + messages.extend( + validate_release( + Path(args.dist), + assets=args.asset, + require_signatures=require_signatures, + verify=verify, + public_key=public_key, + require_production_trust=require_production_trust, + require_rocm_asset_names=args.require_rocm_asset_names, + require_exact_assets=args.require_exact_assets, + ) ) except ReadinessError as error: fail(str(error)) diff --git a/xtask/src/verify_pinned_keys.rs b/xtask/src/verify_pinned_keys.rs index 208ce04192..38583d7dc4 100644 --- a/xtask/src/verify_pinned_keys.rs +++ b/xtask/src/verify_pinned_keys.rs @@ -258,13 +258,20 @@ fn check_pinned_keys(root: &Path, ci_public_key: Option<&str>) -> Result, current_release_body: Option<&str>, ) -> Result> { let env_pem = ci_public_key.unwrap_or_default(); if env_pem.trim().is_empty() { - return Ok(Vec::new()); + return Ok(vec![ + "ci public key: not configured — cross-check skipped".to_owned(), + ]); } let env_body = pem_body(env_pem) .context("the configured CI signing public key is not a valid PUBLIC KEY PEM")?; @@ -381,17 +388,15 @@ mod tests { fn ci_public_key_cross_check() { let current = pem_body(SAMPLE).unwrap(); - // No CI key configured -> nothing to assert. - assert!( - check_ci_public_key(None, Some(¤t)) - .unwrap() - .is_empty() - ); - assert!( - check_ci_public_key(Some(" "), Some(¤t)) - .unwrap() - .is_empty() - ); + // No CI key configured -> nothing to assert, but the skip is reported so an + // unset/rotated-away secret does not read as a passing cross-check. + for absent in [None, Some(" "), Some("")] { + let reported = check_ci_public_key(absent, Some(¤t)).unwrap(); + assert!( + reported.iter().any(|m| m.contains("not configured")), + "absent CI key should report a skip, got {reported:?}" + ); + } // Configured but no canonical current key yet -> skipped, not an error. let skipped = check_ci_public_key(Some(SAMPLE), None).unwrap(); From e711b37663dec97008560470566b6a922b97d9dd Mon Sep 17 00:00:00 2001 From: Roman Sirokov Date: Tue, 6 Oct 2026 09:34:56 +0300 Subject: [PATCH 02/16] fix(release): resolve the signing key from the path variable too Self-review found that making --require-signatures fail closed turned a silent pass into a misleading failure for anyone configuring the key as a file. resolve_signing_key() consulted only the inline ROCM_CLI_SIGNING_PUBLIC_KEY_PEM, so an operator who had set ROCM_CLI_SIGNING_PUBLIC_KEY_PATH -- a documented input this script already accepts in PRODUCTION_TRUST_ENV_NAMES and validate_production_trust() -- was told to "set ROCM_CLI_SIGNING_PUBLIC_KEY_PEM" while the key they had supplied went unused. Resolution now consults --public-key, then the key file named by ROCM_CLI_SIGNING_PUBLIC_KEY_PATH, then the inline PEM. That path-before-PEM order matches how install.sh resolve_public_keys() and xtask/src/package.rs signing_private_key_pem() already resolve their keys. A path that does not exist stays an error rather than falling through to the inline PEM: verifying against a different key than the operator named is the class of failure this gate exists to prevent. Self-test covers the new source and its precedence; the no-key and empty-string cases now clear both variables so they still prove a hard failure rather than passing because one source happened to be unset. Signed-off-by: Roman Sirokov --- docs/release-trust.md | 6 ++++-- scripts/release_readiness.py | 41 ++++++++++++++++++++++++++++++------ 2 files changed, 38 insertions(+), 9 deletions(-) diff --git a/docs/release-trust.md b/docs/release-trust.md index d61ccd2c3d..a008c36ba9 100644 --- a/docs/release-trust.md +++ b/docs/release-trust.md @@ -53,9 +53,11 @@ the archive being published. `--require-signatures` requires signatures that *verify*, not merely exist: each archive must have a non-empty `.sig` sidecar, and that signature must check out against the release signing public key via `cargo xtask verify`. The key is -taken from `--public-key` when given, otherwise from the inline +taken from `--public-key` when given, then from the key file named by +`ROCM_CLI_SIGNING_PUBLIC_KEY_PATH`, then from the inline `ROCM_CLI_SIGNING_PUBLIC_KEY_PEM` that release and nightly CI wire from the -signing-key secret. If neither resolves, the gate fails. It deliberately does +signing-key secret — the same path-before-PEM order `install.sh` and +`cargo xtask package` use. If none resolves, the gate fails. It deliberately does not fall back to checking that a `.sig` file is present: a sidecar produced by the wrong key, truncated, or corrupted would pass such a check, and because an unset GitHub secret expands to the empty string, the fallback would be reached diff --git a/scripts/release_readiness.py b/scripts/release_readiness.py index 1de747c270..b012fbff6f 100644 --- a/scripts/release_readiness.py +++ b/scripts/release_readiness.py @@ -47,9 +47,10 @@ "bin/rocmd", "install.sh", ) +SIGNING_PUBLIC_KEY_PATH_ENV = "ROCM_CLI_SIGNING_PUBLIC_KEY_PATH" SIGNING_PUBLIC_KEY_ENV = "ROCM_CLI_SIGNING_PUBLIC_KEY_PEM" PRODUCTION_TRUST_ENV_NAMES = ( - "ROCM_CLI_SIGNING_PUBLIC_KEY_PATH", + SIGNING_PUBLIC_KEY_PATH_ENV, SIGNING_PUBLIC_KEY_ENV, "ROCM_CLI_METADATA_PUBLIC_KEY_PATH", "ROCM_CLI_METADATA_PUBLIC_KEY_PEM", @@ -466,6 +467,12 @@ def resolve_signing_key(explicit: Path | None) -> tuple[Path | None, str]: ``ROCM_CLI_SIGNING_PUBLIC_KEY_PEM`` itself, so no temporary key file has to be materialized here. + A key file path is preferred over the inline PEM, matching how `install.sh` + and `cargo xtask package` resolve their signing keys. A path that does not + exist is an error from `verify_signature`, not a reason to fall through to + the next source: silently verifying against a different key than the + operator named is the failure mode this gate exists to prevent. + Raises when no key resolves. That is the point of this function: GitHub expands an unset secret to the empty string, so a missing key is indistinguishable from a deliberately absent one. Verification has to fail @@ -474,11 +481,14 @@ def resolve_signing_key(explicit: Path | None) -> tuple[Path | None, str]: """ if explicit is not None: return explicit, f"--public-key {explicit}" + if (path := env_path(SIGNING_PUBLIC_KEY_PATH_ENV)) is not None: + return path, f"${SIGNING_PUBLIC_KEY_PATH_ENV} ({path})" if env_text(SIGNING_PUBLIC_KEY_ENV) is not None: return None, f"${SIGNING_PUBLIC_KEY_ENV}" raise ReadinessError( "signature verification requires a release signing public key: pass " - f"--public-key or set {SIGNING_PUBLIC_KEY_ENV}" + f"--public-key or set {SIGNING_PUBLIC_KEY_PATH_ENV} or " + f"{SIGNING_PUBLIC_KEY_ENV}" ) @@ -939,7 +949,7 @@ def run_self_test(root: Path) -> None: expect_failure( "signature verification with no resolvable key", lambda: run_with_env( - {SIGNING_PUBLIC_KEY_ENV: None}, + {SIGNING_PUBLIC_KEY_ENV: None, SIGNING_PUBLIC_KEY_PATH_ENV: None}, lambda: resolve_signing_key(None), ), ) @@ -949,13 +959,14 @@ def run_self_test(root: Path) -> None: expect_failure( "signature verification with an empty key", lambda: run_with_env( - {SIGNING_PUBLIC_KEY_ENV: ""}, + {SIGNING_PUBLIC_KEY_ENV: "", SIGNING_PUBLIC_KEY_PATH_ENV: ""}, lambda: resolve_signing_key(None), ), ) + inline_pem = "-----BEGIN PUBLIC KEY-----\nself-test\n" env_key, env_source = run_with_env( - {SIGNING_PUBLIC_KEY_ENV: "-----BEGIN PUBLIC KEY-----\nself-test\n"}, + {SIGNING_PUBLIC_KEY_ENV: inline_pem, SIGNING_PUBLIC_KEY_PATH_ENV: None}, lambda: resolve_signing_key(None), ) if env_key is not None or SIGNING_PUBLIC_KEY_ENV not in env_source: @@ -963,9 +974,25 @@ def run_self_test(root: Path) -> None: f"expected the environment key to resolve, got {env_source}" ) + # A key file the operator named must be used, not passed over in favour of + # the inline PEM — and never ignored in favour of failing, which would tell + # them to configure a key they had already configured. + key_path = root / "configured-public-key.pem" + path_key, path_source = run_with_env( + {SIGNING_PUBLIC_KEY_ENV: inline_pem, SIGNING_PUBLIC_KEY_PATH_ENV: key_path}, + lambda: resolve_signing_key(None), + ) + if path_key != key_path or SIGNING_PUBLIC_KEY_PATH_ENV not in path_source: + raise ReadinessError( + f"expected the key path to win over the inline PEM, got {path_source}" + ) + explicit = root / "explicit-public-key.pem" explicit_key, explicit_source = run_with_env( - {SIGNING_PUBLIC_KEY_ENV: "-----BEGIN PUBLIC KEY-----\nself-test\n"}, + { + SIGNING_PUBLIC_KEY_ENV: inline_pem, + SIGNING_PUBLIC_KEY_PATH_ENV: key_path, + }, lambda: resolve_signing_key(explicit), ) if explicit_key != explicit or str(explicit) not in explicit_source: @@ -999,7 +1026,7 @@ def parse_args() -> argparse.Namespace: "--public-key", type=Path, help="Verify detached signatures with this public key instead of the one " - f"in {SIGNING_PUBLIC_KEY_ENV}.", + f"named by {SIGNING_PUBLIC_KEY_PATH_ENV} or {SIGNING_PUBLIC_KEY_ENV}.", ) parser.add_argument( "--require-production-trust", From b1efe5b57e15d1ea0caccf164b94e5ad849437fa Mon Sep 17 00:00:00 2001 From: Roman Sirokov Date: Tue, 6 Oct 2026 11:14:41 +0300 Subject: [PATCH 03/16] fix(release): pin the verify gate with a self-test that can fail `--require-signatures` forcing cryptographic verification is the whole point of this change, and nothing tested it. `main` returns on the `--self-test` branch before the gate, and the self-test's `validate_release` cases pass `verify` explicitly, so the flag-to-verify decision was never reached. Proven rather than assumed: deleting `require_signatures or` from the gate -- which restores the pre-PR weakness, where `--require-signatures` accepts a `.sig` signed by the wrong key whenever no key is configured -- left `--self-test` exiting 0 with "self-test: ok". Extract the decision into `verification_is_required` so the self-test can reach it, and assert each arm: `--require-signatures`, production trust and an explicit `--public-key` each force verification, and nothing else does (an ordinary readiness run must not demand a signing key it has no reason to need). Both mutations now fail the self-test; the unmutated file passes. Not added: a keygen/sign/verify round trip with a wrong-key rejection. The self-test writes a placeholder `.sig` and does no crypto anywhere, so that would introduce an openssl dependency into an otherwise hermetic self-test -- worth doing, but as its own change. Signed-off-by: Roman Sirokov --- scripts/release_readiness.py | 58 +++++++++++++++++++++++++++++++----- 1 file changed, 51 insertions(+), 7 deletions(-) diff --git a/scripts/release_readiness.py b/scripts/release_readiness.py index b012fbff6f..6f8cdc5df5 100644 --- a/scripts/release_readiness.py +++ b/scripts/release_readiness.py @@ -459,6 +459,32 @@ def env_text(name: str) -> str | None: return value +def verification_is_required( + require_signatures: bool, + require_production_trust: bool, + explicit_public_key: Path | None, +) -> bool: + """Whether this run must cryptographically verify signatures. + + Requiring signatures means requiring they verify. Checking only that a + ``.sig`` exists would pass an artifact signed by the wrong key, truncated, + or corrupted -- so whenever signatures are required, so is verification, + and a key that cannot be resolved is a hard failure rather than a quiet + downgrade to the presence check. + + Split out of ``main`` so ``--self-test`` can reach it. ``main`` returns on + the ``--self-test`` branch long before the gate, and the self-test's + ``validate_release`` cases pass ``verify`` explicitly, so nothing exercised + this decision: dropping ``require_signatures`` from it left the self-test + green while removing the gate entirely. + """ + return ( + require_signatures + or require_production_trust + or explicit_public_key is not None + ) + + def resolve_signing_key(explicit: Path | None) -> tuple[Path | None, str]: """Resolve the public key release signatures are verified against. @@ -1000,6 +1026,29 @@ def run_self_test(root: Path) -> None: f"expected --public-key to win over the environment, got {explicit_source}" ) print("release readiness self-test: signing key resolution ok") + + # The verify gate itself. Without these, dropping `require_signatures` + # from `verification_is_required` left `--self-test` green while + # removing the whole point of the gate: `--require-signatures` would + # accept a `.sig` signed by the wrong key whenever no key is + # configured. `main` returns on the `--self-test` branch before the + # gate, and the `validate_release` cases above pass `verify` + # explicitly, so nothing else reaches this decision. + for label, flags in ( + ("--require-signatures", (True, False, None)), + ("--require-production-trust", (False, True, None)), + ("--public-key", (False, False, Path("/nonexistent/key.pem"))), + ): + if not verification_is_required(*flags): + raise ReadinessError( + f"{label} must force cryptographic verification, not just a .sig check" + ) + if verification_is_required(False, False, None): + raise ReadinessError( + "verification must stay off when nothing asks for it, or an ordinary " + "readiness run would demand a signing key it has no reason to need" + ) + print("release readiness self-test: verify gate ok") finally: shutil.rmtree(root, ignore_errors=True) print("release readiness self-test: ok") @@ -1072,13 +1121,8 @@ def main() -> None: require_production_trust = args.require_production_trust or truthy( os.environ.get("ROCM_CLI_REQUIRE_PRODUCTION_TRUST") ) - # Requiring signatures means requiring they verify. Checking only that a - # .sig exists would pass an artifact signed by the wrong key, truncated, or - # corrupted — so whenever signatures are required, so is cryptographic - # verification, and a key that cannot be resolved is a hard failure rather - # than a quiet downgrade to the presence check. - verify = ( - require_signatures or require_production_trust or args.public_key is not None + verify = verification_is_required( + require_signatures, require_production_trust, args.public_key ) public_key: Path | None = None messages: list[str] = [] From b4c9b9ae3c5749ded7a9b91bd93ceb095b86f1d3 Mon Sep 17 00:00:00 2001 From: Roman Sirokov Date: Tue, 6 Oct 2026 16:58:59 +0300 Subject: [PATCH 04/16] fix(release): pin the verify wiring, not just the decision The previous round extracted `verification_is_required` and tested it in isolation. That was not enough, and the comment claiming the self-test now pinned the gate overclaimed: it pinned the decision function while `main`'s use of it stayed untested. Reproduced both holes the review named. Neutralising the call in `main` (`verify = False and verification_is_required(...)`) left `--self-test` exiting 0 -- the original fail-open, where `--require-signatures` accepts a `.sig` signed by the wrong key. Dropping the `resolve_signing_key` call stayed green too. So make the sequence the unit. `resolve_verification` now decides AND resolves the key, returning `(verify, public_key, key_source)`, and `main` holds no copy of that logic -- it cannot drift from what the tests cover because there is nothing left in it to drift. The new self-test case asserts a required verification actually yields a key, and that nothing is resolved when verification is off. Both mutations now fail and name what regressed: neutralise the gate call -> "--require-signatures must force verification through the wiring main uses" skip key resolution -> "verification was required but no key was resolved ... the fail-open this gate closes" The unmutated script passes. Still not covered, and not claimed to be: that `validate_archive` calls `verify_signature`, and the `public_key=None` inline-PEM path CI uses. Both need the keygen/sign round trip noted as a follow-up, rather than another isolated assertion. Signed-off-by: Roman Sirokov --- scripts/release_readiness.py | 73 +++++++++++++++++++++++++++++++++--- 1 file changed, 67 insertions(+), 6 deletions(-) diff --git a/scripts/release_readiness.py b/scripts/release_readiness.py index 6f8cdc5df5..cc23d08b71 100644 --- a/scripts/release_readiness.py +++ b/scripts/release_readiness.py @@ -485,6 +485,35 @@ def verification_is_required( ) +def resolve_verification( + require_signatures: bool, + require_production_trust: bool, + explicit_public_key: Path | None, +) -> tuple[bool, Path | None, str | None]: + """The whole verify wiring: decide, then resolve the key when required. + + Returns ``(verify, public_key, key_source)``; ``key_source`` is ``None`` + when no verification is required. + + This exists because testing the *decision* was not enough. An earlier round + extracted :func:`verification_is_required` and pinned it in isolation, but + nothing exercised ``main``'s use of it -- neutralising the call there + (``verify = False and verification_is_required(...)``) left ``--self-test`` + exiting 0, which is the original fail-open: ``--require-signatures`` would + again accept a ``.sig`` signed by the wrong key. Dropping the + ``resolve_signing_key`` call stayed green for the same reason. + + So the sequence itself is the unit, and ``main`` holds no copy of it. + """ + verify = verification_is_required( + require_signatures, require_production_trust, explicit_public_key + ) + if not verify: + return False, None, None + public_key, key_source = resolve_signing_key(explicit_public_key) + return True, public_key, key_source + + def resolve_signing_key(explicit: Path | None) -> tuple[Path | None, str]: """Resolve the public key release signatures are verified against. @@ -676,6 +705,25 @@ def run_with_env(values: dict[str, str | Path | None], func): os.environ[name] = value +def _assert_verification_wiring(expected_key: Path) -> None: + """`resolve_verification` must decide AND resolve, not just decide.""" + verify, public_key, key_source = resolve_verification(True, False, None) + if not verify: + raise ReadinessError( + "--require-signatures must force verification through the wiring main uses" + ) + if public_key != expected_key or not key_source: + raise ReadinessError( + f"verification was required but no key was resolved: {public_key!r} / {key_source!r}; " + "a required verification that silently resolves no key is the fail-open this gate closes" + ) + off_verify, off_key, off_source = resolve_verification(False, False, None) + if off_verify or off_key is not None or off_source is not None: + raise ReadinessError( + "verification must stay off, and resolve no key, when nothing asks for it" + ) + + def run_self_test(root: Path) -> None: if root.exists(): shutil.rmtree(root) @@ -1049,6 +1097,21 @@ def run_self_test(root: Path) -> None: "readiness run would demand a signing key it has no reason to need" ) print("release readiness self-test: verify gate ok") + # The wiring, not just the decision. Pinning `verification_is_required` + # alone was not enough: `main` could stop using it + # (`verify = False and verification_is_required(...)`) or skip + # `resolve_signing_key` entirely and this self-test stayed green -- + # the original fail-open, where `--require-signatures` accepts a `.sig` + # from the wrong key. `main` now holds no copy of the sequence, so + # these cases cover what it actually does. + run_with_env( + { + SIGNING_PUBLIC_KEY_PATH_ENV: key_path, + SIGNING_PUBLIC_KEY_ENV: None, + }, + lambda: _assert_verification_wiring(key_path), + ) + print("release readiness self-test: verify wiring ok") finally: shutil.rmtree(root, ignore_errors=True) print("release readiness self-test: ok") @@ -1121,14 +1184,12 @@ def main() -> None: require_production_trust = args.require_production_trust or truthy( os.environ.get("ROCM_CLI_REQUIRE_PRODUCTION_TRUST") ) - verify = verification_is_required( - require_signatures, require_production_trust, args.public_key - ) - public_key: Path | None = None messages: list[str] = [] try: - if verify: - public_key, key_source = resolve_signing_key(args.public_key) + verify, public_key, key_source = resolve_verification( + require_signatures, require_production_trust, args.public_key + ) + if key_source is not None: messages.append(f"signature verification key: {key_source}") messages.extend( validate_release( From c654885d5a51c93253a85a284fe955dd711a4fd8 Mon Sep 17 00:00:00 2001 From: Roman Sirokov Date: Wed, 7 Oct 2026 09:49:50 +0300 Subject: [PATCH 05/16] test(release): pin main's use of the verify gate, not just the gate Extracting the decision and then the wiring still left `main`'s call to `resolve_verification` unexercised, because `--self-test` returns long before `main` reaches it. Replacing that call with `verify, public_key, key_source = False, None, None` kept every check green while restoring the original fail-open, where `--require-signatures` accepts a `.sig` signed by the wrong key. Drive `main` itself against a dist that is valid on every other axis, so the gate is the only thing that can reject it: with signatures required and no key resolvable, `main` must exit 1. Mutation results, via `release_readiness.py --self-test` (what ci.yml:412 and :648 run): drop `require_signatures` from verification_is_required -> fails drop `require_production_trust` -> fails make resolve_verification skip resolve_signing_key -> fails neutralise main's resolve_verification call -> fails (was passing) Each fails with its own message. The new case is not vacuous: under the last mutation the same dist is accepted with `release readiness: ok`, and unmutated it is rejected for the key specifically. Signed-off-by: Roman Sirokov --- scripts/release_readiness.py | 58 ++++++++++++++++++++++++++++++++++++ 1 file changed, 58 insertions(+) diff --git a/scripts/release_readiness.py b/scripts/release_readiness.py index cc23d08b71..97766302e4 100644 --- a/scripts/release_readiness.py +++ b/scripts/release_readiness.py @@ -724,6 +724,44 @@ def _assert_verification_wiring(expected_key: Path) -> None: ) +def _assert_main_consults_the_gate(dist: Path) -> None: + """`main` must route its verify decision through `resolve_verification`. + + Pinning the helpers was not enough on its own. `--self-test` returns before + `main` reaches the gate, so swapping that call for constants + (`verify, public_key, key_source = False, None, None`) left every check + green while restoring the original fail-open: `--require-signatures` + accepting a `.sig` signed by the wrong key. So drive `main` itself. With + signatures required and no key resolvable, it must exit 1; the dist is + otherwise valid, so a `main` that skipped the gate would accept it on the + presence check alone and exit 0. + """ + + def attempt() -> None: + saved_argv = sys.argv + sys.argv = ["release_readiness.py", "--dist", str(dist), "--require-signatures"] + try: + main() + finally: + sys.argv = saved_argv + + try: + attempt() + except SystemExit as exit_error: + if exit_error.code == 1: + return + # argparse exits 2; treat anything that is not the readiness failure as + # a broken test rather than a passing gate. + raise ReadinessError( + f"main exited {exit_error.code!r} rather than failing the readiness " + "check; the gate self-test is no longer exercising what it claims" + ) from exit_error + raise ReadinessError( + "main accepted a signature-required run with no key configured; its " + "verify decision no longer goes through resolve_verification" + ) + + def run_self_test(root: Path) -> None: if root.exists(): shutil.rmtree(root) @@ -1112,6 +1150,26 @@ def run_self_test(root: Path) -> None: lambda: _assert_verification_wiring(key_path), ) print("release readiness self-test: verify wiring ok") + + # ...and `main`'s use of that wiring, which none of the above reaches. + # A dist that is valid on every other axis, so the only thing that can + # reject it is the gate. + gate_dist = root / "gate-dist" + gate_dist.mkdir() + gate_archive = gate_dist / "rocm-cli-test-linux-amd64.tar.gz" + create_test_tar(gate_archive, "rocm-cli-test-linux-amd64") + write_sha(gate_archive) + Path(f"{gate_archive}.sig").write_bytes(b"detached signature placeholder\n") + run_with_env( + { + SIGNING_PUBLIC_KEY_PATH_ENV: None, + SIGNING_PUBLIC_KEY_ENV: None, + "ROCM_CLI_REQUIRE_SIGNATURE": None, + "ROCM_CLI_REQUIRE_PRODUCTION_TRUST": None, + }, + lambda: _assert_main_consults_the_gate(gate_dist), + ) + print("release readiness self-test: main consults the verify gate ok") finally: shutil.rmtree(root, ignore_errors=True) print("release readiness self-test: ok") From 90a664bef83d294896bc2d654baf7cc970414e0d Mon Sep 17 00:00:00 2001 From: Roman Sirokov Date: Wed, 7 Oct 2026 13:25:47 +0300 Subject: [PATCH 06/16] test(release): pin that main verifies, not just that it refuses The previous round's case drove only the no-key path. That path fails inside `resolve_verification`, before `validate_release` is reached, so `main` could still pass `verify=False` at that call and verify nothing while printing the key line as though it had. It also accepted any `SystemExit(1)`, so a `main` that skipped the gate and failed for an unrelated reason counted as a pass. Both were reproduced here before changing anything. Record the call instead: with a key resolvable, a `--require-signatures` run must succeed AND have reached `verify_signature`. The refusal case now asserts the message, not just the exit code. Mutation results via `release_readiness.py --self-test` (what ci.yml:412 and :648 run) -- the first three were green before this commit: main passes verify=False to validate_release -> fails gate skipped, main exits 1 for an unrelated reason -> fails verify_signature call removed from validate_archive -> fails drop require_signatures from the decision -> fails drop require_production_trust -> fails resolve_verification skips resolve_signing_key -> fails neutralise main's resolve_verification call -> fails Also prints the signature-verification key immediately instead of collecting it into `messages`, which is only flushed on success: a run that fails verification is exactly when you need to know which key it used. Flushed so it cannot land after the unbuffered stderr of `fail()`. The inner `main()` calls now capture stdout and stderr, so a passing self-test no longer prints `release readiness failed: ...` into CI logs. Comments reworded to state the invariant each case guards rather than the history of how it got there. Signed-off-by: Roman Sirokov --- scripts/release_readiness.py | 164 ++++++++++++++++++++--------------- 1 file changed, 94 insertions(+), 70 deletions(-) diff --git a/scripts/release_readiness.py b/scripts/release_readiness.py index 97766302e4..e76e3f99ef 100644 --- a/scripts/release_readiness.py +++ b/scripts/release_readiness.py @@ -8,6 +8,7 @@ from __future__ import annotations import argparse +import contextlib import hashlib import io import os @@ -472,11 +473,8 @@ def verification_is_required( and a key that cannot be resolved is a hard failure rather than a quiet downgrade to the presence check. - Split out of ``main`` so ``--self-test`` can reach it. ``main`` returns on - the ``--self-test`` branch long before the gate, and the self-test's - ``validate_release`` cases pass ``verify`` explicitly, so nothing exercised - this decision: dropping ``require_signatures`` from it left the self-test - green while removing the gate entirely. + Separate from ``main`` so ``--self-test`` can exercise the decision + directly; ``main`` returns on the ``--self-test`` branch before the gate. """ return ( require_signatures @@ -495,15 +493,9 @@ def resolve_verification( Returns ``(verify, public_key, key_source)``; ``key_source`` is ``None`` when no verification is required. - This exists because testing the *decision* was not enough. An earlier round - extracted :func:`verification_is_required` and pinned it in isolation, but - nothing exercised ``main``'s use of it -- neutralising the call there - (``verify = False and verification_is_required(...)``) left ``--self-test`` - exiting 0, which is the original fail-open: ``--require-signatures`` would - again accept a ``.sig`` signed by the wrong key. Dropping the - ``resolve_signing_key`` call stayed green for the same reason. - - So the sequence itself is the unit, and ``main`` holds no copy of it. + Deciding and resolving belong together: a run that requires verification + but resolves no key would verify nothing while reporting success. Keeping + both here means ``main`` holds no copy of the sequence. """ verify = verification_is_required( require_signatures, require_production_trust, explicit_public_key @@ -724,42 +716,77 @@ def _assert_verification_wiring(expected_key: Path) -> None: ) -def _assert_main_consults_the_gate(dist: Path) -> None: - """`main` must route its verify decision through `resolve_verification`. +def _run_main(dist: Path, *flags: str) -> tuple[int | None, str]: + """Drive `main` in-process. Returns its exit code (`None` if it returned). - Pinning the helpers was not enough on its own. `--self-test` returns before - `main` reaches the gate, so swapping that call for constants - (`verify, public_key, key_source = False, None, None`) left every check - green while restoring the original fail-open: `--require-signatures` - accepting a `.sig` signed by the wrong key. So drive `main` itself. With - signatures required and no key resolvable, it must exit 1; the dist is - otherwise valid, so a `main` that skipped the gate would accept it on the - presence check alone and exit 0. + Stderr is captured so a case that expects `main` to fail does not print + `release readiness failed: ...` into the log of a passing run. """ + saved_argv = sys.argv + sys.argv = ["release_readiness.py", "--dist", str(dist), *flags] + captured = io.StringIO() + try: + with ( + contextlib.redirect_stderr(captured), + contextlib.redirect_stdout(io.StringIO()), + ): + try: + main() + except SystemExit as exit_error: + code = exit_error.code + return (code if isinstance(code, int) else 1), captured.getvalue() + return None, captured.getvalue() + finally: + sys.argv = saved_argv - def attempt() -> None: - saved_argv = sys.argv - sys.argv = ["release_readiness.py", "--dist", str(dist), "--require-signatures"] - try: - main() - finally: - sys.argv = saved_argv - try: - attempt() - except SystemExit as exit_error: - if exit_error.code == 1: - return - # argparse exits 2; treat anything that is not the readiness failure as - # a broken test rather than a passing gate. +def _assert_main_refuses_without_a_key(dist: Path) -> None: + """`--require-signatures` with no key must fail, and say so.""" + code, stderr = _run_main(dist, "--require-signatures") + if code is None: raise ReadinessError( - f"main exited {exit_error.code!r} rather than failing the readiness " - "check; the gate self-test is no longer exercising what it claims" - ) from exit_error - raise ReadinessError( - "main accepted a signature-required run with no key configured; its " - "verify decision no longer goes through resolve_verification" + "main accepted a signature-required run with no key configured; " + "that is the fail-open this gate exists to close" + ) + # The reason matters, not just the exit code: a `main` that skipped the + # gate and happened to fail for some unrelated reason would otherwise pass. + if "requires a release signing public key" not in stderr: + raise ReadinessError( + "main failed, but not on the missing signing key -- the gate case is " + f"no longer exercising what it claims. stderr was: {stderr.strip()!r}" + ) + + +def _assert_main_verifies(dist: Path) -> None: + """`main` must reach `verify_signature`, not merely decide that it should. + + Driving only the refusal path is not enough. It fails inside + `resolve_verification`, before `validate_release` is reached, so `main` + could still pass `verify=False` at that call and never verify anything -- + the original fail-open, with the key line printed as though it had. Record + the call instead: with a key resolvable, a `--require-signatures` run must + both succeed and have asked for verification. + """ + verified: list[Path] = [] + module = sys.modules[__name__] + original = module.verify_signature + module.verify_signature = lambda archive, signature, public_key: verified.append( + archive ) + try: + code, stderr = _run_main(dist, "--require-signatures") + finally: + module.verify_signature = original + if code is not None: + raise ReadinessError( + f"main rejected a run it should have accepted (exit {code}): " + f"{stderr.strip()!r}" + ) + if not verified: + raise ReadinessError( + "main completed a --require-signatures run without verifying any " + "signature; the verify decision is not reaching validate_release" + ) def run_self_test(root: Path) -> None: @@ -1113,13 +1140,7 @@ def run_self_test(root: Path) -> None: ) print("release readiness self-test: signing key resolution ok") - # The verify gate itself. Without these, dropping `require_signatures` - # from `verification_is_required` left `--self-test` green while - # removing the whole point of the gate: `--require-signatures` would - # accept a `.sig` signed by the wrong key whenever no key is - # configured. `main` returns on the `--self-test` branch before the - # gate, and the `validate_release` cases above pass `verify` - # explicitly, so nothing else reaches this decision. + # Each flag must force verification on, and nothing else may turn it on. for label, flags in ( ("--require-signatures", (True, False, None)), ("--require-production-trust", (False, True, None)), @@ -1135,13 +1156,7 @@ def run_self_test(root: Path) -> None: "readiness run would demand a signing key it has no reason to need" ) print("release readiness self-test: verify gate ok") - # The wiring, not just the decision. Pinning `verification_is_required` - # alone was not enough: `main` could stop using it - # (`verify = False and verification_is_required(...)`) or skip - # `resolve_signing_key` entirely and this self-test stayed green -- - # the original fail-open, where `--require-signatures` accepts a `.sig` - # from the wrong key. `main` now holds no copy of the sequence, so - # these cases cover what it actually does. + # Requiring verification must also resolve a key, not just set a flag. run_with_env( { SIGNING_PUBLIC_KEY_PATH_ENV: key_path, @@ -1151,25 +1166,27 @@ def run_self_test(root: Path) -> None: ) print("release readiness self-test: verify wiring ok") - # ...and `main`'s use of that wiring, which none of the above reaches. - # A dist that is valid on every other axis, so the only thing that can - # reject it is the gate. + # `main`'s own behaviour, on a dist that is valid on every other axis so + # the verify gate is the only thing either case can turn on. gate_dist = root / "gate-dist" gate_dist.mkdir() gate_archive = gate_dist / "rocm-cli-test-linux-amd64.tar.gz" create_test_tar(gate_archive, "rocm-cli-test-linux-amd64") write_sha(gate_archive) Path(f"{gate_archive}.sig").write_bytes(b"detached signature placeholder\n") + no_key_env = { + SIGNING_PUBLIC_KEY_PATH_ENV: None, + SIGNING_PUBLIC_KEY_ENV: None, + "ROCM_CLI_REQUIRE_SIGNATURE": None, + "ROCM_CLI_REQUIRE_PRODUCTION_TRUST": None, + } + run_with_env(no_key_env, lambda: _assert_main_refuses_without_a_key(gate_dist)) + print("release readiness self-test: main refuses without a key ok") run_with_env( - { - SIGNING_PUBLIC_KEY_PATH_ENV: None, - SIGNING_PUBLIC_KEY_ENV: None, - "ROCM_CLI_REQUIRE_SIGNATURE": None, - "ROCM_CLI_REQUIRE_PRODUCTION_TRUST": None, - }, - lambda: _assert_main_consults_the_gate(gate_dist), + {**no_key_env, SIGNING_PUBLIC_KEY_PATH_ENV: key_path}, + lambda: _assert_main_verifies(gate_dist), ) - print("release readiness self-test: main consults the verify gate ok") + print("release readiness self-test: main actually verifies ok") finally: shutil.rmtree(root, ignore_errors=True) print("release readiness self-test: ok") @@ -1248,7 +1265,14 @@ def main() -> None: require_signatures, require_production_trust, args.public_key ) if key_source is not None: - messages.append(f"signature verification key: {key_source}") + # Printed now rather than collected into `messages`, which is only + # flushed on success: a run that fails verification is exactly when + # you need to know which key it used. Flushed so it cannot be + # reordered after the unbuffered stderr of `fail()` when piped. + print( + f"release readiness: signature verification key: {key_source}", + flush=True, + ) messages.extend( validate_release( Path(args.dist), From a9ec22a1a391d78b449535d9c21f7040ab6e7509 Mon Sep 17 00:00:00 2001 From: Roman Sirokov Date: Wed, 7 Oct 2026 13:28:05 +0300 Subject: [PATCH 07/16] docs(release): record the verification trigger that was removed `--require-signatures` implying verification replaced an older trigger: a key present in ROCM_CLI_SIGNING_PUBLIC_KEY_PEM used to make a run verify opportunistically, with no flag asked for. That narrowing was never written down, so a reader would expect a configured key to still be checked. Also notes that the key line is printed as soon as the key resolves, so it survives a failed run, and stops describing install.sh (a public-key resolver) and cargo xtask package (a private-key one) as if they resolved the same key. Signed-off-by: Roman Sirokov --- docs/release-trust.md | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/docs/release-trust.md b/docs/release-trust.md index a008c36ba9..95537ba4df 100644 --- a/docs/release-trust.md +++ b/docs/release-trust.md @@ -56,13 +56,25 @@ against the release signing public key via `cargo xtask verify`. The key is taken from `--public-key` when given, then from the key file named by `ROCM_CLI_SIGNING_PUBLIC_KEY_PATH`, then from the inline `ROCM_CLI_SIGNING_PUBLIC_KEY_PEM` that release and nightly CI wire from the -signing-key secret — the same path-before-PEM order `install.sh` and -`cargo xtask package` use. If none resolves, the gate fails. It deliberately does +signing-key secret — the same path-before-PEM precedence `install.sh` applies to +the public key, and `cargo xtask package` to the private one. If none resolves, +the gate fails. It deliberately does not fall back to checking that a `.sig` file is present: a sidecar produced by the wrong key, truncated, or corrupted would pass such a check, and because an unset GitHub secret expands to the empty string, the fallback would be reached silently whenever the secret was removed or rotated away. The log line -`signature verification key:` names the key the run actually used. +`signature verification key:` names the key the run actually used, and is +printed as soon as the key resolves so that a run which then fails verification +still records which trust root it used. + +Verification is driven by what the run asks for, not by what happens to be in +the environment: `--require-signatures`, `--require-production-trust` or an +explicit `--public-key`. Previously a key present in +`ROCM_CLI_SIGNING_PUBLIC_KEY_PEM` was enough to opportunistically verify even +without those flags; that trigger was removed when the flags began to imply +verification, so a readiness run that asks for neither now checks neither +signature presence nor validity. Every `release.yml` and `nightly.yml` +invocation passes `--require-signatures`, so no workflow takes that path. Release CI also enables `--require-rocm-asset-names`, which rejects branch-like, path-like, or otherwise unsupported archive names before upload. From 3738f70b8ee088ee17f46e8b158beafd6dbdb167 Mon Sep 17 00:00:00 2001 From: Roman Sirokov Date: Wed, 7 Oct 2026 14:37:58 +0300 Subject: [PATCH 08/16] test(release): pin which key verification uses, and that the log names it The previous case recorded only that `verify_signature` was reached. It threw away `public_key` and discarded stdout, so three separate edits stayed green: `main` passing `public_key=None`, `verify_signature` dropping its `key_args`, and the key log line going away. Each is silent in production too -- `cargo xtask verify` falls back to the inline PEM when `--public-key` is omitted, so the run would verify against one key while reporting another. Intercept at the subprocess boundary instead of stubbing `verify_signature`, so the real argv is built and the `--public-key` handoff is covered. The case now asserts three things together: that verification happened, that every invocation names the resolved key, and that stdout reports that same key. This needs a key file that exists, since the real `verify_signature` rejects a named key that is not a file. Added one rather than reusing `key_path`, which stays absent on purpose to pin that a named-but- missing key still wins resolution instead of falling through to the PEM. Mutation results via `release_readiness.py --self-test` -- the first three were green before this commit: main passes public_key=None to validate_release -> fails verify_signature drops its key_args -> fails key log line removed, and replaced with a wrong one-> fails main passes verify=False to validate_release -> fails verify_signature call removed from validate_archive-> fails drop require_signatures from the decision -> fails resolve_verification skips resolve_signing_key -> fails neutralise main's resolve_verification call -> fails Signed-off-by: Roman Sirokov --- scripts/release_readiness.py | 98 +++++++++++++++++++++++++----------- 1 file changed, 69 insertions(+), 29 deletions(-) diff --git a/scripts/release_readiness.py b/scripts/release_readiness.py index e76e3f99ef..82e4b611cf 100644 --- a/scripts/release_readiness.py +++ b/scripts/release_readiness.py @@ -716,33 +716,36 @@ def _assert_verification_wiring(expected_key: Path) -> None: ) -def _run_main(dist: Path, *flags: str) -> tuple[int | None, str]: - """Drive `main` in-process. Returns its exit code (`None` if it returned). +def _run_main(dist: Path, *flags: str) -> tuple[int | None, str, str]: + """Drive `main` in-process. - Stderr is captured so a case that expects `main` to fail does not print - `release readiness failed: ...` into the log of a passing run. + Returns ``(exit code or None if it returned, stdout, stderr)``. Both streams + are captured: stderr so a case expecting failure does not print + ``release readiness failed: ...`` into the log of a passing run, and stdout + so cases can assert on what the run claimed it did. """ saved_argv = sys.argv sys.argv = ["release_readiness.py", "--dist", str(dist), *flags] - captured = io.StringIO() + out, err = io.StringIO(), io.StringIO() try: - with ( - contextlib.redirect_stderr(captured), - contextlib.redirect_stdout(io.StringIO()), - ): + with contextlib.redirect_stderr(err), contextlib.redirect_stdout(out): try: main() except SystemExit as exit_error: code = exit_error.code - return (code if isinstance(code, int) else 1), captured.getvalue() - return None, captured.getvalue() + return ( + (code if isinstance(code, int) else 1), + out.getvalue(), + err.getvalue(), + ) + return None, out.getvalue(), err.getvalue() finally: sys.argv = saved_argv def _assert_main_refuses_without_a_key(dist: Path) -> None: """`--require-signatures` with no key must fail, and say so.""" - code, stderr = _run_main(dist, "--require-signatures") + code, _stdout, stderr = _run_main(dist, "--require-signatures") if code is None: raise ReadinessError( "main accepted a signature-required run with no key configured; " @@ -757,36 +760,67 @@ def _assert_main_refuses_without_a_key(dist: Path) -> None: ) -def _assert_main_verifies(dist: Path) -> None: - """`main` must reach `verify_signature`, not merely decide that it should. +def _assert_main_verifies(dist: Path, expected_key: Path) -> None: + """`main` must verify, against the resolved key, and say which key that was. - Driving only the refusal path is not enough. It fails inside - `resolve_verification`, before `validate_release` is reached, so `main` - could still pass `verify=False` at that call and never verify anything -- - the original fail-open, with the key line printed as though it had. Record - the call instead: with a key resolvable, a `--require-signatures` run must - both succeed and have asked for verification. + Three things have to hold together, because each is separately silent: + verification has to happen at all; it has to use the key + `resolve_signing_key` chose, since `cargo xtask verify` falls back to the + inline PEM when `--public-key` is omitted and would then check a different + key than the run reports; and the `signature verification key:` line has to + name that same key, or the log makes a claim nothing backs. """ - verified: list[Path] = [] + # Intercept at the subprocess boundary rather than stubbing + # `verify_signature`, so the real argv is built and the `--public-key` + # handoff is covered too. Stubbing the function instead leaves `key_args` + # unexercised: dropping it stays green while verification silently falls + # back to whatever the inline PEM holds. + argvs: list[list[str]] = [] + + class _Completed: + returncode = 0 + stdout = "" + module = sys.modules[__name__] - original = module.verify_signature - module.verify_signature = lambda archive, signature, public_key: verified.append( - archive + original_run, original_which = module.subprocess.run, module.shutil.which + module.subprocess.run = lambda argv, **_kwargs: ( + argvs.append(list(argv)), + _Completed(), + )[1] + module.shutil.which = lambda name: ( + "/fake/cargo" if name == "cargo" else original_which(name) ) try: - code, stderr = _run_main(dist, "--require-signatures") + code, stdout, stderr = _run_main(dist, "--require-signatures") finally: - module.verify_signature = original + module.subprocess.run, module.shutil.which = original_run, original_which if code is not None: raise ReadinessError( f"main rejected a run it should have accepted (exit {code}): " f"{stderr.strip()!r}" ) - if not verified: + if not argvs: raise ReadinessError( "main completed a --require-signatures run without verifying any " "signature; the verify decision is not reaching validate_release" ) + resolved = str(expected_key.resolve()) + for argv in argvs: + if "--public-key" not in argv or resolved not in argv: + raise ReadinessError( + f"verification ran without --public-key {resolved}: {argv!r}. " + "`cargo xtask verify` falls back to the inline PEM when the flag " + "is omitted, so this would check a different key than the run " + "reports." + ) + expected_line = ( + f"signature verification key: ${SIGNING_PUBLIC_KEY_PATH_ENV} ({expected_key})" + ) + if expected_line not in stdout: + raise ReadinessError( + f"the run did not report the key it used; expected {expected_line!r} " + f"in stdout, got {stdout.strip()!r}" + ) def run_self_test(root: Path) -> None: @@ -1182,9 +1216,15 @@ def run_self_test(root: Path) -> None: } run_with_env(no_key_env, lambda: _assert_main_refuses_without_a_key(gate_dist)) print("release readiness self-test: main refuses without a key ok") + # A key that exists on disk, because this case lets the real + # `verify_signature` run and it rejects a named key that is not a file. + # `key_path` above stays absent on purpose, to pin that a named-but- + # missing key still wins resolution rather than falling through. + real_key = root / "present-public-key.pem" + real_key.write_text(inline_pem, encoding="ascii") run_with_env( - {**no_key_env, SIGNING_PUBLIC_KEY_PATH_ENV: key_path}, - lambda: _assert_main_verifies(gate_dist), + {**no_key_env, SIGNING_PUBLIC_KEY_PATH_ENV: real_key}, + lambda: _assert_main_verifies(gate_dist, real_key), ) print("release readiness self-test: main actually verifies ok") finally: From e02aacf5282fde1becfc4ee2685755596790fee6 Mon Sep 17 00:00:00 2001 From: Roman Sirokov Date: Wed, 7 Oct 2026 15:32:07 +0300 Subject: [PATCH 09/16] fix(xtask): cross-check a path-supplied CI signing key too MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `resolve_signing_key` in scripts/release_readiness.py prefers ROCM_CLI_SIGNING_PUBLIC_KEY_PATH over the inline PEM and verifies release artifacts against whichever it picks. This check read only the PEM, so a path-supplied key was never compared with docs/keys/ -- and the run still logged "ci public key: not configured — cross-check skipped" while that key was in use. The log line this PR added was false in exactly the configuration the PR made reachable. Resolve both sources here, in the same order the gate uses, so the key cross-checked is always the key verified with. A named path that cannot be read is an error rather than a fall-through to the PEM: silently comparing a different key than the operator named is the failure this check exists to catch. Blank counts as absent on both, matching the gate and the shape of an unset GitHub secret. `check_ci_public_key` is unchanged and still takes PEM text, so its existing cases keep their meaning; resolution is a separate pure function taking both values as arguments, so the new test needs no process-environment mutation. NOT YET VERIFIED ON LINUX: the devbox is unreachable (stale Keycloak client secret) and macOS cannot build this workspace. Checked locally: rustfmt parses and reports no change, `Context` is in scope for with_context, and tempfile is an existing xtask dev-dependency used the same way in architecture_doc.rs and e2e_report.rs. `cargo clippy --workspace --all-targets -- -D warnings` and `cargo test -p xtask verify_pinned_keys` still owe a run before this is trusted. Signed-off-by: Roman Sirokov --- xtask/src/verify_pinned_keys.rs | 108 +++++++++++++++++++++++++++++--- 1 file changed, 101 insertions(+), 7 deletions(-) diff --git a/xtask/src/verify_pinned_keys.rs b/xtask/src/verify_pinned_keys.rs index 38583d7dc4..0676c08f9d 100644 --- a/xtask/src/verify_pinned_keys.rs +++ b/xtask/src/verify_pinned_keys.rs @@ -26,9 +26,13 @@ //! than the specific constant) would let a wrong/tampered constant pass whenever //! the canonical body appears anywhere else in the file — e.g. in the `NEXT` slot //! mid-rotation or a stale comment. -//! - **CI cross-check.** When `ROCM_CLI_SIGNING_PUBLIC_KEY_PEM` is set (as in -//! release/nightly CI), it must equal the canonical *current* release key, so the -//! key CI verifies against is exactly the one users pin. +//! - **CI cross-check.** When a CI signing public key is configured — the file +//! named by `ROCM_CLI_SIGNING_PUBLIC_KEY_PATH`, else the inline +//! `ROCM_CLI_SIGNING_PUBLIC_KEY_PEM` release/nightly CI wires from the secret — +//! it must equal the canonical *current* release key, so the key CI verifies +//! against is exactly the one users pin. Both sources are resolved in the same +//! order `scripts/release_readiness.py` uses, so the key cross-checked here is +//! always the key that gate verified with. use std::path::{Path, PathBuf}; @@ -190,9 +194,9 @@ fn extract_constant<'a>(source: &'a str, token: &str, embedding: Embedding) -> O } /// Check every populated canonical key against its pinned sources. `ci_public_key` -/// is the inline PEM CI would verify against (from `ROCM_CLI_SIGNING_PUBLIC_KEY_PEM`) -/// or `None`; it is passed in rather than read here so tests need not mutate the -/// process environment. +/// is the PEM text CI would verify against, already resolved by +/// [`resolve_ci_public_key`] from either key source, or `None`; it is passed in +/// rather than read here so tests need not mutate the process environment. fn check_pinned_keys(root: &Path, ci_public_key: Option<&str>) -> Result> { let mut messages = Vec::new(); let mut current_release_body: Option = None; @@ -294,6 +298,38 @@ fn check_ci_public_key( ]) } +/// Resolve the CI signing public key the way `scripts/release_readiness.py`'s +/// `resolve_signing_key` does: the file named by `ROCM_CLI_SIGNING_PUBLIC_KEY_PATH` +/// first, then the inline `ROCM_CLI_SIGNING_PUBLIC_KEY_PEM`. +/// +/// Reading only the PEM would make the cross-check skip silently whenever the key +/// was supplied by path, while the readiness gate verified release artifacts +/// against that very key — a trust root nobody compared with `docs/keys/`, and +/// reported in the log as "not configured". +/// +/// A named path that cannot be read is an error, not a fall-through to the PEM: +/// cross-checking a different key than the operator named is the failure this +/// check exists to catch. Blank counts as absent, because an unset GitHub secret +/// expands to the empty string. +fn resolve_ci_public_key(path_var: Option<&str>, pem_var: Option<&str>) -> Result> { + if let Some(path) = path_var.filter(|value| !value.trim().is_empty()) { + // The raw value rather than a trimmed one, so this and + // `release_readiness.py` always mean the same file. + let path = Path::new(path); + let pem = std::fs::read_to_string(path).with_context(|| { + format!( + "failed to read the CI signing public key named by \ + ROCM_CLI_SIGNING_PUBLIC_KEY_PATH: {}", + path.display() + ) + })?; + return Ok(Some(pem)); + } + Ok(pem_var + .filter(|value| !value.trim().is_empty()) + .map(str::to_owned)) +} + fn repo_root() -> PathBuf { // `CARGO_MANIFEST_DIR` is the `xtask/` crate dir; its parent is the repo root, // so this is correct regardless of the working directory CI invokes us from. @@ -304,7 +340,9 @@ fn repo_root() -> PathBuf { } pub fn run() -> Result<()> { - let ci_public_key = std::env::var("ROCM_CLI_SIGNING_PUBLIC_KEY_PEM").ok(); + let path_var = std::env::var("ROCM_CLI_SIGNING_PUBLIC_KEY_PATH").ok(); + let pem_var = std::env::var("ROCM_CLI_SIGNING_PUBLIC_KEY_PEM").ok(); + let ci_public_key = resolve_ci_public_key(path_var.as_deref(), pem_var.as_deref())?; for message in check_pinned_keys(&repo_root(), ci_public_key.as_deref())? { println!("pinned key check: {message}"); } @@ -412,4 +450,60 @@ mod tests { // Malformed CI key -> rejected. assert!(check_ci_public_key(Some("not a pem"), Some(¤t)).is_err()); } + + /// The cross-check must see a key supplied by path, not just the inline PEM. + /// + /// `release_readiness.py` prefers `ROCM_CLI_SIGNING_PUBLIC_KEY_PATH` and + /// verifies artifacts against it. Resolving only the PEM here meant that key + /// was never compared with `docs/keys/`, and the run still logged + /// "not configured — cross-check skipped" while a key was in use. + #[test] + fn ci_public_key_resolves_the_path_variable_before_the_inline_pem() { + let dir = tempfile::tempdir().expect("tempdir"); + let key_file = dir.path().join("ci-public-key.pem"); + std::fs::write(&key_file, SAMPLE).expect("write key"); + let named = key_file.to_str().expect("utf-8 path"); + let other = SAMPLE.replace("test", "diff"); + + // Path wins over the PEM, so the key cross-checked is the key verified with. + let resolved = resolve_ci_public_key(Some(named), Some(&other)) + .expect("readable key") + .expect("a key"); + assert_eq!(pem_body(&resolved).unwrap(), pem_body(SAMPLE).unwrap()); + + // ...and that resolved key is what reaches the comparison: a path key that + // disagrees with the canonical one must fail rather than skip. + let canonical = pem_body(&other).unwrap(); + assert!(check_ci_public_key(Some(&resolved), Some(&canonical)).is_err()); + + // The PEM is still used when no path is named. + let from_pem = resolve_ci_public_key(None, Some(SAMPLE)) + .expect("inline pem") + .expect("a key"); + assert_eq!(from_pem, SAMPLE); + + // Blank counts as absent on both, which is the shape of an unset secret. + for blank in [None, Some(""), Some(" ")] { + assert!( + resolve_ci_public_key(blank, blank) + .expect("no key") + .is_none(), + "blank {blank:?} should resolve to no key" + ); + } + assert_eq!( + resolve_ci_public_key(Some(" "), Some(SAMPLE)) + .expect("falls through to pem") + .as_deref(), + Some(SAMPLE) + ); + + // A named key that cannot be read is an error, never a quiet fall-through + // to a different key than the operator named. + let missing = dir.path().join("absent.pem"); + assert!( + resolve_ci_public_key(missing.to_str(), Some(SAMPLE)).is_err(), + "an unreadable named key must fail rather than fall back to the PEM" + ); + } } From 2531243c7e9cdfc508d76feee72d042523f37d25 Mon Sep 17 00:00:00 2001 From: Roman Sirokov Date: Thu, 8 Oct 2026 11:47:22 +0300 Subject: [PATCH 10/16] test(release): drive main on the paths the gate's own promises live on The self-test exercised `main` only on the `_PATH` success path, so four single-line edits in `main` stayed green: key line collected into `messages` (flushed only on success) key line gated on `public_key is not None` main passing False for require_production_trust main passing None for args.public_key The first two break promises `docs/release-trust.md` makes: that the line names the key a run used, including a run that then fails, and that it appears for the inline PEM -- the source release and nightly actually use, where no key file exists to name. The last two leave `--require-production-trust` and `--public-key` pinned only on the `verification_is_required` helper, not on `main`'s call to it. Adds three cases, built on a shared `_run_main_verifying` harness that can make the stubbed verify fail: the key line must survive a failed verification; an inline-PEM run must report the PEM label and pass no `--public-key`; and each remaining trigger must make `main` verify. The failing-verify case also covers the previously untested branch where `cargo xtask verify` returns non-zero -- the PR's headline behaviour, which no automated case reached. Also names both environment triggers in `docs/release-trust.md` and in the missing-key error, since packaging steps export ROCM_CLI_REQUIRE_SIGNATURE and a developer who did got no hint why the gate fired. Mutation results via `release_readiness.py --self-test`; each mutant is asserted to differ from the baseline before running, because a mutant that fails to apply is indistinguishable from one that survives: M14 messages-instead-of-print -> fails (was passing) M26 print gated on public_key -> fails (was passing) M23 False for production trust -> fails (was passing) M24 None for args.public_key -> fails (was passing) M21 verify failure ignored -> fails (was passing) K1/K2/R1/A/C earlier mutations -> fail Signed-off-by: Roman Sirokov --- docs/release-trust.md | 7 +- scripts/release_readiness.py | 153 ++++++++++++++++++++++++++++++----- 2 files changed, 139 insertions(+), 21 deletions(-) diff --git a/docs/release-trust.md b/docs/release-trust.md index 95537ba4df..90f4380c1e 100644 --- a/docs/release-trust.md +++ b/docs/release-trust.md @@ -69,7 +69,12 @@ still records which trust root it used. Verification is driven by what the run asks for, not by what happens to be in the environment: `--require-signatures`, `--require-production-trust` or an -explicit `--public-key`. Previously a key present in +explicit `--public-key` — or the environment equivalents of the first two, +`ROCM_CLI_REQUIRE_SIGNATURE` and `ROCM_CLI_REQUIRE_PRODUCTION_TRUST`, which +`release.yml` and `nightly.yml` set in the same step that runs the gate. Note +that exporting `ROCM_CLI_REQUIRE_SIGNATURE=1` for packaging therefore also +requires a resolvable signing key, where before it needed only a `.sig`. +Previously a key present in `ROCM_CLI_SIGNING_PUBLIC_KEY_PEM` was enough to opportunistically verify even without those flags; that trigger was removed when the flags began to imply verification, so a readiness run that asks for neither now checks neither diff --git a/scripts/release_readiness.py b/scripts/release_readiness.py index 82e4b611cf..6e967c3cc2 100644 --- a/scripts/release_readiness.py +++ b/scripts/release_readiness.py @@ -535,7 +535,10 @@ def resolve_signing_key(explicit: Path | None) -> tuple[Path | None, str]: raise ReadinessError( "signature verification requires a release signing public key: pass " f"--public-key or set {SIGNING_PUBLIC_KEY_PATH_ENV} or " - f"{SIGNING_PUBLIC_KEY_ENV}" + f"{SIGNING_PUBLIC_KEY_ENV}. Verification is requested by " + "--require-signatures, --require-production-trust, --public-key, or the " + "ROCM_CLI_REQUIRE_SIGNATURE / ROCM_CLI_REQUIRE_PRODUCTION_TRUST " + "environment variables -- the last of which packaging steps export." ) @@ -760,26 +763,27 @@ def _assert_main_refuses_without_a_key(dist: Path) -> None: ) -def _assert_main_verifies(dist: Path, expected_key: Path) -> None: - """`main` must verify, against the resolved key, and say which key that was. +def _run_main_verifying( + dist: Path, *flags: str, verification_fails: bool = False +) -> tuple[int | None, str, str, list[list[str]]]: + """Drive `main` with `cargo xtask verify` intercepted at the subprocess call. - Three things have to hold together, because each is separately silent: - verification has to happen at all; it has to use the key - `resolve_signing_key` chose, since `cargo xtask verify` falls back to the - inline PEM when `--public-key` is omitted and would then check a different - key than the run reports; and the `signature verification key:` line has to - name that same key, or the log makes a claim nothing backs. + Returns `(exit code, stdout, stderr, argvs)`, where `argvs` holds every + command verification actually ran. + + Interception sits at the subprocess boundary rather than stubbing + `verify_signature`, so the real argv is built and the `--public-key` handoff + is exercised; stubbing the function leaves `key_args` unreachable, and + dropping it would stay green while verification silently fell back to + whatever the inline PEM holds. """ - # Intercept at the subprocess boundary rather than stubbing - # `verify_signature`, so the real argv is built and the `--public-key` - # handoff is covered too. Stubbing the function instead leaves `key_args` - # unexercised: dropping it stays green while verification silently falls - # back to whatever the inline PEM holds. argvs: list[list[str]] = [] + returncode = 1 if verification_fails else 0 class _Completed: - returncode = 0 - stdout = "" + def __init__(self) -> None: + self.returncode = returncode + self.stdout = "stub: signature did not verify" if returncode else "" module = sys.modules[__name__] original_run, original_which = module.subprocess.run, module.shutil.which @@ -791,9 +795,23 @@ class _Completed: "/fake/cargo" if name == "cargo" else original_which(name) ) try: - code, stdout, stderr = _run_main(dist, "--require-signatures") + code, stdout, stderr = _run_main(dist, *flags) finally: module.subprocess.run, module.shutil.which = original_run, original_which + return code, stdout, stderr, argvs + + +def _assert_main_verifies(dist: Path, expected_key: Path) -> None: + """`main` must verify, against the resolved key, and say which key that was. + + Three things have to hold together, because each is separately silent: + verification has to happen at all; it has to use the key + `resolve_signing_key` chose, since `cargo xtask verify` falls back to the + inline PEM when `--public-key` is omitted and would then check a different + key than the run reports; and the `signature verification key:` line has to + name that same key, or the log makes a claim nothing backs. + """ + code, stdout, stderr, argvs = _run_main_verifying(dist, "--require-signatures") if code is not None: raise ReadinessError( f"main rejected a run it should have accepted (exit {code}): " @@ -823,6 +841,76 @@ class _Completed: ) +def _assert_key_line_survives_a_failed_verification(dist: Path) -> None: + """The key line must appear on the run that fails, not only on the one that passes. + + `docs/release-trust.md` promises it names the key the run actually used; a + failed verification is when that matters most. Collecting the line into + `messages`, which is flushed only on success, would keep every other case + green while deleting it from exactly that run. + """ + code, stdout, _stderr, argvs = _run_main_verifying( + dist, "--require-signatures", verification_fails=True + ) + if code is None or not argvs: + raise ReadinessError( + "main accepted a run whose signature verification failed " + f"(exit {code!r}, {len(argvs)} verification call(s))" + ) + if "signature verification key:" not in stdout: + raise ReadinessError( + "the key line is missing from a run that failed verification, which " + "is the run that most needs it; it must not be deferred to the " + f"success-only message flush. stdout was {stdout.strip()!r}" + ) + + +def _assert_inline_pem_run_omits_the_key_flag(dist: Path) -> None: + """With only the inline PEM set, the run must say so and pass no `--public-key`. + + This is the source release and nightly actually use. `cargo xtask verify` + reads the PEM itself, so materialising a key file would be wrong -- but the + label still has to name the PEM, and gating the line on a resolved path + would silently drop it for every real CI run. + """ + code, stdout, stderr, argvs = _run_main_verifying(dist, "--require-signatures") + if code is not None: + raise ReadinessError( + f"main rejected an inline-PEM run it should have accepted (exit {code}): " + f"{stderr.strip()!r}" + ) + if not argvs: + raise ReadinessError("an inline-PEM run verified nothing") + for argv in argvs: + if "--public-key" in argv: + raise ReadinessError( + f"an inline-PEM run passed --public-key: {argv!r}; `cargo xtask " + "verify` reads the PEM itself and no key file exists to name" + ) + expected_line = f"signature verification key: ${SIGNING_PUBLIC_KEY_ENV}" + if expected_line not in stdout: + raise ReadinessError( + f"an inline-PEM run did not report its key source; expected " + f"{expected_line!r}, got {stdout.strip()!r}" + ) + + +def _assert_main_honours_trigger(dist: Path, *flags: str) -> None: + """Each trigger must force verification *through `main`*, not just in the helper. + + `verification_is_required` is pinned directly, but `main` chooses what to + feed it. Passing a literal `False` for production trust, or `None` for + `--public-key`, leaves those helper cases green while the flag stops doing + anything. + """ + _code, _stdout, _stderr, argvs = _run_main_verifying(dist, *flags) + if not argvs: + raise ReadinessError( + f"{' '.join(flags)} did not make main verify anything; the flag is no " + "longer reaching resolve_verification" + ) + + def run_self_test(root: Path) -> None: if root.exists(): shutil.rmtree(root) @@ -1222,11 +1310,36 @@ def run_self_test(root: Path) -> None: # missing key still wins resolution rather than falling through. real_key = root / "present-public-key.pem" real_key.write_text(inline_pem, encoding="ascii") + path_env = {**no_key_env, SIGNING_PUBLIC_KEY_PATH_ENV: real_key} + run_with_env(path_env, lambda: _assert_main_verifies(gate_dist, real_key)) + print("release readiness self-test: main actually verifies ok") + run_with_env( - {**no_key_env, SIGNING_PUBLIC_KEY_PATH_ENV: real_key}, - lambda: _assert_main_verifies(gate_dist, real_key), + path_env, lambda: _assert_key_line_survives_a_failed_verification(gate_dist) ) - print("release readiness self-test: main actually verifies ok") + print("release readiness self-test: key line survives a failed verify ok") + + # The source release and nightly actually use. + run_with_env( + {**no_key_env, SIGNING_PUBLIC_KEY_ENV: inline_pem}, + lambda: _assert_inline_pem_run_omits_the_key_flag(gate_dist), + ) + print("release readiness self-test: inline-PEM run reports its source ok") + + # The other two triggers, driven through `main` rather than the helper. + run_with_env( + path_env, + lambda: _assert_main_honours_trigger( + gate_dist, "--require-production-trust" + ), + ) + run_with_env( + no_key_env, + lambda: _assert_main_honours_trigger( + gate_dist, "--public-key", str(real_key) + ), + ) + print("release readiness self-test: production-trust and --public-key ok") finally: shutil.rmtree(root, ignore_errors=True) print("release readiness self-test: ok") From ae29152c3211b7608ad927f2257f12595f2e9ffd Mon Sep 17 00:00:00 2001 From: Roman Sirokov Date: Thu, 8 Oct 2026 15:27:44 +0300 Subject: [PATCH 11/16] fix(xtask): treat a named-but-empty CI key as an error, and pin the env names MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Three defects in the cross-check this branch added, all of them mine. A key file named by ROCM_CLI_SIGNING_PUBLIC_KEY_PATH that is empty or only whitespace resolved to `Some("")`. `check_ci_public_key` then trimmed it away and logged "ci public key: not configured — cross-check skipped" before exiting 0 -- a key *was* configured, and the log said otherwise. That is exactly what that function's own doc forbids: "'no key configured' and 'key matches' must not look alike in the log". Only an unset variable counts as absent now; a named path that is unreadable or blank is an error. `run()` read the variable names inline, so nothing covered them. Misspelling one left all six tests green while the check queried a variable nothing sets. The names are now constants, `resolve_ci_public_key` takes an env lookup so the names themselves are exercised, and a new case asserts both appear as key sources in release_readiness.py -- the lookup test alone cannot catch a wrong value, because it matches on the same constants. A non-UTF-8 path fell through to the PEM, because `std::env::var(..).ok()` turns NotUnicode into None. Resolution now goes through `var_os`, so a non-UTF-8 path still names the file the operator meant. Also corrects two statements about which workflow step sets what. The packaging steps export ROCM_CLI_REQUIRE_SIGNATURE=1 (release.yml:133,228, nightly.yml:137,232); ROCM_CLI_REQUIRE_PRODUCTION_TRUST is set once for the whole workflow from a repository variable (release.yml:24, nightly.yml:20). The error hint said "the last of which packaging steps export", which pointed at the wrong one. And the environment triggers now have a test: dropping either `truthy(os.environ.get(...))` term from `main` left --self-test green while a packaging run that exports the variable stopped verifying. Verified on Linux. `cargo test -p xtask verify_pinned_keys` 7/7, `cargo clippy --workspace --all-targets -- -D warnings` exit 0, `--self-test` exit 0. Mutations, each asserted to apply first: empty named key file passed through as Some("") -> fails (was passing) env name misspelled in the constant -> fails (was passing) ROCM_CLI_REQUIRE_SIGNATURE term dropped -> fails (was passing) ROCM_CLI_REQUIRE_PRODUCTION_TRUST term dropped -> fails (was passing) Signed-off-by: Roman Sirokov --- docs/release-trust.md | 11 ++- scripts/release_readiness.py | 34 +++++++- xtask/src/verify_pinned_keys.rs | 150 ++++++++++++++++++++++++-------- 3 files changed, 152 insertions(+), 43 deletions(-) diff --git a/docs/release-trust.md b/docs/release-trust.md index 90f4380c1e..cb578a6fcd 100644 --- a/docs/release-trust.md +++ b/docs/release-trust.md @@ -70,10 +70,13 @@ still records which trust root it used. Verification is driven by what the run asks for, not by what happens to be in the environment: `--require-signatures`, `--require-production-trust` or an explicit `--public-key` — or the environment equivalents of the first two, -`ROCM_CLI_REQUIRE_SIGNATURE` and `ROCM_CLI_REQUIRE_PRODUCTION_TRUST`, which -`release.yml` and `nightly.yml` set in the same step that runs the gate. Note -that exporting `ROCM_CLI_REQUIRE_SIGNATURE=1` for packaging therefore also -requires a resolvable signing key, where before it needed only a `.sig`. +`ROCM_CLI_REQUIRE_SIGNATURE` and `ROCM_CLI_REQUIRE_PRODUCTION_TRUST`. The +packaging steps set `ROCM_CLI_REQUIRE_SIGNATURE=1` on the step that runs the +gate (`release.yml:133,228`, `nightly.yml:137,232`), while production trust is +set once for the whole workflow from a repository variable (`release.yml:24`, +`nightly.yml:20`). Note that exporting `ROCM_CLI_REQUIRE_SIGNATURE=1` for +packaging therefore also requires a resolvable signing key, where before it +needed only a `.sig`. Previously a key present in `ROCM_CLI_SIGNING_PUBLIC_KEY_PEM` was enough to opportunistically verify even without those flags; that trigger was removed when the flags began to imply diff --git a/scripts/release_readiness.py b/scripts/release_readiness.py index 6e967c3cc2..32d057cb19 100644 --- a/scripts/release_readiness.py +++ b/scripts/release_readiness.py @@ -538,7 +538,8 @@ def resolve_signing_key(explicit: Path | None) -> tuple[Path | None, str]: f"{SIGNING_PUBLIC_KEY_ENV}. Verification is requested by " "--require-signatures, --require-production-trust, --public-key, or the " "ROCM_CLI_REQUIRE_SIGNATURE / ROCM_CLI_REQUIRE_PRODUCTION_TRUST " - "environment variables -- the last of which packaging steps export." + "environment variables; the packaging steps export " + "ROCM_CLI_REQUIRE_SIGNATURE=1, so a packaging run needs a key too." ) @@ -895,6 +896,24 @@ def _assert_inline_pem_run_omits_the_key_flag(dist: Path) -> None: ) +def _assert_env_trigger_reaches_the_gate(dist: Path, name: str) -> None: + """The environment equivalents must force verification through `main` too. + + `main` reads them with `truthy(os.environ.get(...))` and ors them into the + flags. Nothing covered that, so dropping either term left the suite green + while a packaging run that exports the variable stopped verifying -- the + very consequence `docs/release-trust.md` spells out. + """ + _code, _stdout, _stderr, argvs = run_with_env( + {name: "1"}, lambda: _run_main_verifying(dist) + ) + if not argvs: + raise ReadinessError( + f"{name}=1 did not make main verify anything; the environment " + "trigger is no longer reaching resolve_verification" + ) + + def _assert_main_honours_trigger(dist: Path, *flags: str) -> None: """Each trigger must force verification *through `main`*, not just in the helper. @@ -1340,6 +1359,19 @@ def run_self_test(root: Path) -> None: ), ) print("release readiness self-test: production-trust and --public-key ok") + + # ...and the environment equivalents, which packaging steps export. + for trigger in ( + "ROCM_CLI_REQUIRE_SIGNATURE", + "ROCM_CLI_REQUIRE_PRODUCTION_TRUST", + ): + run_with_env( + path_env, + lambda trigger=trigger: _assert_env_trigger_reaches_the_gate( + gate_dist, trigger + ), + ) + print("release readiness self-test: environment triggers reach the gate ok") finally: shutil.rmtree(root, ignore_errors=True) print("release readiness self-test: ok") diff --git a/xtask/src/verify_pinned_keys.rs b/xtask/src/verify_pinned_keys.rs index 0676c08f9d..72e9c3dfec 100644 --- a/xtask/src/verify_pinned_keys.rs +++ b/xtask/src/verify_pinned_keys.rs @@ -30,10 +30,13 @@ //! named by `ROCM_CLI_SIGNING_PUBLIC_KEY_PATH`, else the inline //! `ROCM_CLI_SIGNING_PUBLIC_KEY_PEM` release/nightly CI wires from the secret — //! it must equal the canonical *current* release key, so the key CI verifies -//! against is exactly the one users pin. Both sources are resolved in the same -//! order `scripts/release_readiness.py` uses, so the key cross-checked here is -//! always the key that gate verified with. +//! against is exactly the one users pin. Both environment sources are resolved +//! in the same order `scripts/release_readiness.py` uses (path before PEM). An +//! explicit `--public-key` passed to the gate is not visible here, and this +//! runs as its own workflow step, so a *relative* path would resolve against +//! each process's own working directory. No workflow does either today. +use std::ffi::OsString; use std::path::{Path, PathBuf}; use anyhow::{Context, Result, bail}; @@ -43,6 +46,10 @@ const PEM_END: &str = "-----END PUBLIC KEY-----"; const CURRENT_RELEASE_KEY: &str = "release-current"; +/// The key sources, in the order `scripts/release_readiness.py` resolves them. +const CI_PUBLIC_KEY_PATH_ENV: &str = "ROCM_CLI_SIGNING_PUBLIC_KEY_PATH"; +const CI_PUBLIC_KEY_PEM_ENV: &str = "ROCM_CLI_SIGNING_PUBLIC_KEY_PEM"; + /// How a pinned constant embeds its PEM string, so its value span can be isolated. #[derive(Clone, Copy)] enum Embedding { @@ -307,27 +314,42 @@ fn check_ci_public_key( /// against that very key — a trust root nobody compared with `docs/keys/`, and /// reported in the log as "not configured". /// -/// A named path that cannot be read is an error, not a fall-through to the PEM: -/// cross-checking a different key than the operator named is the failure this -/// check exists to catch. Blank counts as absent, because an unset GitHub secret -/// expands to the empty string. -fn resolve_ci_public_key(path_var: Option<&str>, pem_var: Option<&str>) -> Result> { - if let Some(path) = path_var.filter(|value| !value.trim().is_empty()) { - // The raw value rather than a trimmed one, so this and - // `release_readiness.py` always mean the same file. - let path = Path::new(path); - let pem = std::fs::read_to_string(path).with_context(|| { - format!( - "failed to read the CI signing public key named by \ - ROCM_CLI_SIGNING_PUBLIC_KEY_PATH: {}", - path.display() - ) - })?; - return Ok(Some(pem)); +/// A named path that cannot be read, or that holds nothing, is an error rather +/// than a fall-through: the operator named a key, so reporting "not configured" +/// and exiting 0 would be the exact confusion the doc on +/// [`check_ci_public_key`] forbids. Only an *unset* variable counts as absent, +/// because an unset GitHub secret expands to the empty string. +/// +/// `lookup` is injected so the variable names themselves are covered: reading +/// them directly here left `run()` free to query a misspelled name with every +/// test still green. +fn resolve_ci_public_key(lookup: impl Fn(&str) -> Option) -> Result> { + if let Some(raw) = lookup(CI_PUBLIC_KEY_PATH_ENV) { + // `OsString`, not `String`: a non-UTF-8 path must still name the file the + // operator meant, not silently fall through to the inline PEM. + let path = PathBuf::from(&raw); + if !path.as_os_str().is_empty() { + let pem = std::fs::read_to_string(&path).with_context(|| { + format!( + "failed to read the CI signing public key named by \ + {CI_PUBLIC_KEY_PATH_ENV}: {}", + path.display() + ) + })?; + if pem.trim().is_empty() { + bail!( + "the CI signing public key named by {CI_PUBLIC_KEY_PATH_ENV} is \ + empty: {}; a configured key that holds nothing must not be \ + reported as 'not configured'", + path.display() + ); + } + return Ok(Some(pem)); + } } - Ok(pem_var - .filter(|value| !value.trim().is_empty()) - .map(str::to_owned)) + Ok(lookup(CI_PUBLIC_KEY_PEM_ENV) + .map(|raw| raw.to_string_lossy().into_owned()) + .filter(|value| !value.trim().is_empty())) } fn repo_root() -> PathBuf { @@ -340,9 +362,9 @@ fn repo_root() -> PathBuf { } pub fn run() -> Result<()> { - let path_var = std::env::var("ROCM_CLI_SIGNING_PUBLIC_KEY_PATH").ok(); - let pem_var = std::env::var("ROCM_CLI_SIGNING_PUBLIC_KEY_PEM").ok(); - let ci_public_key = resolve_ci_public_key(path_var.as_deref(), pem_var.as_deref())?; + // Wrapped rather than passed as `std::env::var_os` directly: the generic fn + // item cannot satisfy the higher-ranked `Fn(&str)` bound. + let ci_public_key = resolve_ci_public_key(|name: &str| std::env::var_os(name))?; for message in check_pinned_keys(&repo_root(), ci_public_key.as_deref())? { println!("pinned key check: {message}"); } @@ -451,6 +473,26 @@ mod tests { assert!(check_ci_public_key(Some("not a pem"), Some(¤t)).is_err()); } + /// The variable names must be the ones the release gate actually reads. + /// + /// The lookup test below matches on these same constants, so it cannot see a + /// wrong *value*: misspelling one leaves every case green while `run()` + /// queries a variable nothing ever sets, and the cross-check silently skips. + /// Checking them against `release_readiness.py` pins the value and the + /// cross-language agreement at once. + #[test] + fn ci_public_key_env_names_match_the_release_gate() { + let gate = std::fs::read_to_string(repo_root().join("scripts/release_readiness.py")) + .expect("read scripts/release_readiness.py"); + for name in [CI_PUBLIC_KEY_PATH_ENV, CI_PUBLIC_KEY_PEM_ENV] { + assert!( + gate.contains(&format!("\"{name}\"")), + "{name} is not a key source in release_readiness.py, so this \ + cross-check would read a variable the gate never resolves" + ); + } + } + /// The cross-check must see a key supplied by path, not just the inline PEM. /// /// `release_readiness.py` prefers `ROCM_CLI_SIGNING_PUBLIC_KEY_PATH` and @@ -462,11 +504,24 @@ mod tests { let dir = tempfile::tempdir().expect("tempdir"); let key_file = dir.path().join("ci-public-key.pem"); std::fs::write(&key_file, SAMPLE).expect("write key"); - let named = key_file.to_str().expect("utf-8 path"); let other = SAMPLE.replace("test", "diff"); + // A lookup over a fixed map, so the *names* queried are covered too: with + // the variables read directly, `run()` could query a misspelled name and + // every case here would still pass. + let env = |path: Option<&str>, pem: Option<&str>| { + let path = path.map(OsString::from); + let pem = pem.map(OsString::from); + move |name: &str| match name { + CI_PUBLIC_KEY_PATH_ENV => path.clone(), + CI_PUBLIC_KEY_PEM_ENV => pem.clone(), + other => panic!("unexpected variable queried: {other}"), + } + }; + let named = key_file.to_str().expect("utf-8 path"); + // Path wins over the PEM, so the key cross-checked is the key verified with. - let resolved = resolve_ci_public_key(Some(named), Some(&other)) + let resolved = resolve_ci_public_key(env(Some(named), Some(&other))) .expect("readable key") .expect("a key"); assert_eq!(pem_body(&resolved).unwrap(), pem_body(SAMPLE).unwrap()); @@ -477,33 +532,52 @@ mod tests { assert!(check_ci_public_key(Some(&resolved), Some(&canonical)).is_err()); // The PEM is still used when no path is named. - let from_pem = resolve_ci_public_key(None, Some(SAMPLE)) - .expect("inline pem") - .expect("a key"); - assert_eq!(from_pem, SAMPLE); + assert_eq!( + resolve_ci_public_key(env(None, Some(SAMPLE))) + .expect("inline pem") + .as_deref(), + Some(SAMPLE) + ); - // Blank counts as absent on both, which is the shape of an unset secret. + // Unset counts as absent -- the shape of an unset GitHub secret, which + // expands to the empty string. for blank in [None, Some(""), Some(" ")] { assert!( - resolve_ci_public_key(blank, blank) + resolve_ci_public_key(env(None, blank)) .expect("no key") .is_none(), - "blank {blank:?} should resolve to no key" + "blank PEM {blank:?} should resolve to no key" ); } assert_eq!( - resolve_ci_public_key(Some(" "), Some(SAMPLE)) + resolve_ci_public_key(env(Some(""), Some(SAMPLE))) .expect("falls through to pem") .as_deref(), - Some(SAMPLE) + Some(SAMPLE), + "an unset path variable should fall through to the PEM" ); // A named key that cannot be read is an error, never a quiet fall-through // to a different key than the operator named. let missing = dir.path().join("absent.pem"); assert!( - resolve_ci_public_key(missing.to_str(), Some(SAMPLE)).is_err(), + resolve_ci_public_key(env(missing.to_str(), Some(SAMPLE))).is_err(), "an unreadable named key must fail rather than fall back to the PEM" ); + + // A named key that exists but holds nothing must be an error too. Passing + // it on as `Some("")` made `check_ci_public_key` trim it away and report + // "not configured" while a key was in fact configured -- exactly the + // confusion that check's own doc forbids. + for empty in ["", " \n\t "] { + let blank_file = dir.path().join("blank.pem"); + std::fs::write(&blank_file, empty).expect("write blank key"); + let error = resolve_ci_public_key(env(blank_file.to_str(), Some(SAMPLE))) + .expect_err("an empty named key must be an error"); + assert!( + error.to_string().contains("is empty"), + "unexpected error for an empty named key: {error}" + ); + } } } From 173c521e34626ac0c1d8d8c022d928c0af9ba823 Mon Sep 17 00:00:00 2001 From: Roman Sirokov Date: Fri, 9 Oct 2026 11:26:22 +0300 Subject: [PATCH 12/16] fix(xtask): resolve a blank key path like the gate, and test that run() resolves Three gaps, all in the same seam between `run()` and the key comparison. A blank `ROCM_CLI_SIGNING_PUBLIC_KEY_PATH` was not treated as absent. A whitespace-only value made the run fail with "failed to read the CI signing public key named by ...: ", while `release_readiness.py`'s `env_path` strips the value, treats it as unset and falls through to the inline PEM. The doc claimed the two resolved alike. They now do: the value is trimmed before it is taken as a path, lossy replacement characters still count as named so a non-UTF-8 path is unaffected, and the doc states the rule it actually implements -- an empty variable is absent, a variable naming a missing or empty file is an error. Nothing reached `run()`. Its lookup closure could query the wrong variable, answer `None` for everything, or hand `check_pinned_keys` a `None`, and all nine tests stayed green while the path resolution became a silent no-op reporting "not configured -- cross-check skipped" on a run that did have a key. `run_with(root, lookup)` now holds that handoff and is driven by a fake lookup against the real repo root: no key reports the skip, a path key that disagrees with `docs/keys/` fails the run, and one that agrees is reported as a match. What is left in `run()` is `env_lookup`, a pass-through shim with nothing to get wrong, pinned by comparing it with the environment under a name that is always set. The env-name test only asked whether each literal appeared somewhere in `release_readiness.py`. Both appear there anyway, in `PRODUCTION_TRUST_ENV_NAMES` and `validate_production_trust`, so swapping the two constants or pointing either at `ROCM_CLI_METADATA_*` passed -- the first would make release CI read the PEM text as a file path, the second would make the cross-check skip in silence. It now matches the gate's assignment lines, which ties each name to the role the gate gives it. Also pins the message on an unreadable named key. Asserting only `is_err()` let a failed read fall through as an empty string and still pass, telling the operator the key "is empty" and sending them to a file that is fine. Signed-off-by: Roman Sirokov --- xtask/src/verify_pinned_keys.rs | 214 +++++++++++++++++++++++++++----- 1 file changed, 181 insertions(+), 33 deletions(-) diff --git a/xtask/src/verify_pinned_keys.rs b/xtask/src/verify_pinned_keys.rs index 72e9c3dfec..96a648a801 100644 --- a/xtask/src/verify_pinned_keys.rs +++ b/xtask/src/verify_pinned_keys.rs @@ -314,21 +314,29 @@ fn check_ci_public_key( /// against that very key — a trust root nobody compared with `docs/keys/`, and /// reported in the log as "not configured". /// -/// A named path that cannot be read, or that holds nothing, is an error rather -/// than a fall-through: the operator named a key, so reporting "not configured" -/// and exiting 0 would be the exact confusion the doc on -/// [`check_ci_public_key`] forbids. Only an *unset* variable counts as absent, -/// because an unset GitHub secret expands to the empty string. +/// An empty variable counts as absent; a variable naming a missing or empty file +/// is an error. The blank case matches the Python gate exactly: `env_path` and +/// `env_text` both `strip()` the value and treat whatever is left of nothing as +/// unset, which is the shape an unset GitHub secret takes when it expands to the +/// empty string. Erroring on a whitespace-only path here while the gate fell +/// through to the PEM would make the two disagree about which key is in use. +/// +/// Once a path *is* named, a file that cannot be read or that holds nothing is an +/// error rather than a fall-through: the operator named a key, so reporting "not +/// configured" and exiting 0 would be the exact confusion the doc on +/// [`check_ci_public_key`] forbids. /// /// `lookup` is injected so the variable names themselves are covered: reading -/// them directly here left `run()` free to query a misspelled name with every +/// them directly here left callers free to query a misspelled name with every /// test still green. fn resolve_ci_public_key(lookup: impl Fn(&str) -> Option) -> Result> { if let Some(raw) = lookup(CI_PUBLIC_KEY_PATH_ENV) { // `OsString`, not `String`: a non-UTF-8 path must still name the file the - // operator meant, not silently fall through to the inline PEM. - let path = PathBuf::from(&raw); - if !path.as_os_str().is_empty() { + // operator meant, not silently fall through to the inline PEM. Lossy + // replacement characters are not whitespace, so such a path still counts + // as named. + if !raw.to_string_lossy().trim().is_empty() { + let path = PathBuf::from(&raw); let pem = std::fs::read_to_string(&path).with_context(|| { format!( "failed to read the CI signing public key named by \ @@ -361,11 +369,29 @@ fn repo_root() -> PathBuf { .to_path_buf() } +/// A named shim rather than a closure in [`run`]: the generic `std::env::var_os` +/// fn item cannot satisfy the higher-ranked `Fn(&str)` bound, and a closure +/// spelled out at the call site is a place a wrong variable name could live +/// where no test could reach it. Here it is a pass-through with nothing to get +/// wrong, and `env_lookup_passes_the_name_through` pins that it stays one. +fn env_lookup(name: &str) -> Option { + std::env::var_os(name) +} + +/// Resolve the CI key through `lookup`, then run every pinned-key check against +/// `root`, returning the lines the caller should print. +/// +/// Split out of [`run`] so the resolution-to-comparison handoff is testable: with +/// it inlined there, a `run` that queried the wrong variable, resolved nothing, +/// or simply handed `check_pinned_keys` a `None` left every test green while the +/// cross-check silently degraded to "not configured". +fn run_with(root: &Path, lookup: impl Fn(&str) -> Option) -> Result> { + let ci_public_key = resolve_ci_public_key(lookup)?; + check_pinned_keys(root, ci_public_key.as_deref()) +} + pub fn run() -> Result<()> { - // Wrapped rather than passed as `std::env::var_os` directly: the generic fn - // item cannot satisfy the higher-ranked `Fn(&str)` bound. - let ci_public_key = resolve_ci_public_key(|name: &str| std::env::var_os(name))?; - for message in check_pinned_keys(&repo_root(), ci_public_key.as_deref())? { + for message in run_with(&repo_root(), env_lookup)? { println!("pinned key check: {message}"); } Ok(()) @@ -473,22 +499,33 @@ mod tests { assert!(check_ci_public_key(Some("not a pem"), Some(¤t)).is_err()); } - /// The variable names must be the ones the release gate actually reads. + /// Each variable name must be the one the release gate reads *for that role*. + /// + /// The lookup tests below match on these same constants, so they cannot see a + /// wrong *value*: misspelling one leaves every case green while `run` queries + /// a variable nothing ever sets, and the cross-check silently skips. /// - /// The lookup test below matches on these same constants, so it cannot see a - /// wrong *value*: misspelling one leaves every case green while `run()` - /// queries a variable nothing ever sets, and the cross-check silently skips. - /// Checking them against `release_readiness.py` pins the value and the - /// cross-language agreement at once. + /// Matching the gate's assignment lines, not merely the name somewhere in the + /// file, is what makes this bite. Both literals also appear in + /// `PRODUCTION_TRUST_ENV_NAMES` and `validate_production_trust`, so a + /// name-anywhere search passed three real regressions: swapping the two + /// constants (release CI would read the PEM text as a file path), and pointing + /// either at `ROCM_CLI_METADATA_*` (the cross-check would quietly skip). + /// Pinning `BINDING = "NAME"` ties each name to the role the gate gives it. #[test] fn ci_public_key_env_names_match_the_release_gate() { let gate = std::fs::read_to_string(repo_root().join("scripts/release_readiness.py")) .expect("read scripts/release_readiness.py"); - for name in [CI_PUBLIC_KEY_PATH_ENV, CI_PUBLIC_KEY_PEM_ENV] { + for (binding, name) in [ + ("SIGNING_PUBLIC_KEY_PATH_ENV", CI_PUBLIC_KEY_PATH_ENV), + ("SIGNING_PUBLIC_KEY_ENV", CI_PUBLIC_KEY_PEM_ENV), + ] { + let assignment = format!("\n{binding} = \"{name}\"\n"); assert!( - gate.contains(&format!("\"{name}\"")), - "{name} is not a key source in release_readiness.py, so this \ - cross-check would read a variable the gate never resolves" + gate.contains(&assignment), + "release_readiness.py does not bind {binding} to {name}; this \ + cross-check would resolve a different key than the gate verifies \ + with. Expected the line {assignment:?}" ); } } @@ -549,20 +586,31 @@ mod tests { "blank PEM {blank:?} should resolve to no key" ); } - assert_eq!( - resolve_ci_public_key(env(Some(""), Some(SAMPLE))) - .expect("falls through to pem") - .as_deref(), - Some(SAMPLE), - "an unset path variable should fall through to the PEM" - ); + // Same for the path variable, and whitespace counts as blank: the Python + // gate's `env_path` strips the value and falls through to the PEM, so + // erroring here instead would make the two sides disagree about which key + // a release is verified with. + for blank in [None, Some(""), Some(" "), Some("\t\n")] { + assert_eq!( + resolve_ci_public_key(env(blank, Some(SAMPLE))) + .unwrap_or_else(|e| panic!("blank path {blank:?} should not error: {e}")) + .as_deref(), + Some(SAMPLE), + "a blank path variable {blank:?} should fall through to the PEM" + ); + } // A named key that cannot be read is an error, never a quiet fall-through - // to a different key than the operator named. + // to a different key than the operator named. The *message* is pinned too: + // a failed read that fell through as an empty string would still be an + // error, but the operator would be told the key "is empty" and go looking + // at a file that is fine. let missing = dir.path().join("absent.pem"); + let read_error = resolve_ci_public_key(env(missing.to_str(), Some(SAMPLE))) + .expect_err("an unreadable named key must fail rather than fall back to the PEM"); assert!( - resolve_ci_public_key(env(missing.to_str(), Some(SAMPLE))).is_err(), - "an unreadable named key must fail rather than fall back to the PEM" + format!("{read_error:#}").contains("failed to read"), + "an unreadable named key should report the read failure, got: {read_error:#}" ); // A named key that exists but holds nothing must be an error too. Passing @@ -580,4 +628,104 @@ mod tests { ); } } + + /// `run_with` is the whole path `run` takes, so the resolved key must reach + /// the comparison rather than being dropped on the way. + /// + /// Nothing previously ran through this seam. A `run` whose lookup always + /// queried the PEM name, always returned `None`, or simply handed + /// `check_pinned_keys` a `None` turned the path resolution into a silent + /// no-op and reported "not configured — cross-check skipped" on a run that + /// did have a key. Each case below fails under exactly those mutations. + #[test] + fn run_with_carries_the_resolved_key_into_the_comparison() { + let root = repo_root(); + let canonical = + std::fs::read_to_string(root.join("docs/keys/rocm-cli-release-current-public.pem")) + .expect("the canonical current release key is committed"); + let dir = tempfile::tempdir().expect("tempdir"); + + let env = |path: Option<&str>, pem: Option<&str>| { + let path = path.map(OsString::from); + let pem = pem.map(OsString::from); + move |name: &str| match name { + CI_PUBLIC_KEY_PATH_ENV => path.clone(), + CI_PUBLIC_KEY_PEM_ENV => pem.clone(), + other => panic!("unexpected variable queried: {other}"), + } + }; + + // No key anywhere: the run passes, and says the cross-check was skipped. + // This is the only place that message is exercised through the code path + // that actually prints it. + let skipped = run_with(&root, env(None, None)).expect("no key configured is not an error"); + assert!( + skipped + .iter() + .any(|m| m.contains("not configured — cross-check skipped")), + "a run with no key must say the cross-check was skipped, got {skipped:?}" + ); + + // A key named by path must be compared, not skipped: a wrong one fails. + let wrong = dir.path().join("wrong.pem"); + std::fs::write(&wrong, SAMPLE).expect("write wrong key"); + let error = run_with(&root, env(wrong.to_str(), None)) + .expect_err("a path key that disagrees with docs/keys must fail the run"); + assert!( + format!("{error:#}").contains("does not match the canonical current release key"), + "unexpected failure for a mismatched path key: {error:#}" + ); + + // ...and the right one passes, reported as a match rather than a skip. + let right = dir.path().join("right.pem"); + std::fs::write(&right, &canonical).expect("write canonical key"); + let matched = run_with(&root, env(right.to_str(), None)).expect("canonical key matches"); + assert!( + matched + .iter() + .any(|m| m.contains("matches the canonical current release key")), + "a path key equal to docs/keys must be reported as a match, got {matched:?}" + ); + + // The PEM source still reaches the comparison too, and loses to the path. + let via_pem = run_with(&root, env(None, Some(&canonical))).expect("canonical pem matches"); + assert!( + via_pem + .iter() + .any(|m| m.contains("matches the canonical current release key")), + "an inline canonical PEM must be reported as a match, got {via_pem:?}" + ); + assert!( + run_with(&root, env(wrong.to_str(), Some(&canonical))).is_err(), + "the path key must win, so a wrong path key is not rescued by a good PEM" + ); + } + + /// `run`'s only remaining untestable surface is this shim, so pin it. + /// + /// It must pass the name it is given straight to the environment. A shim that + /// ignored its argument, or always answered `None`, would make `run` resolve + /// the wrong variable or no variable at all while every fake-lookup test above + /// stayed green. `PATH` is used because it is set in every environment this + /// runs in, so a shim that ignores the name cannot accidentally agree. + #[test] + fn env_lookup_passes_the_name_through() { + assert_eq!(env_lookup("PATH"), std::env::var_os("PATH")); + assert!( + env_lookup("PATH").is_some(), + "PATH is unset, so this test cannot tell a pass-through from a stub" + ); + for name in [CI_PUBLIC_KEY_PATH_ENV, CI_PUBLIC_KEY_PEM_ENV] { + assert_eq!( + env_lookup(name), + std::env::var_os(name), + "{name} must be read from the environment under its own name" + ); + } + assert_eq!( + env_lookup("ROCM_CLI_VERIFY_PINNED_KEYS_NOT_A_REAL_VARIABLE"), + None, + "an unset variable must resolve to None, not to some other variable" + ); + } } From 39eeceb6bc5a5b9d86585d06acdc175a9d8552af Mon Sep 17 00:00:00 2001 From: Roman Sirokov Date: Fri, 9 Oct 2026 11:26:40 +0300 Subject: [PATCH 13/16] test(release): pin that a configured PEM alone no longer verifies This change removed a trigger: before it, a set `ROCM_CLI_SIGNING_PUBLIC_KEY_PEM` was on its own enough to make a readiness run verify signatures. `docs/release-trust.md` now promises the opposite -- "a readiness run that asks for neither now checks neither" -- and AGENTS.md section 3 requires a promise that something will *not* happen to be tested. Nothing tested it. The only "off" case ran with the PEM unset, so it could not tell the two behaviours apart: putting the old `or env_text(SIGNING_PUBLIC_KEY_ENV) is not None` term back into `verification_is_required` left `--self-test` at exit 0. Re-adding the same trigger one level up, in `main`'s flag assembly, was equally invisible. `_assert_the_inline_pem_alone_triggers_nothing` runs with the PEM set and nothing asking for signatures, and pins both levels, because each is silent on its own: `resolve_verification(False, False, None)` must stay `(False, None, None)`, and `main` -- which assembles what is fed to it -- must run no verification at all. Each of the two mutations above now turns the self-test red, and on the assertion that matches it. Also relabels the block that drives `validate_release` directly with `require_signatures=True, verify=False`. `main` can no longer produce that combination, so "valid signed dist accepted" overclaimed: those cases use placeholder `.sig` bytes the real gate would reject. They are presence-half unit checks and now say so, with a note pointing at the cases that cover verification end to end. Signed-off-by: Roman Sirokov --- scripts/release_readiness.py | 61 ++++++++++++++++++++++++++++++++++-- 1 file changed, 58 insertions(+), 3 deletions(-) diff --git a/scripts/release_readiness.py b/scripts/release_readiness.py index 32d057cb19..2a6ec9ca5d 100644 --- a/scripts/release_readiness.py +++ b/scripts/release_readiness.py @@ -914,6 +914,42 @@ def _assert_env_trigger_reaches_the_gate(dist: Path, name: str) -> None: ) +def _assert_the_inline_pem_alone_triggers_nothing(dist: Path) -> None: + """A set `ROCM_CLI_SIGNING_PUBLIC_KEY_PEM` must not, by itself, turn verification on. + + Before this change, a configured PEM *was* a trigger. Removing it is a + promise that something will no longer happen, and `docs/release-trust.md` + states it: "a readiness run that asks for neither now checks neither". Only + a test that runs with the PEM set can tell the two behaviours apart -- every + other "off" case leaves it unset, so putting the old + ``or env_text(SIGNING_PUBLIC_KEY_ENV) is not None`` term back survives them + all. + + Both levels are pinned, because each is silent on its own: the decision + helper must resolve nothing, and `main` -- which assembles the arguments fed + to it -- must run no verification. + """ + decision = resolve_verification(False, False, None) + if decision != (False, None, None): + raise ReadinessError( + "a bare run with only the inline PEM configured must not verify; " + f"resolve_verification(False, False, None) returned {decision!r}. A " + "configured key is no longer a trigger -- asking for neither flag " + "must check neither." + ) + code, _stdout, stderr, argvs = _run_main_verifying(dist) + if code is not None: + raise ReadinessError( + f"main rejected a bare run it should have accepted (exit {code}): " + f"{stderr.strip()!r}" + ) + if argvs: + raise ReadinessError( + f"a bare run verified {len(argvs)} signature(s) with only the inline " + f"PEM set: {argvs!r}; the PEM is no longer a verification trigger" + ) + + def _assert_main_honours_trigger(dist: Path, *flags: str) -> None: """Each trigger must force verification *through `main`*, not just in the helper. @@ -935,6 +971,16 @@ def run_self_test(root: Path) -> None: shutil.rmtree(root) root.mkdir(parents=True) try: + # The block below, down to "production trust inputs accepted", drives + # `validate_release` directly with `require_signatures=True, verify=False`. + # `main` can no longer produce that combination -- requiring signatures now + # implies verifying them -- so these are unit checks of the *presence* half + # of the signature rule: that a `.sig` must exist next to every archive, and + # that the surrounding asset/sha/name rules behave. They deliberately use + # placeholder `.sig` bytes, which the real gate would reject; nothing here + # claims a signature is valid. End-to-end verification, with the key handoff + # and the real argv, is covered further down by the `_run_main_verifying` + # cases. dist = root / "dist" dist.mkdir() linux_archive = dist / "rocm-cli-test-linux-amd64.tar.gz" @@ -954,7 +1000,7 @@ def run_self_test(root: Path) -> None: require_rocm_asset_names=False, require_exact_assets=False, ) - print("release readiness self-test: valid signed dist accepted") + print("release readiness self-test: dist with a .sig present accepted") exact_dist = root / "exact-dist" exact_dist.mkdir() @@ -975,7 +1021,7 @@ def run_self_test(root: Path) -> None: require_rocm_asset_names=True, require_exact_assets=True, ) - print("release readiness self-test: exact signed dist accepted") + print("release readiness self-test: exact dist with a .sig present accepted") stale_archive = exact_dist / "rocm-cli-v9.9.9-linux-amd64.tar.gz" create_test_tar(stale_archive, "rocm-cli-v9.9.9-linux-amd64") @@ -1339,12 +1385,21 @@ def run_self_test(root: Path) -> None: print("release readiness self-test: key line survives a failed verify ok") # The source release and nightly actually use. + inline_pem_env = {**no_key_env, SIGNING_PUBLIC_KEY_ENV: inline_pem} run_with_env( - {**no_key_env, SIGNING_PUBLIC_KEY_ENV: inline_pem}, + inline_pem_env, lambda: _assert_inline_pem_run_omits_the_key_flag(gate_dist), ) print("release readiness self-test: inline-PEM run reports its source ok") + # ...and the same environment with nothing asking for signatures, which + # is the only shape that can see the removed PEM trigger come back. + run_with_env( + inline_pem_env, + lambda: _assert_the_inline_pem_alone_triggers_nothing(gate_dist), + ) + print("release readiness self-test: a configured PEM alone triggers nothing ok") + # The other two triggers, driven through `main` rather than the helper. run_with_env( path_env, From ed61c3a5d03f78ebd3ebee1bc76d622bc4a6d615 Mon Sep 17 00:00:00 2001 From: Roman Sirokov Date: Tue, 13 Oct 2026 10:12:19 +0300 Subject: [PATCH 14/16] docs(release): cite the workflow steps by name, not by line number juhovainio flagged these as bare `file:line` spans. lychee reads the `:NN` suffix as part of the filename, so neither the docs-links CI job nor the prek hook can validate them -- they were accurate at the time of writing and would rot silently. Cite the steps by name ("Package release bundle", "Package Windows bundle", "Package nightly bundle") with plain relative links to the workflow files, and describe production trust as workflow-level `env:` rather than pointing at a line. All three step names verified present in the files they are attributed to. Signed-off-by: Roman Sirokov --- docs/release-trust.md | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/docs/release-trust.md b/docs/release-trust.md index cb578a6fcd..6a965009be 100644 --- a/docs/release-trust.md +++ b/docs/release-trust.md @@ -71,10 +71,13 @@ Verification is driven by what the run asks for, not by what happens to be in the environment: `--require-signatures`, `--require-production-trust` or an explicit `--public-key` — or the environment equivalents of the first two, `ROCM_CLI_REQUIRE_SIGNATURE` and `ROCM_CLI_REQUIRE_PRODUCTION_TRUST`. The -packaging steps set `ROCM_CLI_REQUIRE_SIGNATURE=1` on the step that runs the -gate (`release.yml:133,228`, `nightly.yml:137,232`), while production trust is -set once for the whole workflow from a repository variable (`release.yml:24`, -`nightly.yml:20`). Note that exporting `ROCM_CLI_REQUIRE_SIGNATURE=1` for +bundle-packaging steps set `ROCM_CLI_REQUIRE_SIGNATURE=1` on the step that runs +the gate — "Package release bundle" and "Package Windows bundle" in +[`release.yml`](../.github/workflows/release.yml), "Package nightly bundle" and +"Package Windows bundle" in [`nightly.yml`](../.github/workflows/nightly.yml) — +while production trust is set once at workflow level, from the +`ROCM_CLI_REQUIRE_PRODUCTION_TRUST` repository variable in each file's top-level +`env:` block. Note that exporting `ROCM_CLI_REQUIRE_SIGNATURE=1` for packaging therefore also requires a resolvable signing key, where before it needed only a `.sig`. Previously a key present in From 39d13211bd608fe046971dc573eca62974b89792 Mon Sep 17 00:00:00 2001 From: Roman Sirokov Date: Tue, 13 Oct 2026 15:29:39 +0300 Subject: [PATCH 15/16] test(xtask): pin the path-before-PEM key order across both gates The env-name cross-check pinned which variable each side reads for each role, but nothing pinned the order they are consulted in. Reversing either implementation alone left both suites green: the Rust lookup tests observe only the Rust resolver, and release_readiness.py's --self-test observes only the Python one. A release would then be verified against the inline PEM while the pinned-key cross-check compared the key named by path (or the reverse) -- two different trust roots, with the run still reporting success and naming only one of them. Establish the order twice and require the two to agree: observe it by running resolve_ci_public_key with both sources set to different keys, and read it out of resolve_signing_key's own body in the gate. The body is sliced to that one function because both constants appear elsewhere in the file, where a file-wide search would order unrelated lines. Signed-off-by: Roman Sirokov --- xtask/src/verify_pinned_keys.rs | 101 ++++++++++++++++++++++++++++++-- 1 file changed, 95 insertions(+), 6 deletions(-) diff --git a/xtask/src/verify_pinned_keys.rs b/xtask/src/verify_pinned_keys.rs index 96a648a801..185cd28e9f 100644 --- a/xtask/src/verify_pinned_keys.rs +++ b/xtask/src/verify_pinned_keys.rs @@ -506,12 +506,15 @@ mod tests { /// a variable nothing ever sets, and the cross-check silently skips. /// /// Matching the gate's assignment lines, not merely the name somewhere in the - /// file, is what makes this bite. Both literals also appear in - /// `PRODUCTION_TRUST_ENV_NAMES` and `validate_production_trust`, so a - /// name-anywhere search passed three real regressions: swapping the two - /// constants (release CI would read the PEM text as a file path), and pointing - /// either at `ROCM_CLI_METADATA_*` (the cross-check would quietly skip). - /// Pinning `BINDING = "NAME"` ties each name to the role the gate gives it. + /// file, is what makes this bite. The literals also appear in the gate's + /// self-test environments, so a name-anywhere search passed three real + /// regressions: swapping the two constants (release CI would read the PEM text + /// as a file path), and pointing either at `ROCM_CLI_METADATA_*` (the + /// cross-check would quietly skip). Pinning `BINDING = "NAME"` ties each name + /// to the role the gate gives it. + /// + /// This pins the names only. [`key_resolution_order_matches_the_release_gate`] + /// pins the order they are consulted in, which is a separate rule. #[test] fn ci_public_key_env_names_match_the_release_gate() { let gate = std::fs::read_to_string(repo_root().join("scripts/release_readiness.py")) @@ -530,6 +533,92 @@ mod tests { } } + /// The source of one top-level `def`, up to the next top-level `def`. + /// + /// Slicing to the function matters: both constants appear a dozen other + /// places in the gate (`PRODUCTION_TRUST_ENV_NAMES`, `validate_production_trust`, + /// the self-test cases), so a file-wide search would order unrelated lines and + /// pass whatever the resolver actually did. + fn python_function_body<'a>(source: &'a str, name: &str) -> &'a str { + let header = format!("\ndef {name}("); + let start = source + .find(&header) + .unwrap_or_else(|| panic!("scripts/release_readiness.py has no `def {name}(`")) + + 1; + let rest = &source[start..]; + let end = rest[1..] + .find("\ndef ") + .map_or(rest.len(), |index| index + 2); + &rest[..end] + } + + /// The path-before-PEM precedence must stay in lockstep across both + /// implementations, not merely the variable names. + /// + /// [`ci_public_key_env_names_match_the_release_gate`] pins *which* variable + /// each side reads for each role, but nothing pinned the *order* they are + /// consulted in. Reversing either side alone left both suites green: the + /// lookup tests below only observe the Rust resolver, and the gate's + /// `--self-test` only observes the Python one. A release would then be + /// verified against the inline PEM while this cross-check compared the key + /// named by path (or the reverse) — two different trust roots, with the run + /// still reporting success and naming only one of them. + /// + /// So the order is established twice, independently: observed by *running* + /// the Rust resolver with both sources set, and read out of the gate's own + /// `resolve_signing_key` body. Both must say path-before-PEM. + #[test] + fn key_resolution_order_matches_the_release_gate() { + // The Rust order, observed rather than read off the source: with both + // sources set to different keys, whichever comes back is the winner. + let dir = tempfile::tempdir().expect("tempdir"); + let key_file = dir.path().join("by-path.pem"); + std::fs::write(&key_file, SAMPLE).expect("write key"); + let inline = SAMPLE.replace("test", "diff"); + + let both_set = |name: &str| match name { + CI_PUBLIC_KEY_PATH_ENV => Some(OsString::from(key_file.as_os_str())), + CI_PUBLIC_KEY_PEM_ENV => Some(OsString::from(inline.as_str())), + other => panic!("unexpected variable queried: {other}"), + }; + let resolved = resolve_ci_public_key(both_set) + .expect("both sources set resolves a key") + .expect("a key"); + assert_eq!( + pem_body(&resolved).unwrap(), + pem_body(SAMPLE).unwrap(), + "resolve_ci_public_key stopped preferring {CI_PUBLIC_KEY_PATH_ENV} over \ + {CI_PUBLIC_KEY_PEM_ENV}. The release gate still resolves the path first, so \ + this cross-check would compare a different key than releases are verified \ + against, and report a match for a key nobody verified with" + ); + + // The Python order, read out of the gate's own resolver. + let gate = std::fs::read_to_string(repo_root().join("scripts/release_readiness.py")) + .expect("read scripts/release_readiness.py"); + let body = python_function_body(&gate, "resolve_signing_key"); + let locate = |needle: &str| { + body.find(needle).unwrap_or_else(|| { + panic!( + "resolve_signing_key no longer resolves its key via `{needle}`, so the \ + two implementations' precedence can no longer be compared. Re-establish \ + the comparison rather than deleting it: the order is the security rule. \ + The body read was:\n{body}" + ) + }) + }; + let path_branch = locate("env_path(SIGNING_PUBLIC_KEY_PATH_ENV)"); + let pem_branch = locate("env_text(SIGNING_PUBLIC_KEY_ENV)"); + assert!( + path_branch < pem_branch, + "release_readiness.py's resolve_signing_key now consults \ + {CI_PUBLIC_KEY_PEM_ENV} before {CI_PUBLIC_KEY_PATH_ENV}, while \ + resolve_ci_public_key above still resolves the path first. The gate would \ + verify a release against one key and this cross-check would compare another. \ + Change both or neither" + ); + } + /// The cross-check must see a key supplied by path, not just the inline PEM. /// /// `release_readiness.py` prefers `ROCM_CLI_SIGNING_PUBLIC_KEY_PATH` and From 4230342fd9fe93a9e8a609d61ef71388a655e175 Mon Sep 17 00:00:00 2001 From: Roman Sirokov Date: Tue, 13 Oct 2026 15:29:52 +0300 Subject: [PATCH 16/16] refactor(release): reuse the signing-key env constants in the trust check validate_production_trust() spelled ROCM_CLI_SIGNING_PUBLIC_KEY_PATH and _PEM as raw strings while the rest of the gate goes through SIGNING_PUBLIC_KEY_PATH_ENV / SIGNING_PUBLIC_KEY_ENV, so a rename would have missed this spot and silently stopped checking the key it names. Also record why the signing-key checks here are now defence in depth: --require-production-trust makes verification_is_required() true, so main resolves the key and fails closed before this function runs. They are kept because the function is still reachable directly, and because a check that fails closed costs nothing when it never fires -- deleting them would leave the production-trust report resting entirely on a guarantee made elsewhere. Signed-off-by: Roman Sirokov --- scripts/release_readiness.py | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/scripts/release_readiness.py b/scripts/release_readiness.py index 2a6ec9ca5d..549558f180 100644 --- a/scripts/release_readiness.py +++ b/scripts/release_readiness.py @@ -560,18 +560,29 @@ def require_existing_env_path(name: str) -> Path: def validate_production_trust() -> list[str]: - """Validate only explicit owner-provided production trust inputs.""" + """Validate only explicit owner-provided production trust inputs. + + The signing-key checks below are defence in depth rather than the live gate. + ``--require-production-trust`` now makes ``verification_is_required`` true, so + ``main`` runs ``resolve_signing_key`` first and already fails closed when no + key resolves; by the time this function runs, a signing key is known to exist. + They are kept because this function is also reachable directly (``--self-test`` + drives it, and ``validate_release`` can be called as a library function), and + because a check that fails closed costs nothing when it never fires. Deleting + them would make the "production trust" report depend entirely on a guarantee + established somewhere else. + """ messages: list[str] = [] require_any( "the release signing public key", - ["ROCM_CLI_SIGNING_PUBLIC_KEY_PATH", "ROCM_CLI_SIGNING_PUBLIC_KEY_PEM"], + [SIGNING_PUBLIC_KEY_PATH_ENV, SIGNING_PUBLIC_KEY_ENV], ) if ( - path := env_path("ROCM_CLI_SIGNING_PUBLIC_KEY_PATH") + path := env_path(SIGNING_PUBLIC_KEY_PATH_ENV) ) is not None and not path.is_file(): raise ReadinessError( - f"ROCM_CLI_SIGNING_PUBLIC_KEY_PATH does not point to a file: {path}" + f"{SIGNING_PUBLIC_KEY_PATH_ENV} does not point to a file: {path}" ) messages.append("release signing public key configured")