From 8c7a2772b6edaa9330cae96fa457d4731aa71267 Mon Sep 17 00:00:00 2001 From: Jussi Elo Date: Thu, 29 Oct 2026 13:42:49 +0000 Subject: [PATCH 1/2] ci(release): always build aws-lc-sys from source in shipped builds aws-lc-sys 0.46 and later detect an AWS-LC already present on the build host (OPENSSL_DIR, OPENSSL_INCLUDE_DIR/OPENSSL_LIB_DIR or pkg-config) and link it when it validates. The manylinux build container installs pkgconfig, so a shipped binary could end up depending on whatever the host provides. Set AWS_LC_SYS_USE_SYSTEM=0 in the release and nightly workflows, and for the Linux and Windows PR bundle builds in ci.yml, so the vendored copy is always built. The manylinux container does not inherit the workflow environment, so forward the variable into each docker run. The locked aws-lc-sys is still 0.41.0, which ignores the variable; it takes effect once the lockfile moves to 0.46 (#792). Signed-off-by: Jussi Elo --- .github/workflows/ci.yml | 9 +++++++++ .github/workflows/nightly.yml | 6 ++++++ .github/workflows/release.yml | 6 ++++++ 3 files changed, 21 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e2fd0c4c71..eef6b622db 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -712,6 +712,10 @@ jobs: - name: Build (release) if: needs.changes.outputs.heavy == 'true' shell: pwsh + # These binaries are packaged as the downloadable PR bundle; see the + # Linux bundle build for why aws-lc-sys must build its vendored copy. + env: + AWS_LC_SYS_USE_SYSTEM: "0" run: cargo build --release -p rocm -p rocmd -p rocm-engine-lemonade -p rocm-engine-vllm -p xtask - name: Test @@ -926,6 +930,10 @@ jobs: Linux-manylinux-pr- - name: Build release binaries + # Same as release.yml: the downloadable bundle must not link an + # AWS-LC the build container happens to provide (aws-lc-sys 0.46+). + env: + AWS_LC_SYS_USE_SYSTEM: "0" run: | mkdir -p "$HOME/.cargo" docker run --rm -v "$PWD":/work -w /work quay.io/pypa/manylinux2014_x86_64 /bin/bash -lc ' @@ -935,6 +943,7 @@ jobs: docker run --rm \ -u "$(id -u):$(id -g)" \ -e CARGO_HOME=/host-cargo -e RUSTUP_HOME=/host-cargo/rustup \ + -e AWS_LC_SYS_USE_SYSTEM \ -v "$HOME/.cargo":/host-cargo -v "$PWD":/work -w /work \ quay.io/pypa/manylinux2014_x86_64 \ /bin/bash -lc ' diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index 4d7f68efcf..e9d00ad47a 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -16,6 +16,11 @@ env: # `CARGO_BUILD_WARNINGS: deny` instead (set at workflow level so it reaches # jobs that don't use setup-rust-toolchain too). RUSTFLAGS: -D warnings + # aws-lc-sys 0.46 and later auto-detect an AWS-LC already installed on the + # build host and link it when found. Nightly binaries must not depend on + # what the runner or build container happens to have, so always build the + # vendored copy. + AWS_LC_SYS_USE_SYSTEM: "0" # REQUIRED, not dormant: every readiness step below runs --require-signatures, # which now means signatures must actually verify, so this is the public key # the nightly is verified against. Removing or blanking the secret fails every @@ -122,6 +127,7 @@ jobs: -e CARGO_HOME=/host-cargo \ -e RUSTUP_HOME=/host-cargo/rustup \ -e ROCM_CLI_RELEASE_REF="${{ github.ref_name }}" \ + -e AWS_LC_SYS_USE_SYSTEM \ -v "$HOME/.cargo":/host-cargo \ -v "$PWD":/work \ -w /work \ diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index bbf4ee877e..e55ab2c5d9 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -20,6 +20,11 @@ env: # `CARGO_BUILD_WARNINGS: deny` instead (set at workflow level so it reaches # jobs that don't use setup-rust-toolchain too). RUSTFLAGS: -D warnings + # aws-lc-sys 0.46 and later auto-detect an AWS-LC already installed on the + # build host and link it when found. Release binaries must not depend on + # what the runner or build container happens to have, so always build the + # vendored copy. + AWS_LC_SYS_USE_SYSTEM: "0" # REQUIRED, not dormant: every readiness step below runs --require-signatures, # which now means signatures must actually verify, so this is the public key # the release is verified against. Removing or blanking the secret fails every @@ -126,6 +131,7 @@ jobs: -e CARGO_HOME=/host-cargo \ -e RUSTUP_HOME=/host-cargo/rustup \ -e ROCM_CLI_RELEASE_REF="${{ steps.version.outputs.value }}" \ + -e AWS_LC_SYS_USE_SYSTEM \ -v "$HOME/.cargo":/host-cargo \ -v "$PWD":/work \ -w /work \ From 208e86f4a46c611c122d1fc7a10b8d76aaabaae0 Mon Sep 17 00:00:00 2001 From: Jussi Elo Date: Thu, 29 Oct 2026 15:31:24 +0000 Subject: [PATCH 2/2] test(ci): pin the vendored aws-lc-sys opt-out in the workflow contract Dropping `-e AWS_LC_SYS_USE_SYSTEM` from a manylinux build container, or changing the value a build resolves, would silently revert that build to host AWS-LC detection with no other symptom. For every shipped release `cargo build` step in release.yml, nightly.yml and ci.yml, resolve the variable the way GitHub does (step env, then job env, then workflow env) and require 0, and require the manylinux docker run that builds to forward it with `-e` or `--env`. The expected number of shipped builds is pinned per workflow, so a new build has to pin the variable too. The checker is also exercised on inline YAML, covering each env layer, the accepted forwarding spellings, unnamed steps, flag order and comments. Signed-off-by: Jussi Elo --- xtask/src/workflow_contract.rs | 286 +++++++++++++++++++++++++++++++++ 1 file changed, 286 insertions(+) diff --git a/xtask/src/workflow_contract.rs b/xtask/src/workflow_contract.rs index 6fe45c31f6..596bf1e14d 100644 --- a/xtask/src/workflow_contract.rs +++ b/xtask/src/workflow_contract.rs @@ -707,6 +707,292 @@ trigger-a-workflow#triggering-a-workflow-from-a-workflow" )); } + /// `step` without its full-line comments, so prose that quotes a command + /// cannot stand in for the command. + fn without_comments(step: &str) -> String { + step.lines() + .filter(|line| !line.trim_start().starts_with('#')) + .collect::>() + .join("\n") + } + + /// Whether `line` runs `cargo build` with `--release` or `-r`, in any flag + /// order and quoted or not. A command wrapped over several lines is not seen. + fn is_release_cargo_build(line: &str) -> bool { + let tokens: Vec<&str> = line + .split_whitespace() + .map(|token| token.trim_matches(|c| c == '\'' || c == '"')) + .collect(); + tokens.windows(2).any(|pair| pair == ["cargo", "build"]) + && tokens + .iter() + .any(|token| matches!(*token, "--release" | "-r")) + } + + /// Every step in `text` that builds shipped release binaries: a release + /// `cargo build` that is not the `e2e-test-hooks` harness build. + /// Named and unnamed (`- run:`) steps both count. + fn shipped_release_build_steps(text: &str) -> Vec { + let lines: Vec<&str> = text.lines().collect(); + lines + .iter() + .enumerate() + .filter(|(_, line)| { + let item = line.trim_start(); + item.starts_with("- name:") || item.starts_with("- run:") + }) + .map(|(i, _)| step_block(&lines, i)) + .filter(|step| { + let code = without_comments(step); + code.lines().any(is_release_cargo_build) && !code.contains("e2e-test-hooks") + }) + .collect() + } + + /// The `docker run` flag tokens before the image name (the first token + /// naming `manylinux`) of the invocation that runs the release build, + /// comments excluded. + fn container_build_flags(step: &str) -> Option> { + let code = without_comments(step); + let invocation = code + .split("docker run") + .skip(1) + .find(|part| part.lines().any(is_release_cargo_build))?; + Some( + invocation + .split_whitespace() + .take_while(|token| !token.contains("manylinux")) + .map(str::to_owned) + .collect(), + ) + } + + /// Where each shipped release build in a workflow gets `AWS_LC_SYS_USE_SYSTEM` + /// from, as one message per way it can fail to be `0`. + /// + /// `aws-lc-sys` 0.46+ links an AWS-LC it detects on the build host + /// (`OPENSSL_DIR` or pkg-config) unless the variable is `0`, so a shipped + /// binary could depend on whatever that host provides. GitHub resolves the + /// value step, then job, then workflow, so each build step is judged on its + /// own against that order. A manylinux container inherits none of them: a + /// dropped `-e AWS_LC_SYS_USE_SYSTEM` reverts that build to host detection + /// with no other symptom. + fn aws_lc_opt_out_violations(workflow: &str, text: &str, expected_steps: usize) -> Vec { + const VAR: &str = "AWS_LC_SYS_USE_SYSTEM"; + let entry = |line: &str| { + line.trim() + .strip_prefix("AWS_LC_SYS_USE_SYSTEM:") + .map(|value| strip_quotes(strip_comment(value).trim())) + }; + let workflow_value = text + .lines() + .filter(|line| indent_of(line) == 2) + .find_map(entry); + let jobs = top_level_block(text, "jobs"); + let job_ids: Vec<&str> = jobs + .lines() + .filter(|line| indent_of(line) == 2 && !line.trim_start().starts_with('#')) + .filter_map(|line| line.trim().strip_suffix(':')) + .collect(); + + let mut violations = Vec::new(); + let mut shipped_steps = 0; + for job in job_ids { + let block = job_block(text, job); + // Entries of a job-level `env:` sit at indent 6; a step's own keys + // are two columns deeper, so they cannot match here. + let job_value = block + .lines() + .filter(|line| indent_of(line) == 6) + .find_map(entry); + for step in shipped_release_build_steps(block) { + shipped_steps += 1; + let value = step_env(&step) + .get(VAR) + .cloned() + .or_else(|| job_value.clone()) + .or_else(|| workflow_value.clone()); + if value.as_deref() != Some("0") { + violations.push(format!( + "{workflow}: a shipped release build resolves `{VAR}` to {value:?} from \ + its step, job and workflow env, not \"0\":\n{step}" + )); + } + let Some(flags) = container_build_flags(&step) else { + continue; + }; + let forwarded = flags.windows(2).any(|pair| { + matches!(pair[0].as_str(), "-e" | "--env") + && (pair[1] == VAR || pair[1] == format!("{VAR}=0")) + }); + if !forwarded { + violations.push(format!( + "{workflow}: the manylinux container build does not forward `{VAR}` \ + (`-e {VAR}`), so it builds against whatever AWS-LC the host \ + provides:\n{step}" + )); + } + } + } + if shipped_steps != expected_steps { + violations.push(format!( + "{workflow}: expected {expected_steps} shipped release build steps, found \ + {shipped_steps}. A step is counted when a line runs `cargo build` with \ + `--release` or `-r`; if a build was added or removed, update the count and \ + make sure it pins `{VAR}`, and if one was reshaped so it is no longer \ + recognised, teach `is_release_cargo_build` its spelling instead" + )); + } + violations + } + + #[test] + fn shipped_release_builds_pin_the_vendored_aws_lc_opt_out() { + // The Linux and the Windows bundle build of each workflow. + for (workflow, expected_steps) in [("release.yml", 2), ("nightly.yml", 2), ("ci.yml", 2)] { + let violations = + aws_lc_opt_out_violations(workflow, &read_workflow(workflow), expected_steps); + assert!(violations.is_empty(), "{}", violations.join("\n\n")); + } + } + + #[test] + fn aws_lc_opt_out_check_resolves_every_env_layer_and_spelling() { + const OFF: &str = "AWS_LC_SYS_USE_SYSTEM: \"0\""; + const ON: &str = "AWS_LC_SYS_USE_SYSTEM: \"1\""; + let workflow = |workflow_env: &str, job_env: &str, steps: &str| { + let job_env = if job_env.is_empty() { + String::new() + } else { + format!(" env:\n {job_env}\n") + }; + format!( + "env:\n {workflow_env}\njobs:\n build:\n runs-on: ubuntu-latest\n{job_env} steps:\n{steps}" + ) + }; + let native = |step_env: &str| { + let env = if step_env.is_empty() { + String::new() + } else { + format!(" env:\n {step_env}\n") + }; + format!(" - name: Build\n{env} run: cargo build --release -p rocm\n") + }; + let container = |flag: &str| { + r" - name: Build + run: | + docker run --rm \ + FLAG \ + quay.io/pypa/manylinux2014_x86_64 \ + /bin/bash -lc 'cargo build --release -p rocm' +" + .replace("FLAG", flag) + }; + let check = + |text: &str, expected: usize| aws_lc_opt_out_violations("t.yml", text, expected); + + assert!(check(&workflow(OFF, "", &native("")), 1).is_empty()); + assert!( + check(&workflow(OFF, ON, &native("")), 1).len() == 1, + "job env overrides the workflow's" + ); + assert!( + check(&workflow(OFF, ON, &native(OFF)), 1).is_empty(), + "step env overrides the job's" + ); + assert!( + check(&workflow(OFF, "", &native(ON)), 1).len() == 1, + "step env overrides the workflow's" + ); + assert!( + check(&workflow("", "", &native("")), 1).len() == 1, + "an unset variable is not 0" + ); + + for flag in [ + "-e AWS_LC_SYS_USE_SYSTEM", + "--env AWS_LC_SYS_USE_SYSTEM", + "-e AWS_LC_SYS_USE_SYSTEM=0", + ] { + assert!( + check(&workflow(OFF, "", &container(flag)), 1).is_empty(), + "{flag}" + ); + } + for flag in [ + "-e CARGO_HOME=/c", + "-e AWS_LC_SYS_USE_SYSTEM=1", + "# -e AWS_LC_SYS_USE_SYSTEM", + ] { + assert!( + check(&workflow(OFF, "", &container(flag)), 1).len() == 1, + "{flag}" + ); + } + + let unnamed = " - run: cargo build --release -p rocm\n"; + assert!(check(&workflow(OFF, "", unnamed), 1).is_empty()); + assert_eq!( + check(&workflow(OFF, "", unnamed), 0).len(), + 1, + "unnamed steps are counted" + ); + assert_eq!( + check(&workflow("", "", unnamed), 1).len(), + 1, + "unnamed steps are checked" + ); + let harness = " - name: Harness\n run: cargo build --release -p rocm --features rocm/e2e-test-hooks\n"; + assert!( + check(&workflow("", "", harness), 0).is_empty(), + "the e2e harness build is not shipped" + ); + for spelling in [ + "cargo build --locked --release -p rocm", + "cargo build -p rocm -r", + ] { + let step = format!(" - run: {spelling}\n"); + assert_eq!(check(&workflow("", "", &step), 1).len(), 1, "{spelling}"); + assert!(check(&workflow(OFF, "", &step), 1).is_empty(), "{spelling}"); + } + assert!( + check(&workflow("", "", " - run: cargo build -p rocm\n"), 0).is_empty(), + "a debug build is not shipped" + ); + let quoting_the_harness = " - name: Build\n # unlike the e2e-test-hooks build\n run: cargo build --release -p rocm\n"; + assert_eq!( + check(&workflow("", "", quoting_the_harness), 1).len(), + 1, + "a comment cannot exempt a shipped build" + ); + + assert_eq!( + check(&workflow(OFF, "", &native("")), 2).len(), + 1, + "a missing shipped build is reported" + ); + let commented = "AWS_LC_SYS_USE_SYSTEM: \"0\" # vendored"; + assert!( + check(&workflow(commented, "", &native("")), 1).is_empty(), + "workflow value with a trailing comment" + ); + assert!( + check(&workflow("", commented, &native("")), 1).is_empty(), + "job value with a trailing comment" + ); + let after_image = " - name: Build\n run: |\n docker run --rm quay.io/pypa/manylinux2014_x86_64 \\\n -e AWS_LC_SYS_USE_SYSTEM \\\n /bin/bash -lc 'cargo build --release -p rocm'\n"; + assert_eq!( + check(&workflow(OFF, "", after_image), 1).len(), + 1, + "a flag after the image is not a docker flag" + ); + let comment_key = workflow(OFF, "", &native("")).replace("jobs:\n", "jobs:\n # note:\n"); + assert!( + check(&comment_key, 1).is_empty(), + "a comment ending in a colon is not a job" + ); + } + #[test] fn every_ci_job_declares_a_timeout() { let ci = read_workflow("ci.yml");