From 96f6275ef9abce21b613d5a34edf4cc2e55cc797 Mon Sep 17 00:00:00 2001 From: Tomas Saaristola Date: Thu, 12 Nov 2026 10:15:02 +0000 Subject: [PATCH 1/3] ci: fetch only the host's crates in the queue lane `cargo fetch --locked` without `--target` downloads crates for every platform. The lane builds, pre-warms and runs the suite on the host and sets no build target, so fetch the host triple's crates only, reading the triple from `cargo -vV`. Offline `cargo metadata` after a host-only fetch into an empty CARGO_HOME resolves the full workspace (386 MB downloaded, against 876 MB for the unrestricted fetch). Also note that the first `cargo --version` relies on rustup's proxy installing the toolchain named by rust-toolchain.toml, and is a no-op where cargo is not rustup-managed. Follow-up to review comments on #1009. Signed-off-by: Tomas Saaristola --- docs/nightly-on-the-queue.md | 6 ++--- scripts/run-e2e-lane.sh | 19 +++++++++++--- xtask/src/workflow_contract.rs | 46 +++++++++++++++++++++++++++++++--- 3 files changed, 61 insertions(+), 10 deletions(-) diff --git a/docs/nightly-on-the-queue.md b/docs/nightly-on-the-queue.md index bae5d09d9..3917a5a68 100644 --- a/docs/nightly-on-the-queue.md +++ b/docs/nightly-on-the-queue.md @@ -117,9 +117,9 @@ run, so a failing channel does not hide the other, and the lane fails if either did. A lane starts cold, so its first steps fetch from the network: the build -packages, the rustup installer, the toolchain and the crates. The lane command -tries each of those up to four times, and never retries a build or a test. The -pre-warm's package downloads and the suite's model downloads are left to the +packages, the rustup installer, the toolchain and the crates for the host +being built. The lane command tries each of those up to four times, and never +retries a build or a test. The pre-warm's package downloads and the suite's model downloads are left to the tools doing them, so a model that downloads more slowly than its scenario waits fails that scenario. diff --git a/scripts/run-e2e-lane.sh b/scripts/run-e2e-lane.sh index 1b70c5a57..05a987770 100755 --- a/scripts/run-e2e-lane.sh +++ b/scripts/run-e2e-lane.sh @@ -158,10 +158,23 @@ bash .github/scripts/gpu-preflight.sh preflight_done=0 # The two fetches in front of the build, each retried on its own. The first cargo -# call in this checkout downloads the toolchain rust-toolchain.toml names; then -# every crate is fetched, so the build below does not meet the network. +# call in this checkout downloads the toolchain rust-toolchain.toml names: that +# relies on rustup's proxy installing it on first call, and is a no-op where +# cargo is not rustup-managed. Then the crates are fetched, so the build below +# does not meet the network. Only the host's are: the build, the pre-warm and +# the suite all run on the host, and nothing here sets a build target. retry 4 30 cargo --version -retry 4 30 cargo fetch --locked +# Captured first, so a cargo that fails here aborts the lane with its own error. +cargo_info="$(cargo -vV)" +host="" +while read -r key value; do + if [[ "${key}" == "host:" ]]; then host="${value}"; fi +done <<<"${cargo_info}" +if [[ -z "${host}" ]]; then + echo "cargo did not report its host triple" >&2 + exit 1 +fi +retry 4 30 cargo fetch --locked --target "${host}" # The e2e-test-hooks feature must match what `cargo xtask e2e` builds for # itself; see the e2e-gpu job in e2e-selfhosted.yml. diff --git a/xtask/src/workflow_contract.rs b/xtask/src/workflow_contract.rs index 8da428bd6..c225adc0f 100644 --- a/xtask/src/workflow_contract.rs +++ b/xtask/src/workflow_contract.rs @@ -9653,9 +9653,12 @@ permissions: apt_install_failures: u32, curl_failures: u32, /// How often `cargo --version` (the toolchain download) and - /// `cargo fetch --locked` fail first, each with status 7. + /// `cargo fetch --locked --target ` fail first, each with + /// status 7. version_failures: u32, fetch_failures: u32, + /// Whether `cargo -vV` prints a `host:` line. + host_line: bool, suite_rc: i32, nightly_rc: Option, channels: Option<&'static str>, @@ -9678,6 +9681,7 @@ permissions: curl_failures: 0, version_failures: 0, fetch_failures: 0, + host_line: true, suite_rc: 0, nightly_rc: None, channels: None, @@ -9825,7 +9829,8 @@ permissions: "#!/bin/bash\necho \"cargo $*\"\n\ case \"$*\" in\n\ \"--version\") {version};;\n\ - \"fetch --locked\") {fetch};;\n\ + \"-vV\") {host};;\n\ + \"fetch --locked --target x86_64-unknown-linux-gnu\") {fetch};;\n\ \"xtask e2e-prewarm\"*) echo \"prewarm HF_HOME=$HF_HOME UV_CACHE_DIR=$UV_CACHE_DIR\";;\n\ \"xtask e2e\")\n\ echo \"suite E2E_INCLUDE_NIGHTLY=${{E2E_INCLUDE_NIGHTLY-unset}} \ @@ -9843,6 +9848,11 @@ permissions: // own status. version = fail_first("cargo-version", case.version_failures, 7).replace('\n', "; "), fetch = fail_first("cargo-fetch", case.fetch_failures, 7).replace('\n', "; "), + host = if case.host_line { + "echo \"host: x86_64-unknown-linux-gnu\"" + } else { + "echo \"release: 1.96.0\"" + }, rc = case.suite_rc, nightly = case.nightly_rc.unwrap_or(case.suite_rc) ), @@ -9906,7 +9916,8 @@ permissions: "preflight ran\n".to_owned(), // The toolchain and the crates, fetched before the build. "cargo --version\n".to_owned(), - "cargo fetch --locked\n".to_owned(), + // `cargo -vV` answers the host triple to the script, so it is not logged. + "cargo fetch --locked --target x86_64-unknown-linux-gnu\n".to_owned(), "cargo build --release -p rocm -p rocmd --features rocm/e2e-test-hooks\n" .to_owned(), ]; @@ -10298,7 +10309,10 @@ permissions: fn a_lane_rides_out_a_failed_toolchain_or_crate_fetch_and_never_builds_without_one() { for (fetch, command) in [ ("version", "cargo --version"), - ("fetch", "cargo fetch --locked"), + ( + "fetch", + "cargo fetch --locked --target x86_64-unknown-linux-gnu", + ), ] { let case = |failures| LaneCase { version_failures: if fetch == "version" { failures } else { 0 }, @@ -10326,4 +10340,28 @@ permissions: assert!(!out.exists()); } } + + /// A cargo that reports no host triple stops the lane before any fetch or + /// build, rather than passing an empty `--target`. + #[cfg(unix)] + #[test] + fn a_lane_stops_when_cargo_reports_no_host_triple() { + let LaneRun { + rc, + log, + out, + dir: _dir, + } = run_lane(&LaneCase { + host_line: false, + ..LaneCase::default() + }); + assert_eq!(rc, 1, "{log}"); + assert!( + log.contains("cargo did not report its host triple"), + "{log}" + ); + assert!(!log.contains("cargo fetch"), "{log}"); + assert!(!log.contains("cargo build"), "{log}"); + assert!(!out.exists()); + } } From 518ed16359c8b31b19222d2bdf53266f0d15360d Mon Sep 17 00:00:00 2001 From: Tomas Saaristola Date: Thu, 12 Nov 2026 13:39:33 +0000 Subject: [PATCH 2/3] ci: refuse a build target in the queue lane, rewrap the lane doc The lane fetches only the host's crates, so a CARGO_BUILD_TARGET set from outside would need crates it skipped and meet the network without a retry. Refuse it up front, and exercise the host-scoped fetch with a non-x86_64 triple. Rewrap the paragraph the earlier reflow left uneven. Signed-off-by: Tomas Saaristola --- docs/nightly-on-the-queue.md | 7 +++--- scripts/run-e2e-lane.sh | 10 +++++++- xtask/src/workflow_contract.rs | 44 +++++++++++++++++++++++++++++++--- 3 files changed, 54 insertions(+), 7 deletions(-) diff --git a/docs/nightly-on-the-queue.md b/docs/nightly-on-the-queue.md index 3917a5a68..13c5bc4b8 100644 --- a/docs/nightly-on-the-queue.md +++ b/docs/nightly-on-the-queue.md @@ -119,9 +119,10 @@ did. A lane starts cold, so its first steps fetch from the network: the build packages, the rustup installer, the toolchain and the crates for the host being built. The lane command tries each of those up to four times, and never -retries a build or a test. The pre-warm's package downloads and the suite's model downloads are left to the -tools doing them, so a model that downloads more slowly than its scenario waits -fails that scenario. +retries a build or a test. The pre-warm's package downloads and the suite's +model downloads are left to the tools doing them, so a model that downloads +more slowly than its scenario waits fails that scenario. The lane fetches only +the host's crates, so it refuses to start with `CARGO_BUILD_TARGET` set. The queue's results are an additional signal, not a replacement: the lanes write only into the queue artifact, and the `e2e-gpu--report` diff --git a/scripts/run-e2e-lane.sh b/scripts/run-e2e-lane.sh index 05a987770..68e8569f8 100755 --- a/scripts/run-e2e-lane.sh +++ b/scripts/run-e2e-lane.sh @@ -30,6 +30,14 @@ out="${LANE_OUT:-/out}" work="${E2E_LANE_WORK:-/var/tmp/e2e-lane}" channels="${E2E_CHANNELS:-release nightly}" +# The lane fetches only the host's crates, so a build target set from outside +# would need crates it skips and meet the network without a retry. Refused +# before anything is installed. This covers the environment variable only. +if [[ -n "${CARGO_BUILD_TARGET:-}" ]]; then + echo "CARGO_BUILD_TARGET is set; unset it: the lane fetches only the host's crates" >&2 + exit 1 +fi + export E2E_INCLUDE_NIGHTLY="${E2E_INCLUDE_NIGHTLY-1}" export E2E_SERVE_TIMEOUT_SECS="${E2E_SERVE_TIMEOUT_SECS:-300}" @@ -162,7 +170,7 @@ preflight_done=0 # relies on rustup's proxy installing it on first call, and is a no-op where # cargo is not rustup-managed. Then the crates are fetched, so the build below # does not meet the network. Only the host's are: the build, the pre-warm and -# the suite all run on the host, and nothing here sets a build target. +# the suite all run on the host (see the CARGO_BUILD_TARGET check at the top). retry 4 30 cargo --version # Captured first, so a cargo that fails here aborts the lane with its own error. cargo_info="$(cargo -vV)" diff --git a/xtask/src/workflow_contract.rs b/xtask/src/workflow_contract.rs index c225adc0f..806baf0a8 100644 --- a/xtask/src/workflow_contract.rs +++ b/xtask/src/workflow_contract.rs @@ -9659,6 +9659,8 @@ permissions: fetch_failures: u32, /// Whether `cargo -vV` prints a `host:` line. host_line: bool, + /// A `CARGO_BUILD_TARGET` in the lane's environment, if any. + build_target: Option<&'static str>, suite_rc: i32, nightly_rc: Option, channels: Option<&'static str>, @@ -9682,6 +9684,7 @@ permissions: version_failures: 0, fetch_failures: 0, host_line: true, + build_target: None, suite_rc: 0, nightly_rc: None, channels: None, @@ -9830,7 +9833,7 @@ permissions: case \"$*\" in\n\ \"--version\") {version};;\n\ \"-vV\") {host};;\n\ - \"fetch --locked --target x86_64-unknown-linux-gnu\") {fetch};;\n\ + \"fetch --locked --target {triple}\") {fetch};;\n\ \"xtask e2e-prewarm\"*) echo \"prewarm HF_HOME=$HF_HOME UV_CACHE_DIR=$UV_CACHE_DIR\";;\n\ \"xtask e2e\")\n\ echo \"suite E2E_INCLUDE_NIGHTLY=${{E2E_INCLUDE_NIGHTLY-unset}} \ @@ -9848,10 +9851,11 @@ permissions: // own status. version = fail_first("cargo-version", case.version_failures, 7).replace('\n', "; "), fetch = fail_first("cargo-fetch", case.fetch_failures, 7).replace('\n', "; "), + triple = format!("{}-unknown-linux-gnu", case.machine), host = if case.host_line { - "echo \"host: x86_64-unknown-linux-gnu\"" + format!("echo \"host: {}-unknown-linux-gnu\"", case.machine) } else { - "echo \"release: 1.96.0\"" + "echo \"release: 1.96.0\"".to_owned() }, rc = case.suite_rc, nightly = case.nightly_rc.unwrap_or(case.suite_rc) @@ -9869,6 +9873,9 @@ permissions: if let Some(channels) = case.channels { lane.env("E2E_CHANNELS", channels); } + if let Some(target) = case.build_target { + lane.env("CARGO_BUILD_TARGET", target); + } let output = lane.output().expect("running the lane script"); let mut log = String::from_utf8_lossy(&output.stdout).into_owned(); log.push_str(&String::from_utf8_lossy(&output.stderr)); @@ -10364,4 +10371,35 @@ permissions: assert!(!log.contains("cargo build"), "{log}"); assert!(!out.exists()); } + + /// The fetch covers only the host's crates, so a build target set from + /// outside is refused up front, and a non-x86_64 host's triple is the one + /// the fetch is scoped to. + #[cfg(unix)] + #[test] + fn a_lane_refuses_a_build_target_and_scopes_the_fetch_to_the_reported_host() { + let LaneRun { + rc, + log, + out, + dir: _dir, + } = run_lane(&LaneCase { + build_target: Some("aarch64-apple-darwin"), + ..LaneCase::default() + }); + assert_eq!(rc, 1, "{log}"); + assert!(log.contains("CARGO_BUILD_TARGET is set"), "{log}"); + assert!(!log.contains("cargo fetch"), "{log}"); + assert!(!out.exists()); + + let LaneRun { rc, log, .. } = run_lane(&LaneCase { + machine: "aarch64", + ..LaneCase::default() + }); + assert_eq!(rc, 0, "{log}"); + assert!( + log.contains("cargo fetch --locked --target aarch64-unknown-linux-gnu\n"), + "{log}" + ); + } } From 2838b5a625b212bb4611444c3e80b73dde85cfe3 Mon Sep 17 00:00:00 2001 From: Tomas Saaristola Date: Thu, 12 Nov 2026 13:50:53 +0000 Subject: [PATCH 3/3] ci: build the stub cargo's host reply outside the format call Clippy's format_in_format_args refused the nested format! introduced with the host-triple stub. Signed-off-by: Tomas Saaristola --- xtask/src/workflow_contract.rs | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/xtask/src/workflow_contract.rs b/xtask/src/workflow_contract.rs index 806baf0a8..67ca29248 100644 --- a/xtask/src/workflow_contract.rs +++ b/xtask/src/workflow_contract.rs @@ -9826,6 +9826,12 @@ permissions: ), ) .expect("stub sha256sum"); + let host_triple = format!("{}-unknown-linux-gnu", case.machine); + let host_reply = if case.host_line { + format!("echo \"host: {host_triple}\"") + } else { + "echo \"release: 1.96.0\"".to_owned() + }; write_fake_executable( &cargo_bin.join("cargo"), format!( @@ -9851,12 +9857,8 @@ permissions: // own status. version = fail_first("cargo-version", case.version_failures, 7).replace('\n', "; "), fetch = fail_first("cargo-fetch", case.fetch_failures, 7).replace('\n', "; "), - triple = format!("{}-unknown-linux-gnu", case.machine), - host = if case.host_line { - format!("echo \"host: {}-unknown-linux-gnu\"", case.machine) - } else { - "echo \"release: 1.96.0\"".to_owned() - }, + triple = host_triple, + host = host_reply, rc = case.suite_rc, nightly = case.nightly_rc.unwrap_or(case.suite_rc) ),