diff --git a/docs/nightly-on-the-queue.md b/docs/nightly-on-the-queue.md index bae5d09d9..13c5bc4b8 100644 --- a/docs/nightly-on-the-queue.md +++ b/docs/nightly-on-the-queue.md @@ -117,11 +117,12 @@ run, so a failing channel does not hide the other, and the lane fails if either did. A lane starts cold, so its first steps fetch from the network: the build -packages, the rustup installer, the toolchain and the crates. The lane command -tries each of those up to four times, and never retries a build or a test. The -pre-warm's package downloads and the suite's model downloads are left to the -tools doing them, so a model that downloads more slowly than its scenario waits -fails that scenario. +packages, the rustup installer, the toolchain and the crates for the host +being built. The lane command tries each of those up to four times, and never +retries a build or a test. The pre-warm's package downloads and the suite's +model downloads are left to the tools doing them, so a model that downloads +more slowly than its scenario waits fails that scenario. The lane fetches only +the host's crates, so it refuses to start with `CARGO_BUILD_TARGET` set. The queue's results are an additional signal, not a replacement: the lanes write only into the queue artifact, and the `e2e-gpu--report` diff --git a/scripts/run-e2e-lane.sh b/scripts/run-e2e-lane.sh index 1b70c5a57..68e8569f8 100755 --- a/scripts/run-e2e-lane.sh +++ b/scripts/run-e2e-lane.sh @@ -30,6 +30,14 @@ out="${LANE_OUT:-/out}" work="${E2E_LANE_WORK:-/var/tmp/e2e-lane}" channels="${E2E_CHANNELS:-release nightly}" +# The lane fetches only the host's crates, so a build target set from outside +# would need crates it skips and meet the network without a retry. Refused +# before anything is installed. This covers the environment variable only. +if [[ -n "${CARGO_BUILD_TARGET:-}" ]]; then + echo "CARGO_BUILD_TARGET is set; unset it: the lane fetches only the host's crates" >&2 + exit 1 +fi + export E2E_INCLUDE_NIGHTLY="${E2E_INCLUDE_NIGHTLY-1}" export E2E_SERVE_TIMEOUT_SECS="${E2E_SERVE_TIMEOUT_SECS:-300}" @@ -158,10 +166,23 @@ bash .github/scripts/gpu-preflight.sh preflight_done=0 # The two fetches in front of the build, each retried on its own. The first cargo -# call in this checkout downloads the toolchain rust-toolchain.toml names; then -# every crate is fetched, so the build below does not meet the network. +# call in this checkout downloads the toolchain rust-toolchain.toml names: that +# relies on rustup's proxy installing it on first call, and is a no-op where +# cargo is not rustup-managed. Then the crates are fetched, so the build below +# does not meet the network. Only the host's are: the build, the pre-warm and +# the suite all run on the host (see the CARGO_BUILD_TARGET check at the top). retry 4 30 cargo --version -retry 4 30 cargo fetch --locked +# Captured first, so a cargo that fails here aborts the lane with its own error. +cargo_info="$(cargo -vV)" +host="" +while read -r key value; do + if [[ "${key}" == "host:" ]]; then host="${value}"; fi +done <<<"${cargo_info}" +if [[ -z "${host}" ]]; then + echo "cargo did not report its host triple" >&2 + exit 1 +fi +retry 4 30 cargo fetch --locked --target "${host}" # The e2e-test-hooks feature must match what `cargo xtask e2e` builds for # itself; see the e2e-gpu job in e2e-selfhosted.yml. diff --git a/xtask/src/workflow_contract.rs b/xtask/src/workflow_contract.rs index 8da428bd6..67ca29248 100644 --- a/xtask/src/workflow_contract.rs +++ b/xtask/src/workflow_contract.rs @@ -9653,9 +9653,14 @@ permissions: apt_install_failures: u32, curl_failures: u32, /// How often `cargo --version` (the toolchain download) and - /// `cargo fetch --locked` fail first, each with status 7. + /// `cargo fetch --locked --target ` fail first, each with + /// status 7. version_failures: u32, fetch_failures: u32, + /// Whether `cargo -vV` prints a `host:` line. + host_line: bool, + /// A `CARGO_BUILD_TARGET` in the lane's environment, if any. + build_target: Option<&'static str>, suite_rc: i32, nightly_rc: Option, channels: Option<&'static str>, @@ -9678,6 +9683,8 @@ permissions: curl_failures: 0, version_failures: 0, fetch_failures: 0, + host_line: true, + build_target: None, suite_rc: 0, nightly_rc: None, channels: None, @@ -9819,13 +9826,20 @@ permissions: ), ) .expect("stub sha256sum"); + let host_triple = format!("{}-unknown-linux-gnu", case.machine); + let host_reply = if case.host_line { + format!("echo \"host: {host_triple}\"") + } else { + "echo \"release: 1.96.0\"".to_owned() + }; write_fake_executable( &cargo_bin.join("cargo"), format!( "#!/bin/bash\necho \"cargo $*\"\n\ case \"$*\" in\n\ \"--version\") {version};;\n\ - \"fetch --locked\") {fetch};;\n\ + \"-vV\") {host};;\n\ + \"fetch --locked --target {triple}\") {fetch};;\n\ \"xtask e2e-prewarm\"*) echo \"prewarm HF_HOME=$HF_HOME UV_CACHE_DIR=$UV_CACHE_DIR\";;\n\ \"xtask e2e\")\n\ echo \"suite E2E_INCLUDE_NIGHTLY=${{E2E_INCLUDE_NIGHTLY-unset}} \ @@ -9843,6 +9857,8 @@ permissions: // own status. version = fail_first("cargo-version", case.version_failures, 7).replace('\n', "; "), fetch = fail_first("cargo-fetch", case.fetch_failures, 7).replace('\n', "; "), + triple = host_triple, + host = host_reply, rc = case.suite_rc, nightly = case.nightly_rc.unwrap_or(case.suite_rc) ), @@ -9859,6 +9875,9 @@ permissions: if let Some(channels) = case.channels { lane.env("E2E_CHANNELS", channels); } + if let Some(target) = case.build_target { + lane.env("CARGO_BUILD_TARGET", target); + } let output = lane.output().expect("running the lane script"); let mut log = String::from_utf8_lossy(&output.stdout).into_owned(); log.push_str(&String::from_utf8_lossy(&output.stderr)); @@ -9906,7 +9925,8 @@ permissions: "preflight ran\n".to_owned(), // The toolchain and the crates, fetched before the build. "cargo --version\n".to_owned(), - "cargo fetch --locked\n".to_owned(), + // `cargo -vV` answers the host triple to the script, so it is not logged. + "cargo fetch --locked --target x86_64-unknown-linux-gnu\n".to_owned(), "cargo build --release -p rocm -p rocmd --features rocm/e2e-test-hooks\n" .to_owned(), ]; @@ -10298,7 +10318,10 @@ permissions: fn a_lane_rides_out_a_failed_toolchain_or_crate_fetch_and_never_builds_without_one() { for (fetch, command) in [ ("version", "cargo --version"), - ("fetch", "cargo fetch --locked"), + ( + "fetch", + "cargo fetch --locked --target x86_64-unknown-linux-gnu", + ), ] { let case = |failures| LaneCase { version_failures: if fetch == "version" { failures } else { 0 }, @@ -10326,4 +10349,59 @@ permissions: assert!(!out.exists()); } } + + /// A cargo that reports no host triple stops the lane before any fetch or + /// build, rather than passing an empty `--target`. + #[cfg(unix)] + #[test] + fn a_lane_stops_when_cargo_reports_no_host_triple() { + let LaneRun { + rc, + log, + out, + dir: _dir, + } = run_lane(&LaneCase { + host_line: false, + ..LaneCase::default() + }); + assert_eq!(rc, 1, "{log}"); + assert!( + log.contains("cargo did not report its host triple"), + "{log}" + ); + assert!(!log.contains("cargo fetch"), "{log}"); + assert!(!log.contains("cargo build"), "{log}"); + assert!(!out.exists()); + } + + /// The fetch covers only the host's crates, so a build target set from + /// outside is refused up front, and a non-x86_64 host's triple is the one + /// the fetch is scoped to. + #[cfg(unix)] + #[test] + fn a_lane_refuses_a_build_target_and_scopes_the_fetch_to_the_reported_host() { + let LaneRun { + rc, + log, + out, + dir: _dir, + } = run_lane(&LaneCase { + build_target: Some("aarch64-apple-darwin"), + ..LaneCase::default() + }); + assert_eq!(rc, 1, "{log}"); + assert!(log.contains("CARGO_BUILD_TARGET is set"), "{log}"); + assert!(!log.contains("cargo fetch"), "{log}"); + assert!(!out.exists()); + + let LaneRun { rc, log, .. } = run_lane(&LaneCase { + machine: "aarch64", + ..LaneCase::default() + }); + assert_eq!(rc, 0, "{log}"); + assert!( + log.contains("cargo fetch --locked --target aarch64-unknown-linux-gnu\n"), + "{log}" + ); + } }