From 554574f22539b56ac743d58466bb2c3a052a66f0 Mon Sep 17 00:00:00 2001 From: Roman Inflianskas Date: Wed, 22 Jul 2026 13:43:49 +0000 Subject: [PATCH 01/11] test(e2e): exercise hosted WSL2 Signed-off-by: Roman Inflianskas --- .github/workflows/ci.yml | 66 +++++++++++++++++- docs/ci-hardware-testing.md | 24 +++++-- tests/e2e-cucumber/README.md | 6 +- tests/e2e-cucumber/features/examine.feature | 10 +++ tests/e2e-cucumber/src/capability.rs | 56 ++++++++++++---- tests/e2e-cucumber/src/expectation.rs | 67 ++++++++++++++++--- tests/e2e-cucumber/tests/e2e/examine_steps.rs | 36 ++++++++++ 7 files changed, 236 insertions(+), 29 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 609be8f0e9..3a534fc1bf 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -26,7 +26,7 @@ on: description: Which runner(s) to target type: choice default: all - options: [all, mock] + options: [all, mock, wsl] # No tier input: the harness resolves pass/xfail/skip per scenario, so a # platform runs one job covering everything applicable to it. @@ -715,6 +715,69 @@ jobs: name: e2e-report path: tests/e2e-cucumber/results/ + # Hosted WSL2 coverage: runs the black-box suite in a real Ubuntu distro on a + # windows-latest VM. GitHub-hosted runners have no AMD GPU, so GPU scenarios + # resolve to skip; this job covers the WSL host boundary, build, and CLI paths. + # Advisory while the hosted WSL image path is proven out. + e2e-wsl: + name: E2E tests (WSL2, no GPU) + timeout-minutes: 30 + runs-on: windows-latest + needs: [changes, build-and-test] + if: >- + always() + && needs.changes.result == 'success' + && ( + (github.event_name != 'workflow_dispatch' + && needs.build-and-test.result == 'success' + && needs.changes.outputs.heavy == 'true') + || (github.event_name == 'workflow_dispatch' + && (inputs.platform == 'all' || inputs.platform == 'wsl')) + ) + continue-on-error: true + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + + - name: Set up Ubuntu on WSL2 + uses: Vampire/setup-wsl@d1da7f2c0322a5ee4f24975344f67fc0f5baf364 # v7.0.0 + with: + distribution: Ubuntu-24.04 + wsl-version: 2 + use-cache: false + additional-packages: build-essential ca-certificates curl libcap-dev pkg-config + + - name: Verify WSL2 host + shell: wsl-bash {0} + run: | + set -euo pipefail + grep -qi microsoft /proc/sys/kernel/osrelease + test -n "${WSL_INTEROP:-}" + printf 'WSL kernel: %s\n' "$(uname -r)" + + - name: Ensure Rust toolchain in WSL2 + shell: wsl-bash {0} + run: | + set -euo pipefail + if ! command -v cargo >/dev/null 2>&1; then + curl --proto '=https' --tlsv1.2 -fsSL https://sh.rustup.rs \ + | sh -s -- -y --no-modify-path --default-toolchain none + fi + "$HOME/.cargo/bin/rustup" show active-toolchain + + - name: Run E2E tests in WSL2 + shell: wsl-bash {0} + run: | + set -euo pipefail + export PATH="$HOME/.cargo/bin:$PATH" + cargo xtask e2e + + - name: Upload WSL2 E2E report + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: e2e-wsl-report + path: tests/e2e-cucumber/results/ + # Consolidate the mock (GitHub-hosted) platform's report into the cross-platform # report: a platform × tier matrix in the run Summary plus a single merged HTML # artifact. `if: always()` so a failing platform still appears; runs on @@ -733,6 +796,7 @@ jobs: needs: - changes - e2e + - e2e-wsl # Consolidate whatever ran. On dispatch `heavy` is unset, so also run when the # trigger was manual; `always()` still lets it collect partial/failed tiers. if: >- diff --git a/docs/ci-hardware-testing.md b/docs/ci-hardware-testing.md index 471deed027..01f87fd853 100644 --- a/docs/ci-hardware-testing.md +++ b/docs/ci-hardware-testing.md @@ -4,7 +4,7 @@ Copyright © Advanced Micro Devices, Inc., or its affiliates. SPDX-License-Identifier: MIT --> -# CI hardware (GPU) testing +# CI hardware (GPU / WSL) testing The hosted CI (`ubuntu-latest`, `windows-latest`) builds and unit-tests every shipping target natively, but GitHub-hosted runners have no AMD GPU. A @@ -32,6 +32,7 @@ separate tier flag or tag filter to maintain. | Job | Workflow | Platform | Runner labels | |---|---|---|---| | `e2e` | `ci.yml` | Mock (no GPU) | GitHub-hosted `ubuntu-latest` | +| `e2e-wsl` | `ci.yml` | Ubuntu on WSL2 (no GPU) | GitHub-hosted `windows-latest` | | `e2e-gpu` | `e2e-selfhosted.yml` | MI300X (AMD Instinct, bare-metal Linux) | self-hosted `[self-hosted, linux, amd-gpu]` | | `e2e-gpu-strix-ubuntu` | `e2e-selfhosted.yml` | Strix Halo (gfx1151) on Ubuntu | self-hosted `[self-hosted, linux, strix-halo, native]` | | `e2e-gpu-strix-windows` | `e2e-selfhosted.yml` | Strix Halo (gfx1151) on native Windows 11 | self-hosted `[self-hosted, windows, strix-halo, native]` | @@ -44,6 +45,16 @@ hardcoded `/home/ubuntu/actions-runner` paths exist only on the native one. resolve to skip here, and known bugs resolve to xfail from `expectations.toml`. It is a required check and must stay green. +`e2e-wsl` is the hosted WSL2 lane, also in `ci.yml`: it registers an Ubuntu +distro under WSL2 on a `windows-latest` VM, builds the CLI inside it, and runs +the same suite. It covers real WSL detection and the Windows-to-WSL execution +boundary — paths no Linux or Windows lane exercises. It is **not** GPU-on-WSL +validation: a hosted runner has no AMD GPU, so `/dev/dxg`, the Windows AMD +driver, and live serving stay uncovered, and GPU scenarios resolve to skip. +Those need a self-hosted Windows machine with both an AMD GPU and a configured +WSL distro; a future GPU-on-WSL lane should stay a separate, non-blocking +platform until its runner, distro, and GPU access are validated end to end. + The three GPU jobs (`e2e-gpu`, `e2e-gpu-strix-ubuntu`, `e2e-gpu-strix-windows`) run on dedicated self-hosted runners with a real AMD GPU attached, so they exercise host/GPU detection, engine `detect`/`capabilities`, and live serving @@ -87,8 +98,9 @@ They can also be triggered manually via `e2e-selfhosted.yml`'s - `platform` (choice: `all`, `app-dev-gpu`, `strix-ubuntu`, `strix-windows`) — which self-hosted job(s) to run. `app-dev-gpu` maps to `e2e-gpu`, `strix-ubuntu` to `e2e-gpu-strix-ubuntu`, and `strix-windows` to - `e2e-gpu-strix-windows`. (The mock lane has its own `platform` input on - `ci.yml`; it is not part of this workflow.) + `e2e-gpu-strix-windows`. (The GitHub-hosted lanes have their own `platform` + input on `ci.yml`, with `mock` and `wsl` options; they are not part of this + workflow.) - `name_filter` (string) — a scenario-name regex forwarded to the cucumber harness (`cargo xtask e2e -- --name `) so a dispatch can run a single scenario instead of the full suite. Empty runs everything applicable @@ -108,7 +120,9 @@ gh workflow run e2e-selfhosted.yml --ref -f platform=app-dev-gpu The three hardware jobs — `e2e-gpu`, `e2e-gpu-strix-ubuntu`, and `e2e-gpu-strix-windows` — all run with `continue-on-error: true`, so a hardware failure that RUNS never gates a PR merge. Their results still surface -in the self-hosted consolidated report for visibility. +in the self-hosted consolidated report for visibility. The hosted `e2e-wsl` +lane is non-blocking on the same terms while the hosted WSL image path is +proven out; only `ci.yml`'s mock `e2e` job is required. **Required-check caveat.** These three job names (plus, historically, a consolidated-report name) are still in `main`'s required-status-check list. @@ -130,6 +144,8 @@ which requires write access to trigger). ## Notes +- `e2e-wsl` reports under its own platform slug even without a GPU, so its + results never collide with the ordinary mock report. - The hardware jobs build and run **release** binaries: they assert functional behavior (device detection, engine launch, policy enforcement), not performance, so this is not a performance benchmark. Release-fidelity, diff --git a/tests/e2e-cucumber/README.md b/tests/e2e-cucumber/README.md index 4519b61e68..5f6f44e1df 100644 --- a/tests/e2e-cucumber/README.md +++ b/tests/e2e-cucumber/README.md @@ -98,7 +98,8 @@ Scenarios carry stable-id and capability tags: | Tag | Meaning | |---|---| | `@id:` | Stable scenario id. Keys the expectation matrix and the report grid; every scenario has one. | -| `@requires-gpu` | Needs a real AMD GPU. Resolves to **skip** (n/a) on a host with none (e.g. the mock job). | +| `@requires-gpu` | Needs a real AMD GPU. Resolves to **skip** (n/a) on a host with none (e.g. the mock or hosted WSL job). | +| `@requires-wsl` | Needs a real WSL host. Resolves to **skip** on native Linux, native Windows, and other environments. | | `@requires-engine:` | Pins the serve engine. Resolves to skip where that engine can't start (e.g. vLLM on a lemonade-only Strix host). | | `@requires-os:` | Premise is OS-specific; skip on other OSes. | | `@serve-timeout:` | Lengthen the serve-readiness wait for a genuinely slow serve (e.g. a large model). | @@ -129,7 +130,8 @@ self-hosted runner can never stall `ci.yml`'s merge-required checks: | Job | Workflow | Platform | Blocking | |---|---|---|---| -| `e2e` | `ci.yml` | Mock (no GPU, GitHub-hosted) | yes | +| `e2e` | `ci.yml` | Mock (no GPU, GitHub-hosted Linux) | yes | +| `e2e-wsl` | `ci.yml` | WSL2 / Ubuntu (no GPU, GitHub-hosted Windows) | no | | `e2e-gpu` | `e2e-selfhosted.yml` | MI300X (self-hosted) | no | | `e2e-gpu-strix-ubuntu` | `e2e-selfhosted.yml` | Strix Halo / Ubuntu (self-hosted) | no | | `e2e-gpu-strix-windows` | `e2e-selfhosted.yml` | Strix Halo / Windows (self-hosted) | no | diff --git a/tests/e2e-cucumber/features/examine.feature b/tests/e2e-cucumber/features/examine.feature index 4ccece7f5a..9410d4fd1b 100644 --- a/tests/e2e-cucumber/features/examine.feature +++ b/tests/e2e-cucumber/features/examine.feature @@ -101,3 +101,13 @@ Feature: GPU detection and system inspection When the user inspects the system without probing frameworks Then the inspection reports that it skipped the frameworks And it still states a verdict for this machine + + # The inverse of `@requires-bare-metal`: WSL2 reports an os_family of `linux`, + # so `@requires-os:linux` cannot express "only where the host really is WSL". + # Runs only on the WSL lane; everywhere else the premise does not exist. + @id:examine-detects-wsl @requires-wsl + Scenario: 11 - System inspection recognizes a WSL host + Given the CLI is running in WSL + When the user inspects the system + Then the inspection reports Linux as the operating system + And the inspection reports that the host is WSL diff --git a/tests/e2e-cucumber/src/capability.rs b/tests/e2e-cucumber/src/capability.rs index 352dffce85..744210b9c9 100644 --- a/tests/e2e-cucumber/src/capability.rs +++ b/tests/e2e-cucumber/src/capability.rs @@ -335,7 +335,8 @@ fn probe_host_capability() -> HostCapability { let has_amd_gpu = gfx_target.is_some(); let available_engines = parse_engines_list(&engines); let effective_serve_engine = effective_serve_engine(gfx_target.as_deref(), &os_family); - let platform_slug = derive_platform_slug(has_amd_gpu, gfx_target.as_deref(), &os_family); + let platform_slug = + derive_platform_slug(has_amd_gpu, gfx_target.as_deref(), &os_family, is_wsl); HostCapability { os_family, @@ -405,32 +406,54 @@ fn parse_engines_list(text: &str) -> Vec { engines } -/// Stable platform identity from hardware. No AMD GPU → "mock". Otherwise a +/// Stable platform identity from hardware and host environment. WSL is a +/// distinct platform even without a GPU, so its report never collides with the +/// ordinary hosted mock column. Otherwise no AMD GPU → "mock"; GPU hosts use a /// coarse slug from the gfx family (data-center → "mi300x"; Strix gfx115x → /// "strix-halo"), falling back to the normalized family. The OS is appended for /// families that ship on more than one OS (Strix Halo runs both Ubuntu and /// Windows on the same gfx1151), so those become distinct grid columns rather /// than colliding into one. -fn derive_platform_slug(has_amd_gpu: bool, gfx_target: Option<&str>, os_family: &str) -> String { +fn derive_platform_slug( + has_amd_gpu: bool, + gfx_target: Option<&str>, + os_family: &str, + is_wsl: bool, +) -> String { + if is_wsl { + return match gfx_target { + Some(t) => format!("{}-wsl", platform_hardware_slug(t)), + None => "wsl".to_owned(), + }; + } if !has_amd_gpu { return "mock".to_owned(); } match gfx_target { Some(t) => { - let f = normalize_family(t); - if f.ends_with("-dcgpu") { - "mi300x".to_owned() - } else if f.starts_with("gfx115") { + let hardware = platform_hardware_slug(t); + if hardware == "strix-halo" { // Same silicon on Ubuntu and Windows — disambiguate by OS. - format!("strix-halo-{}", os_normalized(os_family)) + format!("{hardware}-{}", os_normalized(os_family)) } else { - f + hardware } } None => "mock".to_owned(), } } +fn platform_hardware_slug(gfx_target: &str) -> String { + let family = normalize_family(gfx_target); + if family.ends_with("-dcgpu") { + "mi300x".to_owned() + } else if family.starts_with("gfx115") { + "strix-halo".to_owned() + } else { + family + } +} + /// Short OS token for a platform slug: "windows" / "linux" / else the raw value. fn os_normalized(os_family: &str) -> String { let o = os_family.trim().to_ascii_lowercase(); @@ -571,20 +594,27 @@ Local model engines #[test] fn platform_slug_derivation() { - assert_eq!(derive_platform_slug(false, None, "other"), "mock"); + assert_eq!(derive_platform_slug(false, None, "other", false), "mock"); assert_eq!( - derive_platform_slug(true, Some("gfx942"), "linux"), + derive_platform_slug(true, Some("gfx942"), "linux", false), "mi300x" ); // Strix Halo: same gfx1151 silicon on both OSes → distinct slugs so the // report grid gets a column per platform, not a collision. assert_eq!( - derive_platform_slug(true, Some("gfx1151"), "linux"), + derive_platform_slug(true, Some("gfx1151"), "linux", false), "strix-halo-linux" ); assert_eq!( - derive_platform_slug(true, Some("gfx1151"), "windows"), + derive_platform_slug(true, Some("gfx1151"), "windows", false), "strix-halo-windows" ); + // Hosted WSL has no GPU but must not collide with the ordinary mock + // column. A future WSL GPU lane also remains distinct from native Linux. + assert_eq!(derive_platform_slug(false, None, "linux", true), "wsl"); + assert_eq!( + derive_platform_slug(true, Some("gfx1151"), "linux", true), + "strix-halo-wsl" + ); } } diff --git a/tests/e2e-cucumber/src/expectation.rs b/tests/e2e-cucumber/src/expectation.rs index 1632bb6623..bf24bbca46 100644 --- a/tests/e2e-cucumber/src/expectation.rs +++ b/tests/e2e-cucumber/src/expectation.rs @@ -27,6 +27,7 @@ const REQUIRES_OS_PREFIX: &str = "requires-os:"; const REQUIRES_GPU_TAG: &str = "requires-gpu"; const REQUIRES_NO_GPU_TAG: &str = "requires-no-gpu"; const REQUIRES_BARE_METAL_TAG: &str = "requires-bare-metal"; +const REQUIRES_WSL_TAG: &str = "requires-wsl"; const SERVE_TIMEOUT_PREFIX: &str = "serve-timeout:"; const NIGHTLY_TAG: &str = "nightly"; const LIFECYCLE_TAG: &str = "lifecycle"; @@ -77,6 +78,10 @@ pub struct ScenarioDecl { /// `@requires-os:linux` cannot express this — WSL2 reports an os_family of /// `linux`, so it matches. pub requires_bare_metal: bool, + /// `@requires-wsl`: the exact inverse — the scenario's premise IS a WSL2 + /// host, so it is skipped on native Linux, native Windows and everything + /// else. Same reason `@requires-os:linux` cannot stand in for it. + pub requires_wsl: bool, /// Engine the scenario pins via `@requires-engine:` (if any). pub requires_engine: Option, /// OS the scenario requires via `@requires-os:` (e.g. "linux"), if any — @@ -114,6 +119,7 @@ impl ScenarioDecl { let mut requires_gpu = false; let mut requires_no_gpu = false; let mut requires_bare_metal = false; + let mut requires_wsl = false; let mut requires_engine = None; let mut requires_os = None; let mut serve_timeout_secs = None; @@ -139,6 +145,8 @@ impl ScenarioDecl { requires_no_gpu = true; } else if tag == REQUIRES_BARE_METAL_TAG { requires_bare_metal = true; + } else if tag == REQUIRES_WSL_TAG { + requires_wsl = true; } else if tag == NIGHTLY_TAG { nightly = true; } else if tag == LIFECYCLE_TAG { @@ -152,6 +160,7 @@ impl ScenarioDecl { requires_gpu, requires_no_gpu, requires_bare_metal, + requires_wsl, requires_engine, requires_os, serve_timeout_secs, @@ -399,6 +408,11 @@ pub fn resolve( reason: "requires a bare-metal host; this one is WSL2".to_owned(), }; } + if decl.requires_wsl && !cap.is_wsl { + return Expectation::Skip { + reason: "requires WSL; this host is not running under WSL".to_owned(), + }; + } if let Some(os) = &decl.requires_os && !os.eq_ignore_ascii_case(&cap.os_family) { @@ -503,10 +517,10 @@ mod tests { effective_serve_engine: "lemonade".into(), platform_slug: "strix-halo".into(), }, - // A WSL2 dev box: `os_family` is `linux` and a GPU is visible through - // /dev/dxg, so nothing but `is_wsl` distinguishes it from bare metal. - // That is precisely why `@requires-os:linux` cannot stand in for - // `@requires-bare-metal`. + // A WSL2 dev box with the ROCm passthrough in place: `os_family` is + // `linux` and a GPU is usable, so nothing but `is_wsl` distinguishes + // it from bare metal. That is precisely why `@requires-os:linux` + // cannot stand in for `@requires-bare-metal`. "wsl2" => HostCapability { os_family: "linux".into(), is_wsl: true, @@ -514,7 +528,19 @@ mod tests { has_amd_gpu: true, available_engines: vec!["lemonade".into(), "vllm".into()], effective_serve_engine: "lemonade".into(), - platform_slug: "wsl2".into(), + platform_slug: "strix-halo-wsl".into(), + }, + // The same box without the passthrough: the gfx target is reported + // by the Windows-side driver but ROCm cannot reach it, so the probe + // resolves no usable GPU (see `capability::host_has_usable_gpu`). + "wsl" => HostCapability { + os_family: "linux".into(), + is_wsl: true, + gfx_target: None, + has_amd_gpu: false, + available_engines: vec!["lemonade".into(), "vllm".into()], + effective_serve_engine: "lemonade".into(), + platform_slug: "wsl".into(), }, _ => HostCapability { os_family: "other".into(), @@ -737,10 +763,12 @@ serve_timeout_secs = 90 fn requires_bare_metal_skips_on_wsl_and_runs_everywhere_else() { let m = Expectations::default(); let d = decl(&["id:diagnose-matches-known-symptom", "requires-bare-metal"]); - assert!(matches!( - resolve(&d, &cap("wsl2"), &m, false, false, false), - Expectation::Skip { .. } - )); + for wsl in ["wsl2", "wsl"] { + assert!(matches!( + resolve(&d, &cap(wsl), &m, false, false, false), + Expectation::Skip { .. } + )); + } // Bare-metal hosts of every shape still run it — including the mock lane, // which is where these scenarios earn their keep as a required check. for host in ["mock", "mi300x", "strix-ubuntu", "strix-windows"] { @@ -752,6 +780,27 @@ serve_timeout_secs = 90 } } + /// The exact inverse of the tag above: proving both directions keeps the + /// pair from silently collapsing into one predicate. + #[test] + fn requires_wsl_runs_only_on_wsl_hosts() { + let m = Expectations::default(); + let d = decl(&["id:examine-detects-wsl", "requires-wsl"]); + assert!(d.requires_wsl); + for wsl in ["wsl", "wsl2"] { + assert_eq!( + resolve(&d, &cap(wsl), &m, false, false, false), + Expectation::ExpectPass + ); + } + for platform in ["mock", "mi300x", "strix-ubuntu", "strix-windows"] { + assert!(matches!( + resolve(&d, &cap(platform), &m, false, false, false), + Expectation::Skip { .. } + )); + } + } + #[test] fn requires_os_linux_does_not_stand_in_for_requires_bare_metal() { // The reason the tag has to exist: WSL2 reports os_family "linux", so an diff --git a/tests/e2e-cucumber/tests/e2e/examine_steps.rs b/tests/e2e-cucumber/tests/e2e/examine_steps.rs index dc872cd8bb..38299139ca 100644 --- a/tests/e2e-cucumber/tests/e2e/examine_steps.rs +++ b/tests/e2e-cucumber/tests/e2e/examine_steps.rs @@ -6,6 +6,13 @@ use cucumber::{given, then, when}; use crate::E2eWorld; +fn field_value<'a>(output: &'a str, field: &str) -> Option<&'a str> { + output.lines().find_map(|line| { + let (name, value) = line.trim().split_once(':')?; + (name == field).then(|| value.trim()) + }) +} + #[given("a machine with an AMD GPU")] async fn setup_gpu_machine(world: &mut E2eWorld) { let (stdout, _, _) = crate::run_rocm(world, &["examine"]); @@ -20,6 +27,15 @@ async fn setup_unmanaged_rocm(world: &mut E2eWorld) { world.plant_unmanaged_rocm(); } +#[given("the CLI is running in WSL")] +async fn setup_wsl_host(world: &mut E2eWorld) { + let (stdout, _, _) = crate::run_rocm(world, &["examine"]); + assert!( + field_value(&stdout, "wsl").is_some_and(|value| value.eq_ignore_ascii_case("true")), + "CLI did not detect WSL:\n{stdout}" + ); +} + #[when("the user asks for the version")] async fn user_asks_version(world: &mut E2eWorld) { let (stdout, _, _) = crate::run_rocm(world, &["version"]); @@ -135,6 +151,26 @@ async fn assert_all_engines_listed(world: &mut E2eWorld) { } } +#[then("the inspection reports Linux as the operating system")] +async fn assert_linux_host(world: &mut E2eWorld) { + let output = world.cli_output.as_ref().expect("no command was run"); + assert_eq!( + field_value(output, "os"), + Some("linux"), + "expected Linux in examine output:\n{output}" + ); +} + +#[then("the inspection reports that the host is WSL")] +async fn assert_wsl_host(world: &mut E2eWorld) { + let output = world.cli_output.as_ref().expect("no command was run"); + assert_eq!( + field_value(output, "wsl"), + Some("true"), + "expected WSL in examine output:\n{output}" + ); +} + #[then("the inspection reports which GPU is installed")] async fn assert_gpu_detected(world: &mut E2eWorld) { let output = world.cli_output.as_ref().expect("no command was run"); From 97ff23e06cf8aa73f8572ec3ce148e4ec2b40551 Mon Sep 17 00:00:00 2001 From: Roman Inflianskas Date: Wed, 29 Jul 2026 13:41:52 +0000 Subject: [PATCH 02/11] ci(e2e): run the WSL2 E2E lane on the self-hosted WSL runner Target the Ubuntu-under-WSL2 runner on the Strix Halo box (the `wsl` label; the native Strix Linux runner carries `native`) instead of registering a throwaway distro on a hosted windows-latest VM. The lane now mirrors the other self-hosted Linux lane step for step: stray-serve reclaim, bounded GPU preflight, toolchain bootstrap, shared runtime pre-warm, and the full suite with no hand filtering. GPU preflight is advisory here because GPU access under WSL is what this lane is proving out: when it is unavailable the capability probe resolves those scenarios to skip and the rest of the suite still runs. Dispatch selector is `strix-wsl`, matching the sibling hardware lanes. Signed-off-by: Roman Inflianskas --- .github/workflows/ci.yml | 66 +--------- .github/workflows/e2e-selfhosted.yml | 186 ++++++++++++++++++++++++++- docs/ci-hardware-testing.md | 48 ++++--- tests/e2e-cucumber/README.md | 14 +- 4 files changed, 218 insertions(+), 96 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3a534fc1bf..609be8f0e9 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -26,7 +26,7 @@ on: description: Which runner(s) to target type: choice default: all - options: [all, mock, wsl] + options: [all, mock] # No tier input: the harness resolves pass/xfail/skip per scenario, so a # platform runs one job covering everything applicable to it. @@ -715,69 +715,6 @@ jobs: name: e2e-report path: tests/e2e-cucumber/results/ - # Hosted WSL2 coverage: runs the black-box suite in a real Ubuntu distro on a - # windows-latest VM. GitHub-hosted runners have no AMD GPU, so GPU scenarios - # resolve to skip; this job covers the WSL host boundary, build, and CLI paths. - # Advisory while the hosted WSL image path is proven out. - e2e-wsl: - name: E2E tests (WSL2, no GPU) - timeout-minutes: 30 - runs-on: windows-latest - needs: [changes, build-and-test] - if: >- - always() - && needs.changes.result == 'success' - && ( - (github.event_name != 'workflow_dispatch' - && needs.build-and-test.result == 'success' - && needs.changes.outputs.heavy == 'true') - || (github.event_name == 'workflow_dispatch' - && (inputs.platform == 'all' || inputs.platform == 'wsl')) - ) - continue-on-error: true - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - - - name: Set up Ubuntu on WSL2 - uses: Vampire/setup-wsl@d1da7f2c0322a5ee4f24975344f67fc0f5baf364 # v7.0.0 - with: - distribution: Ubuntu-24.04 - wsl-version: 2 - use-cache: false - additional-packages: build-essential ca-certificates curl libcap-dev pkg-config - - - name: Verify WSL2 host - shell: wsl-bash {0} - run: | - set -euo pipefail - grep -qi microsoft /proc/sys/kernel/osrelease - test -n "${WSL_INTEROP:-}" - printf 'WSL kernel: %s\n' "$(uname -r)" - - - name: Ensure Rust toolchain in WSL2 - shell: wsl-bash {0} - run: | - set -euo pipefail - if ! command -v cargo >/dev/null 2>&1; then - curl --proto '=https' --tlsv1.2 -fsSL https://sh.rustup.rs \ - | sh -s -- -y --no-modify-path --default-toolchain none - fi - "$HOME/.cargo/bin/rustup" show active-toolchain - - - name: Run E2E tests in WSL2 - shell: wsl-bash {0} - run: | - set -euo pipefail - export PATH="$HOME/.cargo/bin:$PATH" - cargo xtask e2e - - - name: Upload WSL2 E2E report - if: always() - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 - with: - name: e2e-wsl-report - path: tests/e2e-cucumber/results/ - # Consolidate the mock (GitHub-hosted) platform's report into the cross-platform # report: a platform × tier matrix in the run Summary plus a single merged HTML # artifact. `if: always()` so a failing platform still appears; runs on @@ -796,7 +733,6 @@ jobs: needs: - changes - e2e - - e2e-wsl # Consolidate whatever ran. On dispatch `heavy` is unset, so also run when the # trigger was manual; `always()` still lets it collect partial/failed tiers. if: >- diff --git a/.github/workflows/e2e-selfhosted.yml b/.github/workflows/e2e-selfhosted.yml index c32f1f9d9e..b6d7d5f4c5 100644 --- a/.github/workflows/e2e-selfhosted.yml +++ b/.github/workflows/e2e-selfhosted.yml @@ -32,7 +32,7 @@ on: description: Which self-hosted runner(s) to target type: choice default: all - options: [all, app-dev-gpu, strix-ubuntu, strix-windows] + options: [all, app-dev-gpu, strix-ubuntu, strix-windows, strix-wsl] name_filter: description: "Scenario-name regex (cucumber --name); empty = full suite" type: string @@ -636,6 +636,189 @@ jobs: name: e2e-gpu-strix-windows-report path: tests/e2e-cucumber/results/ + # WSL2 coverage on real hardware: the runner is an Ubuntu distro running under + # WSL2 on the Strix Halo Windows box, so this lane exercises the WSL host + # boundary AND whatever GPU access WSL exposes. Same suite as every other + # platform — scenarios the host cannot satisfy resolve to skip from the + # capability probe, so nothing is filtered out here. `wsl` in `runs-on` + # disambiguates it from the `native` Strix Linux runner. Non-blocking while + # GPU-on-WSL is proven out. + e2e-wsl: + name: E2E tests (Strix Halo, WSL2) + # 35min: matches the sibling collapsed GPU jobs — one job runs all serves + # plus per-scenario `install sdk`, and the cap must exceed the run so the job + # still writes platform.json. + timeout-minutes: 35 + runs-on: [self-hosted, linux, strix-halo, wsl] + needs: [changes] + # See `e2e-gpu`: no build-and-test gate (cross-workflow); strix-wsl. Gated on + # `serve` like the sibling lanes rather than the broader `heavy`: the + # consolidated report gates on `serve` too, so a heavy-but-not-serve change + # would otherwise run this lane and then discard its artifact unreported. + if: >- + always() + && needs.changes.result == 'success' + && ( + (github.event_name != 'workflow_dispatch' + && needs.changes.outputs.serve == 'true') + || (github.event_name == 'workflow_dispatch' + && (inputs.platform == 'all' || inputs.platform == 'strix-wsl')) + ) + continue-on-error: true + env: + E2E_SERVE_TIMEOUT_SECS: "300" + # Match the other hardware lanes: opt into the platform-adaptive + # large-model scenario only when the manual include_nightly input is on. + E2E_INCLUDE_NIGHTLY: "${{ inputs.include_nightly && '1' || '' }}" + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + + # Fail fast and loudly if this job ever lands on a native Linux runner: + # every WSL-tagged scenario would silently resolve to skip instead. + - name: Verify the host really is WSL2 + run: | + if ! grep -qi microsoft /proc/sys/kernel/osrelease; then + echo "::error::runner is not a WSL host: $(uname -r)" + exit 1 + fi + printf 'WSL kernel: %s\n' "$(uname -r)" + + # The native Strix runners come pre-provisioned; a WSL distro often does + # not. Install the native build deps only when something is missing, and + # only if passwordless sudo is available — otherwise say what is missing + # instead of hanging on a password prompt. + - name: Ensure native build deps + run: | + missing="" + command -v pkg-config >/dev/null 2>&1 || missing="$missing pkg-config" + command -v cc >/dev/null 2>&1 || missing="$missing build-essential" + pkg-config --exists libcap 2>/dev/null || missing="$missing libcap-dev" + if [ -z "$missing" ]; then + echo "native build deps present" + elif sudo -n true 2>/dev/null; then + echo "installing:$missing" + sudo -n apt-get update + # shellcheck disable=SC2086 + sudo -n apt-get install -y $missing + else + echo "::error::missing native build deps:$missing (no passwordless sudo to install them)" + exit 1 + fi + + - name: Reclaim GPU from stray E2E processes + run: | + # Reclaim from any serve leaked by a killed/timed-out prior run + # (see e2e-gpu). Scoped to e2e leftovers only. + pkill -f '/tmp/rocm-e2e.*llama-server' 2>/dev/null || true + pkill -f '/tmp/rocm-e2e.*vllm serve' 2>/dev/null || true + pkill -f 'e2e-shared.*llama-server' 2>/dev/null || true + pkill -f '__engine-serve-http.*rocm-e2e' 2>/dev/null || true + rm -rf /tmp/rocm-e2e-* 2>/dev/null || true + echo "reclaimed" + + # GPU preflight, bounded like the native lanes — but ADVISORY here. GPU + # access under WSL is exactly what this lane is proving out, so an absent + # rocm-smi is a reported condition, not a job failure: the capability probe + # then resolves @requires-gpu scenarios to skip and the rest still runs. A + # GPU that IS present but stays held by a leftover serve still fails, since + # that would corrupt the serve scenarios' results. + - name: GPU preflight (advisory bounded wait) + run: | + MIN_FREE_GIB="${GPU_PREFLIGHT_MIN_FREE_GIB:-8}" + CEILING_SECS="${GPU_PREFLIGHT_CEILING_SECS:-90}" + if ! command -v rocm-smi >/dev/null 2>&1; then + echo "::warning::rocm-smi not found in this WSL distro — GPU scenarios will resolve to skip" + exit 0 + fi + min_free=$(( MIN_FREE_GIB * 1024 * 1024 * 1024 )) + deadline=$(( SECONDS + CEILING_SECS )) + saw_vram=0 + while [ "$SECONDS" -lt "$deadline" ]; do + out=$(timeout 15 rocm-smi --showmeminfo vram 2>/dev/null) || { sleep 5; continue; } + total=$(printf '%s\n' "$out" | grep -i 'VRAM Total Memory' | sed 's/.*: *//' | grep -oE '[0-9]+' | tail -1) + used=$(printf '%s\n' "$out" | grep -i 'VRAM Total Used Memory' | sed 's/.*: *//' | grep -oE '[0-9]+' | tail -1) + if [ -z "$total" ] || [ -z "$used" ]; then + sleep 5; continue + fi + saw_vram=1 + free=$(( total - used )) + if [ "$free" -ge "$min_free" ]; then + echo "GPU ready: $(( free / 1024 / 1024 / 1024 )) GiB free (>= ${MIN_FREE_GIB} GiB)." + exit 0 + fi + echo "waiting: $(( free / 1024 / 1024 / 1024 )) GiB free (< ${MIN_FREE_GIB} GiB)…" + sleep 5 + done + if [ "$saw_vram" -eq 0 ]; then + echo "::warning::rocm-smi reported no VRAM figures under WSL — GPU scenarios will resolve to skip" + exit 0 + fi + echo "::error::GPU preflight failed after ${CEILING_SECS}s: VRAM never dropped below the floor — a serve is likely still holding the GPU" + exit 1 + + # Bootstrap rustup with --no-modify-path so it never writes $HOME/.profile + # (setup-rust-toolchain doesn't expose that flag). rust-toolchain.toml pins + # the exact toolchain, installed on first cargo use. Idempotent. + - name: Ensure Rust toolchain + run: | + if ! command -v cargo >/dev/null 2>&1 && [ ! -x "$HOME/.cargo/bin/cargo" ]; then + curl --proto '=https' --tlsv1.2 -fsSL https://sh.rustup.rs \ + | sh -s -- -y --no-modify-path --default-toolchain none + fi + echo "$HOME/.cargo/bin" >> "$GITHUB_PATH" + + - name: Run E2E tests on Strix Halo WSL2 + run: | + export CARGO_TARGET_DIR="$RUNNER_WORKSPACE/e2e-target" + export E2E_SHARED_CACHE_DIR="$RUNNER_WORKSPACE/e2e-shared" + # Share ONE installed managed runtime across serve/chat scenarios and + # PRE-WARM it in place (mirrors e2e-gpu-strix-ubuntu). Required for + # correctness, not speed: `install sdk` bakes ABSOLUTE paths into the + # runtime manifest, so installing into a per-scenario temp dir leaves + # every later serve pointing at a deleted install root. + prewarm="$RUNNER_WORKSPACE/e2e-prewarm" + export E2E_SHARED_RUNTIMES_DIR="$prewarm/data/runtimes" + + # Build the rocm binary once; reuse for pre-warm + suite so xtask + # doesn't rebuild. + cargo build --release -p rocm + export ROCM_CLI_BINARY="$CARGO_TARGET_DIR/release/rocm" + + # Pre-warm once, serially, in place (no mv/symlink). Skipped once the + # tree is populated (persists across runs on RUNNER_WORKSPACE). + if [ ! -d "$E2E_SHARED_RUNTIMES_DIR/registry" ]; then + echo "pre-warming shared runtime (first run on this runner)…" + mkdir -p "$prewarm"/{data,config,cache} + ROCM_CLI_CONFIG_DIR="$prewarm/config" \ + ROCM_CLI_DATA_DIR="$prewarm/data" \ + ROCM_CLI_CACHE_DIR="$prewarm/cache" \ + HF_HOME="$E2E_SHARED_CACHE_DIR/huggingface" \ + "$ROCM_CLI_BINARY" install sdk + if [ -d "$E2E_SHARED_RUNTIMES_DIR/registry" ]; then + echo "shared runtime pre-warmed at $E2E_SHARED_RUNTIMES_DIR" + else + echo "pre-warm did not produce a runtimes registry; scenarios will install their own" >&2 + fi + else + echo "shared runtime already present at $E2E_SHARED_RUNTIMES_DIR — skipping pre-warm" + fi + + # Optional scenario-name filter for a scoped manual dispatch. + NAME_FILTER="${{ github.event.inputs.name_filter }}" + if [ -n "$NAME_FILTER" ]; then + echo "name filter active: $NAME_FILTER" + cargo xtask e2e -- --name "$NAME_FILTER" + else + cargo xtask e2e + fi + + - name: Upload E2E report + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: e2e-gpu-strix-wsl-report + path: tests/e2e-cucumber/results/ + # Consolidate this workflow's self-hosted platform reports into one GPU-side # cross-platform grid (Summary + merged HTML). Distinct name from ci.yml's # required `E2E consolidated report` so it does NOT collide with that required @@ -651,6 +834,7 @@ jobs: - e2e-gpu - e2e-gpu-strix-ubuntu - e2e-gpu-strix-windows + - e2e-wsl # Gate on `serve`: every lane this report consolidates (the GPU jobs) is now # serve-gated, so a serve-only change runs them and their report must still be # produced. On dispatch `serve` is unset, so also run when the trigger was diff --git a/docs/ci-hardware-testing.md b/docs/ci-hardware-testing.md index 01f87fd853..2ddf6977db 100644 --- a/docs/ci-hardware-testing.md +++ b/docs/ci-hardware-testing.md @@ -32,34 +32,37 @@ separate tier flag or tag filter to maintain. | Job | Workflow | Platform | Runner labels | |---|---|---|---| | `e2e` | `ci.yml` | Mock (no GPU) | GitHub-hosted `ubuntu-latest` | -| `e2e-wsl` | `ci.yml` | Ubuntu on WSL2 (no GPU) | GitHub-hosted `windows-latest` | | `e2e-gpu` | `e2e-selfhosted.yml` | MI300X (AMD Instinct, bare-metal Linux) | self-hosted `[self-hosted, linux, amd-gpu]` | | `e2e-gpu-strix-ubuntu` | `e2e-selfhosted.yml` | Strix Halo (gfx1151) on Ubuntu | self-hosted `[self-hosted, linux, strix-halo, native]` | | `e2e-gpu-strix-windows` | `e2e-selfhosted.yml` | Strix Halo (gfx1151) on native Windows 11 | self-hosted `[self-hosted, windows, strix-halo, native]` | +| `e2e-wsl` | `e2e-selfhosted.yml` | Strix Halo (gfx1151) on Ubuntu under WSL2 | self-hosted `[self-hosted, linux, strix-halo, wsl]` | The Strix Halo lanes pin the extra `native` label because two Linux runners share the `strix-halo` label (a native host and a WSL host) and the jobs' -hardcoded `/home/ubuntu/actions-runner` paths exist only on the native one. +hardcoded `/home/ubuntu/actions-runner` paths exist only on the native one. The +WSL lane pins `wsl` for the same reason, from the other side. `e2e` is the blocking, GitHub-hosted mock job: `@requires-gpu` scenarios resolve to skip here, and known bugs resolve to xfail from `expectations.toml`. It is a required check and must stay green. -`e2e-wsl` is the hosted WSL2 lane, also in `ci.yml`: it registers an Ubuntu -distro under WSL2 on a `windows-latest` VM, builds the CLI inside it, and runs -the same suite. It covers real WSL detection and the Windows-to-WSL execution -boundary — paths no Linux or Windows lane exercises. It is **not** GPU-on-WSL -validation: a hosted runner has no AMD GPU, so `/dev/dxg`, the Windows AMD -driver, and live serving stay uncovered, and GPU scenarios resolve to skip. -Those need a self-hosted Windows machine with both an AMD GPU and a configured -WSL distro; a future GPU-on-WSL lane should stay a separate, non-blocking -platform until its runner, distro, and GPU access are validated end to end. - The three GPU jobs (`e2e-gpu`, `e2e-gpu-strix-ubuntu`, `e2e-gpu-strix-windows`) run on dedicated self-hosted runners with a real AMD GPU attached, so they exercise host/GPU detection, engine `detect`/`capabilities`, and live serving scenarios that the mock job cannot. +`e2e-wsl` runs on an Ubuntu distro hosted in WSL2 on the Strix Halo Windows box +and mirrors the sibling Linux lane step for step: stray-serve reclaim, GPU +preflight, toolchain bootstrap, shared-runtime pre-warm, then the full suite +with no hand filtering. It covers WSL host detection, the Windows-to-WSL +execution boundary, and whatever GPU access WSL exposes on that machine. The +GPU preflight is advisory here precisely because GPU-on-WSL is what the lane is +proving out: where it is unavailable the capability probe resolves those +scenarios to not-applicable and the rest of the suite still runs. Scenarios the +product deliberately routes around on WSL carry `@requires-bare-metal`; the one +scenario whose premise *is* a WSL host carries `@requires-wsl`, and this is the +only lane that runs it. + Each workflow has its own consolidated report job. `ci.yml`'s `e2e-report` covers the mock platform; `e2e-selfhosted.yml`'s `e2e-report` covers the three GPU platforms; `nightly.yml`'s `e2e-report-nightly` covers the same three @@ -95,12 +98,11 @@ authoritative pre-merge build gate. They can also be triggered manually via `e2e-selfhosted.yml`'s `workflow_dispatch`, independent of the `serve` gate, with these inputs: -- `platform` (choice: `all`, `app-dev-gpu`, `strix-ubuntu`, `strix-windows`) — - which self-hosted job(s) to run. `app-dev-gpu` maps to `e2e-gpu`, - `strix-ubuntu` to `e2e-gpu-strix-ubuntu`, and `strix-windows` to - `e2e-gpu-strix-windows`. (The GitHub-hosted lanes have their own `platform` - input on `ci.yml`, with `mock` and `wsl` options; they are not part of this - workflow.) +- `platform` (choice: `all`, `app-dev-gpu`, `strix-ubuntu`, `strix-windows`, + `strix-wsl`) — which self-hosted job(s) to run. `app-dev-gpu` maps to + `e2e-gpu`, `strix-ubuntu` to `e2e-gpu-strix-ubuntu`, `strix-windows` to + `e2e-gpu-strix-windows`, and `strix-wsl` to `e2e-wsl`. (The mock lane has its + own `platform` input on `ci.yml`; it is not part of this workflow.) - `name_filter` (string) — a scenario-name regex forwarded to the cucumber harness (`cargo xtask e2e -- --name `) so a dispatch can run a single scenario instead of the full suite. Empty runs everything applicable @@ -117,12 +119,10 @@ gh workflow run e2e-selfhosted.yml --ref -f platform=app-dev-gpu ## Blocking vs. non-blocking -The three hardware jobs — `e2e-gpu`, `e2e-gpu-strix-ubuntu`, and -`e2e-gpu-strix-windows` — all run with `continue-on-error: true`, so a +The four self-hosted jobs — `e2e-gpu`, `e2e-gpu-strix-ubuntu`, +`e2e-gpu-strix-windows`, and `e2e-wsl` — all run with `continue-on-error: true`, so a hardware failure that RUNS never gates a PR merge. Their results still surface -in the self-hosted consolidated report for visibility. The hosted `e2e-wsl` -lane is non-blocking on the same terms while the hosted WSL image path is -proven out; only `ci.yml`'s mock `e2e` job is required. +in the self-hosted consolidated report for visibility. **Required-check caveat.** These three job names (plus, historically, a consolidated-report name) are still in `main`'s required-status-check list. @@ -144,8 +144,6 @@ which requires write access to trigger). ## Notes -- `e2e-wsl` reports under its own platform slug even without a GPU, so its - results never collide with the ordinary mock report. - The hardware jobs build and run **release** binaries: they assert functional behavior (device detection, engine launch, policy enforcement), not performance, so this is not a performance benchmark. Release-fidelity, diff --git a/tests/e2e-cucumber/README.md b/tests/e2e-cucumber/README.md index 5f6f44e1df..8a335cc0b6 100644 --- a/tests/e2e-cucumber/README.md +++ b/tests/e2e-cucumber/README.md @@ -98,8 +98,9 @@ Scenarios carry stable-id and capability tags: | Tag | Meaning | |---|---| | `@id:` | Stable scenario id. Keys the expectation matrix and the report grid; every scenario has one. | -| `@requires-gpu` | Needs a real AMD GPU. Resolves to **skip** (n/a) on a host with none (e.g. the mock or hosted WSL job). | -| `@requires-wsl` | Needs a real WSL host. Resolves to **skip** on native Linux, native Windows, and other environments. | +| `@requires-gpu` | Needs a usable AMD GPU. Resolves to **skip** (n/a) on a host with none — the mock job, or a WSL host whose ROCm passthrough is incomplete (`driver_status` other than `wsl_rocdxg_ready`), where the gfx target is reported but unreachable. | +| `@requires-bare-metal` | Premise is a host running the in-tree amdgpu driver, so it does not hold under WSL2. Resolves to **skip** there. `@requires-os:linux` cannot express this: WSL2 reports an `os_family` of `linux`. | +| `@requires-wsl` | The inverse: premise **is** a WSL2 host. Resolves to **skip** on native Linux, native Windows, and everything else. | | `@requires-engine:` | Pins the serve engine. Resolves to skip where that engine can't start (e.g. vLLM on a lemonade-only Strix host). | | `@requires-os:` | Premise is OS-specific; skip on other OSes. | | `@serve-timeout:` | Lengthen the serve-readiness wait for a genuinely slow serve (e.g. a large model). | @@ -130,17 +131,20 @@ self-hosted runner can never stall `ci.yml`'s merge-required checks: | Job | Workflow | Platform | Blocking | |---|---|---|---| -| `e2e` | `ci.yml` | Mock (no GPU, GitHub-hosted Linux) | yes | -| `e2e-wsl` | `ci.yml` | WSL2 / Ubuntu (no GPU, GitHub-hosted Windows) | no | +| `e2e` | `ci.yml` | Mock (no GPU, GitHub-hosted) | yes | | `e2e-gpu` | `e2e-selfhosted.yml` | MI300X (self-hosted) | no | | `e2e-gpu-strix-ubuntu` | `e2e-selfhosted.yml` | Strix Halo / Ubuntu (self-hosted) | no | | `e2e-gpu-strix-windows` | `e2e-selfhosted.yml` | Strix Halo / Windows (self-hosted) | no | +| `e2e-wsl` | `e2e-selfhosted.yml` | Strix Halo / Ubuntu under WSL2 (self-hosted) | no | The blocking mock job passes when every applicable scenario is pass-or-xfail with no XPASS or unexpected failure; the GPU jobs are non-blocking. Each workflow consolidates its own platforms: `ci.yml`'s `e2e-report` the mock platform, `e2e-selfhosted.yml`'s `e2e-report` the self-hosted platforms, and -`nightly.yml`'s `e2e-report-nightly` the nightly lanes below. +`nightly.yml`'s `e2e-report-nightly` the nightly lanes below. `e2e-wsl` runs the suite in an +Ubuntu distro under WSL2 on the Strix Halo box, so it is the only lane that +exercises `@requires-wsl` scenarios; `@requires-bare-metal` scenarios resolve to +skip there. The nightly workflow runs three non-blocking jobs — the existing MI300X job and new Strix Halo jobs on Ubuntu and Windows — with `E2E_INCLUDE_NIGHTLY=1`, then From 67f814c4639e7261b5f6f6a4ea7d876db8ecdfdb Mon Sep 17 00:00:00 2001 From: Roman Inflianskas Date: Wed, 12 Aug 2026 10:54:06 +0000 Subject: [PATCH 03/11] test(e2e): key GPU capability on the driver, not the gfx target `has_amd_gpu` was `gfx_target.is_some()`. On WSL the target is read from the Windows-side driver and is reported whether or not ROCm can reach the device, so a distro with /dev/dxg but no librocdxg.so advertises gfx1151 while `rocm serve` refuses with "no usable AMD GPU". Every `@requires-gpu` scenario therefore ran on a host that cannot serve one and failed on its premise instead of resolving to not-applicable -- which is the whole job of the capability probe. On the self-hosted WSL runner that surfaced as `serve-absent-gpu-index-rejected`: it asserts the index-specific rejection, which the GPU-required pre-flight refuses long before the index is validated. Its own comment already said so, and it already carried `@requires-gpu`; the gate just never fired. Take the CLI's own verdict instead: on WSL require `driver_status: wsl_rocdxg_ready`, which it sets only once /dev/dxg, dxcore, librocdxg.so and its ldconfig entry are all in place. Native hosts are unaffected. Signed-off-by: Roman Inflianskas --- docs/ci-hardware-testing.md | 18 +++++ tests/e2e-cucumber/src/capability.rs | 113 +++++++++++++++++++++++---- 2 files changed, 114 insertions(+), 17 deletions(-) diff --git a/docs/ci-hardware-testing.md b/docs/ci-hardware-testing.md index 2ddf6977db..2a3c39f2bc 100644 --- a/docs/ci-hardware-testing.md +++ b/docs/ci-hardware-testing.md @@ -63,6 +63,24 @@ product deliberately routes around on WSL carry `@requires-bare-metal`; the one scenario whose premise *is* a WSL host carries `@requires-wsl`, and this is the only lane that runs it. +### What the WSL distro needs + +The distro needs `pkg-config`, `build-essential` and `libcap-dev` to build the +workspace; the lane installs them itself where it has passwordless sudo, and +otherwise fails with the list of what is missing rather than hanging on a +password prompt. + +GPU coverage additionally needs ROCm's WSL passthrough to be complete — +`/dev/dxg` and dxcore alone are not enough, `librocdxg.so` and its ldconfig +entry must be present too. `rocm examine` reports the verdict as +`driver_status: wsl_rocdxg_ready`; anything else (`wsl_rocdxg_missing`, +`wsl_gpu_plumbing_missing`) means the runtime cannot reach the device even +though `detected_gfx_target` still names it, because that target is read from +the Windows-side driver. The capability probe keys `@requires-gpu` on the +driver verdict rather than the target for exactly this reason, so a distro +without the passthrough runs the non-GPU suite and reports the GPU scenarios as +not applicable, instead of failing them on a premise the host cannot meet. + Each workflow has its own consolidated report job. `ci.yml`'s `e2e-report` covers the mock platform; `e2e-selfhosted.yml`'s `e2e-report` covers the three GPU platforms; `nightly.yml`'s `e2e-report-nightly` covers the same three diff --git a/tests/e2e-cucumber/src/capability.rs b/tests/e2e-cucumber/src/capability.rs index 744210b9c9..10a662a773 100644 --- a/tests/e2e-cucumber/src/capability.rs +++ b/tests/e2e-cucumber/src/capability.rs @@ -102,7 +102,9 @@ pub struct HostCapability { /// First AMD GPU's gfx target from `examine`'s `detected_gfx_target:` line /// (e.g. "gfx942", "gfx1151"), if a real one was reported. pub gfx_target: Option, - /// Whether an AMD GPU was detected (a real `detected_gfx_target` is present). + /// Whether an AMD GPU is present AND usable here, so `@requires-gpu` + /// scenarios can run. A real `detected_gfx_target`, plus a ready ROCm + /// driver path on WSL — see [`host_has_usable_gpu`]. pub has_amd_gpu: bool, /// Engine adapters the binary reports as present. Both builtins are always /// "built-in", so this is NOT the same as "can start here" — use @@ -331,8 +333,13 @@ fn probe_host_capability() -> HostCapability { let examine = run_probe(root, &["examine"]); let engines = run_probe(root, &["engines", "list"]); - let (os_family, is_wsl, gfx_target) = parse_examine_text(&examine); - let has_amd_gpu = gfx_target.is_some(); + let ExamineFacts { + os_family, + is_wsl, + gfx_target, + driver_status, + } = parse_examine_text(&examine); + let has_amd_gpu = host_has_usable_gpu(gfx_target.as_deref(), is_wsl, &driver_status); let available_engines = parse_engines_list(&engines); let effective_serve_engine = effective_serve_engine(gfx_target.as_deref(), &os_family); let platform_slug = @@ -363,15 +370,34 @@ fn run_probe(root: &std::path::Path, args: &[&str]) -> String { } } -/// Extract `(os_family, is_wsl, first_gfx_target)` from the human `rocm examine` -/// text (format string in rocm-core `ExamineSummary`): lines like ` os: linux`, -/// ` detected_gfx_target: gfx942`, ` wsl: false`. A missing/placeholder -/// (``, empty, `none`) gfx target yields `None` (→ treated as no GPU). -/// Tolerant: an unrecognized dump degrades to a mock-like host. -fn parse_examine_text(text: &str) -> (String, bool, Option) { +/// The `examine` fields the capability probe reads. +struct ExamineFacts { + os_family: String, + is_wsl: bool, + gfx_target: Option, + /// `driver_status:` — the CLI's own verdict on whether the runtime can + /// reach the device. Only consulted on WSL (see [`host_has_usable_gpu`]). + driver_status: String, +} + +/// `driver_status` value meaning ROCm's WSL passthrough path is complete. +/// +/// The CLI sets this only when `/dev/dxg`, dxcore, `librocdxg.so` and its +/// ldconfig entry are all present; the other WSL states (`wsl_rocdxg_missing`, +/// `wsl_gpu_plumbing_missing`) mean the runtime cannot reach the GPU. +const WSL_DRIVER_READY: &str = "wsl_rocdxg_ready"; + +/// Extract the probe's facts from the human `rocm examine` text (format string +/// in rocm-core `ExamineSummary`): lines like ` os: linux`, +/// ` detected_gfx_target: gfx942`, ` wsl: false`, ` driver_status: ...`. A +/// missing/placeholder (``, empty, `none`) gfx target yields `None` +/// (→ treated as no GPU). Tolerant: an unrecognized dump degrades to a +/// mock-like host. +fn parse_examine_text(text: &str) -> ExamineFacts { let mut os_family = "other".to_owned(); let mut is_wsl = false; let mut gfx_target = None; + let mut driver_status = String::new(); for line in text.lines() { let line = line.trim(); if let Some(v) = line.strip_prefix("os:") { @@ -383,9 +409,29 @@ fn parse_examine_text(text: &str) -> (String, bool, Option) { } } else if let Some(v) = line.strip_prefix("wsl:") { is_wsl = matches!(v.trim(), "true" | "yes" | "1"); + } else if let Some(v) = line.strip_prefix("driver_status:") { + driver_status = v.trim().to_owned(); } } - (os_family, is_wsl, gfx_target) + ExamineFacts { + os_family, + is_wsl, + gfx_target, + driver_status, + } +} + +/// Whether `@requires-gpu` scenarios can actually run on this host. +/// +/// A detected gfx target is enough on a native host. It is NOT enough under +/// WSL: the target is reported from the Windows-side driver even when ROCm has +/// no path to the device, so a distro missing `librocdxg.so` advertises +/// `gfx1151` while `rocm serve` refuses with "no usable AMD GPU". Taking the +/// target at face value there runs every GPU scenario against a host that +/// cannot serve one — they fail on their premise rather than resolving to +/// not-applicable, which is exactly what the capability probe exists to avoid. +fn host_has_usable_gpu(gfx_target: Option<&str>, is_wsl: bool, driver_status: &str) -> bool { + gfx_target.is_some() && (!is_wsl || driver_status == WSL_DRIVER_READY) } /// Parse engine names from `rocm engines list`. Engine rows are the lines whose @@ -556,10 +602,11 @@ rocm examine wsl: false driver_status: ok "; - let (os, wsl, gfx) = parse_examine_text(text); - assert_eq!(os, "linux"); - assert!(!wsl); - assert_eq!(gfx.as_deref(), Some("gfx942")); + let facts = parse_examine_text(text); + assert_eq!(facts.os_family, "linux"); + assert!(!facts.is_wsl); + assert_eq!(facts.gfx_target.as_deref(), Some("gfx942")); + assert_eq!(facts.driver_status, "ok"); } #[test] @@ -571,9 +618,41 @@ rocm examine detected_gfx_target: wsl: false "; - let (os, _, gfx) = parse_examine_text(text); - assert_eq!(os, "other"); - assert_eq!(gfx, None); + let facts = parse_examine_text(text); + assert_eq!(facts.os_family, "other"); + assert_eq!(facts.gfx_target, None); + } + + /// A WSL distro advertises the Windows-side gfx target whether or not ROCm + /// can reach it, so the driver verdict is what decides. Without this, every + /// `@requires-gpu` scenario runs on a host where `serve` cannot start and + /// fails on its premise instead of resolving to not-applicable. + #[test] + fn wsl_gpu_requires_a_ready_rocdxg_path() { + // Real values from the self-hosted WSL runner before librocdxg was + // installed: /dev/dxg present, the ROCm passthrough library missing. + assert!(!host_has_usable_gpu( + Some("gfx1151"), + true, + "wsl_rocdxg_missing" + )); + assert!(!host_has_usable_gpu( + Some("gfx1151"), + true, + "wsl_gpu_plumbing_missing" + )); + // Complete passthrough: the GPU is usable and the scenarios must run. + assert!(host_has_usable_gpu(Some("gfx1151"), true, WSL_DRIVER_READY)); + // A native host is unaffected by the driver verdict. + assert!(host_has_usable_gpu( + Some("gfx942"), + false, + "amdgpu_available" + )); + assert!(host_has_usable_gpu(Some("gfx942"), false, "")); + // No gfx target is still no GPU, WSL or not. + assert!(!host_has_usable_gpu(None, true, WSL_DRIVER_READY)); + assert!(!host_has_usable_gpu(None, false, "amdgpu_available")); } #[test] From e85be7f2a821c78b57672bf1e987feabf657e243 Mon Sep 17 00:00:00 2001 From: Roman Inflianskas Date: Wed, 12 Aug 2026 14:59:30 +0000 Subject: [PATCH 04/11] test(e2e): record the WSL json/human GPU disagreement as a known bug The WSL lane reproduces EAI-7998 first-hand: `examine --json` reports `has_amd_gpu: false` with an empty `gpus` list on a host whose own summary, in the same document, carries `detected_gfx_target: gfx1151`. The GPU probes are routed out on WSL2 because they read KFD and DRM, neither of which exists there, while the summary reads the target from the Windows-side driver. The ticket had this half of the defect only second-hand, from the comment in capability.rs describing it on MI300X. This lane is the first to hit it directly, which is the lane doing its job. Recorded as an xfail rather than a skip: the two forms are supposed to agree here, so it is a defect, not a premise the platform cannot meet. Signed-off-by: Roman Inflianskas --- tests/e2e-cucumber/expectations.toml | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/tests/e2e-cucumber/expectations.toml b/tests/e2e-cucumber/expectations.toml index da2de69982..1045ae433e 100644 --- a/tests/e2e-cucumber/expectations.toml +++ b/tests/e2e-cucumber/expectations.toml @@ -162,3 +162,18 @@ bug = "EAI-7423" reason = "Lemonade managed serve reaches ready then shuts down immediately, so the benchmark never reaches a live model." serve_timeout_secs = 90 flaky = true + +# --- EAI-7998: `examine --json` disagrees with the human report about the GPU. +# On WSL2 the JSON contradicts ITSELF: `has_amd_gpu: false` and `gpus: []` +# alongside a summary carrying `detected_gfx_target: gfx1151`. The driver/GPU +# probes are routed out on WSL because they read KFD and DRM, neither of which +# exists there, while the summary reads the target from the Windows-side driver. +# +# Scoped to WSL because that is where this lane reproduces it. The ticket also +# records the same disagreement on MI300X, second-hand from the comment in +# capability.rs; if that is confirmed the row should widen rather than gain a +# sibling. Remove it when the two forms agree. --- +[["examine-both-forms-agree-on-gpu"]] +when = { is_wsl = true } +bug = "EAI-7998" +reason = "`examine --json` reports has_amd_gpu:false on a WSL2 host whose own summary names a gfx target." From 28fbd47429a0a32995de0426808bdaeca4369c38 Mon Sep 17 00:00:00 2001 From: Roman Inflianskas Date: Wed, 12 Aug 2026 16:15:40 +0000 Subject: [PATCH 05/11] test(e2e): let the TUI wait budget follow the host, like serve already does `dash-managed-service-metrics` failed on two of four WSL lane runs while passing on every native lane, and passed on re-run. The assertion is not wrong: the TUI wait is a single hardcoded 30s, and this workflow now puts a third lane on the one physical Strix box, so a frame that renders well inside 30s on an idle runner can miss it while a sibling loads a model. `E2E_TUI_TIMEOUT_SECS` overrides it, mirroring `E2E_SERVE_TIMEOUT_SECS` next door -- same reasoning, that the budget is what varies by host, not the assertion. Set to 90s on the three lanes that share the machine. A genuine hang still fails, just later; 0 or an unparseable value falls back to the default rather than disabling the bound. Recorded here rather than as a `flaky = true` xfail row: contention on a shared runner is not a defect in the product, and an xfail entry would tell every later reader that it is. Signed-off-by: Roman Inflianskas --- .github/workflows/e2e-selfhosted.yml | 18 +++++++ docs/ci-hardware-testing.md | 10 ++++ tests/e2e-cucumber/tests/e2e/dash_steps.rs | 47 ++++++++++--------- .../e2e-cucumber/tests/e2e/lifecycle_steps.rs | 6 +-- tests/e2e-cucumber/tests/e2e/tui_driver.rs | 22 ++++++++- 5 files changed, 76 insertions(+), 27 deletions(-) diff --git a/.github/workflows/e2e-selfhosted.yml b/.github/workflows/e2e-selfhosted.yml index b6d7d5f4c5..4632d5d0ad 100644 --- a/.github/workflows/e2e-selfhosted.yml +++ b/.github/workflows/e2e-selfhosted.yml @@ -360,6 +360,12 @@ jobs: TMPDIR: /home/ubuntu/actions-runner/tmp PIP_CACHE_DIR: /home/ubuntu/actions-runner/pip-cache E2E_SERVE_TIMEOUT_SECS: "300" + # The three Strix lanes share one physical machine, and this workflow now + # runs a third of them, so a TUI frame that renders well inside the 30s + # default on an idle runner can miss it while a sibling lane loads a model. + # Raise the wait budget rather than let that read as a product failure; a + # genuine hang still fails, just later. + E2E_TUI_TIMEOUT_SECS: "90" # Match the MI300X dispatch path: opt into the platform-adaptive large-model # scenario only when the manual include_nightly input is enabled. E2E_INCLUDE_NIGHTLY: "${{ inputs.include_nightly && '1' || '' }}" @@ -504,6 +510,12 @@ jobs: ) continue-on-error: true env: + # The three Strix lanes share one physical machine, and this workflow now + # runs a third of them, so a TUI frame that renders well inside the 30s + # default on an idle runner can miss it while a sibling lane loads a model. + # Raise the wait budget rather than let that read as a product failure; a + # genuine hang still fails, just later. + E2E_TUI_TIMEOUT_SECS: "90" # Match the Linux Strix dispatch path: opt into the platform-adaptive # large-model scenario only when the manual input is enabled. E2E_INCLUDE_NIGHTLY: "${{ inputs.include_nightly && '1' || '' }}" @@ -667,6 +679,12 @@ jobs: continue-on-error: true env: E2E_SERVE_TIMEOUT_SECS: "300" + # The three Strix lanes share one physical machine, and this workflow now + # runs a third of them, so a TUI frame that renders well inside the 30s + # default on an idle runner can miss it while a sibling lane loads a model. + # Raise the wait budget rather than let that read as a product failure; a + # genuine hang still fails, just later. + E2E_TUI_TIMEOUT_SECS: "90" # Match the other hardware lanes: opt into the platform-adaptive # large-model scenario only when the manual include_nightly input is on. E2E_INCLUDE_NIGHTLY: "${{ inputs.include_nightly && '1' || '' }}" diff --git a/docs/ci-hardware-testing.md b/docs/ci-hardware-testing.md index 2a3c39f2bc..f9176048cb 100644 --- a/docs/ci-hardware-testing.md +++ b/docs/ci-hardware-testing.md @@ -142,6 +142,16 @@ The four self-hosted jobs — `e2e-gpu`, `e2e-gpu-strix-ubuntu`, hardware failure that RUNS never gates a PR merge. Their results still surface in the self-hosted consolidated report for visibility. +### Timeouts on the shared Strix box + +Three of those lanes — the two native Strix ones and `e2e-wsl` — run on the same +physical machine and can be in flight together, so a wait that is comfortable on +an idle runner can expire while a sibling lane loads a model. Two budgets are +raised there rather than letting contention read as a product failure: +`E2E_SERVE_TIMEOUT_SECS` for serve readiness, and `E2E_TUI_TIMEOUT_SECS` for the +PTY-driven dashboard waits. Both only lengthen how long a wait may take; a +genuine hang still fails, just later. + **Required-check caveat.** These three job names (plus, historically, a consolidated-report name) are still in `main`'s required-status-check list. `continue-on-error` neutralizes a job that ran and failed, but a required check diff --git a/tests/e2e-cucumber/tests/e2e/dash_steps.rs b/tests/e2e-cucumber/tests/e2e/dash_steps.rs index e12fe3962a..5e50522e0d 100644 --- a/tests/e2e-cucumber/tests/e2e/dash_steps.rs +++ b/tests/e2e-cucumber/tests/e2e/dash_steps.rs @@ -12,7 +12,7 @@ use e2e_cucumber::mock_server::{MetricsMode, MockServer, ServiceRecordOptions}; use std::time::{Duration, Instant}; use crate::E2eWorld; -use crate::e2e::tui_driver::{DEFAULT_TIMEOUT, TuiSession}; +use crate::e2e::tui_driver::{TuiSession, default_timeout}; /// The exact prompt `send_managed_model_message` types, and the string the /// corresponding `Then` step (`managed_chat_request_carried_prompt`) asserts @@ -161,7 +161,7 @@ async fn choose_serving(world: &mut E2eWorld) { #[when("the user accepts the local endpoint")] async fn accept_local_endpoint(world: &mut E2eWorld) { let tui = session(world); - tui.wait_for_screen("Your request only leaves this machine", DEFAULT_TIMEOUT) + tui.wait_for_screen("Your request only leaves this machine", default_timeout()) .await .unwrap_or_else(|e| panic!("local endpoint consent did not appear: {e}")); tui.send("y") @@ -171,7 +171,7 @@ async fn accept_local_endpoint(world: &mut E2eWorld) { #[when("the user sends a message to the managed model")] async fn send_managed_model_message(world: &mut E2eWorld) { let tui = session(world); - tui.wait_for_screen("No messages yet.", DEFAULT_TIMEOUT) + tui.wait_for_screen("No messages yet.", default_timeout()) .await .unwrap_or_else(|e| panic!("chat surface never became ready: {e}")); // No `i` here: accepting local-endpoint consent (`accept_chat_consent`) already @@ -189,7 +189,7 @@ async fn send_gpu_message(world: &mut E2eWorld) { let tui = session(world); // Wait for the accepted, empty chat surface before typing so the input is // ready to receive focus. - tui.wait_for_screen("No messages yet.", DEFAULT_TIMEOUT) + tui.wait_for_screen("No messages yet.", default_timeout()) .await .unwrap_or_else(|e| panic!("chat surface never became ready: {e}")); // `i` focuses the input; then the message, then Enter to submit. @@ -203,7 +203,7 @@ async fn send_gpu_message(world: &mut E2eWorld) { async fn quit_tui(world: &mut E2eWorld, surface: &str) { session(world) - .quit_and_wait(DEFAULT_TIMEOUT) + .quit_and_wait(default_timeout()) .await .unwrap_or_else(|e| panic!("{surface} did not exit cleanly: {e}")); } @@ -225,7 +225,7 @@ async fn home_view_displayed(world: &mut E2eWorld) { let tui = session(world); // The Home tab's summary cards (Running / Health / Updates) are drawn at any // size; the wider "GPU UTILIZATION" hero is not, so assert on the cards. - tui.wait_for_screen("Updates", DEFAULT_TIMEOUT) + tui.wait_for_screen("Updates", default_timeout()) .await .unwrap_or_else(|e| panic!("the home view did not appear: {e}")); let screen = tui.screen_text(); @@ -238,7 +238,7 @@ async fn home_view_displayed(world: &mut E2eWorld) { #[then("ROCm setup actions are displayed")] async fn rocm_actions_displayed(world: &mut E2eWorld) { session(world) - .wait_for_screen("Set up / Install ROCm", DEFAULT_TIMEOUT) + .wait_for_screen("Set up / Install ROCm", default_timeout()) .await .unwrap_or_else(|e| panic!("the ROCm setup actions did not appear: {e}")); } @@ -246,7 +246,7 @@ async fn rocm_actions_displayed(world: &mut E2eWorld) { #[then("the assistant's GPU status response is displayed")] async fn gpu_response_displayed(world: &mut E2eWorld) { session(world) - .wait_for_screen("GPU-2 is running hot", DEFAULT_TIMEOUT) + .wait_for_screen("GPU-2 is running hot", default_timeout()) .await .unwrap_or_else(|e| panic!("the assistant's response did not appear: {e}")); } @@ -254,7 +254,7 @@ async fn gpu_response_displayed(world: &mut E2eWorld) { #[then("the managed model's response is displayed")] async fn managed_model_response_displayed(world: &mut E2eWorld) { session(world) - .wait_for_screen("mock response for testing", DEFAULT_TIMEOUT) + .wait_for_screen("mock response for testing", default_timeout()) .await .unwrap_or_else(|e| panic!("the managed model's response did not appear: {e}")); } @@ -272,7 +272,7 @@ async fn managed_chat_request_carried_prompt(world: &mut E2eWorld) { .mock .as_ref() .expect("no mock server running") - .wait_for_chat_request(DEFAULT_TIMEOUT) + .wait_for_chat_request(default_timeout()) .await .unwrap_or_else(|e| panic!("the mock never received a chat request: {e}")); let messages = body @@ -296,14 +296,14 @@ async fn managed_chat_request_carried_prompt(world: &mut E2eWorld) { async fn managed_model_shown_loading(world: &mut E2eWorld) { let model = world.model_name.clone().expect("no model name set"); let tui = session(world); - tui.wait_for_screen(&model, DEFAULT_TIMEOUT) + tui.wait_for_screen(&model, default_timeout()) .await .unwrap_or_else(|e| panic!("the managed model did not appear: {e}")); // The compact Observe table intentionally omits lifecycle status; opening // the selected instance exposes the status a user uses to diagnose startup. tui.send("\r") .unwrap_or_else(|e| panic!("failed to open instance details: {e}")); - tui.wait_for_screen("LOADING", DEFAULT_TIMEOUT) + tui.wait_for_screen("LOADING", default_timeout()) .await .unwrap_or_else(|e| panic!("the loading state did not appear: {e}")); let screen = tui.screen_text(); @@ -320,10 +320,10 @@ async fn managed_model_shown_loading(world: &mut E2eWorld) { async fn managed_model_metrics_displayed(world: &mut E2eWorld) { let model = world.model_name.clone().expect("no model name set"); let tui = session(world); - tui.wait_for_screen(&model, DEFAULT_TIMEOUT) + tui.wait_for_screen(&model, default_timeout()) .await .unwrap_or_else(|e| panic!("the managed model did not appear: {e}")); - tui.wait_for_screen("50ms", DEFAULT_TIMEOUT) + tui.wait_for_screen("50ms", default_timeout()) .await .unwrap_or_else(|e| panic!("TTFT metrics did not appear: {e}")); let screen = tui.screen_text(); @@ -341,7 +341,7 @@ async fn managed_model_metrics_displayed(world: &mut E2eWorld) { #[then("navigation and next-step guidance are displayed")] async fn navigation_guidance_displayed(world: &mut E2eWorld) { let tui = session(world); - tui.wait_for_screen("toggle this help", DEFAULT_TIMEOUT) + tui.wait_for_screen("toggle this help", default_timeout()) .await .unwrap_or_else(|e| panic!("dashboard help did not appear: {e}")); let screen = tui.screen_text(); @@ -354,7 +354,7 @@ async fn navigation_guidance_displayed(world: &mut E2eWorld) { #[then("dashboard destinations are displayed")] async fn dashboard_destinations_displayed(world: &mut E2eWorld) { let tui = session(world); - tui.wait_for_screen("Go to", DEFAULT_TIMEOUT) + tui.wait_for_screen("Go to", default_timeout()) .await .unwrap_or_else(|e| panic!("command palette did not appear: {e}")); let screen = tui.screen_text(); @@ -367,7 +367,7 @@ async fn dashboard_destinations_displayed(world: &mut E2eWorld) { #[then("Serving actions are displayed")] async fn serving_actions_displayed(world: &mut E2eWorld) { session(world) - .wait_for_screen("Serving actions", DEFAULT_TIMEOUT) + .wait_for_screen("Serving actions", default_timeout()) .await .unwrap_or_else(|e| panic!("Serving actions did not appear: {e}")); } @@ -380,7 +380,7 @@ async fn managed_model_displayed(world: &mut E2eWorld) { .expect("no model name set") .to_string(); session(world) - .wait_for_screen(&model, DEFAULT_TIMEOUT) + .wait_for_screen(&model, default_timeout()) .await .unwrap_or_else(|e| panic!("managed model did not appear: {e}")); } @@ -419,7 +419,7 @@ async fn local_endpoint_shown(world: &mut E2eWorld) { .port() .to_string(); session(world) - .wait_for_screen(&port, DEFAULT_TIMEOUT) + .wait_for_screen(&port, default_timeout()) .await .unwrap_or_else(|e| panic!("the detected endpoint was not shown: {e}")); } @@ -431,7 +431,7 @@ async fn privacy_notice_shown(world: &mut E2eWorld) { session(world) .wait_for_screen( "Your request only leaves this machine after you accept.", - DEFAULT_TIMEOUT, + default_timeout(), ) .await .unwrap_or_else(|e| panic!("the privacy notice was not shown: {e}")); @@ -457,7 +457,7 @@ async fn managed_model_scripted_metrics(world: &mut E2eWorld) { #[then("positive generation throughput is displayed for the managed model")] async fn positive_gen_tps_displayed(world: &mut E2eWorld) { session(world) - .wait_for_screen("tok/s", DEFAULT_TIMEOUT) + .wait_for_screen("tok/s", default_timeout()) .await .unwrap_or_else(|e| { panic!("positive gen_tps (\"tok/s\") never appeared after Growing-mode scrapes: {e}") @@ -475,14 +475,15 @@ async fn metrics_endpoint_fails(world: &mut E2eWorld) { // Poll until the daemon delivers at least one 503 to the mock endpoint. // The production instance_tick is 2 s, so this converges in 2–3 s. - let deadline = Instant::now() + DEFAULT_TIMEOUT; + let budget = default_timeout(); + let deadline = Instant::now() + budget; loop { if mock.metrics_failure_count() >= 1 { break; } assert!( Instant::now() < deadline, - "scripted failure was never served within {DEFAULT_TIMEOUT:?}; \ + "scripted failure was never served within {budget:?}; \ check instance_tick and scrape cadence" ); tokio::time::sleep(Duration::from_millis(100)).await; diff --git a/tests/e2e-cucumber/tests/e2e/lifecycle_steps.rs b/tests/e2e-cucumber/tests/e2e/lifecycle_steps.rs index ebea19992c..8d743afa72 100644 --- a/tests/e2e-cucumber/tests/e2e/lifecycle_steps.rs +++ b/tests/e2e-cucumber/tests/e2e/lifecycle_steps.rs @@ -23,7 +23,7 @@ use std::process::Command; use cucumber::{given, then, when}; use crate::E2eWorld; -use crate::e2e::tui_driver::{DEFAULT_TIMEOUT, TuiSession}; +use crate::e2e::tui_driver::{TuiSession, default_timeout}; /// Per-scenario release-lifecycle state. All paths are rooted in the scenario's /// isolated temp dir; `Drop` restores the captured Windows user PATH. @@ -857,7 +857,7 @@ async fn when_quit_installed_chat(world: &mut E2eWorld) { .tui .as_mut() .expect("no installed interactive chat session is open") - .quit_and_wait(DEFAULT_TIMEOUT) + .quit_and_wait(default_timeout()) .await .unwrap_or_else(|e| panic!("installed interactive chat did not exit cleanly: {e}")); } @@ -1008,7 +1008,7 @@ async fn then_installed_chat_displayed(world: &mut E2eWorld) { .tui .as_mut() .expect("no installed interactive chat session is open") - .wait_for_screen("No messages yet.", DEFAULT_TIMEOUT) + .wait_for_screen("No messages yet.", default_timeout()) .await .unwrap_or_else(|e| panic!("installed interactive chat did not become ready: {e}")); } diff --git a/tests/e2e-cucumber/tests/e2e/tui_driver.rs b/tests/e2e-cucumber/tests/e2e/tui_driver.rs index 9996dba4d4..f42641bc80 100644 --- a/tests/e2e-cucumber/tests/e2e/tui_driver.rs +++ b/tests/e2e-cucumber/tests/e2e/tui_driver.rs @@ -54,7 +54,27 @@ const DRAIN_TIMEOUT: Duration = Duration::from_millis(250); /// Default wall-clock budget for a single wait. Generous enough for a cold dash /// start plus the embedded-daemon connect, while still turning a genuine hang /// into a prompt, diagnosable failure rather than a CI-timeout. -pub const DEFAULT_TIMEOUT: Duration = Duration::from_secs(30); +const DEFAULT_TIMEOUT_SECS: u64 = 30; + +/// Wall-clock budget for a single wait, overridable with `E2E_TUI_TIMEOUT_SECS`. +/// +/// Mirrors `E2E_SERVE_TIMEOUT_SECS` in `serving_steps.rs`: the assertion is +/// right, the budget is what varies by host. The self-hosted Strix lanes share +/// one physical machine, so a TUI frame that renders well inside 30s on an idle +/// runner can miss it when a sibling lane is loading a model on the same box — +/// which shows up as an unrelated-looking flake, not as a real hang. +/// +/// Deliberately a wait budget and not a retry: a genuine hang must still fail. +#[must_use] +pub fn default_timeout() -> Duration { + Duration::from_secs( + std::env::var("E2E_TUI_TIMEOUT_SECS") + .ok() + .and_then(|value| value.parse().ok()) + .filter(|secs| *secs > 0) + .unwrap_or(DEFAULT_TIMEOUT_SECS), + ) +} /// A running `rocm` TUI attached to a pseudo-terminal. pub struct TuiSession { From 0894e7df116c13df4417beef60cc50a5a14e5a28 Mon Sep 17 00:00:00 2001 From: Roman Inflianskas Date: Fri, 14 Aug 2026 11:19:31 +0000 Subject: [PATCH 06/11] ci(e2e): give the WSL2 lane a real column in the report grid MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The WSL2 lane uploads `e2e-gpu-strix-wsl-report`, but nothing taught the report tooling that name. It would have passed through `fallback_descriptor` and rendered as "Gpu Strix Wsl" on Linux — a WSL2 host reported as native Linux, the exact miscoloring the artifact-name contract exists to prevent. Teach both halves: `parse_descriptor` maps the artifact to Strix Halo / WSL2, and `label_for_root_report` maps the `strix-halo-wsl` slug back to it so a flattened single-artifact download labels the same way. A bare `wsl` slug (WSL host, no GPU) deliberately stays neutral rather than claiming hardware the run never saw. Add the matching nightly lane too: the nightly grid is only comparable to the per-PR grid while both publish the same platforms, which is what the contract test asserts. Signed-off-by: Roman Inflianskas --- .github/workflows/nightly.yml | 164 +++++++++++++++++++++++++++++++++- crates/e2e-report/src/lib.rs | 9 ++ xtask/src/e2e_report.rs | 42 ++++++--- 3 files changed, 199 insertions(+), 16 deletions(-) diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index 7cca48b490..fc912d2dc1 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -625,11 +625,168 @@ jobs: name: e2e-gpu-strix-windows-report path: tests/e2e-cucumber/results/ + # WSL2 counterpart to e2e-gpu-nightly-strix: the same Strix Halo box again, but + # reached through an Ubuntu distro under WSL2, so this lane exercises the WSL + # host boundary AND whatever GPU access WSL exposes. Mirrors the per-PR + # `e2e-wsl` lane in e2e-selfhosted.yml, opting into the same @nightly + # large-model scenario as the other nightly lanes. `wsl` in `runs-on` + # disambiguates it from the `native` Strix Linux runner; the native lane's + # /home/ubuntu/actions-runner paths deliberately are NOT reused, since they do + # not exist in the WSL distro. + e2e-wsl-nightly: + name: E2E tests (Strix Halo, WSL2, incl. nightly-only) + timeout-minutes: 90 + runs-on: [self-hosted, linux, strix-halo, wsl] + continue-on-error: true + env: + E2E_INCLUDE_NIGHTLY: "1" + E2E_SERVE_TIMEOUT_SECS: "300" + # The Strix lanes share one physical machine, so a TUI frame that renders + # well inside the 30s default on an idle runner can miss it while a sibling + # lane loads a model. Raise the wait budget rather than let that read as a + # product failure; a genuine hang still fails, just later. + E2E_TUI_TIMEOUT_SECS: "90" + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + + # Fail fast and loudly if this job ever lands on a native Linux runner: + # every WSL-tagged scenario would silently resolve to skip instead. + - name: Verify the host really is WSL2 + run: | + if ! grep -qi microsoft /proc/sys/kernel/osrelease; then + echo "::error::runner is not a WSL host: $(uname -r)" + exit 1 + fi + printf 'WSL kernel: %s\n' "$(uname -r)" + + # The native Strix runners come pre-provisioned; a WSL distro often does + # not. Install the native build deps only when something is missing, and + # only if passwordless sudo is available — otherwise say what is missing + # instead of hanging on a password prompt. + - name: Ensure native build deps + run: | + missing="" + command -v pkg-config >/dev/null 2>&1 || missing="$missing pkg-config" + command -v cc >/dev/null 2>&1 || missing="$missing build-essential" + pkg-config --exists libcap 2>/dev/null || missing="$missing libcap-dev" + if [ -z "$missing" ]; then + echo "native build deps present" + elif sudo -n true 2>/dev/null; then + echo "installing:$missing" + sudo -n apt-get update + # shellcheck disable=SC2086 + sudo -n apt-get install -y $missing + else + echo "::error::missing native build deps:$missing (no passwordless sudo to install them)" + exit 1 + fi + + - name: Reclaim GPU from stray E2E processes + run: | + pkill -f '/tmp/rocm-e2e.*llama-server' 2>/dev/null || true + pkill -f '/tmp/rocm-e2e.*vllm serve' 2>/dev/null || true + pkill -f 'e2e-shared.*llama-server' 2>/dev/null || true + pkill -f '__engine-serve-http.*rocm-e2e' 2>/dev/null || true + rm -rf /tmp/rocm-e2e-* 2>/dev/null || true + echo "reclaimed" + + # GPU preflight, bounded like the native lanes — but ADVISORY here. GPU + # access under WSL is exactly what this lane is proving out, so an absent + # rocm-smi is a reported condition, not a job failure: the capability probe + # then resolves @requires-gpu scenarios to skip and the rest still runs. A + # GPU that IS present but stays held by a leftover serve still fails, since + # that would corrupt the serve scenarios' results. + - name: GPU preflight (advisory bounded wait) + run: | + MIN_FREE_GIB="${GPU_PREFLIGHT_MIN_FREE_GIB:-8}" + CEILING_SECS="${GPU_PREFLIGHT_CEILING_SECS:-90}" + if ! command -v rocm-smi >/dev/null 2>&1; then + echo "::warning::rocm-smi not found in this WSL distro — GPU scenarios will resolve to skip" + exit 0 + fi + min_free=$(( MIN_FREE_GIB * 1024 * 1024 * 1024 )) + deadline=$(( SECONDS + CEILING_SECS )) + saw_vram=0 + while [ "$SECONDS" -lt "$deadline" ]; do + out=$(timeout 15 rocm-smi --showmeminfo vram 2>/dev/null) || { sleep 5; continue; } + total=$(printf '%s\n' "$out" | grep -i 'VRAM Total Memory' | sed 's/.*: *//' | grep -oE '[0-9]+' | tail -1) + used=$(printf '%s\n' "$out" | grep -i 'VRAM Total Used Memory' | sed 's/.*: *//' | grep -oE '[0-9]+' | tail -1) + if [ -z "$total" ] || [ -z "$used" ]; then + sleep 5; continue + fi + saw_vram=1 + free=$(( total - used )) + if [ "$free" -ge "$min_free" ]; then + echo "GPU ready: $(( free / 1024 / 1024 / 1024 )) GiB free (>= ${MIN_FREE_GIB} GiB)." + exit 0 + fi + echo "waiting: $(( free / 1024 / 1024 / 1024 )) GiB free (< ${MIN_FREE_GIB} GiB)…" + sleep 5 + done + if [ "$saw_vram" -eq 0 ]; then + echo "::warning::rocm-smi reported no VRAM figures under WSL — GPU scenarios will resolve to skip" + exit 0 + fi + echo "::error::GPU preflight failed after ${CEILING_SECS}s: VRAM never dropped below the floor — a serve is likely still holding the GPU" + exit 1 + + # Bootstrap rustup with --no-modify-path so it never writes $HOME/.profile + # (setup-rust-toolchain doesn't expose that flag). rust-toolchain.toml pins + # the exact toolchain, installed on first cargo use. Idempotent. + - name: Ensure Rust toolchain + run: | + if ! command -v cargo >/dev/null 2>&1 && [ ! -x "$HOME/.cargo/bin/cargo" ]; then + curl --proto '=https' --tlsv1.2 -fsSL https://sh.rustup.rs \ + | sh -s -- -y --no-modify-path --default-toolchain none + fi + echo "$HOME/.cargo/bin" >> "$GITHUB_PATH" + + - name: Run E2E tests on Strix Halo WSL2 (incl. nightly-only) + run: | + export CARGO_TARGET_DIR="$RUNNER_WORKSPACE/e2e-target" + export E2E_SHARED_CACHE_DIR="$RUNNER_WORKSPACE/e2e-shared" + # Share ONE installed managed runtime across serve/chat scenarios and + # PRE-WARM it in place (mirrors e2e-gpu-nightly-strix). Required for + # correctness, not speed: `install sdk` bakes ABSOLUTE paths into the + # runtime manifest, so installing into a per-scenario temp dir leaves + # every later serve pointing at a deleted install root. + prewarm="$RUNNER_WORKSPACE/e2e-prewarm" + export E2E_SHARED_RUNTIMES_DIR="$prewarm/data/runtimes" + + cargo build --release -p rocm + export ROCM_CLI_BINARY="$CARGO_TARGET_DIR/release/rocm" + + if [ ! -d "$E2E_SHARED_RUNTIMES_DIR/registry" ]; then + echo "pre-warming shared runtime (first run on this runner)…" + mkdir -p "$prewarm"/{data,config,cache} + ROCM_CLI_CONFIG_DIR="$prewarm/config" \ + ROCM_CLI_DATA_DIR="$prewarm/data" \ + ROCM_CLI_CACHE_DIR="$prewarm/cache" \ + HF_HOME="$E2E_SHARED_CACHE_DIR/huggingface" \ + "$ROCM_CLI_BINARY" install sdk + if [ -d "$E2E_SHARED_RUNTIMES_DIR/registry" ]; then + echo "shared runtime pre-warmed at $E2E_SHARED_RUNTIMES_DIR" + else + echo "pre-warm did not produce a runtimes registry; scenarios will install their own" >&2 + fi + else + echo "shared runtime already present at $E2E_SHARED_RUNTIMES_DIR — skipping pre-warm" + fi + + cargo xtask e2e + + - name: Upload E2E report + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: e2e-gpu-strix-wsl-report + path: tests/e2e-cucumber/results/ + # The nightly lanes each uploaded a report nobody joined up, so reading a - # nightly meant opening three artifacts and comparing them by hand — on the + # nightly meant opening every artifact and comparing them by hand — on the # run that covers the most scenarios, since only nightly sets # E2E_INCLUDE_NIGHTLY. Same consolidation the per-PR self-hosted workflow - # already does (e2e-selfhosted.yml), against the three nightly artifacts. + # already does (e2e-selfhosted.yml), against the nightly artifacts. e2e-report-nightly: name: E2E consolidated report (nightly) runs-on: ubuntu-latest @@ -643,6 +800,7 @@ jobs: - e2e-gpu-nightly - e2e-gpu-nightly-strix - e2e-gpu-nightly-strix-windows + - e2e-wsl-nightly # Unconditional `always()`, unlike the per-PR workflow: there is no paths # filter to consult on a schedule, and the lanes are `continue-on-error`, so # a failed lane must still be reported. A nightly that broke is precisely @@ -653,7 +811,7 @@ jobs: - uses: actions-rust-lang/setup-rust-toolchain@166cdcfd11aee3cb47222f9ddb555ce30ddb9659 # v1.17.0 - # `*-report` matches the three artifacts above, and will pick up any + # `*-report` matches the artifacts above, and will pick up any # nightly platform added later without a change here — provided it is # named canonically (see the rename note on the upload steps). # Layout note (same as the per-PR job): download-artifact@v8 uses a diff --git a/crates/e2e-report/src/lib.rs b/crates/e2e-report/src/lib.rs index 6116b06c82..69a58e9ad2 100644 --- a/crates/e2e-report/src/lib.rs +++ b/crates/e2e-report/src/lib.rs @@ -414,6 +414,12 @@ fn parse_descriptor(name: &str) -> Descriptor { "gpu" => ("MI300X", "Linux"), "gpu-strix-ubuntu" => ("Strix Halo", "Ubuntu"), "gpu-strix-windows" => ("Strix Halo", "Windows"), + // Same silicon again, third host boundary: an Ubuntu distro under WSL2 on + // the Windows box. It is neither the native Ubuntu nor the Windows lane — + // GPU access goes through the WSL passthrough — so it needs its own OS + // value, not a reuse of "Ubuntu", or the grid would show two rows that + // claim to be the same host and disagree. + "gpu-strix-wsl" => ("Strix Halo", "WSL2"), // `e2e-unknown-report`: a report whose platform.json sidecar was missing // or unrecognized (e.g. a GPU run that errored before writing it). The OS // is genuinely unknown here — a Windows GPU run that erupted early must NOT @@ -2012,6 +2018,9 @@ mod tests { ("e2e-gpu-report", "MI300X", "Linux"), ("e2e-gpu-strix-ubuntu-report", "Strix Halo", "Ubuntu"), ("e2e-gpu-strix-windows-report", "Strix Halo", "Windows"), + // Must not fall through to `fallback_descriptor`, which would render + // "Gpu Strix Wsl" on Linux — a WSL2 host reported as native Linux. + ("e2e-gpu-strix-wsl-report", "Strix Halo", "WSL2"), ] { let d = parse_descriptor(name); assert_eq!( diff --git a/xtask/src/e2e_report.rs b/xtask/src/e2e_report.rs index 353d4c18f6..af4ded9fc0 100644 --- a/xtask/src/e2e_report.rs +++ b/xtask/src/e2e_report.rs @@ -133,6 +133,12 @@ fn label_for_root_report(dir: &Path) -> String { Some("mi300x") => "e2e-gpu-report".to_owned(), Some("strix-halo-linux") => "e2e-gpu-strix-ubuntu-report".to_owned(), Some("strix-halo-windows") => "e2e-gpu-strix-windows-report".to_owned(), + Some("strix-halo-wsl") => "e2e-gpu-strix-wsl-report".to_owned(), + // Note the deliberate absence of a bare `wsl` arm. `derive_platform_slug` + // emits that for a WSL host with no GPU, and mapping it to the Strix WSL + // artifact would assert hardware the run never saw; it falls through to + // the neutral label below instead. + // // Missing sidecar (e.g. a GPU run that errored before writing it) or an // unrecognized slug → neutral identity, never a false "Mock". _ => "e2e-unknown-report".to_owned(), @@ -159,6 +165,7 @@ mod tests { "e2e-gpu-report", "e2e-gpu-strix-ubuntu-report", "e2e-gpu-strix-windows-report", + "e2e-gpu-strix-wsl-report", ]; /// Every `e2e-`-prefixed artifact name an upload step in `file` publishes. @@ -234,8 +241,8 @@ mod tests { #[test] fn the_nightly_lanes_publish_the_same_platforms_as_the_per_pr_lanes() { // The nightly workflow exists to run *more* scenarios on the *same* - // three platforms. If the two ever diverge, the nightly grid is - // comparing different hardware than the PR grid without saying so. + // platforms. If the two ever diverge, the nightly grid is comparing + // different hardware than the PR grid without saying so. let lanes = |file: &str| { let mut names = uploaded_e2e_artifacts( &Path::new(env!("CARGO_MANIFEST_DIR")) @@ -288,6 +295,7 @@ mod tests { ("mi300x", "e2e-gpu-report"), ("strix-halo-linux", "e2e-gpu-strix-ubuntu-report"), ("strix-halo-windows", "e2e-gpu-strix-windows-report"), + ("strix-halo-wsl", "e2e-gpu-strix-wsl-report"), ("mock", "e2e-report"), ] { let tmp = tempfile::tempdir().expect("tempdir"); @@ -326,17 +334,25 @@ mod tests { // An unrecognized slug is likewise neutral, never a false real platform. #[test] fn discover_root_report_unknown_slug_is_neutral() { - let tmp = tempfile::tempdir().expect("tempdir"); - let root = tmp.path(); - std::fs::write(root.join("report.json"), "[]").unwrap(); - std::fs::write( - root.join("platform.json"), - r#"{"platform_slug":"some-future-gpu"}"#, - ) - .unwrap(); + for slug in [ + "some-future-gpu", + // `derive_platform_slug` emits a bare `wsl` for a WSL host where no + // GPU was found. It must NOT collapse into the Strix WSL artifact: + // that lane's report would then claim hardware the run never saw. + "wsl", + ] { + let tmp = tempfile::tempdir().expect("tempdir"); + let root = tmp.path(); + std::fs::write(root.join("report.json"), "[]").unwrap(); + std::fs::write( + root.join("platform.json"), + format!(r#"{{"platform_slug":"{slug}"}}"#), + ) + .unwrap(); - let got = discover(root).expect("discover"); - let names: Vec<&str> = got.iter().map(|(n, _)| n.as_str()).collect(); - assert_eq!(names, vec!["e2e-unknown-report"]); + let got = discover(root).expect("discover"); + let names: Vec<&str> = got.iter().map(|(n, _)| n.as_str()).collect(); + assert_eq!(names, vec!["e2e-unknown-report"], "slug {slug}"); + } } } From e8550ac136f24e1474bacccdbeb1aaf3cb57bc26 Mon Sep 17 00:00:00 2001 From: Michael Roy Date: Sat, 15 Aug 2026 11:59:51 -0700 Subject: [PATCH 07/11] ci(e2e): include WSL lane in merge-queue coverage Signed-off-by: Michael Roy --- .github/workflows/e2e-selfhosted.yml | 2 + docs/ci-hardware-testing.md | 24 +-- tests/e2e-cucumber/README.md | 6 +- xtask/src/workflow_contract.rs | 213 +++++++++++++++++++++++++++ 4 files changed, 231 insertions(+), 14 deletions(-) diff --git a/.github/workflows/e2e-selfhosted.yml b/.github/workflows/e2e-selfhosted.yml index 4632d5d0ad..28677d40b0 100644 --- a/.github/workflows/e2e-selfhosted.yml +++ b/.github/workflows/e2e-selfhosted.yml @@ -688,6 +688,8 @@ jobs: # Match the other hardware lanes: opt into the platform-adaptive # large-model scenario only when the manual include_nightly input is on. E2E_INCLUDE_NIGHTLY: "${{ inputs.include_nightly && '1' || '' }}" + # Heavy @merge-queue serves run only in the merge queue; see e2e-gpu. + E2E_MERGE_QUEUE: "${{ github.event_name == 'merge_group' && '1' || '' }}" steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 diff --git a/docs/ci-hardware-testing.md b/docs/ci-hardware-testing.md index f9176048cb..8124da72bd 100644 --- a/docs/ci-hardware-testing.md +++ b/docs/ci-hardware-testing.md @@ -46,10 +46,11 @@ WSL lane pins `wsl` for the same reason, from the other side. resolve to skip here, and known bugs resolve to xfail from `expectations.toml`. It is a required check and must stay green. -The three GPU jobs (`e2e-gpu`, `e2e-gpu-strix-ubuntu`, `e2e-gpu-strix-windows`) -run on dedicated self-hosted runners with a real AMD GPU attached, so they -exercise host/GPU detection, engine `detect`/`capabilities`, and live serving -scenarios that the mock job cannot. +The four self-hosted jobs (`e2e-gpu`, `e2e-gpu-strix-ubuntu`, +`e2e-gpu-strix-windows`, and `e2e-wsl`) run on AMD GPU systems, so they exercise +host/GPU detection, engine `detect`/`capabilities`, and live serving scenarios +that the mock job cannot. GPU availability is advisory in the WSL lane, as +described below. `e2e-wsl` runs on an Ubuntu distro hosted in WSL2 on the Strix Halo Windows box and mirrors the sibling Linux lane step for step: stray-serve reclaim, GPU @@ -82,17 +83,18 @@ without the passthrough runs the non-GPU suite and reports the GPU scenarios as not applicable, instead of failing them on a premise the host cannot meet. Each workflow has its own consolidated report job. `ci.yml`'s `e2e-report` -covers the mock platform; `e2e-selfhosted.yml`'s `e2e-report` covers the three -GPU platforms; `nightly.yml`'s `e2e-report-nightly` covers the same three -platforms with the `@nightly` scenarios included. Each joins its platforms' +covers the mock platform; +`e2e-selfhosted.yml`'s `e2e-report` covers the four GPU platforms; +`nightly.yml`'s `e2e-report-nightly` covers the same four platforms with the +`@nightly` scenarios included. Each joins its platforms' reports — including partial or failed runs — by scenario id into one HTML report and GitHub step summary. The lane artifacts are named canonically (`e2e-report`, `e2e-gpu-report`, -`e2e-gpu-strix-ubuntu-report`, `e2e-gpu-strix-windows-report`) in every workflow, -because the report derives each platform's name and OS from the artifact name. -An unrecognised name renders as a guessed platform on Linux, which would report -a Windows lane as Linux; `xtask`'s +`e2e-gpu-strix-ubuntu-report`, `e2e-gpu-strix-windows-report`, +`e2e-gpu-strix-wsl-report`) in every workflow, because the report derives each +platform's name and OS from the artifact name. An unrecognised name renders as a +guessed platform on Linux, which would report a Windows lane as Linux; `xtask`'s `every_uploaded_e2e_artifact_has_a_name_the_report_can_label` guards against it. ## Triggers diff --git a/tests/e2e-cucumber/README.md b/tests/e2e-cucumber/README.md index 8a335cc0b6..8b4b403672 100644 --- a/tests/e2e-cucumber/README.md +++ b/tests/e2e-cucumber/README.md @@ -146,9 +146,9 @@ Ubuntu distro under WSL2 on the Strix Halo box, so it is the only lane that exercises `@requires-wsl` scenarios; `@requires-bare-metal` scenarios resolve to skip there. -The nightly workflow runs three non-blocking jobs — the existing MI300X job and -new Strix Halo jobs on Ubuntu and Windows — with `E2E_INCLUDE_NIGHTLY=1`, then -consolidates them into the same cross-platform grid. The +The nightly workflow runs four non-blocking jobs — MI300X plus Strix Halo on +Ubuntu, Windows, and WSL2 — with `E2E_INCLUDE_NIGHTLY=1`, then consolidates them +into the same cross-platform grid. The shared large-model scenario serves `Qwen/Qwen3.6-27B` through vLLM on MI300X and the hardware-verified `unsloth/Qwen3.6-35B-A3B-GGUF:UD-Q4_K_XL` checkpoint through Lemonade on Strix Halo. diff --git a/xtask/src/workflow_contract.rs b/xtask/src/workflow_contract.rs index 9d2c574b42..b282b1524a 100644 --- a/xtask/src/workflow_contract.rs +++ b/xtask/src/workflow_contract.rs @@ -17,6 +17,7 @@ #[cfg(test)] mod tests { + use std::collections::BTreeMap; use std::path::{Path, PathBuf}; fn repo_root() -> PathBuf { @@ -127,6 +128,112 @@ mod tests { .expect("workflow declares a top-level name:") } + /// Extract one top-level job's complete YAML block by its job id. + fn job_block<'a>(text: &'a str, job: &str) -> &'a str { + let marker = format!(" {job}:\n"); + let start = text + .find(&marker) + .unwrap_or_else(|| panic!("workflow defines job `{job}`")); + let rest = &text[start + marker.len()..]; + let end = rest + .match_indices("\n ") + .find_map(|(i, _)| { + rest[i + 1..] + .lines() + .next() + .is_some_and(|line| line.starts_with(" ") && !line.starts_with(" ")) + .then_some(i) + }) + .unwrap_or(rest.len()); + &rest[..end] + } + + /// Extract the direct scalar entries from a named job-level mapping such as + /// `env:`. Nested step mappings cannot satisfy this extractor. + fn job_mapping(block: &str, mapping: &str) -> BTreeMap { + let marker = format!("{mapping}:"); + let lines: Vec<&str> = block.lines().collect(); + let (start, mapping_indent) = lines + .iter() + .enumerate() + .find_map(|(i, line)| { + (line.trim() == marker && indent_of(line) == 4).then_some((i, indent_of(line))) + }) + .unwrap_or_else(|| panic!("job defines top-level mapping `{mapping}`")); + + let mut entries = BTreeMap::new(); + for raw in &lines[start + 1..] { + if raw.trim().is_empty() || raw.trim_start().starts_with('#') { + continue; + } + let line = strip_comment(raw); + let indent = indent_of(line); + if indent <= mapping_indent { + break; + } + if indent != mapping_indent + 2 { + continue; + } + let (key, raw_value) = line + .trim() + .split_once(':') + .unwrap_or_else(|| panic!("mapping entry has a scalar value: `{line}`")); + let value = raw_value.trim(); + let value = value + .strip_prefix('"') + .and_then(|v| v.strip_suffix('"')) + .or_else(|| value.strip_prefix('\'').and_then(|v| v.strip_suffix('\''))) + .unwrap_or(value); + assert!( + entries.insert(key.to_owned(), value.to_owned()).is_none(), + "mapping `{mapping}` contains duplicate key `{key}`" + ); + } + entries + } + + fn markdown_table_rows(text: &str, header: &str) -> Vec> { + let mut lines = text.lines().skip_while(|line| *line != header); + assert_eq!( + lines.next(), + Some(header), + "markdown table `{header}` exists" + ); + let separator = lines.next().expect("markdown table has a separator row"); + assert!( + separator.starts_with("|---"), + "markdown table has a separator row" + ); + lines + .take_while(|line| line.starts_with('|')) + .map(|line| { + line.trim_matches('|') + .split('|') + .map(|cell| cell.trim().to_owned()) + .collect() + }) + .collect() + } + + fn backticked_list_between(text: &str, prefix: &str, suffix: &str) -> Vec { + let section = text + .split_once(prefix) + .unwrap_or_else(|| panic!("section starts with `{prefix}`")) + .1 + .split_once(suffix) + .unwrap_or_else(|| panic!("section ends with `{suffix}`")) + .0; + section + .split('`') + .enumerate() + .filter_map(|(i, item)| (i % 2 == 1).then_some(item.to_owned())) + .collect() + } + + fn normalized_whitespace(text: &str) -> String { + text.split_whitespace().collect::>().join(" ") + } + #[test] fn ci_yml_schedules_no_self_hosted_job() { let ci = read_workflow("ci.yml"); @@ -168,6 +275,89 @@ mod tests { } } + #[test] + fn self_hosted_wsl_enables_merge_queue_scenarios_only_for_merge_group() { + let sh = read_workflow("e2e-selfhosted.yml"); + let wsl = job_block(&sh, "e2e-wsl"); + let env = job_mapping(wsl, "env"); + assert_eq!( + env.get("E2E_MERGE_QUEUE").map(String::as_str), + Some("${{ github.event_name == 'merge_group' && '1' || '' }}"), + "e2e-wsl's job-level env must opt into @merge-queue scenarios for merge_group only" + ); + + let nightly = read_workflow("nightly.yml"); + let nightly_wsl = job_block(&nightly, "e2e-wsl-nightly"); + let nightly_env = job_mapping(nightly_wsl, "env"); + assert!( + !nightly_env.contains_key("E2E_MERGE_QUEUE"), + "the nightly WSL job cannot receive merge_group events and must not opt into @merge-queue scenarios" + ); + } + + #[test] + fn hardware_testing_docs_cover_all_four_self_hosted_platforms() { + let docs = std::fs::read_to_string(repo_root().join("docs/ci-hardware-testing.md")) + .expect("read hardware testing docs"); + let rows = markdown_table_rows(&docs, "| Job | Workflow | Platform | Runner labels |"); + let self_hosted_job_platforms: Vec<(String, String)> = rows + .into_iter() + .filter(|row| { + row.get(1) + .is_some_and(|workflow| workflow == "`e2e-selfhosted.yml`") + }) + .map(|row| (row[0].clone(), row[2].clone())) + .collect(); + assert_eq!( + self_hosted_job_platforms, + vec![ + ( + "`e2e-gpu`".to_owned(), + "MI300X (AMD Instinct, bare-metal Linux)".to_owned(), + ), + ( + "`e2e-gpu-strix-ubuntu`".to_owned(), + "Strix Halo (gfx1151) on Ubuntu".to_owned(), + ), + ( + "`e2e-gpu-strix-windows`".to_owned(), + "Strix Halo (gfx1151) on native Windows 11".to_owned(), + ), + ( + "`e2e-wsl`".to_owned(), + "Strix Halo (gfx1151) on Ubuntu under WSL2".to_owned(), + ), + ], + "hardware testing table must document the four actual self-hosted job/platform rows" + ); + + let artifacts = backticked_list_between( + &docs, + "The lane artifacts are named canonically (", + ") in every workflow", + ); + assert_eq!( + artifacts, + vec![ + "e2e-report", + "e2e-gpu-report", + "e2e-gpu-strix-ubuntu-report", + "e2e-gpu-strix-windows-report", + "e2e-gpu-strix-wsl-report", + ], + "the canonical artifact list must enumerate every report platform exactly once" + ); + + let readme = std::fs::read_to_string(repo_root().join("tests/e2e-cucumber/README.md")) + .expect("read E2E README"); + assert!( + normalized_whitespace(&readme).contains( + "The nightly workflow runs four non-blocking jobs — MI300X plus Strix Halo on Ubuntu, Windows, and WSL2 — with `E2E_INCLUDE_NIGHTLY=1`" + ), + "E2E README must identify all four nightly job platforms" + ); + } + #[test] fn workflows_use_distinct_concurrency_groups() { // Isolation comes from the group KEY differing per workflow. Extract the @@ -250,4 +440,27 @@ permissions: // Must stop at the next key, not swallow permissions. assert!(!g.contains("permissions")); } + + #[test] + fn job_mapping_extractor_ignores_nested_step_env() { + let block = " env:\n TOP_LEVEL: \"expected\"\n steps:\n - name: nested\n env:\n E2E_MERGE_QUEUE: wrong\n"; + let env = job_mapping(block, "env"); + assert_eq!(env.get("TOP_LEVEL").map(String::as_str), Some("expected")); + assert!(!env.contains_key("E2E_MERGE_QUEUE")); + } + + #[test] + fn job_mapping_extractor_ignores_blank_and_full_line_comments() { + let block = " env:\n BEFORE: one\n\n# a YAML comment may be less indented than the mapping\n # or aligned with its entries\n AFTER: two\n steps:\n"; + let env = job_mapping(block, "env"); + assert_eq!(env.get("BEFORE").map(String::as_str), Some("one")); + assert_eq!(env.get("AFTER").map(String::as_str), Some("two")); + } + + #[test] + #[should_panic(expected = "mapping entry has a scalar value")] + fn job_mapping_extractor_rejects_malformed_non_comment_rows() { + let block = " env:\n VALID: one\n MALFORMED\n steps:\n"; + let _ = job_mapping(block, "env"); + } } From dc80dc6d00b33458a6be86b78c0c1b09644dbed5 Mon Sep 17 00:00:00 2001 From: Michael Roy Date: Mon, 17 Aug 2026 11:35:59 -0700 Subject: [PATCH 08/11] fix(e2e): preserve WSL lane contracts Signed-off-by: Michael Roy --- .github/workflows/e2e-selfhosted.yml | 5 ++- .github/workflows/nightly.yml | 7 +++- tests/e2e-cucumber/expectations.toml | 8 ++-- tests/e2e-cucumber/src/expectation.rs | 43 ++++++++++++++++++++++ xtask/src/e2e_report.rs | 53 ++++++++++++++------------- xtask/src/workflow_contract.rs | 47 ++++++++++++++++++++++++ 6 files changed, 132 insertions(+), 31 deletions(-) diff --git a/.github/workflows/e2e-selfhosted.yml b/.github/workflows/e2e-selfhosted.yml index 28677d40b0..62aec0f36c 100644 --- a/.github/workflows/e2e-selfhosted.yml +++ b/.github/workflows/e2e-selfhosted.yml @@ -697,7 +697,10 @@ jobs: # every WSL-tagged scenario would silently resolve to skip instead. - name: Verify the host really is WSL2 run: | - if ! grep -qi microsoft /proc/sys/kernel/osrelease; then + proc_version=$(cat /proc/version 2>/dev/null || true) + if [ ! -e /dev/dxg ] \ + && [ -z "${WSL_DISTRO_NAME:-}" ] \ + && ! printf '%s\n' "$proc_version" | grep -qiE 'microsoft|wsl'; then echo "::error::runner is not a WSL host: $(uname -r)" exit 1 fi diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index fc912d2dc1..071166b3f1 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -435,6 +435,7 @@ jobs: PIP_CACHE_DIR: /home/ubuntu/actions-runner/pip-cache E2E_INCLUDE_NIGHTLY: "1" E2E_SERVE_TIMEOUT_SECS: "300" + E2E_TUI_TIMEOUT_SECS: "90" steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 @@ -530,6 +531,7 @@ jobs: env: E2E_INCLUDE_NIGHTLY: "1" E2E_SERVE_TIMEOUT_SECS: "300" + E2E_TUI_TIMEOUT_SECS: "90" steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 @@ -653,7 +655,10 @@ jobs: # every WSL-tagged scenario would silently resolve to skip instead. - name: Verify the host really is WSL2 run: | - if ! grep -qi microsoft /proc/sys/kernel/osrelease; then + proc_version=$(cat /proc/version 2>/dev/null || true) + if [ ! -e /dev/dxg ] \ + && [ -z "${WSL_DISTRO_NAME:-}" ] \ + && ! printf '%s\n' "$proc_version" | grep -qiE 'microsoft|wsl'; then echo "::error::runner is not a WSL host: $(uname -r)" exit 1 fi diff --git a/tests/e2e-cucumber/expectations.toml b/tests/e2e-cucumber/expectations.toml index 1045ae433e..febd7a4744 100644 --- a/tests/e2e-cucumber/expectations.toml +++ b/tests/e2e-cucumber/expectations.toml @@ -21,7 +21,7 @@ # # Condition predicate grammar (deliberately tiny): # [[]] one table per condition (arrays = OR) -# when = { effective_engine = "vllm", os = "linux", is_wsl = false } +# when = { effective_engine = "vllm", os = "linux", has_amd_gpu = true } # all present keys must match (AND) # bug = "EAI-NNNN" ticket reference (shown in the report) # reason = "..." human explanation (shown in the report) @@ -113,7 +113,7 @@ serve_timeout_secs = 90 # scenario now also runs on the no-GPU mock lane (backed by MockServer, not a # real lemonade serve), where EAI-7423 does not apply and it must expect-pass. [["chat-tool-definitions-accepted"]] -when = { effective_engine = "lemonade", os = "linux", therock_family = "gfx*" } +when = { effective_engine = "lemonade", os = "linux", therock_family = "gfx*", has_amd_gpu = true } bug = "EAI-7423" reason = "Lemonade managed serve reaches ready then shuts down immediately, so the chat request never reaches a live model." serve_timeout_secs = 90 @@ -130,7 +130,7 @@ flaky = true # a REAL GPU host: the scenario also runs on the no-GPU mock lane (MockServer, not # a real lemonade serve), where EAI-7423 does not apply and it must expect-pass. [["chat-end-to-end-local-model"]] -when = { effective_engine = "lemonade", os = "linux", therock_family = "gfx*" } +when = { effective_engine = "lemonade", os = "linux", therock_family = "gfx*", has_amd_gpu = true } bug = "EAI-7423" reason = "Lemonade managed serve reaches ready then shuts down immediately, so the chat request never reaches a live model." serve_timeout_secs = 90 @@ -157,7 +157,7 @@ reason = "Managed runtime path nests recursively (runtimes/wheel/...) on re-prov # MockServer-backed bench scenarios run unxfail'd on every lane, including this # one, and they are what pin the request path and the failure reporting. [["bench-load-real-serve"]] -when = { effective_engine = "lemonade", os = "linux", therock_family = "gfx*" } +when = { effective_engine = "lemonade", os = "linux", therock_family = "gfx*", has_amd_gpu = true } bug = "EAI-7423" reason = "Lemonade managed serve reaches ready then shuts down immediately, so the benchmark never reaches a live model." serve_timeout_secs = 90 diff --git a/tests/e2e-cucumber/src/expectation.rs b/tests/e2e-cucumber/src/expectation.rs index bf24bbca46..d1066af2a3 100644 --- a/tests/e2e-cucumber/src/expectation.rs +++ b/tests/e2e-cucumber/src/expectation.rs @@ -197,6 +197,8 @@ pub struct Condition { #[serde(default)] pub therock_family: Option, #[serde(default)] + pub has_amd_gpu: Option, + #[serde(default)] pub is_wsl: Option, } @@ -219,6 +221,11 @@ impl Condition { return false; } } + if let Some(has_gpu) = self.has_amd_gpu + && has_gpu != cap.has_amd_gpu + { + return false; + } if let Some(w) = self.is_wsl && w != cap.is_wsl { @@ -542,6 +549,17 @@ mod tests { effective_serve_engine: "lemonade".into(), platform_slug: "wsl".into(), }, + // The hosted WSL runner before the ROCm passthrough is complete: + // Windows still reports the gfx target, but the CLI cannot use it. + "wsl-no-passthrough" => HostCapability { + os_family: "linux".into(), + is_wsl: true, + gfx_target: Some("gfx1151".into()), + has_amd_gpu: false, + available_engines: vec!["lemonade".into(), "vllm".into()], + effective_serve_engine: "lemonade".into(), + platform_slug: "strix-halo-wsl".into(), + }, _ => HostCapability { os_family: "other".into(), is_wsl: false, @@ -996,6 +1014,31 @@ reason = "vLLM readiness gap" assert!(!m.is_xfail("nope", &cap("mi300x"), "vllm")); } + #[test] + fn gpu_bug_condition_does_not_match_an_unusable_wsl_gpu() { + let m = Expectations::parse( + r#" +[["chat-end-to-end-local-model"]] +when = { effective_engine = "lemonade", os = "linux", therock_family = "gfx*", has_amd_gpu = true } +bug = "EAI-7423" +reason = "Applies only when the scenario uses a real Lemonade GPU serve." +flaky = true +"#, + ) + .unwrap(); + + assert!(m.is_xfail( + "chat-end-to-end-local-model", + &cap("strix-ubuntu"), + "lemonade" + )); + assert!(!m.is_xfail( + "chat-end-to-end-local-model", + &cap("wsl-no-passthrough"), + "lemonade" + )); + } + #[test] fn glob_matches_family() { assert!(glob_match("gfx94*", "gfx942")); diff --git a/xtask/src/e2e_report.rs b/xtask/src/e2e_report.rs index af4ded9fc0..27122ebbd5 100644 --- a/xtask/src/e2e_report.rs +++ b/xtask/src/e2e_report.rs @@ -133,12 +133,11 @@ fn label_for_root_report(dir: &Path) -> String { Some("mi300x") => "e2e-gpu-report".to_owned(), Some("strix-halo-linux") => "e2e-gpu-strix-ubuntu-report".to_owned(), Some("strix-halo-windows") => "e2e-gpu-strix-windows-report".to_owned(), - Some("strix-halo-wsl") => "e2e-gpu-strix-wsl-report".to_owned(), - // Note the deliberate absence of a bare `wsl` arm. `derive_platform_slug` - // emits that for a WSL host with no GPU, and mapping it to the Strix WSL - // artifact would assert hardware the run never saw; it falls through to - // the neutral label below instead. - // + // This workflow is statically pinned to the Strix WSL runner. A bare + // `wsl` slug means the probe could not recover a gfx target; map it to + // the same identity the workflow's artifact directory carries so a + // flattened single-artifact download cannot change the report column. + Some("wsl" | "strix-halo-wsl") => "e2e-gpu-strix-wsl-report".to_owned(), // Missing sidecar (e.g. a GPU run that errored before writing it) or an // unrecognized slug → neutral identity, never a false "Mock". _ => "e2e-unknown-report".to_owned(), @@ -334,25 +333,29 @@ mod tests { // An unrecognized slug is likewise neutral, never a false real platform. #[test] fn discover_root_report_unknown_slug_is_neutral() { - for slug in [ - "some-future-gpu", - // `derive_platform_slug` emits a bare `wsl` for a WSL host where no - // GPU was found. It must NOT collapse into the Strix WSL artifact: - // that lane's report would then claim hardware the run never saw. - "wsl", - ] { - let tmp = tempfile::tempdir().expect("tempdir"); - let root = tmp.path(); - std::fs::write(root.join("report.json"), "[]").unwrap(); - std::fs::write( - root.join("platform.json"), - format!(r#"{{"platform_slug":"{slug}"}}"#), - ) - .unwrap(); + let tmp = tempfile::tempdir().expect("tempdir"); + let root = tmp.path(); + std::fs::write(root.join("report.json"), "[]").unwrap(); + std::fs::write( + root.join("platform.json"), + r#"{"platform_slug":"some-future-gpu"}"#, + ) + .unwrap(); - let got = discover(root).expect("discover"); - let names: Vec<&str> = got.iter().map(|(n, _)| n.as_str()).collect(); - assert_eq!(names, vec!["e2e-unknown-report"], "slug {slug}"); - } + let got = discover(root).expect("discover"); + let names: Vec<&str> = got.iter().map(|(n, _)| n.as_str()).collect(); + assert_eq!(names, vec!["e2e-unknown-report"]); + } + + #[test] + fn discover_root_bare_wsl_matches_the_static_wsl_artifact() { + let tmp = tempfile::tempdir().expect("tempdir"); + let root = tmp.path(); + std::fs::write(root.join("report.json"), "[]").unwrap(); + std::fs::write(root.join("platform.json"), r#"{"platform_slug":"wsl"}"#).unwrap(); + + let got = discover(root).expect("discover"); + let names: Vec<&str> = got.iter().map(|(n, _)| n.as_str()).collect(); + assert_eq!(names, vec!["e2e-gpu-strix-wsl-report"]); } } diff --git a/xtask/src/workflow_contract.rs b/xtask/src/workflow_contract.rs index b282b1524a..b33416385d 100644 --- a/xtask/src/workflow_contract.rs +++ b/xtask/src/workflow_contract.rs @@ -295,6 +295,53 @@ mod tests { ); } + #[test] + fn wsl_jobs_accept_every_canonical_wsl_detection_signal() { + for (workflow, job) in [ + ("e2e-selfhosted.yml", "e2e-wsl"), + ("nightly.yml", "e2e-wsl-nightly"), + ] { + let text = read_workflow(workflow); + let block = job_block(&text, job); + // Remove shell line-continuation backslashes after whitespace has + // been normalized so the assertion describes the effective test. + let block = normalized_whitespace(block).replace("\\ ", ""); + assert!( + block.contains("proc_version=$(cat /proc/version 2>/dev/null || true)"), + "{workflow} job {job} must read the canonical kernel-version signal" + ); + let distro_signal = ["$", "{", "WSL_DISTRO_NAME:-", "}"].concat(); + let canonical_union = format!( + "if [ ! -e /dev/dxg ] && [ -z \"{distro_signal}\" ] && ! printf '%s\\n' \"$proc_version\" | grep -qiE 'microsoft|wsl'; then" + ); + assert!( + block.contains(&canonical_union), + "{workflow} job {job} must accept the canonical union of WSL signals" + ); + assert!( + !block.contains("/proc/sys/kernel/osrelease"), + "{workflow} job {job} must not use the narrower osrelease-only WSL check" + ); + } + } + + #[test] + fn every_nightly_strix_job_uses_the_shared_machine_tui_budget() { + let nightly = read_workflow("nightly.yml"); + for job in [ + "e2e-gpu-nightly-strix", + "e2e-gpu-nightly-strix-windows", + "e2e-wsl-nightly", + ] { + let env = job_mapping(job_block(&nightly, job), "env"); + assert_eq!( + env.get("E2E_TUI_TIMEOUT_SECS").map(String::as_str), + Some("90"), + "nightly Strix job {job} must use the shared-machine TUI wait budget" + ); + } + } + #[test] fn hardware_testing_docs_cover_all_four_self_hosted_platforms() { let docs = std::fs::read_to_string(repo_root().join("docs/ci-hardware-testing.md")) From 222f9d7640fa6c999e905e4d349066171ea94c48 Mon Sep 17 00:00:00 2001 From: Michael Roy Date: Mon, 17 Aug 2026 12:01:57 -0700 Subject: [PATCH 09/11] fix(e2e): align WSL capability contracts Signed-off-by: Michael Roy --- .github/workflows/e2e-selfhosted.yml | 10 +-- .github/workflows/nightly.yml | 2 +- tests/e2e-cucumber/expectations.toml | 11 +-- tests/e2e-cucumber/src/capability.rs | 115 +++++++++++++++++++++++--- tests/e2e-cucumber/src/expectation.rs | 11 +++ xtask/src/workflow_contract.rs | 31 ++++++- 6 files changed, 157 insertions(+), 23 deletions(-) diff --git a/.github/workflows/e2e-selfhosted.yml b/.github/workflows/e2e-selfhosted.yml index 62aec0f36c..7da4333141 100644 --- a/.github/workflows/e2e-selfhosted.yml +++ b/.github/workflows/e2e-selfhosted.yml @@ -657,10 +657,10 @@ jobs: # GPU-on-WSL is proven out. e2e-wsl: name: E2E tests (Strix Halo, WSL2) - # 35min: matches the sibling collapsed GPU jobs — one job runs all serves - # plus per-scenario `install sdk`, and the cap must exceed the run so the job - # still writes platform.json. - timeout-minutes: 35 + # A manual include_nightly dispatch runs the 2400s large-model readiness + # scenario, so match the dedicated nightly lane's cap and leave room for + # build, runtime pre-warm, the rest of the suite, and platform.json. + timeout-minutes: 90 runs-on: [self-hosted, linux, strix-halo, wsl] needs: [changes] # See `e2e-gpu`: no build-and-test gate (cross-workflow); strix-wsl. Gated on @@ -699,7 +699,7 @@ jobs: run: | proc_version=$(cat /proc/version 2>/dev/null || true) if [ ! -e /dev/dxg ] \ - && [ -z "${WSL_DISTRO_NAME:-}" ] \ + && [ -z "${WSL_DISTRO_NAME+x}" ] \ && ! printf '%s\n' "$proc_version" | grep -qiE 'microsoft|wsl'; then echo "::error::runner is not a WSL host: $(uname -r)" exit 1 diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index 071166b3f1..1c000ba540 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -657,7 +657,7 @@ jobs: run: | proc_version=$(cat /proc/version 2>/dev/null || true) if [ ! -e /dev/dxg ] \ - && [ -z "${WSL_DISTRO_NAME:-}" ] \ + && [ -z "${WSL_DISTRO_NAME+x}" ] \ && ! printf '%s\n' "$proc_version" | grep -qiE 'microsoft|wsl'; then echo "::error::runner is not a WSL host: $(uname -r)" exit 1 diff --git a/tests/e2e-cucumber/expectations.toml b/tests/e2e-cucumber/expectations.toml index febd7a4744..c86821cdb3 100644 --- a/tests/e2e-cucumber/expectations.toml +++ b/tests/e2e-cucumber/expectations.toml @@ -169,11 +169,12 @@ flaky = true # probes are routed out on WSL because they read KFD and DRM, neither of which # exists there, while the summary reads the target from the Windows-side driver. # -# Scoped to WSL because that is where this lane reproduces it. The ticket also -# records the same disagreement on MI300X, second-hand from the comment in -# capability.rs; if that is confirmed the row should widen rather than gain a -# sibling. Remove it when the two forms agree. --- +# Scoped to WSL hosts whose human report actually names a gfx target: without +# one both forms correctly report no GPU, so xfail would turn agreement into a +# stale XPASS. The ticket also records the same disagreement on MI300X, +# second-hand from the comment in capability.rs; if that is confirmed the row +# should widen rather than gain a sibling. Remove it when the two forms agree. --- [["examine-both-forms-agree-on-gpu"]] -when = { is_wsl = true } +when = { is_wsl = true, therock_family = "gfx*" } bug = "EAI-7998" reason = "`examine --json` reports has_amd_gpu:false on a WSL2 host whose own summary names a gfx target." diff --git a/tests/e2e-cucumber/src/capability.rs b/tests/e2e-cucumber/src/capability.rs index 10a662a773..9ca0d6a0a6 100644 --- a/tests/e2e-cucumber/src/capability.rs +++ b/tests/e2e-cucumber/src/capability.rs @@ -431,7 +431,40 @@ fn parse_examine_text(text: &str) -> ExamineFacts { /// cannot serve one — they fail on their premise rather than resolving to /// not-applicable, which is exactly what the capability probe exists to avoid. fn host_has_usable_gpu(gfx_target: Option<&str>, is_wsl: bool, driver_status: &str) -> bool { - gfx_target.is_some() && (!is_wsl || driver_status == WSL_DRIVER_READY) + let hip = std::env::var_os("HIP_VISIBLE_DEVICES"); + let rocr = std::env::var_os("ROCR_VISIBLE_DEVICES"); + host_has_usable_gpu_with_mask( + gfx_target, + is_wsl, + driver_status, + selected_visibility_mask(hip.as_deref(), rocr.as_deref()), + ) +} + +/// Match the product's visibility-mask precedence: HIP wins when present, +/// including an explicitly empty value; ROCR is the fallback. +fn selected_visibility_mask<'a>( + hip: Option<&'a std::ffi::OsStr>, + rocr: Option<&'a std::ffi::OsStr>, +) -> Option<&'a std::ffi::OsStr> { + hip.or(rocr) +} + +/// Pure form of [`host_has_usable_gpu`] for contract tests. +/// +/// The harness knows whether a gfx target exists, but not the product's complete +/// device topology. As in `rocm_core::has_usable_amd_gpu`, only an authoritative +/// empty mask proves zero usable devices; nonempty/opaque masks stay eligible and +/// let the product perform ordinal/UUID validation at launch time. +fn host_has_usable_gpu_with_mask( + gfx_target: Option<&str>, + is_wsl: bool, + driver_status: &str, + visibility_mask: Option<&std::ffi::OsStr>, +) -> bool { + gfx_target.is_some() + && (!is_wsl || driver_status == WSL_DRIVER_READY) + && visibility_mask.is_none_or(|mask| !mask.is_empty()) } /// Parse engine names from `rocm engines list`. Engine rows are the lines whose @@ -631,28 +664,88 @@ rocm examine fn wsl_gpu_requires_a_ready_rocdxg_path() { // Real values from the self-hosted WSL runner before librocdxg was // installed: /dev/dxg present, the ROCm passthrough library missing. - assert!(!host_has_usable_gpu( + assert!(!host_has_usable_gpu_with_mask( Some("gfx1151"), true, - "wsl_rocdxg_missing" + "wsl_rocdxg_missing", + None )); - assert!(!host_has_usable_gpu( + assert!(!host_has_usable_gpu_with_mask( Some("gfx1151"), true, - "wsl_gpu_plumbing_missing" + "wsl_gpu_plumbing_missing", + None )); // Complete passthrough: the GPU is usable and the scenarios must run. - assert!(host_has_usable_gpu(Some("gfx1151"), true, WSL_DRIVER_READY)); + assert!(host_has_usable_gpu_with_mask( + Some("gfx1151"), + true, + WSL_DRIVER_READY, + None + )); // A native host is unaffected by the driver verdict. - assert!(host_has_usable_gpu( + assert!(host_has_usable_gpu_with_mask( Some("gfx942"), false, - "amdgpu_available" + "amdgpu_available", + None + )); + assert!(host_has_usable_gpu_with_mask( + Some("gfx942"), + false, + "", + None )); - assert!(host_has_usable_gpu(Some("gfx942"), false, "")); // No gfx target is still no GPU, WSL or not. - assert!(!host_has_usable_gpu(None, true, WSL_DRIVER_READY)); - assert!(!host_has_usable_gpu(None, false, "amdgpu_available")); + assert!(!host_has_usable_gpu_with_mask( + None, + true, + WSL_DRIVER_READY, + None + )); + assert!(!host_has_usable_gpu_with_mask( + None, + false, + "amdgpu_available", + None + )); + } + + #[test] + fn gpu_capability_honors_the_product_visibility_mask_precedence() { + use std::ffi::OsStr; + + let empty = OsStr::new(""); + let device_zero = OsStr::new("0"); + assert_eq!( + selected_visibility_mask(Some(empty), Some(device_zero)), + Some(empty), + "HIP_VISIBLE_DEVICES must take precedence even when explicitly empty" + ); + assert_eq!( + selected_visibility_mask(None, Some(empty)), + Some(empty), + "ROCR_VISIBLE_DEVICES applies when HIP_VISIBLE_DEVICES is unset" + ); + + assert!(!host_has_usable_gpu_with_mask( + Some("gfx1151"), + true, + WSL_DRIVER_READY, + Some(empty) + )); + assert!(!host_has_usable_gpu_with_mask( + Some("gfx942"), + false, + "amdgpu_available", + Some(empty) + )); + assert!(host_has_usable_gpu_with_mask( + Some("gfx1151"), + true, + WSL_DRIVER_READY, + Some(device_zero) + )); } #[test] diff --git a/tests/e2e-cucumber/src/expectation.rs b/tests/e2e-cucumber/src/expectation.rs index d1066af2a3..99b8c13454 100644 --- a/tests/e2e-cucumber/src/expectation.rs +++ b/tests/e2e-cucumber/src/expectation.rs @@ -1039,6 +1039,17 @@ flaky = true )); } + #[test] + fn gpu_report_disagreement_xfail_requires_a_reported_gfx_target() { + let m = Expectations::parse(include_str!("../expectations.toml")).unwrap(); + assert!(m.is_xfail( + "examine-both-forms-agree-on-gpu", + &cap("wsl-no-passthrough"), + "lemonade" + )); + assert!(!m.is_xfail("examine-both-forms-agree-on-gpu", &cap("wsl"), "lemonade")); + } + #[test] fn glob_matches_family() { assert!(glob_match("gfx94*", "gfx942")); diff --git a/xtask/src/workflow_contract.rs b/xtask/src/workflow_contract.rs index b33416385d..7b234d3511 100644 --- a/xtask/src/workflow_contract.rs +++ b/xtask/src/workflow_contract.rs @@ -192,6 +192,19 @@ mod tests { entries } + fn job_scalar<'a>(block: &'a str, key: &str) -> &'a str { + let marker = format!("{key}:"); + block + .lines() + .find_map(|line| { + (indent_of(line) == 4) + .then(|| line.trim().strip_prefix(&marker)) + .flatten() + .map(str::trim) + }) + .unwrap_or_else(|| panic!("job defines top-level scalar `{key}`")) + } + fn markdown_table_rows(text: &str, header: &str) -> Vec> { let mut lines = text.lines().skip_while(|line| *line != header); assert_eq!( @@ -310,7 +323,7 @@ mod tests { block.contains("proc_version=$(cat /proc/version 2>/dev/null || true)"), "{workflow} job {job} must read the canonical kernel-version signal" ); - let distro_signal = ["$", "{", "WSL_DISTRO_NAME:-", "}"].concat(); + let distro_signal = ["$", "{", "WSL_DISTRO_NAME+x", "}"].concat(); let canonical_union = format!( "if [ ! -e /dev/dxg ] && [ -z \"{distro_signal}\" ] && ! printf '%s\\n' \"$proc_version\" | grep -qiE 'microsoft|wsl'; then" ); @@ -342,6 +355,22 @@ mod tests { } } + #[test] + fn dispatchable_wsl_nightly_run_has_the_full_nightly_job_budget() { + let self_hosted = read_workflow("e2e-selfhosted.yml"); + let nightly = read_workflow("nightly.yml"); + let dispatch_timeout = job_scalar(job_block(&self_hosted, "e2e-wsl"), "timeout-minutes"); + let nightly_timeout = job_scalar(job_block(&nightly, "e2e-wsl-nightly"), "timeout-minutes"); + assert_eq!( + dispatch_timeout, "90", + "the 2400s large-model readiness budget needs the established 90-minute job cap for setup and the remaining suite" + ); + assert_eq!( + dispatch_timeout, nightly_timeout, + "e2e-wsl supports include_nightly, so its job timeout must cover the same 2400s scenario plus setup as e2e-wsl-nightly" + ); + } + #[test] fn hardware_testing_docs_cover_all_four_self_hosted_platforms() { let docs = std::fs::read_to_string(repo_root().join("docs/ci-hardware-testing.md")) From b0ecb1ac99619d0cc6f7c71cf9f9d06f49700a16 Mon Sep 17 00:00:00 2001 From: Michael Roy Date: Mon, 17 Aug 2026 12:30:41 -0700 Subject: [PATCH 10/11] test(ci): normalize workflow line endings Signed-off-by: Michael Roy --- xtask/src/workflow_contract.rs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/xtask/src/workflow_contract.rs b/xtask/src/workflow_contract.rs index 7b234d3511..dade4f5b6a 100644 --- a/xtask/src/workflow_contract.rs +++ b/xtask/src/workflow_contract.rs @@ -31,7 +31,9 @@ mod tests { fn read_workflow(name: &str) -> String { let p = repo_root().join(".github/workflows").join(name); - std::fs::read_to_string(&p).unwrap_or_else(|e| panic!("reading {}: {e}", p.display())) + std::fs::read_to_string(&p) + .unwrap_or_else(|e| panic!("reading {}: {e}", p.display())) + .replace("\r\n", "\n") } /// The self-hosted runner labels that must not appear in a `runs-on`. From 16e65807e5d8a1ca12137f9880b4da807fce3026 Mon Sep 17 00:00:00 2001 From: Michael Roy Date: Mon, 17 Aug 2026 21:46:48 -0700 Subject: [PATCH 11/11] fix(e2e): use dynamic timeout for Observe retry Signed-off-by: Michael Roy --- tests/e2e-cucumber/tests/e2e/dash_steps.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/e2e-cucumber/tests/e2e/dash_steps.rs b/tests/e2e-cucumber/tests/e2e/dash_steps.rs index bbbbda2455..911fcf7435 100644 --- a/tests/e2e-cucumber/tests/e2e/dash_steps.rs +++ b/tests/e2e-cucumber/tests/e2e/dash_steps.rs @@ -129,7 +129,7 @@ async fn open_observe_view(world: &mut E2eWorld) { // repeat it until the Observe tab is actually selected — the `●` marks the // active chip. The step then fails only if the dashboard never gets there, // not if it was slow to start. - tui.send_until("4", "● Observe", DEFAULT_TIMEOUT) + tui.send_until("4", "● Observe", default_timeout()) .await .unwrap_or_else(|e| panic!("failed to switch to the Observe tab: {e}")); }