diff --git a/.github/workflows/e2e-selfhosted.yml b/.github/workflows/e2e-selfhosted.yml index c32f1f9d9e..7da4333141 100644 --- a/.github/workflows/e2e-selfhosted.yml +++ b/.github/workflows/e2e-selfhosted.yml @@ -32,7 +32,7 @@ on: description: Which self-hosted runner(s) to target type: choice default: all - options: [all, app-dev-gpu, strix-ubuntu, strix-windows] + options: [all, app-dev-gpu, strix-ubuntu, strix-windows, strix-wsl] name_filter: description: "Scenario-name regex (cucumber --name); empty = full suite" type: string @@ -360,6 +360,12 @@ jobs: TMPDIR: /home/ubuntu/actions-runner/tmp PIP_CACHE_DIR: /home/ubuntu/actions-runner/pip-cache E2E_SERVE_TIMEOUT_SECS: "300" + # The three Strix lanes share one physical machine, and this workflow now + # runs a third of them, so a TUI frame that renders well inside the 30s + # default on an idle runner can miss it while a sibling lane loads a model. + # Raise the wait budget rather than let that read as a product failure; a + # genuine hang still fails, just later. + E2E_TUI_TIMEOUT_SECS: "90" # Match the MI300X dispatch path: opt into the platform-adaptive large-model # scenario only when the manual include_nightly input is enabled. E2E_INCLUDE_NIGHTLY: "${{ inputs.include_nightly && '1' || '' }}" @@ -504,6 +510,12 @@ jobs: ) continue-on-error: true env: + # The three Strix lanes share one physical machine, and this workflow now + # runs a third of them, so a TUI frame that renders well inside the 30s + # default on an idle runner can miss it while a sibling lane loads a model. + # Raise the wait budget rather than let that read as a product failure; a + # genuine hang still fails, just later. + E2E_TUI_TIMEOUT_SECS: "90" # Match the Linux Strix dispatch path: opt into the platform-adaptive # large-model scenario only when the manual input is enabled. E2E_INCLUDE_NIGHTLY: "${{ inputs.include_nightly && '1' || '' }}" @@ -636,6 +648,200 @@ jobs: name: e2e-gpu-strix-windows-report path: tests/e2e-cucumber/results/ + # WSL2 coverage on real hardware: the runner is an Ubuntu distro running under + # WSL2 on the Strix Halo Windows box, so this lane exercises the WSL host + # boundary AND whatever GPU access WSL exposes. Same suite as every other + # platform — scenarios the host cannot satisfy resolve to skip from the + # capability probe, so nothing is filtered out here. `wsl` in `runs-on` + # disambiguates it from the `native` Strix Linux runner. Non-blocking while + # GPU-on-WSL is proven out. + e2e-wsl: + name: E2E tests (Strix Halo, WSL2) + # A manual include_nightly dispatch runs the 2400s large-model readiness + # scenario, so match the dedicated nightly lane's cap and leave room for + # build, runtime pre-warm, the rest of the suite, and platform.json. + timeout-minutes: 90 + runs-on: [self-hosted, linux, strix-halo, wsl] + needs: [changes] + # See `e2e-gpu`: no build-and-test gate (cross-workflow); strix-wsl. Gated on + # `serve` like the sibling lanes rather than the broader `heavy`: the + # consolidated report gates on `serve` too, so a heavy-but-not-serve change + # would otherwise run this lane and then discard its artifact unreported. + if: >- + always() + && needs.changes.result == 'success' + && ( + (github.event_name != 'workflow_dispatch' + && needs.changes.outputs.serve == 'true') + || (github.event_name == 'workflow_dispatch' + && (inputs.platform == 'all' || inputs.platform == 'strix-wsl')) + ) + continue-on-error: true + env: + E2E_SERVE_TIMEOUT_SECS: "300" + # The three Strix lanes share one physical machine, and this workflow now + # runs a third of them, so a TUI frame that renders well inside the 30s + # default on an idle runner can miss it while a sibling lane loads a model. + # Raise the wait budget rather than let that read as a product failure; a + # genuine hang still fails, just later. + E2E_TUI_TIMEOUT_SECS: "90" + # Match the other hardware lanes: opt into the platform-adaptive + # large-model scenario only when the manual include_nightly input is on. + E2E_INCLUDE_NIGHTLY: "${{ inputs.include_nightly && '1' || '' }}" + # Heavy @merge-queue serves run only in the merge queue; see e2e-gpu. + E2E_MERGE_QUEUE: "${{ github.event_name == 'merge_group' && '1' || '' }}" + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + + # Fail fast and loudly if this job ever lands on a native Linux runner: + # every WSL-tagged scenario would silently resolve to skip instead. + - name: Verify the host really is WSL2 + run: | + proc_version=$(cat /proc/version 2>/dev/null || true) + if [ ! -e /dev/dxg ] \ + && [ -z "${WSL_DISTRO_NAME+x}" ] \ + && ! printf '%s\n' "$proc_version" | grep -qiE 'microsoft|wsl'; then + echo "::error::runner is not a WSL host: $(uname -r)" + exit 1 + fi + printf 'WSL kernel: %s\n' "$(uname -r)" + + # The native Strix runners come pre-provisioned; a WSL distro often does + # not. Install the native build deps only when something is missing, and + # only if passwordless sudo is available — otherwise say what is missing + # instead of hanging on a password prompt. + - name: Ensure native build deps + run: | + missing="" + command -v pkg-config >/dev/null 2>&1 || missing="$missing pkg-config" + command -v cc >/dev/null 2>&1 || missing="$missing build-essential" + pkg-config --exists libcap 2>/dev/null || missing="$missing libcap-dev" + if [ -z "$missing" ]; then + echo "native build deps present" + elif sudo -n true 2>/dev/null; then + echo "installing:$missing" + sudo -n apt-get update + # shellcheck disable=SC2086 + sudo -n apt-get install -y $missing + else + echo "::error::missing native build deps:$missing (no passwordless sudo to install them)" + exit 1 + fi + + - name: Reclaim GPU from stray E2E processes + run: | + # Reclaim from any serve leaked by a killed/timed-out prior run + # (see e2e-gpu). Scoped to e2e leftovers only. + pkill -f '/tmp/rocm-e2e.*llama-server' 2>/dev/null || true + pkill -f '/tmp/rocm-e2e.*vllm serve' 2>/dev/null || true + pkill -f 'e2e-shared.*llama-server' 2>/dev/null || true + pkill -f '__engine-serve-http.*rocm-e2e' 2>/dev/null || true + rm -rf /tmp/rocm-e2e-* 2>/dev/null || true + echo "reclaimed" + + # GPU preflight, bounded like the native lanes — but ADVISORY here. GPU + # access under WSL is exactly what this lane is proving out, so an absent + # rocm-smi is a reported condition, not a job failure: the capability probe + # then resolves @requires-gpu scenarios to skip and the rest still runs. A + # GPU that IS present but stays held by a leftover serve still fails, since + # that would corrupt the serve scenarios' results. + - name: GPU preflight (advisory bounded wait) + run: | + MIN_FREE_GIB="${GPU_PREFLIGHT_MIN_FREE_GIB:-8}" + CEILING_SECS="${GPU_PREFLIGHT_CEILING_SECS:-90}" + if ! command -v rocm-smi >/dev/null 2>&1; then + echo "::warning::rocm-smi not found in this WSL distro — GPU scenarios will resolve to skip" + exit 0 + fi + min_free=$(( MIN_FREE_GIB * 1024 * 1024 * 1024 )) + deadline=$(( SECONDS + CEILING_SECS )) + saw_vram=0 + while [ "$SECONDS" -lt "$deadline" ]; do + out=$(timeout 15 rocm-smi --showmeminfo vram 2>/dev/null) || { sleep 5; continue; } + total=$(printf '%s\n' "$out" | grep -i 'VRAM Total Memory' | sed 's/.*: *//' | grep -oE '[0-9]+' | tail -1) + used=$(printf '%s\n' "$out" | grep -i 'VRAM Total Used Memory' | sed 's/.*: *//' | grep -oE '[0-9]+' | tail -1) + if [ -z "$total" ] || [ -z "$used" ]; then + sleep 5; continue + fi + saw_vram=1 + free=$(( total - used )) + if [ "$free" -ge "$min_free" ]; then + echo "GPU ready: $(( free / 1024 / 1024 / 1024 )) GiB free (>= ${MIN_FREE_GIB} GiB)." + exit 0 + fi + echo "waiting: $(( free / 1024 / 1024 / 1024 )) GiB free (< ${MIN_FREE_GIB} GiB)…" + sleep 5 + done + if [ "$saw_vram" -eq 0 ]; then + echo "::warning::rocm-smi reported no VRAM figures under WSL — GPU scenarios will resolve to skip" + exit 0 + fi + echo "::error::GPU preflight failed after ${CEILING_SECS}s: VRAM never dropped below the floor — a serve is likely still holding the GPU" + exit 1 + + # Bootstrap rustup with --no-modify-path so it never writes $HOME/.profile + # (setup-rust-toolchain doesn't expose that flag). rust-toolchain.toml pins + # the exact toolchain, installed on first cargo use. Idempotent. + - name: Ensure Rust toolchain + run: | + if ! command -v cargo >/dev/null 2>&1 && [ ! -x "$HOME/.cargo/bin/cargo" ]; then + curl --proto '=https' --tlsv1.2 -fsSL https://sh.rustup.rs \ + | sh -s -- -y --no-modify-path --default-toolchain none + fi + echo "$HOME/.cargo/bin" >> "$GITHUB_PATH" + + - name: Run E2E tests on Strix Halo WSL2 + run: | + export CARGO_TARGET_DIR="$RUNNER_WORKSPACE/e2e-target" + export E2E_SHARED_CACHE_DIR="$RUNNER_WORKSPACE/e2e-shared" + # Share ONE installed managed runtime across serve/chat scenarios and + # PRE-WARM it in place (mirrors e2e-gpu-strix-ubuntu). Required for + # correctness, not speed: `install sdk` bakes ABSOLUTE paths into the + # runtime manifest, so installing into a per-scenario temp dir leaves + # every later serve pointing at a deleted install root. + prewarm="$RUNNER_WORKSPACE/e2e-prewarm" + export E2E_SHARED_RUNTIMES_DIR="$prewarm/data/runtimes" + + # Build the rocm binary once; reuse for pre-warm + suite so xtask + # doesn't rebuild. + cargo build --release -p rocm + export ROCM_CLI_BINARY="$CARGO_TARGET_DIR/release/rocm" + + # Pre-warm once, serially, in place (no mv/symlink). Skipped once the + # tree is populated (persists across runs on RUNNER_WORKSPACE). + if [ ! -d "$E2E_SHARED_RUNTIMES_DIR/registry" ]; then + echo "pre-warming shared runtime (first run on this runner)…" + mkdir -p "$prewarm"/{data,config,cache} + ROCM_CLI_CONFIG_DIR="$prewarm/config" \ + ROCM_CLI_DATA_DIR="$prewarm/data" \ + ROCM_CLI_CACHE_DIR="$prewarm/cache" \ + HF_HOME="$E2E_SHARED_CACHE_DIR/huggingface" \ + "$ROCM_CLI_BINARY" install sdk + if [ -d "$E2E_SHARED_RUNTIMES_DIR/registry" ]; then + echo "shared runtime pre-warmed at $E2E_SHARED_RUNTIMES_DIR" + else + echo "pre-warm did not produce a runtimes registry; scenarios will install their own" >&2 + fi + else + echo "shared runtime already present at $E2E_SHARED_RUNTIMES_DIR — skipping pre-warm" + fi + + # Optional scenario-name filter for a scoped manual dispatch. + NAME_FILTER="${{ github.event.inputs.name_filter }}" + if [ -n "$NAME_FILTER" ]; then + echo "name filter active: $NAME_FILTER" + cargo xtask e2e -- --name "$NAME_FILTER" + else + cargo xtask e2e + fi + + - name: Upload E2E report + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: e2e-gpu-strix-wsl-report + path: tests/e2e-cucumber/results/ + # Consolidate this workflow's self-hosted platform reports into one GPU-side # cross-platform grid (Summary + merged HTML). Distinct name from ci.yml's # required `E2E consolidated report` so it does NOT collide with that required @@ -651,6 +857,7 @@ jobs: - e2e-gpu - e2e-gpu-strix-ubuntu - e2e-gpu-strix-windows + - e2e-wsl # Gate on `serve`: every lane this report consolidates (the GPU jobs) is now # serve-gated, so a serve-only change runs them and their report must still be # produced. On dispatch `serve` is unset, so also run when the trigger was diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index 7cca48b490..1c000ba540 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -435,6 +435,7 @@ jobs: PIP_CACHE_DIR: /home/ubuntu/actions-runner/pip-cache E2E_INCLUDE_NIGHTLY: "1" E2E_SERVE_TIMEOUT_SECS: "300" + E2E_TUI_TIMEOUT_SECS: "90" steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 @@ -530,6 +531,7 @@ jobs: env: E2E_INCLUDE_NIGHTLY: "1" E2E_SERVE_TIMEOUT_SECS: "300" + E2E_TUI_TIMEOUT_SECS: "90" steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 @@ -625,11 +627,171 @@ jobs: name: e2e-gpu-strix-windows-report path: tests/e2e-cucumber/results/ + # WSL2 counterpart to e2e-gpu-nightly-strix: the same Strix Halo box again, but + # reached through an Ubuntu distro under WSL2, so this lane exercises the WSL + # host boundary AND whatever GPU access WSL exposes. Mirrors the per-PR + # `e2e-wsl` lane in e2e-selfhosted.yml, opting into the same @nightly + # large-model scenario as the other nightly lanes. `wsl` in `runs-on` + # disambiguates it from the `native` Strix Linux runner; the native lane's + # /home/ubuntu/actions-runner paths deliberately are NOT reused, since they do + # not exist in the WSL distro. + e2e-wsl-nightly: + name: E2E tests (Strix Halo, WSL2, incl. nightly-only) + timeout-minutes: 90 + runs-on: [self-hosted, linux, strix-halo, wsl] + continue-on-error: true + env: + E2E_INCLUDE_NIGHTLY: "1" + E2E_SERVE_TIMEOUT_SECS: "300" + # The Strix lanes share one physical machine, so a TUI frame that renders + # well inside the 30s default on an idle runner can miss it while a sibling + # lane loads a model. Raise the wait budget rather than let that read as a + # product failure; a genuine hang still fails, just later. + E2E_TUI_TIMEOUT_SECS: "90" + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + + # Fail fast and loudly if this job ever lands on a native Linux runner: + # every WSL-tagged scenario would silently resolve to skip instead. + - name: Verify the host really is WSL2 + run: | + proc_version=$(cat /proc/version 2>/dev/null || true) + if [ ! -e /dev/dxg ] \ + && [ -z "${WSL_DISTRO_NAME+x}" ] \ + && ! printf '%s\n' "$proc_version" | grep -qiE 'microsoft|wsl'; then + echo "::error::runner is not a WSL host: $(uname -r)" + exit 1 + fi + printf 'WSL kernel: %s\n' "$(uname -r)" + + # The native Strix runners come pre-provisioned; a WSL distro often does + # not. Install the native build deps only when something is missing, and + # only if passwordless sudo is available — otherwise say what is missing + # instead of hanging on a password prompt. + - name: Ensure native build deps + run: | + missing="" + command -v pkg-config >/dev/null 2>&1 || missing="$missing pkg-config" + command -v cc >/dev/null 2>&1 || missing="$missing build-essential" + pkg-config --exists libcap 2>/dev/null || missing="$missing libcap-dev" + if [ -z "$missing" ]; then + echo "native build deps present" + elif sudo -n true 2>/dev/null; then + echo "installing:$missing" + sudo -n apt-get update + # shellcheck disable=SC2086 + sudo -n apt-get install -y $missing + else + echo "::error::missing native build deps:$missing (no passwordless sudo to install them)" + exit 1 + fi + + - name: Reclaim GPU from stray E2E processes + run: | + pkill -f '/tmp/rocm-e2e.*llama-server' 2>/dev/null || true + pkill -f '/tmp/rocm-e2e.*vllm serve' 2>/dev/null || true + pkill -f 'e2e-shared.*llama-server' 2>/dev/null || true + pkill -f '__engine-serve-http.*rocm-e2e' 2>/dev/null || true + rm -rf /tmp/rocm-e2e-* 2>/dev/null || true + echo "reclaimed" + + # GPU preflight, bounded like the native lanes — but ADVISORY here. GPU + # access under WSL is exactly what this lane is proving out, so an absent + # rocm-smi is a reported condition, not a job failure: the capability probe + # then resolves @requires-gpu scenarios to skip and the rest still runs. A + # GPU that IS present but stays held by a leftover serve still fails, since + # that would corrupt the serve scenarios' results. + - name: GPU preflight (advisory bounded wait) + run: | + MIN_FREE_GIB="${GPU_PREFLIGHT_MIN_FREE_GIB:-8}" + CEILING_SECS="${GPU_PREFLIGHT_CEILING_SECS:-90}" + if ! command -v rocm-smi >/dev/null 2>&1; then + echo "::warning::rocm-smi not found in this WSL distro — GPU scenarios will resolve to skip" + exit 0 + fi + min_free=$(( MIN_FREE_GIB * 1024 * 1024 * 1024 )) + deadline=$(( SECONDS + CEILING_SECS )) + saw_vram=0 + while [ "$SECONDS" -lt "$deadline" ]; do + out=$(timeout 15 rocm-smi --showmeminfo vram 2>/dev/null) || { sleep 5; continue; } + total=$(printf '%s\n' "$out" | grep -i 'VRAM Total Memory' | sed 's/.*: *//' | grep -oE '[0-9]+' | tail -1) + used=$(printf '%s\n' "$out" | grep -i 'VRAM Total Used Memory' | sed 's/.*: *//' | grep -oE '[0-9]+' | tail -1) + if [ -z "$total" ] || [ -z "$used" ]; then + sleep 5; continue + fi + saw_vram=1 + free=$(( total - used )) + if [ "$free" -ge "$min_free" ]; then + echo "GPU ready: $(( free / 1024 / 1024 / 1024 )) GiB free (>= ${MIN_FREE_GIB} GiB)." + exit 0 + fi + echo "waiting: $(( free / 1024 / 1024 / 1024 )) GiB free (< ${MIN_FREE_GIB} GiB)…" + sleep 5 + done + if [ "$saw_vram" -eq 0 ]; then + echo "::warning::rocm-smi reported no VRAM figures under WSL — GPU scenarios will resolve to skip" + exit 0 + fi + echo "::error::GPU preflight failed after ${CEILING_SECS}s: VRAM never dropped below the floor — a serve is likely still holding the GPU" + exit 1 + + # Bootstrap rustup with --no-modify-path so it never writes $HOME/.profile + # (setup-rust-toolchain doesn't expose that flag). rust-toolchain.toml pins + # the exact toolchain, installed on first cargo use. Idempotent. + - name: Ensure Rust toolchain + run: | + if ! command -v cargo >/dev/null 2>&1 && [ ! -x "$HOME/.cargo/bin/cargo" ]; then + curl --proto '=https' --tlsv1.2 -fsSL https://sh.rustup.rs \ + | sh -s -- -y --no-modify-path --default-toolchain none + fi + echo "$HOME/.cargo/bin" >> "$GITHUB_PATH" + + - name: Run E2E tests on Strix Halo WSL2 (incl. nightly-only) + run: | + export CARGO_TARGET_DIR="$RUNNER_WORKSPACE/e2e-target" + export E2E_SHARED_CACHE_DIR="$RUNNER_WORKSPACE/e2e-shared" + # Share ONE installed managed runtime across serve/chat scenarios and + # PRE-WARM it in place (mirrors e2e-gpu-nightly-strix). Required for + # correctness, not speed: `install sdk` bakes ABSOLUTE paths into the + # runtime manifest, so installing into a per-scenario temp dir leaves + # every later serve pointing at a deleted install root. + prewarm="$RUNNER_WORKSPACE/e2e-prewarm" + export E2E_SHARED_RUNTIMES_DIR="$prewarm/data/runtimes" + + cargo build --release -p rocm + export ROCM_CLI_BINARY="$CARGO_TARGET_DIR/release/rocm" + + if [ ! -d "$E2E_SHARED_RUNTIMES_DIR/registry" ]; then + echo "pre-warming shared runtime (first run on this runner)…" + mkdir -p "$prewarm"/{data,config,cache} + ROCM_CLI_CONFIG_DIR="$prewarm/config" \ + ROCM_CLI_DATA_DIR="$prewarm/data" \ + ROCM_CLI_CACHE_DIR="$prewarm/cache" \ + HF_HOME="$E2E_SHARED_CACHE_DIR/huggingface" \ + "$ROCM_CLI_BINARY" install sdk + if [ -d "$E2E_SHARED_RUNTIMES_DIR/registry" ]; then + echo "shared runtime pre-warmed at $E2E_SHARED_RUNTIMES_DIR" + else + echo "pre-warm did not produce a runtimes registry; scenarios will install their own" >&2 + fi + else + echo "shared runtime already present at $E2E_SHARED_RUNTIMES_DIR — skipping pre-warm" + fi + + cargo xtask e2e + + - name: Upload E2E report + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: e2e-gpu-strix-wsl-report + path: tests/e2e-cucumber/results/ + # The nightly lanes each uploaded a report nobody joined up, so reading a - # nightly meant opening three artifacts and comparing them by hand — on the + # nightly meant opening every artifact and comparing them by hand — on the # run that covers the most scenarios, since only nightly sets # E2E_INCLUDE_NIGHTLY. Same consolidation the per-PR self-hosted workflow - # already does (e2e-selfhosted.yml), against the three nightly artifacts. + # already does (e2e-selfhosted.yml), against the nightly artifacts. e2e-report-nightly: name: E2E consolidated report (nightly) runs-on: ubuntu-latest @@ -643,6 +805,7 @@ jobs: - e2e-gpu-nightly - e2e-gpu-nightly-strix - e2e-gpu-nightly-strix-windows + - e2e-wsl-nightly # Unconditional `always()`, unlike the per-PR workflow: there is no paths # filter to consult on a schedule, and the lanes are `continue-on-error`, so # a failed lane must still be reported. A nightly that broke is precisely @@ -653,7 +816,7 @@ jobs: - uses: actions-rust-lang/setup-rust-toolchain@166cdcfd11aee3cb47222f9ddb555ce30ddb9659 # v1.17.0 - # `*-report` matches the three artifacts above, and will pick up any + # `*-report` matches the artifacts above, and will pick up any # nightly platform added later without a change here — provided it is # named canonically (see the rename note on the upload steps). # Layout note (same as the per-PR job): download-artifact@v8 uses a diff --git a/crates/e2e-report/src/lib.rs b/crates/e2e-report/src/lib.rs index 6116b06c82..69a58e9ad2 100644 --- a/crates/e2e-report/src/lib.rs +++ b/crates/e2e-report/src/lib.rs @@ -414,6 +414,12 @@ fn parse_descriptor(name: &str) -> Descriptor { "gpu" => ("MI300X", "Linux"), "gpu-strix-ubuntu" => ("Strix Halo", "Ubuntu"), "gpu-strix-windows" => ("Strix Halo", "Windows"), + // Same silicon again, third host boundary: an Ubuntu distro under WSL2 on + // the Windows box. It is neither the native Ubuntu nor the Windows lane — + // GPU access goes through the WSL passthrough — so it needs its own OS + // value, not a reuse of "Ubuntu", or the grid would show two rows that + // claim to be the same host and disagree. + "gpu-strix-wsl" => ("Strix Halo", "WSL2"), // `e2e-unknown-report`: a report whose platform.json sidecar was missing // or unrecognized (e.g. a GPU run that errored before writing it). The OS // is genuinely unknown here — a Windows GPU run that erupted early must NOT @@ -2012,6 +2018,9 @@ mod tests { ("e2e-gpu-report", "MI300X", "Linux"), ("e2e-gpu-strix-ubuntu-report", "Strix Halo", "Ubuntu"), ("e2e-gpu-strix-windows-report", "Strix Halo", "Windows"), + // Must not fall through to `fallback_descriptor`, which would render + // "Gpu Strix Wsl" on Linux — a WSL2 host reported as native Linux. + ("e2e-gpu-strix-wsl-report", "Strix Halo", "WSL2"), ] { let d = parse_descriptor(name); assert_eq!( diff --git a/docs/ci-hardware-testing.md b/docs/ci-hardware-testing.md index 471deed027..8124da72bd 100644 --- a/docs/ci-hardware-testing.md +++ b/docs/ci-hardware-testing.md @@ -4,7 +4,7 @@ Copyright © Advanced Micro Devices, Inc., or its affiliates. SPDX-License-Identifier: MIT --> -# CI hardware (GPU) testing +# CI hardware (GPU / WSL) testing The hosted CI (`ubuntu-latest`, `windows-latest`) builds and unit-tests every shipping target natively, but GitHub-hosted runners have no AMD GPU. A @@ -35,32 +35,66 @@ separate tier flag or tag filter to maintain. | `e2e-gpu` | `e2e-selfhosted.yml` | MI300X (AMD Instinct, bare-metal Linux) | self-hosted `[self-hosted, linux, amd-gpu]` | | `e2e-gpu-strix-ubuntu` | `e2e-selfhosted.yml` | Strix Halo (gfx1151) on Ubuntu | self-hosted `[self-hosted, linux, strix-halo, native]` | | `e2e-gpu-strix-windows` | `e2e-selfhosted.yml` | Strix Halo (gfx1151) on native Windows 11 | self-hosted `[self-hosted, windows, strix-halo, native]` | +| `e2e-wsl` | `e2e-selfhosted.yml` | Strix Halo (gfx1151) on Ubuntu under WSL2 | self-hosted `[self-hosted, linux, strix-halo, wsl]` | The Strix Halo lanes pin the extra `native` label because two Linux runners share the `strix-halo` label (a native host and a WSL host) and the jobs' -hardcoded `/home/ubuntu/actions-runner` paths exist only on the native one. +hardcoded `/home/ubuntu/actions-runner` paths exist only on the native one. The +WSL lane pins `wsl` for the same reason, from the other side. `e2e` is the blocking, GitHub-hosted mock job: `@requires-gpu` scenarios resolve to skip here, and known bugs resolve to xfail from `expectations.toml`. It is a required check and must stay green. -The three GPU jobs (`e2e-gpu`, `e2e-gpu-strix-ubuntu`, `e2e-gpu-strix-windows`) -run on dedicated self-hosted runners with a real AMD GPU attached, so they -exercise host/GPU detection, engine `detect`/`capabilities`, and live serving -scenarios that the mock job cannot. +The four self-hosted jobs (`e2e-gpu`, `e2e-gpu-strix-ubuntu`, +`e2e-gpu-strix-windows`, and `e2e-wsl`) run on AMD GPU systems, so they exercise +host/GPU detection, engine `detect`/`capabilities`, and live serving scenarios +that the mock job cannot. GPU availability is advisory in the WSL lane, as +described below. + +`e2e-wsl` runs on an Ubuntu distro hosted in WSL2 on the Strix Halo Windows box +and mirrors the sibling Linux lane step for step: stray-serve reclaim, GPU +preflight, toolchain bootstrap, shared-runtime pre-warm, then the full suite +with no hand filtering. It covers WSL host detection, the Windows-to-WSL +execution boundary, and whatever GPU access WSL exposes on that machine. The +GPU preflight is advisory here precisely because GPU-on-WSL is what the lane is +proving out: where it is unavailable the capability probe resolves those +scenarios to not-applicable and the rest of the suite still runs. Scenarios the +product deliberately routes around on WSL carry `@requires-bare-metal`; the one +scenario whose premise *is* a WSL host carries `@requires-wsl`, and this is the +only lane that runs it. + +### What the WSL distro needs + +The distro needs `pkg-config`, `build-essential` and `libcap-dev` to build the +workspace; the lane installs them itself where it has passwordless sudo, and +otherwise fails with the list of what is missing rather than hanging on a +password prompt. + +GPU coverage additionally needs ROCm's WSL passthrough to be complete — +`/dev/dxg` and dxcore alone are not enough, `librocdxg.so` and its ldconfig +entry must be present too. `rocm examine` reports the verdict as +`driver_status: wsl_rocdxg_ready`; anything else (`wsl_rocdxg_missing`, +`wsl_gpu_plumbing_missing`) means the runtime cannot reach the device even +though `detected_gfx_target` still names it, because that target is read from +the Windows-side driver. The capability probe keys `@requires-gpu` on the +driver verdict rather than the target for exactly this reason, so a distro +without the passthrough runs the non-GPU suite and reports the GPU scenarios as +not applicable, instead of failing them on a premise the host cannot meet. Each workflow has its own consolidated report job. `ci.yml`'s `e2e-report` -covers the mock platform; `e2e-selfhosted.yml`'s `e2e-report` covers the three -GPU platforms; `nightly.yml`'s `e2e-report-nightly` covers the same three -platforms with the `@nightly` scenarios included. Each joins its platforms' +covers the mock platform; +`e2e-selfhosted.yml`'s `e2e-report` covers the four GPU platforms; +`nightly.yml`'s `e2e-report-nightly` covers the same four platforms with the +`@nightly` scenarios included. Each joins its platforms' reports — including partial or failed runs — by scenario id into one HTML report and GitHub step summary. The lane artifacts are named canonically (`e2e-report`, `e2e-gpu-report`, -`e2e-gpu-strix-ubuntu-report`, `e2e-gpu-strix-windows-report`) in every workflow, -because the report derives each platform's name and OS from the artifact name. -An unrecognised name renders as a guessed platform on Linux, which would report -a Windows lane as Linux; `xtask`'s +`e2e-gpu-strix-ubuntu-report`, `e2e-gpu-strix-windows-report`, +`e2e-gpu-strix-wsl-report`) in every workflow, because the report derives each +platform's name and OS from the artifact name. An unrecognised name renders as a +guessed platform on Linux, which would report a Windows lane as Linux; `xtask`'s `every_uploaded_e2e_artifact_has_a_name_the_report_can_label` guards against it. ## Triggers @@ -84,11 +118,11 @@ authoritative pre-merge build gate. They can also be triggered manually via `e2e-selfhosted.yml`'s `workflow_dispatch`, independent of the `serve` gate, with these inputs: -- `platform` (choice: `all`, `app-dev-gpu`, `strix-ubuntu`, `strix-windows`) — - which self-hosted job(s) to run. `app-dev-gpu` maps to `e2e-gpu`, - `strix-ubuntu` to `e2e-gpu-strix-ubuntu`, and `strix-windows` to - `e2e-gpu-strix-windows`. (The mock lane has its own `platform` input on - `ci.yml`; it is not part of this workflow.) +- `platform` (choice: `all`, `app-dev-gpu`, `strix-ubuntu`, `strix-windows`, + `strix-wsl`) — which self-hosted job(s) to run. `app-dev-gpu` maps to + `e2e-gpu`, `strix-ubuntu` to `e2e-gpu-strix-ubuntu`, `strix-windows` to + `e2e-gpu-strix-windows`, and `strix-wsl` to `e2e-wsl`. (The mock lane has its + own `platform` input on `ci.yml`; it is not part of this workflow.) - `name_filter` (string) — a scenario-name regex forwarded to the cucumber harness (`cargo xtask e2e -- --name `) so a dispatch can run a single scenario instead of the full suite. Empty runs everything applicable @@ -105,11 +139,21 @@ gh workflow run e2e-selfhosted.yml --ref -f platform=app-dev-gpu ## Blocking vs. non-blocking -The three hardware jobs — `e2e-gpu`, `e2e-gpu-strix-ubuntu`, and -`e2e-gpu-strix-windows` — all run with `continue-on-error: true`, so a +The four self-hosted jobs — `e2e-gpu`, `e2e-gpu-strix-ubuntu`, +`e2e-gpu-strix-windows`, and `e2e-wsl` — all run with `continue-on-error: true`, so a hardware failure that RUNS never gates a PR merge. Their results still surface in the self-hosted consolidated report for visibility. +### Timeouts on the shared Strix box + +Three of those lanes — the two native Strix ones and `e2e-wsl` — run on the same +physical machine and can be in flight together, so a wait that is comfortable on +an idle runner can expire while a sibling lane loads a model. Two budgets are +raised there rather than letting contention read as a product failure: +`E2E_SERVE_TIMEOUT_SECS` for serve readiness, and `E2E_TUI_TIMEOUT_SECS` for the +PTY-driven dashboard waits. Both only lengthen how long a wait may take; a +genuine hang still fails, just later. + **Required-check caveat.** These three job names (plus, historically, a consolidated-report name) are still in `main`'s required-status-check list. `continue-on-error` neutralizes a job that ran and failed, but a required check diff --git a/tests/e2e-cucumber/README.md b/tests/e2e-cucumber/README.md index 4519b61e68..8b4b403672 100644 --- a/tests/e2e-cucumber/README.md +++ b/tests/e2e-cucumber/README.md @@ -98,7 +98,9 @@ Scenarios carry stable-id and capability tags: | Tag | Meaning | |---|---| | `@id:` | Stable scenario id. Keys the expectation matrix and the report grid; every scenario has one. | -| `@requires-gpu` | Needs a real AMD GPU. Resolves to **skip** (n/a) on a host with none (e.g. the mock job). | +| `@requires-gpu` | Needs a usable AMD GPU. Resolves to **skip** (n/a) on a host with none — the mock job, or a WSL host whose ROCm passthrough is incomplete (`driver_status` other than `wsl_rocdxg_ready`), where the gfx target is reported but unreachable. | +| `@requires-bare-metal` | Premise is a host running the in-tree amdgpu driver, so it does not hold under WSL2. Resolves to **skip** there. `@requires-os:linux` cannot express this: WSL2 reports an `os_family` of `linux`. | +| `@requires-wsl` | The inverse: premise **is** a WSL2 host. Resolves to **skip** on native Linux, native Windows, and everything else. | | `@requires-engine:` | Pins the serve engine. Resolves to skip where that engine can't start (e.g. vLLM on a lemonade-only Strix host). | | `@requires-os:` | Premise is OS-specific; skip on other OSes. | | `@serve-timeout:` | Lengthen the serve-readiness wait for a genuinely slow serve (e.g. a large model). | @@ -133,16 +135,20 @@ self-hosted runner can never stall `ci.yml`'s merge-required checks: | `e2e-gpu` | `e2e-selfhosted.yml` | MI300X (self-hosted) | no | | `e2e-gpu-strix-ubuntu` | `e2e-selfhosted.yml` | Strix Halo / Ubuntu (self-hosted) | no | | `e2e-gpu-strix-windows` | `e2e-selfhosted.yml` | Strix Halo / Windows (self-hosted) | no | +| `e2e-wsl` | `e2e-selfhosted.yml` | Strix Halo / Ubuntu under WSL2 (self-hosted) | no | The blocking mock job passes when every applicable scenario is pass-or-xfail with no XPASS or unexpected failure; the GPU jobs are non-blocking. Each workflow consolidates its own platforms: `ci.yml`'s `e2e-report` the mock platform, `e2e-selfhosted.yml`'s `e2e-report` the self-hosted platforms, and -`nightly.yml`'s `e2e-report-nightly` the nightly lanes below. - -The nightly workflow runs three non-blocking jobs — the existing MI300X job and -new Strix Halo jobs on Ubuntu and Windows — with `E2E_INCLUDE_NIGHTLY=1`, then -consolidates them into the same cross-platform grid. The +`nightly.yml`'s `e2e-report-nightly` the nightly lanes below. `e2e-wsl` runs the suite in an +Ubuntu distro under WSL2 on the Strix Halo box, so it is the only lane that +exercises `@requires-wsl` scenarios; `@requires-bare-metal` scenarios resolve to +skip there. + +The nightly workflow runs four non-blocking jobs — MI300X plus Strix Halo on +Ubuntu, Windows, and WSL2 — with `E2E_INCLUDE_NIGHTLY=1`, then consolidates them +into the same cross-platform grid. The shared large-model scenario serves `Qwen/Qwen3.6-27B` through vLLM on MI300X and the hardware-verified `unsloth/Qwen3.6-35B-A3B-GGUF:UD-Q4_K_XL` checkpoint through Lemonade on Strix Halo. diff --git a/tests/e2e-cucumber/expectations.toml b/tests/e2e-cucumber/expectations.toml index da2de69982..c86821cdb3 100644 --- a/tests/e2e-cucumber/expectations.toml +++ b/tests/e2e-cucumber/expectations.toml @@ -21,7 +21,7 @@ # # Condition predicate grammar (deliberately tiny): # [[]] one table per condition (arrays = OR) -# when = { effective_engine = "vllm", os = "linux", is_wsl = false } +# when = { effective_engine = "vllm", os = "linux", has_amd_gpu = true } # all present keys must match (AND) # bug = "EAI-NNNN" ticket reference (shown in the report) # reason = "..." human explanation (shown in the report) @@ -113,7 +113,7 @@ serve_timeout_secs = 90 # scenario now also runs on the no-GPU mock lane (backed by MockServer, not a # real lemonade serve), where EAI-7423 does not apply and it must expect-pass. [["chat-tool-definitions-accepted"]] -when = { effective_engine = "lemonade", os = "linux", therock_family = "gfx*" } +when = { effective_engine = "lemonade", os = "linux", therock_family = "gfx*", has_amd_gpu = true } bug = "EAI-7423" reason = "Lemonade managed serve reaches ready then shuts down immediately, so the chat request never reaches a live model." serve_timeout_secs = 90 @@ -130,7 +130,7 @@ flaky = true # a REAL GPU host: the scenario also runs on the no-GPU mock lane (MockServer, not # a real lemonade serve), where EAI-7423 does not apply and it must expect-pass. [["chat-end-to-end-local-model"]] -when = { effective_engine = "lemonade", os = "linux", therock_family = "gfx*" } +when = { effective_engine = "lemonade", os = "linux", therock_family = "gfx*", has_amd_gpu = true } bug = "EAI-7423" reason = "Lemonade managed serve reaches ready then shuts down immediately, so the chat request never reaches a live model." serve_timeout_secs = 90 @@ -157,8 +157,24 @@ reason = "Managed runtime path nests recursively (runtimes/wheel/...) on re-prov # MockServer-backed bench scenarios run unxfail'd on every lane, including this # one, and they are what pin the request path and the failure reporting. [["bench-load-real-serve"]] -when = { effective_engine = "lemonade", os = "linux", therock_family = "gfx*" } +when = { effective_engine = "lemonade", os = "linux", therock_family = "gfx*", has_amd_gpu = true } bug = "EAI-7423" reason = "Lemonade managed serve reaches ready then shuts down immediately, so the benchmark never reaches a live model." serve_timeout_secs = 90 flaky = true + +# --- EAI-7998: `examine --json` disagrees with the human report about the GPU. +# On WSL2 the JSON contradicts ITSELF: `has_amd_gpu: false` and `gpus: []` +# alongside a summary carrying `detected_gfx_target: gfx1151`. The driver/GPU +# probes are routed out on WSL because they read KFD and DRM, neither of which +# exists there, while the summary reads the target from the Windows-side driver. +# +# Scoped to WSL hosts whose human report actually names a gfx target: without +# one both forms correctly report no GPU, so xfail would turn agreement into a +# stale XPASS. The ticket also records the same disagreement on MI300X, +# second-hand from the comment in capability.rs; if that is confirmed the row +# should widen rather than gain a sibling. Remove it when the two forms agree. --- +[["examine-both-forms-agree-on-gpu"]] +when = { is_wsl = true, therock_family = "gfx*" } +bug = "EAI-7998" +reason = "`examine --json` reports has_amd_gpu:false on a WSL2 host whose own summary names a gfx target." diff --git a/tests/e2e-cucumber/features/examine.feature b/tests/e2e-cucumber/features/examine.feature index 4ccece7f5a..9410d4fd1b 100644 --- a/tests/e2e-cucumber/features/examine.feature +++ b/tests/e2e-cucumber/features/examine.feature @@ -101,3 +101,13 @@ Feature: GPU detection and system inspection When the user inspects the system without probing frameworks Then the inspection reports that it skipped the frameworks And it still states a verdict for this machine + + # The inverse of `@requires-bare-metal`: WSL2 reports an os_family of `linux`, + # so `@requires-os:linux` cannot express "only where the host really is WSL". + # Runs only on the WSL lane; everywhere else the premise does not exist. + @id:examine-detects-wsl @requires-wsl + Scenario: 11 - System inspection recognizes a WSL host + Given the CLI is running in WSL + When the user inspects the system + Then the inspection reports Linux as the operating system + And the inspection reports that the host is WSL diff --git a/tests/e2e-cucumber/src/capability.rs b/tests/e2e-cucumber/src/capability.rs index 352dffce85..9ca0d6a0a6 100644 --- a/tests/e2e-cucumber/src/capability.rs +++ b/tests/e2e-cucumber/src/capability.rs @@ -102,7 +102,9 @@ pub struct HostCapability { /// First AMD GPU's gfx target from `examine`'s `detected_gfx_target:` line /// (e.g. "gfx942", "gfx1151"), if a real one was reported. pub gfx_target: Option, - /// Whether an AMD GPU was detected (a real `detected_gfx_target` is present). + /// Whether an AMD GPU is present AND usable here, so `@requires-gpu` + /// scenarios can run. A real `detected_gfx_target`, plus a ready ROCm + /// driver path on WSL — see [`host_has_usable_gpu`]. pub has_amd_gpu: bool, /// Engine adapters the binary reports as present. Both builtins are always /// "built-in", so this is NOT the same as "can start here" — use @@ -331,11 +333,17 @@ fn probe_host_capability() -> HostCapability { let examine = run_probe(root, &["examine"]); let engines = run_probe(root, &["engines", "list"]); - let (os_family, is_wsl, gfx_target) = parse_examine_text(&examine); - let has_amd_gpu = gfx_target.is_some(); + let ExamineFacts { + os_family, + is_wsl, + gfx_target, + driver_status, + } = parse_examine_text(&examine); + let has_amd_gpu = host_has_usable_gpu(gfx_target.as_deref(), is_wsl, &driver_status); let available_engines = parse_engines_list(&engines); let effective_serve_engine = effective_serve_engine(gfx_target.as_deref(), &os_family); - let platform_slug = derive_platform_slug(has_amd_gpu, gfx_target.as_deref(), &os_family); + let platform_slug = + derive_platform_slug(has_amd_gpu, gfx_target.as_deref(), &os_family, is_wsl); HostCapability { os_family, @@ -362,15 +370,34 @@ fn run_probe(root: &std::path::Path, args: &[&str]) -> String { } } -/// Extract `(os_family, is_wsl, first_gfx_target)` from the human `rocm examine` -/// text (format string in rocm-core `ExamineSummary`): lines like ` os: linux`, -/// ` detected_gfx_target: gfx942`, ` wsl: false`. A missing/placeholder -/// (``, empty, `none`) gfx target yields `None` (→ treated as no GPU). -/// Tolerant: an unrecognized dump degrades to a mock-like host. -fn parse_examine_text(text: &str) -> (String, bool, Option) { +/// The `examine` fields the capability probe reads. +struct ExamineFacts { + os_family: String, + is_wsl: bool, + gfx_target: Option, + /// `driver_status:` — the CLI's own verdict on whether the runtime can + /// reach the device. Only consulted on WSL (see [`host_has_usable_gpu`]). + driver_status: String, +} + +/// `driver_status` value meaning ROCm's WSL passthrough path is complete. +/// +/// The CLI sets this only when `/dev/dxg`, dxcore, `librocdxg.so` and its +/// ldconfig entry are all present; the other WSL states (`wsl_rocdxg_missing`, +/// `wsl_gpu_plumbing_missing`) mean the runtime cannot reach the GPU. +const WSL_DRIVER_READY: &str = "wsl_rocdxg_ready"; + +/// Extract the probe's facts from the human `rocm examine` text (format string +/// in rocm-core `ExamineSummary`): lines like ` os: linux`, +/// ` detected_gfx_target: gfx942`, ` wsl: false`, ` driver_status: ...`. A +/// missing/placeholder (``, empty, `none`) gfx target yields `None` +/// (→ treated as no GPU). Tolerant: an unrecognized dump degrades to a +/// mock-like host. +fn parse_examine_text(text: &str) -> ExamineFacts { let mut os_family = "other".to_owned(); let mut is_wsl = false; let mut gfx_target = None; + let mut driver_status = String::new(); for line in text.lines() { let line = line.trim(); if let Some(v) = line.strip_prefix("os:") { @@ -382,9 +409,62 @@ fn parse_examine_text(text: &str) -> (String, bool, Option) { } } else if let Some(v) = line.strip_prefix("wsl:") { is_wsl = matches!(v.trim(), "true" | "yes" | "1"); + } else if let Some(v) = line.strip_prefix("driver_status:") { + driver_status = v.trim().to_owned(); } } - (os_family, is_wsl, gfx_target) + ExamineFacts { + os_family, + is_wsl, + gfx_target, + driver_status, + } +} + +/// Whether `@requires-gpu` scenarios can actually run on this host. +/// +/// A detected gfx target is enough on a native host. It is NOT enough under +/// WSL: the target is reported from the Windows-side driver even when ROCm has +/// no path to the device, so a distro missing `librocdxg.so` advertises +/// `gfx1151` while `rocm serve` refuses with "no usable AMD GPU". Taking the +/// target at face value there runs every GPU scenario against a host that +/// cannot serve one — they fail on their premise rather than resolving to +/// not-applicable, which is exactly what the capability probe exists to avoid. +fn host_has_usable_gpu(gfx_target: Option<&str>, is_wsl: bool, driver_status: &str) -> bool { + let hip = std::env::var_os("HIP_VISIBLE_DEVICES"); + let rocr = std::env::var_os("ROCR_VISIBLE_DEVICES"); + host_has_usable_gpu_with_mask( + gfx_target, + is_wsl, + driver_status, + selected_visibility_mask(hip.as_deref(), rocr.as_deref()), + ) +} + +/// Match the product's visibility-mask precedence: HIP wins when present, +/// including an explicitly empty value; ROCR is the fallback. +fn selected_visibility_mask<'a>( + hip: Option<&'a std::ffi::OsStr>, + rocr: Option<&'a std::ffi::OsStr>, +) -> Option<&'a std::ffi::OsStr> { + hip.or(rocr) +} + +/// Pure form of [`host_has_usable_gpu`] for contract tests. +/// +/// The harness knows whether a gfx target exists, but not the product's complete +/// device topology. As in `rocm_core::has_usable_amd_gpu`, only an authoritative +/// empty mask proves zero usable devices; nonempty/opaque masks stay eligible and +/// let the product perform ordinal/UUID validation at launch time. +fn host_has_usable_gpu_with_mask( + gfx_target: Option<&str>, + is_wsl: bool, + driver_status: &str, + visibility_mask: Option<&std::ffi::OsStr>, +) -> bool { + gfx_target.is_some() + && (!is_wsl || driver_status == WSL_DRIVER_READY) + && visibility_mask.is_none_or(|mask| !mask.is_empty()) } /// Parse engine names from `rocm engines list`. Engine rows are the lines whose @@ -405,32 +485,54 @@ fn parse_engines_list(text: &str) -> Vec { engines } -/// Stable platform identity from hardware. No AMD GPU → "mock". Otherwise a +/// Stable platform identity from hardware and host environment. WSL is a +/// distinct platform even without a GPU, so its report never collides with the +/// ordinary hosted mock column. Otherwise no AMD GPU → "mock"; GPU hosts use a /// coarse slug from the gfx family (data-center → "mi300x"; Strix gfx115x → /// "strix-halo"), falling back to the normalized family. The OS is appended for /// families that ship on more than one OS (Strix Halo runs both Ubuntu and /// Windows on the same gfx1151), so those become distinct grid columns rather /// than colliding into one. -fn derive_platform_slug(has_amd_gpu: bool, gfx_target: Option<&str>, os_family: &str) -> String { +fn derive_platform_slug( + has_amd_gpu: bool, + gfx_target: Option<&str>, + os_family: &str, + is_wsl: bool, +) -> String { + if is_wsl { + return match gfx_target { + Some(t) => format!("{}-wsl", platform_hardware_slug(t)), + None => "wsl".to_owned(), + }; + } if !has_amd_gpu { return "mock".to_owned(); } match gfx_target { Some(t) => { - let f = normalize_family(t); - if f.ends_with("-dcgpu") { - "mi300x".to_owned() - } else if f.starts_with("gfx115") { + let hardware = platform_hardware_slug(t); + if hardware == "strix-halo" { // Same silicon on Ubuntu and Windows — disambiguate by OS. - format!("strix-halo-{}", os_normalized(os_family)) + format!("{hardware}-{}", os_normalized(os_family)) } else { - f + hardware } } None => "mock".to_owned(), } } +fn platform_hardware_slug(gfx_target: &str) -> String { + let family = normalize_family(gfx_target); + if family.ends_with("-dcgpu") { + "mi300x".to_owned() + } else if family.starts_with("gfx115") { + "strix-halo".to_owned() + } else { + family + } +} + /// Short OS token for a platform slug: "windows" / "linux" / else the raw value. fn os_normalized(os_family: &str) -> String { let o = os_family.trim().to_ascii_lowercase(); @@ -533,10 +635,11 @@ rocm examine wsl: false driver_status: ok "; - let (os, wsl, gfx) = parse_examine_text(text); - assert_eq!(os, "linux"); - assert!(!wsl); - assert_eq!(gfx.as_deref(), Some("gfx942")); + let facts = parse_examine_text(text); + assert_eq!(facts.os_family, "linux"); + assert!(!facts.is_wsl); + assert_eq!(facts.gfx_target.as_deref(), Some("gfx942")); + assert_eq!(facts.driver_status, "ok"); } #[test] @@ -548,9 +651,101 @@ rocm examine detected_gfx_target: wsl: false "; - let (os, _, gfx) = parse_examine_text(text); - assert_eq!(os, "other"); - assert_eq!(gfx, None); + let facts = parse_examine_text(text); + assert_eq!(facts.os_family, "other"); + assert_eq!(facts.gfx_target, None); + } + + /// A WSL distro advertises the Windows-side gfx target whether or not ROCm + /// can reach it, so the driver verdict is what decides. Without this, every + /// `@requires-gpu` scenario runs on a host where `serve` cannot start and + /// fails on its premise instead of resolving to not-applicable. + #[test] + fn wsl_gpu_requires_a_ready_rocdxg_path() { + // Real values from the self-hosted WSL runner before librocdxg was + // installed: /dev/dxg present, the ROCm passthrough library missing. + assert!(!host_has_usable_gpu_with_mask( + Some("gfx1151"), + true, + "wsl_rocdxg_missing", + None + )); + assert!(!host_has_usable_gpu_with_mask( + Some("gfx1151"), + true, + "wsl_gpu_plumbing_missing", + None + )); + // Complete passthrough: the GPU is usable and the scenarios must run. + assert!(host_has_usable_gpu_with_mask( + Some("gfx1151"), + true, + WSL_DRIVER_READY, + None + )); + // A native host is unaffected by the driver verdict. + assert!(host_has_usable_gpu_with_mask( + Some("gfx942"), + false, + "amdgpu_available", + None + )); + assert!(host_has_usable_gpu_with_mask( + Some("gfx942"), + false, + "", + None + )); + // No gfx target is still no GPU, WSL or not. + assert!(!host_has_usable_gpu_with_mask( + None, + true, + WSL_DRIVER_READY, + None + )); + assert!(!host_has_usable_gpu_with_mask( + None, + false, + "amdgpu_available", + None + )); + } + + #[test] + fn gpu_capability_honors_the_product_visibility_mask_precedence() { + use std::ffi::OsStr; + + let empty = OsStr::new(""); + let device_zero = OsStr::new("0"); + assert_eq!( + selected_visibility_mask(Some(empty), Some(device_zero)), + Some(empty), + "HIP_VISIBLE_DEVICES must take precedence even when explicitly empty" + ); + assert_eq!( + selected_visibility_mask(None, Some(empty)), + Some(empty), + "ROCR_VISIBLE_DEVICES applies when HIP_VISIBLE_DEVICES is unset" + ); + + assert!(!host_has_usable_gpu_with_mask( + Some("gfx1151"), + true, + WSL_DRIVER_READY, + Some(empty) + )); + assert!(!host_has_usable_gpu_with_mask( + Some("gfx942"), + false, + "amdgpu_available", + Some(empty) + )); + assert!(host_has_usable_gpu_with_mask( + Some("gfx1151"), + true, + WSL_DRIVER_READY, + Some(device_zero) + )); } #[test] @@ -571,20 +766,27 @@ Local model engines #[test] fn platform_slug_derivation() { - assert_eq!(derive_platform_slug(false, None, "other"), "mock"); + assert_eq!(derive_platform_slug(false, None, "other", false), "mock"); assert_eq!( - derive_platform_slug(true, Some("gfx942"), "linux"), + derive_platform_slug(true, Some("gfx942"), "linux", false), "mi300x" ); // Strix Halo: same gfx1151 silicon on both OSes → distinct slugs so the // report grid gets a column per platform, not a collision. assert_eq!( - derive_platform_slug(true, Some("gfx1151"), "linux"), + derive_platform_slug(true, Some("gfx1151"), "linux", false), "strix-halo-linux" ); assert_eq!( - derive_platform_slug(true, Some("gfx1151"), "windows"), + derive_platform_slug(true, Some("gfx1151"), "windows", false), "strix-halo-windows" ); + // Hosted WSL has no GPU but must not collide with the ordinary mock + // column. A future WSL GPU lane also remains distinct from native Linux. + assert_eq!(derive_platform_slug(false, None, "linux", true), "wsl"); + assert_eq!( + derive_platform_slug(true, Some("gfx1151"), "linux", true), + "strix-halo-wsl" + ); } } diff --git a/tests/e2e-cucumber/src/expectation.rs b/tests/e2e-cucumber/src/expectation.rs index 1632bb6623..99b8c13454 100644 --- a/tests/e2e-cucumber/src/expectation.rs +++ b/tests/e2e-cucumber/src/expectation.rs @@ -27,6 +27,7 @@ const REQUIRES_OS_PREFIX: &str = "requires-os:"; const REQUIRES_GPU_TAG: &str = "requires-gpu"; const REQUIRES_NO_GPU_TAG: &str = "requires-no-gpu"; const REQUIRES_BARE_METAL_TAG: &str = "requires-bare-metal"; +const REQUIRES_WSL_TAG: &str = "requires-wsl"; const SERVE_TIMEOUT_PREFIX: &str = "serve-timeout:"; const NIGHTLY_TAG: &str = "nightly"; const LIFECYCLE_TAG: &str = "lifecycle"; @@ -77,6 +78,10 @@ pub struct ScenarioDecl { /// `@requires-os:linux` cannot express this — WSL2 reports an os_family of /// `linux`, so it matches. pub requires_bare_metal: bool, + /// `@requires-wsl`: the exact inverse — the scenario's premise IS a WSL2 + /// host, so it is skipped on native Linux, native Windows and everything + /// else. Same reason `@requires-os:linux` cannot stand in for it. + pub requires_wsl: bool, /// Engine the scenario pins via `@requires-engine:` (if any). pub requires_engine: Option, /// OS the scenario requires via `@requires-os:` (e.g. "linux"), if any — @@ -114,6 +119,7 @@ impl ScenarioDecl { let mut requires_gpu = false; let mut requires_no_gpu = false; let mut requires_bare_metal = false; + let mut requires_wsl = false; let mut requires_engine = None; let mut requires_os = None; let mut serve_timeout_secs = None; @@ -139,6 +145,8 @@ impl ScenarioDecl { requires_no_gpu = true; } else if tag == REQUIRES_BARE_METAL_TAG { requires_bare_metal = true; + } else if tag == REQUIRES_WSL_TAG { + requires_wsl = true; } else if tag == NIGHTLY_TAG { nightly = true; } else if tag == LIFECYCLE_TAG { @@ -152,6 +160,7 @@ impl ScenarioDecl { requires_gpu, requires_no_gpu, requires_bare_metal, + requires_wsl, requires_engine, requires_os, serve_timeout_secs, @@ -188,6 +197,8 @@ pub struct Condition { #[serde(default)] pub therock_family: Option, #[serde(default)] + pub has_amd_gpu: Option, + #[serde(default)] pub is_wsl: Option, } @@ -210,6 +221,11 @@ impl Condition { return false; } } + if let Some(has_gpu) = self.has_amd_gpu + && has_gpu != cap.has_amd_gpu + { + return false; + } if let Some(w) = self.is_wsl && w != cap.is_wsl { @@ -399,6 +415,11 @@ pub fn resolve( reason: "requires a bare-metal host; this one is WSL2".to_owned(), }; } + if decl.requires_wsl && !cap.is_wsl { + return Expectation::Skip { + reason: "requires WSL; this host is not running under WSL".to_owned(), + }; + } if let Some(os) = &decl.requires_os && !os.eq_ignore_ascii_case(&cap.os_family) { @@ -503,10 +524,10 @@ mod tests { effective_serve_engine: "lemonade".into(), platform_slug: "strix-halo".into(), }, - // A WSL2 dev box: `os_family` is `linux` and a GPU is visible through - // /dev/dxg, so nothing but `is_wsl` distinguishes it from bare metal. - // That is precisely why `@requires-os:linux` cannot stand in for - // `@requires-bare-metal`. + // A WSL2 dev box with the ROCm passthrough in place: `os_family` is + // `linux` and a GPU is usable, so nothing but `is_wsl` distinguishes + // it from bare metal. That is precisely why `@requires-os:linux` + // cannot stand in for `@requires-bare-metal`. "wsl2" => HostCapability { os_family: "linux".into(), is_wsl: true, @@ -514,7 +535,30 @@ mod tests { has_amd_gpu: true, available_engines: vec!["lemonade".into(), "vllm".into()], effective_serve_engine: "lemonade".into(), - platform_slug: "wsl2".into(), + platform_slug: "strix-halo-wsl".into(), + }, + // The same box without the passthrough: the gfx target is reported + // by the Windows-side driver but ROCm cannot reach it, so the probe + // resolves no usable GPU (see `capability::host_has_usable_gpu`). + "wsl" => HostCapability { + os_family: "linux".into(), + is_wsl: true, + gfx_target: None, + has_amd_gpu: false, + available_engines: vec!["lemonade".into(), "vllm".into()], + effective_serve_engine: "lemonade".into(), + platform_slug: "wsl".into(), + }, + // The hosted WSL runner before the ROCm passthrough is complete: + // Windows still reports the gfx target, but the CLI cannot use it. + "wsl-no-passthrough" => HostCapability { + os_family: "linux".into(), + is_wsl: true, + gfx_target: Some("gfx1151".into()), + has_amd_gpu: false, + available_engines: vec!["lemonade".into(), "vllm".into()], + effective_serve_engine: "lemonade".into(), + platform_slug: "strix-halo-wsl".into(), }, _ => HostCapability { os_family: "other".into(), @@ -737,10 +781,12 @@ serve_timeout_secs = 90 fn requires_bare_metal_skips_on_wsl_and_runs_everywhere_else() { let m = Expectations::default(); let d = decl(&["id:diagnose-matches-known-symptom", "requires-bare-metal"]); - assert!(matches!( - resolve(&d, &cap("wsl2"), &m, false, false, false), - Expectation::Skip { .. } - )); + for wsl in ["wsl2", "wsl"] { + assert!(matches!( + resolve(&d, &cap(wsl), &m, false, false, false), + Expectation::Skip { .. } + )); + } // Bare-metal hosts of every shape still run it — including the mock lane, // which is where these scenarios earn their keep as a required check. for host in ["mock", "mi300x", "strix-ubuntu", "strix-windows"] { @@ -752,6 +798,27 @@ serve_timeout_secs = 90 } } + /// The exact inverse of the tag above: proving both directions keeps the + /// pair from silently collapsing into one predicate. + #[test] + fn requires_wsl_runs_only_on_wsl_hosts() { + let m = Expectations::default(); + let d = decl(&["id:examine-detects-wsl", "requires-wsl"]); + assert!(d.requires_wsl); + for wsl in ["wsl", "wsl2"] { + assert_eq!( + resolve(&d, &cap(wsl), &m, false, false, false), + Expectation::ExpectPass + ); + } + for platform in ["mock", "mi300x", "strix-ubuntu", "strix-windows"] { + assert!(matches!( + resolve(&d, &cap(platform), &m, false, false, false), + Expectation::Skip { .. } + )); + } + } + #[test] fn requires_os_linux_does_not_stand_in_for_requires_bare_metal() { // The reason the tag has to exist: WSL2 reports os_family "linux", so an @@ -947,6 +1014,42 @@ reason = "vLLM readiness gap" assert!(!m.is_xfail("nope", &cap("mi300x"), "vllm")); } + #[test] + fn gpu_bug_condition_does_not_match_an_unusable_wsl_gpu() { + let m = Expectations::parse( + r#" +[["chat-end-to-end-local-model"]] +when = { effective_engine = "lemonade", os = "linux", therock_family = "gfx*", has_amd_gpu = true } +bug = "EAI-7423" +reason = "Applies only when the scenario uses a real Lemonade GPU serve." +flaky = true +"#, + ) + .unwrap(); + + assert!(m.is_xfail( + "chat-end-to-end-local-model", + &cap("strix-ubuntu"), + "lemonade" + )); + assert!(!m.is_xfail( + "chat-end-to-end-local-model", + &cap("wsl-no-passthrough"), + "lemonade" + )); + } + + #[test] + fn gpu_report_disagreement_xfail_requires_a_reported_gfx_target() { + let m = Expectations::parse(include_str!("../expectations.toml")).unwrap(); + assert!(m.is_xfail( + "examine-both-forms-agree-on-gpu", + &cap("wsl-no-passthrough"), + "lemonade" + )); + assert!(!m.is_xfail("examine-both-forms-agree-on-gpu", &cap("wsl"), "lemonade")); + } + #[test] fn glob_matches_family() { assert!(glob_match("gfx94*", "gfx942")); diff --git a/tests/e2e-cucumber/tests/e2e/dash_steps.rs b/tests/e2e-cucumber/tests/e2e/dash_steps.rs index 71600ae8ae..911fcf7435 100644 --- a/tests/e2e-cucumber/tests/e2e/dash_steps.rs +++ b/tests/e2e-cucumber/tests/e2e/dash_steps.rs @@ -12,7 +12,7 @@ use e2e_cucumber::mock_server::{MetricsMode, MockServer, ServiceRecordOptions}; use std::time::{Duration, Instant}; use crate::E2eWorld; -use crate::e2e::tui_driver::{DEFAULT_TIMEOUT, TuiSession}; +use crate::e2e::tui_driver::{TuiSession, default_timeout}; /// The exact prompt `send_managed_model_message` types, and the string the /// corresponding `Then` step (`managed_chat_request_carried_prompt`) asserts @@ -129,7 +129,7 @@ async fn open_observe_view(world: &mut E2eWorld) { // repeat it until the Observe tab is actually selected — the `●` marks the // active chip. The step then fails only if the dashboard never gets there, // not if it was slow to start. - tui.send_until("4", "● Observe", DEFAULT_TIMEOUT) + tui.send_until("4", "● Observe", default_timeout()) .await .unwrap_or_else(|e| panic!("failed to switch to the Observe tab: {e}")); } @@ -169,7 +169,7 @@ async fn choose_serving(world: &mut E2eWorld) { #[when("the user accepts the local endpoint")] async fn accept_local_endpoint(world: &mut E2eWorld) { let tui = session(world); - tui.wait_for_screen("Your request only leaves this machine", DEFAULT_TIMEOUT) + tui.wait_for_screen("Your request only leaves this machine", default_timeout()) .await .unwrap_or_else(|e| panic!("local endpoint consent did not appear: {e}")); tui.send("y") @@ -179,7 +179,7 @@ async fn accept_local_endpoint(world: &mut E2eWorld) { #[when("the user sends a message to the managed model")] async fn send_managed_model_message(world: &mut E2eWorld) { let tui = session(world); - tui.wait_for_screen("No messages yet.", DEFAULT_TIMEOUT) + tui.wait_for_screen("No messages yet.", default_timeout()) .await .unwrap_or_else(|e| panic!("chat surface never became ready: {e}")); // No `i` here: accepting local-endpoint consent (`accept_chat_consent`) already @@ -197,7 +197,7 @@ async fn send_gpu_message(world: &mut E2eWorld) { let tui = session(world); // Wait for the accepted, empty chat surface before typing so the input is // ready to receive focus. - tui.wait_for_screen("No messages yet.", DEFAULT_TIMEOUT) + tui.wait_for_screen("No messages yet.", default_timeout()) .await .unwrap_or_else(|e| panic!("chat surface never became ready: {e}")); // `i` focuses the input; then the message, then Enter to submit. @@ -211,7 +211,7 @@ async fn send_gpu_message(world: &mut E2eWorld) { async fn quit_tui(world: &mut E2eWorld, surface: &str) { session(world) - .quit_and_wait(DEFAULT_TIMEOUT) + .quit_and_wait(default_timeout()) .await .unwrap_or_else(|e| panic!("{surface} did not exit cleanly: {e}")); } @@ -233,7 +233,7 @@ async fn home_view_displayed(world: &mut E2eWorld) { let tui = session(world); // The Home tab's summary cards (Running / Health / Updates) are drawn at any // size; the wider "GPU UTILIZATION" hero is not, so assert on the cards. - tui.wait_for_screen("Updates", DEFAULT_TIMEOUT) + tui.wait_for_screen("Updates", default_timeout()) .await .unwrap_or_else(|e| panic!("the home view did not appear: {e}")); let screen = tui.screen_text(); @@ -246,7 +246,7 @@ async fn home_view_displayed(world: &mut E2eWorld) { #[then("ROCm setup actions are displayed")] async fn rocm_actions_displayed(world: &mut E2eWorld) { session(world) - .wait_for_screen("Set up / Install ROCm", DEFAULT_TIMEOUT) + .wait_for_screen("Set up / Install ROCm", default_timeout()) .await .unwrap_or_else(|e| panic!("the ROCm setup actions did not appear: {e}")); } @@ -254,7 +254,7 @@ async fn rocm_actions_displayed(world: &mut E2eWorld) { #[then("the assistant's GPU status response is displayed")] async fn gpu_response_displayed(world: &mut E2eWorld) { session(world) - .wait_for_screen("GPU-2 is running hot", DEFAULT_TIMEOUT) + .wait_for_screen("GPU-2 is running hot", default_timeout()) .await .unwrap_or_else(|e| panic!("the assistant's response did not appear: {e}")); } @@ -262,7 +262,7 @@ async fn gpu_response_displayed(world: &mut E2eWorld) { #[then("the managed model's response is displayed")] async fn managed_model_response_displayed(world: &mut E2eWorld) { session(world) - .wait_for_screen("mock response for testing", DEFAULT_TIMEOUT) + .wait_for_screen("mock response for testing", default_timeout()) .await .unwrap_or_else(|e| panic!("the managed model's response did not appear: {e}")); } @@ -280,7 +280,7 @@ async fn managed_chat_request_carried_prompt(world: &mut E2eWorld) { .mock .as_ref() .expect("no mock server running") - .wait_for_chat_request(DEFAULT_TIMEOUT) + .wait_for_chat_request(default_timeout()) .await .unwrap_or_else(|e| panic!("the mock never received a chat request: {e}")); let messages = body @@ -304,14 +304,14 @@ async fn managed_chat_request_carried_prompt(world: &mut E2eWorld) { async fn managed_model_shown_loading(world: &mut E2eWorld) { let model = world.model_name.clone().expect("no model name set"); let tui = session(world); - tui.wait_for_screen(&model, DEFAULT_TIMEOUT) + tui.wait_for_screen(&model, default_timeout()) .await .unwrap_or_else(|e| panic!("the managed model did not appear: {e}")); // The compact Observe table intentionally omits lifecycle status; opening // the selected instance exposes the status a user uses to diagnose startup. tui.send("\r") .unwrap_or_else(|e| panic!("failed to open instance details: {e}")); - tui.wait_for_screen("LOADING", DEFAULT_TIMEOUT) + tui.wait_for_screen("LOADING", default_timeout()) .await .unwrap_or_else(|e| panic!("the loading state did not appear: {e}")); let screen = tui.screen_text(); @@ -328,10 +328,10 @@ async fn managed_model_shown_loading(world: &mut E2eWorld) { async fn managed_model_metrics_displayed(world: &mut E2eWorld) { let model = world.model_name.clone().expect("no model name set"); let tui = session(world); - tui.wait_for_screen(&model, DEFAULT_TIMEOUT) + tui.wait_for_screen(&model, default_timeout()) .await .unwrap_or_else(|e| panic!("the managed model did not appear: {e}")); - tui.wait_for_screen("50ms", DEFAULT_TIMEOUT) + tui.wait_for_screen("50ms", default_timeout()) .await .unwrap_or_else(|e| panic!("TTFT metrics did not appear: {e}")); let screen = tui.screen_text(); @@ -349,7 +349,7 @@ async fn managed_model_metrics_displayed(world: &mut E2eWorld) { #[then("navigation and next-step guidance are displayed")] async fn navigation_guidance_displayed(world: &mut E2eWorld) { let tui = session(world); - tui.wait_for_screen("toggle this help", DEFAULT_TIMEOUT) + tui.wait_for_screen("toggle this help", default_timeout()) .await .unwrap_or_else(|e| panic!("dashboard help did not appear: {e}")); let screen = tui.screen_text(); @@ -362,7 +362,7 @@ async fn navigation_guidance_displayed(world: &mut E2eWorld) { #[then("dashboard destinations are displayed")] async fn dashboard_destinations_displayed(world: &mut E2eWorld) { let tui = session(world); - tui.wait_for_screen("Go to", DEFAULT_TIMEOUT) + tui.wait_for_screen("Go to", default_timeout()) .await .unwrap_or_else(|e| panic!("command palette did not appear: {e}")); let screen = tui.screen_text(); @@ -375,7 +375,7 @@ async fn dashboard_destinations_displayed(world: &mut E2eWorld) { #[then("Serving actions are displayed")] async fn serving_actions_displayed(world: &mut E2eWorld) { session(world) - .wait_for_screen("Serving actions", DEFAULT_TIMEOUT) + .wait_for_screen("Serving actions", default_timeout()) .await .unwrap_or_else(|e| panic!("Serving actions did not appear: {e}")); } @@ -388,7 +388,7 @@ async fn managed_model_displayed(world: &mut E2eWorld) { .expect("no model name set") .to_string(); session(world) - .wait_for_screen(&model, DEFAULT_TIMEOUT) + .wait_for_screen(&model, default_timeout()) .await .unwrap_or_else(|e| panic!("managed model did not appear: {e}")); } @@ -427,7 +427,7 @@ async fn local_endpoint_shown(world: &mut E2eWorld) { .port() .to_string(); session(world) - .wait_for_screen(&port, DEFAULT_TIMEOUT) + .wait_for_screen(&port, default_timeout()) .await .unwrap_or_else(|e| panic!("the detected endpoint was not shown: {e}")); } @@ -439,7 +439,7 @@ async fn privacy_notice_shown(world: &mut E2eWorld) { session(world) .wait_for_screen( "Your request only leaves this machine after you accept.", - DEFAULT_TIMEOUT, + default_timeout(), ) .await .unwrap_or_else(|e| panic!("the privacy notice was not shown: {e}")); @@ -465,7 +465,7 @@ async fn managed_model_scripted_metrics(world: &mut E2eWorld) { #[then("positive generation throughput is displayed for the managed model")] async fn positive_gen_tps_displayed(world: &mut E2eWorld) { session(world) - .wait_for_screen("tok/s", DEFAULT_TIMEOUT) + .wait_for_screen("tok/s", default_timeout()) .await .unwrap_or_else(|e| { panic!("positive gen_tps (\"tok/s\") never appeared after Growing-mode scrapes: {e}") @@ -483,14 +483,15 @@ async fn metrics_endpoint_fails(world: &mut E2eWorld) { // Poll until the daemon delivers at least one 503 to the mock endpoint. // The production instance_tick is 2 s, so this converges in 2–3 s. - let deadline = Instant::now() + DEFAULT_TIMEOUT; + let budget = default_timeout(); + let deadline = Instant::now() + budget; loop { if mock.metrics_failure_count() >= 1 { break; } assert!( Instant::now() < deadline, - "scripted failure was never served within {DEFAULT_TIMEOUT:?}; \ + "scripted failure was never served within {budget:?}; \ check instance_tick and scrape cadence" ); tokio::time::sleep(Duration::from_millis(100)).await; diff --git a/tests/e2e-cucumber/tests/e2e/examine_steps.rs b/tests/e2e-cucumber/tests/e2e/examine_steps.rs index dc872cd8bb..38299139ca 100644 --- a/tests/e2e-cucumber/tests/e2e/examine_steps.rs +++ b/tests/e2e-cucumber/tests/e2e/examine_steps.rs @@ -6,6 +6,13 @@ use cucumber::{given, then, when}; use crate::E2eWorld; +fn field_value<'a>(output: &'a str, field: &str) -> Option<&'a str> { + output.lines().find_map(|line| { + let (name, value) = line.trim().split_once(':')?; + (name == field).then(|| value.trim()) + }) +} + #[given("a machine with an AMD GPU")] async fn setup_gpu_machine(world: &mut E2eWorld) { let (stdout, _, _) = crate::run_rocm(world, &["examine"]); @@ -20,6 +27,15 @@ async fn setup_unmanaged_rocm(world: &mut E2eWorld) { world.plant_unmanaged_rocm(); } +#[given("the CLI is running in WSL")] +async fn setup_wsl_host(world: &mut E2eWorld) { + let (stdout, _, _) = crate::run_rocm(world, &["examine"]); + assert!( + field_value(&stdout, "wsl").is_some_and(|value| value.eq_ignore_ascii_case("true")), + "CLI did not detect WSL:\n{stdout}" + ); +} + #[when("the user asks for the version")] async fn user_asks_version(world: &mut E2eWorld) { let (stdout, _, _) = crate::run_rocm(world, &["version"]); @@ -135,6 +151,26 @@ async fn assert_all_engines_listed(world: &mut E2eWorld) { } } +#[then("the inspection reports Linux as the operating system")] +async fn assert_linux_host(world: &mut E2eWorld) { + let output = world.cli_output.as_ref().expect("no command was run"); + assert_eq!( + field_value(output, "os"), + Some("linux"), + "expected Linux in examine output:\n{output}" + ); +} + +#[then("the inspection reports that the host is WSL")] +async fn assert_wsl_host(world: &mut E2eWorld) { + let output = world.cli_output.as_ref().expect("no command was run"); + assert_eq!( + field_value(output, "wsl"), + Some("true"), + "expected WSL in examine output:\n{output}" + ); +} + #[then("the inspection reports which GPU is installed")] async fn assert_gpu_detected(world: &mut E2eWorld) { let output = world.cli_output.as_ref().expect("no command was run"); diff --git a/tests/e2e-cucumber/tests/e2e/lifecycle_steps.rs b/tests/e2e-cucumber/tests/e2e/lifecycle_steps.rs index ebea19992c..8d743afa72 100644 --- a/tests/e2e-cucumber/tests/e2e/lifecycle_steps.rs +++ b/tests/e2e-cucumber/tests/e2e/lifecycle_steps.rs @@ -23,7 +23,7 @@ use std::process::Command; use cucumber::{given, then, when}; use crate::E2eWorld; -use crate::e2e::tui_driver::{DEFAULT_TIMEOUT, TuiSession}; +use crate::e2e::tui_driver::{TuiSession, default_timeout}; /// Per-scenario release-lifecycle state. All paths are rooted in the scenario's /// isolated temp dir; `Drop` restores the captured Windows user PATH. @@ -857,7 +857,7 @@ async fn when_quit_installed_chat(world: &mut E2eWorld) { .tui .as_mut() .expect("no installed interactive chat session is open") - .quit_and_wait(DEFAULT_TIMEOUT) + .quit_and_wait(default_timeout()) .await .unwrap_or_else(|e| panic!("installed interactive chat did not exit cleanly: {e}")); } @@ -1008,7 +1008,7 @@ async fn then_installed_chat_displayed(world: &mut E2eWorld) { .tui .as_mut() .expect("no installed interactive chat session is open") - .wait_for_screen("No messages yet.", DEFAULT_TIMEOUT) + .wait_for_screen("No messages yet.", default_timeout()) .await .unwrap_or_else(|e| panic!("installed interactive chat did not become ready: {e}")); } diff --git a/tests/e2e-cucumber/tests/e2e/tui_driver.rs b/tests/e2e-cucumber/tests/e2e/tui_driver.rs index cd4fca1501..3ec8d0d855 100644 --- a/tests/e2e-cucumber/tests/e2e/tui_driver.rs +++ b/tests/e2e-cucumber/tests/e2e/tui_driver.rs @@ -60,7 +60,27 @@ const DRAIN_TIMEOUT: Duration = Duration::from_millis(250); /// Default wall-clock budget for a single wait. Generous enough for a cold dash /// start plus the embedded-daemon connect, while still turning a genuine hang /// into a prompt, diagnosable failure rather than a CI-timeout. -pub const DEFAULT_TIMEOUT: Duration = Duration::from_secs(30); +const DEFAULT_TIMEOUT_SECS: u64 = 30; + +/// Wall-clock budget for a single wait, overridable with `E2E_TUI_TIMEOUT_SECS`. +/// +/// Mirrors `E2E_SERVE_TIMEOUT_SECS` in `serving_steps.rs`: the assertion is +/// right, the budget is what varies by host. The self-hosted Strix lanes share +/// one physical machine, so a TUI frame that renders well inside 30s on an idle +/// runner can miss it when a sibling lane is loading a model on the same box — +/// which shows up as an unrelated-looking flake, not as a real hang. +/// +/// Deliberately a wait budget and not a retry: a genuine hang must still fail. +#[must_use] +pub fn default_timeout() -> Duration { + Duration::from_secs( + std::env::var("E2E_TUI_TIMEOUT_SECS") + .ok() + .and_then(|value| value.parse().ok()) + .filter(|secs| *secs > 0) + .unwrap_or(DEFAULT_TIMEOUT_SECS), + ) +} /// A running `rocm` TUI attached to a pseudo-terminal. pub struct TuiSession { diff --git a/xtask/src/e2e_report.rs b/xtask/src/e2e_report.rs index 353d4c18f6..27122ebbd5 100644 --- a/xtask/src/e2e_report.rs +++ b/xtask/src/e2e_report.rs @@ -133,6 +133,11 @@ fn label_for_root_report(dir: &Path) -> String { Some("mi300x") => "e2e-gpu-report".to_owned(), Some("strix-halo-linux") => "e2e-gpu-strix-ubuntu-report".to_owned(), Some("strix-halo-windows") => "e2e-gpu-strix-windows-report".to_owned(), + // This workflow is statically pinned to the Strix WSL runner. A bare + // `wsl` slug means the probe could not recover a gfx target; map it to + // the same identity the workflow's artifact directory carries so a + // flattened single-artifact download cannot change the report column. + Some("wsl" | "strix-halo-wsl") => "e2e-gpu-strix-wsl-report".to_owned(), // Missing sidecar (e.g. a GPU run that errored before writing it) or an // unrecognized slug → neutral identity, never a false "Mock". _ => "e2e-unknown-report".to_owned(), @@ -159,6 +164,7 @@ mod tests { "e2e-gpu-report", "e2e-gpu-strix-ubuntu-report", "e2e-gpu-strix-windows-report", + "e2e-gpu-strix-wsl-report", ]; /// Every `e2e-`-prefixed artifact name an upload step in `file` publishes. @@ -234,8 +240,8 @@ mod tests { #[test] fn the_nightly_lanes_publish_the_same_platforms_as_the_per_pr_lanes() { // The nightly workflow exists to run *more* scenarios on the *same* - // three platforms. If the two ever diverge, the nightly grid is - // comparing different hardware than the PR grid without saying so. + // platforms. If the two ever diverge, the nightly grid is comparing + // different hardware than the PR grid without saying so. let lanes = |file: &str| { let mut names = uploaded_e2e_artifacts( &Path::new(env!("CARGO_MANIFEST_DIR")) @@ -288,6 +294,7 @@ mod tests { ("mi300x", "e2e-gpu-report"), ("strix-halo-linux", "e2e-gpu-strix-ubuntu-report"), ("strix-halo-windows", "e2e-gpu-strix-windows-report"), + ("strix-halo-wsl", "e2e-gpu-strix-wsl-report"), ("mock", "e2e-report"), ] { let tmp = tempfile::tempdir().expect("tempdir"); @@ -339,4 +346,16 @@ mod tests { let names: Vec<&str> = got.iter().map(|(n, _)| n.as_str()).collect(); assert_eq!(names, vec!["e2e-unknown-report"]); } + + #[test] + fn discover_root_bare_wsl_matches_the_static_wsl_artifact() { + let tmp = tempfile::tempdir().expect("tempdir"); + let root = tmp.path(); + std::fs::write(root.join("report.json"), "[]").unwrap(); + std::fs::write(root.join("platform.json"), r#"{"platform_slug":"wsl"}"#).unwrap(); + + let got = discover(root).expect("discover"); + let names: Vec<&str> = got.iter().map(|(n, _)| n.as_str()).collect(); + assert_eq!(names, vec!["e2e-gpu-strix-wsl-report"]); + } } diff --git a/xtask/src/workflow_contract.rs b/xtask/src/workflow_contract.rs index 196ca3a1c0..c2f9daf5ec 100644 --- a/xtask/src/workflow_contract.rs +++ b/xtask/src/workflow_contract.rs @@ -17,6 +17,7 @@ #[cfg(test)] mod tests { + use std::collections::BTreeMap; use std::path::{Path, PathBuf}; fn repo_root() -> PathBuf { @@ -30,7 +31,9 @@ mod tests { fn read_workflow(name: &str) -> String { let p = repo_root().join(".github/workflows").join(name); - std::fs::read_to_string(&p).unwrap_or_else(|e| panic!("reading {}: {e}", p.display())) + std::fs::read_to_string(&p) + .unwrap_or_else(|e| panic!("reading {}: {e}", p.display())) + .replace("\r\n", "\n") } /// The self-hosted runner labels that must not appear in a `runs-on`. @@ -127,6 +130,125 @@ mod tests { .expect("workflow declares a top-level name:") } + /// Extract one top-level job's complete YAML block by its job id. + fn job_block<'a>(text: &'a str, job: &str) -> &'a str { + let marker = format!(" {job}:\n"); + let start = text + .find(&marker) + .unwrap_or_else(|| panic!("workflow defines job `{job}`")); + let rest = &text[start + marker.len()..]; + let end = rest + .match_indices("\n ") + .find_map(|(i, _)| { + rest[i + 1..] + .lines() + .next() + .is_some_and(|line| line.starts_with(" ") && !line.starts_with(" ")) + .then_some(i) + }) + .unwrap_or(rest.len()); + &rest[..end] + } + + /// Extract the direct scalar entries from a named job-level mapping such as + /// `env:`. Nested step mappings cannot satisfy this extractor. + fn job_mapping(block: &str, mapping: &str) -> BTreeMap { + let marker = format!("{mapping}:"); + let lines: Vec<&str> = block.lines().collect(); + let (start, mapping_indent) = lines + .iter() + .enumerate() + .find_map(|(i, line)| { + (line.trim() == marker && indent_of(line) == 4).then_some((i, indent_of(line))) + }) + .unwrap_or_else(|| panic!("job defines top-level mapping `{mapping}`")); + + let mut entries = BTreeMap::new(); + for raw in &lines[start + 1..] { + if raw.trim().is_empty() || raw.trim_start().starts_with('#') { + continue; + } + let line = strip_comment(raw); + let indent = indent_of(line); + if indent <= mapping_indent { + break; + } + if indent != mapping_indent + 2 { + continue; + } + let (key, raw_value) = line + .trim() + .split_once(':') + .unwrap_or_else(|| panic!("mapping entry has a scalar value: `{line}`")); + let value = raw_value.trim(); + let value = value + .strip_prefix('"') + .and_then(|v| v.strip_suffix('"')) + .or_else(|| value.strip_prefix('\'').and_then(|v| v.strip_suffix('\''))) + .unwrap_or(value); + assert!( + entries.insert(key.to_owned(), value.to_owned()).is_none(), + "mapping `{mapping}` contains duplicate key `{key}`" + ); + } + entries + } + + fn job_scalar<'a>(block: &'a str, key: &str) -> &'a str { + let marker = format!("{key}:"); + block + .lines() + .find_map(|line| { + (indent_of(line) == 4) + .then(|| line.trim().strip_prefix(&marker)) + .flatten() + .map(str::trim) + }) + .unwrap_or_else(|| panic!("job defines top-level scalar `{key}`")) + } + + fn markdown_table_rows(text: &str, header: &str) -> Vec> { + let mut lines = text.lines().skip_while(|line| *line != header); + assert_eq!( + lines.next(), + Some(header), + "markdown table `{header}` exists" + ); + let separator = lines.next().expect("markdown table has a separator row"); + assert!( + separator.starts_with("|---"), + "markdown table has a separator row" + ); + lines + .take_while(|line| line.starts_with('|')) + .map(|line| { + line.trim_matches('|') + .split('|') + .map(|cell| cell.trim().to_owned()) + .collect() + }) + .collect() + } + + fn backticked_list_between(text: &str, prefix: &str, suffix: &str) -> Vec { + let section = text + .split_once(prefix) + .unwrap_or_else(|| panic!("section starts with `{prefix}`")) + .1 + .split_once(suffix) + .unwrap_or_else(|| panic!("section ends with `{suffix}`")) + .0; + section + .split('`') + .enumerate() + .filter_map(|(i, item)| (i % 2 == 1).then_some(item.to_owned())) + .collect() + } + + fn normalized_whitespace(text: &str) -> String { + text.split_whitespace().collect::>().join(" ") + } + /// Extract a top-level YAML value/block, stopping at the next column-zero /// key or comment. This is deliberately small: workflow contract tests /// inspect only checked-in files with the repository's established layout. @@ -420,6 +542,152 @@ trigger-a-workflow#triggering-a-workflow-from-a-workflow" } } + #[test] + fn self_hosted_wsl_enables_merge_queue_scenarios_only_for_merge_group() { + let sh = read_workflow("e2e-selfhosted.yml"); + let wsl = job_block(&sh, "e2e-wsl"); + let env = job_mapping(wsl, "env"); + assert_eq!( + env.get("E2E_MERGE_QUEUE").map(String::as_str), + Some("${{ github.event_name == 'merge_group' && '1' || '' }}"), + "e2e-wsl's job-level env must opt into @merge-queue scenarios for merge_group only" + ); + + let nightly = read_workflow("nightly.yml"); + let nightly_wsl = job_block(&nightly, "e2e-wsl-nightly"); + let nightly_env = job_mapping(nightly_wsl, "env"); + assert!( + !nightly_env.contains_key("E2E_MERGE_QUEUE"), + "the nightly WSL job cannot receive merge_group events and must not opt into @merge-queue scenarios" + ); + } + + #[test] + fn wsl_jobs_accept_every_canonical_wsl_detection_signal() { + for (workflow, job) in [ + ("e2e-selfhosted.yml", "e2e-wsl"), + ("nightly.yml", "e2e-wsl-nightly"), + ] { + let text = read_workflow(workflow); + let block = job_block(&text, job); + // Remove shell line-continuation backslashes after whitespace has + // been normalized so the assertion describes the effective test. + let block = normalized_whitespace(block).replace("\\ ", ""); + assert!( + block.contains("proc_version=$(cat /proc/version 2>/dev/null || true)"), + "{workflow} job {job} must read the canonical kernel-version signal" + ); + let distro_signal = ["$", "{", "WSL_DISTRO_NAME+x", "}"].concat(); + let canonical_union = format!( + "if [ ! -e /dev/dxg ] && [ -z \"{distro_signal}\" ] && ! printf '%s\\n' \"$proc_version\" | grep -qiE 'microsoft|wsl'; then" + ); + assert!( + block.contains(&canonical_union), + "{workflow} job {job} must accept the canonical union of WSL signals" + ); + assert!( + !block.contains("/proc/sys/kernel/osrelease"), + "{workflow} job {job} must not use the narrower osrelease-only WSL check" + ); + } + } + + #[test] + fn every_nightly_strix_job_uses_the_shared_machine_tui_budget() { + let nightly = read_workflow("nightly.yml"); + for job in [ + "e2e-gpu-nightly-strix", + "e2e-gpu-nightly-strix-windows", + "e2e-wsl-nightly", + ] { + let env = job_mapping(job_block(&nightly, job), "env"); + assert_eq!( + env.get("E2E_TUI_TIMEOUT_SECS").map(String::as_str), + Some("90"), + "nightly Strix job {job} must use the shared-machine TUI wait budget" + ); + } + } + + #[test] + fn dispatchable_wsl_nightly_run_has_the_full_nightly_job_budget() { + let self_hosted = read_workflow("e2e-selfhosted.yml"); + let nightly = read_workflow("nightly.yml"); + let dispatch_timeout = job_scalar(job_block(&self_hosted, "e2e-wsl"), "timeout-minutes"); + let nightly_timeout = job_scalar(job_block(&nightly, "e2e-wsl-nightly"), "timeout-minutes"); + assert_eq!( + dispatch_timeout, "90", + "the 2400s large-model readiness budget needs the established 90-minute job cap for setup and the remaining suite" + ); + assert_eq!( + dispatch_timeout, nightly_timeout, + "e2e-wsl supports include_nightly, so its job timeout must cover the same 2400s scenario plus setup as e2e-wsl-nightly" + ); + } + + #[test] + fn hardware_testing_docs_cover_all_four_self_hosted_platforms() { + let docs = std::fs::read_to_string(repo_root().join("docs/ci-hardware-testing.md")) + .expect("read hardware testing docs"); + let rows = markdown_table_rows(&docs, "| Job | Workflow | Platform | Runner labels |"); + let self_hosted_job_platforms: Vec<(String, String)> = rows + .into_iter() + .filter(|row| { + row.get(1) + .is_some_and(|workflow| workflow == "`e2e-selfhosted.yml`") + }) + .map(|row| (row[0].clone(), row[2].clone())) + .collect(); + assert_eq!( + self_hosted_job_platforms, + vec![ + ( + "`e2e-gpu`".to_owned(), + "MI300X (AMD Instinct, bare-metal Linux)".to_owned(), + ), + ( + "`e2e-gpu-strix-ubuntu`".to_owned(), + "Strix Halo (gfx1151) on Ubuntu".to_owned(), + ), + ( + "`e2e-gpu-strix-windows`".to_owned(), + "Strix Halo (gfx1151) on native Windows 11".to_owned(), + ), + ( + "`e2e-wsl`".to_owned(), + "Strix Halo (gfx1151) on Ubuntu under WSL2".to_owned(), + ), + ], + "hardware testing table must document the four actual self-hosted job/platform rows" + ); + + let artifacts = backticked_list_between( + &docs, + "The lane artifacts are named canonically (", + ") in every workflow", + ); + assert_eq!( + artifacts, + vec![ + "e2e-report", + "e2e-gpu-report", + "e2e-gpu-strix-ubuntu-report", + "e2e-gpu-strix-windows-report", + "e2e-gpu-strix-wsl-report", + ], + "the canonical artifact list must enumerate every report platform exactly once" + ); + + let readme = std::fs::read_to_string(repo_root().join("tests/e2e-cucumber/README.md")) + .expect("read E2E README"); + assert!( + normalized_whitespace(&readme).contains( + "The nightly workflow runs four non-blocking jobs — MI300X plus Strix Halo on Ubuntu, Windows, and WSL2 — with `E2E_INCLUDE_NIGHTLY=1`" + ), + "E2E README must identify all four nightly job platforms" + ); + } + #[test] fn workflows_use_distinct_concurrency_groups() { // Isolation comes from the group KEY differing per workflow. Extract the @@ -502,4 +770,27 @@ permissions: // Must stop at the next key, not swallow permissions. assert!(!g.contains("permissions")); } + + #[test] + fn job_mapping_extractor_ignores_nested_step_env() { + let block = " env:\n TOP_LEVEL: \"expected\"\n steps:\n - name: nested\n env:\n E2E_MERGE_QUEUE: wrong\n"; + let env = job_mapping(block, "env"); + assert_eq!(env.get("TOP_LEVEL").map(String::as_str), Some("expected")); + assert!(!env.contains_key("E2E_MERGE_QUEUE")); + } + + #[test] + fn job_mapping_extractor_ignores_blank_and_full_line_comments() { + let block = " env:\n BEFORE: one\n\n# a YAML comment may be less indented than the mapping\n # or aligned with its entries\n AFTER: two\n steps:\n"; + let env = job_mapping(block, "env"); + assert_eq!(env.get("BEFORE").map(String::as_str), Some("one")); + assert_eq!(env.get("AFTER").map(String::as_str), Some("two")); + } + + #[test] + #[should_panic(expected = "mapping entry has a scalar value")] + fn job_mapping_extractor_rejects_malformed_non_comment_rows() { + let block = " env:\n VALID: one\n MALFORMED\n steps:\n"; + let _ = job_mapping(block, "env"); + } }