diff --git a/.github/workflows/e2e-selfhosted.yml b/.github/workflows/e2e-selfhosted.yml index cb33e90b9d..34a3b2afe6 100644 --- a/.github/workflows/e2e-selfhosted.yml +++ b/.github/workflows/e2e-selfhosted.yml @@ -10,8 +10,9 @@ name: E2E self-hosted # # These lanes are non-blocking because none of their check names are in # branch protection's required-status-check list (confirmed 2026-09-11) — not -# because of `continue-on-error`, which is `false` below so a real hardware -# regression shows red instead of always green. PRs #334, #335 and #343 all +# because of `continue-on-error`, which is `false` on every lane but +# one, so a real hardware regression shows red instead of always green. The +# exception is `e2e-gpu-w7900`, `true` only while that hardware is proven out. PRs #334, #335 and #343 all # landed while `E2E tests (Strix Halo, Ubuntu)` reported `cancelled` on the # merge_group ref, which a required check under ALLGREEN would have blocked; # renaming a lane is therefore safe. @@ -42,7 +43,7 @@ on: type: choice default: all # Each value also selectable on its own; `all` covers every lane below. - options: [all, app-dev-gpu, strix-ubuntu, strix-windows, strix-wsl] + options: [all, app-dev-gpu, strix-ubuntu, strix-windows, strix-wsl, w7900] name_filter: description: "Scenario-name regex (cucumber --name); empty = full suite" type: string @@ -1497,16 +1498,158 @@ jobs: path: tests/e2e-cucumber/results/ # The copy-out step above is deliberately best-effort (it must not # report red just because an earlier step already failed the job), - # so this is the only place left to catch a copy that failed for its - # own reason: default `warn` would upload nothing, and the - # consolidated report's `discover()` (xtask/src/e2e_report.rs) simply - # omits any platform with no report.json rather than flagging it, so - # the lane would silently vanish from the report instead of showing - # as a failure. Only the WSL lanes need this: every other lane - # writes report.json directly in-job, with no extra guest-to-host - # copy step that can fail on its own. + # so a copy that failed for its own reason leaves results/ empty. + # `error` makes that a failed step here, in the lane that owns the + # copy, where default `warn` would upload nothing without a word. + # The consolidated report job's missing-report check flags the absent + # artifact too, and is the only signal on a `continue-on-error` lane: + # a failed step there does not turn the lane's check red. Only the + # WSL lanes need this guard: every other lane writes report.json + # directly in-job, with no guest-to-host copy step that can fail on + # its own. if-no-files-found: error + e2e-gpu-w7900: + name: E2E tests (W7900) + # 90min: matches every other Linux GPU lane — same collapsed suite. The card + # is smaller than the Instinct parts, so do not tighten this below their cap. + timeout-minutes: 90 + # `w7900` names the hardware rather than reusing the generic `amd-gpu`: + # GitHub matches a job to any runner whose labels are a superset, so a shared + # label would let this workstation card pick up Instinct-targeted work and + # vice versa. + runs-on: [self-hosted, linux, w7900] + needs: [changes] + if: >- + always() + && needs.changes.result == 'success' + && ( + (github.event_name != 'workflow_dispatch' + && needs.changes.outputs.serve == 'true') + || (github.event_name == 'workflow_dispatch' + && (inputs.platform == 'all' || inputs.platform == 'w7900')) + ) + # Non-blocking while this hardware is proven out, as the MI350P lane is. + # Flip to `false` once it has a track record — that is what the older lanes + # run with, so a real regression on this silicon shows red instead of + # always green. + continue-on-error: true + env: + E2E_SERVE_TIMEOUT_SECS: "300" + E2E_INCLUDE_NIGHTLY: "${{ inputs.include_nightly && '1' || '' }}" + # Full serve matrix at the merge-queue gate, like every other lane (see e2e-gpu). + E2E_MERGE_QUEUE: "${{ github.event_name == 'merge_group' && '1' || '' }}" + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + + # Reclaim the GPU before running: a serve leaked by a killed or + # timed-out prior run (its Drop teardown never executed) keeps an engine + # process on the card and starves this job's serves. Scoped to E2E-owned + # trees: a manual /workload serve is matched only if its command line + # also names an E2E root — see the exposure documented in + # scripts/reclaim-gpu.sh. + - name: Reclaim GPU from stray E2E processes + run: bash scripts/reclaim-gpu.sh + + # Bounded wait, as in e2e-gpu. One card per job, so rocm-smi sees exactly + # the GPU this lane runs on. The floor stays at the shared 16 GiB default, + # which is a flat constant rather than a proportion of capacity: it only + # has to be high enough to catch a leftover serve still holding the card, + # and low enough that a clean host passes on the first poll. e2e-gpu + # uses the same 16 GiB against a very different VRAM size for exactly + # that reason — do NOT scale it down for a smaller part. + - name: GPU preflight (bounded wait for an available GPU) + run: | + MIN_FREE_GIB="${GPU_PREFLIGHT_MIN_FREE_GIB:-16}" + CEILING_SECS="${GPU_PREFLIGHT_CEILING_SECS:-90}" + min_free=$(( MIN_FREE_GIB * 1024 * 1024 * 1024 )) + deadline=$(( SECONDS + CEILING_SECS )) + reason="rocm-smi never returned within its timeout (driver wedged or GPU absent)" + # Check the deadline only after a poll, never before the first one. + # $SECONDS counts whole wall-clock seconds, so a short ceiling can + # lapse before the loop is entered, and the verdict below would then + # describe a reading that was never taken. + first_poll=1 + while [ "$first_poll" -eq 1 ] || [ "$SECONDS" -lt "$deadline" ]; do + first_poll=0 + out=$(timeout 15 rocm-smi --showmeminfo vram 2>/dev/null) || { sleep 5; continue; } + total=$(printf '%s\n' "$out" | grep -i 'VRAM Total Memory' | sed 's/.*: *//' | grep -oE '[0-9]+' | tail -1) + used=$(printf '%s\n' "$out" | grep -i 'VRAM Total Used Memory' | sed 's/.*: *//' | grep -oE '[0-9]+' | tail -1) + if [ -z "$total" ] || [ -z "$used" ]; then + reason="rocm-smi returned no VRAM figures (no AMD GPU detected)" + sleep 5; continue + fi + free=$(( total - used )) + if [ "$free" -ge "$min_free" ]; then + echo "GPU ready: $(( free / 1024 / 1024 / 1024 )) GiB free (>= ${MIN_FREE_GIB} GiB)." + exit 0 + fi + reason="VRAM never dropped below the floor: only $(( free / 1024 / 1024 / 1024 )) GiB free (< ${MIN_FREE_GIB} GiB) — a serve is likely still holding the GPU" + echo "waiting: $(( free / 1024 / 1024 / 1024 )) GiB free (< ${MIN_FREE_GIB} GiB)…" + sleep 5 + done + echo "::error::GPU preflight failed after ${CEILING_SECS}s: ${reason}" + # Name what is holding the card instead of leaving the next reader to + # reconstruct it: --report-holders prints per-process VRAM and the + # engine processes visible here. Never fails the step itself. + bash scripts/reclaim-gpu.sh --report-holders || true + exit 1 + + # cache: false for the same reason as e2e-gpu — the build cache is kept in + # CARGO_TARGET_DIR on the work volume, not in GitHub's cache service. + - uses: actions-rust-lang/setup-rust-toolchain@166cdcfd11aee3cb47222f9ddb555ce30ddb9659 # v1.17.0 + with: + cache: false + + - name: Run E2E tests on W7900 + run: | + # $RUNNER_WORKSPACE is on a volume that outlives any single job, so the + # cargo target dir and the shared caches are still warm on the next one. + export CARGO_TARGET_DIR="$RUNNER_WORKSPACE/e2e-target" + export E2E_SHARED_CACHE_DIR="$RUNNER_WORKSPACE/e2e-shared" + # Same mount as the runtimes it populates, so uv hardlinks into them + # instead of copying, and inside the directory the runner's uv-cache-gc + # initContainer bounds — see the e2e-gpu lane for both mechanisms and + # why a cache one level deeper keeps the links but loses the floor. + runner_root="$(dirname "$(dirname "$RUNNER_WORKSPACE")")" + export E2E_SHARED_UV_CACHE_DIR="$runner_root/uv-cache" + # Pre-warm one shared runtime in place before the suite. Not an + # optimization — `install sdk` bakes absolute paths into the runtime + # manifest, so installing anywhere temporary breaks every later serve. + # See e2e-gpu for the full rationale. + prewarm="$RUNNER_WORKSPACE/e2e-prewarm-multi-arch-v2" + export E2E_SHARED_RUNTIMES_DIR="$prewarm/data/runtimes" + + # See the e2e-gpu lane for why the e2e-test-hooks feature must match + # what `cargo xtask e2e` would build. + cargo build --release -p rocm -p rocmd --features rocm/e2e-test-hooks + export ROCM_CLI_BINARY="$CARGO_TARGET_DIR/release/rocm" + export ROCM_CLI_ROCMD_BINARY="$CARGO_TARGET_DIR/release/rocmd" + + # Pre-warm once, serially, in place (no mv/symlink), and refresh it when + # the channel index has moved on — the tree is a cache, not a one-shot. + # See the e2e-gpu lane and `xtask e2e-prewarm`. + HF_HOME="$E2E_SHARED_CACHE_DIR/huggingface" \ + UV_CACHE_DIR="$E2E_SHARED_UV_CACHE_DIR" \ + cargo xtask e2e-prewarm --channel release --prewarm-dir "$prewarm" + + # Optional scenario-name filter for a scoped manual dispatch — a cheap + # way to exercise this lane against a single scenario without the full suite. + NAME_FILTER="${{ github.event.inputs.name_filter }}" + if [ -n "$NAME_FILTER" ]; then + echo "name filter active: $NAME_FILTER" + cargo xtask e2e -- --name "$NAME_FILTER" + else + cargo xtask e2e + fi + + - name: Upload E2E report + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: e2e-gpu-w7900-report + path: tests/e2e-cucumber/results/ + # Consolidate this workflow's self-hosted platform reports into one GPU-side # cross-platform grid (Summary + merged HTML). Distinct name from ci.yml's # required `E2E consolidated report` so it does NOT collide with that required @@ -1517,12 +1660,17 @@ jobs: name: E2E consolidated report (self-hosted) runs-on: ubuntu-latest timeout-minutes: 15 + # `actions: read` lets the missing-report check list this run's artifacts. + permissions: + contents: read + actions: read needs: - changes - e2e-gpu - e2e-gpu-strix-ubuntu - e2e-gpu-strix-windows - e2e-wsl + - e2e-gpu-w7900 # Gate on `serve`: every lane this report consolidates (the GPU jobs) is now # serve-gated, so a serve-only change runs them and their report must still be # produced. On dispatch `serve` is unset, so also run when the trigger was @@ -1557,6 +1705,56 @@ jobs: --artifacts-dir e2e-artifacts \ --html-out consolidated/index.html >> "$GITHUB_STEP_SUMMARY" + # A lane that dies before writing report.json (preflight, build, a crash + # before results exist) uploads nothing, and `discover()` can only render + # the reports that arrived, so that lane would drop out of the grid with + # no row and no FAIL. On a `continue-on-error` lane nothing else is red + # either. So check it here: every lane that ran (any `needs` result but + # `skipped`) must have uploaded its report artifact. This reads the run's + # artifact list rather than e2e-artifacts/, because download-artifact + # flattens a single match into the root and loses its name. The list also + # holds earlier attempts' artifacts, so a re-run that uploads nothing can + # pass on the first attempt's report: this can miss a lane, but never + # flags one that did upload. A lane with a `matrix.channel` axis is checked + # through its `release` leg (the matrix context does not exist in this + # job), which stands for the whole lane. LANE_REPORTS, and this script's + # behaviour, + # are pinned by xtask's workflow contract tests. The annotation quotes the + # lane's `needs` result verbatim; on a `continue-on-error` lane it may read + # `success` even though the lane failed, so the absent artifact is the + # signal, not that value. `!cancelled()`, not `always()`: a run + # superseded mid-flight cuts lanes off before upload, which is not a gap. + - name: Check every lane that ran uploaded its report + if: ${{ !cancelled() }} + env: + GH_TOKEN: ${{ github.token }} + NEEDS: ${{ toJSON(needs) }} + LANE_REPORTS: | + e2e-gpu=e2e-gpu-report + e2e-gpu-strix-ubuntu=e2e-gpu-strix-ubuntu-release-report + e2e-gpu-strix-windows=e2e-gpu-strix-windows-report + e2e-wsl=e2e-gpu-strix-wsl-report + e2e-gpu-w7900=e2e-gpu-w7900-report + run: | + uploaded=$(gh api --paginate \ + "repos/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}/artifacts" \ + --jq '.artifacts[].name') || { + echo "::error::could not list this run's artifacts, so no lane was checked" + exit 1 + } + missing=0 + while IFS='=' read -r job artifact; do + [ -n "$job" ] || continue + result=$(jq -r --arg job "$job" '.[$job].result // "absent"' <<<"$NEEDS") + [ "$result" = skipped ] && continue + if ! grep -qxF "$artifact" <<<"$uploaded"; then + echo "::error::lane $job ran (needs result: $result) but uploaded no $artifact, so it is missing from the consolidated report" + echo "- :x: \`$job\` ran (needs result: $result) but uploaded no \`$artifact\`; it is missing from the grid above" >> "$GITHUB_STEP_SUMMARY" + missing=1 + fi + done <<<"$LANE_REPORTS" + exit "$missing" + - name: Upload consolidated report if: always() uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index 9cd3e4517e..8f10fe96d0 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -523,7 +523,8 @@ jobs: - name: Reclaim GPU from stray E2E processes run: bash scripts/reclaim-gpu.sh - # Bounded wait for a free GPU. Floor is half of the R9700S's 32GB — high + # Bounded wait for a free GPU. The floor stays at the shared 16 GiB + # default, a flat constant rather than a proportion of capacity — high # enough to catch a leftover serve, low enough that a clean pod passes on # the first poll. Same as the per-PR rad3 lane. - name: GPU preflight (bounded wait for an available GPU) @@ -707,6 +708,116 @@ jobs: name: e2e-gpu-mi350p-${{ matrix.channel }}-report path: tests/e2e-cucumber/results/ + # W7900 counterpart to e2e-gpu-nightly. Mirrors the per-PR e2e-gpu-w7900 lane + # in e2e-selfhosted.yml (w7900 label, shared uv cache, GPU preflight), with + # @nightly scenarios enabled. Keep the two in step: the report crate asserts + # the nightly lanes publish the same platforms as the per-PR lanes. + e2e-gpu-nightly-w7900: + name: E2E tests (W7900, incl. nightly-only) + timeout-minutes: 90 + runs-on: [self-hosted, linux, w7900] + strategy: + fail-fast: false + matrix: + channel: [release, nightly] + continue-on-error: true + env: + E2E_INCLUDE_NIGHTLY: "1" + E2E_SERVE_TIMEOUT_SECS: "300" + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + + # Reclaim the GPU before running: a serve leaked by a killed or + # timed-out prior run (its Drop teardown never executed) keeps an engine + # process on the card and starves this job's serves. Scoped to E2E-owned + # trees: a manual /workload serve is matched only if its command line + # also names an E2E root — see the exposure documented in + # scripts/reclaim-gpu.sh. + - name: Reclaim GPU from stray E2E processes + run: bash scripts/reclaim-gpu.sh + + # Bounded wait for a free GPU. The floor stays at the shared 16 GiB + # default, which is a flat constant rather than a proportion of capacity: + # it only has to be high enough to catch a leftover serve still holding + # the card, and low enough that a clean host passes on the first poll. + # Same as the per-PR W7900 lane — do NOT scale it down for a smaller part. + - name: GPU preflight (bounded wait for an available GPU) + run: | + MIN_FREE_GIB="${GPU_PREFLIGHT_MIN_FREE_GIB:-16}" + CEILING_SECS="${GPU_PREFLIGHT_CEILING_SECS:-90}" + min_free=$(( MIN_FREE_GIB * 1024 * 1024 * 1024 )) + deadline=$(( SECONDS + CEILING_SECS )) + reason="rocm-smi never returned within its timeout (driver wedged or GPU absent)" + # Check the deadline only after a poll, never before the first one. + # $SECONDS counts whole wall-clock seconds, so a short ceiling can + # lapse before the loop is entered, and the verdict below would then + # describe a reading that was never taken. + first_poll=1 + while [ "$first_poll" -eq 1 ] || [ "$SECONDS" -lt "$deadline" ]; do + first_poll=0 + out=$(timeout 15 rocm-smi --showmeminfo vram 2>/dev/null) || { sleep 5; continue; } + total=$(printf '%s\n' "$out" | grep -i 'VRAM Total Memory' | sed 's/.*: *//' | grep -oE '[0-9]+' | tail -1) + used=$(printf '%s\n' "$out" | grep -i 'VRAM Total Used Memory' | sed 's/.*: *//' | grep -oE '[0-9]+' | tail -1) + if [ -z "$total" ] || [ -z "$used" ]; then + reason="rocm-smi returned no VRAM figures (no AMD GPU detected)" + sleep 5; continue + fi + free=$(( total - used )) + if [ "$free" -ge "$min_free" ]; then + echo "GPU ready: $(( free / 1024 / 1024 / 1024 )) GiB free (>= ${MIN_FREE_GIB} GiB)." + exit 0 + fi + reason="VRAM never dropped below the floor: only $(( free / 1024 / 1024 / 1024 )) GiB free (< ${MIN_FREE_GIB} GiB) — a serve is likely still holding the GPU" + echo "waiting: $(( free / 1024 / 1024 / 1024 )) GiB free (< ${MIN_FREE_GIB} GiB)…" + sleep 5 + done + echo "::error::GPU preflight failed after ${CEILING_SECS}s: ${reason}" + # Name what is holding the card instead of leaving the next reader to + # reconstruct it: --report-holders prints per-process VRAM and the + # engine processes visible here. Never fails the step itself. + bash scripts/reclaim-gpu.sh --report-holders || true + exit 1 + + - uses: actions-rust-lang/setup-rust-toolchain@166cdcfd11aee3cb47222f9ddb555ce30ddb9659 # v1.17.0 + with: + cache: false + + - name: Run full E2E on W7900 (incl. nightly-only) + run: | + # $RUNNER_WORKSPACE is on a volume that outlives any single job, so the + # cargo target dir and shared caches stay warm. + export CARGO_TARGET_DIR="$RUNNER_WORKSPACE/e2e-target" + export E2E_SHARED_CACHE_DIR="$RUNNER_WORKSPACE/e2e-shared" + # Same mount as the runtimes it populates, so uv hardlinks into them + # instead of copying, and inside the uv-cache-gc initContainer's floor + # (see e2e-selfhosted.yml e2e-gpu). + runner_root="$(dirname "$(dirname "$RUNNER_WORKSPACE")")" + export E2E_SHARED_UV_CACHE_DIR="$runner_root/uv-cache" + prewarm="$RUNNER_WORKSPACE/e2e-prewarm-multi-arch-v2" + export E2E_SHARED_RUNTIMES_DIR="$prewarm/data/runtimes" + + # See the e2e-gpu lane in e2e-selfhosted.yml for why the + # e2e-test-hooks feature must match what `cargo xtask e2e` would build. + cargo build --release -p rocm -p rocmd --features rocm/e2e-test-hooks + export ROCM_CLI_BINARY="$CARGO_TARGET_DIR/release/rocm" + export ROCM_CLI_ROCMD_BINARY="$CARGO_TARGET_DIR/release/rocmd" + + # Pre-warm once, serially, in place (no mv/symlink), and refresh it when + # the channel index has published a newer runtime — the tree is a cache, + # not a one-shot. See `xtask e2e-prewarm`. + HF_HOME="$E2E_SHARED_CACHE_DIR/huggingface" \ + UV_CACHE_DIR="$E2E_SHARED_UV_CACHE_DIR" \ + cargo xtask e2e-prewarm --channel ${{ matrix.channel }} --prewarm-dir "$prewarm" + + cargo xtask e2e + + - name: Upload E2E report + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: e2e-gpu-w7900-${{ matrix.channel }}-report + path: tests/e2e-cucumber/results/ + # Strix Halo counterpart to e2e-gpu-nightly. The same @nightly large-model # scenario serves the platform-specific Lemonade GGUF on this host. Keep the # runner's storage and runtime setup aligned with e2e-selfhosted.yml's proven @@ -1596,14 +1707,15 @@ jobs: path: tests/e2e-cucumber/results/ # The copy-out step above is deliberately best-effort (it must not # report red just because an earlier step already failed the job), - # so this is the only place left to catch a copy that failed for its - # own reason: default `warn` would upload nothing, and the - # consolidated report's `discover()` (xtask/src/e2e_report.rs) simply - # omits any platform with no report.json rather than flagging it, so - # the lane would silently vanish from the report instead of showing - # as a failure. Only the WSL lanes need this: every other lane - # writes report.json directly in-job, with no extra guest-to-host - # copy step that can fail on its own. + # so a copy that failed for its own reason leaves results/ empty. + # `error` makes that a failed step here, in the lane that owns the + # copy, where default `warn` would upload nothing without a word. + # The consolidated report job's missing-report check flags the absent + # artifact too, and is the only signal on a `continue-on-error` lane: + # a failed step there does not turn the lane's check red. Only the + # WSL lanes need this guard: every other lane writes report.json + # directly in-job, with no guest-to-host copy step that can fail on + # its own. if-no-files-found: error # The nightly lanes each uploaded a report nobody joined up, so reading a @@ -1618,12 +1730,15 @@ jobs: # Narrower than the workflow-level `contents: write`, which exists for the # release-publishing jobs. This one checks out, downloads artifacts, runs an # xtask and uploads; it has no business being able to write to the repo. + # `actions: read` lets the missing-report check list this run's artifacts. permissions: contents: read + actions: read needs: - e2e-gpu-nightly - e2e-gpu-nightly-rad3 - e2e-gpu-nightly-mi350p + - e2e-gpu-nightly-w7900 - e2e-gpu-nightly-strix - e2e-gpu-nightly-strix-windows - e2e-wsl-nightly @@ -1659,6 +1774,57 @@ jobs: --html-out consolidated/index.html > consolidated/support-matrix.md cat consolidated/support-matrix.md >> "$GITHUB_STEP_SUMMARY" + # A lane that dies before writing report.json (preflight, build, a crash + # before results exist) uploads nothing, and `discover()` can only render + # the reports that arrived, so that lane would drop out of the grid with + # no row and no FAIL. On a `continue-on-error` lane nothing else is red + # either. So check it here: every lane that ran (any `needs` result but + # `skipped`) must have uploaded its report artifact. This reads the run's + # artifact list rather than e2e-artifacts/, because download-artifact + # flattens a single match into the root and loses its name. The list also + # holds earlier attempts' artifacts, so a re-run that uploads nothing can + # pass on the first attempt's report: this can miss a lane, but never + # flags one that did upload. Every lane carries a `matrix.channel` axis and is + # checked through its `release` leg (the matrix context does not exist in + # this job), which stands for the whole lane. LANE_REPORTS, and this script's behaviour, + # are pinned by xtask's workflow contract tests. The annotation quotes the + # lane's `needs` result verbatim; on a `continue-on-error` lane it may read + # `success` even though the lane failed, so the absent artifact is the + # signal, not that value. `!cancelled()`, not `always()`: a run + # superseded mid-flight cuts lanes off before upload, which is not a gap. + - name: Check every lane that ran uploaded its report + if: ${{ !cancelled() }} + env: + GH_TOKEN: ${{ github.token }} + NEEDS: ${{ toJSON(needs) }} + LANE_REPORTS: | + e2e-gpu-nightly=e2e-gpu-release-report + e2e-gpu-nightly-rad3=e2e-gpu-rad3-release-report + e2e-gpu-nightly-mi350p=e2e-gpu-mi350p-release-report + e2e-gpu-nightly-w7900=e2e-gpu-w7900-release-report + e2e-gpu-nightly-strix=e2e-gpu-strix-ubuntu-release-report + e2e-gpu-nightly-strix-windows=e2e-gpu-strix-windows-release-report + e2e-wsl-nightly=e2e-gpu-strix-wsl-release-report + run: | + uploaded=$(gh api --paginate \ + "repos/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}/artifacts" \ + --jq '.artifacts[].name') || { + echo "::error::could not list this run's artifacts, so no lane was checked" + exit 1 + } + missing=0 + while IFS='=' read -r job artifact; do + [ -n "$job" ] || continue + result=$(jq -r --arg job "$job" '.[$job].result // "absent"' <<<"$NEEDS") + [ "$result" = skipped ] && continue + if ! grep -qxF "$artifact" <<<"$uploaded"; then + echo "::error::lane $job ran (needs result: $result) but uploaded no $artifact, so it is missing from the consolidated report" + echo "- :x: \`$job\` ran (needs result: $result) but uploaded no \`$artifact\`; it is missing from the grid above" >> "$GITHUB_STEP_SUMMARY" + missing=1 + fi + done <<<"$LANE_REPORTS" + exit "$missing" + - name: Upload consolidated report if: always() uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 diff --git a/crates/e2e-report/src/consolidated.rs b/crates/e2e-report/src/consolidated.rs index fcd218c13a..d03b555966 100644 --- a/crates/e2e-report/src/consolidated.rs +++ b/crates/e2e-report/src/consolidated.rs @@ -71,6 +71,7 @@ fn parse_descriptor(name: &str) -> Descriptor { "gpu" => ("MI300X", "Linux"), "gpu-rad3" => ("R9700", "Linux"), "gpu-mi350p" => ("MI350P", "Linux"), + "gpu-w7900" => ("W7900", "Linux"), "gpu-strix-ubuntu" => ("Strix Halo", "Ubuntu"), "gpu-strix-windows" => ("Strix Halo", "Windows"), // Same silicon again, third host boundary: an Ubuntu distro under WSL2 on @@ -1510,6 +1511,7 @@ mod tests { ("e2e-gpu-report", "MI300X", "Linux"), ("e2e-gpu-rad3-report", "R9700", "Linux"), ("e2e-gpu-mi350p-report", "MI350P", "Linux"), + ("e2e-gpu-w7900-report", "W7900", "Linux"), ("e2e-gpu-strix-ubuntu-report", "Strix Halo", "Ubuntu"), ("e2e-gpu-strix-windows-report", "Strix Halo", "Windows"), // Must not fall through to `fallback_descriptor`, which would render diff --git a/docs/ci-hardware-testing.md b/docs/ci-hardware-testing.md index f94da3e85a..b62701050f 100644 --- a/docs/ci-hardware-testing.md +++ b/docs/ci-hardware-testing.md @@ -36,6 +36,7 @@ separate tier flag or tag filter to maintain. | `e2e-gpu-strix-ubuntu` | `e2e-selfhosted.yml` | Strix Halo (gfx1151) on Ubuntu | self-hosted `[self-hosted, linux, devlab-dispatch, strix-halo]` | | `e2e-gpu-strix-windows` | `e2e-selfhosted.yml` | Strix Halo (gfx1151) on Windows 11 | self-hosted `[self-hosted, windows, devlab-dispatch, strix-halo]` | | `e2e-wsl` | `e2e-selfhosted.yml` | Strix Halo (gfx1151) on Ubuntu under WSL2 | self-hosted `[self-hosted, windows, devlab-dispatch, strix-halo]` | +| `e2e-gpu-w7900` | `e2e-selfhosted.yml` | AMD Radeon PRO W7900 (gfx1100, 48GB) on Linux | self-hosted `[self-hosted, linux, w7900]` | `e2e-gpu-strix-ubuntu` additionally runs a `strategy.matrix.channel: [release, nightly]` axis — two concurrent legs on the same job, channel-suffixed @@ -86,7 +87,7 @@ resolve to skip here, and known bugs resolve to xfail from `expectations.toml`. It is a required check and must stay green. The self-hosted jobs (`e2e-gpu`, `e2e-gpu-strix-ubuntu`, `e2e-gpu-strix-windows`, -and `e2e-wsl`) run on AMD GPU systems, so they +`e2e-wsl`, and `e2e-gpu-w7900`) run on AMD GPU systems, so they exercise host/GPU detection, engine `detect`/`capabilities`, and live serving scenarios that the mock job cannot. GPU availability is advisory in the WSL lane, as described below. @@ -162,8 +163,9 @@ GitHub does not create the wiki's git repository until then and the job fails with that explanation until it exists. The lane artifacts are named canonically (`e2e-report`, `e2e-gpu-report`, -`e2e-gpu-rad3-report`, `e2e-gpu-mi350p-report`, `e2e-gpu-strix-ubuntu-report`, -`e2e-gpu-strix-windows-report`, `e2e-gpu-strix-wsl-report`) in `ci.yml` and +`e2e-gpu-rad3-report`, `e2e-gpu-mi350p-report`, `e2e-gpu-w7900-report`, +`e2e-gpu-strix-ubuntu-report`, `e2e-gpu-strix-windows-report`, +`e2e-gpu-strix-wsl-report`) in `ci.yml` and `e2e-selfhosted.yml`, because the report derives each platform's name and OS from the artifact name. `nightly.yml`'s self-hosted lanes run a `channel: [release, nightly]` matrix and append the channel as the final segment before @@ -208,10 +210,11 @@ They can also be triggered manually via `e2e-selfhosted.yml`'s `workflow_dispatch`, independent of the `serve` gate, with these inputs: - `platform` (choice: `all`, `app-dev-gpu`, `strix-ubuntu`, `strix-windows`, - `strix-wsl`) — which self-hosted job(s) to run. `app-dev-gpu` + `strix-wsl`, `w7900`) — which self-hosted job(s) to run. `app-dev-gpu` maps to `e2e-gpu`, `strix-ubuntu` to `e2e-gpu-strix-ubuntu`, `strix-windows` to - `e2e-gpu-strix-windows`, and `strix-wsl` to `e2e-wsl`. (The mock lane has its own - `platform` input on `ci.yml`; it is not part of this workflow.) + `e2e-gpu-strix-windows`, `strix-wsl` to `e2e-wsl`, and `w7900` to + `e2e-gpu-w7900`. (The mock lane has its own `platform` input on `ci.yml`; it + is not part of this workflow.) - `name_filter` (string) — a scenario-name regex forwarded to the cucumber harness (`cargo xtask e2e -- --name `) so a dispatch can run a single scenario instead of the full suite. Empty runs everything applicable @@ -298,12 +301,18 @@ the pre-warm block is duplicated across multiple jobs in two shells; ## Blocking vs. non-blocking The self-hosted jobs — `e2e-gpu`, `e2e-gpu-strix-ubuntu`, -`e2e-gpu-strix-windows`, and `e2e-wsl` — all run with +`e2e-gpu-strix-windows`, `e2e-wsl`, and `e2e-gpu-w7900` — all run with check names that are absent from branch protection's required-status-check list (see below), so a hardware failure never gates a PR merge no matter how -it reports. All four run with `continue-on-error: false`, so a real -regression shows red instead of always green. Their results also -surface in the self-hosted consolidated report for visibility. +it reports. Four of them run with `continue-on-error: false`, so a real +regression shows red instead of always green; `e2e-gpu-w7900` runs with +`continue-on-error: true` while that hardware is being proven out, and should +be flipped to `false` once it has a track record. Their results also surface in +the self-hosted consolidated report for visibility. A lane that ran but uploaded +no report (it died before writing its results) fails the consolidated report +job, naming the lane, rather than silently dropping out of the grid; on a +`continue-on-error: true` lane that is the only red signal for such a failure. +The nightly report job applies the same check to every nightly lane. ### Timeouts on the Strix lanes diff --git a/tests/e2e-cucumber/README.md b/tests/e2e-cucumber/README.md index 6dbc445abf..b92b7570f7 100644 --- a/tests/e2e-cucumber/README.md +++ b/tests/e2e-cucumber/README.md @@ -176,6 +176,7 @@ self-hosted runner can never stall `ci.yml`'s merge-required checks: | `e2e-gpu-strix-ubuntu` | `e2e-selfhosted.yml` | Strix Halo / Ubuntu (self-hosted) | no | | `e2e-gpu-strix-windows` | `e2e-selfhosted.yml` | Strix Halo / Windows (self-hosted) | no | | `e2e-wsl` | `e2e-selfhosted.yml` | Strix Halo / Ubuntu under WSL2 (self-hosted) | no | +| `e2e-gpu-w7900` | `e2e-selfhosted.yml` | Radeon PRO W7900 (self-hosted) | no | The blocking mock job passes when every applicable scenario is pass-or-xfail with no XPASS or unexpected failure; the GPU jobs are non-blocking. Each workflow @@ -188,15 +189,16 @@ skip there. The nightly workflow covers a strict superset of the hardware in the table above, as non-blocking lanes (`e2e-gpu-nightly`, `e2e-gpu-nightly-rad3`, -`e2e-gpu-nightly-mi350p`, `e2e-gpu-nightly-strix`, +`e2e-gpu-nightly-mi350p`, `e2e-gpu-nightly-w7900`, `e2e-gpu-nightly-strix`, `e2e-gpu-nightly-strix-windows`, `e2e-wsl-nightly`) with `E2E_INCLUDE_NIGHTLY=1`, then consolidates them into the same cross-platform grid. `e2e-gpu-nightly-rad3` and `e2e-gpu-nightly-mi350p` have no per-PR counterpart — both were demoted from per-PR to nightly-only per ROCMAI-125 — so nightly is where their regression coverage now lives. The shared -large-model scenario serves `Qwen/Qwen3.6-27B` through vLLM on MI300X and the -hardware-verified `unsloth/Qwen3.6-35B-A3B-GGUF:UD-Q4_K_XL` checkpoint through -Lemonade on Strix Halo. +large-model scenario follows each lane's serve engine: it serves +`Qwen/Qwen3.6-27B` through vLLM on the Instinct lanes (MI300X, MI350P), and the +`unsloth/Qwen3.6-35B-A3B-GGUF:UD-Q4_K_XL` checkpoint (hardware-verified on +Strix Halo) through Lemonade on every other lane (Strix Halo, R9700, W7900). Use the self-hosted E2E workflow dispatch to run either model independently on a ref (the GPU platform / `include_nightly` / `name_filter` inputs live on diff --git a/tests/e2e-cucumber/features/model_serving.feature b/tests/e2e-cucumber/features/model_serving.feature index 93c3c874e3..04169d8e37 100644 --- a/tests/e2e-cucumber/features/model_serving.feature +++ b/tests/e2e-cucumber/features/model_serving.feature @@ -45,11 +45,12 @@ Feature: Model serving And the response identifies the correct model # Large-model coverage (dogfooding W9): serve a representative large model for - # each GPU platform end-to-end at least once. MI300X uses Qwen/Qwen3.6-27B through - # vLLM; Strix Halo uses the hardware-verified - # unsloth/Qwen3.6-35B-A3B-GGUF:UD-Q4_K_XL checkpoint through Lemonade. These slow - # loads stay off the ordinary per-PR path, and the longer readiness timeout also - # gives the first inference request enough time to complete. + # each GPU platform end-to-end at least once. The model follows the lane's serve + # engine: the vLLM lanes (MI300X, MI350P) use Qwen/Qwen3.6-27B; the Lemonade lanes + # (Strix Halo, R9700, W7900) use the unsloth/Qwen3.6-35B-A3B-GGUF:UD-Q4_K_XL + # checkpoint, hardware-verified on Strix Halo. These slow loads stay off the + # ordinary per-PR path, and the longer readiness timeout also gives the first + # inference request enough time to complete. @id:serve-large-model-inference @requires-gpu @serve-timeout:2400 @nightly Scenario: serve-06 - A large platform-specific model serves and responds to inference Given a managed runtime is active diff --git a/tests/e2e-cucumber/src/capability.rs b/tests/e2e-cucumber/src/capability.rs index b1ae994ba8..548b4700ae 100644 --- a/tests/e2e-cucumber/src/capability.rs +++ b/tests/e2e-cucumber/src/capability.rs @@ -1002,6 +1002,19 @@ Local model engines ("gfx950:sramecc+:xnack-", "mi350p"), ("gfx942", "mi300x"), ("gfx94X-dcgpu", "mi300x"), + // The consumer/workstation RDNA parts slug as themselves, and the + // rad3 (R9700) and W7900 lanes both depend on that: their report + // artifacts are keyed off the bare family in + // `xtask::e2e_report::label_for_root_report`. + // + // Pinned because the fallthrough is the absence of a rule, which no + // other assertion here would notice losing. `rocm_core`'s + // similarly-named `normalize_therock_family` DOES fold these into + // `gfx110X-all`/`gfx120X-all` for wheel selection; aligning the two + // would send both lanes' artifacts to `e2e-unknown-report` with + // every test still green. + ("gfx1100", "gfx1100"), + ("gfx1201", "gfx1201"), ] { assert_eq!( derive_platform_slug(true, Some(gfx_target), "linux", false), diff --git a/tests/e2e-cucumber/tests/e2e/serving_steps.rs b/tests/e2e-cucumber/tests/e2e/serving_steps.rs index 4a382eb833..8b9e5cdeba 100644 --- a/tests/e2e-cucumber/tests/e2e/serving_steps.rs +++ b/tests/e2e-cucumber/tests/e2e/serving_steps.rs @@ -635,13 +635,13 @@ async fn setup_lemonade_hf_checkpoint_model(world: &mut E2eWorld) { #[given("a large model is being served on GPU")] async fn setup_large_gpu_model(world: &mut E2eWorld) { - // Large-model nightly coverage follows the host's serving path. MI300X uses - // the dense BF16 model through vLLM; Strix Halo uses the hardware-verified - // UD-Q4_K_XL GGUF checkpoint through Lemonade. The scenario's @serve-timeout - // tag widens both the readiness poll and heavy-model inference timeout. For - // vLLM it also raises ROCM_CLI_VLLM_READY_TIMEOUT_SECS (see isolate_cmd), - // preventing the CLI's default readiness cap from terminating the server - // mid-load. + // Large-model nightly coverage follows the host's serving path. vLLM hosts + // (MI300X, MI350P) use the dense BF16 model; Lemonade hosts (Strix Halo, + // R9700, W7900) use the UD-Q4_K_XL GGUF checkpoint, hardware-verified on + // Strix Halo. The scenario's @serve-timeout tag widens both the readiness + // poll and heavy-model inference timeout. For vLLM it also raises + // ROCM_CLI_VLLM_READY_TIMEOUT_SECS (see isolate_cmd), preventing the CLI's + // default readiness cap from terminating the server mid-load. let (model, engine, ready_substr) = if e2e_cucumber::capability::host_capability().effective_serve_engine == "lemonade" { // ready_substr is the base name WITHOUT the quant. lemonade serves diff --git a/xtask/src/e2e_report.rs b/xtask/src/e2e_report.rs index bd15d5122c..22148e9b54 100644 --- a/xtask/src/e2e_report.rs +++ b/xtask/src/e2e_report.rs @@ -133,6 +133,10 @@ fn label_for_root_report(dir: &Path) -> String { Some("mi300x") => "e2e-gpu-report".to_owned(), Some("gfx1201") => "e2e-gpu-rad3-report".to_owned(), Some("mi350p") => "e2e-gpu-mi350p-report".to_owned(), + // Like gfx1201 above, gfx1100 has no product-name mapping in + // `platform_hardware_slug` — the raw gfx target IS the slug — so the + // sidecar carries `gfx1100` for the W7900 lane. + Some("gfx1100") => "e2e-gpu-w7900-report".to_owned(), Some("strix-halo-linux") => "e2e-gpu-strix-ubuntu-report".to_owned(), Some("strix-halo-windows") => "e2e-gpu-strix-windows-report".to_owned(), // This workflow is statically pinned to the Strix WSL runner. A bare @@ -202,6 +206,7 @@ mod tests { "e2e-gpu-report", "e2e-gpu-rad3-report", "e2e-gpu-mi350p-report", + "e2e-gpu-w7900-report", "e2e-gpu-strix-ubuntu-report", "e2e-gpu-strix-windows-report", "e2e-gpu-strix-wsl-report", @@ -338,6 +343,7 @@ mod tests { ("mi300x", "e2e-gpu-report"), ("gfx1201", "e2e-gpu-rad3-report"), ("mi350p", "e2e-gpu-mi350p-report"), + ("gfx1100", "e2e-gpu-w7900-report"), ("strix-halo-linux", "e2e-gpu-strix-ubuntu-report"), ("strix-halo-windows", "e2e-gpu-strix-windows-report"), ("strix-halo-wsl", "e2e-gpu-strix-wsl-report"), diff --git a/xtask/src/workflow_contract.rs b/xtask/src/workflow_contract.rs index 6d10f2b843..ac30c6f268 100644 --- a/xtask/src/workflow_contract.rs +++ b/xtask/src/workflow_contract.rs @@ -1020,6 +1020,388 @@ trigger-a-workflow#triggering-a-workflow-from-a-workflow" Some(steps) } + /// The non-empty lines of the `key: |` block scalar in `block`, trimmed. + fn block_scalar_lines(block: &str, key: &str) -> Vec { + let marker = format!("{key}: |"); + let mut lines = block.lines().skip_while(|line| line.trim() != marker); + let head = lines + .next() + .unwrap_or_else(|| panic!("block defines `{marker}`")); + let indent = indent_of(head); + lines + .take_while(|line| line.trim().is_empty() || indent_of(line) > indent) + .map(str::trim) + .filter(|line| !line.is_empty()) + .map(str::to_owned) + .collect() + } + + const REPORT_CHECK_STEP: &str = "- name: Check every lane that ran uploaded its report"; + + /// A step block, found by its exact `- name:` line, including that line. + fn named_step(block: &str, name_line: &str) -> String { + let lines: Vec<&str> = block.lines().collect(); + let at = lines + .iter() + .position(|line| line.trim() == name_line) + .unwrap_or_else(|| panic!("block has a step `{name_line}`")); + let step_indent = indent_of(lines[at]); + let mut step = format!("{}\n", lines[at]); + for body in &lines[at + 1..] { + if !body.trim().is_empty() && indent_of(body) <= step_indent { + break; + } + step.push_str(body); + step.push('\n'); + } + step + } + + /// The consolidated report job of each self-hosted workflow, with its + /// missing-report check step. + fn report_check_steps() -> Vec<(&'static str, String, &'static str, String)> { + self_hosted_workflows() + .into_iter() + .map(|(workflow, text)| { + let report_job = match workflow { + "e2e-selfhosted.yml" => "e2e-report", + "nightly.yml" => "e2e-report-nightly", + other => panic!("no consolidated report job known for {other}"), + }; + let step = named_step(job_block(&text, report_job), REPORT_CHECK_STEP); + (workflow, text, report_job, step) + }) + .collect() + } + + /// The report artifact a lane's upload step names (`name: e2e-*-report`). + fn uploaded_report_artifact(workflow: &str, text: &str, job: &str) -> String { + let names: Vec<&str> = job_block(text, job) + .lines() + .filter_map(|line| line.trim().strip_prefix("name: ")) + .filter(|name| name.starts_with("e2e-") && name.ends_with("-report")) + .collect(); + assert_eq!( + names.len(), + 1, + "{workflow} job `{job}` should upload exactly one `e2e-*-report` artifact, \ + found {names:?}" + ); + // The report job has no `matrix` context, so a lane with a channel axis is + // checked through its `release` leg, which stands for the whole lane. + names[0].replace("${{ matrix.channel }}", "release") + } + + #[test] + fn every_lane_that_runs_is_checked_for_its_report_artifact() { + // A lane that dies before writing report.json uploads nothing, and the + // consolidated grid can only render reports that arrived, so the lane + // drops out with no row and no FAIL — and on a `continue-on-error` lane + // nothing else is red either. The report job's missing-report check is + // what catches that, and it only knows a lane through LANE_REPORTS. + // + // Derived from the lanes, like its siblings: a new lane fails here until + // it is added to the map under the exact artifact name its upload step + // uses — a misspelt name would flag the lane missing on every run, and + // an absent one would leave it uncovered. + let steps = report_check_steps(); + for (workflow, text, report_job, step) in &steps { + let (workflow, report_job) = (*workflow, *report_job); + let block = job_block(text, report_job); + let lanes = self_hosted_e2e_jobs(text); + let expected: Vec = lanes + .iter() + .map(|(job, _)| format!("{job}={}", uploaded_report_artifact(workflow, text, job))) + .collect(); + // Read from the step itself, so a LANE_REPORTS left elsewhere in the + // job after the step is deleted cannot satisfy this. + assert_eq!( + block_scalar_lines(step, "LANE_REPORTS"), + expected, + "{workflow} `{report_job}`: LANE_REPORTS must map every self-hosted lane, \ + in workflow order, to the artifact its upload step names" + ); + for (job, _) in &lanes { + assert!( + block.lines().any(|line| line.trim() == format!("- {job}")), + "{workflow} `{report_job}` must list `{job}` under needs, or the \ + missing-report check cannot see whether it ran" + ); + } + assert!( + block.lines().any(|line| line.trim() == "actions: read"), + "{workflow} `{report_job}` needs `actions: read` to list the run's artifacts" + ); + // `!cancelled()` rather than the implicit `success()`: the check has + // to run precisely when an earlier step or a lane has failed. + let step_lines: Vec<&str> = step.lines().map(str::trim).collect(); + for required in [ + "if: ${{ !cancelled() }}", + "GH_TOKEN: ${{ github.token }}", + "NEEDS: ${{ toJSON(needs) }}", + ] { + assert!( + step_lines.contains(&required), + "{workflow} `{report_job}`'s missing-report check must carry `{required}`" + ); + } + } + // The script is copied into both workflows; a fix applied to one twin + // must not silently miss the other. + let bodies: Vec = steps + .iter() + .map(|(_, _, _, step)| { + dedent(&run_block(step).expect("the check step has a `run: |` block")) + .expect("the check step's script is not empty") + }) + .collect(); + assert_eq!( + bodies[0], bodies[1], + "the missing-report check scripts in e2e-selfhosted.yml and nightly.yml have drifted" + ); + } + + /// Run a report job's missing-report check script under `bash -e`, as a + /// `run:` step does, with a stub `gh` that lists `uploaded` (or fails when + /// `uploaded` is `None`) and a `needs` context of `results`. + #[cfg(unix)] + fn run_report_check( + step: &str, + results: &[(&str, &str)], + uploaded: Option<&[&str]>, + ) -> (i32, String, String) { + use std::os::unix::fs::PermissionsExt; + + let dir = tempfile::tempdir().expect("temp dir"); + let script = dir.path().join("check.sh"); + let listing = dir.path().join("artifacts.txt"); + let summary = dir.path().join("summary.md"); + let stub = dir.path().join("gh"); + std::fs::write( + &script, + dedent(&run_block(step).expect("the check step has a `run: |` block")) + .expect("the check step's script is not empty"), + ) + .expect("write script"); + std::fs::write(&summary, "").expect("write summary"); + match uploaded { + Some(names) => { + std::fs::write(&listing, names.join("\n") + "\n").expect("write listing"); + std::fs::write(&stub, "#!/bin/sh\ncat \"$LISTING\"\n").expect("write gh stub"); + } + None => std::fs::write(&stub, "#!/bin/sh\necho 'HTTP 502' >&2\nexit 1\n") + .expect("write gh stub"), + } + std::fs::set_permissions(&stub, std::fs::Permissions::from_mode(0o755)) + .expect("make the stub executable"); + + let needs: serde_json::Map = results + .iter() + .map(|(job, result)| ((*job).to_owned(), serde_json::json!({ "result": result }))) + .collect(); + let path = format!( + "{}:{}", + dir.path().display(), + std::env::var("PATH").unwrap_or_default() + ); + let out = std::process::Command::new("bash") + .arg("-e") + .arg(&script) + .env("PATH", path) + .env("LISTING", &listing) + .env("NEEDS", serde_json::Value::Object(needs).to_string()) + // A `|` block scalar ends in a newline, as GitHub passes it, so the + // script's read loop sees a trailing empty line it must skip. + .env( + "LANE_REPORTS", + block_scalar_lines(step, "LANE_REPORTS").join("\n") + "\n", + ) + .env("GITHUB_REPOSITORY", "owner/repo") + .env("GITHUB_RUN_ID", "1") + .env("GITHUB_STEP_SUMMARY", &summary) + .output() + .expect("running the check script"); + let mut log = String::from_utf8_lossy(&out.stdout).into_owned(); + log.push_str(&String::from_utf8_lossy(&out.stderr)); + ( + out.status.code().expect("check exited with a status code"), + log, + std::fs::read_to_string(&summary).expect("read summary"), + ) + } + + #[test] + #[cfg(unix)] + fn missing_report_check_flags_only_lanes_that_ran_without_a_report() { + // The contract test above pins the check's inputs; this runs its logic, + // so a check that never fails (or always does) cannot pass for one. + let has_jq = std::process::Command::new("jq") + .arg("--version") + .output() + .is_ok_and(|out| out.status.success()); + if !has_jq { + // A skip reads as a pass, so on CI a missing `jq` must fail loudly + // rather than leave the check's logic silently untested. + assert!( + std::env::var_os("CI").is_none(), + "`jq` is not on PATH, so the missing-report check's logic cannot be \ + exercised; install it on this runner" + ); + eprintln!("skipping: no `jq` on PATH"); + return; + } + let (.., step) = report_check_steps() + .into_iter() + .find(|(workflow, ..)| *workflow == "e2e-selfhosted.yml") + .expect("the per-PR workflow has a report check"); + let lanes: Vec<(String, String)> = block_scalar_lines(&step, "LANE_REPORTS") + .iter() + .map(|line| { + let (job, artifact) = line.split_once('=').expect("`job=artifact`"); + (job.to_owned(), artifact.to_owned()) + }) + .collect(); + let jobs: Vec<&str> = lanes.iter().map(|(job, _)| job.as_str()).collect(); + let every_artifact: Vec<&str> = lanes.iter().map(|(_, a)| a.as_str()).collect(); + let (victim, victim_artifact) = (&lanes[0].0, &lanes[0].1); + let all = |result: &'static str| -> Vec<(&str, &str)> { + jobs.iter().map(|job| (*job, result)).collect() + }; + let without_victim: Vec<&str> = every_artifact + .iter() + .copied() + .filter(|a| a != victim_artifact) + .collect(); + let mut with_changes = all("success"); + with_changes.push(("changes", "success")); + + let (code, log, summary) = run_report_check(&step, &with_changes, Some(&every_artifact)); + assert_eq!( + code, 0, + "every lane reported, so nothing is missing:\n{log}" + ); + assert!( + summary.is_empty(), + "nothing missing, nothing summarised: {summary}" + ); + + let mut failed = all("success"); + failed[0].1 = "failure"; + let (code, log, summary) = run_report_check(&step, &failed, Some(&without_victim)); + assert_eq!( + code, 1, + "a lane that ran and uploaded nothing must fail:\n{log}" + ); + assert!( + log.contains(&format!("::error::lane {victim} ran")) && log.contains(victim_artifact), + "the annotation must name the lane and its artifact:\n{log}" + ); + assert!( + summary.contains(victim.as_str()), + "the summary must name the lane: {summary}" + ); + + // A continue-on-error lane may present as `success`; the missing + // artifact alone has to be enough. + let (code, log, _) = run_report_check(&step, &all("success"), Some(&without_victim)); + assert_eq!( + code, 1, + "a missing report fails whatever the needs result:\n{log}" + ); + + let mut skipped = all("success"); + skipped[0].1 = "skipped"; + let (code, log, _) = run_report_check(&step, &skipped, Some(&without_victim)); + assert_eq!( + code, 0, + "a skipped lane uploads nothing and is not missing:\n{log}" + ); + + // A single-platform dispatch skips most lanes. Skipping one must move + // on to the next lane, not stop checking: skip a middle lane while the + // last one is missing. + assert!( + lanes.len() >= 3, + "these cases need a first, middle and last lane" + ); + let (middle_artifact, (last, last_artifact)) = (&lanes[1].1, lanes.last().unwrap()); + let mut skip_middle = all("success"); + skip_middle[1].1 = "skipped"; + let after_skip: Vec<&str> = every_artifact + .iter() + .copied() + .filter(|a| a != middle_artifact && a != last_artifact) + .collect(); + let (code, log, _) = run_report_check(&step, &skip_middle, Some(&after_skip)); + assert_eq!( + code, 1, + "lanes after a skipped one must still be checked:\n{log}" + ); + assert!( + log.contains(&format!("::error::lane {last} ran")), + "the missing lane after the skipped one must be named:\n{log}" + ); + + // Every missing lane is reported, not just the first one found. + let two_missing: Vec<&str> = without_victim + .iter() + .copied() + .filter(|a| a != last_artifact) + .collect(); + let (code, log, summary) = run_report_check(&step, &all("success"), Some(&two_missing)); + assert_eq!(code, 1, "two lanes missing must fail:\n{log}"); + for lane in [victim, last] { + assert!( + log.contains(&format!("::error::lane {lane} ran")) + && summary.contains(lane.as_str()), + "every missing lane must be named, including `{lane}`:\n{log}\n{summary}" + ); + } + + // A lane absent from `needs` has no result to say it was skipped, so + // it is checked rather than waved through. + let (code, log, _) = run_report_check(&step, &all("success")[1..], Some(&without_victim)); + assert_eq!( + code, 1, + "a lane missing from `needs` must not pass unchecked:\n{log}" + ); + assert!( + log.contains("needs result: absent"), + "the annotation must show the lane had no `needs` entry:\n{log}" + ); + + // `cancelled` is what a lane cut off by its timeout reports: it is + // checked like any lane that ran, and passes only if it still uploaded. + let mut timed_out = all("success"); + timed_out[0].1 = "cancelled"; + let (code, log, _) = run_report_check(&step, &timed_out, Some(&without_victim)); + assert_eq!( + code, 1, + "a timed-out lane that uploaded nothing is missing:\n{log}" + ); + let (code, log, _) = run_report_check(&step, &timed_out, Some(&every_artifact)); + assert_eq!( + code, 0, + "a timed-out lane that still uploaded is not missing:\n{log}" + ); + + let mut near_miss = without_victim.clone(); + let stale = format!("{victim_artifact}-old"); + near_miss.push(&stale); + let (code, log, _) = run_report_check(&step, &all("success"), Some(&near_miss)); + assert_eq!( + code, 1, + "only the exact artifact name satisfies the check:\n{log}" + ); + + let (code, log, _) = run_report_check(&step, &all("success"), None); + assert_eq!(code, 1, "an unlistable run must not pass unchecked:\n{log}"); + assert!( + log.contains("could not list this run's artifacts"), + "an API failure must say so rather than fail silently:\n{log}" + ); + } + /// Every `GPU preflight` step block in `text`, in file order. fn gpu_preflight_steps(text: &str) -> Vec { let lines: Vec<&str> = text.lines().collect(); @@ -1816,8 +2198,8 @@ esac } assert_eq!( lanes.len(), - 8, - "expected 8 shell GPU preflight blocks across both workflows — if a \ + 10, + "expected 10 shell GPU preflight blocks across both workflows — if a \ lane was added or removed, update this count" ); @@ -2225,8 +2607,9 @@ esac for row in &self_hosted_rows { assert_eq!( row[3], "no", - "self-hosted lane `{}` is continue-on-error, so the README must not \ - document it as blocking", + "self-hosted lane `{}` is not a required status check, so the README \ + must not document it as blocking (continue-on-error does not decide \ + this; see the header of e2e-selfhosted.yml)", row[0] ); }