Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
68 changes: 65 additions & 3 deletions .github/workflows/nightly.yml
Original file line number Diff line number Diff line change
Expand Up @@ -404,11 +404,17 @@ jobs:

cargo xtask e2e

# Canonical artifact name, matching the per-PR workflow rather than a
# `-nightly-` variant. `parse_descriptor` in the e2e-report crate maps
# these names to Platform/OS, and an unrecognised one falls back to a
# titlecased platform with the OS hardcoded to Linux — which would have
# labelled the Windows lane below as Linux. Artifacts are scoped to their
# run, so sharing the name with the per-PR workflow collides with nothing.
- name: Upload E2E report
if: always()
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
with:
name: e2e-gpu-nightly-report
name: e2e-gpu-report
path: tests/e2e-cucumber/results/

# Strix Halo counterpart to e2e-gpu-nightly. The same @nightly large-model
Expand Down Expand Up @@ -510,7 +516,7 @@ jobs:
if: always()
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
with:
name: e2e-gpu-nightly-strix-report
name: e2e-gpu-strix-ubuntu-report
path: tests/e2e-cucumber/results/

# Windows counterpart to e2e-gpu-nightly-strix. Keep the PowerShell-native
Expand Down Expand Up @@ -616,5 +622,61 @@ jobs:
if: always()
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
with:
name: e2e-gpu-nightly-strix-windows-report
name: e2e-gpu-strix-windows-report
path: tests/e2e-cucumber/results/

# The nightly lanes each uploaded a report nobody joined up, so reading a
# nightly meant opening three artifacts and comparing them by hand — on the
# run that covers the most scenarios, since only nightly sets
# E2E_INCLUDE_NIGHTLY. Same consolidation the per-PR self-hosted workflow
# already does (e2e-selfhosted.yml), against the three nightly artifacts.
e2e-report-nightly:
name: E2E consolidated report (nightly)
runs-on: ubuntu-latest
timeout-minutes: 15
# Narrower than the workflow-level `contents: write`, which exists for the
# release-publishing jobs. This one checks out, downloads artifacts, runs an
# xtask and uploads; it has no business being able to write to the repo.
permissions:
contents: read
needs:
- e2e-gpu-nightly
- e2e-gpu-nightly-strix
- e2e-gpu-nightly-strix-windows
# Unconditional `always()`, unlike the per-PR workflow: there is no paths
# filter to consult on a schedule, and the lanes are `continue-on-error`, so
# a failed lane must still be reported. A nightly that broke is precisely
# when the grid is worth reading.
if: always()
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1

- uses: actions-rust-lang/setup-rust-toolchain@166cdcfd11aee3cb47222f9ddb555ce30ddb9659 # v1.17.0

# `*-report` matches the three artifacts above, and will pick up any
# nightly platform added later without a change here — provided it is
# named canonically (see the rename note on the upload steps).
# Layout note (same as the per-PR job): download-artifact@v8 uses a
# per-artifact subdir when several match but flattens into the root when
# exactly one does, which is what a partially-failed nightly can produce;
# `discover()` handles both, labeling a root-level report from its
# platform.json slug.
- name: Download all nightly E2E reports
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
with:
pattern: '*-report'
path: e2e-artifacts

- name: Build consolidated report + step summary
run: |
mkdir -p consolidated
cargo xtask e2e-report \
--artifacts-dir e2e-artifacts \
--html-out consolidated/index.html >> "$GITHUB_STEP_SUMMARY"

- name: Upload consolidated report
if: always()
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
with:
name: e2e-consolidated-report-nightly
path: consolidated/
18 changes: 13 additions & 5 deletions docs/ci-hardware-testing.md
Original file line number Diff line number Diff line change
Expand Up @@ -49,11 +49,19 @@ run on dedicated self-hosted runners with a real AMD GPU attached, so they
exercise host/GPU detection, engine `detect`/`capabilities`, and live serving
scenarios that the mock job cannot.

Each workflow has its own consolidated report job (both named `e2e-report`
internally). `ci.yml`'s `E2E consolidated report` covers the mock platform;
`e2e-selfhosted.yml`'s `E2E consolidated report (self-hosted)` covers the three
GPU platforms. Each joins its platforms' reports — including partial or failed
runs — by scenario id into one HTML report and GitHub step summary.
Each workflow has its own consolidated report job. `ci.yml`'s `e2e-report`
covers the mock platform; `e2e-selfhosted.yml`'s `e2e-report` covers the three
GPU platforms; `nightly.yml`'s `e2e-report-nightly` covers the same three
platforms with the `@nightly` scenarios included. Each joins its platforms'
reports — including partial or failed runs — by scenario id into one HTML report
and GitHub step summary.

The lane artifacts are named canonically (`e2e-report`, `e2e-gpu-report`,
`e2e-gpu-strix-ubuntu-report`, `e2e-gpu-strix-windows-report`) in every workflow,
because the report derives each platform's name and OS from the artifact name.
An unrecognised name renders as a guessed platform on Linux, which would report
a Windows lane as Linux; `xtask`'s
`every_uploaded_e2e_artifact_has_a_name_the_report_can_label` guards against it.

## Triggers

Expand Down
10 changes: 6 additions & 4 deletions tests/e2e-cucumber/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -135,12 +135,14 @@ self-hosted runner can never stall `ci.yml`'s merge-required checks:
| `e2e-gpu-strix-windows` | `e2e-selfhosted.yml` | Strix Halo / Windows (self-hosted) | no |

The blocking mock job passes when every applicable scenario is pass-or-xfail with
no XPASS or unexpected failure; the GPU jobs are non-blocking. Each workflow has
its own `e2e-report` job: `ci.yml`'s consolidates the mock platform, and
`e2e-selfhosted.yml`'s consolidates the self-hosted platforms.
no XPASS or unexpected failure; the GPU jobs are non-blocking. Each workflow
consolidates its own platforms: `ci.yml`'s `e2e-report` the mock platform,
`e2e-selfhosted.yml`'s `e2e-report` the self-hosted platforms, and
`nightly.yml`'s `e2e-report-nightly` the nightly lanes below.

The nightly workflow runs three non-blocking jobs — the existing MI300X job and
new Strix Halo jobs on Ubuntu and Windows — with `E2E_INCLUDE_NIGHTLY=1`. The
new Strix Halo jobs on Ubuntu and Windows — with `E2E_INCLUDE_NIGHTLY=1`, then
consolidates them into the same cross-platform grid. The
shared large-model scenario serves `Qwen/Qwen3.6-27B` through vLLM on MI300X and
the hardware-verified `unsloth/Qwen3.6-35B-A3B-GGUF:UD-Q4_K_XL` checkpoint
through Lemonade on Strix Halo.
Expand Down
108 changes: 108 additions & 0 deletions xtask/src/e2e_report.rs
Original file line number Diff line number Diff line change
Expand Up @@ -143,6 +143,114 @@ fn label_for_root_report(dir: &Path) -> String {
mod tests {
use super::*;

/// The artifact names a **workflow may statically upload** and have
/// `parse_descriptor` resolve to a real platform and OS. Anything outside
/// this set falls through to a titlecased platform with the OS hardcoded to
/// Linux, which is how a Windows lane comes to be reported as Linux.
///
/// `e2e-unknown-report` is deliberately absent despite being recognised.
/// It is a runtime-only sentinel: [`label_for_root_report`] emits it for a
/// report whose `platform.json` was missing or carried an unknown slug, and
/// `parse_descriptor` maps it to Unknown/Unknown precisely so it escapes the
/// Linux default. No workflow can name it, so a workflow that did would be
/// claiming an identity it has no business asserting.
const CANONICAL_REPORT_ARTIFACTS: &[&str] = &[
"e2e-report",
"e2e-gpu-report",
"e2e-gpu-strix-ubuntu-report",
"e2e-gpu-strix-windows-report",
];

/// Every `e2e-`-prefixed artifact name an upload step in `file` publishes.
///
/// Deliberately not filtered to `-report`: that is the shape the caller
/// asserts, so filtering on it first would make the population and the
/// assertion the same condition, and an artifact named `e2e-gpu-results`
/// would be invisible to a test claiming to check every e2e artifact.
fn uploaded_e2e_artifacts(path: &Path) -> Vec<String> {
let text = std::fs::read_to_string(path)
.unwrap_or_else(|e| panic!("read {}: {e}", path.display()));
text.lines()
.filter_map(|line| line.trim().strip_prefix("name: "))
.map(str::trim)
// Job and step display names are titlecased (`E2E tests …`), so the
// lowercase prefix picks out artifact names only.
.filter(|name| name.starts_with("e2e-"))
.map(str::to_owned)
.collect()
}

/// Every workflow in `.github/workflows`, so a new one cannot upload under a
/// name nothing checks — the exact hole this guard exists to close.
fn workflow_files() -> Vec<PathBuf> {
let dir = Path::new(env!("CARGO_MANIFEST_DIR"))
.join("..")
.join(".github")
.join("workflows");
let mut files: Vec<PathBuf> = std::fs::read_dir(&dir)
.unwrap_or_else(|e| panic!("read {}: {e}", dir.display()))
.flatten()
.map(|entry| entry.path())
.filter(|path| {
path.extension()
.and_then(|e| e.to_str())
.is_some_and(|e| e == "yml" || e == "yaml")
})
.collect();
assert!(
!files.is_empty(),
"no workflows found under {}",
dir.display()
);
files.sort();
files
}

#[test]
fn every_uploaded_e2e_artifact_has_a_name_the_report_can_label() {
// A misnamed artifact does not fail anything -- it renders under a
// titlecased platform with the OS defaulted to Linux, so a Windows lane
// silently reports as Linux. That is how the nightly artifacts were
// named before they fed a consolidated report, and renaming them is
// what makes the nightly grid correct rather than merely present.
for path in workflow_files() {
let file = path.file_name().unwrap_or_default().to_string_lossy();
for name in uploaded_e2e_artifacts(&path) {
// The consolidated artifacts are outputs, not inputs: nothing
// parses them back into a platform.
if name.starts_with("e2e-consolidated-report") {
continue;
}
assert!(
CANONICAL_REPORT_ARTIFACTS.contains(&name.as_str()),
"{file} uploads `{name}`, which the report cannot map to a \
platform; it would render as a guessed name on Linux. Use one \
of {CANONICAL_REPORT_ARTIFACTS:?} or teach parse_descriptor."
);
}
}
}

#[test]
fn the_nightly_lanes_publish_the_same_platforms_as_the_per_pr_lanes() {
// The nightly workflow exists to run *more* scenarios on the *same*
// three platforms. If the two ever diverge, the nightly grid is
// comparing different hardware than the PR grid without saying so.
let lanes = |file: &str| {
let mut names = uploaded_e2e_artifacts(
&Path::new(env!("CARGO_MANIFEST_DIR"))
.join("..")
.join(".github")
.join("workflows")
.join(file),
);
names.retain(|n| !n.starts_with("e2e-consolidated-report"));
names.sort();
names
};
assert_eq!(lanes("nightly.yml"), lanes("e2e-selfhosted.yml"));
}

#[test]
fn discover_missing_dir_is_empty() {
let got = discover(Path::new("/no/such/dir")).expect("ok");
Expand Down
Loading