diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 65f570109c..d00f1caf16 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -94,6 +94,7 @@ jobs: lint: ${{ steps.filter.outputs.lint || steps.all.outputs.forced }} tpn: ${{ steps.filter.outputs.tpn || steps.all.outputs.forced }} docs: ${{ steps.filter.outputs.docs || steps.all.outputs.forced }} + skills: ${{ steps.filter.outputs.skills || steps.all.outputs.forced }} steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 @@ -136,6 +137,11 @@ jobs: # The containerised control-channel lane; its Dockerfile matches # none of the extension globs above. - 'tests/remote-ssh/**' + # skills/rocm-doctor/reference.md is an e2e FIXTURE: the + # rocm_doctor_skill.feature scenarios parse its catalog table and + # compare it to `rocm fix`. A skill-only edit must therefore run + # the E2E job, which gates on `heavy`. + - 'skills/**' - 'install*' # Pinned-key consistency compares docs/keys/* against the installers, # so a canonical-key-only change must trigger the heavy job that runs @@ -173,6 +179,13 @@ jobs: - '.readthedocs.yaml' - '.github/workflows/**' + # Published Agent Skills, graded by skillscope. The dataset and the + # skill's prose are one unit — a prompt can stop matching because the + # description changed — so any edit under skills/ re-runs the check. + skills: + - 'skills/**' + - '.github/workflows/**' + - name: Force full run off pull requests id: all if: github.event_name != 'pull_request' @@ -1143,6 +1156,88 @@ jobs: if: needs.changes.outputs.lint == 'true' run: cargo xtask powershell-lint --shell powershell + # Grades the published Agent Skills under skills/ with AMD's skillscope + # harness. Only the `structural` command runs: it needs no agent, no API key + # and no network beyond the install, so a red run is never wrong for a + # reason of its own. It is advisory today, not blocking: `Skill checks + # (skillscope)` (this job's `name:`, not the `skill-evals` job id) is not a + # required status check in branch protection, so a failing run does not + # stop a merge. An admin must add that exact context to make it blocking. + # It checks the frontmatter an agent runtime + # parses (a `name` that disagrees with the folder makes the skill unloadable), + # the dataset's coverage bar, and every internal markdown reference — a link + # into a renamed file is not cosmetic here, because the agent follows it, + # finds nothing, and improvises. + # + # `routing` and `behavioral` are deliberately NOT run: both need an + # authenticated `claude` CLI and an ANTHROPIC_API_KEY, and this repo has no + # such secret. Turning them on is a caller change to skillscope's + # `skill-evals.yml` reusable workflow once one exists — they are the half of + # the question that asks whether the skill actually fires, so this job is not + # a substitute for them. + # + # skills/rocm-cli-assistant/ is deliberately out of scope. Despite its path it + # is not a published skill: apps/rocm/src/main.rs embeds it verbatim into the + # chat assistant's system prompt with include_str!, so giving it the YAML + # frontmatter this check requires would inject that frontmatter into a live + # prompt. + skill-evals: + name: Skill checks (skillscope) + runs-on: ubuntu-latest + timeout-minutes: 10 + needs: changes + # A manual E2E dispatch only exercises the E2E jobs; skip the rest. + if: github.event_name != 'workflow_dispatch' && needs.changes.outputs.skills == 'true' + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + + # The `skillscope` step below names skills/rocm-doctor explicitly, so a + # third skills// folder added later would pass this job green + # while never being graded -- the same silent-non-grading failure this + # job exists to catch, just one level deeper. Fail loudly instead: any + # directory under skills/ that isn't graded above or excluded by design + # (rocm-cli-assistant -- see AGENTS.md #7) forces a deliberate edit here. + - name: Guard against an ungraded skills/ directory + run: | + set -euo pipefail + # Without nullglob, a skills/ with no subdirectories leaves the + # pattern unexpanded and the loop runs once on the literal + # `skills/*/`, reporting `skills/* is neither graded ...` -- a + # made-up skill name, from a tree that has no ungraded skill in it. + # With it the loop simply does not run, which is the right answer for + # "nothing to grade". + shopt -s nullglob + known="rocm-doctor rocm-cli-assistant" + for dir in skills/*/; do + name="${dir%/}" + name="${name#skills/}" + case " $known " in + *" $name "*) ;; + *) + echo "::error::skills/$name is neither graded by skill-evals nor excluded by design ($known). Add it to the skillscope step's 'skills:' input, or add it to this guard's exclusion list with the reason, before merging." >&2 + exit 1 + ;; + esac + done + + # The action is only a launcher: it sets up Python and uv, resolves which + # build of the harness to run, and execs it. It imports nothing from + # skillscope, so pinning it to a release SHA cannot break on a payload + # version it predates. What the pin does NOT cover: the composite's own + # steps (setup-python, setup-uv, setup-node) are on their own moving + # tags, and the harness itself is resolved at runtime via + # `uvx --from git+.../skillscope@` with no lockfile. Unlike this + # repo's usual SHA pin (AGENTS.md #6), this one does not reach the code + # that actually runs. install-claude stays off; structural needs no + # agent. + - name: Check skill structure, dataset, and references + uses: amd/skillscope@6dde8e8a34ad5456d3c2ff3418bc81c2cd9d5d69 # v0.1.0 + with: + command: structural + skills: skills/rocm-doctor + args: --skill-files skill-card.md --skill-sections Description,Owner,License + install-claude: "false" + license-headers: name: License header check (hawkeye) runs-on: ubuntu-latest diff --git a/.github/workflows/e2e-selfhosted.yml b/.github/workflows/e2e-selfhosted.yml index f3188e6851..ca9fac71e6 100644 --- a/.github/workflows/e2e-selfhosted.yml +++ b/.github/workflows/e2e-selfhosted.yml @@ -105,6 +105,11 @@ jobs: - '**/*.ps1' - '**/*.feature' - 'tests/e2e-cucumber/**' + # skills/rocm-doctor/reference.md is an e2e FIXTURE: the + # rocm_doctor_skill.feature scenarios parse its catalog table and + # compare it to `rocm fix`. A skill-only edit must therefore run + # this job too, same as ci.yml's `heavy` filter. + - 'skills/**' - 'install*' - 'docs/keys/**' - '.github/workflows/**' diff --git a/.gitignore b/.gitignore index fdec19493e..f94b9a657e 100644 --- a/.gitignore +++ b/.gitignore @@ -13,6 +13,8 @@ active-implementation-log.md current-user-asks.md __pycache__/ /.hawkeye-bin/ +# skillscope writes its JSON run reports here. +/.skillscope/ /.claude /plans/ /workspace/ diff --git a/AGENTS.md b/AGENTS.md index 842f2ab198..1e6ba35b94 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -216,6 +216,7 @@ When changing assistant-adjacent behavior, keep consistency with: - `docs/llm-tool-use.md` - `skills/rocm-cli-assistant/SKILL.md` +- `skills/rocm-doctor/SKILL.md` and `skills/rocm-doctor/reference.md` Required consistency points: @@ -224,6 +225,62 @@ Required consistency points: - avoid invented shell/package-manager commands in assistant behavior paths - preserve built-in assistant constraints and no-CPU-fallback policy +### `skills/rocm-doctor/` — published from here, and a test fixture + +`skills/rocm-cli-assistant/SKILL.md` is compiled into the binary +(`include_str!` in `apps/rocm/src/main.rs`). `skills/rocm-doctor/` is different +on two counts, and both change how you edit it: + +- **This repo is its source of truth.** The skill is a thin driver over the + `rocm` binary, so it is versioned with the binary and lives here. + [`amd/skills`](https://github.com/amd/skills) is still in Phase-1 + incubation for this skill: it carries no automated federation for + `rocm-doctor` yet — `.github/federation.json` there only declares + `AMD-AGI/TraceLens` as a source, and this skill instead sits under + `staging/rocm-doctor`, outside any job's coverage. Until federation picks it + up, the rocm-cli team hand-syncs `staging/rocm-doctor` from this folder + whenever it changes materially. So edit it here, and never edit the + `amd/skills` copy directly — the next hand-sync overwrites it. +- **`reference.md` is an e2e fixture.** `tests/e2e-cucumber/features/rocm_doctor_skill.feature` + parses its closed-catalog table and compares it to what `rocm fix` reports. + The failure catalog itself is authoritative in `crates/rocm-core/src/fix.rs` + (the `RECIPES` list) and `crates/rocm-core/src/diagnose.rs` (each mode's + checker and OS scoping) — adding, renaming, or re-scoping a failure mode + means changing the CLI **first**, then the two docs. That feature is what + catches you if you forget. + +The folder is excluded from `licenserc.toml`: `SKILL.md` must open with YAML +frontmatter for the skill loader, and skills published this way carry no +license headers of their own. The licence is stated in `skill-card.md` +instead. + +Two checks gate it, and they cover different things: + +- **`skill-evals` (skillscope, advisory today)** — the frontmatter an agent + runtime parses, the `evals/evals.json` coverage bar (at least 3 prompts that + should trigger the skill and 2 near misses that should not), the + `skill-card.md` sections, and every internal markdown link. It is not yet a + required status check in branch protection: an admin must add its exact + context, `Skill checks (skillscope)` (the job's `name:`, not the + `skill-evals` job id), before a red run actually blocks a merge. Reproduce + it locally with + `uv tool install git+https://github.com/amd/skillscope@v0.1.0`, then + `skillscope structural --skills-dir 'skills/rocm-doctor' --skill-files + skill-card.md --skill-sections Description,Owner,License`. Add `--external` + to check the outbound URLs too; CI does not, because a rate-limited host is + not a broken link. +- **`rocm_doctor_skill.feature` (e2e, blocking)** — whether the prose still + describes the binary, as above. + +Neither grades whether the skill actually *fires*. That is skillscope's +`routing` and `behavioral`, which need an authenticated `claude` CLI and an +`ANTHROPIC_API_KEY` this repo does not have. The dataset is written and checked +so those can be switched on without further work. + +`skills/rocm-cli-assistant/` is **not** in scope for skillscope: it is embedded +verbatim into the chat system prompt with `include_str!`, so the YAML +frontmatter a published skill needs would end up inside that prompt. + ## 8) Verification Matrix For This Repo Minimum quality gate before upstream-ready status: diff --git a/crates/rocm-core/src/diagnose.rs b/crates/rocm-core/src/diagnose.rs index 656aabb58f..20e8021fae 100644 --- a/crates/rocm-core/src/diagnose.rs +++ b/crates/rocm-core/src/diagnose.rs @@ -105,15 +105,17 @@ impl DiagnoseReport { } } -/// Upstream tracker for a framework key. +/// Upstream tracker for a routing target. +/// +/// Only the targets [`route_when_no_match`] can actually produce are listed. +/// Arms for lemonade / ollama / lm-studio / amdgpu-install were unreachable — +/// the host probe never reports those frameworks — so they described a +/// capability the CLI does not have. Routing an app that merely appears in the +/// symptom text is the caller's job, not the probe's. fn upstream_tracker(target: &str) -> &'static str { match target { "pytorch" => "https://github.com/pytorch/pytorch/issues (tag with rocm label)", "llama-cpp" => "https://github.com/ggml-org/llama.cpp/issues", - "lemonade" => "https://github.com/lemonade-sdk/lemonade/issues", - "ollama" => "https://github.com/ollama/ollama/issues", - "lm-studio" => "https://lmstudio.ai/docs/app (use in-app support; no public repo)", - "amdgpu-install" => "https://repo.radeon.com (raise via your AMD support contact)", _ => "https://github.com/ROCm/ROCm/issues", } } @@ -1018,13 +1020,14 @@ fn check_9_igpu_dgpu_collision(e: &Examination, symptom: &str) -> Diagnosis { "Detected gfx targets: {gfx_targets:?}. Discrete GPU(s): {discrete_targets:?}; integrated APU(s): {apu_targets:?}. Pin HIP_VISIBLE_DEVICES to the discrete GPU — do not assume the higher-numbered gfx target is the dGPU (on RDNA3 the APU can be higher)." ) }; - // Marked auto_applicable below, but `rocm fix fix-9-igpu-dgpu` still needs - // --device-index to actually make the change: without it, both the Linux - // and Windows runners only print the query that finds the index and - // change nothing (see README's --device-index caveat). - let note = format!( - "{note} Without --device-index, `rocm fix` only prints this query and makes no change, despite being marked AUTO." - ); + // Both branches below are marked auto_applicable, but `rocm fix + // fix-9-igpu-dgpu` still needs --device-index to actually make the change: + // without it, the Linux and the Windows runner alike only print the query + // that finds the index and change nothing (see README's --device-index + // caveat). Each branch states that once, in its second note. + // `render_report_text` prints every note on its own line, so saying it + // again here -- appended to the detected-targets note -- would show up as a + // second bullet repeating the first. let fix = if e.os_family == "windows" { Fix { summary: "Pin the HIP runtime to the discrete GPU with HIP_VISIBLE_DEVICES so the iGPU is hidden.".to_owned(), @@ -1038,7 +1041,14 @@ fn check_9_igpu_dgpu_collision(e: &Examination, symptom: &str) -> Diagnosis { fix_id: "fix-9-igpu-dgpu".to_owned(), auto_applicable: true, verify: "powershell -NoProfile -Command \"$env:HIP_VISIBLE_DEVICES=1; python -c \\\"import torch; print(torch.cuda.device_count())\\\"\"".to_owned(), - notes: vec![note], + notes: vec![ + note, + "auto-applicable here means `rocm fix fix-9-igpu-dgpu` has a runner \ + for it — but that runner only pins HIP_VISIBLE_DEVICES when you pass \ + --device-index N. Without it, `rocm fix` just prints the query that \ + identifies which index is the discrete GPU." + .to_owned(), + ], ..Fix::default() } } else { @@ -1053,11 +1063,21 @@ fn check_9_igpu_dgpu_collision(e: &Examination, symptom: &str) -> Diagnosis { ], fix_id: "fix-9-igpu-dgpu".to_owned(), // Matches the `fix-9-igpu-dgpu` FixRecipe in fix.rs (auto_applicable: - // true, runner: run_hip_visible_devices) -- `rocm fix` can already - // carry this out on Linux, so the report must not claim otherwise. + // true, runner: run_hip_visible_devices) -- `rocm fix + // fix-9-igpu-dgpu --device-index N` really does carry this out on + // Linux, so the report must not claim otherwise. An agent branches + // on this flag to decide whether to offer to run the fix or only + // print it, so a wrong value here costs more than a stale sentence. auto_applicable: true, verify: "HIP_VISIBLE_DEVICES=1 python -c \"import torch; print(torch.cuda.device_count())\"".to_owned(), - notes: vec![note], + notes: vec![ + note, + "auto-applicable here means `rocm fix fix-9-igpu-dgpu` has a runner \ + for it — but that runner only pins HIP_VISIBLE_DEVICES when you pass \ + --device-index N. Without it, `rocm fix` just prints the query that \ + identifies which index is the discrete GPU." + .to_owned(), + ], ..Fix::default() } }; @@ -2092,13 +2112,21 @@ fn catalog_covers(e: &Examination) -> bool { .any(|(_, applicable)| applicable.contains(&family)) } +/// Where to send a user when nothing in the catalog matched. +/// +/// Keyed off the *host-detected* framework, which `Examination::probe` only +/// ever sets to `pytorch`, `llama-cpp`, `unknown` or `skipped` — so those two +/// named arms plus the ROCm-core default are the whole reachable set. +/// +/// Adding a framework to `examine.rs`'s probe means adding its arm here, its +/// tracker in [`upstream_tracker`], and its name to the hand-maintained list in +/// `routing_targets_cover_every_framework_the_probe_reports`. That test reads +/// its own list rather than deriving one from `examine.rs`, so it cannot notice +/// a new framework on its own — all three edits are manual. fn route_when_no_match(e: &Examination) -> Route { let target = match e.framework.as_str() { "pytorch" => "pytorch", "llama-cpp" => "llama-cpp", - "lemonade" => "lemonade", - "ollama" => "ollama", - "lm-studio" => "lm-studio", _ => "rocm-core", }; Route { @@ -2792,6 +2820,192 @@ mod tests { assert_eq!(top.score, 100); } + /// The gate the rocm-doctor skill tells an agent to read, and the reason it + /// is not "is `matched` empty?". + /// + /// This cannot be asserted from the e2e suite: `diagnose` scores several + /// checkers from host state alone, with no symptom keyword involved, so on a + /// runner that happens to have (say) `amdgpu` blacklisted the catalog + /// explains the host no matter what symptom is passed. Constructing the + /// `Examination` is the only way to hold the premise still. + #[test] + fn sub_threshold_causes_leave_has_match_false_and_route_upstream() { + // Missing both groups scores 45 -- worth listing, not enough to + // establish. `matched` is NOT empty here, which is the whole point: an + // agent gating on emptiness would propose this fix for a host where + // nothing was established, and never route the user anywhere. + let mut e = linux_base(); + e.in_render_group = Some(false); + e.in_video_group = Some(false); + let report = diagnose(&e, "the office printer keeps jamming on page three"); + + assert!( + !report.matched.is_empty(), + "this test is pointless unless something sub-threshold was listed" + ); + assert!( + report + .matched + .iter() + .all(|d| d.score < report.min_score_for_match), + "expected every cause below {}, got {:?}", + report.min_score_for_match, + report + .matched + .iter() + .map(|d| (&d.id, d.score)) + .collect::>() + ); + assert!( + !report.has_match, + "nothing cleared the threshold, so has_match must be false -- \ + skills/rocm-doctor/ tells an agent to route upstream on exactly this" + ); + assert!( + report.route_when_no_match.url.starts_with("http"), + "the skill's rule is to hand over this tracker when nothing was \ + established, so it must name somewhere to go: {:?}", + report.route_when_no_match + ); + } + + /// The other half: the flag has to discriminate, or asserting it is free. + #[test] + fn an_established_cause_sets_has_match() { + let mut e = linux_base(); + e.in_render_group = Some(false); + e.in_video_group = Some(false); + let report = diagnose(&e, "cannot open /dev/kfd: permission denied"); + assert!( + report.has_match, + "a symptom that matches the catalog must set has_match: {:?}", + report + .matched + .iter() + .map(|d| (&d.id, d.score)) + .collect::>() + ); + } + + /// Routing must stay defined for every framework the probe can report, and + /// must not claim targets it can never reach. + /// + /// The catalog docs previously advertised lemonade / ollama / lm-studio + /// routing that no probe could ever trigger; this pins the reachable set so + /// a re-added arm has to come with a probe that reaches it. + /// + /// The list below is **hand-maintained**: its first four entries mirror what + /// `Examination::probe` sets in `examine.rs`, and the rest are names the + /// probe never returns, kept so a re-added arm for one of them shows up. + /// Neither half is derived from that code, so adding a fifth framework to + /// the probe will not fail this test — see the note on + /// [`route_when_no_match`] for the three places to edit. + #[test] + fn routing_targets_cover_every_framework_the_probe_reports() { + let mut targets = Vec::new(); + // The first four are what `rocm examine` can actually report. The last + // three never come back from the probe, and are here precisely for + // that reason: `route_when_no_match` once carried arms for them, and + // removing those arms is what this test pins. Without these values in + // the loop, restoring `"lemonade" => "lemonade"` changes nothing the + // assertion below observes, and the CLI could go back to advertising + // routing the probe can never reach with every test still green. With + // them, an unreachable arm grows the target set and fails loudly. + for framework in [ + "skipped", + "pytorch", + "llama-cpp", + "unknown", + "lemonade", + "ollama", + "lm-studio", + ] { + let e = Examination { + framework: framework.to_owned(), + ..Examination::default() + }; + let route = route_when_no_match(&e); + assert!( + route.url.starts_with("http"), + "{framework}: routed to a non-URL {:?}", + route.url + ); + targets.push(route.target); + } + targets.sort_unstable(); + targets.dedup(); + assert_eq!( + targets, + vec!["llama-cpp", "pytorch", "rocm-core"], + "the reachable routing targets changed; update the docs in \ + skills/rocm-doctor/reference.md (and the amd/skills copy) to match" + ); + } + + /// Every diagnosis must agree with the fix catalog about whether the CLI + /// can apply the fix itself. + /// + /// An agent following the rocm-doctor skill branches on `auto_applicable` + /// from `diagnose --json` to decide whether to offer to run `rocm fix` or + /// merely print the plan, while `fix::apply` dispatches on `RECIPES`. The + /// two are written in different files and nothing but this test holds them + /// together, so a drift makes the CLI contradict itself: it would advertise + /// a fix as one it can run and then refuse, or the reverse. + /// + /// Both OS families are exercised because the checkers build their `Fix` + /// per-OS and only one branch is taken per run, so a Linux-only test would + /// leave the Windows branch free to drift unobserved. + #[test] + fn every_diagnosis_agrees_with_the_fix_catalog_on_auto_applicability() { + for os in ["linux", "windows"] { + let mut e = Examination { + os_family: os.to_owned(), + ..Examination::default() + }; + // Trip several checkers at once so this covers more of the catalog + // than a single fix. + e.in_render_group = Some(false); + e.in_video_group = Some(false); + e.has_apu = true; + e.has_discrete_amd = true; + e.gpus = vec![ + Gpu { + gfx_target: "gfx1103".to_owned(), + is_amd: true, + is_apu: Some(true), + ..Gpu::default() + }, + Gpu { + gfx_target: "gfx1100".to_owned(), + is_amd: true, + is_apu: Some(false), + ..Gpu::default() + }, + ]; + + let report = diagnose(&e, "torch crashes with a segfault"); + assert!( + !report.matched.is_empty(), + "{os}: expected at least one diagnosis to compare against the catalog" + ); + for d in &report.matched { + let fix = d + .fix + .as_ref() + .unwrap_or_else(|| panic!("{os}/{}: matched with no fix", d.id)); + let catalog = crate::fix::auto_applicable_for(&fix.fix_id).unwrap_or_else(|| { + panic!("{os}/{}: emitted a fix-id not in the catalog", fix.fix_id) + }); + assert_eq!( + fix.auto_applicable, catalog, + "{os}/{}: diagnose reports auto_applicable={}, but the fix catalog \ + (what `rocm fix` actually does) says {catalog}", + fix.fix_id, fix.auto_applicable + ); + } + } + } + #[test] fn path_missing_names_the_versioned_rocm_root() { // A box whose only ROCm is a versioned root used to report an empty @@ -2878,11 +3092,84 @@ mod tests { "note must not repeat the old wrong gfx-number heuristic: {note}" ); assert!( - note.contains("Without --device-index") && note.contains("despite being marked AUTO"), + note.contains("--device-index"), "note must warn that fix-9 is a no-op without --device-index: {note}" ); } + /// The `note:` lines `render_report_text` prints for one diagnosis, in + /// order, with the ` note: ` prefix stripped. + fn rendered_notes(report: &DiagnoseReport, id: &str) -> Vec { + let text = render_report_text(report, report.matched.len()); + let lines: Vec<&str> = text.lines().collect(); + let id_line = lines + .iter() + .position(|l| l.trim_start() == format!("id: {id}")) + .unwrap_or_else(|| panic!("{id} should appear in the rendered report:\n{text}")); + lines[id_line..] + .iter() + .take_while(|l| !l.trim_start().starts_with("apply with:")) + .filter_map(|l| l.trim_start().strip_prefix("note: ")) + .map(str::to_owned) + .collect() + } + + /// An APU + discrete pairing, on the OS family given, that fires fix-9. + fn igpu_dgpu_host(os_family: &str) -> Examination { + Examination { + os_family: os_family.to_owned(), + has_apu: true, + has_discrete_amd: true, + gpus: vec![ + Gpu { + gfx_target: "gfx1103".to_owned(), + is_amd: true, + is_apu: Some(true), + ..Gpu::default() + }, + Gpu { + gfx_target: "gfx1100".to_owned(), + is_amd: true, + is_apu: Some(false), + ..Gpu::default() + }, + ], + ..Examination::default() + } + } + + #[test] + fn fix_9_states_the_device_index_caveat_exactly_once() { + // `render_report_text` prints every note on its own line, so two notes + // that both say "--device-index is required for the auto-apply to do + // anything" reach the user as two bullets saying the same thing. The + // caveat is worth stating -- once. Checked on the rendered lines rather + // than on `Fix::notes`, because the duplicate is only a defect at the + // point where it is printed, and checked on both OS branches, which + // build their `Fix` separately and have drifted apart before. + for os_family in ["linux", "windows"] { + let report = diagnose(&igpu_dgpu_host(os_family), "torch crashes with a segfault"); + let notes = rendered_notes(&report, "fix-9-igpu-dgpu"); + let caveats: Vec<&String> = notes + .iter() + .filter(|n| n.contains("--device-index")) + .collect(); + assert_eq!( + caveats.len(), + 1, + "{os_family}: the --device-index caveat must be stated once, not {}; notes: {notes:#?}", + caveats.len() + ); + let mut seen = std::collections::BTreeSet::new(); + for note in ¬es { + assert!( + seen.insert(note.clone()), + "{os_family}: fix-9 prints the same note twice: {note}" + ); + } + } + } + #[test] fn fix_9_igpu_dgpu_is_auto_applicable_on_linux() { // `check_9_igpu_dgpu_collision`'s Linux/else branch sets diff --git a/crates/rocm-core/src/fix.rs b/crates/rocm-core/src/fix.rs index acf22e88b9..2d6cea6803 100644 --- a/crates/rocm-core/src/fix.rs +++ b/crates/rocm-core/src/fix.rs @@ -757,6 +757,18 @@ fn looks_like_a_diagnosis_position(value: &str) -> bool { !digits.is_empty() && digits.chars().all(|c| c.is_ascii_digit()) } +/// Whether the CLI will apply `fix_id` itself, or `None` if it isn't a known +/// fix. `RECIPES` is the authority: `apply()` dispatches on it, so this is the +/// value any other surface describing a fix has to agree with. +/// +/// Test-only: the one production consumer is `apply()`, which reads the recipe +/// directly. This exists so `diagnose`'s tests can assert the two surfaces +/// agree without exposing `RECIPES`. +#[cfg(test)] +pub(crate) fn auto_applicable_for(fix_id: &str) -> Option { + find_recipe(fix_id).map(|r| r.auto_applicable) +} + /// List every fix-id (id, kind, OS scope, title). #[must_use] pub fn list_recipes() -> String { @@ -899,13 +911,30 @@ pub fn apply(fix_id: &str, opts: &FixOptions) -> i32 { // Consent / environment helpers // --------------------------------------------------------------------------- -fn confirm(prompt: &str, assume_yes: bool) -> bool { +/// The half of the consent decision that needs no I/O: `Some(verdict)` when the +/// answer is already determined, `None` when the user must actually be asked. +/// +/// Split out from [`confirm`] so the two rules that matter — `--yes` approves, +/// and a non-interactive shell *refuses* rather than silently proceeding — are +/// testable without a controlled stdin. A test that drove `confirm` directly +/// would block on `read_line` whenever the suite happened to run with a terminal +/// attached. +const fn consent_without_prompt(assume_yes: bool, stdin_is_terminal: bool) -> Option { if assume_yes { - return true; + return Some(true); } - if !std::io::stdin().is_terminal() { - fail!("Non-interactive shell and --yes not passed; refusing to apply."); - return false; + if !stdin_is_terminal { + return Some(false); + } + None +} + +fn confirm(prompt: &str, assume_yes: bool) -> bool { + if let Some(verdict) = consent_without_prompt(assume_yes, std::io::stdin().is_terminal()) { + if !verdict { + fail!("Non-interactive shell and --yes not passed; refusing to apply."); + } + return verdict; } print!("{prompt} [y/N]: "); let _ = std::io::stdout().flush(); @@ -1043,15 +1072,26 @@ fn run_unset_override_linux() -> i32 { let Some(home) = home_dir() else { return 0; }; - let candidates = [ + report_persistent_override(&[ home.join(".bashrc"), home.join(".bash_profile"), home.join(".zshrc"), home.join(".profile"), home.join(".config").join("fish").join("config.fish"), - ]; - let hits: Vec = candidates - .into_iter() + ]) +} + +/// Report which of `candidates` persist `HSA_OVERRIDE_GFX_VERSION`, and leave +/// every one of them exactly as it was. +/// +/// Takes the paths rather than deriving them from `$HOME`, for the same reason +/// [`pin_device_in_rc_file`] takes its rc path: it makes the promise testable +/// without mutating a process-global. And the promise needs testing — fix-2 is +/// flagged `auto_applicable`, so `skills/rocm-doctor/` has to say plainly that +/// this arm still only reports, and something has to hold that true. +fn report_persistent_override(candidates: &[PathBuf]) -> i32 { + let hits: Vec<&PathBuf> = candidates + .iter() .filter(|f| { std::fs::read_to_string(f).is_ok_and(|b| b.contains("HSA_OVERRIDE_GFX_VERSION")) }) @@ -1087,6 +1127,31 @@ fn run_unset_override_windows(opts: &FixOptions) -> i32 { } let user_val = ps_env_scope("HSA_OVERRIDE_GFX_VERSION", "User"); let machine_val = ps_env_scope("HSA_OVERRIDE_GFX_VERSION", "Machine"); + report_and_clear_override_windows(opts, &user_val, &machine_val, |prompt| { + confirm(prompt, opts.yes) + }) +} + +/// Does the reporting/clearing work for [`run_unset_override_windows`], with +/// the User/Machine scope values and the consent prompt taken as parameters +/// rather than read from `ps_env_scope`/`confirm` directly. That's what makes +/// the decline path testable without a real Windows host -- same reason +/// [`pin_device_in_rc_file`] takes its consent as a closure. +/// +/// A decline must still exit `5`: `skills/rocm-doctor/reference.md` documents +/// `5` as "user declined", and a caller (human or agent) relies on that to +/// tell "nothing happened because you said no" apart from "nothing happened +/// because there was nothing to do". Declining just the User scope can't +/// short-circuit straight to `return 5`, though -- if the Machine scope is +/// also set, the guidance for clearing it (which needs an elevated shell) +/// still has to print unconditionally, so the decline is recorded in a flag +/// and checked only once both scopes have had their say. +fn report_and_clear_override_windows( + opts: &FixOptions, + user_val: &str, + machine_val: &str, + consent: impl FnOnce(&str) -> bool, +) -> i32 { if user_val.is_empty() && machine_val.is_empty() { println!("\nNo persistent HSA_OVERRIDE_GFX_VERSION found in either the User"); println!("or Machine env scope. You're done after closing/reopening shells."); @@ -1099,12 +1164,13 @@ fn run_unset_override_windows(opts: &FixOptions) -> i32 { if !machine_val.is_empty() { println!(" Machine scope: {machine_val}"); } + let mut declined = false; if !user_val.is_empty() { println!("\nClear from the User scope (no admin needed):"); println!(" Will run: setx HSA_OVERRIDE_GFX_VERSION \"\""); if opts.dry_run { println!(" (dry-run; not executed)"); - } else if confirm("Clear HSA_OVERRIDE_GFX_VERSION from User scope?", opts.yes) { + } else if consent("Clear HSA_OVERRIDE_GFX_VERSION from User scope?") { let (rc, out, err) = run("setx", &["HSA_OVERRIDE_GFX_VERSION", ""], RUN_TIMEOUT); relay_output(&out, &err); if rc != 0 { @@ -1112,6 +1178,8 @@ fn run_unset_override_windows(opts: &FixOptions) -> i32 { return 4; } println!("Cleared from User scope. Reopen your terminal for it to take effect."); + } else { + declined = true; } } if !machine_val.is_empty() { @@ -1125,7 +1193,7 @@ fn run_unset_override_windows(opts: &FixOptions) -> i32 { "or remove it through System Properties -> Environment Variables -> System variables. This command does NOT elevate itself." ); } - 0 + if declined { 5 } else { 0 } } /// fix-6: persist the ROCm/HIP bin directory on PATH (with consent). @@ -1270,8 +1338,29 @@ fn run_hip_visible_devices_linux(opts: &FixOptions) -> i32 { fail!("Could not determine your home directory."); return 3; }; + pin_device_in_rc_file(&rc_file, idx, opts, || { + confirm(&format!("Append to {}?", rc_file.display()), opts.yes) + }) +} + +/// The part of fix-9 that actually touches the user's machine: decide whether +/// the rc file already pins a device, honour `--dry-run`, ask for consent, and +/// append. +/// +/// Taking the rc path and the consent decision as parameters keeps this — the +/// one auto-fix path on Linux that really mutates a file — testable end to end +/// against a temp file, with no `$HOME` mutation and no dependency on whether +/// stdin happens to be a terminal. Exit codes are the contract documented in +/// `skills/rocm-doctor/reference.md`: 0 applied/dry-run/already-set, 4 write +/// failed, 5 declined. +fn pin_device_in_rc_file( + rc_file: &Path, + idx: i64, + opts: &FixOptions, + consent: impl FnOnce() -> bool, +) -> i32 { let export_line = format!("export HIP_VISIBLE_DEVICES={idx}"); - if let Ok(existing) = std::fs::read_to_string(&rc_file) + if let Ok(existing) = std::fs::read_to_string(rc_file) && existing.contains("HIP_VISIBLE_DEVICES=") { println!( @@ -1286,11 +1375,11 @@ fn run_hip_visible_devices_linux(opts: &FixOptions) -> i32 { println!("(dry-run; not executed)"); return 0; } - if !confirm(&format!("Append to {}?", rc_file.display()), opts.yes) { + if !consent() { return 5; } if let Err(exc) = append_line( - &rc_file, + rc_file, "# Added by rocm examine (fix-9-igpu-dgpu)", &export_line, ) { @@ -1762,4 +1851,272 @@ mod tests { matching literals and this test's expected set together" ); } + + // ── Consent and mutation contract ────────────────────────────── + // + // `skills/rocm-doctor/reference.md` documents six exit codes, and the skill + // tells an agent it may run an auto-fix because the CLI "refuses on a + // non-interactive shell without --yes" and "confirms before mutating". + // Before these tests, codes 1, 4 and 5 were asserted nowhere in the repo and + // no test ever reached a runner that writes to disk: the two that look like + // they cover it don't — `dry_run_never_mutates_...` picks fix-2, whose Linux + // runner takes no FixOptions and never prompts, and diagnose.feature's + // preview scenario picks print-only fix-1, which returns before any runner. + // + // fix-9 is the auto-fix used here because its Linux runner is the one that + // genuinely appends to a file, and `pin_device_in_rc_file` lets that run + // against a temp path with an injected consent verdict. + + /// A unique scratch path under the workspace's test-artifact dir, following + /// the convention in `lib.rs`'s tests (no `tempfile` dev-dependency here). + fn scratch_dir(name: &str) -> PathBuf { + let dir = PathBuf::from(env!("CARGO_MANIFEST_DIR")) + .join("..") + .join("..") + .join(".rocm-work") + .join("tests") + .join("core") + .join(format!( + "fix-{name}-{}-{:?}", + std::process::id(), + std::thread::current().id() + )); + std::fs::create_dir_all(&dir).expect("failed to create scratch dir"); + dir + } + + fn pinning_opts() -> FixOptions { + FixOptions { + device_index: Some(1), + ..FixOptions::default() + } + } + + #[test] + fn yes_flag_approves_without_prompting() { + assert_eq!(consent_without_prompt(true, false), Some(true)); + assert_eq!(consent_without_prompt(true, true), Some(true)); + } + + #[test] + fn non_interactive_shell_refuses_instead_of_proceeding() { + // The rule the skill relies on: without --yes and with nothing to prompt, + // the answer is a definite NO, never an implicit yes. + assert_eq!(consent_without_prompt(false, false), Some(false)); + } + + #[test] + fn interactive_shell_without_yes_must_actually_ask() { + assert_eq!(consent_without_prompt(false, true), None); + } + + /// `auto_applicable` means "the CLI has a runner", not "the runner mutates". + /// + /// fix-2's Linux arm only reports where the override is persisted; it never + /// edits dotfiles. `auto_applicable_recipes_have_a_runner` cannot catch a + /// regression here, because fix-2 does have a runner — so without this, a + /// doc promising a `--dry-run` preview the Linux user never gets stays + /// green. That is the drift `skills/rocm-doctor/` exists to prevent. + /// + /// Asserted on the files themselves rather than on an exit code: a runner + /// that started stripping the line would still return 0. + #[test] + fn reporting_a_persistent_override_never_edits_the_rc_file() { + let dir = scratch_dir("fix2-report"); + let carries = dir.join(".bashrc"); + let clean = dir.join(".profile"); + let before = "export HSA_OVERRIDE_GFX_VERSION=10.3.0\nexport PATH=$PATH:/x\n"; + std::fs::write(&carries, before).expect("seed rc file"); + std::fs::write(&clean, "# nothing to see\n").expect("seed rc file"); + + let code = report_persistent_override(&[carries.clone(), clean.clone()]); + + assert_eq!(code, 0, "reporting is a success, not a refusal"); + assert_eq!( + std::fs::read_to_string(&carries).expect("rc file still readable"), + before, + "fix-2 on Linux must leave the user's dotfiles byte-identical — \ + skills/rocm-doctor/ tells an agent it only reports" + ); + assert_eq!( + std::fs::read_to_string(&clean).expect("rc file still readable"), + "# nothing to see\n", + "a file that never carried the override must not be touched either" + ); + std::fs::remove_dir_all(&dir).ok(); + } + + /// The Windows arm of fix-2 does mutate (it runs `setx` to clear the User + /// scope), so unlike the Linux arm above, a decline has to come back as + /// `5`, not `0` -- `skills/rocm-doctor/reference.md` documents `5` as + /// "user declined" and an agent tells that apart from "nothing to do" + /// this way. + #[test] + fn windows_decline_on_user_scope_returns_5() { + let code = + report_and_clear_override_windows(&FixOptions::default(), "10.3.0", "", |_| false); + + assert_eq!(code, 5, "a declined User-scope clear must exit 5, not 0"); + } + + /// Declining the User scope must not swallow the Machine-scope guidance: + /// a user with both scopes set still needs the elevated-shell + /// instructions printed, so the decline can't `return 5` on the spot -- + /// it has to fall through and let the Machine-scope block run first. + /// This test can't see stdout, but it pins the return code so a future + /// change that reintroduces an early `return 5` (skipping that block) + /// would have to change this assertion to keep passing. + #[test] + fn windows_decline_with_machine_scope_also_set_still_returns_5() { + let code = + report_and_clear_override_windows(&FixOptions::default(), "10.3.0", "11.0.0", |_| { + false + }); + + assert_eq!(code, 5); + } + + #[test] + fn windows_consent_granted_is_not_treated_as_a_decline() { + // `setx` isn't on this host, so a real run would report failure (`4`) + // -- the point here is only that the consent gate itself was + // reached and answered "yes", not routed as a decline. + let code = + report_and_clear_override_windows(&FixOptions::default(), "10.3.0", "", |_| true); + + assert_ne!( + code, 5, + "granted consent must never be reported as declined" + ); + } + + #[test] + fn windows_dry_run_short_circuits_before_the_consent_gate() { + let opts = FixOptions { + dry_run: true, + ..FixOptions::default() + }; + + let code = report_and_clear_override_windows(&opts, "10.3.0", "", |_| { + panic!("dry-run must not reach the consent gate") + }); + + assert_eq!(code, 0, "a dry-run preview is not a decline"); + } + + #[test] + fn windows_nothing_persisted_in_either_scope_returns_0_without_prompting() { + let code = report_and_clear_override_windows(&FixOptions::default(), "", "", |_| { + panic!("nothing to clear means no consent gate at all") + }); + + assert_eq!(code, 0); + } + + #[test] + fn declining_consent_returns_5_and_writes_nothing() { + let rc = scratch_dir("declined").join(".bashrc"); + std::fs::write(&rc, "# existing\n").expect("seed rc file"); + + let code = pin_device_in_rc_file(&rc, 1, &pinning_opts(), || false); + + assert_eq!(code, 5, "a declined fix must exit 5"); + assert_eq!( + std::fs::read_to_string(&rc).expect("read rc"), + "# existing\n", + "a declined fix must not touch the file" + ); + std::fs::remove_dir_all(rc.parent().expect("parent")).ok(); + } + + #[test] + fn dry_run_returns_0_and_writes_nothing_even_with_consent() { + let rc = scratch_dir("dryrun").join(".bashrc"); + std::fs::write(&rc, "# existing\n").expect("seed rc file"); + let opts = FixOptions { + dry_run: true, + ..pinning_opts() + }; + + // Consent would be granted; --dry-run must short-circuit before it. + let code = pin_device_in_rc_file(&rc, 1, &opts, || { + panic!("dry-run must not reach the consent gate") + }); + + assert_eq!(code, 0); + assert_eq!( + std::fs::read_to_string(&rc).expect("read rc"), + "# existing\n", + "a dry-run must not touch the file" + ); + std::fs::remove_dir_all(rc.parent().expect("parent")).ok(); + } + + #[test] + fn granting_consent_appends_the_export_and_returns_0() { + let rc = scratch_dir("granted").join(".bashrc"); + std::fs::write(&rc, "# existing\n").expect("seed rc file"); + + let code = pin_device_in_rc_file(&rc, 3, &pinning_opts(), || true); + + assert_eq!(code, 0); + let body = std::fs::read_to_string(&rc).expect("read rc"); + assert!( + body.starts_with("# existing\n"), + "existing rc content must be preserved:\n{body}" + ); + assert!( + body.contains("export HIP_VISIBLE_DEVICES=3"), + "expected the export line to be appended:\n{body}" + ); + std::fs::remove_dir_all(rc.parent().expect("parent")).ok(); + } + + #[test] + fn an_rc_file_that_already_pins_a_device_is_left_alone() { + let rc = scratch_dir("already-pinned").join(".bashrc"); + let original = "export HIP_VISIBLE_DEVICES=0\n"; + std::fs::write(&rc, original).expect("seed rc file"); + + let code = pin_device_in_rc_file(&rc, 1, &pinning_opts(), || { + panic!("an already-pinned rc file must not reach the consent gate") + }); + + assert_eq!(code, 0, "already-pinned is success, not failure"); + assert_eq!( + std::fs::read_to_string(&rc).expect("read rc"), + original, + "must not append a second, conflicting pin" + ); + std::fs::remove_dir_all(rc.parent().expect("parent")).ok(); + } + + #[test] + fn a_failed_write_returns_4_rather_than_reporting_success() { + // A directory where the rc file should be: the append fails at open(). + // This is the only way the documented "attempted but the command + // failed" code is reachable for this fix. + let dir = scratch_dir("write-fails"); + let rc = dir.join(".bashrc"); + std::fs::create_dir_all(&rc).expect("create dir in place of rc file"); + + let code = pin_device_in_rc_file(&rc, 1, &pinning_opts(), || true); + + assert_eq!(code, 4, "a failed write must exit 4, not 0"); + std::fs::remove_dir_all(&dir).ok(); + } + + #[test] + fn exit_code_1_is_unreachable_by_construction() { + // apply() returns 1 only for an auto_applicable recipe with no runner. + // `auto_applicable_recipes_have_a_runner` keeps that impossible; assert + // the reachability argument here so reference.md's row 1 is accounted + // for rather than silently untested. + assert!( + RECIPES + .iter() + .all(|r| !r.auto_applicable || r.runner.is_some()), + "an auto-applicable recipe without a runner would make exit 1 reachable" + ); + } } diff --git a/licenserc.toml b/licenserc.toml index 1e42e442c2..2ee3b77de8 100644 --- a/licenserc.toml +++ b/licenserc.toml @@ -1,6 +1,12 @@ ## hawkeye license-header enforcement configuration. ## Run locally: hawkeye check ## Docs: https://github.com/fast/hawkeye +## +## `skills/rocm-doctor/` is excluded. It is a published Agent Skill, not repo +## prose: its SKILL.md must open with a YAML frontmatter block for the skill +## loader, which pushes any header out of hawkeye's detection window, and the +## folder is vendored verbatim into the amd/skills catalog, whose skills carry +## no headers. The licence is stated in the skill's own skill-card.md instead. [header] text = """ @@ -22,6 +28,7 @@ excludes = [ ".claude/**", ".tmp-*/**", "third_party/**", + "skills/rocm-doctor/**", ] [[rules]] diff --git a/skills/rocm-doctor/SKILL.md b/skills/rocm-doctor/SKILL.md new file mode 100644 index 0000000000..b02a8535f2 --- /dev/null +++ b/skills/rocm-doctor/SKILL.md @@ -0,0 +1,208 @@ +--- +name: rocm-doctor +description: >- + Diagnoses why ROCm, the HIP SDK, PyTorch, or llama.cpp is broken on an AMD GPU + on Linux, Windows, or WSL2, then applies a low-risk fix with consent or hands + back the exact next step. Also routes Lemonade, LM Studio, and Ollama problems to the + right upstream channel. Use when the user reports that ROCm or HIP "isn't + working", torch.cuda.is_available() is False, rocminfo / hipInfo can't see the + GPU, or hits hipErrorNoBinaryForGpu, HSA_STATUS_ERROR_INVALID_ISA, "invalid + device function", "no kernel image is available", cannot open /dev/kfd, + permission denied on /dev/kfd, "ROCk module is NOT loaded", a missing + libamdhip64.so / amdhip64_6.dll / hipblas.dll / vcruntime140_1.dll, an + HSA_OVERRIDE_GFX_VERSION page fault, an iGPU+dGPU crash, a container that can't + see the GPU, or an amdgpu-install / DKMS failure. Backed by the `rocm` CLI + (`rocm examine` / `rocm diagnose` / `rocm fix`); this skill is a thin driver + over those commands, not a re-implementation. +--- + +# ROCm Doctor + +Given a "ROCm / PyTorch / llama.cpp isn't working on my AMD GPU" complaint, +identify which **known misconfiguration** is the cause and either fix it (with +consent) or hand back the exact next step. + +This skill does **not** probe or reason on its own. The `rocm` CLI owns the +probe, the closed failure-mode catalog, and the fixes; the skill just drives it +and relays the results. The catalog is a **closed list** — if the symptom +doesn't match a known mode, route the user upstream instead of guessing. + +## Scope gate — check before anything else + +Read the user's symptom and answer one question first: **is this an AMD GPU?** + +If it is **not** — an **NVIDIA / Intel / Apple** GPU — then **stop and decline**: + +- Say plainly that it is **out of scope** for this skill and why (not an AMD GPU). +- Give **no** troubleshooting for it: no commands to run, no driver or CUDA + advice, no diagnostic checklist, no "try this first" — not even generic GPU + suggestions. Point at the vendor's own docs and stop there. +- Do **not** run `rocm examine` / `rocm diagnose` / `rocm fix`. + +Being helpful here means being honest about the boundary — confidently-wrong +advice for a stack this skill does not cover is worse than no advice. Only +continue past this gate when the GPU is AMD. See +[Out of scope](#out-of-scope). + +**WSL2 is in scope** and does not stop this gate. It is a platform family of its +own, with its own catalog entries covering `/dev/dxg`, the DXCore handoff, +ROCDXG, the distro floor, the Windows host driver and WSL 1. Do not decline a +WSL2 user or route them away untouched — run the workflow as for any other +platform. The CLI scopes entries itself, so a bare-metal Linux fix is never +offered there. + +## Prerequisites + +- **The `rocm` CLI.** This skill is only a driver over it; Phase 0 below installs + it with the user's consent if `rocm --version` fails. Nothing else here is + assumed — the CLI does the probing. +- **Platform:** native Linux (in-tree `amdgpu` module + `/dev/kfd`), Windows + (HIP SDK), or WSL2 (`/dev/dxg` + the Windows host driver). NVIDIA/Intel/Apple + GPUs and clean-machine installs are out of scope (see + [Out of scope](#out-of-scope)). +- **No fixed ROCm version, GPU arch (`gfx…`), or container image is assumed** — + `rocm examine`/`diagnose` detect the installed ROCm, the GPU's `gfx` target, and + container context, and match fixes to what they find. Never hand-set + `HSA_OVERRIDE_GFX_VERSION` (or similar footgun env vars) yourself; let the CLI + decide. + +## Workflow + +Only start here once the [Scope gate](#scope-gate--check-before-anything-else) +passes — the GPU is AMD. Linux, Windows and WSL2 all run the same workflow. + +0. **Ensure the `rocm` CLI is present.** Everything below shells out to it, so + check first and install it if missing: + + ``` + rocm --version + ``` + + If that succeeds, skip to step 1. If it's not found, install it **with the + user's consent** (this fetches and runs an installer that drops the `rocm` and + `rocmd` binaries into `~/.local/bin`). Both installers default to the + `release` channel, so no channel argument is needed: + + - **Linux (x86_64 only):** + ``` + curl -fsSL https://raw.githubusercontent.com/ROCm/rocm-cli/main/install.sh | sh + ``` + - **Windows (PowerShell):** + ``` + irm https://raw.githubusercontent.com/ROCm/rocm-cli/main/install.ps1 | iex + ``` + + There is no macOS build. `install.sh` refuses anything but Linux x86_64, so + on any other host hand the user the install page and stop rather than + suggesting the command anyway. To pin an unreleased build instead, pass + `nightly` (`sh -s -- nightly`, or `$env:ROCM_CLI_CHANNEL = "nightly"`). + + After install, confirm `~/.local/bin` is on `PATH` and re-run `rocm --version`. + If it still isn't available, hand the user the install page + (https://github.com/ROCm/rocm-cli) and stop. + +1. **Diagnose.** Pass the user's error text as the symptom: + + ``` + rocm diagnose --symptom "" --json + ``` + + Read the JSON: + - `has_match` — **the gate.** True means a cause cleared the threshold and + you may propose a fix; false means nothing was established, so route + upstream. Do **not** substitute "is `matched` empty?": several checkers + open with a nonzero base score for a merely *potentially* relevant + situation (a container, an APU beside a discrete GPU), so a healthy host + still returns a non-empty `matched` of sub-threshold entries. Gating on + emptiness proposes a fix for a machine with nothing wrong. + - `matched[]` — ranked causes, each with `id`, `title`, `score` (0–100), + `evidence[]`, and a `fix` (with `fix_id`, `summary`, `commands`, `verify`, + `notes`, and the `needs_sudo` / `needs_reboot` / `needs_relogin` / + `auto_applicable` flags). `score >= 75` = high confidence; `50–74` = likely + (confirm one more piece of evidence with the user first). + - `out_of_scope` — set only when the host's platform family has no catalog + entries at all. Linux, Windows and WSL2 are all covered, so this does + **not** fire for WSL2. When it is set, nothing was checked — say so rather + than implying the machine looks fine. First, if the user's symptom clearly + names an app that ships its own runtime (Lemonade, Ollama, LM Studio), + route them to that app's tracker (see + [Framework routing](#framework-routing)) — those trackers apply regardless + of platform. Otherwise relay the `out_of_scope` message and stop. + - `route_when_no_match` — when `has_match` is false, hand the user this + upstream tracker; **do not speculate**. Note the CLI picks this target from + the *host-detected* framework, not from the symptom text — so for an app + named only in the symptom, route it yourself per + [Framework routing](#framework-routing). + +2. **Propose the fix.** Show the top match's `title`, `evidence`, plan, and + `verify` command. Only propose applying it when the user is on board. + +3. **Apply with consent.** For an auto-applicable fix: + + ``` + rocm fix # auto fixes: prompt before changing anything + rocm fix --dry-run # show the exact change, touch nothing + rocm fix --yes # required to apply in a non-interactive shell + ``` + + Only the four auto-applicable fixes are ones the CLI runs itself. The other 20 + are **print-only** (bootloader, kernel, reinstall, Windows driver, …): `rocm + fix ` just prints the plan for the user to run themselves — no prompt, and + the CLI never performs those. + + Of the four, two do not always mutate: + + - **`fix-2-unset-override` mutates on Windows only.** On Linux it reports + where the override is set and which rc files carry it, then stops — it + will not edit the user's dotfiles. So there is no prompt to answer and + nothing for `--dry-run` to preview. Tell the Linux user what to edit; do + not describe it as a change the CLI made or previewed. + - **`fix-9-igpu-dgpu` mutates only when `--device-index N` is passed.** + Without it — on either platform — the runner just prints the + `rocminfo`/`hipInfo` query that identifies the discrete GPU's index and + returns 0; there is no prompt, no `--dry-run` preview, and nothing is + pinned. Once the user knows N, re-run with + `rocm fix fix-9-igpu-dgpu --device-index N`. + +4. **Verify.** Have the user run the `verify` command from the diagnosis. + +Use `rocm examine` (or `rocm examine --json`) when you only need the host state +(GPU, driver, ROCm install, groups, framework) without a diagnosis. + +## Framework routing + +`rocm diagnose` covers frameworks that build against the **system** ROCm/HIP: + +- **PyTorch**, **llama.cpp** — in scope; diagnose normally. + +Apps that ship their **own** ROCm runtime aren't diagnosed here — route the user +to the right tracker. This routing is **yours, not the CLI's**: the host probe +reports only `pytorch`, `llama-cpp` or `unknown`, so `route_when_no_match` can +never name one of these apps. Use the list below. + +- **Lemonade** → https://github.com/lemonade-sdk/lemonade/issues +- **Ollama** → https://github.com/ollama/ollama/issues +- **LM Studio** → in-app support (no public repo) +- Anything else with no catalog match → ROCm core: + https://github.com/ROCm/ROCm/issues (this is what `route_when_no_match` + returns by default). + +## Out of scope + +- **NVIDIA / Intel / Apple Silicon GPUs**, and **fresh installs on a clean + machine** (a setup task, not a diagnosis). Exit cleanly and say so. + +WSL2 is **not** in this list. It is a supported platform family with its own +catalog entries — see [reference.md](reference.md). + +## Rules + +- Never run the workflow — or offer *any* troubleshooting, generic GPU fixes + included — for a non-AMD GPU. State it is out of scope and stop. This does + not apply to WSL2, which is in scope and runs the normal workflow. +- Never invent a fix. If `rocm diagnose` returns no match, route upstream. +- Never run a mutating fix without the user's explicit OK; prefer `--dry-run` + first. New failure modes are added to the CLI catalog, not improvised here. + +See [reference.md](reference.md) for the full closed catalog and the CLI +command/exit-code reference. diff --git a/skills/rocm-doctor/evals/evals.json b/skills/rocm-doctor/evals/evals.json new file mode 100644 index 0000000000..3cc0d31c64 --- /dev/null +++ b/skills/rocm-doctor/evals/evals.json @@ -0,0 +1,81 @@ +{ + "evaluations": [ + { + "id": "rocm-hip-no-binary-for-gpu", + "skill_should_trigger": true, + "prompt": "torch.cuda.is_available() returns False on my AMD GPU and I get 'hipErrorNoBinaryForGpu' when I run my script. What's wrong?", + "expected_behavior": [ + "Drive the `rocm` CLI to diagnose -- check `rocm --version`, offer to install it with the user's consent, then `rocm diagnose` -- instead of applying a ROCm fix invented from general knowledge", + "Hand the user the CLI install path rather than guessing, if the CLI cannot be installed here" + ], + "unexpected_behavior": [ + "Execute a mutating or sudo command without first getting the user's explicit consent" + ], + "logs_contain": [ + "rocm --version" + ] + }, + { + "id": "rocm-permission-denied-kfd", + "skill_should_trigger": true, + "prompt": "I get 'permission denied' opening /dev/kfd and rocminfo can't see my AMD GPU. How do I fix it?", + "note": "The symptom is Linux-only and the runner has no GPU, so this grades the Phase 0 probe and the consent rule, not a successful diagnosis.", + "unexpected_behavior": [ + "Execute a mutating or sudo command (usermod, modprobe, and the like) without first getting the user's explicit consent" + ], + "logs_contain": [ + "rocm --version" + ] + }, + { + "id": "rocm-lemonade-routes-upstream", + "skill_should_trigger": true, + "prompt": "The Lemonade app (from lemonade-sdk) fails to load a model on my Radeon GPU -- its bundled ROCm runtime throws an error. Where should I report this?", + "note": "Lemonade ships its own ROCm runtime. The host probe cannot detect it, so `route_when_no_match` never names it -- the skill hands over this tracker from its own routing table. The URL is the proof that it did.", + "logs_contain": [ + "lemonade-sdk/lemonade/issues" + ] + }, + { + "id": "rocm-nothing-established-routes-upstream", + "skill_should_trigger": true, + "prompt": "My AMD GPU setup feels wrong on Linux -- something about ROCm is off but I can't paste an exact error. Can you work out what's broken and fix it?", + "note": "A vague report the catalog cannot explain. `matched` can still come back non-empty with sub-threshold entries, so an agent gating on emptiness would propose a fix for a healthy host. The rule is to read `has_match` and route upstream when it is false.", + "expected_behavior": [ + "Treat `has_match` being false as 'no cause established' and route the user upstream, rather than proposing a fix from a sub-threshold entry in `matched`", + "Ask the user for the exact error text instead of inventing a diagnosis" + ], + "unexpected_behavior": [ + "Propose or apply a remediation when the CLI established no cause", + "Invent a failure mode that is not in the CLI's closed catalog" + ] + }, + { + "id": "cuda-broken-on-nvidia", + "skill_should_trigger": false, + "note": "Same symptom sentence this skill exists for, on the wrong vendor's GPU. The scope gate makes a non-AMD GPU a hard decline, so nothing should fire.", + "prompt": "torch.cuda.is_available() is False on my NVIDIA RTX 4090 and CUDA seems broken. Help me fix it." + }, + { + "id": "rocm-under-wsl2", + "skill_should_trigger": true, + "note": "ROCm vocabulary and an AMD GPU on WSL2, which is a platform family the catalog covers rather than declines. This grades that the scope gate lets WSL2 through and the normal workflow runs, and that the agent stays on what the CLI returns -- the CLI scopes entries to the platform family itself, so a remediation it did not offer means the agent went around it. Which entries those are is the CLI's business and is not restated here: this prompt's symptom reaches WSL-scoped and cross-platform entries alike.", + "prompt": "I'm running ROCm inside WSL2 on Windows and torch.cuda.is_available() is False for my AMD GPU. How do I fix it?", + "expected_behavior": [ + "Run the normal workflow for WSL2 as for any other platform, driving `rocm diagnose` rather than declining on the platform", + "Keep to the entries the CLI returns for this host, rather than substituting remediations it did not offer" + ], + "unexpected_behavior": [ + "Decline the user because they are on WSL2, or route them away to AMD's ROCm-on-WSL guide untouched", + "Offer a bare-metal Linux remediation the CLI did not return for this host, such as loading `amdgpu` or joining the render group", + "Execute a mutating or sudo command without first getting the user's explicit consent" + ] + }, + { + "id": "nvidia-container-toolkit-unrelated", + "skill_should_trigger": false, + "note": "NVIDIA GPU and a container runtime error, with no ROCm/AMD vocabulary at all -- doesn't match this skill's frontmatter description, so nothing should load. Restores the two-negative floor `rocm-under-wsl2` moved off of.", + "prompt": "My NVIDIA Docker container can't see the GPU -- 'nvidia-container-cli: initialization error: nvml error: driver/library version mismatch'. How do I get the NVIDIA Container Toolkit working?" + } + ] +} diff --git a/skills/rocm-doctor/reference.md b/skills/rocm-doctor/reference.md new file mode 100644 index 0000000000..be7179abb8 --- /dev/null +++ b/skills/rocm-doctor/reference.md @@ -0,0 +1,152 @@ +# ROCm Doctor — reference + +The closed failure-mode catalog and the CLI it drives. The catalog is +authoritative in the `rocm` CLI (`crates/rocm-core`); this file mirrors it for +humans. To add or change a failure mode, change the CLI catalog — not this doc. + +## CLI commands + +### `rocm examine [--json]` + +Inspect the host: GPU + gfx target, driver, ROCm install, render/video groups, +`/dev/kfd` + render devices, kernel modules, framework introspection, recent +amdgpu kernel-log evidence. `--json` emits the **Examination** document for +tooling. + +- It's a general system inspector, so it **always exits 0**. The verdict is the + `status` field: `ok` · `no-amd-gpu` · `wsl` · `unsupported-os` · `degraded`. + +### `rocm diagnose [--symptom ""] [--top N] [--json]` + +Match the host + symptom against the closed catalog. **Always exits 0**; read +the result from `--json`: + +- `matched[]` — ranked `{ id, title, score, evidence[], fix }`. Tiers: + `>= 75` high confidence, `50–74` likely, `< 50` weak. +- `has_match` — whether any entry cleared `min_score_for_match`. **This is the + gate, not whether `matched` is empty.** Several checkers open with a nonzero + base score for a merely *potentially* relevant situation (running in a + container, an APU alongside a discrete GPU), so a healthy host produces a + non-empty `matched` full of sub-threshold entries. Gating on emptiness + proposes a fix for a machine with nothing wrong and never routes upstream. +- `min_score_for_match` (50), `high_confidence_threshold` (75). +- `out_of_scope` — set when the host's platform family has **no catalog entries + at all**, in which case `matched` is empty. Linux, Windows and WSL2 are all + covered, so this now fires only for a platform outside those three. It means + nothing was checked — not a clean bill of health. +- `route_when_no_match` — `{ target, url }` upstream tracker to use when `has_match` is false. + +### `rocm fix [] [--yes] [--dry-run] [--device-index N]` + +Apply a fix by id (run with no id to list). Exit codes: + +| code | meaning | +| --- | --- | +| 0 | applied / dry-run / print-only plan / list | +| 1 | internal error | +| 2 | usage error (incl. unknown fix-id) | +| 3 | not applicable on this host (OS mismatch, negative `--device-index`) — nothing changed | +| 4 | attempted but the command failed | +| 5 | user declined at the prompt | + +Only four fixes are auto-applicable — `fix-2-unset-override`, +`fix-4-render-group`, `fix-6-path`, `fix-9-igpu-dgpu` — and the rest print their +plan for the user to run. Pass the **full** id (`rocm fix fix-2-unset-override`, +not `rocm fix fix-2`; a short id returns exit 2, unknown fix-id). + +**Auto-fix** in the catalog below means only what the CLI reports in +`rocm fix`'s listing: the CLI has a runner for it and will carry it out itself, +rather than printing a plan for the user. It is not a promise that the runner +mutates anything. + +The ones that do mutate print the exact command, honor `--dry-run`, refuse on a +non-interactive shell without `--yes`, and confirm first. **Two exceptions:** + +- **`fix-2-unset-override` mutates on Windows only.** Its Linux runner reports + where the override is set and which rc files carry it, then stops — it will + not edit your dotfiles. So on Linux it never prompts, `--dry-run` has nothing + to preview, and `--yes` is never read. Do not tell a Linux user a dry run + previewed a change that was never going to happen. +- **`fix-9-igpu-dgpu` mutates only when you pass `--device-index N`.** Without + it — on Linux and Windows alike — the runner just prints the + `rocminfo`/`hipInfo` query that identifies which index is the discrete GPU + and returns 0: no prompt, nothing for `--dry-run` to preview, nothing + pinned. Run `rocm fix fix-9-igpu-dgpu --device-index N` (not the bare id) + once you know N. + +## Closed catalog (24 failure modes) + +The OS column is the platform family the CLI scopes an entry to, and WSL2 is a +family of its own — not a flavour of `linux`. An entry reaches a WSL host only +by naming `wsl`, so the bare-metal Linux entries (the `amdgpu` module, +`/dev/kfd`, the render group) stop applying there automatically rather than +reporting confident nonsense. + +| id | OS | Failure mode | Typical signal | Auto-fix | +| --- | --- | --- | --- | --- | +| `fix-1-arch` | linux/windows/wsl | GPU gfx target not in the framework's build arch list | `hipErrorNoBinaryForGpu`, `HSA_STATUS_ERROR_INVALID_ISA`, "invalid device function" | no | +| `fix-2-unset-override` | linux/windows/wsl | `HSA_OVERRIDE_GFX_VERSION` set on a GPU that now has a native wheel | page faults / `OUT_OF_REGISTERS`, override set in env | yes | +| `fix-3-rocm-kernel` | linux | ROCm + distro/kernel form an unsupported triple | ROCm installed but `amdgpu` not loaded; DKMS build failure | no | +| `fix-4-render-group` | linux | User not in `render`/`video` group (or `/dev/kfd` owned by the other group) | cannot open `/dev/kfd`, permission denied | yes | +| `fix-5-amdgpu-load` | linux | `amdgpu` module not loaded (or blacklisted) | "ROCk module is NOT loaded", blacklist entry, Secure Boot | no | +| `fix-6-path` | linux/windows/wsl | ROCm/HIP binaries not on PATH after install | `rocminfo: command not found`, `hipInfo` missing from PATH | yes | +| `fix-7-stale-repos` | linux | Stale/conflicting APT/DNF repos from prior installer runs | apt 404 `repo.radeon.com`, unmet deps, ≥2 ROCm repo files | no | +| `fix-8-wheel-rocm` | linux/windows/wsl | Framework wheel built for a different ROCm major than the system | `libamdhip64.so.X` / `amdhip64_X.dll` load failure | no | +| `fix-9-igpu-dgpu` | linux/windows | iGPU enumerated alongside dGPU, destabilising the runtime | APU + discrete AMD present, `HIP_VISIBLE_DEVICES` unset, crash/segfault | yes | +| `fix-10-container` | linux | Container can't see `/dev/kfd` or `/dev/dri/renderD*` | running in docker/podman, kfd/render devices missing | no | +| `fix-11-iommu` | linux | Multi-GPU hang with IOMMU enabled | ≥2 AMD GPUs, `iommu=` not `pt`, hang/deadlock/timeout | no | +| `fix-12-installer` | linux | `amdgpu-install` left a broken DKMS / repo state | dpkg half-configured, DKMS failed, `--accept-eula` | no | +| `fix-13-hip-sdk-missing` | windows | HIP SDK not installed | no HIP SDK under Program Files, `hipInfo` not recognized | no | +| `fix-14-adrenalin-too-old` | windows | Adrenalin / kernel-mode driver too old for the HIP SDK | `hipInfo` can't enumerate, "driver too old", HSA "no agents found" | no | +| `fix-15-msvc-redist` | windows | MSVC runtime missing (HIP DLLs can't load) | `vcruntime140.dll` / `vcruntime140_1.dll` missing | no | +| `fix-17-torch-dlpack` | linux | `torch-c-dlpack-ext` loads its CUDA prebuilt on a ROCm torch, aborting vLLM's engine start at import time | vLLM engine start fails on import; error names `torch_c_dlpack_ext` or tvm_ffi's `_optional_torch_c_dlpack` | no | +| `fix-19-shm-too-small` | linux/wsl | `/dev/shm` too small for a serving workload, which needs gigabytes where a container and WSL2 both default to 64 MB | reported under 1 GiB; a data-loader worker killed by a bus error, or a failed write to a temporary file, with nothing naming shared memory | no | +| `fix-wsl-1-gpu-not-exposed` | wsl | `/dev/dxg` absent, so the distro cannot reach the GPU at all | no `/dev/dxg`; in a container, the device was never passed through | no | +| `fix-wsl-2-dxcore-missing` | wsl | `/usr/lib/wsl/lib` DXCore shims missing, so the runtime cannot reach the host driver | `/usr/lib/wsl/lib/libdxcore.so` missing (or the directory absent entirely) | no | +| `fix-wsl-3-rocdxg-missing` | wsl | ROCDXG, the ROCm-to-DXCore shim the WSL path runs on, is not installed | `librocdxg.so` not found under any ROCm install | no | +| `fix-wsl-4-rocdxg-not-linked` | wsl | `librocdxg` installed but not in the linker cache, so it is unloadable | `librocdxg` present on disk yet absent from `ldconfig -p` | no | +| `fix-wsl-5-distro-too-old` | wsl | Distro release below the floor the WSL path requires | distro release under the supported floor (e.g. Ubuntu 22.04, whose glibc 2.35 is below the 2.38 / `GLIBCXX_3.4.32` the engines need) | no | +| `fix-wsl-6-host-driver-too-old` | wsl | Windows host driver too old or absent, with the distro side already complete | the Windows host reports no AMD display adapter | no | +| `fix-wsl-7-wsl1` | wsl | Distro running under WSL 1, which exposes no GPU device at all | the running kernel is a WSL 1 kernel | no | + +Two things the numbering does not tell you. `fix-16` is a reserved handle, not a +missing row — ids are stable handles rather than positions. And the `fix-wsl-N` +entries are a parallel series, not a continuation of the numeric one, because +they answer for a different platform family. + +Linux-only: fix-3, -4, -5, -7, -10, -11, -12, -17. Windows-only: fix-13, -14, -15. +WSL-only: fix-wsl-1 through fix-wsl-7. Linux + Windows: fix-9. +Linux + WSL: fix-19. Linux + Windows + WSL: fix-1, -2, -6, -8. + +## Framework routing + +`rocm diagnose` diagnoses frameworks built against the **system** ROCm/HIP: + +- **PyTorch**, **llama.cpp** — in scope. + +Apps that ship their own runtime are routed upstream **by the skill**, from the +table below. The CLI cannot do it: `route_when_no_match` keys off the +host-detected framework, which `rocm examine` only ever reports as `pytorch`, +`llama-cpp`, `unknown` or `skipped`, so it has no arm for these apps. + +- **Lemonade** → https://github.com/lemonade-sdk/lemonade/issues +- **Ollama** → https://github.com/ollama/ollama/issues +- **LM Studio** → in-app support (no public repo) + +`route_when_no_match` itself returns one of: + +- **pytorch** → https://github.com/pytorch/pytorch/issues (tag with the rocm label) +- **llama-cpp** → https://github.com/ggml-org/llama.cpp/issues +- otherwise **rocm-core** → https://github.com/ROCm/ROCm/issues + +## Out of scope + +- NVIDIA / Intel / Apple Silicon GPUs; fresh installs on a clean machine. + +WSL2 used to sit here and no longer does: it is a supported platform family with +its own seven catalog entries. It remains a *distinct* platform — the GPU +arrives over `/dev/dxg` from the Windows host driver, not through the in-tree +`amdgpu` module or `/dev/kfd` — which is why it gets its own entries rather than +inheriting the bare-metal Linux ones. AMD's ROCm-on-WSL guide is still the right +pointer for install questions the catalog does not cover: +https://rocm.docs.amd.com/projects/radeon-ryzen/en/latest/docs/install/installryz/wsl/howto_wsl.html diff --git a/skills/rocm-doctor/skill-card.md b/skills/rocm-doctor/skill-card.md new file mode 100644 index 0000000000..c151cb6d88 --- /dev/null +++ b/skills/rocm-doctor/skill-card.md @@ -0,0 +1,17 @@ +# Skill Card + +## Description + +Diagnoses ROCm / HIP SDK / PyTorch / llama.cpp failures on AMD GPUs (Linux, +Windows, and WSL2) against a closed list of known misconfigurations, and applies a +low-risk fix with consent or routes the user to the right upstream channel. A +thin driver over the `rocm` CLI (`rocm examine` / `rocm diagnose` / `rocm fix`) — +the probe, catalog, and fixes live in the CLI, versioned with the binary. + +## Owner + +rocm-cli team (AMD) + +## License + +MIT diff --git a/tests/e2e-cucumber/README.md b/tests/e2e-cucumber/README.md index 06acf08488..23fb2c8f77 100644 --- a/tests/e2e-cucumber/README.md +++ b/tests/e2e-cucumber/README.md @@ -35,6 +35,7 @@ tests/e2e-cucumber/ │ ├── chat.feature │ ├── examine.feature │ ├── model_serving.feature +│ ├── rocm_doctor_skill.feature # skill↔CLI contract (see below) │ └── runtime_setup.feature │ ├── tests/ # test binary + step modules @@ -43,7 +44,8 @@ tests/e2e-cucumber/ │ ├── chat_steps.rs │ ├── examine_steps.rs │ ├── runtime_steps.rs -│ └── serving_steps.rs +│ ├── serving_steps.rs +│ └── skill_steps.rs │ └── src/ # shared test infrastructure ├── lib.rs @@ -242,3 +244,62 @@ The `.feature` file is both the spec and the test input — cucumber reads it at - **Behavioral language.** Feature files describe what users care about, not implementation details. How steps are implemented (mock vs real, which port, which API) stays in the step functions. - **OS-assigned ports.** The mock server binds to `127.0.0.1:0` to avoid port conflicts between tests. - **A message is asserted with the state it describes.** A `Then` step matching printed output ("no existing SDK was found", "becomes the active default runtime", "stop them with `rocm services stop `") asserts the resulting state in the same scenario — the file that was or was not written, the runtime that is or is not active, the advice's named command actually clearing the condition. Matching the text alone certifies the wording, not the truth of it, which is how a scenario passes over a CLI that is wrong about what it did. See AGENTS.md §3. + +## The rocm-doctor skill contract + +`rocm_doctor_skill.feature` is the one feature that treats a file in this repo as +an expected value. `skills/rocm-doctor/` is a published skill: this repo is its +source of truth, and the [`amd/skills`](https://github.com/amd/skills) catalog +vendors it from here. The skill owns no probe, no catalog and no fixes — all of +that ships inside the binary — so what it does own is a **contract**: which +fix-ids exist, which ones the CLI applies +itself, which machines each is for, which fields a diagnosis carries, which +confidence thresholds an agent reasons about, which verdicts `examine` can +return, and where a report goes when nothing matched. + +`diagnose.feature` covers the same commands, so the division matters: + +| | expected value lives in | catches | +|---|---|---| +| `diagnose.feature` | the test suite (e.g. `CATALOG_FIX_IDS`) | the **CLI** drifting | +| `rocm_doctor_skill.feature` | `skills/rocm-doctor/reference.md` | the **document** drifting | + +Only the second can notice that a published document has gone stale, and that is +the whole reason the feature exists. So anything already proven by +`diagnose.feature` — that a fix for another OS is declined without writing +anything, that the listing is complete, that a report says plainly whether a +cause was established — stays there and is **not** restated here. If a scenario +in this feature does not read `reference.md`, it is in the wrong file. + +Reading that document as test data is a deliberate, narrow exception to +**Black-box only** above: nothing is imported from the rocm-cli codebase — a +*documentation artifact* is read as test data, and that artifact is the thing +under test. `skill_steps.rs` parses the specific claims each scenario names — +the catalog table's cells, the auto-applicable prose line, the framework-routing +bullets, the diagnose fields and thresholds — rather than restating them as +constants, so editing one of those in `reference.md` starts being checked with +no code change, and every one of those parsers asserts it found something — a +heading it depends on being reworded upstream must fail loudly rather than +quietly reduce an assertion to a no-op. Free-standing prose outside those +spots — the failure-mode count in the catalog heading, the Linux-only/ +Windows-only recap line under the table, SKILL.md's "the other N are +print-only" — is not parsed, so it can drift without failing anything here. + +Two rules follow: + +- The catalog is authoritative in `crates/rocm-core/src/fix.rs`. When one of + these scenarios fails, the CLI is right and the docs are what change. +- A skill-only edit must still run this job, so `skills/**` is in the `heavy` + paths filter that gates the `e2e` job. + +No scenario asserts whether a symptom matched, in either direction — that is not +a property this suite can hold still. The catalog scores several checkers from +**host state alone**, with no symptom keyword involved, so a runner with `amdgpu` +blacklisted is explained by `fix-5-amdgpu-load` whatever it is asked about; and +on a host the catalog rules out of scope (WSL2) nothing is scored at all. An +assertion either way would encode one runner's state. + +Claims about *whether* a cause was established therefore live in +`crates/rocm-core/src/diagnose.rs`, where the `Examination` is constructed rather +than probed. What this feature asserts holds on any host, so it is clean on WSL2, +the no-GPU lane, and the GPU lanes alike. diff --git a/tests/e2e-cucumber/features/rocm_doctor_skill.feature b/tests/e2e-cucumber/features/rocm_doctor_skill.feature new file mode 100644 index 0000000000..a315a36874 --- /dev/null +++ b/tests/e2e-cucumber/features/rocm_doctor_skill.feature @@ -0,0 +1,95 @@ +Feature: The ROCm Doctor skill's instructions match the CLI they drive + + # `skills/rocm-doctor/` is a published skill this repo is the source of truth + # for; the amd/skills catalog vendors it from here. The skill owns no probe, no + # catalog and no fixes — all of that ships inside the `rocm` binary — so what + # it DOES own is a contract: which fix-ids exist, which the CLI applies itself, + # which machines each one is for, and what a diagnosis carries. + # + # The centre of this feature is the catalog diff (scenarios 1-3): the table in + # `reference.md` is parsed and compared to what `rocm fix` really reports, so a + # renamed fix-id, a re-scoped OS, a flipped auto-flag, or a 16th failure mode + # cannot merge here and silently break the published skill. Nothing else tests + # that seam — `diagnose.feature` deliberately asserts only the SHAPE of a + # diagnosis so it stays host-independent. + # + # That guarantee covers the table's cells and the auto-applicable prose line — + # what `skill_steps.rs` actually parses. Free-standing prose that only + # restates those same facts in words — the "(N failure modes)" count in the + # catalog heading, the Linux-only/Windows-only recap line beneath the table, + # SKILL.md's "the other M are print-only" — is not parsed here and can go + # stale even while the table itself stays correct. + # + # The catalog is authoritative in `crates/rocm-core`. When one of these fails, + # the CLI is right and `skills/rocm-doctor/reference.md` is what changes. + # + # Scope is deliberately narrow: EVERY scenario here compares the CLI against + # the published document, and nothing else belongs. Claims proven elsewhere + # are not restated. `diagnose.feature` already owns the CLI's own behaviour — + # that a fix meant for another OS is declined without changing anything + # (`@id:fix-inapplicable-here-is-declined-not-attempted`), that the catalog + # listing is complete against an in-test list + # (`@id:fix-catalog-is-complete`), and that a report says plainly whether a + # cause was established (`@id:diagnose-json-states-when-nothing-matched`). + # The exit-code and confirm-before-mutating contract is unit tested in + # `crates/rocm-core/src/fix.rs`, where a declined consent or a failed write + # can be driven directly. + # + # The distinction that earns this feature its place: those tests compare the + # CLI against expectations written in the test suite, so they catch the CLI + # drifting. Only these scenarios catch the DOCUMENT drifting, because only + # here is `reference.md` itself the expected value. + # + # Every scenario is a query: no GPU, no serve, no download, no mutation — so + # they all run on the blocking mock lane and need no capability tags. + + @id:skill-catalog-ids-match-cli + Scenario: skill-01 - Every remediation the skill documents is one the CLI still offers + Given the ROCm Doctor skill as it is published + When an agent asks the CLI which remediations it knows + Then the skill and the CLI describe the same set of remediations + + @id:skill-auto-fix-set-matches-cli + Scenario: skill-02 - The skill agrees with the CLI about which remediations the CLI will run itself + Given the ROCm Doctor skill as it is published + When an agent asks the CLI which remediations it knows + Then the skill and the CLI agree on which ones the CLI applies without help + + @id:skill-os-scope-matches-cli + Scenario: skill-03 - The skill agrees with the CLI about which machines each remediation applies to + Given the ROCm Doctor skill as it is published + When an agent asks the CLI which remediations it knows + Then the skill and the CLI agree on which machines each remediation is for + + @id:skill-diagnosis-matches-what-the-skill-documents + Scenario: skill-04 - A diagnosis carries everything the skill tells an agent to read + Given the ROCm Doctor skill as it is published + And a user who reports a recognised ROCm failure + When an agent asks the CLI to diagnose that report for tooling + Then the diagnosis carries every field the skill names + And its confidence thresholds are the ones the skill reasons about + + @id:skill-examine-verdicts-are-documented + Scenario: skill-05 - Inspecting the machine returns a verdict the skill documents + Given the ROCm Doctor skill as it is published + When an agent inspects the machine for tooling + Then the inspection succeeds whatever it finds + And its verdict is one the skill documents + + # The skill's standing rule is "never invent a fix — if nothing matched, route + # the user upstream". That rule is only followable if the address the CLI hands + # over is one the skill's own routing table gives, so this is the last claim in + # the document that the binary can be held to. + # + # The scenario is about the ADDRESS, not about nothing having matched. Whether + # a report goes unexplained is not a property this suite can hold still: the + # catalog scores several checkers from host state alone, so a runner with (for + # instance) amdgpu blacklisted explains the host whatever symptom it is given. + # That half is pinned in `crates/rocm-core/src/diagnose.rs`, where the host can + # be constructed instead of probed. + @id:skill-escalation-target-matches-cli + Scenario: skill-06 - The upstream tracker the CLI hands back is one the skill documents + Given the ROCm Doctor skill as it is published + And a user who reports a failure with no catalog keyword in it + When an agent asks the CLI to diagnose that report for tooling + Then the CLI routes the report to a tracker the skill documents diff --git a/tests/e2e-cucumber/tests/e2e.rs b/tests/e2e-cucumber/tests/e2e.rs index 2e372d3c86..794cd8cf47 100644 --- a/tests/e2e-cucumber/tests/e2e.rs +++ b/tests/e2e-cucumber/tests/e2e.rs @@ -38,6 +38,7 @@ mod e2e { pub mod runtime_steps; pub mod service_cleanup_steps; pub mod serving_steps; + pub mod skill_steps; pub mod storage_steps; pub mod therock_steps; pub mod tui_driver; @@ -114,6 +115,14 @@ pub struct E2eWorld { /// dir, captured logs). `Some` only for `@lifecycle` scenarios; all its paths /// are rooted in `isolated_root` so teardown removes them with the temp dir. pub lifecycle: Option, + /// Raw text of `skills/rocm-doctor/reference.md`, loaded by the rocm-doctor + /// skill scenarios. That document is the EXPECTED-value fixture for the + /// skill↔CLI contract checks — see `e2e::skill_steps`. + pub skill_reference: Option, + /// The symptom text a rocm-doctor scenario hands to `rocm diagnose`. Its own + /// field rather than borrowing `model_name`, which means a served model and + /// has nothing to do with a user's error report. + pub skill_symptom: Option, } /// One scenario's resolved expectation plus the identity needed to report it. @@ -248,6 +257,8 @@ impl Default for E2eWorld { tui: None, chat_use_mock: false, lifecycle: None, + skill_reference: None, + skill_symptom: None, } } } diff --git a/tests/e2e-cucumber/tests/e2e/skill_steps.rs b/tests/e2e-cucumber/tests/e2e/skill_steps.rs new file mode 100644 index 0000000000..ba2de1be27 --- /dev/null +++ b/tests/e2e-cucumber/tests/e2e/skill_steps.rs @@ -0,0 +1,713 @@ +// Copyright © Advanced Micro Devices, Inc., or its affiliates. +// +// SPDX-License-Identifier: MIT + +//! Contract steps for the `rocm-doctor` skill (`skills/rocm-doctor/`). +//! +//! The skill is a thin driver: the probe, the closed catalog and the fixes all +//! live in `crates/rocm-core` and ship with the binary. What the skill owns is a +//! set of literal claims about how that binary behaves. These steps drive the +//! real `rocm` through the sequence the skill prescribes and check the claims +//! still hold. +//! +//! `reference.md` is read as the EXPECTED-value fixture. That is a deliberate, +//! narrow exception to the suite's black-box rule: nothing is imported from the +//! rocm-cli codebase — a documentation artifact is read as test data, and that +//! artifact is the thing under test. + +use std::collections::{BTreeMap, BTreeSet}; +use std::path::PathBuf; + +use cucumber::{given, then, when}; + +use crate::E2eWorld; + +/// Same symptom the diagnose steps use: it keys off a `LINUX_AND_WINDOWS` +/// checker and so renders identically on either OS. See the rationale on +/// `diagnose_steps::KNOWN_SYMPTOM`. +const KNOWN_SYMPTOM: &str = "HSA_STATUS_ERROR_INVALID_ISA"; + +/// Prose with no catalog keyword in it, so nothing scores *from the symptom*. +/// +/// It does not follow that the report goes unexplained: several checkers score +/// from host state alone, so a machine with a real problem still matches. The +/// route is populated either way, which is what the scenario using this checks. +const UNMATCHED_SYMPTOM: &str = "the office printer keeps jamming on page three"; + +/// The reference states the auto-applicable set twice: once as `yes` cells in +/// the catalog table, and once in prose above it. Both are parsed, so a rename +/// applied to the table and the CLI together still fails if the prose was +/// missed — and neither side is a constant restated in this file. +const AUTO_APPLICABLE_PROSE: &str = "auto-applicable"; + +/// One catalog row, from either side of the comparison. +#[derive(Debug, PartialEq, Eq)] +struct Remediation { + /// Machines it applies to, as each side spells it. Neither side is + /// normalised, so a catalog that invents its own shorthand shows up as a + /// mismatch instead of being quietly translated into agreement. + os_scope: String, + /// Whether the CLI applies it itself, as opposed to printing a plan. + auto: bool, +} + +fn reference_md_path() -> PathBuf { + // /tests/e2e-cucumber -> /skills/rocm-doctor/reference.md + PathBuf::from(env!("CARGO_MANIFEST_DIR")) + .join("..") + .join("..") + .join("skills") + .join("rocm-doctor") + .join("reference.md") +} + +/// Read the closed-catalog table out of the skill's reference doc. +/// +/// Rows look like: +/// `| `fix-1-arch` | linux/windows/wsl | | | no |` +/// Only rows whose first cell is a backticked `fix-*` id are taken, which skips +/// the header, the separator, and the exit-code table further up the file. +fn parse_reference_catalog(md: &str) -> BTreeMap { + let mut out = BTreeMap::new(); + for line in md.lines() { + let line = line.trim(); + if !line.starts_with('|') { + continue; + } + let cells: Vec<&str> = line.trim_matches('|').split('|').map(str::trim).collect(); + if cells.len() < 5 { + continue; + } + let id = cells[0].trim_matches('`'); + if !id.starts_with("fix-") { + continue; + } + // Taken verbatim, never normalised. A shorthand like `both` used to be + // rewritten to `linux/windows` here, which would have silently dropped + // `wsl` from a linux/windows/wsl row -- the exact OS-scope mismatch + // `assert_same_os_scope` exists to catch. Unrewritten, any spelling the + // CLI does not use fails there naming the id and both values, and + // `catalog_os_scopes_use_the_cli_spellings` (tests/skill_reference.rs) + // fails sooner, in the ordinary `cargo test` set. + let os_scope = cells[1].to_owned(); + // An unrecognised cell drops the row rather than aborting the parse, so + // a reworded table surfaces as the scenario's own set diff — naming the + // ids that went missing — instead of a panic from inside the reader. + let auto = match *cells.last().expect("row has cells") { + "yes" => true, + "no" => false, + _ => continue, + }; + out.insert(id.to_owned(), Remediation { os_scope, auto }); + } + assert!( + !out.is_empty(), + "no catalog rows found in {}", + reference_md_path().display() + ); + out +} + +/// The fix-ids the reference's prose names as auto-applicable. +/// +/// The sentence spans two lines and is delimited by em dashes: +/// `Only four fixes are auto-applicable — `fix-2-…`, `fix-4-…` — and the rest…` +/// Bounding on the dashes keeps the `rocm fix fix-2-unset-override` example +/// later in the same paragraph out of the set. +fn documented_auto_prose(md: &str) -> BTreeSet { + let joined = md.replace('\n', " "); + let Some((_, after)) = joined.split_once(AUTO_APPLICABLE_PROSE) else { + panic!( + "reference.md no longer states which fixes are {AUTO_APPLICABLE_PROSE} in prose; \ + the catalog table alone cannot catch a rename that missed the prose" + ) + }; + let (_, inside) = after + .split_once('—') + .expect("the auto-applicable sentence no longer opens with an em dash"); + let (list, _) = inside + .split_once('—') + .expect("the auto-applicable sentence no longer closes with an em dash"); + let ids: BTreeSet = backticked(list) + .into_iter() + .filter(|span| span.starts_with("fix-")) + .map(str::to_owned) + .collect(); + assert!( + !ids.is_empty(), + "no fix-ids parsed out of reference.md's auto-applicable sentence: {list:?}" + ); + ids +} + +/// Read the same shape out of `rocm fix`, whose rows look like: +/// ` [ AUTO] [ linux/windows] fix-2-unset-override -- Unset ...` +fn parse_fix_listing(stdout: &str) -> BTreeMap { + let mut out = BTreeMap::new(); + for line in stdout.lines() { + let line = line.trim(); + let Some(rest) = line.strip_prefix('[') else { + continue; + }; + let Some((marker, rest)) = rest.split_once(']') else { + continue; + }; + // As above: a renamed marker drops the row, and the scenario reports it + // as an id `rocm fix` no longer offers rather than as a parser panic. + let auto = match marker.trim() { + "AUTO" => true, + "PRINT-ONLY" => false, + _ => continue, + }; + let Some((os_scope, rest)) = rest.trim_start().trim_start_matches('[').split_once(']') + else { + continue; + }; + let id = rest.split_whitespace().next().unwrap_or_default(); + if !id.starts_with("fix-") { + continue; + } + out.insert( + id.to_owned(), + Remediation { + os_scope: os_scope.trim().to_owned(), + auto, + }, + ); + } + assert!( + !out.is_empty(), + "no fix rows parsed out of the `rocm fix` listing:\n{stdout}" + ); + out +} + +// ── Reading the rest of the reference as expected values ─────────── +// +// The catalog table above is not the only claim the skill makes. It also names +// the fields a diagnosis carries, the confidence thresholds an agent reasons +// about, the verdicts `examine` can return, and where to send a report nothing +// matched. Those are parsed out of the document too, rather than restated as +// constants here: a constant only ever proves this file and the CLI agree, +// which is not the drift this feature exists to catch. Parsing also means a +// claim ADDED upstream starts being checked without touching this file. +// +// Every parser asserts it found something. A heading or bullet reworded +// upstream must fail loudly — a silent empty parse turns the assertions it +// feeds into no-ops, which is the worst outcome available here. + +/// The body of one `##`/`###` section, selected by how its heading starts. +/// Ends at the next heading of any depth. +fn section<'a>(md: &'a str, heading_starts_with: &str) -> Vec<&'a str> { + let mut body = Vec::new(); + let mut inside = false; + for line in md.lines() { + if let Some(heading) = line.strip_prefix('#') { + if inside { + break; + } + inside = heading + .trim_start_matches('#') + .trim_start() + .starts_with(heading_starts_with); + continue; + } + if inside { + body.push(line); + } + } + assert!( + !body.is_empty(), + "reference.md has no section whose heading starts {heading_starts_with:?}" + ); + body +} + +/// Every backtick-delimited span on a line, in order. +fn backticked(line: &str) -> Vec<&str> { + let mut spans = Vec::new(); + let mut rest = line; + while let Some(open) = rest.find('`') { + let after = &rest[open + 1..]; + let Some(close) = after.find('`') else { break }; + spans.push(&after[..close]); + rest = &after[close + 1..]; + } + spans +} + +/// Whether a backticked span is a bare JSON field name, as opposed to the other +/// things the reference puts in backticks (`{ id, ... }` shapes, `>= 75`, +/// `--json`). +fn is_field_name(span: &str) -> bool { + let name = span.strip_suffix("[]").unwrap_or(span); + !name.is_empty() + && name.starts_with(|c: char| c.is_ascii_lowercase()) + && name + .chars() + .all(|c| c.is_ascii_lowercase() || c.is_ascii_digit() || c == '_') +} + +/// Top-level fields of `diagnose --json` that the reference tells an agent to +/// read. +fn documented_diagnose_fields(md: &str) -> BTreeSet { + let fields: BTreeSet = section(md, "`rocm diagnose") + .iter() + .filter(|line| line.trim_start().starts_with("- ")) + .flat_map(|line| backticked(line)) + .filter(|span| is_field_name(span)) + .map(|span| span.strip_suffix("[]").unwrap_or(span).to_owned()) + .collect(); + assert!( + !fields.is_empty(), + "no diagnose fields parsed out of reference.md" + ); + fields +} + +/// The per-cause shape the reference spells out: ``matched[]`` — ranked +/// `{ id, title, score, evidence[], fix }`. +fn documented_cause_fields(md: &str) -> BTreeSet { + let fields: BTreeSet = section(md, "`rocm diagnose") + .iter() + .flat_map(|line| backticked(line)) + .find(|span| span.trim_start().starts_with('{')) + .unwrap_or_else(|| panic!("reference.md no longer spells out the shape of a matched cause")) + .trim_matches(|c| c == '{' || c == '}') + .split(',') + .map(|field| { + let field = field.trim(); + field.strip_suffix("[]").unwrap_or(field).to_owned() + }) + .filter(|field| !field.is_empty()) + .collect(); + assert!( + !fields.is_empty(), + "no per-cause fields parsed out of reference.md" + ); + fields +} + +/// Thresholds the reference states inline, as ``name` (50)`. +fn documented_thresholds(md: &str) -> BTreeMap { + let mut out = BTreeMap::new(); + for line in section(md, "`rocm diagnose") { + let mut rest = line; + while let Some(open) = rest.find('`') { + let after = &rest[open + 1..]; + let Some(close) = after.find('`') else { break }; + let (name, tail) = (&after[..close], &after[close + 1..]); + rest = tail; + if !is_field_name(name) { + continue; + } + let Some(open_paren) = tail.trim_start().strip_prefix('(') else { + continue; + }; + let Some((value, _)) = open_paren.split_once(')') else { + continue; + }; + if let Ok(parsed) = value.trim().parse::() { + out.insert(name.to_owned(), parsed); + } + } + } + assert!( + !out.is_empty(), + "no confidence thresholds parsed out of reference.md" + ); + out +} + +/// Verdicts the reference enumerates for `examine --json`'s `status`. +fn documented_verdicts(md: &str) -> BTreeSet { + let line = section(md, "`rocm examine") + .into_iter() + .find(|line| line.contains("`status`")) + .unwrap_or_else(|| panic!("reference.md no longer enumerates the `status` verdicts")); + let verdicts: BTreeSet = backticked(line) + .into_iter() + .filter(|span| *span != "status") + .map(str::to_owned) + .collect(); + assert!( + !verdicts.is_empty(), + "no examine verdicts parsed out of reference.md" + ); + verdicts +} + +/// The subset of Framework routing the reference attributes to the **CLI**. +/// +/// The section names two different things: trackers the *skill* hands over from +/// its own table (Lemonade, Ollama, LM Studio — the host probe cannot detect +/// them), and the targets `route_when_no_match` actually returns. Checking the +/// CLI's route against the union of both lets a documented-but-dead CLI target +/// pass silently, which is how the removed lemonade / ollama / lm-studio arms +/// stayed in the document. So the CLI is held to its own list. +fn documented_cli_routes(md: &str) -> BTreeMap> { + let mut out = BTreeMap::new(); + let mut inside = false; + for line in section(md, "Framework routing") { + let trimmed = line.trim(); + if trimmed.contains("`route_when_no_match`") && trimmed.ends_with(':') { + inside = true; + continue; + } + if !inside || trimmed.is_empty() { + continue; + } + let Some(bullet) = trimmed.strip_prefix("- ") else { + break; + }; + let (Some(label), url) = (route_label(bullet), route_url(bullet)) else { + continue; + }; + out.insert(label, url); + } + assert!( + !out.is_empty(), + "reference.md no longer lists the targets `route_when_no_match` returns; \ + without that list the CLI's route can only be checked against the skill's \ + own hand-off table, which is not the same claim" + ); + out +} + +/// The bolded target name on a routing bullet, or the one after the arrow. +fn route_label(bullet: &str) -> Option { + bullet + .split("**") + .nth(1) + .map(normalise_route_name) + .filter(|label| !label.is_empty()) +} + +/// The tracker URL on a routing bullet, where it gives one. +fn route_url(bullet: &str) -> Option { + bullet.find("https://").map(|at| { + bullet[at..] + .split_whitespace() + .next() + .unwrap_or_default() + .to_owned() + }) +} + +/// Lowercase, alphanumerics only: `LM Studio`, `lm-studio` and `lm.studio` all +/// collapse to the same key. +fn normalise_route_name(name: &str) -> String { + name.chars() + .filter(char::is_ascii_alphanumeric) + .map(|c| c.to_ascii_lowercase()) + .collect() +} + +/// The reference text a step loaded, or a panic naming the missing Given. +fn reference(world: &E2eWorld) -> &str { + world + .skill_reference + .as_deref() + .expect("skill reference not loaded") +} + +/// Both sides of a comparison, or a panic explaining which step is missing. +fn both_sides(world: &E2eWorld) -> (BTreeMap, BTreeMap) { + let doc = world + .skill_reference + .as_ref() + .expect("skill reference not loaded"); + let listing = world + .cli_output + .as_ref() + .expect("no `rocm fix` listing captured"); + (parse_reference_catalog(doc), parse_fix_listing(listing)) +} + +/// The parsed diagnosis report, or a panic quoting what was emitted instead. +fn diagnosis(world: &E2eWorld) -> serde_json::Value { + assert_eq!( + world.cli_rc, + Some(0), + "the skill reads the JSON, not the exit code: diagnose must always exit 0" + ); + let output = world.cli_output.as_ref().expect("no diagnose output"); + serde_json::from_str(output).expect("diagnose --json did not emit valid JSON") +} + +// ── Given ────────────────────────────────────────────────────────── + +#[given("the ROCm Doctor skill as it is published")] +async fn load_skill_reference(world: &mut E2eWorld) { + let path = reference_md_path(); + let text = std::fs::read_to_string(&path) + .unwrap_or_else(|e| panic!("failed to read {}: {e}", path.display())); + world.skill_reference = Some(text); +} + +#[given("a user who reports a recognised ROCm failure")] +async fn user_reports_known_failure(world: &mut E2eWorld) { + world.skill_symptom = Some(KNOWN_SYMPTOM.to_owned()); +} + +#[given("a user who reports a failure with no catalog keyword in it")] +async fn user_reports_unmatched_failure(world: &mut E2eWorld) { + world.skill_symptom = Some(UNMATCHED_SYMPTOM.to_owned()); +} + +// ── When ─────────────────────────────────────────────────────────── + +#[when("an agent asks the CLI which remediations it knows")] +async fn agent_lists_remediations(world: &mut E2eWorld) { + let (stdout, _, rc) = crate::run_rocm(world, &["fix"]); + world.cli_output = Some(stdout); + world.cli_rc = Some(rc); +} + +#[when("an agent asks the CLI to diagnose that report for tooling")] +async fn agent_diagnoses_for_tooling(world: &mut E2eWorld) { + let symptom = world.skill_symptom.clone().expect("no symptom set"); + let (stdout, _, rc) = crate::run_rocm(world, &["diagnose", "--symptom", &symptom, "--json"]); + world.cli_output = Some(stdout); + world.cli_rc = Some(rc); +} + +#[when("an agent inspects the machine for tooling")] +async fn agent_examines_for_tooling(world: &mut E2eWorld) { + let (stdout, _, rc) = crate::run_rocm(world, &["examine", "--json"]); + world.cli_output = Some(stdout); + world.cli_rc = Some(rc); +} + +// ── Then ─────────────────────────────────────────────────────────── + +#[then("the skill and the CLI describe the same set of remediations")] +async fn assert_same_ids(world: &mut E2eWorld) { + let (doc, cli) = both_sides(world); + let documented: Vec<&String> = doc.keys().collect(); + let offered: Vec<&String> = cli.keys().collect(); + assert_eq!( + documented, offered, + "the skill's catalog and `rocm fix` disagree.\n\ + The catalog is authoritative in crates/rocm-core — update \ + skills/rocm-doctor/reference.md to match the CLI.\n\ + documented: {documented:?}\n\ + offered: {offered:?}" + ); +} + +#[then("the skill and the CLI agree on which ones the CLI applies without help")] +async fn assert_same_auto_set(world: &mut E2eWorld) { + let (doc, cli) = both_sides(world); + for (id, offered) in &cli { + let documented = doc.get(id).unwrap_or_else(|| { + panic!( + "`rocm fix` offers {id}, which skills/rocm-doctor/reference.md does not document" + ) + }); + assert_eq!( + documented.auto, offered.auto, + "{id}: reference.md says auto-applicable={}, `rocm fix` says {}", + documented.auto, offered.auto + ); + } + // The reference says it twice — `yes` cells above, prose below — and the + // loop only checked the cells. A rename applied to the table and the CLI + // together would leave the prose stale, and the prose is what an agent + // reads before it decides whether to offer to run a fix. + let prose = documented_auto_prose(reference(world)); + let offered: BTreeSet = cli + .iter() + .filter(|(_, r)| r.auto) + .map(|(id, _)| id.clone()) + .collect(); + assert_eq!( + prose, offered, + "reference.md's prose names {prose:?} as auto-applicable, `rocm fix` offers {offered:?}" + ); +} + +#[then("the skill and the CLI agree on which machines each remediation is for")] +async fn assert_same_os_scope(world: &mut E2eWorld) { + let (doc, cli) = both_sides(world); + for (id, offered) in &cli { + let documented = doc + .get(id) + .unwrap_or_else(|| panic!("`rocm fix` offers {id}, undocumented in reference.md")); + assert_eq!( + documented.os_scope, offered.os_scope, + "{id}: reference.md scopes it to {:?}, `rocm fix` to {:?}", + documented.os_scope, offered.os_scope + ); + } +} + +#[then("the diagnosis carries every field the skill names")] +async fn assert_documented_fields_present(world: &mut E2eWorld) { + let documented = documented_diagnose_fields(reference(world)); + let per_cause = documented_cause_fields(reference(world)); + let report = diagnosis(world); + + let missing: Vec<&String> = documented + .iter() + .filter(|field| report.get(field.as_str()).is_none()) + .collect(); + assert!( + missing.is_empty(), + "diagnose --json is missing {missing:?}, which skills/rocm-doctor/reference.md \ + tells an agent to read.\n\ + The CLI is authoritative: if a field was renamed or dropped, the document \ + follows it.\n{report:#}" + ); + + // And the converse. Checking only `documented ⊆ emitted` cannot see a field + // the CLI emits that the document never mentions — which is how `has_match` + // stayed undocumented while the skill told agents to gate on `matched` being + // empty, the one substitute `rocm-core` documents as wrong. An agent reads + // the document, so a field absent from it does not exist as far as the skill + // is concerned. + let emitted = report + .as_object() + .expect("diagnose --json is not a JSON object"); + let undocumented: Vec<&String> = emitted + .keys() + .filter(|field| !documented.contains(field.as_str())) + .collect(); + assert!( + undocumented.is_empty(), + "diagnose --json emits {undocumented:?}, which skills/rocm-doctor/reference.md \ + never names — so an agent following the skill will not read it.\n\ + Document the field, or stop emitting it." + ); + + let matched = report + .get("matched") + .and_then(serde_json::Value::as_array) + .expect("diagnose JSON has no 'matched' array"); + // Deliberately not asserting `matched` is non-empty: on a host the catalog + // rules out of scope (WSL2) an empty list is the correct answer, and the + // routing scenario covers that branch. What must hold is that anything + // offered is fully readable by an agent following the skill. + // + // The individual fields of a `fix` plan are NOT checked here. The reference + // does not spell them out, so there is no documented claim to compare + // against — that is a serialization shape, and `crates/rocm-core` owns it. + for cause in matched { + let absent: Vec<&String> = per_cause + .iter() + .filter(|field| cause.get(field.as_str()).is_none()) + .collect(); + assert!( + absent.is_empty(), + "a matched cause is missing {absent:?}, which reference.md documents \ + a cause as carrying:\n{cause:#}" + ); + } +} + +#[then("its confidence thresholds are the ones the skill reasons about")] +async fn assert_thresholds(world: &mut E2eWorld) { + let documented = documented_thresholds(reference(world)); + let report = diagnosis(world); + for (field, expected) in &documented { + let actual = report + .get(field.as_str()) + .and_then(serde_json::Value::as_i64) + .unwrap_or_else(|| panic!("diagnose JSON has no numeric '{field}'")); + assert_eq!( + actual, *expected, + "{field} is {actual}, but reference.md states {expected} — and the skill's \ + workflow text reasons about that number when it grades a cause" + ); + } +} + +#[then("the CLI routes the report to a tracker the skill documents")] +async fn assert_route_is_documented(world: &mut E2eWorld) { + let documented = documented_cli_routes(reference(world)); + let report = diagnosis(world); + + // Deliberately NOT asserting `has_match == false` here, even though the + // symptom carries no catalog keyword. `diagnose` scores several checkers + // from host state alone: the GitHub-hosted Linux runner ships + // /etc/modprobe.d/blacklist-radeon-instinct.conf with amdgpu unloaded, which + // is `fix-5-amdgpu-load` at score 90 whatever symptom is passed. No symptom + // can hold that premise still, so asserting it here only encodes the + // runner's state. + // + // The premise is pinned where the host can be held still instead -- + // `diagnose::tests::sub_threshold_causes_leave_has_match_false_and_route_upstream` + // builds an Examination whose causes are all sub-threshold and asserts both + // that `has_match` is false and that the route names somewhere to go. + // + // What is left here is the half only this feature can check: that whatever + // target the CLI hands back is one the published document names. + let route = report + .get("route_when_no_match") + .expect("diagnose JSON has no 'route_when_no_match'"); + let target = route + .get("target") + .and_then(serde_json::Value::as_str) + .expect("the route carries no target"); + let url = route + .get("url") + .and_then(serde_json::Value::as_str) + .expect("the route carries no url"); + + assert!( + !url.trim().is_empty(), + "the skill's rule is to route upstream when nothing matched, so the route \ + must name somewhere to go:\n{route:#}" + ); + let known = documented + .get(&normalise_route_name(target)) + .unwrap_or_else(|| { + panic!( + "the CLI routes to {target:?}, which reference.md's Framework routing \ + section does not name. Documented: {:?}", + documented.keys().collect::>() + ) + }); + // `documented` only ever collects bullets from the `route_when_no_match` + // list, and every bullet there carries a `-> https://...` URL, so this is + // Some in practice. It stays an `expect` rather than a bare index so a + // future bullet added without a URL fails loudly here instead of being + // silently skipped. + let expected = known.as_ref().unwrap_or_else(|| { + panic!( + "reference.md names {target} in the `route_when_no_match` list without a \ + URL, so the CLI's {url:?} has nothing to be checked against" + ) + }); + assert!( + url.starts_with(expected.as_str()), + "reference.md sends a {target} report to {expected}, the CLI to {url:?}" + ); +} + +#[then("the inspection succeeds whatever it finds")] +async fn assert_examine_exits_zero(world: &mut E2eWorld) { + assert_eq!( + world.cli_rc, + Some(0), + "the skill reads `status`, not the exit code: examine must always exit 0" + ); +} + +#[then("its verdict is one the skill documents")] +async fn assert_known_verdict(world: &mut E2eWorld) { + let documented = documented_verdicts(reference(world)); + let output = world.cli_output.as_ref().expect("no examine output"); + let report: serde_json::Value = + serde_json::from_str(output).expect("examine --json did not emit valid JSON"); + let status = report + .get("status") + .and_then(serde_json::Value::as_str) + .expect("examine JSON has no 'status'"); + assert!( + documented.contains(status), + "examine reported {status:?}, which the skill does not account for. \ + reference.md enumerates {documented:?}, and an agent that meets a verdict \ + outside that list has no instruction for what to do next." + ); +} diff --git a/tests/e2e-cucumber/tests/feature_naming.rs b/tests/e2e-cucumber/tests/feature_naming.rs index af3b43df03..822181f3da 100644 --- a/tests/e2e-cucumber/tests/feature_naming.rs +++ b/tests/e2e-cucumber/tests/feature_naming.rs @@ -43,6 +43,7 @@ const FEATURE_KEYS: &[(&str, &str)] = &[ ("model_serving.feature", "serve"), ("networking.feature", "networking"), ("remote.feature", "remote"), + ("rocm_doctor_skill.feature", "skill"), // Not `runtime`: `runtime_setup.feature` owns that key, and two files // sharing one key would collide on every index (`runtime-01` in both). ("runtime_lifecycle.feature", "runtime-lifecycle"), diff --git a/tests/e2e-cucumber/tests/skill_reference.rs b/tests/e2e-cucumber/tests/skill_reference.rs new file mode 100644 index 0000000000..9d5d53e151 --- /dev/null +++ b/tests/e2e-cucumber/tests/skill_reference.rs @@ -0,0 +1,91 @@ +// Copyright © Advanced Micro Devices, Inc., or its affiliates. +// +// SPDX-License-Identifier: MIT + +//! Drift guard for the shape of `skills/rocm-doctor/reference.md`'s catalog. +//! +//! The `rocm_doctor_skill.feature` scenarios compare that table against the +//! real `rocm fix` listing, cell for cell, so a cell the table spells its own +//! way is a contract failure. Those scenarios need a built `rocm` binary and +//! only run under `cargo xtask e2e`; this file runs in the ordinary `cargo +//! test` set, so the malformed cell is caught without one. +//! +//! Narrow on purpose: the CLI is authoritative for *which* fixes exist and what +//! they are scoped to, and the feature file already checks that. All this +//! checks is that the document uses the vocabulary the CLI prints, so the +//! comparison there can stay a literal one. + +use std::collections::BTreeSet; +use std::path::{Path, PathBuf}; + +/// The platform atoms `rocm fix` prints, which it builds by joining a recipe's +/// `applies_on` with `/` (see `FixRecipe` in `crates/rocm-core/src/fix.rs`). +/// A catalog cell is any non-empty `/`-joined subset of these. +const PLATFORM_ATOMS: &[&str] = &["linux", "windows", "wsl"]; + +fn reference_md_path() -> PathBuf { + Path::new(env!("CARGO_MANIFEST_DIR")) + .join("..") + .join("..") + .join("skills") + .join("rocm-doctor") + .join("reference.md") +} + +/// `(fix-id, os-scope cell)` for every catalog row, taken the same way +/// `skill_steps::parse_reference_catalog` takes them: a leading `|`, at least +/// five cells, and a backticked `fix-*` id first. +fn catalog_rows(md: &str) -> Vec<(String, String)> { + let mut rows = Vec::new(); + for line in md.lines() { + let line = line.trim(); + if !line.starts_with('|') { + continue; + } + let cells: Vec<&str> = line.trim_matches('|').split('|').map(str::trim).collect(); + if cells.len() < 5 { + continue; + } + let id = cells[0].trim_matches('`'); + if !id.starts_with("fix-") { + continue; + } + rows.push((id.to_owned(), cells[1].to_owned())); + } + rows +} + +#[test] +fn catalog_os_scopes_use_the_cli_spellings() { + // A shorthand such as `both` reads as "every platform" but cannot say + // which, so any mapping of it back onto the CLI's atoms is a guess -- + // `linux/windows` for a row that meant linux/windows/wsl drops `wsl` + // silently, which is precisely the mismatch the skill contract exists to + // catch. The document does not get to invent one: it spells the platforms + // out, the comparison stays literal, and nothing has to be normalised. + let md = std::fs::read_to_string(reference_md_path()) + .unwrap_or_else(|e| panic!("failed to read {}: {e}", reference_md_path().display())); + let rows = catalog_rows(&md); + assert!( + !rows.is_empty(), + "no catalog rows found in {}", + reference_md_path().display() + ); + + let allowed: BTreeSet<&str> = PLATFORM_ATOMS.iter().copied().collect(); + let bad: Vec<&(String, String)> = rows + .iter() + .filter(|(_, scope)| { + scope.is_empty() || scope.split('/').any(|atom| !allowed.contains(atom)) + }) + .collect(); + assert!( + bad.is_empty(), + "skills/rocm-doctor/reference.md scopes these fixes with words `rocm fix` never prints: \ + {bad:?}.\n\ + An OS-scope cell must be a `/`-joined list of {PLATFORM_ATOMS:?}, matching the recipe's \ + `applies_on` in crates/rocm-core/src/fix.rs. Spell the platforms out instead of \ + introducing a shorthand: the skill contract compares these cells literally, and a \ + shorthand would have to be translated -- which is how a dropped platform hides." + ); +}