From 31523d8af98cd9f4f5268a22f9f83262b32c28ce Mon Sep 17 00:00:00 2001 From: Eugene Volen Date: Wed, 11 Nov 2026 14:13:06 +0200 Subject: [PATCH] feat(diagnose): say per match whether the symptom text contributed `rocm diagnose --symptom` (or `--symptom-file`) scores the catalog from the examined host's state and the pasted text together, so a host-state finding can outrank the entry the paste added to. A consumer had to judge from the evidence whether the paste lay behind a match. Each entry in `matched` now carries `symptom_matched`: true when the text contributed to the entry's score or evidence, false when it came from the host's state alone. Any keyword hit counts, however weak, so it records a contribution, not that the entry explains the error. It is decided once, in `diagnose`, by running the catalog again with no symptom and comparing each entry with its counterpart (absent, a different score, or different evidence, since a hit on an entry already at the 100 cap changes only its evidence), rather than by every checker setting a flag it could forget. The field is always serialized and is false when no symptom was given, so its absence means only an older CLI. It is additive: the diagnose JSON carries no version, and `#[serde(default)]` keeps older payloads readable. The text report prints a `source: the examined host's state alone` line under each shown cause the text did not contribute to, once the text contributed to any cause in the report, including one cut off by `--top`. The wording names the examined host because `rocm remote doctor` renders the same report for a machine it reached over the network. ROCMAI-955 Signed-off-by: Eugene Volen --- README.md | 10 + apps/rocm/src/cli.rs | 3 + apps/rocm/src/diagnose_cmd.rs | 1 + apps/rocm/src/remote/doctor.rs | 46 ++++ crates/rocm-core/src/diagnose.rs | 222 +++++++++++++++++- crates/rocm-core/src/diagnose/report.rs | 118 ++++++++++ crates/rocm-core/src/diagnose/scoring.rs | 3 + crates/rocm-core/src/diagnose/windows.rs | 1 + docs/architecture.md | 4 +- docs/manual-testing.md | 28 +++ docs/testing.md | 35 +++ .../rocm-current-cli-carries-on/replies.json | 2 +- .../replies.json | 2 +- .../replies.json | 2 +- .../rocm-vllm-import-error/replies.json | 2 +- .../rocm-vllm-oom-at-startup/replies.json | 2 +- tests/e2e-cucumber/features/diagnose.feature | 44 ++++ .../e2e-cucumber/tests/e2e/diagnose_steps.rs | 189 +++++++++++++++ 18 files changed, 706 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index c08a94cb9f..e84fc07e46 100644 --- a/README.md +++ b/README.md @@ -408,6 +408,16 @@ position. :: cmd.exe: save the error to a file first rocm diagnose --symptom-file error.txt ``` +- With either flag, a finding about the examined host can still outrank the + entry the text added to, so each match says whether the text contributed to + its score or evidence: `symptom_matched` in `--json` (always present, `false` + when neither `--symptom` nor `--symptom-file` was given). Any keyword hit + counts, however weak, so `true` means the text contributed, not that the + match explains the error. In the human-readable output, a `source: the + examined host's state alone` line marks each shown match the text did not + contribute to, once the text contributed to any match in the report. That + includes one past `--top`, so all the rows shown can carry the line when the + only match the text contributed to was cut off. - On Windows, `diagnose` also checks the age of the Visual C++ runtime, not only whether it is there. The runtime in `System32` must be 14.40 or newer (`fix-21-msvc-redist-outdated`). Programs built with a recent toolset, such diff --git a/apps/rocm/src/cli.rs b/apps/rocm/src/cli.rs index 869202fe4c..35225cc1b4 100644 --- a/apps/rocm/src/cli.rs +++ b/apps/rocm/src/cli.rs @@ -100,6 +100,9 @@ pub(crate) enum Command { /// A shell reads this text before `rocm` does, so quotes, `$(...)` /// and a leading `-` in a pasted error can be changed, run, or taken /// as a flag. Use `--symptom-file` to hand over a paste untouched. + /// + /// Each match then says whether this text contributed to it + /// (`symptom_matched` in --json). #[arg(long, value_name = "TEXT")] symptom: Option, /// Read the error text from a file instead, or from standard input diff --git a/apps/rocm/src/diagnose_cmd.rs b/apps/rocm/src/diagnose_cmd.rs index 0ba9254d7e..7558cadeda 100644 --- a/apps/rocm/src/diagnose_cmd.rs +++ b/apps/rocm/src/diagnose_cmd.rs @@ -531,6 +531,7 @@ mod tests { rocm_core::DiagnoseReport { has_match, matched: vec![rocm_core::Diagnosis { + symptom_matched: false, id: id.to_owned(), title: "under test".to_owned(), score, diff --git a/apps/rocm/src/remote/doctor.rs b/apps/rocm/src/remote/doctor.rs index 66e8e279e0..33379fa766 100644 --- a/apps/rocm/src/remote/doctor.rs +++ b/apps/rocm/src/remote/doctor.rs @@ -382,6 +382,7 @@ mod tests { fn real_report_with_a_fix() -> DiagnoseReport { use rocm_core::diagnose::{Diagnosis, Fix}; report(vec![Diagnosis { + symptom_matched: false, id: "dkms-mismatch".to_owned(), title: "DKMS built against another kernel".to_owned(), score: 90, @@ -590,6 +591,7 @@ mod tests { let rendered = render_report( "gpu-box", &report(vec![Diagnosis { + symptom_matched: false, id: "annotated".to_owned(), title: "a fix that explains itself".to_owned(), score: 90, @@ -630,6 +632,7 @@ mod tests { let rendered = render_report( "gpu-box", &report(vec![Diagnosis { + symptom_matched: false, id: "container".to_owned(), title: "container missing devices".to_owned(), score: 90, @@ -704,6 +707,7 @@ mod tests { let rendered = render_report( "gpu-box", &report(vec![Diagnosis { + symptom_matched: false, id: "wheels".to_owned(), title: "wrong wheels".to_owned(), score: 90, @@ -740,4 +744,46 @@ mod tests { let rendered = render_report("gpu-box", &report(vec![]), 5); assert!(rendered.starts_with("Health of gpu-box"), "{rendered}"); } + + /// The note under a cause the symptom did not contribute to describes the + /// examined host, which here is the remote one, and it marks only that + /// cause. + #[test] + fn the_symptom_note_is_true_of_a_remote_host_and_marks_only_its_cause() { + use rocm_core::diagnose::{Diagnosis, SYMPTOM_DID_NOT_CONTRIBUTE_NOTE}; + let row = |id: &str, score: i32, symptom_matched: bool| Diagnosis { + id: id.to_owned(), + title: id.to_owned(), + score, + symptom_matched, + ..Diagnosis::default() + }; + let rendered = render_report( + "gpu-box", + &report(vec![ + row("fix-host-only", 60, false), + row("fix-from-the-paste", 55, true), + ]), + 5, + ); + let block = |id: &str| -> Vec<&str> { + let marker = format!("({id})"); + rendered + .lines() + .skip_while(|l| !l.contains(&marker)) + .take_while(|l| !l.trim().is_empty()) + .collect() + }; + let marked = |id: &str| { + block(id) + .iter() + .any(|l| l.trim() == SYMPTOM_DID_NOT_CONTRIBUTE_NOTE) + }; + assert!(marked("fix-host-only"), "{rendered}"); + assert!(!marked("fix-from-the-paste"), "{rendered}"); + assert!( + !SYMPTOM_DID_NOT_CONTRIBUTE_NOTE.contains("this machine"), + "the note is printed about a remote host too, where `this machine` is the wrong one" + ); + } } diff --git a/crates/rocm-core/src/diagnose.rs b/crates/rocm-core/src/diagnose.rs index a95e9f81f8..83f5533ad9 100644 --- a/crates/rocm-core/src/diagnose.rs +++ b/crates/rocm-core/src/diagnose.rs @@ -43,7 +43,7 @@ mod wsl; pub use engine::{VLLM_IMPORT_CANONICAL_SYMPTOM, VLLM_OOM_CANONICAL_SYMPTOM}; pub use linux::{AMDGPU_MISSING_EVIDENCE, dkms_driver_plan_is_supported}; -pub use report::{render_report_text, upstream_route}; +pub use report::{SYMPTOM_DID_NOT_CONTRIBUTE_NOTE, render_report_text, upstream_route}; /// At/above this score a diagnosis is treated as a real match. pub const MIN_SCORE_FOR_MATCH: i32 = 50; @@ -91,6 +91,25 @@ pub struct Diagnosis { pub score: i32, pub evidence: Vec, pub fix: Option, + /// Whether the symptom text the user supplied contributed to this entry. + /// + /// `true` when, without the symptom, this entry would have a different score + /// or different evidence, or would not be listed at all; `false` when it + /// comes from the examined host's state alone. A host-state finding can + /// outrank the entry the symptom added to, so a reader asking "did the text + /// I pasted contribute to this entry?" reads this rather than the ranking. + /// + /// It records a contribution, not a verdict that the entry explains the + /// error: any keyword hit counts, however weak. Read `score` and `evidence` + /// for how much the text added. + /// + /// Always serialized, and `false` when no symptom was given: nothing was + /// pasted, so nothing in it contributed. Omitting it in that case would make + /// "no symptom" indistinguishable from "a CLI too old to report this field". + /// `#[serde(default)]` so a payload written before the field existed still + /// deserializes. Set by [`diagnose`], never by a checker. + #[serde(default)] + pub symptom_matched: bool, } /// Where to send a report when no catalog entry matches. @@ -200,6 +219,43 @@ pub fn vllm_import_symptom_is_diagnosable(symptom: &str) -> bool { scores_on_every_engine_host(symptom, "fix-17-torch-dlpack", run_all_checks) } +/// Set [`Diagnosis::symptom_matched`] on every entry in `matched`. +/// +/// Decided by running the same catalog again on the same machine with no +/// symptom and comparing, rather than by each checker reporting what it read. +/// Checkers fold the symptom in several ways (keyword tables, single patterns, +/// a keyword whose own weight is replaced by a fixed one, a symptom that lifts a +/// negative host score above zero), and a flag each of them had to set by hand +/// is one a new checker can forget. The comparison asks the question itself: +/// would this entry look the same without the text? Evidence is compared as +/// well as the score because a score is capped at 100, so a keyword hit on an +/// entry already at the cap changes its evidence and not its number. +/// +/// The checkers are pure functions of the examination and the symptom, so the +/// second run reads no more of the machine than the first. +fn mark_symptom_contribution(matched: &mut [Diagnosis], e: &Examination, symptom: &str) { + if symptom.is_empty() { + return; + } + let host_only = run_all_checks(e, ""); + for d in matched { + d.symptom_matched = contributed(d, &host_only); + } +} + +/// Whether the symptom contributed to `d`, given `host_only`, the same catalog +/// run on the same host with no symptom. +/// +/// It did when `d` is absent from that run, or present with a different score +/// or different evidence. Either difference alone is enough: a keyword hit on +/// an entry already at the 100 cap changes only its evidence. +fn contributed(d: &Diagnosis, host_only: &[Diagnosis]) -> bool { + host_only + .iter() + .find(|h| h.id == d.id) + .is_none_or(|h| h.score != d.score || h.evidence != d.evidence) +} + /// Diagnose an examination against the closed catalog. #[must_use] pub fn diagnose(e: &Examination, symptom: &str) -> DiagnoseReport { @@ -221,6 +277,7 @@ pub fn diagnose(e: &Examination, symptom: &str) -> DiagnoseReport { } else { run_all_checks(e, symptom) }; + mark_symptom_contribution(&mut matched, e, symptom); take_applicability_from_the_catalog(&mut matched, platform_family(e)); DiagnoseReport { has_match: any_cleared_threshold(&matched), @@ -507,4 +564,167 @@ mod tests { // And a bare framework mention the engine would reject is not diagnosable. assert!(!vllm_oom_symptom_is_diagnosable("vllm: out of memory")); } + + /// The entry with `id` in `report`, or a panic naming what was listed. + fn entry<'a>(report: &'a DiagnoseReport, id: &str) -> &'a Diagnosis { + report + .matched + .iter() + .find(|d| d.id == id) + .unwrap_or_else(|| { + panic!( + "{id} was not listed: {:?}", + report + .matched + .iter() + .map(|d| (&d.id, d.score)) + .collect::>() + ) + }) + } + + fn with_gpu_override() -> Examination { + let mut e = linux_base(); + e.env + .insert("HSA_OVERRIDE_GFX_VERSION".to_owned(), "11.0.0".to_owned()); + e + } + + const ISA_PASTE: &str = "hipErrorNoBinaryForGpu: HSA_STATUS_ERROR_INVALID_ISA"; + + /// The case the field exists for: a finding about the machine sits beside + /// the entry the paste scores, and the paste contributed to only one of them. + #[test] + fn a_host_state_finding_is_not_credited_to_a_paste_about_something_else() { + let report = diagnose(&with_gpu_override(), ISA_PASTE); + + let host = entry(&report, "fix-2-unset-override"); + assert!( + !host.symptom_matched, + "the override finding comes from the environment alone: {host:?}" + ); + let pasted = entry(&report, "fix-1-arch"); + assert!( + pasted.symptom_matched, + "the ISA entry is scored from the paste: {pasted:?}" + ); + + // And both answers reach the document a consumer reads. + let json = serde_json::to_value(&report).unwrap(); + let field = |id: &str| { + json["matched"] + .as_array() + .unwrap() + .iter() + .find(|d| d["id"] == id) + .map(|d| d["symptom_matched"].clone()) + }; + assert_eq!(field("fix-2-unset-override"), Some(false.into())); + assert_eq!(field("fix-1-arch"), Some(true.into())); + } + + #[test] + fn a_paste_that_raises_a_host_finding_is_credited() { + // The override alone scores 30; a page fault in the paste adds to it. + let report = diagnose(&with_gpu_override(), "memory access page fault"); + let raised = entry(&report, "fix-2-unset-override"); + assert!( + raised.score > 30, + "the paste must have raised it: {raised:?}" + ); + assert!(raised.symptom_matched, "{raised:?}"); + } + + #[test] + fn a_paste_that_lifts_an_entry_the_host_alone_ruled_out_is_credited() { + // The framework already carries this GPU's target, which pulls the + // arch entry below zero on its own, so it is not listed without the + // paste at all. With it, the entry is listed, and only because of it. + let mut e = linux_base(); + e.gpus = vec![crate::examine::Gpu { + gfx_target: "gfx1100".to_owned(), + is_amd: true, + ..crate::examine::Gpu::default() + }]; + e.framework_arch_list = vec!["gfx1100".to_owned()]; + assert!( + diagnose(&e, "") + .matched + .iter() + .all(|d| d.id != "fix-1-arch"), + "the premise: without the paste this entry is not listed" + ); + let report = diagnose(&e, ISA_PASTE); + assert!(entry(&report, "fix-1-arch").symptom_matched); + } + + #[test] + fn with_no_symptom_every_entry_says_so_and_the_field_is_still_written() { + for symptom in ["", " "] { + let report = diagnose(&with_gpu_override(), symptom); + assert!( + !report.matched.is_empty(), + "this test is pointless unless something was listed" + ); + assert!( + report.matched.iter().all(|d| !d.symptom_matched), + "nothing was pasted for {symptom:?}, so nothing in it contributed: {:?}", + report.matched + ); + let json = serde_json::to_value(&report).unwrap(); + for d in json["matched"].as_array().unwrap() { + assert_eq!( + d.get("symptom_matched"), + Some(&serde_json::Value::Bool(false)), + "the field is written, not omitted, when no symptom was given: {d}" + ); + } + } + } + + #[test] + fn a_payload_written_before_the_field_existed_still_reads() { + let old = + r#"{"id":"fix-2-unset-override","title":"t","score":30,"evidence":[],"fix":null}"#; + let d: Diagnosis = serde_json::from_str(old).expect("an older payload deserializes"); + assert!(!d.symptom_matched); + } + + fn finding(id: &str, score: i32, evidence: &[&str]) -> Diagnosis { + Diagnosis { + id: id.to_owned(), + score, + evidence: evidence.iter().map(|&e| e.to_owned()).collect(), + ..Diagnosis::default() + } + } + + /// The three ways the symptom can show in an entry, one at a time, so each + /// half of the comparison has a case that only it decides. + #[test] + fn contribution_is_any_difference_from_the_run_without_the_symptom() { + let host_only = [finding("fix-a", 30, &["host fact"])]; + + // Same entry, same evidence, higher score: the score alone differs. + assert!(contributed( + &finding("fix-a", 55, &["host fact"]), + &host_only + )); + // Same score, extra evidence: what a keyword hit does to an entry + // already at the 100 cap. The evidence alone differs. + assert!(contributed( + &finding("fix-a", 30, &["host fact", "error mentions X"]), + &host_only + )); + // Not listed at all without the symptom. + assert!(contributed( + &finding("fix-b", 50, &["error mentions Y"]), + &host_only + )); + // Identical to the run without it: nothing contributed. + assert!(!contributed( + &finding("fix-a", 30, &["host fact"]), + &host_only + )); + } } diff --git a/crates/rocm-core/src/diagnose/report.rs b/crates/rocm-core/src/diagnose/report.rs index 9e0f355305..0ea6614080 100644 --- a/crates/rocm-core/src/diagnose/report.rs +++ b/crates/rocm-core/src/diagnose/report.rs @@ -70,6 +70,17 @@ fn is_advice_only(fix_id: &str) -> bool { fix_id == crate::windows_driver::DISPLAY_DRIVER_OLD_FIX_ID } +/// The note under a cause the symptom text did not contribute to. +/// +/// Printed when the symptom contributed to some other cause in the same +/// report: the text form of [`super::Diagnosis::symptom_matched`] being +/// `false`. +/// +/// "The examined host", not "this machine": `rocm remote doctor` renders the +/// same report for a machine it reached over the network. +pub const SYMPTOM_DID_NOT_CONTRIBUTE_NOTE: &str = + "source: the examined host's state alone; the symptom text did not contribute to it"; + /// Render the human-facing diagnosis view (mirrors `diagnose.py`'s text output). #[must_use] pub fn render_report_text(report: &DiagnoseReport, top: usize) -> String { @@ -100,6 +111,7 @@ pub fn render_report_text(report: &DiagnoseReport, top: usize) -> String { "\nAdvice about this host follows. It does not explain the symptom, and nothing in it is applied automatically.\n\n", ); } + let symptom_matched_any = report.matched.iter().any(|d| d.symptom_matched); for (i, d) in report.matched.iter().take(top).enumerate() { let tier = if d.score >= HIGH_CONFIDENCE { "HIGH" @@ -124,6 +136,19 @@ pub fn render_report_text(report: &DiagnoseReport, top: usize) -> String { for ev in &d.evidence { let _ = writeln!(out, " - {ev}"); } + // Said only once the symptom contributed to something in this report: + // a finding about the host can outrank the one the symptom added to, + // and the ranking alone reads as if the symptom lay behind it. With no + // symptom, or one that contributed to nothing, every row would carry it + // and it would say nothing. + // + // `symptom_matched_any` reads the whole report, not only the rows + // `--top` shows. When the only entry the symptom contributed to is cut + // off, every row shown carries the note, which is still true of each, + // and the "Showing N of M" line below says more were found. + if symptom_matched_any && !d.symptom_matched { + let _ = writeln!(out, " {SYMPTOM_DID_NOT_CONTRIBUTE_NOTE}"); + } if let Some(fix) = &d.fix { let _ = writeln!(out, " plan: {}", fix.summary); for c in &fix.commands { @@ -216,6 +241,7 @@ mod tests { class: crate::fix::FixClass::DiagnoseOnly, ..Fix::default() }), + ..Diagnosis::default() }, Diagnosis { id: "fix-9-igpu-dgpu".to_owned(), @@ -227,6 +253,7 @@ mod tests { summary: "Pin HIP_VISIBLE_DEVICES.".to_owned(), ..Fix::default() }), + ..Diagnosis::default() }, ], has_match: true, @@ -388,6 +415,97 @@ mod tests { ); } + /// The cause block for `id` in rendered `out`: its header line and every + /// line under it up to the blank line that ends it. + fn block(out: &str, id: &str) -> String { + let marker = format!("({id})"); + out.lines() + .skip_while(|l| !l.contains(&marker)) + .take_while(|l| !l.trim().is_empty()) + .collect::>() + .join("\n") + } + + /// The note and the field it renders are asserted together: the text + /// claims the symptom did not contribute, and the report's own field is + /// what says so. + #[test] + fn a_cause_the_symptom_did_not_contribute_to_says_so_in_the_text() { + let mut e = linux_base(); + e.env + .insert("HSA_OVERRIDE_GFX_VERSION".to_owned(), "11.0.0".to_owned()); + let report = diagnose(&e, "hipErrorNoBinaryForGpu: HSA_STATUS_ERROR_INVALID_ISA"); + let out = render_report_text(&report, 20); + for d in &report.matched { + let shown = block(&out, &d.id); + assert!(!shown.is_empty(), "{} was not rendered:\n{out}", d.id); + assert_eq!( + shown.contains(SYMPTOM_DID_NOT_CONTRIBUTE_NOTE), + !d.symptom_matched, + "the note must appear exactly where the symptom did not contribute ({}):\n{out}", + d.id + ); + } + let host = report + .matched + .iter() + .find(|d| d.id == "fix-2-unset-override") + .expect("the override is listed"); + assert!(!host.symptom_matched); + assert!( + block(&out, &host.id).contains(SYMPTOM_DID_NOT_CONTRIBUTE_NOTE), + "{out}" + ); + } + + #[test] + fn without_a_symptom_no_cause_carries_the_note() { + let mut e = linux_base(); + e.env + .insert("HSA_OVERRIDE_GFX_VERSION".to_owned(), "11.0.0".to_owned()); + let report = diagnose(&e, ""); + assert!(!report.matched.is_empty(), "something must be listed"); + let out = render_report_text(&report, 20); + assert!( + !out.contains(SYMPTOM_DID_NOT_CONTRIBUTE_NOTE), + "with nothing pasted the note would sit on every cause and say nothing:\n{out}" + ); + } + + /// The rule for `--top`: the note is decided per row from the whole + /// report, so when the only entry the symptom contributed to is cut off, + /// the row shown still carries it, which is true of that row, and the + /// truncation line says there is more. + #[test] + fn a_contributed_entry_past_top_still_marks_the_rows_shown() { + let row = |id: &str, score: i32, symptom_matched: bool| Diagnosis { + id: id.to_owned(), + title: id.to_owned(), + score, + symptom_matched, + ..Diagnosis::default() + }; + let report = DiagnoseReport { + matched: vec![ + row("fix-host-only", 60, false), + row("fix-from-the-paste", 40, true), + ], + has_match: true, + min_score_for_match: MIN_SCORE_FOR_MATCH, + high_confidence_threshold: HIGH_CONFIDENCE, + route_when_no_match: upstream_route("rocm-core"), + out_of_scope: None, + model: None, + }; + let out = render_report_text(&report, 1); + assert!( + block(&out, "fix-host-only").contains(SYMPTOM_DID_NOT_CONTRIBUTE_NOTE), + "{out}" + ); + assert!(!out.contains("(fix-from-the-paste)"), "{out}"); + assert!(out.contains("Showing 1 of 2"), "{out}"); + } + #[test] fn no_match_routes_upstream() { let mut e = linux_base(); diff --git a/crates/rocm-core/src/diagnose/scoring.rs b/crates/rocm-core/src/diagnose/scoring.rs index 19871c7a5e..2473403416 100644 --- a/crates/rocm-core/src/diagnose/scoring.rs +++ b/crates/rocm-core/src/diagnose/scoring.rs @@ -465,6 +465,9 @@ pub(super) fn finalize( score: score.min(100), evidence, fix: Some(fix), + // Not a checker's call: `diagnose` sets it by comparing against a run + // with no symptom. + symptom_matched: false, } } diff --git a/crates/rocm-core/src/diagnose/windows.rs b/crates/rocm-core/src/diagnose/windows.rs index 63535a40ad..7c2988651f 100644 --- a/crates/rocm-core/src/diagnose/windows.rs +++ b/crates/rocm-core/src/diagnose/windows.rs @@ -1200,6 +1200,7 @@ mod tests { score, evidence: vec!["synthetic".to_owned()], fix: None, + ..Diagnosis::default() }, ); assert!(score < HIGH_CONFIDENCE); diff --git a/docs/architecture.md b/docs/architecture.md index a303a91f88..6dde8c6385 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -276,12 +276,12 @@ The library every first-party app and engine crate depends on. It holds host exa | [`config_proptests.rs`](../crates/rocm-core/src/config_proptests.rs) | Property tests for the persisted `config.json` (`RocmCliConfig`) and its one-shot migration from the legacy rocm-dash `config.toml`: save and load round trips, and a save that never writes a file load rejects. Each property counts how often the generator reached the edge classes (absent optionals, empty and non-ASCII strings, non-finite floats) and fails a run that never did. Declared from [`config.rs`](../crates/rocm-core/src/config.rs). | flat | | [`confirm.rs`](../crates/rocm-core/src/confirm.rs) | The shared `[y/N]` prompt (`print_prompt`, `read_answer`), `interactive_terminal`, and the `require_yes_or_interactive` guard for `--yes`-gated commands. | domain | | [`crypto.rs`](../crates/rocm-core/src/crypto.rs) | Generic RSASSA-PKCS#1 v1.5 over SHA-256 sign and verify, with no external `openssl` process. | domain | -| [`diagnose.rs`](../crates/rocm-core/src/diagnose.rs) | Root of `rocm diagnose`: the public result types, the score thresholds, `diagnose` itself, and the vLLM symptom classifiers that run the whole catalog. | domain | +| [`diagnose.rs`](../crates/rocm-core/src/diagnose.rs) | Root of `rocm diagnose`: the public result types, the score thresholds, `diagnose` itself (with `mark_symptom_contribution`, which sets each finding's `symptom_matched` by running the catalog again with no symptom and comparing), and the vLLM symptom classifiers that run the whole catalog. | domain | | [`diagnose/catalog.rs`](../crates/rocm-core/src/diagnose/catalog.rs) | The checker table: which check runs on which platform family and in what order, and how the family is resolved from an `Examination`. | domain | | [`diagnose/engine.rs`](../crates/rocm-core/src/diagnose/engine.rs) | Engine-runtime checks on Linux and WSL2: vLLM out-of-memory and import failures, a second code object manager, an undersized `/dev/shm`. | domain | | [`diagnose/host.rs`](../crates/rocm-core/src/diagnose/host.rs) | Host-configuration checks that run on more than one platform: wheel architecture, HSA override, `PATH`, wheel against system ROCm, mixed integrated and discrete GPUs, unsupported GPUs. | domain | | [`diagnose/linux.rs`](../crates/rocm-core/src/diagnose/linux.rs) | Linux-only checks (kernel and amdgpu module, device-node access, package repositories, repo-native install) and the package-manager command helpers. | domain | -| [`diagnose/report.rs`](../crates/rocm-core/src/diagnose/report.rs) | Upstream routing for a failure nothing recognised (`upstream_route`) and the human-readable report (`render_report_text`). | domain | +| [`diagnose/report.rs`](../crates/rocm-core/src/diagnose/report.rs) | Upstream routing for a failure nothing recognised (`upstream_route`) and the human-readable report (`render_report_text`), including the per-entry note under a cause the symptom did not contribute to, whose text is the exported `SYMPTOM_DID_NOT_CONTRIBUTE_NOTE` the e2e steps read too. | domain | | [`diagnose/scoring.rs`](../crates/rocm-core/src/diagnose/scoring.rs) | The keyword tables shared across platforms and the scoring over them, plus helpers that read an `Examination` or build a `Diagnosis`. | domain | | [`diagnose/test_support.rs`](../crates/rocm-core/src/diagnose/test_support.rs) | Test-only `Examination` builders shared by the tests of several `diagnose` modules. | flat | | [`diagnose/windows.rs`](../crates/rocm-core/src/diagnose/windows.rs) | Windows-only checks: the HIP SDK, the Adrenalin driver, the Visual C++ runtime and the display driver date. | domain | diff --git a/docs/manual-testing.md b/docs/manual-testing.md index e31acf3f8a..42fd1933d6 100644 --- a/docs/manual-testing.md +++ b/docs/manual-testing.md @@ -1703,3 +1703,31 @@ Expected result: the same command reading that file still reports the same causes, and a paste with a non-ASCII character in it (`é`, `→`, a typographic quote) reports the same causes as in PowerShell 7. + +## 14. Which Causes a Pasted Error Contributed To + +`rocm diagnose --symptom` scores the catalog from the examined host's state and +the pasted text together, so a finding about the host can outrank the entry the +paste added to. Each match says whether the paste contributed to its score or +evidence; any keyword hit counts, however weak, so this is not a verdict that +the match explains the error. On any host (PowerShell: +`$env:HSA_OVERRIDE_GFX_VERSION = "11.0.0"` instead of the prefix): + +```bash +HSA_OVERRIDE_GFX_VERSION=11.0.0 rocm diagnose --symptom 'HSA_STATUS_ERROR_INVALID_ISA' --json +HSA_OVERRIDE_GFX_VERSION=11.0.0 rocm diagnose --symptom 'HSA_STATUS_ERROR_INVALID_ISA' +HSA_OVERRIDE_GFX_VERSION=11.0.0 rocm diagnose --json +``` + +- In the first, every entry in `matched` carries `symptom_matched`: + `fix-2-unset-override` (raised by the override alone) is `false`, and + `fix-1-arch` (raised by the pasted ISA error) is `true`. +- In the second, `fix-2-unset-override` has a line `source: the examined + host's state alone; the symptom text did not contribute to it`, and + `fix-1-arch` does not. +- In the third, with no `--symptom`, the field is still present and `false` on + every entry, and the text form (the same command without `--json`) prints no + `source:` line. + +The e2e scenarios `diagnose-62`, `diagnose-63` and `diagnose-64` run these three +checks on every lane. diff --git a/docs/testing.md b/docs/testing.md index a2cb15ed81..68073fb335 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -4492,6 +4492,41 @@ top of that feature file: they compare two routes on the same host. cargo xtask e2e -- -n 'diagnose-(59|60|61) ' ``` +### Which causes the pasted error contributed to + +Every entry in `rocm diagnose --json`'s `matched` carries `symptom_matched`: +`true` when the symptom text (from `--symptom` or `--symptom-file`) +contributed to its score or evidence, `false` when it came from the examined +host's state alone, and `false` on every entry when no symptom was given. Any +keyword hit counts, however weak, so the field records a contribution, not that +the entry explains the error. +[diagnose.rs](../crates/rocm-core/src/diagnose.rs) decides it by running the +catalog a second time with no symptom and comparing each entry with its +counterpart (`contributed`: absent, a different score, or different evidence), +so no checker sets it by hand. The text report prints a `source: the examined +host's state alone` line under each shown entry the text did not contribute to, +once the text contributed to any entry in the report, including one cut off by +`--top`. The rules are pinned by unit tests in diagnose.rs, +[report.rs](../crates/rocm-core/src/diagnose/report.rs) and, for the same line +under `rocm remote doctor`, [doctor.rs](../apps/rocm/src/remote/doctor.rs): + +```bash +cargo nextest run -p rocm-core -E 'test(/credited|contribution_is_any|symptom|carries_the_note|says_so_in_the_text|past_top|before_the_field/)' +cargo nextest run -p rocm -E 'test(the_symptom_note_is_true_of_a_remote_host)' +``` + +`diagnose-62` (JSON), `diagnose-63` (text) and `diagnose-64` (no symptom, both +forms) in [diagnose.feature](../tests/e2e-cucumber/features/diagnose.feature) +cover it on every lane. They set an unneeded `HSA_OVERRIDE_GFX_VERSION` on the +CLI. The first two also paste an ISA error, so the override entry must read +`false` and the ISA entry `true` whatever the runner's own state; the third +pastes nothing, so every entry must read `false` and no line is printed. To run +only those three without bypassing the harness's gates: + +```bash +E2E_ONLY_LIFECYCLE=1 E2E_ALSO_RUN_IDS=diagnose-json-says-which-causes-the-paste-contributed-to,diagnose-text-marks-causes-the-paste-did-not-contribute-to,diagnose-without-a-paste-credits-no-cause-to-one cargo xtask e2e +``` + ## Model Fit Preflight `rocm diagnose --model ` answers whether a model will run on this host diff --git a/skills/rocm-doctor/evals/fixtures/rocm-current-cli-carries-on/replies.json b/skills/rocm-doctor/evals/fixtures/rocm-current-cli-carries-on/replies.json index 1d85dbe3a1..d3e6fa0ae3 100644 --- a/skills/rocm-doctor/evals/fixtures/rocm-current-cli-carries-on/replies.json +++ b/skills/rocm-doctor/evals/fixtures/rocm-current-cli-carries-on/replies.json @@ -10,7 +10,7 @@ "rc": 0 }, "diagnose_json": { - "stdout": "{\n \"matched\": [\n {\n \"id\": \"fix-1-arch\",\n \"title\": \"GPU gfx target not in framework's build arch list\",\n \"score\": 45,\n \"evidence\": [\n \"error mentions hipErrorNoBinaryForGpu\"\n ],\n \"fix\": {\n \"summary\": \"Reinstall the framework from a wheel index that includes this GPU's gfx target. Use HSA_OVERRIDE_GFX_VERSION ONLY as a temporary workaround when no native wheel exists.\",\n \"commands\": [\n \"# Recommended: a PyTorch ROCm nightly, which often carries kernels a\",\n \"# release has not shipped yet. Pick the nightly for the ROCm major you\",\n \"# have, not an older one.\",\n \"pip uninstall -y torch torchvision torchaudio\",\n \"pip install --pre torch torchvision torchaudio \\\\\\n --index-url https://download.pytorch.org/whl/nightly/rocm7.14\",\n \"# llama.cpp: rebuild with AMDGPU_TARGETS set to this GPU's gfx.\",\n \"# cmake -B build -DGGML_HIP=ON -DAMDGPU_TARGETS=\"\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": false,\n \"needs_relogin\": false,\n \"fix_id\": \"fix-1-arch\",\n \"auto_applicable\": false,\n \"class\": \"print-only\",\n \"notes\": [\n \"TheRock (rocm/TheRock) ships nightly per-gfx wheels and is the preferred fallback when the official pytorch wheel index does not yet cover your gfx target.\"\n ],\n \"verify\": \"python -c \\\"import torch; print(torch.cuda.is_available(), torch.cuda.get_arch_list())\\\"\"\n }\n }\n ],\n \"has_match\": false,\n \"min_score_for_match\": 50,\n \"high_confidence_threshold\": 75,\n \"route_when_no_match\": {\n \"target\": \"rocm-core\",\n \"url\": \"https://github.com/ROCm/ROCm/issues\"\n },\n \"out_of_scope\": null,\n \"model\": null\n}\n", + "stdout": "{\n \"matched\": [\n {\n \"id\": \"fix-1-arch\",\n \"title\": \"GPU gfx target not in framework's build arch list\",\n \"score\": 45,\n \"evidence\": [\n \"error mentions hipErrorNoBinaryForGpu\"\n ],\n \"fix\": {\n \"summary\": \"Reinstall the framework from a wheel index that includes this GPU's gfx target. Use HSA_OVERRIDE_GFX_VERSION ONLY as a temporary workaround when no native wheel exists.\",\n \"commands\": [\n \"# Recommended: a PyTorch ROCm nightly, which often carries kernels a\",\n \"# release has not shipped yet. Pick the nightly for the ROCm major you\",\n \"# have, not an older one.\",\n \"pip uninstall -y torch torchvision torchaudio\",\n \"pip install --pre torch torchvision torchaudio \\\\\\n --index-url https://download.pytorch.org/whl/nightly/rocm7.14\",\n \"# llama.cpp: rebuild with AMDGPU_TARGETS set to this GPU's gfx.\",\n \"# cmake -B build -DGGML_HIP=ON -DAMDGPU_TARGETS=\"\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": false,\n \"needs_relogin\": false,\n \"fix_id\": \"fix-1-arch\",\n \"auto_applicable\": false,\n \"class\": \"print-only\",\n \"notes\": [\n \"TheRock (rocm/TheRock) ships nightly per-gfx wheels and is the preferred fallback when the official pytorch wheel index does not yet cover your gfx target.\"\n ],\n \"verify\": \"python -c \\\"import torch; print(torch.cuda.is_available(), torch.cuda.get_arch_list())\\\"\"\n },\n \"symptom_matched\": true\n }\n ],\n \"has_match\": false,\n \"min_score_for_match\": 50,\n \"high_confidence_threshold\": 75,\n \"route_when_no_match\": {\n \"target\": \"rocm-core\",\n \"url\": \"https://github.com/ROCm/ROCm/issues\"\n },\n \"out_of_scope\": null,\n \"model\": null\n}\n", "stderr": "", "rc": 0 }, diff --git a/skills/rocm-doctor/evals/fixtures/rocm-no-vendor-wsl-host-gpu-hidden-asks-which-gpu/replies.json b/skills/rocm-doctor/evals/fixtures/rocm-no-vendor-wsl-host-gpu-hidden-asks-which-gpu/replies.json index f455edfc49..c73b2dcc66 100644 --- a/skills/rocm-doctor/evals/fixtures/rocm-no-vendor-wsl-host-gpu-hidden-asks-which-gpu/replies.json +++ b/skills/rocm-doctor/evals/fixtures/rocm-no-vendor-wsl-host-gpu-hidden-asks-which-gpu/replies.json @@ -10,7 +10,7 @@ "rc": 0 }, "diagnose_json": { - "stdout": "{\n \"matched\": [\n {\n \"id\": \"fix-wsl-3-rocdxg-missing\",\n \"title\": \"ROCDXG not installed in the distro\",\n \"score\": 65,\n \"evidence\": [\n \"librocdxg.so was not found under any ROCm install or on LD_LIBRARY_PATH\",\n \"the WSL DXCore handoff is present, so only the distro-side ROCDXG is missing\"\n ],\n \"fix\": {\n \"summary\": \"Install ROCDXG inside the distro: it is the ROCm-to-DXCore shim the WSL path runs on.\",\n \"commands\": [\n \"rocm install driver\",\n \"# Then, once the plan looks right:\",\n \"# rocm install driver --yes\"\n ],\n \"needs_sudo\": true,\n \"needs_reboot\": false,\n \"needs_relogin\": false,\n \"fix_id\": \"fix-wsl-3-rocdxg-missing\",\n \"auto_applicable\": false,\n \"class\": \"print-only\",\n \"notes\": [\n \"This downloads and installs a .deb with sudo, so `rocm fix` prints it rather than running it. `rocm install driver` shows the full plan, and checks the download against a digest pinned for that ROCDXG release.\"\n ],\n \"verify\": \"ldconfig -p | grep librocdxg\"\n }\n }\n ],\n \"has_match\": true,\n \"min_score_for_match\": 50,\n \"high_confidence_threshold\": 75,\n \"route_when_no_match\": {\n \"target\": \"rocm-core\",\n \"url\": \"https://github.com/ROCm/ROCm/issues\"\n },\n \"out_of_scope\": null,\n \"model\": null\n}\n", + "stdout": "{\n \"matched\": [\n {\n \"id\": \"fix-wsl-3-rocdxg-missing\",\n \"title\": \"ROCDXG not installed in the distro\",\n \"score\": 65,\n \"evidence\": [\n \"librocdxg.so was not found under any ROCm install or on LD_LIBRARY_PATH\",\n \"the WSL DXCore handoff is present, so only the distro-side ROCDXG is missing\"\n ],\n \"fix\": {\n \"summary\": \"Install ROCDXG inside the distro: it is the ROCm-to-DXCore shim the WSL path runs on.\",\n \"commands\": [\n \"rocm install driver\",\n \"# Then, once the plan looks right:\",\n \"# rocm install driver --yes\"\n ],\n \"needs_sudo\": true,\n \"needs_reboot\": false,\n \"needs_relogin\": false,\n \"fix_id\": \"fix-wsl-3-rocdxg-missing\",\n \"auto_applicable\": false,\n \"class\": \"print-only\",\n \"notes\": [\n \"This downloads and installs a .deb with sudo, so `rocm fix` prints it rather than running it. `rocm install driver` shows the full plan, and checks the download against a digest pinned for that ROCDXG release.\"\n ],\n \"verify\": \"ldconfig -p | grep librocdxg\"\n },\n \"symptom_matched\": false\n }\n ],\n \"has_match\": true,\n \"min_score_for_match\": 50,\n \"high_confidence_threshold\": 75,\n \"route_when_no_match\": {\n \"target\": \"rocm-core\",\n \"url\": \"https://github.com/ROCm/ROCm/issues\"\n },\n \"out_of_scope\": null,\n \"model\": null\n}\n", "stderr": "", "rc": 0 }, diff --git a/skills/rocm-doctor/evals/fixtures/rocm-older-cli-stops-before-diagnosing/replies.json b/skills/rocm-doctor/evals/fixtures/rocm-older-cli-stops-before-diagnosing/replies.json index 06e020882e..a888429434 100644 --- a/skills/rocm-doctor/evals/fixtures/rocm-older-cli-stops-before-diagnosing/replies.json +++ b/skills/rocm-doctor/evals/fixtures/rocm-older-cli-stops-before-diagnosing/replies.json @@ -10,7 +10,7 @@ "rc": 2 }, "diagnose_json": { - "stdout": "{\n \"matched\": [\n {\n \"id\": \"fix-1-arch\",\n \"title\": \"GPU gfx target not in framework's build arch list\",\n \"score\": 45,\n \"evidence\": [\n \"error mentions hipErrorNoBinaryForGpu\"\n ],\n \"fix\": {\n \"summary\": \"Reinstall the framework from a wheel index that includes this GPU's gfx target. Use HSA_OVERRIDE_GFX_VERSION ONLY as a temporary workaround when no native wheel exists.\",\n \"commands\": [\n \"# Recommended: a PyTorch ROCm nightly, which often carries kernels a\",\n \"# release has not shipped yet. Pick the nightly for the ROCm major you\",\n \"# have, not an older one.\",\n \"pip uninstall -y torch torchvision torchaudio\",\n \"pip install --pre torch torchvision torchaudio \\\\\\n --index-url https://download.pytorch.org/whl/nightly/rocm7.14\",\n \"# llama.cpp: rebuild with AMDGPU_TARGETS set to this GPU's gfx.\",\n \"# cmake -B build -DGGML_HIP=ON -DAMDGPU_TARGETS=\"\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": false,\n \"needs_relogin\": false,\n \"fix_id\": \"fix-1-arch\",\n \"auto_applicable\": false,\n \"class\": \"print-only\",\n \"notes\": [\n \"TheRock (rocm/TheRock) ships nightly per-gfx wheels and is the preferred fallback when the official pytorch wheel index does not yet cover your gfx target.\"\n ],\n \"verify\": \"python -c \\\"import torch; print(torch.cuda.is_available(), torch.cuda.get_arch_list())\\\"\"\n }\n }\n ],\n \"has_match\": false,\n \"min_score_for_match\": 50,\n \"high_confidence_threshold\": 75,\n \"route_when_no_match\": {\n \"target\": \"rocm-core\",\n \"url\": \"https://github.com/ROCm/ROCm/issues\"\n },\n \"out_of_scope\": null,\n \"model\": null\n}\n", + "stdout": "{\n \"matched\": [\n {\n \"id\": \"fix-1-arch\",\n \"title\": \"GPU gfx target not in framework's build arch list\",\n \"score\": 45,\n \"evidence\": [\n \"error mentions hipErrorNoBinaryForGpu\"\n ],\n \"fix\": {\n \"summary\": \"Reinstall the framework from a wheel index that includes this GPU's gfx target. Use HSA_OVERRIDE_GFX_VERSION ONLY as a temporary workaround when no native wheel exists.\",\n \"commands\": [\n \"# Recommended: a PyTorch ROCm nightly, which often carries kernels a\",\n \"# release has not shipped yet. Pick the nightly for the ROCm major you\",\n \"# have, not an older one.\",\n \"pip uninstall -y torch torchvision torchaudio\",\n \"pip install --pre torch torchvision torchaudio \\\\\\n --index-url https://download.pytorch.org/whl/nightly/rocm7.14\",\n \"# llama.cpp: rebuild with AMDGPU_TARGETS set to this GPU's gfx.\",\n \"# cmake -B build -DGGML_HIP=ON -DAMDGPU_TARGETS=\"\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": false,\n \"needs_relogin\": false,\n \"fix_id\": \"fix-1-arch\",\n \"auto_applicable\": false,\n \"class\": \"print-only\",\n \"notes\": [\n \"TheRock (rocm/TheRock) ships nightly per-gfx wheels and is the preferred fallback when the official pytorch wheel index does not yet cover your gfx target.\"\n ],\n \"verify\": \"python -c \\\"import torch; print(torch.cuda.is_available(), torch.cuda.get_arch_list())\\\"\"\n },\n \"symptom_matched\": true\n }\n ],\n \"has_match\": false,\n \"min_score_for_match\": 50,\n \"high_confidence_threshold\": 75,\n \"route_when_no_match\": {\n \"target\": \"rocm-core\",\n \"url\": \"https://github.com/ROCm/ROCm/issues\"\n },\n \"out_of_scope\": null,\n \"model\": null\n}\n", "stderr": "", "rc": 0 }, diff --git a/skills/rocm-doctor/evals/fixtures/rocm-vllm-import-error/replies.json b/skills/rocm-doctor/evals/fixtures/rocm-vllm-import-error/replies.json index 1c84c3611f..eaf2f7a7d9 100644 --- a/skills/rocm-doctor/evals/fixtures/rocm-vllm-import-error/replies.json +++ b/skills/rocm-doctor/evals/fixtures/rocm-vllm-import-error/replies.json @@ -10,7 +10,7 @@ "rc": 0 }, "diagnose_json": { - "stdout": "{\n \"matched\": [\n {\n \"id\": \"fix-17-torch-dlpack\",\n \"title\": \"vLLM engine start aborts on torch-c-dlpack-ext loading its CUDA variant\",\n \"score\": 50,\n \"evidence\": [\n \"error names libtorch_cuda.so, which a ROCm build of torch does not ship\"\n ],\n \"fix\": {\n \"summary\": \"Only if the engine's runtime holds a ROCm build of torch in the 2.4-2.9 range with torch-c-dlpack-ext installed: reinstall the engine so its pinned torch is restored, which moves torch off the versions the extension ships prebuilts for.\",\n \"commands\": [\n \"# --- step 1 of 3, in YOUR shell ---\",\n \"# Opens an INTERACTIVE subshell with the engine's environment active,\",\n \"# and does not return until you leave it. Run this line on its own.\",\n \"rocm engines shell vllm\",\n \"# --- step 2 of 3, INSIDE the subshell step 1 opened ---\",\n \"# Confirm the trigger before changing anything. It has to be the\",\n \"# ENGINE's interpreter, not the one on your PATH -- they are different\",\n \"# interpreters, and only the engine's decides this failure.\",\n \"python -c \\\"import torch; print(torch.__version__, torch.version.hip)\\\"\",\n \"python -c \\\"import importlib.metadata as m; print(m.version('torch-c-dlpack-ext'))\\\"\",\n \"# This entry applies ONLY when torch.version.hip is set, torch.__version__\",\n \"# is in the 2.4-2.9 range, and torch-c-dlpack-ext is installed. Outside\",\n \"# that range the extension raises a handled ImportError and this is not\",\n \"# the failure you are looking at. Then leave the subshell:\",\n \"exit\",\n \"# --- step 3 of 3, back in YOUR OWN shell ---\",\n \"# If all three held, put the engine's pinned torch back. Do NOT run\",\n \"# this from inside the subshell: it replaces the environment that\",\n \"# shell is standing in.\",\n \"rocm engines install vllm --reinstall\"\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": false,\n \"needs_relogin\": false,\n \"fix_id\": \"fix-17-torch-dlpack\",\n \"auto_applicable\": false,\n \"class\": \"print-only\",\n \"notes\": [\n \"Running vLLM on ROCm is not by itself a reason to apply this. The trigger is narrow: a ROCm build of torch in the 2.4-2.9 range (the versions torch-c-dlpack-ext ships prebuilts for), torch without a native __dlpack_c_exchange_api__, and torch-c-dlpack-ext present -- it arrives as a transitive dependency of tilelang, which vLLM pins.\",\n \"The usual way a runtime lands in that range is `rocm install sdk` being re-run after the engine was installed, which overwrites the engine's pinned torch. Reinstalling the engine puts the pin back.\",\n \"The defect is upstream and there is nothing to correct locally: torch-c-dlpack-ext picks its variant from torch.cuda.is_available(), which is True on ROCm because PyTorch reuses the torch.cuda namespace for HIP, and it ships no ROCm variant to pick. tvm_ffi imports it as optional but guards only ImportError/AttributeError, while ctypes.CDLL raises OSError -- so an explicitly optional import kills the process.\",\n \"A service that failed at startup is hidden from a plain `rocm services list`; pass --all to recover its id.\"\n ],\n \"verify\": \"rocm serve --engine vllm # then `rocm services list --all` and `rocm services logs ` to confirm the import no longer aborts\"\n }\n },\n {\n \"id\": \"fix-8-wheel-rocm\",\n \"title\": \"Framework wheel built for a different ROCm major than the system\",\n \"score\": 25,\n \"evidence\": [\n \"ldopen failure\"\n ],\n \"fix\": {\n \"summary\": \"Reinstall the framework from the wheel index that matches the system ROCm major (or upgrade the system ROCm to match the wheel).\",\n \"commands\": [\n \"pip uninstall -y torch torchvision torchaudio\",\n \"# Install the index for the ROCm major `rocm examine` reports:\",\n \"pip install torch torchvision torchaudio --index-url https://download.pytorch.org/whl/rocm7.14\",\n \"# ROCm 10 has no released PyTorch index yet; it is on the nightly channel:\",\n \"pip install --pre torch torchvision torchaudio --index-url https://download.pytorch.org/whl/nightly/rocm10.0\",\n \"# Then re-check:\",\n \"python -c \\\"import torch; print(torch.__version__, torch.version.hip)\\\"\"\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": false,\n \"needs_relogin\": false,\n \"fix_id\": \"fix-8-wheel-rocm\",\n \"auto_applicable\": false,\n \"class\": \"print-only\",\n \"notes\": [],\n \"verify\": \"python -c \\\"import torch; print(torch.cuda.is_available(), torch.version.hip)\\\"\"\n }\n }\n ],\n \"has_match\": true,\n \"min_score_for_match\": 50,\n \"high_confidence_threshold\": 75,\n \"route_when_no_match\": {\n \"target\": \"rocm-core\",\n \"url\": \"https://github.com/ROCm/ROCm/issues\"\n },\n \"out_of_scope\": null,\n \"model\": null\n}\n", + "stdout": "{\n \"matched\": [\n {\n \"id\": \"fix-17-torch-dlpack\",\n \"title\": \"vLLM engine start aborts on torch-c-dlpack-ext loading its CUDA variant\",\n \"score\": 50,\n \"evidence\": [\n \"error names libtorch_cuda.so, which a ROCm build of torch does not ship\"\n ],\n \"fix\": {\n \"summary\": \"Only if the engine's runtime holds a ROCm build of torch in the 2.4-2.9 range with torch-c-dlpack-ext installed: reinstall the engine so its pinned torch is restored, which moves torch off the versions the extension ships prebuilts for.\",\n \"commands\": [\n \"# --- step 1 of 3, in YOUR shell ---\",\n \"# Opens an INTERACTIVE subshell with the engine's environment active,\",\n \"# and does not return until you leave it. Run this line on its own.\",\n \"rocm engines shell vllm\",\n \"# --- step 2 of 3, INSIDE the subshell step 1 opened ---\",\n \"# Confirm the trigger before changing anything. It has to be the\",\n \"# ENGINE's interpreter, not the one on your PATH -- they are different\",\n \"# interpreters, and only the engine's decides this failure.\",\n \"python -c \\\"import torch; print(torch.__version__, torch.version.hip)\\\"\",\n \"python -c \\\"import importlib.metadata as m; print(m.version('torch-c-dlpack-ext'))\\\"\",\n \"# This entry applies ONLY when torch.version.hip is set, torch.__version__\",\n \"# is in the 2.4-2.9 range, and torch-c-dlpack-ext is installed. Outside\",\n \"# that range the extension raises a handled ImportError and this is not\",\n \"# the failure you are looking at. Then leave the subshell:\",\n \"exit\",\n \"# --- step 3 of 3, back in YOUR OWN shell ---\",\n \"# If all three held, put the engine's pinned torch back. Do NOT run\",\n \"# this from inside the subshell: it replaces the environment that\",\n \"# shell is standing in.\",\n \"rocm engines install vllm --reinstall\"\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": false,\n \"needs_relogin\": false,\n \"fix_id\": \"fix-17-torch-dlpack\",\n \"auto_applicable\": false,\n \"class\": \"print-only\",\n \"notes\": [\n \"Running vLLM on ROCm is not by itself a reason to apply this. The trigger is narrow: a ROCm build of torch in the 2.4-2.9 range (the versions torch-c-dlpack-ext ships prebuilts for), torch without a native __dlpack_c_exchange_api__, and torch-c-dlpack-ext present -- it arrives as a transitive dependency of tilelang, which vLLM pins.\",\n \"The usual way a runtime lands in that range is `rocm install sdk` being re-run after the engine was installed, which overwrites the engine's pinned torch. Reinstalling the engine puts the pin back.\",\n \"The defect is upstream and there is nothing to correct locally: torch-c-dlpack-ext picks its variant from torch.cuda.is_available(), which is True on ROCm because PyTorch reuses the torch.cuda namespace for HIP, and it ships no ROCm variant to pick. tvm_ffi imports it as optional but guards only ImportError/AttributeError, while ctypes.CDLL raises OSError -- so an explicitly optional import kills the process.\",\n \"A service that failed at startup is hidden from a plain `rocm services list`; pass --all to recover its id.\"\n ],\n \"verify\": \"rocm serve --engine vllm # then `rocm services list --all` and `rocm services logs ` to confirm the import no longer aborts\"\n },\n \"symptom_matched\": true\n },\n {\n \"id\": \"fix-8-wheel-rocm\",\n \"title\": \"Framework wheel built for a different ROCm major than the system\",\n \"score\": 25,\n \"evidence\": [\n \"ldopen failure\"\n ],\n \"fix\": {\n \"summary\": \"Reinstall the framework from the wheel index that matches the system ROCm major (or upgrade the system ROCm to match the wheel).\",\n \"commands\": [\n \"pip uninstall -y torch torchvision torchaudio\",\n \"# Install the index for the ROCm major `rocm examine` reports:\",\n \"pip install torch torchvision torchaudio --index-url https://download.pytorch.org/whl/rocm7.14\",\n \"# ROCm 10 has no released PyTorch index yet; it is on the nightly channel:\",\n \"pip install --pre torch torchvision torchaudio --index-url https://download.pytorch.org/whl/nightly/rocm10.0\",\n \"# Then re-check:\",\n \"python -c \\\"import torch; print(torch.__version__, torch.version.hip)\\\"\"\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": false,\n \"needs_relogin\": false,\n \"fix_id\": \"fix-8-wheel-rocm\",\n \"auto_applicable\": false,\n \"class\": \"print-only\",\n \"notes\": [],\n \"verify\": \"python -c \\\"import torch; print(torch.cuda.is_available(), torch.version.hip)\\\"\"\n },\n \"symptom_matched\": true\n }\n ],\n \"has_match\": true,\n \"min_score_for_match\": 50,\n \"high_confidence_threshold\": 75,\n \"route_when_no_match\": {\n \"target\": \"rocm-core\",\n \"url\": \"https://github.com/ROCm/ROCm/issues\"\n },\n \"out_of_scope\": null,\n \"model\": null\n}\n", "stderr": "", "rc": 0 }, diff --git a/skills/rocm-doctor/evals/fixtures/rocm-vllm-oom-at-startup/replies.json b/skills/rocm-doctor/evals/fixtures/rocm-vllm-oom-at-startup/replies.json index fa031de7ed..b966d6d705 100644 --- a/skills/rocm-doctor/evals/fixtures/rocm-vllm-oom-at-startup/replies.json +++ b/skills/rocm-doctor/evals/fixtures/rocm-vllm-oom-at-startup/replies.json @@ -10,7 +10,7 @@ "rc": 0 }, "diagnose_json": { - "stdout": "{\n \"matched\": [\n {\n \"id\": \"fix-16-vllm-oom\",\n \"title\": \"vLLM ran the GPU out of memory at startup\",\n \"score\": 95,\n \"evidence\": [\n \"error mentions 'HIP out of memory'\",\n \"error mentions torch.OutOfMemoryError\"\n ],\n \"fix\": {\n \"summary\": \"vLLM reserves ~90% of the GPU's total VRAM by default; on a shared or busy GPU this can collide with memory already in use. Lower the reservation with `--gpu-memory-utilization ` (e.g. 0.5 for a small model), or target a less-busy GPU with `--gpu `. If instead the model genuinely does not fit in this GPU's VRAM, lowering the reservation will not help - use a smaller or quantized model (rocm-cli serves one model on a single GPU; it does not shard a model across GPUs).\",\n \"commands\": [\n \"# Case 1 -- shared/busy GPU (tenancy collision): lower the reservation,\",\n \"# or steer vLLM onto a less-busy device:\",\n \"rocm serve --gpu-memory-utilization 0.5\",\n \"rocm serve --gpu \",\n \"# Case 2 -- the model genuinely does not fit: the reservation is not the\",\n \"# problem; pick a smaller or quantized model (single-GPU serving only).\"\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": false,\n \"needs_relogin\": false,\n \"fix_id\": \"fix-16-vllm-oom\",\n \"auto_applicable\": false,\n \"class\": \"print-only\",\n \"notes\": [\n \"Only lower --gpu-memory-utilization when the GPU is shared or already busy; on a GPU dedicated to this server it cannot create the room a too-large model needs.\",\n \"This entry is keyword-matched from the error text: `rocm diagnose` cannot see per-GPU VRAM or tenancy, so pass the failure with --symptom (or arrive from the `rocm serve` failure note).\"\n ],\n \"verify\": \"rocm serve # re-run and watch for a clean startup\"\n }\n }\n ],\n \"has_match\": true,\n \"min_score_for_match\": 50,\n \"high_confidence_threshold\": 75,\n \"route_when_no_match\": {\n \"target\": \"rocm-core\",\n \"url\": \"https://github.com/ROCm/ROCm/issues\"\n },\n \"out_of_scope\": null,\n \"model\": null\n}\n", + "stdout": "{\n \"matched\": [\n {\n \"id\": \"fix-16-vllm-oom\",\n \"title\": \"vLLM ran the GPU out of memory at startup\",\n \"score\": 95,\n \"evidence\": [\n \"error mentions 'HIP out of memory'\",\n \"error mentions torch.OutOfMemoryError\"\n ],\n \"fix\": {\n \"summary\": \"vLLM reserves ~90% of the GPU's total VRAM by default; on a shared or busy GPU this can collide with memory already in use. Lower the reservation with `--gpu-memory-utilization ` (e.g. 0.5 for a small model), or target a less-busy GPU with `--gpu `. If instead the model genuinely does not fit in this GPU's VRAM, lowering the reservation will not help - use a smaller or quantized model (rocm-cli serves one model on a single GPU; it does not shard a model across GPUs).\",\n \"commands\": [\n \"# Case 1 -- shared/busy GPU (tenancy collision): lower the reservation,\",\n \"# or steer vLLM onto a less-busy device:\",\n \"rocm serve --gpu-memory-utilization 0.5\",\n \"rocm serve --gpu \",\n \"# Case 2 -- the model genuinely does not fit: the reservation is not the\",\n \"# problem; pick a smaller or quantized model (single-GPU serving only).\"\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": false,\n \"needs_relogin\": false,\n \"fix_id\": \"fix-16-vllm-oom\",\n \"auto_applicable\": false,\n \"class\": \"print-only\",\n \"notes\": [\n \"Only lower --gpu-memory-utilization when the GPU is shared or already busy; on a GPU dedicated to this server it cannot create the room a too-large model needs.\",\n \"This entry is keyword-matched from the error text: `rocm diagnose` cannot see per-GPU VRAM or tenancy, so pass the failure with --symptom (or arrive from the `rocm serve` failure note).\"\n ],\n \"verify\": \"rocm serve # re-run and watch for a clean startup\"\n },\n \"symptom_matched\": true\n }\n ],\n \"has_match\": true,\n \"min_score_for_match\": 50,\n \"high_confidence_threshold\": 75,\n \"route_when_no_match\": {\n \"target\": \"rocm-core\",\n \"url\": \"https://github.com/ROCm/ROCm/issues\"\n },\n \"out_of_scope\": null,\n \"model\": null\n}\n", "stderr": "", "rc": 0 }, diff --git a/tests/e2e-cucumber/features/diagnose.feature b/tests/e2e-cucumber/features/diagnose.feature index 28ba19c9c7..7b137d9fc9 100644 --- a/tests/e2e-cucumber/features/diagnose.feature +++ b/tests/e2e-cucumber/features/diagnose.feature @@ -923,3 +923,47 @@ Feature: Diagnosing failures and listing fixes When the user asks the CLI to diagnose the error in each file Then each is refused as a usage error that names the file and says why And nothing is diagnosed + + # A diagnosis scores the catalog from two sources at once: what the examined + # host shows, and the error the user pasted. A finding about the host can + # outrank the entry the paste added to, so the ranking alone does not say + # which entries the paste contributed to, and a tool reading the result needs + # that said per entry. "Contributed" is all it says: any keyword hit counts, + # however weak, so it is not a verdict that the entry explains the error. The + # misconfiguration is an unneeded GPU override in the environment the CLI + # examines, which every lane reads the same way, so this holds on Linux, + # Windows and WSL alike. The paste is an ISA error, which no host state scores + # on its own and which names nothing the override entry reads, so the two + # answers cannot depend on the runner. + @id:diagnose-json-says-which-causes-the-paste-contributed-to + Scenario: diagnose-62 - A tool can tell which causes the pasted error contributed to + Given a machine with a known misconfiguration in its environment + And a pasted error that describes a different failure + When the user asks for a diagnosis with that paste, in machine-readable form + Then the paste is reported as not contributing to the misconfiguration on the machine + And the paste is reported as contributing to the cause it describes + + # The same answer, for a person. Only the cause the paste did not contribute + # to is marked, and the mark appears only because the paste contributed to + # something else in the same report. + @id:diagnose-text-marks-causes-the-paste-did-not-contribute-to + Scenario: diagnose-63 - The human-readable diagnosis marks a cause the pasted error did not contribute to + Given a machine with a known misconfiguration in its environment + And a pasted error that describes a different failure + When the user asks for a diagnosis with that paste + Then the misconfiguration on the machine is marked as found from the host alone + And the cause the pasted error describes carries no such mark + + # With nothing pasted, nothing in a paste contributed to anything. A tool is + # still told so on every entry rather than the answer being left out, because + # a missing answer is what a CLI too old to give one prints. A person is told + # nothing: the mark would sit on every cause and say nothing. The host is the + # one diagnose-62 uses, so at least one cause is listed on every lane, and + # both forms are asserted against it. + @id:diagnose-without-a-paste-credits-no-cause-to-one + Scenario: diagnose-64 - Without a pasted error no cause is credited to one, and none is marked + Given a machine with a known misconfiguration in its environment + When the user asks for a diagnosis without pasting an error, in machine-readable form + Then every listed cause is reported as not contributed to by a paste + When the user asks for a diagnosis without pasting an error + Then no listed cause is marked as found from the host alone diff --git a/tests/e2e-cucumber/tests/e2e/diagnose_steps.rs b/tests/e2e-cucumber/tests/e2e/diagnose_steps.rs index b24eb8b28b..3802ee8b67 100644 --- a/tests/e2e-cucumber/tests/e2e/diagnose_steps.rs +++ b/tests/e2e-cucumber/tests/e2e/diagnose_steps.rs @@ -7,6 +7,11 @@ use cucumber::{given, then, when}; use crate::E2eWorld; use crate::e2e::lifecycle_steps; use crate::e2e::tui_driver::{TuiSession, default_timeout}; +// The note the text report prints under a cause the paste did not contribute +// to (diagnose-63/-64) is read from the library's own constant rather than +// typed again here, so the real binary's output is held to the one string the +// renderer prints from. +use rocm_core::diagnose::SYMPTOM_DID_NOT_CONTRIBUTE_NOTE; /// A symptom string that scores a catalog match on both Linux and Windows. It /// keys off `check_1_arch_not_in_wheel` (a `LINUX_AND_WINDOWS` checker), which @@ -4187,3 +4192,187 @@ async fn report_names_the_rocm_version_leading_version(world: &mut E2eWorld) { "the report's cli_version does not follow from the ref ({first_line})" ); } + +// ── Which causes the pasted error contributed to (diagnose-62/-63/-64) ───── + +/// The entry an unneeded GPU override in the environment raises. It reads the +/// environment and a page-fault paste, never an ISA error. +const ENV_MISCONFIG_FIX_ID: &str = "fix-2-unset-override"; + +/// The entry [`KNOWN_SYMPTOM`] scores. No host state scores it on its own. +const PASTE_FIX_ID: &str = "fix-1-arch"; + +#[given("a machine with a known misconfiguration in its environment")] +async fn machine_with_env_misconfiguration(world: &mut E2eWorld) { + // `rocm diagnose` examines its own environment, so setting the override on + // the CLI child is the whole setup, on every platform the entry covers. + world + .command_env + .push(("HSA_OVERRIDE_GFX_VERSION", "11.0.0".into())); +} + +#[given("a pasted error that describes a different failure")] +async fn pasted_error_for_a_different_failure(world: &mut E2eWorld) { + world.model_name = Some(KNOWN_SYMPTOM.to_owned()); +} + +/// Run `rocm diagnose` with `paste` as the symptom, or with no `--symptom` at +/// all when it is `None`. +fn run_diagnose(world: &mut E2eWorld, paste: Option<&str>, json: bool) { + let mut args = vec!["diagnose", "--top", "50"]; + if let Some(paste) = paste { + args.extend(["--symptom", paste]); + } + if json { + args.push("--json"); + } + // The planted host holds for the whole scenario, and diagnose-64 asks + // twice, but running a command consumes the scenario environment. Put it + // back so the second run examines the same host as the first. + let host_env = world.command_env.clone(); + let (stdout, stderr, rc) = crate::run_rocm_with_scenario_env(world, &args); + world.command_env = host_env; + world.cli_output = Some(stdout); + world.cli_stderr = Some(stderr); + world.cli_rc = Some(rc); +} + +fn diagnose_with_paste(world: &mut E2eWorld, json: bool) { + let paste = world.model_name.clone().expect("no paste set"); + run_diagnose(world, Some(&paste), json); +} + +#[when("the user asks for a diagnosis with that paste, in machine-readable form")] +async fn user_diagnoses_paste_json(world: &mut E2eWorld) { + diagnose_with_paste(world, true); +} + +#[when("the user asks for a diagnosis with that paste")] +async fn user_diagnoses_paste_text(world: &mut E2eWorld) { + diagnose_with_paste(world, false); +} + +#[when("the user asks for a diagnosis without pasting an error, in machine-readable form")] +async fn user_diagnoses_without_paste_json(world: &mut E2eWorld) { + run_diagnose(world, None, true); +} + +#[when("the user asks for a diagnosis without pasting an error")] +async fn user_diagnoses_without_paste_text(world: &mut E2eWorld) { + run_diagnose(world, None, false); +} + +/// The listed causes of the JSON report, each as `(id, symptom_matched)`, +/// failing the step when the run failed or any entry lacks a boolean field. +fn symptom_matched_by_id(world: &E2eWorld) -> Vec<(String, bool)> { + let stdout = world.cli_output.as_deref().expect("no diagnose output"); + let stderr = world.cli_stderr.as_deref().unwrap_or_default(); + assert_eq!( + world.cli_rc, + Some(0), + "diagnose failed:\n{stdout}\n{stderr}" + ); + let report: serde_json::Value = serde_json::from_str(stdout) + .unwrap_or_else(|e| panic!("diagnose --json is not JSON ({e}):\n{stdout}")); + let matched = report["matched"] + .as_array() + .unwrap_or_else(|| panic!("no `matched` list:\n{report:#}")); + matched + .iter() + .map(|d| { + let flag = d["symptom_matched"].as_bool().unwrap_or_else(|| { + panic!("every listed cause says whether the paste contributed to it:\n{d:#}") + }); + (d["id"].as_str().unwrap_or_default().to_owned(), flag) + }) + .collect() +} + +/// `symptom_matched` of the entry `id`, failing the step when it is not listed. +fn symptom_matched_of(world: &E2eWorld, id: &str) -> bool { + let listed = symptom_matched_by_id(world); + listed + .iter() + .find(|(listed_id, _)| listed_id == id) + .map_or_else( + || panic!("{id} was not listed: {listed:?}"), + |(_, flag)| *flag, + ) +} + +#[then("the paste is reported as not contributing to the misconfiguration on the machine")] +async fn assert_paste_did_not_contribute_to_env_misconfig(world: &mut E2eWorld) { + assert!( + !symptom_matched_of(world, ENV_MISCONFIG_FIX_ID), + "{ENV_MISCONFIG_FIX_ID} comes from the environment alone, so the paste must not \ + be credited with it" + ); +} + +#[then("the paste is reported as contributing to the cause it describes")] +async fn assert_paste_contributed_to_its_cause(world: &mut E2eWorld) { + assert!( + symptom_matched_of(world, PASTE_FIX_ID), + "{PASTE_FIX_ID} is scored from the paste, so the paste must be credited with it" + ); +} + +#[then("every listed cause is reported as not contributed to by a paste")] +async fn assert_no_cause_credited_without_a_paste(world: &mut E2eWorld) { + let listed = symptom_matched_by_id(world); + assert!( + listed.iter().any(|(id, _)| id == ENV_MISCONFIG_FIX_ID), + "the planted misconfiguration must be listed, or this asserts nothing: {listed:?}" + ); + assert!( + listed.iter().all(|(_, flag)| !flag), + "nothing was pasted, so no cause may be credited to a paste: {listed:?}" + ); +} + +#[then("the misconfiguration on the machine is marked as found from the host alone")] +async fn assert_env_misconfig_marked(world: &mut E2eWorld) { + let output = world.cli_output.as_deref().expect("no diagnose output"); + assert_eq!(world.cli_rc, Some(0), "diagnose failed:\n{output}"); + let block = cause_block(output, ENV_MISCONFIG_FIX_ID); + assert!( + !block.is_empty(), + "{ENV_MISCONFIG_FIX_ID} was not reported:\n{output}" + ); + assert!( + block + .iter() + .any(|line| line.trim() == SYMPTOM_DID_NOT_CONTRIBUTE_NOTE), + "the cause the paste did not contribute to must say so:\n{output}" + ); +} + +#[then("the cause the pasted error describes carries no such mark")] +async fn assert_paste_cause_unmarked(world: &mut E2eWorld) { + let output = world.cli_output.as_deref().expect("no diagnose output"); + let block = cause_block(output, PASTE_FIX_ID); + assert!( + !block.is_empty(), + "{PASTE_FIX_ID} was not reported:\n{output}" + ); + assert!( + !block + .iter() + .any(|line| line.contains(SYMPTOM_DID_NOT_CONTRIBUTE_NOTE)), + "the cause the paste contributed to must not be marked as found without it:\n{output}" + ); +} + +#[then("no listed cause is marked as found from the host alone")] +async fn assert_no_cause_marked_without_a_paste(world: &mut E2eWorld) { + let output = world.cli_output.as_deref().expect("no diagnose output"); + assert_eq!(world.cli_rc, Some(0), "diagnose failed:\n{output}"); + assert!( + !cause_block(output, ENV_MISCONFIG_FIX_ID).is_empty(), + "the planted misconfiguration must be reported, or this asserts nothing:\n{output}" + ); + assert!( + !output.contains(SYMPTOM_DID_NOT_CONTRIBUTE_NOTE), + "with nothing pasted the mark would sit on every cause and say nothing:\n{output}" + ); +}