Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 10 additions & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -408,6 +408,16 @@ position.
:: cmd.exe: save the error to a file first
rocm diagnose --symptom-file error.txt
```
- With either flag, a finding about the examined host can still outrank the
entry the text added to, so each match says whether the text contributed to
its score or evidence: `symptom_matched` in `--json` (always present, `false`
when neither `--symptom` nor `--symptom-file` was given). Any keyword hit
counts, however weak, so `true` means the text contributed, not that the
match explains the error. In the human-readable output, a `source: the
examined host's state alone` line marks each shown match the text did not
contribute to, once the text contributed to any match in the report. That
includes one past `--top`, so all the rows shown can carry the line when the
only match the text contributed to was cut off.
- On Windows, `diagnose` also checks the age of the Visual C++ runtime, not
only whether it is there. The runtime in `System32` must be 14.40 or newer
(`fix-21-msvc-redist-outdated`). Programs built with a recent toolset, such
Expand Down
3 changes: 3 additions & 0 deletions apps/rocm/src/cli.rs
Original file line number Diff line number Diff line change
Expand Up @@ -100,6 +100,9 @@ pub(crate) enum Command {
/// A shell reads this text before `rocm` does, so quotes, `$(...)`
/// and a leading `-` in a pasted error can be changed, run, or taken
/// as a flag. Use `--symptom-file` to hand over a paste untouched.
///
/// Each match then says whether this text contributed to it
/// (`symptom_matched` in --json).
#[arg(long, value_name = "TEXT")]
symptom: Option<String>,
/// Read the error text from a file instead, or from standard input
Expand Down
1 change: 1 addition & 0 deletions apps/rocm/src/diagnose_cmd.rs
Original file line number Diff line number Diff line change
Expand Up @@ -531,6 +531,7 @@ mod tests {
rocm_core::DiagnoseReport {
has_match,
matched: vec![rocm_core::Diagnosis {
symptom_matched: false,
id: id.to_owned(),
title: "under test".to_owned(),
score,
Expand Down
46 changes: 46 additions & 0 deletions apps/rocm/src/remote/doctor.rs
Original file line number Diff line number Diff line change
Expand Up @@ -382,6 +382,7 @@ mod tests {
fn real_report_with_a_fix() -> DiagnoseReport {
use rocm_core::diagnose::{Diagnosis, Fix};
report(vec![Diagnosis {
symptom_matched: false,
id: "dkms-mismatch".to_owned(),
title: "DKMS built against another kernel".to_owned(),
score: 90,
Expand Down Expand Up @@ -590,6 +591,7 @@ mod tests {
let rendered = render_report(
"gpu-box",
&report(vec![Diagnosis {
symptom_matched: false,
id: "annotated".to_owned(),
title: "a fix that explains itself".to_owned(),
score: 90,
Expand Down Expand Up @@ -630,6 +632,7 @@ mod tests {
let rendered = render_report(
"gpu-box",
&report(vec![Diagnosis {
symptom_matched: false,
id: "container".to_owned(),
title: "container missing devices".to_owned(),
score: 90,
Expand Down Expand Up @@ -704,6 +707,7 @@ mod tests {
let rendered = render_report(
"gpu-box",
&report(vec![Diagnosis {
symptom_matched: false,
id: "wheels".to_owned(),
title: "wrong wheels".to_owned(),
score: 90,
Expand Down Expand Up @@ -740,4 +744,46 @@ mod tests {
let rendered = render_report("gpu-box", &report(vec![]), 5);
assert!(rendered.starts_with("Health of gpu-box"), "{rendered}");
}

/// The note under a cause the symptom did not contribute to describes the
/// examined host, which here is the remote one, and it marks only that
/// cause.
#[test]
fn the_symptom_note_is_true_of_a_remote_host_and_marks_only_its_cause() {
use rocm_core::diagnose::{Diagnosis, SYMPTOM_DID_NOT_CONTRIBUTE_NOTE};
let row = |id: &str, score: i32, symptom_matched: bool| Diagnosis {
id: id.to_owned(),
title: id.to_owned(),
score,
symptom_matched,
..Diagnosis::default()
};
let rendered = render_report(
"gpu-box",
&report(vec![
row("fix-host-only", 60, false),
row("fix-from-the-paste", 55, true),
]),
5,
);
let block = |id: &str| -> Vec<&str> {
let marker = format!("({id})");
rendered
.lines()
.skip_while(|l| !l.contains(&marker))
.take_while(|l| !l.trim().is_empty())
.collect()
};
let marked = |id: &str| {
block(id)
.iter()
.any(|l| l.trim() == SYMPTOM_DID_NOT_CONTRIBUTE_NOTE)
};
assert!(marked("fix-host-only"), "{rendered}");
assert!(!marked("fix-from-the-paste"), "{rendered}");
assert!(
!SYMPTOM_DID_NOT_CONTRIBUTE_NOTE.contains("this machine"),
"the note is printed about a remote host too, where `this machine` is the wrong one"
);
}
}
222 changes: 221 additions & 1 deletion crates/rocm-core/src/diagnose.rs
Original file line number Diff line number Diff line change
Expand Up @@ -43,7 +43,7 @@ mod wsl;

pub use engine::{VLLM_IMPORT_CANONICAL_SYMPTOM, VLLM_OOM_CANONICAL_SYMPTOM};
pub use linux::{AMDGPU_MISSING_EVIDENCE, dkms_driver_plan_is_supported};
pub use report::{render_report_text, upstream_route};
pub use report::{SYMPTOM_DID_NOT_CONTRIBUTE_NOTE, render_report_text, upstream_route};

/// At/above this score a diagnosis is treated as a real match.
pub const MIN_SCORE_FOR_MATCH: i32 = 50;
Expand Down Expand Up @@ -91,6 +91,25 @@ pub struct Diagnosis {
pub score: i32,
pub evidence: Vec<String>,
pub fix: Option<Fix>,
/// Whether the symptom text the user supplied contributed to this entry.
///
/// `true` when, without the symptom, this entry would have a different score
/// or different evidence, or would not be listed at all; `false` when it
/// comes from the examined host's state alone. A host-state finding can
/// outrank the entry the symptom added to, so a reader asking "did the text
/// I pasted contribute to this entry?" reads this rather than the ranking.
///
/// It records a contribution, not a verdict that the entry explains the
/// error: any keyword hit counts, however weak. Read `score` and `evidence`
/// for how much the text added.
///
/// Always serialized, and `false` when no symptom was given: nothing was
/// pasted, so nothing in it contributed. Omitting it in that case would make
/// "no symptom" indistinguishable from "a CLI too old to report this field".
/// `#[serde(default)]` so a payload written before the field existed still
/// deserializes. Set by [`diagnose`], never by a checker.
#[serde(default)]
pub symptom_matched: bool,
}

/// Where to send a report when no catalog entry matches.
Expand Down Expand Up @@ -200,6 +219,43 @@ pub fn vllm_import_symptom_is_diagnosable(symptom: &str) -> bool {
scores_on_every_engine_host(symptom, "fix-17-torch-dlpack", run_all_checks)
}

/// Set [`Diagnosis::symptom_matched`] on every entry in `matched`.
///
/// Decided by running the same catalog again on the same machine with no
/// symptom and comparing, rather than by each checker reporting what it read.
/// Checkers fold the symptom in several ways (keyword tables, single patterns,
/// a keyword whose own weight is replaced by a fixed one, a symptom that lifts a
/// negative host score above zero), and a flag each of them had to set by hand
/// is one a new checker can forget. The comparison asks the question itself:
/// would this entry look the same without the text? Evidence is compared as
/// well as the score because a score is capped at 100, so a keyword hit on an
/// entry already at the cap changes its evidence and not its number.
///
/// The checkers are pure functions of the examination and the symptom, so the
/// second run reads no more of the machine than the first.
fn mark_symptom_contribution(matched: &mut [Diagnosis], e: &Examination, symptom: &str) {
if symptom.is_empty() {
return;
}
let host_only = run_all_checks(e, "");
for d in matched {
d.symptom_matched = contributed(d, &host_only);
}
}

/// Whether the symptom contributed to `d`, given `host_only`, the same catalog
/// run on the same host with no symptom.
///
/// It did when `d` is absent from that run, or present with a different score
/// or different evidence. Either difference alone is enough: a keyword hit on
/// an entry already at the 100 cap changes only its evidence.
fn contributed(d: &Diagnosis, host_only: &[Diagnosis]) -> bool {
host_only
.iter()
.find(|h| h.id == d.id)
.is_none_or(|h| h.score != d.score || h.evidence != d.evidence)
}

/// Diagnose an examination against the closed catalog.
#[must_use]
pub fn diagnose(e: &Examination, symptom: &str) -> DiagnoseReport {
Expand All @@ -221,6 +277,7 @@ pub fn diagnose(e: &Examination, symptom: &str) -> DiagnoseReport {
} else {
run_all_checks(e, symptom)
};
mark_symptom_contribution(&mut matched, e, symptom);
Comment thread
volen-silo marked this conversation as resolved.
Resolved
take_applicability_from_the_catalog(&mut matched, platform_family(e));
DiagnoseReport {
has_match: any_cleared_threshold(&matched),
Expand Down Expand Up @@ -507,4 +564,167 @@ mod tests {
// And a bare framework mention the engine would reject is not diagnosable.
assert!(!vllm_oom_symptom_is_diagnosable("vllm: out of memory"));
}

/// The entry with `id` in `report`, or a panic naming what was listed.
fn entry<'a>(report: &'a DiagnoseReport, id: &str) -> &'a Diagnosis {
report
.matched
.iter()
.find(|d| d.id == id)
.unwrap_or_else(|| {
panic!(
"{id} was not listed: {:?}",
report
.matched
.iter()
.map(|d| (&d.id, d.score))
.collect::<Vec<_>>()
)
})
}

fn with_gpu_override() -> Examination {
let mut e = linux_base();
e.env
.insert("HSA_OVERRIDE_GFX_VERSION".to_owned(), "11.0.0".to_owned());
e
}

const ISA_PASTE: &str = "hipErrorNoBinaryForGpu: HSA_STATUS_ERROR_INVALID_ISA";

/// The case the field exists for: a finding about the machine sits beside
/// the entry the paste scores, and the paste contributed to only one of them.
#[test]
fn a_host_state_finding_is_not_credited_to_a_paste_about_something_else() {
let report = diagnose(&with_gpu_override(), ISA_PASTE);

let host = entry(&report, "fix-2-unset-override");
assert!(
!host.symptom_matched,
"the override finding comes from the environment alone: {host:?}"
);
let pasted = entry(&report, "fix-1-arch");
assert!(
pasted.symptom_matched,
"the ISA entry is scored from the paste: {pasted:?}"
);

// And both answers reach the document a consumer reads.
let json = serde_json::to_value(&report).unwrap();
let field = |id: &str| {
json["matched"]
.as_array()
.unwrap()
.iter()
.find(|d| d["id"] == id)
.map(|d| d["symptom_matched"].clone())
};
assert_eq!(field("fix-2-unset-override"), Some(false.into()));
assert_eq!(field("fix-1-arch"), Some(true.into()));
}

#[test]
fn a_paste_that_raises_a_host_finding_is_credited() {
// The override alone scores 30; a page fault in the paste adds to it.
let report = diagnose(&with_gpu_override(), "memory access page fault");
let raised = entry(&report, "fix-2-unset-override");
assert!(
raised.score > 30,
"the paste must have raised it: {raised:?}"
);
assert!(raised.symptom_matched, "{raised:?}");
}

#[test]
fn a_paste_that_lifts_an_entry_the_host_alone_ruled_out_is_credited() {
// The framework already carries this GPU's target, which pulls the
// arch entry below zero on its own, so it is not listed without the
// paste at all. With it, the entry is listed, and only because of it.
let mut e = linux_base();
e.gpus = vec![crate::examine::Gpu {
gfx_target: "gfx1100".to_owned(),
is_amd: true,
..crate::examine::Gpu::default()
}];
e.framework_arch_list = vec!["gfx1100".to_owned()];
assert!(
diagnose(&e, "")
.matched
.iter()
.all(|d| d.id != "fix-1-arch"),
"the premise: without the paste this entry is not listed"
);
let report = diagnose(&e, ISA_PASTE);
assert!(entry(&report, "fix-1-arch").symptom_matched);
}

#[test]
fn with_no_symptom_every_entry_says_so_and_the_field_is_still_written() {
for symptom in ["", " "] {
let report = diagnose(&with_gpu_override(), symptom);
assert!(
!report.matched.is_empty(),
"this test is pointless unless something was listed"
);
assert!(
report.matched.iter().all(|d| !d.symptom_matched),
"nothing was pasted for {symptom:?}, so nothing in it contributed: {:?}",
report.matched
);
let json = serde_json::to_value(&report).unwrap();
for d in json["matched"].as_array().unwrap() {
assert_eq!(
d.get("symptom_matched"),
Some(&serde_json::Value::Bool(false)),
"the field is written, not omitted, when no symptom was given: {d}"
);
}
}
}

#[test]
fn a_payload_written_before_the_field_existed_still_reads() {
let old =
r#"{"id":"fix-2-unset-override","title":"t","score":30,"evidence":[],"fix":null}"#;
let d: Diagnosis = serde_json::from_str(old).expect("an older payload deserializes");
assert!(!d.symptom_matched);
}

fn finding(id: &str, score: i32, evidence: &[&str]) -> Diagnosis {
Diagnosis {
id: id.to_owned(),
score,
evidence: evidence.iter().map(|&e| e.to_owned()).collect(),
..Diagnosis::default()
}
}

/// The three ways the symptom can show in an entry, one at a time, so each
/// half of the comparison has a case that only it decides.
#[test]
fn contribution_is_any_difference_from_the_run_without_the_symptom() {
let host_only = [finding("fix-a", 30, &["host fact"])];

// Same entry, same evidence, higher score: the score alone differs.
assert!(contributed(
&finding("fix-a", 55, &["host fact"]),
&host_only
));
// Same score, extra evidence: what a keyword hit does to an entry
// already at the 100 cap. The evidence alone differs.
assert!(contributed(
&finding("fix-a", 30, &["host fact", "error mentions X"]),
&host_only
));
// Not listed at all without the symptom.
assert!(contributed(
&finding("fix-b", 50, &["error mentions Y"]),
&host_only
));
// Identical to the run without it: nothing contributed.
assert!(!contributed(
&finding("fix-a", 30, &["host fact"]),
&host_only
));
}
}
Loading
Loading