From 3e6950f3d7bb65f29b2fcc8bacac52f07037fe86 Mon Sep 17 00:00:00 2001 From: Eugene Volen Date: Fri, 14 Aug 2026 07:47:28 +0000 Subject: [PATCH] fix(diagnose): state in the JSON whether a cause was established `rocm diagnose --json` gave a caller no way to ask the question it most needs answered. The obvious substitute -- is `matched` empty? -- is a different question, and on ordinary hardware it gives the wrong answer. `matched` carries every checker that fired at all, including ones scoring too low to act on, and several open with a nonzero score for a situation that is merely potentially relevant: being in a container scores 25 before anything has been inspected, an APU beside a discrete GPU scores 40. So a machine with nothing wrong with it hands back a non-empty list. A caller reading that list as a diagnosis proposes a fix for a healthy machine -- re-launch the container that already has its devices -- and never routes the user upstream, which is the one thing it was supposed to do when it recognised nothing. The verdict already existed as `has_match`, and the CLI already documented it as part of what callers read from `--json`. It was simply never serialized. This emits it, computed through the same predicate the accessor uses so the two cannot answer differently, and defaulted on the way in so existing documents still load. The human report was never affected: it has always labelled these entries WEAK. Nothing else changes. `matched` still carries sub-threshold entries -- the WEAK tier is deliberate and the rendered report depends on it -- and no score moves. A base 25 for "in a container" is the tiering working as designed; the missing verdict was the defect. The e2e scenario meant to pin this asserted that `matched` was non-empty, so it encoded the bug rather than the contract. It now reads the verdict. Four scenarios close the gaps that let this through: a verdict is stated when nothing matched; a platform the catalog does not cover is given no findings, cross-checked against the host report so the covered branch can fail too; a fix meant for another OS is declined with its own exit code rather than attempted; and the catalog is complete, with exactly the four entries the CLI carries out itself marked as such. Two comments in examine.feature still said "Expected to FAIL" for defects fixed earlier, whose expectation rows were removed at the time without the feature file following. Both scenarios pass; the comments now say what each is guarding instead of misreporting it as broken. Signed-off-by: Eugene Volen --- crates/rocm-core/src/diagnose.rs | 116 +++++++- tests/e2e-cucumber/features/diagnose.feature | 62 ++++ tests/e2e-cucumber/features/examine.feature | 31 +- .../e2e-cucumber/tests/e2e/diagnose_steps.rs | 271 +++++++++++++++++- 4 files changed, 462 insertions(+), 18 deletions(-) diff --git a/crates/rocm-core/src/diagnose.rs b/crates/rocm-core/src/diagnose.rs index dbc52d506b..784cdf490b 100644 --- a/crates/rocm-core/src/diagnose.rs +++ b/crates/rocm-core/src/diagnose.rs @@ -59,7 +59,26 @@ pub struct Route { #[derive(Debug, Clone, Serialize, Deserialize)] pub struct DiagnoseReport { /// All nonzero-score diagnoses, highest score first. + /// + /// Every checker that fired at all lands here, including ones scoring below + /// [`MIN_SCORE_FOR_MATCH`] — so a non-empty `matched` does NOT mean a cause + /// was established. Read [`DiagnoseReport::has_match`] for that. pub matched: Vec, + /// Whether any entry in `matched` cleared [`MIN_SCORE_FOR_MATCH`]. + /// + /// Serialized because a JSON consumer cannot otherwise tell a real cause + /// from a weak signal, and the obvious substitute — "is `matched` empty?" — + /// is wrong. Several checkers open with a nonzero base score for a + /// situation that is merely *potentially* relevant (being in a container, + /// having an APU alongside a discrete GPU), so a perfectly healthy host + /// produces a non-empty `matched` full of sub-threshold entries. A caller + /// gating on emptiness proposes a fix for a machine with nothing wrong, and + /// never routes the user to `route_when_no_match`. + /// + /// Computed at construction; [`DiagnoseReport::has_match`] recomputes from + /// `matched` and stays the authority for Rust callers. + #[serde(default)] + pub has_match: bool, pub min_score_for_match: i32, pub high_confidence_threshold: i32, pub route_when_no_match: Route, @@ -70,11 +89,19 @@ pub struct DiagnoseReport { pub out_of_scope: Option, } +/// Whether any diagnosis cleared [`MIN_SCORE_FOR_MATCH`]. +/// +/// One place the rule is written, so the serialized `has_match` field and the +/// [`DiagnoseReport::has_match`] accessor cannot answer differently. +fn any_cleared_threshold(matched: &[Diagnosis]) -> bool { + matched.iter().any(|d| d.score >= MIN_SCORE_FOR_MATCH) +} + impl DiagnoseReport { /// Whether at least one diagnosis cleared [`MIN_SCORE_FOR_MATCH`]. #[must_use] pub fn has_match(&self) -> bool { - self.matched.iter().any(|d| d.score >= MIN_SCORE_FOR_MATCH) + any_cleared_threshold(&self.matched) } } @@ -1310,6 +1337,7 @@ pub fn diagnose(e: &Examination, symptom: &str) -> DiagnoseReport { run_all_checks(e, symptom) }; DiagnoseReport { + has_match: any_cleared_threshold(&matched), matched, min_score_for_match: MIN_SCORE_FOR_MATCH, high_confidence_threshold: HIGH_CONFIDENCE, @@ -1740,6 +1768,91 @@ mod tests { assert!(report.route_when_no_match.url.contains("pytorch/pytorch")); } + #[test] + fn a_healthy_container_reports_no_match_despite_a_nonempty_list() { + // The case `has_match` exists for. `check_10_container_devices` opens at + // 25 for merely being in a container, before it has looked at anything, + // and this fixture adds the 20 for no visible render device (a probe + // that found nothing, not a device that is missing) -- 45, short of the + // threshold and still in `matched`. A caller reading "is `matched` + // empty?" as "did anything match?" would propose re-launching a + // container over what is only a thin probe, and would never route the + // user upstream. + let mut e = linux_base(); + e.in_container = true; + e.container_kind = "docker".to_owned(); + let report = diagnose(&e, ""); + + assert!( + !report.matched.is_empty(), + "fixture must produce an entry for this test to mean anything" + ); + assert!( + report.matched.iter().all(|d| d.score < MIN_SCORE_FOR_MATCH), + "fixture must stay below the threshold; got {:?}", + report + .matched + .iter() + .map(|d| (&d.id, d.score)) + .collect::>() + ); + assert!( + !report.has_match, + "a list of sub-threshold entries is not a match" + ); + } + + #[test] + fn the_serialized_verdict_agrees_with_the_accessor() { + // Rust callers read the method, tooling reads the field. They answer the + // same question, so a host where they disagree would hand the two + // audiences different verdicts. + let mut in_container = linux_base(); + in_container.in_container = true; + let mut render_group_missing = linux_base(); + render_group_missing.in_render_group = Some(false); + render_group_missing.in_video_group = Some(false); + + // The fixtures above all land BELOW the threshold, so on their own they + // would only ever exercise the `false` branch -- and a field that is + // always false serializes correctly by accident. The last one carries a + // symptom that pushes the same finding past HIGH_CONFIDENCE, so `true` + // reaches the wire here too and not only on a bare-metal e2e lane. + let mut real_fault = linux_base(); + real_fault.in_render_group = Some(false); + + let cases = [ + (linux_base(), ""), + (in_container, ""), + (render_group_missing, ""), + (real_fault, "RuntimeError: unable to open /dev/kfd"), + ]; + assert!( + cases + .iter() + .any(|(e, symptom)| diagnose(e, symptom).has_match), + "at least one fixture must clear the threshold, or this only ever \ + proves the false branch" + ); + + for (e, symptom) in cases { + let report = diagnose(&e, symptom); + assert_eq!( + report.has_match, + report.has_match(), + "field and accessor disagree for {:?}", + report.matched.iter().map(|d| &d.id).collect::>() + ); + let json: serde_json::Value = + serde_json::from_str(&serde_json::to_string(&report).unwrap()).unwrap(); + assert_eq!( + json.get("has_match").and_then(serde_json::Value::as_bool), + Some(report.has_match), + "the verdict must survive into the emitted document" + ); + } + } + #[test] fn no_match_default_route_is_rocm_core() { let e = linux_base(); @@ -1818,6 +1931,7 @@ mod tests { let v = serde_json::to_value(&report).unwrap(); for key in [ "matched", + "has_match", "min_score_for_match", "high_confidence_threshold", "route_when_no_match", diff --git a/tests/e2e-cucumber/features/diagnose.feature b/tests/e2e-cucumber/features/diagnose.feature index 6497b1f088..88b2f795f0 100644 --- a/tests/e2e-cucumber/features/diagnose.feature +++ b/tests/e2e-cucumber/features/diagnose.feature @@ -78,3 +78,65 @@ Feature: Diagnosing failures and listing fixes When the user asks the CLI to apply it without agreeing to the change Then the CLI refuses and explains that it needs agreement And the file the fix would have changed is untouched + + # The other half of scenario 3, and the half every host can prove. A caller + # cannot read "did anything match?" off the size of the list: every checker + # that fires at all is reported, including ones scoring too low to act on, + # and several open with a nonzero score for a situation that is merely + # POTENTIALLY relevant — being in a container, having an APU beside a + # discrete GPU. So a healthy machine hands back a non-empty list of things + # that are not wrong with it. A caller treating that as a diagnosis proposes + # a fix for a machine with nothing wrong, and never routes the user onward. + @id:diagnose-json-states-when-nothing-matched + Scenario: 9 - A tool is told plainly when no cause was established + Given a user who hit a failure the CLI does not recognise + When the user asks the CLI to diagnose that symptom in machine-readable form + Then the result states that no cause was established + And the CLI always points to somewhere the problem can be reported + + # Host-agnostic on purpose: the scenario asks the CLI what it makes of this + # platform and then holds it to the matching half of the contract. A caller + # decides whether to diagnose at all from this verdict, and nothing pinned it + # before — the suite only ever SKIPPED the bare-metal scenarios on WSL2, which + # proves nothing about what gets reported there. + # + # Be precise about where each half runs, because the halves are not equal. + # There is NO WSL2 lane in CI (every job pins a native runner), so the + # route-out half is proven only by a developer running the suite on WSL2. + # What CI gets is the covered half plus the cross-check against the host + # report — both of which can fail, which is the bar an assertion has to clear + # to be worth writing. An earlier version of this scenario returned early on a + # covered platform and asserted nothing at all on any lane CI runs. + @id:diagnose-states-whether-the-platform-is-covered + Scenario: 10 - A platform the catalog does not cover says so and routes onward + Given a user who hit a known ROCm failure + When the user asks the CLI to diagnose that symptom in machine-readable form + Then the result says whether this platform is covered + And a platform that is not covered is given no diagnosis + And a platform that is covered gets a verdict that follows the evidence + And the CLI always points to somewhere the problem can be reported + + # A fix that cannot run here is a different outcome from one that failed, and + # from one the user declined — a caller that cannot tell them apart reports a + # broken machine when the truth is "wrong operating system". The scenario + # picks whichever catalog entry belongs to the OTHER platform, so it carries + # the same weight on the Linux and Windows lanes. + @id:fix-inapplicable-here-is-declined-not-attempted + Scenario: 11 - A fix meant for another operating system is declined, not attempted + Given a user who has chosen a fix meant for a different operating system + When the user asks the CLI to apply that fix + Then the CLI declines because the fix does not apply to this machine + And nothing on the machine is changed + + # Scenario 4 proves the listing works; this proves it is COMPLETE. Which + # failure modes exist, and which of them the CLI will carry out itself, are + # part of the published contract rather than private detail — so a mode added + # or removed is a change to what callers were promised, and it should not be + # possible to make it quietly. This is deliberately the brittle test that + # breaks when the catalog changes; that break is the notification. Do not + # loosen it. + @id:fix-catalog-is-complete + Scenario: 12 - The CLI offers every fix its catalog documents + When the user asks the CLI which fixes it offers + Then every fix the catalog documents is listed + And only the fixes the CLI can carry out itself are marked as such diff --git a/tests/e2e-cucumber/features/examine.feature b/tests/e2e-cucumber/features/examine.feature index 0aeef746b0..4ccece7f5a 100644 --- a/tests/e2e-cucumber/features/examine.feature +++ b/tests/e2e-cucumber/features/examine.feature @@ -50,26 +50,27 @@ Feature: GPU detection and system inspection And the inspection does not claim nothing is installed And the inspection suggests setting up a CLI-managed install - # Expected to FAIL. The machine-readable form is a separate code path, not a - # re-rendering of the human one: it answers before the CLI has loaded its paths - # or config, so every CLI-side fact is out of reach. Eleven things the human - # report states have no field in it at all — among them which engine this host - # will serve on and whether an existing ROCm install was found. Tooling reads - # this form; it should not be the weaker of the two. + # The machine-readable form is a separate code path, not a re-rendering of the + # human one: it used to answer before the CLI had loaded its paths or config, + # putting every CLI-side fact out of reach. Eleven things the human report + # states had no field in it at all — among them which engine this host will + # serve on and whether an existing ROCm install was found. Since fixed (those + # facts now travel under `summary`), and this is what holds the two forms + # level: tooling reads this one, and it must not drift back into being the + # weaker of the two. @id:examine-machine-readable-report Scenario: 7 - What the inspection tells a tool matches what it tells a person When the user inspects the system both for reading and for scripting Then the machine-readable form states everything the readable one does - # Expected to FAIL on Instinct. The harness parses the human text rather than - # this form precisely because of this defect, and says so in capability.rs — - # on a real MI300X the machine-readable form reported no AMD GPU on a machine - # that has one. That workaround makes the disagreement load-bearing: every - # host capability the suite resolves comes from scraped text because this form - # could not be trusted. - # - # It is narrower than "any host with a GPU": Strix Halo (gfx1151) agrees, - # MI300X (gfx943) does not. The expectations row is scoped to match. + # The harness parses the human text rather than this form because of a defect + # this scenario caught, and says so in capability.rs — on a real MI300X the + # machine-readable form reported no AMD GPU on a machine that has one, while + # Strix Halo (gfx1151) agreed. That workaround makes the disagreement + # load-bearing: every host capability the suite resolves comes from scraped + # text, so if the two forms ever diverge again, every capability-keyed + # expectation silently resolves against the wrong host. Since fixed; this is + # the guard that keeps it fixed. @id:examine-both-forms-agree-on-gpu Scenario: 8 - Both forms of the inspection agree about the GPU When the user inspects the system both for reading and for scripting diff --git a/tests/e2e-cucumber/tests/e2e/diagnose_steps.rs b/tests/e2e-cucumber/tests/e2e/diagnose_steps.rs index fac3fc4ba4..74272ed191 100644 --- a/tests/e2e-cucumber/tests/e2e/diagnose_steps.rs +++ b/tests/e2e-cucumber/tests/e2e/diagnose_steps.rs @@ -30,6 +30,56 @@ const PREVIEW_FIX_ID: &str = "fix-1-arch"; /// found". This one needs only `--device-index`, which the scenario supplies. const MUTATING_FIX_ID: &str = "fix-9-igpu-dgpu"; +/// Every fix-id in the closed catalog, in the order `rocm fix` lists them. +/// +/// A duplicate of the catalog, on purpose: a test that derived this list from +/// the same source it checks could not notice a change to it. +/// +/// The catalog is a closed list that external tooling reads and reproduces — +/// the ids, their OS scope, and which four are auto-applicable are all part of +/// the CLI's published contract, not private detail. Changing the catalog +/// changes that contract, and this is the assertion that says so out loud. +/// When it fires, update this list along with whatever documents the catalog; +/// do not relax it. +const CATALOG_FIX_IDS: &[&str] = &[ + "fix-1-arch", + "fix-2-unset-override", + "fix-3-rocm-kernel", + "fix-4-render-group", + "fix-5-amdgpu-load", + "fix-6-path", + "fix-7-stale-repos", + "fix-8-wheel-rocm", + "fix-9-igpu-dgpu", + "fix-10-container", + "fix-11-iommu", + "fix-12-installer", + "fix-13-hip-sdk-missing", + "fix-14-adrenalin-too-old", + "fix-15-msvc-redist", +]; + +/// The fixes the CLI carries out itself. Every other entry only prints a plan. +/// Pinned exactly: a mode quietly promoted to AUTO would begin changing +/// machines that callers had been told it only ever advised on. +const AUTO_APPLICABLE_FIX_IDS: &[&str] = &[ + "fix-2-unset-override", + "fix-4-render-group", + "fix-6-path", + "fix-9-igpu-dgpu", +]; + +/// A catalog entry that cannot apply on the host running the suite, whichever +/// host that is. Both are print-only, so the run stops at the OS gate without +/// reaching any recipe that could touch the machine. +const fn fix_id_for_the_other_os() -> &'static str { + if cfg!(windows) { + "fix-5-amdgpu-load" // linux-only + } else { + "fix-13-hip-sdk-missing" // windows-only + } +} + /// Contents planted in the scenario's own shell rc file. The assertion is that /// this survives the run byte for byte. const PLANTED_RC: &str = "# planted by the e2e suite; the fix must not touch this\n"; @@ -85,6 +135,11 @@ async fn user_chose_mutating_fix(world: &mut E2eWorld) { world.model_name = Some(MUTATING_FIX_ID.to_string()); } +#[given("a user who has chosen a fix meant for a different operating system")] +async fn user_chose_fix_for_another_os(world: &mut E2eWorld) { + world.model_name = Some(fix_id_for_the_other_os().to_string()); +} + #[given("a user who refers to a cause by its position in the diagnosis")] async fn user_named_diagnosis_position(world: &mut E2eWorld) { // Quoted deliberately: unquoted, the shell treats `#1` as a comment and the @@ -241,13 +296,225 @@ async fn assert_json_identifies_match(world: &mut E2eWorld) { let output = world.cli_output.as_ref().expect("no diagnose output"); let report: serde_json::Value = serde_json::from_str(output).expect("diagnose --json did not emit valid JSON"); + // NOT `matched` being non-empty: that list also carries entries scoring too + // low to act on, so its size does not answer "was a cause established?". + // `has_match` is the field that does, and it is what a caller must read. + assert_eq!( + report.get("has_match").and_then(serde_json::Value::as_bool), + Some(true), + "a known symptom must be reported as an established cause:\n{output}" + ); + let top = report + .get("matched") + .and_then(|m| m.as_array()) + .and_then(|m| m.first()) + .expect("a report with a match must name it"); + // A cause the caller cannot act on is not actionable: it needs an id to + // refer to and a fix-id to hand to `rocm fix`. + assert!( + top.get("id").and_then(serde_json::Value::as_str).is_some(), + "the matched cause must carry an id:\n{output}" + ); + assert!( + top.pointer("/fix/fix_id") + .and_then(serde_json::Value::as_str) + .is_some(), + "the matched cause must name the fix that applies it:\n{output}" + ); +} + +/// Hold the report to its own arithmetic: `has_match` is true exactly when some +/// cause cleared the threshold the report itself declares. Returns the ids that +/// cleared it. +/// +/// This is the assertion that survives on any host. Pinning a specific verdict +/// would make the scenario a test of the runner's health — a CI box with a +/// genuine fault of its own (a blacklisted amdgpu, a user outside the render +/// group) produces real causes whatever symptom was passed. Self-consistency +/// holds regardless, and it is exactly the property that was broken: the +/// verdict used to be unavailable, so callers inferred it from the list length. +fn assert_verdict_follows_scores<'a>(report: &'a serde_json::Value, output: &str) -> Vec<&'a str> { + let has_match = report + .get("has_match") + .and_then(serde_json::Value::as_bool) + .expect("diagnose JSON must state whether a cause was established"); + // Read the threshold from the document rather than restating 50 here: the + // report publishes it so callers do not have to hardcode it, and a test + // that hardcodes it is not exercising that. + let threshold = report + .get("min_score_for_match") + .and_then(serde_json::Value::as_i64) + .expect("diagnose JSON must publish its match threshold"); + let cleared: Vec<&str> = report + .get("matched") + .and_then(|m| m.as_array()) + .expect("diagnose JSON has no 'matched' array") + .iter() + .filter(|d| { + d.get("score") + .and_then(serde_json::Value::as_i64) + .is_some_and(|s| s >= threshold) + }) + .filter_map(|d| d.get("id").and_then(serde_json::Value::as_str)) + .collect(); + assert_eq!( + has_match, + !cleared.is_empty(), + "the verdict must follow the scores (threshold {threshold}); \ + cleared={cleared:?}\n{output}" + ); + cleared +} + +fn parsed_diagnosis(world: &E2eWorld) -> (serde_json::Value, String) { + let output = world + .cli_output + .as_ref() + .expect("no diagnose output") + .clone(); + let report = serde_json::from_str(&output).expect("diagnose --json did not emit valid JSON"); + (report, output) +} + +#[then("the result states that no cause was established")] +async fn assert_json_states_no_match(world: &mut E2eWorld) { + assert_eq!( + world.cli_rc, + Some(0), + "diagnose should exit 0 (it is a query)" + ); + let (report, output) = parsed_diagnosis(world); + let cleared = assert_verdict_follows_scores(&report, &output); + if !cleared.is_empty() { + // This host has a real fault of its own, so the premise is gone. The + // consistency check above still ran, which is what this scenario is for. + return; + } + assert!( + !report + .get("has_match") + .and_then(serde_json::Value::as_bool) + .unwrap_or(true), + "an unrecognised symptom on a host with no real fault must not report \ + an established cause:\n{output}" + ); +} + +#[then("the result says whether this platform is covered")] +async fn assert_json_states_platform_scope(world: &mut E2eWorld) { + let (report, output) = parsed_diagnosis(world); + // NOT "the key is present": `out_of_scope` is an Option with no + // skip_serializing_if, so serde emits it either way and its mere presence + // proves nothing. Cross-check the verdict against the one the host report + // gives for the same machine — the same trick `examine-both-forms-agree-on-gpu` + // uses, and the only version of this assertion that can fail on a covered + // host. The two are computed by different code paths off the same probe, so + // this is a cross-check rather than a tautology. + let (examine, _, rc) = crate::run_rocm(world, &["examine", "--json"]); + assert_eq!(rc, 0, "examine should exit 0 (it is an inspector)"); + let host: serde_json::Value = + serde_json::from_str(&examine).expect("examine --json did not emit valid JSON"); + let host_says_uncovered = host + .get("status") + .and_then(serde_json::Value::as_str) + .is_some_and(|status| status == "wsl"); + let diagnosis_says_uncovered = report.get("out_of_scope").is_some_and(|v| !v.is_null()); + assert_eq!( + diagnosis_says_uncovered, host_says_uncovered, + "the diagnosis and the host report disagree about whether this platform \ + is covered (diagnosis={diagnosis_says_uncovered}, host={host_says_uncovered})\ + \n{output}\n{examine}" + ); +} + +#[then("a platform that is not covered is given no diagnosis")] +async fn assert_uncovered_platform_gets_no_diagnosis(world: &mut E2eWorld) { + let (report, output) = parsed_diagnosis(world); + let Some(reason) = report + .get("out_of_scope") + .and_then(serde_json::Value::as_str) + else { + return; // covered platform — held to its own half by the next step + }; + assert!( + !reason.trim().is_empty(), + "an out-of-scope verdict must say why:\n{output}" + ); + // The whole point of routing out is to avoid emitting bare-metal findings + // that cannot apply. A verdict with findings attached would be worse than + // no verdict: the caller would act on them. let matched = report .get("matched") .and_then(|m| m.as_array()) .expect("diagnose JSON has no 'matched' array"); assert!( - !matched.is_empty(), - "expected a non-empty 'matched' array for a known symptom:\n{output}" + matched.is_empty(), + "an out-of-scope platform must be given no findings:\n{output}" + ); + assert_eq!( + report.get("has_match").and_then(serde_json::Value::as_bool), + Some(false), + "an out-of-scope platform cannot have an established cause:\n{output}" + ); +} + +#[then("a platform that is covered gets a verdict that follows the evidence")] +async fn assert_covered_platform_verdict_is_consistent(world: &mut E2eWorld) { + let (report, output) = parsed_diagnosis(world); + if report.get("out_of_scope").is_some_and(|v| !v.is_null()) { + return; // uncovered platform — held to its own half by the previous step + } + // The covered branch used to return without asserting anything, which left + // this scenario proving nothing at all on every lane CI actually runs (there + // is no WSL2 runner). This is the half that holds everywhere. + assert_verdict_follows_scores(&report, &output); +} + +#[then("the CLI declines because the fix does not apply to this machine")] +async fn assert_inapplicable_fix_declined(world: &mut E2eWorld) { + // 3 is its own outcome: not a usage error (2), not a failed attempt (4), + // not a refusal by the user (5). A caller that cannot tell them apart + // reports a broken machine when the truth is "wrong operating system". + assert_eq!( + world.cli_rc, + Some(3), + "a fix that does not apply here should exit 3, distinct from 2/4/5" + ); + let output = world.cli_output.as_ref().expect("no fix output"); + assert!( + output.contains("This fix only applies on:"), + "the refusal must say which platforms the fix is for:\n{output}" + ); +} + +#[then("every fix the catalog documents is listed")] +async fn assert_catalog_complete(world: &mut E2eWorld) { + let output = world.cli_output.as_ref().expect("no fix list output"); + let missing: Vec<_> = CATALOG_FIX_IDS + .iter() + .filter(|id| !output.contains(**id)) + .collect(); + assert!( + missing.is_empty(), + "the listing is missing {missing:?}. If the catalog gained or lost a \ + failure mode, update CATALOG_FIX_IDS here and whatever else documents \ + the catalog — do not loosen this assertion:\n{output}" + ); +} + +#[then("only the fixes the CLI can carry out itself are marked as such")] +async fn assert_auto_set_is_exact(world: &mut E2eWorld) { + let output = world.cli_output.as_ref().expect("no fix list output"); + let marked_auto: Vec<&str> = output + .lines() + .filter(|line| line.contains("[ AUTO]")) + .filter_map(|line| CATALOG_FIX_IDS.iter().copied().find(|id| line.contains(id))) + .collect(); + // Exact, not "at least": a mode quietly promoted to AUTO would start + // changing machines that callers were told it only ever advised. + assert_eq!( + marked_auto, AUTO_APPLICABLE_FIX_IDS, + "the set of fixes the CLI applies itself has changed:\n{output}" ); }