Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@ current-user-asks.md
__pycache__/
/.hawkeye-bin/
/.claude
/.supergoal/
/plans/
/workspace/
/e2e-consolidated-report.md
Expand Down
1 change: 1 addition & 0 deletions Cargo.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

47 changes: 45 additions & 2 deletions apps/rocm/src/dash.rs
Original file line number Diff line number Diff line change
Expand Up @@ -66,13 +66,30 @@ pub fn runner_options(
// amd-smi ships inside the managed runtime wheel's bin dir, not on PATH;
// resolve the path so the GPU collector can find it.
amd_smi_binary: Some(rocm_core::resolve_amd_smi_binary()),
// Production always runs the real `/dev/kfd` pre-flight; only daemon
// Production always runs the real GPU-device pre-flight; only daemon
// integration tests with a fake binary skip it.
amd_smi_skip_kfd_preflight: false,
amd_smi_skip_device_preflight: false,
// Runs once per dashboard launch here, not per refresh tick, so the
// extra subprocess spawn is negligible.
amd_smi_gpu_reachable: gpu_reachable_for_preflight(
rocm_core::is_wsl_host(),
rocm_core::has_usable_amd_gpu(),
),
test_clock_offset_path: dash_test_clock_offset_path(),
}
}

/// The verdict may only substitute for the device open on WSL: `has_usable_amd_gpu()`
/// is deliberately fail-open on an unprobeable platform (so it never blocks a
/// launch), which is the wrong polarity for this pre-flight's hang guard — an
/// unknown verdict must NOT skip the real `/dev/kfd` open. Restricting the
/// substitution to WSL (where there is no device node the dash crates can
/// probe directly, only the shared ROCDXG-plumbing probe this same function
/// backs) keeps bare-metal behaviour byte-identical to a plain `/dev/kfd` check.
const fn gpu_reachable_for_preflight(is_wsl_host: bool, has_usable_gpu: bool) -> bool {
is_wsl_host && has_usable_gpu
}

#[cfg(feature = "e2e-test-hooks")]
fn dash_test_clock_offset_path() -> Option<std::path::PathBuf> {
std::env::var_os("ROCM_CLI_DASH_TEST_CLOCK_OFFSET_PATH").map(Into::into)
Expand Down Expand Up @@ -891,6 +908,32 @@ mod tests {
);
}

#[test]
fn runner_options_never_skips_the_device_preflight() {
let p = paths();
let opts = runner_options(&cfg(), &p, false);
assert!(!opts.amd_smi_skip_device_preflight);
}

/// Truth-tables the WSL gate as a pure function, independent of the test
/// host's actual environment: a regression back to the un-gated
/// `has_usable_amd_gpu()` would flip the `(false, true)` bare-metal case
/// to `true`, since the fail-open verdict fires on any unprobeable
/// platform, not just WSL.
///
/// This pins the helper's body only. The call site's binding — that
/// `runner_options` passes `is_wsl_host()` itself rather than a literal —
/// is not pinned by any test: on an ordinary CI host with no usable GPU,
/// `has_usable_amd_gpu()` is already `false`, so a mutant literal `true`
/// there is indistinguishable from the real call from this crate's tests.
#[test]
fn gpu_reachable_for_preflight_only_substitutes_on_wsl() {
assert!(!gpu_reachable_for_preflight(false, true));
assert!(!gpu_reachable_for_preflight(false, false));
assert!(!gpu_reachable_for_preflight(true, false));
assert!(gpu_reachable_for_preflight(true, true));
}

#[test]
fn runner_options_derives_default_bench_csv_from_current_paths() {
let p = paths();
Expand Down
1 change: 1 addition & 0 deletions crates/rocm-dash-collectors/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -28,4 +28,5 @@ reqwest = { version = "0.12", default-features = false }

[dev-dependencies]
tokio = { version = "1", features = ["full"] }
tempfile = "3"
wiremock = "0.6"
107 changes: 77 additions & 30 deletions crates/rocm-dash-collectors/src/amd_smi.rs
Original file line number Diff line number Diff line change
Expand Up @@ -4,11 +4,12 @@

//! amd-smi subprocess + JSON parse.
//!
//! Field paths and the KFD pre-flight check are vendored from the TypeScript
//! Field paths and the GPU-device pre-flight are vendored from the TypeScript
//! `AmdSmiProvider` in instinct-dash. See `../wiki/entities/amd-smi.md`.

use std::ffi::OsString;
use std::io;
use std::path::Path;
use std::time::Duration;

use rocm_dash_core::metrics::{GpuMetrics, GpuSystemInfo};
Expand All @@ -29,46 +30,58 @@ pub struct AmdSmiCollector {
}

impl AmdSmiCollector {
/// Returns `Some` only if `/dev/kfd` is readable AND `amd-smi version` succeeds.
/// Uses an explicit `amd-smi` binary path, and takes a precomputed
/// GPU-reachability verdict for the pre-flight. Returns `Some` only if a
/// supported GPU device is readable (or `gpu_reachable` substitutes for
/// that check) and `amd-smi version` succeeds.
///
/// The KFD pre-flight is mandatory: without it, `amd-smi` blocks in
/// uninterruptible kernel sleep (D-state) that no signal can escape.
pub async fn detect() -> Option<Self> {
Self::detect_with_binary("amd-smi").await
}

/// Like [`detect`](Self::detect) but uses an explicit `amd-smi` binary path.
/// The device pre-flight is mandatory: without an accessible `/dev/kfd` on
/// bare-metal Linux, `amd-smi` can block in uninterruptible kernel sleep
/// (D-state) that no signal can escape.
///
/// The managed ROCm SDK ships `amd-smi` inside the runtime wheel's bin
/// directory rather than on `PATH`, so callers resolve the path or command
/// name (via `rocm_core::resolve_amd_smi_binary`) and pass it here.
pub async fn detect_with_binary(binary: impl Into<OsString>) -> Option<Self> {
Self::detect_with_binary_inner(binary, false).await
///
/// `gpu_reachable` lets the pre-flight pass without a readable `/dev/kfd`
/// (e.g. on WSL, where the real signal is `rocm_core::has_usable_amd_gpu()`
/// rather than any single device node this crate could probe directly).
/// This crate deliberately does not depend on `rocm-core` to compute that
/// verdict itself; the caller (`apps/rocm`, which already depends on it)
/// threads it through instead, so on WSL, `serve`/`examine`/the dashboard
/// never disagree about whether a host's GPU is usable. On bare metal the
/// caller always passes `false`, so this is still a plain `/dev/kfd` check.
pub async fn detect_with_binary(
binary: impl Into<OsString>,
gpu_reachable: bool,
) -> Option<Self> {
Self::detect_with_binary_inner(binary, false, gpu_reachable).await
}

/// Like [`detect_with_binary`](Self::detect_with_binary) but skips the
/// mandatory `/dev/kfd` pre-flight.
/// mandatory GPU-device pre-flight.
///
/// **Test-only.** The KFD pre-flight is a safety guard: against a *real*
/// `amd-smi` on a host without a usable `/dev/kfd`, the process can block in
/// uninterruptible kernel sleep (D-state) that no signal can escape. This
/// entry point exists solely so daemon integration tests can point
/// [`detect_with_binary`](Self::detect_with_binary) at a *fake* script (for
/// which the hang cannot happen) and have it actually run on a GPU-less CI
/// host, instead of short-circuiting to `None` and turning the test into a
/// no-op. Never call it against a real binary in production.
/// **Test-only.** The device pre-flight is a safety guard: against a *real*
/// `amd-smi` on a host without an accessible `/dev/kfd`, the process can
/// block in uninterruptible kernel sleep (D-state) that no signal can
/// escape. This entry point exists solely so daemon integration tests can
/// point [`detect_with_binary`](Self::detect_with_binary) at a *fake*
/// script (for which the hang cannot happen) and have it actually run on a
/// GPU-less CI host, instead of short-circuiting to `None` and turning the
/// test into a no-op. Never call it against a real binary in production.
#[doc(hidden)]
pub async fn detect_with_binary_skipping_kfd_preflight(
pub async fn detect_with_binary_skipping_device_preflight(
binary: impl Into<OsString>,
) -> Option<Self> {
Self::detect_with_binary_inner(binary, true).await
Self::detect_with_binary_inner(binary, true, false).await
}

async fn detect_with_binary_inner(
binary: impl Into<OsString>,
skip_kfd_preflight: bool,
skip_device_preflight: bool,
gpu_reachable: bool,
) -> Option<Self> {
if !skip_kfd_preflight && !kfd_accessible() {
if !preflight_passes(skip_device_preflight, gpu_reachable, Path::new(KFD_DEVICE)) {
return None;
}
let me = Self {
Expand Down Expand Up @@ -134,11 +147,16 @@ impl AmdSmiCollector {
}
}

fn kfd_accessible() -> bool {
std::fs::OpenOptions::new()
.read(true)
.open(KFD_DEVICE)
.is_ok()
/// The device pre-flight passes if the caller already skipped it, already
/// knows the GPU is reachable by some other means (e.g. WSL, via
/// `rocm_core::has_usable_amd_gpu()`), or the bare-metal device node itself is
/// readable.
fn preflight_passes(skip_device_preflight: bool, gpu_reachable: bool, kfd_device: &Path) -> bool {
skip_device_preflight || gpu_reachable || device_accessible(kfd_device)
}

fn device_accessible(path: &Path) -> bool {
std::fs::OpenOptions::new().read(true).open(path).is_ok()
}

fn val_u64(v: Option<&Value>) -> Option<u64> {
Expand Down Expand Up @@ -365,6 +383,35 @@ fn parse_memory(v: Option<&Value>) -> MemoryPartitionMode {
mod tests {
use super::*;

#[test]
fn preflight_skipped_passes_regardless_of_device_or_reachability() {
let missing = Path::new("/nonexistent/kfd");
assert!(preflight_passes(true, false, missing));
}

#[test]
fn precomputed_reachability_satisfies_preflight_without_kfd() {
let missing = Path::new("/nonexistent/kfd");
assert!(preflight_passes(false, true, missing));
}

#[test]
fn readable_kfd_device_satisfies_preflight() {
let dir = tempfile::tempdir().unwrap();
let kfd = dir.path().join("kfd");
std::fs::write(&kfd, []).unwrap();

assert!(preflight_passes(false, false, &kfd));
}

#[test]

Copy link
Copy Markdown
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Two small things here.

Nit (formatting): no blank line after use super::*;. This is the sole outlier among 18 test modules repo-wide, and cargo fmt won't catch it.

Coverage: of five plausible mutants of gpu_device_accessible, this test kills only ||→&&. Surviving: returning unconditional true; a typo in DXG_DEVICE; a call site that passes KFD_DEVICE twice; .write(true) instead of .read(true). There's no assertion of the false direction at all, and DXG_DEVICE has exactly one reference in the crate — the untested call site at line 76. Because the test writes a regular file, the interesting real-device failure modes (EACCES / ENXIO / EBUSY on a char device) are unrepresented, so what it mostly proves is std::fs semantics.

Minimal close: add a gpu_device_accessible(missing, missing) == false case, a KFD-only case, and assert_eq!(DXG_DEVICE, "/dev/dxg").

fn missing_kfd_and_no_reachability_fails_preflight() {
let dir = tempfile::tempdir().unwrap();
let missing_kfd = dir.path().join("missing-kfd");

assert!(!preflight_passes(false, false, &missing_kfd));
}

const SAMPLE_METRIC: &str = r#"{
"gpu_data": [
{
Expand Down Expand Up @@ -528,7 +575,7 @@ mod tests {
#[tokio::test]
#[ignore = "requires a real AMD GPU + amd-smi; run manually on hardware"]
async fn live_processes_no_panic() {
if let Some(c) = AmdSmiCollector::detect().await {
if let Some(c) = AmdSmiCollector::detect_with_binary("amd-smi", false).await {
// Either Ok or Err is acceptable; the contract is "does not panic".
let _ = c.processes().await;
}
Expand Down
Loading
Loading