Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions Cargo.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

4 changes: 3 additions & 1 deletion apps/rocm/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,9 @@ rocm-deps = { path = "../../crates/rocm-deps" }
# rocm-dash unified dashboard launch. The
# telemetry daemon + ratatui-0.30 TUI are launched from the `dash` verb; tokio
# drives the async daemon/TUI from the otherwise-sync `rocm` binary.
# (rocm-dash-core is pulled in transitively by the two below — not a direct dep.)
# rocm-dash-core is a direct dep so the launcher front door can build serving
# `Instance`s from the managed-service registry.
rocm-dash-core = { path = "../../crates/rocm-dash-core" }
rocm-dash-daemon = { path = "../../crates/rocm-dash-daemon" }
rocm-dash-tui = { path = "../../crates/rocm-dash-tui" }
tokio = { workspace = true }
Expand Down
105 changes: 104 additions & 1 deletion apps/rocm/src/dash.rs
Original file line number Diff line number Diff line change
Expand Up @@ -66,6 +66,9 @@ pub fn runner_options(
// amd-smi ships inside the managed runtime wheel's bin dir, not on PATH;
// resolve the path so the GPU collector can find it.
amd_smi_binary: Some(rocm_core::resolve_amd_smi_binary()),
// Production always runs the real `/dev/kfd` pre-flight; only daemon
// integration tests with a fake binary skip it.
amd_smi_skip_kfd_preflight: false,
}
}

Expand Down Expand Up @@ -358,7 +361,12 @@ pub fn run_launcher(chat_mock: bool) -> Result<()> {
let config = RocmCliConfig::load(&paths).unwrap_or_default();
let theme = config.dashboard.tui.theme;
loop {
match rocm_dash_tui::ui::launcher::run_launcher(&theme)? {
// Re-read the managed-service registry on each pass so the front door
// reflects models started (or stopped) during a prior flow. This is a
// cheap status-only file read — no telemetry daemon and no network
// readiness probes, so the front door stays instant.
let serving = launcher_serving_instances(&paths);
match rocm_dash_tui::ui::launcher::run_launcher(&theme, serving)? {
None => return Ok(()),
Some(choice) => match launcher_route(choice) {
LauncherRoute::Focused(focus) => run_focused(focus)?,
Expand All @@ -369,6 +377,28 @@ pub fn run_launcher(chat_mock: bool) -> Result<()> {
}
}

/// Serving instances for the launcher front door, read from the managed-service
/// registry (the same authority `rocm services` reads).
///
/// Deliberately cheap: a status-only registry read with no network readiness
/// probes and no telemetry daemon, so the front door renders instantly.
fn launcher_serving_instances(paths: &AppPaths) -> Vec<rocm_dash_core::metrics::Instance> {
use rocm_dash_daemon::registry::{discover_managed_services, load_service_records};
let records = load_service_records(&paths.services_dir());
discover_managed_services(&records)
.svcs
.into_iter()
.map(|svc| rocm_dash_core::metrics::Instance {
container_id: svc.container_id,
container_name: svc.container_name,
model_name: svc.model_name,
status: svc.status,
port: svc.port,
..Default::default()
})
.collect()
}

/// Entry point for a focused launcher flow (Set up / Serve / Diagnose).
///
/// Opens the dashboard runtime hosting exactly the one overlay for `focus` — no
Expand Down Expand Up @@ -916,4 +946,77 @@ mod tests {
first.preferred_engines.first()
);
}

/// The launcher front-door seam that regressed: a `ready` managed vLLM
/// service recorded on disk must map into a live `Instance` (id, model,
/// port, status) so bare `rocm` shows it instead of "Idle". Reads the same
/// registry `rocm services` reads — status-only, no daemon, no network.
#[test]
fn launcher_serving_instances_maps_ready_registry_record() {
let root = std::env::temp_dir().join(format!(
"rocm-cli-launcher-serving-test-{}-{}",
std::process::id(),
rocm_core::unix_time_millis()
));
let p = AppPaths {
config_dir: root.join("cfg"),
data_dir: root.join("data"),
cache_dir: root.join("cache"),
};
let services_dir = p.services_dir();
std::fs::create_dir_all(&services_dir).unwrap();
std::fs::write(
services_dir.join("svc.json"),
r#"{"service_id":"vllm-launcher","engine":"vllm",
"model_ref":"meta-llama/Llama-3.1-8B","canonical_model_id":"m",
"host":"127.0.0.1","port":11435,
"endpoint_url":"http://127.0.0.1:11435/v1","mode":"managed",
"status":"ready","created_at_unix_ms":1}"#,
)
.unwrap();

let instances = launcher_serving_instances(&p);

assert_eq!(instances.len(), 1, "the ready managed service must surface");
let inst = &instances[0];
assert_eq!(inst.container_id, "vllm-launcher");
assert_eq!(inst.model_name, "meta-llama/Llama-3.1-8B");
assert_eq!(inst.port, Some(11435));
assert_eq!(inst.status, rocm_dash_core::metrics::InstanceStatus::Ready);

let _ = std::fs::remove_dir_all(&root);
}

/// A non-scrapeable record (unbound `port:0`) must NOT surface as a live
/// instance on the front door — the registry is the authority and `:0` is
/// never a real serving endpoint.
#[test]
fn launcher_serving_instances_skips_unbound_record() {
let root = std::env::temp_dir().join(format!(
"rocm-cli-launcher-serving-skip-test-{}-{}",
std::process::id(),
rocm_core::unix_time_millis()
));
let p = AppPaths {
config_dir: root.join("cfg"),
data_dir: root.join("data"),
cache_dir: root.join("cache"),
};
let services_dir = p.services_dir();
std::fs::create_dir_all(&services_dir).unwrap();
std::fs::write(
services_dir.join("svc.json"),
r#"{"service_id":"vllm-unbound","engine":"vllm","model_ref":"m",
"canonical_model_id":"m","host":"127.0.0.1","port":0,
"mode":"managed","status":"ready","created_at_unix_ms":1}"#,
)
.unwrap();

assert!(
launcher_serving_instances(&p).is_empty(),
"an unbound (:0) record must not surface as a live instance"
);

let _ = std::fs::remove_dir_all(&root);
}
}
27 changes: 26 additions & 1 deletion crates/rocm-dash-collectors/src/amd_smi.rs
Original file line number Diff line number Diff line change
Expand Up @@ -43,7 +43,32 @@ impl AmdSmiCollector {
/// directory rather than on `PATH`, so callers resolve the path or command
/// name (via `rocm_core::resolve_amd_smi_binary`) and pass it here.
pub async fn detect_with_binary(binary: impl Into<OsString>) -> Option<Self> {
if !kfd_accessible() {
Self::detect_with_binary_inner(binary, false).await
}

/// Like [`detect_with_binary`](Self::detect_with_binary) but skips the
/// mandatory `/dev/kfd` pre-flight.
///
/// **Test-only.** The KFD pre-flight is a safety guard: against a *real*
/// `amd-smi` on a host without a usable `/dev/kfd`, the process can block in
/// uninterruptible kernel sleep (D-state) that no signal can escape. This
/// entry point exists solely so daemon integration tests can point
/// [`detect_with_binary`](Self::detect_with_binary) at a *fake* script (for
/// which the hang cannot happen) and have it actually run on a GPU-less CI
/// host, instead of short-circuiting to `None` and turning the test into a
/// no-op. Never call it against a real binary in production.
#[doc(hidden)]
pub async fn detect_with_binary_skipping_kfd_preflight(
binary: impl Into<OsString>,
) -> Option<Self> {
Self::detect_with_binary_inner(binary, true).await
}

async fn detect_with_binary_inner(
binary: impl Into<OsString>,
skip_kfd_preflight: bool,
) -> Option<Self> {
if !skip_kfd_preflight && !kfd_accessible() {
return None;
}
let me = Self {
Expand Down
103 changes: 85 additions & 18 deletions crates/rocm-dash-daemon/src/runner.rs
Original file line number Diff line number Diff line change
Expand Up @@ -74,6 +74,14 @@ pub struct RunnerOptions {
/// so the caller resolves it (via `rocm_core::resolve_amd_smi_binary`) and
/// passes it here. `None` falls back to looking up `amd-smi` on `PATH`.
pub amd_smi_binary: Option<OsString>,
/// **Test-only.** Skip the mandatory `/dev/kfd` pre-flight in amd-smi
/// detection so a *fake* `amd_smi_binary` is actually invoked on a GPU-less
/// CI host instead of short-circuiting to "no GPU". Never set in
/// production: the KFD guard prevents a *real* `amd-smi` from hanging in
/// uninterruptible D-state. Only the daemon integration test that points
/// `amd_smi_binary` at a deliberately-slow fake script flips this, so the
/// off-critical-path detection behaviour is genuinely exercised.
pub amd_smi_skip_kfd_preflight: bool,
}

impl Default for RunnerOptions {
Expand All @@ -93,6 +101,7 @@ impl Default for RunnerOptions {
persist_dir: None,
services_dir: None,
amd_smi_binary: None,
amd_smi_skip_kfd_preflight: false,
}
}
}
Expand Down Expand Up @@ -200,23 +209,36 @@ pub async fn run_loop(
// stable between scrapes (mirrors how GPU power drives tokens_per_watt).
let mut per_container_used: HashMap<String, u64> = HashMap::new();

let gpu = match opts.amd_smi_binary.clone() {
Some(binary) => AmdSmiCollector::detect_with_binary(binary).await,
None => AmdSmiCollector::detect().await,
};
let mut gpu_system_info: Option<GpuSystemInfo> = if let Some(g) = &gpu {
let info = g.system_info().await;
info!(
gpus = info.physical_gpu_count,
model = %info.gpu_model,
rocm = info.rocm_version.as_deref().unwrap_or("?"),
"amd-smi detected"
);
Some(info)
} else {
warn!("amd-smi not available (no /dev/kfd or `amd-smi version` failed); GPU disabled");
None
};
// amd-smi GPU detection plus the first `system_info()` can take up to ~15s
// on some hosts (subprocess spawn plus the per-call detect/run timeouts).
// Doing it inline here blocked the run loop's very first tick — and thus
// managed-service discovery and the first snapshot broadcast — behind it, so
// an already-running model did not surface in the dashboard until GPU
// detection finished (a visible ~15-20s "0 models running" lag while
// `rocm services` already reported it). Run detection off the critical path:
// the loop starts ticking immediately (surfacing serving instances within
// one discovery tick) and GPU metrics fill in the moment detection lands.
let amd_smi_binary = opts.amd_smi_binary.clone();
let amd_smi_skip_kfd_preflight = opts.amd_smi_skip_kfd_preflight;
let (gpu_init_tx, mut gpu_init_rx) =
tokio::sync::oneshot::channel::<(Option<AmdSmiCollector>, Option<GpuSystemInfo>)>();
tokio::spawn(async move {
let gpu = match amd_smi_binary {
Some(binary) if amd_smi_skip_kfd_preflight => {
AmdSmiCollector::detect_with_binary_skipping_kfd_preflight(binary).await
}
Some(binary) => AmdSmiCollector::detect_with_binary(binary).await,
None => AmdSmiCollector::detect().await,
};
let info = match &gpu {
Some(g) => Some(g.system_info().await),
None => None,
};
let _ = gpu_init_tx.send((gpu, info));
});
let mut gpu: Option<AmdSmiCollector> = None;
let mut gpu_system_info: Option<GpuSystemInfo> = None;
let mut gpu_init_done = false;

let mut tick_count: u64 = 0;
let mut last_sysinfo_refresh: u64 = 0;
Expand All @@ -234,6 +256,45 @@ pub async fn run_loop(
// every instance refreshed in this cycle serialises Fresh deterministically.
let cycle_at = Utc::now();

// Adopt the background amd-smi detection result the moment it lands,
// without ever blocking the loop while it is still in flight. Until then
// `gpu` stays `None` (GPU panels read "unavailable") but discovery and
// snapshots run every tick, so serving instances appear promptly.
if !gpu_init_done {
match gpu_init_rx.try_recv() {
Ok((detected, info)) => {
gpu_init_done = true;
if let Some(i) = &info {
info!(
gpus = i.physical_gpu_count,
model = %i.gpu_model,
rocm = i.rocm_version.as_deref().unwrap_or("?"),
"amd-smi detected"
);
} else {
warn!(
"amd-smi not available (no /dev/kfd or `amd-smi version` failed); GPU disabled"
);
}
gpu = detected;
gpu_system_info = info;
last_sysinfo_refresh = tick_count;
}
// The detection task dropped its sender without ever sending.
// It has no fallible await today, so this is latent rather than
// a live path — but if it ever regressed, polling `Empty`
// forever would hang `gpu_init_done` at false and suppress the
// honest "unavailable" warning below. Settle as unavailable and
// say so once, so the badge resolves instead of silently never
// clearing.
Err(tokio::sync::oneshot::error::TryRecvError::Closed) => {
gpu_init_done = true;
warn!("amd-smi detection task ended without a result; GPU disabled");
}
Err(tokio::sync::oneshot::error::TryRecvError::Empty) => {}
}
}

let mut warnings = Vec::new();
let gpus = if let Some(g) = &gpu {
match g.metrics().await {
Expand All @@ -243,9 +304,15 @@ pub async fn run_loop(
Vec::new()
}
}
} else {
} else if gpu_init_done {
warnings.push("amd-smi unavailable (no /dev/kfd or binary missing)".into());
Vec::new()
} else {
// Detection is still in flight (spawned off the critical path), so
// `gpu == None` is transient here. Stay silent instead of flashing a
// false "unavailable" alarm on healthy hardware for the ~15s window
// until the first detection result lands.
Vec::new()
};

if gpu.is_some()
Expand Down
Loading
Loading