From 7d399a5519ac8738a01a775d40e36e5b3daa662e Mon Sep 17 00:00:00 2001 From: Roman Sirokov Date: Fri, 21 Aug 2026 07:45:40 +0000 Subject: [PATCH 1/3] fix(dash): seed launcher front door with live serving instances The launcher front door built an empty AppState, so it always rendered the idle variant even when a model was actively serving. Read the managed-service registry (the same authority `rocm services` reads) once per hub-loop pass and seed the AppState's instances from it. Also treat `Ready` as serving everywhere the dashboard counts running instances (`is_serving()`), matching the `Running`+`Ready` treatment already used elsewhere (e.g. home.rs) -- a served model reports `Ready`, not `Running`, so the count previously undercounted actual serving models. Adds apps/rocm's direct dependency on rocm-dash-core (previously only a transitive dep) so the launcher can build `Instance`s from the registry's `DiscoveredService` records. Signed-off-by: Roman Sirokov --- Cargo.lock | 1 + apps/rocm/Cargo.toml | 4 +- apps/rocm/src/dash.rs | 29 +++++++++- crates/rocm-dash-tui/src/ui/launcher.rs | 60 +++++++++++++++++++-- crates/rocm-dash-tui/src/ui/tabs/pane.rs | 4 +- crates/rocm-dash-tui/src/ui/tabs/serving.rs | 30 +++++++++++ 6 files changed, 119 insertions(+), 9 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index b71b3fada0..473b8eb7bc 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3569,6 +3569,7 @@ dependencies = [ "keyring-core", "libc", "rocm-core", + "rocm-dash-core", "rocm-dash-daemon", "rocm-dash-tui", "rocm-deps", diff --git a/apps/rocm/Cargo.toml b/apps/rocm/Cargo.toml index 7c71f68669..b5c0a39b9f 100644 --- a/apps/rocm/Cargo.toml +++ b/apps/rocm/Cargo.toml @@ -22,7 +22,9 @@ rocm-deps = { path = "../../crates/rocm-deps" } # rocm-dash unified dashboard launch. The # telemetry daemon + ratatui-0.30 TUI are launched from the `dash` verb; tokio # drives the async daemon/TUI from the otherwise-sync `rocm` binary. -# (rocm-dash-core is pulled in transitively by the two below — not a direct dep.) +# rocm-dash-core is a direct dep so the launcher front door can build serving +# `Instance`s from the managed-service registry. +rocm-dash-core = { path = "../../crates/rocm-dash-core" } rocm-dash-daemon = { path = "../../crates/rocm-dash-daemon" } rocm-dash-tui = { path = "../../crates/rocm-dash-tui" } tokio = { workspace = true } diff --git a/apps/rocm/src/dash.rs b/apps/rocm/src/dash.rs index 230a4add8a..cecc81a997 100644 --- a/apps/rocm/src/dash.rs +++ b/apps/rocm/src/dash.rs @@ -358,7 +358,12 @@ pub fn run_launcher(chat_mock: bool) -> Result<()> { let config = RocmCliConfig::load(&paths).unwrap_or_default(); let theme = config.dashboard.tui.theme; loop { - match rocm_dash_tui::ui::launcher::run_launcher(&theme)? { + // Re-read the managed-service registry on each pass so the front door + // reflects models started (or stopped) during a prior flow. This is a + // cheap status-only file read — no telemetry daemon and no network + // readiness probes, so the front door stays instant. + let serving = launcher_serving_instances(&paths); + match rocm_dash_tui::ui::launcher::run_launcher(&theme, serving)? { None => return Ok(()), Some(choice) => match launcher_route(choice) { LauncherRoute::Focused(focus) => run_focused(focus)?, @@ -369,6 +374,28 @@ pub fn run_launcher(chat_mock: bool) -> Result<()> { } } +/// Serving instances for the launcher front door, read from the managed-service +/// registry (the same authority `rocm services` reads). +/// +/// Deliberately cheap: a status-only registry read with no network readiness +/// probes and no telemetry daemon, so the front door renders instantly. +fn launcher_serving_instances(paths: &AppPaths) -> Vec { + use rocm_dash_daemon::registry::{discover_managed_services, load_service_records}; + let records = load_service_records(&paths.services_dir()); + discover_managed_services(&records) + .svcs + .into_iter() + .map(|svc| rocm_dash_core::metrics::Instance { + container_id: svc.container_id, + container_name: svc.container_name, + model_name: svc.model_name, + status: svc.status, + port: svc.port, + ..Default::default() + }) + .collect() +} + /// Entry point for a focused launcher flow (Set up / Serve / Diagnose). /// /// Opens the dashboard runtime hosting exactly the one overlay for `focus` — no diff --git a/crates/rocm-dash-tui/src/ui/launcher.rs b/crates/rocm-dash-tui/src/ui/launcher.rs index ccc2a61d4c..144f532538 100644 --- a/crates/rocm-dash-tui/src/ui/launcher.rs +++ b/crates/rocm-dash-tui/src/ui/launcher.rs @@ -16,6 +16,7 @@ use ratatui::style::{Modifier, Style}; use ratatui::text::{Line, Span}; use ratatui::widgets::Paragraph; +use rocm_dash_core::metrics::Instance; #[cfg(test)] use rocm_dash_core::metrics::InstanceStatus; @@ -239,18 +240,38 @@ fn draw_menu(f: &mut Frame, area: Rect, state: &AppState, sel: usize, theme: &Th f.render_widget(Paragraph::new(lines), area); } +/// Build the launcher's render state, seeding `instances` from the serving +/// models the caller discovered in the managed-service registry. +/// +/// Kept separate from the terminal loop so the seeding is unit-testable without +/// a real terminal: an empty `serving` yields the idle front door, a non-empty +/// one makes `is_running` report the running model. +fn launcher_state(theme_name: &str, serving: Vec) -> AppState { + let mut state = AppState::new(String::new(), theme_name.to_string()); + state.instances = serving + .into_iter() + .map(|inst| (inst.container_id.clone(), inst)) + .collect(); + state +} + /// Run the launcher as a synchronous pre-dash screen. /// /// Returns the chosen destination, or `None` when the user quits. On success the /// caller escalates into the existing dash entry points (ponytail: no parallel /// async loop here). /// -/// Renders the idle variant from a fresh `AppState` (no live daemon is started -/// just for the front door); live telemetry appears once the dash is opened. +/// `serving` seeds the status strip / menu so the front door reflects the models +/// the managed-service registry reports as running (the same authority +/// `rocm services` reads) — without starting a live telemetry daemon just for the +/// front door. Live GPU telemetry still appears once the dash is opened. /// /// # Errors /// Propagates terminal setup / event-read I/O errors. -pub fn run_launcher(theme_name: &str) -> std::io::Result> { +pub fn run_launcher( + theme_name: &str, + serving: Vec, +) -> std::io::Result> { use crossterm::event::{self, Event, KeyCode, KeyEventKind}; use crossterm::terminal::{ EnterAlternateScreen, LeaveAlternateScreen, disable_raw_mode, enable_raw_mode, @@ -258,7 +279,7 @@ pub fn run_launcher(theme_name: &str) -> std::io::Result> use ratatui::Terminal; use ratatui::backend::CrosstermBackend; - let state = AppState::new(String::new(), theme_name.to_string()); + let state = launcher_state(theme_name, serving); let theme = state.theme; enable_raw_mode()?; @@ -397,6 +418,37 @@ mod tests { assert!(out.contains("Serve a model"), "menu missing in idle"); } + #[test] + fn launcher_state_seeds_serving_from_registry_instances() { + // Regression: the front door built an empty AppState and always showed + // "Idle — nothing serving" even when a model was running. Seeding from + // the caller's registry-derived instances must make the launcher report + // the running model. + let serving = vec![Instance { + container_id: "vllm-ready".into(), + model_name: "Qwen2.5-0.5B".into(), + status: InstanceStatus::Ready, + port: Some(11435), + ..Default::default() + }]; + let running = launcher_state("default-dark", serving); + assert!( + is_running(&running), + "seeded serving instance must make the launcher report running" + ); + let out = render(&running, 0, 100, 24); + assert!(out.contains("Serving"), "serve line missing: {out:?}"); + assert!( + out.contains("Qwen2.5-0.5B"), + "served model missing: {out:?}" + ); + assert!(out.contains("11435"), "served port missing: {out:?}"); + + // With no serving instances the front door stays honestly idle. + let idle = launcher_state("default-dark", Vec::new()); + assert!(!is_running(&idle), "empty seed must render idle"); + } + #[test] fn choice_mapping_and_counts() { assert_eq!(row_count(), 5); diff --git a/crates/rocm-dash-tui/src/ui/tabs/pane.rs b/crates/rocm-dash-tui/src/ui/tabs/pane.rs index bd7ba1115c..4892d55e95 100644 --- a/crates/rocm-dash-tui/src/ui/tabs/pane.rs +++ b/crates/rocm-dash-tui/src/ui/tabs/pane.rs @@ -20,8 +20,6 @@ use ratatui::style::{Modifier, Style}; use ratatui::text::{Line, Span}; use ratatui::widgets::{Paragraph, Wrap}; -use rocm_dash_core::metrics::InstanceStatus; - use crate::app::{AppState, KeyAction, PaneFocus}; use crate::ui::format; use crate::ui::panel::{self, BoxRole}; @@ -406,7 +404,7 @@ fn live_lines(action: KeyAction, state: &AppState, theme: &Theme) -> Vec = state .instances .values() - .filter(|i| i.status == InstanceStatus::Running) + .filter(|i| i.status.is_serving()) .collect(); let mut lines = vec![head(format!("Running now — {}", running.len()))]; if running.is_empty() { diff --git a/crates/rocm-dash-tui/src/ui/tabs/serving.rs b/crates/rocm-dash-tui/src/ui/tabs/serving.rs index d3afacebc0..c631c3d639 100644 --- a/crates/rocm-dash-tui/src/ui/tabs/serving.rs +++ b/crates/rocm-dash-tui/src/ui/tabs/serving.rs @@ -206,6 +206,36 @@ mod tests { assert!(out.contains("8000"), "running port not shown: {out:?}"); } + #[test] + fn serving_detail_counts_ready_instances_as_running() { + // Regression: a served model reports `Ready` (readiness-probed), not + // `Running`. The detail pane must count it as serving so the count + // matches `rocm services`, which also treats ready as live. + use rocm_dash_core::metrics::{Instance, InstanceStatus}; + let mut s = AppState::new("t".into(), "default-dark".into()); + s.active_tab = ActiveTab::Serving; + s.serving_sel = 2; // "Running instances/services" → OpenServices + s.instances.insert( + "vllm-1".into(), + Instance { + container_name: "vllm-1".into(), + model_name: "Llama-3.1-8B".into(), + status: InstanceStatus::Ready, + port: Some(8000), + ..Default::default() + }, + ); + let out = render(&s, 120, 28); + assert!( + out.contains("Running now — 1"), + "ready instance not counted as running: {out:?}" + ); + assert!( + out.contains("Llama-3.1-8B"), + "ready model not shown inline: {out:?}" + ); + } + #[test] fn serving_verb_action_maps_selection_to_seam() { assert_eq!(verb_action(0), KeyAction::OpenServeWizard); From 72c1ea1940e8083f96e1c68e93c956b4f07e8f9e Mon Sep 17 00:00:00 2001 From: Roman Sirokov Date: Fri, 21 Aug 2026 07:46:54 +0000 Subject: [PATCH 2/3] fix(dash-daemon): run amd-smi detection off the run loop's critical path Detecting amd-smi (`amd-smi version` plus the first `system_info()`) can take up to ~15s on real hardware. Running it inline before the run loop's first tick blocked managed-service discovery and the first snapshot broadcast behind it, so an already-running model did not surface in the dashboard until GPU detection finished -- a visible ~15-20s "0 models running" lag while `rocm services` already reported it live. Spawn detection in the background and adopt the result via a oneshot channel the moment it lands, without ever blocking the loop while it is in flight. The loop now starts ticking immediately, so serving instances appear within one discovery tick; GPU metrics fill in once detection completes. Adds a regression test asserting on ordering (the instance must surface in a snapshot whose gpu_system_info is still None) rather than wall-clock timing, since a pure "arrived within Ns" check would be flaky under subscriber starvation. Signed-off-by: Roman Sirokov --- crates/rocm-dash-daemon/src/runner.rs | 68 +++++++++---- crates/rocm-dash-daemon/tests/end_to_end.rs | 104 ++++++++++++++++++++ 2 files changed, 155 insertions(+), 17 deletions(-) diff --git a/crates/rocm-dash-daemon/src/runner.rs b/crates/rocm-dash-daemon/src/runner.rs index 492a1eb0bc..946a4db1f1 100644 --- a/crates/rocm-dash-daemon/src/runner.rs +++ b/crates/rocm-dash-daemon/src/runner.rs @@ -200,23 +200,32 @@ pub async fn run_loop( // stable between scrapes (mirrors how GPU power drives tokens_per_watt). let mut per_container_used: HashMap = HashMap::new(); - let gpu = match opts.amd_smi_binary.clone() { - Some(binary) => AmdSmiCollector::detect_with_binary(binary).await, - None => AmdSmiCollector::detect().await, - }; - let mut gpu_system_info: Option = if let Some(g) = &gpu { - let info = g.system_info().await; - info!( - gpus = info.physical_gpu_count, - model = %info.gpu_model, - rocm = info.rocm_version.as_deref().unwrap_or("?"), - "amd-smi detected" - ); - Some(info) - } else { - warn!("amd-smi not available (no /dev/kfd or `amd-smi version` failed); GPU disabled"); - None - }; + // amd-smi GPU detection plus the first `system_info()` can take up to ~15s + // on some hosts (subprocess spawn plus the per-call detect/run timeouts). + // Doing it inline here blocked the run loop's very first tick — and thus + // managed-service discovery and the first snapshot broadcast — behind it, so + // an already-running model did not surface in the dashboard until GPU + // detection finished (a visible ~15-20s "0 models running" lag while + // `rocm services` already reported it). Run detection off the critical path: + // the loop starts ticking immediately (surfacing serving instances within + // one discovery tick) and GPU metrics fill in the moment detection lands. + let amd_smi_binary = opts.amd_smi_binary.clone(); + let (gpu_init_tx, mut gpu_init_rx) = + tokio::sync::oneshot::channel::<(Option, Option)>(); + tokio::spawn(async move { + let gpu = match amd_smi_binary { + Some(binary) => AmdSmiCollector::detect_with_binary(binary).await, + None => AmdSmiCollector::detect().await, + }; + let info = match &gpu { + Some(g) => Some(g.system_info().await), + None => None, + }; + let _ = gpu_init_tx.send((gpu, info)); + }); + let mut gpu: Option = None; + let mut gpu_system_info: Option = None; + let mut gpu_init_done = false; let mut tick_count: u64 = 0; let mut last_sysinfo_refresh: u64 = 0; @@ -234,6 +243,31 @@ pub async fn run_loop( // every instance refreshed in this cycle serialises Fresh deterministically. let cycle_at = Utc::now(); + // Adopt the background amd-smi detection result the moment it lands, + // without ever blocking the loop while it is still in flight. Until then + // `gpu` stays `None` (GPU panels read "unavailable") but discovery and + // snapshots run every tick, so serving instances appear promptly. + if !gpu_init_done && let Ok((detected, info)) = gpu_init_rx.try_recv() { + gpu_init_done = true; + if detected.is_some() { + if let Some(i) = &info { + info!( + gpus = i.physical_gpu_count, + model = %i.gpu_model, + rocm = i.rocm_version.as_deref().unwrap_or("?"), + "amd-smi detected" + ); + } + } else { + warn!( + "amd-smi not available (no /dev/kfd or `amd-smi version` failed); GPU disabled" + ); + } + gpu = detected; + gpu_system_info = info; + last_sysinfo_refresh = tick_count; + } + let mut warnings = Vec::new(); let gpus = if let Some(g) = &gpu { match g.metrics().await { diff --git a/crates/rocm-dash-daemon/tests/end_to_end.rs b/crates/rocm-dash-daemon/tests/end_to_end.rs index 2d25f4bc1f..835f1f9d28 100644 --- a/crates/rocm-dash-daemon/tests/end_to_end.rs +++ b/crates/rocm-dash-daemon/tests/end_to_end.rs @@ -319,3 +319,107 @@ async fn scrape_warning_persists_between_scrape_ticks() { handle.abort(); let _ = handle.await; } + +/// Regression: a slow `amd-smi` startup must NOT delay managed-service discovery. +/// +/// GPU detection (`amd-smi version` + the first `system_info()`) can take ~15s on +/// real hardware. It used to run inline before the loop started, so the very +/// first managed-service discovery + snapshot were blocked behind it and an +/// already-running model appeared in the dashboard ~15-20s late (while +/// `rocm services` already listed it). Detection now runs off the critical path, +/// so a `ready` service surfaces on the first discovery tick while GPU detection +/// is still in flight. +/// +/// This asserts on ORDERING rather than wall-clock timing (delivery to an +/// in-process subscriber can be starved while the fake subprocess runs, which +/// would make a pure "arrived within Ns" check flaky): the instance MUST appear +/// in a snapshot whose `gpu_system_info` is still `None`. Post-fix that window +/// exists (many ~100ms ticks before the 4s fake detection lands). Pre-fix it +/// cannot: the loop only starts ticking after detection completes, so the very +/// first snapshot carrying the instance already has `gpu_system_info == Some`. +/// +/// On hosts without `/dev/kfd` the fake is never invoked (detection +/// short-circuits to "no GPU", leaving `gpu_system_info == None` forever), which +/// is exactly the case where this bug cannot occur — the assertion still holds. +#[cfg(unix)] +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn slow_gpu_detection_does_not_delay_service_discovery() { + use std::os::unix::fs::PermissionsExt; + + let dir = tempfile::tempdir().unwrap(); + + // A fake amd-smi that is deliberately slow: whatever the subcommand, sleep + // well past the discovery deadline, then emit trivially-valid JSON so both + // `version` and the `system_info` `--json` calls "succeed". + let fake = dir.path().join("amd-smi-slow"); + std::fs::write(&fake, "#!/bin/sh\nsleep 4\necho '{}'\n").unwrap(); + std::fs::set_permissions(&fake, std::fs::Permissions::from_mode(0o755)).unwrap(); + + // A ready managed vLLM service on disk — the authority `rocm services` reads. + std::fs::write( + dir.path().join("svc.json"), + r#"{"service_id":"vllm-ready-fast","engine":"vllm","model_ref":"m","canonical_model_id":"m", + "host":"127.0.0.1","port":11435,"endpoint_url":"http://127.0.0.1:11435/v1", + "mode":"managed","status":"ready","created_at_unix_ms":1}"#, + ) + .unwrap(); + + // Roomy channel so buffered early snapshots are never dropped (Lagged) even + // if the subscriber is briefly starved while the fake subprocess runs. + let (tx, mut rx) = broadcast::channel::(512); + let services_dir = dir.path().to_path_buf(); + let fake_bin = fake.into_os_string(); + let handle = tokio::spawn(async move { + let opts = runner::RunnerOptions { + services_dir: Some(services_dir), + amd_smi_binary: Some(fake_bin), + disable_vllm_metrics: true, + ..Default::default() + }; + runner::run_loop( + Some(Duration::from_millis(100)), + tx, + Arc::new(Mutex::new(SnapshotRing::new(512))), + Arc::new(Mutex::new(BenchRing::new(4))), + None, + opts, + ) + .await; + }); + + // Walk snapshots in order. The FIRST snapshot that carries the instance + // decides: post-fix it has no GPU info yet (pass); pre-fix it already has + // GPU info because detection ran to completion before the loop started. + // The generous timeout covers detection + any subscriber starvation. + let mut surfaced_before_gpu = None; + let overall = Duration::from_secs(15); + loop { + let ev = match timeout(overall, rx.recv()).await { + Ok(Ok(ev)) => ev, + Ok(Err(broadcast::error::RecvError::Lagged(_))) => continue, + Ok(Err(broadcast::error::RecvError::Closed)) => break, + Err(_) => break, + }; + if let Event::Snapshot(snap) = ev + && snap + .instances + .iter() + .any(|i| i.container_id == "vllm-ready-fast") + { + surfaced_before_gpu = Some(snap.gpu_system_info.is_none()); + break; + } + } + + handle.abort(); + let _ = handle.await; + + match surfaced_before_gpu { + Some(true) => {} // instance surfaced while GPU detection was still in flight + Some(false) => panic!( + "managed service only surfaced after GPU detection completed \ + (the ~15-20s dashboard startup lag regressed)" + ), + None => panic!("ready managed service never surfaced in a snapshot"), + } +} From 422abff3d7df708ab8730ce3b5ec4be1fc5e543b Mon Sep 17 00:00:00 2001 From: Roman Sirokov Date: Thu, 27 Aug 2026 11:24:06 +0300 Subject: [PATCH 3/3] test(dash): exercise off-critical-path detection and launcher front door Address review feedback on the amd-smi-off-critical-path change. - Gate the "amd-smi unavailable" warning behind gpu_init_done so a healthy host never flashes it during the detection window; settle as unavailable (and say so once) if the detection task ever ends without a result. - Add an amd_smi_skip_kfd_preflight test seam so the daemon regression test drives a fake amd-smi through the real detection path instead of short-circuiting on a GPU-less CI host (the /dev/kfd guard stays mandatory in production). The test now genuinely fails if detection moves back onto the critical path, and asserts the surfaced snapshot carries no premature "amd-smi unavailable" warning. - Add direct unit tests for launcher_serving_instances (ready record maps to a live Instance; unbound :0 record is dropped) and a behavioural launcher scenario driving bare rocm through a PTY to prove the front door shows "Serving " rather than "Idle" for a live registry service. Signed-off-by: Roman Sirokov --- apps/rocm/src/dash.rs | 76 +++++++++++++++++++++ crates/rocm-dash-collectors/src/amd_smi.rs | 27 +++++++- crates/rocm-dash-daemon/src/runner.rs | 69 ++++++++++++++----- crates/rocm-dash-daemon/tests/end_to_end.rs | 31 ++++++++- tests/e2e-cucumber/features/dash.feature | 13 ++++ tests/e2e-cucumber/tests/e2e/dash_steps.rs | 44 ++++++++++++ 6 files changed, 238 insertions(+), 22 deletions(-) diff --git a/apps/rocm/src/dash.rs b/apps/rocm/src/dash.rs index cecc81a997..88b63864d1 100644 --- a/apps/rocm/src/dash.rs +++ b/apps/rocm/src/dash.rs @@ -66,6 +66,9 @@ pub fn runner_options( // amd-smi ships inside the managed runtime wheel's bin dir, not on PATH; // resolve the path so the GPU collector can find it. amd_smi_binary: Some(rocm_core::resolve_amd_smi_binary()), + // Production always runs the real `/dev/kfd` pre-flight; only daemon + // integration tests with a fake binary skip it. + amd_smi_skip_kfd_preflight: false, } } @@ -943,4 +946,77 @@ mod tests { first.preferred_engines.first() ); } + + /// The launcher front-door seam that regressed: a `ready` managed vLLM + /// service recorded on disk must map into a live `Instance` (id, model, + /// port, status) so bare `rocm` shows it instead of "Idle". Reads the same + /// registry `rocm services` reads — status-only, no daemon, no network. + #[test] + fn launcher_serving_instances_maps_ready_registry_record() { + let root = std::env::temp_dir().join(format!( + "rocm-cli-launcher-serving-test-{}-{}", + std::process::id(), + rocm_core::unix_time_millis() + )); + let p = AppPaths { + config_dir: root.join("cfg"), + data_dir: root.join("data"), + cache_dir: root.join("cache"), + }; + let services_dir = p.services_dir(); + std::fs::create_dir_all(&services_dir).unwrap(); + std::fs::write( + services_dir.join("svc.json"), + r#"{"service_id":"vllm-launcher","engine":"vllm", + "model_ref":"meta-llama/Llama-3.1-8B","canonical_model_id":"m", + "host":"127.0.0.1","port":11435, + "endpoint_url":"http://127.0.0.1:11435/v1","mode":"managed", + "status":"ready","created_at_unix_ms":1}"#, + ) + .unwrap(); + + let instances = launcher_serving_instances(&p); + + assert_eq!(instances.len(), 1, "the ready managed service must surface"); + let inst = &instances[0]; + assert_eq!(inst.container_id, "vllm-launcher"); + assert_eq!(inst.model_name, "meta-llama/Llama-3.1-8B"); + assert_eq!(inst.port, Some(11435)); + assert_eq!(inst.status, rocm_dash_core::metrics::InstanceStatus::Ready); + + let _ = std::fs::remove_dir_all(&root); + } + + /// A non-scrapeable record (unbound `port:0`) must NOT surface as a live + /// instance on the front door — the registry is the authority and `:0` is + /// never a real serving endpoint. + #[test] + fn launcher_serving_instances_skips_unbound_record() { + let root = std::env::temp_dir().join(format!( + "rocm-cli-launcher-serving-skip-test-{}-{}", + std::process::id(), + rocm_core::unix_time_millis() + )); + let p = AppPaths { + config_dir: root.join("cfg"), + data_dir: root.join("data"), + cache_dir: root.join("cache"), + }; + let services_dir = p.services_dir(); + std::fs::create_dir_all(&services_dir).unwrap(); + std::fs::write( + services_dir.join("svc.json"), + r#"{"service_id":"vllm-unbound","engine":"vllm","model_ref":"m", + "canonical_model_id":"m","host":"127.0.0.1","port":0, + "mode":"managed","status":"ready","created_at_unix_ms":1}"#, + ) + .unwrap(); + + assert!( + launcher_serving_instances(&p).is_empty(), + "an unbound (:0) record must not surface as a live instance" + ); + + let _ = std::fs::remove_dir_all(&root); + } } diff --git a/crates/rocm-dash-collectors/src/amd_smi.rs b/crates/rocm-dash-collectors/src/amd_smi.rs index ce5635e287..f19e20b82d 100644 --- a/crates/rocm-dash-collectors/src/amd_smi.rs +++ b/crates/rocm-dash-collectors/src/amd_smi.rs @@ -43,7 +43,32 @@ impl AmdSmiCollector { /// directory rather than on `PATH`, so callers resolve the path or command /// name (via `rocm_core::resolve_amd_smi_binary`) and pass it here. pub async fn detect_with_binary(binary: impl Into) -> Option { - if !kfd_accessible() { + Self::detect_with_binary_inner(binary, false).await + } + + /// Like [`detect_with_binary`](Self::detect_with_binary) but skips the + /// mandatory `/dev/kfd` pre-flight. + /// + /// **Test-only.** The KFD pre-flight is a safety guard: against a *real* + /// `amd-smi` on a host without a usable `/dev/kfd`, the process can block in + /// uninterruptible kernel sleep (D-state) that no signal can escape. This + /// entry point exists solely so daemon integration tests can point + /// [`detect_with_binary`](Self::detect_with_binary) at a *fake* script (for + /// which the hang cannot happen) and have it actually run on a GPU-less CI + /// host, instead of short-circuiting to `None` and turning the test into a + /// no-op. Never call it against a real binary in production. + #[doc(hidden)] + pub async fn detect_with_binary_skipping_kfd_preflight( + binary: impl Into, + ) -> Option { + Self::detect_with_binary_inner(binary, true).await + } + + async fn detect_with_binary_inner( + binary: impl Into, + skip_kfd_preflight: bool, + ) -> Option { + if !skip_kfd_preflight && !kfd_accessible() { return None; } let me = Self { diff --git a/crates/rocm-dash-daemon/src/runner.rs b/crates/rocm-dash-daemon/src/runner.rs index 946a4db1f1..b4c95979ab 100644 --- a/crates/rocm-dash-daemon/src/runner.rs +++ b/crates/rocm-dash-daemon/src/runner.rs @@ -74,6 +74,14 @@ pub struct RunnerOptions { /// so the caller resolves it (via `rocm_core::resolve_amd_smi_binary`) and /// passes it here. `None` falls back to looking up `amd-smi` on `PATH`. pub amd_smi_binary: Option, + /// **Test-only.** Skip the mandatory `/dev/kfd` pre-flight in amd-smi + /// detection so a *fake* `amd_smi_binary` is actually invoked on a GPU-less + /// CI host instead of short-circuiting to "no GPU". Never set in + /// production: the KFD guard prevents a *real* `amd-smi` from hanging in + /// uninterruptible D-state. Only the daemon integration test that points + /// `amd_smi_binary` at a deliberately-slow fake script flips this, so the + /// off-critical-path detection behaviour is genuinely exercised. + pub amd_smi_skip_kfd_preflight: bool, } impl Default for RunnerOptions { @@ -93,6 +101,7 @@ impl Default for RunnerOptions { persist_dir: None, services_dir: None, amd_smi_binary: None, + amd_smi_skip_kfd_preflight: false, } } } @@ -210,10 +219,14 @@ pub async fn run_loop( // the loop starts ticking immediately (surfacing serving instances within // one discovery tick) and GPU metrics fill in the moment detection lands. let amd_smi_binary = opts.amd_smi_binary.clone(); + let amd_smi_skip_kfd_preflight = opts.amd_smi_skip_kfd_preflight; let (gpu_init_tx, mut gpu_init_rx) = tokio::sync::oneshot::channel::<(Option, Option)>(); tokio::spawn(async move { let gpu = match amd_smi_binary { + Some(binary) if amd_smi_skip_kfd_preflight => { + AmdSmiCollector::detect_with_binary_skipping_kfd_preflight(binary).await + } Some(binary) => AmdSmiCollector::detect_with_binary(binary).await, None => AmdSmiCollector::detect().await, }; @@ -247,25 +260,39 @@ pub async fn run_loop( // without ever blocking the loop while it is still in flight. Until then // `gpu` stays `None` (GPU panels read "unavailable") but discovery and // snapshots run every tick, so serving instances appear promptly. - if !gpu_init_done && let Ok((detected, info)) = gpu_init_rx.try_recv() { - gpu_init_done = true; - if detected.is_some() { - if let Some(i) = &info { - info!( - gpus = i.physical_gpu_count, - model = %i.gpu_model, - rocm = i.rocm_version.as_deref().unwrap_or("?"), - "amd-smi detected" - ); + if !gpu_init_done { + match gpu_init_rx.try_recv() { + Ok((detected, info)) => { + gpu_init_done = true; + if let Some(i) = &info { + info!( + gpus = i.physical_gpu_count, + model = %i.gpu_model, + rocm = i.rocm_version.as_deref().unwrap_or("?"), + "amd-smi detected" + ); + } else { + warn!( + "amd-smi not available (no /dev/kfd or `amd-smi version` failed); GPU disabled" + ); + } + gpu = detected; + gpu_system_info = info; + last_sysinfo_refresh = tick_count; } - } else { - warn!( - "amd-smi not available (no /dev/kfd or `amd-smi version` failed); GPU disabled" - ); + // The detection task dropped its sender without ever sending. + // It has no fallible await today, so this is latent rather than + // a live path — but if it ever regressed, polling `Empty` + // forever would hang `gpu_init_done` at false and suppress the + // honest "unavailable" warning below. Settle as unavailable and + // say so once, so the badge resolves instead of silently never + // clearing. + Err(tokio::sync::oneshot::error::TryRecvError::Closed) => { + gpu_init_done = true; + warn!("amd-smi detection task ended without a result; GPU disabled"); + } + Err(tokio::sync::oneshot::error::TryRecvError::Empty) => {} } - gpu = detected; - gpu_system_info = info; - last_sysinfo_refresh = tick_count; } let mut warnings = Vec::new(); @@ -277,9 +304,15 @@ pub async fn run_loop( Vec::new() } } - } else { + } else if gpu_init_done { warnings.push("amd-smi unavailable (no /dev/kfd or binary missing)".into()); Vec::new() + } else { + // Detection is still in flight (spawned off the critical path), so + // `gpu == None` is transient here. Stay silent instead of flashing a + // false "unavailable" alarm on healthy hardware for the ~15s window + // until the first detection result lands. + Vec::new() }; if gpu.is_some() diff --git a/crates/rocm-dash-daemon/tests/end_to_end.rs b/crates/rocm-dash-daemon/tests/end_to_end.rs index 835f1f9d28..49194f6536 100644 --- a/crates/rocm-dash-daemon/tests/end_to_end.rs +++ b/crates/rocm-dash-daemon/tests/end_to_end.rs @@ -338,9 +338,13 @@ async fn scrape_warning_persists_between_scrape_ticks() { /// cannot: the loop only starts ticking after detection completes, so the very /// first snapshot carrying the instance already has `gpu_system_info == Some`. /// -/// On hosts without `/dev/kfd` the fake is never invoked (detection -/// short-circuits to "no GPU", leaving `gpu_system_info == None` forever), which -/// is exactly the case where this bug cannot occur — the assertion still holds. +/// The fake `amd-smi` is forced through the real detection path via +/// `amd_smi_skip_kfd_preflight` (see `RunnerOptions`): otherwise the `/dev/kfd` +/// pre-flight short-circuits to "no GPU" on GPU-less CI, the fake never runs, +/// and the assertion passes vacuously even with the fix reverted. The surfaced +/// snapshot must also carry NO "amd-smi unavailable" warning — detection is +/// still in flight there, so flashing that alarm would be a false alarm the +/// header ⚠ badge should never show. #[cfg(unix)] #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn slow_gpu_detection_does_not_delay_service_discovery() { @@ -373,6 +377,10 @@ async fn slow_gpu_detection_does_not_delay_service_discovery() { let opts = runner::RunnerOptions { services_dir: Some(services_dir), amd_smi_binary: Some(fake_bin), + // Drive the fake through the real detection path on GPU-less CI: + // without this the `/dev/kfd` pre-flight would short-circuit before + // the fake runs, making the ordering assertion vacuous. + amd_smi_skip_kfd_preflight: true, disable_vllm_metrics: true, ..Default::default() }; @@ -392,6 +400,7 @@ async fn slow_gpu_detection_does_not_delay_service_discovery() { // GPU info because detection ran to completion before the loop started. // The generous timeout covers detection + any subscriber starvation. let mut surfaced_before_gpu = None; + let mut surfaced_without_unavailable_warning = None; let overall = Duration::from_secs(15); loop { let ev = match timeout(overall, rx.recv()).await { @@ -407,6 +416,12 @@ async fn slow_gpu_detection_does_not_delay_service_discovery() { .any(|i| i.container_id == "vllm-ready-fast") { surfaced_before_gpu = Some(snap.gpu_system_info.is_none()); + surfaced_without_unavailable_warning = Some( + !snap + .warnings + .iter() + .any(|w| w.contains("amd-smi unavailable")), + ); break; } } @@ -422,4 +437,14 @@ async fn slow_gpu_detection_does_not_delay_service_discovery() { ), None => panic!("ready managed service never surfaced in a snapshot"), } + + // Detection is still in flight in that first snapshot, so the honest state + // is "GPU pending", never "amd-smi unavailable". Flashing the unavailable + // warning here would be a false alarm on healthy hardware. + assert_eq!( + surfaced_without_unavailable_warning, + Some(true), + "managed service surfaced with a premature `amd-smi unavailable` warning \ + while detection was still in flight" + ); } diff --git a/tests/e2e-cucumber/features/dash.feature b/tests/e2e-cucumber/features/dash.feature index f1fa35e4f1..3fbbfa1505 100644 --- a/tests/e2e-cucumber/features/dash.feature +++ b/tests/e2e-cucumber/features/dash.feature @@ -110,3 +110,16 @@ Feature: Interactive dashboard Then generation throughput is no longer displayed When the user quits the dashboard Then the dashboard exits successfully + + @id:launcher-shows-live-serving-instance @requires-os:linux + Scenario: 10 - The launcher front door shows a live serving model rather than idle + # EAI-8190 regression: bare `rocm` opens the launcher front door, which + # reads the managed-service registry (`launcher_serving_instances`) the same + # way `rocm services` does. A model already serving must surface as + # "Serving ", not the "Idle — nothing serving" state the front door + # showed before the fix, which drove this whole PR. + Given a running managed model is available locally + When the user opens the launcher + Then the launcher shows the model serving + When the user quits the launcher + Then the launcher exits successfully diff --git a/tests/e2e-cucumber/tests/e2e/dash_steps.rs b/tests/e2e-cucumber/tests/e2e/dash_steps.rs index 911fcf7435..49518724af 100644 --- a/tests/e2e-cucumber/tests/e2e/dash_steps.rs +++ b/tests/e2e-cucumber/tests/e2e/dash_steps.rs @@ -108,6 +108,15 @@ async fn open_dashboard(world: &mut E2eWorld) { world.tui = Some(tui); } +#[when("the user opens the launcher")] +async fn open_launcher(world: &mut E2eWorld) { + // Bare `rocm` (no subcommand) opens the launcher front door under an + // interactive terminal — the PTY slave satisfies `interactive_terminal()`. + let tui = TuiSession::spawn(world, &[]) + .unwrap_or_else(|e| panic!("failed to open the launcher: {e}")); + world.tui = Some(tui); +} + #[when("the user opens the ROCm view")] async fn open_rocm_view(world: &mut E2eWorld) { // Dashboard tabs are currently ordered Home, ROCm, Serving, Observe; these @@ -226,6 +235,11 @@ async fn quit_interactive_chat(world: &mut E2eWorld) { quit_tui(world, "interactive chat").await; } +#[when("the user quits the launcher")] +async fn quit_launcher(world: &mut E2eWorld) { + quit_tui(world, "the launcher").await; +} + // ── Then ─────────────────────────────────────────────────────────── #[then("the dashboard home view is displayed")] @@ -408,6 +422,36 @@ async fn dashboard_exited(world: &mut E2eWorld) { assert_tui_opened(world); } +#[then("the launcher shows the model serving")] +async fn launcher_shows_serving(world: &mut E2eWorld) { + let model = world + .model_name + .as_deref() + .expect("no model name set") + .to_string(); + let tui = session(world); + // The front door's status strip renders "Serving " for a live + // registry instance; wait on the model name to synchronise with the first + // paint before inspecting the whole screen. + tui.wait_for_screen(&model, default_timeout()) + .await + .unwrap_or_else(|e| panic!("the launcher never showed the serving model: {e}")); + let screen = tui.screen_text(); + assert!( + screen.contains("Serving"), + "launcher did not show the model as serving:\n{screen}" + ); + assert!( + !screen.contains("Idle — nothing serving"), + "launcher still reported idle despite a live registry instance:\n{screen}" + ); +} + +#[then("the launcher exits successfully")] +async fn launcher_exited(world: &mut E2eWorld) { + assert_tui_opened(world); +} + #[then("interactive chat exits successfully")] async fn interactive_chat_exited(world: &mut E2eWorld) { assert_tui_opened(world);