diff --git a/apps/rocm/src/dash.rs b/apps/rocm/src/dash.rs index 0d2dd8e347..71d2668ebc 100644 --- a/apps/rocm/src/dash.rs +++ b/apps/rocm/src/dash.rs @@ -69,9 +69,20 @@ pub fn runner_options( // Production always runs the real `/dev/kfd` pre-flight; only daemon // integration tests with a fake binary skip it. amd_smi_skip_kfd_preflight: false, + test_clock_offset_path: dash_test_clock_offset_path(), } } +#[cfg(feature = "e2e-test-hooks")] +fn dash_test_clock_offset_path() -> Option { + std::env::var_os("ROCM_CLI_DASH_TEST_CLOCK_OFFSET_PATH").map(Into::into) +} + +#[cfg(not(feature = "e2e-test-hooks"))] +const fn dash_test_clock_offset_path() -> Option { + None +} + /// API key precedence — sourced from the environment ONLY (never TOML/CLI/source/ /// logs); see the chat invariant. /// diff --git a/crates/rocm-dash-daemon/src/runner.rs b/crates/rocm-dash-daemon/src/runner.rs index b4c95979ab..369f9fc8c7 100644 --- a/crates/rocm-dash-daemon/src/runner.rs +++ b/crates/rocm-dash-daemon/src/runner.rs @@ -82,6 +82,11 @@ pub struct RunnerOptions { /// `amd_smi_binary` at a deliberately-slow fake script flips this, so the /// off-critical-path detection behaviour is genuinely exercised. pub amd_smi_skip_kfd_preflight: bool, + /// **Test-only.** When set, cycle timestamps advance from a fixed epoch by + /// the logical tick count plus the signed seconds stored in this file. + /// Production callers leave this unset; E2E scenarios use it to cross + /// observation-validity boundaries without racing wall-clock scheduling. + pub test_clock_offset_path: Option, } impl Default for RunnerOptions { @@ -102,6 +107,7 @@ impl Default for RunnerOptions { services_dir: None, amd_smi_binary: None, amd_smi_skip_kfd_preflight: false, + test_clock_offset_path: None, } } } @@ -122,6 +128,27 @@ const fn vllm_metrics_enabled(opts: &RunnerOptions) -> bool { !opts.disable_vllm_metrics } +fn cycle_timestamp( + epoch: DateTime, + tick: Duration, + tick_count: u64, + offset_path: Option<&std::path::Path>, +) -> DateTime { + let Some(path) = offset_path else { + return Utc::now(); + }; + let tick_millis = tick.as_millis().saturating_mul(u128::from(tick_count)); + let logical_millis = i64::try_from(tick_millis).unwrap_or(i64::MAX); + let offset_secs = std::fs::read_to_string(path) + .ok() + .and_then(|value| value.trim().parse::().ok()) + .unwrap_or_default(); + epoch + .checked_add_signed(chrono::TimeDelta::milliseconds(logical_millis)) + .and_then(|at| at.checked_add_signed(chrono::TimeDelta::seconds(offset_secs))) + .unwrap_or(DateTime::::MAX_UTC) +} + /// Loop forever: tick host + gpu metrics + bench rows, apply through reducer, broadcast. /// /// `tick_override` lets tests run faster than `opts.gpu_tick`; production passes @@ -138,6 +165,7 @@ pub async fn run_loop( let mut host = HostCollector::new(); let tick = tick_override.unwrap_or(opts.gpu_tick); let mut ticker = interval(tick); + let test_clock_epoch = Utc::now(); ticker.set_missed_tick_behavior(MissedTickBehavior::Skip); // Compute multipliers vs the gpu tick. @@ -254,7 +282,12 @@ pub async fn run_loop( // Single wall-clock anchor for this loop iteration. All counter/direct // observations and the assembled Snapshot timestamp share this instant so // every instance refreshed in this cycle serialises Fresh deterministically. - let cycle_at = Utc::now(); + let cycle_at = cycle_timestamp( + test_clock_epoch, + tick, + tick_count, + opts.test_clock_offset_path.as_deref(), + ); // Adopt the background amd-smi detection result the moment it lands, // without ever blocking the loop while it is still in flight. Until then @@ -992,6 +1025,25 @@ mod tests { use super::*; use chrono::TimeZone; + #[test] + fn test_clock_advances_by_ticks_and_external_offset() { + let dir = tempfile::tempdir().unwrap(); + let offset = dir.path().join("offset-secs"); + std::fs::write(&offset, "0").unwrap(); + let epoch = Utc.timestamp_opt(1_700_000_000, 0).unwrap(); + + assert_eq!( + cycle_timestamp(epoch, Duration::from_secs(1), 3, Some(&offset)), + epoch + chrono::TimeDelta::seconds(3) + ); + + std::fs::write(&offset, "7").unwrap(); + assert_eq!( + cycle_timestamp(epoch, Duration::from_secs(1), 4, Some(&offset)), + epoch + chrono::TimeDelta::seconds(11) + ); + } + fn at(secs: i64) -> DateTime { Utc.timestamp_opt(secs, 0).unwrap() } diff --git a/tests/e2e-cucumber/features/dash.feature b/tests/e2e-cucumber/features/dash.feature index 16ad1b2677..fea8b70381 100644 --- a/tests/e2e-cucumber/features/dash.feature +++ b/tests/e2e-cucumber/features/dash.feature @@ -74,12 +74,11 @@ Feature: Interactive dashboard Scenario: dash-08 - Gen throughput stays visible for the validity window after a scrape failure # EAI-7960 principal regression: after establishing a positive gen_tps # baseline through the scripted mock, a single /metrics transport failure - # must NOT immediately clear the displayed "tok/s" value. The contract - # requires the held value to remain visible for the validity window - # clamp(3 x instance_tick, 6 s, 30 s). Current code has no such window - # (runner.rs clears gen_tps on the same tick as the failure), so the - # "generation throughput remains visible" step is the RED assertion. + # must NOT immediately clear the displayed "tok/s" value. The test-only + # logical clock advances with daemon cycles rather than runner wall time, so + # host scheduling cannot consume the validity window before this assertion. Given a managed model exposes scripted serving metrics + And dashboard observation time is deterministic When the user opens the dashboard And the user opens the Observe view Then positive generation throughput is displayed for the managed model @@ -90,17 +89,12 @@ Feature: Interactive dashboard @id:dash-gen-tps-expiry-boundary @requires-os:linux Scenario: dash-09 - Gen throughput expires after the validity window following sustained failure - # EAI-7960 expiry-boundary scenario: two contract boundaries are pinned. - # - # BOUNDARY 1 (held assertion) — immediately after the first failed scrape, - # gen_tps must still be visible (Held). With current code this FAILS (RED) - # because runner.rs clears gen_tps immediately. - # - # BOUNDARY 2 (expired assertion) — after the validity window elapses - # (clamp(3 × instance_tick, 6 s, 30 s) = 6 s for the production 2 s tick), - # gen_tps must be gone from the screen. This step is unreachable today - # because BOUNDARY 1 fails first; it becomes GREEN once the fix is applied. + # EAI-7960 expiry-boundary scenario: immediately after the first failed + # scrape, gen_tps remains visible as Held. Advancing the injected logical + # clock beyond clamp(3 × instance_tick, 6 s, 30 s) then makes the next scrape + # publish an expired value. No wall-clock sleep defines either boundary. Given a managed model exposes scripted serving metrics + And dashboard observation time is deterministic When the user opens the dashboard And the user opens the Observe view Then positive generation throughput is displayed for the managed model diff --git a/tests/e2e-cucumber/tests/e2e/dash_steps.rs b/tests/e2e-cucumber/tests/e2e/dash_steps.rs index d49b9476b1..b6832703f7 100644 --- a/tests/e2e-cucumber/tests/e2e/dash_steps.rs +++ b/tests/e2e-cucumber/tests/e2e/dash_steps.rs @@ -18,6 +18,7 @@ use crate::e2e::tui_driver::{TuiSession, default_timeout}; /// corresponding `Then` step (`managed_chat_request_carried_prompt`) asserts /// the mock actually received — so the two can never silently drift apart. const MANAGED_MODEL_PROMPT: &str = "hello from the terminal"; +const DASH_CLOCK_OFFSET_FILE: &str = "dash-clock-offset-secs"; /// Borrow the scenario's active TUI session, or fail clearly if none was opened. const fn session(world: &mut E2eWorld) -> &mut TuiSession { @@ -628,6 +629,21 @@ async fn managed_model_scripted_metrics(world: &mut E2eWorld) { world.register_mock_service_with(ServiceRecordOptions::default()); } +#[given("dashboard observation time is deterministic")] +async fn dashboard_observation_time_is_deterministic(world: &mut E2eWorld) { + let root = world + .isolated_root + .as_ref() + .expect("scenario has no isolated root") + .path(); + let path = root.join(DASH_CLOCK_OFFSET_FILE); + std::fs::write(&path, "0").expect("failed to initialize dashboard test clock"); + world.command_env.push(( + "ROCM_CLI_DASH_TEST_CLOCK_OFFSET_PATH", + path.into_os_string(), + )); +} + /// The Observe tab's node-throughput hero shows the "tok/s" unit whenever /// `gen_tps` is `Some(_)`. Wait for it to confirm a positive baseline was /// established through at least two successful Growing-mode scrapes. @@ -670,17 +686,10 @@ async fn metrics_endpoint_fails(world: &mut E2eWorld) { // snapshot is painted before the assertion reads the screen. tokio::time::sleep(Duration::from_millis(50)).await; } - -/// EAI-7960 principal regression assertion (must be RED with current code). -/// -/// Contract: the Observe tab must still show "tok/s" immediately after the -/// first failed scrape — the held value must persist for the validity window -/// `clamp(3 × instance_tick, 6 s, 30 s)` before clearing. +/// EAI-7960 principal regression assertion. /// -/// **Current behaviour:** `runner.rs` lines 464-476 clear `gen_tps` on the -/// very tick that the `/metrics` fetch fails — no holding logic exists. The -/// TUI therefore renders "—" the moment the failure propagates, and this -/// assertion **FAILS**, confirming EAI-7960 is reproduced at the PTY seam. +/// The scenario's injected logical clock cannot cross the validity boundary +/// because the host was descheduled; only an explicit scenario advance can. #[then("generation throughput remains visible within the validity window")] async fn gen_tps_held_after_failure(world: &mut E2eWorld) { let screen = session(world).screen_text(); @@ -688,42 +697,41 @@ async fn gen_tps_held_after_failure(world: &mut E2eWorld) { screen.contains("tok/s"), "EAI-7960 REGRESSION: gen throughput (\"tok/s\") was cleared immediately \ after the first failed scrape instead of being held for the validity \ - window (clamp(3 × instance_tick, 6 s, 30 s)).\n\ - Root cause: runner.rs clears gen_tps on the same tick as the failure; \ - no held-value / validity-window logic exists yet.\n\ - This assertion must FAIL (RED) until the fix is applied.\n\n\ + window (clamp(3 × instance_tick, 6 s, 30 s)).\n\n\ Last screen:\n{screen}" ); } // ── EAI-7960: expiry boundary helpers ─────────────────────────────────────── -/// Production validity window: clamp(3 × instance_tick, 6 s, 30 s). -/// -/// With the daemon's 2 s `instance_tick` the lower bound clamp(6 s, 6 s) = 6 s -/// is always reached. An additional buffer of two instance-ticks (4 s) ensures -/// the runner has had enough cycles to propagate the expiry to the TUI. -const VALIDITY_WINDOW: Duration = Duration::from_secs(6); -const VALIDITY_WINDOW_BUFFER: Duration = Duration::from_secs(5); // 2 × instance_tick + render - -/// Sleep for the full observation validity window so the caller can then assert -/// that the held gen_tps has expired. Designed to follow -/// "When the metrics endpoint fails transiently" — at that step's exit at least -/// one 503 has been served, meaning the validity clock has started. +/// Advance the test-only logical clock beyond the six-second validity window, +/// then synchronize on the next failed scrape that publishes the expired value. #[when("the validity window has elapsed")] -async fn validity_window_elapsed(_world: &mut E2eWorld) { - // Sleep the full window + buffer so the daemon has had enough cycles - // after expiry to deliver the snapshot change to the TUI. - tokio::time::sleep(VALIDITY_WINDOW + VALIDITY_WINDOW_BUFFER).await; +async fn validity_window_elapsed(world: &mut E2eWorld) { + let mock = world.mock.as_ref().expect("no mock server running"); + let prior_failures = mock.metrics_failure_count(); + let root = world + .isolated_root + .as_ref() + .expect("scenario has no isolated root") + .path(); + std::fs::write(root.join(DASH_CLOCK_OFFSET_FILE), "7") + .expect("failed to advance dashboard test clock"); + + let budget = default_timeout(); + let deadline = Instant::now() + budget; + while mock.metrics_failure_count() == prior_failures { + assert!( + Instant::now() < deadline, + "no metrics scrape observed after advancing the dashboard clock" + ); + tokio::time::sleep(Duration::from_millis(20)).await; + } + tokio::time::sleep(Duration::from_millis(50)).await; } -/// Assert that gen_tps is no longer rendered on screen (BOUNDARY 2 of the -/// EAI-7960 expiry contract). After the validity window the daemon must clear -/// the held value and the TUI must show "—" in place of the "tok/s" unit. -/// -/// With current code this step is unreachable because BOUNDARY 1 (the "remains -/// visible" assertion) fails first. This step becomes GREEN once the hold/expiry -/// logic is implemented. +/// Assert that gen_tps is no longer rendered after the scenario advances the +/// injected clock beyond the validity boundary and observes the next scrape. #[then("generation throughput is no longer displayed")] async fn gen_tps_no_longer_displayed(world: &mut E2eWorld) { let screen = session(world).screen_text(); diff --git a/tests/e2e-cucumber/tests/e2e/tui_driver.rs b/tests/e2e-cucumber/tests/e2e/tui_driver.rs index b65eb9b9c9..6d644d375f 100644 --- a/tests/e2e-cucumber/tests/e2e/tui_driver.rs +++ b/tests/e2e-cucumber/tests/e2e/tui_driver.rs @@ -181,6 +181,11 @@ impl TuiSession { for (key, value) in world.isolate_env().into_iter().chain(world.pty_env()) { cmd.env(key, value); } + // Behavioural fixtures attached by Given steps apply to PTY commands too, + // just as they do to the piped `run_rocm_with_scenario_env` path. + for (key, value) in &world.command_env { + cmd.env(key, value); + } // Caller-supplied overrides win over the scenario's own isolation // (e.g. a `Given` step's HOME/SHELL for state it planted itself). for (key, value) in extra_env {