Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
11 changes: 11 additions & 0 deletions apps/rocm/src/dash.rs
Original file line number Diff line number Diff line change
Expand Up @@ -69,9 +69,20 @@ pub fn runner_options(
// Production always runs the real `/dev/kfd` pre-flight; only daemon
// integration tests with a fake binary skip it.
amd_smi_skip_kfd_preflight: false,
test_clock_offset_path: dash_test_clock_offset_path(),
}
}

#[cfg(feature = "e2e-test-hooks")]
fn dash_test_clock_offset_path() -> Option<std::path::PathBuf> {
std::env::var_os("ROCM_CLI_DASH_TEST_CLOCK_OFFSET_PATH").map(Into::into)
}

#[cfg(not(feature = "e2e-test-hooks"))]
const fn dash_test_clock_offset_path() -> Option<std::path::PathBuf> {
None
}

/// API key precedence — sourced from the environment ONLY (never TOML/CLI/source/
/// logs); see the chat invariant.
///
Expand Down
54 changes: 53 additions & 1 deletion crates/rocm-dash-daemon/src/runner.rs
Original file line number Diff line number Diff line change
Expand Up @@ -82,6 +82,11 @@ pub struct RunnerOptions {
/// `amd_smi_binary` at a deliberately-slow fake script flips this, so the
/// off-critical-path detection behaviour is genuinely exercised.
pub amd_smi_skip_kfd_preflight: bool,
/// **Test-only.** When set, cycle timestamps advance from a fixed epoch by
/// the logical tick count plus the signed seconds stored in this file.
/// Production callers leave this unset; E2E scenarios use it to cross
/// observation-validity boundaries without racing wall-clock scheduling.
pub test_clock_offset_path: Option<PathBuf>,
}

impl Default for RunnerOptions {
Expand All @@ -102,6 +107,7 @@ impl Default for RunnerOptions {
services_dir: None,
amd_smi_binary: None,
amd_smi_skip_kfd_preflight: false,
test_clock_offset_path: None,
}
}
}
Expand All @@ -122,6 +128,27 @@ const fn vllm_metrics_enabled(opts: &RunnerOptions) -> bool {
!opts.disable_vllm_metrics
}

fn cycle_timestamp(
epoch: DateTime<Utc>,
tick: Duration,
tick_count: u64,
offset_path: Option<&std::path::Path>,
) -> DateTime<Utc> {
let Some(path) = offset_path else {
return Utc::now();
};
let tick_millis = tick.as_millis().saturating_mul(u128::from(tick_count));
let logical_millis = i64::try_from(tick_millis).unwrap_or(i64::MAX);
let offset_secs = std::fs::read_to_string(path)
.ok()
.and_then(|value| value.trim().parse::<i64>().ok())
.unwrap_or_default();
epoch
.checked_add_signed(chrono::TimeDelta::milliseconds(logical_millis))
.and_then(|at| at.checked_add_signed(chrono::TimeDelta::seconds(offset_secs)))
.unwrap_or(DateTime::<Utc>::MAX_UTC)
}

/// Loop forever: tick host + gpu metrics + bench rows, apply through reducer, broadcast.
///
/// `tick_override` lets tests run faster than `opts.gpu_tick`; production passes
Expand All @@ -138,6 +165,7 @@ pub async fn run_loop(
let mut host = HostCollector::new();
let tick = tick_override.unwrap_or(opts.gpu_tick);
let mut ticker = interval(tick);
let test_clock_epoch = Utc::now();
ticker.set_missed_tick_behavior(MissedTickBehavior::Skip);

// Compute multipliers vs the gpu tick.
Expand Down Expand Up @@ -254,7 +282,12 @@ pub async fn run_loop(
// Single wall-clock anchor for this loop iteration. All counter/direct
// observations and the assembled Snapshot timestamp share this instant so
// every instance refreshed in this cycle serialises Fresh deterministically.
let cycle_at = Utc::now();
let cycle_at = cycle_timestamp(
test_clock_epoch,
tick,
tick_count,
opts.test_clock_offset_path.as_deref(),
);

// Adopt the background amd-smi detection result the moment it lands,
// without ever blocking the loop while it is still in flight. Until then
Expand Down Expand Up @@ -992,6 +1025,25 @@ mod tests {
use super::*;
use chrono::TimeZone;

#[test]
fn test_clock_advances_by_ticks_and_external_offset() {
let dir = tempfile::tempdir().unwrap();
let offset = dir.path().join("offset-secs");
std::fs::write(&offset, "0").unwrap();
let epoch = Utc.timestamp_opt(1_700_000_000, 0).unwrap();

assert_eq!(
cycle_timestamp(epoch, Duration::from_secs(1), 3, Some(&offset)),
epoch + chrono::TimeDelta::seconds(3)
);

std::fs::write(&offset, "7").unwrap();
assert_eq!(
cycle_timestamp(epoch, Duration::from_secs(1), 4, Some(&offset)),
epoch + chrono::TimeDelta::seconds(11)
);
}

fn at(secs: i64) -> DateTime<Utc> {
Utc.timestamp_opt(secs, 0).unwrap()
}
Expand Down
24 changes: 9 additions & 15 deletions tests/e2e-cucumber/features/dash.feature
Original file line number Diff line number Diff line change
Expand Up @@ -74,12 +74,11 @@ Feature: Interactive dashboard
Scenario: dash-08 - Gen throughput stays visible for the validity window after a scrape failure
# EAI-7960 principal regression: after establishing a positive gen_tps
# baseline through the scripted mock, a single /metrics transport failure
# must NOT immediately clear the displayed "tok/s" value. The contract
# requires the held value to remain visible for the validity window
# clamp(3 x instance_tick, 6 s, 30 s). Current code has no such window
# (runner.rs clears gen_tps on the same tick as the failure), so the
# "generation throughput remains visible" step is the RED assertion.
# must NOT immediately clear the displayed "tok/s" value. The test-only
# logical clock advances with daemon cycles rather than runner wall time, so
# host scheduling cannot consume the validity window before this assertion.
Given a managed model exposes scripted serving metrics
And dashboard observation time is deterministic
When the user opens the dashboard
And the user opens the Observe view
Then positive generation throughput is displayed for the managed model
Expand All @@ -90,17 +89,12 @@ Feature: Interactive dashboard

@id:dash-gen-tps-expiry-boundary @requires-os:linux
Scenario: dash-09 - Gen throughput expires after the validity window following sustained failure
# EAI-7960 expiry-boundary scenario: two contract boundaries are pinned.
#
# BOUNDARY 1 (held assertion) — immediately after the first failed scrape,
# gen_tps must still be visible (Held). With current code this FAILS (RED)
# because runner.rs clears gen_tps immediately.
#
# BOUNDARY 2 (expired assertion) — after the validity window elapses
# (clamp(3 × instance_tick, 6 s, 30 s) = 6 s for the production 2 s tick),
# gen_tps must be gone from the screen. This step is unreachable today
# because BOUNDARY 1 fails first; it becomes GREEN once the fix is applied.
# EAI-7960 expiry-boundary scenario: immediately after the first failed
# scrape, gen_tps remains visible as Held. Advancing the injected logical
# clock beyond clamp(3 × instance_tick, 6 s, 30 s) then makes the next scrape
# publish an expired value. No wall-clock sleep defines either boundary.
Given a managed model exposes scripted serving metrics
And dashboard observation time is deterministic
When the user opens the dashboard
And the user opens the Observe view
Then positive generation throughput is displayed for the managed model
Expand Down
82 changes: 45 additions & 37 deletions tests/e2e-cucumber/tests/e2e/dash_steps.rs
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@ use crate::e2e::tui_driver::{TuiSession, default_timeout};
/// corresponding `Then` step (`managed_chat_request_carried_prompt`) asserts
/// the mock actually received — so the two can never silently drift apart.
const MANAGED_MODEL_PROMPT: &str = "hello from the terminal";
const DASH_CLOCK_OFFSET_FILE: &str = "dash-clock-offset-secs";

/// Borrow the scenario's active TUI session, or fail clearly if none was opened.
const fn session(world: &mut E2eWorld) -> &mut TuiSession {
Expand Down Expand Up @@ -628,6 +629,21 @@ async fn managed_model_scripted_metrics(world: &mut E2eWorld) {
world.register_mock_service_with(ServiceRecordOptions::default());
}

#[given("dashboard observation time is deterministic")]
async fn dashboard_observation_time_is_deterministic(world: &mut E2eWorld) {
let root = world
.isolated_root
.as_ref()
.expect("scenario has no isolated root")
.path();
let path = root.join(DASH_CLOCK_OFFSET_FILE);
std::fs::write(&path, "0").expect("failed to initialize dashboard test clock");
world.command_env.push((
"ROCM_CLI_DASH_TEST_CLOCK_OFFSET_PATH",
path.into_os_string(),
));
}

/// The Observe tab's node-throughput hero shows the "tok/s" unit whenever
/// `gen_tps` is `Some(_)`. Wait for it to confirm a positive baseline was
/// established through at least two successful Growing-mode scrapes.
Expand Down Expand Up @@ -670,60 +686,52 @@ async fn metrics_endpoint_fails(world: &mut E2eWorld) {
// snapshot is painted before the assertion reads the screen.
tokio::time::sleep(Duration::from_millis(50)).await;
}

/// EAI-7960 principal regression assertion (must be RED with current code).
///
/// Contract: the Observe tab must still show "tok/s" immediately after the
/// first failed scrape — the held value must persist for the validity window
/// `clamp(3 × instance_tick, 6 s, 30 s)` before clearing.
/// EAI-7960 principal regression assertion.
///
/// **Current behaviour:** `runner.rs` lines 464-476 clear `gen_tps` on the
/// very tick that the `/metrics` fetch fails — no holding logic exists. The
/// TUI therefore renders "—" the moment the failure propagates, and this
/// assertion **FAILS**, confirming EAI-7960 is reproduced at the PTY seam.
/// The scenario's injected logical clock cannot cross the validity boundary
/// because the host was descheduled; only an explicit scenario advance can.
#[then("generation throughput remains visible within the validity window")]
async fn gen_tps_held_after_failure(world: &mut E2eWorld) {
let screen = session(world).screen_text();
assert!(
screen.contains("tok/s"),
"EAI-7960 REGRESSION: gen throughput (\"tok/s\") was cleared immediately \
after the first failed scrape instead of being held for the validity \
window (clamp(3 × instance_tick, 6 s, 30 s)).\n\
Root cause: runner.rs clears gen_tps on the same tick as the failure; \
no held-value / validity-window logic exists yet.\n\
This assertion must FAIL (RED) until the fix is applied.\n\n\
window (clamp(3 × instance_tick, 6 s, 30 s)).\n\n\
Last screen:\n{screen}"
);
}

// ── EAI-7960: expiry boundary helpers ───────────────────────────────────────

/// Production validity window: clamp(3 × instance_tick, 6 s, 30 s).
///
/// With the daemon's 2 s `instance_tick` the lower bound clamp(6 s, 6 s) = 6 s
/// is always reached. An additional buffer of two instance-ticks (4 s) ensures
/// the runner has had enough cycles to propagate the expiry to the TUI.
const VALIDITY_WINDOW: Duration = Duration::from_secs(6);
const VALIDITY_WINDOW_BUFFER: Duration = Duration::from_secs(5); // 2 × instance_tick + render

/// Sleep for the full observation validity window so the caller can then assert
/// that the held gen_tps has expired. Designed to follow
/// "When the metrics endpoint fails transiently" — at that step's exit at least
/// one 503 has been served, meaning the validity clock has started.
/// Advance the test-only logical clock beyond the six-second validity window,
/// then synchronize on the next failed scrape that publishes the expired value.
#[when("the validity window has elapsed")]
async fn validity_window_elapsed(_world: &mut E2eWorld) {
// Sleep the full window + buffer so the daemon has had enough cycles
// after expiry to deliver the snapshot change to the TUI.
tokio::time::sleep(VALIDITY_WINDOW + VALIDITY_WINDOW_BUFFER).await;
async fn validity_window_elapsed(world: &mut E2eWorld) {
let mock = world.mock.as_ref().expect("no mock server running");
let prior_failures = mock.metrics_failure_count();
let root = world
.isolated_root
.as_ref()
.expect("scenario has no isolated root")
.path();
std::fs::write(root.join(DASH_CLOCK_OFFSET_FILE), "7")
.expect("failed to advance dashboard test clock");

let budget = default_timeout();
let deadline = Instant::now() + budget;
while mock.metrics_failure_count() == prior_failures {
assert!(
Instant::now() < deadline,
"no metrics scrape observed after advancing the dashboard clock"
);
tokio::time::sleep(Duration::from_millis(20)).await;
}
tokio::time::sleep(Duration::from_millis(50)).await;
}

/// Assert that gen_tps is no longer rendered on screen (BOUNDARY 2 of the
/// EAI-7960 expiry contract). After the validity window the daemon must clear
/// the held value and the TUI must show "—" in place of the "tok/s" unit.
///
/// With current code this step is unreachable because BOUNDARY 1 (the "remains
/// visible" assertion) fails first. This step becomes GREEN once the hold/expiry
/// logic is implemented.
/// Assert that gen_tps is no longer rendered after the scenario advances the
/// injected clock beyond the validity boundary and observes the next scrape.
#[then("generation throughput is no longer displayed")]
async fn gen_tps_no_longer_displayed(world: &mut E2eWorld) {
let screen = session(world).screen_text();
Expand Down
5 changes: 5 additions & 0 deletions tests/e2e-cucumber/tests/e2e/tui_driver.rs
Original file line number Diff line number Diff line change
Expand Up @@ -181,6 +181,11 @@ impl TuiSession {
for (key, value) in world.isolate_env().into_iter().chain(world.pty_env()) {
cmd.env(key, value);
}
// Behavioural fixtures attached by Given steps apply to PTY commands too,
// just as they do to the piped `run_rocm_with_scenario_env` path.
for (key, value) in &world.command_env {
cmd.env(key, value);
}
// Caller-supplied overrides win over the scenario's own isolation
// (e.g. a `Given` step's HOME/SHELL for state it planted itself).
for (key, value) in extra_env {
Expand Down
Loading