From a6559658e9b60db570bbc5de6c88f6fee6d8428b Mon Sep 17 00:00:00 2001 From: fredespi Date: Tue, 25 Aug 2026 18:45:42 +0200 Subject: [PATCH 1/8] test(e2e): pin public-bind endpoint authentication (EAI-7409) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Completes the public-bind contract begun in #293. That change pinned the pre-flight refusal (a non-loopback bind without --allow-public-bind); this pins the other half, and the security property the coverage audit ranked first: after opting in, the endpoint must not be open. `rocm serve --host 0.0.0.0 --allow-public-bind` generates an API key, prints it exactly once with the `Authorization: Bearer` instructions, and hands the serving engine a key file to enforce. The scenario drives a real managed serve and then talks to the endpoint directly: a request with no key and a request with the wrong key must both be refused as unauthorized, and the key that was just issued must be accepted. GPU lane, because the ENGINE enforces the key (the CLI only issues and hands it over) and `serve` has no plan-only mode, so enforcement cannot be observed on the mock lane. The pure key-resolution rules — notably that a loopback bind stays credential-free even when --api-key is passed — are already unit-tested in the CLI, so this covers only what a unit test cannot: that a live public endpoint actually authenticates. Adds an endpoint_api_key field to the test World and exposes three existing serve helpers to the new step module. Signed-off-by: fredespi --- .../e2e-cucumber/features/networking.feature | 20 +++ tests/e2e-cucumber/tests/e2e.rs | 7 + .../tests/e2e/endpoint_auth_steps.rs | 166 ++++++++++++++++++ tests/e2e-cucumber/tests/e2e/serving_steps.rs | 6 +- 4 files changed, 196 insertions(+), 3 deletions(-) create mode 100644 tests/e2e-cucumber/tests/e2e/endpoint_auth_steps.rs diff --git a/tests/e2e-cucumber/features/networking.feature b/tests/e2e-cucumber/features/networking.feature index b0de704911..332564cc70 100644 --- a/tests/e2e-cucumber/features/networking.feature +++ b/tests/e2e-cucumber/features/networking.feature @@ -39,3 +39,23 @@ Feature: Native HTTP networking When the user serves a model bound to a public interface without allowing public binding Then serving is refused before any engine starts And the user is told to allow public binding first + + # The other half of the EAI-7409 contract, and the security property the ticket + # calls out first: opting in to a public bind must not produce an open endpoint. + # The CLI issues an API key, shows it exactly once, and hands the serving engine + # a key file to enforce — so the endpoint must reject callers without the key and + # with the wrong key, while accepting the key it just issued. + # + # GPU lane: the ENGINE enforces the key (the CLI only issues and hands it over), + # so proving enforcement needs a real engine serving a real model. `serve` has no + # plan-only mode that would let the mock lane observe it. The pure key-resolution + # rules (loopback stays credential-free even when a key is passed) are already + # unit-tested in the CLI, so this pins only what a unit test cannot: that a live + # public endpoint actually authenticates. + @id:networking-public-bind-endpoint-enforces-key @requires-gpu + Scenario: 4 - A public endpoint issues a key once and then enforces it + When the user serves a model on a public interface with public binding allowed + Then the CLI shows the endpoint key once and how to send it + And a request without the key is refused as unauthorized + And a request with the wrong key is refused as unauthorized + And a request carrying the issued key is accepted diff --git a/tests/e2e-cucumber/tests/e2e.rs b/tests/e2e-cucumber/tests/e2e.rs index ec3be17f12..584fe3a0c4 100644 --- a/tests/e2e-cucumber/tests/e2e.rs +++ b/tests/e2e-cucumber/tests/e2e.rs @@ -25,6 +25,7 @@ mod e2e { pub mod dash_steps; pub mod dependency_guard_steps; pub mod diagnose_steps; + pub mod endpoint_auth_steps; pub mod engines_steps; pub mod examine_steps; pub mod lifecycle_steps; @@ -47,6 +48,11 @@ pub struct E2eWorld { /// Cache-marker destination discovered from `rocmd`'s own JSON report. pub artifact_marker_path: Option, pub endpoint: Option, + /// The endpoint API key `rocm serve` issued for a public bind, captured from + /// the launch output. `None` for a loopback serve, which is credential-free. + /// Used by the public-bind auth scenario to prove the endpoint really enforces + /// the key it printed (see `e2e::endpoint_auth_steps`). + pub endpoint_api_key: Option, pub model_name: Option, pub chat_response: Option, pub cli_output: Option, @@ -182,6 +188,7 @@ impl Default for E2eWorld { artifact_server: None, artifact_marker_path: None, endpoint: None, + endpoint_api_key: None, model_name: None, chat_response: None, cli_output: None, diff --git a/tests/e2e-cucumber/tests/e2e/endpoint_auth_steps.rs b/tests/e2e-cucumber/tests/e2e/endpoint_auth_steps.rs new file mode 100644 index 0000000000..60ed4be0da --- /dev/null +++ b/tests/e2e-cucumber/tests/e2e/endpoint_auth_steps.rs @@ -0,0 +1,166 @@ +// Copyright © Advanced Micro Devices, Inc., or its affiliates. +// +// SPDX-License-Identifier: MIT + +//! Steps for the public-bind endpoint authentication contract (EAI-7409). +//! +//! `rocm serve --host 0.0.0.0 --allow-public-bind` issues an endpoint API key, +//! prints it exactly once, and hands the serving engine a key file to enforce. +//! These steps drive a real managed serve and then talk to the endpoint directly +//! over HTTP with and without credentials, because the ENGINE is what enforces the +//! key — the CLI only issues it. That is also why these steps are GPU-lane only: +//! there is no plan-only serve mode that would surface enforcement without a real +//! engine and model. Contracts read from the CLI's own output (EAI-8072). + +use std::time::{Duration, Instant}; + +use cucumber::{then, when}; + +use crate::E2eWorld; +use crate::e2e::serving_steps::{ensure_serve_port_free, host_serve_target, serve_timeout_for}; + +/// The port every serve scenario shares (see `serving_steps::SERVE_PORT`). A public +/// bind listens on all interfaces, so the test still reaches it over loopback. +const ENDPOINT_URL: &str = "http://127.0.0.1:11435/v1"; + +/// Pull the issued endpoint key out of the launch output. `serve` prints it as +/// ` api key: ` exactly once, alongside the note and curl example. +fn issued_api_key(output: &str) -> Option { + output + .lines() + .find_map(|line| line.trim().strip_prefix("api key:")) + .map(|key| key.trim().to_owned()) + .filter(|key| !key.is_empty()) +} + +/// GET `/models`, optionally presenting a bearer token. Returns the HTTP +/// status so a step can assert acceptance vs. refusal. +async fn get_models_status(bearer: Option<&str>) -> reqwest::StatusCode { + let client = reqwest::Client::builder() + .timeout(Duration::from_secs(30)) + .build() + .expect("failed to build HTTP client"); + let url = format!("{ENDPOINT_URL}/models"); + let mut request = client.get(&url); + if let Some(token) = bearer { + request = request.bearer_auth(token); + } + request + .send() + .await + .unwrap_or_else(|error| panic!("GET {url} failed: {error}")) + .status() +} + +/// Wait until the authenticated endpoint answers, so the assertions run against a +/// loaded model rather than a still-starting one. Polls WITH the key, because an +/// unauthenticated probe is exactly what the endpoint is meant to reject. +async fn wait_for_authenticated_endpoint(key: &str, timeout_secs: u64) { + let deadline = Instant::now() + Duration::from_secs(timeout_secs); + loop { + if get_models_status(Some(key)).await.is_success() { + return; + } + assert!( + Instant::now() < deadline, + "public endpoint {ENDPOINT_URL} did not answer an authenticated request within {timeout_secs}s" + ); + tokio::time::sleep(Duration::from_secs(5)).await; + } +} + +// ── When ─────────────────────────────────────────────────────────── + +#[when("the user serves a model on a public interface with public binding allowed")] +async fn serve_public_bind(world: &mut E2eWorld) { + let (model, engine, _) = host_serve_target(); + ensure_serve_port_free().await; + // No `--api-key`: the CLI must GENERATE one, which is the behaviour under test. + let stdout = crate::run_rocm_ok( + world, + &[ + "serve", + model, + "--engine", + engine, + "--managed", + "--host", + "0.0.0.0", + "--allow-public-bind", + ], + ); + let key = issued_api_key(&stdout) + .unwrap_or_else(|| panic!("serve did not print an endpoint api key:\n{stdout}")); + wait_for_authenticated_endpoint(&key, serve_timeout_for(world)).await; + world.endpoint = Some(ENDPOINT_URL.to_owned()); + world.endpoint_api_key = Some(key); + world.model_name = Some(model.to_owned()); + world.cli_output = Some(stdout); +} + +// ── Then ─────────────────────────────────────────────────────────── + +#[then("the CLI shows the endpoint key once and how to send it")] +async fn shows_key_once(world: &mut E2eWorld) { + let output = world.cli_output.as_deref().expect("no serve output"); + let key = world + .endpoint_api_key + .as_deref() + .expect("no endpoint api key captured"); + // Shown exactly once: a key echoed on several lines would widen the window in + // which it can be captured from a log. + let shown = output + .lines() + .filter(|line| line.trim().starts_with("api key:")) + .count(); + assert_eq!( + shown, 1, + "expected the key on exactly one line, got {shown}:\n{output}" + ); + assert!( + output.contains("shown only now"), + "expected the show-once note, got:\n{output}" + ); + assert!( + output.contains("Authorization: Bearer"), + "expected the CLI to say how to send the key, got:\n{output}" + ); + assert!( + key.len() >= 16, + "expected a strong generated key, got one of {} chars", + key.len() + ); +} + +#[then("a request without the key is refused as unauthorized")] +async fn unauthenticated_refused(_world: &mut E2eWorld) { + let status = get_models_status(None).await; + assert_eq!( + status, + reqwest::StatusCode::UNAUTHORIZED, + "an unauthenticated request to a public endpoint must be refused, got {status}" + ); +} + +#[then("a request with the wrong key is refused as unauthorized")] +async fn wrong_key_refused(_world: &mut E2eWorld) { + let status = get_models_status(Some("not-the-issued-key")).await; + assert_eq!( + status, + reqwest::StatusCode::UNAUTHORIZED, + "a request with the wrong key must be refused, got {status}" + ); +} + +#[then("a request carrying the issued key is accepted")] +async fn issued_key_accepted(world: &mut E2eWorld) { + let key = world + .endpoint_api_key + .clone() + .expect("no endpoint api key captured"); + let status = get_models_status(Some(&key)).await; + assert!( + status.is_success(), + "the issued key must be accepted, got {status}" + ); +} diff --git a/tests/e2e-cucumber/tests/e2e/serving_steps.rs b/tests/e2e-cucumber/tests/e2e/serving_steps.rs index 46a50f313e..63cdf7797c 100644 --- a/tests/e2e-cucumber/tests/e2e/serving_steps.rs +++ b/tests/e2e-cucumber/tests/e2e/serving_steps.rs @@ -32,7 +32,7 @@ fn serve_timeout_secs() -> u64 { /// scenario (lengthen a genuinely slow serve, e.g. a large model), else an /// `expectations.toml` xfail `serve_timeout_secs` (shorten a known-bug serve so /// it fails fast). -fn serve_timeout_for(world: &E2eWorld) -> u64 { +pub(crate) fn serve_timeout_for(world: &E2eWorld) -> u64 { world .serve_timeout_override .unwrap_or_else(serve_timeout_secs) @@ -101,7 +101,7 @@ const ASSISTANT_PORT: u16 = 8001; /// (see [`wait_for_free_vram`]). Callers that only need the reset can ignore it; /// a serve that then fails to become ready reports it, because "the previous /// engine had not released the GPU yet" is otherwise invisible in the log. -async fn ensure_serve_port_free() -> String { +pub(crate) async fn ensure_serve_port_free() -> String { // Always kill any listener on the shared port — NOT just one that already // answers /v1/models. A prior scenario's vLLM that is still LOADING holds the // port and GPU memory without yet serving /v1/models; if we only checked HTTP @@ -326,7 +326,7 @@ async fn setup_mock_custom_port(world: &mut E2eWorld) { /// exercised exactly once, in the `@nightly` `serve-large-model-inference` /// scenario (Qwen3.6-27B) — never on the per-PR path. Do not raise a serve /// scenario's model unless a smaller one genuinely cannot prove the assertion. -fn host_serve_target() -> (&'static str, &'static str, &'static str) { +pub(crate) fn host_serve_target() -> (&'static str, &'static str, &'static str) { if e2e_cucumber::capability::host_capability().effective_serve_engine == "lemonade" { // GGUF via lemonade's llama.cpp backend; endpoint reports e.g. // Qwen3-0.6B-Q4_0.gguf, so "Qwen3-0.6B" is the distinctive substring. From c736d8e8ea643e9f2aad7b69b1112aeda9e9b79a Mon Sep 17 00:00:00 2001 From: fredespi Date: Tue, 25 Aug 2026 19:53:54 +0200 Subject: [PATCH 2/8] test(e2e): give the public-bind auth scenario its runtime precondition The scenario served straight from a fresh isolated data directory, which has no ROCm runtime configured, so `rocm serve` refused before it could issue a key: "device_policy: gpu_required; no active ROCm runtime is configured". Every other GPU serve scenario opens with `Given a managed runtime is active`, which points the scenario at the runner's shared managed runtime; this one was missing it. Found by a scoped GPU dispatch of the scenario, which is also why the run needed reading rather than trusting: the self-hosted E2E lane is non-blocking, so the workflow run reports success even when the suite inside it fails. Signed-off-by: fredespi --- tests/e2e-cucumber/features/networking.feature | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/e2e-cucumber/features/networking.feature b/tests/e2e-cucumber/features/networking.feature index 332564cc70..c1b32a8cd6 100644 --- a/tests/e2e-cucumber/features/networking.feature +++ b/tests/e2e-cucumber/features/networking.feature @@ -54,6 +54,7 @@ Feature: Native HTTP networking # public endpoint actually authenticates. @id:networking-public-bind-endpoint-enforces-key @requires-gpu Scenario: 4 - A public endpoint issues a key once and then enforces it + Given a managed runtime is active When the user serves a model on a public interface with public binding allowed Then the CLI shows the endpoint key once and how to send it And a request without the key is refused as unauthorized From 8fbcb49ca44939851f133fb5cdb7be893e0f2af7 Mon Sep 17 00:00:00 2001 From: fredespi Date: Tue, 25 Aug 2026 20:06:06 +0200 Subject: [PATCH 3/8] test(e2e): retry the public-bind readiness probe while the engine binds MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A managed serve returns once the supervisor is up, so for the first seconds the engine has not bound the port and the readiness probe's connection is refused. The probe treated any transport error as fatal, so the scenario died on that first refusal before it ever reached its assertions. Split the request helper: the readiness wait now retries until the endpoint answers (or the timeout expires), while the assertions keep treating a refused connection as fatal — by then the endpoint has already answered, so a transport failure there is a real fault rather than a startup race. Signed-off-by: fredespi --- .../tests/e2e/endpoint_auth_steps.rs | 34 +++++++++++++------ 1 file changed, 24 insertions(+), 10 deletions(-) diff --git a/tests/e2e-cucumber/tests/e2e/endpoint_auth_steps.rs b/tests/e2e-cucumber/tests/e2e/endpoint_auth_steps.rs index 60ed4be0da..8301b25078 100644 --- a/tests/e2e-cucumber/tests/e2e/endpoint_auth_steps.rs +++ b/tests/e2e-cucumber/tests/e2e/endpoint_auth_steps.rs @@ -33,32 +33,46 @@ fn issued_api_key(output: &str) -> Option { .filter(|key| !key.is_empty()) } -/// GET `/models`, optionally presenting a bearer token. Returns the HTTP -/// status so a step can assert acceptance vs. refusal. -async fn get_models_status(bearer: Option<&str>) -> reqwest::StatusCode { +/// GET `/models`, optionally presenting a bearer token. `None` means the +/// request never reached a listener (connection refused / timed out), which is +/// distinct from "answered with a status" — the readiness wait tolerates that while +/// the engine is still binding, but an assertion must not. +async fn try_models_status(bearer: Option<&str>) -> Option { let client = reqwest::Client::builder() .timeout(Duration::from_secs(30)) .build() .expect("failed to build HTTP client"); - let url = format!("{ENDPOINT_URL}/models"); - let mut request = client.get(&url); + let mut request = client.get(format!("{ENDPOINT_URL}/models")); if let Some(token) = bearer { request = request.bearer_auth(token); } - request - .send() + request.send().await.ok().map(|response| response.status()) +} + +/// As [`try_models_status`], but a transport failure is fatal. Used by the +/// assertions, which only run once the endpoint has already answered — so at that +/// point a refused connection is a genuine failure, not a startup race. +async fn get_models_status(bearer: Option<&str>) -> reqwest::StatusCode { + try_models_status(bearer) .await - .unwrap_or_else(|error| panic!("GET {url} failed: {error}")) - .status() + .unwrap_or_else(|| panic!("GET {ENDPOINT_URL}/models did not reach the served endpoint")) } /// Wait until the authenticated endpoint answers, so the assertions run against a /// loaded model rather than a still-starting one. Polls WITH the key, because an /// unauthenticated probe is exactly what the endpoint is meant to reject. +/// +/// A managed serve returns as soon as the supervisor is up, so for the first while +/// the engine has not bound the port yet and the connection is simply refused. +/// That is expected here and must be retried — panicking on the first refusal is +/// what made this scenario fail before it ever reached its assertions. async fn wait_for_authenticated_endpoint(key: &str, timeout_secs: u64) { let deadline = Instant::now() + Duration::from_secs(timeout_secs); loop { - if get_models_status(Some(key)).await.is_success() { + if try_models_status(Some(key)) + .await + .is_some_and(|status| status.is_success()) + { return; } assert!( From 272f0e6e3d3bab7bc4c788f0c456c4cf6ad87f0f Mon Sep 17 00:00:00 2001 From: fredespi Date: Tue, 25 Aug 2026 20:48:39 +0200 Subject: [PATCH 4/8] test(e2e): pin the public-bind auth scenario to the vLLM engine MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The scenario was tagged @requires-gpu only, so it also selected the native-Windows GPU lane. There the effective serve engine is lemonade, and the CLI deliberately refuses a public bind on Windows lemonade because the endpoint key cannot be enforced through it — a designed-in refusal, not a defect. The scenario would have driven that refusal and failed deterministically on that lane; the MI300X dispatch could not surface it because that host resolves to vLLM. Add @requires-engine:vllm, the same precedent serve-vllm-default-on-instinct uses. Because vLLM cannot start on Windows or on families that do not prefer it, the scenario now resolves to a skip on the Strix lanes rather than needing an expectations row for behaviour that is working as designed. It also keeps this cold serve off the time-capped Strix lanes. Also correct the show-once comment: the key does legitimately appear a second time inside the curl example line, so the assertion counts the dedicated `api key:` line rather than occurrences of the key. Signed-off-by: fredespi --- tests/e2e-cucumber/features/networking.feature | 10 +++++++++- tests/e2e-cucumber/tests/e2e/endpoint_auth_steps.rs | 10 +++++++--- 2 files changed, 16 insertions(+), 4 deletions(-) diff --git a/tests/e2e-cucumber/features/networking.feature b/tests/e2e-cucumber/features/networking.feature index c1b32a8cd6..44f1326082 100644 --- a/tests/e2e-cucumber/features/networking.feature +++ b/tests/e2e-cucumber/features/networking.feature @@ -52,7 +52,15 @@ Feature: Native HTTP networking # rules (loopback stays credential-free even when a key is passed) are already # unit-tested in the CLI, so this pins only what a unit test cannot: that a live # public endpoint actually authenticates. - @id:networking-public-bind-endpoint-enforces-key @requires-gpu + # + # `@requires-engine:vllm` (same precedent as `serve-vllm-default-on-instinct`) is + # load-bearing, not incidental: on a native-Windows host the effective serve engine + # is lemonade, and the CLI deliberately REFUSES a public bind there because the + # endpoint key cannot be enforced through Windows lemonade. Without this tag the + # scenario would drive that refusal and fail on the Strix-Windows lane — a + # designed-in behaviour, not a bug, so it wants a tag rather than an xfail row. It + # also keeps this cold serve off the time-capped Strix lanes entirely. + @id:networking-public-bind-endpoint-enforces-key @requires-gpu @requires-engine:vllm Scenario: 4 - A public endpoint issues a key once and then enforces it Given a managed runtime is active When the user serves a model on a public interface with public binding allowed diff --git a/tests/e2e-cucumber/tests/e2e/endpoint_auth_steps.rs b/tests/e2e-cucumber/tests/e2e/endpoint_auth_steps.rs index 8301b25078..1c19b07660 100644 --- a/tests/e2e-cucumber/tests/e2e/endpoint_auth_steps.rs +++ b/tests/e2e-cucumber/tests/e2e/endpoint_auth_steps.rs @@ -121,15 +121,19 @@ async fn shows_key_once(world: &mut E2eWorld) { .endpoint_api_key .as_deref() .expect("no endpoint api key captured"); - // Shown exactly once: a key echoed on several lines would widen the window in - // which it can be captured from a log. + // Exactly one dedicated `api key:` line — this is the launch output's single + // disclosure of the secret, matching the "shown only now" promise beneath it. + // The key also appears inside the adjacent `example: curl -H "Authorization: + // Bearer "` usage line, which is deliberate guidance rather than a second + // disclosure, so count only the dedicated line rather than occurrences of the + // key itself. let shown = output .lines() .filter(|line| line.trim().starts_with("api key:")) .count(); assert_eq!( shown, 1, - "expected the key on exactly one line, got {shown}:\n{output}" + "expected exactly one dedicated `api key:` line, got {shown}:\n{output}" ); assert!( output.contains("shown only now"), From 039d0e431640080970545138b32bdc5021bae484 Mon Sep 17 00:00:00 2001 From: fredespi Date: Tue, 25 Aug 2026 21:03:57 +0200 Subject: [PATCH 5/8] test(e2e): make the public-bind readiness timeout diagnosable MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A run on MI300X timed out after the full readiness budget with nothing but "did not answer an authenticated request within 300s", which cannot distinguish the three failures that look identical from outside: the engine never started, it is still loading weights, or it is up and rejecting the key it just issued — the last of which would be a real defect in the contract under test rather than a slow start. Carry the engine's own log tail, the serve stdout, and the last observed endpoint response into the timeout panic, the same diagnostics the other serve steps already collect. Signed-off-by: fredespi --- .../tests/e2e/endpoint_auth_steps.rs | 28 +++++++++++++------ 1 file changed, 19 insertions(+), 9 deletions(-) diff --git a/tests/e2e-cucumber/tests/e2e/endpoint_auth_steps.rs b/tests/e2e-cucumber/tests/e2e/endpoint_auth_steps.rs index 1c19b07660..c7a90b6dfd 100644 --- a/tests/e2e-cucumber/tests/e2e/endpoint_auth_steps.rs +++ b/tests/e2e-cucumber/tests/e2e/endpoint_auth_steps.rs @@ -17,6 +17,8 @@ use std::time::{Duration, Instant}; use cucumber::{then, when}; use crate::E2eWorld; +use e2e_cucumber::serve_log::service_log_tail; + use crate::e2e::serving_steps::{ensure_serve_port_free, host_serve_target, serve_timeout_for}; /// The port every serve scenario shares (see `serving_steps::SERVE_PORT`). A public @@ -66,18 +68,26 @@ async fn get_models_status(bearer: Option<&str>) -> reqwest::StatusCode { /// the engine has not bound the port yet and the connection is simply refused. /// That is expected here and must be retried — panicking on the first refusal is /// what made this scenario fail before it ever reached its assertions. -async fn wait_for_authenticated_endpoint(key: &str, timeout_secs: u64) { +/// +/// On timeout the panic carries the engine's own log tail and the last thing the +/// endpoint did, because "did not answer in Ns" alone cannot distinguish the three +/// failures that look identical from outside: the engine never started, it started +/// but is still loading weights, or it is up and REJECTING the key we were issued +/// (which would be a real defect in the contract under test, not a slow start). +async fn wait_for_authenticated_endpoint(key: &str, timeout_secs: u64, serve_stdout: &str) { let deadline = Instant::now() + Duration::from_secs(timeout_secs); loop { - if try_models_status(Some(key)) - .await - .is_some_and(|status| status.is_success()) - { - return; - } + let last_seen = match try_models_status(Some(key)).await { + Some(status) if status.is_success() => return, + Some(status) => format!("endpoint answered {status} to the issued key"), + None => "connection refused / timed out".to_owned(), + }; assert!( Instant::now() < deadline, - "public endpoint {ENDPOINT_URL} did not answer an authenticated request within {timeout_secs}s" + "public endpoint {ENDPOINT_URL} did not answer an authenticated request within \ + {timeout_secs}s\nlast observed: {last_seen}\n--- SERVE STDOUT ---\n{serve_stdout}\ + \n--- SERVICE LOG (tail) ---\n{}", + service_log_tail(serve_stdout) ); tokio::time::sleep(Duration::from_secs(5)).await; } @@ -105,7 +115,7 @@ async fn serve_public_bind(world: &mut E2eWorld) { ); let key = issued_api_key(&stdout) .unwrap_or_else(|| panic!("serve did not print an endpoint api key:\n{stdout}")); - wait_for_authenticated_endpoint(&key, serve_timeout_for(world)).await; + wait_for_authenticated_endpoint(&key, serve_timeout_for(world), &stdout).await; world.endpoint = Some(ENDPOINT_URL.to_owned()); world.endpoint_api_key = Some(key); world.model_name = Some(model.to_owned()); From 02726d1aa5fb3612a9d433cfa411ffe1e2f81757 Mon Sep 17 00:00:00 2001 From: fredespi Date: Fri, 23 Oct 2026 20:15:41 +0200 Subject: [PATCH 6/8] test(e2e): fix networking-04 scenario naming and reuse the shared serve URL Signed-off-by: fredespi --- .../e2e-cucumber/features/networking.feature | 6 ++-- .../tests/e2e/endpoint_auth_steps.rs | 30 ++++++++++++------- tests/e2e-cucumber/tests/e2e/serving_steps.rs | 2 +- 3 files changed, 23 insertions(+), 15 deletions(-) diff --git a/tests/e2e-cucumber/features/networking.feature b/tests/e2e-cucumber/features/networking.feature index f688a00880..62c0878811 100644 --- a/tests/e2e-cucumber/features/networking.feature +++ b/tests/e2e-cucumber/features/networking.feature @@ -4,8 +4,8 @@ Feature: Native HTTP networking # certificate store), replacing the generated PowerShell script that used to # run under `powershell.exe -ExecutionPolicy Bypass` on Windows. These # scenarios drive the real `rocm` binary end-to-end against the mock server so a - # GET/round-trip reaches a local endpoint over the native stack. They run on the - # mock (no GPU) on every platform the suite runs on, so a regression back to a + # GET/round-trip reaches a local endpoint over the native stack. All but + # networking-04 run on the mock (no GPU) on every platform the suite runs on, so a regression back to a # shell-out networking backend surfaces here rather than only in the field. # `rocm services list` extracts host:port from the service record and issues a @@ -61,7 +61,7 @@ Feature: Native HTTP networking # designed-in behaviour, not a bug, so it wants a tag rather than an xfail row. It # also keeps this cold serve off the time-capped Strix lanes entirely. @id:networking-public-bind-endpoint-enforces-key @requires-gpu @requires-engine:vllm - Scenario: 4 - A public endpoint issues a key once and then enforces it + Scenario: networking-04 - A public endpoint issues a key once and then enforces it Given a managed runtime is active When the user serves a model on a public interface with public binding allowed Then the CLI shows the endpoint key once and how to send it diff --git a/tests/e2e-cucumber/tests/e2e/endpoint_auth_steps.rs b/tests/e2e-cucumber/tests/e2e/endpoint_auth_steps.rs index c7a90b6dfd..083ff20b55 100644 --- a/tests/e2e-cucumber/tests/e2e/endpoint_auth_steps.rs +++ b/tests/e2e-cucumber/tests/e2e/endpoint_auth_steps.rs @@ -19,11 +19,15 @@ use cucumber::{then, when}; use crate::E2eWorld; use e2e_cucumber::serve_log::service_log_tail; -use crate::e2e::serving_steps::{ensure_serve_port_free, host_serve_target, serve_timeout_for}; - -/// The port every serve scenario shares (see `serving_steps::SERVE_PORT`). A public -/// bind listens on all interfaces, so the test still reaches it over loopback. -const ENDPOINT_URL: &str = "http://127.0.0.1:11435/v1"; +use crate::e2e::serving_steps::{ + SERVE_BASE_URL, ensure_serve_port_free, host_serve_target, serve_timeout_for, +}; + +/// A public bind listens on all interfaces, so the test still reaches the shared +/// serve port over loopback. +fn endpoint_url() -> &'static str { + SERVE_BASE_URL.as_str() +} /// Pull the issued endpoint key out of the launch output. `serve` prints it as /// ` api key: ` exactly once, alongside the note and curl example. @@ -44,7 +48,7 @@ async fn try_models_status(bearer: Option<&str>) -> Option .timeout(Duration::from_secs(30)) .build() .expect("failed to build HTTP client"); - let mut request = client.get(format!("{ENDPOINT_URL}/models")); + let mut request = client.get(format!("{}/models", endpoint_url())); if let Some(token) = bearer { request = request.bearer_auth(token); } @@ -55,9 +59,12 @@ async fn try_models_status(bearer: Option<&str>) -> Option /// assertions, which only run once the endpoint has already answered — so at that /// point a refused connection is a genuine failure, not a startup race. async fn get_models_status(bearer: Option<&str>) -> reqwest::StatusCode { - try_models_status(bearer) - .await - .unwrap_or_else(|| panic!("GET {ENDPOINT_URL}/models did not reach the served endpoint")) + try_models_status(bearer).await.unwrap_or_else(|| { + panic!( + "GET {}/models did not reach the served endpoint", + endpoint_url() + ) + }) } /// Wait until the authenticated endpoint answers, so the assertions run against a @@ -82,9 +89,10 @@ async fn wait_for_authenticated_endpoint(key: &str, timeout_secs: u64, serve_std Some(status) => format!("endpoint answered {status} to the issued key"), None => "connection refused / timed out".to_owned(), }; + let endpoint = endpoint_url(); assert!( Instant::now() < deadline, - "public endpoint {ENDPOINT_URL} did not answer an authenticated request within \ + "public endpoint {endpoint} did not answer an authenticated request within \ {timeout_secs}s\nlast observed: {last_seen}\n--- SERVE STDOUT ---\n{serve_stdout}\ \n--- SERVICE LOG (tail) ---\n{}", service_log_tail(serve_stdout) @@ -116,7 +124,7 @@ async fn serve_public_bind(world: &mut E2eWorld) { let key = issued_api_key(&stdout) .unwrap_or_else(|| panic!("serve did not print an endpoint api key:\n{stdout}")); wait_for_authenticated_endpoint(&key, serve_timeout_for(world), &stdout).await; - world.endpoint = Some(ENDPOINT_URL.to_owned()); + world.endpoint = Some(endpoint_url().to_owned()); world.endpoint_api_key = Some(key); world.model_name = Some(model.to_owned()); world.cli_output = Some(stdout); diff --git a/tests/e2e-cucumber/tests/e2e/serving_steps.rs b/tests/e2e-cucumber/tests/e2e/serving_steps.rs index 4874ed710b..43c15e967c 100644 --- a/tests/e2e-cucumber/tests/e2e/serving_steps.rs +++ b/tests/e2e-cucumber/tests/e2e/serving_steps.rs @@ -75,7 +75,7 @@ pub(crate) async fn model_is_ready( /// The OpenAI-compatible base URL every GPU serve scenario serves on (see /// [`SERVE_PORT`]), and the model listing readiness is judged by. -static SERVE_BASE_URL: LazyLock = +pub(crate) static SERVE_BASE_URL: LazyLock = LazyLock::new(|| format!("http://127.0.0.1:{SERVE_PORT}/v1")); static MODELS_URL: LazyLock = LazyLock::new(|| format!("http://127.0.0.1:{SERVE_PORT}/v1/models")); From 90089998d666c494421287b5b3c3a38f00055888 Mon Sep 17 00:00:00 2001 From: fredespi Date: Mon, 26 Oct 2026 10:25:33 +0100 Subject: [PATCH 7/8] test(e2e): update testing.md for networking-04, tighten key-leak check, fix feature header Signed-off-by: fredespi --- docs/testing.md | 11 +++++++---- tests/e2e-cucumber/features/networking.feature | 7 +++++-- .../e2e-cucumber/tests/e2e/endpoint_auth_steps.rs | 14 ++++++++++++++ 3 files changed, 26 insertions(+), 6 deletions(-) diff --git a/docs/testing.md b/docs/testing.md index 83e2178824..db68cfcc66 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -952,10 +952,13 @@ key file (never argv), persisted as a 0600 per-service file (not the OS keychain which is unavailable on headless serving hosts), and printed once as client configuration — it must never appear in `rocm services`, `rocm logs`, or the audit log. Verifying that the running server actually *rejects* an -unauthenticated request requires a live engine; that end-to-end assertion is a -deferred follow-up and is **not yet** exercised by the GPU acceptance scripts -(`scripts/vllm_therock_gpu_test.py`). The unit tests cover key policy, -generation, the 0600 key file, and key-file resolution. +unauthenticated request requires a live engine: the `networking-04` scenario +(`tests/e2e-cucumber/features/networking.feature`) serves a model on a public +bind and checks that a request without the key and one with a wrong key get +401, while the issued key is accepted. It runs on the MI300X vLLM lane only, which +is not a required check; native Windows and lemonade-default hosts skip it. The +unit tests cover key policy, generation, the 0600 key file, and key-file +resolution. The same requirement holds across respawns, because the invariant belongs to the recorded *host*, not to the key file: a service recorded on a public host refuses diff --git a/tests/e2e-cucumber/features/networking.feature b/tests/e2e-cucumber/features/networking.feature index 62c0878811..7ce2fd7af4 100644 --- a/tests/e2e-cucumber/features/networking.feature +++ b/tests/e2e-cucumber/features/networking.feature @@ -5,8 +5,11 @@ Feature: Native HTTP networking # run under `powershell.exe -ExecutionPolicy Bypass` on Windows. These # scenarios drive the real `rocm` binary end-to-end against the mock server so a # GET/round-trip reaches a local endpoint over the native stack. All but - # networking-04 run on the mock (no GPU) on every platform the suite runs on, so a regression back to a - # shell-out networking backend surfaces here rather than only in the field. + # networking-04 run on the mock (no GPU) on every platform the suite runs on, so + # a regression back to a shell-out networking backend surfaces here rather than + # only in the field. networking-04 is a GPU serve scenario: it starts a real + # engine on a public bind and checks key enforcement from the harness's own + # HTTP client, not through the CLI. # `rocm services list` extracts host:port from the service record and issues a # native HTTP GET to `/v1/models` as its readiness probe (served by the mock). diff --git a/tests/e2e-cucumber/tests/e2e/endpoint_auth_steps.rs b/tests/e2e-cucumber/tests/e2e/endpoint_auth_steps.rs index 083ff20b55..ae686c4c25 100644 --- a/tests/e2e-cucumber/tests/e2e/endpoint_auth_steps.rs +++ b/tests/e2e-cucumber/tests/e2e/endpoint_auth_steps.rs @@ -153,6 +153,20 @@ async fn shows_key_once(world: &mut E2eWorld) { shown, 1, "expected exactly one dedicated `api key:` line, got {shown}:\n{output}" ); + // The key may appear only on the `api key:` line and in the curl example; any + // other line carrying it means it leaked elsewhere in the launch output. + let leaked: Vec<&str> = output + .lines() + .filter(|line| line.contains(key)) + .filter(|line| { + let line = line.trim(); + !line.starts_with("api key:") && !line.starts_with("example:") + }) + .collect(); + assert!( + leaked.is_empty(), + "the key leaked onto lines other than the api key line and curl example: {leaked:?}" + ); assert!( output.contains("shown only now"), "expected the show-once note, got:\n{output}" From 95676038e0afc8ea136234fe5f75fda327bc316d Mon Sep 17 00:00:00 2001 From: fredespi Date: Mon, 26 Oct 2026 12:45:11 +0100 Subject: [PATCH 8/8] test(e2e): link networking.feature in testing.md, fail fast when the issued key gets 401 Signed-off-by: fredespi --- docs/testing.md | 2 +- tests/e2e-cucumber/tests/e2e/endpoint_auth_steps.rs | 7 +++++++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/docs/testing.md b/docs/testing.md index db68cfcc66..3409cb291b 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -953,7 +953,7 @@ which is unavailable on headless serving hosts), and printed once as client configuration — it must never appear in `rocm services`, `rocm logs`, or the audit log. Verifying that the running server actually *rejects* an unauthenticated request requires a live engine: the `networking-04` scenario -(`tests/e2e-cucumber/features/networking.feature`) serves a model on a public +([networking.feature](../tests/e2e-cucumber/features/networking.feature)) serves a model on a public bind and checks that a request without the key and one with a wrong key get 401, while the issued key is accepted. It runs on the MI300X vLLM lane only, which is not a required check; native Windows and lemonade-default hosts skip it. The diff --git a/tests/e2e-cucumber/tests/e2e/endpoint_auth_steps.rs b/tests/e2e-cucumber/tests/e2e/endpoint_auth_steps.rs index ae686c4c25..e1b3bc94af 100644 --- a/tests/e2e-cucumber/tests/e2e/endpoint_auth_steps.rs +++ b/tests/e2e-cucumber/tests/e2e/endpoint_auth_steps.rs @@ -86,6 +86,13 @@ async fn wait_for_authenticated_endpoint(key: &str, timeout_secs: u64, serve_std loop { let last_seen = match try_models_status(Some(key)).await { Some(status) if status.is_success() => return, + // A bound engine rejecting the key it was issued is a contract defect, + // not a slow start, so report it now instead of at the deadline. + Some(reqwest::StatusCode::UNAUTHORIZED) => panic!( + "public endpoint {} rejected the issued key with 401\n--- SERVE STDOUT ---\n{serve_stdout}\n--- SERVICE LOG (tail) ---\n{}", + endpoint_url(), + service_log_tail(serve_stdout) + ), Some(status) => format!("endpoint answered {status} to the issued key"), None => "connection refused / timed out".to_owned(), };