From 7579270f28835b0c2eae022548cdbfb44e2d606c Mon Sep 17 00:00:00 2001 From: fredespi Date: Fri, 17 Jul 2026 11:40:46 +0200 Subject: [PATCH 1/4] test(e2e): serve smallest vLLM model (0.5B) in shared GPU serve path host_serve_target's vLLM branch served Qwen2.5-1.5B, feeding the four most-run Instinct serve scenarios (serve-vllm-inference, serve-readiness-contract, serve-default-engine-working-endpoint and its inference half). Qwen2.5-0.5B is the smallest vLLM-preferred catalog entry and is already served+asserted-vLLM by serve-vllm-default-on-instinct, so switching cuts the loaded model ~3x on those lanes with no coverage loss. Weights lazy-download to the shared HF cache, so no CI prewarm list changes. Signed-off-by: fredespi --- tests/e2e-cucumber/tests/e2e/serving_steps.rs | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/tests/e2e-cucumber/tests/e2e/serving_steps.rs b/tests/e2e-cucumber/tests/e2e/serving_steps.rs index 9d4cb05613..58d526b4b6 100644 --- a/tests/e2e-cucumber/tests/e2e/serving_steps.rs +++ b/tests/e2e-cucumber/tests/e2e/serving_steps.rs @@ -281,8 +281,9 @@ fn host_serve_target() -> (&'static str, &'static str, &'static str) { // Qwen3-0.6B-Q4_0.gguf, so "Qwen3-0.6B" is the distinctive substring. ("Qwen3-0.6B-GGUF", "lemonade", "Qwen3-0.6B") } else { - // Safetensors via vLLM; "Qwen2.5-1.5B" is the distinctive substring. - ("Qwen/Qwen2.5-1.5B-Instruct", "vllm", "Qwen2.5-1.5B") + // Safetensors via vLLM; "Qwen2.5-0.5B" is the distinctive substring. + // Smallest vLLM-preferred catalog entry — see user_serves_vllm_capable_default. + ("Qwen/Qwen2.5-0.5B-Instruct", "vllm", "Qwen2.5-0.5B") } } @@ -692,7 +693,7 @@ async fn assert_response_model_correct(world: &mut E2eWorld) { /// Whether a chat response's `model` field identifies the model we served. /// -/// vLLM echoes the exact id we passed (`Qwen/Qwen2.5-1.5B-Instruct`), so a plain +/// vLLM echoes the exact id we passed (`Qwen/Qwen2.5-0.5B-Instruct`), so a plain /// containment holds. Lemonade instead reports the concrete GGUF artifact it /// loaded — e.g. serving `Qwen3-0.6B-GGUF` yields `Qwen3-0.6B-Q4_0.gguf` — so an /// exact/containment check on the catalog name fails even though it IS the right From f326e93abf9df4c6ae2aa758b72005d96db7195b Mon Sep 17 00:00:00 2001 From: fredespi Date: Fri, 17 Jul 2026 11:51:09 +0200 Subject: [PATCH 2/4] docs(e2e): record the GPU serve model-size policy Document, at host_serve_target (where serve models are chosen), that every GPU serve scenario uses the smallest model satisfying its assertion, name the current per-engine floors (vLLM 0.5B, lemonade 0.6B), and note that large-model coverage lives solely in the @nightly serve-large-model scenario. Prevents a future serve scenario from silently pinning a larger model and reintroducing weight-load overhead on the per-PR path. Signed-off-by: fredespi --- tests/e2e-cucumber/tests/e2e/serving_steps.rs | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/tests/e2e-cucumber/tests/e2e/serving_steps.rs b/tests/e2e-cucumber/tests/e2e/serving_steps.rs index 58d526b4b6..b65daf29a8 100644 --- a/tests/e2e-cucumber/tests/e2e/serving_steps.rs +++ b/tests/e2e-cucumber/tests/e2e/serving_steps.rs @@ -275,6 +275,15 @@ async fn setup_mock_custom_port(world: &mut E2eWorld) { /// single-engine steps (`setup_gpu_model`'s old vLLM body and /// `setup_lemonade_model`), but lets a scenario tagged only `@requires-gpu` run /// on whichever engine the platform actually uses. +/// +/// MODEL-SIZE POLICY: every GPU serve scenario uses the SMALLEST model that +/// still satisfies its assertion — real GPU serves are the E2E wall-clock long +/// pole on serial hardware, so weight-load time is pure overhead here. Current +/// floors: vLLM `Qwen2.5-0.5B` (smallest vLLM-preferred catalog entry), lemonade +/// `Qwen3-0.6B-GGUF` (smallest lemonade recipe). Large-model behaviour is +/// exercised exactly once, in the `@nightly` `serve-large-model-inference` +/// scenario (Qwen3.6-27B) — never on the per-PR path. Do not raise a serve +/// scenario's model unless a smaller one genuinely cannot prove the assertion. fn host_serve_target() -> (&'static str, &'static str, &'static str) { if e2e_cucumber::capability::host_capability().effective_serve_engine == "lemonade" { // GGUF via lemonade's llama.cpp backend; endpoint reports e.g. From 4cd2c5311091a6eb1405f339bc1da6803e15fb4e Mon Sep 17 00:00:00 2001 From: fredespi Date: Fri, 17 Jul 2026 13:24:23 +0200 Subject: [PATCH 3/4] test(e2e): scope 0.5B swap to explicit-vLLM path only host_serve_target also feeds the default-engine serve step (no --engine), where the model's own preferred_engines decides which engine runs. Swapping the shared target to the vLLM-preferred 0.5B flipped default-engine resolution on Instinct from lemonade to vLLM, invalidating the EAI-7052 xfail for serve-default-engine-working-endpoint / -inference (they would XPASS and fail reconciliation). Add default_engine_serve_target (stays lemonade-preferred) for that step, keeping the 0.5B size cut only on the explicit --engine vllm path (serve-vllm-inference, serve-readiness-contract). Default-engine resolution is now identical to before the swap; no xfail-matrix change. Signed-off-by: fredespi --- tests/e2e-cucumber/tests/e2e/serving_steps.rs | 26 ++++++++++++++++++- 1 file changed, 25 insertions(+), 1 deletion(-) diff --git a/tests/e2e-cucumber/tests/e2e/serving_steps.rs b/tests/e2e-cucumber/tests/e2e/serving_steps.rs index b65daf29a8..a27198a24c 100644 --- a/tests/e2e-cucumber/tests/e2e/serving_steps.rs +++ b/tests/e2e-cucumber/tests/e2e/serving_steps.rs @@ -296,6 +296,25 @@ fn host_serve_target() -> (&'static str, &'static str, &'static str) { } } +/// The model the default-engine (no `--engine`) serve step should request. +/// +/// This is deliberately NOT `host_serve_target` — that step passes no `--engine`, +/// so the CLI resolves the engine from the model's own `preferred_engines`, and +/// the model choice therefore determines WHICH engine runs. The default-engine +/// xfail matrix depends on that resolution: on an Instinct host the request must +/// resolve to a GGUF recipe on lemonade (EAI-7052 — see expectations.toml), NOT +/// to vLLM. So this stays lemonade-preferred on every host; the smaller +/// vLLM-preferred 0.5B size cut applies only to the explicit-vLLM path. The model +/// hangs at load on Instinct (EAI-7052, xfail) so its size is irrelevant to +/// wall-clock here — behaviour preservation is what matters. +fn default_engine_serve_target() -> &'static str { + if e2e_cucumber::capability::host_capability().effective_serve_engine == "lemonade" { + "Qwen3-0.6B-GGUF" + } else { + "Qwen/Qwen2.5-1.5B-Instruct" + } +} + #[given("a model is being served on GPU")] async fn setup_gpu_model(world: &mut E2eWorld) { // Serve by the canonical HuggingFace ID (not the `qwen2.5` alias) with an @@ -432,7 +451,12 @@ async fn user_serves_default_engine(world: &mut E2eWorld) { // resolved (parsed from the serve plan), not the requested one — hardcoding the // requested model made this time out on hosts where the recipe resolved to a // different model, for reasons unrelated to engine selection. - let model = host_serve_target().0; + // + // Uses `default_engine_serve_target` (a lemonade-preferred model), NOT + // `host_serve_target`: the model here drives engine resolution, and the xfail + // matrix requires this to resolve to lemonade on Instinct (EAI-7052). See that + // fn's doc comment. + let model = default_engine_serve_target(); ensure_serve_port_free().await; let (stdout, _, rc) = crate::run_rocm(world, &["serve", model, "--managed"]); // The model the CLI resolved (what actually gets served) can differ from the From 77076770239c549c981248e419a23eaded6d7ba8 Mon Sep 17 00:00:00 2001 From: Roman Inflianskas Date: Fri, 17 Jul 2026 18:03:09 +0000 Subject: [PATCH 4/4] test(e2e): serve Qwen3.5 0.8B on vLLM Signed-off-by: Roman Inflianskas --- tests/e2e-cucumber/tests/e2e/serving_steps.rs | 22 +++++++++---------- 1 file changed, 10 insertions(+), 12 deletions(-) diff --git a/tests/e2e-cucumber/tests/e2e/serving_steps.rs b/tests/e2e-cucumber/tests/e2e/serving_steps.rs index a27198a24c..08907a0409 100644 --- a/tests/e2e-cucumber/tests/e2e/serving_steps.rs +++ b/tests/e2e-cucumber/tests/e2e/serving_steps.rs @@ -276,11 +276,10 @@ async fn setup_mock_custom_port(world: &mut E2eWorld) { /// `setup_lemonade_model`), but lets a scenario tagged only `@requires-gpu` run /// on whichever engine the platform actually uses. /// -/// MODEL-SIZE POLICY: every GPU serve scenario uses the SMALLEST model that -/// still satisfies its assertion — real GPU serves are the E2E wall-clock long -/// pole on serial hardware, so weight-load time is pure overhead here. Current -/// floors: vLLM `Qwen2.5-0.5B` (smallest vLLM-preferred catalog entry), lemonade -/// `Qwen3-0.6B-GGUF` (smallest lemonade recipe). Large-model behaviour is +/// MODEL-SIZE POLICY: GPU serve scenarios use small models that still satisfy +/// their assertions — real GPU serves are the E2E wall-clock long pole on serial +/// hardware, so weight-load time is pure overhead here. Current paths use vLLM +/// `Qwen3.5-0.8B` and lemonade `Qwen3-0.6B-GGUF`. Large-model behaviour is /// exercised exactly once, in the `@nightly` `serve-large-model-inference` /// scenario (Qwen3.6-27B) — never on the per-PR path. Do not raise a serve /// scenario's model unless a smaller one genuinely cannot prove the assertion. @@ -290,9 +289,8 @@ fn host_serve_target() -> (&'static str, &'static str, &'static str) { // Qwen3-0.6B-Q4_0.gguf, so "Qwen3-0.6B" is the distinctive substring. ("Qwen3-0.6B-GGUF", "lemonade", "Qwen3-0.6B") } else { - // Safetensors via vLLM; "Qwen2.5-0.5B" is the distinctive substring. - // Smallest vLLM-preferred catalog entry — see user_serves_vllm_capable_default. - ("Qwen/Qwen2.5-0.5B-Instruct", "vllm", "Qwen2.5-0.5B") + // Safetensors via vLLM; "Qwen3.5-0.8B" is the distinctive substring. + ("Qwen/Qwen3.5-0.8B", "vllm", "Qwen3.5-0.8B") } } @@ -303,9 +301,9 @@ fn host_serve_target() -> (&'static str, &'static str, &'static str) { /// the model choice therefore determines WHICH engine runs. The default-engine /// xfail matrix depends on that resolution: on an Instinct host the request must /// resolve to a GGUF recipe on lemonade (EAI-7052 — see expectations.toml), NOT -/// to vLLM. So this stays lemonade-preferred on every host; the smaller -/// vLLM-preferred 0.5B size cut applies only to the explicit-vLLM path. The model -/// hangs at load on Instinct (EAI-7052, xfail) so its size is irrelevant to +/// to vLLM. So this stays lemonade-preferred on every host; the smaller model +/// used by vLLM applies only to the explicit-engine path. The model hangs at load +/// on Instinct (EAI-7052, xfail) so its size is irrelevant to /// wall-clock here — behaviour preservation is what matters. fn default_engine_serve_target() -> &'static str { if e2e_cucumber::capability::host_capability().effective_serve_engine == "lemonade" { @@ -726,7 +724,7 @@ async fn assert_response_model_correct(world: &mut E2eWorld) { /// Whether a chat response's `model` field identifies the model we served. /// -/// vLLM echoes the exact id we passed (`Qwen/Qwen2.5-0.5B-Instruct`), so a plain +/// vLLM echoes the exact id we passed (`Qwen/Qwen3.5-0.8B`), so a plain /// containment holds. Lemonade instead reports the concrete GGUF artifact it /// loaded — e.g. serving `Qwen3-0.6B-GGUF` yields `Qwen3-0.6B-Q4_0.gguf` — so an /// exact/containment check on the catalog name fails even though it IS the right