From 70e884e4cbb87cc627fa1b1b6ee23c3073bf85ba Mon Sep 17 00:00:00 2001 From: Jussi Elo Date: Wed, 4 Nov 2026 18:10:17 +0000 Subject: [PATCH 1/4] refactor(serve): move the engine-recipe overrides out of serve_cmd.rs serve_cmd.rs carried about 1,700 production lines, most of them the pure transforms serve() applies to the selected engine's launch recipe hint. Move them to a new serve_recipe.rs: protocol_engine_recipe_hint and render_serve_engine_recipe_lines, the --tool-call-parser, --gpu-memory-utilization, --temperature/--top-p/--max-tokens and --engine-env overrides with their private rewriters and RESERVED_ENGINE_ENV_KEYS, engine_env_ignored_note and engine_recipe_enables_tool_choice. This is a pure move. The function bodies and the 28 tests that exercise them are byte-identical and keep their leaf names. Visibility is widened to pub(crate) only for what serve() and the AIM tests in serve_cmd.rs call (ServeGenerationDefaults and its three fields included); the private rewriters and the reserved-key list stay private. The AIM pre-flight and its tests stay in serve_cmd.rs, since their subject is the AIM checks and they only use protocol_engine_recipe_hint and engine_recipe_with_extra_env to build their input. Docs follow in the next commit. Signed-off-by: Jussi Elo --- apps/rocm/src/main.rs | 1 + apps/rocm/src/serve_cmd.rs | 1035 +------------------------------- apps/rocm/src/serve_recipe.rs | 1051 +++++++++++++++++++++++++++++++++ 3 files changed, 1059 insertions(+), 1028 deletions(-) create mode 100644 apps/rocm/src/serve_recipe.rs diff --git a/apps/rocm/src/main.rs b/apps/rocm/src/main.rs index 637e137474..e33dc60217 100644 --- a/apps/rocm/src/main.rs +++ b/apps/rocm/src/main.rs @@ -52,6 +52,7 @@ mod runtime_services; mod runtime_uninstall; mod runtimes_cmd; mod serve_cmd; +mod serve_recipe; mod serve_summary; mod service_hints; mod service_launch; diff --git a/apps/rocm/src/serve_cmd.rs b/apps/rocm/src/serve_cmd.rs index d32a3164cf..fa31bcc734 100644 --- a/apps/rocm/src/serve_cmd.rs +++ b/apps/rocm/src/serve_cmd.rs @@ -36,8 +36,6 @@ //! to `service_launch.rs` in ROCMAI-91 (Phase 6b), along with its test module; //! this file's `use` list below points there now instead of at the crate root. -use std::fmt::Write as _; - use anyhow::{Context as _, Result, bail}; use rocm_core::{ AppPaths, GfxSupport, ModelRecipeRecord, RocmCliConfig, default_engine_for_platform, @@ -46,8 +44,7 @@ use rocm_core::{ resolve_model_recipe, }; use rocm_engine_protocol::{ - DevicePolicy, ENGINE_RECIPE_CONTRACT_VERSION, EngineMethod, EngineRecipeEndpointHint, - EngineRecipeHint, EngineRecipeUnsupportedCombinationHint, GpuSelection, ResolveModelRequest, + DevicePolicy, EngineMethod, EngineRecipeHint, GpuSelection, ResolveModelRequest, ResolveModelResponse, }; @@ -64,6 +61,12 @@ use crate::gpu_policy::{ serve_gpu_low_memory_warning, validate_pinned_gpu_index, write_unchecked_index_warning, }; use crate::managed_service_status::{find_live_managed_service, live_managed_services_for_model}; +use crate::serve_recipe::{ + ServeGenerationDefaults, engine_env_ignored_note, engine_recipe_enables_tool_choice, + engine_recipe_with_extra_env, engine_recipe_with_generation_defaults, + engine_recipe_with_gpu_memory_utilization_override, engine_recipe_with_tool_call_override, + parse_gpu_memory_utilization, protocol_engine_recipe_hint, render_serve_engine_recipe_lines, +}; use crate::serve_summary; use crate::service_launch::{ drop_orphaned_endpoint_key_on_already_running, e2e_simulate_oom_launch, @@ -220,28 +223,6 @@ fn serve_engine_selection_line(selection: &ServeEngineSelection) -> String { format!(" engine_selection: {}", selection.source) } -fn render_serve_engine_recipe_lines(engine_recipe: &EngineRecipeHint) -> String { - let mut output = String::new(); - let _ = writeln!( - output, - " engine_recipe_contract: {}", - engine_recipe.contract_version - ); - let _ = writeln!( - output, - " engine_recipe_policy: selected-engine required_flags are applied at launch; parser/endpoint metadata is forwarded to the adapter" - ); - let _ = writeln!(output, " engine_recipe_engine: {}", engine_recipe.engine); - if !engine_recipe.required_flags.is_empty() { - let _ = writeln!( - output, - " engine_recipe_required_flags: {}", - engine_recipe.required_flags.join(" ") - ); - } - output -} - /// For AIM, refuses a GPU the catalog has no profile for and pins the catalog's /// `AIM_PROFILE_ID` for the detected GPU family. `host_family` is a normalized /// TheRock family, `None` when it could not be detected (nothing is enforced @@ -420,391 +401,6 @@ fn aim_board_note( )) } -fn protocol_engine_recipe_hint( - recipe: &ModelRecipeRecord, - engine: &str, -) -> Option { - recipe - .engine_recipes - .iter() - .find(|engine_recipe| engine_recipe.engine == engine) - .map(|engine_recipe| EngineRecipeHint { - contract_version: ENGINE_RECIPE_CONTRACT_VERSION.to_owned(), - engine: engine_recipe.engine.clone(), - required_flags: engine_recipe.required_flags.clone(), - parser_settings: engine_recipe.parser_settings.clone(), - extra_env: engine_recipe.extra_env.clone(), - preferred_endpoint: engine_recipe.preferred_endpoint.as_ref().map(|endpoint| { - EngineRecipeEndpointHint { - endpoint_mode: endpoint.endpoint_mode.clone(), - settings: endpoint.settings.clone(), - } - }), - unsupported_combinations: engine_recipe - .unsupported_combinations - .iter() - .map(|combination| EngineRecipeUnsupportedCombinationHint { - combination: combination.combination.clone(), - reason: combination.reason.clone(), - }) - .collect(), - notes: engine_recipe.notes.clone(), - }) -} - -/// Applies an explicit `--tool-call-parser` override to a vLLM engine recipe hint. -/// -/// The TUI chat tab always attaches tool definitions to non-streaming chat -/// requests (`tool_choice: "auto"`). vLLM rejects those with HTTP 400 unless it was -/// started with `--enable-auto-tool-choice` *and* a matching `--tool-call-parser`. -/// The correct parser is model-specific and vLLM does not auto-detect it, so it is -/// never guessed from the model ref: it comes either from authored catalog recipe -/// metadata (already carried in `required_flags`) or from the explicit -/// `--tool-call-parser` serve flag, which this applies. -/// -/// Only vLLM is affected. When an override is supplied it wins over any -/// recipe-authored parser (a single `--tool-call-parser`, no duplication) and a -/// minimal hint is synthesized when none exists (arbitrary HF repos, or a catalog -/// model forced onto a non-preferred engine). With no override the hint passes -/// through unchanged. -fn engine_recipe_with_tool_call_override( - engine: &str, - hint: Option, - tool_call_parser: Option<&str>, -) -> Option { - if !engine.eq_ignore_ascii_case("vllm") { - return hint; - } - let Some(parser) = tool_call_parser - .map(str::trim) - .filter(|value| !value.is_empty()) - else { - return hint; - }; - let mut hint = hint.unwrap_or_else(|| EngineRecipeHint { - contract_version: ENGINE_RECIPE_CONTRACT_VERSION.to_owned(), - engine: engine.to_owned(), - ..EngineRecipeHint::default() - }); - set_vllm_tool_call_parser(&mut hint.required_flags, parser); - Some(hint) -} - -/// Sampling defaults a `rocm serve` invocation can push into a vLLM launch. -#[derive(Debug, Clone, Copy, Default, PartialEq)] -struct ServeGenerationDefaults { - temperature: Option, - top_p: Option, - max_tokens: Option, -} - -impl ServeGenerationDefaults { - const fn is_empty(&self) -> bool { - self.temperature.is_none() && self.top_p.is_none() && self.max_tokens.is_none() - } -} - -/// Applies `rocm serve` generation defaults (`--temperature`/`--top-p`/`--max-tokens`) -/// to the selected engine's launch recipe. -/// -/// vLLM `serve` has no raw `--temperature`/`--top-p` flags; `--override-generation-config` -/// is the supported way to set server-wide sampling defaults, so `--max-tokens` is -/// mapped onto vLLM's `max_new_tokens` output cap. Only supplied values are written, -/// and any values already carried by an authored recipe's -/// `--override-generation-config` are preserved (the CLI-supplied keys win). -/// -/// Lemonade's llama.cpp backend accepts the equivalent `--temperature`, `--top-p`, -/// and `--n-predict` launch flags. A minimal hint is synthesized when none exists. -/// With no defaults supplied the hint passes through unchanged. -fn engine_recipe_with_generation_defaults( - engine: &str, - hint: Option, - defaults: ServeGenerationDefaults, -) -> Result> { - if defaults.is_empty() { - return Ok(hint); - } - let mut hint = hint.unwrap_or_else(|| EngineRecipeHint { - contract_version: ENGINE_RECIPE_CONTRACT_VERSION.to_owned(), - engine: engine.to_owned(), - ..EngineRecipeHint::default() - }); - if engine.eq_ignore_ascii_case("vllm") { - let mut overrides = serde_json::Map::new(); - if let Some(temperature) = defaults.temperature { - overrides.insert("temperature".to_owned(), serde_json::json!(temperature)); - } - if let Some(top_p) = defaults.top_p { - overrides.insert("top_p".to_owned(), serde_json::json!(top_p)); - } - if let Some(max_tokens) = defaults.max_tokens { - overrides.insert("max_new_tokens".to_owned(), serde_json::json!(max_tokens)); - } - set_vllm_override_generation_config(&mut hint.required_flags, &overrides); - } else if engine.eq_ignore_ascii_case("lemonade") { - set_lemonade_generation_defaults(&mut hint.required_flags, defaults); - } else { - bail!( - "generation defaults are not supported by engine `{engine}`; omit --temperature/--top-p/--max-tokens or select vllm/lemonade" - ); - } - Ok(Some(hint)) -} - -fn set_lemonade_generation_defaults(flags: &mut Vec, defaults: ServeGenerationDefaults) { - // Only touch a flag pair when the caller actually supplied that control — - // an unset field must leave any authored recipe value in place rather than - // deleting it, mirroring the vLLM merge semantics in - // `set_vllm_override_generation_config`. - for (name, value) in [ - ( - "--temperature", - defaults.temperature.map(|value| value.to_string()), - ), - ("--top-p", defaults.top_p.map(|value| value.to_string())), - ( - "--n-predict", - defaults.max_tokens.map(|value| value.to_string()), - ), - ] { - let Some(value) = value else { - continue; - }; - let mut rewritten = Vec::with_capacity(flags.len() + 2); - let mut skip_value = false; - for flag in std::mem::take(flags) { - if skip_value { - skip_value = false; - continue; - } - if flag == name { - skip_value = true; - } else { - rewritten.push(flag); - } - } - rewritten.extend([name.to_owned(), value]); - *flags = rewritten; - } -} - -/// Rewrites `flags` so vLLM's `--override-generation-config` carries exactly one -/// merged JSON object: any existing `--override-generation-config ` pair is -/// removed, its keys are used as a base, and `overrides` are layered on top (CLI -/// values win). Emits a single flag pair with the merged, stably-ordered config. -fn set_vllm_override_generation_config( - flags: &mut Vec, - overrides: &serde_json::Map, -) { - let existing = std::mem::take(flags); - let mut rewritten: Vec = Vec::with_capacity(existing.len() + 2); - let mut merged = serde_json::Map::new(); - let mut take_value = false; - for flag in existing { - if take_value { - take_value = false; - if let Ok(serde_json::Value::Object(existing_config)) = - serde_json::from_str::(&flag) - { - for (key, value) in existing_config { - merged.insert(key, value); - } - } else { - eprintln!( - "warning: existing --override-generation-config value is not valid JSON; discarding it" - ); - } - continue; - } - if flag == "--override-generation-config" { - take_value = true; - continue; - } - rewritten.push(flag); - } - for (key, value) in overrides { - merged.insert(key.clone(), value.clone()); - } - rewritten.push("--override-generation-config".to_owned()); - rewritten.push(serde_json::Value::Object(merged).to_string()); - *flags = rewritten; -} - -/// Rewrites `flags` so vLLM tool calling uses exactly `parser`: drops any existing -/// `--tool-call-parser ` pair, ensures `--enable-auto-tool-choice` is -/// present, then appends the new parser flag. -fn set_vllm_tool_call_parser(flags: &mut Vec, parser: &str) { - let existing = std::mem::take(flags); - let mut rewritten: Vec = Vec::with_capacity(existing.len() + 3); - let mut skip_value = false; - for flag in existing { - if skip_value { - // Drop the value that followed the removed `--tool-call-parser`. - skip_value = false; - continue; - } - if flag == "--tool-call-parser" { - skip_value = true; - continue; - } - rewritten.push(flag); - } - if !rewritten - .iter() - .any(|flag| flag == "--enable-auto-tool-choice") - { - rewritten.push("--enable-auto-tool-choice".to_owned()); - } - rewritten.push("--tool-call-parser".to_owned()); - rewritten.push(parser.to_owned()); - *flags = rewritten; -} - -/// Applies an explicit `--gpu-memory-utilization` to the vLLM engine recipe. -/// -/// rocm-cli intentionally ships no default for this: vLLM sizes its KV cache as -/// a fraction of the device's TOTAL VRAM, and any number rocm-cli picked would -/// silently override upstream's and drift from it. So the flag is passed through -/// only when the user asked for one, via `required_flags` (the same channel the -/// `--tool-call-parser` override uses — no protocol change needed). -/// -/// Only vLLM is affected. An explicit value wins over any recipe-authored one, -/// and a minimal hint is synthesized when none exists. -fn engine_recipe_with_gpu_memory_utilization_override( - engine: &str, - hint: Option, - gpu_memory_utilization: Option, -) -> Option { - if !engine.eq_ignore_ascii_case("vllm") { - return hint; - } - let Some(value) = gpu_memory_utilization else { - return hint; - }; - let mut hint = hint.unwrap_or_else(|| EngineRecipeHint { - contract_version: ENGINE_RECIPE_CONTRACT_VERSION.to_owned(), - engine: engine.to_owned(), - ..EngineRecipeHint::default() - }); - set_vllm_gpu_memory_utilization(&mut hint.required_flags, value); - Some(hint) -} - -/// Rewrites `flags` so vLLM receives exactly one `--gpu-memory-utilization -/// ` pair: drops any existing pair, then appends the new one. -fn set_vllm_gpu_memory_utilization(flags: &mut Vec, value: f64) { - let existing = std::mem::take(flags); - let mut rewritten: Vec = Vec::with_capacity(existing.len() + 2); - let mut skip_value = false; - for flag in existing { - if skip_value { - // Drop the value that followed the removed flag. - skip_value = false; - continue; - } - if flag == "--gpu-memory-utilization" { - skip_value = true; - continue; - } - rewritten.push(flag); - } - rewritten.push("--gpu-memory-utilization".to_owned()); - rewritten.push(format!("{value}")); - *flags = rewritten; -} - -/// Parse `rocm serve --gpu-memory-utilization`. Unlike the env-var overrides -/// elsewhere in this file, an explicit CLI value is never silently ignored: a -/// user who types a bad fraction is told so. -fn parse_gpu_memory_utilization(value: Option<&str>) -> Result> { - let Some(raw) = value else { - return Ok(None); - }; - let trimmed = raw.trim(); - let parsed: f64 = trimmed.parse().map_err(|_| { - anyhow::anyhow!( - "--gpu-memory-utilization expects a fraction greater than 0 and at most 1 \ - (e.g. 0.5); got `{trimmed}`" - ) - })?; - if !parsed.is_finite() || parsed <= 0.0 || parsed > 1.0 { - bail!( - "--gpu-memory-utilization must be greater than 0 and at most 1 (a fraction of \ - the GPU's TOTAL VRAM, e.g. 0.5); got `{trimmed}`" - ); - } - Ok(Some(parsed)) -} - -/// Env vars the engine sets itself, which `--engine-env` may not override. -const RESERVED_ENGINE_ENV_KEYS: [&str; 2] = ["HIP_VISIBLE_DEVICES", "VLLM_API_KEY"]; - -/// Applies `--engine-env` overrides to the engine recipe's `extra_env` map, for -/// the one engine that consumes it (`aim`, e.g. `AIM_PROFILE_ID`): `KEY=VALUE` -/// sets a value, and a bare `KEY` forwards `KEY` from the engine process's own -/// environment so a secret such as `HF_TOKEN` never lands in argv or the -/// service record (recorded as an empty value, meaning "forward"). Other engines -/// ignore the flag (see [`engine_env_ignored_note`]) and get their hint back -/// unchanged. A minimal hint is synthesized when none exists. -fn engine_recipe_with_extra_env( - engine: &str, - hint: Option, - engine_env: &[String], -) -> Result> { - if engine_env.is_empty() || !engine.eq_ignore_ascii_case("aim") { - return Ok(hint); - } - let mut hint = hint.unwrap_or_else(|| EngineRecipeHint { - contract_version: ENGINE_RECIPE_CONTRACT_VERSION.to_owned(), - engine: engine.to_owned(), - ..EngineRecipeHint::default() - }); - for entry in engine_env { - let (key, value) = entry.split_once('=').unwrap_or((entry.as_str(), "")); - // A bare `KEY` forwards it from the environment; the hint cannot carry an - // explicitly empty value, so `KEY=` would silently mean the same thing. - if entry.contains('=') && value.is_empty() { - bail!( - "--engine-env `{entry}` sets an empty value; pass `{key}` alone to forward it \ - from your environment, or `{key}=VALUE`" - ); - } - if key.is_empty() || !key.chars().all(|c| c.is_ascii_alphanumeric() || c == '_') { - bail!( - "--engine-env expects `KEY=VALUE` (or a bare `KEY` to forward it from the \ - environment) with a non-empty name of letters, digits and `_`; got `{entry}`" - ); - } - if RESERVED_ENGINE_ENV_KEYS.contains(&key) { - bail!("--engine-env cannot set `{key}`; rocm-cli manages it for the engine"); - } - if value.contains(['\n', '\r']) { - bail!("--engine-env value for `{key}` cannot contain newlines"); - } - hint.extra_env.insert(key.to_owned(), value.to_owned()); - } - Ok(Some(hint)) -} - -/// `--engine-env` note shown when the flag was passed but the selected engine -/// never consumes `EngineRecipeHint::extra_env` (only `aim` does; see -/// `engine_recipe_with_extra_env`), so the override would otherwise be silently -/// dropped with no indication to the user. -fn engine_env_ignored_note(engine: &str, engine_env: &[String]) -> Option { - (!engine_env.is_empty() && !engine.eq_ignore_ascii_case("aim")).then(|| { - format!("--engine-env applies only to engine 'aim'; ignored for engine '{engine}'") - }) -} - -/// Whether the resolved engine recipe launches vLLM with tool calling enabled. -fn engine_recipe_enables_tool_choice(hint: Option<&EngineRecipeHint>) -> bool { - hint.is_some_and(|hint| { - hint.required_flags - .iter() - .any(|flag| flag == "--enable-auto-tool-choice") - }) -} - /// Whether the reuse pre-gate may run its engine/runtime probe. /// /// The pre-gate exists for exactly one purpose: to answer the no-usable-GPU @@ -1718,8 +1314,6 @@ fn collect_serve_notes( #[cfg(test)] mod tests { - use std::collections::BTreeMap; - use super::*; use rocm_core::resolve_builtin_model_recipe; @@ -2361,621 +1955,6 @@ mod tests { ); } - #[test] - fn protocol_engine_recipe_hint_maps_selected_engine_metadata() { - let mut recipe = resolve_builtin_model_recipe("qwen").expect("qwen recipe"); - recipe.engine_recipes = vec![ - rocm_core::ModelRecipeEngineRecord { - engine: "vllm".to_owned(), - required_flags: vec!["--enable-auto-tool-choice".to_owned()], - parser_settings: BTreeMap::from([( - "reasoning_parser".to_owned(), - "qwen3".to_owned(), - )]), - preferred_endpoint: Some(rocm_core::ModelRecipeEndpointRecord { - endpoint_mode: "openai".to_owned(), - settings: BTreeMap::from([("streaming".to_owned(), "true".to_owned())]), - }), - unsupported_combinations: vec![ - rocm_core::ModelRecipeUnsupportedCombinationRecord { - combination: "native Windows GPU serving".to_owned(), - reason: "vLLM ROCm serving is Linux/WSL only".to_owned(), - }, - ], - notes: vec!["adapter hint".to_owned()], - model_id_override: None, - image_digest: None, - generated_at_unix_ms: None, - supported_gfx_families: Vec::new(), - profile_ids: BTreeMap::new(), - requires_hf_token: false, - supported_boards: Vec::new(), - min_gpu_counts: BTreeMap::new(), - extra_env: BTreeMap::new(), - }, - rocm_core::ModelRecipeEngineRecord { - engine: "lemonade".to_owned(), - required_flags: vec!["--reasoning-parser".to_owned(), "qwen3".to_owned()], - parser_settings: BTreeMap::new(), - preferred_endpoint: None, - unsupported_combinations: Vec::new(), - notes: Vec::new(), - model_id_override: None, - image_digest: None, - generated_at_unix_ms: None, - supported_gfx_families: Vec::new(), - profile_ids: BTreeMap::new(), - requires_hf_token: false, - supported_boards: Vec::new(), - min_gpu_counts: BTreeMap::new(), - extra_env: BTreeMap::new(), - }, - ]; - - let hint = protocol_engine_recipe_hint(&recipe, "vllm").expect("vllm hint"); - - assert_eq!(hint.contract_version, ENGINE_RECIPE_CONTRACT_VERSION); - assert_eq!(hint.engine, "vllm"); - assert_eq!( - hint.required_flags, - vec!["--enable-auto-tool-choice".to_owned()] - ); - assert_eq!( - hint.parser_settings - .get("reasoning_parser") - .map(String::as_str), - Some("qwen3") - ); - assert_eq!( - hint.preferred_endpoint - .as_ref() - .map(|endpoint| endpoint.endpoint_mode.as_str()), - Some("openai") - ); - assert_eq!( - hint.preferred_endpoint - .as_ref() - .and_then(|endpoint| endpoint.settings.get("streaming")) - .map(String::as_str), - Some("true") - ); - assert_eq!(hint.unsupported_combinations.len(), 1); - assert_eq!(hint.notes, vec!["adapter hint".to_owned()]); - let serve_lines = render_serve_engine_recipe_lines(&hint); - assert!(serve_lines.contains( - "engine_recipe_policy: selected-engine required_flags are applied at launch" - )); - assert!(serve_lines.contains("engine_recipe_required_flags: --enable-auto-tool-choice")); - assert!(protocol_engine_recipe_hint(&recipe, "unknown-engine").is_none()); - } - - #[test] - fn tool_call_override_synthesizes_hint_for_vllm_without_recipe() { - // Arbitrary HF repo with no catalog recipe: the explicit override is the - // only source of the parser, and a minimal hint is synthesized to carry it. - let hint = engine_recipe_with_tool_call_override("vllm", None, Some("hermes")) - .expect("an override should synthesize a vllm tool-choice hint"); - assert_eq!(hint.engine, "vllm"); - assert_eq!(hint.contract_version, ENGINE_RECIPE_CONTRACT_VERSION); - assert_eq!( - hint.required_flags, - vec![ - "--enable-auto-tool-choice".to_owned(), - "--tool-call-parser".to_owned(), - "hermes".to_owned(), - ] - ); - } - - #[test] - fn tool_call_override_replaces_recipe_authored_parser() { - // Override wins over an authored parser: exactly one `--tool-call-parser`, - // set to the override value, with unrelated flags preserved in order. - let existing = EngineRecipeHint { - contract_version: ENGINE_RECIPE_CONTRACT_VERSION.to_owned(), - engine: "vllm".to_owned(), - required_flags: vec![ - "--reasoning-parser".to_owned(), - "qwen3".to_owned(), - "--enable-auto-tool-choice".to_owned(), - "--tool-call-parser".to_owned(), - "llama3_json".to_owned(), - ], - ..EngineRecipeHint::default() - }; - let hint = - engine_recipe_with_tool_call_override("vllm", Some(existing), Some("hermes")).unwrap(); - assert_eq!( - hint.required_flags, - vec![ - "--reasoning-parser".to_owned(), - "qwen3".to_owned(), - "--enable-auto-tool-choice".to_owned(), - "--tool-call-parser".to_owned(), - "hermes".to_owned(), - ] - ); - assert_eq!( - hint.required_flags - .iter() - .filter(|flag| *flag == "--tool-call-parser") - .count(), - 1 - ); - } - - #[test] - fn tool_call_override_absent_preserves_recipe_flags_without_guessing() { - // No override: authored recipe metadata flows through unchanged and no - // parser is ever guessed from the model ref. - let authored = EngineRecipeHint { - contract_version: ENGINE_RECIPE_CONTRACT_VERSION.to_owned(), - engine: "vllm".to_owned(), - required_flags: vec![ - "--enable-auto-tool-choice".to_owned(), - "--tool-call-parser".to_owned(), - "hermes".to_owned(), - ], - ..EngineRecipeHint::default() - }; - let hint = - engine_recipe_with_tool_call_override("vllm", Some(authored.clone()), None).unwrap(); - assert_eq!(hint.required_flags, authored.required_flags); - - // Unknown model, no recipe, no override: nothing is injected. - assert!(engine_recipe_with_tool_call_override("vllm", None, None).is_none()); - // A blank override is treated as absent. - assert!(engine_recipe_with_tool_call_override("vllm", None, Some(" ")).is_none()); - } - - #[test] - fn tool_call_override_leaves_non_vllm_engines_untouched() { - // The override is vLLM-specific: other engines are never rewritten. - assert!(engine_recipe_with_tool_call_override("lemonade", None, Some("hermes")).is_none()); - let existing = EngineRecipeHint { - contract_version: ENGINE_RECIPE_CONTRACT_VERSION.to_owned(), - engine: "lemonade".to_owned(), - required_flags: vec!["--some-flag".to_owned()], - ..EngineRecipeHint::default() - }; - let hint = engine_recipe_with_tool_call_override( - "lemonade", - Some(existing.clone()), - Some("hermes"), - ) - .unwrap(); - assert_eq!(hint.required_flags, existing.required_flags); - } - - #[test] - fn gpu_memory_utilization_absent_without_explicit_flag() { - // rocm-cli ships no default: with nothing supplied the recipe is left - // alone, so vLLM applies its own default rather than one rocm-cli owns. - assert_eq!(parse_gpu_memory_utilization(None).unwrap(), None); - assert!(engine_recipe_with_gpu_memory_utilization_override("vllm", None, None).is_none()); - let authored = EngineRecipeHint { - contract_version: ENGINE_RECIPE_CONTRACT_VERSION.to_owned(), - engine: "vllm".to_owned(), - required_flags: vec!["--enable-auto-tool-choice".to_owned()], - ..EngineRecipeHint::default() - }; - let hint = engine_recipe_with_gpu_memory_utilization_override("vllm", Some(authored), None) - .unwrap(); - assert!( - !hint - .required_flags - .iter() - .any(|flag| flag == "--gpu-memory-utilization"), - "no default may be injected: {:?}", - hint.required_flags - ); - } - - #[test] - fn gpu_memory_utilization_override_reaches_required_flags() { - let value = parse_gpu_memory_utilization(Some("0.35")).unwrap(); - let hint = engine_recipe_with_gpu_memory_utilization_override("vllm", None, value) - .expect("an explicit value should synthesize a vllm hint"); - assert_eq!(hint.engine, "vllm"); - assert_eq!(hint.contract_version, ENGINE_RECIPE_CONTRACT_VERSION); - assert_eq!( - hint.required_flags, - vec!["--gpu-memory-utilization".to_owned(), "0.35".to_owned()] - ); - } - - #[test] - fn gpu_memory_utilization_override_replaces_authored_value_and_keeps_others() { - let existing = EngineRecipeHint { - contract_version: ENGINE_RECIPE_CONTRACT_VERSION.to_owned(), - engine: "vllm".to_owned(), - required_flags: vec![ - "--enable-auto-tool-choice".to_owned(), - "--gpu-memory-utilization".to_owned(), - "0.8".to_owned(), - "--tool-call-parser".to_owned(), - "hermes".to_owned(), - ], - ..EngineRecipeHint::default() - }; - let hint = engine_recipe_with_gpu_memory_utilization_override( - "vllm", - Some(existing), - parse_gpu_memory_utilization(Some("1.0")).unwrap(), - ) - .unwrap(); - assert_eq!( - hint.required_flags, - vec![ - "--enable-auto-tool-choice".to_owned(), - "--tool-call-parser".to_owned(), - "hermes".to_owned(), - "--gpu-memory-utilization".to_owned(), - "1".to_owned(), - ] - ); - } - - #[test] - fn gpu_memory_utilization_override_leaves_non_vllm_engines_untouched() { - // The override is vLLM-specific: other engines are never rewritten, with - // or without a recipe of their own. - assert!( - engine_recipe_with_gpu_memory_utilization_override("lemonade", None, Some(0.5)) - .is_none() - ); - let existing = EngineRecipeHint { - contract_version: ENGINE_RECIPE_CONTRACT_VERSION.to_owned(), - engine: "lemonade".to_owned(), - required_flags: vec!["--some-flag".to_owned()], - ..EngineRecipeHint::default() - }; - let hint = engine_recipe_with_gpu_memory_utilization_override( - "lemonade", - Some(existing.clone()), - Some(0.5), - ) - .unwrap(); - assert_eq!(hint.required_flags, existing.required_flags); - } - - #[test] - fn gpu_memory_utilization_rejects_out_of_range_and_unparsable_values() { - // An explicit CLI value is never silently ignored (unlike the env-var - // overrides elsewhere): each bad value must produce an actionable error. - for bad in ["0", "0.0", "1.5", "-0.2", "abc", "", "NaN", "inf"] { - let Err(error) = parse_gpu_memory_utilization(Some(bad)) else { - panic!("`{bad}` must be rejected, not silently ignored"); - }; - let message = error.to_string(); - assert!( - message.contains("--gpu-memory-utilization"), - "error for `{bad}` should name the flag: {message}" - ); - } - assert_eq!( - parse_gpu_memory_utilization(Some(" 0.5 ")).unwrap(), - Some(0.5) - ); - assert_eq!(parse_gpu_memory_utilization(Some("1")).unwrap(), Some(1.0)); - } - - #[test] - fn generation_defaults_inject_override_generation_config_for_vllm() { - // vLLM has no raw sampling flags: all three controls collapse into a single - // `--override-generation-config` JSON with `--max-tokens` mapped to the - // engine's `max_new_tokens` output cap. - let hint = engine_recipe_with_generation_defaults( - "vllm", - None, - ServeGenerationDefaults { - temperature: Some(0.5), - top_p: Some(0.25), - max_tokens: Some(128), - }, - ) - .expect("vllm defaults are supported") - .expect("supplied defaults should synthesize a vllm hint"); - assert_eq!(hint.engine, "vllm"); - assert_eq!(hint.required_flags.len(), 2); - assert_eq!(hint.required_flags[0], "--override-generation-config"); - let config: serde_json::Value = - serde_json::from_str(&hint.required_flags[1]).expect("config is valid JSON"); - assert_eq!(config["temperature"], 0.5); - assert_eq!(config["top_p"], 0.25); - assert_eq!(config["max_new_tokens"], 128); - } - - #[test] - fn generation_defaults_include_only_supplied_values() { - // Unset controls are omitted so the engine keeps its own defaults. - let hint = engine_recipe_with_generation_defaults( - "vllm", - None, - ServeGenerationDefaults { - temperature: Some(0.25), - top_p: None, - max_tokens: None, - }, - ) - .expect("vllm defaults are supported") - .expect("a single supplied default still synthesizes a hint"); - let config: serde_json::Value = - serde_json::from_str(&hint.required_flags[1]).expect("config is valid JSON"); - assert_eq!(config["temperature"], 0.25); - assert!(config.get("top_p").is_none()); - assert!(config.get("max_new_tokens").is_none()); - } - - #[test] - fn generation_defaults_merge_with_recipe_authored_config() { - // CLI values win, but authored keys the CLI does not set are preserved and - // exactly one `--override-generation-config` pair remains. - let authored = EngineRecipeHint { - contract_version: ENGINE_RECIPE_CONTRACT_VERSION.to_owned(), - engine: "vllm".to_owned(), - required_flags: vec![ - "--enable-auto-tool-choice".to_owned(), - "--override-generation-config".to_owned(), - "{\"temperature\":0.9,\"repetition_penalty\":1.1}".to_owned(), - ], - ..EngineRecipeHint::default() - }; - let hint = engine_recipe_with_generation_defaults( - "vllm", - Some(authored), - ServeGenerationDefaults { - temperature: Some(0.25), - top_p: Some(0.5), - max_tokens: None, - }, - ) - .expect("vllm defaults are supported") - .unwrap(); - assert_eq!( - hint.required_flags - .iter() - .filter(|flag| *flag == "--override-generation-config") - .count(), - 1 - ); - assert_eq!(hint.required_flags[0], "--enable-auto-tool-choice"); - let config: serde_json::Value = - serde_json::from_str(hint.required_flags.last().unwrap()).unwrap(); - assert_eq!(config["temperature"], 0.25); - assert_eq!(config["top_p"], 0.5); - assert_eq!(config["repetition_penalty"], 1.1); - } - - #[test] - fn generation_defaults_absent_or_non_vllm_pass_through() { - // No controls supplied: the hint flows through unchanged. - assert!( - engine_recipe_with_generation_defaults( - "vllm", - None, - ServeGenerationDefaults::default() - ) - .unwrap() - .is_none() - ); - assert!( - engine_recipe_with_generation_defaults( - "unknown", - None, - ServeGenerationDefaults { - temperature: Some(0.5), - top_p: Some(0.5), - max_tokens: Some(64), - }, - ) - .is_err() - ); - } - - #[test] - fn generation_defaults_reject_aim_instead_of_silently_dropping() { - // AIM's engine code only ever reads `extra_env`, never `required_flags` — a - // container launch has no argv to carry `--override-generation-config` - // into, unlike vLLM's own host process. Writing it into `required_flags` - // anyway would silently drop `--temperature`/`--top-p`/`--max-tokens` for - // `rocm serve --engine aim`, so this must fail fast instead. - let error = engine_recipe_with_generation_defaults( - "aim", - None, - ServeGenerationDefaults { - temperature: Some(0.5), - top_p: None, - max_tokens: None, - }, - ) - .unwrap_err(); - assert!(error.to_string().contains("aim")); - } - - #[test] - fn generation_defaults_translate_to_lemonade_llama_server_flags() { - let hint = engine_recipe_with_generation_defaults( - "lemonade", - None, - ServeGenerationDefaults { - temperature: Some(0.5), - top_p: Some(0.25), - max_tokens: Some(128), - }, - ) - .expect("lemonade defaults are supported") - .expect("defaults synthesize a recipe"); - assert_eq!( - hint.required_flags, - [ - "--temperature", - "0.5", - "--top-p", - "0.25", - "--n-predict", - "128" - ] - ); - } - - #[test] - fn generation_defaults_preserve_unset_lemonade_recipe_flags() { - // Only --temperature is supplied via CLI; an authored --top-p already - // present in the recipe must survive untouched, mirroring the vLLM - // merge behavior instead of being deleted. - let authored = EngineRecipeHint { - contract_version: ENGINE_RECIPE_CONTRACT_VERSION.to_owned(), - engine: "lemonade".to_owned(), - required_flags: vec!["--top-p".to_owned(), "0.9".to_owned()], - ..EngineRecipeHint::default() - }; - let hint = engine_recipe_with_generation_defaults( - "lemonade", - Some(authored), - ServeGenerationDefaults { - temperature: Some(0.5), - top_p: None, - max_tokens: None, - }, - ) - .expect("lemonade defaults are supported") - .expect("supplied defaults should synthesize a hint"); - assert_eq!( - hint.required_flags, - ["--top-p", "0.9", "--temperature", "0.5"] - ); - } - - #[test] - fn extra_env_absent_passes_hint_through_unchanged() { - assert!( - engine_recipe_with_extra_env("aim", None, &[]) - .unwrap() - .is_none() - ); - - let authored = EngineRecipeHint { - contract_version: ENGINE_RECIPE_CONTRACT_VERSION.to_owned(), - engine: "aim".to_owned(), - ..EngineRecipeHint::default() - }; - let hint = engine_recipe_with_extra_env("aim", Some(authored.clone()), &[]).unwrap(); - assert_eq!(hint, Some(authored)); - } - - #[test] - fn extra_env_synthesizes_hint_and_merges_into_authored_one() { - let hint = engine_recipe_with_extra_env("aim", None, &["AIM_PROFILE_ID=foo".to_owned()]) - .unwrap() - .expect("--engine-env should synthesize a hint"); - assert_eq!(hint.engine, "aim"); - assert_eq!( - hint.extra_env.get("AIM_PROFILE_ID"), - Some(&"foo".to_owned()) - ); - - let authored = EngineRecipeHint { - contract_version: ENGINE_RECIPE_CONTRACT_VERSION.to_owned(), - engine: "aim".to_owned(), - extra_env: BTreeMap::from([("EXISTING".to_owned(), "kept".to_owned())]), - ..EngineRecipeHint::default() - }; - let hint = - engine_recipe_with_extra_env("aim", Some(authored), &["AIM_PROFILE_ID=foo".to_owned()]) - .unwrap() - .unwrap(); - assert_eq!(hint.extra_env.get("EXISTING"), Some(&"kept".to_owned())); - assert_eq!( - hint.extra_env.get("AIM_PROFILE_ID"), - Some(&"foo".to_owned()) - ); - } - - #[test] - fn extra_env_bare_key_forwards_from_the_environment_without_recording_a_value() { - let hint = engine_recipe_with_extra_env("aim", None, &["HF_TOKEN".to_owned()]) - .unwrap() - .unwrap(); - assert_eq!(hint.extra_env.get("HF_TOKEN"), Some(&String::new())); - } - - #[test] - fn extra_env_rejects_entry_with_empty_or_malformed_key() { - for entry in ["=some-value", "", "BAD KEY=1", "A-B=1"] { - let error = engine_recipe_with_extra_env("aim", None, &[entry.to_owned()]).unwrap_err(); - assert!(error.to_string().contains("KEY=VALUE"), "{entry}: {error}"); - } - } - - #[test] - fn extra_env_rejects_reserved_keys_and_newlines() { - for entry in ["VLLM_API_KEY=x", "HIP_VISIBLE_DEVICES=0", "VLLM_API_KEY"] { - let error = engine_recipe_with_extra_env("aim", None, &[entry.to_owned()]).unwrap_err(); - assert!(error.to_string().contains("manages it"), "{entry}: {error}"); - } - assert!(engine_recipe_with_extra_env("aim", None, &["A=b\nc".to_owned()]).is_err()); - } - - #[test] - fn extra_env_leaves_the_hint_alone_for_engines_that_ignore_it() { - assert_eq!( - engine_recipe_with_extra_env("vllm", None, &["FOO=bar".to_owned()]).unwrap(), - None, - "a flag reported as ignored must not change what the engine is sent" - ); - } - - #[test] - fn engine_env_distinguishes_a_bare_key_from_an_empty_value() { - let env = |entry: &str| engine_recipe_with_extra_env("aim", None, &[entry.to_owned()]); - let bare = env("HF_TOKEN").unwrap().expect("hint"); - assert_eq!(bare.extra_env.get("HF_TOKEN").map(String::as_str), Some("")); - let err = env("HF_TOKEN=").unwrap_err().to_string(); - assert!( - err.contains("empty value") && err.contains("`HF_TOKEN` alone"), - "{err}" - ); - let set = env("A=b").unwrap().expect("hint"); - assert_eq!(set.extra_env.get("A").map(String::as_str), Some("b")); - } - - #[test] - fn engine_env_ignored_note_is_none_for_aim() { - assert_eq!( - engine_env_ignored_note("aim", &["AIM_PROFILE_ID=foo".to_owned()]), - None - ); - } - - #[test] - fn engine_env_ignored_note_is_none_when_engine_env_is_empty() { - assert_eq!(engine_env_ignored_note("vllm", &[]), None); - } - - #[test] - fn engine_env_ignored_note_warns_for_engines_that_do_not_consume_it() { - let note = engine_env_ignored_note("vllm", &["FOO=bar".to_owned()]) - .expect("vllm does not consume extra_env"); - assert!(note.contains("--engine-env")); - assert!(note.contains("'aim'")); - assert!(note.contains("'vllm'")); - } - - #[test] - fn engine_recipe_enables_tool_choice_reflects_flags() { - assert!(!engine_recipe_enables_tool_choice(None)); - let without = EngineRecipeHint { - contract_version: ENGINE_RECIPE_CONTRACT_VERSION.to_owned(), - engine: "vllm".to_owned(), - required_flags: vec!["--reasoning-parser".to_owned(), "qwen3".to_owned()], - ..EngineRecipeHint::default() - }; - assert!(!engine_recipe_enables_tool_choice(Some(&without))); - let with = engine_recipe_with_tool_call_override("vllm", None, Some("hermes")); - assert!(engine_recipe_enables_tool_choice(with.as_ref())); - } - #[test] fn serve_notes_surface_the_ignored_engine_flag_in_summary_mode() { // The interactive summary is what a default `rocm serve` prints, so a flag diff --git a/apps/rocm/src/serve_recipe.rs b/apps/rocm/src/serve_recipe.rs new file mode 100644 index 0000000000..94976cbe2f --- /dev/null +++ b/apps/rocm/src/serve_recipe.rs @@ -0,0 +1,1051 @@ +// Copyright © Advanced Micro Devices, Inc., or its affiliates. +// +// SPDX-License-Identifier: MIT + +//! Engine-recipe overrides for `rocm serve`. +//! +//! Relocated from `serve_cmd.rs` with no behavior change. These are the pure +//! transforms `serve()` applies, in order, to the selected engine's launch +//! recipe hint: mapping the catalog's engine record onto the protocol hint, +//! then layering the explicit `--tool-call-parser`, `--gpu-memory-utilization`, +//! `--temperature`/`--top-p`/`--max-tokens` and `--engine-env` controls on top. +//! `serve()` is the only production caller; nothing here calls back into it. + +use std::fmt::Write as _; + +use anyhow::{Result, bail}; +use rocm_core::ModelRecipeRecord; +use rocm_engine_protocol::{ + ENGINE_RECIPE_CONTRACT_VERSION, EngineRecipeEndpointHint, EngineRecipeHint, + EngineRecipeUnsupportedCombinationHint, +}; + +pub(crate) fn render_serve_engine_recipe_lines(engine_recipe: &EngineRecipeHint) -> String { + let mut output = String::new(); + let _ = writeln!( + output, + " engine_recipe_contract: {}", + engine_recipe.contract_version + ); + let _ = writeln!( + output, + " engine_recipe_policy: selected-engine required_flags are applied at launch; parser/endpoint metadata is forwarded to the adapter" + ); + let _ = writeln!(output, " engine_recipe_engine: {}", engine_recipe.engine); + if !engine_recipe.required_flags.is_empty() { + let _ = writeln!( + output, + " engine_recipe_required_flags: {}", + engine_recipe.required_flags.join(" ") + ); + } + output +} + +pub(crate) fn protocol_engine_recipe_hint( + recipe: &ModelRecipeRecord, + engine: &str, +) -> Option { + recipe + .engine_recipes + .iter() + .find(|engine_recipe| engine_recipe.engine == engine) + .map(|engine_recipe| EngineRecipeHint { + contract_version: ENGINE_RECIPE_CONTRACT_VERSION.to_owned(), + engine: engine_recipe.engine.clone(), + required_flags: engine_recipe.required_flags.clone(), + parser_settings: engine_recipe.parser_settings.clone(), + extra_env: engine_recipe.extra_env.clone(), + preferred_endpoint: engine_recipe.preferred_endpoint.as_ref().map(|endpoint| { + EngineRecipeEndpointHint { + endpoint_mode: endpoint.endpoint_mode.clone(), + settings: endpoint.settings.clone(), + } + }), + unsupported_combinations: engine_recipe + .unsupported_combinations + .iter() + .map(|combination| EngineRecipeUnsupportedCombinationHint { + combination: combination.combination.clone(), + reason: combination.reason.clone(), + }) + .collect(), + notes: engine_recipe.notes.clone(), + }) +} + +/// Applies an explicit `--tool-call-parser` override to a vLLM engine recipe hint. +/// +/// The TUI chat tab always attaches tool definitions to non-streaming chat +/// requests (`tool_choice: "auto"`). vLLM rejects those with HTTP 400 unless it was +/// started with `--enable-auto-tool-choice` *and* a matching `--tool-call-parser`. +/// The correct parser is model-specific and vLLM does not auto-detect it, so it is +/// never guessed from the model ref: it comes either from authored catalog recipe +/// metadata (already carried in `required_flags`) or from the explicit +/// `--tool-call-parser` serve flag, which this applies. +/// +/// Only vLLM is affected. When an override is supplied it wins over any +/// recipe-authored parser (a single `--tool-call-parser`, no duplication) and a +/// minimal hint is synthesized when none exists (arbitrary HF repos, or a catalog +/// model forced onto a non-preferred engine). With no override the hint passes +/// through unchanged. +pub(crate) fn engine_recipe_with_tool_call_override( + engine: &str, + hint: Option, + tool_call_parser: Option<&str>, +) -> Option { + if !engine.eq_ignore_ascii_case("vllm") { + return hint; + } + let Some(parser) = tool_call_parser + .map(str::trim) + .filter(|value| !value.is_empty()) + else { + return hint; + }; + let mut hint = hint.unwrap_or_else(|| EngineRecipeHint { + contract_version: ENGINE_RECIPE_CONTRACT_VERSION.to_owned(), + engine: engine.to_owned(), + ..EngineRecipeHint::default() + }); + set_vllm_tool_call_parser(&mut hint.required_flags, parser); + Some(hint) +} + +/// Sampling defaults a `rocm serve` invocation can push into a vLLM launch. +#[derive(Debug, Clone, Copy, Default, PartialEq)] +pub(crate) struct ServeGenerationDefaults { + pub(crate) temperature: Option, + pub(crate) top_p: Option, + pub(crate) max_tokens: Option, +} + +impl ServeGenerationDefaults { + const fn is_empty(&self) -> bool { + self.temperature.is_none() && self.top_p.is_none() && self.max_tokens.is_none() + } +} + +/// Applies `rocm serve` generation defaults (`--temperature`/`--top-p`/`--max-tokens`) +/// to the selected engine's launch recipe. +/// +/// vLLM `serve` has no raw `--temperature`/`--top-p` flags; `--override-generation-config` +/// is the supported way to set server-wide sampling defaults, so `--max-tokens` is +/// mapped onto vLLM's `max_new_tokens` output cap. Only supplied values are written, +/// and any values already carried by an authored recipe's +/// `--override-generation-config` are preserved (the CLI-supplied keys win). +/// +/// Lemonade's llama.cpp backend accepts the equivalent `--temperature`, `--top-p`, +/// and `--n-predict` launch flags. A minimal hint is synthesized when none exists. +/// With no defaults supplied the hint passes through unchanged. +pub(crate) fn engine_recipe_with_generation_defaults( + engine: &str, + hint: Option, + defaults: ServeGenerationDefaults, +) -> Result> { + if defaults.is_empty() { + return Ok(hint); + } + let mut hint = hint.unwrap_or_else(|| EngineRecipeHint { + contract_version: ENGINE_RECIPE_CONTRACT_VERSION.to_owned(), + engine: engine.to_owned(), + ..EngineRecipeHint::default() + }); + if engine.eq_ignore_ascii_case("vllm") { + let mut overrides = serde_json::Map::new(); + if let Some(temperature) = defaults.temperature { + overrides.insert("temperature".to_owned(), serde_json::json!(temperature)); + } + if let Some(top_p) = defaults.top_p { + overrides.insert("top_p".to_owned(), serde_json::json!(top_p)); + } + if let Some(max_tokens) = defaults.max_tokens { + overrides.insert("max_new_tokens".to_owned(), serde_json::json!(max_tokens)); + } + set_vllm_override_generation_config(&mut hint.required_flags, &overrides); + } else if engine.eq_ignore_ascii_case("lemonade") { + set_lemonade_generation_defaults(&mut hint.required_flags, defaults); + } else { + bail!( + "generation defaults are not supported by engine `{engine}`; omit --temperature/--top-p/--max-tokens or select vllm/lemonade" + ); + } + Ok(Some(hint)) +} + +fn set_lemonade_generation_defaults(flags: &mut Vec, defaults: ServeGenerationDefaults) { + // Only touch a flag pair when the caller actually supplied that control — + // an unset field must leave any authored recipe value in place rather than + // deleting it, mirroring the vLLM merge semantics in + // `set_vllm_override_generation_config`. + for (name, value) in [ + ( + "--temperature", + defaults.temperature.map(|value| value.to_string()), + ), + ("--top-p", defaults.top_p.map(|value| value.to_string())), + ( + "--n-predict", + defaults.max_tokens.map(|value| value.to_string()), + ), + ] { + let Some(value) = value else { + continue; + }; + let mut rewritten = Vec::with_capacity(flags.len() + 2); + let mut skip_value = false; + for flag in std::mem::take(flags) { + if skip_value { + skip_value = false; + continue; + } + if flag == name { + skip_value = true; + } else { + rewritten.push(flag); + } + } + rewritten.extend([name.to_owned(), value]); + *flags = rewritten; + } +} + +/// Rewrites `flags` so vLLM's `--override-generation-config` carries exactly one +/// merged JSON object: any existing `--override-generation-config ` pair is +/// removed, its keys are used as a base, and `overrides` are layered on top (CLI +/// values win). Emits a single flag pair with the merged, stably-ordered config. +fn set_vllm_override_generation_config( + flags: &mut Vec, + overrides: &serde_json::Map, +) { + let existing = std::mem::take(flags); + let mut rewritten: Vec = Vec::with_capacity(existing.len() + 2); + let mut merged = serde_json::Map::new(); + let mut take_value = false; + for flag in existing { + if take_value { + take_value = false; + if let Ok(serde_json::Value::Object(existing_config)) = + serde_json::from_str::(&flag) + { + for (key, value) in existing_config { + merged.insert(key, value); + } + } else { + eprintln!( + "warning: existing --override-generation-config value is not valid JSON; discarding it" + ); + } + continue; + } + if flag == "--override-generation-config" { + take_value = true; + continue; + } + rewritten.push(flag); + } + for (key, value) in overrides { + merged.insert(key.clone(), value.clone()); + } + rewritten.push("--override-generation-config".to_owned()); + rewritten.push(serde_json::Value::Object(merged).to_string()); + *flags = rewritten; +} + +/// Rewrites `flags` so vLLM tool calling uses exactly `parser`: drops any existing +/// `--tool-call-parser ` pair, ensures `--enable-auto-tool-choice` is +/// present, then appends the new parser flag. +fn set_vllm_tool_call_parser(flags: &mut Vec, parser: &str) { + let existing = std::mem::take(flags); + let mut rewritten: Vec = Vec::with_capacity(existing.len() + 3); + let mut skip_value = false; + for flag in existing { + if skip_value { + // Drop the value that followed the removed `--tool-call-parser`. + skip_value = false; + continue; + } + if flag == "--tool-call-parser" { + skip_value = true; + continue; + } + rewritten.push(flag); + } + if !rewritten + .iter() + .any(|flag| flag == "--enable-auto-tool-choice") + { + rewritten.push("--enable-auto-tool-choice".to_owned()); + } + rewritten.push("--tool-call-parser".to_owned()); + rewritten.push(parser.to_owned()); + *flags = rewritten; +} + +/// Applies an explicit `--gpu-memory-utilization` to the vLLM engine recipe. +/// +/// rocm-cli intentionally ships no default for this: vLLM sizes its KV cache as +/// a fraction of the device's TOTAL VRAM, and any number rocm-cli picked would +/// silently override upstream's and drift from it. So the flag is passed through +/// only when the user asked for one, via `required_flags` (the same channel the +/// `--tool-call-parser` override uses — no protocol change needed). +/// +/// Only vLLM is affected. An explicit value wins over any recipe-authored one, +/// and a minimal hint is synthesized when none exists. +pub(crate) fn engine_recipe_with_gpu_memory_utilization_override( + engine: &str, + hint: Option, + gpu_memory_utilization: Option, +) -> Option { + if !engine.eq_ignore_ascii_case("vllm") { + return hint; + } + let Some(value) = gpu_memory_utilization else { + return hint; + }; + let mut hint = hint.unwrap_or_else(|| EngineRecipeHint { + contract_version: ENGINE_RECIPE_CONTRACT_VERSION.to_owned(), + engine: engine.to_owned(), + ..EngineRecipeHint::default() + }); + set_vllm_gpu_memory_utilization(&mut hint.required_flags, value); + Some(hint) +} + +/// Rewrites `flags` so vLLM receives exactly one `--gpu-memory-utilization +/// ` pair: drops any existing pair, then appends the new one. +fn set_vllm_gpu_memory_utilization(flags: &mut Vec, value: f64) { + let existing = std::mem::take(flags); + let mut rewritten: Vec = Vec::with_capacity(existing.len() + 2); + let mut skip_value = false; + for flag in existing { + if skip_value { + // Drop the value that followed the removed flag. + skip_value = false; + continue; + } + if flag == "--gpu-memory-utilization" { + skip_value = true; + continue; + } + rewritten.push(flag); + } + rewritten.push("--gpu-memory-utilization".to_owned()); + rewritten.push(format!("{value}")); + *flags = rewritten; +} + +/// Parse `rocm serve --gpu-memory-utilization`. Unlike the env-var overrides +/// elsewhere in this file, an explicit CLI value is never silently ignored: a +/// user who types a bad fraction is told so. +pub(crate) fn parse_gpu_memory_utilization(value: Option<&str>) -> Result> { + let Some(raw) = value else { + return Ok(None); + }; + let trimmed = raw.trim(); + let parsed: f64 = trimmed.parse().map_err(|_| { + anyhow::anyhow!( + "--gpu-memory-utilization expects a fraction greater than 0 and at most 1 \ + (e.g. 0.5); got `{trimmed}`" + ) + })?; + if !parsed.is_finite() || parsed <= 0.0 || parsed > 1.0 { + bail!( + "--gpu-memory-utilization must be greater than 0 and at most 1 (a fraction of \ + the GPU's TOTAL VRAM, e.g. 0.5); got `{trimmed}`" + ); + } + Ok(Some(parsed)) +} + +/// Env vars the engine sets itself, which `--engine-env` may not override. +const RESERVED_ENGINE_ENV_KEYS: [&str; 2] = ["HIP_VISIBLE_DEVICES", "VLLM_API_KEY"]; + +/// Applies `--engine-env` overrides to the engine recipe's `extra_env` map, for +/// the one engine that consumes it (`aim`, e.g. `AIM_PROFILE_ID`): `KEY=VALUE` +/// sets a value, and a bare `KEY` forwards `KEY` from the engine process's own +/// environment so a secret such as `HF_TOKEN` never lands in argv or the +/// service record (recorded as an empty value, meaning "forward"). Other engines +/// ignore the flag (see [`engine_env_ignored_note`]) and get their hint back +/// unchanged. A minimal hint is synthesized when none exists. +pub(crate) fn engine_recipe_with_extra_env( + engine: &str, + hint: Option, + engine_env: &[String], +) -> Result> { + if engine_env.is_empty() || !engine.eq_ignore_ascii_case("aim") { + return Ok(hint); + } + let mut hint = hint.unwrap_or_else(|| EngineRecipeHint { + contract_version: ENGINE_RECIPE_CONTRACT_VERSION.to_owned(), + engine: engine.to_owned(), + ..EngineRecipeHint::default() + }); + for entry in engine_env { + let (key, value) = entry.split_once('=').unwrap_or((entry.as_str(), "")); + // A bare `KEY` forwards it from the environment; the hint cannot carry an + // explicitly empty value, so `KEY=` would silently mean the same thing. + if entry.contains('=') && value.is_empty() { + bail!( + "--engine-env `{entry}` sets an empty value; pass `{key}` alone to forward it \ + from your environment, or `{key}=VALUE`" + ); + } + if key.is_empty() || !key.chars().all(|c| c.is_ascii_alphanumeric() || c == '_') { + bail!( + "--engine-env expects `KEY=VALUE` (or a bare `KEY` to forward it from the \ + environment) with a non-empty name of letters, digits and `_`; got `{entry}`" + ); + } + if RESERVED_ENGINE_ENV_KEYS.contains(&key) { + bail!("--engine-env cannot set `{key}`; rocm-cli manages it for the engine"); + } + if value.contains(['\n', '\r']) { + bail!("--engine-env value for `{key}` cannot contain newlines"); + } + hint.extra_env.insert(key.to_owned(), value.to_owned()); + } + Ok(Some(hint)) +} + +/// `--engine-env` note shown when the flag was passed but the selected engine +/// never consumes `EngineRecipeHint::extra_env` (only `aim` does; see +/// `engine_recipe_with_extra_env`), so the override would otherwise be silently +/// dropped with no indication to the user. +pub(crate) fn engine_env_ignored_note(engine: &str, engine_env: &[String]) -> Option { + (!engine_env.is_empty() && !engine.eq_ignore_ascii_case("aim")).then(|| { + format!("--engine-env applies only to engine 'aim'; ignored for engine '{engine}'") + }) +} + +/// Whether the resolved engine recipe launches vLLM with tool calling enabled. +pub(crate) fn engine_recipe_enables_tool_choice(hint: Option<&EngineRecipeHint>) -> bool { + hint.is_some_and(|hint| { + hint.required_flags + .iter() + .any(|flag| flag == "--enable-auto-tool-choice") + }) +} + +#[cfg(test)] +mod tests { + use std::collections::BTreeMap; + + use super::*; + use rocm_core::resolve_builtin_model_recipe; + + #[test] + fn protocol_engine_recipe_hint_maps_selected_engine_metadata() { + let mut recipe = resolve_builtin_model_recipe("qwen").expect("qwen recipe"); + recipe.engine_recipes = vec![ + rocm_core::ModelRecipeEngineRecord { + engine: "vllm".to_owned(), + required_flags: vec!["--enable-auto-tool-choice".to_owned()], + parser_settings: BTreeMap::from([( + "reasoning_parser".to_owned(), + "qwen3".to_owned(), + )]), + preferred_endpoint: Some(rocm_core::ModelRecipeEndpointRecord { + endpoint_mode: "openai".to_owned(), + settings: BTreeMap::from([("streaming".to_owned(), "true".to_owned())]), + }), + unsupported_combinations: vec![ + rocm_core::ModelRecipeUnsupportedCombinationRecord { + combination: "native Windows GPU serving".to_owned(), + reason: "vLLM ROCm serving is Linux/WSL only".to_owned(), + }, + ], + notes: vec!["adapter hint".to_owned()], + model_id_override: None, + image_digest: None, + generated_at_unix_ms: None, + supported_gfx_families: Vec::new(), + profile_ids: BTreeMap::new(), + requires_hf_token: false, + supported_boards: Vec::new(), + min_gpu_counts: BTreeMap::new(), + extra_env: BTreeMap::new(), + }, + rocm_core::ModelRecipeEngineRecord { + engine: "lemonade".to_owned(), + required_flags: vec!["--reasoning-parser".to_owned(), "qwen3".to_owned()], + parser_settings: BTreeMap::new(), + preferred_endpoint: None, + unsupported_combinations: Vec::new(), + notes: Vec::new(), + model_id_override: None, + image_digest: None, + generated_at_unix_ms: None, + supported_gfx_families: Vec::new(), + profile_ids: BTreeMap::new(), + requires_hf_token: false, + supported_boards: Vec::new(), + min_gpu_counts: BTreeMap::new(), + extra_env: BTreeMap::new(), + }, + ]; + + let hint = protocol_engine_recipe_hint(&recipe, "vllm").expect("vllm hint"); + + assert_eq!(hint.contract_version, ENGINE_RECIPE_CONTRACT_VERSION); + assert_eq!(hint.engine, "vllm"); + assert_eq!( + hint.required_flags, + vec!["--enable-auto-tool-choice".to_owned()] + ); + assert_eq!( + hint.parser_settings + .get("reasoning_parser") + .map(String::as_str), + Some("qwen3") + ); + assert_eq!( + hint.preferred_endpoint + .as_ref() + .map(|endpoint| endpoint.endpoint_mode.as_str()), + Some("openai") + ); + assert_eq!( + hint.preferred_endpoint + .as_ref() + .and_then(|endpoint| endpoint.settings.get("streaming")) + .map(String::as_str), + Some("true") + ); + assert_eq!(hint.unsupported_combinations.len(), 1); + assert_eq!(hint.notes, vec!["adapter hint".to_owned()]); + let serve_lines = render_serve_engine_recipe_lines(&hint); + assert!(serve_lines.contains( + "engine_recipe_policy: selected-engine required_flags are applied at launch" + )); + assert!(serve_lines.contains("engine_recipe_required_flags: --enable-auto-tool-choice")); + assert!(protocol_engine_recipe_hint(&recipe, "unknown-engine").is_none()); + } + + #[test] + fn tool_call_override_synthesizes_hint_for_vllm_without_recipe() { + // Arbitrary HF repo with no catalog recipe: the explicit override is the + // only source of the parser, and a minimal hint is synthesized to carry it. + let hint = engine_recipe_with_tool_call_override("vllm", None, Some("hermes")) + .expect("an override should synthesize a vllm tool-choice hint"); + assert_eq!(hint.engine, "vllm"); + assert_eq!(hint.contract_version, ENGINE_RECIPE_CONTRACT_VERSION); + assert_eq!( + hint.required_flags, + vec![ + "--enable-auto-tool-choice".to_owned(), + "--tool-call-parser".to_owned(), + "hermes".to_owned(), + ] + ); + } + + #[test] + fn tool_call_override_replaces_recipe_authored_parser() { + // Override wins over an authored parser: exactly one `--tool-call-parser`, + // set to the override value, with unrelated flags preserved in order. + let existing = EngineRecipeHint { + contract_version: ENGINE_RECIPE_CONTRACT_VERSION.to_owned(), + engine: "vllm".to_owned(), + required_flags: vec![ + "--reasoning-parser".to_owned(), + "qwen3".to_owned(), + "--enable-auto-tool-choice".to_owned(), + "--tool-call-parser".to_owned(), + "llama3_json".to_owned(), + ], + ..EngineRecipeHint::default() + }; + let hint = + engine_recipe_with_tool_call_override("vllm", Some(existing), Some("hermes")).unwrap(); + assert_eq!( + hint.required_flags, + vec![ + "--reasoning-parser".to_owned(), + "qwen3".to_owned(), + "--enable-auto-tool-choice".to_owned(), + "--tool-call-parser".to_owned(), + "hermes".to_owned(), + ] + ); + assert_eq!( + hint.required_flags + .iter() + .filter(|flag| *flag == "--tool-call-parser") + .count(), + 1 + ); + } + + #[test] + fn tool_call_override_absent_preserves_recipe_flags_without_guessing() { + // No override: authored recipe metadata flows through unchanged and no + // parser is ever guessed from the model ref. + let authored = EngineRecipeHint { + contract_version: ENGINE_RECIPE_CONTRACT_VERSION.to_owned(), + engine: "vllm".to_owned(), + required_flags: vec![ + "--enable-auto-tool-choice".to_owned(), + "--tool-call-parser".to_owned(), + "hermes".to_owned(), + ], + ..EngineRecipeHint::default() + }; + let hint = + engine_recipe_with_tool_call_override("vllm", Some(authored.clone()), None).unwrap(); + assert_eq!(hint.required_flags, authored.required_flags); + + // Unknown model, no recipe, no override: nothing is injected. + assert!(engine_recipe_with_tool_call_override("vllm", None, None).is_none()); + // A blank override is treated as absent. + assert!(engine_recipe_with_tool_call_override("vllm", None, Some(" ")).is_none()); + } + + #[test] + fn tool_call_override_leaves_non_vllm_engines_untouched() { + // The override is vLLM-specific: other engines are never rewritten. + assert!(engine_recipe_with_tool_call_override("lemonade", None, Some("hermes")).is_none()); + let existing = EngineRecipeHint { + contract_version: ENGINE_RECIPE_CONTRACT_VERSION.to_owned(), + engine: "lemonade".to_owned(), + required_flags: vec!["--some-flag".to_owned()], + ..EngineRecipeHint::default() + }; + let hint = engine_recipe_with_tool_call_override( + "lemonade", + Some(existing.clone()), + Some("hermes"), + ) + .unwrap(); + assert_eq!(hint.required_flags, existing.required_flags); + } + + #[test] + fn gpu_memory_utilization_absent_without_explicit_flag() { + // rocm-cli ships no default: with nothing supplied the recipe is left + // alone, so vLLM applies its own default rather than one rocm-cli owns. + assert_eq!(parse_gpu_memory_utilization(None).unwrap(), None); + assert!(engine_recipe_with_gpu_memory_utilization_override("vllm", None, None).is_none()); + let authored = EngineRecipeHint { + contract_version: ENGINE_RECIPE_CONTRACT_VERSION.to_owned(), + engine: "vllm".to_owned(), + required_flags: vec!["--enable-auto-tool-choice".to_owned()], + ..EngineRecipeHint::default() + }; + let hint = engine_recipe_with_gpu_memory_utilization_override("vllm", Some(authored), None) + .unwrap(); + assert!( + !hint + .required_flags + .iter() + .any(|flag| flag == "--gpu-memory-utilization"), + "no default may be injected: {:?}", + hint.required_flags + ); + } + + #[test] + fn gpu_memory_utilization_override_reaches_required_flags() { + let value = parse_gpu_memory_utilization(Some("0.35")).unwrap(); + let hint = engine_recipe_with_gpu_memory_utilization_override("vllm", None, value) + .expect("an explicit value should synthesize a vllm hint"); + assert_eq!(hint.engine, "vllm"); + assert_eq!(hint.contract_version, ENGINE_RECIPE_CONTRACT_VERSION); + assert_eq!( + hint.required_flags, + vec!["--gpu-memory-utilization".to_owned(), "0.35".to_owned()] + ); + } + + #[test] + fn gpu_memory_utilization_override_replaces_authored_value_and_keeps_others() { + let existing = EngineRecipeHint { + contract_version: ENGINE_RECIPE_CONTRACT_VERSION.to_owned(), + engine: "vllm".to_owned(), + required_flags: vec![ + "--enable-auto-tool-choice".to_owned(), + "--gpu-memory-utilization".to_owned(), + "0.8".to_owned(), + "--tool-call-parser".to_owned(), + "hermes".to_owned(), + ], + ..EngineRecipeHint::default() + }; + let hint = engine_recipe_with_gpu_memory_utilization_override( + "vllm", + Some(existing), + parse_gpu_memory_utilization(Some("1.0")).unwrap(), + ) + .unwrap(); + assert_eq!( + hint.required_flags, + vec![ + "--enable-auto-tool-choice".to_owned(), + "--tool-call-parser".to_owned(), + "hermes".to_owned(), + "--gpu-memory-utilization".to_owned(), + "1".to_owned(), + ] + ); + } + + #[test] + fn gpu_memory_utilization_override_leaves_non_vllm_engines_untouched() { + // The override is vLLM-specific: other engines are never rewritten, with + // or without a recipe of their own. + assert!( + engine_recipe_with_gpu_memory_utilization_override("lemonade", None, Some(0.5)) + .is_none() + ); + let existing = EngineRecipeHint { + contract_version: ENGINE_RECIPE_CONTRACT_VERSION.to_owned(), + engine: "lemonade".to_owned(), + required_flags: vec!["--some-flag".to_owned()], + ..EngineRecipeHint::default() + }; + let hint = engine_recipe_with_gpu_memory_utilization_override( + "lemonade", + Some(existing.clone()), + Some(0.5), + ) + .unwrap(); + assert_eq!(hint.required_flags, existing.required_flags); + } + + #[test] + fn gpu_memory_utilization_rejects_out_of_range_and_unparsable_values() { + // An explicit CLI value is never silently ignored (unlike the env-var + // overrides elsewhere): each bad value must produce an actionable error. + for bad in ["0", "0.0", "1.5", "-0.2", "abc", "", "NaN", "inf"] { + let Err(error) = parse_gpu_memory_utilization(Some(bad)) else { + panic!("`{bad}` must be rejected, not silently ignored"); + }; + let message = error.to_string(); + assert!( + message.contains("--gpu-memory-utilization"), + "error for `{bad}` should name the flag: {message}" + ); + } + assert_eq!( + parse_gpu_memory_utilization(Some(" 0.5 ")).unwrap(), + Some(0.5) + ); + assert_eq!(parse_gpu_memory_utilization(Some("1")).unwrap(), Some(1.0)); + } + + #[test] + fn generation_defaults_inject_override_generation_config_for_vllm() { + // vLLM has no raw sampling flags: all three controls collapse into a single + // `--override-generation-config` JSON with `--max-tokens` mapped to the + // engine's `max_new_tokens` output cap. + let hint = engine_recipe_with_generation_defaults( + "vllm", + None, + ServeGenerationDefaults { + temperature: Some(0.5), + top_p: Some(0.25), + max_tokens: Some(128), + }, + ) + .expect("vllm defaults are supported") + .expect("supplied defaults should synthesize a vllm hint"); + assert_eq!(hint.engine, "vllm"); + assert_eq!(hint.required_flags.len(), 2); + assert_eq!(hint.required_flags[0], "--override-generation-config"); + let config: serde_json::Value = + serde_json::from_str(&hint.required_flags[1]).expect("config is valid JSON"); + assert_eq!(config["temperature"], 0.5); + assert_eq!(config["top_p"], 0.25); + assert_eq!(config["max_new_tokens"], 128); + } + + #[test] + fn generation_defaults_include_only_supplied_values() { + // Unset controls are omitted so the engine keeps its own defaults. + let hint = engine_recipe_with_generation_defaults( + "vllm", + None, + ServeGenerationDefaults { + temperature: Some(0.25), + top_p: None, + max_tokens: None, + }, + ) + .expect("vllm defaults are supported") + .expect("a single supplied default still synthesizes a hint"); + let config: serde_json::Value = + serde_json::from_str(&hint.required_flags[1]).expect("config is valid JSON"); + assert_eq!(config["temperature"], 0.25); + assert!(config.get("top_p").is_none()); + assert!(config.get("max_new_tokens").is_none()); + } + + #[test] + fn generation_defaults_merge_with_recipe_authored_config() { + // CLI values win, but authored keys the CLI does not set are preserved and + // exactly one `--override-generation-config` pair remains. + let authored = EngineRecipeHint { + contract_version: ENGINE_RECIPE_CONTRACT_VERSION.to_owned(), + engine: "vllm".to_owned(), + required_flags: vec![ + "--enable-auto-tool-choice".to_owned(), + "--override-generation-config".to_owned(), + "{\"temperature\":0.9,\"repetition_penalty\":1.1}".to_owned(), + ], + ..EngineRecipeHint::default() + }; + let hint = engine_recipe_with_generation_defaults( + "vllm", + Some(authored), + ServeGenerationDefaults { + temperature: Some(0.25), + top_p: Some(0.5), + max_tokens: None, + }, + ) + .expect("vllm defaults are supported") + .unwrap(); + assert_eq!( + hint.required_flags + .iter() + .filter(|flag| *flag == "--override-generation-config") + .count(), + 1 + ); + assert_eq!(hint.required_flags[0], "--enable-auto-tool-choice"); + let config: serde_json::Value = + serde_json::from_str(hint.required_flags.last().unwrap()).unwrap(); + assert_eq!(config["temperature"], 0.25); + assert_eq!(config["top_p"], 0.5); + assert_eq!(config["repetition_penalty"], 1.1); + } + + #[test] + fn generation_defaults_absent_or_non_vllm_pass_through() { + // No controls supplied: the hint flows through unchanged. + assert!( + engine_recipe_with_generation_defaults( + "vllm", + None, + ServeGenerationDefaults::default() + ) + .unwrap() + .is_none() + ); + assert!( + engine_recipe_with_generation_defaults( + "unknown", + None, + ServeGenerationDefaults { + temperature: Some(0.5), + top_p: Some(0.5), + max_tokens: Some(64), + }, + ) + .is_err() + ); + } + + #[test] + fn generation_defaults_reject_aim_instead_of_silently_dropping() { + // AIM's engine code only ever reads `extra_env`, never `required_flags` — a + // container launch has no argv to carry `--override-generation-config` + // into, unlike vLLM's own host process. Writing it into `required_flags` + // anyway would silently drop `--temperature`/`--top-p`/`--max-tokens` for + // `rocm serve --engine aim`, so this must fail fast instead. + let error = engine_recipe_with_generation_defaults( + "aim", + None, + ServeGenerationDefaults { + temperature: Some(0.5), + top_p: None, + max_tokens: None, + }, + ) + .unwrap_err(); + assert!(error.to_string().contains("aim")); + } + + #[test] + fn generation_defaults_translate_to_lemonade_llama_server_flags() { + let hint = engine_recipe_with_generation_defaults( + "lemonade", + None, + ServeGenerationDefaults { + temperature: Some(0.5), + top_p: Some(0.25), + max_tokens: Some(128), + }, + ) + .expect("lemonade defaults are supported") + .expect("defaults synthesize a recipe"); + assert_eq!( + hint.required_flags, + [ + "--temperature", + "0.5", + "--top-p", + "0.25", + "--n-predict", + "128" + ] + ); + } + + #[test] + fn generation_defaults_preserve_unset_lemonade_recipe_flags() { + // Only --temperature is supplied via CLI; an authored --top-p already + // present in the recipe must survive untouched, mirroring the vLLM + // merge behavior instead of being deleted. + let authored = EngineRecipeHint { + contract_version: ENGINE_RECIPE_CONTRACT_VERSION.to_owned(), + engine: "lemonade".to_owned(), + required_flags: vec!["--top-p".to_owned(), "0.9".to_owned()], + ..EngineRecipeHint::default() + }; + let hint = engine_recipe_with_generation_defaults( + "lemonade", + Some(authored), + ServeGenerationDefaults { + temperature: Some(0.5), + top_p: None, + max_tokens: None, + }, + ) + .expect("lemonade defaults are supported") + .expect("supplied defaults should synthesize a hint"); + assert_eq!( + hint.required_flags, + ["--top-p", "0.9", "--temperature", "0.5"] + ); + } + + #[test] + fn extra_env_absent_passes_hint_through_unchanged() { + assert!( + engine_recipe_with_extra_env("aim", None, &[]) + .unwrap() + .is_none() + ); + + let authored = EngineRecipeHint { + contract_version: ENGINE_RECIPE_CONTRACT_VERSION.to_owned(), + engine: "aim".to_owned(), + ..EngineRecipeHint::default() + }; + let hint = engine_recipe_with_extra_env("aim", Some(authored.clone()), &[]).unwrap(); + assert_eq!(hint, Some(authored)); + } + + #[test] + fn extra_env_synthesizes_hint_and_merges_into_authored_one() { + let hint = engine_recipe_with_extra_env("aim", None, &["AIM_PROFILE_ID=foo".to_owned()]) + .unwrap() + .expect("--engine-env should synthesize a hint"); + assert_eq!(hint.engine, "aim"); + assert_eq!( + hint.extra_env.get("AIM_PROFILE_ID"), + Some(&"foo".to_owned()) + ); + + let authored = EngineRecipeHint { + contract_version: ENGINE_RECIPE_CONTRACT_VERSION.to_owned(), + engine: "aim".to_owned(), + extra_env: BTreeMap::from([("EXISTING".to_owned(), "kept".to_owned())]), + ..EngineRecipeHint::default() + }; + let hint = + engine_recipe_with_extra_env("aim", Some(authored), &["AIM_PROFILE_ID=foo".to_owned()]) + .unwrap() + .unwrap(); + assert_eq!(hint.extra_env.get("EXISTING"), Some(&"kept".to_owned())); + assert_eq!( + hint.extra_env.get("AIM_PROFILE_ID"), + Some(&"foo".to_owned()) + ); + } + + #[test] + fn extra_env_bare_key_forwards_from_the_environment_without_recording_a_value() { + let hint = engine_recipe_with_extra_env("aim", None, &["HF_TOKEN".to_owned()]) + .unwrap() + .unwrap(); + assert_eq!(hint.extra_env.get("HF_TOKEN"), Some(&String::new())); + } + + #[test] + fn extra_env_rejects_entry_with_empty_or_malformed_key() { + for entry in ["=some-value", "", "BAD KEY=1", "A-B=1"] { + let error = engine_recipe_with_extra_env("aim", None, &[entry.to_owned()]).unwrap_err(); + assert!(error.to_string().contains("KEY=VALUE"), "{entry}: {error}"); + } + } + + #[test] + fn extra_env_rejects_reserved_keys_and_newlines() { + for entry in ["VLLM_API_KEY=x", "HIP_VISIBLE_DEVICES=0", "VLLM_API_KEY"] { + let error = engine_recipe_with_extra_env("aim", None, &[entry.to_owned()]).unwrap_err(); + assert!(error.to_string().contains("manages it"), "{entry}: {error}"); + } + assert!(engine_recipe_with_extra_env("aim", None, &["A=b\nc".to_owned()]).is_err()); + } + + #[test] + fn extra_env_leaves_the_hint_alone_for_engines_that_ignore_it() { + assert_eq!( + engine_recipe_with_extra_env("vllm", None, &["FOO=bar".to_owned()]).unwrap(), + None, + "a flag reported as ignored must not change what the engine is sent" + ); + } + + #[test] + fn engine_env_distinguishes_a_bare_key_from_an_empty_value() { + let env = |entry: &str| engine_recipe_with_extra_env("aim", None, &[entry.to_owned()]); + let bare = env("HF_TOKEN").unwrap().expect("hint"); + assert_eq!(bare.extra_env.get("HF_TOKEN").map(String::as_str), Some("")); + let err = env("HF_TOKEN=").unwrap_err().to_string(); + assert!( + err.contains("empty value") && err.contains("`HF_TOKEN` alone"), + "{err}" + ); + let set = env("A=b").unwrap().expect("hint"); + assert_eq!(set.extra_env.get("A").map(String::as_str), Some("b")); + } + + #[test] + fn engine_env_ignored_note_is_none_for_aim() { + assert_eq!( + engine_env_ignored_note("aim", &["AIM_PROFILE_ID=foo".to_owned()]), + None + ); + } + + #[test] + fn engine_env_ignored_note_is_none_when_engine_env_is_empty() { + assert_eq!(engine_env_ignored_note("vllm", &[]), None); + } + + #[test] + fn engine_env_ignored_note_warns_for_engines_that_do_not_consume_it() { + let note = engine_env_ignored_note("vllm", &["FOO=bar".to_owned()]) + .expect("vllm does not consume extra_env"); + assert!(note.contains("--engine-env")); + assert!(note.contains("'aim'")); + assert!(note.contains("'vllm'")); + } + + #[test] + fn engine_recipe_enables_tool_choice_reflects_flags() { + assert!(!engine_recipe_enables_tool_choice(None)); + let without = EngineRecipeHint { + contract_version: ENGINE_RECIPE_CONTRACT_VERSION.to_owned(), + engine: "vllm".to_owned(), + required_flags: vec!["--reasoning-parser".to_owned(), "qwen3".to_owned()], + ..EngineRecipeHint::default() + }; + assert!(!engine_recipe_enables_tool_choice(Some(&without))); + let with = engine_recipe_with_tool_call_override("vllm", None, Some("hermes")); + assert!(engine_recipe_enables_tool_choice(with.as_ref())); + } +} From 07e6c05e6bf45892892ce868869143ce95d90004 Mon Sep 17 00:00:00 2001 From: Jussi Elo Date: Wed, 4 Nov 2026 18:16:33 +0000 Subject: [PATCH 2/4] docs: describe serve_recipe.rs and point comments at its new home Rewrite the serve_cmd.rs entry in docs/architecture.md, add one for the new serve_recipe.rs, and update the one clause that called ServeGenerationDefaults private. Update serve_cmd.rs's module doc, which still named the engine-recipe overrides as part of that file, and the two comments that cite the moved code by file (parse_gpu_memory_utilization's "elsewhere in this file" and the fix.rs test comment naming serve_cmd.rs). Comments and docs only. Signed-off-by: Jussi Elo --- apps/rocm/src/serve_cmd.rs | 6 +++++- apps/rocm/src/serve_recipe.rs | 2 +- crates/rocm-core/src/fix.rs | 2 +- docs/architecture.md | 5 +++-- 4 files changed, 10 insertions(+), 5 deletions(-) diff --git a/apps/rocm/src/serve_cmd.rs b/apps/rocm/src/serve_cmd.rs index fa31bcc734..aef148658f 100644 --- a/apps/rocm/src/serve_cmd.rs +++ b/apps/rocm/src/serve_cmd.rs @@ -2,7 +2,11 @@ // // SPDX-License-Identifier: MIT -//! `rocm serve` command handler and engine-recipe overrides. +//! `rocm serve` command handler and the AIM pre-flight that runs inside it. +//! +//! The engine-recipe overrides `serve()` applies to the launch hint +//! (`--tool-call-parser`, `--gpu-memory-utilization`, generation defaults and +//! `--engine-env`) live in `serve_recipe.rs`; this file imports them by name. //! //! Originally a mechanical relocation from `main.rs` with no behavior change — //! the `dispatch()` call site stays byte-identical (`serve(ServeArgs { .. })`, diff --git a/apps/rocm/src/serve_recipe.rs b/apps/rocm/src/serve_recipe.rs index 94976cbe2f..26fef59c44 100644 --- a/apps/rocm/src/serve_recipe.rs +++ b/apps/rocm/src/serve_recipe.rs @@ -336,7 +336,7 @@ fn set_vllm_gpu_memory_utilization(flags: &mut Vec, value: f64) { } /// Parse `rocm serve --gpu-memory-utilization`. Unlike the env-var overrides -/// elsewhere in this file, an explicit CLI value is never silently ignored: a +/// in `serve_cmd.rs`, an explicit CLI value is never silently ignored: a /// user who types a bad fraction is told so. pub(crate) fn parse_gpu_memory_utilization(value: Option<&str>) -> Result> { let Some(raw) = value else { diff --git a/crates/rocm-core/src/fix.rs b/crates/rocm-core/src/fix.rs index c44c5366da..00fcaee884 100644 --- a/crates/rocm-core/src/fix.rs +++ b/crates/rocm-core/src/fix.rs @@ -3131,7 +3131,7 @@ mod tests { // `rocm serve`'s `parse_gpu_memory_utilization` rejects `<= 0` and // accepts `1`, so the domain is (0, 1]. The behaviour itself is proved // by `gpu_memory_utilization_rejects_out_of_range_and_unparsable_values` - // in `apps/rocm/src/serve_cmd.rs`, which rejects `0`/`0.0` and accepts + // in `apps/rocm/src/serve_recipe.rs`, which rejects `0`/`0.0` and accepts // `1`; this assertion pins only the wording that advertises it, which // has no behaviour of its own. `<0-1>` advertises `0`, and this // hint prints in three places (pre-launch note, engine OOM hint, and the diff --git a/docs/architecture.md b/docs/architecture.md index 4dd3148e6d..996a996105 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -15,7 +15,7 @@ This is the living module map for rocm-cli. It's a contributor-facing reference New subcommands and subsystems default to their own file from day one — they should not grow inside `main.rs`/`lib.rs` waiting for a future extraction pass. Two extraction patterns already exist in the codebase; use whichever fits: - **Full domain extraction** — a subsystem's domain implementation moves into its own file that owns its own types (structs/enums), not just relocated functions; that ownership is usually what distinguishes this pattern from mechanical relocation below, though the Phase 6a variant described next shows ownership alone doesn't fully tell them apart — every `apps/rocm` module is a private `mod x;` regardless of pattern, so module privacy alone doesn't tell the two apart either. Full-domain-extraction modules are typically reached via qualified paths (e.g. `comfyui::render_status(...)`); mechanically relocated modules with no owned types are reached the same way (e.g. [`endpoint_keys.rs`](../apps/rocm/src/endpoint_keys.rs), [`logging.rs`](../apps/rocm/src/logging.rs)). The exception — reached unqualified through `use crate::x::{...}` re-imports, matching their dispatch call sites' pre-extraction form — is limited to the clusters that moved as a dispatch-adjacent group; the block of `use crate::x::{...}` re-imports near the top of [`main.rs`](../apps/rocm/src/main.rs) is the current list of the modules it reaches that way (a few of them re-imported for reasons other than dispatch, as the comment above the block explains). [`service_launch.rs`](../apps/rocm/src/service_launch.rs) (as of Phase 6b, ROCMAI-91) uses the same unqualified re-import form for its cross-module callers, but it is not a dispatch-adjacent cluster in the same sense — one of its callers, `automations.rs`, is not a dispatch call site, and its pre-extraction form was a qualified `crate::` call rather than an unqualified import, so the re-import is new rather than kept. This call-site form, not type ownership or dispatch-fn placement, is what actually separates the Phase 6a variant below from full domain extraction. Where a subsystem has a dedicated clap subcommand, its command *enum* usually stays in `main.rs` as part of the `Command`/`Cli` definition (e.g. `ComfyuiCommand`, `RuntimesCommand`) — but the dispatch function itself does not always stay with it: `comfyui()` does, while `engines()` (Phase 6a) and `runtimes()` (Phase 6c, ROCMAI-89) moved out to `engines_cmd.rs`/`runtimes_cmd.rs` respectively, following the mechanical-relocation re-import form below rather than full domain extraction's qualified-path form. Not every domain-extracted module has a dedicated command enum at all (`providers.rs` has none; it's invoked from the existing chat/config command flows). In library crates (`crates/rocm-core`) the module is `pub mod x;` plus a `pub use x::{...};` re-export, since it's part of the crate's public API. This is the default for new subsystems. Examples: [`apps/rocm/src/therock.rs`](../apps/rocm/src/therock.rs) (TheRock SDK install orchestration, with its update planning and other clusters in `therock/` child modules and no command enum of its own), [`comfyui.rs`](../apps/rocm/src/comfyui.rs) (`ComfyuiCommand`), [`providers.rs`](../apps/rocm/src/providers.rs) (no dedicated command enum); `crates/rocm-core`'s [`diagnose.rs`](../crates/rocm-core/src/diagnose.rs)/[`examine.rs`](../crates/rocm-core/src/examine.rs). -- **Mechanical relocation** — a `pub(crate) fn` moves out verbatim, with shared types/config staying at the crate root and reached via `crate::`. Used for dispatch-adjacent clusters where a minimal, easy-to-review diff matters more than full extraction. Example: [`apps/rocm/src/update_cmd.rs`](../apps/rocm/src/update_cmd.rs). Variant: when the cluster is large enough to own types (part of `main.rs`'s Phase 6a split, ROCMAI-82, `main.rs`'s Phase 6b split, ROCMAI-91, `main.rs`'s Phase 6c split, ROCMAI-89, and Phase 6e's follow-up, ROCMAI-726), those types move with it instead of staying in `main.rs` — [`engines_cmd.rs`](../apps/rocm/src/engines_cmd.rs)'s types are used nowhere else, but [`driver_install.rs`](../apps/rocm/src/driver_install.rs)'s `DriverInstallResult`/`DriverInstallError` are `pub(crate)` and read from [`install_cmd.rs`](../apps/rocm/src/install_cmd.rs)'s `install()` (`result.output`/`result.executed`, `error.source`/`error.executed`); [`freeform_execution.rs`](../apps/rocm/src/freeform_execution.rs)'s `FreeformExecution` is `pub(crate)` and read from `main.rs`'s `execute_freeform_next_action()` (`execution.action`) — it was already `pub(crate)` before the split, unlike the two functions that return/consume it, which widened from module-private; [`serve_cmd.rs`](../apps/rocm/src/serve_cmd.rs)'s `ServeEngineSelection`/`ServeArgs` both stay `pub(crate)` and are used from `main.rs` (`ServeArgs`) and [`model_fit.rs`](../apps/rocm/src/model_fit.rs) (`assess_model_for_host` calls `select_serve_engine` and reads the resulting `ServeEngineSelection`'s `.engine`/`.source` fields) — and `ServeGenerationDefaults` is private; [`service_launch.rs`](../apps/rocm/src/service_launch.rs)'s `AttachOutcome`/`RawModeGuard`/`ManagedSpawn`/`AutostartClaim` are private, and its `ManagedLaunchReport` is `pub(crate)` and read from [`serve_cmd.rs`](../apps/rocm/src/serve_cmd.rs) for the interactive summary; [`runtimes_cmd.rs`](../apps/rocm/src/runtimes_cmd.rs)'s `RuntimeActivationResult` and `ActiveRuntimeMarker` are `pub(crate)` and read from [`update_cmd.rs`](../apps/rocm/src/update_cmd.rs)'s `apply_runtime_update`/`append_update_activate_summary` and from [`storage.rs`](../apps/rocm/src/storage.rs) respectively; its sibling [`runtime_install_finalize.rs`](../apps/rocm/src/runtime_install_finalize.rs)'s `SdkInstallConsents` is `pub(crate)` and read from `install_cmd.rs`'s `install()`. The pattern is still "mechanical relocation" rather than full domain extraction because call sites keep their pre-extraction unqualified-re-import form — the exception named above — rather than switching to the qualified paths full-domain-extraction modules use. The top-level dispatch fn (`dispatch()`) staying in `main.rs` is not what makes it so: full domain extraction usually keeps its dispatch fn in `main.rs` too (see `comfyui()` above). +- **Mechanical relocation** — a `pub(crate) fn` moves out verbatim, with shared types/config staying at the crate root and reached via `crate::`. Used for dispatch-adjacent clusters where a minimal, easy-to-review diff matters more than full extraction. Example: [`apps/rocm/src/update_cmd.rs`](../apps/rocm/src/update_cmd.rs). Variant: when the cluster is large enough to own types (part of `main.rs`'s Phase 6a split, ROCMAI-82, `main.rs`'s Phase 6b split, ROCMAI-91, `main.rs`'s Phase 6c split, ROCMAI-89, and Phase 6e's follow-up, ROCMAI-726), those types move with it instead of staying in `main.rs` — [`engines_cmd.rs`](../apps/rocm/src/engines_cmd.rs)'s types are used nowhere else, but [`driver_install.rs`](../apps/rocm/src/driver_install.rs)'s `DriverInstallResult`/`DriverInstallError` are `pub(crate)` and read from [`install_cmd.rs`](../apps/rocm/src/install_cmd.rs)'s `install()` (`result.output`/`result.executed`, `error.source`/`error.executed`); [`freeform_execution.rs`](../apps/rocm/src/freeform_execution.rs)'s `FreeformExecution` is `pub(crate)` and read from `main.rs`'s `execute_freeform_next_action()` (`execution.action`) — it was already `pub(crate)` before the split, unlike the two functions that return/consume it, which widened from module-private; [`serve_cmd.rs`](../apps/rocm/src/serve_cmd.rs)'s `ServeEngineSelection`/`ServeArgs` both stay `pub(crate)` and are used from `main.rs` (`ServeArgs`) and [`model_fit.rs`](../apps/rocm/src/model_fit.rs) (`assess_model_for_host` calls `select_serve_engine` and reads the resulting `ServeEngineSelection`'s `.engine`/`.source` fields) — and `ServeGenerationDefaults` moved out with the engine-recipe overrides to [`serve_recipe.rs`](../apps/rocm/src/serve_recipe.rs), where it and its fields are `pub(crate)` because `serve()` builds it; [`service_launch.rs`](../apps/rocm/src/service_launch.rs)'s `AttachOutcome`/`RawModeGuard`/`ManagedSpawn`/`AutostartClaim` are private, and its `ManagedLaunchReport` is `pub(crate)` and read from [`serve_cmd.rs`](../apps/rocm/src/serve_cmd.rs) for the interactive summary; [`runtimes_cmd.rs`](../apps/rocm/src/runtimes_cmd.rs)'s `RuntimeActivationResult` and `ActiveRuntimeMarker` are `pub(crate)` and read from [`update_cmd.rs`](../apps/rocm/src/update_cmd.rs)'s `apply_runtime_update`/`append_update_activate_summary` and from [`storage.rs`](../apps/rocm/src/storage.rs) respectively; its sibling [`runtime_install_finalize.rs`](../apps/rocm/src/runtime_install_finalize.rs)'s `SdkInstallConsents` is `pub(crate)` and read from `install_cmd.rs`'s `install()`. The pattern is still "mechanical relocation" rather than full domain extraction because call sites keep their pre-extraction unqualified-re-import form — the exception named above — rather than switching to the qualified paths full-domain-extraction modules use. The top-level dispatch fn (`dispatch()`) staying in `main.rs` is not what makes it so: full domain extraction usually keeps its dispatch fn in `main.rs` too (see `comfyui()` above). There is no file-line-count CI gate enforcing this — `too_many_lines = "allow"` in the workspace `Cargo.toml` is a deliberate, function-level choice, not an oversight. This convention is the guardrail instead. @@ -69,7 +69,8 @@ Each per-crate module inventory lists one file per line. Flat inventories (every - [`runtime_services.rs`](../apps/rocm/src/runtime_services.rs) — full domain extraction; reads the live local servers at activation time, classifies each against the runtime being activated, renders the report, and moves stale ones with `--restart-services`. `fn runtimes()` lives in [`runtimes_cmd.rs`](../apps/rocm/src/runtimes_cmd.rs) and `RuntimesCommand` in [`main.rs`](../apps/rocm/src/main.rs); this file owns `ServiceRuntimeState`/`RuntimeServiceEntry`/`FailedServiceRestart`/`UnconfirmedServiceRestart`/`ServiceRestartOutcome`/`RuntimeServiceReconciliation`/`ServiceRuntimePin`, reached from `runtimes_cmd.rs` via `crate::runtime_services`. - [`runtime_uninstall.rs`](../apps/rocm/src/runtime_uninstall.rs) — full domain extraction, crate-level (split out of [`runtimes_cmd.rs`](../apps/rocm/src/runtimes_cmd.rs)): `rocm runtimes uninstall` and the storage pruning that removes a registered runtime. `plan_runtime_uninstall` builds the plan (`RuntimeUninstallPlan`, with the `InstallRootDecision` for its folder), `print_runtime_uninstall_plan` shows it, `confirm_and_revalidate_runtime_uninstall` asks and then checks that nothing changed while the prompt was open (`revalidate_runtime_uninstall_plan`), and `apply_runtime_uninstall` unregisters the runtime, clears the active and default state that named it and removes its install root when `should_remove_runtime_install_root` says ROCm CLI owns it (`local_runtime_manifest_matches`, `ensure_runtime_install_root_is_safe_to_remove`). `uninstall_runtime` is the non-interactive entry point. It reads the registry, the active-runtime marker and the path comparison from [`runtime_registry.rs`](../apps/rocm/src/runtime_registry.rs), and does not depend on `runtimes_cmd.rs`, which calls into it from `runtimes()` and re-exports `uninstall_runtime`, `should_remove_runtime_install_root` and `InstallRootDecision`. The planning and apply entry points, `RuntimeUninstallPlan`, `RuntimeUninstallConfirmation` and `RuntimeUninstallResult` (with its fields) are `pub(crate)` for `runtimes()`; `revalidate_runtime_uninstall_plan` and the folder-safety helpers are private. Its tests cover planning, revalidation, the folder-removal rules and uninstalling a read-only imported runtime; the uninstall tests that first activate a runtime stay with the activation tests in `runtimes_cmd.rs`. - [`runtimes_cmd.rs`](../apps/rocm/src/runtimes_cmd.rs) — mechanical relocation, owns types (Phase 6c, ROCMAI-89): `runtimes()` and its activate/rollback handlers (the activation transaction, including `write_active_runtime_marker`), plus the runtime list rendering; the import and adopt handlers it dispatches to live in [`runtime_adopt.rs`](../apps/rocm/src/runtime_adopt.rs) and the uninstall plan, confirmation and apply in [`runtime_uninstall.rs`](../apps/rocm/src/runtime_uninstall.rs); registry selection and manifest validation moved to [`runtime_registry.rs`](../apps/rocm/src/runtime_registry.rs), which this module re-exports for the callers that import them from here. `RuntimesCommand` stayed in [`main.rs`](../apps/rocm/src/main.rs) (part of the `Command`/`Cli` clap definition, the same way `EnginesCommand` did in Phase 6a). `RuntimeActivationResult` is `pub(crate)`, read from [`update_cmd.rs`](../apps/rocm/src/update_cmd.rs)'s `apply_runtime_update`/`append_update_activate_summary`; `uninstall_runtime`, `should_remove_runtime_install_root` and `InstallRootDecision` (from `runtime_uninstall.rs`) and `ActiveRuntimeMarker` and `active_runtime_marker_path` (from `runtime_registry.rs`) are re-exported for [`storage.rs`](../apps/rocm/src/storage.rs) and [`config_cmd.rs`](../apps/rocm/src/config_cmd.rs); `recover_setup_runtime_registration`, `newest_installed_runtime_manifest` and `setup_runtime_paths` (from `runtime_adopt.rs`) and `normalize_path_for_compare` (from `runtime_registry.rs`) are re-exported for the modules that import them from here, such as [`engines_cmd.rs`](../apps/rocm/src/engines_cmd.rs), [`engine_runtime_select.rs`](../apps/rocm/src/engine_runtime_select.rs) and [`examine_report_text.rs`](../apps/rocm/src/examine_report_text.rs). The dependency on the sibling [`runtime_install_finalize.rs`](../apps/rocm/src/runtime_install_finalize.rs) above runs the other way: that module calls into this one (`activate_runtime`, `newest_installed_runtime_manifest`, `recover_setup_runtime_registration`), while this module reaches it only from its test module. -- [`serve_cmd.rs`](../apps/rocm/src/serve_cmd.rs) — mechanical relocation, owns types (Phase 6a): `serve()`, plus engine-recipe overrides and the AIM pre-flight that runs inside it: the Hugging Face token check, the GPU-family profile check and pin, the multi-GPU refusal (`aim_gpu_preflight`), and AIM-aware default engine selection (`recipe_preferred_engine`). It also holds `collect_serve_notes`, the GPU/device warnings folded into the interactive summary, which was relocated here from `main.rs` with its two tests. +- [`serve_cmd.rs`](../apps/rocm/src/serve_cmd.rs) — mechanical relocation, owns types: `serve()`, plus the engine selection that runs ahead of it (`select_serve_engine`, `recipe_preferred_engine`, `serve_model_ref_for_engine`) and the AIM pre-flight that runs inside it: the Hugging Face token check, the GPU-family profile check and pin, the multi-GPU refusal (`aim_gpu_preflight`), and the board note. The engine-recipe overrides `serve()` applies to the launch hint live in [`serve_recipe.rs`](../apps/rocm/src/serve_recipe.rs); this module imports them by name. It also holds `collect_serve_notes`, the GPU/device warnings folded into the interactive summary, which was relocated here from `main.rs` with its two tests. Its tests cover engine selection, the AIM pre-flight and the reuse pre-gate; the AIM tests build their hint with `protocol_engine_recipe_hint` and `engine_recipe_with_extra_env` from `serve_recipe.rs`, so those two are `pub(crate)`. +- [`serve_recipe.rs`](../apps/rocm/src/serve_recipe.rs) — mechanical relocation, owns a type (split out of [`serve_cmd.rs`](../apps/rocm/src/serve_cmd.rs), which imports the entry points by name): the pure transforms `serve()` applies to the selected engine's launch recipe hint. `protocol_engine_recipe_hint` maps the catalog's engine record onto the protocol hint, and `render_serve_engine_recipe_lines` prints it for the serve plan. The explicit controls are then layered on top: `--tool-call-parser` (`engine_recipe_with_tool_call_override`), `--gpu-memory-utilization` (`parse_gpu_memory_utilization`, `engine_recipe_with_gpu_memory_utilization_override`), `--temperature`/`--top-p`/`--max-tokens` (`ServeGenerationDefaults`, `engine_recipe_with_generation_defaults`) and `--engine-env` (`engine_recipe_with_extra_env`, `engine_env_ignored_note`, with the env names it refuses to override in `RESERVED_ENGINE_ENV_KEYS`); `engine_recipe_enables_tool_choice` reads the result back. The per-flag rewriters (`set_vllm_tool_call_parser`, `set_vllm_gpu_memory_utilization`, `set_vllm_override_generation_config`, `set_lemonade_generation_defaults`) are private. The entry points above, and `ServeGenerationDefaults`, are `pub(crate)` for `serve()` and for the AIM tests in `serve_cmd.rs`; `RESERVED_ENGINE_ENV_KEYS` is private. Among first-party crates the module depends only on `rocm_core` and `rocm_engine_protocol`, and nothing in it calls back into `serve_cmd.rs`. Its tests cover each override on its own. - [`serve_summary.rs`](../apps/rocm/src/serve_summary.rs) — full domain extraction. - [`service_hints.rs`](../apps/rocm/src/service_hints.rs) — shared CLI-output component: the `rocm services stop|restart|remove --yes` hint text, shared by [`remote/mod.rs`](../apps/rocm/src/remote/mod.rs), [`serve_summary.rs`](../apps/rocm/src/serve_summary.rs), [`service_launch.rs`](../apps/rocm/src/service_launch.rs), [`service_records.rs`](../apps/rocm/src/service_records.rs), [`services_cmd.rs`](../apps/rocm/src/services_cmd.rs), and [`services_text.rs`](../apps/rocm/src/services_text.rs) so they cannot go out of sync with each other, as `serve_summary.rs`'s stop row did from the day it was added — #417. It also holds the helpers that map a service tool name to the command word, verb and past tense those hints and messages use (`service_action_command`, `service_action_verb`, `service_action_past_tense`) and to its `SandboxToolArg` (`sandbox_tool_arg_from_service_tool`); [`services_text.rs`](../apps/rocm/src/services_text.rs) reads `service_action_past_tense` from here rather than through the crate root. - [`service_launch.rs`](../apps/rocm/src/service_launch.rs) — mechanical relocation, owns types (Phase 6b, ROCMAI-91): spawning/attaching to a managed service process and streaming its log, plus the endpoint-key/auth checks that gate a public bind — `AttachOutcome`/`RawModeGuard`/`ManagedSpawn`/`AutostartClaim` are its private types; `ManagedLaunchReport` is `pub(crate)`, since [`serve_cmd.rs`](../apps/rocm/src/serve_cmd.rs) reads its fields directly for the interactive summary. Most of this module's functions are `pub(crate)` for the same reason — other modules, including [`serve_cmd.rs`](../apps/rocm/src/serve_cmd.rs), [`automations.rs`](../apps/rocm/src/automations.rs), and [`services_text.rs`](../apps/rocm/src/services_text.rs), call into it; `main.rs` does not. `ServicesCommand` stayed in [`main.rs`](../apps/rocm/src/main.rs) rather than moving alongside, and `services()` lives in [`services_cmd.rs`](../apps/rocm/src/services_cmd.rs); the record removal and pruning it dispatches to now lives in [`service_records.rs`](../apps/rocm/src/service_records.rs), and only stop/restart are also reachable from the chat/LLM sandbox tool-calling path, out of scope for Phase 6b. From 0d4e97c3004c1adb2fd50dc9d060a32762a1086c Mon Sep 17 00:00:00 2001 From: Jussi Elo Date: Wed, 4 Nov 2026 20:17:52 +0000 Subject: [PATCH 3/4] docs: fix the reason given for the pub(crate) recipe entry points The serve_cmd.rs entry in docs/architecture.md said protocol_engine_recipe_hint and engine_recipe_with_extra_env are pub(crate) because the AIM tests use them. All the entry points are pub(crate) because serve() calls them; those two are also used by the AIM tests. Say so. Signed-off-by: Jussi Elo --- docs/architecture.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/architecture.md b/docs/architecture.md index 996a996105..9851b7220e 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -69,7 +69,7 @@ Each per-crate module inventory lists one file per line. Flat inventories (every - [`runtime_services.rs`](../apps/rocm/src/runtime_services.rs) — full domain extraction; reads the live local servers at activation time, classifies each against the runtime being activated, renders the report, and moves stale ones with `--restart-services`. `fn runtimes()` lives in [`runtimes_cmd.rs`](../apps/rocm/src/runtimes_cmd.rs) and `RuntimesCommand` in [`main.rs`](../apps/rocm/src/main.rs); this file owns `ServiceRuntimeState`/`RuntimeServiceEntry`/`FailedServiceRestart`/`UnconfirmedServiceRestart`/`ServiceRestartOutcome`/`RuntimeServiceReconciliation`/`ServiceRuntimePin`, reached from `runtimes_cmd.rs` via `crate::runtime_services`. - [`runtime_uninstall.rs`](../apps/rocm/src/runtime_uninstall.rs) — full domain extraction, crate-level (split out of [`runtimes_cmd.rs`](../apps/rocm/src/runtimes_cmd.rs)): `rocm runtimes uninstall` and the storage pruning that removes a registered runtime. `plan_runtime_uninstall` builds the plan (`RuntimeUninstallPlan`, with the `InstallRootDecision` for its folder), `print_runtime_uninstall_plan` shows it, `confirm_and_revalidate_runtime_uninstall` asks and then checks that nothing changed while the prompt was open (`revalidate_runtime_uninstall_plan`), and `apply_runtime_uninstall` unregisters the runtime, clears the active and default state that named it and removes its install root when `should_remove_runtime_install_root` says ROCm CLI owns it (`local_runtime_manifest_matches`, `ensure_runtime_install_root_is_safe_to_remove`). `uninstall_runtime` is the non-interactive entry point. It reads the registry, the active-runtime marker and the path comparison from [`runtime_registry.rs`](../apps/rocm/src/runtime_registry.rs), and does not depend on `runtimes_cmd.rs`, which calls into it from `runtimes()` and re-exports `uninstall_runtime`, `should_remove_runtime_install_root` and `InstallRootDecision`. The planning and apply entry points, `RuntimeUninstallPlan`, `RuntimeUninstallConfirmation` and `RuntimeUninstallResult` (with its fields) are `pub(crate)` for `runtimes()`; `revalidate_runtime_uninstall_plan` and the folder-safety helpers are private. Its tests cover planning, revalidation, the folder-removal rules and uninstalling a read-only imported runtime; the uninstall tests that first activate a runtime stay with the activation tests in `runtimes_cmd.rs`. - [`runtimes_cmd.rs`](../apps/rocm/src/runtimes_cmd.rs) — mechanical relocation, owns types (Phase 6c, ROCMAI-89): `runtimes()` and its activate/rollback handlers (the activation transaction, including `write_active_runtime_marker`), plus the runtime list rendering; the import and adopt handlers it dispatches to live in [`runtime_adopt.rs`](../apps/rocm/src/runtime_adopt.rs) and the uninstall plan, confirmation and apply in [`runtime_uninstall.rs`](../apps/rocm/src/runtime_uninstall.rs); registry selection and manifest validation moved to [`runtime_registry.rs`](../apps/rocm/src/runtime_registry.rs), which this module re-exports for the callers that import them from here. `RuntimesCommand` stayed in [`main.rs`](../apps/rocm/src/main.rs) (part of the `Command`/`Cli` clap definition, the same way `EnginesCommand` did in Phase 6a). `RuntimeActivationResult` is `pub(crate)`, read from [`update_cmd.rs`](../apps/rocm/src/update_cmd.rs)'s `apply_runtime_update`/`append_update_activate_summary`; `uninstall_runtime`, `should_remove_runtime_install_root` and `InstallRootDecision` (from `runtime_uninstall.rs`) and `ActiveRuntimeMarker` and `active_runtime_marker_path` (from `runtime_registry.rs`) are re-exported for [`storage.rs`](../apps/rocm/src/storage.rs) and [`config_cmd.rs`](../apps/rocm/src/config_cmd.rs); `recover_setup_runtime_registration`, `newest_installed_runtime_manifest` and `setup_runtime_paths` (from `runtime_adopt.rs`) and `normalize_path_for_compare` (from `runtime_registry.rs`) are re-exported for the modules that import them from here, such as [`engines_cmd.rs`](../apps/rocm/src/engines_cmd.rs), [`engine_runtime_select.rs`](../apps/rocm/src/engine_runtime_select.rs) and [`examine_report_text.rs`](../apps/rocm/src/examine_report_text.rs). The dependency on the sibling [`runtime_install_finalize.rs`](../apps/rocm/src/runtime_install_finalize.rs) above runs the other way: that module calls into this one (`activate_runtime`, `newest_installed_runtime_manifest`, `recover_setup_runtime_registration`), while this module reaches it only from its test module. -- [`serve_cmd.rs`](../apps/rocm/src/serve_cmd.rs) — mechanical relocation, owns types: `serve()`, plus the engine selection that runs ahead of it (`select_serve_engine`, `recipe_preferred_engine`, `serve_model_ref_for_engine`) and the AIM pre-flight that runs inside it: the Hugging Face token check, the GPU-family profile check and pin, the multi-GPU refusal (`aim_gpu_preflight`), and the board note. The engine-recipe overrides `serve()` applies to the launch hint live in [`serve_recipe.rs`](../apps/rocm/src/serve_recipe.rs); this module imports them by name. It also holds `collect_serve_notes`, the GPU/device warnings folded into the interactive summary, which was relocated here from `main.rs` with its two tests. Its tests cover engine selection, the AIM pre-flight and the reuse pre-gate; the AIM tests build their hint with `protocol_engine_recipe_hint` and `engine_recipe_with_extra_env` from `serve_recipe.rs`, so those two are `pub(crate)`. +- [`serve_cmd.rs`](../apps/rocm/src/serve_cmd.rs) — mechanical relocation, owns types: `serve()`, plus the engine selection that runs ahead of it (`select_serve_engine`, `recipe_preferred_engine`, `serve_model_ref_for_engine`) and the AIM pre-flight that runs inside it: the Hugging Face token check, the GPU-family profile check and pin, the multi-GPU refusal (`aim_gpu_preflight`), and the board note. The engine-recipe overrides `serve()` applies to the launch hint live in [`serve_recipe.rs`](../apps/rocm/src/serve_recipe.rs); this module imports them by name. It also holds `collect_serve_notes`, the GPU/device warnings folded into the interactive summary, which was relocated here from `main.rs` with its two tests. Its tests cover engine selection, the AIM pre-flight and the reuse pre-gate; the AIM tests build their hint with `protocol_engine_recipe_hint` and `engine_recipe_with_extra_env` from `serve_recipe.rs`, which `serve()` also calls. - [`serve_recipe.rs`](../apps/rocm/src/serve_recipe.rs) — mechanical relocation, owns a type (split out of [`serve_cmd.rs`](../apps/rocm/src/serve_cmd.rs), which imports the entry points by name): the pure transforms `serve()` applies to the selected engine's launch recipe hint. `protocol_engine_recipe_hint` maps the catalog's engine record onto the protocol hint, and `render_serve_engine_recipe_lines` prints it for the serve plan. The explicit controls are then layered on top: `--tool-call-parser` (`engine_recipe_with_tool_call_override`), `--gpu-memory-utilization` (`parse_gpu_memory_utilization`, `engine_recipe_with_gpu_memory_utilization_override`), `--temperature`/`--top-p`/`--max-tokens` (`ServeGenerationDefaults`, `engine_recipe_with_generation_defaults`) and `--engine-env` (`engine_recipe_with_extra_env`, `engine_env_ignored_note`, with the env names it refuses to override in `RESERVED_ENGINE_ENV_KEYS`); `engine_recipe_enables_tool_choice` reads the result back. The per-flag rewriters (`set_vllm_tool_call_parser`, `set_vllm_gpu_memory_utilization`, `set_vllm_override_generation_config`, `set_lemonade_generation_defaults`) are private. The entry points above, and `ServeGenerationDefaults`, are `pub(crate)` for `serve()` and for the AIM tests in `serve_cmd.rs`; `RESERVED_ENGINE_ENV_KEYS` is private. Among first-party crates the module depends only on `rocm_core` and `rocm_engine_protocol`, and nothing in it calls back into `serve_cmd.rs`. Its tests cover each override on its own. - [`serve_summary.rs`](../apps/rocm/src/serve_summary.rs) — full domain extraction. - [`service_hints.rs`](../apps/rocm/src/service_hints.rs) — shared CLI-output component: the `rocm services stop|restart|remove --yes` hint text, shared by [`remote/mod.rs`](../apps/rocm/src/remote/mod.rs), [`serve_summary.rs`](../apps/rocm/src/serve_summary.rs), [`service_launch.rs`](../apps/rocm/src/service_launch.rs), [`service_records.rs`](../apps/rocm/src/service_records.rs), [`services_cmd.rs`](../apps/rocm/src/services_cmd.rs), and [`services_text.rs`](../apps/rocm/src/services_text.rs) so they cannot go out of sync with each other, as `serve_summary.rs`'s stop row did from the day it was added — #417. It also holds the helpers that map a service tool name to the command word, verb and past tense those hints and messages use (`service_action_command`, `service_action_verb`, `service_action_past_tense`) and to its `SandboxToolArg` (`sandbox_tool_arg_from_service_tool`); [`services_text.rs`](../apps/rocm/src/services_text.rs) reads `service_action_past_tense` from here rather than through the crate root. From e8740e717730afd96471e437a48549d51bd36b17 Mon Sep 17 00:00:00 2001 From: Jussi Elo Date: Thu, 5 Nov 2026 13:31:26 +0000 Subject: [PATCH 4/4] docs: drop the history note from serve_recipe.rs and restore the Phase 6a label Review notes on the serve_recipe.rs split. The module doc no longer says the code was relocated with no behavior change, since that sentence describes the move rather than the module and will age. The serve_cmd.rs entry in docs/architecture.md gets its "(Phase 6a)" label back, which the split had dropped from the entry. Signed-off-by: Jussi Elo --- apps/rocm/src/serve_recipe.rs | 10 +++++----- docs/architecture.md | 2 +- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/apps/rocm/src/serve_recipe.rs b/apps/rocm/src/serve_recipe.rs index 26fef59c44..c2cb412ac1 100644 --- a/apps/rocm/src/serve_recipe.rs +++ b/apps/rocm/src/serve_recipe.rs @@ -4,11 +4,11 @@ //! Engine-recipe overrides for `rocm serve`. //! -//! Relocated from `serve_cmd.rs` with no behavior change. These are the pure -//! transforms `serve()` applies, in order, to the selected engine's launch -//! recipe hint: mapping the catalog's engine record onto the protocol hint, -//! then layering the explicit `--tool-call-parser`, `--gpu-memory-utilization`, -//! `--temperature`/`--top-p`/`--max-tokens` and `--engine-env` controls on top. +//! These are the pure transforms `serve()` applies, in order, to the selected +//! engine's launch recipe hint: mapping the catalog's engine record onto the +//! protocol hint, then layering the explicit `--tool-call-parser`, +//! `--gpu-memory-utilization`, `--temperature`/`--top-p`/`--max-tokens` and +//! `--engine-env` controls on top. //! `serve()` is the only production caller; nothing here calls back into it. use std::fmt::Write as _; diff --git a/docs/architecture.md b/docs/architecture.md index de0c7e30b8..2b54397e56 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -70,7 +70,7 @@ Each per-crate module inventory lists one file per line. Flat inventories (every - [`runtime_services.rs`](../apps/rocm/src/runtime_services.rs) — full domain extraction; reads the live local servers at activation time, classifies each against the runtime being activated, renders the report, and moves stale ones with `--restart-services`. `fn runtimes()` lives in [`runtimes_cmd.rs`](../apps/rocm/src/runtimes_cmd.rs) and `RuntimesCommand` in [`main.rs`](../apps/rocm/src/main.rs); this file owns `ServiceRuntimeState`/`RuntimeServiceEntry`/`FailedServiceRestart`/`UnconfirmedServiceRestart`/`ServiceRestartOutcome`/`RuntimeServiceReconciliation`/`ServiceRuntimePin`, reached from `runtimes_cmd.rs` via `crate::runtime_services`. - [`runtime_uninstall.rs`](../apps/rocm/src/runtime_uninstall.rs) — full domain extraction, crate-level (split out of [`runtimes_cmd.rs`](../apps/rocm/src/runtimes_cmd.rs)): `rocm runtimes uninstall` and the storage pruning that removes a registered runtime. `plan_runtime_uninstall` builds the plan (`RuntimeUninstallPlan`, with the `InstallRootDecision` for its folder), `print_runtime_uninstall_plan` shows it, `confirm_and_revalidate_runtime_uninstall` asks and then checks that nothing changed while the prompt was open (`revalidate_runtime_uninstall_plan`), and `apply_runtime_uninstall` unregisters the runtime, clears the active and default state that named it and removes its install root when `should_remove_runtime_install_root` says ROCm CLI owns it (`local_runtime_manifest_matches`, `ensure_runtime_install_root_is_safe_to_remove`). `uninstall_runtime` is the non-interactive entry point. It reads the registry, the active-runtime marker and the path comparison from [`runtime_registry.rs`](../apps/rocm/src/runtime_registry.rs), and does not depend on `runtimes_cmd.rs`, which calls into it from `runtimes()` and re-exports `uninstall_runtime`, `should_remove_runtime_install_root` and `InstallRootDecision`. The planning and apply entry points, `RuntimeUninstallPlan`, `RuntimeUninstallConfirmation` and `RuntimeUninstallResult` (with its fields) are `pub(crate)` for `runtimes()`; `revalidate_runtime_uninstall_plan` and the folder-safety helpers are private. Its tests cover planning, revalidation, the folder-removal rules and uninstalling a read-only imported runtime; the uninstall tests that first activate a runtime stay with the activation tests in `runtimes_cmd.rs`. - [`runtimes_cmd.rs`](../apps/rocm/src/runtimes_cmd.rs) — mechanical relocation, owns types (Phase 6c, ROCMAI-89): `runtimes()` and its activate/rollback handlers (the activation transaction, including `write_active_runtime_marker`), plus the runtime list rendering; the import and adopt handlers it dispatches to live in [`runtime_adopt.rs`](../apps/rocm/src/runtime_adopt.rs) and the uninstall plan, confirmation and apply in [`runtime_uninstall.rs`](../apps/rocm/src/runtime_uninstall.rs); registry selection and manifest validation moved to [`runtime_registry.rs`](../apps/rocm/src/runtime_registry.rs), which this module re-exports for the callers that import them from here. `RuntimesCommand` stayed in [`main.rs`](../apps/rocm/src/main.rs) (part of the `Command`/`Cli` clap definition, the same way `EnginesCommand` did in Phase 6a). `RuntimeActivationResult` is `pub(crate)`, read from [`update_cmd.rs`](../apps/rocm/src/update_cmd.rs)'s `apply_runtime_update`/`append_update_activate_summary`; `uninstall_runtime`, `should_remove_runtime_install_root` and `InstallRootDecision` (from `runtime_uninstall.rs`) and `ActiveRuntimeMarker` and `active_runtime_marker_path` (from `runtime_registry.rs`) are re-exported for [`storage.rs`](../apps/rocm/src/storage.rs) and [`config_cmd.rs`](../apps/rocm/src/config_cmd.rs); `recover_setup_runtime_registration`, `newest_installed_runtime_manifest` and `setup_runtime_paths` (from `runtime_adopt.rs`) and `normalize_path_for_compare` (from `runtime_registry.rs`) are re-exported for the modules that import them from here, such as [`engines_cmd.rs`](../apps/rocm/src/engines_cmd.rs), [`engine_runtime_select.rs`](../apps/rocm/src/engine_runtime_select.rs), [`examine_cmd.rs`](../apps/rocm/src/examine_cmd.rs) and [`examine_report_text.rs`](../apps/rocm/src/examine_report_text.rs). [`engine_install_env.rs`](../apps/rocm/src/engine_install_env.rs) is deliberately not one of them: it imports from `runtime_adopt.rs` and `runtime_registry.rs` directly, because going through this module would lead back to it via `runtime_services.rs`. The dependency on the sibling [`runtime_install_finalize.rs`](../apps/rocm/src/runtime_install_finalize.rs) above runs the other way: that module calls into this one (`activate_runtime`, `newest_installed_runtime_manifest`, `recover_setup_runtime_registration`), while this module reaches it only from its test module. -- [`serve_cmd.rs`](../apps/rocm/src/serve_cmd.rs) — mechanical relocation, owns types: `serve()`, plus the engine selection that runs ahead of it (`select_serve_engine`, `recipe_preferred_engine`, `serve_model_ref_for_engine`) and the AIM pre-flight that runs inside it: the Hugging Face token check, the GPU-family profile check and pin, the multi-GPU refusal (`aim_gpu_preflight`), and the board note. The engine-recipe overrides `serve()` applies to the launch hint live in [`serve_recipe.rs`](../apps/rocm/src/serve_recipe.rs); this module imports them by name. It also holds `collect_serve_notes`, the GPU/device warnings folded into the interactive summary, which was relocated here from `main.rs` with its two tests. Its tests cover engine selection, the AIM pre-flight and the reuse pre-gate; the AIM tests build their hint with `protocol_engine_recipe_hint` and `engine_recipe_with_extra_env` from `serve_recipe.rs`, which `serve()` also calls. +- [`serve_cmd.rs`](../apps/rocm/src/serve_cmd.rs) — mechanical relocation, owns types (Phase 6a): `serve()`, plus the engine selection that runs ahead of it (`select_serve_engine`, `recipe_preferred_engine`, `serve_model_ref_for_engine`) and the AIM pre-flight that runs inside it: the Hugging Face token check, the GPU-family profile check and pin, the multi-GPU refusal (`aim_gpu_preflight`), and the board note. The engine-recipe overrides `serve()` applies to the launch hint live in [`serve_recipe.rs`](../apps/rocm/src/serve_recipe.rs); this module imports them by name. It also holds `collect_serve_notes`, the GPU/device warnings folded into the interactive summary, which was relocated here from `main.rs` with its two tests. Its tests cover engine selection, the AIM pre-flight and the reuse pre-gate; the AIM tests build their hint with `protocol_engine_recipe_hint` and `engine_recipe_with_extra_env` from `serve_recipe.rs`, which `serve()` also calls. - [`serve_recipe.rs`](../apps/rocm/src/serve_recipe.rs) — mechanical relocation, owns a type (split out of [`serve_cmd.rs`](../apps/rocm/src/serve_cmd.rs), which imports the entry points by name): the pure transforms `serve()` applies to the selected engine's launch recipe hint. `protocol_engine_recipe_hint` maps the catalog's engine record onto the protocol hint, and `render_serve_engine_recipe_lines` prints it for the serve plan. The explicit controls are then layered on top: `--tool-call-parser` (`engine_recipe_with_tool_call_override`), `--gpu-memory-utilization` (`parse_gpu_memory_utilization`, `engine_recipe_with_gpu_memory_utilization_override`), `--temperature`/`--top-p`/`--max-tokens` (`ServeGenerationDefaults`, `engine_recipe_with_generation_defaults`) and `--engine-env` (`engine_recipe_with_extra_env`, `engine_env_ignored_note`, with the env names it refuses to override in `RESERVED_ENGINE_ENV_KEYS`); `engine_recipe_enables_tool_choice` reads the result back. The per-flag rewriters (`set_vllm_tool_call_parser`, `set_vllm_gpu_memory_utilization`, `set_vllm_override_generation_config`, `set_lemonade_generation_defaults`) are private. The entry points above, and `ServeGenerationDefaults`, are `pub(crate)` for `serve()` and for the AIM tests in `serve_cmd.rs`; `RESERVED_ENGINE_ENV_KEYS` is private. Among first-party crates the module depends only on `rocm_core` and `rocm_engine_protocol`, and nothing in it calls back into `serve_cmd.rs`. Its tests cover each override on its own. - [`serve_summary.rs`](../apps/rocm/src/serve_summary.rs) — full domain extraction. - [`service_hints.rs`](../apps/rocm/src/service_hints.rs) — shared CLI-output component: the `rocm services stop|restart|remove --yes` hint text, shared by [`remote/mod.rs`](../apps/rocm/src/remote/mod.rs), [`serve_summary.rs`](../apps/rocm/src/serve_summary.rs), [`service_launch.rs`](../apps/rocm/src/service_launch.rs), [`service_records.rs`](../apps/rocm/src/service_records.rs), [`services_cmd.rs`](../apps/rocm/src/services_cmd.rs), and [`services_text.rs`](../apps/rocm/src/services_text.rs) so they cannot go out of sync with each other, as `serve_summary.rs`'s stop row did from the day it was added — #417. It also holds the helpers that map a service tool name to the command word, verb and past tense those hints and messages use (`service_action_command`, `service_action_verb`, `service_action_past_tense`) and to its `SandboxToolArg` (`sandbox_tool_arg_from_service_tool`); [`services_text.rs`](../apps/rocm/src/services_text.rs) reads `service_action_past_tense` from here rather than through the crate root.