diff --git a/docs/architecture.md b/docs/architecture.md index 66bf0b7b80..c83c83a3b3 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -55,4 +55,4 @@ Pinned versions of the third-party runtimes rocm-cli manages (from workspace-roo ### `engines/lemonade`, `engines/vllm` — inference engine adapters -`engines/vllm` follows full domain extraction: `runtime.rs` (TheRock/managed-runtime resolution), `install.rs` (ROCm/vLLM build-variant discovery, install execution, runtime-repair assessment), `process.rs` (spawn/env/path setup, readiness polling), `state.rs` (service state file I/O). `lib.rs` keeps only CLI/envelope dispatch and the few items shared across those modules. `engines/lemonade`'s `lib.rs` is **not yet modularized** — see EAI-7768. +`engines/vllm` follows full domain extraction: `runtime.rs` (TheRock/managed-runtime resolution), `install.rs` (ROCm/vLLM build-variant discovery, install execution, runtime-repair assessment), `process.rs` (spawn/env/path setup, readiness polling), `state.rs` (service state file I/O). `lib.rs` keeps only CLI/envelope dispatch and the few items shared across those modules. `engines/lemonade` follows the same pattern across six modules: `install.rs` (embeddable-runtime archive download/verify/extract, cache locking, manifest persistence), `backend_alignment.rs` (ROCm-SDK-to-llama.cpp-backend Tier 1 / Tier 2 / revert alignment, backend discovery/install/selection), `process.rs` (`lemond`/llama-server spawn, env/path setup, readiness and health polling), `runtime_dir.rs` (`XDG_RUNTIME_DIR` tier1/2/3 hardening for the spawned child), `direct_llama.rs` (Hugging Face checkpoint/GGUF resolution and the direct-llama-server serve path), `state.rs` (service state file I/O, device-policy/GPU-selection parsing, port/endpoint helpers). `lib.rs` keeps only CLI/envelope dispatch and the service-lifecycle orchestration that calls across those modules. diff --git a/engines/lemonade/src/backend_alignment.rs b/engines/lemonade/src/backend_alignment.rs new file mode 100644 index 0000000000..16f2d63eaa --- /dev/null +++ b/engines/lemonade/src/backend_alignment.rs @@ -0,0 +1,1785 @@ +// Copyright © Advanced Micro Devices, Inc., or its affiliates. +// +// SPDX-License-Identifier: MIT + +use anyhow::{Context, Result, bail}; +use rocm_core::{AppPaths, RocmCliConfig, active_managed_therock_version, runtime_is_linux}; +use serde_json::Value; +use std::fs; +use std::path::{Path, PathBuf}; +use std::process::{Command as ProcessCommand, Stdio}; +use std::time::Duration; + +use crate::direct_llama::find_llama_server_binary_for_backend; +use crate::install::{LemonadeInstallManifest, read_manifest}; +use crate::process::{ + LemonadeProcessEnvironment, apply_lemonade_process_environment, hide_child_console_window, + lemonade_process_environment, run_lemonade_backend_install, spawn_lemond, + wait_for_lemonade_cli_status, +}; +use crate::state::{free_local_port, terminate_pid}; +use crate::{DEFAULT_HOST, LLAMACPP_RECIPE, ROCM_BACKEND_NAME}; + +/// Resource file inside the extracted Lemonade embeddable pinning which +/// llama.cpp build (and paired ROCm version) `llamacpp:rocm` downloads. Its own +/// header comment invites hand-editing: "You can modify these values to pin +/// specific versions without rebuilding the application." +const BACKEND_VERSIONS_RESOURCE: &str = "resources/backend_versions.json"; + +/// Preferred llama.cpp backends, best first. Lemonade reports per-GPU support; +/// we pick the highest-priority backend it considers supported on this host. +/// GPU backends only — `cpu` is intentionally excluded so the router path never +/// selects CPU under the GPU-required policy (AGENTS.md §6, matching +/// `DIRECT_LLAMA_SERVER_BACKENDS`). When only CPU is usable, selection returns +/// `None` and the caller fails with actionable guidance. +const LLAMACPP_BACKEND_PRIORITY: [&str; 2] = ["rocm", "vulkan"]; + +#[derive(Debug, Clone)] +pub(crate) struct LemonadeRuntime { + pub(crate) manifest: LemonadeInstallManifest, +} + +/// The variable that opts a machine out of rocm-cli aligning Lemonade's +/// `llamacpp:rocm` backend to the active ROCm SDK. +/// +/// `resources/backend_versions.json` is a documented, first-party +/// customization point — its own header comment invites users to hand-edit it +/// to pin specific versions without rebuilding — so alignment running +/// unconditionally on every plain install would silently overwrite a user's +/// manual pin. Mirrors vLLM's `ROCM_CLI_DISABLE_TORCH_ALIGNMENT` +/// (`rocm_core::TORCH_ALIGNMENT_DISABLED_ENV`): presence is the signal, so any +/// value — including the empty string — disables the alignment. +pub const LEMONADE_BACKEND_ALIGNMENT_DISABLED_ENV: &str = + "ROCM_CLI_DISABLE_LEMONADE_BACKEND_ALIGNMENT"; + +/// Whether the user has opted out of rocm-cli aligning Lemonade's backend. +fn lemonade_backend_alignment_disabled() -> bool { + std::env::var_os(LEMONADE_BACKEND_ALIGNMENT_DISABLED_ENV).is_some() +} + +/// Point Lemonade's `llamacpp:rocm` backend at the ROCm version rocm-cli already has +/// installed and active, instead of Lemonade's hardcoded pin (which does not track +/// whatever the user separately installed via `rocm install sdk`). +/// +/// Best-effort and defensive at every step: if there is no managed rocm-cli SDK, the +/// pinned version already matches, or an aligned attempt cannot be verified to actually +/// resolve its GPU library, this falls back to (or reverts to) today's unmodified +/// pinned-version install — never a regression, and never a silent CPU fallback +/// (AGENTS.md §6): an aligned backend is only kept once [`rocm_backend_resolves`] +/// confirms it. +/// +/// Returns the aligned version string when one was applied and verified, so the caller +/// can surface it in the install response. +pub(crate) fn prepare_llamacpp_backend_for_active_rocm( + paths: &AppPaths, + manifest: &mut LemonadeInstallManifest, +) -> Result> { + prepare_llamacpp_backend_for_active_rocm_impl( + manifest, + lemonade_backend_alignment_disabled(), + || active_rocm_sdk_version_for_alignment(paths), + read_backend_versions_therock_version, + best_llamacpp_backend_for_host, + install_best_llamacpp_backend, + |manifest, backend_versions_path, target_version, pinned_version, disabled| { + align_llamacpp_backend_to_version( + manifest, + backend_versions_path, + target_version, + pinned_version, + disabled, + try_llamacpp_backend_alignment, + install_best_llamacpp_backend, + latest_llamacpp_rocm_stable_tag, + ) + }, + ) +} + +/// The gate and lookup chain behind [`prepare_llamacpp_backend_for_active_rocm`], +/// isolated from its real dependencies (the active-SDK/config lookup, the packaged +/// pin file, and the `lemond`-spawning backend probe) so this crate's own unit tests +/// can drive the disabled early return and the probe-error fallback -- the two +/// branches that decide whether any alignment is even attempted, and previously had +/// no coverage outside a `@nightly @requires-gpu` e2e lane. `disabled` is +/// [`lemonade_backend_alignment_disabled`]'s resolved value, passed in for the same +/// reason `align` (below) takes it as a parameter: a test can drive both branches +/// without touching process environment. +/// +/// This function, not `align`, is the real production enforcement point for the +/// opt-out: the single production call site passes `disabled` straight through to +/// `align`, but every one of *this* function's own skip branches returns before +/// `align` is ever reached, so `align`'s production call always sees `disabled == +/// false` here as well as there. `align`'s own `if disabled` branch exists only so +/// its unit tests can pin the state machine's half of the contract (the pin and +/// llama.cpp tag staying untouched) without duplicating this whole call chain. +#[allow(clippy::too_many_arguments)] +fn prepare_llamacpp_backend_for_active_rocm_impl( + manifest: &mut LemonadeInstallManifest, + disabled: bool, + mut active_rocm_sdk_version: impl FnMut() -> Result>, + mut pinned_version_lookup: impl FnMut(&Path) -> Option, + mut selected_backend_lookup: impl FnMut(&LemonadeInstallManifest) -> Result>, + mut fallback_install: impl FnMut(&mut LemonadeInstallManifest, bool) -> Result<()>, + mut align: impl FnMut( + &mut LemonadeInstallManifest, + &Path, + &str, + &str, + bool, + ) -> Result>, +) -> Result> { + // Announced here, unconditionally, rather than only where `align` happens to be + // reached: every one of the skip branches below -- not Linux, no active SDK + // version, unreadable pin, pin already matches, the vulkan-fallback probe -- + // would otherwise leave a user who set the variable unable to tell it took + // effect, exactly the silence this message exists to avoid. + if disabled { + eprintln!( + "Lemonade backend alignment is disabled by {LEMONADE_BACKEND_ALIGNMENT_DISABLED_ENV}; \ + using whatever backend_versions.json already pins." + ); + // Checked earliest, before even the cheap version/pin lookups below, let alone + // the vulkan-gate probe (a real `lemond` spawn) -- none of that work is worth + // doing when the opt-out means its result can't change what happens next. + fallback_install(manifest, false)?; + return Ok(None); + } + // Alignment is only ever verifiable on Linux ([`rocm_backend_resolves`] always + // reports unresolved elsewhere), so attempting it on Windows can only burn up to + // three multi-GB backend installs and a network round-trip for a guaranteed-futile + // outcome. Skip straight to the ordinary pinned-version install. + if !runtime_is_linux() { + fallback_install(manifest, false)?; + return Ok(None); + } + + let target_version = active_rocm_sdk_version().unwrap_or_else(|error| { + eprintln!( + "Warning: could not determine the active ROCm SDK version to align Lemonade's \ + backend with: {error:#}" + ); + None + }); + let Some(target_version) = target_version else { + fallback_install(manifest, false)?; + return Ok(None); + }; + + let backend_versions_path = manifest.runtime_dir.join(BACKEND_VERSIONS_RESOURCE); + let Some(pinned_version) = pinned_version_lookup(&backend_versions_path) else { + fallback_install(manifest, false)?; + return Ok(None); + }; + if pinned_version == target_version { + fallback_install(manifest, false)?; + return Ok(None); + } + + // Verification below requires the backend Lemonade selects to be `rocm` + // specifically ([`try_llamacpp_backend_alignment`]'s `has_usable_binary` gate) -- + // it falls back to `vulkan` on hosts where Lemonade's ROCm build is unsupported + // (WSL2 being the documented case), and no tier can ever pass there. Check which + // backend this host actually gets *before* patching `backend_versions.json`, the + // same guaranteed-futile case already short-circuited for Windows above: otherwise + // every install on such a host burns two forced reinstalls plus a GitHub round-trip + // discovering what this cheap query already knows. + let selected_backend = match selected_backend_lookup(manifest) { + Ok(selected_backend) => selected_backend, + Err(error) => { + eprintln!( + "Warning: could not determine which llama.cpp backend this host would select; \ + skipping ROCm backend alignment: {error:#}" + ); + fallback_install(manifest, false)?; + return Ok(None); + } + }; + if !should_attempt_llamacpp_backend_alignment(selected_backend.as_deref()) { + fallback_install(manifest, false)?; + return Ok(None); + } + + align( + manifest, + &backend_versions_path, + &target_version, + &pinned_version, + disabled, + ) +} + +/// The Tier 1 / Tier 2 / revert state machine, isolated from the config and active-SDK +/// lookups above so it can be exercised in tests against a temp `backend_versions.json` +/// with the install/align/latest-tag steps injected, instead of spawning a real +/// `lemond` and reaching GitHub. +/// +/// `disabled` is the resolved value of [`lemonade_backend_alignment_disabled`], passed +/// in rather than read here so a test can drive both branches without touching process +/// environment (mirrors vLLM's `torch_alignment_disabled` parameter). The real +/// production enforcement of the opt-out lives one level up, in +/// [`prepare_llamacpp_backend_for_active_rocm_impl`]: its own early return skips +/// every lookup this function depends on, so its single production call site always +/// passes `disabled == false` here. This function's own `if disabled` branch exists +/// only so its unit tests can pin this half of the contract (the pin and llama.cpp +/// tag staying untouched when the flag is set) without spinning up the whole call +/// chain above it. The user-facing announcement lives in +/// [`prepare_llamacpp_backend_for_active_rocm_impl`] as well, so it fires on every +/// skip path the opt-out affects, not only the one this function's own gate reaches. +#[allow(clippy::too_many_arguments)] +fn align_llamacpp_backend_to_version( + manifest: &mut LemonadeInstallManifest, + backend_versions_path: &Path, + target_version: &str, + pinned_version: &str, + disabled: bool, + mut align: impl FnMut(&mut LemonadeInstallManifest, &str, bool, &str) -> bool, + mut fallback_install: impl FnMut(&mut LemonadeInstallManifest, bool) -> Result<()>, + mut latest_tag: impl FnMut() -> Result, +) -> Result> { + if disabled { + fallback_install(manifest, false)?; + return Ok(None); + } + if let Err(error) = + write_backend_versions_therock_version(backend_versions_path, target_version) + { + eprintln!("Warning: could not pin Lemonade's backend to ROCm {target_version}: {error:#}"); + fallback_install(manifest, false)?; + return Ok(None); + } + + // Tier 1: keep Lemonade's own pinned llama.cpp build, just point it at the active + // ROCm version. Works when that specific build's release actually shipped a + // matching ROCm-version asset. Always forces a reinstall: this whole function + // only runs when the pin actually changed (the `pinned_version == target_version` + // check above), so a backend already on disk was installed against the OLD + // pin and must not be trusted just because it happens to still resolve. + if align(manifest, target_version, true, "its pinned llama.cpp build") { + return Ok(Some(target_version.to_owned())); + } + + // Tier 2: the pinned build may simply be too old to ever have shipped a + // matching ROCm-version asset — Lemonade only started publishing per-ROCm- + // version builds partway through its release history, and the embeddable's + // pin (frozen at whatever build was current when this Lemonade version was + // cut) does not track that. Point at the current newest build instead, still + // paired with the active ROCm version, forcing a real reinstall attempt + // regardless of whatever Tier 1 left on disk. + let pinned_tag = read_backend_versions_llamacpp_tag(backend_versions_path); + let latest_tag_applied = match latest_tag() { + Ok(latest_tag) if pinned_tag.as_deref() != Some(latest_tag.as_str()) => { + match write_backend_versions_llamacpp_tag(backend_versions_path, &latest_tag) { + Ok(()) => true, + Err(error) => { + eprintln!( + "Warning: could not pin Lemonade's backend to llama.cpp {latest_tag}: \ + {error:#}" + ); + false + } + } + } + // The pinned tag already is the latest — Tier 1 already tried it. + Ok(_) => false, + Err(error) => { + eprintln!("Warning: could not determine Lemonade's latest llama.cpp build: {error:#}"); + false + } + }; + if latest_tag_applied && align(manifest, target_version, true, "the latest llama.cpp build") { + return Ok(Some(target_version.to_owned())); + } + + // Neither tier produced a verified GPU backend. Revert everything — including + // the llama.cpp build tag, so the fallback install below isn't itself + // misdirected — and retry once more with Lemonade's original pinned defaults. + // Every step below is best-effort, matching its sibling warnings: a failure to + // restore the pin still lets the fallback install run rather than hard-failing + // the whole install, though it does leave the pin holding the unverified + // target version until a later run corrects it. + eprintln!( + "Warning: could not align Lemonade's ROCm backend to {target_version}; reverting to the \ + default pinned version." + ); + if let Err(error) = + write_backend_versions_therock_version(backend_versions_path, pinned_version) + { + eprintln!( + "Warning: could not restore Lemonade's default pinned backend version \ + ({pinned_version}): {error:#}" + ); + } + if latest_tag_applied { + // Best-effort, matching the therock.version restore above: a failure here + // must not skip the fallback reinstall below. Restore the original tag when + // there was one; otherwise Tier 2 pinned a key that did not exist before, so + // removing it (not writing some placeholder) is what "reverted" means. + let restore_result = match pinned_tag.as_deref() { + Some(tag) => write_backend_versions_llamacpp_tag(backend_versions_path, tag), + None => remove_backend_versions_llamacpp_tag(backend_versions_path), + }; + if let Err(error) = restore_result { + eprintln!( + "Warning: could not restore Lemonade's default pinned llama.cpp build tag: \ + {error:#}" + ); + } + } + let aside_dirs = set_aside_rocm_llamacpp_backend_dirs(manifest); + match fallback_install(manifest, true) { + Ok(()) => { + resolve_rocm_llamacpp_backend_dirs_aside(manifest, &aside_dirs, true); + Ok(None) + } + Err(error) => { + // Restore the pre-alignment backend before propagating: a failed + // fallback install must not leave the host with no ROCm backend at all. + resolve_rocm_llamacpp_backend_dirs_aside(manifest, &aside_dirs, false); + Err(error) + } + } +} + +/// Attempt one llama.cpp backend install aligned to `target_version`'s ROCm pairing +/// (already written into `resources/backend_versions.json`), then verify — never +/// simply trust Lemonade's own reported success — that the resulting GPU backend +/// actually resolves against rocm-cli's active ROCm SDK (AGENTS.md §6: no silent CPU +/// fallback). `force_reinstall` must be `true` for every caller here: alignment is +/// only ever attempted when the pin actually changed, and a skipped "already +/// installed" reinstall would leave a stale, pre-alignment binary that can still +/// pass the `ldd`-resolves check below, reporting success for a version that was +/// never actually installed. Returns whether a verified backend is now in place; on +/// success, `manifest.backend_name` is updated to match it. +fn try_llamacpp_backend_alignment( + manifest: &mut LemonadeInstallManifest, + target_version: &str, + force_reinstall: bool, + attempt_label: &str, +) -> bool { + debug_assert!( + force_reinstall, + "every alignment attempt must force a reinstall; see the doc comment above" + ); + let aside_dirs = set_aside_rocm_llamacpp_backend_dirs(manifest); + let install_result = install_best_llamacpp_backend(manifest, force_reinstall); + // `ensure_best_llamacpp_backend` records the backend it attempted into + // `manifest.backend_name` before the fallible install step runs, so this is accurate + // even when `install_result` is `Err` below. Scoping to it — rather than scanning + // every backend directory — means a stale `rocm-*` directory left by an earlier + // attempt can never be mistaken for this round's verification, including when this + // round actually picked `vulkan`. + let backend_name = manifest.backend_name.clone(); + let has_usable_binary = backend_name == ROCM_BACKEND_NAME + && lemonade_process_environment().is_ok_and(|process_env| { + find_llama_server_binary_for_backend(manifest, &backend_name) + .is_some_and(|binary| rocm_backend_resolves(&binary, &process_env)) + }); + let succeeded = llamacpp_backend_alignment_succeeded(&install_result, has_usable_binary); + resolve_rocm_llamacpp_backend_dirs_aside(manifest, &aside_dirs, succeeded); + if !succeeded { + match install_result { + Ok(()) => eprintln!( + "Warning: {attempt_label} installed for ROCm {target_version}, but its GPU \ + backend does not resolve against rocm-cli's active ROCm SDK; not using it." + ), + Err(error) if has_usable_binary => eprintln!( + "Warning: could not install {attempt_label} for ROCm {target_version} \ + ({error:#}); a backend binary is present but may predate this alignment \ + attempt, so it is not trusted without a completed install." + ), + Err(error) => eprintln!( + "Warning: could not install {attempt_label} for ROCm {target_version}: {error:#}" + ), + } + return false; + } + eprintln!("Aligned Lemonade's ROCm backend to {target_version} using {attempt_label}."); + true +} + +/// Whether an alignment attempt counts as a verified success: the install must +/// have actually completed, not merely have left behind some binary that +/// happens to resolve. `ldd`-based verification cannot tell a stale build +/// (paired with a different ROCm version, possibly left over from before this +/// attempt) from a version-matched one, so a failed install must never be +/// forgiven by a leftover binary passing that check. +const fn llamacpp_backend_alignment_succeeded( + install_result: &Result<()>, + has_usable_binary: bool, +) -> bool { + install_result.is_ok() && has_usable_binary +} + +/// GitHub repo backing Lemonade's `llamacpp:rocm-stable` builds. Bumping +/// `resources/backend_versions.json`'s `therock.version` alone doesn't get a +/// ROCm-version-matched build if the pinned tag predates that ROCm version's +/// support: Lemonade only started publishing per-ROCm-version assets partway +/// through this repo's release history, and the embeddable's own pin can be +/// (and, at the time of writing, is) older than that. +const LLAMACPP_ROCM_STABLE_REPO: &str = "lemonade-sdk/llama.cpp"; + +/// The newest published release tag for [`LLAMACPP_ROCM_STABLE_REPO`] (e.g. +/// `b10952`), queried directly rather than assumed — the whole reason Tier 2 +/// exists is that the embeddable's own pinned tag cannot be trusted to have a +/// matching-ROCm-version asset. +fn latest_llamacpp_rocm_stable_tag() -> Result { + let url = format!("https://api.github.com/repos/{LLAMACPP_ROCM_STABLE_REPO}/releases/latest"); + let timeout = Duration::from_secs(15); + // `timeout_connect` takes precedence over `timeout` and defaults to 30s, so without + // it a host that blackholes rather than refuses would stall well past the intended + // ceiling. + let agent = ureq::AgentBuilder::new() + .timeout_connect(timeout) + .timeout(timeout) + .build(); + let response = agent + .get(&url) + .set("User-Agent", "rocm-cli") + .call() + .with_context(|| format!("failed to query {url}"))?; + let body: Value = response + .into_json() + .with_context(|| format!("failed to parse response from {url}"))?; + body.get("tag_name") + .and_then(Value::as_str) + .map(str::to_owned) + .with_context(|| format!("{url} response is missing 'tag_name'")) +} + +/// Read `resources/backend_versions.json`'s pinned `llamacpp.rocm-stable` build tag. +fn read_backend_versions_llamacpp_tag(path: &Path) -> Option { + let bytes = fs::read(path).ok()?; + let value: Value = serde_json::from_slice(&bytes).ok()?; + value + .get("llamacpp")? + .get("rocm-stable")? + .as_str() + .map(str::to_owned) +} + +/// Overwrite `resources/backend_versions.json`'s `llamacpp.rocm-stable` build tag in +/// place, preserving every other key. +fn write_backend_versions_llamacpp_tag(path: &Path, tag: &str) -> Result<()> { + let bytes = fs::read(path).with_context(|| format!("failed to read {}", path.display()))?; + let mut value: Value = serde_json::from_slice(&bytes) + .with_context(|| format!("failed to parse {}", path.display()))?; + value + .get_mut("llamacpp") + .and_then(Value::as_object_mut) + .with_context(|| format!("{} has no 'llamacpp' object to patch", path.display()))? + .insert("rocm-stable".to_owned(), Value::String(tag.to_owned())); + fs::write(path, serde_json::to_vec_pretty(&value)?) + .with_context(|| format!("failed to write {}", path.display())) +} + +/// Remove `resources/backend_versions.json`'s `llamacpp.rocm-stable` key entirely, +/// restoring the "never pinned" state — the counterpart to +/// [`write_backend_versions_llamacpp_tag`] for reverting Tier 2's pin when there was +/// no prior tag to restore it to. +fn remove_backend_versions_llamacpp_tag(path: &Path) -> Result<()> { + let bytes = fs::read(path).with_context(|| format!("failed to read {}", path.display()))?; + let mut value: Value = serde_json::from_slice(&bytes) + .with_context(|| format!("failed to parse {}", path.display()))?; + value + .get_mut("llamacpp") + .and_then(Value::as_object_mut) + .with_context(|| format!("{} has no 'llamacpp' object to patch", path.display()))? + .remove("rocm-stable"); + fs::write(path, serde_json::to_vec_pretty(&value)?) + .with_context(|| format!("failed to write {}", path.display())) +} + +/// The active rocm-cli-managed ROCm SDK version to align Lemonade's backend to, +/// restricted to plain `X.Y.Z` release versions. Nightly builds carry a date suffix +/// (e.g. `7.14.0a20260601`) that does not correspond to any llama.cpp build tag +/// Lemonade publishes, so those are left alone rather than guessed at. +/// +/// Errors (rather than defaulting) when the config can't be loaded: an empty default +/// config would discard the active runtime key and let version selection guess the +/// most-recently-installed runtime, then mutate Lemonade based on that guess. +fn active_rocm_sdk_version_for_alignment(paths: &AppPaths) -> Result> { + let config = RocmCliConfig::load(paths)?; + let Some(version) = active_managed_therock_version(paths, &config)? else { + return Ok(None); + }; + Ok(looks_like_plain_semver(&version).then_some(version)) +} + +/// Whether `version` is a plain `major.minor.patch` string with no build/date suffix +/// (e.g. `10.0.0`, not `7.14.0a20260601`). +fn looks_like_plain_semver(version: &str) -> bool { + let parts: Vec<&str> = version.split('.').collect(); + parts.len() == 3 + && parts + .iter() + .all(|part| !part.is_empty() && part.chars().all(|c| c.is_ascii_digit())) +} + +/// Read `resources/backend_versions.json`'s `therock.version` — Lemonade's pin for +/// which private ROCm runtime its `llamacpp:rocm` backend downloads. +fn read_backend_versions_therock_version(path: &Path) -> Option { + let bytes = fs::read(path).ok()?; + let value: Value = serde_json::from_slice(&bytes).ok()?; + value + .get("therock")? + .get("version")? + .as_str() + .map(str::to_owned) +} + +/// Overwrite `resources/backend_versions.json`'s `therock.version` in place, +/// preserving every other key. Lemonade's own comment in that file invites exactly +/// this: "You can modify these values to pin specific versions without rebuilding the +/// application." +fn write_backend_versions_therock_version(path: &Path, version: &str) -> Result<()> { + let bytes = fs::read(path).with_context(|| format!("failed to read {}", path.display()))?; + let mut value: Value = serde_json::from_slice(&bytes) + .with_context(|| format!("failed to parse {}", path.display()))?; + value + .get_mut("therock") + .and_then(Value::as_object_mut) + .with_context(|| format!("{} has no 'therock' object to patch", path.display()))? + .insert("version".to_owned(), Value::String(version.to_owned())); + fs::write(path, serde_json::to_vec_pretty(&value)?) + .with_context(|| format!("failed to write {}", path.display())) +} + +/// ROCm shared-library sonames the `llamacpp:rocm` backend's `libggml-hip.so` links +/// against. If any fail to resolve, the backend would silently fall back to CPU +/// inference at runtime instead of failing loudly — unacceptable under the +/// GPU-required policy (AGENTS.md §6), so this must be verified, not assumed from +/// version numbers matching. +/// +/// Linux-only, like the `ldd`-based verification it exists for: the non-Linux +/// `rocm_backend_resolves` stub below never references it, so leaving it +/// ungated makes it dead code — and a build failure under `-D warnings` — on +/// every other target. +#[cfg(target_os = "linux")] +const ROCM_BACKEND_REQUIRED_SONAMES: [&str; 4] = [ + "libhipblas.so", + "librocblas.so", + "libamdhip64.so", + "libhsa-runtime64.so", +]; + +/// Whether the ROCm GPU backend beside `llama_server_binary` actually resolves its +/// required ROCm libraries under `process_env` (rocm-cli's injected ROCm environment), +/// checked with `ldd` rather than assumed. `false` on any I/O/parse failure. +#[cfg(target_os = "linux")] +fn rocm_backend_resolves( + llama_server_binary: &Path, + process_env: &LemonadeProcessEnvironment, +) -> bool { + let Some(backend_dir) = llama_server_binary.parent() else { + return false; + }; + let hip_backend = backend_dir.join("libggml-hip.so"); + if !hip_backend.is_file() { + return false; + } + let mut command = ProcessCommand::new("ldd"); + command + .arg(&hip_backend) + .stdin(Stdio::null()) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()); + if apply_lemonade_process_environment(&mut command, process_env).is_err() { + return false; + } + let Ok(output) = command.output() else { + return false; + }; + output.status.success() + && ldd_output_resolves_all( + &String::from_utf8_lossy(&output.stdout), + &ROCM_BACKEND_REQUIRED_SONAMES, + ) +} + +/// Off Linux, `ldd`-based verification is not exercised: a version-aligned backend is +/// never accepted without it, so this always reports unresolved. +#[cfg(not(target_os = "linux"))] +const fn rocm_backend_resolves( + _llama_server_binary: &Path, + _process_env: &LemonadeProcessEnvironment, +) -> bool { + false +} + +/// Whether every `required` soname prefix appears in `ldd` output with a resolved +/// path (i.e. not `=> not found`). Linux-only: see [`ROCM_BACKEND_REQUIRED_SONAMES`]. +#[cfg(target_os = "linux")] +fn ldd_output_resolves_all(output: &str, required: &[&str]) -> bool { + required.iter().all(|soname| { + output.lines().any(|line| { + let line = line.trim(); + line.starts_with(soname) && !line.contains("not found") + }) + }) +} + +fn install_best_llamacpp_backend( + manifest: &mut LemonadeInstallManifest, + force_reinstall: bool, +) -> Result<()> { + let port = free_local_port()?; + let log_path_buf = manifest.runtime_dir.join("install-lemond.log"); + let log_path = Some(log_path_buf.as_path()); + let process_env = lemonade_process_environment()?; + let mut child = spawn_lemond(manifest, DEFAULT_HOST, port, log_path, &process_env)?; + let result = (|| -> Result { + wait_for_lemonade_cli_status( + manifest, + DEFAULT_HOST, + port, + Duration::from_secs(30), + log_path, + &process_env, + )?; + ensure_best_llamacpp_backend(manifest, DEFAULT_HOST, port, &process_env, force_reinstall) + })(); + let _ = terminate_pid(child.id(), true); + let _ = child.wait(); + manifest.backend_name = result?; + Ok(()) +} + +/// Whether the vulkan-gate probe's result means alignment could possibly succeed -- +/// only when the host would select Lemonade's `rocm` backend by itself. Any other +/// selection (including the probe itself failing to determine one) means every +/// alignment tier is guaranteed to fail ([`try_llamacpp_backend_alignment`]'s +/// `has_usable_binary` gate requires the `rocm` backend specifically), so callers +/// should skip straight to an ordinary install instead of patching +/// `backend_versions.json` and burning two forced reinstalls plus a GitHub round-trip +/// discovering what this decision already knows. A pure function so this gate is +/// unit-testable without spawning the real `lemond` process +/// [`best_llamacpp_backend_for_host`] needs. +fn should_attempt_llamacpp_backend_alignment(selected_backend: Option<&str>) -> bool { + selected_backend == Some(ROCM_BACKEND_NAME) +} + +/// Which llama.cpp backend Lemonade would select on this host +/// (`LLAMACPP_BACKEND_PRIORITY`), without installing anything -- mirrors +/// [`install_best_llamacpp_backend`]'s spawn/query steps but stops short of the +/// install, so the ROCm backend alignment dance can check whether it is even +/// reachable before patching `backend_versions.json` or forcing a single reinstall. +fn best_llamacpp_backend_for_host(manifest: &LemonadeInstallManifest) -> Result> { + let port = free_local_port()?; + let log_path_buf = manifest.runtime_dir.join("install-lemond.log"); + let log_path = Some(log_path_buf.as_path()); + let process_env = lemonade_process_environment()?; + let mut child = spawn_lemond(manifest, DEFAULT_HOST, port, log_path, &process_env)?; + let result = (|| -> Result> { + wait_for_lemonade_cli_status( + manifest, + DEFAULT_HOST, + port, + Duration::from_secs(30), + log_path, + &process_env, + )?; + let listing = run_lemonade_backends_list(manifest, &process_env)?; + let backends = parse_llamacpp_backend_statuses(&listing); + Ok(select_best_llamacpp_backend(&backends).map(|(name, _)| name)) + })(); + let _ = terminate_pid(child.id(), true); + let _ = child.wait(); + result +} + +/// Ask Lemonade which llama.cpp backends it supports on this GPU, choose the best +/// one (`LLAMACPP_BACKEND_PRIORITY`), install it if necessary, and return its name. +/// Retry the backend install itself once, rather than the whole Lemonade runtime +/// preparation. The embeddable download already has bounded transport retries; +/// repeating that outer operation would redo deterministic failures and may +/// re-extract a healthy runtime. A backend subprocess can instead fail after a +/// completed download when its connection to lemond is interrupted, and a second +/// call can reuse the backend cache immediately without a delay. +pub(crate) fn install_llamacpp_backend_with_retry( + mut install: impl FnMut() -> Result<()>, +) -> Result<()> { + match install() { + Ok(()) => Ok(()), + Err(first_error) => { + eprintln!("Lemonade backend installation failed; retrying once: {first_error:#}"); + install().with_context(|| { + "Lemonade backend installation failed again; run `rocm engines install \ + lemonade --reinstall` and then retry `rocm serve`" + }) + } + } +} + +pub(crate) fn ensure_best_llamacpp_backend( + manifest: &mut LemonadeInstallManifest, + host: &str, + port: u16, + process_env: &LemonadeProcessEnvironment, + force_reinstall: bool, +) -> Result { + let listing = run_lemonade_backends_list(manifest, process_env)?; + let backends = parse_llamacpp_backend_statuses(&listing); + let Some((backend, already_installed)) = select_best_llamacpp_backend(&backends) else { + bail!( + "Lemonade reports no supported GPU llama.cpp backend for this host (status: {}). \ + The GPU-required policy does not fall back to CPU; install a ROCm or Vulkan \ + backend (e.g. `rocm engines install lemonade`) or verify the GPU driver with \ + `rocm examine`.", + describe_llamacpp_backends(&backends) + ); + }; + // Record the attempted backend before the fallible install step below, so a caller + // verifying the result can scope its check to this backend even when the install + // itself errors out. + manifest.backend_name = backend.clone(); + if already_installed && !force_reinstall { + eprintln!("Using installed Lemonade {LLAMACPP_RECIPE}:{backend} backend."); + } else { + eprintln!("Installing Lemonade {LLAMACPP_RECIPE}:{backend} backend..."); + install_llamacpp_backend_with_retry(|| { + run_lemonade_backend_install(manifest, host, port, &backend, process_env) + })?; + } + Ok(backend) +} + +/// Parse `lemonade backends` table output into `(backend, status)` pairs for the +/// `llamacpp` recipe only. Status is one of `installed`/`installable`/`unsupported`. +fn parse_llamacpp_backend_statuses(output: &str) -> Vec<(String, String)> { + const RECIPES: [&str; 8] = [ + "flm", + "kokoro", + "llamacpp", + "ryzenai-llm", + "sd-cpp", + "vllm", + "whispercpp", + "embeddings", + ]; + const STATUSES: [&str; 3] = ["installed", "installable", "unsupported"]; + let mut current_recipe = ""; + let mut result = Vec::new(); + for line in output.lines() { + let tokens: Vec<&str> = line.split_whitespace().collect(); + let Some(&first) = tokens.first() else { + continue; + }; + if first == "Recipe" || first.starts_with("---") { + continue; + } + // A row either starts with a recipe name (recipe + its first backend) or, + // for grouped recipes, with a backend name continuing the previous recipe. + let (backend, rest) = if RECIPES.contains(&first) { + current_recipe = match RECIPES.iter().find(|r| **r == first) { + Some(r) => r, + None => current_recipe, + }; + match tokens.get(1) { + Some(backend) => (*backend, &tokens[2..]), + None => continue, + } + } else { + (first, &tokens[1..]) + }; + if current_recipe != LLAMACPP_RECIPE { + continue; + } + if let Some(status) = rest.iter().find(|t| STATUSES.contains(t)) { + result.push((backend.to_owned(), (*status).to_owned())); + } + } + result +} + +/// Choose the highest-priority backend Lemonade considers usable (installed or +/// installable). Returns `(backend, already_installed)`. +fn select_best_llamacpp_backend(backends: &[(String, String)]) -> Option<(String, bool)> { + for candidate in LLAMACPP_BACKEND_PRIORITY { + if let Some((name, status)) = backends + .iter() + .find(|(b, s)| b == candidate && (s == "installed" || s == "installable")) + { + return Some((name.clone(), status == "installed")); + } + } + None +} + +fn describe_llamacpp_backends(backends: &[(String, String)]) -> String { + if backends.is_empty() { + return "none reported".to_owned(); + } + backends + .iter() + .map(|(b, s)| format!("{b}={s}")) + .collect::>() + .join(", ") +} + +fn run_lemonade_backends_list( + manifest: &LemonadeInstallManifest, + process_env: &LemonadeProcessEnvironment, +) -> Result { + let mut command = ProcessCommand::new(&manifest.lemonade); + command + .arg("backends") + .stdin(Stdio::null()) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()); + apply_lemonade_process_environment(&mut command, process_env)?; + hide_child_console_window(&mut command); + let output = command + .output() + .with_context(|| format!("failed to run {}", manifest.lemonade.display()))?; + if !output.status.success() { + bail!( + "Lemonade backends query failed ({}): {}", + output.status, + String::from_utf8_lossy(&output.stderr).trim() + ); + } + Ok(String::from_utf8_lossy(&output.stdout).into_owned()) +} + +pub(crate) fn resolve_runtime() -> Result { + let paths = AppPaths::discover()?; + let manifest = read_manifest(&paths)?; + if !manifest.lemond.is_file() { + bail!( + "Lemonade runtime is missing {}; run `rocm engines install lemonade`", + manifest.lemond.display() + ); + } + Ok(LemonadeRuntime { manifest }) +} + +/// The ROCm-family llama.cpp backend directory names under `bin/llamacpp/`. Lemonade's +/// installer extracts each build into its own build-numbered subdirectory rather than +/// replacing one in place, so all three names can accumulate across attempts. +pub(crate) const ROCM_LLAMACPP_BACKEND_DIRS: [&str; 3] = ["rocm-stable", "rocm-nightly", "rocm"]; + +/// Suffix appended to a ROCm-family backend directory moved aside by +/// [`set_aside_rocm_llamacpp_backend_dirs`]. +const BACKEND_DIR_ASIDE_SUFFIX: &str = ".pre-alignment"; + +/// Move every existing ROCm-family llama.cpp backend directory aside (rather than +/// deleting it outright) before a forced alignment reinstall, so a failed install can +/// be recovered from via [`resolve_rocm_llamacpp_backend_dirs_aside`] instead of +/// leaving the host with no ROCm backend at all. Lemonade's installer extracts each +/// build into its own build-numbered subdirectory rather than replacing one in place, +/// so a tier that installs a different `therock.version`/llama.cpp tag than a prior +/// attempt (or the pinned default) can otherwise leave two builds side by side -- +/// after which [`crate::direct_llama::find_binary_in`]'s directory-order fallback, used by both alignment +/// verification and `rocm serve`, may resolve to whichever build a rejected tier +/// installed instead of the one just verified. +/// +/// Returns the `(original, aside)` pairs actually moved; only those need resolving. +fn set_aside_rocm_llamacpp_backend_dirs( + manifest: &LemonadeInstallManifest, +) -> Vec<(PathBuf, PathBuf)> { + let llamacpp_dir = manifest.runtime_dir.join("bin").join("llamacpp"); + ROCM_LLAMACPP_BACKEND_DIRS + .iter() + .filter_map(|backend| { + let original = llamacpp_dir.join(backend); + if !original.exists() { + return None; + } + let aside = llamacpp_dir.join(format!("{backend}{BACKEND_DIR_ASIDE_SUFFIX}")); + // A leftover aside path from an earlier, interrupted attempt must not + // block this rename. + let _ = fs::remove_dir_all(&aside); + match fs::rename(&original, &aside) { + Ok(()) => Some((original, aside)), + Err(error) => { + eprintln!( + "Warning: could not move {} aside before backend alignment: {error:#}", + original.display() + ); + None + } + } + }) + .collect() +} + +/// Resolve the aside directories from [`set_aside_rocm_llamacpp_backend_dirs`]: on +/// success, discard them -- the fresh install replaced what they held. On failure, +/// clear every [`ROCM_LLAMACPP_BACKEND_DIRS`] name (not just the ones that were +/// actually asided) before moving each `aside` back into place: a failed install can +/// land its output under a *different* ROCm-family name than the one that was +/// asided (Lemonade extracts each release into its own build-numbered directory +/// rather than replacing one in place), and [`crate::direct_llama::find_binary_in`]'s priority-ordered +/// lookup would otherwise let that stray, unverified directory shadow the good +/// build this function just restored. Best-effort: a failure here just risks the +/// same stale-directory ambiguity the aside exists to prevent, not the reinstall +/// itself. +fn resolve_rocm_llamacpp_backend_dirs_aside( + manifest: &LemonadeInstallManifest, + aside_dirs: &[(PathBuf, PathBuf)], + success: bool, +) { + if success { + for (_, aside) in aside_dirs { + let _ = fs::remove_dir_all(aside); + } + return; + } + let llamacpp_dir = manifest.runtime_dir.join("bin").join("llamacpp"); + for backend in ROCM_LLAMACPP_BACKEND_DIRS { + let _ = fs::remove_dir_all(llamacpp_dir.join(backend)); + } + for (original, aside) in aside_dirs { + let _ = fs::rename(aside, original); + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::install::platform_binary_name; + use anyhow::anyhow; + use serde_json::json; + use std::fs; + + /// A fresh scratch directory under the crate's `target/`. The base is + /// `CARGO_MANIFEST_DIR`, a compile-time constant, so the path never derives from a + /// runtime environment read. + fn scratch_dir(tag: &str) -> PathBuf { + let dir = Path::new(env!("CARGO_MANIFEST_DIR")) + .join("target") + .join(format!("lemonade-fs-test-{tag}")); + let _ = fs::remove_dir_all(&dir); + fs::create_dir_all(&dir).unwrap(); + dir + } + + #[test] + fn backend_install_succeeds_without_retry() { + let mut attempts = 0; + + install_llamacpp_backend_with_retry(|| { + attempts += 1; + Ok(()) + }) + .unwrap(); + + assert_eq!(attempts, 1); + } + + #[test] + fn backend_install_recovers_on_the_second_attempt() { + let mut attempts = 0; + + install_llamacpp_backend_with_retry(|| { + attempts += 1; + if attempts == 1 { + bail!("first backend connection was interrupted"); + } + Ok(()) + }) + .unwrap(); + + assert_eq!(attempts, 2); + } + + #[test] + fn backend_install_stops_after_one_retry_with_reinstall_guidance() { + let mut attempts = 0; + + let error = install_llamacpp_backend_with_retry(|| { + attempts += 1; + if attempts == 1 { + bail!("first backend connection was interrupted"); + } + bail!("second backend connection was interrupted"); + }) + .unwrap_err(); + let rendered = format!("{error:#}"); + + assert_eq!(attempts, 2); + assert!( + rendered.contains("second backend connection was interrupted"), + "{rendered}" + ); + assert!( + !rendered.contains("first backend connection was interrupted"), + "the terminal error must be the retry's failure: {rendered}" + ); + assert!( + rendered.contains("rocm engines install lemonade --reinstall"), + "{rendered}" + ); + assert!(rendered.contains("retry `rocm serve`"), "{rendered}"); + } + + fn test_manifest(runtime_dir: PathBuf) -> LemonadeInstallManifest { + LemonadeInstallManifest { + env_id: "test".to_owned(), + version: rocm_deps::LEMONADE_VERSION.to_owned(), + runtime_dir, + lemond: PathBuf::from("lemond"), + lemonade: PathBuf::from("lemonade"), + backend_recipe: LLAMACPP_RECIPE.to_owned(), + backend_name: ROCM_BACKEND_NAME.to_owned(), + installed_at_unix_ms: 0, + } + } + + #[test] + fn plain_semver_accepts_release_versions_only() { + assert!(looks_like_plain_semver("10.0.0")); + assert!(looks_like_plain_semver("7.13.0")); + assert!(!looks_like_plain_semver("7.14.0a20260601")); + assert!(!looks_like_plain_semver("10.0")); + assert!(!looks_like_plain_semver("10.0.0.1")); + assert!(!looks_like_plain_semver("")); + } + + #[test] + fn backend_versions_therock_version_round_trips() { + let dir = scratch_dir("backend-versions-round-trip"); + let path = dir.join("backend_versions.json"); + fs::write( + &path, + serde_json::to_vec_pretty(&json!({ + "llamacpp": { "rocm-stable": "b9752" }, + "therock": { "version": "7.13.0", "architectures": ["gfx1151"] }, + })) + .unwrap(), + ) + .unwrap(); + + assert_eq!( + read_backend_versions_therock_version(&path), + Some("7.13.0".to_owned()) + ); + + write_backend_versions_therock_version(&path, "10.0.0").unwrap(); + assert_eq!( + read_backend_versions_therock_version(&path), + Some("10.0.0".to_owned()) + ); + + // Every other key, including sibling keys under `therock`, is preserved. + let value: Value = serde_json::from_slice(&fs::read(&path).unwrap()).unwrap(); + assert_eq!(value["llamacpp"]["rocm-stable"], "b9752"); + assert_eq!(value["therock"]["architectures"][0], "gfx1151"); + + fs::remove_dir_all(&dir).ok(); + } + + #[test] + fn backend_versions_therock_version_missing_object_is_an_error() { + let dir = scratch_dir("backend-versions-missing-therock"); + let path = dir.join("backend_versions.json"); + fs::write( + &path, + serde_json::to_vec_pretty(&json!({"llamacpp": {}})).unwrap(), + ) + .unwrap(); + + assert_eq!(read_backend_versions_therock_version(&path), None); + assert!(write_backend_versions_therock_version(&path, "10.0.0").is_err()); + + fs::remove_dir_all(&dir).ok(); + } + + #[test] + fn backend_versions_llamacpp_tag_round_trips() { + let dir = scratch_dir("backend-versions-tag-round-trip"); + let path = dir.join("backend_versions.json"); + fs::write( + &path, + serde_json::to_vec_pretty(&json!({ + "llamacpp": { "rocm-stable": "b9752", "vulkan": "b9747" }, + "therock": { "version": "7.13.0" }, + })) + .unwrap(), + ) + .unwrap(); + + assert_eq!( + read_backend_versions_llamacpp_tag(&path), + Some("b9752".to_owned()) + ); + + write_backend_versions_llamacpp_tag(&path, "b10952").unwrap(); + assert_eq!( + read_backend_versions_llamacpp_tag(&path), + Some("b10952".to_owned()) + ); + + // Sibling keys, including the unrelated `llamacpp.vulkan` tag and the + // whole `therock` object, are preserved. + let value: Value = serde_json::from_slice(&fs::read(&path).unwrap()).unwrap(); + assert_eq!(value["llamacpp"]["vulkan"], "b9747"); + assert_eq!(value["therock"]["version"], "7.13.0"); + + fs::remove_dir_all(&dir).ok(); + } + + #[test] + fn backend_versions_llamacpp_tag_missing_object_is_an_error() { + let dir = scratch_dir("backend-versions-missing-llamacpp"); + let path = dir.join("backend_versions.json"); + fs::write( + &path, + serde_json::to_vec_pretty(&json!({"therock": {}})).unwrap(), + ) + .unwrap(); + + assert_eq!(read_backend_versions_llamacpp_tag(&path), None); + assert!(write_backend_versions_llamacpp_tag(&path, "b10952").is_err()); + + fs::remove_dir_all(&dir).ok(); + } + + fn write_backend_versions_fixture(path: &Path, therock_version: &str, llamacpp_tag: &str) { + fs::write( + path, + serde_json::to_vec_pretty(&json!({ + "llamacpp": { "rocm-stable": llamacpp_tag }, + "therock": { "version": therock_version }, + })) + .unwrap(), + ) + .unwrap(); + } + + #[test] + fn set_aside_then_discard_removes_every_rocm_family_dir_but_not_vulkan() { + // The bug this guards: Tier 2 installing a different llama.cpp tag than a + // prior attempt left two build directories side by side, and the + // directory-order lookup in `find_binary_in` could resolve to whichever one + // a rejected tier installed instead of the one alignment just verified. + let dir = scratch_dir("aside-then-discard-rocm-backend-dirs"); + let runtime_dir = dir.join("runtime"); + let llamacpp = runtime_dir.join("bin").join("llamacpp"); + let server = platform_binary_name("llama-server"); + for backend in ["rocm-stable", "rocm-nightly", "rocm", "vulkan"] { + let backend_dir = llamacpp.join(backend); + fs::create_dir_all(&backend_dir).unwrap(); + fs::write(backend_dir.join(&server), b"x").unwrap(); + } + let manifest = test_manifest(runtime_dir); + + let aside_dirs = set_aside_rocm_llamacpp_backend_dirs(&manifest); + for backend in ["rocm-stable", "rocm-nightly", "rocm"] { + assert!( + !llamacpp.join(backend).exists(), + "{backend} should have been moved aside" + ); + } + resolve_rocm_llamacpp_backend_dirs_aside(&manifest, &aside_dirs, true); + + for backend in ["rocm-stable", "rocm-nightly", "rocm"] { + assert!( + !llamacpp.join(backend).exists(), + "{backend} should still be gone after a discard" + ); + assert!( + !llamacpp + .join(format!("{backend}{BACKEND_DIR_ASIDE_SUFFIX}")) + .exists(), + "{backend}'s aside copy should have been discarded on success" + ); + } + assert!( + llamacpp.join("vulkan").join(&server).is_file(), + "vulkan is untouched by ROCm alignment and must survive" + ); + fs::remove_dir_all(&dir).ok(); + } + + #[test] + fn resolve_rocm_llamacpp_backend_dirs_aside_restores_on_failure() { + // The bug this guards: a failed final install (e.g. the revert branch's + // fallback_install erroring out) must never leave the host with zero ROCm + // backend directories -- the whole reason to set builds aside instead of + // deleting them outright. + let dir = scratch_dir("aside-restores-on-failure"); + let runtime_dir = dir.join("runtime"); + let llamacpp = runtime_dir.join("bin").join("llamacpp"); + let server = platform_binary_name("llama-server"); + let rocm_dir = llamacpp.join("rocm"); + fs::create_dir_all(&rocm_dir).unwrap(); + fs::write(rocm_dir.join(&server), b"original").unwrap(); + let manifest = test_manifest(runtime_dir); + + let aside_dirs = set_aside_rocm_llamacpp_backend_dirs(&manifest); + assert!( + !rocm_dir.exists(), + "the original must be moved aside before the install is attempted" + ); + // Simulate a failed install leaving nothing behind at the original path. + resolve_rocm_llamacpp_backend_dirs_aside(&manifest, &aside_dirs, false); + + assert_eq!( + fs::read(rocm_dir.join(&server)).unwrap(), + b"original", + "the pre-existing backend must be restored after a failed install" + ); + assert!( + !llamacpp + .join(format!("rocm{BACKEND_DIR_ASIDE_SUFFIX}")) + .exists(), + "the aside copy must be moved back, not left behind" + ); + fs::remove_dir_all(&dir).ok(); + } + + #[test] + fn resolve_rocm_llamacpp_backend_dirs_aside_removes_a_failed_installs_stray_residue() { + // The bug this guards: a failed install can land its (broken) output under a + // DIFFERENT ROCm-family name than the one that was asided -- Lemonade + // extracts each release into its own build-numbered directory rather than + // replacing one in place. Only cleaning up the asided name left that stray + // directory on disk, where `find_binary_in`'s priority order + // (rocm-stable > rocm-nightly > rocm) could shadow the just-restored good + // build with it. + let dir = scratch_dir("aside-removes-stray-residue-in-different-dir"); + let runtime_dir = dir.join("runtime"); + let llamacpp = runtime_dir.join("bin").join("llamacpp"); + let server = platform_binary_name("llama-server"); + let rocm_dir = llamacpp.join("rocm"); + fs::create_dir_all(&rocm_dir).unwrap(); + fs::write(rocm_dir.join(&server), b"original").unwrap(); + let manifest = test_manifest(runtime_dir); + + let aside_dirs = set_aside_rocm_llamacpp_backend_dirs(&manifest); + // Simulate a failed install that landed its (unverified) output in + // `rocm-stable` instead of `rocm`, the name that was actually asided. + let stray_dir = llamacpp.join("rocm-stable"); + fs::create_dir_all(&stray_dir).unwrap(); + fs::write(stray_dir.join(&server), b"stray").unwrap(); + + resolve_rocm_llamacpp_backend_dirs_aside(&manifest, &aside_dirs, false); + + assert!( + !stray_dir.exists(), + "the failed install's residue in a different backend dir must be removed" + ); + assert_eq!( + fs::read(rocm_dir.join(&server)).unwrap(), + b"original", + "the pre-existing backend must still be restored" + ); + fs::remove_dir_all(&dir).ok(); + } + + #[test] + fn llamacpp_backend_alignment_requires_both_a_successful_install_and_a_usable_binary() { + // The bug this guards: a failed install (e.g. Tier 1's pinned build 404ing + // for a ROCm version it predates) must never be forgiven by a leftover + // binary from a PREVIOUS, different-version install that happens to still + // pass the ldd-resolves check -- `ldd` cannot tell the two apart. + assert!(llamacpp_backend_alignment_succeeded(&Ok(()), true)); + assert!(!llamacpp_backend_alignment_succeeded(&Ok(()), false)); + assert!(!llamacpp_backend_alignment_succeeded( + &Err(anyhow!("boom")), + true + )); + assert!(!llamacpp_backend_alignment_succeeded( + &Err(anyhow!("boom")), + false + )); + } + + #[test] + fn should_attempt_llamacpp_backend_alignment_requires_the_rocm_backend_specifically() { + // The bug this guards: alignment is guaranteed to fail on a host that falls + // back to vulkan (WSL2 being the documented case), so it must be skipped for + // any selection other than exactly `rocm` -- including the probe itself + // failing to determine one at all. + assert!(should_attempt_llamacpp_backend_alignment(Some( + ROCM_BACKEND_NAME + ))); + assert!(!should_attempt_llamacpp_backend_alignment(Some("vulkan"))); + assert!(!should_attempt_llamacpp_backend_alignment(None)); + } + + #[test] + fn align_honors_the_disabled_flag_without_touching_the_pin_or_the_injected_steps() { + // This pins `align`'s own half of the opt-out contract in isolation: the pin + // and llama.cpp tag stay byte-identical and `align`/`latest_tag` are never + // called. The real production gate -- whether this function is even reached + // with `disabled == true` in the first place -- lives one level up, in + // `prepare_llamacpp_backend_for_active_rocm_impl`; see + // `prepare_disabled_early_return_skips_every_lookup_and_installs_unforced` + // below for that half. + let dir = scratch_dir("align-disabled"); + let path = dir.join("backend_versions.json"); + write_backend_versions_fixture(&path, "7.13.0", "b9752"); + let mut manifest = test_manifest(dir.clone()); + + let result = align_llamacpp_backend_to_version( + &mut manifest, + &path, + "10.0.0", + "7.13.0", + true, + |_manifest, _target, _force_reinstall, _label| { + panic!("disabled means no alignment attempt of any kind") + }, + |_manifest, force_reinstall| { + assert!( + !force_reinstall, + "the disabled path installs whatever is already pinned, not a fresh reinstall" + ); + Ok(()) + }, + || panic!("disabled means tier 2's tag lookup must not run either"), + ); + + assert_eq!(result.unwrap(), None); + assert_eq!( + read_backend_versions_therock_version(&path), + Some("7.13.0".to_owned()), + "the packaged pin must survive untouched when alignment is disabled" + ); + assert_eq!( + read_backend_versions_llamacpp_tag(&path), + Some("b9752".to_owned()), + "the packaged llama.cpp tag must survive untouched when alignment is disabled" + ); + fs::remove_dir_all(&dir).ok(); + } + + #[test] + fn prepare_disabled_early_return_skips_every_lookup_and_installs_unforced() { + // The real production gate for the opt-out: previously this was covered only + // by a @nightly @requires-gpu e2e scenario, which every per-PR and + // merge-queue lane skips -- so a regression that silently re-enabled + // alignment here shipped green everywhere that actually gates a merge. Every + // lookup the disabled path is supposed to skip panics if reached; the + // unforced install is the falsifiable positive. + let mut manifest = test_manifest(PathBuf::from("unused-runtime-dir")); + + let result = prepare_llamacpp_backend_for_active_rocm_impl( + &mut manifest, + true, + || panic!("disabled means the active SDK version must never be probed"), + |_path| panic!("disabled means the packaged pin must never be read"), + |_manifest| panic!("disabled means the vulkan-fallback probe must never run"), + |_manifest, force_reinstall| { + assert!( + !force_reinstall, + "the disabled path installs whatever is already pinned, not a fresh reinstall" + ); + Ok(()) + }, + |_manifest, _path, _target, _pinned, _disabled| { + panic!("disabled means align must never be reached") + }, + ); + + assert_eq!(result.unwrap(), None); + } + + #[test] + fn prepare_probe_error_falls_back_to_an_unforced_install_without_reaching_align() { + // The bug this guards: a host where the vulkan-fallback probe itself errors + // (e.g. `lemond` failing to start) must degrade to an ordinary install + // rather than propagating the error or attempting alignment blind -- the + // same "skip, don't fail" contract every other probe/lookup failure in this + // gate already gets. + let mut manifest = test_manifest(PathBuf::from("unused-runtime-dir")); + + let result = prepare_llamacpp_backend_for_active_rocm_impl( + &mut manifest, + false, + || Ok(Some("10.0.0".to_owned())), + |_path| Some("7.13.0".to_owned()), + |_manifest| Err(anyhow!("lemond failed to start")), + |_manifest, force_reinstall| { + assert!( + !force_reinstall, + "a probe error falls back to an unforced install, not a reinstall" + ); + Ok(()) + }, + |_manifest, _path, _target, _pinned, _disabled| { + panic!("a probe error must skip align entirely") + }, + ); + + assert_eq!(result.unwrap(), None); + } + + #[test] + fn align_tier1_success_keeps_target_version_pinned() { + let dir = scratch_dir("align-tier1-success"); + let path = dir.join("backend_versions.json"); + write_backend_versions_fixture(&path, "7.13.0", "b9752"); + let mut manifest = test_manifest(dir.clone()); + + let result = align_llamacpp_backend_to_version( + &mut manifest, + &path, + "10.0.0", + "7.13.0", + false, + |_manifest, _target, force_reinstall, _label| { + // A stale backend installed against the OLD pin must not be able to + // fake success just because it happens to still resolve -- Tier 1 + // must force a real reinstall attempt every time. + assert!(force_reinstall, "tier 1 must force a reinstall"); + true + }, + |_manifest, _force_reinstall| panic!("tier 1 succeeded; no fallback install"), + || panic!("tier 1 succeeded; tier 2's tag lookup must not run"), + ); + + assert_eq!(result.unwrap(), Some("10.0.0".to_owned())); + assert_eq!( + read_backend_versions_therock_version(&path), + Some("10.0.0".to_owned()) + ); + fs::remove_dir_all(&dir).ok(); + } + + #[test] + fn align_falls_through_to_tier2_and_succeeds() { + let dir = scratch_dir("align-tier2-success"); + let path = dir.join("backend_versions.json"); + write_backend_versions_fixture(&path, "7.13.0", "b9752"); + let mut manifest = test_manifest(dir.clone()); + let mut align_calls = 0; + + let result = align_llamacpp_backend_to_version( + &mut manifest, + &path, + "10.0.0", + "7.13.0", + false, + |_manifest, _target, force_reinstall, _label| { + // Both tiers force a reinstall now, so the mock can no longer use + // force_reinstall itself to distinguish tier 1 from tier 2 -- use + // call order instead. Tier 1 fails; Tier 2 succeeds. + assert!( + force_reinstall, + "every alignment attempt forces a reinstall" + ); + align_calls += 1; + align_calls > 1 + }, + |_manifest, _force_reinstall| panic!("tier 2 succeeded; no fallback install"), + || Ok("b10952".to_owned()), + ); + + assert_eq!(result.unwrap(), Some("10.0.0".to_owned())); + assert_eq!(align_calls, 2, "both tiers were attempted"); + assert_eq!( + read_backend_versions_therock_version(&path), + Some("10.0.0".to_owned()) + ); + assert_eq!( + read_backend_versions_llamacpp_tag(&path), + Some("b10952".to_owned()), + "tier 2 pins the newer llama.cpp tag" + ); + fs::remove_dir_all(&dir).ok(); + } + + #[test] + fn align_reverts_the_llamacpp_tag_when_tier2_pinned_a_newer_one_and_still_fails() { + // Regression test: the tag restore used to be silently skipped whenever + // deleted, and nothing caught it -- both tiers must fail here so the + // revert path actually runs, and the tag must come back to its original + // value rather than staying on Tier 2's newer pin. + let dir = scratch_dir("align-revert-restores-tag"); + let path = dir.join("backend_versions.json"); + write_backend_versions_fixture(&path, "7.13.0", "b9752"); + let mut manifest = test_manifest(dir.clone()); + + let result = align_llamacpp_backend_to_version( + &mut manifest, + &path, + "10.0.0", + "7.13.0", + false, + |_manifest, _target, _force_reinstall, _label| false, + |_manifest, force_reinstall| { + assert!(force_reinstall); + Ok(()) + }, + || Ok("b10952".to_owned()), + ); + + assert_eq!(result.unwrap(), None); + assert_eq!( + read_backend_versions_therock_version(&path), + Some("7.13.0".to_owned()) + ); + assert_eq!( + read_backend_versions_llamacpp_tag(&path), + Some("b9752".to_owned()), + "the llama.cpp tag must be restored to its original value, not left on Tier 2's pin" + ); + fs::remove_dir_all(&dir).ok(); + } + + #[test] + fn align_removes_the_llamacpp_tag_on_revert_when_none_was_pinned_before() { + // The resource file may have no `llamacpp.rocm-stable` key at all (an + // embeddable that never shipped a pin). Tier 2 still writes one; reverting + // must remove it again rather than leaving it in place with no original + // value to restore it to. + let dir = scratch_dir("align-revert-removes-tag"); + let path = dir.join("backend_versions.json"); + fs::write( + &path, + serde_json::to_vec_pretty(&json!({ + "llamacpp": {}, + "therock": { "version": "7.13.0" }, + })) + .unwrap(), + ) + .unwrap(); + let mut manifest = test_manifest(dir.clone()); + + let result = align_llamacpp_backend_to_version( + &mut manifest, + &path, + "10.0.0", + "7.13.0", + false, + |_manifest, _target, _force_reinstall, _label| false, + |_manifest, force_reinstall| { + assert!(force_reinstall); + Ok(()) + }, + || Ok("b10952".to_owned()), + ); + + assert_eq!(result.unwrap(), None); + assert_eq!( + read_backend_versions_llamacpp_tag(&path), + None, + "no tag was pinned before Tier 2; reverting must remove it, not invent a value" + ); + fs::remove_dir_all(&dir).ok(); + } + + #[test] + fn align_skips_tier2_reinstall_when_pinned_tag_is_already_latest() { + let dir = scratch_dir("align-tier2-tag-unchanged"); + let path = dir.join("backend_versions.json"); + write_backend_versions_fixture(&path, "7.13.0", "b9752"); + let mut manifest = test_manifest(dir.clone()); + let mut align_calls = 0; + + let result = align_llamacpp_backend_to_version( + &mut manifest, + &path, + "10.0.0", + "7.13.0", + false, + |_manifest, _target, _force_reinstall, _label| { + align_calls += 1; + false + }, + |_manifest, force_reinstall| { + assert!( + force_reinstall, + "the revert fallback always forces a reinstall" + ); + Ok(()) + }, + // Already the latest tag: Tier 2's own reinstall must not be attempted. + || Ok("b9752".to_owned()), + ); + + assert_eq!(result.unwrap(), None); + assert_eq!(align_calls, 1, "only tier 1 was attempted"); + assert_eq!( + read_backend_versions_therock_version(&path), + Some("7.13.0".to_owned()), + "reverted to the original pinned version" + ); + fs::remove_dir_all(&dir).ok(); + } + + #[test] + fn align_reverts_and_still_runs_fallback_when_the_restore_write_fails() { + // Regression test: a failure while restoring the original pinned version must + // not abort the install — the fallback reinstall below it is the recovery + // path, and every sibling warning in this function is best-effort. + let dir = scratch_dir("align-revert-write-fails"); + let path = dir.join("backend_versions.json"); + write_backend_versions_fixture(&path, "7.13.0", "b9752"); + let mut manifest = test_manifest(dir.clone()); + let mut fallback_called = false; + + // Corrupt the file's `therock` object between tier attempts and the revert, so + // the revert's own write fails. + let result = align_llamacpp_backend_to_version( + &mut manifest, + &path, + "10.0.0", + "7.13.0", + false, + |_manifest, _target, _force_reinstall, _label| { + fs::write( + &path, + serde_json::to_vec_pretty(&json!({"llamacpp": {}})).unwrap(), + ) + .unwrap(); + false + }, + |_manifest, force_reinstall| { + fallback_called = true; + assert!(force_reinstall); + Ok(()) + }, + || bail!("network unavailable"), + ); + + assert_eq!( + result.unwrap(), + None, + "never propagates the restore-write failure" + ); + assert!(fallback_called, "the fallback reinstall still ran"); + fs::remove_dir_all(&dir).ok(); + } + + #[test] + fn align_revert_restores_the_backend_dir_when_the_fallback_install_fails() { + // The bug this guards: the revert branch used to delete the ROCm backend + // directories before the fallback install, with no recovery if that install + // then failed -- leaving the host with no ROCm backend at all. Both tiers + // fail here (forcing the revert), and the fallback install also fails, so + // the pre-existing backend must come back rather than staying deleted. + let dir = scratch_dir("align-revert-fallback-install-fails"); + let path = dir.join("backend_versions.json"); + write_backend_versions_fixture(&path, "7.13.0", "b9752"); + let mut manifest = test_manifest(dir.clone()); + let llamacpp = dir.join("bin").join("llamacpp"); + let server = platform_binary_name("llama-server"); + let rocm_dir = llamacpp.join("rocm"); + fs::create_dir_all(&rocm_dir).unwrap(); + fs::write(rocm_dir.join(&server), b"pre-existing").unwrap(); + + let result = align_llamacpp_backend_to_version( + &mut manifest, + &path, + "10.0.0", + "7.13.0", + false, + |_manifest, _target, _force_reinstall, _label| false, + |_manifest, force_reinstall| { + assert!(force_reinstall); + bail!("network unavailable") + }, + || Ok("b10952".to_owned()), + ); + + assert!( + result.is_err(), + "the fallback install's failure must be propagated" + ); + assert_eq!( + fs::read(rocm_dir.join(&server)).unwrap(), + b"pre-existing", + "the pre-existing backend must be restored, not left deleted" + ); + fs::remove_dir_all(&dir).ok(); + } + + #[test] + #[cfg(target_os = "linux")] + fn ldd_output_resolves_all_requires_every_soname_resolved() { + let resolved = "\ +\tlibggml-base.so.0 => /opt/rocm/lib/libggml-base.so.0 (0x00007f0)\n\ +\tlibhipblas.so.3 => /opt/rocm/lib/libhipblas.so.3 (0x00007f1)\n\ +\tlibrocblas.so.5 => /opt/rocm/lib/librocblas.so.5 (0x00007f2)\n\ +\tlibamdhip64.so.7 => /opt/rocm/lib/libamdhip64.so.7 (0x00007f3)\n\ +\tlibhsa-runtime64.so.1 => /opt/rocm/lib/libhsa-runtime64.so.1 (0x00007f4)\n"; + assert!(ldd_output_resolves_all( + resolved, + &ROCM_BACKEND_REQUIRED_SONAMES + )); + + let missing_one = "\ +\tlibhipblas.so.3 => not found\n\ +\tlibrocblas.so.5 => /opt/rocm/lib/librocblas.so.5 (0x00007f2)\n\ +\tlibamdhip64.so.7 => /opt/rocm/lib/libamdhip64.so.7 (0x00007f3)\n\ +\tlibhsa-runtime64.so.1 => /opt/rocm/lib/libhsa-runtime64.so.1 (0x00007f4)\n"; + assert!(!ldd_output_resolves_all( + missing_one, + &ROCM_BACKEND_REQUIRED_SONAMES + )); + } + + #[test] + fn parses_llamacpp_backends_from_table() { + let output = "\ +Recipe Backend Status Message/Version Action +---------------------------------------------------------------------------------------------------- +kokoro cpu installable Backend is supported but not installed. lemonade backends install kokoro:cpu + metal unsupported Requires macOS - +llamacpp cpu installable Backend is supported but not installed. lemonade backends install llamacpp:cpu + metal unsupported Requires macOS - + rocm unsupported Unsupported GPU - + system unsupported Requires Linux - + vulkan installable Backend is supported but not installed. lemonade backends install llamacpp:vulkan +vllm rocm unsupported Requires Linux - +"; + let backends = parse_llamacpp_backend_statuses(output); + assert_eq!( + backends, + vec![ + ("cpu".to_owned(), "installable".to_owned()), + ("metal".to_owned(), "unsupported".to_owned()), + ("rocm".to_owned(), "unsupported".to_owned()), + ("system".to_owned(), "unsupported".to_owned()), + ("vulkan".to_owned(), "installable".to_owned()), + ] + ); + } + + #[test] + fn selects_vulkan_when_rocm_unsupported() { + let backends = vec![ + ("cpu".to_owned(), "installable".to_owned()), + ("rocm".to_owned(), "unsupported".to_owned()), + ("vulkan".to_owned(), "installable".to_owned()), + ]; + assert_eq!( + select_best_llamacpp_backend(&backends), + Some(("vulkan".to_owned(), false)) + ); + } + + #[test] + fn prefers_installed_rocm_when_supported() { + let backends = vec![ + ("rocm".to_owned(), "installed".to_owned()), + ("vulkan".to_owned(), "installable".to_owned()), + ]; + assert_eq!( + select_best_llamacpp_backend(&backends), + Some(("rocm".to_owned(), true)) + ); + } + + #[test] + fn never_selects_cpu_when_no_gpu_backend() { + // Under the GPU-required policy there is no silent CPU fallback: when only + // a CPU backend is usable, selection returns nothing and the caller fails. + let backends = vec![ + ("rocm".to_owned(), "unsupported".to_owned()), + ("cpu".to_owned(), "installable".to_owned()), + ]; + assert_eq!(select_best_llamacpp_backend(&backends), None); + } + + #[test] + fn selects_nothing_when_all_unsupported() { + let backends = vec![("rocm".to_owned(), "unsupported".to_owned())]; + assert_eq!(select_best_llamacpp_backend(&backends), None); + } +} diff --git a/engines/lemonade/src/direct_llama.rs b/engines/lemonade/src/direct_llama.rs new file mode 100644 index 0000000000..79721f5297 --- /dev/null +++ b/engines/lemonade/src/direct_llama.rs @@ -0,0 +1,775 @@ +// Copyright © Advanced Micro Devices, Inc., or its affiliates. +// +// SPDX-License-Identifier: MIT + +use anyhow::{Result, anyhow, bail}; +use rocm_core::AppPaths; +use std::fs; +use std::path::{Path, PathBuf}; +use std::time::Duration; + +use crate::backend_alignment::{LemonadeRuntime, ROCM_LLAMACPP_BACKEND_DIRS}; +use crate::install::{LemonadeInstallManifest, platform_binary_name}; +use crate::process::{ + LemonadeProcessEnvironment, run_lemonade_model_load, run_lemonade_pull, + serve_direct_llama_server, spawn_lemond, wait_for_lemonade_cli_status, +}; +use crate::state::{free_local_port, resolve_lemonade_model_ref, terminate_pid}; +use crate::{ + DEFAULT_HOST, DEFAULT_MODEL, DEFAULT_MODEL_GGUF, DEFAULT_MODEL_REPO_DIR, ROCM_BACKEND_NAME, + ServeHttpRequest, +}; + +/// Serve a canonical Hugging Face checkpoint under its exact name. Requires an explicit +/// `:variant`; downloads the GGUF, then runs whichever packaged GPU llama-server backend +/// is installed (e.g. `vulkan` on WSL2, `rocm-stable` on native ROCm) directly on the +/// file with `--alias`. This bypasses Lemonade's model registry, whose naming rules +/// cannot preserve an `owner/repo:variant` name. +pub(crate) fn serve_hf_checkpoint( + request: &ServeHttpRequest, + runtime: &LemonadeRuntime, + process_env: &LemonadeProcessEnvironment, + log_path: Option<&Path>, + checkpoint: &HfCheckpoint, +) -> Result<()> { + // A variant is required: a bare `owner/repo` would need Lemonade's interactive + // variant menu (which cannot be answered from this non-interactive path) and, if a + // GGUF happened to be cached, risks silently serving the wrong quantization. + if checkpoint.variant.is_none() { + bail!( + "`{model}` needs an explicit quantization variant to serve; use `owner/repo:variant`, e.g. `{model}:Q4_K_M` (see the repo's GGUF files on Hugging Face for the available variants)", + model = request.model_ref + ); + } + let Some(server) = find_llama_server_binary(&runtime.manifest) else { + bail!( + "no GPU llama-server backend is installed under {}; run `rocm engines install lemonade`", + runtime + .manifest + .runtime_dir + .join("bin") + .join("llamacpp") + .display() + ); + }; + ensure_hf_checkpoint_downloaded(request, runtime, process_env, log_path)?; + // Never silently pick between quantizations: if the variant matches more than one + // cached GGUF, ask the user to disambiguate instead of choosing one arbitrarily. + let paths = AppPaths::discover()?; + let matches = hf_checkpoint_gguf_matches(&paths, checkpoint); + if matches.len() > 1 { + let names = matches + .iter() + .filter_map(|path| path.file_name().and_then(|name| name.to_str())) + .collect::>() + .join(", "); + bail!( + "variant `{}` matches multiple files ({names}); specify an exact quantization or filename, e.g. `{}:{}`", + checkpoint.variant.as_deref().unwrap_or_default(), + request.model_ref, + matches + .first() + .and_then(|path| path.file_name()) + .and_then(|name| name.to_str()) + .unwrap_or(".gguf") + ); + } + serve_direct_llama_server( + request, + runtime, + process_env, + &server, + log_path, + &anyhow!( + "serving canonical Hugging Face checkpoint `{}` directly", + request.model_ref + ), + ) +} + +/// Ensure a canonical Hugging Face checkpoint's GGUF is in the HF hub cache, pulling +/// it through a short-lived `lemond` if needed. The bare `pull owner/repo:variant` +/// only downloads (and registers a derived name we ignore); the file is what we serve. +fn ensure_hf_checkpoint_downloaded( + request: &ServeHttpRequest, + runtime: &LemonadeRuntime, + process_env: &LemonadeProcessEnvironment, + log_path: Option<&Path>, +) -> Result<()> { + let paths = AppPaths::discover()?; + if direct_llama_model_path(&paths, &request.model_ref).is_some() { + return Ok(()); + } + let download_port = free_local_port()?; + let mut child = spawn_lemond( + &runtime.manifest, + DEFAULT_HOST, + download_port, + log_path, + process_env, + )?; + let result = (|| -> Result<()> { + wait_for_lemonade_cli_status( + &runtime.manifest, + DEFAULT_HOST, + download_port, + Duration::from_secs(30), + log_path, + process_env, + )?; + run_lemonade_pull( + &runtime.manifest, + DEFAULT_HOST, + download_port, + &request.model_ref, + log_path, + process_env, + )?; + if direct_llama_model_path(&paths, &request.model_ref).is_some() { + Ok(()) + } else { + bail!("Lemonade did not download `{}`", request.model_ref) + } + })(); + let _ = terminate_pid(child.id(), true); + let _ = child.wait(); + result +} + +pub(crate) fn ensure_direct_llama_model_available( + request: &ServeHttpRequest, + runtime: &LemonadeRuntime, + process_env: &LemonadeProcessEnvironment, + log_path: Option<&Path>, +) -> Result<()> { + let paths = AppPaths::discover()?; + if direct_llama_model_path(&paths, &request.model_ref).is_some() { + return Ok(()); + } + + let download_port = free_local_port()?; + let mut child = spawn_lemond( + &runtime.manifest, + DEFAULT_HOST, + download_port, + log_path, + process_env, + )?; + let result = (|| -> Result<()> { + wait_for_lemonade_cli_status( + &runtime.manifest, + DEFAULT_HOST, + download_port, + Duration::from_secs(30), + log_path, + process_env, + )?; + let _ = run_lemonade_model_load( + &runtime.manifest, + DEFAULT_HOST, + download_port, + &request.model_ref, + ROCM_BACKEND_NAME, + None, + log_path, + process_env, + ); + if direct_llama_model_path(&paths, &request.model_ref).is_some() { + Ok(()) + } else { + bail!( + "Lemonade did not download `{}` for direct ROCm serving", + request.model_ref + ) + } + })(); + let _ = terminate_pid(child.id(), true); + let _ = child.wait(); + result +} + +/// A canonical Hugging Face checkpoint reference: `owner/repo` with an optional +/// `:variant` suffix (a quantization label such as `BF16`/`Q4_0`, or an exact +/// `file.gguf`). +#[derive(Debug, Clone, PartialEq, Eq)] +pub(crate) struct HfCheckpoint { + owner: String, + repo: String, + variant: Option, +} + +/// Parse a model reference as a canonical Hugging Face checkpoint. Returns `None` for +/// anything not of the form `owner/repo[:variant]` — built-in Lemonade aliases and +/// bare registered names all fall through unchanged. +/// +/// Every component is validated against the Hugging Face id / quantization charset +/// (`[A-Za-z0-9._-]`, and never `.`/`..`). This is a security boundary as well as a +/// correctness one: the components are used to build a filesystem path into the model +/// cache, so rejecting path separators and traversal tokens prevents a crafted model +/// reference from escaping the cache directory. +pub(crate) fn parse_hf_checkpoint(model_ref: &str) -> Option { + let trimmed = model_ref.trim(); + if trimmed.is_empty() { + return None; + } + // Split off an optional `:variant` suffix on the first colon; the owner/repo pair + // never contains a colon, so any remainder is the variant. + let (repo_path, variant) = match trimmed.split_once(':') { + Some((path, variant)) => (path, Some(variant)), + None => (trimmed, None), + }; + let (owner, repo) = repo_path.split_once('/')?; + if !is_safe_hf_component(owner) || !is_safe_hf_component(repo) { + return None; + } + if variant.is_some_and(|variant| !is_safe_hf_component(variant)) { + return None; + } + Some(HfCheckpoint { + owner: owner.to_owned(), + repo: repo.to_owned(), + variant: variant.map(str::to_owned), + }) +} + +/// A safe Hugging Face path component: a non-empty run of `[A-Za-z0-9._-]` that is not +/// a `.`/`..` traversal token. Matches the Hugging Face id and quantization-label +/// charset and, by excluding `/`, `\`, and traversal tokens, keeps a user-supplied +/// component from escaping the model cache directory when used to build a path. +fn is_safe_hf_component(component: &str) -> bool { + !component.is_empty() + && component != "." + && component != ".." + && component + .chars() + .all(|c| c.is_ascii_alphanumeric() || matches!(c, '.' | '_' | '-')) +} + +/// Packaged llama.cpp backends whose `llama-server` we can drive directly, best first. +/// GPU backends only — `cpu` is intentionally excluded so the direct-serve path never +/// silently serves on CPU under a GPU-required policy (AGENTS.md §6). An explicit +/// allowlist rather than a directory scan, so the path is built only from constants. +const DIRECT_LLAMA_SERVER_BACKENDS: [&str; 4] = ["rocm-stable", "rocm-nightly", "rocm", "vulkan"]; + +/// Find an installed Lemonade `llama-server` binary under `bin/llamacpp//`, +/// checking known backends in priority order. This lets direct serving work wherever +/// Lemonade installed a backend (e.g. `vulkan` on WSL2), not just the ROCm build. +/// +/// Lemonade's backend installer extracts some llama.cpp releases straight into +/// `/`, but others land one level deeper under a build-numbered directory +/// (e.g. `rocm-stable/llama-b9752/llama-server`), so both layouts are checked. +pub(crate) fn find_llama_server_binary(manifest: &LemonadeInstallManifest) -> Option { + let llamacpp_dir = manifest.runtime_dir.join("bin").join("llamacpp"); + let binary = platform_binary_name("llama-server"); + DIRECT_LLAMA_SERVER_BACKENDS + .into_iter() + .find_map(|backend| find_binary_in(&llamacpp_dir.join(backend), &binary)) +} + +/// Like [`find_llama_server_binary`], but scoped to the single backend Lemonade +/// actually attempted (`manifest.backend_name`, one of `LLAMACPP_BACKEND_PRIORITY`'s +/// `"rocm"` / `"vulkan"`), rather than scanning every known backend directory in +/// priority order. Lemonade reports the generic `"rocm"` — never the more specific +/// `rocm-stable`/`rocm-nightly` — but may extract the actual build into either +/// versioned directory, so `"rocm"` maps to the whole ROCm family; `"vulkan"` never +/// does. This is what lets alignment verification reject a stale ROCm directory left +/// by an earlier attempt when this round actually picked `vulkan` (or the reverse). +pub(crate) fn find_llama_server_binary_for_backend( + manifest: &LemonadeInstallManifest, + backend_name: &str, +) -> Option { + let llamacpp_dir = manifest.runtime_dir.join("bin").join("llamacpp"); + let binary = platform_binary_name("llama-server"); + let family: &[&str] = if backend_name == ROCM_BACKEND_NAME { + &ROCM_LLAMACPP_BACKEND_DIRS + } else { + &["vulkan"] + }; + family + .iter() + .find_map(|backend| find_binary_in(&llamacpp_dir.join(backend), &binary)) +} + +/// Look for `binary` directly in `dir`, then one level down in each subdirectory. +fn find_binary_in(dir: &Path, binary: &str) -> Option { + let direct = dir.join(binary); + if direct.is_file() { + return Some(direct); + } + fs::read_dir(dir) + .ok()? + .filter_map(Result::ok) + .map(|entry| entry.path()) + .filter(|path| path.is_dir()) + .map(|subdir| subdir.join(binary)) + .find(|candidate| candidate.is_file()) +} + +/// The backend label for a packaged llama-server, taken from the nearest ancestor +/// directory matching a known backend name (e.g. `vulkan`, `rocm-stable`) so it still +/// resolves correctly when Lemonade nests the binary under a build-numbered +/// subdirectory; falls back to the ROCm backend name. +pub(crate) fn llama_server_backend_label(server: &Path) -> String { + server + .ancestors() + .filter_map(|dir| dir.file_name()) + .filter_map(|name| name.to_str()) + .find(|name| DIRECT_LLAMA_SERVER_BACKENDS.contains(name)) + .unwrap_or(ROCM_BACKEND_NAME) + .to_owned() +} + +pub(crate) fn direct_llama_model_path(paths: &AppPaths, model_ref: &str) -> Option { + let as_path = PathBuf::from(model_ref); + if as_path.is_file() { + return Some(as_path); + } + if let Some(checkpoint) = parse_hf_checkpoint(model_ref) { + return hf_cache_roots(paths) + .into_iter() + .find_map(|root| find_hf_checkpoint_gguf(&root, &checkpoint)); + } + if resolve_lemonade_model_ref(model_ref) != DEFAULT_MODEL { + return None; + } + hf_cache_roots(paths) + .into_iter() + .find_map(find_default_qwen_gguf) +} + +fn hf_cache_roots(paths: &AppPaths) -> Vec { + hf_cache_roots_from(paths, |name| std::env::var_os(name).map(PathBuf::from)) +} + +fn hf_cache_roots_from(paths: &AppPaths, mut env_path: F) -> Vec +where + F: FnMut(&str) -> Option, +{ + let mut roots = Vec::new(); + if let Some(hub_cache) = env_path("HUGGINGFACE_HUB_CACHE") { + push_hf_cache_root(&mut roots, hub_cache); + } + if let Some(hf_home) = env_path("HF_HOME") { + push_hf_cache_root(&mut roots, hf_home.join("hub")); + } + push_hf_cache_root(&mut roots, paths.cache_dir.join("huggingface").join("hub")); + if let Some(home) = env_path("HOME") { + push_hf_cache_root( + &mut roots, + home.join(".cache").join("huggingface").join("hub"), + ); + } + roots +} + +fn push_hf_cache_root(roots: &mut Vec, path: PathBuf) { + if path.as_os_str().is_empty() || roots.iter().any(|existing| existing == &path) { + return; + } + roots.push(path); +} + +fn find_default_qwen_gguf(cache_root: PathBuf) -> Option { + let snapshots = cache_root.join(DEFAULT_MODEL_REPO_DIR).join("snapshots"); + let entries = fs::read_dir(snapshots).ok()?; + entries + .flatten() + .map(|entry| entry.path().join(DEFAULT_MODEL_GGUF)) + .find(|path| path.is_file()) +} + +/// Locate the downloaded GGUF for a Hugging Face checkpoint inside one hub cache root, +/// selecting the file that matches the requested `:variant` (if any). +fn find_hf_checkpoint_gguf(cache_root: &Path, checkpoint: &HfCheckpoint) -> Option { + select_gguf_for_variant( + collect_hf_checkpoint_ggufs(cache_root, checkpoint), + checkpoint.variant.as_deref(), + ) +} + +/// All cached GGUF files for a checkpoint's repo under one hub cache root (unfiltered +/// by variant). +fn collect_hf_checkpoint_ggufs(cache_root: &Path, checkpoint: &HfCheckpoint) -> Vec { + let repo_dir = format!("models--{}--{}", checkpoint.owner, checkpoint.repo); + let snapshots = cache_root.join(repo_dir).join("snapshots"); + let mut ggufs = Vec::new(); + if let Ok(entries) = fs::read_dir(snapshots) { + for entry in entries.flatten() { + collect_gguf_files(&entry.path(), &mut ggufs, 0); + } + } + ggufs +} + +/// Every cached GGUF matching a checkpoint's variant, across all hub cache roots, +/// deduplicated by file name. Used to detect an ambiguous variant (more than one match) +/// so the caller can refuse to pick arbitrarily. +fn hf_checkpoint_gguf_matches(paths: &AppPaths, checkpoint: &HfCheckpoint) -> Vec { + let mut matches = Vec::new(); + let mut seen_names = std::collections::HashSet::new(); + for root in hf_cache_roots(paths) { + let ggufs = collect_hf_checkpoint_ggufs(&root, checkpoint); + for gguf in filter_ggufs_by_variant(ggufs, checkpoint.variant.as_deref()) { + if let Some(name) = gguf.file_name().and_then(|name| name.to_str()) + && seen_names.insert(name.to_owned()) + { + matches.push(gguf); + } + } + } + matches +} + +/// Collect `*.gguf` files under a snapshot revision directory. GGUFs usually sit at the +/// snapshot root, but sharded variants live one folder down, so recurse a few levels. +fn collect_gguf_files(dir: &Path, out: &mut Vec, depth: usize) { + if depth > 3 { + return; + } + let Ok(entries) = fs::read_dir(dir) else { + return; + }; + for entry in entries.flatten() { + let path = entry.path(); + if path.is_dir() { + collect_gguf_files(&path, out, depth + 1); + } else if path + .extension() + .and_then(|ext| ext.to_str()) + .is_some_and(|ext| ext.eq_ignore_ascii_case("gguf")) + { + out.push(path); + } + } +} + +/// Pick a single GGUF for a requested variant (the first, in sorted order, of the +/// matches). Sorting is by path — deterministic but not by mtime — which is fine +/// because the caller rejects an ambiguous variant before relying on this pick. +fn select_gguf_for_variant(ggufs: Vec, variant: Option<&str>) -> Option { + filter_ggufs_by_variant(ggufs, variant).into_iter().next() +} + +/// All GGUFs matching a requested variant, sorted by path. With no variant, every file +/// matches. With a variant, a case-insensitive exact filename match (`:model.gguf`) +/// wins outright; otherwise every file whose name contains the quantization label +/// (`:BF16`, `:Q4_0`) matches — more than one indicates an ambiguous variant. +fn filter_ggufs_by_variant(mut ggufs: Vec, variant: Option<&str>) -> Vec { + ggufs.sort(); + ggufs.dedup(); + let Some(variant) = variant else { + return ggufs; + }; + let file_name = |path: &Path| -> Option { + path.file_name() + .and_then(|name| name.to_str()) + .map(str::to_owned) + }; + if let Some(exact) = ggufs + .iter() + .find(|path| file_name(path).is_some_and(|name| name.eq_ignore_ascii_case(variant))) + { + return vec![exact.clone()]; + } + let variant_lower = variant.to_ascii_lowercase(); + ggufs + .into_iter() + .filter(|path| { + file_name(path).is_some_and(|name| name.to_ascii_lowercase().contains(&variant_lower)) + }) + .collect() +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::LLAMACPP_RECIPE; + use std::fs; + + /// A fresh scratch directory under the crate's `target/`. The base is + /// `CARGO_MANIFEST_DIR`, a compile-time constant, so the path never derives from a + /// runtime environment read. + fn scratch_dir(tag: &str) -> PathBuf { + let dir = Path::new(env!("CARGO_MANIFEST_DIR")) + .join("target") + .join(format!("lemonade-fs-test-{tag}")); + let _ = fs::remove_dir_all(&dir); + fs::create_dir_all(&dir).unwrap(); + dir + } + + #[test] + fn parses_canonical_hugging_face_checkpoints() { + assert_eq!( + parse_hf_checkpoint("LiquidAI/LFM2.5-230M-GGUF:Q4_0"), + Some(HfCheckpoint { + owner: "LiquidAI".to_owned(), + repo: "LFM2.5-230M-GGUF".to_owned(), + variant: Some("Q4_0".to_owned()), + }) + ); + assert_eq!( + parse_hf_checkpoint("unsloth/Qwen3-0.6B-GGUF"), + Some(HfCheckpoint { + owner: "unsloth".to_owned(), + repo: "Qwen3-0.6B-GGUF".to_owned(), + variant: None, + }) + ); + } + + #[test] + fn rejects_non_checkpoint_model_refs() { + // Built-in aliases and bare registered names have no owner/repo pair. + assert!(parse_hf_checkpoint("qwen").is_none()); + assert!(parse_hf_checkpoint(DEFAULT_MODEL).is_none()); + assert!(parse_hf_checkpoint("").is_none()); + assert!(parse_hf_checkpoint("/model.gguf").is_none()); + assert!(parse_hf_checkpoint("owner/").is_none()); + assert!(parse_hf_checkpoint("owner/repo/extra").is_none()); + assert!(parse_hf_checkpoint("owner/repo:").is_none()); + } + + #[test] + fn rejects_path_traversal_in_checkpoint_components() { + // A crafted reference must not be able to escape the model cache directory. + assert!(parse_hf_checkpoint("../../etc/passwd").is_none()); + assert!(parse_hf_checkpoint("..:Q4_0").is_none()); + assert!(parse_hf_checkpoint("owner/..").is_none()); + assert!(parse_hf_checkpoint("owner/repo:../secret").is_none()); + assert!(parse_hf_checkpoint(r"owner\..\..--x/repo").is_none()); + assert!(parse_hf_checkpoint("owner/repo:a/b.gguf").is_none()); + } + + #[test] + fn selects_gguf_by_variant() { + // Real filenames use the quant label without the `-GGUF` repo suffix, so a + // substring match on the `:variant` selects the right file. + let ggufs = vec![ + PathBuf::from("/cache/LFM2.5-230M-Q4_0.gguf"), + PathBuf::from("/cache/LFM2.5-230M-Q8_0.gguf"), + ]; + assert_eq!( + select_gguf_for_variant(ggufs.clone(), Some("q4_0")), + Some(PathBuf::from("/cache/LFM2.5-230M-Q4_0.gguf")) + ); + assert_eq!( + select_gguf_for_variant(ggufs.clone(), Some("LFM2.5-230M-Q8_0.gguf")), + Some(PathBuf::from("/cache/LFM2.5-230M-Q8_0.gguf")) + ); + assert_eq!( + select_gguf_for_variant(ggufs.clone(), None), + Some(PathBuf::from("/cache/LFM2.5-230M-Q4_0.gguf")) + ); + assert_eq!(select_gguf_for_variant(ggufs, Some("Q2_K")), None); + } + + #[test] + fn ambiguous_variant_matches_multiple_ggufs() { + // A partial label like `Q4` matches several quants; the caller must refuse to + // pick. A full label or exact filename resolves to exactly one. + let ggufs = vec![ + PathBuf::from("/cache/LFM2.5-230M-Q4_0.gguf"), + PathBuf::from("/cache/LFM2.5-230M-Q4_K_M.gguf"), + ]; + assert_eq!(filter_ggufs_by_variant(ggufs.clone(), Some("Q4")).len(), 2); + assert_eq!( + filter_ggufs_by_variant(ggufs.clone(), Some("Q4_0")).len(), + 1 + ); + assert_eq!( + filter_ggufs_by_variant(ggufs, Some("LFM2.5-230M-Q4_K_M.gguf")).len(), + 1 + ); + } + + #[test] + fn find_llama_server_binary_prefers_gpu_and_skips_cpu() { + let dir = scratch_dir("find-server"); + let runtime_dir = dir.join("runtime"); + let llamacpp = runtime_dir.join("bin").join("llamacpp"); + for backend in ["cpu", "vulkan"] { + let backend_dir = llamacpp.join(backend); + fs::create_dir_all(&backend_dir).unwrap(); + fs::write(backend_dir.join(platform_binary_name("llama-server")), b"x").unwrap(); + } + let manifest = test_manifest(runtime_dir); + // Both cpu and vulkan exist; the GPU backend is chosen and cpu is never selected. + assert_eq!( + find_llama_server_binary(&manifest), + Some( + llamacpp + .join("vulkan") + .join(platform_binary_name("llama-server")) + ) + ); + // With only cpu present, nothing is returned (no silent CPU fallback). + fs::remove_dir_all(llamacpp.join("vulkan")).unwrap(); + assert_eq!(find_llama_server_binary(&manifest), None); + fs::remove_dir_all(&dir).ok(); + } + + #[test] + fn find_llama_server_binary_finds_build_numbered_nesting() { + // Some Lemonade llama.cpp releases extract one level deeper than + // `/llama-server`, e.g. `rocm-stable/llama-b9752/llama-server`. + let dir = scratch_dir("find-server-nested"); + let runtime_dir = dir.join("runtime"); + let llamacpp = runtime_dir.join("bin").join("llamacpp"); + let nested = llamacpp.join("rocm-stable").join("llama-b9752"); + fs::create_dir_all(&nested).unwrap(); + let server = nested.join(platform_binary_name("llama-server")); + fs::write(&server, b"x").unwrap(); + let manifest = test_manifest(runtime_dir); + assert_eq!(find_llama_server_binary(&manifest), Some(server.clone())); + assert_eq!(llama_server_backend_label(&server), "rocm-stable"); + fs::remove_dir_all(&dir).ok(); + } + + #[test] + fn find_llama_server_binary_follows_backend_priority() { + let dir = scratch_dir("find-server-priority"); + let runtime_dir = dir.join("runtime"); + let llamacpp = runtime_dir.join("bin").join("llamacpp"); + let server = platform_binary_name("llama-server"); + let install = |backend: &str| { + let backend_dir = llamacpp.join(backend); + fs::create_dir_all(&backend_dir).unwrap(); + fs::write(backend_dir.join(&server), b"x").unwrap(); + }; + let manifest = test_manifest(runtime_dir); + + // Nightly-only host (the resilient case that replaced the hardcoded + // rocm-stable direct-serve path): the nightly backend is selected over vulkan. + install("rocm-nightly"); + install("vulkan"); + assert_eq!( + find_llama_server_binary(&manifest), + Some(llamacpp.join("rocm-nightly").join(&server)) + ); + + // With rocm-stable also present, it wins (highest priority). + install("rocm-stable"); + assert_eq!( + find_llama_server_binary(&manifest), + Some(llamacpp.join("rocm-stable").join(&server)) + ); + fs::remove_dir_all(&dir).ok(); + } + + #[test] + fn collect_gguf_files_recurses_and_filters() { + let dir = scratch_dir("collect-gguf"); + fs::create_dir_all(dir.join("shard")).unwrap(); + fs::write(dir.join("model-Q4_0.gguf"), b"x").unwrap(); + fs::write(dir.join("notes.txt"), b"x").unwrap(); + fs::write(dir.join("shard").join("model-Q8_0.gguf"), b"x").unwrap(); + let mut found = Vec::new(); + collect_gguf_files(&dir, &mut found, 0); + let mut names = found + .iter() + .filter_map(|path| path.file_name().and_then(|name| name.to_str())) + .map(str::to_owned) + .collect::>(); + names.sort(); + assert_eq!(names, vec!["model-Q4_0.gguf", "model-Q8_0.gguf"]); + fs::remove_dir_all(&dir).ok(); + } + + fn test_manifest(runtime_dir: PathBuf) -> LemonadeInstallManifest { + LemonadeInstallManifest { + env_id: "test".to_owned(), + version: rocm_deps::LEMONADE_VERSION.to_owned(), + runtime_dir, + lemond: PathBuf::from("lemond"), + lemonade: PathBuf::from("lemonade"), + backend_recipe: LLAMACPP_RECIPE.to_owned(), + backend_name: ROCM_BACKEND_NAME.to_owned(), + installed_at_unix_ms: 0, + } + } + + #[test] + fn find_llama_server_binary_for_backend_scopes_to_rocm_family() { + let dir = scratch_dir("find-server-scoped-rocm"); + let runtime_dir = dir.join("runtime"); + let llamacpp = runtime_dir.join("bin").join("llamacpp"); + let server = platform_binary_name("llama-server"); + // A stale vulkan install from an earlier attempt must never satisfy a "rocm" + // scoped lookup. + let vulkan_dir = llamacpp.join("vulkan"); + fs::create_dir_all(&vulkan_dir).unwrap(); + fs::write(vulkan_dir.join(&server), b"x").unwrap(); + let manifest = test_manifest(runtime_dir); + assert_eq!( + find_llama_server_binary_for_backend(&manifest, ROCM_BACKEND_NAME), + None + ); + + // Any directory in the ROCm family satisfies it, matching the unscoped scan's + // priority order. + let nightly_dir = llamacpp.join("rocm-nightly"); + fs::create_dir_all(&nightly_dir).unwrap(); + fs::write(nightly_dir.join(&server), b"x").unwrap(); + assert_eq!( + find_llama_server_binary_for_backend(&manifest, ROCM_BACKEND_NAME), + Some(nightly_dir.join(&server)) + ); + fs::remove_dir_all(&dir).ok(); + } + + #[test] + fn find_llama_server_binary_for_backend_scopes_to_vulkan_only() { + let dir = scratch_dir("find-server-scoped-vulkan"); + let runtime_dir = dir.join("runtime"); + let llamacpp = runtime_dir.join("bin").join("llamacpp"); + let server = platform_binary_name("llama-server"); + // A stale rocm-stable install from an earlier attempt must never satisfy a + // "vulkan" scoped lookup, even though it would win the unscoped priority scan. + let rocm_dir = llamacpp.join("rocm-stable"); + fs::create_dir_all(&rocm_dir).unwrap(); + fs::write(rocm_dir.join(&server), b"x").unwrap(); + let manifest = test_manifest(runtime_dir); + assert_eq!( + find_llama_server_binary_for_backend(&manifest, "vulkan"), + None + ); + + let vulkan_dir = llamacpp.join("vulkan"); + fs::create_dir_all(&vulkan_dir).unwrap(); + fs::write(vulkan_dir.join(&server), b"x").unwrap(); + assert_eq!( + find_llama_server_binary_for_backend(&manifest, "vulkan"), + Some(vulkan_dir.join(&server)) + ); + fs::remove_dir_all(&dir).ok(); + } + + #[test] + fn direct_qwen_lookup_checks_huggingface_cache_env() { + let paths = AppPaths { + config_dir: PathBuf::from("config"), + data_dir: PathBuf::from("data"), + cache_dir: PathBuf::from("rocm-cache"), + }; + let roots = hf_cache_roots_from(&paths, |name| match name { + "HUGGINGFACE_HUB_CACHE" => Some(PathBuf::from("hf-hub")), + "HF_HOME" => Some(PathBuf::from("hf-home")), + "HOME" => Some(PathBuf::from("home")), + _ => None, + }); + + assert_eq!( + roots, + vec![ + PathBuf::from("hf-hub"), + PathBuf::from("hf-home").join("hub"), + PathBuf::from("rocm-cache").join("huggingface").join("hub"), + PathBuf::from("home") + .join(".cache") + .join("huggingface") + .join("hub"), + ] + ); + } +} diff --git a/engines/lemonade/src/install.rs b/engines/lemonade/src/install.rs new file mode 100644 index 0000000000..e03b7e8cd0 --- /dev/null +++ b/engines/lemonade/src/install.rs @@ -0,0 +1,1956 @@ +// Copyright © Advanced Micro Devices, Inc., or its affiliates. +// +// SPDX-License-Identifier: MIT + +use anyhow::{Context, Result, bail}; +use rocm_core::{ + AppPaths, download_file_to_path, normalize_runtime_path_for_host, runtime_is_windows, +}; +use serde::{Deserialize, Serialize}; +use sha2::{Digest, Sha256}; +use std::ffi::OsString; +use std::fs; +use std::io::Read; +use std::path::{Path, PathBuf}; +use std::time::Duration; + +use crate::{ENGINE_NAME, LLAMACPP_RECIPE, ROCM_BACKEND_NAME, current_unix_millis}; + +/// Records the extracted embeddable version inside the runtime tree itself +/// (not the CLI's own manifest) so the version check survives even when the +/// manifest lives under a data dir that is not shared with the runtime tree +/// (e.g. each e2e scenario's isolated data dir vs. a shared runtime tree). +/// Living inside `runtime_dir` rather than beside it also means a wipe of the +/// tree removes the marker in the same operation — "marker present implies +/// extraction completed" holds by construction. Without this marker, +/// [`prepare_embeddable`] can never prove the shared tree is already current, +/// wipes it, and re-extracts on every call — destroying any llama.cpp backend +/// installed into it in the process. +const RUNTIME_VERSION_MARKER: &str = "runtime-version.txt"; + +/// Archive paths currently being validated, published, or extracted by this +/// process, with a condvar to wake waiters. Keyed per archive so an install of +/// one archive does not serialize behind an unrelated one — the download window +/// is minutes long, and the cross-process guarantee comes from the per-archive +/// file lock, not from this set. +static ARCHIVE_CACHE_BUSY: std::sync::Mutex> = + std::sync::Mutex::new(std::collections::BTreeSet::new()); +static ARCHIVE_CACHE_BUSY_SIGNAL: std::sync::Condvar = std::sync::Condvar::new(); + +/// How deep the extracted tree is searched for the embeddable root before the +/// search gives up. Official archives place the root at the top level, so this +/// only bounds the work spent looking; exceeding it is not an error. +const MAX_EMBEDDABLE_SEARCH_DEPTH: usize = 4; + +/// Recursion backstop for copying the embeddable tree. Containment is enforced +/// per entry, so this exists only to fail loudly instead of overflowing the +/// stack on a pathological tree; it is far above any plausible archive layout. +const MAX_COPY_RECURSION_DEPTH: usize = 64; + +/// Digests of the pinned embeddable archives. Unlike the archive names and +/// URLs, these cannot be derived from the version in `runtime-deps.toml`: each +/// release has its own digest, so bumping the pin means recording the new pair +/// here as well. +const EMBEDDABLE_WINDOWS_SHA256: &str = + "50a133bbc35c4f3f8971eafef2c9fe56c4bbbfb0f1032bf8728324d8d8c8a0e1"; +const EMBEDDABLE_LINUX_SHA256: &str = + "bdfd3c3e5d6eda5101c8a32f36e6dd9236ceb9ab2eb66734c32628e6e86e18ac"; + +/// Embeddable asset token and archive extension for the host this adapter runs +/// on. Only `windows-x64` and `ubuntu-x64` are wired up here; the release also +/// publishes `macos-arm64` and `ubuntu-arm64`, which this adapter never selects. +const fn embeddable_os_arch() -> (&'static str, &'static str) { + if runtime_is_windows() { + ("windows-x64", "zip") + } else { + ("ubuntu-x64", "tar.gz") + } +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub(crate) struct LemonadeInstallManifest { + pub(crate) env_id: String, + pub(crate) version: String, + pub(crate) runtime_dir: PathBuf, + pub(crate) lemond: PathBuf, + pub(crate) lemonade: PathBuf, + pub(crate) backend_recipe: String, + pub(crate) backend_name: String, + pub(crate) installed_at_unix_ms: u128, +} + +/// Whether the embeddable archive must be extracted over the runtime tree. +/// +/// Pure so the upgrade case is testable without a real archive. `installed` +/// is the version recorded for the tree already on disk, if any. +fn needs_extraction( + reinstall: bool, + installed: Option<&str>, + wanted: &str, + server_present: bool, +) -> bool { + reinstall || !server_present || installed != Some(wanted) +} + +/// The Lemonade runtime version last extracted into `runtime_dir`, if any. +fn installed_runtime_version(runtime_dir: &Path) -> Option { + fs::read_to_string(runtime_dir.join(RUNTIME_VERSION_MARKER)) + .ok() + .map(|s| s.trim().to_owned()) +} + +/// Record `version` as installed into `runtime_dir`. Marker lives inside +/// `runtime_dir` (not beside it) so `remove_dir_all(runtime_dir)` clears it as +/// part of the same wipe that removes the tree it describes — "marker present +/// implies extraction completed" holds by construction, with no separate +/// bookkeeping to keep in sync. +fn record_runtime_version(runtime_dir: &Path, version: &str) -> Result<()> { + fs::write(runtime_dir.join(RUNTIME_VERSION_MARKER), version).with_context(|| { + format!( + "failed to record installed Lemonade runtime version under {}", + runtime_dir.display() + ) + }) +} + +pub(crate) fn prepare_embeddable( + paths: &AppPaths, + env_root: Option<&Path>, + reinstall: bool, +) -> Result { + prepare_embeddable_with( + paths, + env_root, + reinstall, + rocm_deps::LEMONADE_VERSION, + embeddable_archive_sha256(), + download_file, + ) +} + +/// Same as [`prepare_embeddable`], but with the archive version, its expected +/// sha256, and the download step all injectable — the seam a test uses to +/// exercise the real decision of which source `installed_version` trusts, +/// rather than reimplementing that decision inline. +fn prepare_embeddable_with( + paths: &AppPaths, + env_root: Option<&Path>, + reinstall: bool, + version: &str, + expected_sha256: &str, + download: F, +) -> Result +where + F: FnMut(&str, &Path) -> Result<()>, +{ + let root = lemonade_root(paths, env_root); + let version = version.to_owned(); + let archive_name = embeddable_archive_name(&version); + let archive_url = embeddable_url(&version); + let downloads = root.join("downloads"); + let archive = downloads.join(&archive_name); + fs::create_dir_all(&downloads)?; + // Held until the archive bytes have been extracted and copied out: the + // verified bytes and the extracted bytes must be the same read. + let archive_guard = ensure_cached_archive(&archive_url, &archive, expected_sha256, download)?; + let runtime_dir = runtime_dir_in(&root); + // The runtime directory is not version-scoped, so a bump downloads a new + // archive into a tree that already holds `lemond` from the previous + // version. Without comparing versions the extraction would be skipped and + // the old binaries reported as the new version. + // + // The version is read from a marker inside `runtime_dir` (the same shared + // tree it describes), not from the CLI's own manifest: the manifest lives + // under `paths` (the per-invocation data dir), which is not guaranteed to + // be the same data dir that last extracted into this `runtime_dir` (e.g. + // each e2e scenario gets an isolated data dir but shares one runtime + // tree). Sourcing the check from the isolated manifest would report + // "unknown version" on every call, wipe the shared tree, and re-extract + // over it — destroying any llama.cpp backend already installed there. + let installed_version = installed_runtime_version(&runtime_dir); + if needs_extraction( + reinstall, + installed_version.as_deref(), + &version, + lemond_path_in(&runtime_dir).is_file(), + ) { + if runtime_dir.exists() { + fs::remove_dir_all(&runtime_dir) + .with_context(|| format!("failed to clear {}", runtime_dir.display()))?; + } + let extract_root = root + .join("extract") + .join(format!("{}", current_unix_millis())); + fs::create_dir_all(&extract_root)?; + extract_archive(&archive, &extract_root)?; + let embeddable_root = find_embeddable_root(&extract_root)?; + copy_tree(&embeddable_root, &runtime_dir)?; + fs::remove_dir_all(&extract_root).ok(); + record_runtime_version(&runtime_dir, &version)?; + } + drop(archive_guard); + let lemond = lemond_path_in(&runtime_dir); + let lemonade = lemonade_path_in(&runtime_dir); + if !lemond.is_file() || !lemonade.is_file() { + bail!( + "Lemonade embeddable extraction did not produce lemonade/lemond binaries in {}", + runtime_dir.display() + ); + } + Ok(LemonadeInstallManifest { + env_id: format!("lemonade-embeddable-{version}"), + version, + runtime_dir, + lemond, + lemonade, + backend_recipe: LLAMACPP_RECIPE.to_owned(), + // Resolved during install once Lemonade reports per-GPU backend support. + backend_name: ROCM_BACKEND_NAME.to_owned(), + installed_at_unix_ms: current_unix_millis(), + }) +} + +fn manifest_path(paths: &AppPaths) -> PathBuf { + paths.engine_manifests_dir(ENGINE_NAME).join("runtime.json") +} + +fn lemonade_root(paths: &AppPaths, env_root: Option<&Path>) -> PathBuf { + env_root.map_or_else( + || paths.engine_dir(ENGINE_NAME), + |root| normalize_runtime_path_for_host(root).join(ENGINE_NAME), + ) +} + +fn runtime_dir_in(root: &Path) -> PathBuf { + root.join("runtime") +} + +pub(crate) fn read_manifest(paths: &AppPaths) -> Result { + let path = manifest_path(paths); + let bytes = fs::read(&path).with_context(|| format!("failed to read {}", path.display()))?; + serde_json::from_slice(&bytes).with_context(|| format!("failed to parse {}", path.display())) +} + +pub(crate) fn write_manifest(paths: &AppPaths, manifest: &LemonadeInstallManifest) -> Result<()> { + let path = manifest_path(paths); + if let Some(parent) = path.parent() { + fs::create_dir_all(parent)?; + } + fs::write(&path, serde_json::to_vec_pretty(manifest)?)?; + Ok(()) +} + +fn lemond_path_in(runtime_dir: &Path) -> PathBuf { + runtime_dir.join(platform_binary_name("lemond")) +} + +fn lemonade_path_in(runtime_dir: &Path) -> PathBuf { + runtime_dir.join(platform_binary_name("lemonade")) +} + +pub(crate) fn platform_binary_name(name: &str) -> String { + if runtime_is_windows() { + format!("{name}.exe") + } else { + name.to_owned() + } +} + +fn embeddable_archive_name(version: &str) -> String { + let (os_arch, extension) = embeddable_os_arch(); + rocm_deps::lemonade_archive_name(version, os_arch, extension) +} + +fn embeddable_url(version: &str) -> String { + rocm_deps::lemonade_download_url(version, &embeddable_archive_name(version)) +} + +const fn embeddable_archive_sha256() -> &'static str { + if runtime_is_windows() { + EMBEDDABLE_WINDOWS_SHA256 + } else { + EMBEDDABLE_LINUX_SHA256 + } +} + +fn download_file(url: &str, destination: &Path) -> Result<()> { + download_file_to_path(url, destination, Duration::from_mins(15)) +} + +fn verify_sha256(path: &Path, expected: &str) -> Result<()> { + let mut file = fs::File::open(path) + .with_context(|| format!("failed to open {} for verification", path.display()))?; + let mut hasher = Sha256::new(); + std::io::copy(&mut file, &mut hasher) + .with_context(|| format!("failed to hash {}", path.display()))?; + let actual = format!("{:x}", hasher.finalize()); + if actual != expected { + bail!( + "SHA-256 mismatch for {}: expected {expected}, got {actual}", + path.display() + ); + } + Ok(()) +} + +fn publish_verified_archive( + temporary: tempfile::NamedTempFile, + archive: &Path, + expected_sha256: &str, +) -> Result<()> { + match temporary.persist_noclobber(archive) { + Ok(_) => Ok(()), + Err(error) if error.error.kind() == std::io::ErrorKind::AlreadyExists => { + let publication_error = error.error; + verify_sha256(archive, expected_sha256).with_context(|| { + format!( + "concurrent archive publication at {} failed verification after publish conflict ({publication_error})", + archive.display() + ) + }) + } + Err(error) => Err(error.error) + .with_context(|| format!("failed to publish verified archive {}", archive.display())), + } +} + +fn ensure_cached_archive( + url: &str, + archive: &Path, + expected_sha256: &str, + download: F, +) -> Result +where + F: FnMut(&str, &Path) -> Result<()>, +{ + ensure_cached_archive_after_invalid(url, archive, expected_sha256, download, || {}) +} + +#[derive(Debug)] +struct ArchiveCacheLock { + file: fs::File, +} + +impl Drop for ArchiveCacheLock { + fn drop(&mut self) { + let _ = self.file.unlock(); + } +} + +/// In-process exclusion for one archive path. +#[derive(Debug)] +struct ArchiveCacheProcessLock { + key: PathBuf, +} + +impl Drop for ArchiveCacheProcessLock { + fn drop(&mut self) { + let mut busy = ARCHIVE_CACHE_BUSY + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + busy.remove(&self.key); + drop(busy); + ARCHIVE_CACHE_BUSY_SIGNAL.notify_all(); + } +} + +fn lock_archive_cache_in_process(archive: &Path) -> ArchiveCacheProcessLock { + let key = archive.to_path_buf(); + let mut busy = ARCHIVE_CACHE_BUSY + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + if busy.contains(&key) { + // The holder may be inside a download window minutes long; without this + // the command sits silent and looks hung. + eprintln!( + "Waiting for another install in this process to finish with {}...", + archive.display() + ); + } + while busy.contains(&key) { + busy = ARCHIVE_CACHE_BUSY_SIGNAL + .wait(busy) + .unwrap_or_else(std::sync::PoisonError::into_inner); + } + busy.insert(key.clone()); + ArchiveCacheProcessLock { key } +} + +/// Exclusive access to one cached archive, held by the caller for as long as +/// the verified bytes are relied upon. +/// +/// Verification and use have to happen under the same lock: releasing it at the +/// end of verification would leave a window in which a co-resident writer could +/// swap the archive before it is read again for extraction. +#[derive(Debug)] +struct ArchiveCacheGuard { + _process: ArchiveCacheProcessLock, + _file: ArchiveCacheLock, +} + +fn archive_lock_path(archive: &Path) -> Result { + let parent = archive + .parent() + .with_context(|| format!("archive path has no parent: {}", archive.display()))?; + let file_name = archive + .file_name() + .with_context(|| format!("archive path has no file name: {}", archive.display()))?; + let mut lock_name = OsString::from("."); + lock_name.push(file_name); + lock_name.push(".lock"); + Ok(parent.join(lock_name)) +} + +#[cfg(test)] +fn mark_archive_lock_test_stage(stage: &str) { + const ROLE: &str = "ROCM_LEMONADE_ARCHIVE_LOCK_TEST_ROLE"; + const DIR: &str = "ROCM_LEMONADE_ARCHIVE_LOCK_TEST_DIR"; + if let (Ok(role), Some(dir)) = (std::env::var(ROLE), std::env::var_os(DIR)) { + fs::write(PathBuf::from(dir).join(format!("{role}-{stage}")), b"") + .expect("write archive lock test marker"); + } +} + +#[cfg(test)] +fn start_archive_lock_test_heartbeat() -> Option> { + const ROLE: &str = "ROCM_LEMONADE_ARCHIVE_LOCK_TEST_ROLE"; + const DIR: &str = "ROCM_LEMONADE_ARCHIVE_LOCK_TEST_DIR"; + if std::env::var(ROLE).as_deref() != Ok("recoverer") { + return None; + } + let dir = std::env::var_os(DIR)?; + Some(std::thread::spawn(move || { + std::thread::sleep(Duration::from_millis(100)); + fs::write(PathBuf::from(dir).join("recoverer-heartbeat"), b"") + .expect("write archive lock test heartbeat"); + })) +} + +fn lock_archive_cache(archive: &Path) -> Result { + let lock_path = archive_lock_path(archive)?; + let file = fs::OpenOptions::new() + .read(true) + .write(true) + .create(true) + .truncate(false) + .open(&lock_path) + .with_context(|| format!("failed to open archive cache lock {}", lock_path.display()))?; + #[cfg(test)] + mark_archive_lock_test_stage("before-lock"); + #[cfg(test)] + let heartbeat = start_archive_lock_test_heartbeat(); + // `lock` blocks with no timeout, and the holder may be minutes into a + // download, so announce the wait instead of leaving the command silent. + // Probing first keeps the common uncontended case quiet. + if file.try_lock().is_err() { + eprintln!( + "Waiting for another process to finish with {}...", + archive.display() + ); + file.lock().with_context(|| { + format!( + "failed to acquire archive cache lock {}", + lock_path.display() + ) + })?; + } + #[cfg(test)] + { + mark_archive_lock_test_stage("after-lock"); + if let Some(heartbeat) = heartbeat { + heartbeat + .join() + .expect("archive lock test heartbeat panicked"); + } + } + Ok(ArchiveCacheLock { file }) +} + +fn ensure_cached_archive_after_invalid( + url: &str, + archive: &Path, + expected_sha256: &str, + mut download: F, + after_invalid: I, +) -> Result +where + F: FnMut(&str, &Path) -> Result<()>, + I: FnOnce(), +{ + let parent = archive + .parent() + .with_context(|| format!("archive path has no parent: {}", archive.display()))?; + fs::create_dir_all(parent).with_context(|| { + format!( + "failed to create archive cache directory {}", + parent.display() + ) + })?; + let process_guard = lock_archive_cache_in_process(archive); + let archive_guard = lock_archive_cache(archive)?; + let guard = ArchiveCacheGuard { + _process: process_guard, + _file: archive_guard, + }; + + match fs::symlink_metadata(archive) { + Ok(metadata) if metadata.file_type().is_file() => { + if verify_sha256(archive, expected_sha256).is_ok() { + eprintln!("Using verified cached {}.", archive.display()); + return Ok(guard); + } + after_invalid(); + match fs::remove_file(archive) { + Ok(()) => {} + Err(error) if error.kind() == std::io::ErrorKind::NotFound => {} + Err(error) => { + return Err(error).with_context(|| { + format!("failed to remove invalid {}", archive.display()) + }); + } + } + } + Ok(_) => bail!( + "archive cache path is not a regular file: {}", + archive.display() + ), + Err(error) if error.kind() == std::io::ErrorKind::NotFound => {} + Err(error) => return Err(error.into()), + } + + let temporary = tempfile::Builder::new() + .prefix(".lemonade-download-") + .tempfile_in(parent) + .with_context(|| { + format!( + // Owner-only permissions are a Unix guarantee: `tempfile` + // creates with mode 0600 there, while on Windows the file + // inherits the parent directory's DACL. + "failed to create temporary download beside {}", + archive.display() + ) + })?; + let temporary_path = temporary.path().to_path_buf(); + eprintln!("Downloading {}...", archive.display()); + download(url, &temporary_path)?; + verify_sha256(&temporary_path, expected_sha256)?; + publish_verified_archive(temporary, archive, expected_sha256)?; + Ok(guard) +} + +/// Resolves an archive entry name to a path inside `destination`. +/// +/// Entry names come from the archive, so they are untrusted: this rejects +/// absolute paths, drive-relative and UNC names, `..` traversal, and anything +/// else that is not a plain sequence of normal components. The returned path is +/// built component-by-component, so it cannot leave `destination`. +fn archive_entry_destination(destination: &Path, name: &Path) -> Result { + let mut resolved = destination.to_path_buf(); + let mut components = 0usize; + for component in name.components() { + match component { + std::path::Component::Normal(part) => { + // Reject separators smuggled through a single component (a + // Windows-style `a\..\b` read on Unix is one component). + let text = part.to_string_lossy(); + if text.contains('/') || text.contains('\\') || text == ".." { + bail!("archive entry has an unsafe name: {}", name.display()); + } + resolved.push(part); + components += 1; + } + std::path::Component::CurDir => {} + std::path::Component::ParentDir + | std::path::Component::RootDir + | std::path::Component::Prefix(_) => { + bail!( + "archive entry escapes the extraction root: {}", + name.display() + ); + } + } + } + if components == 0 { + bail!("archive entry has an empty name"); + } + Ok(resolved) +} + +/// Creates the parent directories for an extracted entry, inside `destination`. +fn create_entry_parent(destination: &Path, path: &Path) -> Result<()> { + let Some(parent) = path.parent() else { + return Ok(()); + }; + if !parent.starts_with(destination) { + bail!( + "archive entry escapes the extraction root: {}", + path.display() + ); + } + fs::create_dir_all(parent).with_context(|| format!("failed to create {}", parent.display()))?; + Ok(()) +} + +/// Writes one extracted regular file, refusing to follow an existing entry. +/// +/// `create_new` matters: without it a directory entry extracted earlier — or a +/// symlink planted by a co-resident process between entries — would be written +/// through rather than rejected. +fn write_entry_file(path: &Path, reader: &mut dyn Read) -> Result<()> { + let mut file = fs::OpenOptions::new() + .write(true) + .create_new(true) + .open(path) + .with_context(|| format!("failed to create {}", path.display()))?; + std::io::copy(reader, &mut file) + .with_context(|| format!("failed to write {}", path.display()))?; + Ok(()) +} + +#[cfg(unix)] +fn set_entry_mode(path: &Path, mode: Option) -> Result<()> { + use std::os::unix::fs::PermissionsExt; + + let Some(mode) = mode else { + return Ok(()); + }; + // Keep the executable bit the runtime needs; drop setuid/setgid/sticky and + // any group/other write bit the archive happened to carry. + let mode = mode & 0o755; + fs::set_permissions(path, fs::Permissions::from_mode(mode)) + .with_context(|| format!("failed to set permissions on {}", path.display())) +} + +#[cfg(not(unix))] +#[expect( + clippy::unnecessary_wraps, + reason = "mirrors the Unix signature; Windows has no mode bits to apply" +)] +const fn set_entry_mode(_path: &Path, _mode: Option) -> Result<()> { + Ok(()) +} + +/// Extracts a gzip-compressed tar archive in-process. +fn extract_tar_gz(archive: &Path, destination: &Path) -> Result<()> { + let file = + fs::File::open(archive).with_context(|| format!("failed to open {}", archive.display()))?; + let mut tar = tar::Archive::new(flate2::read::GzDecoder::new(std::io::BufReader::new(file))); + for entry in tar + .entries() + .with_context(|| format!("failed to read {}", archive.display()))? + { + let mut entry = entry.with_context(|| format!("failed to read {}", archive.display()))?; + let name = entry + .path() + .with_context(|| { + format!( + "archive entry has an undecodable name in {}", + archive.display() + ) + })? + .into_owned(); + let kind = entry.header().entry_type(); + // A pax *global* header carries archive-wide metadata, not a member: its + // name (`pax_global_header` by convention) is not a path to create. The + // `tar` crate consumes long-name, long-link and pax *local* headers + // itself, but hands this one to the caller, so it has to be skipped + // explicitly — `git archive` and `tar --format=pax` both emit it, and + // treating it as a member would reject an otherwise valid archive. + if kind == tar::EntryType::XGlobalHeader { + continue; + } + let path = archive_entry_destination(destination, &name)?; + match kind { + tar::EntryType::Directory => { + fs::create_dir_all(&path) + .with_context(|| format!("failed to create {}", path.display()))?; + } + tar::EntryType::Regular | tar::EntryType::Continuous => { + create_entry_parent(destination, &path)?; + let mode = entry.header().mode().ok(); + write_entry_file(&path, &mut entry)?; + set_entry_mode(&path, mode)?; + } + // A sparse entry would expand correctly — the crate resolves the + // sparse map — but a few-byte header can declare a multi-gigabyte + // logical size, so accepting one turns a digest-pinned archive into + // an unbounded write. No Lemonade artifact stores files sparsely, so + // this stays rejected until one needs it. + tar::EntryType::GNUSparse => bail!( + "archive contains a sparse entry at {}, which is not supported", + name.display() + ), + // Links, devices, fifos and sockets are the entry kinds that let an + // archive reach outside its own tree; the official artifacts carry + // none of them. See the link policy on `extract_archive` for what + // to do if a future release ships one. + other => bail!( + "archive contains an unsupported entry ({other:?}) at {}", + name.display() + ), + } + } + Ok(()) +} + +/// Extracts a zip archive in-process. +fn extract_zip(archive: &Path, destination: &Path) -> Result<()> { + let file = + fs::File::open(archive).with_context(|| format!("failed to open {}", archive.display()))?; + let mut zip = zip::ZipArchive::new(std::io::BufReader::new(file)) + .with_context(|| format!("failed to read {}", archive.display()))?; + for index in 0..zip.len() { + let mut entry = zip + .by_index(index) + .with_context(|| format!("failed to read entry {index} of {}", archive.display()))?; + // `enclosed_name` returns None for absolute or `..`-bearing names; the + // raw name is still used for diagnostics and for the trailing-slash + // directory convention. + let raw = entry.name().to_owned(); + // See the link policy on `extract_archive`. + if entry.is_symlink() { + bail!("archive contains symlink at {raw}"); + } + let name = entry + .enclosed_name() + .with_context(|| format!("archive entry escapes the extraction root: {raw}"))?; + let path = archive_entry_destination(destination, &name)?; + if entry.is_dir() { + fs::create_dir_all(&path) + .with_context(|| format!("failed to create {}", path.display()))?; + continue; + } + // No third entry kind to reject here: `zip` defines `is_file` as + // "neither a directory nor a symlink", and both are handled above. + create_entry_parent(destination, &path)?; + let mode = entry.unix_mode(); + write_entry_file(&path, &mut entry)?; + set_entry_mode(&path, mode)?; + } + Ok(()) +} + +/// Extracts the embeddable archive into `destination`. +/// +/// Extraction is done in-process rather than by shelling out to `tar`: the +/// external binary was resolved from `PATH` (so the extractor unpacking +/// untrusted bytes was itself unpinned), and its traversal behaviour varied +/// between GNU tar, the bsdtar shipped as Windows `tar.exe`, and busybox tar. +/// +/// # Link policy +/// +/// Extraction writes directories and regular files and nothing else: a link +/// entry is rejected rather than resolved, here and again when the extracted +/// tree is searched and copied out. That is deliberately conservative. The +/// v10.10.0 artifacts contain no links, and refusing them removes the whole +/// class of "entry points outside the tree" bugs instead of trying to reason +/// about each one. +/// +/// The cost is that it fails closed on an authentic archive: if a future +/// Lemonade release ships, say, `libfoo.so -> libfoo.so.1`, installs stop with +/// "archive contains symlink at ...". That is a signal to widen the policy +/// deliberately — resolve the target and accept it only if it stays inside the +/// extraction root — not to drop the check. +fn extract_archive(archive: &Path, destination: &Path) -> Result<()> { + let is_zip = archive + .extension() + .is_some_and(|extension| extension.eq_ignore_ascii_case("zip")); + if is_zip { + extract_zip(archive, destination) + } else { + extract_tar_gz(archive, destination) + } +} + +fn find_embeddable_root(extract_root: &Path) -> Result { + let extract_root = fs::canonicalize(extract_root) + .with_context(|| format!("failed to canonicalize {}", extract_root.display()))?; + let mut candidates = Vec::new(); + collect_embeddable_roots(&extract_root, &extract_root, &mut candidates, 0)?; + candidates.into_iter().next().with_context(|| { + format!( + "no Lemonade embeddable root found in {}", + extract_root.display() + ) + }) +} + +fn collect_embeddable_roots( + extract_root: &Path, + path: &Path, + candidates: &mut Vec, + depth: usize, +) -> Result<()> { + if depth > MAX_EMBEDDABLE_SEARCH_DEPTH { + return Ok(()); + } + let metadata = fs::symlink_metadata(path)?; + // See the link policy on `extract_archive`. + if metadata.file_type().is_symlink() { + bail!("archive contains symlink at {}", path.display()); + } + if !metadata.file_type().is_dir() { + bail!("archive contains non-directory entry at {}", path.display()); + } + let canonical_path = fs::canonicalize(path)?; + if !canonical_path.starts_with(extract_root) { + bail!("archive entry escapes extraction root: {}", path.display()); + } + + let lemond = lemond_path_in(path); + let lemonade = lemonade_path_in(path); + let lemond_type = fs::symlink_metadata(&lemond) + .ok() + .map(|metadata| metadata.file_type()); + let lemonade_type = fs::symlink_metadata(&lemonade) + .ok() + .map(|metadata| metadata.file_type()); + if lemond_type.as_ref().is_some_and(fs::FileType::is_symlink) + || lemonade_type.as_ref().is_some_and(fs::FileType::is_symlink) + { + bail!( + "archive contains symlinked Lemonade binary in {}", + path.display() + ); + } + if lemond_type.as_ref().is_some_and(fs::FileType::is_file) + && lemonade_type.as_ref().is_some_and(fs::FileType::is_file) + { + candidates.push(canonical_path); + return Ok(()); + } + + for entry in fs::read_dir(path)? { + let entry = entry?; + let file_type = entry.file_type()?; + if file_type.is_symlink() { + bail!("archive contains symlink at {}", entry.path().display()); + } + if file_type.is_dir() { + collect_embeddable_roots(extract_root, &entry.path(), candidates, depth + 1)?; + } else if !file_type.is_file() { + bail!( + "archive contains non-regular entry at {}", + entry.path().display() + ); + } + } + Ok(()) +} + +fn destination_child(destination: &Path, name: &std::ffi::OsStr) -> Result { + let component = Path::new(name); + if component.components().count() != 1 + || !matches!( + component.components().next(), + Some(std::path::Component::Normal(_)) + ) + { + bail!( + "archive entry is not one destination component: {}", + Path::new(name).display() + ); + } + Ok(destination.join(component)) +} + +fn copy_tree(source: &Path, destination: &Path) -> Result<()> { + let source = fs::canonicalize(source) + .with_context(|| format!("failed to canonicalize {}", source.display()))?; + if !fs::symlink_metadata(&source)?.file_type().is_dir() { + bail!("copy source is not a directory: {}", source.display()); + } + fs::create_dir(destination) + .with_context(|| format!("failed to create {}", destination.display()))?; + let destination = fs::canonicalize(destination) + .with_context(|| format!("failed to canonicalize {}", destination.display()))?; + copy_tree_entries(&source, &source, &destination, &destination, 0) +} + +fn copy_tree_entries( + source_root: &Path, + source: &Path, + destination_root: &Path, + destination: &Path, + depth: usize, +) -> Result<()> { + if depth > MAX_COPY_RECURSION_DEPTH { + bail!( + "archive tree is deeper than {MAX_COPY_RECURSION_DEPTH} levels at {}", + source.display() + ); + } + for entry in fs::read_dir(source)? { + let entry = entry?; + let source_path = entry.path(); + let file_type = entry.file_type()?; + // See the link policy on `extract_archive`. + if file_type.is_symlink() { + bail!("refusing to copy symlink {}", source_path.display()); + } + if !file_type.is_dir() && !file_type.is_file() { + bail!( + "refusing to copy non-regular entry {}", + source_path.display() + ); + } + let canonical_source = fs::canonicalize(&source_path)?; + if !canonical_source.starts_with(source_root) { + bail!( + "copy source escapes extraction root: {}", + source_path.display() + ); + } + let destination_path = destination_child(destination, &entry.file_name())?; + if !destination_path.starts_with(destination_root) { + bail!( + "copy destination escapes runtime root: {}", + destination_path.display() + ); + } + if file_type.is_dir() { + fs::create_dir(&destination_path) + .with_context(|| format!("failed to create {}", destination_path.display()))?; + let canonical_destination = fs::canonicalize(&destination_path)?; + if !canonical_destination.starts_with(destination_root) { + bail!( + "copy destination escapes runtime root: {}", + destination_path.display() + ); + } + copy_tree_entries( + source_root, + &canonical_source, + destination_root, + &canonical_destination, + depth + 1, + )?; + } else { + fs::copy(&canonical_source, &destination_path).with_context(|| { + format!( + "failed to copy {} to {}", + canonical_source.display(), + destination_path.display() + ) + })?; + if !cfg!(windows) { + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + let metadata = fs::symlink_metadata(&canonical_source)?; + fs::set_permissions( + &destination_path, + fs::Permissions::from_mode(metadata.permissions().mode()), + )?; + } + } + } + } + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + use std::io::Write; + use std::process::{Command as ProcessCommand, Stdio}; + use std::time::Duration; + + /// A fresh scratch directory under the crate's `target/`. The base is + /// `CARGO_MANIFEST_DIR`, a compile-time constant, so the path never derives from a + /// runtime environment read. + fn scratch_dir(tag: &str) -> PathBuf { + let dir = Path::new(env!("CARGO_MANIFEST_DIR")) + .join("target") + .join(format!("lemonade-fs-test-{tag}")); + let _ = fs::remove_dir_all(&dir); + fs::create_dir_all(&dir).unwrap(); + dir + } + + #[test] + fn embeddable_package_matches_runtime_os() { + let version = rocm_deps::LEMONADE_VERSION; + let (os_arch, extension) = embeddable_os_arch(); + let suffix = format!("{os_arch}.{extension}"); + assert!(embeddable_archive_name(version).ends_with(&suffix)); + assert!(embeddable_url(version).ends_with(&suffix)); + if runtime_is_windows() { + assert_eq!(suffix, "windows-x64.zip"); + assert_eq!(platform_binary_name("lemond"), "lemond.exe"); + } else { + assert_eq!(suffix, "ubuntu-x64.tar.gz"); + assert_eq!(platform_binary_name("lemond"), "lemond"); + } + } + + #[test] + fn sha256_verification_accepts_matching_content_and_rejects_mismatch() { + let dir = scratch_dir("archive-digest"); + let archive = dir.join("archive"); + fs::write(&archive, b"verified archive").unwrap(); + let expected = format!("{:x}", Sha256::digest(b"verified archive")); + + verify_sha256(&archive, &expected).unwrap(); + let error = verify_sha256(&archive, &"0".repeat(64)).unwrap_err(); + assert!(error.to_string().contains("SHA-256 mismatch")); + fs::remove_dir_all(&dir).ok(); + } + + #[test] + fn repeated_valid_archive_publication_is_idempotent() { + let dir = scratch_dir("valid-publication-race"); + let archive = dir.join("archive"); + let content = b"verified archive"; + let expected = format!("{:x}", Sha256::digest(content)); + + for _ in 0..2 { + let mut temporary = tempfile::NamedTempFile::new_in(&dir).unwrap(); + temporary.write_all(content).unwrap(); + verify_sha256(temporary.path(), &expected).unwrap(); + publish_verified_archive(temporary, &archive, &expected).unwrap(); + } + + assert_eq!(fs::read(&archive).unwrap(), content); + assert_eq!(fs::read_dir(&dir).unwrap().count(), 1); + fs::remove_dir_all(&dir).ok(); + } + + #[test] + fn archive_publication_conflict_with_foreign_bytes_fails_closed() { + let dir = scratch_dir("invalid-publication-race"); + let archive = dir.join("archive"); + let content = b"verified archive"; + let expected = format!("{:x}", Sha256::digest(content)); + let mut temporary = tempfile::NamedTempFile::new_in(&dir).unwrap(); + temporary.write_all(content).unwrap(); + verify_sha256(temporary.path(), &expected).unwrap(); + + fs::write(&archive, b"unverified concurrent archive").unwrap(); + let error = publish_verified_archive(temporary, &archive, &expected).unwrap_err(); + let error = format!("{error:#}"); + + assert!(error.contains("concurrent archive publication")); + assert!(error.contains("failed verification after publish conflict")); + assert!(error.contains("SHA-256 mismatch")); + assert_eq!( + fs::read(&archive).unwrap(), + b"unverified concurrent archive" + ); + assert_eq!(fs::read_dir(&dir).unwrap().count(), 1); + fs::remove_dir_all(&dir).ok(); + } + + #[test] + fn poisoned_cached_archive_is_removed_and_redownloaded() { + let dir = scratch_dir("poisoned-cache"); + let archive = dir.join("archive"); + fs::write(&archive, b"poisoned").unwrap(); + let expected = format!("{:x}", Sha256::digest(b"fresh archive")); + let mut downloads = 0; + + ensure_cached_archive("test://archive", &archive, &expected, |_, destination| { + downloads += 1; + fs::write(destination, b"fresh archive")?; + Ok(()) + }) + .unwrap(); + + assert_eq!(downloads, 1); + assert_eq!(fs::read(&archive).unwrap(), b"fresh archive"); + fs::remove_dir_all(&dir).ok(); + } + + #[test] + fn download_with_wrong_content_is_rejected_and_leaves_no_archive() { + let dir = scratch_dir("bad-content-download"); + let archive = dir.join("archive"); + let expected = format!("{:x}", Sha256::digest(b"fresh archive")); + + // A download that succeeds at the transport level but returns the wrong + // bytes must not be published, even though nothing errored. + let error = ensure_cached_archive("test://archive", &archive, &expected, |_, path| { + fs::write(path, b"attacker archive")?; + Ok(()) + }) + .unwrap_err(); + + let error = format!("{error:#}"); + assert!(error.contains("SHA-256 mismatch"), "{error}"); + assert!(!archive.exists(), "unverified bytes were published"); + // Only the lock file may remain; the temporary download must be gone. + let leftovers: Vec<_> = fs::read_dir(&dir) + .unwrap() + .map(|entry| entry.unwrap().file_name()) + .filter(|name| name != ".archive.lock") + .collect(); + assert!(leftovers.is_empty(), "leftovers: {leftovers:?}"); + fs::remove_dir_all(&dir).ok(); + } + + #[test] + fn archive_guard_outlives_verification_and_is_scoped_to_one_archive() { + let dir = scratch_dir("archive-guard-scope"); + let archive = dir.join("archive"); + let other = dir.join("other-archive"); + let content = b"fresh archive"; + let expected = format!("{:x}", Sha256::digest(content)); + + // The guard returned by verification is what callers hold across + // extraction; while it is alive the same archive must not be lockable. + let guard = ensure_cached_archive("test://archive", &archive, &expected, |_, path| { + fs::write(path, content)?; + Ok(()) + }) + .unwrap(); + + let blocked = std::thread::spawn(move || { + let _second = lock_archive_cache_in_process(&archive); + }); + std::thread::sleep(Duration::from_millis(50)); + assert!( + !blocked.is_finished(), + "a second in-process lock on the held archive was granted" + ); + + // A different archive must not queue behind it. + let unrelated = std::thread::spawn(move || { + let _other = lock_archive_cache_in_process(&other); + }); + let deadline = std::time::Instant::now() + Duration::from_secs(10); + while !unrelated.is_finished() { + assert!( + std::time::Instant::now() < deadline, + "an unrelated archive serialized behind the held one" + ); + std::thread::sleep(Duration::from_millis(10)); + } + unrelated.join().unwrap(); + + drop(guard); + blocked.join().unwrap(); + fs::remove_dir_all(&dir).ok(); + } + + #[test] + fn poisoned_process_mutex_is_recovered_before_digest_validation() { + let _ = std::thread::spawn(|| { + let _guard = ARCHIVE_CACHE_BUSY.lock().unwrap(); + panic!("poison archive cache process mutex"); + }) + .join(); + let dir = scratch_dir("poisoned-process-mutex"); + let archive = dir.join("archive"); + fs::write(&archive, b"poisoned").unwrap(); + let expected = format!("{:x}", Sha256::digest(b"fresh archive")); + + ensure_cached_archive("test://archive", &archive, &expected, |_, destination| { + fs::write(destination, b"fresh archive")?; + Ok(()) + }) + .unwrap(); + + verify_sha256(&archive, &expected).unwrap(); + fs::remove_dir_all(&dir).ok(); + } + + const ARCHIVE_LOCK_TEST_ROLE: &str = "ROCM_LEMONADE_ARCHIVE_LOCK_TEST_ROLE"; + const ARCHIVE_LOCK_TEST_DIR: &str = "ROCM_LEMONADE_ARCHIVE_LOCK_TEST_DIR"; + const ARCHIVE_LOCK_TEST_TIMEOUT: Duration = Duration::from_secs(10); + + struct ChildGuard(Option); + + impl ChildGuard { + fn spawn(test_exe: &Path, dir: &Path, role: &str) -> Self { + let child = ProcessCommand::new(test_exe) + .args([ + "--exact", + "install::tests::archive_cache_subprocess_helper", + "--nocapture", + ]) + .env(ARCHIVE_LOCK_TEST_ROLE, role) + .env(ARCHIVE_LOCK_TEST_DIR, dir) + .stdin(Stdio::null()) + .stdout(Stdio::null()) + .stderr(Stdio::null()) + .spawn() + .unwrap(); + Self(Some(child)) + } + + fn is_running(&mut self) -> bool { + self.0.as_mut().unwrap().try_wait().unwrap().is_none() + } + + fn wait_until_exit(&mut self, timeout: Duration) -> std::process::ExitStatus { + let deadline = std::time::Instant::now() + timeout; + loop { + if let Some(status) = self.0.as_mut().unwrap().try_wait().unwrap() { + self.0.take(); + return status; + } + assert!( + std::time::Instant::now() < deadline, + "timed out waiting for child process {}", + self.0.as_ref().unwrap().id() + ); + std::thread::sleep(Duration::from_millis(10)); + } + } + } + + impl Drop for ChildGuard { + fn drop(&mut self) { + if let Some(mut child) = self.0.take() { + if child.try_wait().ok().flatten().is_none() { + let _ = child.kill(); + } + let _ = child.wait(); + } + } + } + + #[test] + fn archive_cache_subprocess_helper() { + let Ok(role) = std::env::var(ARCHIVE_LOCK_TEST_ROLE) else { + return; + }; + let dir = PathBuf::from(std::env::var_os(ARCHIVE_LOCK_TEST_DIR).unwrap()); + let archive = dir.join("archive"); + let expected = format!("{:x}", Sha256::digest(b"fresh archive")); + + ensure_cached_archive_after_invalid( + "test://archive", + &archive, + &expected, + |_, destination| { + fs::write(destination, b"fresh archive")?; + Ok(()) + }, + || { + if role == "validator" { + fs::write(dir.join("validator-paused"), b"").unwrap(); + let deadline = std::time::Instant::now() + ARCHIVE_LOCK_TEST_TIMEOUT; + while !dir.join("release-validator").is_file() { + assert!( + std::time::Instant::now() < deadline, + "timed out waiting to release validator" + ); + std::thread::sleep(Duration::from_millis(10)); + } + } + }, + ) + .unwrap(); + verify_sha256(&archive, &expected).unwrap(); + } + + fn wait_for_test_file(path: &Path, timeout: Duration) { + let deadline = std::time::Instant::now() + timeout; + while !path.is_file() { + assert!( + std::time::Instant::now() < deadline, + "timed out waiting for {}", + path.display() + ); + std::thread::sleep(Duration::from_millis(10)); + } + } + + #[test] + fn interprocess_lock_serializes_poisoned_validation_and_recovery() { + let dir = scratch_dir("interprocess-archive-lock"); + let archive = dir.join("archive"); + fs::write(&archive, b"poisoned").unwrap(); + let test_exe = std::env::current_exe().unwrap(); + + let mut validator = ChildGuard::spawn(&test_exe, &dir, "validator"); + wait_for_test_file(&dir.join("validator-paused"), ARCHIVE_LOCK_TEST_TIMEOUT); + let mut recoverer = ChildGuard::spawn(&test_exe, &dir, "recoverer"); + wait_for_test_file( + &dir.join("recoverer-before-lock"), + ARCHIVE_LOCK_TEST_TIMEOUT, + ); + wait_for_test_file(&dir.join("recoverer-heartbeat"), ARCHIVE_LOCK_TEST_TIMEOUT); + assert!( + !dir.join("recoverer-after-lock").is_file(), + "recoverer acquired the inter-process lock before release" + ); + assert!( + recoverer.is_running(), + "recoverer exited while waiting for the inter-process lock" + ); + + fs::write(dir.join("release-validator"), b"").unwrap(); + assert!( + validator + .wait_until_exit(ARCHIVE_LOCK_TEST_TIMEOUT) + .success(), + "validator subprocess failed" + ); + assert!( + recoverer + .wait_until_exit(ARCHIVE_LOCK_TEST_TIMEOUT) + .success(), + "recoverer subprocess failed" + ); + assert!(dir.join("recoverer-after-lock").is_file()); + let expected = format!("{:x}", Sha256::digest(b"fresh archive")); + verify_sha256(&archive, &expected).unwrap(); + fs::remove_dir_all(&dir).ok(); + } + + #[test] + fn failed_download_removes_private_partial_file() { + let dir = scratch_dir("partial-download"); + let archive = dir.join("archive"); + let expected = format!("{:x}", Sha256::digest(b"complete")); + + let error = + ensure_cached_archive("test://archive", &archive, &expected, |_, destination| { + fs::write(destination, b"partial")?; + bail!("download interrupted") + }) + .unwrap_err(); + + assert!(error.to_string().contains("download interrupted")); + assert!(!archive.exists()); + assert_eq!( + fs::read_dir(&dir) + .unwrap() + .filter( + |entry| entry.as_ref().unwrap().path() != archive_lock_path(&archive).unwrap() + ) + .count(), + 0 + ); + fs::remove_dir_all(&dir).ok(); + } + + #[test] + fn valid_embeddable_tree_is_found_and_copied() { + let dir = scratch_dir("valid-embeddable-tree"); + let extract = dir.join("extract"); + let package = extract.join("package"); + fs::create_dir_all(package.join("lib")).unwrap(); + fs::write(lemond_path_in(&package), b"lemond").unwrap(); + fs::write(lemonade_path_in(&package), b"lemonade").unwrap(); + fs::write(package.join("lib").join("backend"), b"backend").unwrap(); + + let found = find_embeddable_root(&extract).unwrap(); + assert_eq!(found, fs::canonicalize(&package).unwrap()); + let destination = dir.join("runtime"); + copy_tree(&found, &destination).unwrap(); + assert_eq!( + fs::read(destination.join("lib").join("backend")).unwrap(), + b"backend" + ); + fs::remove_dir_all(&dir).ok(); + } + + #[test] + fn deeply_nested_embeddable_tree_is_copied() { + let dir = scratch_dir("deep-embeddable-tree"); + let extract = dir.join("extract"); + let package = extract.join("package"); + let nested = package + .join("lib") + .join("site-packages") + .join("backend") + .join("kernels") + .join("gfx"); + fs::create_dir_all(&nested).unwrap(); + fs::write(lemond_path_in(&package), b"lemond").unwrap(); + fs::write(lemonade_path_in(&package), b"lemonade").unwrap(); + fs::write(nested.join("kernel.so"), b"kernel").unwrap(); + + let found = find_embeddable_root(&extract).unwrap(); + let destination = dir.join("runtime"); + copy_tree(&found, &destination).unwrap(); + assert_eq!( + fs::read( + destination + .join("lib") + .join("site-packages") + .join("backend") + .join("kernels") + .join("gfx") + .join("kernel.so") + ) + .unwrap(), + b"kernel" + ); + fs::remove_dir_all(&dir).ok(); + } + + #[cfg(unix)] + #[test] + fn embeddable_tree_rejects_external_file_symlink() { + use std::os::unix::fs::symlink; + + let dir = scratch_dir("external-file-symlink"); + let extract = dir.join("extract"); + let package = extract.join("package"); + fs::create_dir_all(&package).unwrap(); + fs::write(lemond_path_in(&package), b"lemond").unwrap(); + fs::write(lemonade_path_in(&package), b"lemonade").unwrap(); + let outside = dir.join("outside-file"); + fs::write(&outside, b"secret").unwrap(); + symlink(&outside, package.join("linked-file")).unwrap(); + + let found = find_embeddable_root(&extract).unwrap(); + let error = copy_tree(&found, &dir.join("runtime")).unwrap_err(); + assert!(error.to_string().contains("symlink")); + fs::remove_dir_all(&dir).ok(); + } + + /// Windows counterpart to the Unix symlink rejection tests. + /// + /// Creating a symlink on Windows needs privilege, but `mklink /J` creates a + /// directory junction unelevated, and Rust reports junctions as symlinks + /// (the reparse tag carries the name-surrogate bit) — so this exercises the + /// same rejection branch on the platform where the zip artifact is + /// first-class. + #[cfg(windows)] + #[test] + fn embeddable_tree_rejects_external_directory_junction() { + let dir = scratch_dir("external-directory-junction"); + let extract = dir.join("extract"); + let package = extract.join("package"); + fs::create_dir_all(&package).unwrap(); + fs::write(lemond_path_in(&package), b"lemond").unwrap(); + fs::write(lemonade_path_in(&package), b"lemonade").unwrap(); + let outside = dir.join("outside-directory"); + fs::create_dir_all(&outside).unwrap(); + fs::write(outside.join("secret.txt"), b"secret").unwrap(); + + let junction = package.join("linked-directory"); + let status = ProcessCommand::new("cmd") + .args(["/C", "mklink", "/J"]) + .arg(&junction) + .arg(&outside) + .stdin(Stdio::null()) + .stdout(Stdio::null()) + .stderr(Stdio::null()) + .status() + .unwrap(); + assert!(status.success(), "mklink /J failed"); + assert!( + fs::symlink_metadata(&junction) + .unwrap() + .file_type() + .is_symlink(), + "junction was not reported as a symlink" + ); + + let found = find_embeddable_root(&extract).unwrap(); + let error = copy_tree(&found, &dir.join("runtime")).unwrap_err(); + assert!(error.to_string().contains("symlink"), "{error}"); + assert!(!dir.join("runtime").join("linked-directory").exists()); + fs::remove_dir_all(&dir).ok(); + } + + #[cfg(unix)] + #[test] + fn embeddable_tree_rejects_external_directory_symlink() { + use std::os::unix::fs::symlink; + + let dir = scratch_dir("external-directory-symlink"); + let extract = dir.join("extract"); + fs::create_dir_all(&extract).unwrap(); + let outside = dir.join("outside-package"); + fs::create_dir_all(&outside).unwrap(); + fs::write(lemond_path_in(&outside), b"lemond").unwrap(); + fs::write(lemonade_path_in(&outside), b"lemonade").unwrap(); + symlink(&outside, extract.join("linked-package")).unwrap(); + + let error = find_embeddable_root(&extract).unwrap_err(); + assert!(error.to_string().contains("symlink")); + fs::remove_dir_all(&dir).ok(); + } + + #[test] + fn destination_child_requires_one_normal_component() { + let destination = Path::new("runtime"); + assert_eq!( + destination_child(destination, std::ffi::OsStr::new("bin")).unwrap(), + destination.join("bin") + ); + assert!(destination_child(destination, std::ffi::OsStr::new("..")).is_err()); + assert!(destination_child(destination, std::ffi::OsStr::new(".")).is_err()); + assert!(destination_child(destination, std::ffi::OsStr::new("a/b")).is_err()); + } + + #[test] + fn a_version_change_forces_re_extraction_over_the_existing_runtime() { + let pin = rocm_deps::LEMONADE_VERSION; + // Same version already unpacked: nothing to do. + assert!(!needs_extraction(false, Some(pin), pin, true)); + // A bump must not be silently skipped just because `lemond` exists. + assert!(needs_extraction(false, Some("10.10.0"), pin, true)); + // Nothing recorded, nothing unpacked, or an explicit reinstall. + assert!(needs_extraction(false, None, pin, true)); + assert!(needs_extraction(false, Some(pin), pin, false)); + assert!(needs_extraction(true, Some(pin), pin, true)); + } + + #[test] + fn runtime_version_marker_round_trips_through_needs_extraction() { + // Unit coverage for the helpers themselves and their wiring into + // `needs_extraction` — NOT a regression test for the bug this marker + // fixes (see `prepare_embeddable_trusts_the_runtime_tree_over_an_absent_manifest` + // below for that): this never calls `prepare_embeddable`, so it can't + // constrain which source it reads `installed_version` from. + let dir = tempfile::tempdir().unwrap(); + let runtime_dir = dir.path(); + + assert_eq!(installed_runtime_version(runtime_dir), None); + + record_runtime_version(runtime_dir, "7.13.0").unwrap(); + let installed = installed_runtime_version(runtime_dir); + assert_eq!(installed.as_deref(), Some("7.13.0")); + assert!(!needs_extraction( + false, + installed.as_deref(), + "7.13.0", + true + )); + assert!(needs_extraction( + false, + installed.as_deref(), + "7.14.0", + true + )); + } + + #[test] + fn prepare_embeddable_trusts_the_runtime_tree_over_an_absent_manifest() { + // The actual regression: `prepare_embeddable` must source + // `installed_version` from the marker inside `runtime_dir`, not from + // `paths`' manifest. An isolated caller (fresh `AppPaths`, no + // manifest on disk) sharing a runtime tree that a *different* caller + // already extracted must recognize it as current and skip + // re-extraction. Reverting to a manifest read (the original bug) + // makes every isolated caller miss, wipe, and re-extract — destroying + // any llama.cpp backend already installed there. Calling + // `prepare_embeddable_with` (the real function, seamed for an + // injectable download step) rather than reimplementing its decision + // inline is what makes this test fail under that reversion. + let env_root = tempfile::tempdir().unwrap(); + let paths = AppPaths { + config_dir: env_root.path().join("config"), + data_dir: env_root.path().join("data"), // no manifest ever written here + cache_dir: env_root.path().join("cache"), + }; + let version = "7.13.0"; + let root = lemonade_root(&paths, Some(env_root.path())); + let runtime_dir = runtime_dir_in(&root); + fs::create_dir_all(&runtime_dir).unwrap(); + fs::write(lemond_path_in(&runtime_dir), b"fake lemond").unwrap(); + fs::write(lemonade_path_in(&runtime_dir), b"fake lemonade").unwrap(); + record_runtime_version(&runtime_dir, version).unwrap(); + let sentinel = runtime_dir.join("llamacpp-rocm-backend.sentinel"); + fs::write(&sentinel, b"a backend a prior scenario installed").unwrap(); + + let archive_bytes = b"never parsed unless extraction wrongly triggers"; + let expected_sha256 = format!("{:x}", Sha256::digest(archive_bytes)); + + let manifest = prepare_embeddable_with( + &paths, + Some(env_root.path()), + false, + version, + &expected_sha256, + |_, destination| { + fs::write(destination, archive_bytes)?; + Ok(()) + }, + ) + .unwrap(); + + assert_eq!(manifest.version, version); + assert!( + sentinel.is_file(), + "prepare_embeddable wiped the shared runtime tree instead of recognizing it as current" + ); + } + + #[test] + fn prepare_embeddable_records_the_version_after_a_real_extraction() { + // The write-side sibling of the test above: an isolated caller + // extracting into an empty `runtime_dir` for the first time must + // leave a marker behind, or the *next* isolated caller sharing this + // tree sees no marker, wipes, and re-extracts on every call — the + // same defect the read-side regression guards against, from the + // write side instead. Dropping `record_runtime_version`'s call site + // in `prepare_embeddable` fails this test. + let env_root = tempfile::tempdir().unwrap(); + let paths = AppPaths { + config_dir: env_root.path().join("config"), + data_dir: env_root.path().join("data"), + cache_dir: env_root.path().join("cache"), + }; + let version = "7.13.0"; + + // A real, well-formed archive (unlike the stub above): this test + // exercises the extraction branch, so `extract_archive` must succeed. + // `extract_archive` dispatches on the destination's extension, which + // `prepare_embeddable_with` derives from the current platform (see + // `embeddable_os_arch`) — so the bytes built here must be in that + // same format, not always a tar.gz. + let (_, extension) = embeddable_os_arch(); + let archive_dir = tempfile::tempdir().unwrap(); + let archive = archive_dir.path().join(format!("embeddable.{extension}")); + let entry_names = [ + platform_binary_name("lemond"), + platform_binary_name("lemonade"), + ]; + if runtime_is_windows() { + let file = fs::File::create(&archive).unwrap(); + let mut writer = zip::ZipWriter::new(file); + let options = zip::write::SimpleFileOptions::default(); + for name in &entry_names { + writer + .start_file(format!("embeddable/bin/{name}"), options) + .unwrap(); + writer.write_all(b"hello").unwrap(); + } + writer.finish().unwrap(); + } else { + let file = fs::File::create(&archive).unwrap(); + let encoder = flate2::write::GzEncoder::new(file, flate2::Compression::fast()); + let mut builder = tar::Builder::new(encoder); + for name in &entry_names { + let mut header = tar::Header::new_ustar(); + header.set_size(5); + header.set_mode(0o755); + header.set_entry_type(tar::EntryType::Regular); + header.set_path(format!("embeddable/bin/{name}")).unwrap(); + header.set_cksum(); + builder.append(&header, &b"hello"[..]).unwrap(); + } + builder.into_inner().unwrap().finish().unwrap(); + } + let expected_sha256 = format!("{:x}", Sha256::digest(fs::read(&archive).unwrap())); + + let root = lemonade_root(&paths, Some(env_root.path())); + let runtime_dir = runtime_dir_in(&root); + assert_eq!(installed_runtime_version(&runtime_dir), None); + + prepare_embeddable_with( + &paths, + Some(env_root.path()), + false, + version, + &expected_sha256, + |_, destination| { + fs::copy(&archive, destination)?; + Ok(()) + }, + ) + .unwrap(); + + assert_eq!( + installed_runtime_version(&runtime_dir).as_deref(), + Some(version), + "prepare_embeddable extracted the archive but never recorded the version" + ); + } + + #[test] + fn lemonade_root_uses_requested_engine_root() { + let paths = AppPaths { + config_dir: PathBuf::from("C:/Users/test/.rocm"), + data_dir: PathBuf::from("C:/Users/test/.rocm"), + cache_dir: PathBuf::from("C:/Users/test/.rocm/cache"), + }; + let engine_root = PathBuf::from("D:/path/to/therock_venvs/engines"); + + assert_eq!( + lemonade_root(&paths, Some(&engine_root)), + normalize_runtime_path_for_host(&engine_root).join(ENGINE_NAME) + ); + } + + /// Writes a `.tar.gz` whose single entry carries `entry_name` verbatim. + /// + /// The name is stamped into the raw header bytes because `tar::Builder` + /// refuses to serialize `..` or absolute paths — the archives this guards + /// against are hostile ones, which no well-behaved writer would produce. + fn write_raw_named_tar_gz(path: &Path, entry_name: &str, body: &[u8]) { + let file = fs::File::create(path).unwrap(); + let encoder = flate2::write::GzEncoder::new(file, flate2::Compression::fast()); + let mut builder = tar::Builder::new(encoder); + let mut header = tar::Header::new_ustar(); + header.set_size(body.len() as u64); + header.set_mode(0o644); + header.set_entry_type(tar::EntryType::Regular); + let name = header.as_old_mut(); + let bytes = entry_name.as_bytes(); + assert!(bytes.len() < name.name.len()); + name.name[..bytes.len()].copy_from_slice(bytes); + header.set_cksum(); + builder.append(&header, body).unwrap(); + builder.into_inner().unwrap().finish().unwrap(); + } + + const EXTRACT_PATH_TEST_DIR: &str = "ROCM_LEMONADE_EXTRACT_PATH_TEST_DIR"; + + /// Child half of [`extract_archive_does_not_execute_a_path_supplied_tar`]. + /// + /// Runs in its own process so the hostile `PATH` can be set with + /// `Command::env` — mutating `PATH` in-process would need `unsafe`, which + /// the workspace denies. + #[test] + fn extract_path_subprocess_helper() { + let Some(dir) = std::env::var_os(EXTRACT_PATH_TEST_DIR) else { + return; + }; + let dir = PathBuf::from(dir); + let _ = extract_archive(&dir.join("archive.tar.gz"), &dir.join("extract")); + } + + #[cfg(unix)] + #[test] + fn extract_archive_does_not_execute_a_path_supplied_tar() { + use std::os::unix::fs::PermissionsExt; + + let dir = scratch_dir("path-supplied-tar"); + let archive = dir.join("archive.tar.gz"); + fs::create_dir_all(dir.join("extract")).unwrap(); + write_raw_named_tar_gz(&archive, "payload.txt", b"real"); + + let fake_bin = dir.join("bin"); + fs::create_dir_all(&fake_bin).unwrap(); + let marker = dir.join("impostor-ran"); + let fake_tar = fake_bin.join("tar"); + fs::write( + &fake_tar, + format!("#!/bin/sh\ntouch '{}'\nexit 0\n", marker.display()), + ) + .unwrap(); + fs::set_permissions(&fake_tar, fs::Permissions::from_mode(0o755)).unwrap(); + + let mut entries = vec![fake_bin]; + entries.extend(std::env::split_paths( + &std::env::var_os("PATH").unwrap_or_default(), + )); + let status = ProcessCommand::new(std::env::current_exe().unwrap()) + .args([ + "--exact", + "install::tests::extract_path_subprocess_helper", + "--nocapture", + ]) + .env(EXTRACT_PATH_TEST_DIR, &dir) + .env("PATH", std::env::join_paths(entries).unwrap()) + .stdin(Stdio::null()) + .stdout(Stdio::null()) + .stderr(Stdio::null()) + .status() + .unwrap(); + assert!(status.success(), "helper subprocess failed"); + + assert!( + !marker.exists(), + "extraction executed a PATH-supplied tar instead of a pinned in-process extractor" + ); + fs::remove_dir_all(&dir).ok(); + } + + fn extract_hostile_tar_entry(tag: &str, entry_name: &str) -> String { + let dir = scratch_dir(tag); + let archive = dir.join("hostile.tar.gz"); + let destination = dir.join("extract"); + fs::create_dir_all(&destination).unwrap(); + write_raw_named_tar_gz(&archive, entry_name, b"pwned"); + + let error = extract_archive(&archive, &destination).unwrap_err(); + assert_eq!( + fs::read_dir(&destination).unwrap().count(), + 0, + "hostile entry was written into the extraction root" + ); + fs::remove_dir_all(&dir).ok(); + format!("{error:#}") + } + + #[test] + fn extract_archive_rejects_parent_traversal_entry() { + let error = extract_hostile_tar_entry("traversal-parent", "../escaped.txt"); + assert!(error.contains("escapes the extraction root"), "{error}"); + } + + #[test] + fn extract_archive_rejects_nested_parent_traversal_entry() { + let error = extract_hostile_tar_entry("traversal-nested", "inner/../../escaped.txt"); + assert!(error.contains("escapes the extraction root"), "{error}"); + } + + #[test] + fn extract_archive_rejects_absolute_entry() { + let error = extract_hostile_tar_entry("traversal-absolute", "/etc/escaped.txt"); + assert!(error.contains("escapes the extraction root"), "{error}"); + } + + #[test] + fn extract_archive_rejects_symlink_entry() { + let dir = scratch_dir("traversal-symlink"); + let archive = dir.join("hostile.tar.gz"); + let destination = dir.join("extract"); + fs::create_dir_all(&destination).unwrap(); + + let file = fs::File::create(&archive).unwrap(); + let encoder = flate2::write::GzEncoder::new(file, flate2::Compression::fast()); + let mut builder = tar::Builder::new(encoder); + let mut header = tar::Header::new_ustar(); + header.set_size(0); + header.set_mode(0o777); + header.set_entry_type(tar::EntryType::Symlink); + header.set_link_name("/etc/passwd").unwrap(); + header.set_path("escape-link").unwrap(); + header.set_cksum(); + builder.append(&header, std::io::empty()).unwrap(); + builder.into_inner().unwrap().finish().unwrap(); + + let error = format!("{:#}", extract_archive(&archive, &destination).unwrap_err()); + assert!(error.contains("unsupported entry"), "{error}"); + assert!(!destination.join("escape-link").exists()); + fs::remove_dir_all(&dir).ok(); + } + + #[test] + fn extract_archive_writes_a_well_formed_tree() { + let dir = scratch_dir("extract-well-formed"); + let archive = dir.join("good.tar.gz"); + let destination = dir.join("extract"); + fs::create_dir_all(&destination).unwrap(); + + let file = fs::File::create(&archive).unwrap(); + let encoder = flate2::write::GzEncoder::new(file, flate2::Compression::fast()); + let mut builder = tar::Builder::new(encoder); + let mut header = tar::Header::new_ustar(); + header.set_size(5); + header.set_mode(0o755); + header.set_entry_type(tar::EntryType::Regular); + header.set_path("embeddable/bin/lemond").unwrap(); + header.set_cksum(); + builder.append(&header, &b"hello"[..]).unwrap(); + builder.into_inner().unwrap().finish().unwrap(); + + extract_archive(&archive, &destination).unwrap(); + let extracted = destination.join("embeddable").join("bin").join("lemond"); + assert_eq!(fs::read(&extracted).unwrap(), b"hello"); + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + let mode = fs::metadata(&extracted).unwrap().permissions().mode(); + assert_eq!(mode & 0o111, 0o111, "executable bit was not preserved"); + } + fs::remove_dir_all(&dir).ok(); + } + + #[test] + fn extract_archive_rejects_hostile_zip_entry() { + let dir = scratch_dir("traversal-zip"); + let archive = dir.join("hostile.zip"); + let destination = dir.join("extract"); + fs::create_dir_all(&destination).unwrap(); + + let file = fs::File::create(&archive).unwrap(); + let mut writer = zip::ZipWriter::new(file); + let options = zip::write::SimpleFileOptions::default(); + // `add_directory`/`start_file` normalize names, so the traversal name is + // written through the raw entry API a hostile producer would use. + writer + .start_file("../escaped.txt", options) + .or_else(|_| writer.start_file("placeholder.txt", options)) + .unwrap(); + writer.write_all(b"pwned").unwrap(); + writer.finish().unwrap(); + + // If the zip writer refused to emit the traversal name there is nothing + // hostile to assert on; the tar cases cover the shared validator. + let mut zip = zip::ZipArchive::new(fs::File::open(&archive).unwrap()).unwrap(); + let hostile = zip.by_index(0).unwrap().name().to_owned(); + if hostile == "../escaped.txt" { + let error = format!("{:#}", extract_archive(&archive, &destination).unwrap_err()); + assert!(error.contains("escapes the extraction root"), "{error}"); + assert!(!dir.join("escaped.txt").exists()); + } else { + extract_archive(&archive, &destination).unwrap(); + assert!(destination.join("placeholder.txt").is_file()); + } + fs::remove_dir_all(&dir).ok(); + } + + /// A pax global header is archive-wide metadata, not a member, and the + /// `tar` crate hands it to the caller rather than consuming it the way it + /// consumes long-name and pax local headers. Treating it as a member + /// rejects any archive written by `git archive` or `tar --format=pax`. + #[test] + fn extract_archive_skips_pax_global_header() { + let dir = scratch_dir("pax-global-header"); + let archive = dir.join("pax.tar.gz"); + let destination = dir.join("extract"); + fs::create_dir_all(&destination).unwrap(); + + let file = fs::File::create(&archive).unwrap(); + let encoder = flate2::write::GzEncoder::new(file, flate2::Compression::fast()); + let mut builder = tar::Builder::new(encoder); + + let record = b"52 comment=0000000000000000000000000000000000000000\n"; + let mut global = tar::Header::new_ustar(); + global.set_size(record.len() as u64); + global.set_mode(0o644); + global.set_entry_type(tar::EntryType::XGlobalHeader); + global.set_path("pax_global_header").unwrap(); + global.set_cksum(); + builder.append(&global, &record[..]).unwrap(); + + let mut header = tar::Header::new_ustar(); + header.set_size(5); + header.set_mode(0o755); + header.set_entry_type(tar::EntryType::Regular); + header.set_path("embeddable/bin/lemond").unwrap(); + header.set_cksum(); + builder.append(&header, &b"hello"[..]).unwrap(); + builder.into_inner().unwrap().finish().unwrap(); + + extract_archive(&archive, &destination).unwrap(); + assert_eq!( + fs::read(destination.join("embeddable").join("bin").join("lemond")).unwrap(), + b"hello" + ); + assert!( + !destination.join("pax_global_header").exists(), + "the global header was written out as a member" + ); + fs::remove_dir_all(&dir).ok(); + } + + /// Zip counterpart to `extract_archive_rejects_symlink_entry`: the zip path + /// has its own link check, keyed on the entry's unix mode. + #[test] + fn extract_archive_rejects_zip_symlink_entry() { + let dir = scratch_dir("zip-symlink"); + let archive = dir.join("hostile.zip"); + let destination = dir.join("extract"); + fs::create_dir_all(&destination).unwrap(); + + let file = fs::File::create(&archive).unwrap(); + let mut writer = zip::ZipWriter::new(file); + writer + .add_symlink( + "escape-link", + "/etc/passwd", + zip::write::SimpleFileOptions::default(), + ) + .unwrap(); + writer.finish().unwrap(); + + let error = format!("{:#}", extract_archive(&archive, &destination).unwrap_err()); + assert!(error.contains("symlink"), "{error}"); + assert!(!destination.join("escape-link").exists()); + fs::remove_dir_all(&dir).ok(); + } +} diff --git a/engines/lemonade/src/lib.rs b/engines/lemonade/src/lib.rs index 154d90c0f8..a517111433 100644 --- a/engines/lemonade/src/lib.rs +++ b/engines/lemonade/src/lib.rs @@ -4,115 +4,48 @@ use anyhow::{Context, Result, anyhow, bail}; use clap::{Parser, Subcommand}; -use rocm_core::{ - AppPaths, DEFAULT_LOCAL_PORT, RocmCliConfig, active_managed_therock_environment, - active_managed_therock_version, download_file_to_path, format_http_base_url, - http_get_text_with_auth, normalize_runtime_path_for_host, prepend_runtime_paths, - require_nonempty, runtime_is_linux, runtime_is_windows, -}; +use rocm_core::{AppPaths, DEFAULT_LOCAL_PORT, normalize_runtime_path_for_host, runtime_is_linux}; use rocm_engine_protocol::{ - DetectRequest, DetectResponse, DevicePolicy, ENGINE_RECIPE_CONTRACT_VERSION, EndpointRequest, - EndpointResponse, EngineCapabilities, EngineDeviceAvailability, EngineMethod, EngineRecipeHint, - EngineRequestEnvelope, EngineResponseEnvelope, GpuSelection, HealthcheckRequest, - HealthcheckResponse, InstallRequest, InstallResponse, LaunchRequest, LaunchResponse, - LogsRequest, LogsResponse, ResolveModelRequest, ResolveModelResponse, StopRequest, - StopResponse, + DetectRequest, DetectResponse, DevicePolicy, EndpointRequest, EndpointResponse, + EngineCapabilities, EngineMethod, EngineRequestEnvelope, EngineResponseEnvelope, + HealthcheckRequest, HealthcheckResponse, InstallRequest, InstallResponse, LaunchRequest, + LaunchResponse, LogsRequest, LogsResponse, ResolveModelRequest, ResolveModelResponse, + StopRequest, StopResponse, }; use serde::{Deserialize, Serialize}; use serde_json::{Value, json}; -use sha2::{Digest, Sha256}; -use std::collections::{VecDeque, hash_map::DefaultHasher}; -#[cfg(unix)] -use std::ffi::OsStr; -use std::ffi::OsString; -use std::fs; -use std::hash::{Hash, Hasher}; -use std::io::{BufRead, Read, Seek, Write}; -use std::net::TcpListener; -use std::path::{Path, PathBuf}; -use std::process::{Command as ProcessCommand, Stdio}; +use std::io::Read; +use std::path::PathBuf; use std::time::{Duration, SystemTime, UNIX_EPOCH}; -const ENGINE_NAME: &str = "lemonade"; -const DEFAULT_HOST: &str = "127.0.0.1"; -const DEFAULT_MODEL: &str = "Qwen3-4B-Instruct-2507-GGUF"; -const DEFAULT_MODEL_REPO_DIR: &str = "models--unsloth--Qwen3-4B-Instruct-2507-GGUF"; -const DEFAULT_MODEL_GGUF: &str = "Qwen3-4B-Instruct-2507-Q4_K_M.gguf"; -const LLAMACPP_RECIPE: &str = "llamacpp"; -const ROCM_BACKEND_NAME: &str = "rocm"; -/// Resource file inside the extracted Lemonade embeddable pinning which -/// llama.cpp build (and paired ROCm version) `llamacpp:rocm` downloads. Its own -/// header comment invites hand-editing: "You can modify these values to pin -/// specific versions without rebuilding the application." -const BACKEND_VERSIONS_RESOURCE: &str = "resources/backend_versions.json"; -/// Records the extracted embeddable version inside the runtime tree itself -/// (not the CLI's own manifest) so the version check survives even when the -/// manifest lives under a data dir that is not shared with the runtime tree -/// (e.g. each e2e scenario's isolated data dir vs. a shared runtime tree). -/// Living inside `runtime_dir` rather than beside it also means a wipe of the -/// tree removes the marker in the same operation — "marker present implies -/// extraction completed" holds by construction. Without this marker, -/// [`prepare_embeddable`] can never prove the shared tree is already current, -/// wipes it, and re-extracts on every call — destroying any llama.cpp backend -/// installed into it in the process. -const RUNTIME_VERSION_MARKER: &str = "runtime-version.txt"; +mod backend_alignment; +mod direct_llama; +mod install; +mod process; +mod runtime_dir; +mod state; + +// Re-exported so this stays reachable at its pre-split crate-root path — +// `backend_alignment` is a private module, but this constant was `pub` at +// the crate root before the split, and `apps/rocm` depends on this crate's +// lib target. +pub use backend_alignment::LEMONADE_BACKEND_ALIGNMENT_DISABLED_ENV; + +pub(crate) const ENGINE_NAME: &str = "lemonade"; +pub(crate) const DEFAULT_HOST: &str = "127.0.0.1"; +pub(crate) const DEFAULT_MODEL: &str = "Qwen3-4B-Instruct-2507-GGUF"; +pub(crate) const DEFAULT_MODEL_REPO_DIR: &str = "models--unsloth--Qwen3-4B-Instruct-2507-GGUF"; +pub(crate) const DEFAULT_MODEL_GGUF: &str = "Qwen3-4B-Instruct-2507-Q4_K_M.gguf"; +pub(crate) const LLAMACPP_RECIPE: &str = "llamacpp"; +pub(crate) const ROCM_BACKEND_NAME: &str = "rocm"; #[cfg(feature = "e2e-test-hooks")] const BACKEND_INSTALL_FAILURE_TEST_ENV: &str = "ROCM_E2E_LEMONADE_BACKEND_INSTALL_FAILURE"; -/// Preferred llama.cpp backends, best first. Lemonade reports per-GPU support; -/// we pick the highest-priority backend it considers supported on this host. -/// GPU backends only — `cpu` is intentionally excluded so the router path never -/// selects CPU under the GPU-required policy (AGENTS.md §6, matching -/// `DIRECT_LLAMA_SERVER_BACKENDS`). When only CPU is usable, selection returns -/// `None` and the caller fails with actionable guidance. -const LLAMACPP_BACKEND_PRIORITY: [&str; 2] = ["rocm", "vulkan"]; const DEFAULT_LOG_TAIL_LINES: usize = 200; /// How long a stop waits for the server to actually exit after each signal /// before reporting a timeout (or, under `force`, escalating to `SIGKILL`). const STOP_GRACE: Duration = Duration::from_secs(10); /// Lemonade state identifies the actual llama-server process directly. const STOP_SCOPE: rocm_core::KillScope = rocm_core::KillScope::Single; -const STARTUP_FAILURE_LOG_TAIL_LINES: usize = 80; -/// Maximum bytes to read from the end of a log file when extracting tail lines. -/// Prevents reading entire gigabyte-sized logs on startup timeout. -const MAX_TAIL_READ: u64 = 4 * 1024 * 1024; // 4MB -/// Archive paths currently being validated, published, or extracted by this -/// process, with a condvar to wake waiters. Keyed per archive so an install of -/// one archive does not serialize behind an unrelated one — the download window -/// is minutes long, and the cross-process guarantee comes from the per-archive -/// file lock, not from this set. -static ARCHIVE_CACHE_BUSY: std::sync::Mutex> = - std::sync::Mutex::new(std::collections::BTreeSet::new()); -static ARCHIVE_CACHE_BUSY_SIGNAL: std::sync::Condvar = std::sync::Condvar::new(); - -/// How deep the extracted tree is searched for the embeddable root before the -/// search gives up. Official archives place the root at the top level, so this -/// only bounds the work spent looking; exceeding it is not an error. -const MAX_EMBEDDABLE_SEARCH_DEPTH: usize = 4; - -/// Recursion backstop for copying the embeddable tree. Containment is enforced -/// per entry, so this exists only to fail loudly instead of overflowing the -/// stack on a pathological tree; it is far above any plausible archive layout. -const MAX_COPY_RECURSION_DEPTH: usize = 64; - -/// Digests of the pinned embeddable archives. Unlike the archive names and -/// URLs, these cannot be derived from the version in `runtime-deps.toml`: each -/// release has its own digest, so bumping the pin means recording the new pair -/// here as well. -const EMBEDDABLE_WINDOWS_SHA256: &str = - "50a133bbc35c4f3f8971eafef2c9fe56c4bbbfb0f1032bf8728324d8d8c8a0e1"; -const EMBEDDABLE_LINUX_SHA256: &str = - "bdfd3c3e5d6eda5101c8a32f36e6dd9236ceb9ab2eb66734c32628e6e86e18ac"; - -/// Embeddable asset token and archive extension for the host this adapter runs -/// on. Only `windows-x64` and `ubuntu-x64` are wired up here; the release also -/// publishes `macos-arm64` and `ubuntu-arm64`, which this adapter never selects. -const fn embeddable_os_arch() -> (&'static str, &'static str) { - if runtime_is_windows() { - ("windows-x64", "zip") - } else { - ("ubuntu-x64", "tar.gz") - } -} #[derive(Parser)] #[command(name = "rocm-engine-lemonade")] @@ -175,50 +108,19 @@ enum CommandKind { }, } -#[derive(Debug, Clone, Serialize, Deserialize)] -struct LemonadeInstallManifest { - env_id: String, - version: String, - runtime_dir: PathBuf, - lemond: PathBuf, - lemonade: PathBuf, - backend_recipe: String, - backend_name: String, - installed_at_unix_ms: u128, -} - -#[derive(Debug, Clone)] -struct LemonadeRuntime { - manifest: LemonadeInstallManifest, -} - -#[derive(Debug, Clone, Default)] -struct LemonadeProcessEnvironment { - rocm_root: Option, - path_entries: Vec, - library_entries: Vec, - gpu_indices: Vec, -} - #[derive(Debug, Clone)] -struct ServeHttpRequest { - service_id: String, - model_ref: String, - host: String, - port: u16, - device_policy: DevicePolicy, - gpu_indices: Vec, - runtime_id: Option, - env_id: Option, - state_path: PathBuf, - log_path: Option, - engine_recipe: Option, -} - -#[derive(Debug, Clone)] -struct ServiceFiles { - state_path: PathBuf, - log_path: PathBuf, +pub(crate) struct ServeHttpRequest { + pub(crate) service_id: String, + pub(crate) model_ref: String, + pub(crate) host: String, + pub(crate) port: u16, + pub(crate) device_policy: DevicePolicy, + pub(crate) gpu_indices: Vec, + pub(crate) runtime_id: Option, + pub(crate) env_id: Option, + pub(crate) state_path: PathBuf, + pub(crate) log_path: Option, + pub(crate) engine_recipe: Option, } pub fn run_cli() -> Result<()> { @@ -260,10 +162,12 @@ pub fn run_cli() -> Result<()> { model_ref, host, port, - device_policy: Some(parse_device_policy_arg(device_policy.as_deref())?), + device_policy: Some(crate::state::parse_device_policy_arg( + device_policy.as_deref(), + )?), endpoint_mode: Some("openai".to_owned()), engine_recipe: None, - gpu_selection: parse_gpu_selection_arg(gpu.as_deref())?, + gpu_selection: crate::state::parse_gpu_selection_arg(gpu.as_deref())?, })?)?, CommandKind::Stdio => { let envelope = read_request()?; @@ -286,13 +190,13 @@ pub fn run_cli() -> Result<()> { model_ref, host, port, - device_policy: parse_device_policy_arg(device_policy.as_deref())?, - gpu_indices: parse_gpu_indices_arg(gpu.as_deref())?, + device_policy: crate::state::parse_device_policy_arg(device_policy.as_deref())?, + gpu_indices: crate::state::parse_gpu_indices_arg(gpu.as_deref())?, runtime_id, env_id, state_path, log_path, - engine_recipe: parse_engine_recipe_json(engine_recipe_json)?, + engine_recipe: crate::state::parse_engine_recipe_json(engine_recipe_json)?, })?, } Ok(()) @@ -314,7 +218,7 @@ pub fn builtin_serve_http( env_id: Option, state_path: PathBuf, log_path: Option, - engine_recipe: Option, + engine_recipe: Option, ) -> Result<()> { serve_http(ServeHttpRequest { service_id, @@ -404,7 +308,7 @@ fn capabilities() -> EngineCapabilities { } fn detect_response() -> DetectResponse { - let runtime = resolve_runtime().ok(); + let runtime = crate::backend_alignment::resolve_runtime().ok(); let mut notes = Vec::new(); if let Some(runtime) = runtime.as_ref() { notes.push(format!( @@ -435,7 +339,7 @@ fn detect_response() -> DetectResponse { python_version: None, torch_version: None, transformers_version: None, - available_devices: vec![gpu_availability_device(runtime.is_some())], + available_devices: vec![crate::state::gpu_availability_device(runtime.is_some())], capabilities: capabilities(), notes, } @@ -450,7 +354,9 @@ fn install_response(request: InstallRequest) -> Result { // and exercising it must not download or alter a runtime on the test host. #[cfg(feature = "e2e-test-hooks")] if std::env::var_os(BACKEND_INSTALL_FAILURE_TEST_ENV).is_some() { - install_llamacpp_backend_with_retry(|| bail!("scripted Lemonade backend install failure"))?; + crate::backend_alignment::install_llamacpp_backend_with_retry(|| { + bail!("scripted Lemonade backend install failure") + })?; } eprintln!( "Preparing Lemonade embeddable {}...", @@ -460,10 +366,12 @@ fn install_response(request: InstallRequest) -> Result { .env_root .as_deref() .map(normalize_runtime_path_for_host); - let mut manifest = prepare_embeddable(&paths, env_root.as_deref(), request.reinstall)?; + let mut manifest = + crate::install::prepare_embeddable(&paths, env_root.as_deref(), request.reinstall)?; eprintln!("Detecting best supported Lemonade llama.cpp backend..."); - let aligned_version = prepare_llamacpp_backend_for_active_rocm(&paths, &mut manifest)?; - write_manifest(&paths, &manifest)?; + let aligned_version = + crate::backend_alignment::prepare_llamacpp_backend_for_active_rocm(&paths, &mut manifest)?; + crate::install::write_manifest(&paths, &manifest)?; let mut warnings = vec![ "Lemonade is installed as a rocm-cli managed embeddable runtime".to_owned(), format!( @@ -491,15 +399,15 @@ fn install_response(request: InstallRequest) -> Result { ), ], capabilities: capabilities(), - lock_hash: manifest_lock_hash(&manifest), + lock_hash: crate::state::manifest_lock_hash(&manifest), warnings, }) } fn resolve_model_response(request: ResolveModelRequest) -> Result { - let device_policy = normalize_device_policy(request.device_policy)?; - let engine_recipe = accepted_engine_recipe(request.engine_recipe)?; - let canonical_model_id = resolve_lemonade_model_ref(&request.model_ref); + let device_policy = crate::state::normalize_device_policy(request.device_policy)?; + let engine_recipe = crate::state::accepted_engine_recipe(request.engine_recipe)?; + let canonical_model_id = crate::state::resolve_lemonade_model_ref(&request.model_ref); Ok(ResolveModelResponse { canonical_model_id, task: "chat-completions".to_owned(), @@ -524,25 +432,27 @@ fn resolve_model_response(request: ResolveModelRequest) -> Result Result { - require_nonempty(&request.service_id, "service_id")?; - require_nonempty(&request.model_ref, "model_ref")?; - request.device_policy = Some(normalize_device_policy(request.device_policy.clone())?); - request.engine_recipe = accepted_engine_recipe(request.engine_recipe)?; - let runtime = resolve_runtime()?; + rocm_core::require_nonempty(&request.service_id, "service_id")?; + rocm_core::require_nonempty(&request.model_ref, "model_ref")?; + request.device_policy = Some(crate::state::normalize_device_policy( + request.device_policy.clone(), + )?); + request.engine_recipe = crate::state::accepted_engine_recipe(request.engine_recipe)?; + let runtime = crate::backend_alignment::resolve_runtime()?; let paths = AppPaths::discover()?; paths.ensure()?; - fs::create_dir_all(paths.engine_logs_dir(ENGINE_NAME))?; - fs::create_dir_all(paths.engine_state_dir(ENGINE_NAME))?; + std::fs::create_dir_all(paths.engine_logs_dir(ENGINE_NAME))?; + std::fs::create_dir_all(paths.engine_state_dir(ENGINE_NAME))?; let log_path = paths .engine_logs_dir(ENGINE_NAME) .join(format!("{}.log", request.service_id)); let state_path = paths .engine_state_dir(ENGINE_NAME) .join(format!("{}.json", request.service_id)); - let endpoint_url = endpoint_url(&request.host, request.port); + let endpoint_url = crate::state::endpoint_url(&request.host, request.port); let serve_request = ServeHttpRequest { service_id: request.service_id.clone(), - model_ref: resolve_lemonade_model_ref(&request.model_ref), + model_ref: crate::state::resolve_lemonade_model_ref(&request.model_ref), host: request.host.clone(), port: request.port, device_policy: request @@ -558,16 +468,16 @@ fn launch_service(mut request: LaunchRequest) -> Result { }; let current_exe = std::env::current_exe().context("failed to discover current Lemonade engine binary")?; - let args = serve_http_command_args(&serve_request); - write_running_state( + let args = crate::state::serve_http_command_args(&serve_request); + crate::state::write_running_state( &serve_request, &runtime, std::process::id(), None, "starting", )?; - let wrapper_pid = spawn_serve_http_background(¤t_exe, &args)?; - merge_json_state( + let wrapper_pid = crate::state::spawn_serve_http_background(¤t_exe, &args)?; + crate::state::merge_json_state( &state_path, &json!({ "pid": wrapper_pid, @@ -587,30 +497,41 @@ fn launch_service(mut request: LaunchRequest) -> Result { } fn serve_http(mut request: ServeHttpRequest) -> Result<()> { - require_gpu_required(&request.device_policy)?; + crate::state::require_gpu_required(&request.device_policy)?; // Verify a real GPU is available before launching anything: zero usable // devices fails fast, `auto` resolves to the first present device (never an // assumed device 0), and an explicit `--gpu` index is rejected when it is not // actually present. This runs before any server process is spawned. - request.gpu_indices = resolve_serve_gpu_indices(&request.gpu_indices)?; - let mut runtime = resolve_runtime()?; - let mut process_env = lemonade_process_environment()?; + request.gpu_indices = crate::state::resolve_serve_gpu_indices(&request.gpu_indices)?; + let mut runtime = crate::backend_alignment::resolve_runtime()?; + let mut process_env = crate::process::lemonade_process_environment()?; process_env.gpu_indices = request.gpu_indices.clone(); let log_path = request.log_path.as_deref(); - write_running_state(&request, &runtime, std::process::id(), None, "starting")?; + crate::state::write_running_state(&request, &runtime, std::process::id(), None, "starting")?; // A canonical Hugging Face checkpoint (owner/repo:variant) cannot be served under its // exact name through Lemonade's model router (Lemonade renames registered models and // its registry has several naming quirks). Download the GGUF and run a packaged // llama-server on it directly with `--alias`, which serves it under exactly that name. - if let Some(checkpoint) = parse_hf_checkpoint(&request.model_ref) { - return serve_hf_checkpoint(&request, &runtime, &process_env, log_path, &checkpoint); + if let Some(checkpoint) = crate::direct_llama::parse_hf_checkpoint(&request.model_ref) { + return crate::direct_llama::serve_hf_checkpoint( + &request, + &runtime, + &process_env, + log_path, + &checkpoint, + ); } if runtime_is_linux() - && let Some(server) = find_llama_server_binary(&runtime.manifest) + && let Some(server) = crate::direct_llama::find_llama_server_binary(&runtime.manifest) { - ensure_direct_llama_model_available(&request, &runtime, &process_env, log_path)?; - let backend = llama_server_backend_label(&server); - return serve_direct_llama_server( + crate::direct_llama::ensure_direct_llama_model_available( + &request, + &runtime, + &process_env, + log_path, + )?; + let backend = crate::direct_llama::llama_server_backend_label(&server); + return crate::process::serve_direct_llama_server( &request, &runtime, &process_env, @@ -619,21 +540,21 @@ fn serve_http(mut request: ServeHttpRequest) -> Result<()> { &anyhow!("using Lemonade packaged {backend} llama-server directly on Linux"), ); } - let mut child = spawn_lemond( + let mut child = crate::process::spawn_lemond( &runtime.manifest, &request.host, request.port, log_path, &process_env, )?; - write_running_state( + crate::state::write_running_state( &request, &runtime, std::process::id(), Some(child.id()), "running", )?; - wait_for_lemonade_cli_status( + crate::process::wait_for_lemonade_cli_status( &runtime.manifest, &request.host, request.port, @@ -642,7 +563,7 @@ fn serve_http(mut request: ServeHttpRequest) -> Result<()> { &process_env, ) .context("Lemonade server did not become ready")?; - let backend = ensure_best_llamacpp_backend( + let backend = crate::backend_alignment::ensure_best_llamacpp_backend( &mut runtime.manifest, &request.host, request.port, @@ -650,7 +571,7 @@ fn serve_http(mut request: ServeHttpRequest) -> Result<()> { false, ) .context("failed to select a supported Lemonade llama.cpp backend")?; - let load_result = run_lemonade_model_load( + let load_result = crate::process::run_lemonade_model_load( &runtime.manifest, &request.host, request.port, @@ -661,21 +582,26 @@ fn serve_http(mut request: ServeHttpRequest) -> Result<()> { &process_env, ); let router_ready = load_result.is_ok() - && query_loaded_model_endpoint( - &endpoint_url(&request.host, request.port), + && crate::process::query_loaded_model_endpoint( + &crate::state::endpoint_url(&request.host, request.port), &request.model_ref, &backend, ) .unwrap_or(false) - && query_chat_smoke_endpoint(&request.host, request.port, &request.model_ref) - .unwrap_or(false); + && crate::process::query_chat_smoke_endpoint( + &request.host, + request.port, + &request.model_ref, + ) + .unwrap_or(false); if let Err(error) = load_result { if runtime_is_linux() - && let Some(direct_server) = find_llama_server_binary(&runtime.manifest) + && let Some(direct_server) = + crate::direct_llama::find_llama_server_binary(&runtime.manifest) { - let _ = terminate_pid(child.id(), true); + let _ = crate::state::terminate_pid(child.id(), true); let _ = child.wait(); - return serve_direct_llama_server( + return crate::process::serve_direct_llama_server( &request, &runtime, &process_env, @@ -696,11 +622,12 @@ fn serve_http(mut request: ServeHttpRequest) -> Result<()> { "Lemonade load completed but the endpoint did not report a {LLAMACPP_RECIPE}:{backend}-loaded model" ); if runtime_is_linux() - && let Some(direct_server) = find_llama_server_binary(&runtime.manifest) + && let Some(direct_server) = + crate::direct_llama::find_llama_server_binary(&runtime.manifest) { - let _ = terminate_pid(child.id(), true); + let _ = crate::state::terminate_pid(child.id(), true); let _ = child.wait(); - return serve_direct_llama_server( + return crate::process::serve_direct_llama_server( &request, &runtime, &process_env, @@ -716,7 +643,7 @@ fn serve_http(mut request: ServeHttpRequest) -> Result<()> { ) }); } - merge_json_state( + crate::state::merge_json_state( &request.state_path, &json!({ "status": "ready", @@ -734,7 +661,7 @@ fn serve_http(mut request: ServeHttpRequest) -> Result<()> { }), )?; let status = child.wait().context("failed waiting for Lemonade server")?; - mark_json_status( + crate::state::mark_json_status( &request.state_path, if status.success() { "stopped" @@ -749,198 +676,35 @@ fn serve_http(mut request: ServeHttpRequest) -> Result<()> { } } -/// Serve a canonical Hugging Face checkpoint under its exact name. Requires an explicit -/// `:variant`; downloads the GGUF, then runs whichever packaged GPU llama-server backend -/// is installed (e.g. `vulkan` on WSL2, `rocm-stable` on native ROCm) directly on the -/// file with `--alias`. This bypasses Lemonade's model registry, whose naming rules -/// cannot preserve an `owner/repo:variant` name. -fn serve_hf_checkpoint( - request: &ServeHttpRequest, - runtime: &LemonadeRuntime, - process_env: &LemonadeProcessEnvironment, - log_path: Option<&Path>, - checkpoint: &HfCheckpoint, -) -> Result<()> { - // A variant is required: a bare `owner/repo` would need Lemonade's interactive - // variant menu (which cannot be answered from this non-interactive path) and, if a - // GGUF happened to be cached, risks silently serving the wrong quantization. - if checkpoint.variant.is_none() { - bail!( - "`{model}` needs an explicit quantization variant to serve; use `owner/repo:variant`, e.g. `{model}:Q4_K_M` (see the repo's GGUF files on Hugging Face for the available variants)", - model = request.model_ref - ); - } - let Some(server) = find_llama_server_binary(&runtime.manifest) else { - bail!( - "no GPU llama-server backend is installed under {}; run `rocm engines install lemonade`", - runtime - .manifest - .runtime_dir - .join("bin") - .join("llamacpp") - .display() - ); - }; - ensure_hf_checkpoint_downloaded(request, runtime, process_env, log_path)?; - // Never silently pick between quantizations: if the variant matches more than one - // cached GGUF, ask the user to disambiguate instead of choosing one arbitrarily. - let paths = AppPaths::discover()?; - let matches = hf_checkpoint_gguf_matches(&paths, checkpoint); - if matches.len() > 1 { - let names = matches - .iter() - .filter_map(|path| path.file_name().and_then(|name| name.to_str())) - .collect::>() - .join(", "); - bail!( - "variant `{}` matches multiple files ({names}); specify an exact quantization or filename, e.g. `{}:{}`", - checkpoint.variant.as_deref().unwrap_or_default(), - request.model_ref, - matches - .first() - .and_then(|path| path.file_name()) - .and_then(|name| name.to_str()) - .unwrap_or(".gguf") - ); - } - serve_direct_llama_server( - request, - runtime, - process_env, - &server, - log_path, - &anyhow!( - "serving canonical Hugging Face checkpoint `{}` directly", - request.model_ref - ), - ) -} - -/// Ensure a canonical Hugging Face checkpoint's GGUF is in the HF hub cache, pulling -/// it through a short-lived `lemond` if needed. The bare `pull owner/repo:variant` -/// only downloads (and registers a derived name we ignore); the file is what we serve. -fn ensure_hf_checkpoint_downloaded( - request: &ServeHttpRequest, - runtime: &LemonadeRuntime, - process_env: &LemonadeProcessEnvironment, - log_path: Option<&Path>, -) -> Result<()> { - let paths = AppPaths::discover()?; - if direct_llama_model_path(&paths, &request.model_ref).is_some() { - return Ok(()); - } - let download_port = free_local_port()?; - let mut child = spawn_lemond( - &runtime.manifest, - DEFAULT_HOST, - download_port, - log_path, - process_env, - )?; - let result = (|| -> Result<()> { - wait_for_lemonade_cli_status( - &runtime.manifest, - DEFAULT_HOST, - download_port, - Duration::from_secs(30), - log_path, - process_env, - )?; - run_lemonade_pull( - &runtime.manifest, - DEFAULT_HOST, - download_port, - &request.model_ref, - log_path, - process_env, - )?; - if direct_llama_model_path(&paths, &request.model_ref).is_some() { - Ok(()) - } else { - bail!("Lemonade did not download `{}`", request.model_ref) - } - })(); - let _ = terminate_pid(child.id(), true); - let _ = child.wait(); - result -} - -fn ensure_direct_llama_model_available( - request: &ServeHttpRequest, - runtime: &LemonadeRuntime, - process_env: &LemonadeProcessEnvironment, - log_path: Option<&Path>, -) -> Result<()> { - let paths = AppPaths::discover()?; - if direct_llama_model_path(&paths, &request.model_ref).is_some() { - return Ok(()); - } - - let download_port = free_local_port()?; - let mut child = spawn_lemond( - &runtime.manifest, - DEFAULT_HOST, - download_port, - log_path, - process_env, - )?; - let result = (|| -> Result<()> { - wait_for_lemonade_cli_status( - &runtime.manifest, - DEFAULT_HOST, - download_port, - Duration::from_secs(30), - log_path, - process_env, - )?; - let _ = run_lemonade_model_load( - &runtime.manifest, - DEFAULT_HOST, - download_port, - &request.model_ref, - ROCM_BACKEND_NAME, - None, - log_path, - process_env, - ); - if direct_llama_model_path(&paths, &request.model_ref).is_some() { - Ok(()) - } else { - bail!( - "Lemonade did not download `{}` for direct ROCm serving", - request.model_ref - ) - } - })(); - let _ = terminate_pid(child.id(), true); - let _ = child.wait(); - result -} - fn healthcheck_service(request: HealthcheckRequest) -> Result { - require_nonempty(&request.service_id, "service_id")?; - let files = service_files(&request.service_id)?; - let state = read_service_state(&files.state_path).ok(); - let endpoint_url = state.as_ref().and_then(endpoint_url_from_state); + rocm_core::require_nonempty(&request.service_id, "service_id")?; + let files = crate::state::service_files(&request.service_id)?; + let state = crate::state::read_service_state(&files.state_path).ok(); + let endpoint_url = state + .as_ref() + .and_then(crate::state::endpoint_url_from_state); let state_status = state .as_ref() - .and_then(|value| value_string(value, "status")) + .and_then(|value| crate::state::value_string(value, "status")) .unwrap_or_else(|| "unknown".to_owned()); let model_ref = state .as_ref() .and_then(|value| { - value_string(value, "canonical_model_id").or_else(|| value_string(value, "model_ref")) + crate::state::value_string(value, "canonical_model_id") + .or_else(|| crate::state::value_string(value, "model_ref")) }) .unwrap_or_default(); let backend = state .as_ref() - .and_then(|value| value_string(value, "backend_requested")) + .and_then(|value| crate::state::value_string(value, "backend_requested")) .unwrap_or_else(|| ROCM_BACKEND_NAME.to_owned()); let listed = state_status == "ready" && !model_ref.is_empty() && endpoint_url .as_deref() - .map(|endpoint| query_loaded_model_endpoint(endpoint, &model_ref, &backend)) + .map(|endpoint| { + crate::process::query_loaded_model_endpoint(endpoint, &model_ref, &backend) + }) .transpose() .unwrap_or(None) .unwrap_or(false); @@ -949,10 +713,15 @@ fn healthcheck_service(request: HealthcheckRequest) -> Result Result, - endpoint_url: &str, - model_ref: &str, -) -> bool { - let Some((host, port)) = parse_http_endpoint(endpoint_url) else { - return false; - }; - rocm_core::engine_state_inference_verified( - state_path, - state, - &format_http_base_url(&host, port), - model_ref, - rocm_engine_protocol::resolve_endpoint_api_key().as_deref(), - ) -} - -/// The device string reported once a model is loaded. Reflects the backend that -/// actually ran (`rocm`, `vulkan`, …) and the pinned GPU ordinal when known, -/// rather than a hardcoded value, so the report matches observed execution. -fn reported_device(state: Option<&Value>, backend: &str) -> String { - match state.and_then(first_gpu_index_from_state) { - Some(index) => format!("{backend} gpu {index}"), - None => format!("{backend} gpu"), - } -} - -/// The first pinned GPU ordinal recorded in service state, if any. -fn first_gpu_index_from_state(state: &Value) -> Option { - state - .get("gpu_indices") - .and_then(Value::as_array) - .and_then(|indices| indices.first()) - .and_then(Value::as_u64) - .and_then(|value| u32::try_from(value).ok()) -} - fn endpoint_response(request: EndpointRequest) -> Result { - require_nonempty(&request.service_id, "service_id")?; - let files = service_files(&request.service_id)?; - let state = read_service_state(&files.state_path) + rocm_core::require_nonempty(&request.service_id, "service_id")?; + let files = crate::state::service_files(&request.service_id)?; + let state = crate::state::read_service_state(&files.state_path) .with_context(|| format!("service state not found for `{}`", request.service_id))?; - let endpoint_url = endpoint_url_from_state(&state) + let endpoint_url = crate::state::endpoint_url_from_state(&state) .with_context(|| format!("service `{}` has no endpoint URL", request.service_id))?; Ok(EndpointResponse { endpoint_url, @@ -1027,13 +753,13 @@ fn endpoint_response(request: EndpointRequest) -> Result { } fn logs_response(request: LogsRequest) -> Result { - require_nonempty(&request.service_id, "service_id")?; - let files = service_files(&request.service_id)?; + rocm_core::require_nonempty(&request.service_id, "service_id")?; + let files = crate::state::service_files(&request.service_id)?; let limit = request.tail_lines.unwrap_or(DEFAULT_LOG_TAIL_LINES); Ok(LogsResponse { log_path: files.log_path.display().to_string(), recent_lines: if files.log_path.is_file() { - tail_lines(&files.log_path, limit)? + crate::state::tail_lines(&files.log_path, limit)? } else { Vec::new() }, @@ -1041,14 +767,14 @@ fn logs_response(request: LogsRequest) -> Result { } fn stop_service(request: StopRequest) -> Result { - require_nonempty(&request.service_id, "service_id")?; - let files = service_files(&request.service_id)?; - let state = read_service_state(&files.state_path).ok(); + rocm_core::require_nonempty(&request.service_id, "service_id")?; + let files = crate::state::service_files(&request.service_id)?; + let state = crate::state::read_service_state(&files.state_path).ok(); // Verify the recorded PID still belongs to our server before signalling it, // then wait for it to actually exit so the reported result is truthful. let outcome = state .as_ref() - .and_then(identity_from_state) + .and_then(crate::state::identity_from_state) .map(|identity| { rocm_core::terminate_verified(&identity, STOP_SCOPE, STOP_GRACE, request.force) }); @@ -1057,6830 +783,42 @@ fn stop_service(request: StopRequest) -> Result { None => (false, false), }; if stopped { - mark_json_status(&files.state_path, "stopped")?; + crate::state::mark_json_status(&files.state_path, "stopped")?; } Ok(StopResponse { stopped, graceful }) } -/// Whether the embeddable archive must be extracted over the runtime tree. -/// -/// Pure so the upgrade case is testable without a real archive. `installed` -/// is the version recorded for the tree already on disk, if any. -fn needs_extraction( - reinstall: bool, - installed: Option<&str>, - wanted: &str, - server_present: bool, -) -> bool { - reinstall || !server_present || installed != Some(wanted) -} - -/// The Lemonade runtime version last extracted into `runtime_dir`, if any. -fn installed_runtime_version(runtime_dir: &Path) -> Option { - fs::read_to_string(runtime_dir.join(RUNTIME_VERSION_MARKER)) - .ok() - .map(|s| s.trim().to_owned()) -} - -/// Record `version` as installed into `runtime_dir`. Marker lives inside -/// `runtime_dir` (not beside it) so `remove_dir_all(runtime_dir)` clears it as -/// part of the same wipe that removes the tree it describes — "marker present -/// implies extraction completed" holds by construction, with no separate -/// bookkeeping to keep in sync. -fn record_runtime_version(runtime_dir: &Path, version: &str) -> Result<()> { - fs::write(runtime_dir.join(RUNTIME_VERSION_MARKER), version).with_context(|| { - format!( - "failed to record installed Lemonade runtime version under {}", - runtime_dir.display() - ) - }) -} - -fn prepare_embeddable( - paths: &AppPaths, - env_root: Option<&Path>, - reinstall: bool, -) -> Result { - prepare_embeddable_with( - paths, - env_root, - reinstall, - rocm_deps::LEMONADE_VERSION, - embeddable_archive_sha256(), - download_file, - ) -} - -/// Same as [`prepare_embeddable`], but with the archive version, its expected -/// sha256, and the download step all injectable — the seam a test uses to -/// exercise the real decision of which source `installed_version` trusts, -/// rather than reimplementing that decision inline. -fn prepare_embeddable_with( - paths: &AppPaths, - env_root: Option<&Path>, - reinstall: bool, - version: &str, - expected_sha256: &str, - download: F, -) -> Result -where - F: FnMut(&str, &Path) -> Result<()>, -{ - let root = lemonade_root(paths, env_root); - let version = version.to_owned(); - let archive_name = embeddable_archive_name(&version); - let archive_url = embeddable_url(&version); - let downloads = root.join("downloads"); - let archive = downloads.join(&archive_name); - fs::create_dir_all(&downloads)?; - // Held until the archive bytes have been extracted and copied out: the - // verified bytes and the extracted bytes must be the same read. - let archive_guard = ensure_cached_archive(&archive_url, &archive, expected_sha256, download)?; - let runtime_dir = runtime_dir_in(&root); - // The runtime directory is not version-scoped, so a bump downloads a new - // archive into a tree that already holds `lemond` from the previous - // version. Without comparing versions the extraction would be skipped and - // the old binaries reported as the new version. - // - // The version is read from a marker inside `runtime_dir` (the same shared - // tree it describes), not from the CLI's own manifest: the manifest lives - // under `paths` (the per-invocation data dir), which is not guaranteed to - // be the same data dir that last extracted into this `runtime_dir` (e.g. - // each e2e scenario gets an isolated data dir but shares one runtime - // tree). Sourcing the check from the isolated manifest would report - // "unknown version" on every call, wipe the shared tree, and re-extract - // over it — destroying any llama.cpp backend already installed there. - let installed_version = installed_runtime_version(&runtime_dir); - if needs_extraction( - reinstall, - installed_version.as_deref(), - &version, - lemond_path_in(&runtime_dir).is_file(), - ) { - if runtime_dir.exists() { - fs::remove_dir_all(&runtime_dir) - .with_context(|| format!("failed to clear {}", runtime_dir.display()))?; - } - let extract_root = root - .join("extract") - .join(format!("{}", current_unix_millis())); - fs::create_dir_all(&extract_root)?; - extract_archive(&archive, &extract_root)?; - let embeddable_root = find_embeddable_root(&extract_root)?; - copy_tree(&embeddable_root, &runtime_dir)?; - fs::remove_dir_all(&extract_root).ok(); - record_runtime_version(&runtime_dir, &version)?; - } - drop(archive_guard); - let lemond = lemond_path_in(&runtime_dir); - let lemonade = lemonade_path_in(&runtime_dir); - if !lemond.is_file() || !lemonade.is_file() { - bail!( - "Lemonade embeddable extraction did not produce lemonade/lemond binaries in {}", - runtime_dir.display() - ); - } - Ok(LemonadeInstallManifest { - env_id: format!("lemonade-embeddable-{version}"), - version, - runtime_dir, - lemond, - lemonade, - backend_recipe: LLAMACPP_RECIPE.to_owned(), - // Resolved during install once Lemonade reports per-GPU backend support. - backend_name: ROCM_BACKEND_NAME.to_owned(), - installed_at_unix_ms: current_unix_millis(), - }) -} - -/// The variable that opts a machine out of rocm-cli aligning Lemonade's -/// `llamacpp:rocm` backend to the active ROCm SDK. -/// -/// `resources/backend_versions.json` is a documented, first-party -/// customization point — its own header comment invites users to hand-edit it -/// to pin specific versions without rebuilding — so alignment running -/// unconditionally on every plain install would silently overwrite a user's -/// manual pin. Mirrors vLLM's `ROCM_CLI_DISABLE_TORCH_ALIGNMENT` -/// (`rocm_core::TORCH_ALIGNMENT_DISABLED_ENV`): presence is the signal, so any -/// value — including the empty string — disables the alignment. -pub const LEMONADE_BACKEND_ALIGNMENT_DISABLED_ENV: &str = - "ROCM_CLI_DISABLE_LEMONADE_BACKEND_ALIGNMENT"; - -/// Whether the user has opted out of rocm-cli aligning Lemonade's backend. -fn lemonade_backend_alignment_disabled() -> bool { - std::env::var_os(LEMONADE_BACKEND_ALIGNMENT_DISABLED_ENV).is_some() -} - -/// Point Lemonade's `llamacpp:rocm` backend at the ROCm version rocm-cli already has -/// installed and active, instead of Lemonade's hardcoded pin (which does not track -/// whatever the user separately installed via `rocm install sdk`). -/// -/// Best-effort and defensive at every step: if there is no managed rocm-cli SDK, the -/// pinned version already matches, or an aligned attempt cannot be verified to actually -/// resolve its GPU library, this falls back to (or reverts to) today's unmodified -/// pinned-version install — never a regression, and never a silent CPU fallback -/// (AGENTS.md §6): an aligned backend is only kept once [`rocm_backend_resolves`] -/// confirms it. -/// -/// Returns the aligned version string when one was applied and verified, so the caller -/// can surface it in the install response. -fn prepare_llamacpp_backend_for_active_rocm( - paths: &AppPaths, - manifest: &mut LemonadeInstallManifest, -) -> Result> { - prepare_llamacpp_backend_for_active_rocm_impl( - manifest, - lemonade_backend_alignment_disabled(), - || active_rocm_sdk_version_for_alignment(paths), - read_backend_versions_therock_version, - best_llamacpp_backend_for_host, - install_best_llamacpp_backend, - |manifest, backend_versions_path, target_version, pinned_version, disabled| { - align_llamacpp_backend_to_version( - manifest, - backend_versions_path, - target_version, - pinned_version, - disabled, - try_llamacpp_backend_alignment, - install_best_llamacpp_backend, - latest_llamacpp_rocm_stable_tag, - ) - }, - ) -} - -/// The gate and lookup chain behind [`prepare_llamacpp_backend_for_active_rocm`], -/// isolated from its real dependencies (the active-SDK/config lookup, the packaged -/// pin file, and the `lemond`-spawning backend probe) so this crate's own unit tests -/// can drive the disabled early return and the probe-error fallback -- the two -/// branches that decide whether any alignment is even attempted, and previously had -/// no coverage outside a `@nightly @requires-gpu` e2e lane. `disabled` is -/// [`lemonade_backend_alignment_disabled`]'s resolved value, passed in for the same -/// reason `align` (below) takes it as a parameter: a test can drive both branches -/// without touching process environment. -/// -/// This function, not `align`, is the real production enforcement point for the -/// opt-out: the single production call site passes `disabled` straight through to -/// `align`, but every one of *this* function's own skip branches returns before -/// `align` is ever reached, so `align`'s production call always sees `disabled == -/// false` here as well as there. `align`'s own `if disabled` branch exists only so -/// its unit tests can pin the state machine's half of the contract (the pin and -/// llama.cpp tag staying untouched) without duplicating this whole call chain. -#[allow(clippy::too_many_arguments)] -fn prepare_llamacpp_backend_for_active_rocm_impl( - manifest: &mut LemonadeInstallManifest, - disabled: bool, - mut active_rocm_sdk_version: impl FnMut() -> Result>, - mut pinned_version_lookup: impl FnMut(&Path) -> Option, - mut selected_backend_lookup: impl FnMut(&LemonadeInstallManifest) -> Result>, - mut fallback_install: impl FnMut(&mut LemonadeInstallManifest, bool) -> Result<()>, - mut align: impl FnMut( - &mut LemonadeInstallManifest, - &Path, - &str, - &str, - bool, - ) -> Result>, -) -> Result> { - // Announced here, unconditionally, rather than only where `align` happens to be - // reached: every one of the skip branches below -- not Linux, no active SDK - // version, unreadable pin, pin already matches, the vulkan-fallback probe -- - // would otherwise leave a user who set the variable unable to tell it took - // effect, exactly the silence this message exists to avoid. - if disabled { - eprintln!( - "Lemonade backend alignment is disabled by {LEMONADE_BACKEND_ALIGNMENT_DISABLED_ENV}; \ - using whatever backend_versions.json already pins." - ); - // Checked earliest, before even the cheap version/pin lookups below, let alone - // the vulkan-gate probe (a real `lemond` spawn) -- none of that work is worth - // doing when the opt-out means its result can't change what happens next. - fallback_install(manifest, false)?; - return Ok(None); - } - // Alignment is only ever verifiable on Linux ([`rocm_backend_resolves`] always - // reports unresolved elsewhere), so attempting it on Windows can only burn up to - // three multi-GB backend installs and a network round-trip for a guaranteed-futile - // outcome. Skip straight to the ordinary pinned-version install. - if !runtime_is_linux() { - fallback_install(manifest, false)?; - return Ok(None); - } - - let target_version = active_rocm_sdk_version().unwrap_or_else(|error| { - eprintln!( - "Warning: could not determine the active ROCm SDK version to align Lemonade's \ - backend with: {error:#}" - ); - None - }); - let Some(target_version) = target_version else { - fallback_install(manifest, false)?; - return Ok(None); - }; - - let backend_versions_path = manifest.runtime_dir.join(BACKEND_VERSIONS_RESOURCE); - let Some(pinned_version) = pinned_version_lookup(&backend_versions_path) else { - fallback_install(manifest, false)?; - return Ok(None); - }; - if pinned_version == target_version { - fallback_install(manifest, false)?; - return Ok(None); - } - - // Verification below requires the backend Lemonade selects to be `rocm` - // specifically ([`try_llamacpp_backend_alignment`]'s `has_usable_binary` gate) -- - // it falls back to `vulkan` on hosts where Lemonade's ROCm build is unsupported - // (WSL2 being the documented case), and no tier can ever pass there. Check which - // backend this host actually gets *before* patching `backend_versions.json`, the - // same guaranteed-futile case already short-circuited for Windows above: otherwise - // every install on such a host burns two forced reinstalls plus a GitHub round-trip - // discovering what this cheap query already knows. - let selected_backend = match selected_backend_lookup(manifest) { - Ok(selected_backend) => selected_backend, - Err(error) => { - eprintln!( - "Warning: could not determine which llama.cpp backend this host would select; \ - skipping ROCm backend alignment: {error:#}" - ); - fallback_install(manifest, false)?; - return Ok(None); - } - }; - if !should_attempt_llamacpp_backend_alignment(selected_backend.as_deref()) { - fallback_install(manifest, false)?; - return Ok(None); - } - - align( - manifest, - &backend_versions_path, - &target_version, - &pinned_version, - disabled, - ) -} - -/// The Tier 1 / Tier 2 / revert state machine, isolated from the config and active-SDK -/// lookups above so it can be exercised in tests against a temp `backend_versions.json` -/// with the install/align/latest-tag steps injected, instead of spawning a real -/// `lemond` and reaching GitHub. -/// -/// `disabled` is the resolved value of [`lemonade_backend_alignment_disabled`], passed -/// in rather than read here so a test can drive both branches without touching process -/// environment (mirrors vLLM's `torch_alignment_disabled` parameter). The real -/// production enforcement of the opt-out lives one level up, in -/// [`prepare_llamacpp_backend_for_active_rocm_impl`]: its own early return skips -/// every lookup this function depends on, so its single production call site always -/// passes `disabled == false` here. This function's own `if disabled` branch exists -/// only so its unit tests can pin this half of the contract (the pin and llama.cpp -/// tag staying untouched when the flag is set) without spinning up the whole call -/// chain above it. The user-facing announcement lives in -/// [`prepare_llamacpp_backend_for_active_rocm_impl`] as well, so it fires on every -/// skip path the opt-out affects, not only the one this function's own gate reaches. -#[allow(clippy::too_many_arguments)] -fn align_llamacpp_backend_to_version( - manifest: &mut LemonadeInstallManifest, - backend_versions_path: &Path, - target_version: &str, - pinned_version: &str, - disabled: bool, - mut align: impl FnMut(&mut LemonadeInstallManifest, &str, bool, &str) -> bool, - mut fallback_install: impl FnMut(&mut LemonadeInstallManifest, bool) -> Result<()>, - mut latest_tag: impl FnMut() -> Result, -) -> Result> { - if disabled { - fallback_install(manifest, false)?; - return Ok(None); - } - if let Err(error) = - write_backend_versions_therock_version(backend_versions_path, target_version) - { - eprintln!("Warning: could not pin Lemonade's backend to ROCm {target_version}: {error:#}"); - fallback_install(manifest, false)?; - return Ok(None); - } - - // Tier 1: keep Lemonade's own pinned llama.cpp build, just point it at the active - // ROCm version. Works when that specific build's release actually shipped a - // matching ROCm-version asset. Always forces a reinstall: this whole function - // only runs when the pin actually changed (the `pinned_version == target_version` - // check above), so a backend already on disk was installed against the OLD - // pin and must not be trusted just because it happens to still resolve. - if align(manifest, target_version, true, "its pinned llama.cpp build") { - return Ok(Some(target_version.to_owned())); - } - - // Tier 2: the pinned build may simply be too old to ever have shipped a - // matching ROCm-version asset — Lemonade only started publishing per-ROCm- - // version builds partway through its release history, and the embeddable's - // pin (frozen at whatever build was current when this Lemonade version was - // cut) does not track that. Point at the current newest build instead, still - // paired with the active ROCm version, forcing a real reinstall attempt - // regardless of whatever Tier 1 left on disk. - let pinned_tag = read_backend_versions_llamacpp_tag(backend_versions_path); - let latest_tag_applied = match latest_tag() { - Ok(latest_tag) if pinned_tag.as_deref() != Some(latest_tag.as_str()) => { - match write_backend_versions_llamacpp_tag(backend_versions_path, &latest_tag) { - Ok(()) => true, - Err(error) => { - eprintln!( - "Warning: could not pin Lemonade's backend to llama.cpp {latest_tag}: \ - {error:#}" - ); - false - } - } - } - // The pinned tag already is the latest — Tier 1 already tried it. - Ok(_) => false, - Err(error) => { - eprintln!("Warning: could not determine Lemonade's latest llama.cpp build: {error:#}"); - false - } - }; - if latest_tag_applied && align(manifest, target_version, true, "the latest llama.cpp build") { - return Ok(Some(target_version.to_owned())); - } - - // Neither tier produced a verified GPU backend. Revert everything — including - // the llama.cpp build tag, so the fallback install below isn't itself - // misdirected — and retry once more with Lemonade's original pinned defaults. - // Every step below is best-effort, matching its sibling warnings: a failure to - // restore the pin still lets the fallback install run rather than hard-failing - // the whole install, though it does leave the pin holding the unverified - // target version until a later run corrects it. - eprintln!( - "Warning: could not align Lemonade's ROCm backend to {target_version}; reverting to the \ - default pinned version." - ); - if let Err(error) = - write_backend_versions_therock_version(backend_versions_path, pinned_version) - { - eprintln!( - "Warning: could not restore Lemonade's default pinned backend version \ - ({pinned_version}): {error:#}" - ); - } - if latest_tag_applied { - // Best-effort, matching the therock.version restore above: a failure here - // must not skip the fallback reinstall below. Restore the original tag when - // there was one; otherwise Tier 2 pinned a key that did not exist before, so - // removing it (not writing some placeholder) is what "reverted" means. - let restore_result = match pinned_tag.as_deref() { - Some(tag) => write_backend_versions_llamacpp_tag(backend_versions_path, tag), - None => remove_backend_versions_llamacpp_tag(backend_versions_path), - }; - if let Err(error) = restore_result { - eprintln!( - "Warning: could not restore Lemonade's default pinned llama.cpp build tag: \ - {error:#}" - ); - } - } - let aside_dirs = set_aside_rocm_llamacpp_backend_dirs(manifest); - match fallback_install(manifest, true) { - Ok(()) => { - resolve_rocm_llamacpp_backend_dirs_aside(manifest, &aside_dirs, true); - Ok(None) - } - Err(error) => { - // Restore the pre-alignment backend before propagating: a failed - // fallback install must not leave the host with no ROCm backend at all. - resolve_rocm_llamacpp_backend_dirs_aside(manifest, &aside_dirs, false); - Err(error) - } - } -} - -/// Attempt one llama.cpp backend install aligned to `target_version`'s ROCm pairing -/// (already written into `resources/backend_versions.json`), then verify — never -/// simply trust Lemonade's own reported success — that the resulting GPU backend -/// actually resolves against rocm-cli's active ROCm SDK (AGENTS.md §6: no silent CPU -/// fallback). `force_reinstall` must be `true` for every caller here: alignment is -/// only ever attempted when the pin actually changed, and a skipped "already -/// installed" reinstall would leave a stale, pre-alignment binary that can still -/// pass the `ldd`-resolves check below, reporting success for a version that was -/// never actually installed. Returns whether a verified backend is now in place; on -/// success, `manifest.backend_name` is updated to match it. -fn try_llamacpp_backend_alignment( - manifest: &mut LemonadeInstallManifest, - target_version: &str, - force_reinstall: bool, - attempt_label: &str, -) -> bool { - debug_assert!( - force_reinstall, - "every alignment attempt must force a reinstall; see the doc comment above" - ); - let aside_dirs = set_aside_rocm_llamacpp_backend_dirs(manifest); - let install_result = install_best_llamacpp_backend(manifest, force_reinstall); - // `ensure_best_llamacpp_backend` records the backend it attempted into - // `manifest.backend_name` before the fallible install step runs, so this is accurate - // even when `install_result` is `Err` below. Scoping to it — rather than scanning - // every backend directory — means a stale `rocm-*` directory left by an earlier - // attempt can never be mistaken for this round's verification, including when this - // round actually picked `vulkan`. - let backend_name = manifest.backend_name.clone(); - let has_usable_binary = backend_name == ROCM_BACKEND_NAME - && lemonade_process_environment().is_ok_and(|process_env| { - find_llama_server_binary_for_backend(manifest, &backend_name) - .is_some_and(|binary| rocm_backend_resolves(&binary, &process_env)) - }); - let succeeded = llamacpp_backend_alignment_succeeded(&install_result, has_usable_binary); - resolve_rocm_llamacpp_backend_dirs_aside(manifest, &aside_dirs, succeeded); - if !succeeded { - match install_result { - Ok(()) => eprintln!( - "Warning: {attempt_label} installed for ROCm {target_version}, but its GPU \ - backend does not resolve against rocm-cli's active ROCm SDK; not using it." - ), - Err(error) if has_usable_binary => eprintln!( - "Warning: could not install {attempt_label} for ROCm {target_version} \ - ({error:#}); a backend binary is present but may predate this alignment \ - attempt, so it is not trusted without a completed install." - ), - Err(error) => eprintln!( - "Warning: could not install {attempt_label} for ROCm {target_version}: {error:#}" - ), - } - return false; - } - eprintln!("Aligned Lemonade's ROCm backend to {target_version} using {attempt_label}."); - true -} - -/// Whether an alignment attempt counts as a verified success: the install must -/// have actually completed, not merely have left behind some binary that -/// happens to resolve. `ldd`-based verification cannot tell a stale build -/// (paired with a different ROCm version, possibly left over from before this -/// attempt) from a version-matched one, so a failed install must never be -/// forgiven by a leftover binary passing that check. -const fn llamacpp_backend_alignment_succeeded( - install_result: &Result<()>, - has_usable_binary: bool, -) -> bool { - install_result.is_ok() && has_usable_binary -} - -/// GitHub repo backing Lemonade's `llamacpp:rocm-stable` builds. Bumping -/// `resources/backend_versions.json`'s `therock.version` alone doesn't get a -/// ROCm-version-matched build if the pinned tag predates that ROCm version's -/// support: Lemonade only started publishing per-ROCm-version assets partway -/// through this repo's release history, and the embeddable's own pin can be -/// (and, at the time of writing, is) older than that. -const LLAMACPP_ROCM_STABLE_REPO: &str = "lemonade-sdk/llama.cpp"; - -/// The newest published release tag for [`LLAMACPP_ROCM_STABLE_REPO`] (e.g. -/// `b10952`), queried directly rather than assumed — the whole reason Tier 2 -/// exists is that the embeddable's own pinned tag cannot be trusted to have a -/// matching-ROCm-version asset. -fn latest_llamacpp_rocm_stable_tag() -> Result { - let url = format!("https://api.github.com/repos/{LLAMACPP_ROCM_STABLE_REPO}/releases/latest"); - let timeout = Duration::from_secs(15); - // `timeout_connect` takes precedence over `timeout` and defaults to 30s, so without - // it a host that blackholes rather than refuses would stall well past the intended - // ceiling. - let agent = ureq::AgentBuilder::new() - .timeout_connect(timeout) - .timeout(timeout) - .build(); - let response = agent - .get(&url) - .set("User-Agent", "rocm-cli") - .call() - .with_context(|| format!("failed to query {url}"))?; - let body: Value = response - .into_json() - .with_context(|| format!("failed to parse response from {url}"))?; - body.get("tag_name") - .and_then(Value::as_str) - .map(str::to_owned) - .with_context(|| format!("{url} response is missing 'tag_name'")) -} - -/// Read `resources/backend_versions.json`'s pinned `llamacpp.rocm-stable` build tag. -fn read_backend_versions_llamacpp_tag(path: &Path) -> Option { - let bytes = fs::read(path).ok()?; - let value: Value = serde_json::from_slice(&bytes).ok()?; - value - .get("llamacpp")? - .get("rocm-stable")? - .as_str() - .map(str::to_owned) -} - -/// Overwrite `resources/backend_versions.json`'s `llamacpp.rocm-stable` build tag in -/// place, preserving every other key. -fn write_backend_versions_llamacpp_tag(path: &Path, tag: &str) -> Result<()> { - let bytes = fs::read(path).with_context(|| format!("failed to read {}", path.display()))?; - let mut value: Value = serde_json::from_slice(&bytes) - .with_context(|| format!("failed to parse {}", path.display()))?; - value - .get_mut("llamacpp") - .and_then(Value::as_object_mut) - .with_context(|| format!("{} has no 'llamacpp' object to patch", path.display()))? - .insert("rocm-stable".to_owned(), Value::String(tag.to_owned())); - fs::write(path, serde_json::to_vec_pretty(&value)?) - .with_context(|| format!("failed to write {}", path.display())) -} - -/// Remove `resources/backend_versions.json`'s `llamacpp.rocm-stable` key entirely, -/// restoring the "never pinned" state — the counterpart to -/// [`write_backend_versions_llamacpp_tag`] for reverting Tier 2's pin when there was -/// no prior tag to restore it to. -fn remove_backend_versions_llamacpp_tag(path: &Path) -> Result<()> { - let bytes = fs::read(path).with_context(|| format!("failed to read {}", path.display()))?; - let mut value: Value = serde_json::from_slice(&bytes) - .with_context(|| format!("failed to parse {}", path.display()))?; - value - .get_mut("llamacpp") - .and_then(Value::as_object_mut) - .with_context(|| format!("{} has no 'llamacpp' object to patch", path.display()))? - .remove("rocm-stable"); - fs::write(path, serde_json::to_vec_pretty(&value)?) - .with_context(|| format!("failed to write {}", path.display())) -} - -/// The active rocm-cli-managed ROCm SDK version to align Lemonade's backend to, -/// restricted to plain `X.Y.Z` release versions. Nightly builds carry a date suffix -/// (e.g. `7.14.0a20260601`) that does not correspond to any llama.cpp build tag -/// Lemonade publishes, so those are left alone rather than guessed at. -/// -/// Errors (rather than defaulting) when the config can't be loaded: an empty default -/// config would discard the active runtime key and let version selection guess the -/// most-recently-installed runtime, then mutate Lemonade based on that guess. -fn active_rocm_sdk_version_for_alignment(paths: &AppPaths) -> Result> { - let config = RocmCliConfig::load(paths)?; - let Some(version) = active_managed_therock_version(paths, &config)? else { - return Ok(None); - }; - Ok(looks_like_plain_semver(&version).then_some(version)) -} - -/// Whether `version` is a plain `major.minor.patch` string with no build/date suffix -/// (e.g. `10.0.0`, not `7.14.0a20260601`). -fn looks_like_plain_semver(version: &str) -> bool { - let parts: Vec<&str> = version.split('.').collect(); - parts.len() == 3 - && parts - .iter() - .all(|part| !part.is_empty() && part.chars().all(|c| c.is_ascii_digit())) -} - -/// Read `resources/backend_versions.json`'s `therock.version` — Lemonade's pin for -/// which private ROCm runtime its `llamacpp:rocm` backend downloads. -fn read_backend_versions_therock_version(path: &Path) -> Option { - let bytes = fs::read(path).ok()?; - let value: Value = serde_json::from_slice(&bytes).ok()?; - value - .get("therock")? - .get("version")? - .as_str() - .map(str::to_owned) -} - -/// Overwrite `resources/backend_versions.json`'s `therock.version` in place, -/// preserving every other key. Lemonade's own comment in that file invites exactly -/// this: "You can modify these values to pin specific versions without rebuilding the -/// application." -fn write_backend_versions_therock_version(path: &Path, version: &str) -> Result<()> { - let bytes = fs::read(path).with_context(|| format!("failed to read {}", path.display()))?; - let mut value: Value = serde_json::from_slice(&bytes) - .with_context(|| format!("failed to parse {}", path.display()))?; - value - .get_mut("therock") - .and_then(Value::as_object_mut) - .with_context(|| format!("{} has no 'therock' object to patch", path.display()))? - .insert("version".to_owned(), Value::String(version.to_owned())); - fs::write(path, serde_json::to_vec_pretty(&value)?) - .with_context(|| format!("failed to write {}", path.display())) -} - -/// ROCm shared-library sonames the `llamacpp:rocm` backend's `libggml-hip.so` links -/// against. If any fail to resolve, the backend would silently fall back to CPU -/// inference at runtime instead of failing loudly — unacceptable under the -/// GPU-required policy (AGENTS.md §6), so this must be verified, not assumed from -/// version numbers matching. -/// -/// Linux-only, like the `ldd`-based verification it exists for: the non-Linux -/// `rocm_backend_resolves` stub below never references it, so leaving it -/// ungated makes it dead code — and a build failure under `-D warnings` — on -/// every other target. -#[cfg(target_os = "linux")] -const ROCM_BACKEND_REQUIRED_SONAMES: [&str; 4] = [ - "libhipblas.so", - "librocblas.so", - "libamdhip64.so", - "libhsa-runtime64.so", -]; - -/// Whether the ROCm GPU backend beside `llama_server_binary` actually resolves its -/// required ROCm libraries under `process_env` (rocm-cli's injected ROCm environment), -/// checked with `ldd` rather than assumed. `false` on any I/O/parse failure. -#[cfg(target_os = "linux")] -fn rocm_backend_resolves( - llama_server_binary: &Path, - process_env: &LemonadeProcessEnvironment, -) -> bool { - let Some(backend_dir) = llama_server_binary.parent() else { - return false; - }; - let hip_backend = backend_dir.join("libggml-hip.so"); - if !hip_backend.is_file() { - return false; - } - let mut command = ProcessCommand::new("ldd"); - command - .arg(&hip_backend) - .stdin(Stdio::null()) - .stdout(Stdio::piped()) - .stderr(Stdio::piped()); - if apply_lemonade_process_environment(&mut command, process_env).is_err() { - return false; - } - let Ok(output) = command.output() else { - return false; - }; - output.status.success() - && ldd_output_resolves_all( - &String::from_utf8_lossy(&output.stdout), - &ROCM_BACKEND_REQUIRED_SONAMES, - ) +pub(crate) fn current_unix_millis() -> u128 { + SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap_or_default() + .as_millis() } -/// Off Linux, `ldd`-based verification is not exercised: a version-aligned backend is -/// never accepted without it, so this always reports unresolved. -#[cfg(not(target_os = "linux"))] -const fn rocm_backend_resolves( - _llama_server_binary: &Path, - _process_env: &LemonadeProcessEnvironment, -) -> bool { - false +fn read_request() -> Result { + let mut buffer = String::new(); + std::io::stdin() + .read_to_string(&mut buffer) + .context("failed to read stdin for engine request")?; + serde_json::from_str(&buffer).context("failed to parse engine request envelope") } -/// Whether every `required` soname prefix appears in `ldd` output with a resolved -/// path (i.e. not `=> not found`). Linux-only: see [`ROCM_BACKEND_REQUIRED_SONAMES`]. -#[cfg(target_os = "linux")] -fn ldd_output_resolves_all(output: &str, required: &[&str]) -> bool { - required.iter().all(|soname| { - output.lines().any(|line| { - let line = line.trim(); - line.starts_with(soname) && !line.contains("not found") - }) - }) -} +fn print_json(value: &T) -> Result<()> { + use std::io::Write; -fn install_best_llamacpp_backend( - manifest: &mut LemonadeInstallManifest, - force_reinstall: bool, -) -> Result<()> { - let port = free_local_port()?; - let log_path_buf = manifest.runtime_dir.join("install-lemond.log"); - let log_path = Some(log_path_buf.as_path()); - let process_env = lemonade_process_environment()?; - let mut child = spawn_lemond(manifest, DEFAULT_HOST, port, log_path, &process_env)?; - let result = (|| -> Result { - wait_for_lemonade_cli_status( - manifest, - DEFAULT_HOST, - port, - Duration::from_secs(30), - log_path, - &process_env, - )?; - ensure_best_llamacpp_backend(manifest, DEFAULT_HOST, port, &process_env, force_reinstall) - })(); - let _ = terminate_pid(child.id(), true); - let _ = child.wait(); - manifest.backend_name = result?; + let stdout = std::io::stdout(); + let mut handle = stdout.lock(); + serde_json::to_writer_pretty(&mut handle, value)?; + writeln!(&mut handle)?; Ok(()) } -/// Whether the vulkan-gate probe's result means alignment could possibly succeed -- -/// only when the host would select Lemonade's `rocm` backend by itself. Any other -/// selection (including the probe itself failing to determine one) means every -/// alignment tier is guaranteed to fail ([`try_llamacpp_backend_alignment`]'s -/// `has_usable_binary` gate requires the `rocm` backend specifically), so callers -/// should skip straight to an ordinary install instead of patching -/// `backend_versions.json` and burning two forced reinstalls plus a GitHub round-trip -/// discovering what this decision already knows. A pure function so this gate is -/// unit-testable without spawning the real `lemond` process -/// [`best_llamacpp_backend_for_host`] needs. -fn should_attempt_llamacpp_backend_alignment(selected_backend: Option<&str>) -> bool { - selected_backend == Some(ROCM_BACKEND_NAME) -} - -/// Which llama.cpp backend Lemonade would select on this host -/// (`LLAMACPP_BACKEND_PRIORITY`), without installing anything -- mirrors -/// [`install_best_llamacpp_backend`]'s spawn/query steps but stops short of the -/// install, so the ROCm backend alignment dance can check whether it is even -/// reachable before patching `backend_versions.json` or forcing a single reinstall. -fn best_llamacpp_backend_for_host(manifest: &LemonadeInstallManifest) -> Result> { - let port = free_local_port()?; - let log_path_buf = manifest.runtime_dir.join("install-lemond.log"); - let log_path = Some(log_path_buf.as_path()); - let process_env = lemonade_process_environment()?; - let mut child = spawn_lemond(manifest, DEFAULT_HOST, port, log_path, &process_env)?; - let result = (|| -> Result> { - wait_for_lemonade_cli_status( - manifest, - DEFAULT_HOST, - port, - Duration::from_secs(30), - log_path, - &process_env, - )?; - let listing = run_lemonade_backends_list(manifest, &process_env)?; - let backends = parse_llamacpp_backend_statuses(&listing); - Ok(select_best_llamacpp_backend(&backends).map(|(name, _)| name)) - })(); - let _ = terminate_pid(child.id(), true); - let _ = child.wait(); - result -} - -/// Ask Lemonade which llama.cpp backends it supports on this GPU, choose the best -/// one (`LLAMACPP_BACKEND_PRIORITY`), install it if necessary, and return its name. -/// Retry the backend install itself once, rather than the whole Lemonade runtime -/// preparation. The embeddable download already has bounded transport retries; -/// repeating that outer operation would redo deterministic failures and may -/// re-extract a healthy runtime. A backend subprocess can instead fail after a -/// completed download when its connection to lemond is interrupted, and a second -/// call can reuse the backend cache immediately without a delay. -fn install_llamacpp_backend_with_retry(mut install: impl FnMut() -> Result<()>) -> Result<()> { - match install() { - Ok(()) => Ok(()), - Err(first_error) => { - eprintln!("Lemonade backend installation failed; retrying once: {first_error:#}"); - install().with_context(|| { - "Lemonade backend installation failed again; run `rocm engines install \ - lemonade --reinstall` and then retry `rocm serve`" - }) - } - } -} - -fn ensure_best_llamacpp_backend( - manifest: &mut LemonadeInstallManifest, - host: &str, - port: u16, - process_env: &LemonadeProcessEnvironment, - force_reinstall: bool, -) -> Result { - let listing = run_lemonade_backends_list(manifest, process_env)?; - let backends = parse_llamacpp_backend_statuses(&listing); - let Some((backend, already_installed)) = select_best_llamacpp_backend(&backends) else { - bail!( - "Lemonade reports no supported GPU llama.cpp backend for this host (status: {}). \ - The GPU-required policy does not fall back to CPU; install a ROCm or Vulkan \ - backend (e.g. `rocm engines install lemonade`) or verify the GPU driver with \ - `rocm examine`.", - describe_llamacpp_backends(&backends) - ); - }; - // Record the attempted backend before the fallible install step below, so a caller - // verifying the result can scope its check to this backend even when the install - // itself errors out. - manifest.backend_name = backend.clone(); - if already_installed && !force_reinstall { - eprintln!("Using installed Lemonade {LLAMACPP_RECIPE}:{backend} backend."); - } else { - eprintln!("Installing Lemonade {LLAMACPP_RECIPE}:{backend} backend..."); - install_llamacpp_backend_with_retry(|| { - run_lemonade_backend_install(manifest, host, port, &backend, process_env) - })?; - } - Ok(backend) -} - -/// Parse `lemonade backends` table output into `(backend, status)` pairs for the -/// `llamacpp` recipe only. Status is one of `installed`/`installable`/`unsupported`. -fn parse_llamacpp_backend_statuses(output: &str) -> Vec<(String, String)> { - const RECIPES: [&str; 8] = [ - "flm", - "kokoro", - "llamacpp", - "ryzenai-llm", - "sd-cpp", - "vllm", - "whispercpp", - "embeddings", - ]; - const STATUSES: [&str; 3] = ["installed", "installable", "unsupported"]; - let mut current_recipe = ""; - let mut result = Vec::new(); - for line in output.lines() { - let tokens: Vec<&str> = line.split_whitespace().collect(); - let Some(&first) = tokens.first() else { - continue; - }; - if first == "Recipe" || first.starts_with("---") { - continue; - } - // A row either starts with a recipe name (recipe + its first backend) or, - // for grouped recipes, with a backend name continuing the previous recipe. - let (backend, rest) = if RECIPES.contains(&first) { - current_recipe = match RECIPES.iter().find(|r| **r == first) { - Some(r) => r, - None => current_recipe, - }; - match tokens.get(1) { - Some(backend) => (*backend, &tokens[2..]), - None => continue, - } - } else { - (first, &tokens[1..]) - }; - if current_recipe != LLAMACPP_RECIPE { - continue; - } - if let Some(status) = rest.iter().find(|t| STATUSES.contains(t)) { - result.push((backend.to_owned(), (*status).to_owned())); - } - } - result -} - -/// Choose the highest-priority backend Lemonade considers usable (installed or -/// installable). Returns `(backend, already_installed)`. -fn select_best_llamacpp_backend(backends: &[(String, String)]) -> Option<(String, bool)> { - for candidate in LLAMACPP_BACKEND_PRIORITY { - if let Some((name, status)) = backends - .iter() - .find(|(b, s)| b == candidate && (s == "installed" || s == "installable")) - { - return Some((name.clone(), status == "installed")); - } - } - None -} - -fn describe_llamacpp_backends(backends: &[(String, String)]) -> String { - if backends.is_empty() { - return "none reported".to_owned(); - } - backends - .iter() - .map(|(b, s)| format!("{b}={s}")) - .collect::>() - .join(", ") -} - -fn run_lemonade_backends_list( - manifest: &LemonadeInstallManifest, - process_env: &LemonadeProcessEnvironment, -) -> Result { - let mut command = ProcessCommand::new(&manifest.lemonade); - command - .arg("backends") - .stdin(Stdio::null()) - .stdout(Stdio::piped()) - .stderr(Stdio::piped()); - apply_lemonade_process_environment(&mut command, process_env)?; - hide_child_console_window(&mut command); - let output = command - .output() - .with_context(|| format!("failed to run {}", manifest.lemonade.display()))?; - if !output.status.success() { - bail!( - "Lemonade backends query failed ({}): {}", - output.status, - String::from_utf8_lossy(&output.stderr).trim() - ); - } - Ok(String::from_utf8_lossy(&output.stdout).into_owned()) -} - -fn resolve_runtime() -> Result { - let paths = AppPaths::discover()?; - let manifest = read_manifest(&paths)?; - if !manifest.lemond.is_file() { - bail!( - "Lemonade runtime is missing {}; run `rocm engines install lemonade`", - manifest.lemond.display() - ); - } - Ok(LemonadeRuntime { manifest }) -} - -fn manifest_path(paths: &AppPaths) -> PathBuf { - paths.engine_manifests_dir(ENGINE_NAME).join("runtime.json") -} - -fn lemonade_root(paths: &AppPaths, env_root: Option<&Path>) -> PathBuf { - env_root.map_or_else( - || paths.engine_dir(ENGINE_NAME), - |root| normalize_runtime_path_for_host(root).join(ENGINE_NAME), - ) -} - -fn runtime_dir_in(root: &Path) -> PathBuf { - root.join("runtime") -} - -fn read_manifest(paths: &AppPaths) -> Result { - let path = manifest_path(paths); - let bytes = fs::read(&path).with_context(|| format!("failed to read {}", path.display()))?; - serde_json::from_slice(&bytes).with_context(|| format!("failed to parse {}", path.display())) -} - -fn write_manifest(paths: &AppPaths, manifest: &LemonadeInstallManifest) -> Result<()> { - let path = manifest_path(paths); - if let Some(parent) = path.parent() { - fs::create_dir_all(parent)?; - } - fs::write(&path, serde_json::to_vec_pretty(manifest)?)?; - Ok(()) -} - -fn lemond_path_in(runtime_dir: &Path) -> PathBuf { - runtime_dir.join(platform_binary_name("lemond")) -} - -fn lemonade_path_in(runtime_dir: &Path) -> PathBuf { - runtime_dir.join(platform_binary_name("lemonade")) -} - -fn platform_binary_name(name: &str) -> String { - if runtime_is_windows() { - format!("{name}.exe") - } else { - name.to_owned() - } -} - -fn embeddable_archive_name(version: &str) -> String { - let (os_arch, extension) = embeddable_os_arch(); - rocm_deps::lemonade_archive_name(version, os_arch, extension) -} - -fn embeddable_url(version: &str) -> String { - rocm_deps::lemonade_download_url(version, &embeddable_archive_name(version)) -} - -const fn embeddable_archive_sha256() -> &'static str { - if runtime_is_windows() { - EMBEDDABLE_WINDOWS_SHA256 - } else { - EMBEDDABLE_LINUX_SHA256 - } -} - -fn download_file(url: &str, destination: &Path) -> Result<()> { - download_file_to_path(url, destination, Duration::from_mins(15)) -} - -fn verify_sha256(path: &Path, expected: &str) -> Result<()> { - let mut file = fs::File::open(path) - .with_context(|| format!("failed to open {} for verification", path.display()))?; - let mut hasher = Sha256::new(); - std::io::copy(&mut file, &mut hasher) - .with_context(|| format!("failed to hash {}", path.display()))?; - let actual = format!("{:x}", hasher.finalize()); - if actual != expected { - bail!( - "SHA-256 mismatch for {}: expected {expected}, got {actual}", - path.display() - ); - } - Ok(()) -} - -fn publish_verified_archive( - temporary: tempfile::NamedTempFile, - archive: &Path, - expected_sha256: &str, -) -> Result<()> { - match temporary.persist_noclobber(archive) { - Ok(_) => Ok(()), - Err(error) if error.error.kind() == std::io::ErrorKind::AlreadyExists => { - let publication_error = error.error; - verify_sha256(archive, expected_sha256).with_context(|| { - format!( - "concurrent archive publication at {} failed verification after publish conflict ({publication_error})", - archive.display() - ) - }) - } - Err(error) => Err(error.error) - .with_context(|| format!("failed to publish verified archive {}", archive.display())), - } -} - -fn ensure_cached_archive( - url: &str, - archive: &Path, - expected_sha256: &str, - download: F, -) -> Result -where - F: FnMut(&str, &Path) -> Result<()>, -{ - ensure_cached_archive_after_invalid(url, archive, expected_sha256, download, || {}) -} - -#[derive(Debug)] -struct ArchiveCacheLock { - file: fs::File, -} - -impl Drop for ArchiveCacheLock { - fn drop(&mut self) { - let _ = self.file.unlock(); - } -} - -/// In-process exclusion for one archive path. -#[derive(Debug)] -struct ArchiveCacheProcessLock { - key: PathBuf, -} - -impl Drop for ArchiveCacheProcessLock { - fn drop(&mut self) { - let mut busy = ARCHIVE_CACHE_BUSY - .lock() - .unwrap_or_else(std::sync::PoisonError::into_inner); - busy.remove(&self.key); - drop(busy); - ARCHIVE_CACHE_BUSY_SIGNAL.notify_all(); - } -} - -fn lock_archive_cache_in_process(archive: &Path) -> ArchiveCacheProcessLock { - let key = archive.to_path_buf(); - let mut busy = ARCHIVE_CACHE_BUSY - .lock() - .unwrap_or_else(std::sync::PoisonError::into_inner); - if busy.contains(&key) { - // The holder may be inside a download window minutes long; without this - // the command sits silent and looks hung. - eprintln!( - "Waiting for another install in this process to finish with {}...", - archive.display() - ); - } - while busy.contains(&key) { - busy = ARCHIVE_CACHE_BUSY_SIGNAL - .wait(busy) - .unwrap_or_else(std::sync::PoisonError::into_inner); - } - busy.insert(key.clone()); - ArchiveCacheProcessLock { key } -} - -/// Exclusive access to one cached archive, held by the caller for as long as -/// the verified bytes are relied upon. -/// -/// Verification and use have to happen under the same lock: releasing it at the -/// end of verification would leave a window in which a co-resident writer could -/// swap the archive before it is read again for extraction. -#[derive(Debug)] -struct ArchiveCacheGuard { - _process: ArchiveCacheProcessLock, - _file: ArchiveCacheLock, -} - -fn archive_lock_path(archive: &Path) -> Result { - let parent = archive - .parent() - .with_context(|| format!("archive path has no parent: {}", archive.display()))?; - let file_name = archive - .file_name() - .with_context(|| format!("archive path has no file name: {}", archive.display()))?; - let mut lock_name = OsString::from("."); - lock_name.push(file_name); - lock_name.push(".lock"); - Ok(parent.join(lock_name)) -} - -#[cfg(test)] -fn mark_archive_lock_test_stage(stage: &str) { - const ROLE: &str = "ROCM_LEMONADE_ARCHIVE_LOCK_TEST_ROLE"; - const DIR: &str = "ROCM_LEMONADE_ARCHIVE_LOCK_TEST_DIR"; - if let (Ok(role), Some(dir)) = (std::env::var(ROLE), std::env::var_os(DIR)) { - fs::write(PathBuf::from(dir).join(format!("{role}-{stage}")), b"") - .expect("write archive lock test marker"); - } -} - -#[cfg(test)] -fn start_archive_lock_test_heartbeat() -> Option> { - const ROLE: &str = "ROCM_LEMONADE_ARCHIVE_LOCK_TEST_ROLE"; - const DIR: &str = "ROCM_LEMONADE_ARCHIVE_LOCK_TEST_DIR"; - if std::env::var(ROLE).as_deref() != Ok("recoverer") { - return None; - } - let dir = std::env::var_os(DIR)?; - Some(std::thread::spawn(move || { - std::thread::sleep(Duration::from_millis(100)); - fs::write(PathBuf::from(dir).join("recoverer-heartbeat"), b"") - .expect("write archive lock test heartbeat"); - })) -} - -fn lock_archive_cache(archive: &Path) -> Result { - let lock_path = archive_lock_path(archive)?; - let file = fs::OpenOptions::new() - .read(true) - .write(true) - .create(true) - .truncate(false) - .open(&lock_path) - .with_context(|| format!("failed to open archive cache lock {}", lock_path.display()))?; - #[cfg(test)] - mark_archive_lock_test_stage("before-lock"); - #[cfg(test)] - let heartbeat = start_archive_lock_test_heartbeat(); - // `lock` blocks with no timeout, and the holder may be minutes into a - // download, so announce the wait instead of leaving the command silent. - // Probing first keeps the common uncontended case quiet. - if file.try_lock().is_err() { - eprintln!( - "Waiting for another process to finish with {}...", - archive.display() - ); - file.lock().with_context(|| { - format!( - "failed to acquire archive cache lock {}", - lock_path.display() - ) - })?; - } - #[cfg(test)] - { - mark_archive_lock_test_stage("after-lock"); - if let Some(heartbeat) = heartbeat { - heartbeat - .join() - .expect("archive lock test heartbeat panicked"); - } - } - Ok(ArchiveCacheLock { file }) -} - -fn ensure_cached_archive_after_invalid( - url: &str, - archive: &Path, - expected_sha256: &str, - mut download: F, - after_invalid: I, -) -> Result -where - F: FnMut(&str, &Path) -> Result<()>, - I: FnOnce(), -{ - let parent = archive - .parent() - .with_context(|| format!("archive path has no parent: {}", archive.display()))?; - fs::create_dir_all(parent).with_context(|| { - format!( - "failed to create archive cache directory {}", - parent.display() - ) - })?; - let process_guard = lock_archive_cache_in_process(archive); - let archive_guard = lock_archive_cache(archive)?; - let guard = ArchiveCacheGuard { - _process: process_guard, - _file: archive_guard, - }; - - match fs::symlink_metadata(archive) { - Ok(metadata) if metadata.file_type().is_file() => { - if verify_sha256(archive, expected_sha256).is_ok() { - eprintln!("Using verified cached {}.", archive.display()); - return Ok(guard); - } - after_invalid(); - match fs::remove_file(archive) { - Ok(()) => {} - Err(error) if error.kind() == std::io::ErrorKind::NotFound => {} - Err(error) => { - return Err(error).with_context(|| { - format!("failed to remove invalid {}", archive.display()) - }); - } - } - } - Ok(_) => bail!( - "archive cache path is not a regular file: {}", - archive.display() - ), - Err(error) if error.kind() == std::io::ErrorKind::NotFound => {} - Err(error) => return Err(error.into()), - } - - let temporary = tempfile::Builder::new() - .prefix(".lemonade-download-") - .tempfile_in(parent) - .with_context(|| { - format!( - // Owner-only permissions are a Unix guarantee: `tempfile` - // creates with mode 0600 there, while on Windows the file - // inherits the parent directory's DACL. - "failed to create temporary download beside {}", - archive.display() - ) - })?; - let temporary_path = temporary.path().to_path_buf(); - eprintln!("Downloading {}...", archive.display()); - download(url, &temporary_path)?; - verify_sha256(&temporary_path, expected_sha256)?; - publish_verified_archive(temporary, archive, expected_sha256)?; - Ok(guard) -} - -/// Resolves an archive entry name to a path inside `destination`. -/// -/// Entry names come from the archive, so they are untrusted: this rejects -/// absolute paths, drive-relative and UNC names, `..` traversal, and anything -/// else that is not a plain sequence of normal components. The returned path is -/// built component-by-component, so it cannot leave `destination`. -fn archive_entry_destination(destination: &Path, name: &Path) -> Result { - let mut resolved = destination.to_path_buf(); - let mut components = 0usize; - for component in name.components() { - match component { - std::path::Component::Normal(part) => { - // Reject separators smuggled through a single component (a - // Windows-style `a\..\b` read on Unix is one component). - let text = part.to_string_lossy(); - if text.contains('/') || text.contains('\\') || text == ".." { - bail!("archive entry has an unsafe name: {}", name.display()); - } - resolved.push(part); - components += 1; - } - std::path::Component::CurDir => {} - std::path::Component::ParentDir - | std::path::Component::RootDir - | std::path::Component::Prefix(_) => { - bail!( - "archive entry escapes the extraction root: {}", - name.display() - ); - } - } - } - if components == 0 { - bail!("archive entry has an empty name"); - } - Ok(resolved) -} - -/// Creates the parent directories for an extracted entry, inside `destination`. -fn create_entry_parent(destination: &Path, path: &Path) -> Result<()> { - let Some(parent) = path.parent() else { - return Ok(()); - }; - if !parent.starts_with(destination) { - bail!( - "archive entry escapes the extraction root: {}", - path.display() - ); - } - fs::create_dir_all(parent).with_context(|| format!("failed to create {}", parent.display()))?; - Ok(()) -} - -/// Writes one extracted regular file, refusing to follow an existing entry. -/// -/// `create_new` matters: without it a directory entry extracted earlier — or a -/// symlink planted by a co-resident process between entries — would be written -/// through rather than rejected. -fn write_entry_file(path: &Path, reader: &mut dyn Read) -> Result<()> { - let mut file = fs::OpenOptions::new() - .write(true) - .create_new(true) - .open(path) - .with_context(|| format!("failed to create {}", path.display()))?; - std::io::copy(reader, &mut file) - .with_context(|| format!("failed to write {}", path.display()))?; - Ok(()) -} - -#[cfg(unix)] -fn set_entry_mode(path: &Path, mode: Option) -> Result<()> { - use std::os::unix::fs::PermissionsExt; - - let Some(mode) = mode else { - return Ok(()); - }; - // Keep the executable bit the runtime needs; drop setuid/setgid/sticky and - // any group/other write bit the archive happened to carry. - let mode = mode & 0o755; - fs::set_permissions(path, fs::Permissions::from_mode(mode)) - .with_context(|| format!("failed to set permissions on {}", path.display())) -} - -#[cfg(not(unix))] -#[expect( - clippy::unnecessary_wraps, - reason = "mirrors the Unix signature; Windows has no mode bits to apply" -)] -const fn set_entry_mode(_path: &Path, _mode: Option) -> Result<()> { - Ok(()) -} - -/// Extracts a gzip-compressed tar archive in-process. -fn extract_tar_gz(archive: &Path, destination: &Path) -> Result<()> { - let file = - fs::File::open(archive).with_context(|| format!("failed to open {}", archive.display()))?; - let mut tar = tar::Archive::new(flate2::read::GzDecoder::new(std::io::BufReader::new(file))); - for entry in tar - .entries() - .with_context(|| format!("failed to read {}", archive.display()))? - { - let mut entry = entry.with_context(|| format!("failed to read {}", archive.display()))?; - let name = entry - .path() - .with_context(|| { - format!( - "archive entry has an undecodable name in {}", - archive.display() - ) - })? - .into_owned(); - let kind = entry.header().entry_type(); - // A pax *global* header carries archive-wide metadata, not a member: its - // name (`pax_global_header` by convention) is not a path to create. The - // `tar` crate consumes long-name, long-link and pax *local* headers - // itself, but hands this one to the caller, so it has to be skipped - // explicitly — `git archive` and `tar --format=pax` both emit it, and - // treating it as a member would reject an otherwise valid archive. - if kind == tar::EntryType::XGlobalHeader { - continue; - } - let path = archive_entry_destination(destination, &name)?; - match kind { - tar::EntryType::Directory => { - fs::create_dir_all(&path) - .with_context(|| format!("failed to create {}", path.display()))?; - } - tar::EntryType::Regular | tar::EntryType::Continuous => { - create_entry_parent(destination, &path)?; - let mode = entry.header().mode().ok(); - write_entry_file(&path, &mut entry)?; - set_entry_mode(&path, mode)?; - } - // A sparse entry would expand correctly — the crate resolves the - // sparse map — but a few-byte header can declare a multi-gigabyte - // logical size, so accepting one turns a digest-pinned archive into - // an unbounded write. No Lemonade artifact stores files sparsely, so - // this stays rejected until one needs it. - tar::EntryType::GNUSparse => bail!( - "archive contains a sparse entry at {}, which is not supported", - name.display() - ), - // Links, devices, fifos and sockets are the entry kinds that let an - // archive reach outside its own tree; the official artifacts carry - // none of them. See the link policy on `extract_archive` for what - // to do if a future release ships one. - other => bail!( - "archive contains an unsupported entry ({other:?}) at {}", - name.display() - ), - } - } - Ok(()) -} - -/// Extracts a zip archive in-process. -fn extract_zip(archive: &Path, destination: &Path) -> Result<()> { - let file = - fs::File::open(archive).with_context(|| format!("failed to open {}", archive.display()))?; - let mut zip = zip::ZipArchive::new(std::io::BufReader::new(file)) - .with_context(|| format!("failed to read {}", archive.display()))?; - for index in 0..zip.len() { - let mut entry = zip - .by_index(index) - .with_context(|| format!("failed to read entry {index} of {}", archive.display()))?; - // `enclosed_name` returns None for absolute or `..`-bearing names; the - // raw name is still used for diagnostics and for the trailing-slash - // directory convention. - let raw = entry.name().to_owned(); - // See the link policy on `extract_archive`. - if entry.is_symlink() { - bail!("archive contains symlink at {raw}"); - } - let name = entry - .enclosed_name() - .with_context(|| format!("archive entry escapes the extraction root: {raw}"))?; - let path = archive_entry_destination(destination, &name)?; - if entry.is_dir() { - fs::create_dir_all(&path) - .with_context(|| format!("failed to create {}", path.display()))?; - continue; - } - // No third entry kind to reject here: `zip` defines `is_file` as - // "neither a directory nor a symlink", and both are handled above. - create_entry_parent(destination, &path)?; - let mode = entry.unix_mode(); - write_entry_file(&path, &mut entry)?; - set_entry_mode(&path, mode)?; - } - Ok(()) -} - -/// Extracts the embeddable archive into `destination`. -/// -/// Extraction is done in-process rather than by shelling out to `tar`: the -/// external binary was resolved from `PATH` (so the extractor unpacking -/// untrusted bytes was itself unpinned), and its traversal behaviour varied -/// between GNU tar, the bsdtar shipped as Windows `tar.exe`, and busybox tar. -/// -/// # Link policy -/// -/// Extraction writes directories and regular files and nothing else: a link -/// entry is rejected rather than resolved, here and again when the extracted -/// tree is searched and copied out. That is deliberately conservative. The -/// v10.10.0 artifacts contain no links, and refusing them removes the whole -/// class of "entry points outside the tree" bugs instead of trying to reason -/// about each one. -/// -/// The cost is that it fails closed on an authentic archive: if a future -/// Lemonade release ships, say, `libfoo.so -> libfoo.so.1`, installs stop with -/// "archive contains symlink at ...". That is a signal to widen the policy -/// deliberately — resolve the target and accept it only if it stays inside the -/// extraction root — not to drop the check. -fn extract_archive(archive: &Path, destination: &Path) -> Result<()> { - let is_zip = archive - .extension() - .is_some_and(|extension| extension.eq_ignore_ascii_case("zip")); - if is_zip { - extract_zip(archive, destination) - } else { - extract_tar_gz(archive, destination) - } -} - -fn windows_child_path(path: &Path) -> String { - let raw = path.display().to_string(); - let normalized = raw.replace('\\', "/"); - let bytes = normalized.as_bytes(); - if bytes.len() >= 3 && bytes[0] == b'/' && bytes[1].is_ascii_alphabetic() && bytes[2] == b'/' { - let drive = (bytes[1] as char).to_ascii_uppercase(); - let rest = normalized[3..].replace('/', "\\"); - return format!("{drive}:\\{rest}"); - } - if bytes.len() == 2 && bytes[0] == b'/' && bytes[1].is_ascii_alphabetic() { - let drive = (bytes[1] as char).to_ascii_uppercase(); - return format!("{drive}:\\"); - } - raw -} - -fn find_embeddable_root(extract_root: &Path) -> Result { - let extract_root = fs::canonicalize(extract_root) - .with_context(|| format!("failed to canonicalize {}", extract_root.display()))?; - let mut candidates = Vec::new(); - collect_embeddable_roots(&extract_root, &extract_root, &mut candidates, 0)?; - candidates.into_iter().next().with_context(|| { - format!( - "no Lemonade embeddable root found in {}", - extract_root.display() - ) - }) -} - -fn collect_embeddable_roots( - extract_root: &Path, - path: &Path, - candidates: &mut Vec, - depth: usize, -) -> Result<()> { - if depth > MAX_EMBEDDABLE_SEARCH_DEPTH { - return Ok(()); - } - let metadata = fs::symlink_metadata(path)?; - // See the link policy on `extract_archive`. - if metadata.file_type().is_symlink() { - bail!("archive contains symlink at {}", path.display()); - } - if !metadata.file_type().is_dir() { - bail!("archive contains non-directory entry at {}", path.display()); - } - let canonical_path = fs::canonicalize(path)?; - if !canonical_path.starts_with(extract_root) { - bail!("archive entry escapes extraction root: {}", path.display()); - } - - let lemond = lemond_path_in(path); - let lemonade = lemonade_path_in(path); - let lemond_type = fs::symlink_metadata(&lemond) - .ok() - .map(|metadata| metadata.file_type()); - let lemonade_type = fs::symlink_metadata(&lemonade) - .ok() - .map(|metadata| metadata.file_type()); - if lemond_type.as_ref().is_some_and(fs::FileType::is_symlink) - || lemonade_type.as_ref().is_some_and(fs::FileType::is_symlink) - { - bail!( - "archive contains symlinked Lemonade binary in {}", - path.display() - ); - } - if lemond_type.as_ref().is_some_and(fs::FileType::is_file) - && lemonade_type.as_ref().is_some_and(fs::FileType::is_file) - { - candidates.push(canonical_path); - return Ok(()); - } - - for entry in fs::read_dir(path)? { - let entry = entry?; - let file_type = entry.file_type()?; - if file_type.is_symlink() { - bail!("archive contains symlink at {}", entry.path().display()); - } - if file_type.is_dir() { - collect_embeddable_roots(extract_root, &entry.path(), candidates, depth + 1)?; - } else if !file_type.is_file() { - bail!( - "archive contains non-regular entry at {}", - entry.path().display() - ); - } - } - Ok(()) -} - -fn destination_child(destination: &Path, name: &std::ffi::OsStr) -> Result { - let component = Path::new(name); - if component.components().count() != 1 - || !matches!( - component.components().next(), - Some(std::path::Component::Normal(_)) - ) - { - bail!( - "archive entry is not one destination component: {}", - Path::new(name).display() - ); - } - Ok(destination.join(component)) -} - -fn copy_tree(source: &Path, destination: &Path) -> Result<()> { - let source = fs::canonicalize(source) - .with_context(|| format!("failed to canonicalize {}", source.display()))?; - if !fs::symlink_metadata(&source)?.file_type().is_dir() { - bail!("copy source is not a directory: {}", source.display()); - } - fs::create_dir(destination) - .with_context(|| format!("failed to create {}", destination.display()))?; - let destination = fs::canonicalize(destination) - .with_context(|| format!("failed to canonicalize {}", destination.display()))?; - copy_tree_entries(&source, &source, &destination, &destination, 0) -} - -fn copy_tree_entries( - source_root: &Path, - source: &Path, - destination_root: &Path, - destination: &Path, - depth: usize, -) -> Result<()> { - if depth > MAX_COPY_RECURSION_DEPTH { - bail!( - "archive tree is deeper than {MAX_COPY_RECURSION_DEPTH} levels at {}", - source.display() - ); - } - for entry in fs::read_dir(source)? { - let entry = entry?; - let source_path = entry.path(); - let file_type = entry.file_type()?; - // See the link policy on `extract_archive`. - if file_type.is_symlink() { - bail!("refusing to copy symlink {}", source_path.display()); - } - if !file_type.is_dir() && !file_type.is_file() { - bail!( - "refusing to copy non-regular entry {}", - source_path.display() - ); - } - let canonical_source = fs::canonicalize(&source_path)?; - if !canonical_source.starts_with(source_root) { - bail!( - "copy source escapes extraction root: {}", - source_path.display() - ); - } - let destination_path = destination_child(destination, &entry.file_name())?; - if !destination_path.starts_with(destination_root) { - bail!( - "copy destination escapes runtime root: {}", - destination_path.display() - ); - } - if file_type.is_dir() { - fs::create_dir(&destination_path) - .with_context(|| format!("failed to create {}", destination_path.display()))?; - let canonical_destination = fs::canonicalize(&destination_path)?; - if !canonical_destination.starts_with(destination_root) { - bail!( - "copy destination escapes runtime root: {}", - destination_path.display() - ); - } - copy_tree_entries( - source_root, - &canonical_source, - destination_root, - &canonical_destination, - depth + 1, - )?; - } else { - fs::copy(&canonical_source, &destination_path).with_context(|| { - format!( - "failed to copy {} to {}", - canonical_source.display(), - destination_path.display() - ) - })?; - if !cfg!(windows) { - #[cfg(unix)] - { - use std::os::unix::fs::PermissionsExt; - let metadata = fs::symlink_metadata(&canonical_source)?; - fs::set_permissions( - &destination_path, - fs::Permissions::from_mode(metadata.permissions().mode()), - )?; - } - } - } - } - Ok(()) -} - -fn spawn_lemond( - manifest: &LemonadeInstallManifest, - host: &str, - port: u16, - log_path: Option<&Path>, - process_env: &LemonadeProcessEnvironment, -) -> Result { - #[cfg(windows)] - if let Some(log_path) = log_path { - let args = vec![ - child_process_path(&manifest.runtime_dir), - "--host".to_owned(), - host.to_owned(), - "--port".to_owned(), - port.to_string(), - ]; - let pid = rocm_core::spawn_hidden_console_with_log(&manifest.lemond, &args, &[], log_path) - .with_context(|| format!("failed to start {}", manifest.lemond.display()))?; - return Ok(LemondChild::Pid(pid)); - } - - let mut command = ProcessCommand::new(&manifest.lemond); - command - .arg(child_process_path(&manifest.runtime_dir)) - .arg("--host") - .arg(host) - .arg("--port") - .arg(port.to_string()) - .stdin(Stdio::null()); - apply_lemonade_process_environment(&mut command, process_env)?; - if let Some(log_path) = log_path { - if let Some(parent) = log_path.parent() { - fs::create_dir_all(parent)?; - } - let log = fs::File::create(log_path) - .with_context(|| format!("failed to create {}", log_path.display()))?; - command.stdout(Stdio::from(log.try_clone()?)); - command.stderr(Stdio::from(log)); - } else { - command.stdout(Stdio::inherit()).stderr(Stdio::inherit()); - } - hide_child_console_window(&mut command); - command - .spawn() - .with_context(|| format!("failed to start {}", manifest.lemond.display())) - .map(LemondChild::Child) -} - -enum LemondChild { - Child(std::process::Child), - #[cfg(windows)] - Pid(u32), -} - -impl LemondChild { - fn id(&self) -> u32 { - match self { - Self::Child(child) => child.id(), - #[cfg(windows)] - Self::Pid(pid) => *pid, - } - } - - fn wait(&mut self) -> Result { - match self { - Self::Child(child) => { - let status = child.wait().context("failed waiting for Lemonade server")?; - Ok(LemondExitStatus { - success: status.success(), - description: status.to_string(), - }) - } - #[cfg(windows)] - Self::Pid(pid) => { - let code = rocm_core::wait_for_process_exit(*pid)?; - Ok(LemondExitStatus { - success: code == 0, - description: format!("exit code {code}"), - }) - } - } - } -} - -struct LemondExitStatus { - success: bool, - description: String, -} - -impl LemondExitStatus { - const fn success(&self) -> bool { - self.success - } -} - -impl std::fmt::Display for LemondExitStatus { - fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - formatter.write_str(&self.description) - } -} - -#[cfg(windows)] -fn hide_child_console_window(command: &mut ProcessCommand) { - use std::os::windows::process::CommandExt; - - const CREATE_NO_WINDOW: u32 = 0x0800_0000; - command.creation_flags(CREATE_NO_WINDOW); -} - -#[cfg(not(windows))] -const fn hide_child_console_window(_command: &mut ProcessCommand) {} - -fn child_process_path(path: &Path) -> String { - if runtime_is_windows() { - windows_child_path(path) - } else { - path.display().to_string() - } -} - -fn lemonade_process_environment() -> Result { - let paths = AppPaths::discover()?; - let config = RocmCliConfig::load(&paths).unwrap_or_default(); - let Some(env) = active_managed_therock_environment(&paths, &config)? else { - return Ok(LemonadeProcessEnvironment::default()); - }; - Ok(LemonadeProcessEnvironment { - rocm_root: env.rocm_root, - path_entries: env.path_entries, - library_entries: env.library_entries, - gpu_indices: Vec::new(), - }) -} - -fn apply_lemonade_process_environment( - command: &mut ProcessCommand, - env: &LemonadeProcessEnvironment, -) -> Result<()> { - let vars = lemonade_process_environment_vars(env, &ParentRuntimeEnvironment::current())?; - apply_environment_vars(command, &vars); - Ok(()) -} - -/// Apply an already-resolved variable set. Callers that spawn repeatedly (the -/// readiness poll) resolve once and reuse, so preparing the runtime directory -/// does not repeat its syscalls on every attempt. -fn apply_environment_vars(command: &mut ProcessCommand, vars: &[(&'static str, OsString)]) { - for (key, value) in vars { - command.env(key, value); - } -} - -/// The parts of *this* process's environment that decide where the child's -/// runtime directory goes. -/// -/// Captured into a value instead of being read inside -/// [`lemonade_process_environment_vars`] so the precedence is testable without -/// mutating process-global env vars, which is `unsafe` and racy under parallel -/// tests in edition 2024. -#[derive(Debug, Clone, Default)] -struct ParentRuntimeEnvironment { - xdg_runtime_dir: Option, - home: Option, - user: Option, - temp_dir: PathBuf, -} - -impl ParentRuntimeEnvironment { - fn current() -> Self { - Self { - xdg_runtime_dir: std::env::var_os("XDG_RUNTIME_DIR"), - home: std::env::var_os("HOME"), - // An empty `USER` must fall through to `LOGNAME`, not short-circuit it. - user: std::env::var("USER") - .ok() - .filter(|v| !v.is_empty()) - .or_else(|| std::env::var("LOGNAME").ok().filter(|v| !v.is_empty())), - temp_dir: std::env::temp_dir(), - } - } -} - -/// Leaf directory the engine gets inside a fallback runtime root, so a crashing -/// or misbehaving server cannot disturb the telemetry socket that shares the -/// same root. -const RUNTIME_DIR_LEAF: &str = "lemonade"; - -/// Guarantee the child a usable `XDG_RUNTIME_DIR`. -/// -/// `lemond` refuses to start unless it can resolve a writable runtime directory -/// from `XDG_RUNTIME_DIR` or `RUNTIME_DIRECTORY`. Neither exists on a headless -/// host: `XDG_RUNTIME_DIR` is populated by `pam_systemd` at login, so it is -/// absent for every non-login process (cron jobs, CI runners, `systemd-run`, a -/// bare container exec), and `RUNTIME_DIRECTORY` only exists for a systemd unit -/// that declares `RuntimeDirectory=`. There, a managed serve failed -/// deterministically before the server ever came up. -/// -/// A value already present in the parent environment is passed through -/// unchanged — an operator's choice always wins, and it is the tier the rest of -/// this precedence exists to fall back from. -fn child_runtime_dir_var( - parent: &ParentRuntimeEnvironment, -) -> Result> { - if !runtime_is_linux() { - return Ok(None); - } - if let Some(existing) = parent - .xdg_runtime_dir - .as_ref() - .filter(|value| !value.is_empty()) - { - return Ok(Some(("XDG_RUNTIME_DIR", existing.clone()))); - } - let has_home = parent.home.as_ref().is_some_and(|value| !value.is_empty()); - let dir = rocm_core::user_runtime_dir( - None, - parent.home.clone(), - parent.user.clone(), - parent.temp_dir.clone(), - RUNTIME_DIR_LEAF, - RUNTIME_DIR_LEAF, - ); - let prepare = if has_home { - create_private_runtime_dir(&dir) - } else { - create_private_tier3_runtime_dir(&dir) - }; - prepare.with_context(|| { - format!( - "preparing a runtime directory for the Lemonade server at {} \ - (XDG_RUNTIME_DIR is unset in this environment)", - dir.display() - ) - })?; - Ok(Some(("XDG_RUNTIME_DIR", dir.into_os_string()))) -} - -#[cfg(unix)] -fn create_private_tier3_runtime_dir(dir: &Path) -> Result<()> { - let user_parent = dir - .parent() - .ok_or_else(|| anyhow!("{} has no tier-3 user parent", dir.display()))?; - let temp_root = user_parent.parent().ok_or_else(|| { - anyhow!( - "{} has no temporary-directory parent", - user_parent.display() - ) - })?; - let temp_root = open_safe_tier3_root(temp_root)?; - let user_component = single_directory_component(user_parent)?; - let user_parent = create_private_owned_directory_at(&temp_root, user_component, user_parent)?; - let leaf_component = single_directory_component(dir)?; - create_private_owned_directory_at(&user_parent, leaf_component, dir)?; - Ok(()) -} - -#[cfg(unix)] -fn single_directory_component(path: &Path) -> Result<&OsStr> { - path.file_name() - .filter(|component| Path::new(component).components().count() == 1) - .ok_or_else(|| anyhow!("{} has no safe final directory component", path.display())) -} - -#[cfg(unix)] -#[allow(unsafe_code)] -fn open_safe_tier3_root(temp_root: &Path) -> Result { - use std::ffi::CString; - use std::io::ErrorKind; - use std::os::fd::{AsRawFd, FromRawFd}; - use std::os::unix::ffi::OsStrExt; - use std::os::unix::fs::OpenOptionsExt; - use std::path::Component; - - if !temp_root.is_absolute() { - bail!( - "tier-3 temporary root {} is relative or otherwise unstable", - temp_root.display() - ); - } - let mut components = temp_root.components(); - if components.next() != Some(Component::RootDir) { - bail!( - "tier-3 temporary root {} has no filesystem-root anchor", - temp_root.display() - ); - } - - let mut directory = fs::OpenOptions::new() - .read(true) - .custom_flags(libc::O_DIRECTORY | libc::O_NOFOLLOW) - .open(Path::new("/")) - .context("opening filesystem root without following symlinks")?; - let current_euid = effective_user_id(); - let mut traversed = PathBuf::from("/"); - let mut components = components.peekable(); - while let Some(component) = components.next() { - let Component::Normal(component) = component else { - bail!( - "tier-3 temporary root {} contains a relative or unstable component", - temp_root.display() - ); - }; - validate_runtime_ancestry_container(&traversed, &directory, current_euid)?; - let child_path = traversed.join(component); - let component = CString::new(component.as_bytes()).with_context(|| { - format!( - "tier-3 temporary root {} contains a NUL byte", - temp_root.display() - ) - })?; - let is_final = components.peek().is_none(); - let raw_fd = match unsafe_openat_directory(directory.as_raw_fd(), &component) { - Ok(fd) => fd, - Err(error) if is_final && error.kind() == ErrorKind::NotFound => { - match unsafe_mkdirat(directory.as_raw_fd(), &component, 0o700) { - Ok(()) => {} - Err(error) if error.kind() == ErrorKind::AlreadyExists => {} - Err(error) => { - return Err(error).with_context(|| { - format!("creating tier-3 temporary root {}", temp_root.display()) - }); - } - } - unsafe_openat_directory(directory.as_raw_fd(), &component).with_context(|| { - format!( - "opening newly created tier-3 temporary root {}", - temp_root.display() - ) - })? - } - Err(error) => { - return Err(error).with_context(|| { - format!( - "opening {} while securely traversing tier-3 temporary-root ancestry", - child_path.display() - ) - }); - } - }; - // SAFETY: `unsafe_openat_directory` returned a new owned descriptor, - // transferred exactly once into a `File` that closes it on drop. - directory = unsafe { fs::File::from_raw_fd(raw_fd) }; - validate_runtime_directory_type(&child_path, &unsafe_fstat(directory.as_raw_fd())?)?; - traversed = child_path; - } - - let metadata = unsafe_fstat(directory.as_raw_fd())?; - if !directory_prevents_cross_uid_entry_replacement( - metadata.st_uid, - metadata.st_mode, - current_euid, - filesystem_is_read_only(directory.as_raw_fd())?, - ) { - bail!( - "tier-3 temporary root {} is unsafe: trusted owners require no group/other write or sticky protection; foreign owners require a read-only filesystem", - temp_root.display() - ); - } - Ok(directory) -} - -#[cfg(unix)] -const fn directory_prevents_cross_uid_entry_replacement( - uid: u32, - mode: u32, - current_euid: u32, - filesystem_read_only: bool, -) -> bool { - if uid == 0 || uid == current_euid { - return mode & 0o022 == 0 || mode & libc::S_ISVTX != 0; - } - filesystem_read_only -} - -#[cfg(unix)] -fn validate_runtime_ancestry_container( - path: &Path, - directory: &fs::File, - current_euid: u32, -) -> Result<()> { - use std::os::fd::AsRawFd; - - let metadata = unsafe_fstat(directory.as_raw_fd())?; - validate_runtime_directory_type(path, &metadata)?; - if !directory_prevents_cross_uid_entry_replacement( - metadata.st_uid, - metadata.st_mode, - current_euid, - filesystem_is_read_only(directory.as_raw_fd())?, - ) { - bail!( - "tier-3 temporary-root ancestor {} permits cross-UID rename or replacement", - path.display() - ); - } - Ok(()) -} - -#[cfg(unix)] -fn filesystem_is_read_only(fd: std::os::fd::RawFd) -> Result { - Ok(unsafe_fstatvfs(fd)?.f_flag & libc::ST_RDONLY != 0) -} - -#[cfg(unix)] -fn validate_runtime_directory_type(path: &Path, metadata: &libc::stat) -> Result<()> { - if metadata.st_mode & libc::S_IFMT != libc::S_IFDIR { - bail!("{} is not a directory", path.display()); - } - Ok(()) -} - -#[cfg(not(unix))] -fn create_private_tier3_runtime_dir(dir: &Path) -> Result<()> { - create_private_runtime_dir(dir) -} - -#[cfg(unix)] -#[allow(unsafe_code)] -fn create_private_owned_directory_at( - parent: &fs::File, - component: &OsStr, - display_path: &Path, -) -> Result { - use std::ffi::CString; - use std::io::ErrorKind; - use std::os::fd::{AsRawFd, FromRawFd}; - use std::os::unix::ffi::OsStrExt; - - let component = CString::new(component.as_bytes()).with_context(|| { - format!( - "{} contains a NUL byte in its final component", - display_path.display() - ) - })?; - let mkdir_result = unsafe_mkdirat(parent.as_raw_fd(), &component, 0o700); - if let Err(error) = mkdir_result - && error.kind() != ErrorKind::AlreadyExists - { - return Err(error).with_context(|| format!("creating {}", display_path.display())); - } - - let raw_fd = unsafe_openat_directory(parent.as_raw_fd(), &component).with_context(|| { - format!( - "opening {} relative to its validated parent without following symlinks", - display_path.display() - ) - })?; - // SAFETY: `unsafe_openat_directory` returned a new owned descriptor, and - // this is its single transfer into a type that closes it on drop. - let directory = unsafe { fs::File::from_raw_fd(raw_fd) }; - let current_euid = effective_user_id(); - validate_runtime_directory_owner( - display_path, - &unsafe_fstat(directory.as_raw_fd())?, - current_euid, - )?; - unsafe_fchmod(directory.as_raw_fd(), 0o700) - .with_context(|| format!("restricting {} to mode 0700", display_path.display()))?; - - let secured = unsafe_fstat(directory.as_raw_fd())?; - validate_runtime_directory_owner(display_path, &secured, current_euid)?; - if secured.st_mode & 0o777 != 0o700 { - bail!( - "{} is not private after chmod; expected mode 0700", - display_path.display() - ); - } - Ok(directory) -} - -#[cfg(unix)] -#[allow(unsafe_code)] -fn unsafe_fstat(fd: std::os::fd::RawFd) -> std::io::Result { - let mut metadata = std::mem::MaybeUninit::::uninit(); - // SAFETY: `metadata` points to writable storage for one `libc::stat`, and - // `fd` is borrowed from a live `File`. On success fstat initializes it. - if unsafe { libc::fstat(fd, metadata.as_mut_ptr()) } == 0 { - // SAFETY: successful fstat initialized the full `libc::stat` value. - Ok(unsafe { metadata.assume_init() }) - } else { - Err(std::io::Error::last_os_error()) - } -} - -#[cfg(unix)] -#[allow(unsafe_code)] -fn unsafe_fstatvfs(fd: std::os::fd::RawFd) -> std::io::Result { - let mut metadata = std::mem::MaybeUninit::::uninit(); - // SAFETY: `metadata` points to writable storage for one `libc::statvfs`, - // and `fd` is borrowed from a live directory `File`. On success fstatvfs - // initializes it. - if unsafe { libc::fstatvfs(fd, metadata.as_mut_ptr()) } == 0 { - // SAFETY: successful fstatvfs initialized the full value. - Ok(unsafe { metadata.assume_init() }) - } else { - Err(std::io::Error::last_os_error()) - } -} - -#[cfg(unix)] -#[allow(unsafe_code)] -fn unsafe_fchmod(fd: std::os::fd::RawFd, mode: libc::mode_t) -> std::io::Result<()> { - // SAFETY: `fd` is borrowed from a live directory `File`; fchmod does not - // retain it, and the supplied mode is a valid permission bitmask. - if unsafe { libc::fchmod(fd, mode) } == 0 { - Ok(()) - } else { - Err(std::io::Error::last_os_error()) - } -} - -#[cfg(unix)] -#[allow(unsafe_code)] -fn unsafe_mkdirat( - parent_fd: std::os::fd::RawFd, - component: &std::ffi::CStr, - mode: libc::mode_t, -) -> std::io::Result<()> { - // SAFETY: `component` is NUL-terminated and lives for the call; `parent_fd` - // is borrowed from an open directory descriptor; mkdirat does not retain - // either argument. - if unsafe { libc::mkdirat(parent_fd, component.as_ptr(), mode) } == 0 { - Ok(()) - } else { - Err(std::io::Error::last_os_error()) - } -} - -#[cfg(unix)] -#[allow(unsafe_code)] -fn unsafe_openat_directory( - parent_fd: std::os::fd::RawFd, - component: &std::ffi::CStr, -) -> std::io::Result { - // SAFETY: `component` is NUL-terminated and lives for the call; `parent_fd` - // is an open directory descriptor. On success openat returns a new owned fd. - let fd = unsafe { - libc::openat( - parent_fd, - component.as_ptr(), - libc::O_RDONLY | libc::O_DIRECTORY | libc::O_NOFOLLOW | libc::O_CLOEXEC, - ) - }; - if fd >= 0 { - Ok(fd) - } else { - Err(std::io::Error::last_os_error()) - } -} - -#[cfg(unix)] -fn validate_runtime_directory_owner( - dir: &Path, - metadata: &libc::stat, - current_euid: u32, -) -> Result<()> { - if metadata.st_mode & libc::S_IFMT != libc::S_IFDIR { - bail!("{} is not a directory", dir.display()); - } - if metadata.st_uid != current_euid { - bail!( - "{} is owned by uid {}, not current euid {}", - dir.display(), - metadata.st_uid, - current_euid - ); - } - Ok(()) -} - -#[cfg(unix)] -#[allow(unsafe_code)] -fn effective_user_id() -> u32 { - // SAFETY: `geteuid` takes no arguments, has no caller-side preconditions, - // and only returns process identity state. - unsafe { libc::geteuid() } -} - -/// Create `dir` (and its parents) restricted to the current user. -/// -/// This is the existing recursive behavior used for the HOME-based fallback, -/// where legitimate symlinked HOME or `.rocm` paths retain normal filesystem -/// semantics. Tier 3 uses [`create_private_tier3_runtime_dir`] instead so its -/// predictable parent is validated separately without following symlinks. -/// -/// The leaf itself is still refused when it is a symlink, and an existing leaf -/// is tightened explicitly because `DirBuilder::mode` only affects directories -/// it creates. -fn create_private_runtime_dir(dir: &Path) -> Result<()> { - #[cfg(unix)] - { - use std::os::unix::fs::{DirBuilderExt, PermissionsExt}; - fs::DirBuilder::new() - .recursive(true) - .mode(0o700) - .create(dir) - .with_context(|| format!("creating {}", dir.display()))?; - if fs::symlink_metadata(dir)?.file_type().is_symlink() { - bail!( - "{} is a symlink, so it may point somewhere another user controls", - dir.display() - ); - } - fs::set_permissions(dir, fs::Permissions::from_mode(0o700)).with_context(|| { - format!( - "restricting {} to mode 0700 — it is most likely owned by another user", - dir.display() - ) - })?; - } - #[cfg(not(unix))] - fs::create_dir_all(dir)?; - Ok(()) -} - -fn lemonade_process_environment_vars( - env: &LemonadeProcessEnvironment, - parent: &ParentRuntimeEnvironment, -) -> Result> { - let mut vars = Vec::new(); - if let Some(rocm_root) = env.rocm_root.as_ref() { - vars.push(("ROCM_PATH", rocm_root.as_os_str().to_owned())); - } - if let Some(runtime_dir) = child_runtime_dir_var(parent)? { - vars.push(runtime_dir); - } - let mut path_entries = env.path_entries.clone(); - if runtime_is_windows() { - path_entries.extend(env.library_entries.iter().cloned()); - } - if let Some(path) = prepend_runtime_paths(&path_entries, std::env::var_os("PATH"))? { - vars.push(("PATH", path)); - } - if runtime_is_linux() - && let Some(ld_library_path) = - prepend_runtime_paths(&env.library_entries, std::env::var_os("LD_LIBRARY_PATH"))? - { - vars.push(("LD_LIBRARY_PATH", ld_library_path)); - } - if let Some(csv) = rocm_engine_protocol::gpu_indices_to_csv(&env.gpu_indices) { - vars.push(("HIP_VISIBLE_DEVICES", OsString::from(csv))); - } - // When `rocm serve` protects a public endpoint, the key arrives via - // `ROCM_SERVE_API_KEY[_FILE]`. Lemonade's server gates /api,/v0,/v1 on - // `LEMONADE_API_KEY`, and its own CLI clients (used here for the readiness - // `status` probe) auto-present the same env var — so translating it once here - // both secures the server and keeps our readiness checks authenticated. - if let Some(api_key) = rocm_engine_protocol::resolve_endpoint_api_key() { - vars.push(("LEMONADE_API_KEY", OsString::from(api_key))); - } - Ok(vars) -} - -fn push_existing_path(paths: &mut Vec, path: PathBuf) { - if !path.exists() || paths.iter().any(|existing| existing == &path) { - return; - } - paths.push(path); -} - -fn wait_for_lemonade_cli_status( - manifest: &LemonadeInstallManifest, - host: &str, - port: u16, - timeout: Duration, - log_path: Option<&Path>, - process_env: &LemonadeProcessEnvironment, -) -> Result<()> { - let start = std::time::Instant::now(); - let mut last_status = None; - // Resolved once: the poll runs twice a second, and resolving also prepares - // the child's runtime directory on disk. - let env_vars = - lemonade_process_environment_vars(process_env, &ParentRuntimeEnvironment::current())?; - while start.elapsed() < timeout { - let mut command = ProcessCommand::new(&manifest.lemonade); - command - .arg("--host") - .arg(host) - .arg("--port") - .arg(port.to_string()) - .arg("status") - .stdin(Stdio::null()) - .stdout(Stdio::null()) - .stderr(Stdio::null()); - apply_environment_vars(&mut command, &env_vars); - hide_child_console_window(&mut command); - match command.status() { - Ok(status) if status.success() => return Ok(()), - Ok(status) => last_status = Some(status.to_string()), - Err(error) => last_status = Some(error.to_string()), - } - std::thread::sleep(Duration::from_millis(500)); - } - let startup_log_summary = summarize_startup_log_tail(log_path, STARTUP_FAILURE_LOG_TAIL_LINES); - bail!( - "Lemonade server did not become ready: {}; {}", - last_status.unwrap_or_else(|| "not checked".to_owned()), - startup_log_summary - ) -} - -fn summarize_startup_log_tail(log_path: Option<&Path>, limit: usize) -> String { - let Some(log_path) = log_path else { - return "no Lemonade startup log path was configured".to_owned(); - }; - if !log_path.is_file() { - return format!("Lemonade startup log not found at {}", log_path.display()); - } - match tail_lines(log_path, limit) { - Ok(lines) if lines.is_empty() => { - format!("Lemonade startup log {} is empty", log_path.display()) - } - Ok(lines) => format!( - "Lemonade startup log tail ({}):\n{}", - log_path.display(), - lines.join("\n") - ), - Err(error) => format!( - "failed to read Lemonade startup log {}: {error}", - log_path.display() - ), - } -} - -fn run_lemonade_backend_install( - manifest: &LemonadeInstallManifest, - host: &str, - port: u16, - backend: &str, - process_env: &LemonadeProcessEnvironment, -) -> Result<()> { - let mut command = ProcessCommand::new(&manifest.lemonade); - command - .arg("--host") - .arg(host) - .arg("--port") - .arg(port.to_string()) - .arg("backends") - .arg("install") - .arg(format!("{LLAMACPP_RECIPE}:{backend}")) - .stdin(Stdio::null()) - .stdout(Stdio::inherit()) - .stderr(Stdio::inherit()); - apply_lemonade_process_environment(&mut command, process_env)?; - hide_child_console_window(&mut command); - let status = command - .status() - .with_context(|| format!("failed to run {}", manifest.lemonade.display()))?; - if !status.success() { - bail!("Lemonade backend install failed with status {status}"); - } - Ok(()) -} - -fn lemonade_pull_args(host: &str, port: u16, checkpoint_ref: &str) -> Vec { - vec![ - "--host".to_owned(), - host.to_owned(), - "--port".to_owned(), - port.to_string(), - "pull".to_owned(), - checkpoint_ref.to_owned(), - ] -} - -/// Download a canonical Hugging Face checkpoint (`owner/repo:variant`) through Lemonade -/// so its GGUF lands in the HF hub cache. Runs non-interactively, so a `:variant` is -/// required — a bare `owner/repo` triggers Lemonade's interactive variant menu, which -/// cannot be answered here. -fn run_lemonade_pull( - manifest: &LemonadeInstallManifest, - host: &str, - port: u16, - checkpoint_ref: &str, - log_path: Option<&Path>, - process_env: &LemonadeProcessEnvironment, -) -> Result<()> { - let mut command = ProcessCommand::new(&manifest.lemonade); - command - .args(lemonade_pull_args(host, port, checkpoint_ref)) - .stdin(Stdio::null()); - if let Some(log_path) = log_path { - let log = fs::OpenOptions::new() - .create(true) - .append(true) - .open(log_path) - .with_context(|| format!("failed to open {}", log_path.display()))?; - command.stdout(Stdio::from(log.try_clone()?)); - command.stderr(Stdio::from(log)); - } else { - command.stdout(Stdio::inherit()).stderr(Stdio::inherit()); - } - apply_lemonade_process_environment(&mut command, process_env)?; - hide_child_console_window(&mut command); - let status = command - .status() - .with_context(|| format!("failed to run {}", manifest.lemonade.display()))?; - if !status.success() { - bail!("Lemonade pull of `{checkpoint_ref}` failed with status {status}"); - } - Ok(()) -} - -#[allow(clippy::too_many_arguments)] -fn run_lemonade_model_load( - manifest: &LemonadeInstallManifest, - host: &str, - port: u16, - model_ref: &str, - backend: &str, - engine_recipe: Option<&EngineRecipeHint>, - log_path: Option<&Path>, - process_env: &LemonadeProcessEnvironment, -) -> Result<()> { - let mut command = ProcessCommand::new(&manifest.lemonade); - command - .args(lemonade_model_load_args( - host, - port, - model_ref, - backend, - engine_recipe, - )) - .stdin(Stdio::null()); - if let Some(log_path) = log_path { - let log = fs::OpenOptions::new() - .create(true) - .append(true) - .open(log_path) - .with_context(|| format!("failed to open {}", log_path.display()))?; - command.stdout(Stdio::from(log.try_clone()?)); - command.stderr(Stdio::from(log)); - } else { - command.stdout(Stdio::inherit()).stderr(Stdio::inherit()); - } - apply_lemonade_process_environment(&mut command, process_env)?; - hide_child_console_window(&mut command); - let status = command - .status() - .with_context(|| format!("failed to run {}", manifest.lemonade.display()))?; - if !status.success() { - bail!("Lemonade model load failed with status {status}"); - } - Ok(()) -} - -/// Run one of Lemonade's packaged `llama-server` binaries (`server`) directly on a -/// GGUF file, exposing the model under `request.model_ref` via `--alias`. This is how -/// a canonical Hugging Face name is served under exactly that name — Lemonade's router -/// renames registered models, but an llama-server alias is a free-form string. -fn serve_direct_llama_server( - request: &ServeHttpRequest, - runtime: &LemonadeRuntime, - process_env: &LemonadeProcessEnvironment, - server: &Path, - log_path: Option<&Path>, - reason: &anyhow::Error, -) -> Result<()> { - let paths = AppPaths::discover()?; - let model_path = direct_llama_model_path(&paths, &request.model_ref).with_context(|| { - format!( - "Lemonade downloaded model `{}` was not found for direct serving", - request.model_ref - ) - })?; - if !server.is_file() { - bail!("Lemonade llama-server is missing at {}", server.display()); - } - let backend = llama_server_backend_label(server); - - if let Some(log_path) = log_path - && let Some(parent) = log_path.parent() - { - fs::create_dir_all(parent)?; - } - if let Some(log_path) = log_path { - let mut log = fs::OpenOptions::new() - .create(true) - .append(true) - .open(log_path) - .with_context(|| format!("failed to open {}", log_path.display()))?; - writeln!( - log, - "\nLaunching Lemonade packaged {backend} llama-server directly: {reason:#}" - ) - .ok(); - } - - let mut direct_env = process_env.clone(); - if let Some(server_dir) = server.parent() { - push_existing_path(&mut direct_env.path_entries, server_dir.to_path_buf()); - push_existing_path(&mut direct_env.library_entries, server_dir.to_path_buf()); - } - - let mut command = ProcessCommand::new(server); - command - .arg("-m") - .arg(&model_path) - .arg("--host") - .arg(&request.host) - .arg("--port") - .arg(request.port.to_string()) - .arg("--n-gpu-layers") - .arg("999") - .arg("--alias") - .arg(&request.model_ref) - .stdin(Stdio::null()); - if let Some(engine_recipe) = request.engine_recipe.as_ref() { - command.args(&engine_recipe.required_flags); - } - // Packaged llama-server is raw llama.cpp — it does not read `LEMONADE_API_KEY`. - // When `rocm serve` protects a public endpoint, hand llama-server the *existing* - // CLI-managed 0600 key file via `--api-key-file` (a path, not the value, so the - // secret never lands in the process table). Reusing the managed file rather than - // writing a copy keeps the key's lifecycle owned by `rocm serve` — created before - // spawn, deleted on stop — with no stale plaintext copy left behind on teardown. - if let Some(key_file) = rocm_engine_protocol::resolve_endpoint_api_key_file() { - command.arg("--api-key-file").arg(&key_file); - } - if let Some(log_path) = log_path { - let log = fs::OpenOptions::new() - .create(true) - .append(true) - .open(log_path) - .with_context(|| format!("failed to open {}", log_path.display()))?; - command.stdout(Stdio::from(log.try_clone()?)); - command.stderr(Stdio::from(log)); - } else { - command.stdout(Stdio::inherit()).stderr(Stdio::inherit()); - } - apply_lemonade_process_environment(&mut command, &direct_env)?; - hide_child_console_window(&mut command); - let mut child = command - .spawn() - .with_context(|| format!("failed to start {}", server.display()))?; - write_running_state( - request, - runtime, - std::process::id(), - Some(child.id()), - "running", - )?; - // If the server never becomes ready (or fails the smoke test), don't leak the child - // or leave the state stuck at `running`: kill it and mark the service failed. - let readiness = (|| -> Result<()> { - wait_for_openai_models_ready( - &request.host, - request.port, - &request.model_ref, - Duration::from_mins(2), - )?; - if !query_chat_smoke_endpoint(&request.host, request.port, &request.model_ref)? { - bail!("Lemonade packaged llama-server did not pass a chat-completion smoke test"); - } - Ok(()) - })(); - if let Err(error) = readiness { - let _ = terminate_pid(child.id(), true); - let _ = child.wait(); - let _ = mark_json_status(&request.state_path, "failed"); - return Err(error); - } - merge_json_state( - &request.state_path, - &json!({ - "status": "ready", - "server_pid": child.id(), - // Identity token for the server PID, captured while the child is alive. - "server_start_ticks": rocm_core::process_start_ticks(child.id()), - "backend_state": "ready", - "backend_requested": backend, - "backend_mode": "lemonade-packaged-llama-server", - "load_response": { - "status": "loaded", - "method": "lemonade-packaged-llama-server", - "model_name": request.model_ref, - "model_path": model_path, - "llamacpp_backend": backend - }, - }), - )?; - let status = child - .wait() - .context("failed waiting for Lemonade packaged llama-server")?; - mark_json_status( - &request.state_path, - if status.success() { - "stopped" - } else { - "failed" - }, - )?; - if status.success() { - Ok(()) - } else { - bail!("Lemonade packaged llama-server exited with status {status}") - } -} - -/// A canonical Hugging Face checkpoint reference: `owner/repo` with an optional -/// `:variant` suffix (a quantization label such as `BF16`/`Q4_0`, or an exact -/// `file.gguf`). -#[derive(Debug, Clone, PartialEq, Eq)] -struct HfCheckpoint { - owner: String, - repo: String, - variant: Option, -} - -/// Parse a model reference as a canonical Hugging Face checkpoint. Returns `None` for -/// anything not of the form `owner/repo[:variant]` — built-in Lemonade aliases and -/// bare registered names all fall through unchanged. -/// -/// Every component is validated against the Hugging Face id / quantization charset -/// (`[A-Za-z0-9._-]`, and never `.`/`..`). This is a security boundary as well as a -/// correctness one: the components are used to build a filesystem path into the model -/// cache, so rejecting path separators and traversal tokens prevents a crafted model -/// reference from escaping the cache directory. -fn parse_hf_checkpoint(model_ref: &str) -> Option { - let trimmed = model_ref.trim(); - if trimmed.is_empty() { - return None; - } - // Split off an optional `:variant` suffix on the first colon; the owner/repo pair - // never contains a colon, so any remainder is the variant. - let (repo_path, variant) = match trimmed.split_once(':') { - Some((path, variant)) => (path, Some(variant)), - None => (trimmed, None), - }; - let (owner, repo) = repo_path.split_once('/')?; - if !is_safe_hf_component(owner) || !is_safe_hf_component(repo) { - return None; - } - if variant.is_some_and(|variant| !is_safe_hf_component(variant)) { - return None; - } - Some(HfCheckpoint { - owner: owner.to_owned(), - repo: repo.to_owned(), - variant: variant.map(str::to_owned), - }) -} - -/// A safe Hugging Face path component: a non-empty run of `[A-Za-z0-9._-]` that is not -/// a `.`/`..` traversal token. Matches the Hugging Face id and quantization-label -/// charset and, by excluding `/`, `\`, and traversal tokens, keeps a user-supplied -/// component from escaping the model cache directory when used to build a path. -fn is_safe_hf_component(component: &str) -> bool { - !component.is_empty() - && component != "." - && component != ".." - && component - .chars() - .all(|c| c.is_ascii_alphanumeric() || matches!(c, '.' | '_' | '-')) -} - -/// Packaged llama.cpp backends whose `llama-server` we can drive directly, best first. -/// GPU backends only — `cpu` is intentionally excluded so the direct-serve path never -/// silently serves on CPU under a GPU-required policy (AGENTS.md §6). An explicit -/// allowlist rather than a directory scan, so the path is built only from constants. -const DIRECT_LLAMA_SERVER_BACKENDS: [&str; 4] = ["rocm-stable", "rocm-nightly", "rocm", "vulkan"]; - -/// Find an installed Lemonade `llama-server` binary under `bin/llamacpp//`, -/// checking known backends in priority order. This lets direct serving work wherever -/// Lemonade installed a backend (e.g. `vulkan` on WSL2), not just the ROCm build. -/// -/// Lemonade's backend installer extracts some llama.cpp releases straight into -/// `/`, but others land one level deeper under a build-numbered directory -/// (e.g. `rocm-stable/llama-b9752/llama-server`), so both layouts are checked. -fn find_llama_server_binary(manifest: &LemonadeInstallManifest) -> Option { - let llamacpp_dir = manifest.runtime_dir.join("bin").join("llamacpp"); - let binary = platform_binary_name("llama-server"); - DIRECT_LLAMA_SERVER_BACKENDS - .into_iter() - .find_map(|backend| find_binary_in(&llamacpp_dir.join(backend), &binary)) -} - -/// Like [`find_llama_server_binary`], but scoped to the single backend Lemonade -/// actually attempted (`manifest.backend_name`, one of `LLAMACPP_BACKEND_PRIORITY`'s -/// `"rocm"` / `"vulkan"`), rather than scanning every known backend directory in -/// priority order. Lemonade reports the generic `"rocm"` — never the more specific -/// `rocm-stable`/`rocm-nightly` — but may extract the actual build into either -/// versioned directory, so `"rocm"` maps to the whole ROCm family; `"vulkan"` never -/// does. This is what lets alignment verification reject a stale ROCm directory left -/// by an earlier attempt when this round actually picked `vulkan` (or the reverse). -fn find_llama_server_binary_for_backend( - manifest: &LemonadeInstallManifest, - backend_name: &str, -) -> Option { - let llamacpp_dir = manifest.runtime_dir.join("bin").join("llamacpp"); - let binary = platform_binary_name("llama-server"); - let family: &[&str] = if backend_name == ROCM_BACKEND_NAME { - &ROCM_LLAMACPP_BACKEND_DIRS - } else { - &["vulkan"] - }; - family - .iter() - .find_map(|backend| find_binary_in(&llamacpp_dir.join(backend), &binary)) -} - -/// The ROCm-family llama.cpp backend directory names under `bin/llamacpp/`. Lemonade's -/// installer extracts each build into its own build-numbered subdirectory rather than -/// replacing one in place, so all three names can accumulate across attempts. -const ROCM_LLAMACPP_BACKEND_DIRS: [&str; 3] = ["rocm-stable", "rocm-nightly", "rocm"]; - -/// Suffix appended to a ROCm-family backend directory moved aside by -/// [`set_aside_rocm_llamacpp_backend_dirs`]. -const BACKEND_DIR_ASIDE_SUFFIX: &str = ".pre-alignment"; - -/// Move every existing ROCm-family llama.cpp backend directory aside (rather than -/// deleting it outright) before a forced alignment reinstall, so a failed install can -/// be recovered from via [`resolve_rocm_llamacpp_backend_dirs_aside`] instead of -/// leaving the host with no ROCm backend at all. Lemonade's installer extracts each -/// build into its own build-numbered subdirectory rather than replacing one in place, -/// so a tier that installs a different `therock.version`/llama.cpp tag than a prior -/// attempt (or the pinned default) can otherwise leave two builds side by side -- -/// after which [`find_binary_in`]'s directory-order fallback, used by both alignment -/// verification and `rocm serve`, may resolve to whichever build a rejected tier -/// installed instead of the one just verified. -/// -/// Returns the `(original, aside)` pairs actually moved; only those need resolving. -fn set_aside_rocm_llamacpp_backend_dirs( - manifest: &LemonadeInstallManifest, -) -> Vec<(PathBuf, PathBuf)> { - let llamacpp_dir = manifest.runtime_dir.join("bin").join("llamacpp"); - ROCM_LLAMACPP_BACKEND_DIRS - .iter() - .filter_map(|backend| { - let original = llamacpp_dir.join(backend); - if !original.exists() { - return None; - } - let aside = llamacpp_dir.join(format!("{backend}{BACKEND_DIR_ASIDE_SUFFIX}")); - // A leftover aside path from an earlier, interrupted attempt must not - // block this rename. - let _ = fs::remove_dir_all(&aside); - match fs::rename(&original, &aside) { - Ok(()) => Some((original, aside)), - Err(error) => { - eprintln!( - "Warning: could not move {} aside before backend alignment: {error:#}", - original.display() - ); - None - } - } - }) - .collect() -} - -/// Resolve the aside directories from [`set_aside_rocm_llamacpp_backend_dirs`]: on -/// success, discard them -- the fresh install replaced what they held. On failure, -/// clear every [`ROCM_LLAMACPP_BACKEND_DIRS`] name (not just the ones that were -/// actually asided) before moving each `aside` back into place: a failed install can -/// land its output under a *different* ROCm-family name than the one that was -/// asided (Lemonade extracts each release into its own build-numbered directory -/// rather than replacing one in place), and [`find_binary_in`]'s priority-ordered -/// lookup would otherwise let that stray, unverified directory shadow the good -/// build this function just restored. Best-effort: a failure here just risks the -/// same stale-directory ambiguity the aside exists to prevent, not the reinstall -/// itself. -fn resolve_rocm_llamacpp_backend_dirs_aside( - manifest: &LemonadeInstallManifest, - aside_dirs: &[(PathBuf, PathBuf)], - success: bool, -) { - if success { - for (_, aside) in aside_dirs { - let _ = fs::remove_dir_all(aside); - } - return; - } - let llamacpp_dir = manifest.runtime_dir.join("bin").join("llamacpp"); - for backend in ROCM_LLAMACPP_BACKEND_DIRS { - let _ = fs::remove_dir_all(llamacpp_dir.join(backend)); - } - for (original, aside) in aside_dirs { - let _ = fs::rename(aside, original); - } -} - -/// Look for `binary` directly in `dir`, then one level down in each subdirectory. -fn find_binary_in(dir: &Path, binary: &str) -> Option { - let direct = dir.join(binary); - if direct.is_file() { - return Some(direct); - } - fs::read_dir(dir) - .ok()? - .filter_map(Result::ok) - .map(|entry| entry.path()) - .filter(|path| path.is_dir()) - .map(|subdir| subdir.join(binary)) - .find(|candidate| candidate.is_file()) -} - -/// The backend label for a packaged llama-server, taken from the nearest ancestor -/// directory matching a known backend name (e.g. `vulkan`, `rocm-stable`) so it still -/// resolves correctly when Lemonade nests the binary under a build-numbered -/// subdirectory; falls back to the ROCm backend name. -fn llama_server_backend_label(server: &Path) -> String { - server - .ancestors() - .filter_map(|dir| dir.file_name()) - .filter_map(|name| name.to_str()) - .find(|name| DIRECT_LLAMA_SERVER_BACKENDS.contains(name)) - .unwrap_or(ROCM_BACKEND_NAME) - .to_owned() -} - -fn direct_llama_model_path(paths: &AppPaths, model_ref: &str) -> Option { - let as_path = PathBuf::from(model_ref); - if as_path.is_file() { - return Some(as_path); - } - if let Some(checkpoint) = parse_hf_checkpoint(model_ref) { - return hf_cache_roots(paths) - .into_iter() - .find_map(|root| find_hf_checkpoint_gguf(&root, &checkpoint)); - } - if resolve_lemonade_model_ref(model_ref) != DEFAULT_MODEL { - return None; - } - hf_cache_roots(paths) - .into_iter() - .find_map(find_default_qwen_gguf) -} - -fn hf_cache_roots(paths: &AppPaths) -> Vec { - hf_cache_roots_from(paths, |name| std::env::var_os(name).map(PathBuf::from)) -} - -fn hf_cache_roots_from(paths: &AppPaths, mut env_path: F) -> Vec -where - F: FnMut(&str) -> Option, -{ - let mut roots = Vec::new(); - if let Some(hub_cache) = env_path("HUGGINGFACE_HUB_CACHE") { - push_hf_cache_root(&mut roots, hub_cache); - } - if let Some(hf_home) = env_path("HF_HOME") { - push_hf_cache_root(&mut roots, hf_home.join("hub")); - } - push_hf_cache_root(&mut roots, paths.cache_dir.join("huggingface").join("hub")); - if let Some(home) = env_path("HOME") { - push_hf_cache_root( - &mut roots, - home.join(".cache").join("huggingface").join("hub"), - ); - } - roots -} - -fn push_hf_cache_root(roots: &mut Vec, path: PathBuf) { - if path.as_os_str().is_empty() || roots.iter().any(|existing| existing == &path) { - return; - } - roots.push(path); -} - -fn find_default_qwen_gguf(cache_root: PathBuf) -> Option { - let snapshots = cache_root.join(DEFAULT_MODEL_REPO_DIR).join("snapshots"); - let entries = fs::read_dir(snapshots).ok()?; - entries - .flatten() - .map(|entry| entry.path().join(DEFAULT_MODEL_GGUF)) - .find(|path| path.is_file()) -} - -/// Locate the downloaded GGUF for a Hugging Face checkpoint inside one hub cache root, -/// selecting the file that matches the requested `:variant` (if any). -fn find_hf_checkpoint_gguf(cache_root: &Path, checkpoint: &HfCheckpoint) -> Option { - select_gguf_for_variant( - collect_hf_checkpoint_ggufs(cache_root, checkpoint), - checkpoint.variant.as_deref(), - ) -} - -/// All cached GGUF files for a checkpoint's repo under one hub cache root (unfiltered -/// by variant). -fn collect_hf_checkpoint_ggufs(cache_root: &Path, checkpoint: &HfCheckpoint) -> Vec { - let repo_dir = format!("models--{}--{}", checkpoint.owner, checkpoint.repo); - let snapshots = cache_root.join(repo_dir).join("snapshots"); - let mut ggufs = Vec::new(); - if let Ok(entries) = fs::read_dir(snapshots) { - for entry in entries.flatten() { - collect_gguf_files(&entry.path(), &mut ggufs, 0); - } - } - ggufs -} - -/// Every cached GGUF matching a checkpoint's variant, across all hub cache roots, -/// deduplicated by file name. Used to detect an ambiguous variant (more than one match) -/// so the caller can refuse to pick arbitrarily. -fn hf_checkpoint_gguf_matches(paths: &AppPaths, checkpoint: &HfCheckpoint) -> Vec { - let mut matches = Vec::new(); - let mut seen_names = std::collections::HashSet::new(); - for root in hf_cache_roots(paths) { - let ggufs = collect_hf_checkpoint_ggufs(&root, checkpoint); - for gguf in filter_ggufs_by_variant(ggufs, checkpoint.variant.as_deref()) { - if let Some(name) = gguf.file_name().and_then(|name| name.to_str()) - && seen_names.insert(name.to_owned()) - { - matches.push(gguf); - } - } - } - matches -} - -/// Collect `*.gguf` files under a snapshot revision directory. GGUFs usually sit at the -/// snapshot root, but sharded variants live one folder down, so recurse a few levels. -fn collect_gguf_files(dir: &Path, out: &mut Vec, depth: usize) { - if depth > 3 { - return; - } - let Ok(entries) = fs::read_dir(dir) else { - return; - }; - for entry in entries.flatten() { - let path = entry.path(); - if path.is_dir() { - collect_gguf_files(&path, out, depth + 1); - } else if path - .extension() - .and_then(|ext| ext.to_str()) - .is_some_and(|ext| ext.eq_ignore_ascii_case("gguf")) - { - out.push(path); - } - } -} - -/// Pick a single GGUF for a requested variant (the first, in sorted order, of the -/// matches). Sorting is by path — deterministic but not by mtime — which is fine -/// because the caller rejects an ambiguous variant before relying on this pick. -fn select_gguf_for_variant(ggufs: Vec, variant: Option<&str>) -> Option { - filter_ggufs_by_variant(ggufs, variant).into_iter().next() -} - -/// All GGUFs matching a requested variant, sorted by path. With no variant, every file -/// matches. With a variant, a case-insensitive exact filename match (`:model.gguf`) -/// wins outright; otherwise every file whose name contains the quantization label -/// (`:BF16`, `:Q4_0`) matches — more than one indicates an ambiguous variant. -fn filter_ggufs_by_variant(mut ggufs: Vec, variant: Option<&str>) -> Vec { - ggufs.sort(); - ggufs.dedup(); - let Some(variant) = variant else { - return ggufs; - }; - let file_name = |path: &Path| -> Option { - path.file_name() - .and_then(|name| name.to_str()) - .map(str::to_owned) - }; - if let Some(exact) = ggufs - .iter() - .find(|path| file_name(path).is_some_and(|name| name.eq_ignore_ascii_case(variant))) - { - return vec![exact.clone()]; - } - let variant_lower = variant.to_ascii_lowercase(); - ggufs - .into_iter() - .filter(|path| { - file_name(path).is_some_and(|name| name.to_ascii_lowercase().contains(&variant_lower)) - }) - .collect() -} - -fn wait_for_openai_models_ready( - host: &str, - port: u16, - model_ref: &str, - timeout: Duration, -) -> Result<()> { - let endpoint = format_http_base_url(host, port); - let endpoint_api_key = rocm_engine_protocol::resolve_endpoint_api_key(); - let start = std::time::Instant::now(); - let mut last_error = None; - while start.elapsed() < timeout { - match http_get_text_with_auth( - &endpoint, - "/v1/models", - endpoint_api_key.as_deref(), - Duration::from_secs(3), - ) - .and_then(|body| parse_models_ready(&body, model_ref)) - { - Ok(true) => return Ok(()), - Ok(false) => last_error = Some("model was not reported by /v1/models".to_owned()), - Err(error) => last_error = Some(error.to_string()), - } - std::thread::sleep(Duration::from_millis(500)); - } - bail!( - "Lemonade packaged llama-server did not become ready: {}", - last_error.unwrap_or_else(|| "not checked".to_owned()) - ) -} - -fn parse_models_ready(body: &str, model_ref: &str) -> Result { - let value = serde_json::from_str::(body.trim()) - .context("failed to parse /v1/models response")?; - Ok(models_payload_has_loaded_model(&value, model_ref)) -} - -fn models_payload_has_loaded_model(value: &Value, model_ref: &str) -> bool { - value - .get("data") - .or_else(|| value.get("models")) - .and_then(Value::as_array) - .is_some_and(|models| { - models.iter().any(|model| { - ["id", "model", "name"] - .into_iter() - .filter_map(|field| model.get(field).and_then(Value::as_str)) - .any(|loaded| model_names_match(loaded, model_ref)) - }) - }) -} - -/// Whether `/v1/models` advertises `model_ref` as ready to serve on GPU. A stock -/// `llama-server` (direct-serve) entry has no `recipe_options` and is accepted by name, -/// since the direct-serve path only runs GPU backends. A Lemonade-router entry carries -/// `recipe_options`, so its `llamacpp_backend` must match — this keeps a merely -/// registered-but-unloaded model (empty `recipe_options`) from reading as ready. -fn models_payload_has_ready_model(value: &Value, model_ref: &str, backend: &str) -> bool { - value - .get("data") - .or_else(|| value.get("models")) - .and_then(Value::as_array) - .is_some_and(|models| { - models.iter().any(|model| { - let name_matches = ["id", "model", "name"] - .into_iter() - .filter_map(|field| model.get(field).and_then(Value::as_str)) - .any(|loaded| model_names_match(loaded, model_ref)); - name_matches && model_reports_ready_backend(model, backend) - }) - }) -} - -/// A `/v1/models` entry is servable on GPU when it carries no `recipe_options` (a stock -/// `llama-server` direct-serve entry — GPU-only by construction) or its reported -/// `llamacpp_backend` matches the expected backend. -fn model_reports_ready_backend(model: &Value, backend: &str) -> bool { - match model.get("recipe_options") { - None => true, - Some(options) => options - .get("llamacpp_backend") - .and_then(Value::as_str) - .is_some_and(|loaded| lemonade_backend_matches(loaded, backend)), - } -} - -fn lemonade_model_load_args( - host: &str, - port: u16, - model_ref: &str, - backend: &str, - engine_recipe: Option<&EngineRecipeHint>, -) -> Vec { - let mut args = vec![ - "--host".to_owned(), - host.to_owned(), - "--port".to_owned(), - port.to_string(), - "load".to_owned(), - model_ref.to_owned(), - "--llamacpp".to_owned(), - backend.to_owned(), - "--save-options".to_owned(), - ]; - if let Some(engine_recipe) = engine_recipe.filter(|recipe| !recipe.required_flags.is_empty()) { - args.extend([ - "--llamacpp-args".to_owned(), - engine_recipe.required_flags.join(" "), - ]); - } - args -} - -fn query_health_json(host: &str, port: u16) -> Result { - let endpoint = format_http_base_url(host, port); - let endpoint_api_key = rocm_engine_protocol::resolve_endpoint_api_key(); - let body = http_get_text_with_auth( - &endpoint, - "/v1/health", - endpoint_api_key.as_deref(), - Duration::from_secs(3), - ) - .with_context(|| format!("failed to query Lemonade health at {endpoint}/v1/health"))?; - serde_json::from_str(&body).context("failed to parse Lemonade health JSON") -} - -fn query_loaded_model_endpoint(endpoint_url: &str, model_ref: &str, backend: &str) -> Result { - let (host, port) = parse_http_endpoint(endpoint_url) - .with_context(|| format!("unsupported endpoint URL `{endpoint_url}`"))?; - // Lemonade's router reports readiness via `/v1/health` (`all_models_loaded`). The - // direct-serve path runs a stock `llama-server` that has no such field, and instead - // advertises the model in `/v1/models`; fall back to that (name + backend) so both - // serving modes are recognized as ready. - if let Ok(health) = query_health_json(&host, port) - && health_has_loaded_model(&health, model_ref, backend) - { - return Ok(true); - } - let endpoint = format_http_base_url(&host, port); - let endpoint_api_key = rocm_engine_protocol::resolve_endpoint_api_key(); - let body = http_get_text_with_auth( - &endpoint, - "/v1/models", - endpoint_api_key.as_deref(), - Duration::from_secs(3), - ) - .with_context(|| format!("failed to query Lemonade models at {endpoint}/v1/models"))?; - let models = - serde_json::from_str::(&body).context("failed to parse Lemonade /v1/models JSON")?; - Ok(models_payload_has_ready_model(&models, model_ref, backend)) -} - -/// Post-load smoke test: the freshly loaded model must actually complete a chat -/// request. Deliberately stricter than the readiness probe — only a `200` counts, -/// so a refusal (wrong model name, unsupported request) fails the serve instead -/// of being reported as a working service. -fn query_chat_smoke_endpoint(host: &str, port: u16, model_ref: &str) -> Result { - let status = rocm_core::openai_chat_completion_status( - &format_http_base_url(host, port), - model_ref, - rocm_engine_protocol::resolve_endpoint_api_key().as_deref(), - rocm_core::INFERENCE_PROBE_TIMEOUT, - )?; - Ok(status == 200) -} - -fn health_has_loaded_model(health: &Value, model_ref: &str, backend: &str) -> bool { - let model_ref = model_ref.trim(); - if model_ref.is_empty() { - return false; - } - health - .get("all_models_loaded") - .and_then(Value::as_array) - .into_iter() - .flatten() - .any(|model| { - let name_matches = ["model_name", "id", "name"] - .into_iter() - .filter_map(|field| model.get(field).and_then(Value::as_str)) - .any(|loaded| model_names_match(loaded, model_ref)); - let backend_matches = model - .get("recipe_options") - .and_then(|options| options.get("llamacpp_backend")) - .and_then(Value::as_str) - .is_some_and(|loaded| lemonade_backend_matches(loaded, backend)); - name_matches && backend_matches - }) -} - -fn model_names_match(left: &str, right: &str) -> bool { - left.eq_ignore_ascii_case(right) - || resolve_lemonade_model_ref(left).eq_ignore_ascii_case(&resolve_lemonade_model_ref(right)) -} - -fn lemonade_backend_matches(value: &str, backend: &str) -> bool { - value - .trim() - .to_ascii_lowercase() - .starts_with(&backend.to_ascii_lowercase()) -} - -fn serve_http_command_args(request: &ServeHttpRequest) -> Vec { - let mut args = vec![ - "serve-http".to_owned(), - request.service_id.clone(), - request.model_ref.clone(), - "--host".to_owned(), - request.host.clone(), - "--port".to_owned(), - request.port.to_string(), - "--device-policy".to_owned(), - device_policy_name(&request.device_policy).to_owned(), - "--state-path".to_owned(), - request.state_path.display().to_string(), - ]; - if let Some(runtime_id) = request.runtime_id.as_deref() { - args.extend(["--runtime-id".to_owned(), runtime_id.to_owned()]); - } - if let Some(env_id) = request.env_id.as_deref() { - args.extend(["--env-id".to_owned(), env_id.to_owned()]); - } - if let Some(log_path) = request.log_path.as_ref() { - args.extend(["--log-path".to_owned(), log_path.display().to_string()]); - } - if let Some(csv) = rocm_engine_protocol::gpu_indices_to_csv(&request.gpu_indices) { - args.extend(["--gpu".to_owned(), csv]); - } - if let Some(engine_recipe) = request.engine_recipe.as_ref() { - args.extend([ - "--engine-recipe-json".to_owned(), - serde_json::to_string(engine_recipe).expect("engine recipe serializes"), - ]); - } - args -} - -#[cfg(windows)] -fn spawn_serve_http_background(current_exe: &Path, serve_args: &[String]) -> Result { - rocm_core::spawn_detached_no_inherit(current_exe, serve_args, &[]) - .context("failed to launch Lemonade serve-http background process") -} - -#[cfg(not(windows))] -fn spawn_serve_http_background(current_exe: &Path, serve_args: &[String]) -> Result { - let child = ProcessCommand::new(current_exe) - .args(serve_args) - .stdin(Stdio::null()) - .stdout(Stdio::null()) - .stderr(Stdio::null()) - .spawn() - .context("failed to launch Lemonade serve-http background process")?; - Ok(child.id()) -} - -fn write_running_state( - request: &ServeHttpRequest, - runtime: &LemonadeRuntime, - pid: u32, - server_pid: Option, - status: &str, -) -> Result<()> { - write_state( - &request.state_path, - &json!({ - "service_id": request.service_id, - "engine": ENGINE_NAME, - "status": status, - "pid": pid, - "server_pid": server_pid, - "model_ref": request.model_ref, - "canonical_model_id": request.model_ref, - "host": request.host, - "port": request.port, - "endpoint_url": endpoint_url(&request.host, request.port), - "device_policy": device_policy_name(&request.device_policy), - "gpu_indices": request.gpu_indices, - "runtime_id": request.runtime_id.as_deref().unwrap_or(runtime.manifest.env_id.as_str()), - "env_id": request.env_id.as_deref().unwrap_or(runtime.manifest.env_id.as_str()), - "runtime_kind": "lemonade_embeddable", - "runtime_executable": runtime.manifest.lemond, - "log_path": request.log_path.as_ref().map(|path| path.display().to_string()), - "engine_recipe": request.engine_recipe, - "started_at_unix_ms": current_unix_millis(), - // Kernel start-times captured while each PID is alive. Paired with the - // PIDs, they identify these exact processes across PID recycling so a - // later stop never signals a reused PID. - "start_ticks": rocm_core::process_start_ticks(pid), - "server_start_ticks": server_pid.and_then(rocm_core::process_start_ticks) - }), - ) -} - -fn service_files(service_id: &str) -> Result { - let paths = AppPaths::discover()?; - Ok(ServiceFiles { - state_path: paths - .engine_state_dir(ENGINE_NAME) - .join(format!("{service_id}.json")), - log_path: paths - .engine_logs_dir(ENGINE_NAME) - .join(format!("{service_id}.log")), - }) -} - -fn endpoint_url(host: &str, port: u16) -> String { - format!("{}/v1", format_http_base_url(host, port)) -} - -fn endpoint_url_from_state(state: &Value) -> Option { - value_string(state, "endpoint_url").or_else(|| { - let host = value_string(state, "host")?; - let port = value_u32(state, "port")?; - let port = u16::try_from(port).ok()?; - Some(endpoint_url(&host, port)) - }) -} - -fn read_service_state(path: &Path) -> Result { - let text = - fs::read_to_string(path).with_context(|| format!("failed to read {}", path.display()))?; - serde_json::from_str(&text).with_context(|| format!("failed to parse {}", path.display())) -} - -fn write_state(path: &Path, value: &Value) -> Result<()> { - if let Some(parent) = path.parent() { - fs::create_dir_all(parent)?; - } - fs::write(path, serde_json::to_vec_pretty(value)?) - .with_context(|| format!("failed to write {}", path.display())) -} - -fn merge_json_state(path: &Path, patch: &Value) -> Result<()> { - let mut value = read_service_state(path).unwrap_or_else(|_| json!({})); - if !value.is_object() { - value = json!({}); - } - let object = value.as_object_mut().expect("object checked above"); - if let Some(patch) = patch.as_object() { - for (key, value) in patch { - object.insert(key.clone(), value.clone()); - } - } - write_state(path, &value) -} - -fn mark_json_status(path: &Path, status: &str) -> Result<()> { - merge_json_state( - path, - &json!({ - "engine": ENGINE_NAME, - "status": status, - "stopped_at_unix_ms": current_unix_millis(), - }), - ) -} - -fn value_string(value: &Value, key: &str) -> Option { - value - .get(key) - .and_then(Value::as_str) - .filter(|value| !value.trim().is_empty()) - .map(ToOwned::to_owned) -} - -fn value_u32(value: &Value, key: &str) -> Option { - value - .get(key) - .and_then(Value::as_u64) - .and_then(|value| u32::try_from(value).ok()) -} - -/// Reconstruct the identity (PID + kernel start-time) of the process a stop -/// should terminate. The server PID is preferred, falling back to the launcher -/// PID, each paired with its recorded start-time. `start_ticks` is absent in -/// pre-existing state files, in which case verification degrades to best-effort. -fn identity_from_state(state: &Value) -> Option { - if let Some(server_pid) = value_u32(state, "server_pid") { - let start_ticks = state.get("server_start_ticks").and_then(Value::as_u64); - return Some(rocm_core::ProcessIdentity::new(server_pid, start_ticks)); - } - let pid = value_u32(state, "pid")?; - let start_ticks = state.get("start_ticks").and_then(Value::as_u64); - Some(rocm_core::ProcessIdentity::new(pid, start_ticks)) -} - -fn tail_lines(path: &Path, limit: usize) -> Result> { - if limit == 0 { - return Ok(Vec::new()); - } - let file = - fs::File::open(path).with_context(|| format!("failed to open {}", path.display()))?; - let metadata = file - .metadata() - .with_context(|| format!("failed to stat {}", path.display()))?; - let file_size = metadata.len(); - - // For small files, read normally to preserve exact line count. - if file_size <= MAX_TAIL_READ { - let reader = std::io::BufReader::new(file); - let mut lines = VecDeque::with_capacity(limit); - for line in reader.lines() { - let line = line.with_context(|| format!("failed to read {}", path.display()))?; - if lines.len() == limit { - lines.pop_front(); - } - lines.push_back(line); - } - return Ok(lines.into_iter().collect()); - } - - // For large files, seek near the end and read only the final chunk. - // This prevents reading multi-gigabyte logs during timeout errors. - let mut file = file; - let seek_pos = file_size.saturating_sub(MAX_TAIL_READ); - file.seek(std::io::SeekFrom::Start(seek_pos)) - .with_context(|| format!("failed to seek in {}", path.display()))?; - - let reader = std::io::BufReader::new(file); - let mut lines = VecDeque::with_capacity(limit); - let mut skipped_first = seek_pos == 0; - for line in reader.lines() { - let line = line.with_context(|| format!("failed to read {}", path.display()))?; - // Skip the first line after seeking, as it may be partial. - if !skipped_first { - skipped_first = true; - continue; - } - if lines.len() == limit { - lines.pop_front(); - } - lines.push_back(line); - } - Ok(lines.into_iter().collect()) -} - -fn terminate_pid(pid: u32, _force: bool) -> bool { - rocm_core::terminate_process(pid).is_ok() -} - -fn free_local_port() -> Result { - let listener = TcpListener::bind((DEFAULT_HOST, 0)).context("failed to reserve local port")?; - Ok(listener.local_addr()?.port()) -} - -fn parse_http_endpoint(endpoint_url: &str) -> Option<(String, u16)> { - let without_scheme = endpoint_url.trim().strip_prefix("http://")?; - let authority = without_scheme.split('/').next()?.trim(); - if let Some(rest) = authority.strip_prefix('[') { - let end = rest.find(']')?; - let host = rest[..end].to_owned(); - let port = rest[end + 1..].strip_prefix(':')?.parse().ok()?; - return Some((host, port)); - } - let (host, port) = authority.rsplit_once(':')?; - Some((host.to_owned(), port.parse().ok()?)) -} - -fn normalize_device_policy(policy: Option) -> Result { - match policy.unwrap_or(DevicePolicy::GpuRequired) { - DevicePolicy::GpuRequired | DevicePolicy::GpuPreferred => Ok(DevicePolicy::GpuRequired), - DevicePolicy::CpuOnly => { - bail!("Lemonade adapter requires ROCm GPU execution; no CPU fallback is used") - } - } -} - -fn require_gpu_required(policy: &DevicePolicy) -> Result<()> { - match policy { - DevicePolicy::GpuRequired | DevicePolicy::GpuPreferred => Ok(()), - DevicePolicy::CpuOnly => { - bail!("Lemonade adapter requires ROCm GPU execution; no CPU fallback is used") - } - } -} - -/// Probe real GPU availability and resolve the device ordinal(s) to pin before a -/// GPU-required launch. Returns the resolved indices, or an error that stops -/// serving before any process is spawned: -/// - no usable device → fail with actionable guidance (no CPU/GPU-0 fallback); -/// - `auto` (empty `requested`) → the first present, visible device; -/// - explicit index not among the usable devices → fail. -/// -/// When availability cannot be probed on this platform, the requested selection -/// is passed through unchanged so serving is not blocked on that basis. -fn resolve_serve_gpu_indices(requested: &[u32]) -> Result> { - resolve_gpu_indices_against(requested, rocm_core::usable_amd_gpu_indices()) -} - -/// Pure resolution used by [`resolve_serve_gpu_indices`], split out so the -/// auto/explicit/no-device policy can be unit-tested without real hardware. -/// `usable` is the probe result: `None` (unprobeable) passes the request -/// through; `Some(_)` is authoritative. -fn resolve_gpu_indices_against(requested: &[u32], usable: Option>) -> Result> { - let Some(usable) = usable else { - return Ok(requested.to_vec()); - }; - if usable.is_empty() { - bail!( - "no usable AMD GPU detected; `rocm serve` requires a GPU under the default \ - GPU-required policy and does not fall back to CPU. Check the driver with \ - `rocm examine`, confirm /dev/kfd is present, and ensure HIP_VISIBLE_DEVICES / \ - ROCR_VISIBLE_DEVICES are not masking every device." - ); - } - if requested.is_empty() { - return Ok(vec![usable[0]]); - } - for index in requested { - if !usable.contains(index) { - let visible = usable - .iter() - .map(u32::to_string) - .collect::>() - .join(", "); - bail!( - "requested GPU {index} is not available on this host; usable GPU indices: [{visible}]" - ); - } - } - Ok(requested.to_vec()) -} - -/// GPU device availability for `detect`, sourced from the real device probe. When -/// the probe is authoritative it reports true/false with a reason; when it cannot -/// determine availability on this platform, it falls back to whether the runtime -/// is installed so `detect` stays informative. -fn gpu_availability_device(runtime_installed: bool) -> EngineDeviceAvailability { - match rocm_core::usable_amd_gpu_indices() { - Some(indices) if !indices.is_empty() => EngineDeviceAvailability { - kind: "rocm_gpu".to_owned(), - available: true, - reason: None, - }, - Some(_) => EngineDeviceAvailability { - kind: "rocm_gpu".to_owned(), - available: false, - reason: Some( - "no AMD GPU detected, or every device is masked by \ - HIP_VISIBLE_DEVICES / ROCR_VISIBLE_DEVICES" - .to_owned(), - ), - }, - None => EngineDeviceAvailability { - kind: "rocm_gpu".to_owned(), - available: runtime_installed, - reason: if runtime_installed { - None - } else { - Some("Lemonade embeddable runtime is not installed".to_owned()) - }, - }, - } -} - -fn parse_device_policy_arg(value: Option<&str>) -> Result { - match value.unwrap_or("gpu_required") { - "gpu" | "gpu_required" | "gpu_preferred" => Ok(DevicePolicy::GpuRequired), - "cpu" | "cpu_only" => Ok(DevicePolicy::CpuOnly), - other => bail!("unknown device policy `{other}`"), - } -} - -/// Parse a `--gpu` CLI value into an optional `GpuSelection` for `LaunchRequest`. -fn parse_gpu_selection_arg(value: Option<&str>) -> Result> { - value - .map(|raw| GpuSelection::parse_cli_value(raw).map_err(anyhow::Error::msg)) - .transpose() -} - -/// Parse a `--gpu` CLI value into explicit device ordinals (empty for `auto`). -fn parse_gpu_indices_arg(value: Option<&str>) -> Result> { - Ok(rocm_engine_protocol::launch_gpu_indices( - parse_gpu_selection_arg(value)?.as_ref(), - )) -} - -const fn device_policy_name(policy: &DevicePolicy) -> &'static str { - match policy { - DevicePolicy::GpuRequired => "gpu_required", - DevicePolicy::GpuPreferred => "gpu_preferred", - DevicePolicy::CpuOnly => "cpu_only", - } -} - -fn accepted_engine_recipe( - engine_recipe: Option, -) -> Result> { - if let Some(hint) = &engine_recipe { - if hint.engine != ENGINE_NAME { - bail!( - "engine_recipe target `{}` does not match adapter `{}`", - hint.engine, - ENGINE_NAME - ); - } - if hint.contract_version != ENGINE_RECIPE_CONTRACT_VERSION { - bail!( - "engine_recipe contract `{}` is unsupported; expected `{}`", - hint.contract_version, - ENGINE_RECIPE_CONTRACT_VERSION - ); - } - } - Ok(engine_recipe) -} - -fn parse_engine_recipe_json(value: Option) -> Result> { - value - .map(|text| { - serde_json::from_str::(&text) - .context("failed to parse engine recipe JSON") - }) - .transpose() - .and_then(accepted_engine_recipe) -} - -fn resolve_lemonade_model_ref(model_ref: &str) -> String { - let trimmed = model_ref.trim(); - let lower = trimmed.to_ascii_lowercase(); - // Only exact, recognized shorthand aliases map to the bundled assistant GGUF. - // A syntactically valid `owner/repo` Hugging Face checkpoint must never be - // silently rewritten here — e.g. `Qwen/Qwen2.5-1.5B-Instruct` is a real - // checkpoint id, not a shorthand, and is passed through unchanged so the - // serve path can honour (or explicitly reject) it rather than substituting - // an unrelated model behind the user's back. - if trimmed.is_empty() - || matches!( - lower.as_str(), - "qwen" - | "assistant" - | "default" - | "small" - | "lemonade-qwen" - | "qwen-gguf" - | "qwen3-4b" - | "qwen3-4b-instruct" - | "qwen3-4b-instruct-2507-gguf" - ) - { - DEFAULT_MODEL.to_owned() - } else if matches!( - lower.as_str(), - "tiny" | "qwen-smoke" | "lemonade-tiny" | "qwen3-0.6b-gguf" - ) { - "Qwen3-0.6B-GGUF".to_owned() - } else { - trimmed.to_owned() - } -} - -fn manifest_lock_hash(manifest: &LemonadeInstallManifest) -> String { - let mut hasher = DefaultHasher::new(); - manifest.env_id.hash(&mut hasher); - manifest.version.hash(&mut hasher); - manifest.runtime_dir.hash(&mut hasher); - manifest.lemond.hash(&mut hasher); - format!("{:016x}", hasher.finish()) -} - -fn current_unix_millis() -> u128 { - SystemTime::now() - .duration_since(UNIX_EPOCH) - .unwrap_or_default() - .as_millis() -} - -fn read_request() -> Result { - let mut buffer = String::new(); - std::io::stdin() - .read_to_string(&mut buffer) - .context("failed to read stdin for engine request")?; - serde_json::from_str(&buffer).context("failed to parse engine request envelope") -} - -fn print_json(value: &T) -> Result<()> { - let stdout = std::io::stdout(); - let mut handle = stdout.lock(); - serde_json::to_writer_pretty(&mut handle, value)?; - writeln!(&mut handle)?; - Ok(()) -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn stop_scope_targets_only_the_recorded_lemonade_server() { - assert_eq!(STOP_SCOPE, rocm_core::KillScope::Single); - } - - #[test] - fn identity_from_state_prefers_server_pid_and_its_start_ticks() { - let state = json!({ - "pid": 100, - "start_ticks": 111_u64, - "server_pid": 200, - "server_start_ticks": 222_u64, - }); - let identity = identity_from_state(&state).expect("identity"); - assert_eq!(identity.pid, 200); - assert_eq!(identity.start_ticks, Some(222)); - } - - #[test] - fn identity_from_state_falls_back_to_launcher_pid() { - let state = json!({ "pid": 100, "start_ticks": 111_u64 }); - let identity = identity_from_state(&state).expect("identity"); - assert_eq!(identity.pid, 100); - assert_eq!(identity.start_ticks, Some(111)); - } - - #[test] - fn identity_from_legacy_state_has_no_start_ticks() { - // Pre-existing state files carry only PIDs; verification must degrade. - let state = json!({ "server_pid": 200 }); - let identity = identity_from_state(&state).expect("identity"); - assert_eq!(identity.pid, 200); - assert_eq!(identity.start_ticks, None); - } - - #[test] - fn identity_from_state_without_any_pid_is_none() { - assert!(identity_from_state(&json!({ "status": "running" })).is_none()); - } - - #[test] - fn qwen_alias_resolves_to_validated_assistant_gguf_model() { - assert_eq!(resolve_lemonade_model_ref("qwen"), DEFAULT_MODEL); - assert_eq!(resolve_lemonade_model_ref("qwen-smoke"), "Qwen3-0.6B-GGUF"); - } - - #[test] - fn canonical_hugging_face_name_passes_through_unchanged() { - assert_eq!( - resolve_lemonade_model_ref("LiquidAI/LFM2.5-230M-GGUF:Q4_0"), - "LiquidAI/LFM2.5-230M-GGUF:Q4_0" - ); - // Regression (EAI-7370): a fully-qualified checkpoint id that merely - // contains a known-alias substring must not be swapped for the bundled - // default. `Qwen/Qwen2.5-1.5B-Instruct` was silently served as - // `Qwen3-4B-Instruct-2507-GGUF`; it must now pass through verbatim. - assert_eq!( - resolve_lemonade_model_ref("Qwen/Qwen2.5-1.5B-Instruct"), - "Qwen/Qwen2.5-1.5B-Instruct" - ); - } - - #[test] - fn parses_canonical_hugging_face_checkpoints() { - assert_eq!( - parse_hf_checkpoint("LiquidAI/LFM2.5-230M-GGUF:Q4_0"), - Some(HfCheckpoint { - owner: "LiquidAI".to_owned(), - repo: "LFM2.5-230M-GGUF".to_owned(), - variant: Some("Q4_0".to_owned()), - }) - ); - assert_eq!( - parse_hf_checkpoint("unsloth/Qwen3-0.6B-GGUF"), - Some(HfCheckpoint { - owner: "unsloth".to_owned(), - repo: "Qwen3-0.6B-GGUF".to_owned(), - variant: None, - }) - ); - } - - #[test] - fn rejects_non_checkpoint_model_refs() { - // Built-in aliases and bare registered names have no owner/repo pair. - assert!(parse_hf_checkpoint("qwen").is_none()); - assert!(parse_hf_checkpoint(DEFAULT_MODEL).is_none()); - assert!(parse_hf_checkpoint("").is_none()); - assert!(parse_hf_checkpoint("/model.gguf").is_none()); - assert!(parse_hf_checkpoint("owner/").is_none()); - assert!(parse_hf_checkpoint("owner/repo/extra").is_none()); - assert!(parse_hf_checkpoint("owner/repo:").is_none()); - } - - #[test] - fn rejects_path_traversal_in_checkpoint_components() { - // A crafted reference must not be able to escape the model cache directory. - assert!(parse_hf_checkpoint("../../etc/passwd").is_none()); - assert!(parse_hf_checkpoint("..:Q4_0").is_none()); - assert!(parse_hf_checkpoint("owner/..").is_none()); - assert!(parse_hf_checkpoint("owner/repo:../secret").is_none()); - assert!(parse_hf_checkpoint(r"owner\..\..--x/repo").is_none()); - assert!(parse_hf_checkpoint("owner/repo:a/b.gguf").is_none()); - } - - #[test] - fn selects_gguf_by_variant() { - // Real filenames use the quant label without the `-GGUF` repo suffix, so a - // substring match on the `:variant` selects the right file. - let ggufs = vec![ - PathBuf::from("/cache/LFM2.5-230M-Q4_0.gguf"), - PathBuf::from("/cache/LFM2.5-230M-Q8_0.gguf"), - ]; - assert_eq!( - select_gguf_for_variant(ggufs.clone(), Some("q4_0")), - Some(PathBuf::from("/cache/LFM2.5-230M-Q4_0.gguf")) - ); - assert_eq!( - select_gguf_for_variant(ggufs.clone(), Some("LFM2.5-230M-Q8_0.gguf")), - Some(PathBuf::from("/cache/LFM2.5-230M-Q8_0.gguf")) - ); - assert_eq!( - select_gguf_for_variant(ggufs.clone(), None), - Some(PathBuf::from("/cache/LFM2.5-230M-Q4_0.gguf")) - ); - assert_eq!(select_gguf_for_variant(ggufs, Some("Q2_K")), None); - } - - #[test] - fn ambiguous_variant_matches_multiple_ggufs() { - // A partial label like `Q4` matches several quants; the caller must refuse to - // pick. A full label or exact filename resolves to exactly one. - let ggufs = vec![ - PathBuf::from("/cache/LFM2.5-230M-Q4_0.gguf"), - PathBuf::from("/cache/LFM2.5-230M-Q4_K_M.gguf"), - ]; - assert_eq!(filter_ggufs_by_variant(ggufs.clone(), Some("Q4")).len(), 2); - assert_eq!( - filter_ggufs_by_variant(ggufs.clone(), Some("Q4_0")).len(), - 1 - ); - assert_eq!( - filter_ggufs_by_variant(ggufs, Some("LFM2.5-230M-Q4_K_M.gguf")).len(), - 1 - ); - } - - #[test] - fn models_ready_accepts_direct_serve_and_gates_router_backend() { - // Direct-serve (stock llama-server): no `recipe_options` → ready by name. - let direct = json!({"data": [{"id": "LiquidAI/LFM2.5-230M-GGUF:Q4_0"}]}); - assert!(models_payload_has_ready_model( - &direct, - "LiquidAI/LFM2.5-230M-GGUF:Q4_0", - "vulkan" - )); - // Router entry loaded on the matching backend → ready. - let loaded = json!({"data": [{ - "id": "Qwen3-0.6B-GGUF", - "recipe_options": {"llamacpp_backend": "vulkan"} - }]}); - assert!(models_payload_has_ready_model( - &loaded, - "Qwen3-0.6B-GGUF", - "vulkan" - )); - // Registered but not loaded (empty `recipe_options`) → not ready. - let registered = json!({"data": [{ - "id": "Qwen3-0.6B-GGUF", - "recipe_options": {} - }]}); - assert!(!models_payload_has_ready_model( - ®istered, - "Qwen3-0.6B-GGUF", - "vulkan" - )); - } - - #[test] - fn find_llama_server_binary_prefers_gpu_and_skips_cpu() { - let dir = scratch_dir("find-server"); - let runtime_dir = dir.join("runtime"); - let llamacpp = runtime_dir.join("bin").join("llamacpp"); - for backend in ["cpu", "vulkan"] { - let backend_dir = llamacpp.join(backend); - fs::create_dir_all(&backend_dir).unwrap(); - fs::write(backend_dir.join(platform_binary_name("llama-server")), b"x").unwrap(); - } - let manifest = test_manifest(runtime_dir); - // Both cpu and vulkan exist; the GPU backend is chosen and cpu is never selected. - assert_eq!( - find_llama_server_binary(&manifest), - Some( - llamacpp - .join("vulkan") - .join(platform_binary_name("llama-server")) - ) - ); - // With only cpu present, nothing is returned (no silent CPU fallback). - fs::remove_dir_all(llamacpp.join("vulkan")).unwrap(); - assert_eq!(find_llama_server_binary(&manifest), None); - fs::remove_dir_all(&dir).ok(); - } - - #[test] - fn find_llama_server_binary_finds_build_numbered_nesting() { - // Some Lemonade llama.cpp releases extract one level deeper than - // `/llama-server`, e.g. `rocm-stable/llama-b9752/llama-server`. - let dir = scratch_dir("find-server-nested"); - let runtime_dir = dir.join("runtime"); - let llamacpp = runtime_dir.join("bin").join("llamacpp"); - let nested = llamacpp.join("rocm-stable").join("llama-b9752"); - fs::create_dir_all(&nested).unwrap(); - let server = nested.join(platform_binary_name("llama-server")); - fs::write(&server, b"x").unwrap(); - let manifest = test_manifest(runtime_dir); - assert_eq!(find_llama_server_binary(&manifest), Some(server.clone())); - assert_eq!(llama_server_backend_label(&server), "rocm-stable"); - fs::remove_dir_all(&dir).ok(); - } - - #[test] - fn find_llama_server_binary_follows_backend_priority() { - let dir = scratch_dir("find-server-priority"); - let runtime_dir = dir.join("runtime"); - let llamacpp = runtime_dir.join("bin").join("llamacpp"); - let server = platform_binary_name("llama-server"); - let install = |backend: &str| { - let backend_dir = llamacpp.join(backend); - fs::create_dir_all(&backend_dir).unwrap(); - fs::write(backend_dir.join(&server), b"x").unwrap(); - }; - let manifest = test_manifest(runtime_dir); - - // Nightly-only host (the resilient case that replaced the hardcoded - // rocm-stable direct-serve path): the nightly backend is selected over vulkan. - install("rocm-nightly"); - install("vulkan"); - assert_eq!( - find_llama_server_binary(&manifest), - Some(llamacpp.join("rocm-nightly").join(&server)) - ); - - // With rocm-stable also present, it wins (highest priority). - install("rocm-stable"); - assert_eq!( - find_llama_server_binary(&manifest), - Some(llamacpp.join("rocm-stable").join(&server)) - ); - fs::remove_dir_all(&dir).ok(); - } - - #[test] - fn collect_gguf_files_recurses_and_filters() { - let dir = scratch_dir("collect-gguf"); - fs::create_dir_all(dir.join("shard")).unwrap(); - fs::write(dir.join("model-Q4_0.gguf"), b"x").unwrap(); - fs::write(dir.join("notes.txt"), b"x").unwrap(); - fs::write(dir.join("shard").join("model-Q8_0.gguf"), b"x").unwrap(); - let mut found = Vec::new(); - collect_gguf_files(&dir, &mut found, 0); - let mut names = found - .iter() - .filter_map(|path| path.file_name().and_then(|name| name.to_str())) - .map(str::to_owned) - .collect::>(); - names.sort(); - assert_eq!(names, vec!["model-Q4_0.gguf", "model-Q8_0.gguf"]); - fs::remove_dir_all(&dir).ok(); - } - - /// Answer `count` chat requests on a loopback port with the given status, - /// recording how many arrived. - fn spawn_chat_endpoint( - status_line: &'static str, - count: usize, - ) -> (u16, std::thread::JoinHandle) { - let listener = TcpListener::bind("127.0.0.1:0").expect("bind loopback"); - let port = listener.local_addr().expect("local addr").port(); - let handle = std::thread::spawn(move || { - let mut served = 0; - for _ in 0..count { - let Ok((mut stream, _)) = listener.accept() else { - break; - }; - stream.set_read_timeout(Some(Duration::from_secs(2))).ok(); - let mut buffer = [0_u8; 1024]; - let _ = stream.read(&mut buffer); - let body = r#"{"choices":[{"message":{"content":"ok"}}]}"#; - let _ = write!( - stream, - "{status_line}\r\nContent-Type: application/json\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{}", - body.len(), - body - ); - served += 1; - } - served - }); - (port, handle) - } - - #[test] - fn inference_verification_latches_into_the_state_file() { - // First check probes and records the verdict; the second reads the latch - // and leaves the model alone. - let (port, server) = spawn_chat_endpoint("HTTP/1.1 200 OK", 1); - let dir = scratch_dir("inference-latch"); - let state_path = dir.join("state.json"); - fs::write(&state_path, json!({"status": "ready"}).to_string()).expect("seed state"); - let endpoint = format_http_base_url("127.0.0.1", port); - - let state = read_service_state(&state_path).ok(); - assert!(inference_verified( - &state_path, - state.as_ref(), - &endpoint, - DEFAULT_MODEL - )); - - let state = read_service_state(&state_path).expect("state readable"); - assert!( - state - .get(rocm_core::INFERENCE_VERIFIED_STATE_KEY) - .and_then(Value::as_u64) - .is_some(), - "a passing probe is latched so later healthchecks skip it" - ); - assert!(inference_verified( - &state_path, - Some(&state), - &endpoint, - DEFAULT_MODEL - )); - - assert_eq!( - server.join().expect("server thread"), - 1, - "the latched check must not send a second inference request" - ); - fs::remove_dir_all(&dir).ok(); - } - - #[test] - fn inference_verification_withheld_while_the_endpoint_cannot_serve() { - // The reported failure: the model is listed but inference still 5xxs. - let (port, server) = spawn_chat_endpoint("HTTP/1.1 503 Service Unavailable", 1); - let dir = scratch_dir("inference-unverified"); - let state_path = dir.join("state.json"); - fs::write(&state_path, json!({"status": "ready"}).to_string()).expect("seed state"); - let endpoint = format_http_base_url("127.0.0.1", port); - - let state = read_service_state(&state_path).ok(); - assert!(!inference_verified( - &state_path, - state.as_ref(), - &endpoint, - DEFAULT_MODEL - )); - - let state = read_service_state(&state_path).expect("state readable"); - assert!( - state.get(rocm_core::INFERENCE_VERIFIED_STATE_KEY).is_none(), - "nothing is latched until inference actually answers" - ); - - server.join().expect("server thread"); - fs::remove_dir_all(&dir).ok(); - } - - /// A fresh scratch directory under the crate's `target/`. The base is - /// `CARGO_MANIFEST_DIR`, a compile-time constant, so the path never derives from a - /// runtime environment read. - fn scratch_dir(tag: &str) -> PathBuf { - let dir = Path::new(env!("CARGO_MANIFEST_DIR")) - .join("target") - .join(format!("lemonade-fs-test-{tag}")); - let _ = fs::remove_dir_all(&dir); - fs::create_dir_all(&dir).unwrap(); - dir - } - - #[test] - fn backend_install_succeeds_without_retry() { - let mut attempts = 0; - - install_llamacpp_backend_with_retry(|| { - attempts += 1; - Ok(()) - }) - .unwrap(); - - assert_eq!(attempts, 1); - } - - #[test] - fn backend_install_recovers_on_the_second_attempt() { - let mut attempts = 0; - - install_llamacpp_backend_with_retry(|| { - attempts += 1; - if attempts == 1 { - bail!("first backend connection was interrupted"); - } - Ok(()) - }) - .unwrap(); - - assert_eq!(attempts, 2); - } - - #[test] - fn backend_install_stops_after_one_retry_with_reinstall_guidance() { - let mut attempts = 0; - - let error = install_llamacpp_backend_with_retry(|| { - attempts += 1; - if attempts == 1 { - bail!("first backend connection was interrupted"); - } - bail!("second backend connection was interrupted"); - }) - .unwrap_err(); - let rendered = format!("{error:#}"); - - assert_eq!(attempts, 2); - assert!( - rendered.contains("second backend connection was interrupted"), - "{rendered}" - ); - assert!( - !rendered.contains("first backend connection was interrupted"), - "the terminal error must be the retry's failure: {rendered}" - ); - assert!( - rendered.contains("rocm engines install lemonade --reinstall"), - "{rendered}" - ); - assert!(rendered.contains("retry `rocm serve`"), "{rendered}"); - } - - fn test_manifest(runtime_dir: PathBuf) -> LemonadeInstallManifest { - LemonadeInstallManifest { - env_id: "test".to_owned(), - version: rocm_deps::LEMONADE_VERSION.to_owned(), - runtime_dir, - lemond: PathBuf::from("lemond"), - lemonade: PathBuf::from("lemonade"), - backend_recipe: LLAMACPP_RECIPE.to_owned(), - backend_name: ROCM_BACKEND_NAME.to_owned(), - installed_at_unix_ms: 0, - } - } - - #[test] - fn plain_semver_accepts_release_versions_only() { - assert!(looks_like_plain_semver("10.0.0")); - assert!(looks_like_plain_semver("7.13.0")); - assert!(!looks_like_plain_semver("7.14.0a20260601")); - assert!(!looks_like_plain_semver("10.0")); - assert!(!looks_like_plain_semver("10.0.0.1")); - assert!(!looks_like_plain_semver("")); - } - - #[test] - fn backend_versions_therock_version_round_trips() { - let dir = scratch_dir("backend-versions-round-trip"); - let path = dir.join("backend_versions.json"); - fs::write( - &path, - serde_json::to_vec_pretty(&json!({ - "llamacpp": { "rocm-stable": "b9752" }, - "therock": { "version": "7.13.0", "architectures": ["gfx1151"] }, - })) - .unwrap(), - ) - .unwrap(); - - assert_eq!( - read_backend_versions_therock_version(&path), - Some("7.13.0".to_owned()) - ); - - write_backend_versions_therock_version(&path, "10.0.0").unwrap(); - assert_eq!( - read_backend_versions_therock_version(&path), - Some("10.0.0".to_owned()) - ); - - // Every other key, including sibling keys under `therock`, is preserved. - let value: Value = serde_json::from_slice(&fs::read(&path).unwrap()).unwrap(); - assert_eq!(value["llamacpp"]["rocm-stable"], "b9752"); - assert_eq!(value["therock"]["architectures"][0], "gfx1151"); - - fs::remove_dir_all(&dir).ok(); - } - - #[test] - fn backend_versions_therock_version_missing_object_is_an_error() { - let dir = scratch_dir("backend-versions-missing-therock"); - let path = dir.join("backend_versions.json"); - fs::write( - &path, - serde_json::to_vec_pretty(&json!({"llamacpp": {}})).unwrap(), - ) - .unwrap(); - - assert_eq!(read_backend_versions_therock_version(&path), None); - assert!(write_backend_versions_therock_version(&path, "10.0.0").is_err()); - - fs::remove_dir_all(&dir).ok(); - } - - #[test] - fn backend_versions_llamacpp_tag_round_trips() { - let dir = scratch_dir("backend-versions-tag-round-trip"); - let path = dir.join("backend_versions.json"); - fs::write( - &path, - serde_json::to_vec_pretty(&json!({ - "llamacpp": { "rocm-stable": "b9752", "vulkan": "b9747" }, - "therock": { "version": "7.13.0" }, - })) - .unwrap(), - ) - .unwrap(); - - assert_eq!( - read_backend_versions_llamacpp_tag(&path), - Some("b9752".to_owned()) - ); - - write_backend_versions_llamacpp_tag(&path, "b10952").unwrap(); - assert_eq!( - read_backend_versions_llamacpp_tag(&path), - Some("b10952".to_owned()) - ); - - // Sibling keys, including the unrelated `llamacpp.vulkan` tag and the - // whole `therock` object, are preserved. - let value: Value = serde_json::from_slice(&fs::read(&path).unwrap()).unwrap(); - assert_eq!(value["llamacpp"]["vulkan"], "b9747"); - assert_eq!(value["therock"]["version"], "7.13.0"); - - fs::remove_dir_all(&dir).ok(); - } - - #[test] - fn backend_versions_llamacpp_tag_missing_object_is_an_error() { - let dir = scratch_dir("backend-versions-missing-llamacpp"); - let path = dir.join("backend_versions.json"); - fs::write( - &path, - serde_json::to_vec_pretty(&json!({"therock": {}})).unwrap(), - ) - .unwrap(); - - assert_eq!(read_backend_versions_llamacpp_tag(&path), None); - assert!(write_backend_versions_llamacpp_tag(&path, "b10952").is_err()); - - fs::remove_dir_all(&dir).ok(); - } - - fn write_backend_versions_fixture(path: &Path, therock_version: &str, llamacpp_tag: &str) { - fs::write( - path, - serde_json::to_vec_pretty(&json!({ - "llamacpp": { "rocm-stable": llamacpp_tag }, - "therock": { "version": therock_version }, - })) - .unwrap(), - ) - .unwrap(); - } - - #[test] - fn find_llama_server_binary_for_backend_scopes_to_rocm_family() { - let dir = scratch_dir("find-server-scoped-rocm"); - let runtime_dir = dir.join("runtime"); - let llamacpp = runtime_dir.join("bin").join("llamacpp"); - let server = platform_binary_name("llama-server"); - // A stale vulkan install from an earlier attempt must never satisfy a "rocm" - // scoped lookup. - let vulkan_dir = llamacpp.join("vulkan"); - fs::create_dir_all(&vulkan_dir).unwrap(); - fs::write(vulkan_dir.join(&server), b"x").unwrap(); - let manifest = test_manifest(runtime_dir); - assert_eq!( - find_llama_server_binary_for_backend(&manifest, ROCM_BACKEND_NAME), - None - ); - - // Any directory in the ROCm family satisfies it, matching the unscoped scan's - // priority order. - let nightly_dir = llamacpp.join("rocm-nightly"); - fs::create_dir_all(&nightly_dir).unwrap(); - fs::write(nightly_dir.join(&server), b"x").unwrap(); - assert_eq!( - find_llama_server_binary_for_backend(&manifest, ROCM_BACKEND_NAME), - Some(nightly_dir.join(&server)) - ); - fs::remove_dir_all(&dir).ok(); - } - - #[test] - fn find_llama_server_binary_for_backend_scopes_to_vulkan_only() { - let dir = scratch_dir("find-server-scoped-vulkan"); - let runtime_dir = dir.join("runtime"); - let llamacpp = runtime_dir.join("bin").join("llamacpp"); - let server = platform_binary_name("llama-server"); - // A stale rocm-stable install from an earlier attempt must never satisfy a - // "vulkan" scoped lookup, even though it would win the unscoped priority scan. - let rocm_dir = llamacpp.join("rocm-stable"); - fs::create_dir_all(&rocm_dir).unwrap(); - fs::write(rocm_dir.join(&server), b"x").unwrap(); - let manifest = test_manifest(runtime_dir); - assert_eq!( - find_llama_server_binary_for_backend(&manifest, "vulkan"), - None - ); - - let vulkan_dir = llamacpp.join("vulkan"); - fs::create_dir_all(&vulkan_dir).unwrap(); - fs::write(vulkan_dir.join(&server), b"x").unwrap(); - assert_eq!( - find_llama_server_binary_for_backend(&manifest, "vulkan"), - Some(vulkan_dir.join(&server)) - ); - fs::remove_dir_all(&dir).ok(); - } - - #[test] - fn set_aside_then_discard_removes_every_rocm_family_dir_but_not_vulkan() { - // The bug this guards: Tier 2 installing a different llama.cpp tag than a - // prior attempt left two build directories side by side, and the - // directory-order lookup in `find_binary_in` could resolve to whichever one - // a rejected tier installed instead of the one alignment just verified. - let dir = scratch_dir("aside-then-discard-rocm-backend-dirs"); - let runtime_dir = dir.join("runtime"); - let llamacpp = runtime_dir.join("bin").join("llamacpp"); - let server = platform_binary_name("llama-server"); - for backend in ["rocm-stable", "rocm-nightly", "rocm", "vulkan"] { - let backend_dir = llamacpp.join(backend); - fs::create_dir_all(&backend_dir).unwrap(); - fs::write(backend_dir.join(&server), b"x").unwrap(); - } - let manifest = test_manifest(runtime_dir); - - let aside_dirs = set_aside_rocm_llamacpp_backend_dirs(&manifest); - for backend in ["rocm-stable", "rocm-nightly", "rocm"] { - assert!( - !llamacpp.join(backend).exists(), - "{backend} should have been moved aside" - ); - } - resolve_rocm_llamacpp_backend_dirs_aside(&manifest, &aside_dirs, true); - - for backend in ["rocm-stable", "rocm-nightly", "rocm"] { - assert!( - !llamacpp.join(backend).exists(), - "{backend} should still be gone after a discard" - ); - assert!( - !llamacpp - .join(format!("{backend}{BACKEND_DIR_ASIDE_SUFFIX}")) - .exists(), - "{backend}'s aside copy should have been discarded on success" - ); - } - assert!( - llamacpp.join("vulkan").join(&server).is_file(), - "vulkan is untouched by ROCm alignment and must survive" - ); - fs::remove_dir_all(&dir).ok(); - } - - #[test] - fn resolve_rocm_llamacpp_backend_dirs_aside_restores_on_failure() { - // The bug this guards: a failed final install (e.g. the revert branch's - // fallback_install erroring out) must never leave the host with zero ROCm - // backend directories -- the whole reason to set builds aside instead of - // deleting them outright. - let dir = scratch_dir("aside-restores-on-failure"); - let runtime_dir = dir.join("runtime"); - let llamacpp = runtime_dir.join("bin").join("llamacpp"); - let server = platform_binary_name("llama-server"); - let rocm_dir = llamacpp.join("rocm"); - fs::create_dir_all(&rocm_dir).unwrap(); - fs::write(rocm_dir.join(&server), b"original").unwrap(); - let manifest = test_manifest(runtime_dir); - - let aside_dirs = set_aside_rocm_llamacpp_backend_dirs(&manifest); - assert!( - !rocm_dir.exists(), - "the original must be moved aside before the install is attempted" - ); - // Simulate a failed install leaving nothing behind at the original path. - resolve_rocm_llamacpp_backend_dirs_aside(&manifest, &aside_dirs, false); - - assert_eq!( - fs::read(rocm_dir.join(&server)).unwrap(), - b"original", - "the pre-existing backend must be restored after a failed install" - ); - assert!( - !llamacpp - .join(format!("rocm{BACKEND_DIR_ASIDE_SUFFIX}")) - .exists(), - "the aside copy must be moved back, not left behind" - ); - fs::remove_dir_all(&dir).ok(); - } - - #[test] - fn resolve_rocm_llamacpp_backend_dirs_aside_removes_a_failed_installs_stray_residue() { - // The bug this guards: a failed install can land its (broken) output under a - // DIFFERENT ROCm-family name than the one that was asided -- Lemonade - // extracts each release into its own build-numbered directory rather than - // replacing one in place. Only cleaning up the asided name left that stray - // directory on disk, where `find_binary_in`'s priority order - // (rocm-stable > rocm-nightly > rocm) could shadow the just-restored good - // build with it. - let dir = scratch_dir("aside-removes-stray-residue-in-different-dir"); - let runtime_dir = dir.join("runtime"); - let llamacpp = runtime_dir.join("bin").join("llamacpp"); - let server = platform_binary_name("llama-server"); - let rocm_dir = llamacpp.join("rocm"); - fs::create_dir_all(&rocm_dir).unwrap(); - fs::write(rocm_dir.join(&server), b"original").unwrap(); - let manifest = test_manifest(runtime_dir); - - let aside_dirs = set_aside_rocm_llamacpp_backend_dirs(&manifest); - // Simulate a failed install that landed its (unverified) output in - // `rocm-stable` instead of `rocm`, the name that was actually asided. - let stray_dir = llamacpp.join("rocm-stable"); - fs::create_dir_all(&stray_dir).unwrap(); - fs::write(stray_dir.join(&server), b"stray").unwrap(); - - resolve_rocm_llamacpp_backend_dirs_aside(&manifest, &aside_dirs, false); - - assert!( - !stray_dir.exists(), - "the failed install's residue in a different backend dir must be removed" - ); - assert_eq!( - fs::read(rocm_dir.join(&server)).unwrap(), - b"original", - "the pre-existing backend must still be restored" - ); - fs::remove_dir_all(&dir).ok(); - } - - #[test] - fn llamacpp_backend_alignment_requires_both_a_successful_install_and_a_usable_binary() { - // The bug this guards: a failed install (e.g. Tier 1's pinned build 404ing - // for a ROCm version it predates) must never be forgiven by a leftover - // binary from a PREVIOUS, different-version install that happens to still - // pass the ldd-resolves check -- `ldd` cannot tell the two apart. - assert!(llamacpp_backend_alignment_succeeded(&Ok(()), true)); - assert!(!llamacpp_backend_alignment_succeeded(&Ok(()), false)); - assert!(!llamacpp_backend_alignment_succeeded( - &Err(anyhow!("boom")), - true - )); - assert!(!llamacpp_backend_alignment_succeeded( - &Err(anyhow!("boom")), - false - )); - } - - #[test] - fn should_attempt_llamacpp_backend_alignment_requires_the_rocm_backend_specifically() { - // The bug this guards: alignment is guaranteed to fail on a host that falls - // back to vulkan (WSL2 being the documented case), so it must be skipped for - // any selection other than exactly `rocm` -- including the probe itself - // failing to determine one at all. - assert!(should_attempt_llamacpp_backend_alignment(Some( - ROCM_BACKEND_NAME - ))); - assert!(!should_attempt_llamacpp_backend_alignment(Some("vulkan"))); - assert!(!should_attempt_llamacpp_backend_alignment(None)); - } - - #[test] - fn align_honors_the_disabled_flag_without_touching_the_pin_or_the_injected_steps() { - // This pins `align`'s own half of the opt-out contract in isolation: the pin - // and llama.cpp tag stay byte-identical and `align`/`latest_tag` are never - // called. The real production gate -- whether this function is even reached - // with `disabled == true` in the first place -- lives one level up, in - // `prepare_llamacpp_backend_for_active_rocm_impl`; see - // `prepare_disabled_early_return_skips_every_lookup_and_installs_unforced` - // below for that half. - let dir = scratch_dir("align-disabled"); - let path = dir.join("backend_versions.json"); - write_backend_versions_fixture(&path, "7.13.0", "b9752"); - let mut manifest = test_manifest(dir.clone()); - - let result = align_llamacpp_backend_to_version( - &mut manifest, - &path, - "10.0.0", - "7.13.0", - true, - |_manifest, _target, _force_reinstall, _label| { - panic!("disabled means no alignment attempt of any kind") - }, - |_manifest, force_reinstall| { - assert!( - !force_reinstall, - "the disabled path installs whatever is already pinned, not a fresh reinstall" - ); - Ok(()) - }, - || panic!("disabled means tier 2's tag lookup must not run either"), - ); - - assert_eq!(result.unwrap(), None); - assert_eq!( - read_backend_versions_therock_version(&path), - Some("7.13.0".to_owned()), - "the packaged pin must survive untouched when alignment is disabled" - ); - assert_eq!( - read_backend_versions_llamacpp_tag(&path), - Some("b9752".to_owned()), - "the packaged llama.cpp tag must survive untouched when alignment is disabled" - ); - fs::remove_dir_all(&dir).ok(); - } - - #[test] - fn prepare_disabled_early_return_skips_every_lookup_and_installs_unforced() { - // The real production gate for the opt-out: previously this was covered only - // by a @nightly @requires-gpu e2e scenario, which every per-PR and - // merge-queue lane skips -- so a regression that silently re-enabled - // alignment here shipped green everywhere that actually gates a merge. Every - // lookup the disabled path is supposed to skip panics if reached; the - // unforced install is the falsifiable positive. - let mut manifest = test_manifest(PathBuf::from("unused-runtime-dir")); - - let result = prepare_llamacpp_backend_for_active_rocm_impl( - &mut manifest, - true, - || panic!("disabled means the active SDK version must never be probed"), - |_path| panic!("disabled means the packaged pin must never be read"), - |_manifest| panic!("disabled means the vulkan-fallback probe must never run"), - |_manifest, force_reinstall| { - assert!( - !force_reinstall, - "the disabled path installs whatever is already pinned, not a fresh reinstall" - ); - Ok(()) - }, - |_manifest, _path, _target, _pinned, _disabled| { - panic!("disabled means align must never be reached") - }, - ); - - assert_eq!(result.unwrap(), None); - } - - #[test] - fn prepare_probe_error_falls_back_to_an_unforced_install_without_reaching_align() { - // The bug this guards: a host where the vulkan-fallback probe itself errors - // (e.g. `lemond` failing to start) must degrade to an ordinary install - // rather than propagating the error or attempting alignment blind -- the - // same "skip, don't fail" contract every other probe/lookup failure in this - // gate already gets. - let mut manifest = test_manifest(PathBuf::from("unused-runtime-dir")); - - let result = prepare_llamacpp_backend_for_active_rocm_impl( - &mut manifest, - false, - || Ok(Some("10.0.0".to_owned())), - |_path| Some("7.13.0".to_owned()), - |_manifest| Err(anyhow!("lemond failed to start")), - |_manifest, force_reinstall| { - assert!( - !force_reinstall, - "a probe error falls back to an unforced install, not a reinstall" - ); - Ok(()) - }, - |_manifest, _path, _target, _pinned, _disabled| { - panic!("a probe error must skip align entirely") - }, - ); - - assert_eq!(result.unwrap(), None); - } - - #[test] - fn align_tier1_success_keeps_target_version_pinned() { - let dir = scratch_dir("align-tier1-success"); - let path = dir.join("backend_versions.json"); - write_backend_versions_fixture(&path, "7.13.0", "b9752"); - let mut manifest = test_manifest(dir.clone()); - - let result = align_llamacpp_backend_to_version( - &mut manifest, - &path, - "10.0.0", - "7.13.0", - false, - |_manifest, _target, force_reinstall, _label| { - // A stale backend installed against the OLD pin must not be able to - // fake success just because it happens to still resolve -- Tier 1 - // must force a real reinstall attempt every time. - assert!(force_reinstall, "tier 1 must force a reinstall"); - true - }, - |_manifest, _force_reinstall| panic!("tier 1 succeeded; no fallback install"), - || panic!("tier 1 succeeded; tier 2's tag lookup must not run"), - ); - - assert_eq!(result.unwrap(), Some("10.0.0".to_owned())); - assert_eq!( - read_backend_versions_therock_version(&path), - Some("10.0.0".to_owned()) - ); - fs::remove_dir_all(&dir).ok(); - } - - #[test] - fn align_falls_through_to_tier2_and_succeeds() { - let dir = scratch_dir("align-tier2-success"); - let path = dir.join("backend_versions.json"); - write_backend_versions_fixture(&path, "7.13.0", "b9752"); - let mut manifest = test_manifest(dir.clone()); - let mut align_calls = 0; - - let result = align_llamacpp_backend_to_version( - &mut manifest, - &path, - "10.0.0", - "7.13.0", - false, - |_manifest, _target, force_reinstall, _label| { - // Both tiers force a reinstall now, so the mock can no longer use - // force_reinstall itself to distinguish tier 1 from tier 2 -- use - // call order instead. Tier 1 fails; Tier 2 succeeds. - assert!( - force_reinstall, - "every alignment attempt forces a reinstall" - ); - align_calls += 1; - align_calls > 1 - }, - |_manifest, _force_reinstall| panic!("tier 2 succeeded; no fallback install"), - || Ok("b10952".to_owned()), - ); - - assert_eq!(result.unwrap(), Some("10.0.0".to_owned())); - assert_eq!(align_calls, 2, "both tiers were attempted"); - assert_eq!( - read_backend_versions_therock_version(&path), - Some("10.0.0".to_owned()) - ); - assert_eq!( - read_backend_versions_llamacpp_tag(&path), - Some("b10952".to_owned()), - "tier 2 pins the newer llama.cpp tag" - ); - fs::remove_dir_all(&dir).ok(); - } - - #[test] - fn align_reverts_the_llamacpp_tag_when_tier2_pinned_a_newer_one_and_still_fails() { - // Regression test: the tag restore used to be silently skipped whenever - // deleted, and nothing caught it -- both tiers must fail here so the - // revert path actually runs, and the tag must come back to its original - // value rather than staying on Tier 2's newer pin. - let dir = scratch_dir("align-revert-restores-tag"); - let path = dir.join("backend_versions.json"); - write_backend_versions_fixture(&path, "7.13.0", "b9752"); - let mut manifest = test_manifest(dir.clone()); - - let result = align_llamacpp_backend_to_version( - &mut manifest, - &path, - "10.0.0", - "7.13.0", - false, - |_manifest, _target, _force_reinstall, _label| false, - |_manifest, force_reinstall| { - assert!(force_reinstall); - Ok(()) - }, - || Ok("b10952".to_owned()), - ); - - assert_eq!(result.unwrap(), None); - assert_eq!( - read_backend_versions_therock_version(&path), - Some("7.13.0".to_owned()) - ); - assert_eq!( - read_backend_versions_llamacpp_tag(&path), - Some("b9752".to_owned()), - "the llama.cpp tag must be restored to its original value, not left on Tier 2's pin" - ); - fs::remove_dir_all(&dir).ok(); - } - - #[test] - fn align_removes_the_llamacpp_tag_on_revert_when_none_was_pinned_before() { - // The resource file may have no `llamacpp.rocm-stable` key at all (an - // embeddable that never shipped a pin). Tier 2 still writes one; reverting - // must remove it again rather than leaving it in place with no original - // value to restore it to. - let dir = scratch_dir("align-revert-removes-tag"); - let path = dir.join("backend_versions.json"); - fs::write( - &path, - serde_json::to_vec_pretty(&json!({ - "llamacpp": {}, - "therock": { "version": "7.13.0" }, - })) - .unwrap(), - ) - .unwrap(); - let mut manifest = test_manifest(dir.clone()); - - let result = align_llamacpp_backend_to_version( - &mut manifest, - &path, - "10.0.0", - "7.13.0", - false, - |_manifest, _target, _force_reinstall, _label| false, - |_manifest, force_reinstall| { - assert!(force_reinstall); - Ok(()) - }, - || Ok("b10952".to_owned()), - ); - - assert_eq!(result.unwrap(), None); - assert_eq!( - read_backend_versions_llamacpp_tag(&path), - None, - "no tag was pinned before Tier 2; reverting must remove it, not invent a value" - ); - fs::remove_dir_all(&dir).ok(); - } - - #[test] - fn align_skips_tier2_reinstall_when_pinned_tag_is_already_latest() { - let dir = scratch_dir("align-tier2-tag-unchanged"); - let path = dir.join("backend_versions.json"); - write_backend_versions_fixture(&path, "7.13.0", "b9752"); - let mut manifest = test_manifest(dir.clone()); - let mut align_calls = 0; - - let result = align_llamacpp_backend_to_version( - &mut manifest, - &path, - "10.0.0", - "7.13.0", - false, - |_manifest, _target, _force_reinstall, _label| { - align_calls += 1; - false - }, - |_manifest, force_reinstall| { - assert!( - force_reinstall, - "the revert fallback always forces a reinstall" - ); - Ok(()) - }, - // Already the latest tag: Tier 2's own reinstall must not be attempted. - || Ok("b9752".to_owned()), - ); - - assert_eq!(result.unwrap(), None); - assert_eq!(align_calls, 1, "only tier 1 was attempted"); - assert_eq!( - read_backend_versions_therock_version(&path), - Some("7.13.0".to_owned()), - "reverted to the original pinned version" - ); - fs::remove_dir_all(&dir).ok(); - } - - #[test] - fn align_reverts_and_still_runs_fallback_when_the_restore_write_fails() { - // Regression test: a failure while restoring the original pinned version must - // not abort the install — the fallback reinstall below it is the recovery - // path, and every sibling warning in this function is best-effort. - let dir = scratch_dir("align-revert-write-fails"); - let path = dir.join("backend_versions.json"); - write_backend_versions_fixture(&path, "7.13.0", "b9752"); - let mut manifest = test_manifest(dir.clone()); - let mut fallback_called = false; - - // Corrupt the file's `therock` object between tier attempts and the revert, so - // the revert's own write fails. - let result = align_llamacpp_backend_to_version( - &mut manifest, - &path, - "10.0.0", - "7.13.0", - false, - |_manifest, _target, _force_reinstall, _label| { - fs::write( - &path, - serde_json::to_vec_pretty(&json!({"llamacpp": {}})).unwrap(), - ) - .unwrap(); - false - }, - |_manifest, force_reinstall| { - fallback_called = true; - assert!(force_reinstall); - Ok(()) - }, - || bail!("network unavailable"), - ); - - assert_eq!( - result.unwrap(), - None, - "never propagates the restore-write failure" - ); - assert!(fallback_called, "the fallback reinstall still ran"); - fs::remove_dir_all(&dir).ok(); - } - - #[test] - fn align_revert_restores_the_backend_dir_when_the_fallback_install_fails() { - // The bug this guards: the revert branch used to delete the ROCm backend - // directories before the fallback install, with no recovery if that install - // then failed -- leaving the host with no ROCm backend at all. Both tiers - // fail here (forcing the revert), and the fallback install also fails, so - // the pre-existing backend must come back rather than staying deleted. - let dir = scratch_dir("align-revert-fallback-install-fails"); - let path = dir.join("backend_versions.json"); - write_backend_versions_fixture(&path, "7.13.0", "b9752"); - let mut manifest = test_manifest(dir.clone()); - let llamacpp = dir.join("bin").join("llamacpp"); - let server = platform_binary_name("llama-server"); - let rocm_dir = llamacpp.join("rocm"); - fs::create_dir_all(&rocm_dir).unwrap(); - fs::write(rocm_dir.join(&server), b"pre-existing").unwrap(); - - let result = align_llamacpp_backend_to_version( - &mut manifest, - &path, - "10.0.0", - "7.13.0", - false, - |_manifest, _target, _force_reinstall, _label| false, - |_manifest, force_reinstall| { - assert!(force_reinstall); - bail!("network unavailable") - }, - || Ok("b10952".to_owned()), - ); - - assert!( - result.is_err(), - "the fallback install's failure must be propagated" - ); - assert_eq!( - fs::read(rocm_dir.join(&server)).unwrap(), - b"pre-existing", - "the pre-existing backend must be restored, not left deleted" - ); - fs::remove_dir_all(&dir).ok(); - } - - #[test] - #[cfg(target_os = "linux")] - fn ldd_output_resolves_all_requires_every_soname_resolved() { - let resolved = "\ -\tlibggml-base.so.0 => /opt/rocm/lib/libggml-base.so.0 (0x00007f0)\n\ -\tlibhipblas.so.3 => /opt/rocm/lib/libhipblas.so.3 (0x00007f1)\n\ -\tlibrocblas.so.5 => /opt/rocm/lib/librocblas.so.5 (0x00007f2)\n\ -\tlibamdhip64.so.7 => /opt/rocm/lib/libamdhip64.so.7 (0x00007f3)\n\ -\tlibhsa-runtime64.so.1 => /opt/rocm/lib/libhsa-runtime64.so.1 (0x00007f4)\n"; - assert!(ldd_output_resolves_all( - resolved, - &ROCM_BACKEND_REQUIRED_SONAMES - )); - - let missing_one = "\ -\tlibhipblas.so.3 => not found\n\ -\tlibrocblas.so.5 => /opt/rocm/lib/librocblas.so.5 (0x00007f2)\n\ -\tlibamdhip64.so.7 => /opt/rocm/lib/libamdhip64.so.7 (0x00007f3)\n\ -\tlibhsa-runtime64.so.1 => /opt/rocm/lib/libhsa-runtime64.so.1 (0x00007f4)\n"; - assert!(!ldd_output_resolves_all( - missing_one, - &ROCM_BACKEND_REQUIRED_SONAMES - )); - } - - #[test] - fn lemonade_pull_builds_checkpoint_command() { - assert_eq!( - lemonade_pull_args("127.0.0.1", 11435, "LiquidAI/LFM2.5-230M-GGUF:Q4_0"), - vec![ - "--host", - "127.0.0.1", - "--port", - "11435", - "pull", - "LiquidAI/LFM2.5-230M-GGUF:Q4_0", - ] - ); - } - - #[test] - fn embeddable_package_matches_runtime_os() { - let version = rocm_deps::LEMONADE_VERSION; - let (os_arch, extension) = embeddable_os_arch(); - let suffix = format!("{os_arch}.{extension}"); - assert!(embeddable_archive_name(version).ends_with(&suffix)); - assert!(embeddable_url(version).ends_with(&suffix)); - if runtime_is_windows() { - assert_eq!(suffix, "windows-x64.zip"); - assert_eq!(platform_binary_name("lemond"), "lemond.exe"); - } else { - assert_eq!(suffix, "ubuntu-x64.tar.gz"); - assert_eq!(platform_binary_name("lemond"), "lemond"); - } - } - - #[test] - fn sha256_verification_accepts_matching_content_and_rejects_mismatch() { - let dir = scratch_dir("archive-digest"); - let archive = dir.join("archive"); - fs::write(&archive, b"verified archive").unwrap(); - let expected = format!("{:x}", Sha256::digest(b"verified archive")); - - verify_sha256(&archive, &expected).unwrap(); - let error = verify_sha256(&archive, &"0".repeat(64)).unwrap_err(); - assert!(error.to_string().contains("SHA-256 mismatch")); - fs::remove_dir_all(&dir).ok(); - } - - #[test] - fn repeated_valid_archive_publication_is_idempotent() { - let dir = scratch_dir("valid-publication-race"); - let archive = dir.join("archive"); - let content = b"verified archive"; - let expected = format!("{:x}", Sha256::digest(content)); - - for _ in 0..2 { - let mut temporary = tempfile::NamedTempFile::new_in(&dir).unwrap(); - temporary.write_all(content).unwrap(); - verify_sha256(temporary.path(), &expected).unwrap(); - publish_verified_archive(temporary, &archive, &expected).unwrap(); - } - - assert_eq!(fs::read(&archive).unwrap(), content); - assert_eq!(fs::read_dir(&dir).unwrap().count(), 1); - fs::remove_dir_all(&dir).ok(); - } - - #[test] - fn archive_publication_conflict_with_foreign_bytes_fails_closed() { - let dir = scratch_dir("invalid-publication-race"); - let archive = dir.join("archive"); - let content = b"verified archive"; - let expected = format!("{:x}", Sha256::digest(content)); - let mut temporary = tempfile::NamedTempFile::new_in(&dir).unwrap(); - temporary.write_all(content).unwrap(); - verify_sha256(temporary.path(), &expected).unwrap(); - - fs::write(&archive, b"unverified concurrent archive").unwrap(); - let error = publish_verified_archive(temporary, &archive, &expected).unwrap_err(); - let error = format!("{error:#}"); - - assert!(error.contains("concurrent archive publication")); - assert!(error.contains("failed verification after publish conflict")); - assert!(error.contains("SHA-256 mismatch")); - assert_eq!( - fs::read(&archive).unwrap(), - b"unverified concurrent archive" - ); - assert_eq!(fs::read_dir(&dir).unwrap().count(), 1); - fs::remove_dir_all(&dir).ok(); - } - - #[test] - fn poisoned_cached_archive_is_removed_and_redownloaded() { - let dir = scratch_dir("poisoned-cache"); - let archive = dir.join("archive"); - fs::write(&archive, b"poisoned").unwrap(); - let expected = format!("{:x}", Sha256::digest(b"fresh archive")); - let mut downloads = 0; - - ensure_cached_archive("test://archive", &archive, &expected, |_, destination| { - downloads += 1; - fs::write(destination, b"fresh archive")?; - Ok(()) - }) - .unwrap(); - - assert_eq!(downloads, 1); - assert_eq!(fs::read(&archive).unwrap(), b"fresh archive"); - fs::remove_dir_all(&dir).ok(); - } - - #[test] - fn download_with_wrong_content_is_rejected_and_leaves_no_archive() { - let dir = scratch_dir("bad-content-download"); - let archive = dir.join("archive"); - let expected = format!("{:x}", Sha256::digest(b"fresh archive")); - - // A download that succeeds at the transport level but returns the wrong - // bytes must not be published, even though nothing errored. - let error = ensure_cached_archive("test://archive", &archive, &expected, |_, path| { - fs::write(path, b"attacker archive")?; - Ok(()) - }) - .unwrap_err(); - - let error = format!("{error:#}"); - assert!(error.contains("SHA-256 mismatch"), "{error}"); - assert!(!archive.exists(), "unverified bytes were published"); - // Only the lock file may remain; the temporary download must be gone. - let leftovers: Vec<_> = fs::read_dir(&dir) - .unwrap() - .map(|entry| entry.unwrap().file_name()) - .filter(|name| name != ".archive.lock") - .collect(); - assert!(leftovers.is_empty(), "leftovers: {leftovers:?}"); - fs::remove_dir_all(&dir).ok(); - } - - #[test] - fn archive_guard_outlives_verification_and_is_scoped_to_one_archive() { - let dir = scratch_dir("archive-guard-scope"); - let archive = dir.join("archive"); - let other = dir.join("other-archive"); - let content = b"fresh archive"; - let expected = format!("{:x}", Sha256::digest(content)); - - // The guard returned by verification is what callers hold across - // extraction; while it is alive the same archive must not be lockable. - let guard = ensure_cached_archive("test://archive", &archive, &expected, |_, path| { - fs::write(path, content)?; - Ok(()) - }) - .unwrap(); - - let blocked = std::thread::spawn(move || { - let _second = lock_archive_cache_in_process(&archive); - }); - std::thread::sleep(Duration::from_millis(50)); - assert!( - !blocked.is_finished(), - "a second in-process lock on the held archive was granted" - ); - - // A different archive must not queue behind it. - let unrelated = std::thread::spawn(move || { - let _other = lock_archive_cache_in_process(&other); - }); - let deadline = std::time::Instant::now() + Duration::from_secs(10); - while !unrelated.is_finished() { - assert!( - std::time::Instant::now() < deadline, - "an unrelated archive serialized behind the held one" - ); - std::thread::sleep(Duration::from_millis(10)); - } - unrelated.join().unwrap(); - - drop(guard); - blocked.join().unwrap(); - fs::remove_dir_all(&dir).ok(); - } - - #[test] - fn poisoned_process_mutex_is_recovered_before_digest_validation() { - let _ = std::thread::spawn(|| { - let _guard = ARCHIVE_CACHE_BUSY.lock().unwrap(); - panic!("poison archive cache process mutex"); - }) - .join(); - let dir = scratch_dir("poisoned-process-mutex"); - let archive = dir.join("archive"); - fs::write(&archive, b"poisoned").unwrap(); - let expected = format!("{:x}", Sha256::digest(b"fresh archive")); - - ensure_cached_archive("test://archive", &archive, &expected, |_, destination| { - fs::write(destination, b"fresh archive")?; - Ok(()) - }) - .unwrap(); - - verify_sha256(&archive, &expected).unwrap(); - fs::remove_dir_all(&dir).ok(); - } - - const ARCHIVE_LOCK_TEST_ROLE: &str = "ROCM_LEMONADE_ARCHIVE_LOCK_TEST_ROLE"; - const ARCHIVE_LOCK_TEST_DIR: &str = "ROCM_LEMONADE_ARCHIVE_LOCK_TEST_DIR"; - const ARCHIVE_LOCK_TEST_TIMEOUT: Duration = Duration::from_secs(10); - - struct ChildGuard(Option); - - impl ChildGuard { - fn spawn(test_exe: &Path, dir: &Path, role: &str) -> Self { - let child = ProcessCommand::new(test_exe) - .args([ - "--exact", - "tests::archive_cache_subprocess_helper", - "--nocapture", - ]) - .env(ARCHIVE_LOCK_TEST_ROLE, role) - .env(ARCHIVE_LOCK_TEST_DIR, dir) - .stdin(Stdio::null()) - .stdout(Stdio::null()) - .stderr(Stdio::null()) - .spawn() - .unwrap(); - Self(Some(child)) - } - - fn is_running(&mut self) -> bool { - self.0.as_mut().unwrap().try_wait().unwrap().is_none() - } - - fn wait_until_exit(&mut self, timeout: Duration) -> std::process::ExitStatus { - let deadline = std::time::Instant::now() + timeout; - loop { - if let Some(status) = self.0.as_mut().unwrap().try_wait().unwrap() { - self.0.take(); - return status; - } - assert!( - std::time::Instant::now() < deadline, - "timed out waiting for child process {}", - self.0.as_ref().unwrap().id() - ); - std::thread::sleep(Duration::from_millis(10)); - } - } - } - - impl Drop for ChildGuard { - fn drop(&mut self) { - if let Some(mut child) = self.0.take() { - if child.try_wait().ok().flatten().is_none() { - let _ = child.kill(); - } - let _ = child.wait(); - } - } - } - - #[test] - fn archive_cache_subprocess_helper() { - let Ok(role) = std::env::var(ARCHIVE_LOCK_TEST_ROLE) else { - return; - }; - let dir = PathBuf::from(std::env::var_os(ARCHIVE_LOCK_TEST_DIR).unwrap()); - let archive = dir.join("archive"); - let expected = format!("{:x}", Sha256::digest(b"fresh archive")); - - ensure_cached_archive_after_invalid( - "test://archive", - &archive, - &expected, - |_, destination| { - fs::write(destination, b"fresh archive")?; - Ok(()) - }, - || { - if role == "validator" { - fs::write(dir.join("validator-paused"), b"").unwrap(); - let deadline = std::time::Instant::now() + ARCHIVE_LOCK_TEST_TIMEOUT; - while !dir.join("release-validator").is_file() { - assert!( - std::time::Instant::now() < deadline, - "timed out waiting to release validator" - ); - std::thread::sleep(Duration::from_millis(10)); - } - } - }, - ) - .unwrap(); - verify_sha256(&archive, &expected).unwrap(); - } - - fn wait_for_test_file(path: &Path, timeout: Duration) { - let deadline = std::time::Instant::now() + timeout; - while !path.is_file() { - assert!( - std::time::Instant::now() < deadline, - "timed out waiting for {}", - path.display() - ); - std::thread::sleep(Duration::from_millis(10)); - } - } - - #[test] - fn interprocess_lock_serializes_poisoned_validation_and_recovery() { - let dir = scratch_dir("interprocess-archive-lock"); - let archive = dir.join("archive"); - fs::write(&archive, b"poisoned").unwrap(); - let test_exe = std::env::current_exe().unwrap(); - - let mut validator = ChildGuard::spawn(&test_exe, &dir, "validator"); - wait_for_test_file(&dir.join("validator-paused"), ARCHIVE_LOCK_TEST_TIMEOUT); - let mut recoverer = ChildGuard::spawn(&test_exe, &dir, "recoverer"); - wait_for_test_file( - &dir.join("recoverer-before-lock"), - ARCHIVE_LOCK_TEST_TIMEOUT, - ); - wait_for_test_file(&dir.join("recoverer-heartbeat"), ARCHIVE_LOCK_TEST_TIMEOUT); - assert!( - !dir.join("recoverer-after-lock").is_file(), - "recoverer acquired the inter-process lock before release" - ); - assert!( - recoverer.is_running(), - "recoverer exited while waiting for the inter-process lock" - ); - - fs::write(dir.join("release-validator"), b"").unwrap(); - assert!( - validator - .wait_until_exit(ARCHIVE_LOCK_TEST_TIMEOUT) - .success(), - "validator subprocess failed" - ); - assert!( - recoverer - .wait_until_exit(ARCHIVE_LOCK_TEST_TIMEOUT) - .success(), - "recoverer subprocess failed" - ); - assert!(dir.join("recoverer-after-lock").is_file()); - let expected = format!("{:x}", Sha256::digest(b"fresh archive")); - verify_sha256(&archive, &expected).unwrap(); - fs::remove_dir_all(&dir).ok(); - } - - #[test] - fn failed_download_removes_private_partial_file() { - let dir = scratch_dir("partial-download"); - let archive = dir.join("archive"); - let expected = format!("{:x}", Sha256::digest(b"complete")); - - let error = - ensure_cached_archive("test://archive", &archive, &expected, |_, destination| { - fs::write(destination, b"partial")?; - bail!("download interrupted") - }) - .unwrap_err(); - - assert!(error.to_string().contains("download interrupted")); - assert!(!archive.exists()); - assert_eq!( - fs::read_dir(&dir) - .unwrap() - .filter( - |entry| entry.as_ref().unwrap().path() != archive_lock_path(&archive).unwrap() - ) - .count(), - 0 - ); - fs::remove_dir_all(&dir).ok(); - } - - #[test] - fn valid_embeddable_tree_is_found_and_copied() { - let dir = scratch_dir("valid-embeddable-tree"); - let extract = dir.join("extract"); - let package = extract.join("package"); - fs::create_dir_all(package.join("lib")).unwrap(); - fs::write(lemond_path_in(&package), b"lemond").unwrap(); - fs::write(lemonade_path_in(&package), b"lemonade").unwrap(); - fs::write(package.join("lib").join("backend"), b"backend").unwrap(); - - let found = find_embeddable_root(&extract).unwrap(); - assert_eq!(found, fs::canonicalize(&package).unwrap()); - let destination = dir.join("runtime"); - copy_tree(&found, &destination).unwrap(); - assert_eq!( - fs::read(destination.join("lib").join("backend")).unwrap(), - b"backend" - ); - fs::remove_dir_all(&dir).ok(); - } - - #[test] - fn deeply_nested_embeddable_tree_is_copied() { - let dir = scratch_dir("deep-embeddable-tree"); - let extract = dir.join("extract"); - let package = extract.join("package"); - let nested = package - .join("lib") - .join("site-packages") - .join("backend") - .join("kernels") - .join("gfx"); - fs::create_dir_all(&nested).unwrap(); - fs::write(lemond_path_in(&package), b"lemond").unwrap(); - fs::write(lemonade_path_in(&package), b"lemonade").unwrap(); - fs::write(nested.join("kernel.so"), b"kernel").unwrap(); - - let found = find_embeddable_root(&extract).unwrap(); - let destination = dir.join("runtime"); - copy_tree(&found, &destination).unwrap(); - assert_eq!( - fs::read( - destination - .join("lib") - .join("site-packages") - .join("backend") - .join("kernels") - .join("gfx") - .join("kernel.so") - ) - .unwrap(), - b"kernel" - ); - fs::remove_dir_all(&dir).ok(); - } - - #[cfg(unix)] - #[test] - fn embeddable_tree_rejects_external_file_symlink() { - use std::os::unix::fs::symlink; - - let dir = scratch_dir("external-file-symlink"); - let extract = dir.join("extract"); - let package = extract.join("package"); - fs::create_dir_all(&package).unwrap(); - fs::write(lemond_path_in(&package), b"lemond").unwrap(); - fs::write(lemonade_path_in(&package), b"lemonade").unwrap(); - let outside = dir.join("outside-file"); - fs::write(&outside, b"secret").unwrap(); - symlink(&outside, package.join("linked-file")).unwrap(); - - let found = find_embeddable_root(&extract).unwrap(); - let error = copy_tree(&found, &dir.join("runtime")).unwrap_err(); - assert!(error.to_string().contains("symlink")); - fs::remove_dir_all(&dir).ok(); - } - - /// Windows counterpart to the Unix symlink rejection tests. - /// - /// Creating a symlink on Windows needs privilege, but `mklink /J` creates a - /// directory junction unelevated, and Rust reports junctions as symlinks - /// (the reparse tag carries the name-surrogate bit) — so this exercises the - /// same rejection branch on the platform where the zip artifact is - /// first-class. - #[cfg(windows)] - #[test] - fn embeddable_tree_rejects_external_directory_junction() { - let dir = scratch_dir("external-directory-junction"); - let extract = dir.join("extract"); - let package = extract.join("package"); - fs::create_dir_all(&package).unwrap(); - fs::write(lemond_path_in(&package), b"lemond").unwrap(); - fs::write(lemonade_path_in(&package), b"lemonade").unwrap(); - let outside = dir.join("outside-directory"); - fs::create_dir_all(&outside).unwrap(); - fs::write(outside.join("secret.txt"), b"secret").unwrap(); - - let junction = package.join("linked-directory"); - let status = ProcessCommand::new("cmd") - .args(["/C", "mklink", "/J"]) - .arg(&junction) - .arg(&outside) - .stdin(Stdio::null()) - .stdout(Stdio::null()) - .stderr(Stdio::null()) - .status() - .unwrap(); - assert!(status.success(), "mklink /J failed"); - assert!( - fs::symlink_metadata(&junction) - .unwrap() - .file_type() - .is_symlink(), - "junction was not reported as a symlink" - ); - - let found = find_embeddable_root(&extract).unwrap(); - let error = copy_tree(&found, &dir.join("runtime")).unwrap_err(); - assert!(error.to_string().contains("symlink"), "{error}"); - assert!(!dir.join("runtime").join("linked-directory").exists()); - fs::remove_dir_all(&dir).ok(); - } - - #[cfg(unix)] - #[test] - fn embeddable_tree_rejects_external_directory_symlink() { - use std::os::unix::fs::symlink; - - let dir = scratch_dir("external-directory-symlink"); - let extract = dir.join("extract"); - fs::create_dir_all(&extract).unwrap(); - let outside = dir.join("outside-package"); - fs::create_dir_all(&outside).unwrap(); - fs::write(lemond_path_in(&outside), b"lemond").unwrap(); - fs::write(lemonade_path_in(&outside), b"lemonade").unwrap(); - symlink(&outside, extract.join("linked-package")).unwrap(); - - let error = find_embeddable_root(&extract).unwrap_err(); - assert!(error.to_string().contains("symlink")); - fs::remove_dir_all(&dir).ok(); - } - - #[test] - fn destination_child_requires_one_normal_component() { - let destination = Path::new("runtime"); - assert_eq!( - destination_child(destination, std::ffi::OsStr::new("bin")).unwrap(), - destination.join("bin") - ); - assert!(destination_child(destination, std::ffi::OsStr::new("..")).is_err()); - assert!(destination_child(destination, std::ffi::OsStr::new(".")).is_err()); - assert!(destination_child(destination, std::ffi::OsStr::new("a/b")).is_err()); - } - - #[test] - fn a_version_change_forces_re_extraction_over_the_existing_runtime() { - let pin = rocm_deps::LEMONADE_VERSION; - // Same version already unpacked: nothing to do. - assert!(!needs_extraction(false, Some(pin), pin, true)); - // A bump must not be silently skipped just because `lemond` exists. - assert!(needs_extraction(false, Some("10.10.0"), pin, true)); - // Nothing recorded, nothing unpacked, or an explicit reinstall. - assert!(needs_extraction(false, None, pin, true)); - assert!(needs_extraction(false, Some(pin), pin, false)); - assert!(needs_extraction(true, Some(pin), pin, true)); - } - - #[test] - fn runtime_version_marker_round_trips_through_needs_extraction() { - // Unit coverage for the helpers themselves and their wiring into - // `needs_extraction` — NOT a regression test for the bug this marker - // fixes (see `prepare_embeddable_trusts_the_runtime_tree_over_an_absent_manifest` - // below for that): this never calls `prepare_embeddable`, so it can't - // constrain which source it reads `installed_version` from. - let dir = tempfile::tempdir().unwrap(); - let runtime_dir = dir.path(); - - assert_eq!(installed_runtime_version(runtime_dir), None); - - record_runtime_version(runtime_dir, "7.13.0").unwrap(); - let installed = installed_runtime_version(runtime_dir); - assert_eq!(installed.as_deref(), Some("7.13.0")); - assert!(!needs_extraction( - false, - installed.as_deref(), - "7.13.0", - true - )); - assert!(needs_extraction( - false, - installed.as_deref(), - "7.14.0", - true - )); - } - - #[test] - fn prepare_embeddable_trusts_the_runtime_tree_over_an_absent_manifest() { - // The actual regression: `prepare_embeddable` must source - // `installed_version` from the marker inside `runtime_dir`, not from - // `paths`' manifest. An isolated caller (fresh `AppPaths`, no - // manifest on disk) sharing a runtime tree that a *different* caller - // already extracted must recognize it as current and skip - // re-extraction. Reverting to a manifest read (the original bug) - // makes every isolated caller miss, wipe, and re-extract — destroying - // any llama.cpp backend already installed there. Calling - // `prepare_embeddable_with` (the real function, seamed for an - // injectable download step) rather than reimplementing its decision - // inline is what makes this test fail under that reversion. - let env_root = tempfile::tempdir().unwrap(); - let paths = AppPaths { - config_dir: env_root.path().join("config"), - data_dir: env_root.path().join("data"), // no manifest ever written here - cache_dir: env_root.path().join("cache"), - }; - let version = "7.13.0"; - let root = lemonade_root(&paths, Some(env_root.path())); - let runtime_dir = runtime_dir_in(&root); - fs::create_dir_all(&runtime_dir).unwrap(); - fs::write(lemond_path_in(&runtime_dir), b"fake lemond").unwrap(); - fs::write(lemonade_path_in(&runtime_dir), b"fake lemonade").unwrap(); - record_runtime_version(&runtime_dir, version).unwrap(); - let sentinel = runtime_dir.join("llamacpp-rocm-backend.sentinel"); - fs::write(&sentinel, b"a backend a prior scenario installed").unwrap(); - - let archive_bytes = b"never parsed unless extraction wrongly triggers"; - let expected_sha256 = format!("{:x}", Sha256::digest(archive_bytes)); - - let manifest = prepare_embeddable_with( - &paths, - Some(env_root.path()), - false, - version, - &expected_sha256, - |_, destination| { - fs::write(destination, archive_bytes)?; - Ok(()) - }, - ) - .unwrap(); - - assert_eq!(manifest.version, version); - assert!( - sentinel.is_file(), - "prepare_embeddable wiped the shared runtime tree instead of recognizing it as current" - ); - } - - #[test] - fn prepare_embeddable_records_the_version_after_a_real_extraction() { - // The write-side sibling of the test above: an isolated caller - // extracting into an empty `runtime_dir` for the first time must - // leave a marker behind, or the *next* isolated caller sharing this - // tree sees no marker, wipes, and re-extracts on every call — the - // same defect the read-side regression guards against, from the - // write side instead. Dropping `record_runtime_version`'s call site - // in `prepare_embeddable` fails this test. - let env_root = tempfile::tempdir().unwrap(); - let paths = AppPaths { - config_dir: env_root.path().join("config"), - data_dir: env_root.path().join("data"), - cache_dir: env_root.path().join("cache"), - }; - let version = "7.13.0"; - - // A real, well-formed archive (unlike the stub above): this test - // exercises the extraction branch, so `extract_archive` must succeed. - // `extract_archive` dispatches on the destination's extension, which - // `prepare_embeddable_with` derives from the current platform (see - // `embeddable_os_arch`) — so the bytes built here must be in that - // same format, not always a tar.gz. - let (_, extension) = embeddable_os_arch(); - let archive_dir = tempfile::tempdir().unwrap(); - let archive = archive_dir.path().join(format!("embeddable.{extension}")); - let entry_names = [ - platform_binary_name("lemond"), - platform_binary_name("lemonade"), - ]; - if runtime_is_windows() { - let file = fs::File::create(&archive).unwrap(); - let mut writer = zip::ZipWriter::new(file); - let options = zip::write::SimpleFileOptions::default(); - for name in &entry_names { - writer - .start_file(format!("embeddable/bin/{name}"), options) - .unwrap(); - writer.write_all(b"hello").unwrap(); - } - writer.finish().unwrap(); - } else { - let file = fs::File::create(&archive).unwrap(); - let encoder = flate2::write::GzEncoder::new(file, flate2::Compression::fast()); - let mut builder = tar::Builder::new(encoder); - for name in &entry_names { - let mut header = tar::Header::new_ustar(); - header.set_size(5); - header.set_mode(0o755); - header.set_entry_type(tar::EntryType::Regular); - header.set_path(format!("embeddable/bin/{name}")).unwrap(); - header.set_cksum(); - builder.append(&header, &b"hello"[..]).unwrap(); - } - builder.into_inner().unwrap().finish().unwrap(); - } - let expected_sha256 = format!("{:x}", Sha256::digest(fs::read(&archive).unwrap())); - - let root = lemonade_root(&paths, Some(env_root.path())); - let runtime_dir = runtime_dir_in(&root); - assert_eq!(installed_runtime_version(&runtime_dir), None); - - prepare_embeddable_with( - &paths, - Some(env_root.path()), - false, - version, - &expected_sha256, - |_, destination| { - fs::copy(&archive, destination)?; - Ok(()) - }, - ) - .unwrap(); - - assert_eq!( - installed_runtime_version(&runtime_dir).as_deref(), - Some(version), - "prepare_embeddable extracted the archive but never recorded the version" - ); - } +#[cfg(test)] +mod tests { + use super::*; #[test] - fn lemonade_root_uses_requested_engine_root() { - let paths = AppPaths { - config_dir: PathBuf::from("C:/Users/test/.rocm"), - data_dir: PathBuf::from("C:/Users/test/.rocm"), - cache_dir: PathBuf::from("C:/Users/test/.rocm/cache"), - }; - let engine_root = PathBuf::from("D:/path/to/therock_venvs/engines"); - - assert_eq!( - lemonade_root(&paths, Some(&engine_root)), - normalize_runtime_path_for_host(&engine_root).join(ENGINE_NAME) - ); - } - - #[test] - fn windows_child_path_maps_ape_drive_paths() { - assert_eq!( - windows_child_path(Path::new("/D/path/to/rocm-cli/file.zip")), - r"D:\path\to\rocm-cli\file.zip" - ); - assert_eq!(windows_child_path(Path::new("/c")), r"C:\"); - } - - #[test] - fn lemonade_model_load_uses_selected_backend() { - let args = lemonade_model_load_args("127.0.0.1", 11435, DEFAULT_MODEL, "vulkan", None); - assert_eq!( - args, - vec![ - "--host", - "127.0.0.1", - "--port", - "11435", - "load", - DEFAULT_MODEL, - "--llamacpp", - "vulkan", - "--save-options", - ] - ); - } - - #[test] - fn lemonade_model_load_forwards_llamacpp_recipe_flags() { - let recipe = EngineRecipeHint { - required_flags: vec![ - "--temperature".to_owned(), - "0.5".to_owned(), - "--top-p".to_owned(), - "0.25".to_owned(), - "--n-predict".to_owned(), - "128".to_owned(), - ], - ..EngineRecipeHint::default() - }; - let args = - lemonade_model_load_args("127.0.0.1", 11435, DEFAULT_MODEL, "vulkan", Some(&recipe)); - assert_eq!( - args[args.len() - 2..], - [ - "--llamacpp-args", - "--temperature 0.5 --top-p 0.25 --n-predict 128" - ] - ); - } - - #[test] - fn parses_llamacpp_backends_from_table() { - let output = "\ -Recipe Backend Status Message/Version Action ----------------------------------------------------------------------------------------------------- -kokoro cpu installable Backend is supported but not installed. lemonade backends install kokoro:cpu - metal unsupported Requires macOS - -llamacpp cpu installable Backend is supported but not installed. lemonade backends install llamacpp:cpu - metal unsupported Requires macOS - - rocm unsupported Unsupported GPU - - system unsupported Requires Linux - - vulkan installable Backend is supported but not installed. lemonade backends install llamacpp:vulkan -vllm rocm unsupported Requires Linux - -"; - let backends = parse_llamacpp_backend_statuses(output); - assert_eq!( - backends, - vec![ - ("cpu".to_owned(), "installable".to_owned()), - ("metal".to_owned(), "unsupported".to_owned()), - ("rocm".to_owned(), "unsupported".to_owned()), - ("system".to_owned(), "unsupported".to_owned()), - ("vulkan".to_owned(), "installable".to_owned()), - ] - ); - } - - #[test] - fn selects_vulkan_when_rocm_unsupported() { - let backends = vec![ - ("cpu".to_owned(), "installable".to_owned()), - ("rocm".to_owned(), "unsupported".to_owned()), - ("vulkan".to_owned(), "installable".to_owned()), - ]; - assert_eq!( - select_best_llamacpp_backend(&backends), - Some(("vulkan".to_owned(), false)) - ); - } - - #[test] - fn prefers_installed_rocm_when_supported() { - let backends = vec![ - ("rocm".to_owned(), "installed".to_owned()), - ("vulkan".to_owned(), "installable".to_owned()), - ]; - assert_eq!( - select_best_llamacpp_backend(&backends), - Some(("rocm".to_owned(), true)) - ); - } - - #[test] - fn never_selects_cpu_when_no_gpu_backend() { - // Under the GPU-required policy there is no silent CPU fallback: when only - // a CPU backend is usable, selection returns nothing and the caller fails. - let backends = vec![ - ("rocm".to_owned(), "unsupported".to_owned()), - ("cpu".to_owned(), "installable".to_owned()), - ]; - assert_eq!(select_best_llamacpp_backend(&backends), None); - } - - #[test] - fn selects_nothing_when_all_unsupported() { - let backends = vec![("rocm".to_owned(), "unsupported".to_owned())]; - assert_eq!(select_best_llamacpp_backend(&backends), None); - } - - #[test] - fn serve_gpu_resolution_fails_fast_when_no_device_usable() { - // No-device / masked-device path: the probe authoritatively reports zero - // usable GPUs, so serving is refused before any process is spawned. - let error = resolve_gpu_indices_against(&[], Some(vec![])) - .expect_err("zero usable devices must be rejected"); - assert!(error.to_string().contains("no usable AMD GPU")); - } - - #[test] - fn serve_gpu_resolution_auto_pins_first_present_device() { - // `auto` (empty request) resolves to the first present device — never an - // assumed device 0 when device 0 is not present. - assert_eq!( - resolve_gpu_indices_against(&[], Some(vec![1, 2])).unwrap(), - vec![1] - ); - } - - #[test] - fn serve_gpu_resolution_validates_explicit_index() { - // An explicitly requested present device is honored. - assert_eq!( - resolve_gpu_indices_against(&[2], Some(vec![0, 1, 2])).unwrap(), - vec![2] - ); - // A requested device that is not present is rejected rather than silently - // remapped to another GPU. - let error = resolve_gpu_indices_against(&[3], Some(vec![0, 1])) - .expect_err("absent device must be rejected"); - assert!( - error - .to_string() - .contains("requested GPU 3 is not available") - ); - } - - #[test] - fn serve_gpu_resolution_passes_through_when_unprobeable() { - // When availability cannot be determined, the request is passed through - // unchanged so serving is not blocked on that basis. - assert_eq!( - resolve_gpu_indices_against(&[], None).unwrap(), - Vec::::new() - ); - assert_eq!(resolve_gpu_indices_against(&[1], None).unwrap(), vec![1]); - } - - #[test] - fn reported_device_reflects_backend_and_pinned_ordinal() { - let state = json!({ "gpu_indices": [1] }); - assert_eq!(reported_device(Some(&state), "vulkan"), "vulkan gpu 1"); - // Without a recorded ordinal the backend is still reported truthfully. - assert_eq!(reported_device(None, "rocm"), "rocm gpu"); - } - - #[test] - fn direct_qwen_lookup_checks_huggingface_cache_env() { - let paths = AppPaths { - config_dir: PathBuf::from("config"), - data_dir: PathBuf::from("data"), - cache_dir: PathBuf::from("rocm-cache"), - }; - let roots = hf_cache_roots_from(&paths, |name| match name { - "HUGGINGFACE_HUB_CACHE" => Some(PathBuf::from("hf-hub")), - "HF_HOME" => Some(PathBuf::from("hf-home")), - "HOME" => Some(PathBuf::from("home")), - _ => None, - }); - - assert_eq!( - roots, - vec![ - PathBuf::from("hf-hub"), - PathBuf::from("hf-home").join("hub"), - PathBuf::from("rocm-cache").join("huggingface").join("hub"), - PathBuf::from("home") - .join(".cache") - .join("huggingface") - .join("hub"), - ] - ); - } - - #[test] - fn device_policy_rejects_cpu_without_fallback() { - let error = normalize_device_policy(Some(DevicePolicy::CpuOnly)) - .expect_err("cpu should be rejected") - .to_string(); - assert!(error.contains("no CPU fallback")); - } - - #[test] - fn endpoint_parser_supports_ipv6_loopback() { - assert_eq!( - parse_http_endpoint("http://[::1]:11435/v1"), - Some(("::1".to_owned(), 11435)) - ); - } - - #[test] - fn serve_http_args_preserve_runtime_selection() { - let request = ServeHttpRequest { - service_id: "svc".to_owned(), - model_ref: DEFAULT_MODEL.to_owned(), - host: "127.0.0.1".to_owned(), - port: 11435, - device_policy: DevicePolicy::GpuRequired, - gpu_indices: Vec::new(), - runtime_id: Some("runtime".to_owned()), - env_id: Some("env".to_owned()), - state_path: PathBuf::from("state.json"), - log_path: Some(PathBuf::from("service.log")), - engine_recipe: None, - }; - let args = serve_http_command_args(&request); - assert!(args.contains(&"--runtime-id".to_owned())); - assert!(args.contains(&"runtime".to_owned())); - assert!(args.contains(&"--env-id".to_owned())); - assert!(args.contains(&"env".to_owned())); - assert!(!args.iter().any(|arg| arg == "cpu")); - } - - #[test] - fn health_parser_requires_loaded_requested_model() { - let unloaded = json!({ - "status": "ok", - "model_loaded": null, - "all_models_loaded": [] - }); - assert!(!health_has_loaded_model(&unloaded, DEFAULT_MODEL, "vulkan")); - - let loaded = json!({ - "status": "ok", - "model_loaded": DEFAULT_MODEL, - "all_models_loaded": [{ - "model_name": DEFAULT_MODEL, - "recipe": "llamacpp", - "recipe_options": { - "llamacpp_backend": "vulkan" - } - }] - }); - assert!(health_has_loaded_model(&loaded, "lemonade-qwen", "vulkan")); - assert!(!health_has_loaded_model(&loaded, "lemonade-qwen", "rocm")); - } - /// Writes a `.tar.gz` whose single entry carries `entry_name` verbatim. - /// - /// The name is stamped into the raw header bytes because `tar::Builder` - /// refuses to serialize `..` or absolute paths — the archives this guards - /// against are hostile ones, which no well-behaved writer would produce. - fn write_raw_named_tar_gz(path: &Path, entry_name: &str, body: &[u8]) { - let file = fs::File::create(path).unwrap(); - let encoder = flate2::write::GzEncoder::new(file, flate2::Compression::fast()); - let mut builder = tar::Builder::new(encoder); - let mut header = tar::Header::new_ustar(); - header.set_size(body.len() as u64); - header.set_mode(0o644); - header.set_entry_type(tar::EntryType::Regular); - let name = header.as_old_mut(); - let bytes = entry_name.as_bytes(); - assert!(bytes.len() < name.name.len()); - name.name[..bytes.len()].copy_from_slice(bytes); - header.set_cksum(); - builder.append(&header, body).unwrap(); - builder.into_inner().unwrap().finish().unwrap(); - } - - const EXTRACT_PATH_TEST_DIR: &str = "ROCM_LEMONADE_EXTRACT_PATH_TEST_DIR"; - - /// Child half of [`extract_archive_does_not_execute_a_path_supplied_tar`]. - /// - /// Runs in its own process so the hostile `PATH` can be set with - /// `Command::env` — mutating `PATH` in-process would need `unsafe`, which - /// the workspace denies. - #[test] - fn extract_path_subprocess_helper() { - let Some(dir) = std::env::var_os(EXTRACT_PATH_TEST_DIR) else { - return; - }; - let dir = PathBuf::from(dir); - let _ = extract_archive(&dir.join("archive.tar.gz"), &dir.join("extract")); - } - - #[cfg(unix)] - #[test] - fn extract_archive_does_not_execute_a_path_supplied_tar() { - use std::os::unix::fs::PermissionsExt; - - let dir = scratch_dir("path-supplied-tar"); - let archive = dir.join("archive.tar.gz"); - fs::create_dir_all(dir.join("extract")).unwrap(); - write_raw_named_tar_gz(&archive, "payload.txt", b"real"); - - let fake_bin = dir.join("bin"); - fs::create_dir_all(&fake_bin).unwrap(); - let marker = dir.join("impostor-ran"); - let fake_tar = fake_bin.join("tar"); - fs::write( - &fake_tar, - format!("#!/bin/sh\ntouch '{}'\nexit 0\n", marker.display()), - ) - .unwrap(); - fs::set_permissions(&fake_tar, fs::Permissions::from_mode(0o755)).unwrap(); - - let mut entries = vec![fake_bin]; - entries.extend(std::env::split_paths( - &std::env::var_os("PATH").unwrap_or_default(), - )); - let status = ProcessCommand::new(std::env::current_exe().unwrap()) - .args([ - "--exact", - "tests::extract_path_subprocess_helper", - "--nocapture", - ]) - .env(EXTRACT_PATH_TEST_DIR, &dir) - .env("PATH", std::env::join_paths(entries).unwrap()) - .stdin(Stdio::null()) - .stdout(Stdio::null()) - .stderr(Stdio::null()) - .status() - .unwrap(); - assert!(status.success(), "helper subprocess failed"); - - assert!( - !marker.exists(), - "extraction executed a PATH-supplied tar instead of a pinned in-process extractor" - ); - fs::remove_dir_all(&dir).ok(); - } - - fn extract_hostile_tar_entry(tag: &str, entry_name: &str) -> String { - let dir = scratch_dir(tag); - let archive = dir.join("hostile.tar.gz"); - let destination = dir.join("extract"); - fs::create_dir_all(&destination).unwrap(); - write_raw_named_tar_gz(&archive, entry_name, b"pwned"); - - let error = extract_archive(&archive, &destination).unwrap_err(); - assert_eq!( - fs::read_dir(&destination).unwrap().count(), - 0, - "hostile entry was written into the extraction root" - ); - fs::remove_dir_all(&dir).ok(); - format!("{error:#}") - } - - #[test] - fn extract_archive_rejects_parent_traversal_entry() { - let error = extract_hostile_tar_entry("traversal-parent", "../escaped.txt"); - assert!(error.contains("escapes the extraction root"), "{error}"); - } - - #[test] - fn extract_archive_rejects_nested_parent_traversal_entry() { - let error = extract_hostile_tar_entry("traversal-nested", "inner/../../escaped.txt"); - assert!(error.contains("escapes the extraction root"), "{error}"); - } - - #[test] - fn extract_archive_rejects_absolute_entry() { - let error = extract_hostile_tar_entry("traversal-absolute", "/etc/escaped.txt"); - assert!(error.contains("escapes the extraction root"), "{error}"); - } - - #[test] - fn extract_archive_rejects_symlink_entry() { - let dir = scratch_dir("traversal-symlink"); - let archive = dir.join("hostile.tar.gz"); - let destination = dir.join("extract"); - fs::create_dir_all(&destination).unwrap(); - - let file = fs::File::create(&archive).unwrap(); - let encoder = flate2::write::GzEncoder::new(file, flate2::Compression::fast()); - let mut builder = tar::Builder::new(encoder); - let mut header = tar::Header::new_ustar(); - header.set_size(0); - header.set_mode(0o777); - header.set_entry_type(tar::EntryType::Symlink); - header.set_link_name("/etc/passwd").unwrap(); - header.set_path("escape-link").unwrap(); - header.set_cksum(); - builder.append(&header, std::io::empty()).unwrap(); - builder.into_inner().unwrap().finish().unwrap(); - - let error = format!("{:#}", extract_archive(&archive, &destination).unwrap_err()); - assert!(error.contains("unsupported entry"), "{error}"); - assert!(!destination.join("escape-link").exists()); - fs::remove_dir_all(&dir).ok(); - } - - #[test] - fn extract_archive_writes_a_well_formed_tree() { - let dir = scratch_dir("extract-well-formed"); - let archive = dir.join("good.tar.gz"); - let destination = dir.join("extract"); - fs::create_dir_all(&destination).unwrap(); - - let file = fs::File::create(&archive).unwrap(); - let encoder = flate2::write::GzEncoder::new(file, flate2::Compression::fast()); - let mut builder = tar::Builder::new(encoder); - let mut header = tar::Header::new_ustar(); - header.set_size(5); - header.set_mode(0o755); - header.set_entry_type(tar::EntryType::Regular); - header.set_path("embeddable/bin/lemond").unwrap(); - header.set_cksum(); - builder.append(&header, &b"hello"[..]).unwrap(); - builder.into_inner().unwrap().finish().unwrap(); - - extract_archive(&archive, &destination).unwrap(); - let extracted = destination.join("embeddable").join("bin").join("lemond"); - assert_eq!(fs::read(&extracted).unwrap(), b"hello"); - #[cfg(unix)] - { - use std::os::unix::fs::PermissionsExt; - let mode = fs::metadata(&extracted).unwrap().permissions().mode(); - assert_eq!(mode & 0o111, 0o111, "executable bit was not preserved"); - } - fs::remove_dir_all(&dir).ok(); - } - - #[test] - fn extract_archive_rejects_hostile_zip_entry() { - let dir = scratch_dir("traversal-zip"); - let archive = dir.join("hostile.zip"); - let destination = dir.join("extract"); - fs::create_dir_all(&destination).unwrap(); - - let file = fs::File::create(&archive).unwrap(); - let mut writer = zip::ZipWriter::new(file); - let options = zip::write::SimpleFileOptions::default(); - // `add_directory`/`start_file` normalize names, so the traversal name is - // written through the raw entry API a hostile producer would use. - writer - .start_file("../escaped.txt", options) - .or_else(|_| writer.start_file("placeholder.txt", options)) - .unwrap(); - writer.write_all(b"pwned").unwrap(); - writer.finish().unwrap(); - - // If the zip writer refused to emit the traversal name there is nothing - // hostile to assert on; the tar cases cover the shared validator. - let mut zip = zip::ZipArchive::new(fs::File::open(&archive).unwrap()).unwrap(); - let hostile = zip.by_index(0).unwrap().name().to_owned(); - if hostile == "../escaped.txt" { - let error = format!("{:#}", extract_archive(&archive, &destination).unwrap_err()); - assert!(error.contains("escapes the extraction root"), "{error}"); - assert!(!dir.join("escaped.txt").exists()); - } else { - extract_archive(&archive, &destination).unwrap(); - assert!(destination.join("placeholder.txt").is_file()); - } - fs::remove_dir_all(&dir).ok(); - } - - /// A pax global header is archive-wide metadata, not a member, and the - /// `tar` crate hands it to the caller rather than consuming it the way it - /// consumes long-name and pax local headers. Treating it as a member - /// rejects any archive written by `git archive` or `tar --format=pax`. - #[test] - fn extract_archive_skips_pax_global_header() { - let dir = scratch_dir("pax-global-header"); - let archive = dir.join("pax.tar.gz"); - let destination = dir.join("extract"); - fs::create_dir_all(&destination).unwrap(); - - let file = fs::File::create(&archive).unwrap(); - let encoder = flate2::write::GzEncoder::new(file, flate2::Compression::fast()); - let mut builder = tar::Builder::new(encoder); - - let record = b"52 comment=0000000000000000000000000000000000000000\n"; - let mut global = tar::Header::new_ustar(); - global.set_size(record.len() as u64); - global.set_mode(0o644); - global.set_entry_type(tar::EntryType::XGlobalHeader); - global.set_path("pax_global_header").unwrap(); - global.set_cksum(); - builder.append(&global, &record[..]).unwrap(); - - let mut header = tar::Header::new_ustar(); - header.set_size(5); - header.set_mode(0o755); - header.set_entry_type(tar::EntryType::Regular); - header.set_path("embeddable/bin/lemond").unwrap(); - header.set_cksum(); - builder.append(&header, &b"hello"[..]).unwrap(); - builder.into_inner().unwrap().finish().unwrap(); - - extract_archive(&archive, &destination).unwrap(); - assert_eq!( - fs::read(destination.join("embeddable").join("bin").join("lemond")).unwrap(), - b"hello" - ); - assert!( - !destination.join("pax_global_header").exists(), - "the global header was written out as a member" - ); - fs::remove_dir_all(&dir).ok(); - } - - /// Zip counterpart to `extract_archive_rejects_symlink_entry`: the zip path - /// has its own link check, keyed on the entry's unix mode. - #[test] - fn extract_archive_rejects_zip_symlink_entry() { - let dir = scratch_dir("zip-symlink"); - let archive = dir.join("hostile.zip"); - let destination = dir.join("extract"); - fs::create_dir_all(&destination).unwrap(); - - let file = fs::File::create(&archive).unwrap(); - let mut writer = zip::ZipWriter::new(file); - writer - .add_symlink( - "escape-link", - "/etc/passwd", - zip::write::SimpleFileOptions::default(), - ) - .unwrap(); - writer.finish().unwrap(); - - let error = format!("{:#}", extract_archive(&archive, &destination).unwrap_err()); - assert!(error.contains("symlink"), "{error}"); - assert!(!destination.join("escape-link").exists()); - fs::remove_dir_all(&dir).ok(); - } - - // `XDG_RUNTIME_DIR` is a Linux/freedesktop concept and the Lemonade server - // only consults it there, so these are Linux-only by construction. - #[cfg(target_os = "linux")] - mod child_runtime_dir { - use super::*; - - fn value_of(vars: &[(&'static str, OsString)], key: &str) -> Option { - vars.iter() - .find(|(name, _)| *name == key) - .map(|(_, value)| value.clone()) - } - - fn runtime_scratch() -> tempfile::TempDir { - tempfile::Builder::new() - .prefix(".lemonade-runtime-test-") - .tempdir_in(std::env::current_dir().expect("current directory")) - .expect("private-ancestry scratch") - } - fn vars_for(parent: &ParentRuntimeEnvironment) -> Vec<(&'static str, OsString)> { - lemonade_process_environment_vars(&LemonadeProcessEnvironment::default(), parent) - .expect("building the child environment must succeed") - } - - fn assert_private_dir(dir: &Path) { - use std::os::unix::fs::PermissionsExt; - assert!( - dir.is_dir(), - "{} must exist before the server is spawned", - dir.display() - ); - let mode = fs::metadata(dir).expect("metadata").permissions().mode() & 0o777; - assert_eq!( - mode, - 0o700, - "{} must be private to this user", - dir.display() - ); - } - - /// Regression: on a headless host `pam_systemd` never runs, so - /// `XDG_RUNTIME_DIR` is unset and the Lemonade server exits during - /// startup with "Unable to resolve writable runtime directory". The - /// child must be handed a directory that already exists. - #[test] - fn is_synthesized_when_the_parent_has_none() { - let scratch = runtime_scratch(); - let home = scratch.path().join("home"); - let parent = ParentRuntimeEnvironment { - xdg_runtime_dir: None, - home: Some(home.clone().into_os_string()), - user: Some("alice".to_owned()), - temp_dir: scratch.path().join("tmp"), - }; - - let value = value_of(&vars_for(&parent), "XDG_RUNTIME_DIR") - .expect("the child must be given a runtime directory"); - - // Tier 2 of the shared precedence, with a `lemonade` leaf so the - // engine never shares a directory with the telemetry socket. - let dir = PathBuf::from(value); - assert_eq!(dir, home.join(".rocm").join("data").join("lemonade")); - assert_private_dir(&dir); - } - - /// An exported-but-empty variable is as good as unset, and must not be - /// mistaken for an operator's choice. - #[test] - fn is_synthesized_when_the_parent_value_is_empty() { - let scratch = runtime_scratch(); - let parent = ParentRuntimeEnvironment { - xdg_runtime_dir: Some(OsString::new()), - home: Some(scratch.path().join("home").into_os_string()), - user: Some("alice".to_owned()), - temp_dir: scratch.path().join("tmp"), - }; - - let value = value_of(&vars_for(&parent), "XDG_RUNTIME_DIR") - .expect("an empty value must fall through, not be passed on"); - assert_eq!( - PathBuf::from(value), - scratch - .path() - .join("home") - .join(".rocm") - .join("data") - .join("lemonade") - ); - } - - /// Tier 2 lives under the operator's HOME and must retain normal path - /// semantics; homes (or `.rocm`) are legitimately symlinked on some - /// systems. The no-follow hardening is intentionally tier-3-only. - #[test] - fn permits_a_symlinked_home_path_for_tier2() { - let scratch = runtime_scratch(); - let real_home = scratch.path().join("real-home"); - let linked_home = scratch.path().join("linked-home"); - fs::create_dir_all(&real_home).expect("real home"); - std::os::unix::fs::symlink(&real_home, &linked_home).expect("linked home"); - let parent = ParentRuntimeEnvironment { - xdg_runtime_dir: None, - home: Some(linked_home.clone().into_os_string()), - user: Some("alice".to_owned()), - temp_dir: scratch.path().join("tmp"), - }; - - let value = value_of(&vars_for(&parent), "XDG_RUNTIME_DIR") - .expect("the child must be given a runtime directory"); - - let dir = PathBuf::from(value); - assert_eq!(dir, linked_home.join(".rocm").join("data").join("lemonade")); - assert_private_dir(&dir); - assert!(real_home.join(".rocm/data/lemonade").is_dir()); - } - /// Tier 3: neither `XDG_RUNTIME_DIR` nor `HOME` — a user-named subdir of - /// the temp dir, so the parent is one we create and own, not `/tmp`. - #[test] - fn falls_back_to_a_user_owned_temp_subdir_without_home() { - let scratch = runtime_scratch(); - let temp_dir = scratch.path().join("tmp"); - let parent = ParentRuntimeEnvironment { - xdg_runtime_dir: None, - home: None, - user: Some("alice".to_owned()), - temp_dir: temp_dir.clone(), - }; - - let value = value_of(&vars_for(&parent), "XDG_RUNTIME_DIR") - .expect("the child must be given a runtime directory"); - - let dir = PathBuf::from(value); - assert_eq!(dir, temp_dir.join("rocm-alice").join("lemonade")); - assert_private_dir(&dir); - } - - /// The tier-3 fallback lands under a shared temp dir, so the target may - /// already exist as a symlink someone else planted. Chmod would follow - /// it, so the resolver refuses rather than loosening another directory. - #[test] - fn refuses_a_symlinked_runtime_dir() { - let scratch = runtime_scratch(); - let temp_dir = scratch.path().join("tmp"); - let elsewhere = scratch.path().join("elsewhere"); - fs::create_dir_all(temp_dir.join("rocm-alice")).expect("runtime root"); - fs::create_dir_all(&elsewhere).expect("elsewhere"); - std::os::unix::fs::symlink(&elsewhere, temp_dir.join("rocm-alice").join("lemonade")) - .expect("symlink"); - - let parent = ParentRuntimeEnvironment { - xdg_runtime_dir: None, - home: None, - user: Some("alice".to_owned()), - temp_dir, - }; - let error = format!( - "{:#}", - lemonade_process_environment_vars(&LemonadeProcessEnvironment::default(), &parent) - .expect_err("a symlinked runtime dir must be refused") - ); - assert!(error.contains("symlink"), "{error}"); - } - - /// Tier 3 uses a predictable user parent under a shared temp directory. - /// Refuse that parent when it is a symlink: recursively creating the - /// leaf would otherwise follow it into an attacker-controlled tree. - #[test] - fn refuses_a_symlinked_tier3_user_parent() { - let scratch = runtime_scratch(); - let temp_dir = scratch.path().join("tmp"); - let elsewhere = scratch.path().join("elsewhere"); - fs::create_dir_all(&temp_dir).expect("temp root"); - fs::create_dir_all(&elsewhere).expect("elsewhere"); - std::os::unix::fs::symlink(&elsewhere, temp_dir.join("rocm-alice")).expect("symlink"); - - let parent = ParentRuntimeEnvironment { - xdg_runtime_dir: None, - home: None, - user: Some("alice".to_owned()), - temp_dir, - }; - let error = format!( - "{:#}", - lemonade_process_environment_vars(&LemonadeProcessEnvironment::default(), &parent) - .expect_err("a symlinked tier-3 user parent must be refused") - ); - - assert!(error.contains("symlink"), "{error}"); - assert!( - !elsewhere.join("lemonade").exists(), - "the resolver followed the tier-3 user-parent symlink" - ); - } - - /// A pre-existing tier-3 user parent may have been created with a - /// permissive umask. The resolver must secure that predictable parent, - /// not merely the Lemonade leaf beneath it. - #[test] - fn tightens_a_preexisting_non_private_tier3_user_parent() { - use std::os::unix::fs::PermissionsExt; - - let scratch = runtime_scratch(); - let temp_dir = scratch.path().join("tmp"); - let user_parent = temp_dir.join("rocm-alice"); - fs::create_dir_all(&user_parent).expect("user parent"); - fs::set_permissions(&user_parent, fs::Permissions::from_mode(0o755)) - .expect("make parent non-private"); - let parent = ParentRuntimeEnvironment { - xdg_runtime_dir: None, - home: None, - user: Some("alice".to_owned()), - temp_dir, - }; - - let value = value_of(&vars_for(&parent), "XDG_RUNTIME_DIR") - .expect("the child must be given a runtime directory"); - - assert_eq!(PathBuf::from(value), user_parent.join("lemonade")); - assert_private_dir(&user_parent); - assert_private_dir(&user_parent.join("lemonade")); - } - - /// A tier-3 root without sticky-directory protection lets another UID - /// rename or replace our predictable user component after validation. - #[test] - fn refuses_a_group_or_other_writable_non_sticky_temp_root() { - use std::os::unix::fs::PermissionsExt; - - let scratch = runtime_scratch(); - let temp_dir = scratch.path().join("unsafe-tmp"); - fs::create_dir(&temp_dir).expect("temp root"); - fs::set_permissions(&temp_dir, fs::Permissions::from_mode(0o777)) - .expect("make root unsafe"); - let parent = ParentRuntimeEnvironment { - xdg_runtime_dir: None, - home: None, - user: Some("alice".to_owned()), - temp_dir: temp_dir.clone(), - }; - - let error = format!( - "{:#}", - lemonade_process_environment_vars(&LemonadeProcessEnvironment::default(), &parent) - .expect_err("a non-sticky writable temp root must be refused") - ); - - assert!(error.contains("sticky"), "{error}"); - assert!(!temp_dir.join("rocm-alice").exists()); - } - - /// A private final temp root is still unstable when an ancestor lets - /// another UID replace the path entry that names it. - #[test] - fn refuses_a_private_temp_root_beneath_a_non_sticky_writable_parent() { - use std::os::unix::fs::PermissionsExt; - - let scratch = runtime_scratch(); - let unsafe_parent = scratch.path().join("unsafe-parent"); - let temp_dir = unsafe_parent.join("private-tmp"); - fs::create_dir(&unsafe_parent).expect("unsafe parent"); - fs::set_permissions(&unsafe_parent, fs::Permissions::from_mode(0o777)) - .expect("make ancestor unsafe"); - fs::create_dir(&temp_dir).expect("private temp root"); - fs::set_permissions(&temp_dir, fs::Permissions::from_mode(0o700)) - .expect("make final root private"); - let parent = ParentRuntimeEnvironment { - xdg_runtime_dir: None, - home: None, - user: Some("alice".to_owned()), - temp_dir: temp_dir.clone(), - }; - - let error = format!( - "{:#}", - lemonade_process_environment_vars(&LemonadeProcessEnvironment::default(), &parent) - .expect_err("unsafe temp-root ancestry must be refused") - ); - - assert!(error.contains("ancestor"), "{error}"); - assert!(!temp_dir.join("rocm-alice").exists()); - } - - #[test] - fn accepts_a_private_temp_root_with_private_ancestry() { - use std::os::unix::fs::PermissionsExt; - - let scratch = runtime_scratch(); - let private_parent = scratch.path().join("private-parent"); - let temp_dir = private_parent.join("private-tmp"); - fs::create_dir(&private_parent).expect("private parent"); - fs::set_permissions(&private_parent, fs::Permissions::from_mode(0o700)) - .expect("private ancestor mode"); - fs::create_dir(&temp_dir).expect("private temp root"); - fs::set_permissions(&temp_dir, fs::Permissions::from_mode(0o700)) - .expect("private root mode"); - let parent = ParentRuntimeEnvironment { - xdg_runtime_dir: None, - home: None, - user: Some("alice".to_owned()), - temp_dir: temp_dir.clone(), - }; - - let value = value_of(&vars_for(&parent), "XDG_RUNTIME_DIR") - .expect("private ancestry must be accepted"); - - assert_eq!(PathBuf::from(value), temp_dir.join("rocm-alice/lemonade")); - } - - #[test] - fn accepts_standard_sticky_temp_root_semantics() { - use std::os::unix::fs::PermissionsExt; - - let scratch = runtime_scratch(); - let temp_dir = scratch.path().join("sticky-tmp"); - fs::create_dir(&temp_dir).expect("sticky temp root"); - fs::set_permissions(&temp_dir, fs::Permissions::from_mode(0o1777)) - .expect("standard sticky temp mode"); - let parent = ParentRuntimeEnvironment { - xdg_runtime_dir: None, - home: None, - user: Some("alice".to_owned()), - temp_dir: temp_dir.clone(), - }; - - let value = value_of(&vars_for(&parent), "XDG_RUNTIME_DIR") - .expect("trusted sticky temp semantics must be accepted"); - - assert_eq!(PathBuf::from(value), temp_dir.join("rocm-alice/lemonade")); - assert_private_dir(&temp_dir.join("rocm-alice")); - } - - #[test] - fn accepts_the_host_standard_tmp_when_its_invariants_are_safe() { - use std::os::fd::AsRawFd; - use std::os::unix::fs::MetadataExt; - - let path = Path::new("/tmp"); - let metadata = fs::metadata(path).expect("/tmp metadata"); - let directory = fs::File::open(path).expect("open /tmp"); - if !directory_prevents_cross_uid_entry_replacement( - metadata.uid(), - metadata.mode(), - effective_user_id(), - filesystem_is_read_only(directory.as_raw_fd()).expect("/tmp filesystem flags"), - ) { - return; - } - - open_safe_tier3_root(path).expect("safe host /tmp must be accepted"); - } - - #[test] - fn refuses_a_relative_tier3_temp_root() { - let parent = ParentRuntimeEnvironment { - xdg_runtime_dir: None, - home: None, - user: Some("alice".to_owned()), - temp_dir: PathBuf::from("relative-tmp"), - }; - - let error = format!( - "{:#}", - lemonade_process_environment_vars(&LemonadeProcessEnvironment::default(), &parent) - .expect_err("relative tier-3 roots must be refused") - ); - - assert!(error.contains("relative"), "{error}"); - } - - #[test] - fn refuses_symlinked_temp_root_ancestry() { - let scratch = runtime_scratch(); - let real_parent = scratch.path().join("real-parent"); - let linked_parent = scratch.path().join("linked-parent"); - fs::create_dir(&real_parent).expect("real parent"); - std::os::unix::fs::symlink(&real_parent, &linked_parent).expect("linked ancestry"); - let parent = ParentRuntimeEnvironment { - xdg_runtime_dir: None, - home: None, - user: Some("alice".to_owned()), - temp_dir: linked_parent.join("private-tmp"), - }; - - let error = format!( - "{:#}", - lemonade_process_environment_vars(&LemonadeProcessEnvironment::default(), &parent) - .expect_err("symlinked tier-3 ancestry must be refused") - ); - - assert!(error.contains("ancestry"), "{error}"); - assert!(!real_parent.join("private-tmp").exists()); - } - - /// Sticky mode does not constrain the directory owner. A sticky root - /// owned by an unrelated non-root UID therefore cannot protect our - /// component from that UID replacing it. - #[test] - fn refuses_an_untrusted_foreign_owned_sticky_temp_root_when_available() { - use std::os::unix::fs::MetadataExt; - - let current_euid = effective_user_id(); - let candidate = [Path::new("/tmp"), Path::new("/var/tmp")] - .into_iter() - .filter_map(|path| fs::metadata(path).ok().map(|metadata| (path, metadata))) - .find(|(_, metadata)| { - metadata.uid() != 0 - && metadata.uid() != current_euid - && metadata.mode() & libc::S_ISVTX != 0 - }); - let Some((path, _)) = candidate else { - return; - }; - - let error = format!( - "{:#}", - open_safe_tier3_root(path) - .expect_err("an untrusted sticky-root owner must be refused") - ); - assert!(error.contains("sticky"), "{error}"); - assert!(error.contains("owner"), "{error}"); - } - - #[test] - fn ancestry_safety_rejects_foreign_owned_mode_0555_on_writable_filesystem() { - assert!(!directory_prevents_cross_uid_entry_replacement( - 1234, 0o555, 1000, false - )); - } - - #[test] - fn ancestry_safety_rejects_foreign_owned_mode_0755() { - assert!(!directory_prevents_cross_uid_entry_replacement( - 1234, 0o755, 1000, false - )); - } - - #[test] - fn ancestry_safety_accepts_foreign_owned_writable_on_read_only_filesystem() { - assert!(directory_prevents_cross_uid_entry_replacement( - 1234, 0o777, 1000, true - )); - } - - #[test] - fn ancestry_safety_accepts_trusted_owned_mode_0755() { - assert!(directory_prevents_cross_uid_entry_replacement( - 0, 0o755, 1000, false - )); - } - - #[test] - fn ancestry_safety_rejects_trusted_owned_mode_0777_without_sticky() { - assert!(!directory_prevents_cross_uid_entry_replacement( - 1000, 0o777, 1000, false - )); - } - - #[test] - fn ancestry_safety_accepts_trusted_owned_mode_1777() { - assert!(directory_prevents_cross_uid_entry_replacement( - 1000, 0o1777, 1000, false - )); - } - - /// Once the temp root is open, replacing its pathname must not redirect - /// creation into the replacement tree. This deterministically models - /// the rename-and-replace window from the original implementation. - #[test] - fn descriptor_relative_creation_resists_temp_root_replacement() { - let scratch = runtime_scratch(); - let temp_dir = scratch.path().join("tmp"); - fs::create_dir(&temp_dir).expect("temp root"); - let root = open_safe_tier3_root(&temp_dir).expect("open root"); - - let original_root = scratch.path().join("original-tmp"); - fs::rename(&temp_dir, &original_root).expect("rename open root"); - fs::create_dir(&temp_dir).expect("replacement root"); - let elsewhere = scratch.path().join("elsewhere"); - fs::create_dir(&elsewhere).expect("elsewhere"); - std::os::unix::fs::symlink(&elsewhere, temp_dir.join("rocm-alice")) - .expect("attacker replacement"); - - create_private_owned_directory_at( - &root, - OsStr::new("rocm-alice"), - &temp_dir.join("rocm-alice"), - ) - .expect("descriptor-relative creation"); - - assert_private_dir(&original_root.join("rocm-alice")); - assert!( - fs::symlink_metadata(temp_dir.join("rocm-alice")) - .expect("replacement symlink") - .file_type() - .is_symlink() - ); - assert!(!elsewhere.join("rocm-alice").exists()); - } - - /// Leaf creation stays anchored to the validated user-parent object even - /// if its pathname is replaced with a symlink after it was opened. - #[test] - fn descriptor_relative_leaf_creation_resists_user_parent_replacement() { - let scratch = runtime_scratch(); - let temp_dir = scratch.path().join("tmp"); - fs::create_dir(&temp_dir).expect("temp root"); - let root = open_safe_tier3_root(&temp_dir).expect("open root"); - let user_path = temp_dir.join("rocm-alice"); - let user_parent = - create_private_owned_directory_at(&root, OsStr::new("rocm-alice"), &user_path) - .expect("create user parent"); - - let original_user_parent = temp_dir.join("original-rocm-alice"); - fs::rename(&user_path, &original_user_parent).expect("rename user parent"); - let elsewhere = scratch.path().join("elsewhere"); - fs::create_dir(&elsewhere).expect("elsewhere"); - std::os::unix::fs::symlink(&elsewhere, &user_path).expect("replacement symlink"); - - create_private_owned_directory_at( - &user_parent, - OsStr::new("lemonade"), - &user_path.join("lemonade"), - ) - .expect("descriptor-relative leaf creation"); - - assert_private_dir(&original_user_parent.join("lemonade")); - assert!(!elsewhere.join("lemonade").exists()); - } - - /// Ownership can be exercised without root when the host exposes a - /// system-owned temp/root directory. The preparation routine must - /// reject it before changing its permissions. - #[test] - fn rejects_a_foreign_owned_tier3_user_parent_when_available() { - use std::os::unix::fs::{MetadataExt, OpenOptionsExt}; - - let current_euid = effective_user_id(); - let foreign = [ - (Path::new("/"), OsStr::new("tmp")), - (Path::new("/"), OsStr::new("var")), - (Path::new("/"), OsStr::new("run")), - ] - .into_iter() - .filter_map(|(parent, component)| { - let path = parent.join(component); - fs::metadata(&path) - .ok() - .map(|metadata| (parent, component, path, metadata)) - }) - .find(|(_, _, _, metadata)| metadata.is_dir() && metadata.uid() != current_euid); - let Some((parent_path, component, path, metadata)) = foreign else { - return; - }; - let original_mode = metadata.mode(); - let parent = fs::OpenOptions::new() - .read(true) - .custom_flags(libc::O_DIRECTORY | libc::O_NOFOLLOW) - .open(parent_path) - .expect("open foreign parent"); - - let error = format!( - "{:#}", - create_private_owned_directory_at(&parent, component, &path) - .expect_err("foreign ownership must be refused") - ); - assert!(error.contains("owned by uid"), "{error}"); - assert!(error.contains(¤t_euid.to_string()), "{error}"); - assert_eq!( - fs::metadata(&path).expect("metadata after refusal").mode(), - original_mode, - "foreign directory permissions changed before ownership was validated" - ); - } - /// A value the operator (or systemd) already set is handed to the child - /// unchanged: we never relocate a working runtime directory, and never - /// create anything of our own beside it. - #[test] - fn passes_an_existing_parent_value_through_unchanged() { - let scratch = tempfile::tempdir().expect("scratch"); - let existing = scratch.path().join("run").join("user").join("1000"); - let home = scratch.path().join("home"); - let parent = ParentRuntimeEnvironment { - xdg_runtime_dir: Some(existing.clone().into_os_string()), - home: Some(home.clone().into_os_string()), - user: Some("alice".to_owned()), - temp_dir: scratch.path().join("tmp"), - }; - - let value = value_of(&vars_for(&parent), "XDG_RUNTIME_DIR") - .expect("the child must be given a runtime directory"); - - assert_eq!(PathBuf::from(value), existing); - assert!( - !home.exists(), - "no fallback directory may be created when the parent already has one" - ); - } + fn stop_scope_targets_only_the_recorded_lemonade_server() { + assert_eq!(STOP_SCOPE, rocm_core::KillScope::Single); } } diff --git a/engines/lemonade/src/process.rs b/engines/lemonade/src/process.rs new file mode 100644 index 0000000000..ce89a1f478 --- /dev/null +++ b/engines/lemonade/src/process.rs @@ -0,0 +1,924 @@ +// Copyright © Advanced Micro Devices, Inc., or its affiliates. +// +// SPDX-License-Identifier: MIT + +use anyhow::{Context, Result, bail}; +use rocm_core::{ + AppPaths, RocmCliConfig, active_managed_therock_environment, format_http_base_url, + http_get_text_with_auth, prepend_runtime_paths, runtime_is_linux, runtime_is_windows, +}; +use rocm_engine_protocol::EngineRecipeHint; +use serde_json::{Value, json}; +use std::ffi::OsString; +use std::fs; +use std::io::Write; +use std::path::{Path, PathBuf}; +use std::process::{Command as ProcessCommand, Stdio}; +use std::time::Duration; + +use crate::backend_alignment::LemonadeRuntime; +use crate::direct_llama::{direct_llama_model_path, llama_server_backend_label}; +use crate::install::LemonadeInstallManifest; +use crate::runtime_dir::{ParentRuntimeEnvironment, child_runtime_dir_var}; +use crate::state::{ + mark_json_status, merge_json_state, parse_http_endpoint, resolve_lemonade_model_ref, + tail_lines, terminate_pid, write_running_state, +}; +use crate::{LLAMACPP_RECIPE, ServeHttpRequest}; + +const STARTUP_FAILURE_LOG_TAIL_LINES: usize = 80; + +#[derive(Debug, Clone, Default)] +pub(crate) struct LemonadeProcessEnvironment { + rocm_root: Option, + path_entries: Vec, + library_entries: Vec, + pub(crate) gpu_indices: Vec, +} + +fn windows_child_path(path: &Path) -> String { + let raw = path.display().to_string(); + let normalized = raw.replace('\\', "/"); + let bytes = normalized.as_bytes(); + if bytes.len() >= 3 && bytes[0] == b'/' && bytes[1].is_ascii_alphabetic() && bytes[2] == b'/' { + let drive = (bytes[1] as char).to_ascii_uppercase(); + let rest = normalized[3..].replace('/', "\\"); + return format!("{drive}:\\{rest}"); + } + if bytes.len() == 2 && bytes[0] == b'/' && bytes[1].is_ascii_alphabetic() { + let drive = (bytes[1] as char).to_ascii_uppercase(); + return format!("{drive}:\\"); + } + raw +} + +pub(crate) fn spawn_lemond( + manifest: &LemonadeInstallManifest, + host: &str, + port: u16, + log_path: Option<&Path>, + process_env: &LemonadeProcessEnvironment, +) -> Result { + #[cfg(windows)] + if let Some(log_path) = log_path { + let args = vec![ + child_process_path(&manifest.runtime_dir), + "--host".to_owned(), + host.to_owned(), + "--port".to_owned(), + port.to_string(), + ]; + let pid = rocm_core::spawn_hidden_console_with_log(&manifest.lemond, &args, &[], log_path) + .with_context(|| format!("failed to start {}", manifest.lemond.display()))?; + return Ok(LemondChild::Pid(pid)); + } + + let mut command = ProcessCommand::new(&manifest.lemond); + command + .arg(child_process_path(&manifest.runtime_dir)) + .arg("--host") + .arg(host) + .arg("--port") + .arg(port.to_string()) + .stdin(Stdio::null()); + apply_lemonade_process_environment(&mut command, process_env)?; + if let Some(log_path) = log_path { + if let Some(parent) = log_path.parent() { + fs::create_dir_all(parent)?; + } + let log = fs::File::create(log_path) + .with_context(|| format!("failed to create {}", log_path.display()))?; + command.stdout(Stdio::from(log.try_clone()?)); + command.stderr(Stdio::from(log)); + } else { + command.stdout(Stdio::inherit()).stderr(Stdio::inherit()); + } + hide_child_console_window(&mut command); + command + .spawn() + .with_context(|| format!("failed to start {}", manifest.lemond.display())) + .map(LemondChild::Child) +} + +pub(crate) enum LemondChild { + Child(std::process::Child), + #[cfg(windows)] + Pid(u32), +} + +impl LemondChild { + pub(crate) fn id(&self) -> u32 { + match self { + Self::Child(child) => child.id(), + #[cfg(windows)] + Self::Pid(pid) => *pid, + } + } + + pub(crate) fn wait(&mut self) -> Result { + match self { + Self::Child(child) => { + let status = child.wait().context("failed waiting for Lemonade server")?; + Ok(LemondExitStatus { + success: status.success(), + description: status.to_string(), + }) + } + #[cfg(windows)] + Self::Pid(pid) => { + let code = rocm_core::wait_for_process_exit(*pid)?; + Ok(LemondExitStatus { + success: code == 0, + description: format!("exit code {code}"), + }) + } + } + } +} + +pub(crate) struct LemondExitStatus { + success: bool, + description: String, +} + +impl LemondExitStatus { + pub(crate) const fn success(&self) -> bool { + self.success + } +} + +impl std::fmt::Display for LemondExitStatus { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + formatter.write_str(&self.description) + } +} + +#[cfg(windows)] +pub(crate) fn hide_child_console_window(command: &mut ProcessCommand) { + use std::os::windows::process::CommandExt; + + const CREATE_NO_WINDOW: u32 = 0x0800_0000; + command.creation_flags(CREATE_NO_WINDOW); +} + +#[cfg(not(windows))] +pub(crate) const fn hide_child_console_window(_command: &mut ProcessCommand) {} + +fn child_process_path(path: &Path) -> String { + if runtime_is_windows() { + windows_child_path(path) + } else { + path.display().to_string() + } +} + +pub(crate) fn lemonade_process_environment() -> Result { + let paths = AppPaths::discover()?; + let config = RocmCliConfig::load(&paths).unwrap_or_default(); + let Some(env) = active_managed_therock_environment(&paths, &config)? else { + return Ok(LemonadeProcessEnvironment::default()); + }; + Ok(LemonadeProcessEnvironment { + rocm_root: env.rocm_root, + path_entries: env.path_entries, + library_entries: env.library_entries, + gpu_indices: Vec::new(), + }) +} + +pub(crate) fn apply_lemonade_process_environment( + command: &mut ProcessCommand, + env: &LemonadeProcessEnvironment, +) -> Result<()> { + let vars = lemonade_process_environment_vars(env, &ParentRuntimeEnvironment::current())?; + apply_environment_vars(command, &vars); + Ok(()) +} + +/// Apply an already-resolved variable set. Callers that spawn repeatedly (the +/// readiness poll) resolve once and reuse, so preparing the runtime directory +/// does not repeat its syscalls on every attempt. +fn apply_environment_vars(command: &mut ProcessCommand, vars: &[(&'static str, OsString)]) { + for (key, value) in vars { + command.env(key, value); + } +} + +pub(crate) fn lemonade_process_environment_vars( + env: &LemonadeProcessEnvironment, + parent: &ParentRuntimeEnvironment, +) -> Result> { + let mut vars = Vec::new(); + if let Some(rocm_root) = env.rocm_root.as_ref() { + vars.push(("ROCM_PATH", rocm_root.as_os_str().to_owned())); + } + if let Some(runtime_dir) = child_runtime_dir_var(parent)? { + vars.push(runtime_dir); + } + let mut path_entries = env.path_entries.clone(); + if runtime_is_windows() { + path_entries.extend(env.library_entries.iter().cloned()); + } + if let Some(path) = prepend_runtime_paths(&path_entries, std::env::var_os("PATH"))? { + vars.push(("PATH", path)); + } + if runtime_is_linux() + && let Some(ld_library_path) = + prepend_runtime_paths(&env.library_entries, std::env::var_os("LD_LIBRARY_PATH"))? + { + vars.push(("LD_LIBRARY_PATH", ld_library_path)); + } + if let Some(csv) = rocm_engine_protocol::gpu_indices_to_csv(&env.gpu_indices) { + vars.push(("HIP_VISIBLE_DEVICES", OsString::from(csv))); + } + // When `rocm serve` protects a public endpoint, the key arrives via + // `ROCM_SERVE_API_KEY[_FILE]`. Lemonade's server gates /api,/v0,/v1 on + // `LEMONADE_API_KEY`, and its own CLI clients (used here for the readiness + // `status` probe) auto-present the same env var — so translating it once here + // both secures the server and keeps our readiness checks authenticated. + if let Some(api_key) = rocm_engine_protocol::resolve_endpoint_api_key() { + vars.push(("LEMONADE_API_KEY", OsString::from(api_key))); + } + Ok(vars) +} + +fn push_existing_path(paths: &mut Vec, path: PathBuf) { + if !path.exists() || paths.iter().any(|existing| existing == &path) { + return; + } + paths.push(path); +} + +pub(crate) fn wait_for_lemonade_cli_status( + manifest: &LemonadeInstallManifest, + host: &str, + port: u16, + timeout: Duration, + log_path: Option<&Path>, + process_env: &LemonadeProcessEnvironment, +) -> Result<()> { + let start = std::time::Instant::now(); + let mut last_status = None; + // Resolved once: the poll runs twice a second, and resolving also prepares + // the child's runtime directory on disk. + let env_vars = + lemonade_process_environment_vars(process_env, &ParentRuntimeEnvironment::current())?; + while start.elapsed() < timeout { + let mut command = ProcessCommand::new(&manifest.lemonade); + command + .arg("--host") + .arg(host) + .arg("--port") + .arg(port.to_string()) + .arg("status") + .stdin(Stdio::null()) + .stdout(Stdio::null()) + .stderr(Stdio::null()); + apply_environment_vars(&mut command, &env_vars); + hide_child_console_window(&mut command); + match command.status() { + Ok(status) if status.success() => return Ok(()), + Ok(status) => last_status = Some(status.to_string()), + Err(error) => last_status = Some(error.to_string()), + } + std::thread::sleep(Duration::from_millis(500)); + } + let startup_log_summary = summarize_startup_log_tail(log_path, STARTUP_FAILURE_LOG_TAIL_LINES); + bail!( + "Lemonade server did not become ready: {}; {}", + last_status.unwrap_or_else(|| "not checked".to_owned()), + startup_log_summary + ) +} + +fn summarize_startup_log_tail(log_path: Option<&Path>, limit: usize) -> String { + let Some(log_path) = log_path else { + return "no Lemonade startup log path was configured".to_owned(); + }; + if !log_path.is_file() { + return format!("Lemonade startup log not found at {}", log_path.display()); + } + match tail_lines(log_path, limit) { + Ok(lines) if lines.is_empty() => { + format!("Lemonade startup log {} is empty", log_path.display()) + } + Ok(lines) => format!( + "Lemonade startup log tail ({}):\n{}", + log_path.display(), + lines.join("\n") + ), + Err(error) => format!( + "failed to read Lemonade startup log {}: {error}", + log_path.display() + ), + } +} + +pub(crate) fn run_lemonade_backend_install( + manifest: &LemonadeInstallManifest, + host: &str, + port: u16, + backend: &str, + process_env: &LemonadeProcessEnvironment, +) -> Result<()> { + let mut command = ProcessCommand::new(&manifest.lemonade); + command + .arg("--host") + .arg(host) + .arg("--port") + .arg(port.to_string()) + .arg("backends") + .arg("install") + .arg(format!("{LLAMACPP_RECIPE}:{backend}")) + .stdin(Stdio::null()) + .stdout(Stdio::inherit()) + .stderr(Stdio::inherit()); + apply_lemonade_process_environment(&mut command, process_env)?; + hide_child_console_window(&mut command); + let status = command + .status() + .with_context(|| format!("failed to run {}", manifest.lemonade.display()))?; + if !status.success() { + bail!("Lemonade backend install failed with status {status}"); + } + Ok(()) +} + +fn lemonade_pull_args(host: &str, port: u16, checkpoint_ref: &str) -> Vec { + vec![ + "--host".to_owned(), + host.to_owned(), + "--port".to_owned(), + port.to_string(), + "pull".to_owned(), + checkpoint_ref.to_owned(), + ] +} + +/// Download a canonical Hugging Face checkpoint (`owner/repo:variant`) through Lemonade +/// so its GGUF lands in the HF hub cache. Runs non-interactively, so a `:variant` is +/// required — a bare `owner/repo` triggers Lemonade's interactive variant menu, which +/// cannot be answered here. +pub(crate) fn run_lemonade_pull( + manifest: &LemonadeInstallManifest, + host: &str, + port: u16, + checkpoint_ref: &str, + log_path: Option<&Path>, + process_env: &LemonadeProcessEnvironment, +) -> Result<()> { + let mut command = ProcessCommand::new(&manifest.lemonade); + command + .args(lemonade_pull_args(host, port, checkpoint_ref)) + .stdin(Stdio::null()); + if let Some(log_path) = log_path { + let log = fs::OpenOptions::new() + .create(true) + .append(true) + .open(log_path) + .with_context(|| format!("failed to open {}", log_path.display()))?; + command.stdout(Stdio::from(log.try_clone()?)); + command.stderr(Stdio::from(log)); + } else { + command.stdout(Stdio::inherit()).stderr(Stdio::inherit()); + } + apply_lemonade_process_environment(&mut command, process_env)?; + hide_child_console_window(&mut command); + let status = command + .status() + .with_context(|| format!("failed to run {}", manifest.lemonade.display()))?; + if !status.success() { + bail!("Lemonade pull of `{checkpoint_ref}` failed with status {status}"); + } + Ok(()) +} + +#[allow(clippy::too_many_arguments)] +pub(crate) fn run_lemonade_model_load( + manifest: &LemonadeInstallManifest, + host: &str, + port: u16, + model_ref: &str, + backend: &str, + engine_recipe: Option<&EngineRecipeHint>, + log_path: Option<&Path>, + process_env: &LemonadeProcessEnvironment, +) -> Result<()> { + let mut command = ProcessCommand::new(&manifest.lemonade); + command + .args(lemonade_model_load_args( + host, + port, + model_ref, + backend, + engine_recipe, + )) + .stdin(Stdio::null()); + if let Some(log_path) = log_path { + let log = fs::OpenOptions::new() + .create(true) + .append(true) + .open(log_path) + .with_context(|| format!("failed to open {}", log_path.display()))?; + command.stdout(Stdio::from(log.try_clone()?)); + command.stderr(Stdio::from(log)); + } else { + command.stdout(Stdio::inherit()).stderr(Stdio::inherit()); + } + apply_lemonade_process_environment(&mut command, process_env)?; + hide_child_console_window(&mut command); + let status = command + .status() + .with_context(|| format!("failed to run {}", manifest.lemonade.display()))?; + if !status.success() { + bail!("Lemonade model load failed with status {status}"); + } + Ok(()) +} + +/// Run one of Lemonade's packaged `llama-server` binaries (`server`) directly on a +/// GGUF file, exposing the model under `request.model_ref` via `--alias`. This is how +/// a canonical Hugging Face name is served under exactly that name — Lemonade's router +/// renames registered models, but an llama-server alias is a free-form string. +pub(crate) fn serve_direct_llama_server( + request: &ServeHttpRequest, + runtime: &LemonadeRuntime, + process_env: &LemonadeProcessEnvironment, + server: &Path, + log_path: Option<&Path>, + reason: &anyhow::Error, +) -> Result<()> { + let paths = AppPaths::discover()?; + let model_path = direct_llama_model_path(&paths, &request.model_ref).with_context(|| { + format!( + "Lemonade downloaded model `{}` was not found for direct serving", + request.model_ref + ) + })?; + if !server.is_file() { + bail!("Lemonade llama-server is missing at {}", server.display()); + } + let backend = llama_server_backend_label(server); + + if let Some(log_path) = log_path + && let Some(parent) = log_path.parent() + { + fs::create_dir_all(parent)?; + } + if let Some(log_path) = log_path { + let mut log = fs::OpenOptions::new() + .create(true) + .append(true) + .open(log_path) + .with_context(|| format!("failed to open {}", log_path.display()))?; + writeln!( + log, + "\nLaunching Lemonade packaged {backend} llama-server directly: {reason:#}" + ) + .ok(); + } + + let mut direct_env = process_env.clone(); + if let Some(server_dir) = server.parent() { + push_existing_path(&mut direct_env.path_entries, server_dir.to_path_buf()); + push_existing_path(&mut direct_env.library_entries, server_dir.to_path_buf()); + } + + let mut command = ProcessCommand::new(server); + command + .arg("-m") + .arg(&model_path) + .arg("--host") + .arg(&request.host) + .arg("--port") + .arg(request.port.to_string()) + .arg("--n-gpu-layers") + .arg("999") + .arg("--alias") + .arg(&request.model_ref) + .stdin(Stdio::null()); + if let Some(engine_recipe) = request.engine_recipe.as_ref() { + command.args(&engine_recipe.required_flags); + } + // Packaged llama-server is raw llama.cpp — it does not read `LEMONADE_API_KEY`. + // When `rocm serve` protects a public endpoint, hand llama-server the *existing* + // CLI-managed 0600 key file via `--api-key-file` (a path, not the value, so the + // secret never lands in the process table). Reusing the managed file rather than + // writing a copy keeps the key's lifecycle owned by `rocm serve` — created before + // spawn, deleted on stop — with no stale plaintext copy left behind on teardown. + if let Some(key_file) = rocm_engine_protocol::resolve_endpoint_api_key_file() { + command.arg("--api-key-file").arg(&key_file); + } + if let Some(log_path) = log_path { + let log = fs::OpenOptions::new() + .create(true) + .append(true) + .open(log_path) + .with_context(|| format!("failed to open {}", log_path.display()))?; + command.stdout(Stdio::from(log.try_clone()?)); + command.stderr(Stdio::from(log)); + } else { + command.stdout(Stdio::inherit()).stderr(Stdio::inherit()); + } + apply_lemonade_process_environment(&mut command, &direct_env)?; + hide_child_console_window(&mut command); + let mut child = command + .spawn() + .with_context(|| format!("failed to start {}", server.display()))?; + write_running_state( + request, + runtime, + std::process::id(), + Some(child.id()), + "running", + )?; + // If the server never becomes ready (or fails the smoke test), don't leak the child + // or leave the state stuck at `running`: kill it and mark the service failed. + let readiness = (|| -> Result<()> { + wait_for_openai_models_ready( + &request.host, + request.port, + &request.model_ref, + Duration::from_mins(2), + )?; + if !query_chat_smoke_endpoint(&request.host, request.port, &request.model_ref)? { + bail!("Lemonade packaged llama-server did not pass a chat-completion smoke test"); + } + Ok(()) + })(); + if let Err(error) = readiness { + let _ = terminate_pid(child.id(), true); + let _ = child.wait(); + let _ = mark_json_status(&request.state_path, "failed"); + return Err(error); + } + merge_json_state( + &request.state_path, + &json!({ + "status": "ready", + "server_pid": child.id(), + // Identity token for the server PID, captured while the child is alive. + "server_start_ticks": rocm_core::process_start_ticks(child.id()), + "backend_state": "ready", + "backend_requested": backend, + "backend_mode": "lemonade-packaged-llama-server", + "load_response": { + "status": "loaded", + "method": "lemonade-packaged-llama-server", + "model_name": request.model_ref, + "model_path": model_path, + "llamacpp_backend": backend + }, + }), + )?; + let status = child + .wait() + .context("failed waiting for Lemonade packaged llama-server")?; + mark_json_status( + &request.state_path, + if status.success() { + "stopped" + } else { + "failed" + }, + )?; + if status.success() { + Ok(()) + } else { + bail!("Lemonade packaged llama-server exited with status {status}") + } +} + +fn wait_for_openai_models_ready( + host: &str, + port: u16, + model_ref: &str, + timeout: Duration, +) -> Result<()> { + let endpoint = format_http_base_url(host, port); + let endpoint_api_key = rocm_engine_protocol::resolve_endpoint_api_key(); + let start = std::time::Instant::now(); + let mut last_error = None; + while start.elapsed() < timeout { + match http_get_text_with_auth( + &endpoint, + "/v1/models", + endpoint_api_key.as_deref(), + Duration::from_secs(3), + ) + .and_then(|body| parse_models_ready(&body, model_ref)) + { + Ok(true) => return Ok(()), + Ok(false) => last_error = Some("model was not reported by /v1/models".to_owned()), + Err(error) => last_error = Some(error.to_string()), + } + std::thread::sleep(Duration::from_millis(500)); + } + bail!( + "Lemonade packaged llama-server did not become ready: {}", + last_error.unwrap_or_else(|| "not checked".to_owned()) + ) +} + +fn parse_models_ready(body: &str, model_ref: &str) -> Result { + let value = serde_json::from_str::(body.trim()) + .context("failed to parse /v1/models response")?; + Ok(models_payload_has_loaded_model(&value, model_ref)) +} + +fn models_payload_has_loaded_model(value: &Value, model_ref: &str) -> bool { + value + .get("data") + .or_else(|| value.get("models")) + .and_then(Value::as_array) + .is_some_and(|models| { + models.iter().any(|model| { + ["id", "model", "name"] + .into_iter() + .filter_map(|field| model.get(field).and_then(Value::as_str)) + .any(|loaded| model_names_match(loaded, model_ref)) + }) + }) +} + +/// Whether `/v1/models` advertises `model_ref` as ready to serve on GPU. A stock +/// `llama-server` (direct-serve) entry has no `recipe_options` and is accepted by name, +/// since the direct-serve path only runs GPU backends. A Lemonade-router entry carries +/// `recipe_options`, so its `llamacpp_backend` must match — this keeps a merely +/// registered-but-unloaded model (empty `recipe_options`) from reading as ready. +fn models_payload_has_ready_model(value: &Value, model_ref: &str, backend: &str) -> bool { + value + .get("data") + .or_else(|| value.get("models")) + .and_then(Value::as_array) + .is_some_and(|models| { + models.iter().any(|model| { + let name_matches = ["id", "model", "name"] + .into_iter() + .filter_map(|field| model.get(field).and_then(Value::as_str)) + .any(|loaded| model_names_match(loaded, model_ref)); + name_matches && model_reports_ready_backend(model, backend) + }) + }) +} + +/// A `/v1/models` entry is servable on GPU when it carries no `recipe_options` (a stock +/// `llama-server` direct-serve entry — GPU-only by construction) or its reported +/// `llamacpp_backend` matches the expected backend. +fn model_reports_ready_backend(model: &Value, backend: &str) -> bool { + match model.get("recipe_options") { + None => true, + Some(options) => options + .get("llamacpp_backend") + .and_then(Value::as_str) + .is_some_and(|loaded| lemonade_backend_matches(loaded, backend)), + } +} + +fn lemonade_model_load_args( + host: &str, + port: u16, + model_ref: &str, + backend: &str, + engine_recipe: Option<&EngineRecipeHint>, +) -> Vec { + let mut args = vec![ + "--host".to_owned(), + host.to_owned(), + "--port".to_owned(), + port.to_string(), + "load".to_owned(), + model_ref.to_owned(), + "--llamacpp".to_owned(), + backend.to_owned(), + "--save-options".to_owned(), + ]; + if let Some(engine_recipe) = engine_recipe.filter(|recipe| !recipe.required_flags.is_empty()) { + args.extend([ + "--llamacpp-args".to_owned(), + engine_recipe.required_flags.join(" "), + ]); + } + args +} + +fn query_health_json(host: &str, port: u16) -> Result { + let endpoint = format_http_base_url(host, port); + let endpoint_api_key = rocm_engine_protocol::resolve_endpoint_api_key(); + let body = http_get_text_with_auth( + &endpoint, + "/v1/health", + endpoint_api_key.as_deref(), + Duration::from_secs(3), + ) + .with_context(|| format!("failed to query Lemonade health at {endpoint}/v1/health"))?; + serde_json::from_str(&body).context("failed to parse Lemonade health JSON") +} + +pub(crate) fn query_loaded_model_endpoint( + endpoint_url: &str, + model_ref: &str, + backend: &str, +) -> Result { + let (host, port) = parse_http_endpoint(endpoint_url) + .with_context(|| format!("unsupported endpoint URL `{endpoint_url}`"))?; + // Lemonade's router reports readiness via `/v1/health` (`all_models_loaded`). The + // direct-serve path runs a stock `llama-server` that has no such field, and instead + // advertises the model in `/v1/models`; fall back to that (name + backend) so both + // serving modes are recognized as ready. + if let Ok(health) = query_health_json(&host, port) + && health_has_loaded_model(&health, model_ref, backend) + { + return Ok(true); + } + let endpoint = format_http_base_url(&host, port); + let endpoint_api_key = rocm_engine_protocol::resolve_endpoint_api_key(); + let body = http_get_text_with_auth( + &endpoint, + "/v1/models", + endpoint_api_key.as_deref(), + Duration::from_secs(3), + ) + .with_context(|| format!("failed to query Lemonade models at {endpoint}/v1/models"))?; + let models = + serde_json::from_str::(&body).context("failed to parse Lemonade /v1/models JSON")?; + Ok(models_payload_has_ready_model(&models, model_ref, backend)) +} + +/// Post-load smoke test: the freshly loaded model must actually complete a chat +/// request. Deliberately stricter than the readiness probe — only a `200` counts, +/// so a refusal (wrong model name, unsupported request) fails the serve instead +/// of being reported as a working service. +pub(crate) fn query_chat_smoke_endpoint(host: &str, port: u16, model_ref: &str) -> Result { + let status = rocm_core::openai_chat_completion_status( + &format_http_base_url(host, port), + model_ref, + rocm_engine_protocol::resolve_endpoint_api_key().as_deref(), + rocm_core::INFERENCE_PROBE_TIMEOUT, + )?; + Ok(status == 200) +} + +fn health_has_loaded_model(health: &Value, model_ref: &str, backend: &str) -> bool { + let model_ref = model_ref.trim(); + if model_ref.is_empty() { + return false; + } + health + .get("all_models_loaded") + .and_then(Value::as_array) + .into_iter() + .flatten() + .any(|model| { + let name_matches = ["model_name", "id", "name"] + .into_iter() + .filter_map(|field| model.get(field).and_then(Value::as_str)) + .any(|loaded| model_names_match(loaded, model_ref)); + let backend_matches = model + .get("recipe_options") + .and_then(|options| options.get("llamacpp_backend")) + .and_then(Value::as_str) + .is_some_and(|loaded| lemonade_backend_matches(loaded, backend)); + name_matches && backend_matches + }) +} + +fn model_names_match(left: &str, right: &str) -> bool { + left.eq_ignore_ascii_case(right) + || resolve_lemonade_model_ref(left).eq_ignore_ascii_case(&resolve_lemonade_model_ref(right)) +} + +fn lemonade_backend_matches(value: &str, backend: &str) -> bool { + value + .trim() + .to_ascii_lowercase() + .starts_with(&backend.to_ascii_lowercase()) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::DEFAULT_MODEL; + + #[test] + fn models_ready_accepts_direct_serve_and_gates_router_backend() { + // Direct-serve (stock llama-server): no `recipe_options` → ready by name. + let direct = json!({"data": [{"id": "LiquidAI/LFM2.5-230M-GGUF:Q4_0"}]}); + assert!(models_payload_has_ready_model( + &direct, + "LiquidAI/LFM2.5-230M-GGUF:Q4_0", + "vulkan" + )); + // Router entry loaded on the matching backend → ready. + let loaded = json!({"data": [{ + "id": "Qwen3-0.6B-GGUF", + "recipe_options": {"llamacpp_backend": "vulkan"} + }]}); + assert!(models_payload_has_ready_model( + &loaded, + "Qwen3-0.6B-GGUF", + "vulkan" + )); + // Registered but not loaded (empty `recipe_options`) → not ready. + let registered = json!({"data": [{ + "id": "Qwen3-0.6B-GGUF", + "recipe_options": {} + }]}); + assert!(!models_payload_has_ready_model( + ®istered, + "Qwen3-0.6B-GGUF", + "vulkan" + )); + } + + #[test] + fn lemonade_pull_builds_checkpoint_command() { + assert_eq!( + lemonade_pull_args("127.0.0.1", 11435, "LiquidAI/LFM2.5-230M-GGUF:Q4_0"), + vec![ + "--host", + "127.0.0.1", + "--port", + "11435", + "pull", + "LiquidAI/LFM2.5-230M-GGUF:Q4_0", + ] + ); + } + + #[test] + fn windows_child_path_maps_ape_drive_paths() { + assert_eq!( + windows_child_path(Path::new("/D/path/to/rocm-cli/file.zip")), + r"D:\path\to\rocm-cli\file.zip" + ); + assert_eq!(windows_child_path(Path::new("/c")), r"C:\"); + } + + #[test] + fn lemonade_model_load_uses_selected_backend() { + let args = lemonade_model_load_args("127.0.0.1", 11435, DEFAULT_MODEL, "vulkan", None); + assert_eq!( + args, + vec![ + "--host", + "127.0.0.1", + "--port", + "11435", + "load", + DEFAULT_MODEL, + "--llamacpp", + "vulkan", + "--save-options", + ] + ); + } + + #[test] + fn lemonade_model_load_forwards_llamacpp_recipe_flags() { + let recipe = EngineRecipeHint { + required_flags: vec![ + "--temperature".to_owned(), + "0.5".to_owned(), + "--top-p".to_owned(), + "0.25".to_owned(), + "--n-predict".to_owned(), + "128".to_owned(), + ], + ..EngineRecipeHint::default() + }; + let args = + lemonade_model_load_args("127.0.0.1", 11435, DEFAULT_MODEL, "vulkan", Some(&recipe)); + assert_eq!( + args[args.len() - 2..], + [ + "--llamacpp-args", + "--temperature 0.5 --top-p 0.25 --n-predict 128" + ] + ); + } + + #[test] + fn health_parser_requires_loaded_requested_model() { + let unloaded = json!({ + "status": "ok", + "model_loaded": null, + "all_models_loaded": [] + }); + assert!(!health_has_loaded_model(&unloaded, DEFAULT_MODEL, "vulkan")); + + let loaded = json!({ + "status": "ok", + "model_loaded": DEFAULT_MODEL, + "all_models_loaded": [{ + "model_name": DEFAULT_MODEL, + "recipe": "llamacpp", + "recipe_options": { + "llamacpp_backend": "vulkan" + } + }] + }); + assert!(health_has_loaded_model(&loaded, "lemonade-qwen", "vulkan")); + assert!(!health_has_loaded_model(&loaded, "lemonade-qwen", "rocm")); + } +} diff --git a/engines/lemonade/src/runtime_dir.rs b/engines/lemonade/src/runtime_dir.rs new file mode 100644 index 0000000000..1e880994f0 --- /dev/null +++ b/engines/lemonade/src/runtime_dir.rs @@ -0,0 +1,1082 @@ +// Copyright © Advanced Micro Devices, Inc., or its affiliates. +// +// SPDX-License-Identifier: MIT + +use anyhow::{Context, Result}; +#[cfg(unix)] +use anyhow::{anyhow, bail}; +use rocm_core::runtime_is_linux; +#[cfg(unix)] +use std::ffi::OsStr; +use std::ffi::OsString; +use std::fs; +use std::path::{Path, PathBuf}; + +/// The parts of *this* process's environment that decide where the child's +/// runtime directory goes. +/// +/// Captured into a value instead of being read inside +/// [`crate::process::lemonade_process_environment_vars`] so the precedence is +/// testable without mutating process-global env vars, which is `unsafe` and +/// racy under parallel tests in edition 2024. +#[derive(Debug, Clone, Default)] +pub(crate) struct ParentRuntimeEnvironment { + xdg_runtime_dir: Option, + home: Option, + user: Option, + temp_dir: PathBuf, +} + +impl ParentRuntimeEnvironment { + pub(crate) fn current() -> Self { + Self { + xdg_runtime_dir: std::env::var_os("XDG_RUNTIME_DIR"), + home: std::env::var_os("HOME"), + // An empty `USER` must fall through to `LOGNAME`, not short-circuit it. + user: std::env::var("USER") + .ok() + .filter(|v| !v.is_empty()) + .or_else(|| std::env::var("LOGNAME").ok().filter(|v| !v.is_empty())), + temp_dir: std::env::temp_dir(), + } + } +} + +/// Leaf directory the engine gets inside a fallback runtime root, so a crashing +/// or misbehaving server cannot disturb the telemetry socket that shares the +/// same root. +const RUNTIME_DIR_LEAF: &str = "lemonade"; + +/// Guarantee the child a usable `XDG_RUNTIME_DIR`. +/// +/// `lemond` refuses to start unless it can resolve a writable runtime directory +/// from `XDG_RUNTIME_DIR` or `RUNTIME_DIRECTORY`. Neither exists on a headless +/// host: `XDG_RUNTIME_DIR` is populated by `pam_systemd` at login, so it is +/// absent for every non-login process (cron jobs, CI runners, `systemd-run`, a +/// bare container exec), and `RUNTIME_DIRECTORY` only exists for a systemd unit +/// that declares `RuntimeDirectory=`. There, a managed serve failed +/// deterministically before the server ever came up. +/// +/// A value already present in the parent environment is passed through +/// unchanged — an operator's choice always wins, and it is the tier the rest of +/// this precedence exists to fall back from. +pub(crate) fn child_runtime_dir_var( + parent: &ParentRuntimeEnvironment, +) -> Result> { + if !runtime_is_linux() { + return Ok(None); + } + if let Some(existing) = parent + .xdg_runtime_dir + .as_ref() + .filter(|value| !value.is_empty()) + { + return Ok(Some(("XDG_RUNTIME_DIR", existing.clone()))); + } + let has_home = parent.home.as_ref().is_some_and(|value| !value.is_empty()); + let dir = rocm_core::user_runtime_dir( + None, + parent.home.clone(), + parent.user.clone(), + parent.temp_dir.clone(), + RUNTIME_DIR_LEAF, + RUNTIME_DIR_LEAF, + ); + let prepare = if has_home { + create_private_runtime_dir(&dir) + } else { + create_private_tier3_runtime_dir(&dir) + }; + prepare.with_context(|| { + format!( + "preparing a runtime directory for the Lemonade server at {} \ + (XDG_RUNTIME_DIR is unset in this environment)", + dir.display() + ) + })?; + Ok(Some(("XDG_RUNTIME_DIR", dir.into_os_string()))) +} + +#[cfg(unix)] +fn create_private_tier3_runtime_dir(dir: &Path) -> Result<()> { + let user_parent = dir + .parent() + .ok_or_else(|| anyhow!("{} has no tier-3 user parent", dir.display()))?; + let temp_root = user_parent.parent().ok_or_else(|| { + anyhow!( + "{} has no temporary-directory parent", + user_parent.display() + ) + })?; + let temp_root = open_safe_tier3_root(temp_root)?; + let user_component = single_directory_component(user_parent)?; + let user_parent = create_private_owned_directory_at(&temp_root, user_component, user_parent)?; + let leaf_component = single_directory_component(dir)?; + create_private_owned_directory_at(&user_parent, leaf_component, dir)?; + Ok(()) +} + +#[cfg(unix)] +fn single_directory_component(path: &Path) -> Result<&OsStr> { + path.file_name() + .filter(|component| Path::new(component).components().count() == 1) + .ok_or_else(|| anyhow!("{} has no safe final directory component", path.display())) +} + +#[cfg(unix)] +#[allow(unsafe_code)] +fn open_safe_tier3_root(temp_root: &Path) -> Result { + use std::ffi::CString; + use std::io::ErrorKind; + use std::os::fd::{AsRawFd, FromRawFd}; + use std::os::unix::ffi::OsStrExt; + use std::os::unix::fs::OpenOptionsExt; + use std::path::Component; + + if !temp_root.is_absolute() { + bail!( + "tier-3 temporary root {} is relative or otherwise unstable", + temp_root.display() + ); + } + let mut components = temp_root.components(); + if components.next() != Some(Component::RootDir) { + bail!( + "tier-3 temporary root {} has no filesystem-root anchor", + temp_root.display() + ); + } + + let mut directory = fs::OpenOptions::new() + .read(true) + .custom_flags(libc::O_DIRECTORY | libc::O_NOFOLLOW) + .open(Path::new("/")) + .context("opening filesystem root without following symlinks")?; + let current_euid = effective_user_id(); + let mut traversed = PathBuf::from("/"); + let mut components = components.peekable(); + while let Some(component) = components.next() { + let Component::Normal(component) = component else { + bail!( + "tier-3 temporary root {} contains a relative or unstable component", + temp_root.display() + ); + }; + validate_runtime_ancestry_container(&traversed, &directory, current_euid)?; + let child_path = traversed.join(component); + let component = CString::new(component.as_bytes()).with_context(|| { + format!( + "tier-3 temporary root {} contains a NUL byte", + temp_root.display() + ) + })?; + let is_final = components.peek().is_none(); + let raw_fd = match unsafe_openat_directory(directory.as_raw_fd(), &component) { + Ok(fd) => fd, + Err(error) if is_final && error.kind() == ErrorKind::NotFound => { + match unsafe_mkdirat(directory.as_raw_fd(), &component, 0o700) { + Ok(()) => {} + Err(error) if error.kind() == ErrorKind::AlreadyExists => {} + Err(error) => { + return Err(error).with_context(|| { + format!("creating tier-3 temporary root {}", temp_root.display()) + }); + } + } + unsafe_openat_directory(directory.as_raw_fd(), &component).with_context(|| { + format!( + "opening newly created tier-3 temporary root {}", + temp_root.display() + ) + })? + } + Err(error) => { + return Err(error).with_context(|| { + format!( + "opening {} while securely traversing tier-3 temporary-root ancestry", + child_path.display() + ) + }); + } + }; + // SAFETY: `unsafe_openat_directory` returned a new owned descriptor, + // transferred exactly once into a `File` that closes it on drop. + directory = unsafe { fs::File::from_raw_fd(raw_fd) }; + validate_runtime_directory_type(&child_path, &unsafe_fstat(directory.as_raw_fd())?)?; + traversed = child_path; + } + + let metadata = unsafe_fstat(directory.as_raw_fd())?; + if !directory_prevents_cross_uid_entry_replacement( + metadata.st_uid, + metadata.st_mode, + current_euid, + filesystem_is_read_only(directory.as_raw_fd())?, + ) { + bail!( + "tier-3 temporary root {} is unsafe: trusted owners require no group/other write or sticky protection; foreign owners require a read-only filesystem", + temp_root.display() + ); + } + Ok(directory) +} + +#[cfg(unix)] +const fn directory_prevents_cross_uid_entry_replacement( + uid: u32, + mode: u32, + current_euid: u32, + filesystem_read_only: bool, +) -> bool { + if uid == 0 || uid == current_euid { + return mode & 0o022 == 0 || mode & libc::S_ISVTX != 0; + } + filesystem_read_only +} + +#[cfg(unix)] +fn validate_runtime_ancestry_container( + path: &Path, + directory: &fs::File, + current_euid: u32, +) -> Result<()> { + use std::os::fd::AsRawFd; + + let metadata = unsafe_fstat(directory.as_raw_fd())?; + validate_runtime_directory_type(path, &metadata)?; + if !directory_prevents_cross_uid_entry_replacement( + metadata.st_uid, + metadata.st_mode, + current_euid, + filesystem_is_read_only(directory.as_raw_fd())?, + ) { + bail!( + "tier-3 temporary-root ancestor {} permits cross-UID rename or replacement", + path.display() + ); + } + Ok(()) +} + +#[cfg(unix)] +fn filesystem_is_read_only(fd: std::os::fd::RawFd) -> Result { + Ok(unsafe_fstatvfs(fd)?.f_flag & libc::ST_RDONLY != 0) +} + +#[cfg(unix)] +fn validate_runtime_directory_type(path: &Path, metadata: &libc::stat) -> Result<()> { + if metadata.st_mode & libc::S_IFMT != libc::S_IFDIR { + bail!("{} is not a directory", path.display()); + } + Ok(()) +} + +#[cfg(not(unix))] +fn create_private_tier3_runtime_dir(dir: &Path) -> Result<()> { + create_private_runtime_dir(dir) +} + +#[cfg(unix)] +#[allow(unsafe_code)] +fn create_private_owned_directory_at( + parent: &fs::File, + component: &OsStr, + display_path: &Path, +) -> Result { + use std::ffi::CString; + use std::io::ErrorKind; + use std::os::fd::{AsRawFd, FromRawFd}; + use std::os::unix::ffi::OsStrExt; + + let component = CString::new(component.as_bytes()).with_context(|| { + format!( + "{} contains a NUL byte in its final component", + display_path.display() + ) + })?; + let mkdir_result = unsafe_mkdirat(parent.as_raw_fd(), &component, 0o700); + if let Err(error) = mkdir_result + && error.kind() != ErrorKind::AlreadyExists + { + return Err(error).with_context(|| format!("creating {}", display_path.display())); + } + + let raw_fd = unsafe_openat_directory(parent.as_raw_fd(), &component).with_context(|| { + format!( + "opening {} relative to its validated parent without following symlinks", + display_path.display() + ) + })?; + // SAFETY: `unsafe_openat_directory` returned a new owned descriptor, and + // this is its single transfer into a type that closes it on drop. + let directory = unsafe { fs::File::from_raw_fd(raw_fd) }; + let current_euid = effective_user_id(); + validate_runtime_directory_owner( + display_path, + &unsafe_fstat(directory.as_raw_fd())?, + current_euid, + )?; + unsafe_fchmod(directory.as_raw_fd(), 0o700) + .with_context(|| format!("restricting {} to mode 0700", display_path.display()))?; + + let secured = unsafe_fstat(directory.as_raw_fd())?; + validate_runtime_directory_owner(display_path, &secured, current_euid)?; + if secured.st_mode & 0o777 != 0o700 { + bail!( + "{} is not private after chmod; expected mode 0700", + display_path.display() + ); + } + Ok(directory) +} + +#[cfg(unix)] +#[allow(unsafe_code)] +fn unsafe_fstat(fd: std::os::fd::RawFd) -> std::io::Result { + let mut metadata = std::mem::MaybeUninit::::uninit(); + // SAFETY: `metadata` points to writable storage for one `libc::stat`, and + // `fd` is borrowed from a live `File`. On success fstat initializes it. + if unsafe { libc::fstat(fd, metadata.as_mut_ptr()) } == 0 { + // SAFETY: successful fstat initialized the full `libc::stat` value. + Ok(unsafe { metadata.assume_init() }) + } else { + Err(std::io::Error::last_os_error()) + } +} + +#[cfg(unix)] +#[allow(unsafe_code)] +fn unsafe_fstatvfs(fd: std::os::fd::RawFd) -> std::io::Result { + let mut metadata = std::mem::MaybeUninit::::uninit(); + // SAFETY: `metadata` points to writable storage for one `libc::statvfs`, + // and `fd` is borrowed from a live directory `File`. On success fstatvfs + // initializes it. + if unsafe { libc::fstatvfs(fd, metadata.as_mut_ptr()) } == 0 { + // SAFETY: successful fstatvfs initialized the full value. + Ok(unsafe { metadata.assume_init() }) + } else { + Err(std::io::Error::last_os_error()) + } +} + +#[cfg(unix)] +#[allow(unsafe_code)] +fn unsafe_fchmod(fd: std::os::fd::RawFd, mode: libc::mode_t) -> std::io::Result<()> { + // SAFETY: `fd` is borrowed from a live directory `File`; fchmod does not + // retain it, and the supplied mode is a valid permission bitmask. + if unsafe { libc::fchmod(fd, mode) } == 0 { + Ok(()) + } else { + Err(std::io::Error::last_os_error()) + } +} + +#[cfg(unix)] +#[allow(unsafe_code)] +fn unsafe_mkdirat( + parent_fd: std::os::fd::RawFd, + component: &std::ffi::CStr, + mode: libc::mode_t, +) -> std::io::Result<()> { + // SAFETY: `component` is NUL-terminated and lives for the call; `parent_fd` + // is borrowed from an open directory descriptor; mkdirat does not retain + // either argument. + if unsafe { libc::mkdirat(parent_fd, component.as_ptr(), mode) } == 0 { + Ok(()) + } else { + Err(std::io::Error::last_os_error()) + } +} + +#[cfg(unix)] +#[allow(unsafe_code)] +fn unsafe_openat_directory( + parent_fd: std::os::fd::RawFd, + component: &std::ffi::CStr, +) -> std::io::Result { + // SAFETY: `component` is NUL-terminated and lives for the call; `parent_fd` + // is an open directory descriptor. On success openat returns a new owned fd. + let fd = unsafe { + libc::openat( + parent_fd, + component.as_ptr(), + libc::O_RDONLY | libc::O_DIRECTORY | libc::O_NOFOLLOW | libc::O_CLOEXEC, + ) + }; + if fd >= 0 { + Ok(fd) + } else { + Err(std::io::Error::last_os_error()) + } +} + +#[cfg(unix)] +fn validate_runtime_directory_owner( + dir: &Path, + metadata: &libc::stat, + current_euid: u32, +) -> Result<()> { + if metadata.st_mode & libc::S_IFMT != libc::S_IFDIR { + bail!("{} is not a directory", dir.display()); + } + if metadata.st_uid != current_euid { + bail!( + "{} is owned by uid {}, not current euid {}", + dir.display(), + metadata.st_uid, + current_euid + ); + } + Ok(()) +} + +#[cfg(unix)] +#[allow(unsafe_code)] +fn effective_user_id() -> u32 { + // SAFETY: `geteuid` takes no arguments, has no caller-side preconditions, + // and only returns process identity state. + unsafe { libc::geteuid() } +} + +/// Create `dir` (and its parents) restricted to the current user. +/// +/// This is the existing recursive behavior used for the HOME-based fallback, +/// where legitimate symlinked HOME or `.rocm` paths retain normal filesystem +/// semantics. Tier 3 uses [`create_private_tier3_runtime_dir`] instead so its +/// predictable parent is validated separately without following symlinks. +/// +/// The leaf itself is still refused when it is a symlink, and an existing leaf +/// is tightened explicitly because `DirBuilder::mode` only affects directories +/// it creates. +fn create_private_runtime_dir(dir: &Path) -> Result<()> { + #[cfg(unix)] + { + use std::os::unix::fs::{DirBuilderExt, PermissionsExt}; + fs::DirBuilder::new() + .recursive(true) + .mode(0o700) + .create(dir) + .with_context(|| format!("creating {}", dir.display()))?; + if fs::symlink_metadata(dir)?.file_type().is_symlink() { + bail!( + "{} is a symlink, so it may point somewhere another user controls", + dir.display() + ); + } + fs::set_permissions(dir, fs::Permissions::from_mode(0o700)).with_context(|| { + format!( + "restricting {} to mode 0700 — it is most likely owned by another user", + dir.display() + ) + })?; + } + #[cfg(not(unix))] + fs::create_dir_all(dir)?; + Ok(()) +} + +#[cfg(test)] +mod tests { + #[cfg(target_os = "linux")] + use super::*; + #[cfg(target_os = "linux")] + use crate::process::{LemonadeProcessEnvironment, lemonade_process_environment_vars}; + + // `XDG_RUNTIME_DIR` is a Linux/freedesktop concept and the Lemonade server + // only consults it there, so these are Linux-only by construction. + #[cfg(target_os = "linux")] + mod child_runtime_dir { + use super::*; + + fn value_of(vars: &[(&'static str, OsString)], key: &str) -> Option { + vars.iter() + .find(|(name, _)| *name == key) + .map(|(_, value)| value.clone()) + } + + fn runtime_scratch() -> tempfile::TempDir { + tempfile::Builder::new() + .prefix(".lemonade-runtime-test-") + .tempdir_in(std::env::current_dir().expect("current directory")) + .expect("private-ancestry scratch") + } + fn vars_for(parent: &ParentRuntimeEnvironment) -> Vec<(&'static str, OsString)> { + lemonade_process_environment_vars(&LemonadeProcessEnvironment::default(), parent) + .expect("building the child environment must succeed") + } + + fn assert_private_dir(dir: &Path) { + use std::os::unix::fs::PermissionsExt; + assert!( + dir.is_dir(), + "{} must exist before the server is spawned", + dir.display() + ); + let mode = fs::metadata(dir).expect("metadata").permissions().mode() & 0o777; + assert_eq!( + mode, + 0o700, + "{} must be private to this user", + dir.display() + ); + } + + /// Regression: on a headless host `pam_systemd` never runs, so + /// `XDG_RUNTIME_DIR` is unset and the Lemonade server exits during + /// startup with "Unable to resolve writable runtime directory". The + /// child must be handed a directory that already exists. + #[test] + fn is_synthesized_when_the_parent_has_none() { + let scratch = runtime_scratch(); + let home = scratch.path().join("home"); + let parent = ParentRuntimeEnvironment { + xdg_runtime_dir: None, + home: Some(home.clone().into_os_string()), + user: Some("alice".to_owned()), + temp_dir: scratch.path().join("tmp"), + }; + + let value = value_of(&vars_for(&parent), "XDG_RUNTIME_DIR") + .expect("the child must be given a runtime directory"); + + // Tier 2 of the shared precedence, with a `lemonade` leaf so the + // engine never shares a directory with the telemetry socket. + let dir = PathBuf::from(value); + assert_eq!(dir, home.join(".rocm").join("data").join("lemonade")); + assert_private_dir(&dir); + } + + /// An exported-but-empty variable is as good as unset, and must not be + /// mistaken for an operator's choice. + #[test] + fn is_synthesized_when_the_parent_value_is_empty() { + let scratch = runtime_scratch(); + let parent = ParentRuntimeEnvironment { + xdg_runtime_dir: Some(OsString::new()), + home: Some(scratch.path().join("home").into_os_string()), + user: Some("alice".to_owned()), + temp_dir: scratch.path().join("tmp"), + }; + + let value = value_of(&vars_for(&parent), "XDG_RUNTIME_DIR") + .expect("an empty value must fall through, not be passed on"); + assert_eq!( + PathBuf::from(value), + scratch + .path() + .join("home") + .join(".rocm") + .join("data") + .join("lemonade") + ); + } + + /// Tier 2 lives under the operator's HOME and must retain normal path + /// semantics; homes (or `.rocm`) are legitimately symlinked on some + /// systems. The no-follow hardening is intentionally tier-3-only. + #[test] + fn permits_a_symlinked_home_path_for_tier2() { + let scratch = runtime_scratch(); + let real_home = scratch.path().join("real-home"); + let linked_home = scratch.path().join("linked-home"); + fs::create_dir_all(&real_home).expect("real home"); + std::os::unix::fs::symlink(&real_home, &linked_home).expect("linked home"); + let parent = ParentRuntimeEnvironment { + xdg_runtime_dir: None, + home: Some(linked_home.clone().into_os_string()), + user: Some("alice".to_owned()), + temp_dir: scratch.path().join("tmp"), + }; + + let value = value_of(&vars_for(&parent), "XDG_RUNTIME_DIR") + .expect("the child must be given a runtime directory"); + + let dir = PathBuf::from(value); + assert_eq!(dir, linked_home.join(".rocm").join("data").join("lemonade")); + assert_private_dir(&dir); + assert!(real_home.join(".rocm/data/lemonade").is_dir()); + } + /// Tier 3: neither `XDG_RUNTIME_DIR` nor `HOME` — a user-named subdir of + /// the temp dir, so the parent is one we create and own, not `/tmp`. + #[test] + fn falls_back_to_a_user_owned_temp_subdir_without_home() { + let scratch = runtime_scratch(); + let temp_dir = scratch.path().join("tmp"); + let parent = ParentRuntimeEnvironment { + xdg_runtime_dir: None, + home: None, + user: Some("alice".to_owned()), + temp_dir: temp_dir.clone(), + }; + + let value = value_of(&vars_for(&parent), "XDG_RUNTIME_DIR") + .expect("the child must be given a runtime directory"); + + let dir = PathBuf::from(value); + assert_eq!(dir, temp_dir.join("rocm-alice").join("lemonade")); + assert_private_dir(&dir); + } + + /// The tier-3 fallback lands under a shared temp dir, so the target may + /// already exist as a symlink someone else planted. Chmod would follow + /// it, so the resolver refuses rather than loosening another directory. + #[test] + fn refuses_a_symlinked_runtime_dir() { + let scratch = runtime_scratch(); + let temp_dir = scratch.path().join("tmp"); + let elsewhere = scratch.path().join("elsewhere"); + fs::create_dir_all(temp_dir.join("rocm-alice")).expect("runtime root"); + fs::create_dir_all(&elsewhere).expect("elsewhere"); + std::os::unix::fs::symlink(&elsewhere, temp_dir.join("rocm-alice").join("lemonade")) + .expect("symlink"); + + let parent = ParentRuntimeEnvironment { + xdg_runtime_dir: None, + home: None, + user: Some("alice".to_owned()), + temp_dir, + }; + let error = format!( + "{:#}", + lemonade_process_environment_vars(&LemonadeProcessEnvironment::default(), &parent) + .expect_err("a symlinked runtime dir must be refused") + ); + assert!(error.contains("symlink"), "{error}"); + } + + /// Tier 3 uses a predictable user parent under a shared temp directory. + /// Refuse that parent when it is a symlink: recursively creating the + /// leaf would otherwise follow it into an attacker-controlled tree. + #[test] + fn refuses_a_symlinked_tier3_user_parent() { + let scratch = runtime_scratch(); + let temp_dir = scratch.path().join("tmp"); + let elsewhere = scratch.path().join("elsewhere"); + fs::create_dir_all(&temp_dir).expect("temp root"); + fs::create_dir_all(&elsewhere).expect("elsewhere"); + std::os::unix::fs::symlink(&elsewhere, temp_dir.join("rocm-alice")).expect("symlink"); + + let parent = ParentRuntimeEnvironment { + xdg_runtime_dir: None, + home: None, + user: Some("alice".to_owned()), + temp_dir, + }; + let error = format!( + "{:#}", + lemonade_process_environment_vars(&LemonadeProcessEnvironment::default(), &parent) + .expect_err("a symlinked tier-3 user parent must be refused") + ); + + assert!(error.contains("symlink"), "{error}"); + assert!( + !elsewhere.join("lemonade").exists(), + "the resolver followed the tier-3 user-parent symlink" + ); + } + + /// A pre-existing tier-3 user parent may have been created with a + /// permissive umask. The resolver must secure that predictable parent, + /// not merely the Lemonade leaf beneath it. + #[test] + fn tightens_a_preexisting_non_private_tier3_user_parent() { + use std::os::unix::fs::PermissionsExt; + + let scratch = runtime_scratch(); + let temp_dir = scratch.path().join("tmp"); + let user_parent = temp_dir.join("rocm-alice"); + fs::create_dir_all(&user_parent).expect("user parent"); + fs::set_permissions(&user_parent, fs::Permissions::from_mode(0o755)) + .expect("make parent non-private"); + let parent = ParentRuntimeEnvironment { + xdg_runtime_dir: None, + home: None, + user: Some("alice".to_owned()), + temp_dir, + }; + + let value = value_of(&vars_for(&parent), "XDG_RUNTIME_DIR") + .expect("the child must be given a runtime directory"); + + assert_eq!(PathBuf::from(value), user_parent.join("lemonade")); + assert_private_dir(&user_parent); + assert_private_dir(&user_parent.join("lemonade")); + } + + /// A tier-3 root without sticky-directory protection lets another UID + /// rename or replace our predictable user component after validation. + #[test] + fn refuses_a_group_or_other_writable_non_sticky_temp_root() { + use std::os::unix::fs::PermissionsExt; + + let scratch = runtime_scratch(); + let temp_dir = scratch.path().join("unsafe-tmp"); + fs::create_dir(&temp_dir).expect("temp root"); + fs::set_permissions(&temp_dir, fs::Permissions::from_mode(0o777)) + .expect("make root unsafe"); + let parent = ParentRuntimeEnvironment { + xdg_runtime_dir: None, + home: None, + user: Some("alice".to_owned()), + temp_dir: temp_dir.clone(), + }; + + let error = format!( + "{:#}", + lemonade_process_environment_vars(&LemonadeProcessEnvironment::default(), &parent) + .expect_err("a non-sticky writable temp root must be refused") + ); + + assert!(error.contains("sticky"), "{error}"); + assert!(!temp_dir.join("rocm-alice").exists()); + } + + /// A private final temp root is still unstable when an ancestor lets + /// another UID replace the path entry that names it. + #[test] + fn refuses_a_private_temp_root_beneath_a_non_sticky_writable_parent() { + use std::os::unix::fs::PermissionsExt; + + let scratch = runtime_scratch(); + let unsafe_parent = scratch.path().join("unsafe-parent"); + let temp_dir = unsafe_parent.join("private-tmp"); + fs::create_dir(&unsafe_parent).expect("unsafe parent"); + fs::set_permissions(&unsafe_parent, fs::Permissions::from_mode(0o777)) + .expect("make ancestor unsafe"); + fs::create_dir(&temp_dir).expect("private temp root"); + fs::set_permissions(&temp_dir, fs::Permissions::from_mode(0o700)) + .expect("make final root private"); + let parent = ParentRuntimeEnvironment { + xdg_runtime_dir: None, + home: None, + user: Some("alice".to_owned()), + temp_dir: temp_dir.clone(), + }; + + let error = format!( + "{:#}", + lemonade_process_environment_vars(&LemonadeProcessEnvironment::default(), &parent) + .expect_err("unsafe temp-root ancestry must be refused") + ); + + assert!(error.contains("ancestor"), "{error}"); + assert!(!temp_dir.join("rocm-alice").exists()); + } + + #[test] + fn accepts_a_private_temp_root_with_private_ancestry() { + use std::os::unix::fs::PermissionsExt; + + let scratch = runtime_scratch(); + let private_parent = scratch.path().join("private-parent"); + let temp_dir = private_parent.join("private-tmp"); + fs::create_dir(&private_parent).expect("private parent"); + fs::set_permissions(&private_parent, fs::Permissions::from_mode(0o700)) + .expect("private ancestor mode"); + fs::create_dir(&temp_dir).expect("private temp root"); + fs::set_permissions(&temp_dir, fs::Permissions::from_mode(0o700)) + .expect("private root mode"); + let parent = ParentRuntimeEnvironment { + xdg_runtime_dir: None, + home: None, + user: Some("alice".to_owned()), + temp_dir: temp_dir.clone(), + }; + + let value = value_of(&vars_for(&parent), "XDG_RUNTIME_DIR") + .expect("private ancestry must be accepted"); + + assert_eq!(PathBuf::from(value), temp_dir.join("rocm-alice/lemonade")); + } + + #[test] + fn accepts_standard_sticky_temp_root_semantics() { + use std::os::unix::fs::PermissionsExt; + + let scratch = runtime_scratch(); + let temp_dir = scratch.path().join("sticky-tmp"); + fs::create_dir(&temp_dir).expect("sticky temp root"); + fs::set_permissions(&temp_dir, fs::Permissions::from_mode(0o1777)) + .expect("standard sticky temp mode"); + let parent = ParentRuntimeEnvironment { + xdg_runtime_dir: None, + home: None, + user: Some("alice".to_owned()), + temp_dir: temp_dir.clone(), + }; + + let value = value_of(&vars_for(&parent), "XDG_RUNTIME_DIR") + .expect("trusted sticky temp semantics must be accepted"); + + assert_eq!(PathBuf::from(value), temp_dir.join("rocm-alice/lemonade")); + assert_private_dir(&temp_dir.join("rocm-alice")); + } + + #[test] + fn accepts_the_host_standard_tmp_when_its_invariants_are_safe() { + use std::os::fd::AsRawFd; + use std::os::unix::fs::MetadataExt; + + let path = Path::new("/tmp"); + let metadata = fs::metadata(path).expect("/tmp metadata"); + let directory = fs::File::open(path).expect("open /tmp"); + if !directory_prevents_cross_uid_entry_replacement( + metadata.uid(), + metadata.mode(), + effective_user_id(), + filesystem_is_read_only(directory.as_raw_fd()).expect("/tmp filesystem flags"), + ) { + return; + } + + open_safe_tier3_root(path).expect("safe host /tmp must be accepted"); + } + + #[test] + fn refuses_a_relative_tier3_temp_root() { + let parent = ParentRuntimeEnvironment { + xdg_runtime_dir: None, + home: None, + user: Some("alice".to_owned()), + temp_dir: PathBuf::from("relative-tmp"), + }; + + let error = format!( + "{:#}", + lemonade_process_environment_vars(&LemonadeProcessEnvironment::default(), &parent) + .expect_err("relative tier-3 roots must be refused") + ); + + assert!(error.contains("relative"), "{error}"); + } + + #[test] + fn refuses_symlinked_temp_root_ancestry() { + let scratch = runtime_scratch(); + let real_parent = scratch.path().join("real-parent"); + let linked_parent = scratch.path().join("linked-parent"); + fs::create_dir(&real_parent).expect("real parent"); + std::os::unix::fs::symlink(&real_parent, &linked_parent).expect("linked ancestry"); + let parent = ParentRuntimeEnvironment { + xdg_runtime_dir: None, + home: None, + user: Some("alice".to_owned()), + temp_dir: linked_parent.join("private-tmp"), + }; + + let error = format!( + "{:#}", + lemonade_process_environment_vars(&LemonadeProcessEnvironment::default(), &parent) + .expect_err("symlinked tier-3 ancestry must be refused") + ); + + assert!(error.contains("ancestry"), "{error}"); + assert!(!real_parent.join("private-tmp").exists()); + } + + /// Sticky mode does not constrain the directory owner. A sticky root + /// owned by an unrelated non-root UID therefore cannot protect our + /// component from that UID replacing it. + #[test] + fn refuses_an_untrusted_foreign_owned_sticky_temp_root_when_available() { + use std::os::unix::fs::MetadataExt; + + let current_euid = effective_user_id(); + let candidate = [Path::new("/tmp"), Path::new("/var/tmp")] + .into_iter() + .filter_map(|path| fs::metadata(path).ok().map(|metadata| (path, metadata))) + .find(|(_, metadata)| { + metadata.uid() != 0 + && metadata.uid() != current_euid + && metadata.mode() & libc::S_ISVTX != 0 + }); + let Some((path, _)) = candidate else { + return; + }; + + let error = format!( + "{:#}", + open_safe_tier3_root(path) + .expect_err("an untrusted sticky-root owner must be refused") + ); + assert!(error.contains("sticky"), "{error}"); + assert!(error.contains("owner"), "{error}"); + } + + #[test] + fn ancestry_safety_rejects_foreign_owned_mode_0555_on_writable_filesystem() { + assert!(!directory_prevents_cross_uid_entry_replacement( + 1234, 0o555, 1000, false + )); + } + + #[test] + fn ancestry_safety_rejects_foreign_owned_mode_0755() { + assert!(!directory_prevents_cross_uid_entry_replacement( + 1234, 0o755, 1000, false + )); + } + + #[test] + fn ancestry_safety_accepts_foreign_owned_writable_on_read_only_filesystem() { + assert!(directory_prevents_cross_uid_entry_replacement( + 1234, 0o777, 1000, true + )); + } + + #[test] + fn ancestry_safety_accepts_trusted_owned_mode_0755() { + assert!(directory_prevents_cross_uid_entry_replacement( + 0, 0o755, 1000, false + )); + } + + #[test] + fn ancestry_safety_rejects_trusted_owned_mode_0777_without_sticky() { + assert!(!directory_prevents_cross_uid_entry_replacement( + 1000, 0o777, 1000, false + )); + } + + #[test] + fn ancestry_safety_accepts_trusted_owned_mode_1777() { + assert!(directory_prevents_cross_uid_entry_replacement( + 1000, 0o1777, 1000, false + )); + } + + /// Once the temp root is open, replacing its pathname must not redirect + /// creation into the replacement tree. This deterministically models + /// the rename-and-replace window from the original implementation. + #[test] + fn descriptor_relative_creation_resists_temp_root_replacement() { + let scratch = runtime_scratch(); + let temp_dir = scratch.path().join("tmp"); + fs::create_dir(&temp_dir).expect("temp root"); + let root = open_safe_tier3_root(&temp_dir).expect("open root"); + + let original_root = scratch.path().join("original-tmp"); + fs::rename(&temp_dir, &original_root).expect("rename open root"); + fs::create_dir(&temp_dir).expect("replacement root"); + let elsewhere = scratch.path().join("elsewhere"); + fs::create_dir(&elsewhere).expect("elsewhere"); + std::os::unix::fs::symlink(&elsewhere, temp_dir.join("rocm-alice")) + .expect("attacker replacement"); + + create_private_owned_directory_at( + &root, + OsStr::new("rocm-alice"), + &temp_dir.join("rocm-alice"), + ) + .expect("descriptor-relative creation"); + + assert_private_dir(&original_root.join("rocm-alice")); + assert!( + fs::symlink_metadata(temp_dir.join("rocm-alice")) + .expect("replacement symlink") + .file_type() + .is_symlink() + ); + assert!(!elsewhere.join("rocm-alice").exists()); + } + + /// Leaf creation stays anchored to the validated user-parent object even + /// if its pathname is replaced with a symlink after it was opened. + #[test] + fn descriptor_relative_leaf_creation_resists_user_parent_replacement() { + let scratch = runtime_scratch(); + let temp_dir = scratch.path().join("tmp"); + fs::create_dir(&temp_dir).expect("temp root"); + let root = open_safe_tier3_root(&temp_dir).expect("open root"); + let user_path = temp_dir.join("rocm-alice"); + let user_parent = + create_private_owned_directory_at(&root, OsStr::new("rocm-alice"), &user_path) + .expect("create user parent"); + + let original_user_parent = temp_dir.join("original-rocm-alice"); + fs::rename(&user_path, &original_user_parent).expect("rename user parent"); + let elsewhere = scratch.path().join("elsewhere"); + fs::create_dir(&elsewhere).expect("elsewhere"); + std::os::unix::fs::symlink(&elsewhere, &user_path).expect("replacement symlink"); + + create_private_owned_directory_at( + &user_parent, + OsStr::new("lemonade"), + &user_path.join("lemonade"), + ) + .expect("descriptor-relative leaf creation"); + + assert_private_dir(&original_user_parent.join("lemonade")); + assert!(!elsewhere.join("lemonade").exists()); + } + + /// Ownership can be exercised without root when the host exposes a + /// system-owned temp/root directory. The preparation routine must + /// reject it before changing its permissions. + #[test] + fn rejects_a_foreign_owned_tier3_user_parent_when_available() { + use std::os::unix::fs::{MetadataExt, OpenOptionsExt}; + + let current_euid = effective_user_id(); + let foreign = [ + (Path::new("/"), OsStr::new("tmp")), + (Path::new("/"), OsStr::new("var")), + (Path::new("/"), OsStr::new("run")), + ] + .into_iter() + .filter_map(|(parent, component)| { + let path = parent.join(component); + fs::metadata(&path) + .ok() + .map(|metadata| (parent, component, path, metadata)) + }) + .find(|(_, _, _, metadata)| metadata.is_dir() && metadata.uid() != current_euid); + let Some((parent_path, component, path, metadata)) = foreign else { + return; + }; + let original_mode = metadata.mode(); + let parent = fs::OpenOptions::new() + .read(true) + .custom_flags(libc::O_DIRECTORY | libc::O_NOFOLLOW) + .open(parent_path) + .expect("open foreign parent"); + + let error = format!( + "{:#}", + create_private_owned_directory_at(&parent, component, &path) + .expect_err("foreign ownership must be refused") + ); + assert!(error.contains("owned by uid"), "{error}"); + assert!(error.contains(¤t_euid.to_string()), "{error}"); + assert_eq!( + fs::metadata(&path).expect("metadata after refusal").mode(), + original_mode, + "foreign directory permissions changed before ownership was validated" + ); + } + /// A value the operator (or systemd) already set is handed to the child + /// unchanged: we never relocate a working runtime directory, and never + /// create anything of our own beside it. + #[test] + fn passes_an_existing_parent_value_through_unchanged() { + let scratch = tempfile::tempdir().expect("scratch"); + let existing = scratch.path().join("run").join("user").join("1000"); + let home = scratch.path().join("home"); + let parent = ParentRuntimeEnvironment { + xdg_runtime_dir: Some(existing.clone().into_os_string()), + home: Some(home.clone().into_os_string()), + user: Some("alice".to_owned()), + temp_dir: scratch.path().join("tmp"), + }; + + let value = value_of(&vars_for(&parent), "XDG_RUNTIME_DIR") + .expect("the child must be given a runtime directory"); + + assert_eq!(PathBuf::from(value), existing); + assert!( + !home.exists(), + "no fallback directory may be created when the parent already has one" + ); + } + } +} diff --git a/engines/lemonade/src/state.rs b/engines/lemonade/src/state.rs new file mode 100644 index 0000000000..55c49cab59 --- /dev/null +++ b/engines/lemonade/src/state.rs @@ -0,0 +1,810 @@ +// Copyright © Advanced Micro Devices, Inc., or its affiliates. +// +// SPDX-License-Identifier: MIT + +use anyhow::{Context, Result, bail}; +use rocm_core::{AppPaths, format_http_base_url}; +use rocm_engine_protocol::{ + DevicePolicy, ENGINE_RECIPE_CONTRACT_VERSION, EngineDeviceAvailability, EngineRecipeHint, + GpuSelection, +}; +use serde_json::{Value, json}; +use std::collections::{VecDeque, hash_map::DefaultHasher}; +use std::fs; +use std::hash::{Hash, Hasher}; +use std::io::{BufRead, Seek}; +use std::net::TcpListener; +use std::path::{Path, PathBuf}; +#[cfg(not(windows))] +use std::process::{Command as ProcessCommand, Stdio}; + +use crate::backend_alignment::LemonadeRuntime; +use crate::install::LemonadeInstallManifest; +use crate::{DEFAULT_HOST, DEFAULT_MODEL, ENGINE_NAME, ServeHttpRequest, current_unix_millis}; + +/// Maximum bytes to read from the end of a log file when extracting tail lines. +/// Prevents reading entire gigabyte-sized logs on startup timeout. +const MAX_TAIL_READ: u64 = 4 * 1024 * 1024; // 4MB + +#[derive(Debug, Clone)] +pub(crate) struct ServiceFiles { + pub(crate) state_path: PathBuf, + pub(crate) log_path: PathBuf, +} + +/// Whether a real inference request has succeeded against this service. +/// +/// Latch and backoff bookkeeping lives in `rocm-core` so both engines share one +/// implementation — what counts as *listed* differs per engine, what counts as +/// *serving* does not. +pub(crate) fn inference_verified( + state_path: &Path, + state: Option<&Value>, + endpoint_url: &str, + model_ref: &str, +) -> bool { + let Some((host, port)) = parse_http_endpoint(endpoint_url) else { + return false; + }; + rocm_core::engine_state_inference_verified( + state_path, + state, + &format_http_base_url(&host, port), + model_ref, + rocm_engine_protocol::resolve_endpoint_api_key().as_deref(), + ) +} + +/// The device string reported once a model is loaded. Reflects the backend that +/// actually ran (`rocm`, `vulkan`, …) and the pinned GPU ordinal when known, +/// rather than a hardcoded value, so the report matches observed execution. +pub(crate) fn reported_device(state: Option<&Value>, backend: &str) -> String { + match state.and_then(first_gpu_index_from_state) { + Some(index) => format!("{backend} gpu {index}"), + None => format!("{backend} gpu"), + } +} + +/// The first pinned GPU ordinal recorded in service state, if any. +fn first_gpu_index_from_state(state: &Value) -> Option { + state + .get("gpu_indices") + .and_then(Value::as_array) + .and_then(|indices| indices.first()) + .and_then(Value::as_u64) + .and_then(|value| u32::try_from(value).ok()) +} + +pub(crate) fn serve_http_command_args(request: &ServeHttpRequest) -> Vec { + let mut args = vec![ + "serve-http".to_owned(), + request.service_id.clone(), + request.model_ref.clone(), + "--host".to_owned(), + request.host.clone(), + "--port".to_owned(), + request.port.to_string(), + "--device-policy".to_owned(), + device_policy_name(&request.device_policy).to_owned(), + "--state-path".to_owned(), + request.state_path.display().to_string(), + ]; + if let Some(runtime_id) = request.runtime_id.as_deref() { + args.extend(["--runtime-id".to_owned(), runtime_id.to_owned()]); + } + if let Some(env_id) = request.env_id.as_deref() { + args.extend(["--env-id".to_owned(), env_id.to_owned()]); + } + if let Some(log_path) = request.log_path.as_ref() { + args.extend(["--log-path".to_owned(), log_path.display().to_string()]); + } + if let Some(csv) = rocm_engine_protocol::gpu_indices_to_csv(&request.gpu_indices) { + args.extend(["--gpu".to_owned(), csv]); + } + if let Some(engine_recipe) = request.engine_recipe.as_ref() { + args.extend([ + "--engine-recipe-json".to_owned(), + serde_json::to_string(engine_recipe).expect("engine recipe serializes"), + ]); + } + args +} + +#[cfg(windows)] +pub(crate) fn spawn_serve_http_background( + current_exe: &Path, + serve_args: &[String], +) -> Result { + rocm_core::spawn_detached_no_inherit(current_exe, serve_args, &[]) + .context("failed to launch Lemonade serve-http background process") +} + +#[cfg(not(windows))] +pub(crate) fn spawn_serve_http_background( + current_exe: &Path, + serve_args: &[String], +) -> Result { + let child = ProcessCommand::new(current_exe) + .args(serve_args) + .stdin(Stdio::null()) + .stdout(Stdio::null()) + .stderr(Stdio::null()) + .spawn() + .context("failed to launch Lemonade serve-http background process")?; + Ok(child.id()) +} + +pub(crate) fn write_running_state( + request: &ServeHttpRequest, + runtime: &LemonadeRuntime, + pid: u32, + server_pid: Option, + status: &str, +) -> Result<()> { + write_state( + &request.state_path, + &json!({ + "service_id": request.service_id, + "engine": ENGINE_NAME, + "status": status, + "pid": pid, + "server_pid": server_pid, + "model_ref": request.model_ref, + "canonical_model_id": request.model_ref, + "host": request.host, + "port": request.port, + "endpoint_url": endpoint_url(&request.host, request.port), + "device_policy": device_policy_name(&request.device_policy), + "gpu_indices": request.gpu_indices, + "runtime_id": request.runtime_id.as_deref().unwrap_or(runtime.manifest.env_id.as_str()), + "env_id": request.env_id.as_deref().unwrap_or(runtime.manifest.env_id.as_str()), + "runtime_kind": "lemonade_embeddable", + "runtime_executable": runtime.manifest.lemond, + "log_path": request.log_path.as_ref().map(|path| path.display().to_string()), + "engine_recipe": request.engine_recipe, + "started_at_unix_ms": current_unix_millis(), + // Kernel start-times captured while each PID is alive. Paired with the + // PIDs, they identify these exact processes across PID recycling so a + // later stop never signals a reused PID. + "start_ticks": rocm_core::process_start_ticks(pid), + "server_start_ticks": server_pid.and_then(rocm_core::process_start_ticks) + }), + ) +} + +pub(crate) fn service_files(service_id: &str) -> Result { + let paths = AppPaths::discover()?; + Ok(ServiceFiles { + state_path: paths + .engine_state_dir(ENGINE_NAME) + .join(format!("{service_id}.json")), + log_path: paths + .engine_logs_dir(ENGINE_NAME) + .join(format!("{service_id}.log")), + }) +} + +pub(crate) fn endpoint_url(host: &str, port: u16) -> String { + format!("{}/v1", format_http_base_url(host, port)) +} + +pub(crate) fn endpoint_url_from_state(state: &Value) -> Option { + value_string(state, "endpoint_url").or_else(|| { + let host = value_string(state, "host")?; + let port = value_u32(state, "port")?; + let port = u16::try_from(port).ok()?; + Some(endpoint_url(&host, port)) + }) +} + +pub(crate) fn read_service_state(path: &Path) -> Result { + let text = + fs::read_to_string(path).with_context(|| format!("failed to read {}", path.display()))?; + serde_json::from_str(&text).with_context(|| format!("failed to parse {}", path.display())) +} + +fn write_state(path: &Path, value: &Value) -> Result<()> { + if let Some(parent) = path.parent() { + fs::create_dir_all(parent)?; + } + fs::write(path, serde_json::to_vec_pretty(value)?) + .with_context(|| format!("failed to write {}", path.display())) +} + +pub(crate) fn merge_json_state(path: &Path, patch: &Value) -> Result<()> { + let mut value = read_service_state(path).unwrap_or_else(|_| json!({})); + if !value.is_object() { + value = json!({}); + } + let object = value.as_object_mut().expect("object checked above"); + if let Some(patch) = patch.as_object() { + for (key, value) in patch { + object.insert(key.clone(), value.clone()); + } + } + write_state(path, &value) +} + +pub(crate) fn mark_json_status(path: &Path, status: &str) -> Result<()> { + merge_json_state( + path, + &json!({ + "engine": ENGINE_NAME, + "status": status, + "stopped_at_unix_ms": current_unix_millis(), + }), + ) +} + +pub(crate) fn value_string(value: &Value, key: &str) -> Option { + value + .get(key) + .and_then(Value::as_str) + .filter(|value| !value.trim().is_empty()) + .map(ToOwned::to_owned) +} + +fn value_u32(value: &Value, key: &str) -> Option { + value + .get(key) + .and_then(Value::as_u64) + .and_then(|value| u32::try_from(value).ok()) +} + +/// Reconstruct the identity (PID + kernel start-time) of the process a stop +/// should terminate. The server PID is preferred, falling back to the launcher +/// PID, each paired with its recorded start-time. `start_ticks` is absent in +/// pre-existing state files, in which case verification degrades to best-effort. +pub(crate) fn identity_from_state(state: &Value) -> Option { + if let Some(server_pid) = value_u32(state, "server_pid") { + let start_ticks = state.get("server_start_ticks").and_then(Value::as_u64); + return Some(rocm_core::ProcessIdentity::new(server_pid, start_ticks)); + } + let pid = value_u32(state, "pid")?; + let start_ticks = state.get("start_ticks").and_then(Value::as_u64); + Some(rocm_core::ProcessIdentity::new(pid, start_ticks)) +} + +pub(crate) fn tail_lines(path: &Path, limit: usize) -> Result> { + if limit == 0 { + return Ok(Vec::new()); + } + let file = + fs::File::open(path).with_context(|| format!("failed to open {}", path.display()))?; + let metadata = file + .metadata() + .with_context(|| format!("failed to stat {}", path.display()))?; + let file_size = metadata.len(); + + // For small files, read normally to preserve exact line count. + if file_size <= MAX_TAIL_READ { + let reader = std::io::BufReader::new(file); + let mut lines = VecDeque::with_capacity(limit); + for line in reader.lines() { + let line = line.with_context(|| format!("failed to read {}", path.display()))?; + if lines.len() == limit { + lines.pop_front(); + } + lines.push_back(line); + } + return Ok(lines.into_iter().collect()); + } + + // For large files, seek near the end and read only the final chunk. + // This prevents reading multi-gigabyte logs during timeout errors. + let mut file = file; + let seek_pos = file_size.saturating_sub(MAX_TAIL_READ); + file.seek(std::io::SeekFrom::Start(seek_pos)) + .with_context(|| format!("failed to seek in {}", path.display()))?; + + let reader = std::io::BufReader::new(file); + let mut lines = VecDeque::with_capacity(limit); + let mut skipped_first = seek_pos == 0; + for line in reader.lines() { + let line = line.with_context(|| format!("failed to read {}", path.display()))?; + // Skip the first line after seeking, as it may be partial. + if !skipped_first { + skipped_first = true; + continue; + } + if lines.len() == limit { + lines.pop_front(); + } + lines.push_back(line); + } + Ok(lines.into_iter().collect()) +} + +pub(crate) fn terminate_pid(pid: u32, _force: bool) -> bool { + rocm_core::terminate_process(pid).is_ok() +} + +pub(crate) fn free_local_port() -> Result { + let listener = TcpListener::bind((DEFAULT_HOST, 0)).context("failed to reserve local port")?; + Ok(listener.local_addr()?.port()) +} + +pub(crate) fn parse_http_endpoint(endpoint_url: &str) -> Option<(String, u16)> { + let without_scheme = endpoint_url.trim().strip_prefix("http://")?; + let authority = without_scheme.split('/').next()?.trim(); + if let Some(rest) = authority.strip_prefix('[') { + let end = rest.find(']')?; + let host = rest[..end].to_owned(); + let port = rest[end + 1..].strip_prefix(':')?.parse().ok()?; + return Some((host, port)); + } + let (host, port) = authority.rsplit_once(':')?; + Some((host.to_owned(), port.parse().ok()?)) +} + +pub(crate) fn normalize_device_policy(policy: Option) -> Result { + match policy.unwrap_or(DevicePolicy::GpuRequired) { + DevicePolicy::GpuRequired | DevicePolicy::GpuPreferred => Ok(DevicePolicy::GpuRequired), + DevicePolicy::CpuOnly => { + bail!("Lemonade adapter requires ROCm GPU execution; no CPU fallback is used") + } + } +} + +pub(crate) fn require_gpu_required(policy: &DevicePolicy) -> Result<()> { + match policy { + DevicePolicy::GpuRequired | DevicePolicy::GpuPreferred => Ok(()), + DevicePolicy::CpuOnly => { + bail!("Lemonade adapter requires ROCm GPU execution; no CPU fallback is used") + } + } +} + +/// Probe real GPU availability and resolve the device ordinal(s) to pin before a +/// GPU-required launch. Returns the resolved indices, or an error that stops +/// serving before any process is spawned: +/// - no usable device → fail with actionable guidance (no CPU/GPU-0 fallback); +/// - `auto` (empty `requested`) → the first present, visible device; +/// - explicit index not among the usable devices → fail. +/// +/// When availability cannot be probed on this platform, the requested selection +/// is passed through unchanged so serving is not blocked on that basis. +pub(crate) fn resolve_serve_gpu_indices(requested: &[u32]) -> Result> { + resolve_gpu_indices_against(requested, rocm_core::usable_amd_gpu_indices()) +} + +/// Pure resolution used by [`resolve_serve_gpu_indices`], split out so the +/// auto/explicit/no-device policy can be unit-tested without real hardware. +/// `usable` is the probe result: `None` (unprobeable) passes the request +/// through; `Some(_)` is authoritative. +fn resolve_gpu_indices_against(requested: &[u32], usable: Option>) -> Result> { + let Some(usable) = usable else { + return Ok(requested.to_vec()); + }; + if usable.is_empty() { + bail!( + "no usable AMD GPU detected; `rocm serve` requires a GPU under the default \ + GPU-required policy and does not fall back to CPU. Check the driver with \ + `rocm examine`, confirm /dev/kfd is present, and ensure HIP_VISIBLE_DEVICES / \ + ROCR_VISIBLE_DEVICES are not masking every device." + ); + } + if requested.is_empty() { + return Ok(vec![usable[0]]); + } + for index in requested { + if !usable.contains(index) { + let visible = usable + .iter() + .map(u32::to_string) + .collect::>() + .join(", "); + bail!( + "requested GPU {index} is not available on this host; usable GPU indices: [{visible}]" + ); + } + } + Ok(requested.to_vec()) +} + +/// GPU device availability for `detect`, sourced from the real device probe. When +/// the probe is authoritative it reports true/false with a reason; when it cannot +/// determine availability on this platform, it falls back to whether the runtime +/// is installed so `detect` stays informative. +pub(crate) fn gpu_availability_device(runtime_installed: bool) -> EngineDeviceAvailability { + match rocm_core::usable_amd_gpu_indices() { + Some(indices) if !indices.is_empty() => EngineDeviceAvailability { + kind: "rocm_gpu".to_owned(), + available: true, + reason: None, + }, + Some(_) => EngineDeviceAvailability { + kind: "rocm_gpu".to_owned(), + available: false, + reason: Some( + "no AMD GPU detected, or every device is masked by \ + HIP_VISIBLE_DEVICES / ROCR_VISIBLE_DEVICES" + .to_owned(), + ), + }, + None => EngineDeviceAvailability { + kind: "rocm_gpu".to_owned(), + available: runtime_installed, + reason: if runtime_installed { + None + } else { + Some("Lemonade embeddable runtime is not installed".to_owned()) + }, + }, + } +} + +pub(crate) fn parse_device_policy_arg(value: Option<&str>) -> Result { + match value.unwrap_or("gpu_required") { + "gpu" | "gpu_required" | "gpu_preferred" => Ok(DevicePolicy::GpuRequired), + "cpu" | "cpu_only" => Ok(DevicePolicy::CpuOnly), + other => bail!("unknown device policy `{other}`"), + } +} + +/// Parse a `--gpu` CLI value into an optional `GpuSelection` for `LaunchRequest`. +pub(crate) fn parse_gpu_selection_arg(value: Option<&str>) -> Result> { + value + .map(|raw| GpuSelection::parse_cli_value(raw).map_err(anyhow::Error::msg)) + .transpose() +} + +/// Parse a `--gpu` CLI value into explicit device ordinals (empty for `auto`). +pub(crate) fn parse_gpu_indices_arg(value: Option<&str>) -> Result> { + Ok(rocm_engine_protocol::launch_gpu_indices( + parse_gpu_selection_arg(value)?.as_ref(), + )) +} + +const fn device_policy_name(policy: &DevicePolicy) -> &'static str { + match policy { + DevicePolicy::GpuRequired => "gpu_required", + DevicePolicy::GpuPreferred => "gpu_preferred", + DevicePolicy::CpuOnly => "cpu_only", + } +} + +pub(crate) fn accepted_engine_recipe( + engine_recipe: Option, +) -> Result> { + if let Some(hint) = &engine_recipe { + if hint.engine != ENGINE_NAME { + bail!( + "engine_recipe target `{}` does not match adapter `{}`", + hint.engine, + ENGINE_NAME + ); + } + if hint.contract_version != ENGINE_RECIPE_CONTRACT_VERSION { + bail!( + "engine_recipe contract `{}` is unsupported; expected `{}`", + hint.contract_version, + ENGINE_RECIPE_CONTRACT_VERSION + ); + } + } + Ok(engine_recipe) +} + +pub(crate) fn parse_engine_recipe_json(value: Option) -> Result> { + value + .map(|text| { + serde_json::from_str::(&text) + .context("failed to parse engine recipe JSON") + }) + .transpose() + .and_then(accepted_engine_recipe) +} + +pub(crate) fn resolve_lemonade_model_ref(model_ref: &str) -> String { + let trimmed = model_ref.trim(); + let lower = trimmed.to_ascii_lowercase(); + // Only exact, recognized shorthand aliases map to the bundled assistant GGUF. + // A syntactically valid `owner/repo` Hugging Face checkpoint must never be + // silently rewritten here — e.g. `Qwen/Qwen2.5-1.5B-Instruct` is a real + // checkpoint id, not a shorthand, and is passed through unchanged so the + // serve path can honour (or explicitly reject) it rather than substituting + // an unrelated model behind the user's back. + if trimmed.is_empty() + || matches!( + lower.as_str(), + "qwen" + | "assistant" + | "default" + | "small" + | "lemonade-qwen" + | "qwen-gguf" + | "qwen3-4b" + | "qwen3-4b-instruct" + | "qwen3-4b-instruct-2507-gguf" + ) + { + DEFAULT_MODEL.to_owned() + } else if matches!( + lower.as_str(), + "tiny" | "qwen-smoke" | "lemonade-tiny" | "qwen3-0.6b-gguf" + ) { + "Qwen3-0.6B-GGUF".to_owned() + } else { + trimmed.to_owned() + } +} + +pub(crate) fn manifest_lock_hash(manifest: &LemonadeInstallManifest) -> String { + let mut hasher = DefaultHasher::new(); + manifest.env_id.hash(&mut hasher); + manifest.version.hash(&mut hasher); + manifest.runtime_dir.hash(&mut hasher); + manifest.lemond.hash(&mut hasher); + format!("{:016x}", hasher.finish()) +} + +#[cfg(test)] +mod tests { + use super::*; + use std::io::{Read, Write}; + use std::time::Duration; + + /// A fresh scratch directory under the crate's `target/`. The base is + /// `CARGO_MANIFEST_DIR`, a compile-time constant, so the path never derives from a + /// runtime environment read. + fn scratch_dir(tag: &str) -> PathBuf { + let dir = Path::new(env!("CARGO_MANIFEST_DIR")) + .join("target") + .join(format!("lemonade-fs-test-{tag}")); + let _ = fs::remove_dir_all(&dir); + fs::create_dir_all(&dir).unwrap(); + dir + } + + #[test] + fn identity_from_state_prefers_server_pid_and_its_start_ticks() { + let state = json!({ + "pid": 100, + "start_ticks": 111_u64, + "server_pid": 200, + "server_start_ticks": 222_u64, + }); + let identity = identity_from_state(&state).expect("identity"); + assert_eq!(identity.pid, 200); + assert_eq!(identity.start_ticks, Some(222)); + } + + #[test] + fn identity_from_state_falls_back_to_launcher_pid() { + let state = json!({ "pid": 100, "start_ticks": 111_u64 }); + let identity = identity_from_state(&state).expect("identity"); + assert_eq!(identity.pid, 100); + assert_eq!(identity.start_ticks, Some(111)); + } + + #[test] + fn identity_from_legacy_state_has_no_start_ticks() { + // Pre-existing state files carry only PIDs; verification must degrade. + let state = json!({ "server_pid": 200 }); + let identity = identity_from_state(&state).expect("identity"); + assert_eq!(identity.pid, 200); + assert_eq!(identity.start_ticks, None); + } + + #[test] + fn identity_from_state_without_any_pid_is_none() { + assert!(identity_from_state(&json!({ "status": "running" })).is_none()); + } + + #[test] + fn qwen_alias_resolves_to_validated_assistant_gguf_model() { + assert_eq!(resolve_lemonade_model_ref("qwen"), DEFAULT_MODEL); + assert_eq!(resolve_lemonade_model_ref("qwen-smoke"), "Qwen3-0.6B-GGUF"); + } + + #[test] + fn canonical_hugging_face_name_passes_through_unchanged() { + assert_eq!( + resolve_lemonade_model_ref("LiquidAI/LFM2.5-230M-GGUF:Q4_0"), + "LiquidAI/LFM2.5-230M-GGUF:Q4_0" + ); + // Regression (EAI-7370): a fully-qualified checkpoint id that merely + // contains a known-alias substring must not be swapped for the bundled + // default. `Qwen/Qwen2.5-1.5B-Instruct` was silently served as + // `Qwen3-4B-Instruct-2507-GGUF`; it must now pass through verbatim. + assert_eq!( + resolve_lemonade_model_ref("Qwen/Qwen2.5-1.5B-Instruct"), + "Qwen/Qwen2.5-1.5B-Instruct" + ); + } + + /// Answer `count` chat requests on a loopback port with the given status, + /// recording how many arrived. + fn spawn_chat_endpoint( + status_line: &'static str, + count: usize, + ) -> (u16, std::thread::JoinHandle) { + let listener = TcpListener::bind("127.0.0.1:0").expect("bind loopback"); + let port = listener.local_addr().expect("local addr").port(); + let handle = std::thread::spawn(move || { + let mut served = 0; + for _ in 0..count { + let Ok((mut stream, _)) = listener.accept() else { + break; + }; + stream.set_read_timeout(Some(Duration::from_secs(2))).ok(); + let mut buffer = [0_u8; 1024]; + let _ = stream.read(&mut buffer); + let body = r#"{"choices":[{"message":{"content":"ok"}}]}"#; + let _ = write!( + stream, + "{status_line}\r\nContent-Type: application/json\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{}", + body.len(), + body + ); + served += 1; + } + served + }); + (port, handle) + } + + #[test] + fn inference_verification_latches_into_the_state_file() { + // First check probes and records the verdict; the second reads the latch + // and leaves the model alone. + let (port, server) = spawn_chat_endpoint("HTTP/1.1 200 OK", 1); + let dir = scratch_dir("inference-latch"); + let state_path = dir.join("state.json"); + fs::write(&state_path, json!({"status": "ready"}).to_string()).expect("seed state"); + let endpoint = format_http_base_url("127.0.0.1", port); + + let state = read_service_state(&state_path).ok(); + assert!(inference_verified( + &state_path, + state.as_ref(), + &endpoint, + DEFAULT_MODEL + )); + + let state = read_service_state(&state_path).expect("state readable"); + assert!( + state + .get(rocm_core::INFERENCE_VERIFIED_STATE_KEY) + .and_then(Value::as_u64) + .is_some(), + "a passing probe is latched so later healthchecks skip it" + ); + assert!(inference_verified( + &state_path, + Some(&state), + &endpoint, + DEFAULT_MODEL + )); + + assert_eq!( + server.join().expect("server thread"), + 1, + "the latched check must not send a second inference request" + ); + fs::remove_dir_all(&dir).ok(); + } + + #[test] + fn inference_verification_withheld_while_the_endpoint_cannot_serve() { + // The reported failure: the model is listed but inference still 5xxs. + let (port, server) = spawn_chat_endpoint("HTTP/1.1 503 Service Unavailable", 1); + let dir = scratch_dir("inference-unverified"); + let state_path = dir.join("state.json"); + fs::write(&state_path, json!({"status": "ready"}).to_string()).expect("seed state"); + let endpoint = format_http_base_url("127.0.0.1", port); + + let state = read_service_state(&state_path).ok(); + assert!(!inference_verified( + &state_path, + state.as_ref(), + &endpoint, + DEFAULT_MODEL + )); + + let state = read_service_state(&state_path).expect("state readable"); + assert!( + state.get(rocm_core::INFERENCE_VERIFIED_STATE_KEY).is_none(), + "nothing is latched until inference actually answers" + ); + + server.join().expect("server thread"); + fs::remove_dir_all(&dir).ok(); + } + + #[test] + fn serve_gpu_resolution_fails_fast_when_no_device_usable() { + // No-device / masked-device path: the probe authoritatively reports zero + // usable GPUs, so serving is refused before any process is spawned. + let error = resolve_gpu_indices_against(&[], Some(vec![])) + .expect_err("zero usable devices must be rejected"); + assert!(error.to_string().contains("no usable AMD GPU")); + } + + #[test] + fn serve_gpu_resolution_auto_pins_first_present_device() { + // `auto` (empty request) resolves to the first present device — never an + // assumed device 0 when device 0 is not present. + assert_eq!( + resolve_gpu_indices_against(&[], Some(vec![1, 2])).unwrap(), + vec![1] + ); + } + + #[test] + fn serve_gpu_resolution_validates_explicit_index() { + // An explicitly requested present device is honored. + assert_eq!( + resolve_gpu_indices_against(&[2], Some(vec![0, 1, 2])).unwrap(), + vec![2] + ); + // A requested device that is not present is rejected rather than silently + // remapped to another GPU. + let error = resolve_gpu_indices_against(&[3], Some(vec![0, 1])) + .expect_err("absent device must be rejected"); + assert!( + error + .to_string() + .contains("requested GPU 3 is not available") + ); + } + + #[test] + fn serve_gpu_resolution_passes_through_when_unprobeable() { + // When availability cannot be determined, the request is passed through + // unchanged so serving is not blocked on that basis. + assert_eq!( + resolve_gpu_indices_against(&[], None).unwrap(), + Vec::::new() + ); + assert_eq!(resolve_gpu_indices_against(&[1], None).unwrap(), vec![1]); + } + + #[test] + fn reported_device_reflects_backend_and_pinned_ordinal() { + let state = json!({ "gpu_indices": [1] }); + assert_eq!(reported_device(Some(&state), "vulkan"), "vulkan gpu 1"); + // Without a recorded ordinal the backend is still reported truthfully. + assert_eq!(reported_device(None, "rocm"), "rocm gpu"); + } + + #[test] + fn device_policy_rejects_cpu_without_fallback() { + let error = normalize_device_policy(Some(DevicePolicy::CpuOnly)) + .expect_err("cpu should be rejected") + .to_string(); + assert!(error.contains("no CPU fallback")); + } + + #[test] + fn endpoint_parser_supports_ipv6_loopback() { + assert_eq!( + parse_http_endpoint("http://[::1]:11435/v1"), + Some(("::1".to_owned(), 11435)) + ); + } + + #[test] + fn serve_http_args_preserve_runtime_selection() { + let request = ServeHttpRequest { + service_id: "svc".to_owned(), + model_ref: DEFAULT_MODEL.to_owned(), + host: "127.0.0.1".to_owned(), + port: 11435, + device_policy: DevicePolicy::GpuRequired, + gpu_indices: Vec::new(), + runtime_id: Some("runtime".to_owned()), + env_id: Some("env".to_owned()), + state_path: PathBuf::from("state.json"), + log_path: Some(PathBuf::from("service.log")), + engine_recipe: None, + }; + let args = serve_http_command_args(&request); + assert!(args.contains(&"--runtime-id".to_owned())); + assert!(args.contains(&"runtime".to_owned())); + assert!(args.contains(&"--env-id".to_owned())); + assert!(args.contains(&"env".to_owned())); + assert!(!args.iter().any(|arg| arg == "cpu")); + } +}