From 767b4d089dc73b39c60c4b358495f15731db2b3b Mon Sep 17 00:00:00 2001 From: pmoutsias-amd Date: Wed, 26 Aug 2026 09:45:10 -0400 Subject: [PATCH 01/23] Add Sphinx documentation site Adds a Sphinx documentation site under docs/rocm-docs/, single-sourced from README.md and CONTRIBUTING.md via MyST {include} directives so the docs and repo content stay in sync automatically. Covers installation, getting started, command reference, demos, and contributing/license, and is set up to publish with the rocm-docs-core theme. Updates some links in README.md and CONTRIBUTING.md to be absolute so they render correctly when reused via {include}. Signed-off-by: pmoutsia_amdeng --- .gitignore | 4 +++ CONTRIBUTING.md | 8 ++--- README.md | 8 ++--- docs/rocm-docs/about/contributing.md | 13 ++++++++ docs/rocm-docs/about/license.md | 5 +++ docs/rocm-docs/commands.md | 6 ++++ docs/rocm-docs/conf.py | 31 +++++++++++++++++++ docs/rocm-docs/demos.md | 6 ++++ docs/rocm-docs/getting-started.md | 20 ++++++++++++ docs/rocm-docs/index.rst | 43 ++++++++++++++++++++++++++ docs/rocm-docs/install/installation.md | 22 +++++++++++++ docs/rocm-docs/sphinx/_toc.yml.in | 18 +++++++++++ docs/rocm-docs/sphinx/requirements.txt | 1 + 13 files changed, 177 insertions(+), 8 deletions(-) create mode 100644 docs/rocm-docs/about/contributing.md create mode 100644 docs/rocm-docs/about/license.md create mode 100644 docs/rocm-docs/commands.md create mode 100644 docs/rocm-docs/conf.py create mode 100644 docs/rocm-docs/demos.md create mode 100644 docs/rocm-docs/getting-started.md create mode 100644 docs/rocm-docs/index.rst create mode 100644 docs/rocm-docs/install/installation.md create mode 100644 docs/rocm-docs/sphinx/_toc.yml.in create mode 100644 docs/rocm-docs/sphinx/requirements.txt diff --git a/.gitignore b/.gitignore index 6a19bd1e30..fdec19493e 100644 --- a/.gitignore +++ b/.gitignore @@ -17,3 +17,7 @@ __pycache__/ /plans/ /workspace/ /e2e-consolidated-report.md + +# Sphinx docs: _toc.yml is generated from _toc.yml.in at build time +docs/rocm-docs/sphinx/_toc.yml +docs/rocm-docs/_build/ diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 45be3df779..ad8bfafa1e 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -10,11 +10,11 @@ Thank you for your interest in contributing. This document explains how to get s ## Code of Conduct -By participating in this project you agree to abide by the [Code of Conduct](CODE_OF_CONDUCT.md). +By participating in this project you agree to abide by the [Code of Conduct](https://github.com/ROCm/rocm-cli/blob/main/CODE_OF_CONDUCT.md). ## Issue Tracking -Bug reports and feature requests are tracked as [GitHub Issues](../../issues). Before opening a new issue, search existing ones to avoid duplicates. +Bug reports and feature requests are tracked as [GitHub Issues](https://github.com/ROCm/rocm-cli/issues). Before opening a new issue, search existing ones to avoid duplicates. ## Branch and Commit Naming @@ -104,8 +104,8 @@ See `docs/commit-signatures.md` for GPG signing, GitHub "Verified" status, and t ## Reporting Security Issues -Do **not** open a public GitHub Issue for security vulnerabilities. See [SECURITY.md](SECURITY.md) for the responsible disclosure process. +Do **not** open a public GitHub Issue for security vulnerabilities. See [SECURITY.md](https://github.com/ROCm/rocm-cli/blob/main/SECURITY.md) for the responsible disclosure process. ## License -By contributing you agree that your contributions will be licensed under the [MIT License](LICENSE.TXT). +By contributing you agree that your contributions will be licensed under the [MIT License](https://github.com/ROCm/rocm-cli/blob/main/LICENSE.TXT). diff --git a/README.md b/README.md index bae9b4c09b..91c3a9c744 100644 --- a/README.md +++ b/README.md @@ -124,7 +124,7 @@ Or copy the release binaries onto your `PATH`: install -m 0755 target/release/rocm target/release/rocmd ~/.local/bin/ ``` -See [CONTRIBUTING.md](CONTRIBUTING.md) for the full development setup, test +See [CONTRIBUTING.md](https://github.com/ROCm/rocm-cli/blob/main/CONTRIBUTING.md) for the full development setup, test commands, and commit-signing requirements. ## First run @@ -175,7 +175,7 @@ Live mode reads telemetry over a Unix domain socket, so it requires Linux or WSL. Use `rocm dash --demo` for a synthetic session that runs anywhere without a GPU or daemon. -## Getting started +## Configure ROCm and serve a model Before serving a model, ensure a managed ROCm runtime is configured: @@ -196,7 +196,7 @@ rocm serve qwen `qwen` is a built-in alias for a small assistant model that serves out of the box. You can also serve any compatible Hugging Face model directly — see -[Model serving](#model-serving) for the GGUF-vs-safetensors rule, since which +[Model serving](https://github.com/ROCm/rocm-cli#model-serving) for the GGUF-vs-safetensors rule, since which form works depends on the engine your GPU selects. Use `rocm model` to see available model recipes and their GPU memory @@ -399,7 +399,7 @@ rocm dash [--demo] [--replay ] Full-screen TUI with Home, ROCm, Serving, Observe, and Chat tabs — GPU utilization graphs, active serving instances, benchmark results, guided actions, and a chat tab backed by any configured provider. See -[Interactive interfaces](#interactive-interfaces) for the tab breakdown. +[Interactive interfaces](https://github.com/ROCm/rocm-cli#interactive-interfaces) for the tab breakdown. - `--demo` runs a deterministic synthetic session with no GPU or daemon needed, works on all platforms. diff --git a/docs/rocm-docs/about/contributing.md b/docs/rocm-docs/about/contributing.md new file mode 100644 index 0000000000..2ec7735cd7 --- /dev/null +++ b/docs/rocm-docs/about/contributing.md @@ -0,0 +1,13 @@ +# Contributing + +```{include} ../../../CONTRIBUTING.md +:start-after: "# Contributing to ROCm CLI" +``` + +## More docs + + +```{include} ../../../README.md +:start-after: "## More docs" +``` diff --git a/docs/rocm-docs/about/license.md b/docs/rocm-docs/about/license.md new file mode 100644 index 0000000000..4c8505e0ba --- /dev/null +++ b/docs/rocm-docs/about/license.md @@ -0,0 +1,5 @@ +# License + +```{literalinclude} ../../../LICENSE.TXT +:language: text +``` diff --git a/docs/rocm-docs/commands.md b/docs/rocm-docs/commands.md new file mode 100644 index 0000000000..7c8de14ae5 --- /dev/null +++ b/docs/rocm-docs/commands.md @@ -0,0 +1,6 @@ +# Command reference + +```{include} ../../README.md +:start-after: "## Commands" +:end-before: "## Contributing" +``` diff --git a/docs/rocm-docs/conf.py b/docs/rocm-docs/conf.py new file mode 100644 index 0000000000..7a7235514b --- /dev/null +++ b/docs/rocm-docs/conf.py @@ -0,0 +1,31 @@ +# Configuration file for the Sphinx documentation builder. +# pylint: disable=invalid-name + +# -- Project information --------------------------------------------------- + +project = "ROCm CLI" +author = "Advanced Micro Devices, Inc." +# pylint: disable=redefined-builtin +copyright = "Copyright (c) Advanced Micro Devices, Inc. All rights reserved." +# pylint: enable=redefined-builtin + +# -- General configuration ------------------------------------------------- + +html_theme = "rocm_docs_theme" +html_theme_options = {"flavor": "rocm"} +html_title = "ROCm CLI documentation" +suppress_warnings = ["etoc.toctree"] +external_toc_path = "./sphinx/_toc.yml" +external_projects_current_project = "rocm-cli" +extensions = [ + "rocm_docs", + "sphinx_design", +] +exclude_patterns = [] + +# -- Sphinx setup ---------------------------------------------------------- + + +def setup(app): + """Sphinx setup""" + return {"parallel_read_safe": True, "parallel_write_safe": True} diff --git a/docs/rocm-docs/demos.md b/docs/rocm-docs/demos.md new file mode 100644 index 0000000000..5a3582ed72 --- /dev/null +++ b/docs/rocm-docs/demos.md @@ -0,0 +1,6 @@ +# Demos + +```{include} ../../README.md +:start-after: "## Demos" +:end-before: " | Platform | Prebuilt binary | Notes | |---|---|---| | Linux (x86_64) | Yes | Full support, including the live dashboard and both inference engines | | Windows (x86_64) | Yes | CLI and Lemonade serving; no live dashboard or vLLM | | WSL2 (x86_64) | Yes (Linux binary) | Full support, including the live dashboard; see [docs/wsl.md](docs/wsl.md) for setup | | macOS | No | No official installer, release, CI, or QA coverage | + Live dashboard telemetry requires Linux or WSL2 (see [Interactive interfaces](#interactive-interfaces)). vLLM serving is Linux/WSL2 diff --git a/docs/rocm-docs/install/installation.md b/docs/rocm-docs/install/installation.md index f744267bb1..8f2ceeaa04 100644 --- a/docs/rocm-docs/install/installation.md +++ b/docs/rocm-docs/install/installation.md @@ -2,12 +2,10 @@ ROCm CLI ships as a single prebuilt binary. Platform support: -| Platform | Prebuilt binary | Notes | -|---|---|---| -| Linux (x86_64) | Yes | Full support, including the live dashboard and both inference engines | -| Windows (x86_64) | Yes | CLI and Lemonade serving; no live dashboard or vLLM | -| WSL2 (x86_64) | Yes (Linux binary) | Full support, including the live dashboard; see `docs/wsl.md` for setup | -| macOS | No | No official installer, release, CI, or QA coverage | +```{include} ../../../README.md +:start-after: "" +:end-before: "" +``` Live dashboard telemetry requires Linux or WSL2 (see [Interactive interfaces](../getting-started.md#interactive-interfaces)). vLLM From 540239ef3aec55f62dd42e1e72595ec181f06b6d Mon Sep 17 00:00:00 2001 From: pmoutsias-amd Date: Wed, 26 Aug 2026 09:45:10 -0400 Subject: [PATCH 03/23] fix(docs): address review feedback on Sphinx docs site - Add the MIT license header to the 7 new files hawkeye flagged as missing it. - Fix the WSL2 platform-support-table link to docs/wsl.md: it's now an absolute GitHub URL so it resolves when the table is pulled into install/installation.md via MyST include, matching the pattern already used for the other README anchors. - Pin rocm-docs-core to the exact version that resolved locally (1.40.0) instead of an open->= range. Signed-off-by: pmoutsia_amdeng --- README.md | 2 +- docs/rocm-docs/about/contributing.md | 6 ++++++ docs/rocm-docs/about/license.md | 6 ++++++ docs/rocm-docs/commands.md | 6 ++++++ docs/rocm-docs/conf.py | 4 ++++ docs/rocm-docs/demos.md | 6 ++++++ docs/rocm-docs/getting-started.md | 6 ++++++ docs/rocm-docs/install/installation.md | 8 +++++++- docs/rocm-docs/sphinx/requirements.txt | 2 +- 9 files changed, 43 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 39d32073ec..9f1c4f205e 100644 --- a/README.md +++ b/README.md @@ -32,7 +32,7 @@ Lemonade and vLLM. |---|---|---| | Linux (x86_64) | Yes | Full support, including the live dashboard and both inference engines | | Windows (x86_64) | Yes | CLI and Lemonade serving; no live dashboard or vLLM | -| WSL2 (x86_64) | Yes (Linux binary) | Full support, including the live dashboard; see [docs/wsl.md](docs/wsl.md) for setup | +| WSL2 (x86_64) | Yes (Linux binary) | Full support, including the live dashboard; see [docs/wsl.md](https://github.com/ROCm/rocm-cli/blob/main/docs/wsl.md) for setup | | macOS | No | No official installer, release, CI, or QA coverage | diff --git a/docs/rocm-docs/about/contributing.md b/docs/rocm-docs/about/contributing.md index 2ec7735cd7..7a86854ecd 100644 --- a/docs/rocm-docs/about/contributing.md +++ b/docs/rocm-docs/about/contributing.md @@ -1,3 +1,9 @@ + + # Contributing ```{include} ../../../CONTRIBUTING.md diff --git a/docs/rocm-docs/about/license.md b/docs/rocm-docs/about/license.md index 4c8505e0ba..16436f19f2 100644 --- a/docs/rocm-docs/about/license.md +++ b/docs/rocm-docs/about/license.md @@ -1,3 +1,9 @@ + + # License ```{literalinclude} ../../../LICENSE.TXT diff --git a/docs/rocm-docs/commands.md b/docs/rocm-docs/commands.md index 7c8de14ae5..96ef428f44 100644 --- a/docs/rocm-docs/commands.md +++ b/docs/rocm-docs/commands.md @@ -1,3 +1,9 @@ + + # Command reference ```{include} ../../README.md diff --git a/docs/rocm-docs/conf.py b/docs/rocm-docs/conf.py index 7a7235514b..e7cf09da33 100644 --- a/docs/rocm-docs/conf.py +++ b/docs/rocm-docs/conf.py @@ -1,3 +1,7 @@ +# Copyright © Advanced Micro Devices, Inc., or its affiliates. +# +# SPDX-License-Identifier: MIT + # Configuration file for the Sphinx documentation builder. # pylint: disable=invalid-name diff --git a/docs/rocm-docs/demos.md b/docs/rocm-docs/demos.md index 5a3582ed72..9540185de0 100644 --- a/docs/rocm-docs/demos.md +++ b/docs/rocm-docs/demos.md @@ -1,3 +1,9 @@ + + # Demos ```{include} ../../README.md diff --git a/docs/rocm-docs/getting-started.md b/docs/rocm-docs/getting-started.md index f0dec21ba7..6a50539b7a 100644 --- a/docs/rocm-docs/getting-started.md +++ b/docs/rocm-docs/getting-started.md @@ -1,3 +1,9 @@ + + # Getting started with ROCm CLI ```{include} ../../README.md diff --git a/docs/rocm-docs/install/installation.md b/docs/rocm-docs/install/installation.md index 8f2ceeaa04..5f1cb3dc86 100644 --- a/docs/rocm-docs/install/installation.md +++ b/docs/rocm-docs/install/installation.md @@ -1,3 +1,9 @@ + + # Installing ROCm CLI ROCm CLI ships as a single prebuilt binary. Platform support: @@ -9,7 +15,7 @@ ROCm CLI ships as a single prebuilt binary. Platform support: Live dashboard telemetry requires Linux or WSL2 (see [Interactive interfaces](../getting-started.md#interactive-interfaces)). vLLM -serving is Linux/WSL2 only (see `docs/vllm.md`). +serving is Linux or WSL2 only (see `docs/vllm.md`). ```{include} ../../../README.md :start-after: "## Installation" diff --git a/docs/rocm-docs/sphinx/requirements.txt b/docs/rocm-docs/sphinx/requirements.txt index 33f88dcf1e..d330c976ac 100644 --- a/docs/rocm-docs/sphinx/requirements.txt +++ b/docs/rocm-docs/sphinx/requirements.txt @@ -1 +1 @@ -rocm-docs-core>=1.7.1 +rocm-docs-core==1.40.0 From fda13f1b0193306de29008032df590a6539192fa Mon Sep 17 00:00:00 2001 From: pmoutsias-amd Date: Wed, 26 Aug 2026 10:27:19 -0400 Subject: [PATCH 04/23] fix(docs): add ReadTheDocs config and a CI job to build docs with -W MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Add .readthedocs.yaml so RTD builds against docs/rocm-docs/conf.py with the same pinned rocm-docs-core toolchain CI uses. - Add a docs-build CI job that runs sphinx-build -W (warnings as errors) against the same source RTD points at, gated on the new `docs` path-filter category so it only runs when doc sources, README.md/CONTRIBUTING.md (single-sourced via MyST includes), or the build config change. - Suppress the myst.header warning from :start-after:/:end-before: README includes stripping the anchor heading, which is cosmetic: docutils re-normalizes the resulting heading depth in the rendered output. - Scope external_projects to [] instead of the implicit "all" default, which was fetching intersphinx inventories for every project in rocm_docs' bundled catalog (~90 projects) - several of which 404 or have moved upstream. No page here uses a cross-project intersphinx role, so this is lossless and makes -W builds deterministic instead of flaky against infrastructure this repo doesn't control. - Filter the "current project not found in projects" warning: rocm-cli isn't registered in rocm-docs-core's shared projects catalog yet, so external_projects_current_project can never resolve until that's added upstream. rocm_docs.projects handles the unresolved project as None everywhere it's used, so this is informational, not a defect. Signed-off-by: pmoutsia_amdeng --- .github/workflows/ci.yml | 41 ++++++++++++++++++++++++++++++++++++++++ .readthedocs.yaml | 13 +++++++++++++ docs/rocm-docs/conf.py | 31 +++++++++++++++++++++++++++++- 3 files changed, 84 insertions(+), 1 deletion(-) create mode 100644 .readthedocs.yaml diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index fab8bc6347..8a940010dc 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -82,6 +82,7 @@ jobs: heavy: ${{ steps.filter.outputs.heavy || steps.all.outputs.forced }} lint: ${{ steps.filter.outputs.lint || steps.all.outputs.forced }} tpn: ${{ steps.filter.outputs.tpn || steps.all.outputs.forced }} + docs: ${{ steps.filter.outputs.docs || steps.all.outputs.forced }} steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 @@ -148,6 +149,15 @@ jobs: - 'ruff.toml' - 'PSScriptAnalyzerSettings.psd1' - '.github/workflows/**' + # Sphinx docs site: the doc sources themselves, plus README.md and + # CONTRIBUTING.md, which several pages single-source via MyST + # `{include}` directives, plus the build config/toolchain pins. + docs: + - 'docs/rocm-docs/**' + - 'README.md' + - 'CONTRIBUTING.md' + - '.readthedocs.yaml' + - '.github/workflows/**' - name: Force full run off pull requests id: all @@ -618,6 +628,37 @@ jobs: run: | "${HAWKEYE_INSTALL_DIR}/hawkeye" check + docs-build: + name: Sphinx docs build (-W) + runs-on: ubuntu-latest + timeout-minutes: 20 + needs: [changes] + # A manual E2E dispatch only exercises the E2E jobs; skip the rest. Off a + # dispatch, `needs.changes` never ran, so short-circuit on event_name first. + if: github.event_name != 'workflow_dispatch' && needs.changes.outputs.docs == 'true' + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + + - name: Cache pip packages + id: cache-pip + uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + with: + path: ~/.cache/pip + key: ${{ runner.os }}-docs-pip-${{ hashFiles('docs/rocm-docs/sphinx/requirements.txt') }} + + - name: Install Sphinx toolchain + run: | + python3 -m venv .docs-venv + .docs-venv/bin/pip install -q -r docs/rocm-docs/sphinx/requirements.txt + + # Same source Sphinx points at (docs/rocm-docs/conf.py) and the same + # -W (warnings-as-errors) mode read-the-docs builds with, so a broken + # MyST include, dead xref, or toctree gap fails CI instead of surfacing + # as a silent RTD build failure after merge. + - name: Build docs (warnings as errors) + run: | + .docs-venv/bin/sphinx-build -b html docs/rocm-docs /tmp/rocm-cli-docs -W + commit-signatures: name: Commit signatures + sign-off runs-on: ubuntu-latest diff --git a/.readthedocs.yaml b/.readthedocs.yaml new file mode 100644 index 0000000000..98f09d9343 --- /dev/null +++ b/.readthedocs.yaml @@ -0,0 +1,13 @@ +version: 2 + +build: + os: ubuntu-24.04 + tools: + python: "3.12" + +sphinx: + configuration: docs/rocm-docs/conf.py + +python: + install: + - requirements: docs/rocm-docs/sphinx/requirements.txt diff --git a/docs/rocm-docs/conf.py b/docs/rocm-docs/conf.py index e7cf09da33..7ea49c43f2 100644 --- a/docs/rocm-docs/conf.py +++ b/docs/rocm-docs/conf.py @@ -5,6 +5,25 @@ # Configuration file for the Sphinx documentation builder. # pylint: disable=invalid-name +import logging + + +class _SuppressUnregisteredProjectWarning(logging.Filter): + """rocm-cli isn't registered in rocm-docs-core's shared projects.yaml + catalog yet, so `external_projects_current_project` can never resolve + until that's added upstream. rocm_docs.projects treats an unresolved + current project as None everywhere it's used (doxygen_html, theme + version banner), so the warning is purely informational, not a defect + here, and would otherwise permanently block a `-W` build.""" + + def filter(self, record: logging.LogRecord) -> bool: + return "not found in projects" not in record.getMessage() + + +logging.getLogger("sphinx.rocm_docs.projects").addFilter( + _SuppressUnregisteredProjectWarning() +) + # -- Project information --------------------------------------------------- project = "ROCm CLI" @@ -18,9 +37,19 @@ html_theme = "rocm_docs_theme" html_theme_options = {"flavor": "rocm"} html_title = "ROCm CLI documentation" -suppress_warnings = ["etoc.toctree"] +# myst.header: `:start-after:` on included README sections strips the heading +# line itself, so the first heading in the chunk renders one level below its +# expected depth; docutils re-normalizes this in the rendered output, so it's +# cosmetic here. +suppress_warnings = ["etoc.toctree", "myst.header"] external_toc_path = "./sphinx/_toc.yml" external_projects_current_project = "rocm-cli" +# No page here cross-references another ROCm project via intersphinx, and the +# default ("all") tries to fetch inventories for every project in +# rocm_docs' bundled projects.yaml — several of which 404 or have moved on +# rocm.docs.amd.com, which would make a `-W` (warnings-as-errors) CI build +# flaky against failures this repo can't fix. +external_projects = [] extensions = [ "rocm_docs", "sphinx_design", From 87f346075b9458af952fb15ea92dabcc84603dbc Mon Sep 17 00:00:00 2001 From: pmoutsias-amd Date: Wed, 26 Aug 2026 10:53:47 -0400 Subject: [PATCH 05/23] docs: use docs-relative cross-page links consistently README.md's own #model-serving and #interactive-interfaces anchors were rewritten to absolute GitHub URLs so they'd still work once pulled into the docs site, but that breaks in-page anchor scrolling when the README is viewed on GitHub. Revert both to relative anchors and instead apply the cut-and-reauthor pattern already used for the CONTRIBUTING.md link in installation.md: narrow the surrounding {include} in getting-started.md and commands.md around each sentence and hand-author a docs-relative replacement (commands.md#model-serving, getting-started.md#interactive-interfaces). Signed-off-by: pmoutsia_amdeng --- README.md | 4 ++-- docs/rocm-docs/commands.md | 9 +++++++++ docs/rocm-docs/getting-started.md | 9 +++++++++ 3 files changed, 20 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 9f1c4f205e..164677e3ab 100644 --- a/README.md +++ b/README.md @@ -198,7 +198,7 @@ rocm serve qwen `qwen` is a built-in alias for a small assistant model that serves out of the box. You can also serve any compatible Hugging Face model directly — see -[Model serving](https://github.com/ROCm/rocm-cli#model-serving) for the GGUF-vs-safetensors rule, since which +[Model serving](#model-serving) for the GGUF-vs-safetensors rule, since which form works depends on the engine your GPU selects. Use `rocm model` to see available model recipes and their GPU memory @@ -401,7 +401,7 @@ rocm dash [--demo] [--replay ] Full-screen TUI with Home, ROCm, Serving, Observe, and Chat tabs — GPU utilization graphs, active serving instances, benchmark results, guided actions, and a chat tab backed by any configured provider. See -[Interactive interfaces](https://github.com/ROCm/rocm-cli#interactive-interfaces) for the tab breakdown. +[Interactive interfaces](#interactive-interfaces) for the tab breakdown. - `--demo` runs a deterministic synthetic session with no GPU or daemon needed, works on all platforms. diff --git a/docs/rocm-docs/commands.md b/docs/rocm-docs/commands.md index 96ef428f44..02e3680530 100644 --- a/docs/rocm-docs/commands.md +++ b/docs/rocm-docs/commands.md @@ -8,5 +8,14 @@ SPDX-License-Identifier: MIT ```{include} ../../README.md :start-after: "## Commands" +:end-before: "and a chat tab backed by any configured provider." +``` + +and a chat tab backed by any configured provider. See +[Interactive interfaces](getting-started.md#interactive-interfaces) for the +tab breakdown. + +```{include} ../../README.md +:start-after: "for the tab breakdown." :end-before: "## Contributing" ``` diff --git a/docs/rocm-docs/getting-started.md b/docs/rocm-docs/getting-started.md index 6a50539b7a..1f0de0be87 100644 --- a/docs/rocm-docs/getting-started.md +++ b/docs/rocm-docs/getting-started.md @@ -15,6 +15,15 @@ SPDX-License-Identifier: MIT ```{include} ../../README.md :start-after: "## Configure ROCm and serve a model" +:end-before: "You can also serve any compatible Hugging Face model directly" +``` + +You can also serve any compatible Hugging Face model directly — see +[Model serving](commands.md#model-serving) for the GGUF-vs-safetensors rule, +since which form works depends on the engine your GPU selects. + +```{include} ../../README.md +:start-after: "form works depends on the engine your GPU selects." :end-before: "## Commands" ``` From 518580a1aeba0f3bb23f037c44a7a53be06f3c6e Mon Sep 17 00:00:00 2001 From: pmoutsias-amd Date: Wed, 26 Aug 2026 12:59:50 -0400 Subject: [PATCH 06/23] docs: use descriptive nav title for Demos section Avoids the sidebar rendering as duplicated Demos > Demos for the single-entry section by overriding the nav link text to match the homepage tile's wording. Signed-off-by: pmoutsias-amd --- docs/rocm-docs/sphinx/_toc.yml.in | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/rocm-docs/sphinx/_toc.yml.in b/docs/rocm-docs/sphinx/_toc.yml.in index 740e6f8b25..7534a1eaa8 100644 --- a/docs/rocm-docs/sphinx/_toc.yml.in +++ b/docs/rocm-docs/sphinx/_toc.yml.in @@ -3,6 +3,7 @@ subtrees: - caption: Demos entries: - file: demos + title: See ROCm CLI in action - caption: Install entries: - file: install/installation From f307a60f363a1ddf542446856d690d5cbe6d73b5 Mon Sep 17 00:00:00 2001 From: pmoutsias-amd Date: Wed, 26 Aug 2026 13:13:06 -0400 Subject: [PATCH 07/23] docs: fix sentence fragments in intro blurb Joins the "single prebuilt binary" fragments into one complete sentence in both README.md and the standalone index.rst homepage copy, and adds a lead-in sentence before the bare `rocm` code fence in the Getting Started "First run" section. Signed-off-by: pmoutsias-amd --- README.md | 8 +++++--- docs/rocm-docs/index.rst | 6 +++--- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index 164677e3ab..9772e85d63 100644 --- a/README.md +++ b/README.md @@ -23,9 +23,9 @@ SPDX-License-Identifier: MIT ROCm CLI is a command-line tool for setting up and running local AI on AMD GPUs, with a full-screen TUI dashboard for GPU telemetry, model serving, and chat. -A single prebuilt binary for Linux and Windows (x86_64). No Python, Rust, or -existing ROCm install required. Ships with inference engine adapters for -Lemonade and vLLM. +It ships as a single prebuilt binary for Linux and Windows (x86_64), needs no +Python, Rust, or existing ROCm install, and includes inference engine adapters +for Lemonade and vLLM. | Platform | Prebuilt binary | Notes | @@ -131,6 +131,8 @@ commands, and commit-signing requirements. ## First run +Launch ROCm CLI with no arguments: + ``` rocm ``` diff --git a/docs/rocm-docs/index.rst b/docs/rocm-docs/index.rst index fc00cb6664..f5143d0069 100644 --- a/docs/rocm-docs/index.rst +++ b/docs/rocm-docs/index.rst @@ -10,9 +10,9 @@ ROCm CLI is a command-line tool for setting up and running local AI on AMD GPUs, with a full-screen TUI dashboard for GPU telemetry, model serving, and chat. -A single prebuilt binary for Linux and Windows (x86_64). No Python, Rust, or -existing ROCm install required. Ships with inference engine adapters for -Lemonade and vLLM. +It ships as a single prebuilt binary for Linux and Windows (x86_64), needs no +Python, Rust, or existing ROCm install, and includes inference engine +adapters for Lemonade and vLLM. .. important:: From 77ea773c240263b1a1fa609ae9b70959c3fb5644 Mon Sep 17 00:00:00 2001 From: pmoutsias-amd Date: Wed, 26 Aug 2026 14:05:26 -0400 Subject: [PATCH 08/23] docs: style guide pass on README and CONTRIBUTING Sentence-case headings, expand DCO acronym, replace banned/informal wording (may, e.g.), normalize slash usage, lowercase version placeholder. Signed-off-by: pmoutsias-amd --- CONTRIBUTING.md | 12 ++++++------ README.md | 22 +++++++++++----------- 2 files changed, 17 insertions(+), 17 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index ad8bfafa1e..a9e1c9fa86 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -12,11 +12,11 @@ Thank you for your interest in contributing. This document explains how to get s By participating in this project you agree to abide by the [Code of Conduct](https://github.com/ROCm/rocm-cli/blob/main/CODE_OF_CONDUCT.md). -## Issue Tracking +## Issue tracking Bug reports and feature requests are tracked as [GitHub Issues](https://github.com/ROCm/rocm-cli/issues). Before opening a new issue, search existing ones to avoid duplicates. -## Branch and Commit Naming +## Branch and commit naming Use [Conventional Commits](https://www.conventionalcommits.org/) for commit messages: @@ -34,7 +34,7 @@ feat/vllm-multi-gpu fix/vram-probe-fallback ``` -## Development Setup +## Development setup Prerequisites: Rust (see `rust-toolchain.toml` for the pinned version) and [uv](https://github.com/astral-sh/uv) (for prek and scripts). @@ -74,7 +74,7 @@ cargo install cargo-about@0.9.1 --locked --features cli # optional, for THIRD_ See `docs/testing.md` for the full test guide and `docs/manual-testing.md` for manual QA steps. -## Making Changes +## Making changes 1. **Fork** the repository and create a branch from `main`. 2. **Make your changes.** Keep commits focused — one logical change per commit. @@ -84,7 +84,7 @@ See `docs/testing.md` for the full test guide and `docs/manual-testing.md` for m ### Commit signing and sign-off -Commits must be both cryptographically **signed** and carry a DCO **`Signed-off-by`** trailer. Use `git commit -s` to add the trailer automatically. This is enforced by the prek hooks and by a blocking CI check. +Commits must be both cryptographically **signed** and carry a Developer Certificate of Origin (DCO) **`Signed-off-by`** trailer. Use `git commit -s` to add the trailer automatically. This is enforced by the prek hooks and by a blocking CI check. Enable SSH signing once with: @@ -102,7 +102,7 @@ See `docs/commit-signatures.md` for GPG signing, GitHub "Verified" status, and t - No secrets, credentials, or internal hostnames in committed files - Third-party dependencies declared in `Cargo.lock`; license headers present on new source files (see `licenserc.toml`) -## Reporting Security Issues +## Reporting security issues Do **not** open a public GitHub Issue for security vulnerabilities. See [SECURITY.md](https://github.com/ROCm/rocm-cli/blob/main/SECURITY.md) for the responsible disclosure process. diff --git a/README.md b/README.md index 9772e85d63..c108b131d6 100644 --- a/README.md +++ b/README.md @@ -37,12 +37,12 @@ for Lemonade and vLLM. Live dashboard telemetry requires Linux or WSL2 (see -[Interactive interfaces](#interactive-interfaces)). vLLM serving is Linux/WSL2 +[Interactive interfaces](#interactive-interfaces)). vLLM serving is Linux or WSL2 only (see [docs/vllm.md](docs/vllm.md)). > [!IMPORTANT] > **Tech Preview** -- This software is provided as-is, without warranty or -> guarantee of stability. APIs, commands, and behavior may change without +> guarantee of stability. APIs, commands, and behavior might change without > notice. Intended for experimentation and early feedback only. ## Demos @@ -73,7 +73,7 @@ The installer downloads a prebuilt bundle, verifies its SHA-256 checksum, installs the `rocm` and `rocmd` binaries into `~/.local/bin`, and adds that directory to your shell `PATH`. Rerun it any time to upgrade. -### Linux / WSL (x86_64) +### Linux and WSL (x86_64) Only nightly builds are published today, so pass the `nightly` channel: @@ -226,7 +226,7 @@ requirements. ``` rocm install sdk [--channel release|nightly] [--format wheel|tarball] - [--version X.Y.Z | --build-date YYYY-MM-DD] + [--version x.y.z | --build-date YYYY-MM-DD] [--family gfx110X-all] [--prefix PATH] [--dry-run] rocm install driver [--dkms] [--yes] [--dry-run] [--reconcile] @@ -254,7 +254,7 @@ rocm runtimes adopt --python [--root ] [--runtime-id ID] ``` `adopt` registers an existing TheRock-based Python environment as a managed -runtime. It does not work with standard ROCm package installs (e.g. +runtime. It does not work with standard ROCm package installs (for example, `/opt/rocm`); use `rocm install sdk` instead. ### Disk space @@ -342,15 +342,15 @@ to stop the server instead. `--managed` is the explicit form of the default background behavior. `--no-smoke-test` skips the post-startup inference probe. Which model form to pass depends on the engine your GPU selects. The Lemonade -engine (Ryzen AI / Radeon) serves llama.cpp **GGUF** models — pass a GGUF repo -with an explicit quantization variant, e.g. +engine (Ryzen AI or Radeon) serves llama.cpp **GGUF** models — pass a GGUF repo +with an explicit quantization variant, for example, `rocm serve unsloth/Qwen3-0.6B-GGUF:Q4_0`. The vLLM engine (Instinct) serves -**safetensors** repos, e.g. `rocm serve Qwen/Qwen2.5-1.5B-Instruct`. A +**safetensors** repos, such as `rocm serve Qwen/Qwen2.5-1.5B-Instruct`. A safetensors-only id has no GGUF build, so serving it through Lemonade fails rather than silently substituting a different model. Short -aliases from `rocm model` may not resolve with all engines. +aliases from `rocm model` might not resolve with all engines. -Some models (e.g., Llama) are gated and require HuggingFace authentication. +Some models (such as Llama) are gated and require HuggingFace authentication. Log in with `huggingface-cli login` or set `HF_TOKEN` in your environment before serving gated models. @@ -375,7 +375,7 @@ rocm model [--verbose] the hardware path they target — Strix Halo (Lemonade / llama.cpp) and MI300X (vLLM) — using canonical Hugging Face ids, and shows for each the quantization that fits a single GPU. Strix Halo entries use the `owner/repo:variant` GGUF form -(e.g. `unsloth/Qwen3.6-35B-A3B-GGUF:Q4_K_M`) that `rocm serve` needs; MI300X +(for example, `unsloth/Qwen3.6-35B-A3B-GGUF:Q4_K_M`) that `rocm serve` needs; MI300X entries serve at BF16. This catalog ships inside the binary, so it is available offline; when a recipe index is configured instead, the header names it. `--verbose` also lists the other recipes (the default assistant and smoke/test From d6a45e98815dbcc212fbba98eabae09bb4cbccfd Mon Sep 17 00:00:00 2001 From: Eugene Volen <150254791+volen-silo@users.noreply.github.com> Date: Thu, 27 Aug 2026 05:50:45 +0000 Subject: [PATCH 09/23] fix(lemonade): retry interrupted backend setup (#249) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix(lemonade): retry interrupted backend setup Signed-off-by: Eugene Volen * ci(e2e): build prebuilt lanes with the test-hooks feature The suite's deterministic failure seams are compiled out without `rocm/e2e-test-hooks`. `cargo xtask e2e` passes the feature only when it builds the binaries itself, so every lane that pre-builds and exports ROCM_CLI_BINARY silently tested a binary without those seams. The scripted-Lemonade-failure scenario then never reached its premise and failed as a regression on the one lane that selects it. Pass the feature on all prebuilt e2e lanes, and add a workflow-contract test so a lane cannot drift back. Release and packaging builds keep the feature off — a shipped binary must not carry a failure-injection seam. Also name the cause in the retry assertion, which otherwise reports a baffling missing announcement rather than the real misconfiguration. Signed-off-by: Eugene Volen --------- Signed-off-by: Eugene Volen Signed-off-by: pmoutsias-amd --- .github/workflows/e2e-selfhosted.yml | 26 +++++- .github/workflows/nightly.yml | 20 +++- apps/rocm/Cargo.toml | 3 + apps/rocm/src/main.rs | 7 +- engines/lemonade/Cargo.toml | 3 + engines/lemonade/src/lib.rs | 92 ++++++++++++++++++- .../features/model_serving.feature | 12 +++ tests/e2e-cucumber/tests/e2e.rs | 28 ++++++ tests/e2e-cucumber/tests/e2e/serving_steps.rs | 65 +++++++++++++ xtask/src/e2e.rs | 11 ++- xtask/src/workflow_contract.rs | 45 +++++++++ 11 files changed, 299 insertions(+), 13 deletions(-) diff --git a/.github/workflows/e2e-selfhosted.yml b/.github/workflows/e2e-selfhosted.yml index b531ab8234..09b6154317 100644 --- a/.github/workflows/e2e-selfhosted.yml +++ b/.github/workflows/e2e-selfhosted.yml @@ -279,7 +279,15 @@ jobs: # Build the rocm and rocmd binaries ONCE and reuse them for both the # pre-warm and suite so xtask does not rebuild. Honors # CARGO_TARGET_DIR set above. - cargo build --release -p rocm -p rocmd + # + # `--features rocm/e2e-test-hooks` must match what `cargo xtask e2e` + # builds when it builds for itself. The suite's deterministic failure + # seams (e.g. the scripted Lemonade backend-install failure) are + # compiled out without it, so a pre-built binary that omits the feature + # leaves those scenarios unable to reach their premise — they then fail + # as regressions on whichever lane happens to select them. Every lane + # that pre-builds and exports ROCM_CLI_BINARY must pass it. + cargo build --release -p rocm -p rocmd --features rocm/e2e-test-hooks export ROCM_CLI_BINARY="$CARGO_TARGET_DIR/release/rocm" export ROCM_CLI_ROCMD_BINARY="$CARGO_TARGET_DIR/release/rocmd" @@ -444,7 +452,9 @@ jobs: export E2E_SHARED_RUNTIMES_DIR="$prewarm/data/runtimes" # Build the rocm and rocmd binaries once; reuse them for pre-warm + suite. - cargo build --release -p rocm -p rocmd + # See the e2e-gpu lane for why the e2e-test-hooks feature must match + # what `cargo xtask e2e` would build. + cargo build --release -p rocm -p rocmd --features rocm/e2e-test-hooks export ROCM_CLI_BINARY="$CARGO_TARGET_DIR/release/rocm" export ROCM_CLI_ROCMD_BINARY="$CARGO_TARGET_DIR/release/rocmd" @@ -591,7 +601,9 @@ jobs: # Build the rocm and rocmd binaries once; reuse them for pre-warm + suite. # This job does not set CARGO_TARGET_DIR, so the binaries land in # the default target\release (fall back to it when the env var is unset). - cargo build --release -p rocm -p rocmd + # See the e2e-gpu lane for why the e2e-test-hooks feature must match + # what `cargo xtask e2e` would build. + cargo build --release -p rocm -p rocmd --features rocm/e2e-test-hooks if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } $targetDir = if ($env:CARGO_TARGET_DIR) { $env:CARGO_TARGET_DIR } else { "target" } $env:ROCM_CLI_BINARY = "$targetDir\release\rocm.exe" @@ -776,7 +788,9 @@ jobs: export E2E_SHARED_RUNTIMES_DIR="$prewarm/data/runtimes" # Build the rocm and rocmd binaries once; reuse them for pre-warm + suite. - cargo build --release -p rocm -p rocmd + # See the e2e-gpu lane for why the e2e-test-hooks feature must match + # what `cargo xtask e2e` would build. + cargo build --release -p rocm -p rocmd --features rocm/e2e-test-hooks export ROCM_CLI_BINARY="$CARGO_TARGET_DIR/release/rocm" export ROCM_CLI_ROCMD_BINARY="$CARGO_TARGET_DIR/release/rocmd" @@ -903,7 +917,9 @@ jobs: prewarm="$RUNNER_WORKSPACE/e2e-prewarm" export E2E_SHARED_RUNTIMES_DIR="$prewarm/data/runtimes" - cargo build --release -p rocm -p rocmd + # See the e2e-gpu lane for why the e2e-test-hooks feature must match + # what `cargo xtask e2e` would build. + cargo build --release -p rocm -p rocmd --features rocm/e2e-test-hooks export ROCM_CLI_BINARY="$CARGO_TARGET_DIR/release/rocm" export ROCM_CLI_ROCMD_BINARY="$CARGO_TARGET_DIR/release/rocmd" diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index eea00b3e7e..a7a687d7c3 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -385,7 +385,9 @@ jobs: export E2E_SHARED_RUNTIMES_DIR="$prewarm/data/runtimes" # Build both binaries once; reuse them for pre-warm + suite. - cargo build --release -p rocm -p rocmd + # See the e2e-gpu lane in e2e-selfhosted.yml for why the + # e2e-test-hooks feature must match what `cargo xtask e2e` would build. + cargo build --release -p rocm -p rocmd --features rocm/e2e-test-hooks export ROCM_CLI_BINARY="$CARGO_TARGET_DIR/release/rocm" export ROCM_CLI_ROCMD_BINARY="$CARGO_TARGET_DIR/release/rocmd" @@ -484,7 +486,9 @@ jobs: prewarm="$RUNNER_WORKSPACE/e2e-prewarm" export E2E_SHARED_RUNTIMES_DIR="$prewarm/data/runtimes" - cargo build --release -p rocm -p rocmd + # See the e2e-gpu lane in e2e-selfhosted.yml for why the + # e2e-test-hooks feature must match what `cargo xtask e2e` would build. + cargo build --release -p rocm -p rocmd --features rocm/e2e-test-hooks export ROCM_CLI_BINARY="$CARGO_TARGET_DIR/release/rocm" export ROCM_CLI_ROCMD_BINARY="$CARGO_TARGET_DIR/release/rocmd" @@ -578,7 +582,9 @@ jobs: prewarm="$RUNNER_WORKSPACE/e2e-prewarm" export E2E_SHARED_RUNTIMES_DIR="$prewarm/data/runtimes" - cargo build --release -p rocm -p rocmd + # See the e2e-gpu lane in e2e-selfhosted.yml for why the + # e2e-test-hooks feature must match what `cargo xtask e2e` would build. + cargo build --release -p rocm -p rocmd --features rocm/e2e-test-hooks export ROCM_CLI_BINARY="$CARGO_TARGET_DIR/release/rocm" export ROCM_CLI_ROCMD_BINARY="$CARGO_TARGET_DIR/release/rocmd" @@ -672,7 +678,9 @@ jobs: $prewarm = "$env:RUNNER_WORKSPACE\e2e-prewarm" $env:E2E_SHARED_RUNTIMES_DIR = "$prewarm\data\runtimes" - cargo build --release -p rocm -p rocmd + # See the e2e-gpu lane in e2e-selfhosted.yml for why the + # e2e-test-hooks feature must match what `cargo xtask e2e` would build. + cargo build --release -p rocm -p rocmd --features rocm/e2e-test-hooks if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } $targetDir = if ($env:CARGO_TARGET_DIR) { $env:CARGO_TARGET_DIR } else { "target" } $env:ROCM_CLI_BINARY = "$targetDir\release\rocm.exe" @@ -824,7 +832,9 @@ jobs: prewarm="$RUNNER_WORKSPACE/e2e-prewarm" export E2E_SHARED_RUNTIMES_DIR="$prewarm/data/runtimes" - cargo build --release -p rocm -p rocmd + # See the e2e-gpu lane in e2e-selfhosted.yml for why the + # e2e-test-hooks feature must match what `cargo xtask e2e` would build. + cargo build --release -p rocm -p rocmd --features rocm/e2e-test-hooks export ROCM_CLI_BINARY="$CARGO_TARGET_DIR/release/rocm" export ROCM_CLI_ROCMD_BINARY="$CARGO_TARGET_DIR/release/rocmd" diff --git a/apps/rocm/Cargo.toml b/apps/rocm/Cargo.toml index 7c71f68669..db8b886225 100644 --- a/apps/rocm/Cargo.toml +++ b/apps/rocm/Cargo.toml @@ -10,6 +10,9 @@ publish.workspace = true [lints] workspace = true +[features] +e2e-test-hooks = ["rocm-engine-lemonade/e2e-test-hooks"] + [dependencies] anyhow.workspace = true clap.workspace = true diff --git a/apps/rocm/src/main.rs b/apps/rocm/src/main.rs index 630ad57e53..5356167ff4 100644 --- a/apps/rocm/src/main.rs +++ b/apps/rocm/src/main.rs @@ -4853,8 +4853,13 @@ fn serve(args: ServeArgs) -> Result<()> { // BEFORE preparing or launching any engine (no wasted engine download, and an // actionable message instead of a late engine crash). The engine enforces the // same rule as a backstop. Skipped for cpu_only; permissive when availability - // cannot be probed on this platform (probe returns `None`). + // cannot be probed on this platform (probe returns `None`). The E2E-only + // backend-failure scenario bypasses this host precondition so the black-box + // test reaches Lemonade's backend boundary without real GPU hardware. + let scripted_backend_failure = cfg!(feature = "e2e-test-hooks") + && std::env::var_os("ROCM_E2E_LEMONADE_BACKEND_INSTALL_FAILURE").is_some(); if !cpu_only + && !scripted_backend_failure && let Some(usable) = rocm_core::usable_amd_gpu_indices() && usable.is_empty() { diff --git a/engines/lemonade/Cargo.toml b/engines/lemonade/Cargo.toml index d2c28fb7bb..94e37252ef 100644 --- a/engines/lemonade/Cargo.toml +++ b/engines/lemonade/Cargo.toml @@ -10,6 +10,9 @@ publish.workspace = true [lints] workspace = true +[features] +e2e-test-hooks = [] + [[bin]] name = "rocm-engine-lemonade" path = "src/main.rs" diff --git a/engines/lemonade/src/lib.rs b/engines/lemonade/src/lib.rs index 315a06c0d9..94090b4646 100644 --- a/engines/lemonade/src/lib.rs +++ b/engines/lemonade/src/lib.rs @@ -40,6 +40,8 @@ const DEFAULT_MODEL_REPO_DIR: &str = "models--unsloth--Qwen3-4B-Instruct-2507-GG const DEFAULT_MODEL_GGUF: &str = "Qwen3-4B-Instruct-2507-Q4_K_M.gguf"; const LLAMACPP_RECIPE: &str = "llamacpp"; const ROCM_BACKEND_NAME: &str = "rocm"; +#[cfg(feature = "e2e-test-hooks")] +const BACKEND_INSTALL_FAILURE_TEST_ENV: &str = "ROCM_E2E_LEMONADE_BACKEND_INSTALL_FAILURE"; /// Preferred llama.cpp backends, best first. Lemonade reports per-GPU support; /// we pick the highest-priority backend it considers supported on this host. /// GPU backends only — `cpu` is intentionally excluded so the router path never @@ -426,6 +428,14 @@ fn detect_response() -> DetectResponse { fn install_response(request: InstallRequest) -> Result { let paths = AppPaths::discover()?; paths.ensure()?; + // Debug builds expose a deterministic failure seam for the black-box CLI + // scenario that pins retry count and terminal recovery guidance. Keep it at + // the backend phase: the real defect happens after the embeddable is ready, + // and exercising it must not download or alter a runtime on the test host. + #[cfg(feature = "e2e-test-hooks")] + if std::env::var_os(BACKEND_INSTALL_FAILURE_TEST_ENV).is_some() { + install_llamacpp_backend_with_retry(|| bail!("scripted Lemonade backend install failure"))?; + } eprintln!( "Preparing Lemonade embeddable {}...", rocm_deps::LEMONADE_VERSION @@ -1136,6 +1146,25 @@ fn install_best_llamacpp_backend(manifest: &mut LemonadeInstallManifest) -> Resu /// Ask Lemonade which llama.cpp backends it supports on this GPU, choose the best /// one (`LLAMACPP_BACKEND_PRIORITY`), install it if necessary, and return its name. +/// Retry the backend install itself once, rather than the whole Lemonade runtime +/// preparation. The embeddable download already has bounded transport retries; +/// repeating that outer operation would redo deterministic failures and may +/// re-extract a healthy runtime. A backend subprocess can instead fail after a +/// completed download when its connection to lemond is interrupted, and a second +/// call can reuse the backend cache immediately without a delay. +fn install_llamacpp_backend_with_retry(mut install: impl FnMut() -> Result<()>) -> Result<()> { + match install() { + Ok(()) => Ok(()), + Err(first_error) => { + eprintln!("Lemonade backend installation failed; retrying once: {first_error:#}"); + install().with_context(|| { + "Lemonade backend installation failed again; run `rocm engines install \ + lemonade --reinstall` and then retry `rocm serve`" + }) + } + } +} + fn ensure_best_llamacpp_backend( manifest: &LemonadeInstallManifest, host: &str, @@ -1157,7 +1186,9 @@ fn ensure_best_llamacpp_backend( eprintln!("Using installed Lemonade {LLAMACPP_RECIPE}:{backend} backend."); } else { eprintln!("Installing Lemonade {LLAMACPP_RECIPE}:{backend} backend..."); - run_lemonade_backend_install(manifest, host, port, &backend, process_env)?; + install_llamacpp_backend_with_retry(|| { + run_lemonade_backend_install(manifest, host, port, &backend, process_env) + })?; } Ok(backend) } @@ -4364,6 +4395,65 @@ mod tests { dir } + #[test] + fn backend_install_succeeds_without_retry() { + let mut attempts = 0; + + install_llamacpp_backend_with_retry(|| { + attempts += 1; + Ok(()) + }) + .unwrap(); + + assert_eq!(attempts, 1); + } + + #[test] + fn backend_install_recovers_on_the_second_attempt() { + let mut attempts = 0; + + install_llamacpp_backend_with_retry(|| { + attempts += 1; + if attempts == 1 { + bail!("first backend connection was interrupted"); + } + Ok(()) + }) + .unwrap(); + + assert_eq!(attempts, 2); + } + + #[test] + fn backend_install_stops_after_one_retry_with_reinstall_guidance() { + let mut attempts = 0; + + let error = install_llamacpp_backend_with_retry(|| { + attempts += 1; + if attempts == 1 { + bail!("first backend connection was interrupted"); + } + bail!("second backend connection was interrupted"); + }) + .unwrap_err(); + let rendered = format!("{error:#}"); + + assert_eq!(attempts, 2); + assert!( + rendered.contains("second backend connection was interrupted"), + "{rendered}" + ); + assert!( + !rendered.contains("first backend connection was interrupted"), + "the terminal error must be the retry's failure: {rendered}" + ); + assert!( + rendered.contains("rocm engines install lemonade --reinstall"), + "{rendered}" + ); + assert!(rendered.contains("retry `rocm serve`"), "{rendered}"); + } + fn test_manifest(runtime_dir: PathBuf) -> LemonadeInstallManifest { LemonadeInstallManifest { env_id: "test".to_owned(), diff --git a/tests/e2e-cucumber/features/model_serving.feature b/tests/e2e-cucumber/features/model_serving.feature index 854b9badd5..219ffc137c 100644 --- a/tests/e2e-cucumber/features/model_serving.feature +++ b/tests/e2e-cucumber/features/model_serving.feature @@ -164,3 +164,15 @@ Feature: Model serving When the user serves a model selecting both a runtime and an environment Then serving is refused before any engine starts And the user is told the two selectors cannot be combined + + # The failure is injected at Lemonade's backend-install boundary in debug/test + # builds, after the CLI has selected Lemonade but before any runtime download or + # machine mutation. That makes the user-visible retry and final recovery command + # deterministic on the blocking no-GPU lane rather than relying on a real 3 GiB + # transfer to fail at just the right moment. + @id:serve-lemonade-preparation-recovery @requires-no-gpu + Scenario: 16 - Repeated Lemonade preparation failure gives the user a recovery path + Given Lemonade preparation cannot complete + When the user serves a model with Lemonade + Then serving stops after one automatic retry + And the user is told how to reinstall Lemonade and retry serving diff --git a/tests/e2e-cucumber/tests/e2e.rs b/tests/e2e-cucumber/tests/e2e.rs index 83f9eb678a..f7eddc2830 100644 --- a/tests/e2e-cucumber/tests/e2e.rs +++ b/tests/e2e-cucumber/tests/e2e.rs @@ -77,6 +77,10 @@ pub struct E2eWorld { /// a run whose failure is already the expected outcome — see the relaunch /// budget in `setup_gpu_model`. pub expect_xfail: bool, + /// Extra environment for this scenario's next `rocm` invocation. A Given step + /// records a behavioral precondition here; the When step remains a plain user + /// action and consumes the fixture without exposing its mechanism in Gherkin. + pub command_env: Vec<(&'static str, std::ffi::OsString)>, /// The interactive dash/chat TUI spawned under a pseudo-terminal for this /// scenario, if any (see `e2e::tui_driver`). Torn down in `Drop` before the /// mock server and isolated directory so the child process never outlives @@ -193,6 +197,7 @@ impl Default for E2eWorld { legacy_rocm_path: None, serve_timeout_override: None, expect_xfail: false, + command_env: Vec::new(), tui: None, chat_use_mock: false, lifecycle: None, @@ -618,6 +623,29 @@ pub fn run_rocm_with_env( ) } +/// Run `rocm` with the behavioral fixture established by a Given step, then +/// consume it so it cannot leak into a later action in the same scenario. +pub fn run_rocm_with_scenario_env(world: &mut E2eWorld, args: &[&str]) -> (String, String, i32) { + let binary = rocm_binary(); + let mut cmd = std::process::Command::new(&binary); + cmd.args(args); + world.isolate_cmd(&mut cmd); + for (key, value) in std::mem::take(&mut world.command_env) { + cmd.env(key, value); + } + let output = cmd + .output() + .unwrap_or_else(|e| panic!("failed to run {binary}: {e}")); + let rc = output.status.code().unwrap_or(-1); + let stdout = String::from_utf8_lossy(&output.stdout).to_string(); + record_command(world.current_scenario.as_deref(), args, rc, &stdout); + ( + stdout, + String::from_utf8_lossy(&output.stderr).to_string(), + rc, + ) +} + /// Append one `rocm` invocation to `results/commands.jsonl` so the consolidated /// report can build a command × platform coverage table tied to real results. /// Best-effort: a recording failure must never fail a scenario. diff --git a/tests/e2e-cucumber/tests/e2e/serving_steps.rs b/tests/e2e-cucumber/tests/e2e/serving_steps.rs index 6b0fc74b01..fdcb49e020 100644 --- a/tests/e2e-cucumber/tests/e2e/serving_steps.rs +++ b/tests/e2e-cucumber/tests/e2e/serving_steps.rs @@ -790,6 +790,31 @@ async fn user_serves_vllm_capable_default(world: &mut E2eWorld) { world.cli_rc = Some(rc); } +#[given("Lemonade preparation cannot complete")] +async fn lemonade_preparation_cannot_complete(world: &mut E2eWorld) { + world.command_env.push(( + "ROCM_E2E_LEMONADE_BACKEND_INSTALL_FAILURE", + "repeated".into(), + )); +} + +#[when("the user serves a model with Lemonade")] +async fn user_serves_with_failing_lemonade_preparation(world: &mut E2eWorld) { + let (stdout, stderr, rc) = crate::run_rocm_with_scenario_env( + world, + &[ + "serve", + "Qwen3-0.6B-GGUF", + "--engine", + "lemonade", + "--managed", + ], + ); + world.cli_output = Some(stdout); + world.cli_stderr = Some(stderr); + world.cli_rc = Some(rc); +} + #[when("the user sends a chat completion request")] async fn user_sends_completion(world: &mut E2eWorld) { crate::send_chat(world).await; @@ -903,6 +928,46 @@ async fn when_cli_reports_ready(world: &mut E2eWorld) { // ── Then ─────────────────────────────────────────────────────────── +#[then("serving stops after one automatic retry")] +async fn assert_lemonade_preparation_retry_is_bounded(world: &mut E2eWorld) { + let output = serve_output(world); + assert_ne!( + world.cli_rc, + Some(0), + "serve unexpectedly succeeded:\n{output}" + ); + // The scripted failure seam also waives serve's no-GPU pre-flight, so this + // refusal can only appear when the seam is compiled out. Name that cause: + // otherwise a lane that pre-builds `rocm` without the feature reports a + // baffling "no retry announcement" instead of its real misconfiguration. + assert!( + !output.contains("no usable AMD GPU detected"), + "serve stopped at the no-GPU pre-flight, so the binary under test was \ + built without the `rocm/e2e-test-hooks` feature and never reached \ + Lemonade preparation. E2E lanes that pre-build `rocm` and export \ + ROCM_CLI_BINARY must pass `--features rocm/e2e-test-hooks`, matching \ + what `cargo xtask e2e` builds for itself:\n{output}" + ); + assert_eq!( + output.matches("retrying once").count(), + 1, + "expected exactly one retry announcement:\n{output}" + ); +} + +#[then("the user is told how to reinstall Lemonade and retry serving")] +async fn assert_lemonade_recovery_guidance(world: &mut E2eWorld) { + let output = serve_output(world); + assert!( + output.contains("rocm engines install lemonade --reinstall"), + "expected a forced-reinstall recovery command:\n{output}" + ); + assert!( + output.contains("retry `rocm serve`"), + "expected guidance to retry serving:\n{output}" + ); +} + #[then("an inference request succeeds immediately")] async fn assert_inference_succeeds_now(world: &mut E2eWorld) { // No extra wait: the CLI already reported ready, so inference must work now. diff --git a/xtask/src/e2e.rs b/xtask/src/e2e.rs index 87d242a01c..05cf29bbb4 100644 --- a/xtask/src/e2e.rs +++ b/xtask/src/e2e.rs @@ -85,7 +85,16 @@ pub fn run(args: &[String]) -> Result<()> { if binaries.build_release { let status = Command::new(&cargo) - .args(["build", "--release", "-p", "rocm", "-p", "rocmd"]) + .args([ + "build", + "--release", + "-p", + "rocm", + "-p", + "rocmd", + "--features", + "rocm/e2e-test-hooks", + ]) .current_dir(&root) .status() .context("failed to run `cargo build --release -p rocm -p rocmd`")?; diff --git a/xtask/src/workflow_contract.rs b/xtask/src/workflow_contract.rs index 6a5016e77f..c551422aee 100644 --- a/xtask/src/workflow_contract.rs +++ b/xtask/src/workflow_contract.rs @@ -211,6 +211,39 @@ mod tests { ); } + /// Every lane that pre-builds `rocm` and hands it to the suite via + /// `ROCM_CLI_BINARY` must enable the same test-hook feature `cargo xtask e2e` + /// enables when it builds for itself. + /// + /// The suite's deterministic failure seams (e.g. the scripted Lemonade + /// backend-install failure) are `#[cfg(feature = "e2e-test-hooks")]`. A lane + /// that omits the feature ships a binary in which those seams do not exist, + /// so the scenarios relying on them cannot reach their premise and fail as + /// regressions — but only on whichever lane happens to select them, which is + /// what made this divergence so hard to read the first time. Pin it here so a + /// new lane copying an existing block cannot silently reintroduce it. + fn assert_prebuilt_e2e_lanes_enable_test_hooks(workflow: &str, text: &str) { + let blocks: Vec<_> = multiline_run_blocks(text) + .into_iter() + .filter(|block| block.contains("ROCM_CLI_BINARY") && invokes_e2e(block)) + .collect(); + assert!( + !blocks.is_empty(), + "{workflow} must contain prebuilt E2E run blocks" + ); + for block in blocks { + assert!( + block.contains( + "cargo build --release -p rocm -p rocmd --features rocm/e2e-test-hooks" + ), + "{workflow} prebuilt E2E lane must build with \ + `--features rocm/e2e-test-hooks`, matching what `cargo xtask e2e` \ + builds for itself; without it the suite's scripted failure seams \ + are compiled out:\n{block}" + ); + } + } + /// Extract one top-level job's complete YAML block by its job id. fn job_block<'a>(text: &'a str, job: &str) -> &'a str { let marker = format!(" {job}:\n"); @@ -821,6 +854,18 @@ trigger-a-workflow#triggering-a-workflow-from-a-workflow" assert_prebuilt_e2e_lanes_export_rocmd("nightly.yml", &workflow); } + #[test] + fn self_hosted_prebuilt_e2e_lanes_enable_test_hooks() { + let workflow = read_workflow("e2e-selfhosted.yml"); + assert_prebuilt_e2e_lanes_enable_test_hooks("e2e-selfhosted.yml", &workflow); + } + + #[test] + fn nightly_prebuilt_e2e_lanes_enable_test_hooks() { + let workflow = read_workflow("nightly.yml"); + assert_prebuilt_e2e_lanes_enable_test_hooks("nightly.yml", &workflow); + } + // Extractor guards: prove the helpers actually parse multiline forms, so the // contract tests above can't silently false-pass on a shape they don't handle. #[test] From 7cd8902be524a8a95fa217eed761d9fab10c4c72 Mon Sep 17 00:00:00 2001 From: Roman Inflianskas Date: Thu, 27 Aug 2026 07:55:14 +0000 Subject: [PATCH 10/23] ci(dependabot): track pre-commit hook versions (#235) The hook pins in .pre-commit-config.yaml were bumped by hand and drifted between releases. Dependabot's pre-commit ecosystem resolves each rev against the hook repository's tags, and skips the builtin and local blocks, which have no upstream release to track. Weekly and grouped, with the same 7-day cooldown as the other two ecosystems: a hook is executable code that runs on every contributor's machine at commit time and in the prek CI job, so a compromised release would run before anyone reads the bump. Signed-off-by: Roman Inflianskas Signed-off-by: pmoutsias-amd --- .github/dependabot.yml | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 7e7bec2b72..0fa35ec263 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -72,3 +72,24 @@ updates: update-types: - minor - patch + + # The hook pins in `.pre-commit-config.yaml` were bumped by hand, so they + # drifted. Dependabot resolves each `rev` against the hook repository's tags + # and leaves the `builtin` and `local` blocks alone -- prek's native hooks and + # the `cargo xtask` hooks have no upstream release to track. Grouped into one + # PR: this is a short list of linters whose individual bumps carry no review + # signal worth a PR each. + - package-ecosystem: pre-commit + directory: / + schedule: + interval: weekly + # Same reasoning as the two blocks above, and it lands harder here: a hook is + # executable code that runs on every contributor's machine at commit time and + # in the `prek` CI job, so a compromised release executes before anyone has + # read the bump. Cooldown delays only version updates, never security updates. + cooldown: + default-days: 7 + groups: + pre-commit: + patterns: + - "*" From d1253c5668457954e30e5ea33b3d439de9574abf Mon Sep 17 00:00:00 2001 From: Roman Inflianskas Date: Thu, 27 Aug 2026 07:55:39 +0000 Subject: [PATCH 11/23] fix: raise the supported Linux minimum to Ubuntu 24.04 (#261) * fix: raise the supported Linux minimum to Ubuntu 24.04 Every published Lemonade embeddable, v10.2.0 through v11.5.2, is linked against GLIBC_2.38 and GLIBCXX_3.4.32. Ubuntu 22.04 ships glibc 2.35, so the Lemonade engine cannot start there at all, yet README.md advertised Linux x86_64 as "full support ... both inference engines" and docs/wsl.md named 22.04 as a supported WSL base. Document a single minimum of Ubuntu 24.04 for Linux and WSL2, keeping the glibc number as the reason so the requirement stays meaningful on other distributions, and drop 22.04 from the WSL preflight's supported set so the automated check matches the prerequisite it enforces. Fixes #258 Signed-off-by: Roman Inflianskas * fix(wsl): accept Ubuntu releases after 24.04 Signed-off-by: Michael Roy --------- Signed-off-by: Roman Inflianskas Signed-off-by: Michael Roy Co-authored-by: Michael Roy Signed-off-by: pmoutsias-amd --- README.md | 11 ++++++++-- docs/wsl.md | 5 ++++- scripts/wsl_preflight.py | 45 ++++++++++++++++++++++++++++++++++------ 3 files changed, 52 insertions(+), 9 deletions(-) diff --git a/README.md b/README.md index bae9b4c09b..834ac314b8 100644 --- a/README.md +++ b/README.md @@ -29,15 +29,22 @@ Lemonade and vLLM. | Platform | Prebuilt binary | Notes | |---|---|---| -| Linux (x86_64) | Yes | Full support, including the live dashboard and both inference engines | +| Linux (x86_64) | Yes | Ubuntu 24.04 or newer; full support, including the live dashboard and both inference engines | | Windows (x86_64) | Yes | CLI and Lemonade serving; no live dashboard or vLLM | -| WSL2 (x86_64) | Yes (Linux binary) | Full support, including the live dashboard; see [docs/wsl.md](docs/wsl.md) for setup | +| WSL2 (x86_64) | Yes (Linux binary) | Ubuntu 24.04 or newer; full support, including the live dashboard; see [docs/wsl.md](docs/wsl.md) for setup | | macOS | No | No official installer, release, CI, or QA coverage | Live dashboard telemetry requires Linux or WSL2 (see [Interactive interfaces](#interactive-interfaces)). vLLM serving is Linux/WSL2 only (see [docs/vllm.md](docs/vllm.md)). +The minimum supported Linux release, native or under WSL2, is Ubuntu 24.04. On +other distributions the equivalent requirement is glibc 2.38 with +`GLIBCXX_3.4.32`: that is what the Lemonade engine is linked against, and every +published build of it needs those versions, so there is no older release to fall +back to. Ubuntu 22.04 ships glibc 2.35 and cannot run it; Ubuntu 24.04 provides +glibc 2.39 and `GLIBCXX_3.4.33`. + > [!IMPORTANT] > **Tech Preview** -- This software is provided as-is, without warranty or > guarantee of stability. APIs, commands, and behavior may change without diff --git a/docs/wsl.md b/docs/wsl.md index 1b9a90a169..24a5367966 100644 --- a/docs/wsl.md +++ b/docs/wsl.md @@ -14,7 +14,10 @@ virtual environments and ROCDXG (`librocdxg`). The AMD WSL path is: 1. Windows 11 with the AMD Adrenalin WSL-capable driver. -2. WSL2 with Ubuntu 24.04 or Ubuntu 22.04. +2. WSL2 with Ubuntu 24.04 or newer. Ubuntu 22.04 is not supported: it ships + glibc 2.35, below the glibc 2.38 / `GLIBCXX_3.4.32` floor that every + published Lemonade embeddable requires, so the Lemonade engine cannot start + there. Ubuntu 24.04 provides glibc 2.39 and `GLIBCXX_3.4.33`. 3. ROCDXG (`librocdxg`) installed inside WSL. 4. A TheRock runtime installed by `rocm-cli` into a managed Python venv. diff --git a/scripts/wsl_preflight.py b/scripts/wsl_preflight.py index 436b0db2be..042aab4e15 100644 --- a/scripts/wsl_preflight.py +++ b/scripts/wsl_preflight.py @@ -258,11 +258,20 @@ def evaluate( sdk_headers = state.get("windows_sdk_headers") or [] version_id = str(os_release.get("VERSION_ID") or "") - supported_ubuntu = os_release.get("ID") == "ubuntu" and version_id in { - "22.04", - "24.04", - "26.04", - } + # 22.04 is deliberately unsupported: its glibc 2.35 is below the glibc 2.38 / + # GLIBCXX_3.4.32 floor every published Lemonade embeddable is linked + # against, so the Lemonade engine cannot start on it. See docs/wsl.md. + version_parts = version_id.split(".") + ubuntu_version = None + if len(version_parts) == 2 and all( + part.isascii() and part.isdecimal() for part in version_parts + ): + ubuntu_version = (int(version_parts[0]), int(version_parts[1])) + supported_ubuntu = ( + os_release.get("ID") == "ubuntu" + and ubuntu_version is not None + and ubuntu_version >= (24, 4) + ) build_tools = all( tools.get(name) for name in ["git", "cmake", "gcc", "g++", "make"] ) @@ -270,7 +279,9 @@ def evaluate( checks = [ Check("wsl", bool(state.get("is_wsl")), "WSL marker or /dev/dxg detected"), Check( - "ubuntu", supported_ubuntu, f"Ubuntu VERSION_ID={version_id or ''}" + "ubuntu", + supported_ubuntu, + f"Ubuntu 24.04 or newer (VERSION_ID={version_id or ''})", ), Check("dxg_device", bool(paths.get("/dev/dxg")), "/dev/dxg"), Check( @@ -391,6 +402,28 @@ def self_test() -> None: checks = evaluate(ready_state, require_rocm_tools=False) assert all(check.ok for check in checks if check.required), checks + for version_id in ["24.04", "24.10", "25.04", "26.04", "28.04"]: + supported_state = json.loads(json.dumps(ready_state)) + supported_state["os_release"]["VERSION_ID"] = version_id + checks = evaluate(supported_state, require_rocm_tools=False) + assert any(check.name == "ubuntu" and check.ok for check in checks), checks + assert all(check.ok for check in checks if check.required), checks + + # 22.04 is below the Lemonade glibc floor, and malformed or unknown + # versions must fail closed rather than report an otherwise perfect host ready. + for version_id in ["22.04", "24.04.1", "unknown", ""]: + unsupported_state = json.loads(json.dumps(ready_state)) + unsupported_state["os_release"]["VERSION_ID"] = version_id + checks = evaluate(unsupported_state, require_rocm_tools=False) + assert any(check.name == "ubuntu" and not check.ok for check in checks), checks + assert not all(check.ok for check in checks if check.required), checks + + missing_version_state = json.loads(json.dumps(ready_state)) + del missing_version_state["os_release"]["VERSION_ID"] + checks = evaluate(missing_version_state, require_rocm_tools=False) + assert any(check.name == "ubuntu" and not check.ok for check in checks), checks + assert not all(check.ok for check in checks if check.required), checks + def main() -> int: parser = argparse.ArgumentParser(description=__doc__) From e641dd269140cf2053bad088e8c33522a1041bf3 Mon Sep 17 00:00:00 2001 From: Eugene Volen <150254791+volen-silo@users.noreply.github.com> Date: Thu, 27 Aug 2026 09:24:33 +0000 Subject: [PATCH 12/23] ci(e2e): repair a pre-warm tree that records a folder elsewhere (#316) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A scenario reaches the shared pre-warm tree through a symlink at its own `data/runtimes`. An `install sdk` run that way writes the link's path — a per-scenario temp dir — into the manifest that lands in the SHARED registry, and into the venv's console-script shebangs. The temp dir goes away with the scenario and the shared runtime is left naming a folder that no longer exists, so later, unrelated runs fail. Drop such a runtime before deciding whether the tree is fresh, so the pre-warm reinstalls instead of serving a dead one. `rocm update` cannot see this: it compares versions against the index, not the runtime against the disk. Removing the folder is the repair, not a precaution. A poisoned venv keeps a working `bin/python` — a symlink to the base interpreter, still present — so the install reuses it and audits already-satisfied packages rather than reinstalling them, leaving every shebang pointing at the folder that went away. Measured on uv 0.9.30: reinstalling over a poisoned venv reports success and repairs nothing. The signal is an install root outside the tree, deliberately not `status=unusable`. Unusable has many causes — a missing rocm_sdk probe block alone reports it — and this deletes what it selects, so a healthy multi-GiB runtime must not hang on a validation detail. Read-only runtimes are exempt: `runtimes adopt` records an external folder on purpose. Refs: #315 Signed-off-by: Eugene Volen Signed-off-by: pmoutsias-amd --- xtask/src/e2e_prewarm.rs | 359 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 359 insertions(+) diff --git a/xtask/src/e2e_prewarm.rs b/xtask/src/e2e_prewarm.rs index acc2bffae4..a8d9f3eeff 100644 --- a/xtask/src/e2e_prewarm.rs +++ b/xtask/src/e2e_prewarm.rs @@ -143,6 +143,132 @@ pub fn decide(update_report: &str, channel: &str) -> Decision { } } +/// A managed runtime in the shared tree that records an install root somewhere +/// else, so serving against it cannot work. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct PoisonedRuntime { + pub runtime_key: String, + pub format: String, + pub install_root: String, +} + +/// Managed runtimes in the pre-warm tree whose recorded install root is not in +/// that tree. +/// +/// Pure so every branch is unit-testable without a GPU, a network, or an install +/// — same contract as [`decide`]. +/// +/// A scenario reaches the shared tree through a symlink at its own +/// `data/runtimes` (`E2eWorld::use_shared_runtimes`). `install sdk` run that way +/// writes the *link's* path — a per-scenario temp dir — into the manifest that +/// lands in the SHARED registry, and into the venv's console-script shebangs. Once +/// the scenario's temp dir is gone the shared runtime records a path that no +/// longer exists, and every later run that resolves it fails. +/// +/// The signal is deliberately "install root is outside the tree" and NOT +/// `status=unusable`. Unusable has many causes — a missing `rocm_sdk` probe block +/// alone reports it — and this function's caller DELETES what it returns. Keying +/// a multi-GiB delete off a status that a future tightening of +/// `validate_runtime_manifest_for_activation` could start emitting for healthy +/// runtimes is not a trade worth making. An out-of-tree install root, for a +/// runtime this tree is supposed to own, has exactly one cause. +/// +/// `mode=read-only` is exempt: `runtimes import` / `runtimes adopt` record an +/// external folder on purpose, and that is what read-only means. +#[must_use] +pub fn assess(runtimes_list_report: &str, runtimes_dir: &Path) -> Vec { + // The recorded root of a poisoned runtime does not exist, so it cannot be + // canonicalized. Compare textually instead, against both the path as given and + // its resolved form, so a caller that passes a path through a symlinked parent + // still matches the roots the CLI wrote. + let mut roots = vec![runtimes_dir.to_path_buf()]; + if let Ok(resolved) = runtimes_dir.canonicalize() + && resolved != *runtimes_dir + { + roots.push(resolved); + } + + let lines: Vec<&str> = runtimes_list_report.lines().collect(); + let mut poisoned = Vec::new(); + for (index, line) in lines.iter().enumerate() { + let Some(entry) = RuntimeEntry::parse(line) else { + continue; + }; + if entry.read_only { + continue; + } + // `install_root:` is written immediately under its entry by + // `render_runtimes_text`. Without it there is nothing to judge, and + // guessing a path we are about to delete is not acceptable. + let Some(install_root) = lines + .get(index + 1) + .and_then(|next| next.trim().strip_prefix("install_root: ")) + .map(str::trim) + .filter(|value| !value.is_empty()) + else { + continue; + }; + if roots + .iter() + .any(|root| Path::new(install_root).starts_with(root)) + { + continue; + } + poisoned.push(PoisonedRuntime { + runtime_key: entry.runtime_key, + format: entry.format, + install_root: install_root.to_owned(), + }); + } + poisoned +} + +/// One ` {marker} runtime_id=… format=… mode=… status=…` line from +/// `rocm runtimes list`. +/// +/// `status=` is last and its value carries spaces and parentheses +/// (`unusable (install root is missing: /x)`), so it is not read here — every +/// field this needs appears before it and is space-free. +struct RuntimeEntry { + runtime_key: String, + format: String, + read_only: bool, +} + +impl RuntimeEntry { + fn parse(line: &str) -> Option { + let mut fields = line.split_whitespace(); + let first = fields.next()?; + // The active/rollback markers are separate tokens; anything else is the key. + let runtime_key = if matches!(first, "*" | "-") { + fields.next()? + } else { + first + }; + let mut format = None; + let mut mode = None; + let mut is_entry = false; + for field in fields { + match field.split_once('=') { + Some(("runtime_id", _)) => is_entry = true, + Some(("format", value)) => format = Some(value.to_owned()), + Some(("mode", value)) => mode = Some(value.to_owned()), + _ => {} + } + } + // `runtime_id=` and `mode=` together separate a real entry from the + // header lines, which use `active_runtime_id:` / `registry:` shapes. + if !is_entry { + return None; + } + Some(Self { + runtime_key: runtime_key.to_owned(), + format: format?, + read_only: mode.as_deref() == Some("read-only"), + }) + } +} + /// One ` runtime format=… channel=… … status=…` line from `rocm update`. /// /// Both shapes that renderer emits are handled: the full report line, and the @@ -188,6 +314,12 @@ pub fn run(channel: &str, keep: usize, prewarm_dir: &Path) -> Result<()> { .with_context(|| format!("failed to create pre-warm directory {}", dir.display()))?; } + // Before asking whether the tree is FRESH, make sure what is in it is + // actually usable from this tree. `rocm update` compares versions against the + // index; it cannot see that a runtime records a folder somewhere else. + // Repairing first means `decide` reads a registry with nothing dead in it. + repair_poisoned_runtimes(&rocm, prewarm_dir)?; + let decision = match probe(&rocm, prewarm_dir) { Ok(report) => decide(&report, channel), Err(error) => { @@ -262,6 +394,94 @@ pub fn run(channel: &str, keep: usize, prewarm_dir: &Path) -> Result<()> { Ok(()) } +/// Drop any managed runtime in the shared tree that records an install root +/// outside it, so the pre-warm reinstalls instead of serving a dead one. +/// +/// Deleting the folder is not belt-and-braces, it is the repair. A poisoned venv +/// keeps a working `bin/python` — that is a symlink to the base interpreter, which +/// is still there — so `ensure_uv_venv` REUSES it, and an already-satisfied +/// package is audited rather than reinstalled, leaving every console-script +/// shebang still pointing at the folder that went away. Measured with uv 0.9.30: +/// re-running the install over a poisoned venv reports success and repairs +/// nothing; only removing the folder first does. +/// +/// Failure to repair is fatal, unlike the rest of this module. Everywhere else a +/// conservative fallback keeps the lane green; here the alternative is serving +/// against a runtime already known to be broken, which fails later, elsewhere, and +/// for reasons that name none of this. +fn repair_poisoned_runtimes(rocm: &Path, prewarm_dir: &Path) -> Result<()> { + let runtimes_dir = prewarm_dir.join("data").join("runtimes"); + if !runtimes_dir.is_dir() { + return Ok(()); + } + + let listing = match list_runtimes(rocm, prewarm_dir) { + Ok(listing) => listing, + Err(error) => { + // Same floor as everywhere else in this module: an unreadable report + // must not delete anything, and must not redden the lane. + println!( + "pre-warm: could not list runtimes ({error:#}); leaving the shared tree untouched" + ); + return Ok(()); + } + }; + + for runtime in assess(&listing, &runtimes_dir) { + println!( + "::warning::pre-warm: removing the shared runtime {} — it records an install root \ + outside this tree ({}), which a scenario that installed through its own \ + `data/runtimes` symlink would have written. A reinstall follows. See rocm-cli#315.", + runtime.runtime_key, runtime.install_root + ); + + // Drops the registry entry, the active marker, and the config pointers. + // Tolerates the recorded folder being absent, which it is. + rocm_command(rocm, prewarm_dir) + .args(["runtimes", "uninstall", &runtime.runtime_key]) + .status_ok("rocm runtimes uninstall")?; + + // The physical tree the CLI could not reach: it removed what the manifest + // POINTED at, and the files are where the manifest should have said. + let planted = runtimes_dir + .join(&runtime.format) + .join(&runtime.runtime_key); + if !planted.starts_with(&runtimes_dir) { + bail!( + "refusing to remove {}: outside the pre-warm tree at {}", + planted.display(), + runtimes_dir.display() + ); + } + if planted.is_dir() { + std::fs::remove_dir_all(&planted).with_context(|| { + format!( + "failed to remove the poisoned runtime folder {}", + planted.display() + ) + })?; + println!("pre-warm: removed {}", planted.display()); + } + } + Ok(()) +} + +/// Ask the CLI what it has registered. Read-only. +fn list_runtimes(rocm: &Path, prewarm_dir: &Path) -> Result { + let output = rocm_command(rocm, prewarm_dir) + .args(["runtimes", "list"]) + .output() + .context("failed to run `rocm runtimes list`")?; + if !output.status.success() { + bail!( + "`rocm runtimes list` exited with {}: {}", + output.status, + String::from_utf8_lossy(&output.stderr).trim() + ); + } + Ok(String::from_utf8_lossy(&output.stdout).into_owned()) +} + /// Ask the CLI whether the installed runtimes are current. Check-only: plain /// `rocm update` without `--apply` never mutates state. fn probe(rocm: &Path, prewarm_dir: &Path) -> Result { @@ -372,6 +592,145 @@ source: index\n" ) } + /// A real `rocm runtimes list` on a tree holding all three shapes that matter, + /// captured verbatim from the built binary rather than written by hand: one + /// poisoned managed runtime (an install root under a per-scenario temp dir that + /// is gone), one healthy managed runtime inside the tree, and one read-only + /// runtime adopted from outside it. + /// + /// Note that ALL THREE report `status=unusable` here. That is the whole reason + /// [`assess`] keys off the install root instead: the healthy one is unusable + /// only for want of a `rocm_sdk` probe block, and treating that as poison would + /// delete a multi-GiB runtime that a reinstall would have kept. + const MIXED: &str = "\ +registered ROCm runtimes + active_runtime_id: + active_runtime_key: + previous_runtime_key: + registry: /w/e2e-prewarm/data/runtimes/registry + marker: /w/e2e-prewarm/data/runtimes/active.json + installed: + adopted-external-env runtime_id=external-adopted version=7.14.0 format=wheel family=gfx94X-dcgpu mode=read-only status=unusable (pip runtime manifest is missing rocm_sdk probe data) + install_root: /opt/external-rocm + release-wheel-gfx94x-dcgpu-7-15-0 runtime_id=therock-release-gfx94x-dcgpu version=7.15.0 format=wheel family=gfx94X-dcgpu mode=managed status=unusable (pip runtime manifest is missing rocm_sdk probe data) + install_root: /w/e2e-prewarm/data/runtimes/wheel/release-wheel-gfx94x-dcgpu-7-15-0 + release-wheel-gfx94x-dcgpu-7-13-0 runtime_id=therock-release-gfx94x-dcgpu version=7.13.0 format=wheel family=gfx94X-dcgpu mode=managed status=unusable (install root is missing: /tmp/rocm-e2e-7MidR2/data/runtimes/wheel/release-wheel-gfx94x-dcgpu-7-13-0) + install_root: /tmp/rocm-e2e-7MidR2/data/runtimes/wheel/release-wheel-gfx94x-dcgpu-7-13-0 +"; + + /// A real `rocm runtimes list` on an empty tree, captured from the binary. + const NO_RUNTIMES: &str = "\ +registered ROCm runtimes + active_runtime_id: + active_runtime_key: + previous_runtime_key: + registry: /w/e2e-prewarm/data/runtimes/registry + marker: /w/e2e-prewarm/data/runtimes/active.json + installed: none + next step: rocm install sdk --channel release --format wheel +"; + + fn prewarm_runtimes_dir() -> &'static Path { + Path::new("/w/e2e-prewarm/data/runtimes") + } + + #[test] + fn a_runtime_recording_a_scenario_temp_dir_is_poisoned() { + assert_eq!( + assess(MIXED, prewarm_runtimes_dir()), + vec![PoisonedRuntime { + runtime_key: "release-wheel-gfx94x-dcgpu-7-13-0".to_owned(), + format: "wheel".to_owned(), + install_root: + "/tmp/rocm-e2e-7MidR2/data/runtimes/wheel/release-wheel-gfx94x-dcgpu-7-13-0" + .to_owned(), + }] + ); + } + + #[test] + fn an_unusable_runtime_inside_the_tree_is_left_alone() { + // The healthy-but-unusable entry in MIXED. Deleting it would throw away a + // multi-GiB install over a validation detail a reinstall would have fixed. + let poisoned = assess(MIXED, prewarm_runtimes_dir()); + assert!( + !poisoned + .iter() + .any(|runtime| runtime.runtime_key.ends_with("7-15-0")), + "an in-tree runtime must never be removed for being unusable: {poisoned:?}" + ); + } + + #[test] + fn a_read_only_runtime_adopted_from_outside_is_left_alone() { + // `runtimes adopt` records an external folder on purpose. Removing it would + // delete a runtime the tree never owned. + let poisoned = assess(MIXED, prewarm_runtimes_dir()); + assert!( + !poisoned + .iter() + .any(|runtime| runtime.runtime_key == "adopted-external-env"), + "a read-only adopted runtime must be exempt: {poisoned:?}" + ); + } + + #[test] + fn an_empty_tree_has_nothing_to_repair() { + assert!(assess(NO_RUNTIMES, prewarm_runtimes_dir()).is_empty()); + } + + #[test] + fn a_report_that_cannot_be_read_removes_nothing() { + // The conservative floor: never delete on a shape this does not recognise. + assert!(assess("", prewarm_runtimes_dir()).is_empty()); + assert!(assess("totally unexpected output\n", prewarm_runtimes_dir()).is_empty()); + } + + #[test] + fn an_entry_without_its_install_root_line_removes_nothing() { + // Guessing the folder to delete from the key alone is not acceptable. + let text = " installed:\n k runtime_id=r version=1 format=wheel family=f \ +mode=managed status=ready\n"; + assert!(assess(text, prewarm_runtimes_dir()).is_empty()); + } + + #[test] + fn the_active_and_rollback_markers_do_not_hide_an_entry() { + // `render_runtimes_text` prefixes the active runtime with `*` and the + // rollback target with `-`; a poisoned runtime is usually the active one. + for marker in ["*", "-"] { + let text = format!( + " installed:\n {marker} k runtime_id=r version=1 format=wheel family=f \ +mode=managed status=ready\n install_root: /tmp/rocm-e2e-XXXX/data/runtimes/wheel/k\n" + ); + let poisoned = assess(&text, prewarm_runtimes_dir()); + assert_eq!(poisoned.len(), 1, "marker {marker} hid the entry"); + assert_eq!(poisoned[0].runtime_key, "k"); + } + } + + #[test] + fn header_lines_are_not_read_as_entries() { + // `active_runtime_id:` is one character away from the `runtime_id=` field + // that identifies a real entry. + assert!(RuntimeEntry::parse(" active_runtime_id: ").is_none()); + assert!(RuntimeEntry::parse(" registry: /w/e2e-prewarm/data/runtimes/registry").is_none()); + assert!(RuntimeEntry::parse(" installed: none").is_none()); + assert!(RuntimeEntry::parse(" install_root: /tmp/x").is_none()); + } + + #[test] + fn a_tarball_runtime_reports_its_own_format() { + // The folder to remove is `//`, so the format has to + // survive parsing — removing the wheel path for a tarball runtime would + // silently repair nothing. + let text = " installed:\n k runtime_id=r version=1 format=tarball family=f \ +mode=managed status=ready\n install_root: /tmp/rocm-e2e-XXXX/data/runtimes/tarball/k\n"; + let poisoned = assess(text, prewarm_runtimes_dir()); + assert_eq!(poisoned.len(), 1); + assert_eq!(poisoned[0].format, "tarball"); + } + #[test] fn no_managed_runtime_installs() { assert_eq!(decide(EMPTY, "release"), Decision::Install); From 98bd916fa084f7b46bc876a424aa3b526bacec33 Mon Sep 17 00:00:00 2001 From: Eugene Volen <150254791+volen-silo@users.noreply.github.com> Date: Thu, 27 Aug 2026 10:19:56 +0000 Subject: [PATCH 13/23] fix(install): record the folder the files land in, not the link (#317) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix(install): record the folder the files land in, not the link `install sdk` built the install root by joining onto the data dir and recorded whatever that produced. Reaching `data/runtimes` through a symlink is enough to make that name a route rather than a place: correct while the link exists, and wrong the moment it goes. The path outlives the command. It goes into the registry manifest, the sidecar beside the runtime, and — through uv — the `#!` line of every console script in the environment. So a runtime installed that way keeps reporting itself installed at a folder that is not there, while the files sit untouched next door and every entry point fails to start. Resolve the root before anything is written to it. Only the root: `python_executable` is derived from it and must keep the venv's own `bin/python`, which is itself a symlink to the base interpreter, so resolving that would record the system Python instead. The adopt path already draws the line in the same place, canonicalizing the install root next to `absolute_existing_file_path_preserving_symlink` for the interpreter; this gives the install path the same treatment, `--prefix` included. The E2E harness creates exactly this shape when a scenario opts into the shared pre-warmed runtime, so the shared tree on a runner was what got poisoned — but nothing about the fault is test-only. Any data dir reached through a link has it. Refs: #315 Signed-off-by: Eugene Volen * fix(vllm): name the missing interpreter behind a failed launch Two messages described a stale runtime as an absence. A spawn that fails on a console script whose `#!` interpreter is gone reports ENOENT against the script, so the error named a file that is plainly there. Say which interpreter is missing when that is what happened, and stay quiet when the ordinary reading is right — a genuinely absent file, a binary, or a live interpreter. Resolution drops a registered runtime whose recorded interpreter is not there. That is correct — a runtime that cannot run is not a candidate — but it left the failure describing an empty registry while `rocm runtimes list` prints the entry. Name the manifests that were passed over and the interpreters they record. Both are additive: the note only ever decorates an error already being returned, and a registry it cannot read yields no note rather than replacing the original failure. First tests for this area, which had none. Refs: #315 Signed-off-by: Eugene Volen * test(install): compare resolved paths, not verbatim ones, on Windows The Windows lane failed on the prefix rather than the behaviour. `canonicalize` hands back `\\?\C:\…` there, so a test that built its expectation that way compared a verbatim path against the plain one the resolver deliberately returns, and three assertions turned on the difference. Build the expectations with the resolver instead, which strips the prefix for the same reason the product does: a stored path is later compared against ordinary ones, and `\\?\C:\…` never `starts_with`-matches `C:\…`. That the prefix really is stripped stays asserted on its own, so this does not become the function agreeing with itself. This also removes a second Windows-only trap: `temp_dir()` there can return an 8.3 short path, which `canonicalize` expands, so the two sides disagreed on the folder name as well as the prefix. Same fault in the scenario's step definitions, where the comparison would have failed only on the Strix Windows lane. That crate cannot reach `rocm-core`, and a dependency for six lines of string handling is the worse trade, so it strips the prefix locally with the reason recorded. Refs: #315 Signed-off-by: Eugene Volen * fix(install): give up on a parent component consistently across hosts The guard against resolving a `..` past a missing directory never fired on Windows, because the two platforms disagree about what such a path means. Windows collapses `..` lexically, so `/missing/..` "exists" and canonicalizes to ``; Unix walks the path through the filesystem and finds nothing. The walk therefore stopped a level early on Windows, resolved to the wrong folder, and re-attached the tail — recording a different folder depending on the host, which is precisely what this function exists to prevent. Check for the parent component before asking whether the candidate exists, so both hosts give up in the same place. The give-up stays narrow: a `..` whose parent is really there is unambiguous everywhere and still resolves, which is now pinned by its own test so a future tightening cannot quietly leave ordinary paths unresolved. Found by the Windows lane; it does not reproduce on Linux. Refs: #315 Signed-off-by: Eugene Volen --------- Signed-off-by: Eugene Volen Signed-off-by: pmoutsias-amd --- apps/rocm/src/therock.rs | 135 ++++++++++++- crates/rocm-core/src/disk_space.rs | 2 +- crates/rocm-core/src/lib.rs | 10 +- crates/rocm-core/src/runtime.rs | 190 ++++++++++++++++++ engines/vllm/src/lib.rs | 188 ++++++++++++++++- .../features/runtime_setup.feature | 27 +++ tests/e2e-cucumber/tests/e2e.rs | 82 ++++++-- tests/e2e-cucumber/tests/e2e/runtime_steps.rs | 89 ++++++++ 8 files changed, 684 insertions(+), 39 deletions(-) diff --git a/apps/rocm/src/therock.rs b/apps/rocm/src/therock.rs index 38d370b123..cf5fcb7e10 100644 --- a/apps/rocm/src/therock.rs +++ b/apps/rocm/src/therock.rs @@ -832,7 +832,7 @@ fn install_wheel_runtime( &resolution.family, Some(&resolution.latest_version), ); - let install_root = prefix.unwrap_or_else(|| managed_runtime_root(paths, "wheel", &runtime_key)); + let install_root = resolved_install_root(paths, "wheel", &runtime_key, prefix); let manifest_path = runtime_manifest_path(paths, &runtime_key); let mut output = String::new(); @@ -1047,8 +1047,7 @@ fn install_tarball_runtime( &artifact.family, Some(&artifact.version), ); - let install_root = - prefix.unwrap_or_else(|| managed_runtime_root(paths, "tarball", &runtime_key)); + let install_root = resolved_install_root(paths, "tarball", &runtime_key, prefix); let manifest_path = runtime_manifest_path(paths, &runtime_key); let cache_path = paths.cache_dir.join("therock").join(&artifact.file_name); @@ -3472,6 +3471,34 @@ fn managed_runtime_root(paths: &AppPaths, format: &str, runtime_key: &str) -> Pa .join(runtime_key) } +/// Where to build a runtime, resolved to its real location on disk first. +/// +/// `install sdk` writes this path into three places that all outlive the command: +/// the registry manifest, the sidecar beside the runtime, and — via `uv` — the +/// `#!` line of every console script in the venv. So the path handed to the +/// installer has to name where the files land, not the route taken to get there. +/// Reaching `data/runtimes` through a symlink is enough to make those differ, and +/// once the link goes the runtime reports itself installed at a folder that is not +/// there while the files sit untouched next door. +/// +/// Applies to `--prefix` too, which has the identical failure mode. +/// +/// Only the root is resolved. `python_executable` is derived from it and must +/// keep the venv's own `bin/python`, which is itself a symlink to the base +/// interpreter — resolving that would record the system Python and break venv +/// semantics. The adopt path already draws the line in the same place: see +/// `main::adopt_runtime_from_probe`, which canonicalizes the install root next to +/// `absolute_existing_file_path_preserving_symlink` for the interpreter. +fn resolved_install_root( + paths: &AppPaths, + format: &str, + runtime_key: &str, + prefix: Option, +) -> PathBuf { + let requested = prefix.unwrap_or_else(|| managed_runtime_root(paths, format, runtime_key)); + rocm_core::resolve_path_through_symlinks(&requested) +} + fn runtime_registry_dir(paths: &AppPaths) -> PathBuf { paths.data_dir.join("runtimes").join("registry") } @@ -4248,6 +4275,108 @@ mod tests { assert_eq!(args, vec!["-m", "venv", "/mnt/envs/my-env"]); } + /// A data dir whose `runtimes` folder is a symlink to somewhere else, plus the + /// real folder it points at — both canonical, so an assertion cannot pass or + /// fail on whether the machine's temp dir is itself behind a link. + /// + /// Deliberately not `test_paths`: that builds paths by plain `join` and never + /// canonicalizes, which is exactly the property under test here. + #[cfg(unix)] + fn linked_runtimes_paths(name: &str) -> (PathBuf, AppPaths, PathBuf) { + let root = std::env::temp_dir().join(format!( + "rocm-cli-linked-runtimes-{name}-{}-{}", + std::process::id(), + unix_time_millis() + )); + fs::create_dir_all(root.join("data")).unwrap(); + let root = root.canonicalize().unwrap(); + let real = root.join("data").join("real-runtimes"); + fs::create_dir_all(&real).unwrap(); + std::os::unix::fs::symlink(&real, root.join("data").join("runtimes")).unwrap(); + let paths = AppPaths { + config_dir: root.join("config"), + data_dir: root.join("data"), + cache_dir: root.join("cache"), + }; + (root, paths, real) + } + + #[test] + #[cfg(unix)] + fn install_root_resolves_a_symlinked_runtimes_folder() { + // The e2e harness points a scenario's `data/runtimes` at a shared tree this + // way. Recording the link's path made the runtime name a folder that + // vanished with the scenario, while the files stayed where they were + // written (rocm-cli#315). + let (root, paths, real) = linked_runtimes_paths("resolves"); + let runtime_key = "release-wheel-gfx120x-all-7-14-0"; + + let resolved = resolved_install_root(&paths, "wheel", runtime_key, None); + + assert_eq!(resolved, real.join("wheel").join(runtime_key)); + assert!( + !resolved.starts_with(paths.data_dir.join("runtimes")), + "the recorded root must not be expressed through the link: {}", + resolved.display() + ); + fs::remove_dir_all(&root).ok(); + } + + #[test] + #[cfg(unix)] + fn install_root_resolves_a_symlinked_prefix() { + // `--prefix` has the identical failure mode, so it gets the identical fix. + let (root, paths, real) = linked_runtimes_paths("prefix"); + let prefix = paths.data_dir.join("runtimes").join("chosen-env"); + + let resolved = resolved_install_root(&paths, "wheel", "unused-key", Some(prefix)); + + assert_eq!(resolved, real.join("chosen-env")); + fs::remove_dir_all(&root).ok(); + } + + #[test] + #[cfg(unix)] + fn install_root_resolves_the_tarball_format_too() { + let (root, paths, real) = linked_runtimes_paths("tarball"); + let runtime_key = "release-tarball-gfx120x-all-7-14-0"; + + let resolved = resolved_install_root(&paths, "tarball", runtime_key, None); + + assert_eq!(resolved, real.join("tarball").join(runtime_key)); + fs::remove_dir_all(&root).ok(); + } + + #[test] + fn install_root_is_unchanged_when_nothing_is_linked() { + // Guards against gratuitous rewriting: on an ordinary tree the recorded + // root must still be the plain layout path, so existing installs and the + // paths compared against them do not move. + let root = std::env::temp_dir().join(format!( + "rocm-cli-plain-runtimes-{}-{}", + std::process::id(), + unix_time_millis() + )); + fs::create_dir_all(root.join("data")).unwrap(); + // Resolved rather than canonicalized: on Windows `canonicalize` returns a + // verbatim `\\?\C:\…` path, so the expectation would carry a prefix the + // resolver deliberately strips and the test would fail on that rather than + // on whether the layout path moved. + let root = rocm_core::resolve_path_through_symlinks(&root); + let paths = AppPaths { + config_dir: root.join("config"), + data_dir: root.join("data"), + cache_dir: root.join("cache"), + }; + let runtime_key = "release-wheel-gfx120x-all-7-14-0"; + + assert_eq!( + resolved_install_root(&paths, "wheel", runtime_key, None), + managed_runtime_root(&paths, "wheel", runtime_key) + ); + fs::remove_dir_all(&root).ok(); + } + #[test] fn managed_uv_cache_sits_under_the_data_dir_for_generated_runtime_folders() { let (_root, paths) = test_paths("managed-uv-cache"); diff --git a/crates/rocm-core/src/disk_space.rs b/crates/rocm-core/src/disk_space.rs index 2d4e90a58d..4abe85266a 100644 --- a/crates/rocm-core/src/disk_space.rs +++ b/crates/rocm-core/src/disk_space.rs @@ -130,7 +130,7 @@ fn nearest_existing_ancestor(path: &Path) -> Option { /// map back to `\\server\share`. /// /// Pure string handling so it is exercised on every platform, not only Windows. -fn strip_verbatim_prefix(path: &Path) -> PathBuf { +pub(crate) fn strip_verbatim_prefix(path: &Path) -> PathBuf { let text = path.to_string_lossy(); if let Some(rest) = text.strip_prefix(r"\\?\UNC\") { return PathBuf::from(format!(r"\\{rest}")); diff --git a/crates/rocm-core/src/lib.rs b/crates/rocm-core/src/lib.rs index 2379c08b6e..e0369ecab5 100644 --- a/crates/rocm-core/src/lib.rs +++ b/crates/rocm-core/src/lib.rs @@ -59,11 +59,11 @@ pub use runtime::{ normalize_runtime_path_for_host, normalize_runtime_path_for_storage, normalize_runtime_path_text_for_host, normalize_runtime_path_text_for_platform, normalize_runtime_path_text_for_storage, platform_binary_name, prepend_runtime_path, - runtime_directory_label, runtime_drive_root_for_key, runtime_drive_roots, runtime_exe_suffix, - runtime_home_dir, runtime_install_root_is_protected, runtime_is_linux, runtime_is_windows, - runtime_os_name, runtime_path_for_child, runtime_path_for_windows_child, - runtime_path_is_same_or_inside, runtime_path_list_join, runtime_path_list_split, - runtime_path_sort_key, runtime_path_text_is_absolute_for_host, + resolve_path_through_symlinks, runtime_directory_label, runtime_drive_root_for_key, + runtime_drive_roots, runtime_exe_suffix, runtime_home_dir, runtime_install_root_is_protected, + runtime_is_linux, runtime_is_windows, runtime_os_name, runtime_path_for_child, + runtime_path_for_windows_child, runtime_path_is_same_or_inside, runtime_path_list_join, + runtime_path_list_split, runtime_path_sort_key, runtime_path_text_is_absolute_for_host, runtime_path_text_is_absolute_for_platform, runtime_paths_equivalent, runtime_python_activation_hint, runtime_python_activation_script, runtime_python_bin_dir_name, runtime_python_env_bin_dir, runtime_python_executable_in_env, runtime_python_executable_name, diff --git a/crates/rocm-core/src/runtime.rs b/crates/rocm-core/src/runtime.rs index bbab3b5c24..bb5ccd9347 100644 --- a/crates/rocm-core/src/runtime.rs +++ b/crates/rocm-core/src/runtime.rs @@ -626,6 +626,78 @@ pub fn normalize_runtime_path_text_for_host(value: &str) -> String { normalize_runtime_path_text(value) } +/// Resolve `path` to the real location on disk, keeping any trailing components +/// that do not exist yet. +/// +/// For a path that is about to be *recorded* rather than merely used. The +/// filesystem happily reaches a folder through a symlink, so an absolute path +/// built by joining onto a linked ancestor names a route rather than a place: +/// correct now, and wrong the moment the link goes. Persisting it — in a manifest, +/// or in the shebang of a console script — outlives whatever made the route valid. +/// +/// This is the resolve half of the pipeline the runtime manifest goes through: +/// resolve, then [`normalize_runtime_path_for_storage`], then persist. +/// +/// The leaf usually does not exist yet — it is where something is about to be +/// installed — so this walks up to the deepest ancestor that does, canonicalizes +/// that, and re-attaches the rest. Same shape as `disk_space::nearest_existing_ancestor`, +/// except that one answers "which filesystem is this on?" and so discards the +/// walked components, while this needs them back. +/// +/// Best-effort by design: a path that cannot be resolved comes back absolutized +/// but otherwise unchanged, which is what the caller would have used anyway. +/// Cases that give up deliberately rather than guess: +/// +/// * a `..` in the not-yet-existing tail, where re-attaching after resolving an +/// ancestor could name a different place than the caller meant; +/// * a relative path with no readable current directory. +/// +/// On Windows this also strips the verbatim `\\?\` prefix that `canonicalize` +/// returns, since a stored path is later compared against ordinary ones. Windows +/// directory junctions resolve through the same call, so no separate branch. +pub fn resolve_path_through_symlinks(path: &Path) -> PathBuf { + let absolute = if path.is_absolute() { + path.to_path_buf() + } else { + match std::env::current_dir() { + Ok(cwd) => cwd.join(path), + Err(_) => return path.to_path_buf(), + } + }; + + let mut tail: Vec = Vec::new(); + let mut candidate = absolute.as_path(); + loop { + // Checked BEFORE `exists`, because the two platforms disagree about what a + // `..` past a missing directory even means. Windows collapses it lexically, + // so `/missing/..` "exists" and canonicalizes to ``; Unix walks + // the path through the filesystem, so it does not exist at all. Resolving + // here would therefore record a different folder depending on the host — + // and the whole point of this function is that the folder it returns is the + // one the files land in. Give up instead. + if candidate.components().next_back() == Some(std::path::Component::ParentDir) { + return absolute; + } + if candidate.exists() { + let Ok(resolved) = candidate.canonicalize() else { + return absolute; + }; + let mut resolved = crate::disk_space::strip_verbatim_prefix(&resolved); + for component in tail.iter().rev() { + resolved.push(component); + } + return resolved; + } + // `file_name` is None at the root. Nothing left to walk up to, so keep the + // caller's path. + let (Some(name), Some(parent)) = (candidate.file_name(), candidate.parent()) else { + return absolute; + }; + tail.push(name.to_os_string()); + candidate = parent; + } +} + pub fn normalize_runtime_path_for_storage(path: &Path) -> PathBuf { PathBuf::from(normalize_runtime_path_text_for_storage( &path.display().to_string(), @@ -755,6 +827,124 @@ mod tests { use super::*; use std::fs; + fn scratch_dir(name: &str) -> PathBuf { + let root = std::env::temp_dir().join(format!( + "rocm-core-resolve-{name}-{}-{:?}", + std::process::id(), + std::thread::current().id() + )); + fs::remove_dir_all(&root).ok(); + fs::create_dir_all(&root).unwrap(); + // Resolved up front, so an assertion cannot turn on whether the platform's + // temp dir is itself reached through a link. + // + // Deliberately the function under test rather than a bare `canonicalize`: + // on Windows `canonicalize` hands back a verbatim `\\?\C:\…` path, which + // would make every expectation here verbatim while the function correctly + // returns a plain one — the tests would then fail on the prefix rather than + // on the behaviour they are about. That the prefix really is stripped is + // asserted separately, in `resolving_never_yields_a_windows_verbatim_prefix`. + resolve_path_through_symlinks(&root) + } + + #[test] + fn resolving_keeps_components_that_do_not_exist_yet() { + // The common case: the leaf is where something is about to be installed. + let root = scratch_dir("missing-leaf"); + assert_eq!( + resolve_path_through_symlinks(&root.join("a").join("b").join("c")), + root.join("a").join("b").join("c") + ); + fs::remove_dir_all(&root).ok(); + } + + #[test] + #[cfg(unix)] + fn resolving_follows_a_linked_ancestor_and_reattaches_the_rest() { + let root = scratch_dir("linked-ancestor"); + let real = root.join("real"); + fs::create_dir_all(&real).unwrap(); + std::os::unix::fs::symlink(&real, root.join("link")).unwrap(); + + assert_eq!( + resolve_path_through_symlinks(&root.join("link").join("wheel").join("key")), + real.join("wheel").join("key") + ); + fs::remove_dir_all(&root).ok(); + } + + #[test] + #[cfg(unix)] + fn resolving_leaves_a_dangling_link_alone() { + // Nothing to resolve to, so the caller's path is the best answer available. + let root = scratch_dir("dangling"); + std::os::unix::fs::symlink(root.join("nowhere"), root.join("link")).unwrap(); + let requested = root.join("link").join("child"); + + assert_eq!(resolve_path_through_symlinks(&requested), requested); + fs::remove_dir_all(&root).ok(); + } + + #[test] + #[cfg(unix)] + fn resolving_is_idempotent() { + let root = scratch_dir("idempotent"); + let real = root.join("real"); + fs::create_dir_all(&real).unwrap(); + std::os::unix::fs::symlink(&real, root.join("link")).unwrap(); + let once = resolve_path_through_symlinks(&root.join("link").join("leaf")); + + assert_eq!(resolve_path_through_symlinks(&once), once); + fs::remove_dir_all(&root).ok(); + } + + #[test] + fn resolving_gives_up_rather_than_guess_at_a_parent_component() { + // A `..` reached past a missing directory means different things on + // different hosts: Windows collapses it lexically, so `/missing/..` + // resolves to ``, while Unix walks the filesystem and finds nothing. + // Resolving it would record a different folder depending on the host, so + // the path comes back untouched on both. + let root = scratch_dir("parent-component"); + let requested = root.join("missing").join("..").join("sibling"); + + assert_eq!(resolve_path_through_symlinks(&requested), requested); + fs::remove_dir_all(&root).ok(); + } + + #[test] + fn resolving_still_handles_a_parent_component_the_filesystem_can_walk() { + // The give-up above must not spread to a `..` whose parent is really + // there — that one is unambiguous on every host, and refusing it would + // leave an ordinary path unresolved. + let root = scratch_dir("parent-that-exists"); + let nested = root.join("a").join("b"); + fs::create_dir_all(&nested).unwrap(); + let requested = nested.join("..").join("b").join("leaf"); + + assert_eq!( + resolve_path_through_symlinks(&requested), + nested.join("leaf") + ); + fs::remove_dir_all(&root).ok(); + } + + #[test] + fn resolving_never_yields_a_windows_verbatim_prefix() { + // A stored path is later compared against ordinary ones, and `\\?\C:\…` + // never `starts_with`-matches `C:\…`. Trivially true off Windows; the + // prefix stripping itself is covered in `disk_space`. + let root = scratch_dir("verbatim"); + let resolved = resolve_path_through_symlinks(&root.join("leaf")); + + assert!( + !resolved.to_string_lossy().starts_with(r"\\?\"), + "{}", + resolved.display() + ); + fs::remove_dir_all(&root).ok(); + } + #[test] fn platform_binary_name_follows_runtime_host() { let name = platform_binary_name("rocm"); diff --git a/engines/vllm/src/lib.rs b/engines/vllm/src/lib.rs index 7971d2b4fa..0ad0e48192 100644 --- a/engines/vllm/src/lib.rs +++ b/engines/vllm/src/lib.rs @@ -443,10 +443,14 @@ fn install_response(request: InstallRequest) -> Result { Some(assessed) => assessed, None => resolve_managed_runtime_python(Some(&request.runtime_id))?.with_context( || { - format!( + let base = format!( "runtime `{}` did not resolve to a managed TheRock Python environment for automatic vLLM install", request.runtime_id - ) + ); + match describe_skipped_managed_runtimes(Some(&request.runtime_id)) { + Some(skipped) => format!("{base}\n\n{skipped}"), + None => base, + } }, )?, }; @@ -853,9 +857,48 @@ the ROCm SDK's bundled numa uses renamed symbol versions and cannot satisfy it, command.stderr(Stdio::from(log)); } - command - .spawn() - .with_context(|| format!("failed to spawn vLLM command {}", runtime.command.display())) + command.spawn().map_err(|error| { + let base = anyhow::Error::new(error).context(format!( + "failed to spawn vLLM command {}", + runtime.command.display() + )); + match stale_interpreter_hint(&runtime.command) { + Some(hint) => base.context(hint), + None => base, + } + }) +} + +/// Explain a spawn that failed on a script whose `#!` interpreter is gone. +/// +/// The kernel reports the missing *interpreter* as ENOENT against the *script*, +/// so the raw error names a file that is plainly there. A virtualenv whose folder +/// was recorded under one path and now lives at another produces exactly this: the +/// entry points still exist, and not one of them can start. +/// +/// Returns `None` whenever the ordinary reading is the right one — a genuinely +/// absent file, a binary, or a shebang whose interpreter is present — so this only +/// ever speaks up when it has something to add. +fn stale_interpreter_hint(command: &Path) -> Option { + if !command.is_file() { + return None; + } + let head = fs::read(command).ok()?; + let first_line = head.split(|byte| *byte == b'\n').next()?; + let text = String::from_utf8_lossy(first_line); + let shebang = text.strip_prefix("#!")?.trim(); + // `#!/usr/bin/env python` names the launcher, not the interpreter; the path + // that goes stale is the direct one a venv writes. + let interpreter = shebang.split_whitespace().next()?; + if Path::new(interpreter).is_file() { + return None; + } + Some(format!( + "`{}` is present, but the interpreter on its `#!` line is not: {interpreter}. \ + Its environment was recorded at a folder that is no longer there, so none of \ + its entry points can start. Reinstall it with `rocm install sdk`.", + command.display() + )) } fn healthcheck_service(request: HealthcheckRequest) -> Result { @@ -1017,9 +1060,11 @@ fn resolve_vllm_runtime(runtime_id: Option<&str>) -> Result { }); } - bail!( - "vLLM is not installed in a Linux/WSL ROCm Python environment. Install/build vLLM against a ROCm-capable Python environment, then set ROCM_CLI_VLLM_COMMAND, set ROCM_CLI_VLLM_PYTHON, or install it into the active rocm-cli TheRock runtime. Native Windows is skipped; no CPU fallback is used." - ) + let base = "vLLM is not installed in a Linux/WSL ROCm Python environment. Install/build vLLM against a ROCm-capable Python environment, then set ROCM_CLI_VLLM_COMMAND, set ROCM_CLI_VLLM_PYTHON, or install it into the active rocm-cli TheRock runtime. Native Windows is skipped; no CPU fallback is used."; + match describe_skipped_managed_runtimes(runtime_id) { + Some(skipped) => bail!("{base}\n\n{skipped}"), + None => bail!("{base}"), + } } fn runtime_from_python( @@ -1133,6 +1178,56 @@ struct ManagedRuntimeCandidate { sdk_library_paths: Vec, } +/// Registered runtimes that matched the request but were passed over because the +/// interpreter they record is not there, phrased for the end of an error message. +/// +/// [`collect_managed_runtime_candidates`] drops those silently, which is right for +/// resolution — a runtime that cannot run is not a candidate — but leaves the +/// failure describing a registry that looks empty while `rocm runtimes list` +/// happily prints the entry. Naming the manifest and the interpreter turns +/// "nothing resolved" into something actionable. +/// +/// Best-effort: this only ever decorates an error that is already being returned, +/// so any problem reading the registry yields no note rather than replacing the +/// original failure. +fn describe_skipped_managed_runtimes(runtime_id: Option<&str>) -> Option { + let paths = AppPaths::discover().ok()?; + let registry = paths.data_dir.join("runtimes").join("registry"); + let mut skipped = Vec::new(); + for entry in fs::read_dir(®istry).ok()? { + let Ok(entry) = entry else { continue }; + let path = entry.path(); + if path.extension().and_then(|value| value.to_str()) != Some("json") { + continue; + } + let Ok(bytes) = fs::read(&path) else { continue }; + let Ok(manifest) = serde_json::from_slice::(&bytes) else { + continue; + }; + if !runtime_matches(&manifest, runtime_id) { + continue; + } + let Some(python) = manifest.python_executable.as_ref() else { + continue; + }; + if python.is_file() { + continue; + } + let key = manifest.runtime_key.as_deref().unwrap_or(""); + skipped.push(format!(" {key}: {} is missing", python.display())); + } + if skipped.is_empty() { + return None; + } + skipped.sort(); + Some(format!( + "These registered runtimes were skipped because the Python interpreter they \ + record is not there:\n{}\nReinstall one with `rocm install sdk`, or drop it with \ + `rocm runtimes uninstall `.", + skipped.join("\n") + )) +} + fn collect_managed_runtime_candidates( runtime_id: Option<&str>, ) -> Result> { @@ -2046,6 +2141,83 @@ fn tail_lines(path: &Path, limit: usize) -> Result> { mod tests { use super::*; + /// A throwaway directory for the shebang cases. Scoped per test and per + /// thread so the suite can keep running these concurrently. + fn shebang_scratch(name: &str) -> std::path::PathBuf { + let root = std::env::temp_dir().join(format!( + "vllm-shebang-{name}-{}-{:?}", + std::process::id(), + std::thread::current().id() + )); + fs::remove_dir_all(&root).ok(); + fs::create_dir_all(&root).expect("create scratch dir"); + root + } + + #[test] + fn a_dead_shebang_interpreter_is_named() { + // The reported confusion: ENOENT against a script that is plainly there, + // because the kernel reports the missing interpreter against the script. + let root = shebang_scratch("dead"); + let script = root.join("vllm"); + fs::write(&script, "#!/gone/bin/python\nprint()\n").unwrap(); + + let hint = stale_interpreter_hint(&script).expect("a dead interpreter must be named"); + + assert!(hint.contains("/gone/bin/python"), "{hint}"); + assert!(hint.contains("is present"), "{hint}"); + fs::remove_dir_all(&root).ok(); + } + + #[test] + fn a_live_shebang_interpreter_gets_no_hint() { + // Nothing to add: the ordinary reading of the error is the right one. + let root = shebang_scratch("live"); + let interpreter = root.join("python"); + fs::write(&interpreter, "").unwrap(); + let script = root.join("vllm"); + fs::write(&script, format!("#!{}\n", interpreter.display())).unwrap(); + + assert!(stale_interpreter_hint(&script).is_none()); + fs::remove_dir_all(&root).ok(); + } + + #[test] + fn a_genuinely_missing_command_gets_no_hint() { + let root = shebang_scratch("absent"); + assert!(stale_interpreter_hint(&root.join("not-there")).is_none()); + fs::remove_dir_all(&root).ok(); + } + + #[test] + fn a_command_with_no_shebang_gets_no_hint() { + // A real binary. Reading its first bytes must not produce a hint. + let root = shebang_scratch("binary"); + let binary = root.join("vllm"); + fs::write(&binary, [0x7f, b'E', b'L', b'F', 0x02, 0x01, 0x01, 0x00]).unwrap(); + + assert!(stale_interpreter_hint(&binary).is_none()); + fs::remove_dir_all(&root).ok(); + } + + #[test] + fn shebang_arguments_do_not_hide_the_interpreter() { + // `#!/path/python -X foo` names the interpreter first; the flags are not + // part of the path being checked. + let root = shebang_scratch("args"); + let script = root.join("vllm"); + fs::write(&script, "#!/gone/bin/python -X utf8\n").unwrap(); + + let hint = stale_interpreter_hint(&script).expect("interpreter must still be found"); + + assert!(hint.contains("/gone/bin/python"), "{hint}"); + assert!( + !hint.contains("-X"), + "the flags are not part of the path: {hint}" + ); + fs::remove_dir_all(&root).ok(); + } + /// Answer `count` chat requests on a loopback port with the given status, /// reporting how many arrived. fn spawn_chat_endpoint( diff --git a/tests/e2e-cucumber/features/runtime_setup.feature b/tests/e2e-cucumber/features/runtime_setup.feature index b4e861d694..b66cb92e86 100644 --- a/tests/e2e-cucumber/features/runtime_setup.feature +++ b/tests/e2e-cucumber/features/runtime_setup.feature @@ -47,6 +47,33 @@ Feature: Runtime configuration When the user checks for runtime updates Then the report states the runtime's freshness against the channel index + # The install used to record the path it was handed rather than the folder the + # files land in, so reaching `data/runtimes` through a link made the runtime name + # a folder that disappeared with the link — taking every console-script shebang + # in the environment with it, while the files stayed where they were written + # (rocm-cli#315). The E2E harness itself creates exactly that link when a scenario + # opts into the shared pre-warmed runtime, so the shared tree on a runner was the + # thing being poisoned. Previewing the install is enough to pin this and needs no + # GPU and no download: the planned folder is resolved before the preview prints + # it, so a regression shows up in the plan. `--family` is supplied because + # without a GPU there is no target to detect. + # + # `@nightly` is not about this scenario's own cost — it runs in about eight + # seconds, nearly all of it resolving the channel index. It is that the no-GPU + # mock lane runs 64 scenarios at once, and that much concurrent network work is + # enough to push `eai-7960-gen-tps-held-after-scrape-failure` and + # `eai-7960-gen-tps-expiry-boundary` past the validity window they assert on + # (measured: both fail 3/3 with this scenario on the mock lane, and pass with the + # very same scenario once the suite is serialized). Those two are timing-fragile + # under load, which is their own problem to fix; until then this runs on the + # nightly lanes, where a GPU is present and scenarios are serialized. + @id:runtime-install-records-the-real-folder @nightly + Scenario: 8 - Previewing an install through a linked runtimes folder names the real folder + Given a machine whose runtimes folder is a link to somewhere else + When the user previews an SDK install + Then the planned runtime folder is inside the folder the link points at + And the planned runtime folder is not expressed through the link + # Linux-only: the step adopts a standard `/opt/rocm` install with a Unix python # path. On Windows those paths don't exist (the CLI resolves `/usr/bin/python3` # to a bogus `C:/usr/bin/python3` and errors on the missing path before it can diff --git a/tests/e2e-cucumber/tests/e2e.rs b/tests/e2e-cucumber/tests/e2e.rs index f7eddc2830..9a28e44d36 100644 --- a/tests/e2e-cucumber/tests/e2e.rs +++ b/tests/e2e-cucumber/tests/e2e.rs @@ -8,7 +8,7 @@ // than splitting the step API into sync/async and mut/non-mut variants. #![allow(clippy::unused_async, clippy::needless_pass_by_ref_mut)] -use std::path::PathBuf; +use std::path::{Path, PathBuf}; use cucumber::{World as _, WriterExt as _}; use e2e_cucumber::cli_failure_report; @@ -278,6 +278,42 @@ impl E2eWorld { } } + /// Create `link` as a directory link to `target`. + /// + /// Self-hosted Windows runners commonly lack `SeCreateSymbolicLinkPrivilege`, + /// so a failed symlink falls back to a directory junction, which needs neither + /// developer mode nor an elevated token and looks the same to the CLI. + #[allow(unused_variables)] + fn link_directory(target: &Path, link: &Path) -> std::io::Result<()> { + #[cfg(unix)] + { + std::os::unix::fs::symlink(target, link) + } + #[cfg(windows)] + { + std::os::windows::fs::symlink_dir(target, link).or_else(|_| { + let status = std::process::Command::new("cmd") + .args(["/C", "mklink", "/J"]) + .arg(link) + .arg(target) + .status()?; + if status.success() { + Ok(()) + } else { + Err(std::io::Error::other(format!( + "mklink /J exited with {status}" + ))) + } + }) + } + #[cfg(not(any(unix, windows)))] + { + Err(std::io::Error::other( + "directory links are unsupported here", + )) + } + } + /// Opt this scenario into the shared managed-runtime tree (see /// [`shared_runtimes_dir`]): replace its empty isolated `data/runtimes` with a /// directory link to the shared dir, so an `install sdk` here populates the @@ -299,31 +335,33 @@ impl E2eWorld { } let link = data.join("runtimes"); let _ = std::fs::remove_dir_all(&link); - #[cfg(unix)] - let res = std::os::unix::fs::symlink(&shared, &link); - #[cfg(windows)] - let res = std::os::windows::fs::symlink_dir(&shared, &link).or_else(|_| { - // Self-hosted Windows runners commonly lack SeCreateSymbolicLinkPrivilege. - // Directory junctions need no developer mode or elevated token and expose - // the same shared runtime tree to the scenario's isolated data directory. - let status = std::process::Command::new("cmd") - .args(["/C", "mklink", "/J"]) - .arg(&link) - .arg(&shared) - .status()?; - if status.success() { - Ok(()) - } else { - Err(std::io::Error::other(format!( - "mklink /J exited with {status}" - ))) - } - }); - if res.is_err() { + if Self::link_directory(&shared, &link).is_err() { let _ = std::fs::create_dir_all(&link); } } + /// Point this scenario's `data/runtimes` at a sibling folder inside its own + /// isolated tree, and return that folder. + /// + /// The same shape as [`use_shared_runtimes`] without the shared tree: for + /// scenarios that need to observe what the CLI does when its runtimes folder is + /// reached through a link, but must not write anywhere a later run can see. + pub fn link_runtimes_within_scenario(&self) -> std::io::Result { + let root = self + .isolated_root + .as_ref() + .expect("scenario has no isolated root") + .path(); + let data = root.join("data"); + std::fs::create_dir_all(&data)?; + let real = data.join("real-runtimes"); + std::fs::create_dir_all(&real)?; + let link = data.join("runtimes"); + let _ = std::fs::remove_dir_all(&link); + Self::link_directory(&real, &link)?; + Ok(real) + } + /// Plant a fake pre-existing (non-CLI) ROCm install in the scenario's isolated /// tree and record its path so `isolate_cmd` exports it as `ROCM_PATH`. The /// CLI's `detect_legacy_rocm_summary` treats any directory containing a known diff --git a/tests/e2e-cucumber/tests/e2e/runtime_steps.rs b/tests/e2e-cucumber/tests/e2e/runtime_steps.rs index 84e6079c73..bec98c56c1 100644 --- a/tests/e2e-cucumber/tests/e2e/runtime_steps.rs +++ b/tests/e2e-cucumber/tests/e2e/runtime_steps.rs @@ -18,6 +18,95 @@ async fn setup_no_runtimes(world: &mut E2eWorld) { #[given("a machine with a standard ROCm install")] async fn setup_standard_rocm(_world: &mut E2eWorld) {} +#[given("a machine whose runtimes folder is a link to somewhere else")] +async fn setup_linked_runtimes_folder(world: &mut E2eWorld) { + world + .link_runtimes_within_scenario() + .expect("failed to link the scenario's runtimes folder"); +} + +/// The folder the scenario's `data/runtimes` link points at, resolved so it can be +/// compared against a path the CLI resolved. +/// +/// The verbatim prefix has to come back off. `canonicalize` returns `\\?\C:\…` on +/// Windows and the CLI records a plain path, so comparing the two raw would fail on +/// the prefix rather than on the folder — and only on the Windows lane, long after +/// this was written. The CLI strips it for the same reason (`rocm-core`'s +/// `strip_verbatim_prefix`); this crate cannot reach that helper, and a dependency +/// on `rocm-core` for six lines of string handling is the worse trade. +fn linked_runtimes_target(world: &E2eWorld) -> std::path::PathBuf { + let real = world + .isolated_root + .as_ref() + .expect("scenario has no isolated root") + .path() + .join("data") + .join("real-runtimes"); + let resolved = real.canonicalize().unwrap_or(real); + let text = resolved.to_string_lossy(); + if let Some(rest) = text.strip_prefix(r"\\?\UNC\") { + return std::path::PathBuf::from(format!(r"\\{rest}")); + } + match text.strip_prefix(r"\\?\") { + Some(rest) => std::path::PathBuf::from(rest), + None => resolved.clone(), + } +} + +#[when("the user previews an SDK install")] +async fn user_previews_sdk_install(world: &mut E2eWorld) { + // `--family` because a host with no AMD GPU has no target to detect, and the + // preview resolves the install folder before it needs one. `--dry-run` keeps + // this to a plan: no venv, no multi-GiB download. + let stdout = crate::run_rocm_ok( + world, + &["install", "sdk", "--family", "gfx110X-all", "--dry-run"], + ); + world.cli_output = Some(stdout); +} + +/// The ` target: ` line of the install preview. +fn planned_runtime_folder(world: &E2eWorld) -> String { + let output = world.cli_output.as_deref().expect("no install preview"); + output + .lines() + .find_map(|line| line.trim().strip_prefix("target: ")) + .unwrap_or_else(|| panic!("no planned runtime folder in the preview:\n{output}")) + .trim() + .to_owned() +} + +#[then("the planned runtime folder is inside the folder the link points at")] +async fn assert_planned_folder_is_the_real_one(world: &mut E2eWorld) { + let planned = planned_runtime_folder(world); + let real = linked_runtimes_target(world); + assert!( + std::path::Path::new(&planned).starts_with(&real), + "the install would record {planned}, which is not inside {}", + real.display() + ); +} + +#[then("the planned runtime folder is not expressed through the link")] +async fn assert_planned_folder_avoids_the_link(world: &mut E2eWorld) { + // The failure this pins: a folder named through the link reads as valid until + // the link goes, and takes the environment's console-script shebangs with it. + let planned = planned_runtime_folder(world); + let link = world + .isolated_root + .as_ref() + .expect("scenario has no isolated root") + .path() + .join("data") + .join("runtimes"); + assert!( + !std::path::Path::new(&planned).starts_with(&link), + "the install would record {planned}, which names the link at {} rather than \ + the folder it points at", + link.display() + ); +} + #[given("a managed runtime is active")] async fn setup_active_runtime(world: &mut E2eWorld) { // On a no-GPU host this is a no-op: a managed TheRock SDK runtime can only be From 131690de9811fc863f31e2cfa1219de7be6c8476 Mon Sep 17 00:00:00 2001 From: Roman Date: Thu, 27 Aug 2026 12:44:44 +0000 Subject: [PATCH 14/23] fix(install): resolve driver repo version and package release in plans (#307) * fix(install): resolve driver repo version and package release in plans The driver install plan stored the repo version and package release as raw shell parameter-expansion templates (${ROCM_CLI_AMDGPU_VERSION:-7.2.4} and ${ROCM_CLI_AMDGPU_PACKAGE_RELEASE:-70204}). Two problems followed. The human-readable plan summary printed the version template verbatim on its "repo_version:" line, leaking an unexpanded shell placeholder into user-facing dry-run output. And the Debian/Ubuntu apt source line wrapped the template in single quotes, so the shell never expanded it and the literal placeholder would have been written into /etc/apt/sources.list.d/amdgpu.list. Resolve both templates once, at plan-build time, to their effective values (the env var when set and non-empty, matching shell :- default semantics, otherwise the documented default) and bake the concrete values into every rendered URL, repo path, and command. The resolver is generalized to resolve_shell_default_template and hardened to pass bare ${VAR} and nested-default shapes through unchanged rather than emitting a partially rewritten value. Add unit tests for the resolver (default, env override, empty-as-unset, non-template pass-through, bare-var and nested-default pass-through) and update every distro plan test to assert the concrete values, plus a render-level regression test and an end-to-end scenario asserting the dry-run summary shows the resolved version rather than the raw placeholder. Tests that read these process env vars serialize on a shared lock and save/restore prior values so they stay deterministic under edition-2024 env semantics. Signed-off-by: Roman Sirokov * test(install): guard the two remaining env-reading driver-plan tests build_driver_install_plan now resolves the repo-version template via std::env::var at the top of the function, so every test calling it reads process env. The sweep that added ScopedTestEnv::with_amd_overrides_cleared() to the other call sites missed windows_install_driver_is_validate_only and wsl_install_driver_uses_rocdxg_guidance_without_dkms, leaving two unguarded readers racing the guarded mutators in the same test binary. Add the guard to both. No production change. Signed-off-by: Roman Sirokov --------- Signed-off-by: Roman Sirokov Signed-off-by: pmoutsias-amd --- apps/rocm/src/main.rs | 349 ++++++++++++++---- tests/e2e-cucumber/features/examine.feature | 11 + tests/e2e-cucumber/tests/e2e/examine_steps.rs | 27 ++ 3 files changed, 310 insertions(+), 77 deletions(-) diff --git a/apps/rocm/src/main.rs b/apps/rocm/src/main.rs index 5356167ff4..4cf40c2d6a 100644 --- a/apps/rocm/src/main.rs +++ b/apps/rocm/src/main.rs @@ -2820,7 +2820,7 @@ struct DriverInstallPlan { os_id: String, version_id: String, codename: String, - repo_version_expr: String, + repo_version: String, reason: String, preflight_checks: Vec, commands: Vec, @@ -2900,7 +2900,17 @@ fn build_driver_install_plan( os_release_text: &str, dkms: bool, ) -> DriverInstallPlan { - let repo_version_expr = "${ROCM_CLI_AMDGPU_VERSION:-7.2.4}".to_owned(); + // Resolve the AMD graphics version and amdgpu-install package release once, + // here at plan-build time, so the concrete values are baked into both the + // human-readable summary and every command the plan runs. Keeping shell + // `${VAR:-default}` templates in the commands used to be load-bearing, but + // AMD's apt `sources.list` line embeds the template inside POSIX single + // quotes, which suppress all expansion — so the literal `${...}` would land + // in the repo file. Resolving up front fixes that and keeps the summary and + // the executed commands in agreement. + let repo_version = resolve_shell_default_template("${ROCM_CLI_AMDGPU_VERSION:-7.2.4}"); + let package_release = + resolve_shell_default_template("${ROCM_CLI_AMDGPU_PACKAGE_RELEASE:-70204}"); if examine.os == "windows" { return DriverInstallPlan { supported: false, @@ -2909,7 +2919,7 @@ fn build_driver_install_plan( os_id: "windows".to_owned(), version_id: String::new(), codename: String::new(), - repo_version_expr, + repo_version, reason: "Windows driver install is validate-only in rocm-cli; use `rocm examine` to inspect the AMD display driver.".to_owned(), preflight_checks: Vec::new(), commands: Vec::new(), @@ -2924,7 +2934,7 @@ fn build_driver_install_plan( os_id: "wsl".to_owned(), version_id: String::new(), codename: String::new(), - repo_version_expr, + repo_version, reason: "WSL uses the Windows host driver plus ROCDXG; run `scripts/wsl_setup_rocdxg.sh` inside WSL instead of installing Linux DKMS.".to_owned(), preflight_checks: Vec::new(), commands: Vec::new(), @@ -2945,7 +2955,7 @@ fn build_driver_install_plan( os_id, version_id, codename, - repo_version_expr, + repo_version, dkms, true, ), @@ -2955,7 +2965,7 @@ fn build_driver_install_plan( os_id, version_id, repo_codename.to_owned(), - repo_version_expr, + repo_version, dkms, false, ); @@ -2974,7 +2984,8 @@ fn build_driver_install_plan( os_id, version_id, codename, - repo_version_expr, + repo_version, + package_release, dkms, DnfDriverDistro::Rhel, ), @@ -2982,7 +2993,8 @@ fn build_driver_install_plan( os_id, version_id, codename, - repo_version_expr, + repo_version, + package_release, dkms, DnfDriverDistro::Oracle, ), @@ -2990,19 +3002,21 @@ fn build_driver_install_plan( os_id, version_id, codename, - repo_version_expr, + repo_version, + package_release, dkms, DnfDriverDistro::Rocky, ), ("sles" | "sle", "15.7") => { - sles_driver_plan(os_id, version_id, codename, repo_version_expr, dkms) + sles_driver_plan(os_id, version_id, codename, repo_version, package_release, dkms) } _ => driver_plan_via_id_like( &os_id, &version_id, &id_like, &codename, - &repo_version_expr, + &repo_version, + &package_release, dkms, ) .unwrap_or_else(|| DriverInstallPlan { @@ -3012,7 +3026,7 @@ fn build_driver_install_plan( os_id, version_id, codename, - repo_version_expr, + repo_version, reason: "Linux DKMS driver install is currently planned only for AMD-documented Ubuntu, Debian, RHEL, Oracle Linux, SLES, and Rocky versions; no commands were guessed for this distro.".to_owned(), preflight_checks: Vec::new(), commands: Vec::new(), @@ -3035,7 +3049,8 @@ fn driver_plan_via_id_like( version_id: &str, id_like: &str, codename: &str, - repo_version_expr: &str, + repo_version: &str, + package_release: &str, dkms: bool, ) -> Option { let likes: Vec = id_like @@ -3061,7 +3076,7 @@ fn driver_plan_via_id_like( os_id.to_owned(), version_id.to_owned(), codename, - repo_version_expr.to_owned(), + repo_version.to_owned(), dkms, true, )); @@ -3075,7 +3090,7 @@ fn driver_plan_via_id_like( os_id.to_owned(), version_id.to_owned(), repo_codename.to_owned(), - repo_version_expr.to_owned(), + repo_version.to_owned(), dkms, false, )); @@ -3095,7 +3110,8 @@ fn driver_plan_via_id_like( os_id.to_owned(), version_id.to_owned(), codename.to_owned(), - repo_version_expr.to_owned(), + repo_version.to_owned(), + package_release.to_owned(), dkms, DnfDriverDistro::Generic, )); @@ -3129,7 +3145,7 @@ fn apt_driver_plan( os_id: String, version_id: String, codename: String, - repo_version_expr: String, + repo_version: String, dkms: bool, include_linux_modules_extra: bool, ) -> DriverInstallPlan { @@ -3160,7 +3176,7 @@ fn apt_driver_plan( driver_command( DriverCommandPhase::Prepare, &format!( - "printf '%s\\n' 'deb [arch=amd64 signed-by=/etc/apt/keyrings/rocm.gpg] https://repo.radeon.com/graphics/{repo_version_expr}/ubuntu {codename} main' | sudo tee /etc/apt/sources.list.d/amdgpu.list >/dev/null" + "printf '%s\\n' 'deb [arch=amd64 signed-by=/etc/apt/keyrings/rocm.gpg] https://repo.radeon.com/graphics/{repo_version}/ubuntu {codename} main' | sudo tee /etc/apt/sources.list.d/amdgpu.list >/dev/null" ), ), driver_command( @@ -3190,7 +3206,7 @@ fn apt_driver_plan( os_id, version_id, codename, - repo_version_expr, + repo_version, reason: if dkms { "Plan uses AMD's package-manager DKMS flow and requires explicit approval before execution." } else { @@ -3222,7 +3238,8 @@ fn dnf_driver_plan( os_id: String, version_id: String, codename: String, - repo_version_expr: String, + repo_version: String, + package_release: String, dkms: bool, distro: DnfDriverDistro, ) -> DriverInstallPlan { @@ -3253,7 +3270,7 @@ fn dnf_driver_plan( DriverCommandPhase::Prepare, &format!( "sudo dnf install -y {}", - amdgpu_install_rpm_url(&repo_version_expr, &version_id, distro) + amdgpu_install_rpm_url(&repo_version, &package_release, &version_id, distro) ), )); commands.push(driver_command( @@ -3281,7 +3298,7 @@ fn dnf_driver_plan( os_id, version_id, codename, - repo_version_expr, + repo_version, reason: if dkms { "Plan uses AMD's documented DNF DKMS flow and requires explicit approval before execution." } else { @@ -3316,7 +3333,8 @@ fn sles_driver_plan( os_id: String, version_id: String, codename: String, - repo_version_expr: String, + repo_version: String, + package_release: String, dkms: bool, ) -> DriverInstallPlan { let mut commands = Vec::new(); @@ -3343,7 +3361,7 @@ fn sles_driver_plan( DriverCommandPhase::Prepare, &format!( "sudo zypper --no-gpg-checks install -y {}", - amdgpu_install_sles_rpm_url(&repo_version_expr, &version_id) + amdgpu_install_sles_rpm_url(&repo_version, &package_release, &version_id) ), ), driver_command(DriverCommandPhase::Prepare, "sudo zypper refresh"), @@ -3367,7 +3385,7 @@ fn sles_driver_plan( os_id, version_id, codename, - repo_version_expr, + repo_version, reason: if dkms { "Plan uses AMD's documented SLES DKMS flow and requires explicit approval before execution." } else { @@ -3413,7 +3431,8 @@ fn rhel_kernel_prepare_commands(version_id: &str) -> Vec<&'static str> { } fn amdgpu_install_rpm_url( - repo_version_expr: &str, + repo_version: &str, + package_release: &str, version_id: &str, distro: DnfDriverDistro, ) -> String { @@ -3421,16 +3440,20 @@ fn amdgpu_install_rpm_url( DnfDriverDistro::Rhel => "rhel", DnfDriverDistro::Oracle | DnfDriverDistro::Rocky | DnfDriverDistro::Generic => "el", }; - let repo_version = dnf_repo_version_path(version_id); + let repo_version_path = dnf_repo_version_path(version_id); let el_major = linux_major_version(version_id); format!( - "https://repo.radeon.com/amdgpu-install/{repo_version_expr}/{repo_family}/{repo_version}/amdgpu-install-{repo_version_expr}.${{ROCM_CLI_AMDGPU_PACKAGE_RELEASE:-70204}}-1.el{el_major}.noarch.rpm" + "https://repo.radeon.com/amdgpu-install/{repo_version}/{repo_family}/{repo_version_path}/amdgpu-install-{repo_version}.{package_release}-1.el{el_major}.noarch.rpm" ) } -fn amdgpu_install_sles_rpm_url(repo_version_expr: &str, version_id: &str) -> String { +fn amdgpu_install_sles_rpm_url( + repo_version: &str, + package_release: &str, + version_id: &str, +) -> String { format!( - "https://repo.radeon.com/amdgpu-install/{repo_version_expr}/sle/{version_id}/amdgpu-install-{repo_version_expr}.${{ROCM_CLI_AMDGPU_PACKAGE_RELEASE:-70204}}-1.noarch.rpm" + "https://repo.radeon.com/amdgpu-install/{repo_version}/sle/{version_id}/amdgpu-install-{repo_version}.{package_release}-1.noarch.rpm" ) } @@ -3457,6 +3480,38 @@ fn driver_command(phase: DriverCommandPhase, command: &str) -> DriverPlanCommand } } +/// Resolve a `${VAR:-default}` shell parameter-expansion template to its +/// effective value: the value of `VAR` when it is set and non-empty (matching +/// the shell `:-` semantics), otherwise the literal default. This is resolved +/// once at plan-build time so the concrete value is baked into both the +/// human-readable summary and the commands the plan runs, rather than leaking an +/// unexpanded `${...}` placeholder into user-facing output or depending on the +/// runtime shell — which, for the single-quoted apt `sources.list` line, would +/// never expand it at all. +/// +/// Only a single, flat `${VAR:-default}` template is recognized. Anything else — +/// a bare `${VAR}`, a `${VAR:=x}`/`${VAR-x}` form, or a nested default such as +/// `${A:-${B:-x}}` whose default itself contains `${` — is returned unchanged, so +/// an unresolvable shape degrades to its literal input rather than to a +/// half-resolved string. +fn resolve_shell_default_template(expr: &str) -> String { + let Some(inner) = expr.strip_prefix("${").and_then(|s| s.strip_suffix('}')) else { + return expr.to_owned(); + }; + let Some((var, default)) = inner.split_once(":-") else { + return expr.to_owned(); + }; + if default.contains("${") { + // Nested or embedded templates are beyond this flat matcher; return the + // input untouched rather than emitting a partially resolved string. + return expr.to_owned(); + } + std::env::var(var) + .ok() + .filter(|value| !value.is_empty()) + .unwrap_or_else(|| default.to_owned()) +} + fn render_driver_install_plan(plan: &DriverInstallPlan, yes: bool, dry_run: bool) -> String { let mut output = String::new(); let _ = writeln!(output, "driver install plan"); @@ -3476,7 +3531,7 @@ fn render_driver_install_plan(plan: &DriverInstallPlan, yes: bool, dry_run: bool empty_as_unknown(&plan.version_id) ); let _ = writeln!(output, " codename: {}", empty_as_unknown(&plan.codename)); - let _ = writeln!(output, " repo_version: {}", plan.repo_version_expr); + let _ = writeln!(output, " repo_version: {}", plan.repo_version); let _ = writeln!(output, " reason: {}", plan.reason); if !plan.preflight_checks.is_empty() { let _ = writeln!(output, " preflight_checks:"); @@ -17825,6 +17880,80 @@ mod tests { use super::*; use serde_json::json; + /// Serializes and isolates tests that read or mutate process-global env vars. + /// + /// `std::env::set_var`/`remove_var` are `unsafe` in edition 2024 because + /// concurrent env mutation races with any concurrent `env::var` read anywhere + /// in the process, and `cargo test` runs these functions multi-threaded in one + /// binary. A unique variable name does not make a mutation safe — the hazard is + /// the mutation racing a read, not a name collision. Every test that touches + /// env therefore holds this one process-wide lock, and each mutation is saved + /// and restored on drop so it cannot leak into another test. + struct ScopedTestEnv { + _lock: std::sync::MutexGuard<'static, ()>, + saved: Vec<(String, Option)>, + } + + impl ScopedTestEnv { + fn new() -> Self { + static LOCK: Mutex<()> = Mutex::new(()); + let lock = LOCK + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + Self { + _lock: lock, + saved: Vec::new(), + } + } + + /// Clear the two AMD driver-install override vars for the duration of the + /// test, so a value exported in the developer's or runner's shell cannot + /// leak into a plan and make its resolved `repo_version`/package release + /// disagree with the assertion. + fn with_amd_overrides_cleared() -> Self { + let mut env = Self::new(); + env.clear("ROCM_CLI_AMDGPU_VERSION"); + env.clear("ROCM_CLI_AMDGPU_PACKAGE_RELEASE"); + env + } + + fn save(&mut self, key: &str) { + if !self.saved.iter().any(|(saved_key, _)| saved_key == key) { + self.saved.push((key.to_owned(), std::env::var(key).ok())); + } + } + + #[allow(unsafe_code)] // std::env::set_var is unsafe in edition 2024 + fn set(&mut self, key: &str, value: &str) { + self.save(key); + unsafe { + std::env::set_var(key, value); + } + } + + #[allow(unsafe_code)] // std::env::remove_var is unsafe in edition 2024 + fn clear(&mut self, key: &str) { + self.save(key); + unsafe { + std::env::remove_var(key); + } + } + } + + impl Drop for ScopedTestEnv { + #[allow(unsafe_code)] // std::env::set_var/remove_var are unsafe in edition 2024 + fn drop(&mut self) { + for (key, previous) in self.saved.iter().rev() { + unsafe { + match previous { + Some(value) => std::env::set_var(key, value), + None => std::env::remove_var(key), + } + } + } + } + } + // The previous `daemon_run_argv_targets_rocmd_run_with_automations` unit test // only re-asserted the literals `daemon_run_argv()` returns, so it tested // nothing real. The intended real behavior — that this argv actually drives @@ -23972,6 +24101,7 @@ install therock"; #[test] fn driver_plan_ubuntu_2404_uses_official_dkms_commands() { + let _env = ScopedTestEnv::with_amd_overrides_cleared(); let os_release = r#" ID=ubuntu VERSION_ID="24.04" @@ -24138,6 +24268,7 @@ VERSION_CODENAME=noble #[test] fn driver_plan_default_linux_preflight_has_no_execution_commands() { + let _env = ScopedTestEnv::with_amd_overrides_cleared(); let os_release = r#" ID=ubuntu VERSION_ID="24.04" @@ -24155,8 +24286,85 @@ VERSION_CODENAME=noble assert!(rendered.contains("add --dkms")); } + #[test] + fn resolve_shell_default_template_uses_default_when_env_unset() { + let _env = ScopedTestEnv::new(); + // A made-up variable name that nothing else sets, cleared under the lock, + // isolates the default path. + assert_eq!( + resolve_shell_default_template("${ROCM_CLI_TEST_UNSET_REPO_VERSION:-7.2.4}"), + "7.2.4" + ); + } + + #[test] + fn resolve_shell_default_template_prefers_env_value_when_set() { + let mut env = ScopedTestEnv::new(); + let var = "ROCM_CLI_TEST_REPO_VERSION_OVERRIDE"; + env.set(var, "9.9.9"); + assert_eq!( + resolve_shell_default_template(&format!("${{{var}:-7.2.4}}")), + "9.9.9" + ); + } + + #[test] + fn resolve_shell_default_template_treats_empty_env_as_unset() { + let mut env = ScopedTestEnv::new(); + let var = "ROCM_CLI_TEST_REPO_VERSION_EMPTY"; + env.set(var, ""); + assert_eq!( + resolve_shell_default_template(&format!("${{{var}:-7.2.4}}")), + "7.2.4" + ); + } + + #[test] + fn resolve_shell_default_template_passes_through_non_template() { + assert_eq!(resolve_shell_default_template("7.2.4"), "7.2.4"); + } + + #[test] + fn resolve_shell_default_template_leaves_bare_var_untouched() { + let _env = ScopedTestEnv::new(); + // No `:-default`, so there is nothing to resolve to; the input must pass + // through unchanged rather than being partially rewritten. + assert_eq!( + resolve_shell_default_template("${ROCM_CLI_TEST_UNSET_REPO_VERSION}"), + "${ROCM_CLI_TEST_UNSET_REPO_VERSION}" + ); + } + + #[test] + fn resolve_shell_default_template_leaves_nested_default_untouched() { + let _env = ScopedTestEnv::new(); + // A nested default is beyond the flat matcher; returning the literal + // input keeps a `${B:-x}` fragment from leaking as a "resolved" value. + assert_eq!( + resolve_shell_default_template("${ROCM_CLI_TEST_UNSET_A:-${ROCM_CLI_TEST_UNSET_B:-x}}"), + "${ROCM_CLI_TEST_UNSET_A:-${ROCM_CLI_TEST_UNSET_B:-x}}" + ); + } + + #[test] + fn driver_plan_dry_run_repo_version_line_is_resolved() { + let _env = ScopedTestEnv::with_amd_overrides_cleared(); + // Regression for the dry-run output leaking the raw shell placeholder on + // the `repo_version:` line instead of the effective version. + let os_release = r#" +ID=rhel +VERSION_ID="9.7" +"#; + let plan = build_driver_install_plan(&test_examine("linux", false), os_release, true); + let rendered = render_driver_install_plan(&plan, false, true); + + assert!(rendered.contains("repo_version: 7.2.4")); + assert!(!rendered.contains("repo_version: ${ROCM_CLI_AMDGPU_VERSION:-7.2.4}")); + } + #[test] fn driver_plan_debian_12_omits_linux_modules_extra() { + let _env = ScopedTestEnv::with_amd_overrides_cleared(); let os_release = r#" ID=debian VERSION_ID="12" @@ -24175,6 +24383,7 @@ VERSION_CODENAME=bookworm #[test] fn driver_plan_rhel_97_uses_documented_dnf_commands() { + let _env = ScopedTestEnv::with_amd_overrides_cleared(); let os_release = r#" ID=rhel VERSION_ID="9.7" @@ -24189,16 +24398,15 @@ VERSION_ID="9.7" assert!(rendered.contains("kernel-headers-$(uname -r)")); assert!(rendered.contains("kernel-devel-$(uname -r)")); assert!(rendered.contains("kernel-devel-matched-$(uname -r)")); - assert!(rendered.contains( - "repo.radeon.com/amdgpu-install/${ROCM_CLI_AMDGPU_VERSION:-7.2.4}/rhel/9.7/" - )); - assert!(rendered.contains("amdgpu-install-${ROCM_CLI_AMDGPU_VERSION:-7.2.4}.${ROCM_CLI_AMDGPU_PACKAGE_RELEASE:-70204}-1.el9.noarch.rpm")); + assert!(rendered.contains("repo.radeon.com/amdgpu-install/7.2.4/rhel/9.7/")); + assert!(rendered.contains("amdgpu-install-7.2.4.70204-1.el9.noarch.rpm")); assert!(rendered.contains("Execute: sudo dnf install -y amdgpu-dkms")); assert!(rendered.contains("approval: required")); } #[test] fn driver_plan_oracle_linux_101_uses_el_10_uek_flow() { + let _env = ScopedTestEnv::with_amd_overrides_cleared(); let os_release = r#" ID=ol VERSION_ID="10.1" @@ -24209,17 +24417,14 @@ VERSION_ID="10.1" assert!(plan.supported); assert!(rendered.contains("approval: not required")); assert!(rendered.contains("kernel-uek-devel-$(uname -r)")); - assert!( - rendered.contains( - "repo.radeon.com/amdgpu-install/${ROCM_CLI_AMDGPU_VERSION:-7.2.4}/el/10/" - ) - ); - assert!(rendered.contains("amdgpu-install-${ROCM_CLI_AMDGPU_VERSION:-7.2.4}.${ROCM_CLI_AMDGPU_PACKAGE_RELEASE:-70204}-1.el10.noarch.rpm")); + assert!(rendered.contains("repo.radeon.com/amdgpu-install/7.2.4/el/10/")); + assert!(rendered.contains("amdgpu-install-7.2.4.70204-1.el10.noarch.rpm")); assert!(rendered.contains("dry run only")); } #[test] fn driver_plan_rocky_97_uses_el_dnf_flow() { + let _env = ScopedTestEnv::with_amd_overrides_cleared(); let os_release = r#" ID=rocky VERSION_ID="9.7" @@ -24232,16 +24437,13 @@ VERSION_ID="9.7" rendered .contains("sudo dnf install -y kernel-headers kernel-devel kernel-devel-matched") ); - assert!( - rendered.contains( - "repo.radeon.com/amdgpu-install/${ROCM_CLI_AMDGPU_VERSION:-7.2.4}/el/9.7/" - ) - ); + assert!(rendered.contains("repo.radeon.com/amdgpu-install/7.2.4/el/9.7/")); assert!(rendered.contains("Execute: sudo dnf install -y amdgpu-dkms")); } #[test] fn driver_plan_rocky_94_uses_el_dnf_flow() { + let _env = ScopedTestEnv::with_amd_overrides_cleared(); // Rocky 9.x point releases must resolve like RHEL 9.x, not just 9.7. let os_release = r#" ID=rocky @@ -24252,17 +24454,14 @@ VERSION_ID="9.4" assert!(plan.supported); assert!(plan.mutating); - assert!( - rendered.contains( - "repo.radeon.com/amdgpu-install/${ROCM_CLI_AMDGPU_VERSION:-7.2.4}/el/9.4/" - ) - ); - assert!(rendered.contains("amdgpu-install-${ROCM_CLI_AMDGPU_VERSION:-7.2.4}.${ROCM_CLI_AMDGPU_PACKAGE_RELEASE:-70204}-1.el9.noarch.rpm")); + assert!(rendered.contains("repo.radeon.com/amdgpu-install/7.2.4/el/9.4/")); + assert!(rendered.contains("amdgpu-install-7.2.4.70204-1.el9.noarch.rpm")); assert!(rendered.contains("Execute: sudo dnf install -y amdgpu-dkms")); } #[test] fn driver_plan_rocky_8_and_10_remain_unsupported() { + let _env = ScopedTestEnv::with_amd_overrides_cleared(); // AMD documents Rocky Linux 9 only; keep the driver matrix scoped to 9.x. for version in ["8.10", "10.0"] { let os_release = format!("\nID=rocky\nVERSION_ID=\"{version}\"\n"); @@ -24278,6 +24477,7 @@ VERSION_ID="9.4" #[test] fn driver_plan_debian_uses_intended_ubuntu_suite() { + let _env = ScopedTestEnv::with_amd_overrides_cleared(); // AMD's documented Debian install deliberately serves Debian from the // Ubuntu-suite graphics tree (Debian 12 -> jammy). Lock that in and // ensure the plan explains the mapping is intentional. @@ -24291,9 +24491,7 @@ VERSION_CODENAME=bookworm assert!(plan.supported); assert_eq!(plan.codename, "jammy"); - assert!(rendered.contains( - "https://repo.radeon.com/graphics/${ROCM_CLI_AMDGPU_VERSION:-7.2.4}/ubuntu jammy main" - )); + assert!(rendered.contains("https://repo.radeon.com/graphics/7.2.4/ubuntu jammy main")); assert!( plan.reason .contains("intentionally uses AMD's Ubuntu-suite repository") @@ -24302,6 +24500,7 @@ VERSION_CODENAME=bookworm #[test] fn driver_plan_sles_157_uses_documented_zypper_commands() { + let _env = ScopedTestEnv::with_amd_overrides_cleared(); let os_release = r#" ID=sles VERSION_ID="15.7" @@ -24314,9 +24513,7 @@ VERSION_ID="15.7" assert!(rendered.contains("SUSEConnect")); assert!(rendered.contains("sle-module-desktop-applications/15.7/x86_64")); assert!(rendered.contains("sudo zypper install -y kernel-default-devel")); - assert!(rendered.contains( - "repo.radeon.com/amdgpu-install/${ROCM_CLI_AMDGPU_VERSION:-7.2.4}/sle/15.7/" - )); + assert!(rendered.contains("repo.radeon.com/amdgpu-install/7.2.4/sle/15.7/")); assert!(rendered.contains("sudo zypper --no-gpg-checks install -y")); assert!(rendered.contains("Execute: sudo zypper install -y amdgpu-dkms")); assert!(rendered.contains("approval: required")); @@ -24324,6 +24521,7 @@ VERSION_ID="15.7" #[test] fn driver_plan_unsupported_linux_is_non_mutating() { + let _env = ScopedTestEnv::with_amd_overrides_cleared(); let os_release = r#" ID=fedora VERSION_ID="41" @@ -24341,6 +24539,7 @@ VERSION_ID="41" #[test] fn windows_install_driver_is_validate_only() { + let _env = ScopedTestEnv::with_amd_overrides_cleared(); let plan = build_driver_install_plan(&test_examine("windows", false), "", true); let rendered = render_driver_install_plan(&plan, false, true); @@ -24357,6 +24556,7 @@ VERSION_ID="41" #[test] fn wsl_install_driver_uses_rocdxg_guidance_without_dkms() { + let _env = ScopedTestEnv::with_amd_overrides_cleared(); let plan = build_driver_install_plan(&test_examine("linux", true), "", true); let rendered = render_driver_install_plan(&plan, false, false); @@ -24375,6 +24575,7 @@ VERSION_ID="41" #[test] fn driver_plan_ubuntu_derivative_via_id_like_matches_ubuntu_plan() { + let _env = ScopedTestEnv::with_amd_overrides_cleared(); // Pop!_OS reports its own ID but reuses Ubuntu's version + repositories. let os_release = r#" ID=pop @@ -24390,14 +24591,13 @@ ID_LIKE="ubuntu debian" assert_eq!(plan.policy, "linux_official_amd_dkms_wrapper"); // Ubuntu-family derivatives ship the Ubuntu kernel, so linux-modules-extra applies. assert!(rendered.contains("linux-modules-extra-$(uname -r)")); - assert!(rendered.contains( - "https://repo.radeon.com/graphics/${ROCM_CLI_AMDGPU_VERSION:-7.2.4}/ubuntu jammy main" - )); + assert!(rendered.contains("https://repo.radeon.com/graphics/7.2.4/ubuntu jammy main")); assert!(rendered.contains("Execute: sudo apt-get install -y amdgpu-dkms")); } #[test] fn driver_plan_debian_derivative_via_id_like_matches_debian_plan() { + let _env = ScopedTestEnv::with_amd_overrides_cleared(); // A Debian derivative (e.g. LMDE) that shares Debian's version scheme. let os_release = r#" ID=lmde @@ -24409,15 +24609,14 @@ ID_LIKE=debian assert!(plan.supported); // Debian-family maps to the Ubuntu jammy repo and omits linux-modules-extra. - assert!(rendered.contains( - "https://repo.radeon.com/graphics/${ROCM_CLI_AMDGPU_VERSION:-7.2.4}/ubuntu jammy main" - )); + assert!(rendered.contains("https://repo.radeon.com/graphics/7.2.4/ubuntu jammy main")); assert!(!rendered.contains("linux-modules-extra-$(uname -r)")); assert!(rendered.contains("amdgpu-dkms")); } #[test] fn driver_plan_almalinux_via_id_like_uses_el_9_flow() { + let _env = ScopedTestEnv::with_amd_overrides_cleared(); // AlmaLinux is a RHEL rebuild: standard kernel, served from the el/ path. let os_release = r#" ID=almalinux @@ -24431,13 +24630,9 @@ ID_LIKE="rhel centos fedora" assert!(plan.mutating); assert_eq!(plan.policy, "linux_official_amd_dkms_wrapper"); // EL rebuilds use the vendor-neutral el/ repo path, not rhel/. - assert!( - rendered.contains( - "repo.radeon.com/amdgpu-install/${ROCM_CLI_AMDGPU_VERSION:-7.2.4}/el/9.6/" - ) - ); + assert!(rendered.contains("repo.radeon.com/amdgpu-install/7.2.4/el/9.6/")); assert!(!rendered.contains("/rhel/9.6/")); - assert!(rendered.contains("amdgpu-install-${ROCM_CLI_AMDGPU_VERSION:-7.2.4}.${ROCM_CLI_AMDGPU_PACKAGE_RELEASE:-70204}-1.el9.noarch.rpm")); + assert!(rendered.contains("amdgpu-install-7.2.4.70204-1.el9.noarch.rpm")); // el9 uses the version-aware standard-kernel prepare commands. assert!(rendered.contains("kernel-devel-matched-$(uname -r)")); assert!(rendered.contains("Execute: sudo dnf install -y amdgpu-dkms")); @@ -24445,6 +24640,7 @@ ID_LIKE="rhel centos fedora" #[test] fn driver_plan_almalinux_8_via_id_like_uses_el_major_path() { + let _env = ScopedTestEnv::with_amd_overrides_cleared(); let os_release = r#" ID=almalinux VERSION_ID="8.10" @@ -24455,10 +24651,7 @@ ID_LIKE="rhel centos fedora" assert!(plan.supported); // EL 8 is served from the major-version path (el/8), matching AMD docs. - assert!( - rendered - .contains("repo.radeon.com/amdgpu-install/${ROCM_CLI_AMDGPU_VERSION:-7.2.4}/el/8/") - ); + assert!(rendered.contains("repo.radeon.com/amdgpu-install/7.2.4/el/8/")); assert!(rendered.contains("-1.el8.noarch.rpm")); // el8 has no kernel-devel-matched package. assert!(!rendered.contains("kernel-devel-matched")); @@ -24467,6 +24660,7 @@ ID_LIKE="rhel centos fedora" #[test] fn driver_plan_id_like_with_unsupported_version_stays_unsupported() { + let _env = ScopedTestEnv::with_amd_overrides_cleared(); // A Debian-family derivative whose VERSION_ID does not align with any // AMD-documented Debian version must not fabricate a plan. let os_release = r#" @@ -24485,6 +24679,7 @@ ID_LIKE=debian #[test] fn driver_plan_exact_id_takes_precedence_over_id_like() { + let _env = ScopedTestEnv::with_amd_overrides_cleared(); // An exact RHEL match must keep the rhel/ path even though ID_LIKE=fedora. let os_release = r#" ID=rhel @@ -24495,14 +24690,13 @@ ID_LIKE=fedora let rendered = render_driver_install_plan(&plan, false, false); assert!(plan.supported); - assert!(rendered.contains( - "repo.radeon.com/amdgpu-install/${ROCM_CLI_AMDGPU_VERSION:-7.2.4}/rhel/9.7/" - )); + assert!(rendered.contains("repo.radeon.com/amdgpu-install/7.2.4/rhel/9.7/")); assert!(!rendered.contains("/el/9.7/")); } #[test] fn driver_plan_oracle_linux_off_arm_version_stays_unsupported() { + let _env = ScopedTestEnv::with_amd_overrides_cleared(); // Oracle Linux reports `ID_LIKE=fedora` (not rhel) and boots UEK. An OL // version outside the exact `ol` arm must NOT be captured by the EL // fallback, which would emit non-UEK kernel commands that cannot install. @@ -24523,6 +24717,7 @@ ID_LIKE=fedora #[test] fn driver_plan_opensuse_leap_stays_unsupported() { + let _env = ScopedTestEnv::with_amd_overrides_cleared(); // openSUSE Leap shares SLES's version scheme but has no SUSEConnect/SCC // entitlement, so it must not be matched to the SLES plan. let os_release = r#" diff --git a/tests/e2e-cucumber/features/examine.feature b/tests/e2e-cucumber/features/examine.feature index 9410d4fd1b..f4e2a9a7d1 100644 --- a/tests/e2e-cucumber/features/examine.feature +++ b/tests/e2e-cucumber/features/examine.feature @@ -111,3 +111,14 @@ Feature: GPU detection and system inspection When the user inspects the system Then the inspection reports Linux as the operating system And the inspection reports that the host is WSL + + # The dry-run plan used to print the raw `${ROCM_CLI_AMDGPU_VERSION:-...}` shell + # placeholder on its `repo_version:` line instead of the effective version, so + # the preview a user reviews before approving disagreed with what the install + # would actually pull. The plan is rendered on every Linux host regardless of + # GPU (the driver is not yet installed when you preview it), so the mock lane + # pins this without hardware. + @id:install-driver-dry-run-resolves-repo-version @requires-os:linux + Scenario: 12 - The driver install dry-run shows the effective repo version + When the user previews the driver install plan + Then the plan's repo version is a concrete version, not a shell placeholder diff --git a/tests/e2e-cucumber/tests/e2e/examine_steps.rs b/tests/e2e-cucumber/tests/e2e/examine_steps.rs index 38299139ca..d232b352d2 100644 --- a/tests/e2e-cucumber/tests/e2e/examine_steps.rs +++ b/tests/e2e-cucumber/tests/e2e/examine_steps.rs @@ -63,6 +63,15 @@ async fn user_asks_help(world: &mut E2eWorld) { world.cli_output = Some(stdout); } +#[when("the user previews the driver install plan")] +async fn user_previews_driver_install_plan(world: &mut E2eWorld) { + // `--dry-run` renders the plan and returns before touching the system, so + // this is safe to run on any Linux host including the no-GPU mock lane. + let (stdout, _, rc) = crate::run_rocm(world, &["install", "driver", "--dry-run"]); + world.cli_output = Some(stdout); + world.cli_rc = Some(rc); +} + #[then("a version string is returned")] async fn assert_version_returned(world: &mut E2eWorld) { let output = world.cli_output.as_ref().expect("no command was run"); @@ -72,6 +81,24 @@ async fn assert_version_returned(world: &mut E2eWorld) { ); } +#[then("the plan's repo version is a concrete version, not a shell placeholder")] +async fn assert_driver_plan_repo_version_resolved(world: &mut E2eWorld) { + let output = world.cli_output.as_ref().expect("no command was run"); + let repo_version = field_value(output, "repo_version") + .unwrap_or_else(|| panic!("no repo_version line in driver install plan:\n{output}")); + assert!( + !repo_version.contains("${"), + "repo_version still shows an unresolved shell placeholder: {repo_version:?}\n{output}" + ); + assert!( + repo_version + .chars() + .next() + .is_some_and(|first| first.is_ascii_digit()), + "repo_version is not a concrete version string: {repo_version:?}\n{output}" + ); +} + #[then("the subcommands are listed in alphabetical order")] async fn assert_subcommands_alphabetical(world: &mut E2eWorld) { let output = world.cli_output.as_ref().expect("no help output"); From 7fa0f8233e8cf554cd7436aab123b61990de5fdc Mon Sep 17 00:00:00 2001 From: Roman Date: Thu, 27 Aug 2026 12:45:08 +0000 Subject: [PATCH 15/23] fix(cli): accept negative float flag values in space form (EAI-8243) (#306) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix(cli): accept negative float flag values in space form The space form of negative-number float flags (e.g. `serve --temperature -1`) was rejected by clap with a confusing "unexpected argument '-1'" error, because clap parsed the leading-dash token as a flag rather than a value. Only the equals form (`--temperature=-1`) reached the range validator that reports the valid range. Enable `allow_negative_numbers` on the affected float flags (`--temperature` and `--top-p` on both `chat` and `serve`) so the space form reaches the value parser and both forms validate identically, surfacing a clear range-validation message instead of an unexpected argument error. Add regression tests covering both the space and equals forms for `--temperature` and `--top-p` on `chat` and `serve`. Signed-off-by: Roman Sirokov * fix(cli): extend negative-value fix to --max-tokens and add serve scenario The negative-number-as-flag gotcha is a tokenizer-level ambiguity that fires before any value parser runs, so it is not float-specific: `--max-tokens -1` (space form) on both chat and serve was still rejected as an unexpected argument while `--max-tokens=-1` reached parse_positive_u32. Add allow_negative_numbers to --max-tokens on both commands so the space form reaches the value parser and reports a clear error, matching the --temperature/--top-p behavior fixed here. Per AGENTS.md §3, the user-observable stderr change is now covered by an ungated Gherkin scenario, @id:serve-negative-temperature-rejected, in model_serving.feature. The value parser runs inside argument parsing before engine selection or GPU pre-flight, so the scenario needs no GPU and no engine and gates every PR (mirrors the ungated @id:fix-position-argument-rejected precedent). Also: reword the shared rationale comment to cover all three numeric value flags (and note clap only treats a token as negative when the whole remainder parses as one, so missing/malformed values still error at parse time), give the chat regression test a symmetric comment, and extend both space-form regression tests with the --max-tokens cases. Signed-off-by: Roman Sirokov * test(e2e): number the negative-temperature serve scenario The new scenario was the only unnumbered title in model_serving.feature; every other scenario uses 'Scenario: - '. Assign the next free number (15) to keep the file consistent. No behavior change. Signed-off-by: Roman Sirokov <roman.sirokov@amd.com> --------- Signed-off-by: Roman Sirokov <roman.sirokov@amd.com> Signed-off-by: pmoutsias-amd <peter.moutsias@amd.com> --- apps/rocm/src/main.rs | 75 +++++++++++++++++-- .../features/model_serving.feature | 10 +++ tests/e2e-cucumber/tests/e2e/serving_steps.rs | 26 +++++++ 3 files changed, 105 insertions(+), 6 deletions(-) diff --git a/apps/rocm/src/main.rs b/apps/rocm/src/main.rs index 4cf40c2d6a..31f3ebe10d 100644 --- a/apps/rocm/src/main.rs +++ b/apps/rocm/src/main.rs @@ -257,13 +257,19 @@ echo \"Summarize this\" | rocm chat --provider anthropic")] #[arg(long)] prompt: Option<String>, /// Sampling temperature to send with the request (>= 0.0). Higher is more random. - #[arg(long, value_parser = parse_non_negative_temperature)] + // `allow_negative_numbers` on the numeric value flags below (`--temperature`, + // `--top-p`, `--max-tokens`) lets their space form (e.g. `--temperature -1`) reach + // the value parser for a clear range error, instead of clap rejecting the token as + // an unexpected argument. clap only treats a `-…` token as a number when the whole + // remainder parses as one, so a missing value (`--temperature --top-p 0.5`) or a + // malformed token (`--temperature -0.5abc`) still errors at parse time. + #[arg(long, allow_negative_numbers = true, value_parser = parse_non_negative_temperature)] temperature: Option<f32>, /// Nucleus sampling probability to send with the request (0.0-1.0). - #[arg(long = "top-p", value_parser = parse_unit_interval)] + #[arg(long = "top-p", allow_negative_numbers = true, value_parser = parse_unit_interval)] top_p: Option<f32>, /// Maximum number of tokens to generate in the response. - #[arg(long = "max-tokens", value_parser = parse_positive_u32)] + #[arg(long = "max-tokens", allow_negative_numbers = true, value_parser = parse_positive_u32)] max_tokens: Option<u32>, #[arg( long, @@ -398,13 +404,19 @@ rocm serve qwen --verbose --device gpu_required")] #[arg(long, value_name = "FRACTION", allow_hyphen_values = true)] gpu_memory_utilization: Option<String>, /// Default sampling temperature for generation (>= 0.0). - #[arg(long, value_parser = parse_non_negative_temperature)] + // `allow_negative_numbers` on the numeric value flags below (`--temperature`, + // `--top-p`, `--max-tokens`) lets their space form (e.g. `--temperature -1`) reach + // the value parser for a clear range error, instead of clap rejecting the token as + // an unexpected argument. clap only treats a `-…` token as a number when the whole + // remainder parses as one, so a missing value (`--temperature --top-p 0.5`) or a + // malformed token (`--temperature -0.5abc`) still errors at parse time. + #[arg(long, allow_negative_numbers = true, value_parser = parse_non_negative_temperature)] temperature: Option<f32>, /// Default nucleus sampling probability for generation (0.0-1.0). - #[arg(long = "top-p", value_parser = parse_unit_interval)] + #[arg(long = "top-p", allow_negative_numbers = true, value_parser = parse_unit_interval)] top_p: Option<f32>, /// Default maximum number of tokens to generate per response. - #[arg(long = "max-tokens", value_parser = parse_positive_u32)] + #[arg(long = "max-tokens", allow_negative_numbers = true, value_parser = parse_positive_u32)] max_tokens: Option<u32>, /// API key that clients must present to a public (non-loopback) endpoint. /// When binding a public interface and this is omitted, a strong key is @@ -18533,6 +18545,57 @@ mod tests { ); } + #[test] + fn serve_negative_sampling_space_form_reaches_range_validator() { + // The space form (`--temperature -1`) must reach the value parser and + // report a value error, not be rejected by clap as an unexpected argument. + // This is the classic negative-number-as-flag gotcha; `allow_negative_numbers` + // makes both forms validate identically. `--max-tokens` shares the gotcha even + // though it is a positive integer: the ambiguity is at the tokenizer, before the + // value parser runs, so the space form must reach `parse_positive_u32` too. + for args in [ + &["--temperature", "-1"][..], + &["--temperature=-1"][..], + &["--top-p", "-0.5"][..], + &["--top-p=-0.5"][..], + &["--max-tokens", "-1"][..], + &["--max-tokens=-1"][..], + ] { + assert_eq!( + parse_serve(args) + .expect_err("negative sampling value is rejected") + .kind(), + clap::error::ErrorKind::ValueValidation, + "expected range validation for {args:?}" + ); + } + } + + #[test] + fn chat_negative_sampling_space_form_reaches_range_validator() { + // Mirrors `serve_negative_sampling_space_form_reaches_range_validator`: the + // same numeric value flags carry `allow_negative_numbers` on `chat`, so the + // space form reaches the value parser instead of clap's unexpected-argument path. + for args in [ + &["--temperature", "-1"][..], + &["--temperature=-1"][..], + &["--top-p", "-0.5"][..], + &["--top-p=-0.5"][..], + &["--max-tokens", "-1"][..], + &["--max-tokens=-1"][..], + ] { + let mut argv = vec!["rocm", "chat", "--prompt", "hi"]; + argv.extend_from_slice(args); + assert_eq!( + Cli::try_parse_from(argv) + .expect_err("negative sampling value is rejected") + .kind(), + clap::error::ErrorKind::ValueValidation, + "expected range validation for {args:?}" + ); + } + } + #[test] fn chat_parses_generation_flags() { let cli = Cli::try_parse_from([ diff --git a/tests/e2e-cucumber/features/model_serving.feature b/tests/e2e-cucumber/features/model_serving.feature index 219ffc137c..222d8c90b9 100644 --- a/tests/e2e-cucumber/features/model_serving.feature +++ b/tests/e2e-cucumber/features/model_serving.feature @@ -135,6 +135,16 @@ Feature: Model serving Then serving is refused before any engine starts And the user is told no AMD GPU was detected + # Parse-time refusal: `--temperature -1` (space form) must reach the value + # parser and report the range error, not clap's "unexpected argument". The + # check runs inside argument parsing, before engine selection or any GPU + # pre-flight, so it needs no GPU and no engine and gates every PR (ungated). + @id:serve-negative-temperature-rejected + Scenario: 15 - Serving with a temperature below zero is refused with a clear reason + When the user serves a model with a negative sampling temperature + Then serving is refused before any engine starts + And the CLI explains that temperature cannot be negative + # The masked-device path: on a real GPU host where every device is hidden, the # GPU-required serve must treat it as "no GPU" and refuse, not fall back. Runs on # GPU hardware (Strix Halo / Instinct). diff --git a/tests/e2e-cucumber/tests/e2e/serving_steps.rs b/tests/e2e-cucumber/tests/e2e/serving_steps.rs index fdcb49e020..9db8e959ef 100644 --- a/tests/e2e-cucumber/tests/e2e/serving_steps.rs +++ b/tests/e2e-cucumber/tests/e2e/serving_steps.rs @@ -832,6 +832,21 @@ async fn user_serves_gpu_required(world: &mut E2eWorld) { world.cli_rc = Some(rc); } +/// Serve with a negative `--temperature` in the space form. The value parser +/// runs inside argument parsing, before engine selection or any GPU pre-flight, +/// so a bogus model name never gets that far — the refusal needs no GPU and no +/// engine, which is what lets this scenario gate every PR. +#[when("the user serves a model with a negative sampling temperature")] +async fn user_serves_negative_temperature(world: &mut E2eWorld) { + let (stdout, stderr, rc) = crate::run_rocm( + world, + &["serve", "sampling-check-model", "--temperature", "-1"], + ); + world.cli_output = Some(stdout); + world.cli_stderr = Some(stderr); + world.cli_rc = Some(rc); +} + /// Serve with every GPU hidden from the CLI and engine via an empty /// `HIP_VISIBLE_DEVICES`, so a GPU host presents as having no usable device. #[when("the user serves a model with every GPU masked from view")] @@ -1014,6 +1029,17 @@ async fn assert_no_gpu_message(world: &mut E2eWorld) { ); } +#[then("the CLI explains that temperature cannot be negative")] +async fn assert_negative_temperature_message(world: &mut E2eWorld) { + let output = serve_output(world); + assert!( + output + .to_lowercase() + .contains("temperature must be a finite value >= 0.0"), + "expected a temperature range explanation, got:\n{output}" + ); +} + #[then("the user is told that GPU index is unavailable")] async fn assert_absent_index_message(world: &mut E2eWorld) { let output = serve_output(world).to_lowercase(); From f4618e9d80afa4543a8ae237eacc3eaf957e59bf Mon Sep 17 00:00:00 2001 From: Roman <roman.sirokov@amd.com> Date: Thu, 27 Aug 2026 12:45:15 +0000 Subject: [PATCH 16/23] fix(serve): validate --device via clap possible values (exit 2) (#304) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix(serve): validate --device via clap possible values (exit 2) Model `rocm serve --device` as a clap `ValueEnum` (`DevicePolicyArg`) so an invalid value is rejected by clap's usage validation (exit code 2) with the valid choices listed, instead of parsing as a free-form string and failing later in application logic with a generic "unsupported device policy" error (exit code 1). This makes `--device` consistent with every other enum-style argument in the CLI (`--engine`, config set-*, engines install, etc.), all of which use clap possible values. The historical aliases (`auto`/`gpu` for gpu_required, `cpu` for cpu_only) stay accepted for backward compatibility but are hidden from the advertised list, and the intentional cpu_only app-level rejection (exit 1 with a GPU-required message) is preserved. Adds regression tests asserting invalid `--device` is a clap InvalidValue usage error and that every value and alias still parses, and reworks the device possible-values sync test to read clap's structural possible values rather than a hand-written doc string. Signed-off-by: Roman Sirokov <roman.sirokov@amd.com> * fix(serve): correct --device help text and hide cpu_only choice The per-variant ValueEnum doc comments render in `serve --help`, shell completions, and the invalid-value suggestion list. `gpu_preferred` previously read "Prefer a ROCm GPU when one is available", which is misleading: parse_device_policy maps gpu_preferred to GpuRequired and rocm serve has no CPU fallback path. Reword it to state it behaves identically to gpu_required. cpu_only is always rejected (rocm serve requires GPU execution), so hide it from the advertised choices with #[value(hide = true)] while keeping it parseable so the deliberate rejection message is preserved. Add device_policy_arg_maps_through_parse_device_policy to guard the DevicePolicyArg::as_policy_str -> parse_device_policy mapping (gpu_required and gpu_preferred resolve to GpuRequired; cpu_only is rejected), closing the previously untested conversion path. Signed-off-by: Roman Sirokov <roman.sirokov@amd.com> * docs(serve): stop advertising hidden cpu_only device value Hiding `cpu_only` from `--device` help/completion left three surfaces still presenting it as a real choice. Align them with what the CLI now advertises: - README serve synopsis dropped `cpu_only` from the --device value list. - Removed the '--gpu is ignored with --device cpu_only (the model runs on CPU)' sentence: cpu_only is rejected (exit 1), never runs on CPU, so the line advertised a fallback path that does not exist (AGENTS.md §6). - Corrected the drifted test comment to note cpu_only is #[value(hide = true)] and that serve_device_help_lists_match_device_policy_names compares against the full DevicePolicy set (hidden entries included). Doc/comment only; no behavior change. Signed-off-by: Roman Sirokov <roman.sirokov@amd.com> --------- Signed-off-by: Roman Sirokov <roman.sirokov@amd.com> Signed-off-by: pmoutsias-amd <peter.moutsias@amd.com> --- README.md | 5 +- apps/rocm/src/main.rs | 162 ++++++++++++++++++++++++++++++------------ 2 files changed, 119 insertions(+), 48 deletions(-) diff --git a/README.md b/README.md index 834ac314b8..b2e3c6690e 100644 --- a/README.md +++ b/README.md @@ -302,7 +302,7 @@ Start a local OpenAI-compatible model server: ``` rocm serve <model> [--engine lemonade|vllm] - [--device gpu_required|gpu_preferred|cpu_only] + [--device gpu_required|gpu_preferred] [--gpu auto|<index>] [--runtime-id KEY | --env-id ID] [--host HOST] [--port PORT] @@ -363,8 +363,7 @@ not already used by another rocm-cli server (managed or foreground), falling back to the GPU with the most free memory. Pass a single index (`--gpu 1`) to pin a specific device. The selected GPU is exposed to the engine via `HIP_VISIBLE_DEVICES`. Serving one -model across multiple GPUs is not supported. `--gpu` is ignored with -`--device cpu_only` (the model runs on CPU). Because selection uses the +model across multiple GPUs is not supported. Because selection uses the `amd-smi` ordinal but is applied via `HIP_VISIBLE_DEVICES`, rocm-cli warns when `ROCR_VISIBLE_DEVICES` is set, since the two orderings can diverge. diff --git a/apps/rocm/src/main.rs b/apps/rocm/src/main.rs index 31f3ebe10d..375e851ca8 100644 --- a/apps/rocm/src/main.rs +++ b/apps/rocm/src/main.rs @@ -352,9 +352,9 @@ rocm serve qwen --verbose --device gpu_required")] /// Engine to use. #[arg(long, value_parser = SUPPORTED_ENGINES)] engine: Option<String>, - /// Device policy [possible values: gpu_required, gpu_preferred, cpu_only]. + /// Device policy. #[arg(long)] - device: Option<String>, + device: Option<DevicePolicyArg>, /// GPU device to serve on: `auto` (default; first free GPU) or a single /// index like `1`. #[arg(long, value_name = "INDEX|auto")] @@ -990,6 +990,45 @@ enum TelemetryModeArg { Off, } +/// User-facing `rocm serve --device` policy. +/// +/// Modeled as a clap `ValueEnum` so invalid input is rejected by clap's usage +/// validation (exit code 2, with the valid choices listed) rather than a generic +/// application error (exit code 1) — matching how every other enum-style CLI +/// argument behaves. The historical aliases (`auto`/`gpu` for `gpu_required`, +/// `cpu` for `cpu_only`) stay accepted for backward compatibility but are hidden +/// from the advertised list. +#[derive(Clone, Copy, Debug, Eq, PartialEq, ValueEnum)] +#[value(rename_all = "snake_case")] +enum DevicePolicyArg { + /// Require a ROCm GPU; fail if none is usable (the default). + #[value(alias = "auto", alias = "gpu")] + GpuRequired, + /// Accepted for compatibility; behaves identically to `gpu_required` + /// (rocm serve has no CPU fallback path). + GpuPreferred, + /// Accepted but always rejected: rocm serve has no CPU fallback path, so + /// this exits with an error. Hidden from `--help`, shell completions and the + /// invalid-value suggestion list so it is never advertised as usable, while + /// still parsing so the deliberate rejection message is preserved. + #[value(alias = "cpu", hide = true)] + CpuOnly, +} + +impl DevicePolicyArg { + /// Canonical policy string understood by [`parse_device_policy`]. Routing the + /// parsed variant back through the string keeps `parse_device_policy` the + /// single choke point that enforces the `cpu_only` rejection across every + /// caller, rather than adding a second conversion path that could bypass it. + const fn as_policy_str(self) -> &'static str { + match self { + Self::GpuRequired => "gpu_required", + Self::GpuPreferred => "gpu_preferred", + Self::CpuOnly => "cpu_only", + } + } +} + #[derive(Clone, Copy, Debug, Eq, PartialEq, ValueEnum)] #[value(rename_all = "snake_case")] enum SandboxToolArg { @@ -4771,7 +4810,7 @@ fn engine_recipe_enables_tool_choice(hint: Option<&EngineRecipeHint>) -> bool { struct ServeArgs { model: String, engine: Option<String>, - device: Option<String>, + device: Option<DevicePolicyArg>, gpu: Option<String>, runtime_id: Option<String>, env_id: Option<String>, @@ -4910,7 +4949,7 @@ fn serve(args: ServeArgs) -> Result<()> { } else { None }; - let device_policy = parse_device_policy(device.as_deref())?; + let device_policy = parse_device_policy(device.as_ref().map(|policy| policy.as_policy_str()))?; let gpu_selection = parse_gpu_selection(gpu.as_deref())?; // CPU-only serving never pins a GPU, so skip GPU resolution entirely and // surface the explicit `--gpu` as ignored rather than printing a device the @@ -18345,23 +18384,7 @@ mod tests { } } - fn possible_values_listed_in_help(help: &str) -> Vec<String> { - let marker = "[possible values:"; - let start = help - .find(marker) - .unwrap_or_else(|| panic!("help text missing `{marker}`: {help:?}")); - let rest = &help[start + marker.len()..]; - let end = rest - .find(']') - .unwrap_or_else(|| panic!("unterminated possible-values list in help: {help:?}")); - rest[..end] - .split(',') - .map(|value| value.trim().to_owned()) - .filter(|value| !value.is_empty()) - .collect() - } - - fn serve_arg_help(arg_id: &str) -> String { + fn serve_possible_values(arg_id: &str) -> Vec<String> { let cli = Cli::command(); let serve = cli .find_subcommand("serve") @@ -18370,32 +18393,25 @@ mod tests { .get_arguments() .find(|arg| arg.get_id().as_str() == arg_id) .unwrap_or_else(|| panic!("serve has no `{arg_id}` argument")) - .get_help() - .map(ToString::to_string) - .unwrap_or_default() + .get_possible_values() + .iter() + .map(|value| value.get_name().to_owned()) + .collect() } - // `--engine` restricts its input to `SUPPORTED_ENGINES` via a clap - // `value_parser`, so the possible values are advertised in `--help` and shell - // completion structurally (not a hand-written doc string). `--device` stays - // free-form (it accepts aliases such as `auto`/`gpu` plus the intentional - // `cpu_only` rejection), so its advertised list is hand-written and guarded - // by a sync test below. Both guards keep the advertised lists honest. + // Both `--engine` and `--device` restrict their input to a fixed set via a + // clap `value_parser`/`ValueEnum`, so invalid input is rejected as a usage + // error (exit code 2) with the accepted choices listed. Values are advertised + // in `--help` and shell completion structurally (not a hand-written doc + // string), with one deliberate exception: `--device cpu_only` is + // `#[value(hide = true)]` — still accepted so its exit-1 rejection message + // survives, but kept out of help and completion. The sync tests below keep the + // advertised lists honest; `serve_device_help_lists_match_device_policy_names` + // compares against the full `DevicePolicy` set (hidden entries included, via + // `get_possible_values`), so a dropped or renamed variant still fails. #[test] fn serve_engine_help_lists_match_engine_inventory() { - let cli = Cli::command(); - let serve = cli - .find_subcommand("serve") - .expect("serve subcommand exists"); - let engine_arg = serve - .get_arguments() - .find(|arg| arg.get_id() == "engine") - .expect("serve has an `engine` argument"); - let mut listed: Vec<String> = engine_arg - .get_possible_values() - .iter() - .map(|value| value.get_name().to_owned()) - .collect(); + let mut listed = serve_possible_values("engine"); let mut expected: Vec<String> = builtin_engine_inventory() .iter() .map(|(name, _)| (*name).to_owned()) @@ -18410,7 +18426,7 @@ mod tests { #[test] fn serve_device_help_lists_match_device_policy_names() { - let mut listed = possible_values_listed_in_help(&serve_arg_help("device")); + let mut listed = serve_possible_values("device"); let mut expected: Vec<String> = [ DevicePolicy::GpuRequired, DevicePolicy::GpuPreferred, @@ -18423,7 +18439,63 @@ mod tests { expected.sort(); assert_eq!( listed, expected, - "serve --device help possible-values must stay in sync with DevicePolicy names" + "serve --device possible-values must stay in sync with DevicePolicy names" + ); + } + + #[test] + fn serve_rejects_unknown_device_policy_as_usage_error() { + // An invalid `--device` must fail clap's value validation (a usage error, + // exit code 2) rather than parsing as a free-form string and failing later + // in application logic (exit code 1). This keeps `--device` consistent with + // every other enum-style argument and lists the valid choices in the error. + let error = parse_serve(&["--device", "bogus"]).expect_err("invalid device rejected"); + assert_eq!(error.kind(), clap::error::ErrorKind::InvalidValue); + } + + #[test] + fn serve_accepts_device_policy_values_and_aliases() { + for (value, expected) in [ + ("gpu_required", DevicePolicyArg::GpuRequired), + ("gpu_preferred", DevicePolicyArg::GpuPreferred), + ("cpu_only", DevicePolicyArg::CpuOnly), + // Historical aliases stay accepted for backward compatibility. + ("auto", DevicePolicyArg::GpuRequired), + ("gpu", DevicePolicyArg::GpuRequired), + ("cpu", DevicePolicyArg::CpuOnly), + ] { + let cli = parse_serve(&["--device", value]).expect("device value parses"); + match cli.command { + Some(Command::Serve { device, .. }) => { + assert_eq!(device, Some(expected), "device value `{value}`"); + } + other => panic!("expected Serve, got {other:?}"), + } + } + } + + #[test] + fn device_policy_arg_maps_through_parse_device_policy() { + // `as_policy_str` feeds the parsed variant back through + // `parse_device_policy`, the single choke point enforcing the strict + // no-CPU policy. Guard the mapping so a mis-typed arm (e.g. `CpuOnly => + // "gpu_required"`) cannot silently turn the deliberate CPU rejection into + // a GPU-required serve while every other test stays green. + assert_eq!( + parse_device_policy(Some(DevicePolicyArg::GpuRequired.as_policy_str())) + .expect("gpu_required parses"), + DevicePolicy::GpuRequired + ); + assert_eq!( + parse_device_policy(Some(DevicePolicyArg::GpuPreferred.as_policy_str())) + .expect("gpu_preferred parses"), + DevicePolicy::GpuRequired + ); + // `cpu_only` must still be rejected outright rather than mapped to a + // GPU policy. + assert!( + parse_device_policy(Some(DevicePolicyArg::CpuOnly.as_policy_str())).is_err(), + "cpu_only must be rejected" ); } From 61a28a32e94cc9d9d3e9e1f15327120fec96c297 Mon Sep 17 00:00:00 2001 From: Roman <roman.sirokov@amd.com> Date: Thu, 27 Aug 2026 12:45:31 +0000 Subject: [PATCH 17/23] fix(dash): launcher shows live serving instances; amd-smi detection off critical path (eai-8190) (#295) * fix(dash): seed launcher front door with live serving instances The launcher front door built an empty AppState, so it always rendered the idle variant even when a model was actively serving. Read the managed-service registry (the same authority `rocm services` reads) once per hub-loop pass and seed the AppState's instances from it. Also treat `Ready` as serving everywhere the dashboard counts running instances (`is_serving()`), matching the `Running`+`Ready` treatment already used elsewhere (e.g. home.rs) -- a served model reports `Ready`, not `Running`, so the count previously undercounted actual serving models. Adds apps/rocm's direct dependency on rocm-dash-core (previously only a transitive dep) so the launcher can build `Instance`s from the registry's `DiscoveredService` records. Signed-off-by: Roman Sirokov <roman.sirokov@amd.com> * fix(dash-daemon): run amd-smi detection off the run loop's critical path Detecting amd-smi (`amd-smi version` plus the first `system_info()`) can take up to ~15s on real hardware. Running it inline before the run loop's first tick blocked managed-service discovery and the first snapshot broadcast behind it, so an already-running model did not surface in the dashboard until GPU detection finished -- a visible ~15-20s "0 models running" lag while `rocm services` already reported it live. Spawn detection in the background and adopt the result via a oneshot channel the moment it lands, without ever blocking the loop while it is in flight. The loop now starts ticking immediately, so serving instances appear within one discovery tick; GPU metrics fill in once detection completes. Adds a regression test asserting on ordering (the instance must surface in a snapshot whose gpu_system_info is still None) rather than wall-clock timing, since a pure "arrived within Ns" check would be flaky under subscriber starvation. Signed-off-by: Roman Sirokov <roman.sirokov@amd.com> * test(dash): exercise off-critical-path detection and launcher front door Address review feedback on the amd-smi-off-critical-path change. - Gate the "amd-smi unavailable" warning behind gpu_init_done so a healthy host never flashes it during the detection window; settle as unavailable (and say so once) if the detection task ever ends without a result. - Add an amd_smi_skip_kfd_preflight test seam so the daemon regression test drives a fake amd-smi through the real detection path instead of short-circuiting on a GPU-less CI host (the /dev/kfd guard stays mandatory in production). The test now genuinely fails if detection moves back onto the critical path, and asserts the surfaced snapshot carries no premature "amd-smi unavailable" warning. - Add direct unit tests for launcher_serving_instances (ready record maps to a live Instance; unbound :0 record is dropped) and a behavioural launcher scenario driving bare rocm through a PTY to prove the front door shows "Serving <model>" rather than "Idle" for a live registry service. Signed-off-by: Roman Sirokov <roman.sirokov@amd.com> --------- Signed-off-by: Roman Sirokov <roman.sirokov@amd.com> Signed-off-by: pmoutsias-amd <peter.moutsias@amd.com> --- Cargo.lock | 1 + apps/rocm/Cargo.toml | 4 +- apps/rocm/src/dash.rs | 105 +++++++++++++++- crates/rocm-dash-collectors/src/amd_smi.rs | 27 +++- crates/rocm-dash-daemon/src/runner.rs | 103 +++++++++++++--- crates/rocm-dash-daemon/tests/end_to_end.rs | 129 ++++++++++++++++++++ crates/rocm-dash-tui/src/ui/launcher.rs | 60 ++++++++- crates/rocm-dash-tui/src/ui/tabs/pane.rs | 4 +- crates/rocm-dash-tui/src/ui/tabs/serving.rs | 30 +++++ tests/e2e-cucumber/features/dash.feature | 13 ++ tests/e2e-cucumber/tests/e2e/dash_steps.rs | 44 +++++++ 11 files changed, 492 insertions(+), 28 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index b71b3fada0..473b8eb7bc 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3569,6 +3569,7 @@ dependencies = [ "keyring-core", "libc", "rocm-core", + "rocm-dash-core", "rocm-dash-daemon", "rocm-dash-tui", "rocm-deps", diff --git a/apps/rocm/Cargo.toml b/apps/rocm/Cargo.toml index db8b886225..944e83e8b3 100644 --- a/apps/rocm/Cargo.toml +++ b/apps/rocm/Cargo.toml @@ -25,7 +25,9 @@ rocm-deps = { path = "../../crates/rocm-deps" } # rocm-dash unified dashboard launch. The # telemetry daemon + ratatui-0.30 TUI are launched from the `dash` verb; tokio # drives the async daemon/TUI from the otherwise-sync `rocm` binary. -# (rocm-dash-core is pulled in transitively by the two below — not a direct dep.) +# rocm-dash-core is a direct dep so the launcher front door can build serving +# `Instance`s from the managed-service registry. +rocm-dash-core = { path = "../../crates/rocm-dash-core" } rocm-dash-daemon = { path = "../../crates/rocm-dash-daemon" } rocm-dash-tui = { path = "../../crates/rocm-dash-tui" } tokio = { workspace = true } diff --git a/apps/rocm/src/dash.rs b/apps/rocm/src/dash.rs index 230a4add8a..88b63864d1 100644 --- a/apps/rocm/src/dash.rs +++ b/apps/rocm/src/dash.rs @@ -66,6 +66,9 @@ pub fn runner_options( // amd-smi ships inside the managed runtime wheel's bin dir, not on PATH; // resolve the path so the GPU collector can find it. amd_smi_binary: Some(rocm_core::resolve_amd_smi_binary()), + // Production always runs the real `/dev/kfd` pre-flight; only daemon + // integration tests with a fake binary skip it. + amd_smi_skip_kfd_preflight: false, } } @@ -358,7 +361,12 @@ pub fn run_launcher(chat_mock: bool) -> Result<()> { let config = RocmCliConfig::load(&paths).unwrap_or_default(); let theme = config.dashboard.tui.theme; loop { - match rocm_dash_tui::ui::launcher::run_launcher(&theme)? { + // Re-read the managed-service registry on each pass so the front door + // reflects models started (or stopped) during a prior flow. This is a + // cheap status-only file read — no telemetry daemon and no network + // readiness probes, so the front door stays instant. + let serving = launcher_serving_instances(&paths); + match rocm_dash_tui::ui::launcher::run_launcher(&theme, serving)? { None => return Ok(()), Some(choice) => match launcher_route(choice) { LauncherRoute::Focused(focus) => run_focused(focus)?, @@ -369,6 +377,28 @@ pub fn run_launcher(chat_mock: bool) -> Result<()> { } } +/// Serving instances for the launcher front door, read from the managed-service +/// registry (the same authority `rocm services` reads). +/// +/// Deliberately cheap: a status-only registry read with no network readiness +/// probes and no telemetry daemon, so the front door renders instantly. +fn launcher_serving_instances(paths: &AppPaths) -> Vec<rocm_dash_core::metrics::Instance> { + use rocm_dash_daemon::registry::{discover_managed_services, load_service_records}; + let records = load_service_records(&paths.services_dir()); + discover_managed_services(&records) + .svcs + .into_iter() + .map(|svc| rocm_dash_core::metrics::Instance { + container_id: svc.container_id, + container_name: svc.container_name, + model_name: svc.model_name, + status: svc.status, + port: svc.port, + ..Default::default() + }) + .collect() +} + /// Entry point for a focused launcher flow (Set up / Serve / Diagnose). /// /// Opens the dashboard runtime hosting exactly the one overlay for `focus` — no @@ -916,4 +946,77 @@ mod tests { first.preferred_engines.first() ); } + + /// The launcher front-door seam that regressed: a `ready` managed vLLM + /// service recorded on disk must map into a live `Instance` (id, model, + /// port, status) so bare `rocm` shows it instead of "Idle". Reads the same + /// registry `rocm services` reads — status-only, no daemon, no network. + #[test] + fn launcher_serving_instances_maps_ready_registry_record() { + let root = std::env::temp_dir().join(format!( + "rocm-cli-launcher-serving-test-{}-{}", + std::process::id(), + rocm_core::unix_time_millis() + )); + let p = AppPaths { + config_dir: root.join("cfg"), + data_dir: root.join("data"), + cache_dir: root.join("cache"), + }; + let services_dir = p.services_dir(); + std::fs::create_dir_all(&services_dir).unwrap(); + std::fs::write( + services_dir.join("svc.json"), + r#"{"service_id":"vllm-launcher","engine":"vllm", + "model_ref":"meta-llama/Llama-3.1-8B","canonical_model_id":"m", + "host":"127.0.0.1","port":11435, + "endpoint_url":"http://127.0.0.1:11435/v1","mode":"managed", + "status":"ready","created_at_unix_ms":1}"#, + ) + .unwrap(); + + let instances = launcher_serving_instances(&p); + + assert_eq!(instances.len(), 1, "the ready managed service must surface"); + let inst = &instances[0]; + assert_eq!(inst.container_id, "vllm-launcher"); + assert_eq!(inst.model_name, "meta-llama/Llama-3.1-8B"); + assert_eq!(inst.port, Some(11435)); + assert_eq!(inst.status, rocm_dash_core::metrics::InstanceStatus::Ready); + + let _ = std::fs::remove_dir_all(&root); + } + + /// A non-scrapeable record (unbound `port:0`) must NOT surface as a live + /// instance on the front door — the registry is the authority and `:0` is + /// never a real serving endpoint. + #[test] + fn launcher_serving_instances_skips_unbound_record() { + let root = std::env::temp_dir().join(format!( + "rocm-cli-launcher-serving-skip-test-{}-{}", + std::process::id(), + rocm_core::unix_time_millis() + )); + let p = AppPaths { + config_dir: root.join("cfg"), + data_dir: root.join("data"), + cache_dir: root.join("cache"), + }; + let services_dir = p.services_dir(); + std::fs::create_dir_all(&services_dir).unwrap(); + std::fs::write( + services_dir.join("svc.json"), + r#"{"service_id":"vllm-unbound","engine":"vllm","model_ref":"m", + "canonical_model_id":"m","host":"127.0.0.1","port":0, + "mode":"managed","status":"ready","created_at_unix_ms":1}"#, + ) + .unwrap(); + + assert!( + launcher_serving_instances(&p).is_empty(), + "an unbound (:0) record must not surface as a live instance" + ); + + let _ = std::fs::remove_dir_all(&root); + } } diff --git a/crates/rocm-dash-collectors/src/amd_smi.rs b/crates/rocm-dash-collectors/src/amd_smi.rs index ce5635e287..f19e20b82d 100644 --- a/crates/rocm-dash-collectors/src/amd_smi.rs +++ b/crates/rocm-dash-collectors/src/amd_smi.rs @@ -43,7 +43,32 @@ impl AmdSmiCollector { /// directory rather than on `PATH`, so callers resolve the path or command /// name (via `rocm_core::resolve_amd_smi_binary`) and pass it here. pub async fn detect_with_binary(binary: impl Into<OsString>) -> Option<Self> { - if !kfd_accessible() { + Self::detect_with_binary_inner(binary, false).await + } + + /// Like [`detect_with_binary`](Self::detect_with_binary) but skips the + /// mandatory `/dev/kfd` pre-flight. + /// + /// **Test-only.** The KFD pre-flight is a safety guard: against a *real* + /// `amd-smi` on a host without a usable `/dev/kfd`, the process can block in + /// uninterruptible kernel sleep (D-state) that no signal can escape. This + /// entry point exists solely so daemon integration tests can point + /// [`detect_with_binary`](Self::detect_with_binary) at a *fake* script (for + /// which the hang cannot happen) and have it actually run on a GPU-less CI + /// host, instead of short-circuiting to `None` and turning the test into a + /// no-op. Never call it against a real binary in production. + #[doc(hidden)] + pub async fn detect_with_binary_skipping_kfd_preflight( + binary: impl Into<OsString>, + ) -> Option<Self> { + Self::detect_with_binary_inner(binary, true).await + } + + async fn detect_with_binary_inner( + binary: impl Into<OsString>, + skip_kfd_preflight: bool, + ) -> Option<Self> { + if !skip_kfd_preflight && !kfd_accessible() { return None; } let me = Self { diff --git a/crates/rocm-dash-daemon/src/runner.rs b/crates/rocm-dash-daemon/src/runner.rs index 492a1eb0bc..b4c95979ab 100644 --- a/crates/rocm-dash-daemon/src/runner.rs +++ b/crates/rocm-dash-daemon/src/runner.rs @@ -74,6 +74,14 @@ pub struct RunnerOptions { /// so the caller resolves it (via `rocm_core::resolve_amd_smi_binary`) and /// passes it here. `None` falls back to looking up `amd-smi` on `PATH`. pub amd_smi_binary: Option<OsString>, + /// **Test-only.** Skip the mandatory `/dev/kfd` pre-flight in amd-smi + /// detection so a *fake* `amd_smi_binary` is actually invoked on a GPU-less + /// CI host instead of short-circuiting to "no GPU". Never set in + /// production: the KFD guard prevents a *real* `amd-smi` from hanging in + /// uninterruptible D-state. Only the daemon integration test that points + /// `amd_smi_binary` at a deliberately-slow fake script flips this, so the + /// off-critical-path detection behaviour is genuinely exercised. + pub amd_smi_skip_kfd_preflight: bool, } impl Default for RunnerOptions { @@ -93,6 +101,7 @@ impl Default for RunnerOptions { persist_dir: None, services_dir: None, amd_smi_binary: None, + amd_smi_skip_kfd_preflight: false, } } } @@ -200,23 +209,36 @@ pub async fn run_loop( // stable between scrapes (mirrors how GPU power drives tokens_per_watt). let mut per_container_used: HashMap<String, u64> = HashMap::new(); - let gpu = match opts.amd_smi_binary.clone() { - Some(binary) => AmdSmiCollector::detect_with_binary(binary).await, - None => AmdSmiCollector::detect().await, - }; - let mut gpu_system_info: Option<GpuSystemInfo> = if let Some(g) = &gpu { - let info = g.system_info().await; - info!( - gpus = info.physical_gpu_count, - model = %info.gpu_model, - rocm = info.rocm_version.as_deref().unwrap_or("?"), - "amd-smi detected" - ); - Some(info) - } else { - warn!("amd-smi not available (no /dev/kfd or `amd-smi version` failed); GPU disabled"); - None - }; + // amd-smi GPU detection plus the first `system_info()` can take up to ~15s + // on some hosts (subprocess spawn plus the per-call detect/run timeouts). + // Doing it inline here blocked the run loop's very first tick — and thus + // managed-service discovery and the first snapshot broadcast — behind it, so + // an already-running model did not surface in the dashboard until GPU + // detection finished (a visible ~15-20s "0 models running" lag while + // `rocm services` already reported it). Run detection off the critical path: + // the loop starts ticking immediately (surfacing serving instances within + // one discovery tick) and GPU metrics fill in the moment detection lands. + let amd_smi_binary = opts.amd_smi_binary.clone(); + let amd_smi_skip_kfd_preflight = opts.amd_smi_skip_kfd_preflight; + let (gpu_init_tx, mut gpu_init_rx) = + tokio::sync::oneshot::channel::<(Option<AmdSmiCollector>, Option<GpuSystemInfo>)>(); + tokio::spawn(async move { + let gpu = match amd_smi_binary { + Some(binary) if amd_smi_skip_kfd_preflight => { + AmdSmiCollector::detect_with_binary_skipping_kfd_preflight(binary).await + } + Some(binary) => AmdSmiCollector::detect_with_binary(binary).await, + None => AmdSmiCollector::detect().await, + }; + let info = match &gpu { + Some(g) => Some(g.system_info().await), + None => None, + }; + let _ = gpu_init_tx.send((gpu, info)); + }); + let mut gpu: Option<AmdSmiCollector> = None; + let mut gpu_system_info: Option<GpuSystemInfo> = None; + let mut gpu_init_done = false; let mut tick_count: u64 = 0; let mut last_sysinfo_refresh: u64 = 0; @@ -234,6 +256,45 @@ pub async fn run_loop( // every instance refreshed in this cycle serialises Fresh deterministically. let cycle_at = Utc::now(); + // Adopt the background amd-smi detection result the moment it lands, + // without ever blocking the loop while it is still in flight. Until then + // `gpu` stays `None` (GPU panels read "unavailable") but discovery and + // snapshots run every tick, so serving instances appear promptly. + if !gpu_init_done { + match gpu_init_rx.try_recv() { + Ok((detected, info)) => { + gpu_init_done = true; + if let Some(i) = &info { + info!( + gpus = i.physical_gpu_count, + model = %i.gpu_model, + rocm = i.rocm_version.as_deref().unwrap_or("?"), + "amd-smi detected" + ); + } else { + warn!( + "amd-smi not available (no /dev/kfd or `amd-smi version` failed); GPU disabled" + ); + } + gpu = detected; + gpu_system_info = info; + last_sysinfo_refresh = tick_count; + } + // The detection task dropped its sender without ever sending. + // It has no fallible await today, so this is latent rather than + // a live path — but if it ever regressed, polling `Empty` + // forever would hang `gpu_init_done` at false and suppress the + // honest "unavailable" warning below. Settle as unavailable and + // say so once, so the badge resolves instead of silently never + // clearing. + Err(tokio::sync::oneshot::error::TryRecvError::Closed) => { + gpu_init_done = true; + warn!("amd-smi detection task ended without a result; GPU disabled"); + } + Err(tokio::sync::oneshot::error::TryRecvError::Empty) => {} + } + } + let mut warnings = Vec::new(); let gpus = if let Some(g) = &gpu { match g.metrics().await { @@ -243,9 +304,15 @@ pub async fn run_loop( Vec::new() } } - } else { + } else if gpu_init_done { warnings.push("amd-smi unavailable (no /dev/kfd or binary missing)".into()); Vec::new() + } else { + // Detection is still in flight (spawned off the critical path), so + // `gpu == None` is transient here. Stay silent instead of flashing a + // false "unavailable" alarm on healthy hardware for the ~15s window + // until the first detection result lands. + Vec::new() }; if gpu.is_some() diff --git a/crates/rocm-dash-daemon/tests/end_to_end.rs b/crates/rocm-dash-daemon/tests/end_to_end.rs index 2d25f4bc1f..49194f6536 100644 --- a/crates/rocm-dash-daemon/tests/end_to_end.rs +++ b/crates/rocm-dash-daemon/tests/end_to_end.rs @@ -319,3 +319,132 @@ async fn scrape_warning_persists_between_scrape_ticks() { handle.abort(); let _ = handle.await; } + +/// Regression: a slow `amd-smi` startup must NOT delay managed-service discovery. +/// +/// GPU detection (`amd-smi version` + the first `system_info()`) can take ~15s on +/// real hardware. It used to run inline before the loop started, so the very +/// first managed-service discovery + snapshot were blocked behind it and an +/// already-running model appeared in the dashboard ~15-20s late (while +/// `rocm services` already listed it). Detection now runs off the critical path, +/// so a `ready` service surfaces on the first discovery tick while GPU detection +/// is still in flight. +/// +/// This asserts on ORDERING rather than wall-clock timing (delivery to an +/// in-process subscriber can be starved while the fake subprocess runs, which +/// would make a pure "arrived within Ns" check flaky): the instance MUST appear +/// in a snapshot whose `gpu_system_info` is still `None`. Post-fix that window +/// exists (many ~100ms ticks before the 4s fake detection lands). Pre-fix it +/// cannot: the loop only starts ticking after detection completes, so the very +/// first snapshot carrying the instance already has `gpu_system_info == Some`. +/// +/// The fake `amd-smi` is forced through the real detection path via +/// `amd_smi_skip_kfd_preflight` (see `RunnerOptions`): otherwise the `/dev/kfd` +/// pre-flight short-circuits to "no GPU" on GPU-less CI, the fake never runs, +/// and the assertion passes vacuously even with the fix reverted. The surfaced +/// snapshot must also carry NO "amd-smi unavailable" warning — detection is +/// still in flight there, so flashing that alarm would be a false alarm the +/// header ⚠ badge should never show. +#[cfg(unix)] +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn slow_gpu_detection_does_not_delay_service_discovery() { + use std::os::unix::fs::PermissionsExt; + + let dir = tempfile::tempdir().unwrap(); + + // A fake amd-smi that is deliberately slow: whatever the subcommand, sleep + // well past the discovery deadline, then emit trivially-valid JSON so both + // `version` and the `system_info` `--json` calls "succeed". + let fake = dir.path().join("amd-smi-slow"); + std::fs::write(&fake, "#!/bin/sh\nsleep 4\necho '{}'\n").unwrap(); + std::fs::set_permissions(&fake, std::fs::Permissions::from_mode(0o755)).unwrap(); + + // A ready managed vLLM service on disk — the authority `rocm services` reads. + std::fs::write( + dir.path().join("svc.json"), + r#"{"service_id":"vllm-ready-fast","engine":"vllm","model_ref":"m","canonical_model_id":"m", + "host":"127.0.0.1","port":11435,"endpoint_url":"http://127.0.0.1:11435/v1", + "mode":"managed","status":"ready","created_at_unix_ms":1}"#, + ) + .unwrap(); + + // Roomy channel so buffered early snapshots are never dropped (Lagged) even + // if the subscriber is briefly starved while the fake subprocess runs. + let (tx, mut rx) = broadcast::channel::<Event>(512); + let services_dir = dir.path().to_path_buf(); + let fake_bin = fake.into_os_string(); + let handle = tokio::spawn(async move { + let opts = runner::RunnerOptions { + services_dir: Some(services_dir), + amd_smi_binary: Some(fake_bin), + // Drive the fake through the real detection path on GPU-less CI: + // without this the `/dev/kfd` pre-flight would short-circuit before + // the fake runs, making the ordering assertion vacuous. + amd_smi_skip_kfd_preflight: true, + disable_vllm_metrics: true, + ..Default::default() + }; + runner::run_loop( + Some(Duration::from_millis(100)), + tx, + Arc::new(Mutex::new(SnapshotRing::new(512))), + Arc::new(Mutex::new(BenchRing::new(4))), + None, + opts, + ) + .await; + }); + + // Walk snapshots in order. The FIRST snapshot that carries the instance + // decides: post-fix it has no GPU info yet (pass); pre-fix it already has + // GPU info because detection ran to completion before the loop started. + // The generous timeout covers detection + any subscriber starvation. + let mut surfaced_before_gpu = None; + let mut surfaced_without_unavailable_warning = None; + let overall = Duration::from_secs(15); + loop { + let ev = match timeout(overall, rx.recv()).await { + Ok(Ok(ev)) => ev, + Ok(Err(broadcast::error::RecvError::Lagged(_))) => continue, + Ok(Err(broadcast::error::RecvError::Closed)) => break, + Err(_) => break, + }; + if let Event::Snapshot(snap) = ev + && snap + .instances + .iter() + .any(|i| i.container_id == "vllm-ready-fast") + { + surfaced_before_gpu = Some(snap.gpu_system_info.is_none()); + surfaced_without_unavailable_warning = Some( + !snap + .warnings + .iter() + .any(|w| w.contains("amd-smi unavailable")), + ); + break; + } + } + + handle.abort(); + let _ = handle.await; + + match surfaced_before_gpu { + Some(true) => {} // instance surfaced while GPU detection was still in flight + Some(false) => panic!( + "managed service only surfaced after GPU detection completed \ + (the ~15-20s dashboard startup lag regressed)" + ), + None => panic!("ready managed service never surfaced in a snapshot"), + } + + // Detection is still in flight in that first snapshot, so the honest state + // is "GPU pending", never "amd-smi unavailable". Flashing the unavailable + // warning here would be a false alarm on healthy hardware. + assert_eq!( + surfaced_without_unavailable_warning, + Some(true), + "managed service surfaced with a premature `amd-smi unavailable` warning \ + while detection was still in flight" + ); +} diff --git a/crates/rocm-dash-tui/src/ui/launcher.rs b/crates/rocm-dash-tui/src/ui/launcher.rs index ccc2a61d4c..144f532538 100644 --- a/crates/rocm-dash-tui/src/ui/launcher.rs +++ b/crates/rocm-dash-tui/src/ui/launcher.rs @@ -16,6 +16,7 @@ use ratatui::style::{Modifier, Style}; use ratatui::text::{Line, Span}; use ratatui::widgets::Paragraph; +use rocm_dash_core::metrics::Instance; #[cfg(test)] use rocm_dash_core::metrics::InstanceStatus; @@ -239,18 +240,38 @@ fn draw_menu(f: &mut Frame, area: Rect, state: &AppState, sel: usize, theme: &Th f.render_widget(Paragraph::new(lines), area); } +/// Build the launcher's render state, seeding `instances` from the serving +/// models the caller discovered in the managed-service registry. +/// +/// Kept separate from the terminal loop so the seeding is unit-testable without +/// a real terminal: an empty `serving` yields the idle front door, a non-empty +/// one makes `is_running` report the running model. +fn launcher_state(theme_name: &str, serving: Vec<Instance>) -> AppState { + let mut state = AppState::new(String::new(), theme_name.to_string()); + state.instances = serving + .into_iter() + .map(|inst| (inst.container_id.clone(), inst)) + .collect(); + state +} + /// Run the launcher as a synchronous pre-dash screen. /// /// Returns the chosen destination, or `None` when the user quits. On success the /// caller escalates into the existing dash entry points (ponytail: no parallel /// async loop here). /// -/// Renders the idle variant from a fresh `AppState` (no live daemon is started -/// just for the front door); live telemetry appears once the dash is opened. +/// `serving` seeds the status strip / menu so the front door reflects the models +/// the managed-service registry reports as running (the same authority +/// `rocm services` reads) — without starting a live telemetry daemon just for the +/// front door. Live GPU telemetry still appears once the dash is opened. /// /// # Errors /// Propagates terminal setup / event-read I/O errors. -pub fn run_launcher(theme_name: &str) -> std::io::Result<Option<LauncherChoice>> { +pub fn run_launcher( + theme_name: &str, + serving: Vec<Instance>, +) -> std::io::Result<Option<LauncherChoice>> { use crossterm::event::{self, Event, KeyCode, KeyEventKind}; use crossterm::terminal::{ EnterAlternateScreen, LeaveAlternateScreen, disable_raw_mode, enable_raw_mode, @@ -258,7 +279,7 @@ pub fn run_launcher(theme_name: &str) -> std::io::Result<Option<LauncherChoice>> use ratatui::Terminal; use ratatui::backend::CrosstermBackend; - let state = AppState::new(String::new(), theme_name.to_string()); + let state = launcher_state(theme_name, serving); let theme = state.theme; enable_raw_mode()?; @@ -397,6 +418,37 @@ mod tests { assert!(out.contains("Serve a model"), "menu missing in idle"); } + #[test] + fn launcher_state_seeds_serving_from_registry_instances() { + // Regression: the front door built an empty AppState and always showed + // "Idle — nothing serving" even when a model was running. Seeding from + // the caller's registry-derived instances must make the launcher report + // the running model. + let serving = vec![Instance { + container_id: "vllm-ready".into(), + model_name: "Qwen2.5-0.5B".into(), + status: InstanceStatus::Ready, + port: Some(11435), + ..Default::default() + }]; + let running = launcher_state("default-dark", serving); + assert!( + is_running(&running), + "seeded serving instance must make the launcher report running" + ); + let out = render(&running, 0, 100, 24); + assert!(out.contains("Serving"), "serve line missing: {out:?}"); + assert!( + out.contains("Qwen2.5-0.5B"), + "served model missing: {out:?}" + ); + assert!(out.contains("11435"), "served port missing: {out:?}"); + + // With no serving instances the front door stays honestly idle. + let idle = launcher_state("default-dark", Vec::new()); + assert!(!is_running(&idle), "empty seed must render idle"); + } + #[test] fn choice_mapping_and_counts() { assert_eq!(row_count(), 5); diff --git a/crates/rocm-dash-tui/src/ui/tabs/pane.rs b/crates/rocm-dash-tui/src/ui/tabs/pane.rs index bd7ba1115c..4892d55e95 100644 --- a/crates/rocm-dash-tui/src/ui/tabs/pane.rs +++ b/crates/rocm-dash-tui/src/ui/tabs/pane.rs @@ -20,8 +20,6 @@ use ratatui::style::{Modifier, Style}; use ratatui::text::{Line, Span}; use ratatui::widgets::{Paragraph, Wrap}; -use rocm_dash_core::metrics::InstanceStatus; - use crate::app::{AppState, KeyAction, PaneFocus}; use crate::ui::format; use crate::ui::panel::{self, BoxRole}; @@ -406,7 +404,7 @@ fn live_lines(action: KeyAction, state: &AppState, theme: &Theme) -> Vec<Line<'s let running: Vec<_> = state .instances .values() - .filter(|i| i.status == InstanceStatus::Running) + .filter(|i| i.status.is_serving()) .collect(); let mut lines = vec![head(format!("Running now — {}", running.len()))]; if running.is_empty() { diff --git a/crates/rocm-dash-tui/src/ui/tabs/serving.rs b/crates/rocm-dash-tui/src/ui/tabs/serving.rs index d3afacebc0..c631c3d639 100644 --- a/crates/rocm-dash-tui/src/ui/tabs/serving.rs +++ b/crates/rocm-dash-tui/src/ui/tabs/serving.rs @@ -206,6 +206,36 @@ mod tests { assert!(out.contains("8000"), "running port not shown: {out:?}"); } + #[test] + fn serving_detail_counts_ready_instances_as_running() { + // Regression: a served model reports `Ready` (readiness-probed), not + // `Running`. The detail pane must count it as serving so the count + // matches `rocm services`, which also treats ready as live. + use rocm_dash_core::metrics::{Instance, InstanceStatus}; + let mut s = AppState::new("t".into(), "default-dark".into()); + s.active_tab = ActiveTab::Serving; + s.serving_sel = 2; // "Running instances/services" → OpenServices + s.instances.insert( + "vllm-1".into(), + Instance { + container_name: "vllm-1".into(), + model_name: "Llama-3.1-8B".into(), + status: InstanceStatus::Ready, + port: Some(8000), + ..Default::default() + }, + ); + let out = render(&s, 120, 28); + assert!( + out.contains("Running now — 1"), + "ready instance not counted as running: {out:?}" + ); + assert!( + out.contains("Llama-3.1-8B"), + "ready model not shown inline: {out:?}" + ); + } + #[test] fn serving_verb_action_maps_selection_to_seam() { assert_eq!(verb_action(0), KeyAction::OpenServeWizard); diff --git a/tests/e2e-cucumber/features/dash.feature b/tests/e2e-cucumber/features/dash.feature index f1fa35e4f1..3fbbfa1505 100644 --- a/tests/e2e-cucumber/features/dash.feature +++ b/tests/e2e-cucumber/features/dash.feature @@ -110,3 +110,16 @@ Feature: Interactive dashboard Then generation throughput is no longer displayed When the user quits the dashboard Then the dashboard exits successfully + + @id:launcher-shows-live-serving-instance @requires-os:linux + Scenario: 10 - The launcher front door shows a live serving model rather than idle + # EAI-8190 regression: bare `rocm` opens the launcher front door, which + # reads the managed-service registry (`launcher_serving_instances`) the same + # way `rocm services` does. A model already serving must surface as + # "Serving <model>", not the "Idle — nothing serving" state the front door + # showed before the fix, which drove this whole PR. + Given a running managed model is available locally + When the user opens the launcher + Then the launcher shows the model serving + When the user quits the launcher + Then the launcher exits successfully diff --git a/tests/e2e-cucumber/tests/e2e/dash_steps.rs b/tests/e2e-cucumber/tests/e2e/dash_steps.rs index 911fcf7435..49518724af 100644 --- a/tests/e2e-cucumber/tests/e2e/dash_steps.rs +++ b/tests/e2e-cucumber/tests/e2e/dash_steps.rs @@ -108,6 +108,15 @@ async fn open_dashboard(world: &mut E2eWorld) { world.tui = Some(tui); } +#[when("the user opens the launcher")] +async fn open_launcher(world: &mut E2eWorld) { + // Bare `rocm` (no subcommand) opens the launcher front door under an + // interactive terminal — the PTY slave satisfies `interactive_terminal()`. + let tui = TuiSession::spawn(world, &[]) + .unwrap_or_else(|e| panic!("failed to open the launcher: {e}")); + world.tui = Some(tui); +} + #[when("the user opens the ROCm view")] async fn open_rocm_view(world: &mut E2eWorld) { // Dashboard tabs are currently ordered Home, ROCm, Serving, Observe; these @@ -226,6 +235,11 @@ async fn quit_interactive_chat(world: &mut E2eWorld) { quit_tui(world, "interactive chat").await; } +#[when("the user quits the launcher")] +async fn quit_launcher(world: &mut E2eWorld) { + quit_tui(world, "the launcher").await; +} + // ── Then ─────────────────────────────────────────────────────────── #[then("the dashboard home view is displayed")] @@ -408,6 +422,36 @@ async fn dashboard_exited(world: &mut E2eWorld) { assert_tui_opened(world); } +#[then("the launcher shows the model serving")] +async fn launcher_shows_serving(world: &mut E2eWorld) { + let model = world + .model_name + .as_deref() + .expect("no model name set") + .to_string(); + let tui = session(world); + // The front door's status strip renders "Serving <model>" for a live + // registry instance; wait on the model name to synchronise with the first + // paint before inspecting the whole screen. + tui.wait_for_screen(&model, default_timeout()) + .await + .unwrap_or_else(|e| panic!("the launcher never showed the serving model: {e}")); + let screen = tui.screen_text(); + assert!( + screen.contains("Serving"), + "launcher did not show the model as serving:\n{screen}" + ); + assert!( + !screen.contains("Idle — nothing serving"), + "launcher still reported idle despite a live registry instance:\n{screen}" + ); +} + +#[then("the launcher exits successfully")] +async fn launcher_exited(world: &mut E2eWorld) { + assert_tui_opened(world); +} + #[then("interactive chat exits successfully")] async fn interactive_chat_exited(world: &mut E2eWorld) { assert_tui_opened(world); From 908671e21a59b4cf11486b3962e695a75b3c07d7 Mon Sep 17 00:00:00 2001 From: Roman <roman.sirokov@amd.com> Date: Thu, 27 Aug 2026 12:46:22 +0000 Subject: [PATCH 18/23] Fix diagnose emitting invalid UNKNOWN render-group remediation (#302) check_4_render_group built the render-group remediation command from the /dev/kfd owner group obtained via `stat -c %G`. When the device GID has no matching group name, stat prints the literal "UNKNOWN", and the existing guard only rejected empty strings. That let UNKNOWN leak into the suggested fix, producing `sudo usermod -a -G UNKNOWN,video "$USER"` in the plan, summary, and --json output -- a command that fails because no such group exists. Reject "UNKNOWN" (case-insensitive) alongside empty values so the group falls back to "render". This makes diagnose's fallback coincide with the "render,video" that fix.rs hardcodes in the UNKNOWN/empty case; it does not unify the two group sources, which are still computed independently and can diverge when /dev/kfd has a real group name that is not "render". Add a regression test covering both the "UNKNOWN" and "unknown" sentinel casings falling back to the render group. Signed-off-by: Roman Sirokov <roman.sirokov@amd.com> Signed-off-by: pmoutsias-amd <peter.moutsias@amd.com> --- crates/rocm-core/src/diagnose.rs | 42 +++++++++++++++++++++++++++++++- 1 file changed, 41 insertions(+), 1 deletion(-) diff --git a/crates/rocm-core/src/diagnose.rs b/crates/rocm-core/src/diagnose.rs index 784cdf490b..ab8f0fe63f 100644 --- a/crates/rocm-core/src/diagnose.rs +++ b/crates/rocm-core/src/diagnose.rs @@ -575,11 +575,15 @@ fn check_4_render_group(e: &Examination, symptom: &str) -> Diagnosis { if score <= 0 { return zero("fix-4-render-group", "User missing render/video group"); } + // `stat -c %G` prints the literal "UNKNOWN" when the device GID has no + // matching group name (common in containers). Treat that the same as an + // empty value and fall back to the render group, so the suggested command + // never names a group that does not exist. let kfd_group = e .kfd .as_ref() .map(|k| k.owner_group.clone()) - .filter(|g| !g.is_empty()) + .filter(|g| !g.is_empty() && !g.eq_ignore_ascii_case("UNKNOWN")) .unwrap_or_else(|| "render".to_owned()); let fix = Fix { summary: format!("Add the current user to '{kfd_group}' (and 'video' for safety) and log out/in."), @@ -1604,6 +1608,42 @@ mod tests { assert_eq!(top.score, 60); // 35 render + 25 kfd } + #[test] + fn unknown_kfd_group_falls_back_to_render() { + // `stat -c %G /dev/kfd` prints "UNKNOWN" when the GID has no group + // name. That value must not leak into the remediation command, which + // would otherwise suggest `usermod -a -G UNKNOWN,video` -- a group that + // does not exist. Fall back to the render group instead. Exercise both + // casings so the case-insensitive guard cannot silently regress to an + // exact-match check while this test stays green. + for sentinel in ["UNKNOWN", "unknown"] { + let mut e = linux_base(); + e.in_render_group = Some(false); + e.kfd = Some(Device { + path: "/dev/kfd".to_owned(), + exists: true, + mode: "crw-rw----".to_owned(), + owner_group: sentinel.to_owned(), + user_can_write: Some(false), + ..Device::default() + }); + let report = diagnose(&e, ""); + let top = &report.matched[0]; + assert_eq!(top.id, "fix-4-render-group"); + let fix = top.fix.as_ref().expect("fix-4 carries a fix"); + assert_eq!( + fix.commands, + vec!["sudo usermod -a -G render,video \"$USER\"".to_owned()], + "{sentinel} group must fall back to render, not leak into the command" + ); + assert!( + !fix.summary.to_uppercase().contains("UNKNOWN"), + "summary must not name the bogus {sentinel} group:\n{}", + fix.summary + ); + } + } + #[test] fn arch_not_in_wheel_strong_signal() { let mut e = linux_base(); From 9169fbbd2f943603fa74a8187b074c54cdb20888 Mon Sep 17 00:00:00 2001 From: pmoutsias-amd <peter.moutsias@amd.com> Date: Thu, 27 Aug 2026 11:43:30 -0400 Subject: [PATCH 19/23] docs(readme): reframe install channels around published release, document rocm examine, describe Configuration section Signed-off-by: pmoutsias-amd <peter.moutsias@amd.com> --- README.md | 48 ++++++++++++++++++++++++++++++++++++------------ 1 file changed, 36 insertions(+), 12 deletions(-) diff --git a/README.md b/README.md index 43429113c8..54d4625c82 100644 --- a/README.md +++ b/README.md @@ -82,28 +82,30 @@ directory to your shell `PATH`. Rerun it any time to upgrade. ### Linux and WSL (x86_64) -Only nightly builds are published today, so pass the `nightly` channel: - ```bash -curl -fsSL https://raw.githubusercontent.com/ROCm/rocm-cli/main/install.sh | sh -s -- nightly +curl -fsSL https://raw.githubusercontent.com/ROCm/rocm-cli/main/install.sh | sh ``` -Once a stable release exists, omit the argument to track the default `release` -channel instead: +This tracks the default `release` channel. For nightly builds, pass the +`nightly` channel instead: ```bash -curl -fsSL https://raw.githubusercontent.com/ROCm/rocm-cli/main/install.sh | sh +curl -fsSL https://raw.githubusercontent.com/ROCm/rocm-cli/main/install.sh | sh -s -- nightly ``` ### Windows (x86_64, PowerShell) ```powershell -$env:ROCM_CLI_CHANNEL = "nightly" irm https://raw.githubusercontent.com/ROCm/rocm-cli/main/install.ps1 | iex ``` -Drop the `ROCM_CLI_CHANNEL` line to track the default `release` channel once a -stable release is published. +This tracks the default `release` channel. For nightly builds, set +`ROCM_CLI_CHANNEL` to `nightly` first: + +```powershell +$env:ROCM_CLI_CHANNEL = "nightly" +irm https://raw.githubusercontent.com/ROCm/rocm-cli/main/install.ps1 | iex +``` ## Build from source @@ -177,7 +179,7 @@ five tabs (switch with `Tab`/`Shift+Tab` or number keys `1`–`5`): | Tab | What it shows | |---|---| | **Home** | At-a-glance status: GPU, active runtime, running servers | -| **ROCm** | Guided ROCm/runtime actions with inline details | +| **ROCm** | Guided ROCm and runtime actions with inline details | | **Serving** | Start, inspect, and manage model servers | | **Observe** | Live GPU utilization, instances, and benchmark telemetry | | **Chat** | Assistant chat backed by a local server or configured provider | @@ -229,6 +231,23 @@ requirements. ## Commands +### Examine + +``` +rocm examine [--json] [--framework auto|pytorch|llama-cpp|skip] +``` + +Checks this computer's GPU, ROCm install, engines, and managed setup +folders — the command to run first to see whether a system is ready, and +what `rocm install sdk` and `rocm serve` will see. `--json` emits a +machine-readable report for diagnosis tooling instead of the human-readable +summary. `--framework` controls which ML framework the `--json` report probes +for its ROCm build and compiled GPU architectures: `auto` (the default) tries +PyTorch, then falls back to llama.cpp; `pytorch` or `llama-cpp` probe only +that framework; `skip` runs no framework probe at all, which is fastest and +still enough to answer GPU and driver questions. `--framework` only affects +the JSON report, not the human-readable one. + ### ROCm installation ``` @@ -378,13 +397,13 @@ rocm model [--verbose] ``` `rocm model` prints a curated catalog of popular open-weight models grouped by -the hardware path they target — Strix Halo (Lemonade / llama.cpp) and MI300X +the hardware path they target — Strix Halo (Lemonade and llama.cpp) and MI300X (vLLM) — using canonical Hugging Face ids, and shows for each the quantization that fits a single GPU. Strix Halo entries use the `owner/repo:variant` GGUF form (for example, `unsloth/Qwen3.6-35B-A3B-GGUF:Q4_K_M`) that `rocm serve` needs; MI300X entries serve at BF16. This catalog ships inside the binary, so it is available offline; when a recipe index is configured instead, the header names it. -`--verbose` also lists the other recipes (the default assistant and smoke/test +`--verbose` also lists the other recipes (the default assistant and smoke-test paths) that `rocm serve` can still resolve but that are hidden from the list. The catalog is only a starting point: you can serve any compatible Hugging Face @@ -453,6 +472,11 @@ Optional background checks that can propose or apply changes automatically. ### Configuration +Show or change rocm-cli's saved settings — the default engine and runtime, +which runtime each engine prefers, local GPU telemetry opt-in, and the +provider used for chat, automations, and ambiguous natural-language plans +(including enabling providers and storing their API keys). + ``` rocm config show rocm config set-default-engine <engine> From b7300c311afbc9d57ea9eb5f8caa5f44afa2360d Mon Sep 17 00:00:00 2001 From: pmoutsias-amd <peter.moutsias@amd.com> Date: Thu, 27 Aug 2026 12:23:45 -0400 Subject: [PATCH 20/23] docs(readme): remove rocm model from command reference Not one of the commands being hardened for this release; team decision was to drop it from the reference for now rather than document it. Signed-off-by: pmoutsias-amd <peter.moutsias@amd.com> --- README.md | 26 +------------------------- 1 file changed, 1 insertion(+), 25 deletions(-) diff --git a/README.md b/README.md index 54d4625c82..26edffe22e 100644 --- a/README.md +++ b/README.md @@ -212,9 +212,6 @@ box. You can also serve any compatible Hugging Face model directly — see [Model serving](#model-serving) for the GGUF-vs-safetensors rule, since which form works depends on the engine your GPU selects. -Use `rocm model` to see available model recipes and their GPU memory -requirements. - ## Quick reference | Command | Description | @@ -373,8 +370,7 @@ with an explicit quantization variant, for example, `rocm serve unsloth/Qwen3-0.6B-GGUF:Q4_0`. The vLLM engine (Instinct) serves **safetensors** repos, such as `rocm serve Qwen/Qwen2.5-1.5B-Instruct`. A safetensors-only id has no GGUF build, so serving it through Lemonade fails -rather than silently substituting a different model. Short -aliases from `rocm model` might not resolve with all engines. +rather than silently substituting a different model. Some models (such as Llama) are gated and require HuggingFace authentication. Log in with `huggingface-cli login` or set `HF_TOKEN` in your environment @@ -390,26 +386,6 @@ model across multiple GPUs is not supported. Because selection uses the `amd-smi` ordinal but is applied via `HIP_VISIBLE_DEVICES`, rocm-cli warns when `ROCR_VISIBLE_DEVICES` is set, since the two orderings can diverge. -Show recommended models and hardware compatibility: - -``` -rocm model [--verbose] -``` - -`rocm model` prints a curated catalog of popular open-weight models grouped by -the hardware path they target — Strix Halo (Lemonade and llama.cpp) and MI300X -(vLLM) — using canonical Hugging Face ids, and shows for each the quantization -that fits a single GPU. Strix Halo entries use the `owner/repo:variant` GGUF form -(for example, `unsloth/Qwen3.6-35B-A3B-GGUF:Q4_K_M`) that `rocm serve` needs; MI300X -entries serve at BF16. This catalog ships inside the binary, so it is available -offline; when a recipe index is configured instead, the header names it. -`--verbose` also lists the other recipes (the default assistant and smoke-test -paths) that `rocm serve` can still resolve but that are hidden from the list. - -The catalog is only a starting point: you can serve any compatible Hugging Face -model by passing its id to `rocm serve` — `owner/repo` for vLLM, or -`owner/repo:<quant>` for a Lemonade GGUF. - Manage background servers started with `--managed`: ``` From 482e76370184baf7e1bdd5b6b2cc8b6dea8075b3 Mon Sep 17 00:00:00 2001 From: pmoutsias-amd <peter.moutsias@amd.com> Date: Thu, 27 Aug 2026 13:36:42 -0400 Subject: [PATCH 21/23] docs(conf): switch to rocm-ai flavor, enable repo and download buttons Signed-off-by: pmoutsias-amd <peter.moutsias@amd.com> --- docs/rocm-docs/conf.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/docs/rocm-docs/conf.py b/docs/rocm-docs/conf.py index 7ea49c43f2..42d262aaf1 100644 --- a/docs/rocm-docs/conf.py +++ b/docs/rocm-docs/conf.py @@ -35,7 +35,11 @@ def filter(self, record: logging.LogRecord) -> bool: # -- General configuration ------------------------------------------------- html_theme = "rocm_docs_theme" -html_theme_options = {"flavor": "rocm"} +html_theme_options = { + "flavor": "rocm-ai", + "use_repository_button": True, + "use_download_button": True, +} html_title = "ROCm CLI documentation" # myst.header: `:start-after:` on included README sections strips the heading # line itself, so the first heading in the chunk renders one level below its From b893d040f709dd9546d8673553d6d6a610daf6fe Mon Sep 17 00:00:00 2001 From: pmoutsias-amd <peter.moutsias@amd.com> Date: Thu, 27 Aug 2026 14:19:17 -0400 Subject: [PATCH 22/23] docs(conf): revert repository/download header buttons use_repository_button and use_download_button rely on rocm-docs-core resolving a repository_url from the current git branch. GitHub Actions' pull_request checkout leaves a detached HEAD on the synthetic merge ref, so get_branch() returns an empty URL, and sphinx_book_theme crashes trying to unpack it, failing the -W build. Signed-off-by: pmoutsias-amd <peter.moutsias@amd.com> --- docs/rocm-docs/conf.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/docs/rocm-docs/conf.py b/docs/rocm-docs/conf.py index 42d262aaf1..e1d4f9251a 100644 --- a/docs/rocm-docs/conf.py +++ b/docs/rocm-docs/conf.py @@ -37,8 +37,6 @@ def filter(self, record: logging.LogRecord) -> bool: html_theme = "rocm_docs_theme" html_theme_options = { "flavor": "rocm-ai", - "use_repository_button": True, - "use_download_button": True, } html_title = "ROCm CLI documentation" # myst.header: `:start-after:` on included README sections strips the heading From 8414709ccba80f9e38989bc92c1c6f0709cd6ce6 Mon Sep 17 00:00:00 2001 From: pmoutsias-amd <peter.moutsias@amd.com> Date: Thu, 27 Aug 2026 14:33:10 -0400 Subject: [PATCH 23/23] docs(conf): fix flavor fallback and pin repository buttons - Broaden the known-warning suppression filter to also cover the "rocm-ai" flavor, which isn't in the published rocm-docs-core release yet and would otherwise fail the -W build even though rocm_docs.theme already falls back to "rocm" gracefully. - Re-enable use_repository_button/use_download_button, pinning repository_url/repository_branch explicitly instead of letting rocm-docs-core infer them from the local git branch. CI's pull_request checkout leaves a detached HEAD on the synthetic merge ref, which resolves to an empty repository_url and crashes sphinx_book_theme's repository button; use_download_button was never actually affected by this, it only reads local source files. Verified by building against the pinned rocm-docs-core==1.40.0 in a clean venv (matching CI) -- build succeeds with 0 warnings. Signed-off-by: pmoutsias-amd <peter.moutsias@amd.com> --- docs/rocm-docs/conf.py | 35 +++++++++++++++++++++++++++-------- 1 file changed, 27 insertions(+), 8 deletions(-) diff --git a/docs/rocm-docs/conf.py b/docs/rocm-docs/conf.py index e1d4f9251a..72cc906ee4 100644 --- a/docs/rocm-docs/conf.py +++ b/docs/rocm-docs/conf.py @@ -8,21 +8,32 @@ import logging -class _SuppressUnregisteredProjectWarning(logging.Filter): +class _SuppressKnownUnreleasedRocmDocsCoreWarnings(logging.Filter): """rocm-cli isn't registered in rocm-docs-core's shared projects.yaml catalog yet, so `external_projects_current_project` can never resolve - until that's added upstream. rocm_docs.projects treats an unresolved - current project as None everywhere it's used (doxygen_html, theme - version banner), so the warning is purely informational, not a defect - here, and would otherwise permanently block a `-W` build.""" + until that's added upstream. Separately, the "rocm-ai" flavor used here + was only added to rocm-docs-core after its last release, so the + published package doesn't recognize it yet and falls back to "rocm". + Both are purely informational until rocm-docs-core catches up, and + would otherwise permanently block a `-W` build.""" + + _IGNORED_SUBSTRINGS = ( + "not found in projects", + 'Unsupported theme flavor "rocm-ai"', + ) def filter(self, record: logging.LogRecord) -> bool: - return "not found in projects" not in record.getMessage() + message = record.getMessage() + return not any(s in message for s in self._IGNORED_SUBSTRINGS) -logging.getLogger("sphinx.rocm_docs.projects").addFilter( - _SuppressUnregisteredProjectWarning() +_suppress_known_unreleased_rocm_docs_core_warnings = ( + _SuppressKnownUnreleasedRocmDocsCoreWarnings() ) +for _logger_name in ("sphinx.rocm_docs.projects", "sphinx.rocm_docs.theme"): + logging.getLogger(_logger_name).addFilter( + _suppress_known_unreleased_rocm_docs_core_warnings + ) # -- Project information --------------------------------------------------- @@ -37,6 +48,14 @@ def filter(self, record: logging.LogRecord) -> bool: html_theme = "rocm_docs_theme" html_theme_options = { "flavor": "rocm-ai", + "use_repository_button": True, + "use_download_button": True, + # Pin these instead of letting rocm-docs-core infer them from the local + # git branch: CI's pull_request checkout leaves a detached HEAD on the + # synthetic merge ref, which resolves to an empty repository_url and + # crashes sphinx_book_theme's repository button. + "repository_url": "https://github.com/ROCm/rocm-cli", + "repository_branch": "main", } html_title = "ROCm CLI documentation" # myst.header: `:start-after:` on included README sections strips the heading