From dfe6bc18962bc819d03ba61b40b0652c905e322e Mon Sep 17 00:00:00 2001 From: bong-water-water-bong Date: Fri, 25 Sep 2026 18:40:44 -0300 Subject: [PATCH 1/2] ONNX backend: 1bit serve --device onnx runs ryzenai-server (ONNX Runtime GenAI) third_party/ryzenai-server (lemonade-sdk, MIT) is the engine's ONNX Runtime GenAI backend, run as a child like llama-server. scripts/build-onnx.sh builds it against Microsoft's ONNX Runtime GenAI 0.11.2 and ONNX Runtime 1.23.2 (MIT, pinned, sha256-checked; CPU), or AMD's Ryzen AI Software when RYZENAI_INSTALL_PATH names it (NPU, hybrid). -DONEBIT_ONNX builds it with the engine. A directory with a genai_config.json goes to --device onnx. Co-Authored-By: Claude Opus 5.5 --- .gitmodules | 3 ++ CMakeLists.txt | 11 ++++++ app/serve.cpp | 35 ++++++++++++++++- scripts/build-onnx.sh | 78 ++++++++++++++++++++++++++++++++++++++ third_party/ryzenai-server | 1 + 5 files changed, 127 insertions(+), 1 deletion(-) create mode 100755 scripts/build-onnx.sh create mode 160000 third_party/ryzenai-server diff --git a/.gitmodules b/.gitmodules index 5b61d06c..1dad7983 100644 --- a/.gitmodules +++ b/.gitmodules @@ -77,3 +77,6 @@ url = https://github.com/1bit-MONSTER/lemonade.git branch = main shallow = true +[submodule "third_party/ryzenai-server"] + path = third_party/ryzenai-server + url = https://github.com/lemonade-sdk/ryzenai-server diff --git a/CMakeLists.txt b/CMakeLists.txt index d3f19e30..c8640b0f 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -162,6 +162,17 @@ endif() # Off by default: it needs LibTorch. Builds third_party/comfyui.cpp (GPL-3.0) with # scripts/build-comfyui.sh as its own program; `1bit comfy ` runs it as a # child process. The engine never links it, so the engine itself stays Apache-2.0. +option(ONEBIT_ONNX "Build ryzenai-server (third_party/ryzenai-server) for --device onnx" OFF) +if(ONEBIT_ONNX) + set(ONEBIT_ONNX_SERVER "${CMAKE_BINARY_DIR}/onnx/ryzenai-server") + add_custom_target(onnx_server + COMMAND ${CMAKE_SOURCE_DIR}/scripts/build-onnx.sh ${CMAKE_BINARY_DIR}/onnx + BYPRODUCTS ${ONEBIT_ONNX_SERVER} + USES_TERMINAL) + add_dependencies(onebit onnx_server) + target_compile_definitions(onebit PRIVATE ONEBIT_ONNX_SERVER="${ONEBIT_ONNX_SERVER}") +endif() + option(ONEBIT_COMFYUI "Build ComfyUI.cpp (third_party/comfyui.cpp) for 1bit comfy" OFF) if(ONEBIT_COMFYUI) # fetch the pinned submodule at configure time when it is missing diff --git a/app/serve.cpp b/app/serve.cpp index e4e941ef..be0df5d9 100644 --- a/app/serve.cpp +++ b/app/serve.cpp @@ -174,6 +174,22 @@ std::string default_lean_server(const std::string& device) { #endif } +// ryzenai-server (third_party/ryzenai-server, MIT): ONNX Runtime GenAI models, on the CPU with +// Microsoft's ONNX Runtime, or the NPU / hybrid where AMD's Ryzen AI Software provides it +std::string default_onnx() { + if (const char* e = std::getenv("ONEBIT_ONNX_SERVER"); e && *e) return e; +#ifdef ONEBIT_ONNX_SERVER + return ONEBIT_ONNX_SERVER; +#else + return "ryzenai-server"; +#endif +} + +// an ONNX Runtime GenAI model directory (genai_config.json beside the graph) +bool is_onnx_model_dir(const std::string& dir) { + return fs::is_directory(dir) && fs::exists(fs::path(dir) / "genai_config.json"); +} + std::string default_zinc() { if (const char* e = std::getenv("ONEBIT_ZINC"); e && *e) return e; #ifdef ONEBIT_ZINC_SERVER @@ -482,6 +498,15 @@ Launch launch_for(const Options& o, const std::string& device, int child_port) { std::vector& env = l.env; bool& drop_model = l.drop_model; std::string& set_model = l.set_model; + if (device == "onnx") { + // ryzenai-server: `-m --port

`; the execution mode (CPU, NPU, hybrid) comes from + // the model's genai_config.json + if (o.parallel > 1 || !o.mtp.empty() || o.lean || !o.prefill_device.empty()) + throw std::runtime_error("--device onnx takes no --parallel, --mtp, --lean or --prefill-device"); + argv = {default_onnx(), "-m", o.model, "--port", std::to_string(child_port)}; + if (o.ctx_size > 0) { argv.push_back("--ctx-size"); argv.push_back(std::to_string(o.ctx_size)); } + return l; + } if (device == "mlx") { // lemon-mlx-engine: ` --port

`; it picks the model by // Hugging Face id, so requests carry that id (docs/apple.md). @@ -883,7 +908,7 @@ int serve_child(const Options& o) { void usage(FILE* out) { std::fprintf(out, "usage: 1bit serve -m [--port 8000] [--host 127.0.0.1]\n" - " [--device auto|npu|vulkan|hrx|rocm|zinc|mlx] [--ctx-size N] [--alias NAME]\n" + " [--device auto|npu|vulkan|hrx|rocm|zinc|mlx|onnx] [--ctx-size N] [--alias NAME]\n" " [--llama-server PATH] [--zinc PATH] [--hrx-libhsa PATH] [--mlx-server PATH]\n" " [--prefill-device hrx] [--prefill-min-tokens N] (with --device vulkan)\n" " [--lean] ROCmFPX formats: ROCmFP4 on vulkan, ROCmI4 with --device rocm\n" @@ -895,6 +920,7 @@ void usage(FILE* out) { " [--npu-opt KEY=VALUE ...] an option for a private NPU route (docs/npu.md)\n" " [--laya-model DIR] with --device auto, the Laya scorer picks each request's device (docs/laya.md)\n" " : an NPU model directory (model.q4nx + npu/, or one a private NPU route serves),\n" + " an ONNX Runtime GenAI directory (genai_config.json: --device onnx),\n" " a .gguf file, or with --device mlx a Hugging Face id (mlx-community/...)\n"); } @@ -938,6 +964,13 @@ int run_serve(int argc, char** argv) { } if (o.model.empty()) { usage(stderr); return 2; } + if (is_onnx_model_dir(o.model)) { + if (o.device != "auto" && o.device != "onnx") + throw std::runtime_error("an ONNX Runtime GenAI model directory (genai_config.json) runs on --device onnx"); + Options onnx = o; + onnx.device = "onnx"; + return serve_child(onnx); + } if (fs::is_directory(o.model)) { if (o.lean) throw std::runtime_error("--lean serves a .gguf (ROCmFP4 or ROCmI4)"); if (o.device != "auto" && o.device != "npu") diff --git a/scripts/build-onnx.sh b/scripts/build-onnx.sh new file mode 100755 index 00000000..21937112 --- /dev/null +++ b/scripts/build-onnx.sh @@ -0,0 +1,78 @@ +#!/usr/bin/env bash +# Copyright 2026 bong-water-water-bong +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# build-onnx.sh +# +# Builds ryzenai-server, pinned in third_party/ryzenai-server (lemonade-sdk/ryzenai-server, MIT), +# into /ryzenai-server, with the libraries it loads beside it. It is the engine's ONNX +# Runtime GenAI backend (`1bit serve --device onnx`, docs/onnx.md), run as a child process. +# +# By default it builds against Microsoft's ONNX Runtime GenAI 0.11.2 and ONNX Runtime 1.23.2 +# (both MIT), fetched pinned and checked against their sha256: models run on the CPU. For the +# NPU and hybrid modes, AMD's Ryzen AI Software (its own licence, installed separately) provides +# ONNX Runtime GenAI with the Vitis AI provider: set RYZENAI_INSTALL_PATH to it, and the build +# uses that instead. +set -euo pipefail +prefix=${1:?usage: build-onnx.sh } +root=$(cd "$(dirname "$0")/.." && pwd) +src=$root/third_party/ryzenai-server +if [ ! -f "$src/CMakeLists.txt" ] && git -C "$root" rev-parse --git-dir > /dev/null 2>&1; then + echo "fetching third_party/ryzenai-server" + git -C "$root" submodule update --init third_party/ryzenai-server +fi +[ -f "$src/CMakeLists.txt" ] || { echo "third_party/ryzenai-server is empty: git submodule update --init third_party/ryzenai-server"; exit 1; } +mkdir -p "$prefix" +prefix=$(cd "$prefix" && pwd) +work=$prefix/work +mkdir -p "$work" + +OGA=https://github.com/microsoft/onnxruntime-genai/releases/download/v0.11.2/onnxruntime-genai-0.11.2-linux-x64.tar.gz +OGA_SHA=0f63ef0fd3ba6a5c49b0b7d58e28e734c8a95deb8162221f50c716773341b286 +ORT=https://github.com/microsoft/onnxruntime/releases/download/v1.23.2/onnxruntime-linux-x64-1.23.2.tgz +ORT_SHA=1fa4dcaef22f6f7d5cd81b28c2800414350c10116f5fdd46a2160082551c5f9b + +fetch() { # fetch : download once, verify every time + local f="$work/$3" + if ! { [ -f "$f" ] && echo "$2 $f" | sha256sum -c --quiet 2>/dev/null; }; then + curl -fsSL -o "$f.part" "$1" + echo "$2 $f.part" | sha256sum -c --quiet || { echo "build-onnx.sh: $1: checksum mismatch"; exit 1; } + mv "$f.part" "$f" + fi + echo "$f" +} + +if [ -n "${RYZENAI_INSTALL_PATH:-}" ]; then + oga_root=$RYZENAI_INSTALL_PATH # Ryzen AI Software: NPU, hybrid and CPU + ort_lib="" +else + # Microsoft's releases, in the layout ryzenai-server's build looks for (/deployment/lib) + rm -rf "$work/oga" "$work/ort" && mkdir -p "$work/oga" "$work/ort" + tar xzf "$(fetch $OGA $OGA_SHA oga.tgz)" -C "$work/oga" --strip-components=1 + tar xzf "$(fetch $ORT $ORT_SHA ort.tgz)" -C "$work/ort" --strip-components=1 + oga_root=$work/oga-root + mkdir -p "$oga_root/deployment" + ln -sfn "$work/oga/lib" "$oga_root/deployment/lib" + ort_lib=$work/ort/lib +fi + +# DT_RPATH $ORIGIN: the binary finds its libraries beside it, and so does ONNX Runtime GenAI when +# it loads libonnxruntime.so (a RUNPATH would not cover that dlopen) +cmake -S "$src" -B "$work/build" -DCMAKE_BUILD_TYPE=Release -DOGA_ROOT="$oga_root" \ + "-DCMAKE_EXE_LINKER_FLAGS=-Wl,--disable-new-dtags -Wl,-rpath,\$ORIGIN" > "$work/cmake.log" +cmake --build "$work/build" -j"$(nproc)" > "$work/build.log" 2>&1 || { tail -20 "$work/build.log"; exit 1; } +cp -a "$work/build/bin/." "$prefix/" +[ -n "$ort_lib" ] && cp -a "$ort_lib"/libonnxruntime.so* "$prefix/" +echo "built $prefix/ryzenai-server (ryzenai-server $(git -C "$src" rev-parse --short=12 HEAD 2>/dev/null || echo unknown))" diff --git a/third_party/ryzenai-server b/third_party/ryzenai-server new file mode 160000 index 00000000..69765cc5 --- /dev/null +++ b/third_party/ryzenai-server @@ -0,0 +1 @@ +Subproject commit 69765cc591968521af80a89bc3eab83004f48668 From 084b3a4552f28e119827709afe031c4770ac015c Mon Sep 17 00:00:00 2001 From: bong-water-water-bong Date: Fri, 25 Sep 2026 18:42:25 -0300 Subject: [PATCH 2/2] docs/onnx.md, NOTICE: the ONNX Runtime GenAI backend and its dependencies Co-Authored-By: Claude Opus 5.5 --- NOTICE | 17 ++++++++++++++ docs/onnx.md | 60 +++++++++++++++++++++++++++++++++++++++++++++++++ docs/windows.md | 2 +- tools/site.py | 2 +- 4 files changed, 79 insertions(+), 2 deletions(-) create mode 100644 docs/onnx.md diff --git a/NOTICE b/NOTICE index c7b5d2be..434b6083 100644 --- a/NOTICE +++ b/NOTICE @@ -89,6 +89,23 @@ Lemonade (third_party/lemonade) built as its own program (scripts/build-lemonade.sh) that runs `1bit serve` as a backend. None of its code is linked into this product. +ryzenai-server (third_party/ryzenai-server) + https://github.com/lemonade-sdk/ryzenai-server + Copyright (c) 2026 lemonade-sdk. License: MIT. + Built as a separate program (scripts/build-onnx.sh) that `1bit serve --device + onnx` runs as a child process; none of its code is linked into this product. + Its build fetches cpp-httplib (MIT), nlohmann/json (MIT) and the ONNX Runtime + GenAI headers (MIT). The AMD_LICENSE file in its repository covers AMD's Ryzen + AI Software, which this product does not ship: the NPU and hybrid modes use it + only when the user installs it (RYZENAI_INSTALL_PATH). + +ONNX Runtime GenAI and ONNX Runtime + https://github.com/microsoft/onnxruntime-genai + https://github.com/microsoft/onnxruntime + Copyright (c) Microsoft Corporation. License: MIT. + scripts/build-onnx.sh fetches the release libraries (ONNX Runtime GenAI 0.11.2, + ONNX Runtime 1.23.2, pinned by sha256) and places them beside ryzenai-server. + ComfyUI.cpp (third_party/comfyui.cpp) https://github.com/1bit-MONSTER/comfyui.cpp A C++ port of ComfyUI (https://github.com/Comfy-Org/ComfyUI, Copyright (C) diff --git a/docs/onnx.md b/docs/onnx.md new file mode 100644 index 00000000..dbb76ee6 --- /dev/null +++ b/docs/onnx.md @@ -0,0 +1,60 @@ + + +# ONNX Runtime GenAI + +`1bit serve --device onnx` serves ONNX Runtime GenAI models: a directory with `genai_config.json`, `model.onnx` and the tokenizer, the format of Lemonade's ONNX models. The engine runs [ryzenai-server](https://github.com/lemonade-sdk/ryzenai-server) (MIT, pinned in `third_party/ryzenai-server`) as a child process, the way it runs llama-server, and forwards the OpenAI API to it. A directory with a `genai_config.json` goes to `--device onnx` on its own: + +```sh +cmake -B build -DONEBIT_ONNX=ON # builds ryzenai-server into build/onnx/ +build/1bit serve -m ~/models/oga/Qwen2.5-0.5B-Instruct-cpu-onnx +``` + +## Where it runs + +The execution mode comes from the model's `genai_config.json`: + +| mode | runtime | how to get it | +| --- | --- | --- | +| CPU | Microsoft's ONNX Runtime GenAI 0.11.2 + ONNX Runtime 1.23.2 (MIT) | `scripts/build-onnx.sh`: fetched pinned, sha256-checked | +| NPU, hybrid (NPU + iGPU) | AMD's Ryzen AI Software (the Vitis AI execution provider) | its own licence and download; set `RYZENAI_INSTALL_PATH` to it before building | + +The engine does not ship Ryzen AI Software. With it installed, the same build serves Lemonade's `-npu` and `-hybrid` ONNX models. + +## Build + +`scripts/build-onnx.sh ` (run by `-DONEBIT_ONNX=ON`): +- **Libraries.** It fetches the two MIT releases, or uses `$RYZENAI_INSTALL_PATH`. +- **Build.** It builds ryzenai-server against them. +- **Output.** It puts the binary and the libraries it loads in ``. The binary carries `DT_RPATH $ORIGIN`, so ONNX Runtime GenAI finds `libonnxruntime.so` beside it when it loads it at run time. + +`ONEBIT_ONNX_SERVER` or `--device onnx` with another `ryzenai-server` on `PATH` overrides the build's. + +## Measured + +On Strix Halo, 2026-09-25, with `amd/Qwen2.5-0.5B-Instruct-quantized_int4-float16-cpu-onnx` (CPU, int4) and `1bit serve -m

` (no `--device`): +- **Chat.** "The capital of France is Paris." and a list of three colors, at 125–129 tok/s decode. +- **Streaming.** A reply streamed as SSE. +- **Model list.** `/v1/models` names the model with device `onnx`. +- **Shutdown.** Stopping `1bit serve` stopped the backend. + +llama-server's extra routes (`/tokenize`, `/slots`, …) answer 501 on `--device onnx`. + +## Next + +- **Windows.** `ryzenai-server.exe` in `scripts/build-windows.sh`: ONNX Runtime GenAI's Windows builds include DirectML, the GPU on Windows. +- **The Ryzen AI NPU.** On a machine with Ryzen AI Software installed. diff --git a/docs/windows.md b/docs/windows.md index c12b7b09..3cec43de 100644 --- a/docs/windows.md +++ b/docs/windows.md @@ -62,5 +62,5 @@ It has not run on a real Windows machine yet. ## Next -- **An ONNX Runtime GenAI backend:** Lemonade's ONNX and hybrid models, and the Ryzen AI NPU on Windows through the Vitis AI execution provider. +- **The ONNX backend on Windows:** `--device onnx` works on Linux ([ONNX Runtime](onnx.md)); `ryzenai-server.exe` in this script is next. - **The NPU lane on Windows XRT.** diff --git a/tools/site.py b/tools/site.py index fe4bdbef..927bf283 100644 --- a/tools/site.py +++ b/tools/site.py @@ -77,7 +77,7 @@ ("Start", [("serve", "1bit serve"), ("lemonade", "Lemonade")]), ("Devices", [("npu", "NPU"), ("hrx", "HRX + Vulkan"), ("vulkan", "Vulkan (upstream)"), ("lean", "Lean (ROCmFP4, ROCmI4)"), ("zinc", "ZINC"), ("apple", "Apple Silicon")]), - ("Components", [("moe-streaming", "MoE streaming"), ("laya", "Laya router"), ("registry", "Model registry"), ("arch-gaps", "Architecture gaps"), ("windows", "Windows"), ("comfyui", "ComfyUI.cpp"), ("tokenizers", "Tokenizers"), ("kernel", "Linux kernel")]), + ("Components", [("moe-streaming", "MoE streaming"), ("laya", "Laya router"), ("registry", "Model registry"), ("arch-gaps", "Architecture gaps"), ("windows", "Windows"), ("onnx", "ONNX Runtime"), ("comfyui", "ComfyUI.cpp"), ("tokenizers", "Tokenizers"), ("kernel", "Linux kernel")]), ("Project", [("PORTING", "Porting map")]), ]