ONNX backend: 1bit serve --device onnx runs ONNX Runtime GenAI models (ryzenai-server)

.gitmodules

@@ -77,3 +77,6 @@
 	url = https://github.com/1bit-MONSTER/lemonade.git
 	branch = main
 	shallow = true
+[submodule "third_party/ryzenai-server"]
+	path = third_party/ryzenai-server
+	url = https://github.com/lemonade-sdk/ryzenai-server

CMakeLists.txt

@@ -162,6 +162,17 @@ endif()
 # Off by default: it needs LibTorch. Builds third_party/comfyui.cpp (GPL-3.0) with
 # scripts/build-comfyui.sh as its own program; `1bit comfy <workflow.json>` runs it as a
 # child process. The engine never links it, so the engine itself stays Apache-2.0.
+option(ONEBIT_ONNX "Build ryzenai-server (third_party/ryzenai-server) for --device onnx" OFF)
+if(ONEBIT_ONNX)
+    set(ONEBIT_ONNX_SERVER "${CMAKE_BINARY_DIR}/onnx/ryzenai-server")
+    add_custom_target(onnx_server
+        COMMAND ${CMAKE_SOURCE_DIR}/scripts/build-onnx.sh ${CMAKE_BINARY_DIR}/onnx
+        BYPRODUCTS ${ONEBIT_ONNX_SERVER}
+        USES_TERMINAL)
+    add_dependencies(onebit onnx_server)
+    target_compile_definitions(onebit PRIVATE ONEBIT_ONNX_SERVER="${ONEBIT_ONNX_SERVER}")
+endif()
+
 option(ONEBIT_COMFYUI "Build ComfyUI.cpp (third_party/comfyui.cpp) for 1bit comfy" OFF)
 if(ONEBIT_COMFYUI)
     # fetch the pinned submodule at configure time when it is missing

NOTICE

@@ -89,6 +89,23 @@ Lemonade (third_party/lemonade)
   built as its own program (scripts/build-lemonade.sh) that runs `1bit serve`
   as a backend. None of its code is linked into this product.
 
+ryzenai-server (third_party/ryzenai-server)
+  https://github.com/lemonade-sdk/ryzenai-server
+  Copyright (c) 2026 lemonade-sdk. License: MIT.
+  Built as a separate program (scripts/build-onnx.sh) that `1bit serve --device
+  onnx` runs as a child process; none of its code is linked into this product.
+  Its build fetches cpp-httplib (MIT), nlohmann/json (MIT) and the ONNX Runtime
+  GenAI headers (MIT). The AMD_LICENSE file in its repository covers AMD's Ryzen
+  AI Software, which this product does not ship: the NPU and hybrid modes use it
+  only when the user installs it (RYZENAI_INSTALL_PATH).
+
+ONNX Runtime GenAI and ONNX Runtime
+  https://github.com/microsoft/onnxruntime-genai
+  https://github.com/microsoft/onnxruntime
+  Copyright (c) Microsoft Corporation. License: MIT.
+  scripts/build-onnx.sh fetches the release libraries (ONNX Runtime GenAI 0.11.2,
+  ONNX Runtime 1.23.2, pinned by sha256) and places them beside ryzenai-server.
+
 ComfyUI.cpp (third_party/comfyui.cpp)
   https://github.com/1bit-MONSTER/comfyui.cpp
   A C++ port of ComfyUI (https://github.com/Comfy-Org/ComfyUI, Copyright (C)

app/serve.cpp

@@ -174,6 +174,22 @@ std::string default_lean_server(const std::string& device) {
 #endif
 }
 
+// ryzenai-server (third_party/ryzenai-server, MIT): ONNX Runtime GenAI models, on the CPU with
+// Microsoft's ONNX Runtime, or the NPU / hybrid where AMD's Ryzen AI Software provides it
+std::string default_onnx() {
+    if (const char* e = std::getenv("ONEBIT_ONNX_SERVER"); e && *e) return e;
+#ifdef ONEBIT_ONNX_SERVER
+    return ONEBIT_ONNX_SERVER;
+#else
+    return "ryzenai-server";
+#endif
+}
+
+// an ONNX Runtime GenAI model directory (genai_config.json beside the graph)
+bool is_onnx_model_dir(const std::string& dir) {
+    return fs::is_directory(dir) && fs::exists(fs::path(dir) / "genai_config.json");
+}
+
 std::string default_zinc() {
     if (const char* e = std::getenv("ONEBIT_ZINC"); e && *e) return e;
 #ifdef ONEBIT_ZINC_SERVER
@@ -482,6 +498,15 @@ Launch launch_for(const Options& o, const std::string& device, int child_port) {
     std::vector<std::string>& env = l.env;
     bool& drop_model = l.drop_model;
     std::string& set_model = l.set_model;
+    if (device == "onnx") {
+        // ryzenai-server: `-m <dir> --port <p>`; the execution mode (CPU, NPU, hybrid) comes from
+        // the model's genai_config.json
+        if (o.parallel > 1 || !o.mtp.empty() || o.lean || !o.prefill_device.empty())
+            throw std::runtime_error("--device onnx takes no --parallel, --mtp, --lean or --prefill-device");
+        argv = {default_onnx(), "-m", o.model, "--port", std::to_string(child_port)};
+        if (o.ctx_size > 0) { argv.push_back("--ctx-size"); argv.push_back(std::to_string(o.ctx_size)); }
+        return l;
+    }
     if (device == "mlx") {
         // lemon-mlx-engine: `<server> <hf id> --port <p>`; it picks the model by
         // Hugging Face id, so requests carry that id (docs/apple.md).
@@ -883,7 +908,7 @@ int serve_child(const Options& o) {
 void usage(FILE* out) {
     std::fprintf(out,
                  "usage: 1bit serve -m <model> [--port 8000] [--host 127.0.0.1]\n"
-                 "                  [--device auto|npu|vulkan|hrx|rocm|zinc|mlx] [--ctx-size N] [--alias NAME]\n"
+                 "                  [--device auto|npu|vulkan|hrx|rocm|zinc|mlx|onnx] [--ctx-size N] [--alias NAME]\n"
                  "                  [--llama-server PATH] [--zinc PATH] [--hrx-libhsa PATH] [--mlx-server PATH]\n"
                  "                  [--prefill-device hrx] [--prefill-min-tokens N]   (with --device vulkan)\n"
                  "                  [--lean]   ROCmFPX formats: ROCmFP4 on vulkan, ROCmI4 with --device rocm\n"
@@ -895,6 +920,7 @@ void usage(FILE* out) {
                  "                  [--npu-opt KEY=VALUE ...]   an option for a private NPU route (docs/npu.md)\n"
                  "                  [--laya-model DIR]   with --device auto, the Laya scorer picks each request's device (docs/laya.md)\n"
                  "  <model>: an NPU model directory (model.q4nx + npu/, or one a private NPU route serves),\n"
+                 "           an ONNX Runtime GenAI directory (genai_config.json: --device onnx),\n"
                  "           a .gguf file, or with --device mlx a Hugging Face id (mlx-community/...)\n");
 }
 
@@ -938,6 +964,13 @@ int run_serve(int argc, char** argv) {
     }
     if (o.model.empty()) { usage(stderr); return 2; }
 
+    if (is_onnx_model_dir(o.model)) {
+        if (o.device != "auto" && o.device != "onnx")
+            throw std::runtime_error("an ONNX Runtime GenAI model directory (genai_config.json) runs on --device onnx");
+        Options onnx = o;
+        onnx.device = "onnx";
+        return serve_child(onnx);
+    }
     if (fs::is_directory(o.model)) {
         if (o.lean) throw std::runtime_error("--lean serves a .gguf (ROCmFP4 or ROCmI4)");
         if (o.device != "auto" && o.device != "npu")

docs/onnx.md

@@ -0,0 +1,60 @@
+<!--
+Copyright 2026 bong-water-water-bong
+SPDX-License-Identifier: Apache-2.0
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+-->
+
+# ONNX Runtime GenAI
+
+`1bit serve --device onnx` serves ONNX Runtime GenAI models: a directory with `genai_config.json`, `model.onnx` and the tokenizer, the format of Lemonade's ONNX models. The engine runs [ryzenai-server](https://github.com/lemonade-sdk/ryzenai-server) (MIT, pinned in `third_party/ryzenai-server`) as a child process, the way it runs llama-server, and forwards the OpenAI API to it. A directory with a `genai_config.json` goes to `--device onnx` on its own:
+
+```sh
+cmake -B build -DONEBIT_ONNX=ON          # builds ryzenai-server into build/onnx/
+build/1bit serve -m ~/models/oga/Qwen2.5-0.5B-Instruct-cpu-onnx
+```
+
+## Where it runs
+
+The execution mode comes from the model's `genai_config.json`:
+
+| mode | runtime | how to get it |
+| --- | --- | --- |
+| CPU | Microsoft's ONNX Runtime GenAI 0.11.2 + ONNX Runtime 1.23.2 (MIT) | `scripts/build-onnx.sh`: fetched pinned, sha256-checked |
+| NPU, hybrid (NPU + iGPU) | AMD's Ryzen AI Software (the Vitis AI execution provider) | its own licence and download; set `RYZENAI_INSTALL_PATH` to it before building |
+
+The engine does not ship Ryzen AI Software. With it installed, the same build serves Lemonade's `-npu` and `-hybrid` ONNX models.
+
+## Build
+
+`scripts/build-onnx.sh <prefix>` (run by `-DONEBIT_ONNX=ON`):
+- **Libraries.** It fetches the two MIT releases, or uses `$RYZENAI_INSTALL_PATH`.
+- **Build.** It builds ryzenai-server against them.
+- **Output.** It puts the binary and the libraries it loads in `<prefix>`. The binary carries `DT_RPATH $ORIGIN`, so ONNX Runtime GenAI finds `libonnxruntime.so` beside it when it loads it at run time.
+
+`ONEBIT_ONNX_SERVER` or `--device onnx` with another `ryzenai-server` on `PATH` overrides the build's.
+
+## Measured
+
+On Strix Halo, 2026-09-25, with `amd/Qwen2.5-0.5B-Instruct-quantized_int4-float16-cpu-onnx` (CPU, int4) and `1bit serve -m <dir>` (no `--device`):
+- **Chat.** "The capital of France is Paris." and a list of three colors, at 125–129 tok/s decode.
+- **Streaming.** A reply streamed as SSE.
+- **Model list.** `/v1/models` names the model with device `onnx`.
+- **Shutdown.** Stopping `1bit serve` stopped the backend.
+
+llama-server's extra routes (`/tokenize`, `/slots`, …) answer 501 on `--device onnx`.
+
+## Next
+
+- **Windows.** `ryzenai-server.exe` in `scripts/build-windows.sh`: ONNX Runtime GenAI's Windows builds include DirectML, the GPU on Windows.
+- **The Ryzen AI NPU.** On a machine with Ryzen AI Software installed.

docs/windows.md

@@ -62,5 +62,5 @@ It has not run on a real Windows machine yet.
 
 ## Next
 
-- **An ONNX Runtime GenAI backend:** Lemonade's ONNX and hybrid models, and the Ryzen AI NPU on Windows through the Vitis AI execution provider.
+- **The ONNX backend on Windows:** `--device onnx` works on Linux ([ONNX Runtime](onnx.md)); `ryzenai-server.exe` in this script is next.
 - **The NPU lane on Windows XRT.**

scripts/build-onnx.sh

@@ -0,0 +1,78 @@
+#!/usr/bin/env bash
+# Copyright 2026 bong-water-water-bong
+# SPDX-License-Identifier: Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# build-onnx.sh <prefix>
+#
+# Builds ryzenai-server, pinned in third_party/ryzenai-server (lemonade-sdk/ryzenai-server, MIT),
+# into <prefix>/ryzenai-server, with the libraries it loads beside it. It is the engine's ONNX
+# Runtime GenAI backend (`1bit serve --device onnx`, docs/onnx.md), run as a child process.
+#
+# By default it builds against Microsoft's ONNX Runtime GenAI 0.11.2 and ONNX Runtime 1.23.2
+# (both MIT), fetched pinned and checked against their sha256: models run on the CPU. For the
+# NPU and hybrid modes, AMD's Ryzen AI Software (its own licence, installed separately) provides
+# ONNX Runtime GenAI with the Vitis AI provider: set RYZENAI_INSTALL_PATH to it, and the build
+# uses that instead.
+set -euo pipefail
+prefix=${1:?usage: build-onnx.sh <prefix>}
+root=$(cd "$(dirname "$0")/.." && pwd)
+src=$root/third_party/ryzenai-server
+if [ ! -f "$src/CMakeLists.txt" ] && git -C "$root" rev-parse --git-dir > /dev/null 2>&1; then
+    echo "fetching third_party/ryzenai-server"
+    git -C "$root" submodule update --init third_party/ryzenai-server
+fi
+[ -f "$src/CMakeLists.txt" ] || { echo "third_party/ryzenai-server is empty: git submodule update --init third_party/ryzenai-server"; exit 1; }
+mkdir -p "$prefix"
+prefix=$(cd "$prefix" && pwd)
+work=$prefix/work
+mkdir -p "$work"
+
+OGA=https://github.com/microsoft/onnxruntime-genai/releases/download/v0.11.2/onnxruntime-genai-0.11.2-linux-x64.tar.gz
+OGA_SHA=0f63ef0fd3ba6a5c49b0b7d58e28e734c8a95deb8162221f50c716773341b286
+ORT=https://github.com/microsoft/onnxruntime/releases/download/v1.23.2/onnxruntime-linux-x64-1.23.2.tgz
+ORT_SHA=1fa4dcaef22f6f7d5cd81b28c2800414350c10116f5fdd46a2160082551c5f9b
+
+fetch() {   # fetch <url> <sha256> <file>: download once, verify every time
+    local f="$work/$3"
+    if ! { [ -f "$f" ] && echo "$2  $f" | sha256sum -c --quiet 2>/dev/null; }; then
+        curl -fsSL -o "$f.part" "$1"
+        echo "$2  $f.part" | sha256sum -c --quiet || { echo "build-onnx.sh: $1: checksum mismatch"; exit 1; }
+        mv "$f.part" "$f"
+    fi
+    echo "$f"
+}
+
+if [ -n "${RYZENAI_INSTALL_PATH:-}" ]; then
+    oga_root=$RYZENAI_INSTALL_PATH       # Ryzen AI Software: NPU, hybrid and CPU
+    ort_lib=""
+else
+    # Microsoft's releases, in the layout ryzenai-server's build looks for (<root>/deployment/lib)
+    rm -rf "$work/oga" "$work/ort" && mkdir -p "$work/oga" "$work/ort"
+    tar xzf "$(fetch $OGA $OGA_SHA oga.tgz)" -C "$work/oga" --strip-components=1
+    tar xzf "$(fetch $ORT $ORT_SHA ort.tgz)" -C "$work/ort" --strip-components=1
+    oga_root=$work/oga-root
+    mkdir -p "$oga_root/deployment"
+    ln -sfn "$work/oga/lib" "$oga_root/deployment/lib"
+    ort_lib=$work/ort/lib
+fi
+
+# DT_RPATH $ORIGIN: the binary finds its libraries beside it, and so does ONNX Runtime GenAI when
+# it loads libonnxruntime.so (a RUNPATH would not cover that dlopen)
+cmake -S "$src" -B "$work/build" -DCMAKE_BUILD_TYPE=Release -DOGA_ROOT="$oga_root" \
+    "-DCMAKE_EXE_LINKER_FLAGS=-Wl,--disable-new-dtags -Wl,-rpath,\$ORIGIN" > "$work/cmake.log"
+cmake --build "$work/build" -j"$(nproc)" > "$work/build.log" 2>&1 || { tail -20 "$work/build.log"; exit 1; }
+cp -a "$work/build/bin/." "$prefix/"
+[ -n "$ort_lib" ] && cp -a "$ort_lib"/libonnxruntime.so* "$prefix/"
+echo "built $prefix/ryzenai-server (ryzenai-server $(git -C "$src" rev-parse --short=12 HEAD 2>/dev/null || echo unknown))"

tools/site.py

@@ -77,7 +77,7 @@
     ("Start", [("serve", "1bit serve"), ("lemonade", "Lemonade")]),
     ("Devices", [("npu", "NPU"), ("hrx", "HRX + Vulkan"), ("vulkan", "Vulkan (upstream)"), ("lean", "Lean (ROCmFP4, ROCmI4)"),
                  ("zinc", "ZINC"), ("apple", "Apple Silicon")]),
-    ("Components", [("moe-streaming", "MoE streaming"), ("laya", "Laya router"), ("registry", "Model registry"), ("arch-gaps", "Architecture gaps"), ("windows", "Windows"), ("comfyui", "ComfyUI.cpp"), ("tokenizers", "Tokenizers"), ("kernel", "Linux kernel")]),
+    ("Components", [("moe-streaming", "MoE streaming"), ("laya", "Laya router"), ("registry", "Model registry"), ("arch-gaps", "Architecture gaps"), ("windows", "Windows"), ("onnx", "ONNX Runtime"), ("comfyui", "ComfyUI.cpp"), ("tokenizers", "Tokenizers"), ("kernel", "Linux kernel")]),
     ("Project", [("PORTING", "Porting map")]),
 ]