bong-water-water-bong · Sep 24, 2026 · Sep 24, 2026 · Sep 24, 2026 · Sep 24, 2026
diff --git a/app/serve.cpp b/app/serve.cpp
@@ -13,7 +13,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-// `1bit serve -m <model> [--port 8000] [--device auto|npu|vulkan|hrx|zinc] [--lean]`
+// `1bit serve -m <model> [--port 8000] [--device auto|npu|vulkan|hrx|rocm|zinc] [--lean] [--mtp HEAD]`
 //
 // The engine's one front door (docs/serve.md): one model per process behind an
 // OpenAI-compatible API. It is how the engine runs inside Lemonade: Lemonade's
@@ -34,9 +34,13 @@
 //                                                  long prompt prefixes on HRX0 over one
 //                                                  shared KV cache (docs/hrx.md)
 //   a .gguf, --device zinc                      -> this build's zinc
+//   a .gguf, --device rocm                      -> the ROCm build's llama-server on ROCm0
+//                                                  (ONEBIT_LEAN_ROCM: any GGUF, and ROCmI4 with
+//                                                  the gfx1151 W4A4 path; docs/lean.md)
 //   a .gguf, --lean                             -> the lean build's llama-server (ROCmFPX's
-//                                                  formats) on Vulkan0; with --device rocm,
-//                                                  its ROCm build on ROCm0 (docs/lean.md)
+//                                                  formats) on Vulkan0 (docs/lean.md)
+//   --mtp <head.gguf> on any llama.cpp route    -> multi-token prediction: the model's MTP
+//                                                  head drafts, the model verifies (docs/serve.md)
 //   a Hugging Face id, --device mlx (macOS)     -> lemon-mlx-engine's server
 // For a .gguf, the engine starts that server as a private child on a loopback
 // port and forwards the OpenAI routes to it, streaming included. `auto` picks
@@ -89,6 +93,8 @@ struct Options {
     std::string prefill_device;
     int prefill_min_tokens = 0;
     bool lean = false;
+    std::string mtp;
+    int mtp_max = 0;
 };
 
 // HRX dlopens the HSA runtime, and a distro libhsa rejects gfx1151's
@@ -132,7 +138,7 @@ std::string default_lean_server(const std::string& device) {
 #ifdef ONEBIT_LEAN_ROCM_SERVER
         return ONEBIT_LEAN_ROCM_SERVER;
 #else
-        throw std::runtime_error("--lean --device rocm needs a build with -DONEBIT_LEAN=ON -DONEBIT_LEAN_ROCM=ON");
+        throw std::runtime_error("--device rocm needs a build with -DONEBIT_LEAN=ON -DONEBIT_LEAN_ROCM=ON");
 #endif
     }
 #ifdef ONEBIT_LEAN_SERVER
@@ -320,11 +326,12 @@ int serve_child(const Options& o) {
         // Hugging Face id, so requests carry that id (docs/apple.md).
         argv = {o.mlx.empty() ? default_mlx() : o.mlx, o.model, "--port", std::to_string(child_port)};
         set_model = o.model;
-    } else if (o.lean) {
-        // ROCmFP4 on Vulkan0, or ROCmI4 on ROCm0 (the W4A4 path lives in that build)
+    } else if (o.lean || device == "rocm") {
+        // --lean: ROCmFPX's formats on Vulkan0. --device rocm: the ROCm build (ROCmFPX's tree,
+        // which reads every GGUF) on ROCm0; ROCmI4 files use its W4A4 path there.
         if (device != "vulkan" && device != "rocm")
             throw std::runtime_error("--lean runs on --device vulkan or rocm");
-        if (!o.prefill_device.empty()) throw std::runtime_error("--lean does not take --prefill-device");
+        if (!o.prefill_device.empty()) throw std::runtime_error("--prefill-device works with --device vulkan only");
         argv = {o.llama_server.empty() ? default_lean_server(device) : o.llama_server,
                 "-m", o.model, "--host", "127.0.0.1", "--port", std::to_string(child_port),
                 "--device", device == "rocm" ? "ROCm0" : "Vulkan0", "-ngl", "99", "--jinja"};
@@ -354,7 +361,13 @@ int serve_child(const Options& o) {
         env.push_back("RADV_PERFTEST=coop_matrix");
         drop_model = true;  // zinc rejects any model id but its own
     } else {
-        throw std::runtime_error("--device " + o.device + " cannot run a .gguf (vulkan, hrx or zinc)");
+        throw std::runtime_error("--device " + o.device + " cannot run a .gguf (vulkan, hrx, rocm or zinc)");
+    }
+    if (!o.mtp.empty()) {
+        // the MTP head drafts tokens on the same device; the model checks them in one batch
+        if (device == "zinc" || device == "mlx") throw std::runtime_error("--mtp works on the llama.cpp devices (vulkan, hrx, rocm)");
+        argv.insert(argv.end(), {"--spec-type", "draft-mtp", "-md", o.mtp, "-ngld", "99"});
+        if (o.mtp_max > 0) { argv.push_back("--spec-draft-n-max"); argv.push_back(std::to_string(o.mtp_max)); }
     }
 
     const std::string id = model_id(o);
@@ -419,10 +432,11 @@ int serve_child(const Options& o) {
 void usage(FILE* out) {
     std::fprintf(out,
                  "usage: 1bit serve -m <model> [--port 8000] [--host 127.0.0.1]\n"
-                 "                  [--device auto|npu|vulkan|hrx|zinc|mlx] [--ctx-size N] [--alias NAME]\n"
+                 "                  [--device auto|npu|vulkan|hrx|rocm|zinc|mlx] [--ctx-size N] [--alias NAME]\n"
                  "                  [--llama-server PATH] [--zinc PATH] [--hrx-libhsa PATH] [--mlx-server PATH]\n"
                  "                  [--prefill-device hrx] [--prefill-min-tokens N]   (with --device vulkan)\n"
                  "                  [--lean]   ROCmFPX formats: ROCmFP4 on vulkan, ROCmI4 with --device rocm\n"
+                 "                  [--mtp HEAD.gguf] [--mtp-max N]   multi-token prediction (vulkan, hrx, rocm)\n"
                  "  <model>: an NPU model directory (model.q4nx + npu/), a .gguf file, or with\n"
                  "           --device mlx a Hugging Face id (mlx-community/...)\n");
 }
@@ -450,6 +464,8 @@ int run_serve(int argc, char** argv) {
         else if (a == "--prefill-device") o.prefill_device = next();
         else if (a == "--prefill-min-tokens") o.prefill_min_tokens = std::stoi(next());
         else if (a == "--lean") o.lean = true;
+        else if (a == "--mtp") o.mtp = next();
+        else if (a == "--mtp-max") o.mtp_max = std::stoi(next());
         else if (a == "-h" || a == "--help") { usage(stdout); return 0; }
         else throw std::runtime_error("unknown option " + a);
     }

diff --git a/docs/lean.md b/docs/lean.md
@@ -49,6 +49,38 @@ MTP chat is decode speed with the MTP head on three prompts (code / prose / shor
 - The ROCmFPX files were quantized without an importance matrix; UD-Q4_K_XL was
   made with one. An imatrix would narrow the accuracy gap somewhat.
 
+## Round 2: with Unsloth's imatrix, and what to use (2026-09-24)
+
+Round 1's ROCmFPX files had no importance matrix. Rebuilt with Unsloth's own
+(`imatrix_unsloth.gguf`, the one UD-Q4_K_XL was made with), next to Unsloth's smaller files:
+
+| File | Size | Mean KLD | Same top token | Best decode, tok/s |
+|---|---|---|---|---|
+| UD-Q4_K_XL (default) | 16.4 GiB | **0.008** | **95.3%** | 12.2 (Vulkan); 35.0 with `--mtp` |
+| UD-IQ4_XS (Unsloth) | 13.3 GiB | 0.019 | 93.3% | 15.0 (Vulkan) |
+| **UD-Q3_K_XL (Unsloth)** | 12.2 GiB | 0.028 | 92.1% | **15.8 (Vulkan)** |
+| ROCmI4 + imatrix | 13.9 GiB | 0.036 | 91.3% | 13.5 (ROCm W4A4; prompt 455) |
+| ROCmFP4 + imatrix | 13.8 GiB | 0.045 | 89.6% | 14.1 (Vulkan) |
+| ROCmFP2 + imatrix | 8.6 GiB | 0.341 | 75.5% | 21.2 (Vulkan) |
+
+- **For a smaller, faster file, use Unsloth's UD-Q3_K_XL on the default route** (no
+  `--lean` needed): it is smaller, faster and closer to the model than either
+  4-bit ROCmFPX format, even with the imatrix.
+- **`--lean` keeps two jobs:** ROCmI4 for the fastest prompt processing (455 tok/s
+  with W4A4 on ROCm), and ROCmFP2 when memory is the limit (8.6 GiB, 21 tok/s, at a
+  large accuracy cost: 75% top-token agreement).
+- **MTP beats every format change:** `--mtp` on the default file gives 2.4-2.9x
+  (docs/serve.md), more than any quant here.
+
+## Two backends at once
+
+`Vulkan0`, `ROCm0` and `HRX0` are one GPU. Decode on both at the same time
+(UD-Q4_K_XL, tg256): Vulkan alone 12.2 tok/s, ROCm alone 11.9; **Vulkan + ROCm together
+7.5 + 6.8 = 14.3 (+18% total)**; two Vulkan processes 6.3 + 6.3 = 12.7 (+4%). A single
+stream uses about 200 of the bus's roughly 256 GB/s, and two different drivers fill the
+gap better than two copies of one. Each stream slows down, so it pays for serving
+several requests at once (one per backend), not for one chat.
+
 ## Build
 
 ```

diff --git a/docs/serve.md b/docs/serve.md
@@ -23,9 +23,10 @@ OpenAI client.
 
 ```sh
 1bit serve -m <model> [--port 8000] [--host 127.0.0.1]
-           [--device auto|npu|vulkan|hrx|zinc|mlx] [--ctx-size N] [--alias NAME]
+           [--device auto|npu|vulkan|hrx|rocm|zinc|mlx] [--ctx-size N] [--alias NAME]
            [--llama-server PATH] [--zinc PATH] [--hrx-libhsa PATH] [--mlx-server PATH]
            [--prefill-device hrx] [--prefill-min-tokens N] [--lean]
+           [--mtp HEAD.gguf] [--mtp-max N]
 ```
 
 One model per process:
@@ -46,7 +47,7 @@ One model per process:
 | `.gguf` | `hrx` | the HRX build's llama-server on `HRX0` (docs/hrx.md) |
 | `.gguf` | `vulkan --prefill-device hrx` | the HRX build's llama-server decoding on `Vulkan0`, long prompt prefixes prefilled on `HRX0` over one shared KV cache (docs/hrx.md, "Prefill on HRX, decode on Vulkan") |
 | ROCmFP4 `.gguf` | `auto`, `vulkan` with `--lean` | the lean (ROCmFPX) build's llama-server on `Vulkan0` (docs/lean.md) |
-| ROCmI4 `.gguf` | `rocm` with `--lean` | the lean ROCm build's llama-server on `ROCm0`, W4A4 (docs/lean.md) |
+| `.gguf` | `rocm` | the ROCm build's llama-server on `ROCm0` (ROCmFPX's tree, `ONEBIT_LEAN_ROCM`); ROCmI4 files take its W4A4 path (docs/lean.md) |
 | `.gguf` | `zinc` | this build's ZINC (Vulkan, ROCm or CUDA, whichever it was built for; docs/zinc.md) |
 | Hugging Face id | `mlx` | lemon-mlx-engine's server, on Apple Silicon (docs/apple.md) |
 
@@ -69,6 +70,25 @@ else the build's copy, else the first one under `/opt/rocm-therock`.
 The child binaries default to this build's (`-DONEBIT_HRX`, `-DONEBIT_ZINC`),
 then `$ONEBIT_LLAMA_SERVER` / `$ONEBIT_ZINC`, then `llama-server` / `zinc` on PATH.
 
+## Multi-token prediction (`--mtp`)
+
+`--mtp <head.gguf>` turns on llama-server's `draft-mtp` speculative decoding: the model's
+own MTP head (Unsloth ships it as `MTP/mtp-<model>-*.gguf`) drafts tokens on the same
+device, and the model checks them in one batch. `--mtp-max N` caps the draft length.
+It works on the llama.cpp devices (`vulkan`, `hrx`, `rocm`).
+
+Measured through `1bit serve` on Strix Halo, Qwen3.8-27B UD-Q4_K_XL with its Q4_0 MTP
+head, decode tok/s on three chat prompts (code / prose / short), 2026-09-24:
+
+| Route | Without `--mtp` | With `--mtp` |
+|---|---|---|
+| `--device vulkan` (upstream pin) | 12.2 / 12.0 / 12.0 | **35.0 / 28.3 / 28.9** |
+| `--device rocm` | 11.8 / 11.8 / 12.2 | 38.8 / 20.6 / 16.9 |
+| `--device rocm --mtp-max 3` | | 27.1 / 19.2 / 19.9 |
+
+Vulkan with MTP is the pick: 2.4-2.9x on every prompt, same file, same accuracy (a
+drafted token is kept only when the model agrees). ROCm edges it on code only.
+
 ## Verified (Strix Halo, 2026-09-23)
 
 `tests/serve_e2e.sh` with Qwen3-0.6B: the Q4_K_M GGUF for the GPU devices and the Q4NX model directory for the NPU. The test checks `/health` 200,