From 14134340f2277c9ae7fb16f4201da451091bd76d Mon Sep 17 00:00:00 2001 From: pi-zerocopy Date: Sun, 1 Nov 2026 00:38:18 -0300 Subject: [PATCH] tests: the shared-KV route, with no model, no GPU and no dma-buf The shared route had no test in this tree because the honest one needs a dma-buf exporter, i.e. this box. That left the plumbing itself untested: the listen -> share -> receive-fds -> provider -> adopt order, the descriptors surviving exec, the share_id coming back, the socket being cleaned up, and the fallback. So the two fakes learn the protocol instead: - tests/fake_backend.py gains ?action=share (connect to the engine's socket, send [uint32 length][JSON] with one SCM_RIGHTS fd, return a share_id and the context params) and ?action=adopt (refuse anything but that share_id, count the tokens). Any readable fd stands in for the dma-buf -- the plumbing does not care what the bytes are. - tests/fake_kvshare_provider.py is the reference provider's shape without a llama_context: it insists on --tokens/--layout/--fds, fails unless each fd is a usable descriptor in region order, and leaves what it was handed in FAKE_KVSHARE_MARKER (serve consumes the provider's stdout and deletes the slot directory, so the proof has to go somewhere stable). - tests/hybrid_kvshare_route.sh (CTest hybrid_kvshare_route) runs 11 checks against `1bit serve` with ONEBIT_KV_SHARE=1: the handoff happens rather than the file route, no fallback line, one readable fd per region, the socket is gone, and a provider that refuses still leaves the request answered by the backend. No model, no GPU, no exporter: this is the half of target 1 a normal CI can run. --- CMakeLists.txt | 5 +++ tests/fake_backend.py | 38 ++++++++++++++++ tests/fake_kvshare_provider.py | 66 ++++++++++++++++++++++++++++ tests/hybrid_kvshare_route.sh | 80 ++++++++++++++++++++++++++++++++++ 4 files changed, 189 insertions(+) create mode 100755 tests/fake_kvshare_provider.py create mode 100755 tests/hybrid_kvshare_route.sh diff --git a/CMakeLists.txt b/CMakeLists.txt index 8df06700..56505767 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -170,6 +170,11 @@ add_test(NAME hrx_cwsr_route COMMAND ${CMAKE_SOURCE_DIR}/tests/hrx_cwsr_route.sh # --device hybrid (docs/hybrid.md): the handoff with a fake backend and a fake provider; and, with a model, # end to end on the CPU llama-server (ONEBIT_SERVE_TEST_LLAMA_SERVER, else the build's own; skips without one) add_test(NAME hybrid_route COMMAND ${CMAKE_SOURCE_DIR}/tests/hybrid_route.sh $) +# target 1's shared route (ONEBIT_KV_SHARE=1): the fake backend speaks ?action=share/adopt and the fake +# provider fails unless it was handed one readable fd per region -- so the socket, the inherited fds, the +# share_id and the fallback are all pinned with no model, no GPU and no dma-buf. Needs no exporter, so CI can +# run it (the GPU gate in ~/wt/zc-probes cannot). +add_test(NAME hybrid_kvshare_route COMMAND ${CMAKE_SOURCE_DIR}/tests/hybrid_kvshare_route.sh $) set(ONEBIT_SERVE_TEST_LLAMA_SERVER "" CACHE FILEPATH "llama-server for hybrid_e2e (default: the build's own)") if(ONEBIT_SERVE_TEST_GGUF) add_test(NAME hybrid_e2e diff --git a/tests/fake_backend.py b/tests/fake_backend.py index 4ac21073..ab91ede4 100755 --- a/tests/fake_backend.py +++ b/tests/fake_backend.py @@ -77,6 +77,44 @@ def do_POST(self): return if self.path.startswith("/slots/"): slot, _, action = self.path[len("/slots/"):].partition("?action=") + if action == "share": + # the shared-KV protocol (fork 1bit-MONSTER/llama.cpp#164, engine ONEBIT_KV_SHARE=1): + # connect to the engine's socket and send [uint32 length][JSON] with one fd, then report + # the export id. No dma-buf here -- any readable fd proves the plumbing (the real one is + # a dma-buf the provider mmaps). tests/hybrid_kvshare_route.sh + path = str(req.get("socket", "")) + import struct + fd = os.open(__file__, os.O_RDONLY) # a real fd the provider can pread() + header = json.dumps({ + "n_regions": 1, + "params": {"n_ctx": 4096, "n_batch": 2048, "n_ubatch": 512, "n_seq_max": 1, + "flash_attn": 1, "type_k": 1, "type_v": 1}, + "regions": [{"offset": 0, "size": 4096, "layout": 1234567}], + }).encode() + sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) + try: + sock.connect(path) + sock.sendmsg([struct.pack(" --model --layout
--fds +# +# It fails unless it was given a real, readable fd per region -- i.e. unless the engine really received the +# fds over the socket and handed them on through exec -- and it says how many bytes it could pread from each, +# which is what the test asserts on. Print one JSON line, exit 0, exactly like the real provider. +import json, os, sys + +args = sys.argv[1:] +if "--tokens" not in args or "--layout" not in args or "--fds" not in args: + print("fake_kvshare_provider: the shared route needs --tokens/--layout/--fds (got %s)" % (args,), + file=sys.stderr) + sys.exit(2) + +opt = {args[i]: args[i + 1] for i in range(0, len(args) - 1, 2)} +with open(opt["--tokens"]) as f: + ids = [int(line) for line in f if line.strip()] +with open(opt["--layout"]) as f: + header = json.load(f) +fds = [int(x) for x in opt["--fds"].split(",") if x] + +regions = header.get("regions") or [] +if len(fds) != len(regions): + print("fake_kvshare_provider: %d fd(s) for %d region(s)" % (len(fds), len(regions)), file=sys.stderr) + sys.exit(1) + +# the provider's whole job here: the fds must be usable descriptors, in region order +readable = [] +for fd, region in zip(fds, regions): + try: + data = os.pread(fd, min(16, int(region.get("size", 1))), int(region.get("offset", 0))) + except OSError as e: + print("fake_kvshare_provider: fd %d is not usable: %s" % (fd, e), file=sys.stderr) + sys.exit(1) + if not data: + print("fake_kvshare_provider: fd %d read 0 bytes" % fd, file=sys.stderr) + sys.exit(1) + readable.append(len(data)) + +params = header.get("params") or {} +# serve reads our stdout for its numbers and does not echo it, and it deletes the ids file and the slot +# directory as soon as we exit -- so the proof of what we were handed goes where the test asks for it +marker = os.environ.get("FAKE_KVSHARE_MARKER") +if marker: + with open(marker, "w") as f: + json.dump({"fds": len(fds), "readable": readable, "n_ctx": params.get("n_ctx")}, f) +print(json.dumps({"n_tokens": len(ids), "prefill_ms": 1.0, "fds": len(fds), "readable": readable, + "n_ctx": params.get("n_ctx")}), flush=True) diff --git a/tests/hybrid_kvshare_route.sh b/tests/hybrid_kvshare_route.sh new file mode 100755 index 00000000..c28e611e --- /dev/null +++ b/tests/hybrid_kvshare_route.sh @@ -0,0 +1,80 @@ +#!/usr/bin/env bash +# Copyright 2026 bong-water-water-bong +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# The shared-KV route (ONEBIT_KV_SHARE=1, docs/hybrid.md) without a model, a GPU or a dma-buf: the fake +# backend speaks the two endpoints the real llama-server has (fork 1bit-MONSTER/llama.cpp#164: ?action=share +# sends [uint32 length][JSON] plus the region fds over a unix socket and returns a share_id; ?action=adopt +# takes the tokens back and refuses a wrong share_id), and fake_kvshare_provider.py plays the provider: it +# fails unless it was handed one readable fd per region, which is only true if the engine received the fds +# over the socket and passed them on through exec. +# +# What this pins down that the GPU gate cannot (that one needs an HRX box): the listen -> share -> fds -> +# provider -> adopt order, the inherited descriptors, that adopt carries the share_id, that the socket is +# cleaned up, and the fallback when a provider fails. +# +# usage: tests/hybrid_kvshare_route.sh path/to/1bit +set -uo pipefail + +bin=${1:?usage: hybrid_kvshare_route.sh path/to/1bit} +here=$(cd "$(dirname "$0")" && pwd) +scratch=$(mktemp -d) +touch "$scratch/tiny.gguf" +fail=0 +check() { if eval "$2"; then echo "ok $1"; else echo "FAIL $1"; fail=1; fi; } +pid= +cleanup() { [ -n "$pid" ] && { kill "$pid" 2>/dev/null; wait "$pid" 2>/dev/null; }; rm -rf "$scratch"; } +trap cleanup EXIT + +start() { # start [env...] + local log=$1; shift + port=$(python3 -c 'import socket; s=socket.socket(); s.bind(("127.0.0.1", 0)); print(s.getsockname()[1])') + env FAKE_KVSHARE_MARKER="$scratch/provider.json" "$@" "$bin" serve -m "$scratch/tiny.gguf" --device hybrid --port "$port" --alias hybrid-model \ + --llama-server "$here/fake_backend.py" --no-mem-check >"$log" 2>&1 & + pid=$! + api="http://127.0.0.1:$port" + local code=000 + for _ in $(seq 1 100); do code=$(curl -s -o /dev/null -w '%{http_code}' "$api/health"); [ "$code" = 200 ] && break; sleep 0.1; done + [ "$code" = 200 ] +} +stop() { kill "$pid" 2>/dev/null; wait "$pid" 2>/dev/null; pid=; } +ask() { curl -s "$api/v1/chat/completions" -H 'Content-Type: application/json' \ + -d '{"model": "hybrid-model", "messages": [{"role": "user", "content": "What is the capital of France?"}]}'; } + +check "serve starts with the shared route on" \ + 'start "$scratch/a.log" ONEBIT_KV_SHARE=1 ONEBIT_HYBRID_PREFILL="$here/fake_kvshare_provider.py"' +reply=$(ask) +check "the reply comes through" '[[ "$reply" == *Paris* ]]' +check "the request was pinned to slot 0" '[[ "$reply" == *"\"fake_id_slot\":0"* ]]' +sleep 0.3 +check "the handoff ran (share + adopt, not the file route)" 'grep -q "hybrid: .* share 0\.[0-9]* ms, first byte" "$scratch/a.log"' +check "no fallback happened" '! grep -q "the backend prefills this request itself" "$scratch/a.log"' +check "the provider got one readable fd per region" \ + 'python3 -c "import json,sys; d=json.load(open(sys.argv[1])); sys.exit(0 if d[\"fds\"]==1 and d[\"readable\"]==[16] and d[\"n_ctx\"]==4096 else 1)" "$scratch/provider.json" 2>/dev/null' +check "the socket is gone from the slot directory" '! ls /tmp/1bit-hybrid-*/kvshare-*.sock 2>/dev/null | grep -q .' +stop + +# the adopt must carry the share_id: the fake backend refuses a wrong or missing one, so a provider that runs +# but whose id does not come back leaves the request to the backend -- that is the fallback this checks +check "serve starts with a provider that refuses the protocol" \ + 'start "$scratch/b.log" ONEBIT_KV_SHARE=1 ONEBIT_HYBRID_PREFILL=/bin/false' +reply=$(ask) +check "the reply still comes through" '[[ "$reply" == *Paris* ]]' +check "the fallback is named once" 'grep -q "the backend prefills this request itself" "$scratch/b.log"' +check "no share/adopt line was reported" '! grep -q "hybrid: .* share 0\.[0-9]* ms, first byte" "$scratch/b.log"' +stop + +if [ $fail -ne 0 ]; then echo "--- log (tail)"; tail -30 "$scratch/a.log"; echo FAIL; exit 1; fi +echo PASS