From ebe59c5deaefa4d14578dcb6d3a28b60d759adef Mon Sep 17 00:00:00 2001 From: Peter van Velzen Date: Mon, 7 Sep 2026 16:47:50 +0200 Subject: [PATCH 1/3] server : pass id_slot through the Anthropic /v1/messages conversion server_chat_convert_anthropic_to_oai copies a small whitelist of common params from the Anthropic request into the internal OAI-style body and left id_slot out. Every other completion endpoint honours id_slot; a proxy that pins a session to the slot holding its KV cache needs the same on /v1/messages, or it silently falls back to automatic slot selection and, on hybrid/recurrent models, pays a full re-prefill. Adds a test asserting a request with id_slot set is served by that slot, observed via GET /slots. --- tools/server/server-chat.cpp | 6 ++-- .../tests/unit/test_compat_anthropic.py | 29 +++++++++++++++++++ 2 files changed, 33 insertions(+), 2 deletions(-) diff --git a/tools/server/server-chat.cpp b/tools/server/server-chat.cpp index a6fe3c6ba61..d7f86690feb 100644 --- a/tools/server/server-chat.cpp +++ b/tools/server/server-chat.cpp @@ -575,8 +575,10 @@ json server_chat_convert_anthropic_to_oai(const json & body) { oai_body["max_tokens"] = 4096; } - // Pass through common params - for (const auto & key : {"temperature", "top_p", "top_k", "stream", "chat_template_kwargs"}) { + // Pass through common params. id_slot: explicit slot pin, honoured by the + // completion handler on every other endpoint; a proxy pinning a session + // to the slot holding its KV cache needs it to survive the conversion. + for (const auto & key : {"temperature", "top_p", "top_k", "stream", "chat_template_kwargs", "id_slot"}) { if (body.contains(key)) { oai_body[key] = body.at(key); } diff --git a/tools/server/tests/unit/test_compat_anthropic.py b/tools/server/tests/unit/test_compat_anthropic.py index d292f83cf0d..b500afa0740 100644 --- a/tools/server/tests/unit/test_compat_anthropic.py +++ b/tools/server/tests/unit/test_compat_anthropic.py @@ -701,6 +701,35 @@ def test_anthropic_top_k(): assert res.body["type"] == "message" +def test_anthropic_id_slot(): + """Test that id_slot (llama.cpp specific) is honoured by the Anthropic endpoint""" + server.n_slots = 2 + server.server_slots = True + server.start() + + # on an idle server automatic selection picks the LAST slot (the LRU scan + # compares with <= and every slot starts with the same t_last_used), so + # pinning slot 0 is what shows whether id_slot survived the conversion + res = server.make_request("POST", "/v1/messages", data={ + "model": "test", + "max_tokens": 8, + "id_slot": 0, + "messages": [ + {"role": "user", "content": "Hello"} + ] + }) + + assert res.status_code == 200 + assert res.body["type"] == "message" + + slots_res = server.make_request("GET", "/slots") + assert slots_res.status_code == 200 + slot_0 = next(slot for slot in slots_res.body if slot["id"] == 0) + slot_1 = next(slot for slot in slots_res.body if slot["id"] == 1) + assert slot_0["n_prompt_tokens"] > 0, "request should have been served by slot 0" + assert slot_1.get("n_prompt_tokens", 0) == 0, "slot 1 should not have served the request" + + # Error handling tests def test_anthropic_missing_messages(): From 35afb60162d1dbe6134796ce99c83eb59a6f5639 Mon Sep 17 00:00:00 2001 From: Peter van Velzen Date: Thu, 24 Sep 2026 22:22:06 +0200 Subject: [PATCH 2/3] server : shorten comments --- tools/server/server-chat.cpp | 4 +--- tools/server/tests/unit/test_compat_anthropic.py | 4 +--- 2 files changed, 2 insertions(+), 6 deletions(-) diff --git a/tools/server/server-chat.cpp b/tools/server/server-chat.cpp index 1936fe45181..c5473417c30 100644 --- a/tools/server/server-chat.cpp +++ b/tools/server/server-chat.cpp @@ -589,9 +589,7 @@ json server_chat_convert_anthropic_to_oai(const json & body) { oai_body["max_tokens"] = 4096; } - // Pass through common params. id_slot: explicit slot pin, honoured by the - // completion handler on every other endpoint; a proxy pinning a session - // to the slot holding its KV cache needs it to survive the conversion. + // Pass through common params for (const auto & key : {"temperature", "top_p", "top_k", "stream", "chat_template_kwargs", "id_slot"}) { if (body.contains(key)) { oai_body[key] = body.at(key); diff --git a/tools/server/tests/unit/test_compat_anthropic.py b/tools/server/tests/unit/test_compat_anthropic.py index b500afa0740..e8d5ad55aa7 100644 --- a/tools/server/tests/unit/test_compat_anthropic.py +++ b/tools/server/tests/unit/test_compat_anthropic.py @@ -707,9 +707,7 @@ def test_anthropic_id_slot(): server.server_slots = True server.start() - # on an idle server automatic selection picks the LAST slot (the LRU scan - # compares with <= and every slot starts with the same t_last_used), so - # pinning slot 0 is what shows whether id_slot survived the conversion + # idle server picks the last slot by default, so slot 0 shows that id_slot is used res = server.make_request("POST", "/v1/messages", data={ "model": "test", "max_tokens": 8, From 6a87a38ecd3975132632219722c48f865ddebf54 Mon Sep 17 00:00:00 2001 From: Xuan Son Nguyen Date: Tue, 20 Oct 2026 11:44:57 +0200 Subject: [PATCH 3/3] rm test --- .../tests/unit/test_compat_anthropic.py | 27 ------------------- 1 file changed, 27 deletions(-) diff --git a/tools/server/tests/unit/test_compat_anthropic.py b/tools/server/tests/unit/test_compat_anthropic.py index e8d5ad55aa7..d292f83cf0d 100644 --- a/tools/server/tests/unit/test_compat_anthropic.py +++ b/tools/server/tests/unit/test_compat_anthropic.py @@ -701,33 +701,6 @@ def test_anthropic_top_k(): assert res.body["type"] == "message" -def test_anthropic_id_slot(): - """Test that id_slot (llama.cpp specific) is honoured by the Anthropic endpoint""" - server.n_slots = 2 - server.server_slots = True - server.start() - - # idle server picks the last slot by default, so slot 0 shows that id_slot is used - res = server.make_request("POST", "/v1/messages", data={ - "model": "test", - "max_tokens": 8, - "id_slot": 0, - "messages": [ - {"role": "user", "content": "Hello"} - ] - }) - - assert res.status_code == 200 - assert res.body["type"] == "message" - - slots_res = server.make_request("GET", "/slots") - assert slots_res.status_code == 200 - slot_0 = next(slot for slot in slots_res.body if slot["id"] == 0) - slot_1 = next(slot for slot in slots_res.body if slot["id"] == 1) - assert slot_0["n_prompt_tokens"] > 0, "request should have been served by slot 0" - assert slot_1.get("n_prompt_tokens", 0) == 0, "slot 1 should not have served the request" - - # Error handling tests def test_anthropic_missing_messages():