From 3ce65e8a4c1809c5689972a5310ecba2fa676852 Mon Sep 17 00:00:00 2001 From: Tom Vaucourt <34662901+T0mSIlver@users.noreply.github.com> Date: Sat, 24 Oct 2026 11:01:53 +0000 Subject: [PATCH 1/3] server : reject a session.update with another sample rate Assisted-by: Claude Code --- tools/server/README.md | 4 ++-- tools/server/server-context.cpp | 7 +++++++ tools/server/tests/unit/test_realtime.py | 5 +++++ 3 files changed, 14 insertions(+), 2 deletions(-) diff --git a/tools/server/README.md b/tools/server/README.md index 56d1ea5fe885..deda7c5b4ff9 100644 --- a/tools/server/README.md +++ b/tools/server/README.md @@ -1588,7 +1588,7 @@ Audio must be PCM16, mono, at the model sample rate. The sample rate is returned *Client events:* -- `session.update`: accepted for compatibility, the server replies with `session.updated` +- `session.update`: accepted for compatibility, the server replies with `session.updated`. If `session.audio.input.format.rate` is set to a rate other than the model sample rate, the server replies with an `error` instead - `input_audio_buffer.append`: `audio` holds base64 PCM16 samples - `input_audio_buffer.commit`: flushes the audio received so far, so the words still held back by the model's delay are produced. The server replies with `input_audio_buffer.committed` and starts a new item. With `"final": true`, the input ends instead and the last item is completed once the stream drains. - `input_audio_buffer.clear`: accepted for compatibility, audio already sent cannot be discarded @@ -1603,7 +1603,7 @@ Closing the websocket also ends the input. - `input_audio_buffer.committed`: `item_id` of the committed item, `previous_item_id` of the item committed before it (`null` for the first one) - `error`: `error.message` describes the problem -Note that words held back by the model's delay at the time of a commit are attributed to the next item. +The `completed` event of a commit includes the words held back by the model's delay at the time of the commit. ### POST `/v1/responses`: OpenAI-compatible Responses API diff --git a/tools/server/server-context.cpp b/tools/server/server-context.cpp index a5cba1d71d46..9283d01fd9a8 100644 --- a/tools/server/server-context.cpp +++ b/tools/server/server-context.cpp @@ -6106,6 +6106,13 @@ void server_routes::init_routes() { } const std::string type = json_value(ev, "type", std::string()); if (type == "session.update" || type == "transcription_session.update") { + // audio sent at another rate would still be read at the model rate + const int32_t rate = mtmd_get_audio_sample_rate(ctx_server.mctx); + const json format = ev.value("session", json()).value("audio", json()).value("input", json()).value("format", json()); + if (json_value(format, "rate", rate) != rate) { + send_error_event(string_format("audio.input.format.rate must be %d", rate), "invalid_request_error"); + continue; + } send_event(json {{"type", "session.updated"}, {"session", session_config}}); } else if (type == "input_audio_buffer.append") { std::string pcm; diff --git a/tools/server/tests/unit/test_realtime.py b/tools/server/tests/unit/test_realtime.py index bd1be39b30c8..ee3acef2dee1 100644 --- a/tools/server/tests/unit/test_realtime.py +++ b/tools/server/tests/unit/test_realtime.py @@ -81,6 +81,8 @@ def test_realtime_control_events(): start_voxtral_rt() events = run_realtime([ {"type": "session.update", "session": {}}, + {"type": "session.update", "session": {"audio": {"input": {"format": {"type": "audio/pcm", "rate": 24000}}}}}, + {"type": "session.update", "session": {"audio": {"input": {"format": {"type": "audio/pcm", "rate": 16000}}}}}, {"type": "input_audio_buffer.clear"}, {"type": "not.an.event"}, {"type": "input_audio_buffer.commit", "final": True}, @@ -88,11 +90,14 @@ def test_realtime_control_events(): assert [ev["type"] for ev in events if ev["type"] != EV_DELTA] == [ "session.created", "session.updated", + "error", + "session.updated", "input_audio_buffer.cleared", "error", EV_COMPLETED, ] assert events[1]["session"] == events[0]["session"] + assert "must be 16000" in events[2]["error"]["message"] # ref: https://github.com/ggml-org/llama.cpp/issues/30968 From 817ccf6bd73f831e63251e807fe94d5c24c2fa8c Mon Sep 17 00:00:00 2001 From: Tom Vaucourt <34662901+T0mSIlver@users.noreply.github.com> Date: Sat, 24 Oct 2026 11:25:49 +0000 Subject: [PATCH 2/3] server : reject a realtime rate that is not an integer Assisted-by: Claude Code --- tools/server/server-context.cpp | 3 ++- tools/server/tests/unit/test_realtime.py | 3 +++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/tools/server/server-context.cpp b/tools/server/server-context.cpp index 9283d01fd9a8..a1aeb21a194d 100644 --- a/tools/server/server-context.cpp +++ b/tools/server/server-context.cpp @@ -6109,7 +6109,8 @@ void server_routes::init_routes() { // audio sent at another rate would still be read at the model rate const int32_t rate = mtmd_get_audio_sample_rate(ctx_server.mctx); const json format = ev.value("session", json()).value("audio", json()).value("input", json()).value("format", json()); - if (json_value(format, "rate", rate) != rate) { + // json_value() would turn a string or a fraction into the default + if (format.contains("rate") && !(format.at("rate").is_number_integer() && format.at("rate").get() == rate)) { send_error_event(string_format("audio.input.format.rate must be %d", rate), "invalid_request_error"); continue; } diff --git a/tools/server/tests/unit/test_realtime.py b/tools/server/tests/unit/test_realtime.py index 0d799877e1bf..cc3669bb0c62 100644 --- a/tools/server/tests/unit/test_realtime.py +++ b/tools/server/tests/unit/test_realtime.py @@ -82,6 +82,7 @@ def test_realtime_control_events(): events = run_realtime([ {"type": "session.update", "session": {}}, {"type": "session.update", "session": {"audio": {"input": {"format": {"type": "audio/pcm", "rate": 24000}}}}}, + {"type": "session.update", "session": {"audio": {"input": {"format": {"type": "audio/pcm", "rate": "24000"}}}}}, {"type": "session.update", "session": {"audio": {"input": {"format": {"type": "audio/pcm", "rate": 16000}}}}}, {"type": "input_audio_buffer.clear"}, {"type": "not.an.event"}, @@ -91,6 +92,7 @@ def test_realtime_control_events(): "session.created", "session.updated", "error", + "error", "session.updated", "input_audio_buffer.cleared", "error", @@ -98,6 +100,7 @@ def test_realtime_control_events(): ] assert events[1]["session"] == events[0]["session"] assert "must be 16000" in events[2]["error"]["message"] + assert "must be 16000" in events[3]["error"]["message"] # ref: https://github.com/ggml-org/llama.cpp/issues/30968 From ee7bc7876219e1f699f431e2add0254f433124e6 Mon Sep 17 00:00:00 2001 From: Tom Vaucourt <34662901+T0mSIlver@users.noreply.github.com> Date: Sat, 24 Oct 2026 12:12:39 +0000 Subject: [PATCH 3/3] server : fix the comment on the realtime rate check Assisted-by: Claude Code --- tools/server/server-context.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/server/server-context.cpp b/tools/server/server-context.cpp index a1aeb21a194d..916b8d035374 100644 --- a/tools/server/server-context.cpp +++ b/tools/server/server-context.cpp @@ -6109,7 +6109,7 @@ void server_routes::init_routes() { // audio sent at another rate would still be read at the model rate const int32_t rate = mtmd_get_audio_sample_rate(ctx_server.mctx); const json format = ev.value("session", json()).value("audio", json()).value("input", json()).value("format", json()); - // json_value() would turn a string or a fraction into the default + // json_value() would fall back to the default for a string and truncate a fraction if (format.contains("rate") && !(format.at("rate").is_number_integer() && format.at("rate").get() == rate)) { send_error_event(string_format("audio.input.format.rate must be %d", rate), "invalid_request_error"); continue;