diff --git a/tools/server/README.md b/tools/server/README.md index 56d1ea5fe885..deda7c5b4ff9 100644 --- a/tools/server/README.md +++ b/tools/server/README.md @@ -1588,7 +1588,7 @@ Audio must be PCM16, mono, at the model sample rate. The sample rate is returned *Client events:* -- `session.update`: accepted for compatibility, the server replies with `session.updated` +- `session.update`: accepted for compatibility, the server replies with `session.updated`. If `session.audio.input.format.rate` is set to a rate other than the model sample rate, the server replies with an `error` instead - `input_audio_buffer.append`: `audio` holds base64 PCM16 samples - `input_audio_buffer.commit`: flushes the audio received so far, so the words still held back by the model's delay are produced. The server replies with `input_audio_buffer.committed` and starts a new item. With `"final": true`, the input ends instead and the last item is completed once the stream drains. - `input_audio_buffer.clear`: accepted for compatibility, audio already sent cannot be discarded @@ -1603,7 +1603,7 @@ Closing the websocket also ends the input. - `input_audio_buffer.committed`: `item_id` of the committed item, `previous_item_id` of the item committed before it (`null` for the first one) - `error`: `error.message` describes the problem -Note that words held back by the model's delay at the time of a commit are attributed to the next item. +The `completed` event of a commit includes the words held back by the model's delay at the time of the commit. ### POST `/v1/responses`: OpenAI-compatible Responses API diff --git a/tools/server/server-context.cpp b/tools/server/server-context.cpp index a5cba1d71d46..916b8d035374 100644 --- a/tools/server/server-context.cpp +++ b/tools/server/server-context.cpp @@ -6106,6 +6106,14 @@ void server_routes::init_routes() { } const std::string type = json_value(ev, "type", std::string()); if (type == "session.update" || type == "transcription_session.update") { + // audio sent at another rate would still be read at the model rate + const int32_t rate = mtmd_get_audio_sample_rate(ctx_server.mctx); + const json format = ev.value("session", json()).value("audio", json()).value("input", json()).value("format", json()); + // json_value() would fall back to the default for a string and truncate a fraction + if (format.contains("rate") && !(format.at("rate").is_number_integer() && format.at("rate").get() == rate)) { + send_error_event(string_format("audio.input.format.rate must be %d", rate), "invalid_request_error"); + continue; + } send_event(json {{"type", "session.updated"}, {"session", session_config}}); } else if (type == "input_audio_buffer.append") { std::string pcm; diff --git a/tools/server/tests/unit/test_realtime.py b/tools/server/tests/unit/test_realtime.py index 608904b070ad..cc3669bb0c62 100644 --- a/tools/server/tests/unit/test_realtime.py +++ b/tools/server/tests/unit/test_realtime.py @@ -81,6 +81,9 @@ def test_realtime_control_events(): start_voxtral_rt() events = run_realtime([ {"type": "session.update", "session": {}}, + {"type": "session.update", "session": {"audio": {"input": {"format": {"type": "audio/pcm", "rate": 24000}}}}}, + {"type": "session.update", "session": {"audio": {"input": {"format": {"type": "audio/pcm", "rate": "24000"}}}}}, + {"type": "session.update", "session": {"audio": {"input": {"format": {"type": "audio/pcm", "rate": 16000}}}}}, {"type": "input_audio_buffer.clear"}, {"type": "not.an.event"}, {"type": "input_audio_buffer.commit", "final": True}, @@ -88,11 +91,16 @@ def test_realtime_control_events(): assert [ev["type"] for ev in events if ev["type"] != EV_DELTA] == [ "session.created", "session.updated", + "error", + "error", + "session.updated", "input_audio_buffer.cleared", "error", EV_COMPLETED, ] assert events[1]["session"] == events[0]["session"] + assert "must be 16000" in events[2]["error"]["message"] + assert "must be 16000" in events[3]["error"]["message"] # ref: https://github.com/ggml-org/llama.cpp/issues/30968