From 511b54d1154a972d4e4a89421e2174c579d2922e Mon Sep 17 00:00:00 2001 From: Daksha1611 Date: Sat, 17 Oct 2026 14:53:45 +0530 Subject: [PATCH 1/2] Return a list of token lists for a text array with one string The /v1/tokenize response nested the tokens only when the tokenizer returned more than one row, so "text": ["hello world"] was answered with the flat list used for "text": "hello world". TokenizeRequest records whether text was an array of strings, and parseTokenizeResponse() takes that as a new last parameter and nests accordingly. The LLM servable, embeddings and rerank callers pass it through. A single string and arrays of two or more strings are unchanged. Adds request and response tests and documents the list form. --- docs/model_server_rest_api_tokenizer.md | 12 +++++ src/embeddings/embeddings_calculator_ov.cc | 2 +- src/llm/servable.cpp | 2 +- src/rerank/rerank_calculator_ov.cc | 2 +- src/test/tokenize_parser_test.cpp | 57 ++++++++++++++++++++++ src/tokenize/tokenize_parser.cpp | 8 ++- src/tokenize/tokenize_parser.hpp | 4 +- 7 files changed, 81 insertions(+), 6 deletions(-) diff --git a/docs/model_server_rest_api_tokenizer.md b/docs/model_server_rest_api_tokenizer.md index 679fa8ac90..7db283f2ee 100644 --- a/docs/model_server_rest_api_tokenizer.md +++ b/docs/model_server_rest_api_tokenizer.md @@ -25,6 +25,18 @@ Response: } ``` +When `text` is a list of strings, the response holds one list of tokens per string, also when the list has a single string: + +```bash +curl http://localhost:8000/v1/tokenize -H "Content-Type: application/json" -d '{ "model": "Qwen/Qwen3-8B", "text": ["hello world", "hi"]}' +``` +Response: +```json +{ + "tokens":[[14990,1879],[6023]] +} +``` + It's possible to use additional parameters: - `pad_to_max_length` - whether to pad the sequence to the maximum length. Default is False. - `max_length` - maximum length of the sequence. If specified, it truncates the tokens to the provided number. Must be a positive integer and cannot exceed 1,000,000 tokens, or the model's maximum context length if that is greater than 1,000,000. diff --git a/src/embeddings/embeddings_calculator_ov.cc b/src/embeddings/embeddings_calculator_ov.cc index de53e4fcdf..c8c5406c32 100644 --- a/src/embeddings/embeddings_calculator_ov.cc +++ b/src/embeddings/embeddings_calculator_ov.cc @@ -167,7 +167,7 @@ class EmbeddingsCalculatorOV : public CalculatorBase { } StringBuffer responseBuffer; - auto responseStatus = ovms::TokenizeParser::parseTokenizeResponse(responseBuffer, tokens, tokenizeRequest.parameters); + auto responseStatus = ovms::TokenizeParser::parseTokenizeResponse(responseBuffer, tokens, tokenizeRequest.parameters, tokenizeRequest.inputIsBatch); if (!responseStatus.ok()) { return responseStatus; } diff --git a/src/llm/servable.cpp b/src/llm/servable.cpp index 1f5aa2ecf9..40822ec796 100644 --- a/src/llm/servable.cpp +++ b/src/llm/servable.cpp @@ -132,7 +132,7 @@ absl::Status GenAiServable::processTokenizeRequest(std::shared_ptr>(&tokenizeRequest.input)) { auto tokens = rerank_session->getTokenizer().encode(*strings, tokenizeRequest.parameters); StringBuffer buffer; - status = TokenizeParser::parseTokenizeResponse(buffer, tokens, tokenizeRequest.parameters); + status = TokenizeParser::parseTokenizeResponse(buffer, tokens, tokenizeRequest.parameters, tokenizeRequest.inputIsBatch); cc->Outputs().Tag(OUTPUT_TAG_NAME).Add(new std::string(buffer.GetString()), cc->InputTimestamp()); return absl::OkStatus(); } else { diff --git a/src/test/tokenize_parser_test.cpp b/src/test/tokenize_parser_test.cpp index 22a8b340dc..a1ccfe244f 100644 --- a/src/test/tokenize_parser_test.cpp +++ b/src/test/tokenize_parser_test.cpp @@ -351,3 +351,60 @@ TEST(TokenizeDeserialization, positiveTokenizeMaxLengthWithSmallModelContext) { ASSERT_EQ(status, absl::OkStatus()); ASSERT_EQ(request.parameters["max_length"].as(), 1000000); } + +TEST(TokenizeDeserialization, singleStringIsNotABatch) { + rapidjson::Document d; + d.Parse(R"({"model": "embeddings", "text": "one"})"); + ovms::TokenizeRequest request; + ASSERT_EQ(ovms::TokenizeParser::parseTokenizeRequest(d, request), absl::OkStatus()); + EXPECT_FALSE(request.inputIsBatch); +} + +TEST(TokenizeDeserialization, arrayWithOneStringIsABatch) { + rapidjson::Document d; + d.Parse(R"({"model": "embeddings", "text": ["one"]})"); + ovms::TokenizeRequest request; + ASSERT_EQ(ovms::TokenizeParser::parseTokenizeRequest(d, request), absl::OkStatus()); + EXPECT_TRUE(request.inputIsBatch); +} + +TEST(TokenizeDeserialization, arrayWithManyStringsIsABatch) { + rapidjson::Document d; + d.Parse(R"({"model": "embeddings", "text": ["one", "two"]})"); + ovms::TokenizeRequest request; + ASSERT_EQ(ovms::TokenizeParser::parseTokenizeRequest(d, request), absl::OkStatus()); + EXPECT_TRUE(request.inputIsBatch); +} + +namespace { +// Tokenizer output for `rows` inputs of two tokens each: row i is [10 * i + 1, 10 * i + 2]. +ov::genai::TokenizedInputs twoTokensPerRow(size_t rows) { + ov::Tensor inputIds(ov::element::i64, ov::Shape{rows, 2}); + ov::Tensor attentionMask(ov::element::i64, ov::Shape{rows, 2}); + for (size_t row = 0; row < rows; row++) { + for (size_t column = 0; column < 2; column++) { + inputIds.data()[row * 2 + column] = 10 * row + column + 1; + attentionMask.data()[row * 2 + column] = 1; + } + } + return ov::genai::TokenizedInputs{inputIds, attentionMask}; +} + +std::string tokenizeResponse(size_t rows, bool inputIsBatch) { + rapidjson::StringBuffer buffer; + EXPECT_EQ(ovms::TokenizeParser::parseTokenizeResponse(buffer, twoTokensPerRow(rows), {}, inputIsBatch), absl::OkStatus()); + return buffer.GetString(); +} +} // namespace + +TEST(TokenizeResponse, singleStringGivesFlatList) { + EXPECT_EQ(tokenizeResponse(1, false), R"({"tokens":[1,2]})"); +} + +TEST(TokenizeResponse, arrayWithOneStringGivesListOfLists) { + EXPECT_EQ(tokenizeResponse(1, true), R"({"tokens":[[1,2]]})"); +} + +TEST(TokenizeResponse, arrayWithManyStringsGivesListOfLists) { + EXPECT_EQ(tokenizeResponse(2, true), R"({"tokens":[[1,2],[11,12]]})"); +} diff --git a/src/tokenize/tokenize_parser.cpp b/src/tokenize/tokenize_parser.cpp index d85528c8e4..8a5471aca4 100644 --- a/src/tokenize/tokenize_parser.cpp +++ b/src/tokenize/tokenize_parser.cpp @@ -22,7 +22,7 @@ #include "src/port/rapidjson_writer.hpp" namespace ovms { -absl::Status TokenizeParser::parseTokenizeResponse(rapidjson::StringBuffer& buffer, const ov::genai::TokenizedInputs& tokens, const ov::AnyMap& parameters) { +absl::Status TokenizeParser::parseTokenizeResponse(rapidjson::StringBuffer& buffer, const ov::genai::TokenizedInputs& tokens, const ov::AnyMap& parameters, bool inputIsBatch) { rapidjson::Writer writer(buffer); writer.StartObject(); @@ -35,7 +35,9 @@ absl::Status TokenizeParser::parseTokenizeResponse(rapidjson::StringBuffer& buff return absl::InvalidArgumentError("Invalid input ids tensor shape"); } - const bool isBatched = outputShape[0] > 1; + // The response follows the shape of the request: a list of strings always gets a list of token lists, + // including a list with a single string. + const bool isBatched = inputIsBatch || outputShape[0] > 1; if (isBatched) { writer.StartArray(); } @@ -75,6 +77,8 @@ std::variant TokenizeParser::validateTokenizeReque auto inputVariant = std::get(parsedInput); if (std::holds_alternative>(inputVariant)) { request.input = std::get>(inputVariant); + auto textIt = parsedJson.FindMember("text"); + request.inputIsBatch = textIt != parsedJson.MemberEnd() && textIt->value.IsArray(); } else { request.input = std::get>>(inputVariant); } diff --git a/src/tokenize/tokenize_parser.hpp b/src/tokenize/tokenize_parser.hpp index 6d91244f77..7cb5be5bad 100644 --- a/src/tokenize/tokenize_parser.hpp +++ b/src/tokenize/tokenize_parser.hpp @@ -44,6 +44,8 @@ struct TokenizeRequest { using InputDataType = std::variant, std::vector>, std::vector>>; InputDataType input; ov::AnyMap parameters = {}; + // True when "text" was given as an array of strings, even if the array holds a single string. + bool inputIsBatch = false; }; class TokenizeParser { @@ -51,7 +53,7 @@ class TokenizeParser { static constexpr const char* TOKENIZE_ENDPOINT_SUFFIX = "tokenize"; static constexpr size_t MAX_LENGTH_LIMIT = 1000000; static std::variant parseInput(rapidjson::Document& parsedJson, const std::string& field_name); - static absl::Status parseTokenizeResponse(rapidjson::StringBuffer& buffer, const ov::genai::TokenizedInputs& tokens, const ov::AnyMap& parameters = {}); + static absl::Status parseTokenizeResponse(rapidjson::StringBuffer& buffer, const ov::genai::TokenizedInputs& tokens, const ov::AnyMap& parameters = {}, bool inputIsBatch = false); static absl::Status parseTokenizeRequest(rapidjson::Document& parsedJson, TokenizeRequest& request, std::optional maxModelLength = std::nullopt); static std::variant validateTokenizeRequest(rapidjson::Document& parsedJson, std::optional maxModelLength = std::nullopt); static bool isTokenizeEndpoint(const std::string& uri); From e9a376fc591d027cc1ed6688bad47037b327a2e2 Mon Sep 17 00:00:00 2001 From: Daksha1611 Date: Tue, 20 Oct 2026 18:07:32 +0530 Subject: [PATCH 2/2] Report the unbatched form from parseInput parseInput() now returns a plain string for an unbatched input and an array of strings for a batch, so validateTokenizeRequest() no longer looks at the JSON again to tell them apart. Errors are returned as an absl::StatusOr status, with the same messages as before. The embeddings request parser is updated for the new return type. Adds tests for the values parseInput() returns. --- src/embeddings/embeddings_api.cpp | 21 ++++++------ src/test/tokenize_parser_test.cpp | 29 ++++++++++++++++ src/tokenize/tokenize_parser.cpp | 56 +++++++++++++++++-------------- src/tokenize/tokenize_parser.hpp | 7 ++-- 4 files changed, 75 insertions(+), 38 deletions(-) diff --git a/src/embeddings/embeddings_api.cpp b/src/embeddings/embeddings_api.cpp index a38d3e8e0c..3b40c7da62 100644 --- a/src/embeddings/embeddings_api.cpp +++ b/src/embeddings/embeddings_api.cpp @@ -48,17 +48,18 @@ std::variant EmbeddingsRequest::fromJson(rapidjs auto parsedInput = TokenizeParser::parseInput(*parsedJson, "input"); - if (std::holds_alternative(parsedInput)) { - return std::get(parsedInput); + if (!parsedInput.ok()) { + return std::string(parsedInput.status().message()); + } + auto& inputVariant = parsedInput.value(); + if (std::holds_alternative(inputVariant)) { + request.input = std::vector{std::get(inputVariant)}; + } else if (std::holds_alternative>(inputVariant)) { + request.input = std::get>(inputVariant); + } else if (std::holds_alternative>>(inputVariant)) { + request.input = std::get>>(inputVariant); } else { - auto inputVariant = std::get(parsedInput); - if (std::holds_alternative>(inputVariant)) { - request.input = std::get>(inputVariant); - } else if (std::holds_alternative>>(inputVariant)) { - request.input = std::get>>(inputVariant); - } else { - return "input must be either array of strings or array of array of integers"; - } + return "input must be either array of strings or array of array of integers"; } auto it = parsedJson->FindMember("encoding_format"); diff --git a/src/test/tokenize_parser_test.cpp b/src/test/tokenize_parser_test.cpp index a1ccfe244f..6a2de58756 100644 --- a/src/test/tokenize_parser_test.cpp +++ b/src/test/tokenize_parser_test.cpp @@ -352,6 +352,35 @@ TEST(TokenizeDeserialization, positiveTokenizeMaxLengthWithSmallModelContext) { ASSERT_EQ(request.parameters["max_length"].as(), 1000000); } +TEST(TokenizeParseInput, plainStringIsReturnedAsUnbatched) { + rapidjson::Document d; + d.Parse(R"({"text": "one"})"); + auto input = ovms::TokenizeParser::parseInput(d, "text"); + ASSERT_TRUE(input.ok()); + auto* text = std::get_if(&input.value()); + ASSERT_NE(text, nullptr); + EXPECT_EQ(*text, "one"); +} + +TEST(TokenizeParseInput, arrayWithOneStringIsReturnedAsBatch) { + rapidjson::Document d; + d.Parse(R"({"text": ["one"]})"); + auto input = ovms::TokenizeParser::parseInput(d, "text"); + ASSERT_TRUE(input.ok()); + auto* texts = std::get_if>(&input.value()); + ASSERT_NE(texts, nullptr); + ASSERT_EQ(texts->size(), 1); + EXPECT_EQ(texts->at(0), "one"); +} + +TEST(TokenizeParseInput, invalidInputReturnsErrorStatus) { + rapidjson::Document d; + d.Parse(R"({"text": []})"); + auto input = ovms::TokenizeParser::parseInput(d, "text"); + ASSERT_FALSE(input.ok()); + EXPECT_EQ(input.status().message(), "text array should not be empty"); +} + TEST(TokenizeDeserialization, singleStringIsNotABatch) { rapidjson::Document d; d.Parse(R"({"model": "embeddings", "text": "one"})"); diff --git a/src/tokenize/tokenize_parser.cpp b/src/tokenize/tokenize_parser.cpp index 8a5471aca4..714f4928be 100644 --- a/src/tokenize/tokenize_parser.cpp +++ b/src/tokenize/tokenize_parser.cpp @@ -70,18 +70,17 @@ std::variant TokenizeParser::validateTokenizeReque return "Received json is not an object"; auto parsedInput = parseInput(parsedJson, "text"); - - if (std::holds_alternative(parsedInput)) { - return std::get(parsedInput); + if (!parsedInput.ok()) { + return std::string(parsedInput.status().message()); + } + auto& inputVariant = parsedInput.value(); + if (auto* text = std::get_if(&inputVariant)) { + request.input = std::vector{std::move(*text)}; + } else if (auto* texts = std::get_if>(&inputVariant)) { + request.input = std::move(*texts); + request.inputIsBatch = true; } else { - auto inputVariant = std::get(parsedInput); - if (std::holds_alternative>(inputVariant)) { - request.input = std::get>(inputVariant); - auto textIt = parsedJson.FindMember("text"); - request.inputIsBatch = textIt != parsedJson.MemberEnd() && textIt->value.IsArray(); - } else { - request.input = std::get>>(inputVariant); - } + request.input = std::get>>(inputVariant); } auto it = parsedJson.FindMember("max_length"); @@ -141,7 +140,7 @@ std::variant TokenizeParser::validateTokenizeReque return request; } -std::variant TokenizeParser::parseInput(rapidjson::Document& parsedJson, const std::string& field_name) { +absl::StatusOr TokenizeParser::parseInput(rapidjson::Document& parsedJson, const std::string& field_name) { enum class InputType { NONE, STRING, @@ -153,28 +152,30 @@ std::variant TokenizeParser::parseI std::vector input_strings; std::vector> input_string_vectors; std::vector> input_tokens; + bool isSingleString = false; auto it = parsedJson.FindMember(field_name.c_str()); if (it != parsedJson.MemberEnd()) { if (it->value.IsString()) { input_strings.push_back(it->value.GetString()); + isSingleString = true; } else if (it->value.IsArray()) { if (it->value.GetArray().Size() == 0) { - return field_name + " array should not be empty"; + return absl::InvalidArgumentError(field_name + " array should not be empty"); } InputType input_type = InputType::NONE; for (auto& input : it->value.GetArray()) { if (input.IsArray()) { const auto array = input.GetArray(); if (array.Size() == 0) { - return "inner arrays in " + field_name + " should not be empty"; + return absl::InvalidArgumentError("inner arrays in " + field_name + " should not be empty"); } if (input_type != InputType::NONE && input_type != InputType::INT_VEC && input_type != InputType::STRING_VEC) - return field_name + " must be homogeneous"; + return absl::InvalidArgumentError(field_name + " must be homogeneous"); if (array[0].IsInt()) { if (input_type == InputType::STRING_VEC) - return field_name + " must be homogeneous"; + return absl::InvalidArgumentError(field_name + " must be homogeneous"); input_type = InputType::INT_VEC; std::vector ints; ints.reserve(array.Size()); @@ -182,12 +183,12 @@ std::variant TokenizeParser::parseI if (val.IsInt()) ints.push_back(val.GetInt()); else - return field_name + " must be homogeneous"; + return absl::InvalidArgumentError(field_name + " must be homogeneous"); } input_tokens.emplace_back(std::move(ints)); } else if (array[0].IsString()) { if (input_type == InputType::INT_VEC) - return field_name + " must be homogeneous"; + return absl::InvalidArgumentError(field_name + " must be homogeneous"); input_type = InputType::STRING_VEC; std::vector strings; strings.reserve(array.Size()); @@ -195,44 +196,47 @@ std::variant TokenizeParser::parseI if (val.IsString()) strings.push_back(val.GetString()); else - return field_name + " must be homogeneous"; + return absl::InvalidArgumentError(field_name + " must be homogeneous"); } input_string_vectors.emplace_back(std::move(strings)); } else { - return "every element in " + field_name + " array should be either string or int"; + return absl::InvalidArgumentError("every element in " + field_name + " array should be either string or int"); } } else if (input.IsString()) { if (input_type != InputType::NONE && input_type != InputType::STRING) - return field_name + " must be homogeneous"; + return absl::InvalidArgumentError(field_name + " must be homogeneous"); input_type = InputType::STRING; input_strings.push_back(input.GetString()); } else if (input.IsInt()) { if (input_type != InputType::NONE && input_type != InputType::INT) - return field_name + " must be homogeneous"; + return absl::InvalidArgumentError(field_name + " must be homogeneous"); input_type = InputType::INT; if (input_tokens.size() == 0) { input_tokens.push_back(std::vector()); } input_tokens[0].push_back(input.GetInt()); } else { - return "every element in " + field_name + " array should be either string or int"; + return absl::InvalidArgumentError("every element in " + field_name + " array should be either string or int"); } } } else { - return field_name + " should be string, array of strings or array of integers"; + return absl::InvalidArgumentError(field_name + " should be string, array of strings or array of integers"); } } else { - return field_name + " field is required"; + return absl::InvalidArgumentError(field_name + " field is required"); } if (input_strings.size() > 0) { + if (isSingleString) { + return std::move(input_strings.front()); + } return input_strings; } else if (input_tokens.size() > 0) { return input_tokens; } else if (input_string_vectors.size() > 0) { return input_string_vectors; } else { - return field_name + " field is required"; + return absl::InvalidArgumentError(field_name + " field is required"); } } diff --git a/src/tokenize/tokenize_parser.hpp b/src/tokenize/tokenize_parser.hpp index 7cb5be5bad..f3a8d8ab7c 100644 --- a/src/tokenize/tokenize_parser.hpp +++ b/src/tokenize/tokenize_parser.hpp @@ -35,13 +35,16 @@ #include "src/port/rapidjson_stringbuffer.hpp" #pragma warning(push) #pragma warning(disable : 6001 6385 6386) +#include "absl/status/statusor.h" #include "absl/strings/escaping.h" #pragma warning(pop) namespace ovms { struct TokenizeRequest { - using InputDataType = std::variant, std::vector>, std::vector>>; + // A plain string is the unbatched form. parseInput() returns it as such, and the request parsers turn it into + // a vector of one string and clear inputIsBatch, so a parsed request's input never holds a std::string. + using InputDataType = std::variant, std::vector>, std::vector>>; InputDataType input; ov::AnyMap parameters = {}; // True when "text" was given as an array of strings, even if the array holds a single string. @@ -52,7 +55,7 @@ class TokenizeParser { public: static constexpr const char* TOKENIZE_ENDPOINT_SUFFIX = "tokenize"; static constexpr size_t MAX_LENGTH_LIMIT = 1000000; - static std::variant parseInput(rapidjson::Document& parsedJson, const std::string& field_name); + static absl::StatusOr parseInput(rapidjson::Document& parsedJson, const std::string& field_name); static absl::Status parseTokenizeResponse(rapidjson::StringBuffer& buffer, const ov::genai::TokenizedInputs& tokens, const ov::AnyMap& parameters = {}, bool inputIsBatch = false); static absl::Status parseTokenizeRequest(rapidjson::Document& parsedJson, TokenizeRequest& request, std::optional maxModelLength = std::nullopt); static std::variant validateTokenizeRequest(rapidjson::Document& parsedJson, std::optional maxModelLength = std::nullopt);