diff --git a/models/ggml-vocab-qwen2.gguf b/models/ggml-vocab-qwen2.gguf index 541e475bc945..30910a41824e 100644 Binary files a/models/ggml-vocab-qwen2.gguf and b/models/ggml-vocab-qwen2.gguf differ diff --git a/models/ggml-vocab-qwen2.gguf.inp b/models/ggml-vocab-qwen2.gguf.inp index 86b934e4020f..502e39a4d8d2 100644 --- a/models/ggml-vocab-qwen2.gguf.inp +++ b/models/ggml-vocab-qwen2.gguf.inp @@ -110,3 +110,5 @@ __ggml_vocab_test__ 🚀 (normal) 😶‍🌫️ (multiple emojis concatenated) ✅ 🦙🦙 3 33 333 3333 33333 333333 3333333 33333333 3.3 3..3 3...3 កាន់តែពិសេសអាច😁 ?我想在apple工作1314151天～ ------======= нещо на Български ''''''```````""""......!!!!!!?????? I've been 'told he's there, 'RE you sure? 'M not sure I'll make it, 'D you like some tea? We'Ve a'lL __ggml_vocab_test__ +café +__ggml_vocab_test__ diff --git a/models/ggml-vocab-qwen2.gguf.out b/models/ggml-vocab-qwen2.gguf.out index 595d59a44963..1b0da5917cff 100644 --- a/models/ggml-vocab-qwen2.gguf.out +++ b/models/ggml-vocab-qwen2.gguf.out @@ -44,3 +44,4 @@ 34 90063 128324 2560 2347 198 4710 14731 65497 7847 1572 2303 78672 10947 145836 320 8252 8 26525 114 378 235 149921 30543 320 35673 99066 97534 8 25521 227 11162 99 247 149955 220 18 220 18 18 220 18 18 18 220 18 18 18 18 220 18 18 18 18 18 220 18 18 18 18 18 18 220 18 18 18 18 18 18 18 220 18 18 18 18 18 18 18 18 220 18 13 18 220 18 496 18 220 18 1112 18 220 146394 97529 241 44258 233 146568 44258 224 147603 20879 115 146280 44258 223 146280 147272 97529 227 144534 937 104100 18493 22377 99257 16 18 16 19 16 20 16 35727 21216 55460 53237 18658 14144 1456 13073 63471 33594 3038 133178 79012 3355 4605 4605 13874 13874 73594 3014 3014 28149 17085 2928 26610 7646 358 3003 1012 364 83 813 566 594 1052 11 364 787 498 2704 30 364 44 537 2704 358 3278 1281 432 11 364 35 498 1075 1045 15243 30 1205 6 42612 264 63866 43 + 924 58858 diff --git a/src/llama-vocab.cpp b/src/llama-vocab.cpp index 05d2527dc073..c956217e877d 100644 --- a/src/llama-vocab.cpp +++ b/src/llama-vocab.cpp @@ -628,7 +628,17 @@ struct llm_tokenizer_bpe_session { } } - virtual void tokenize(const std::string & text, std::vector & output) { + virtual void tokenize(const std::string & text_in, std::vector & output) { + // normalize before the pre-tokenizer when the vocab requests NFC + std::string text_nfc; + const bool apply_nfc = vocab.get_normalizer_opts().nfc; + if (apply_nfc) { + for (const uint32_t cpt : unicode_cpts_normalize_nfc(unicode_cpts_from_utf8(text_in))) { + text_nfc += unicode_cpt_to_utf8(cpt); + } + } + const std::string & text = apply_nfc ? text_nfc : text_in; + int final_prev_index = -1; auto word_collection = unicode_regex_split(text, tokenizer.regex_exprs, tokenizer.byte_encode);