diff --git a/conversion/base.py b/conversion/base.py index 0f3bd9a7b755..b481b13804f0 100644 --- a/conversion/base.py +++ b/conversion/base.py @@ -2505,6 +2505,14 @@ def _set_vocab_plamo(self): with open(tokenizer_config_path, "r", encoding="utf-8") as f: tokenizer_config = json.load(f) + tokenizer_class = tokenizer_config.get("tokenizer_class") + if tokenizer_class == "Plamo2Tokenizer": + tokenizer_model = "plamo2" + elif tokenizer_class == "Plamo3Tokenizer": + tokenizer_model = "plamo3" + else: + raise ValueError(f"Unsupported PLaMo tokenizer class: {tokenizer_class}") + # Load tokens from JSONL file (actually a list format) tokens = [] scores = [] @@ -2544,7 +2552,7 @@ def _set_vocab_plamo(self): scores.append(-1000.0) toktypes.append(gguf.TokenType.UNUSED) - self.gguf_writer.add_tokenizer_model("plamo2") + self.gguf_writer.add_tokenizer_model(tokenizer_model) self.gguf_writer.add_tokenizer_pre("default") self.gguf_writer.add_token_list(tokens) self.gguf_writer.add_token_scores(scores) diff --git a/include/llama.h b/include/llama.h index 9bca8ee1063d..260247e8276f 100644 --- a/include/llama.h +++ b/include/llama.h @@ -78,6 +78,7 @@ extern "C" { LLAMA_VOCAB_TYPE_RWKV = 5, // RWKV tokenizer based on greedy tokenization LLAMA_VOCAB_TYPE_PLAMO2 = 6, // PLaMo-2 tokenizer based on Aho-Corasick with dynamic programming LLAMA_VOCAB_TYPE_TEST = 7, // Dummy tokenizer for testing: rolling hash of fixed-size chunks -> tokens, tokens -> hex + LLAMA_VOCAB_TYPE_PLAMO3 = 8, // PLaMo-3 tokenizer with pre-segmentation and dynamic programming }; enum llama_rope_type { diff --git a/src/llama-vocab.cpp b/src/llama-vocab.cpp index 015e98374167..f2c248c3d382 100644 --- a/src/llama-vocab.cpp +++ b/src/llama-vocab.cpp @@ -1395,7 +1395,7 @@ struct llm_tokenizer_rwkv_session { }; struct llm_tokenizer_plamo2 : llm_tokenizer { - llm_tokenizer_plamo2(const llama_vocab & vocab) { + llm_tokenizer_plamo2(const llama_vocab & vocab, bool pre_segment) : pre_segment_(pre_segment) { build(vocab); } @@ -1538,11 +1538,87 @@ struct llm_tokenizer_plamo2 : llm_tokenizer { std::vector encode(const std::string & text) const { std::vector unicode_data = unicode_cpts_from_utf8(text); - // Skip the first code point if it is a BOM (Byte Order Mark) - if (!unicode_data.empty() && unicode_data[0] == 0xFEFF) { - unicode_data.erase(unicode_data.begin()); + // The PLaMo-3 tokenizer keeps a leading U+FEFF in the input. + if (!pre_segment_) { + if (!unicode_data.empty() && unicode_data[0] == 0xFEFF) { + unicode_data.erase(unicode_data.begin()); + } + return encode_cpts(unicode_data); + } + + const size_t n = unicode_data.size(); + std::vector cut(n + 1, false); + + // pass 1: <|plamo:...|> + { + static constexpr uint32_t prefix[] = { '<', '|', 'p', 'l', 'a', 'm', 'o', ':' }; + const size_t prefix_len = std::size(prefix); + size_t i = 0; + while (i + prefix_len <= n) { + if (!std::equal(prefix, prefix + prefix_len, unicode_data.begin() + i)) { + i++; + continue; + } + // An empty body is valid. + size_t j = i + prefix_len; + // Treat U+001C..U+001F as whitespace (equivalent to Python \s). + while (j < n && j - (i + prefix_len) < 64 && unicode_data[j] != '|' && + (unicode_data[j] < 0x1C || unicode_data[j] > 0x1F) && + !unicode_cpt_flags_from_cpt(unicode_data[j]).is_whitespace) { + j++; + } + if (j + 1 < n && unicode_data[j] == '|' && unicode_data[j + 1] == '>') { + cut[i] = true; + cut[j + 2] = true; + i = j + 2; + } else { + i++; + } + } + } + + // pass 2: runs of repeated characters / spaces (a run never crosses a boundary from pass 1) + { + size_t i = 0; + while (i < n) { + const uint32_t c = unicode_data[i]; + size_t run = 1; + while (i + run < n && unicode_data[i + run] == c && !cut[i + run]) { + run++; + } + + const bool is_repeated_chars = c != '\n' && run >= 4; + const bool is_spaces = c == ' ' && run >= 2; + if (is_repeated_chars || is_spaces) { + cut[i] = true; + cut[i + run] = true; + } + + // a run that does not match cannot match at any later position either (it only gets shorter) + i += run; + } + } + + std::vector output; + size_t seg_start = 0; + for (size_t seg_end = 0; seg_end <= n; ++seg_end) { + // U+EE00 is the tokenizer's private-use boundary marker; literal occurrences split segments and are not emitted. + const bool is_boundary = seg_end < n && unicode_data[seg_end] == 0xEE00; + if (seg_end == n || cut[seg_end] || is_boundary) { + if (seg_start < seg_end) { + const std::vector segment(unicode_data.begin() + seg_start, + unicode_data.begin() + seg_end); + const std::vector tokens = encode_cpts(segment); + output.insert(output.end(), tokens.begin(), tokens.end()); + } + seg_start = seg_end + is_boundary; + } } + return output; + } + + std::vector encode_cpts(const std::vector & unicode_data) const { if (unicode_data.empty()) { return {}; } @@ -1661,6 +1737,8 @@ struct llm_tokenizer_plamo2 : llm_tokenizer { // Flattened table representing the Trie structure // Each row contains: [piece_length, token_id, score, piece_id] std::vector> table_; + + const bool pre_segment_; }; struct llm_tokenizer_plamo2_session { @@ -2127,10 +2205,10 @@ void llama_vocab::impl::load(llama_model_loader & ml, const LLM_KV & kv) { special_sep_id = LLAMA_TOKEN_NULL; special_pad_id = LLAMA_TOKEN_NULL; special_mask_id = LLAMA_TOKEN_NULL; - } else if (tokenizer_model == "plamo2") { - type = LLAMA_VOCAB_TYPE_PLAMO2; + } else if (tokenizer_model == "plamo2" || tokenizer_model == "plamo3") { + type = tokenizer_model == "plamo2" ? LLAMA_VOCAB_TYPE_PLAMO2 : LLAMA_VOCAB_TYPE_PLAMO3; - // PLaMo-2 default special tokens (these will be overridden by model config) + // PLaMo default special tokens (these will be overridden by model config) special_bos_id = 1; // <|plamo:bos|> special_eos_id = 2; // <|plamo:eos|> special_unk_id = 0; // <|plamo:unk|> @@ -3183,6 +3261,7 @@ std::string llama_vocab::impl::type_name() const{ case LLAMA_VOCAB_TYPE_UGM: return "UGM"; case LLAMA_VOCAB_TYPE_RWKV: return "RWKV"; case LLAMA_VOCAB_TYPE_PLAMO2: return "PLaMo2"; + case LLAMA_VOCAB_TYPE_PLAMO3: return "PLaMo3"; case LLAMA_VOCAB_TYPE_TEST: return "TEST"; default: return "unknown"; } @@ -3270,7 +3349,10 @@ void llama_vocab::impl::init_tokenizer(enum llama_vocab_type type) { tokenizer = std::make_unique(vocab); break; case LLAMA_VOCAB_TYPE_PLAMO2: - tokenizer = std::make_unique(vocab); + tokenizer = std::make_unique(vocab, false); + break; + case LLAMA_VOCAB_TYPE_PLAMO3: + tokenizer = std::make_unique(vocab, true); break; case LLAMA_VOCAB_TYPE_TEST: tokenizer = std::make_unique(); @@ -3632,6 +3714,7 @@ std::vector llama_vocab::impl::tokenize( } } break; case LLAMA_VOCAB_TYPE_PLAMO2: + case LLAMA_VOCAB_TYPE_PLAMO3: { if (add_special && add_bos) { GGML_ASSERT(special_bos_id != LLAMA_TOKEN_NULL); @@ -3803,8 +3886,9 @@ int32_t llama_vocab::impl::token_to_piece(llama_token token, char * buf, int32_t std::string result = format("%x", token); return _try_copy(result.data(), result.size()); } - case LLAMA_VOCAB_TYPE_PLAMO2: { - // PLaMo-2 uses similar token handling as BPE/SPM + case LLAMA_VOCAB_TYPE_PLAMO2: + case LLAMA_VOCAB_TYPE_PLAMO3: { + // PLaMo uses similar token handling as BPE/SPM if (vocab.is_byte(token)) { // Handle byte tokens like <0xXX> if (token_text.length() == 6 && token_text.substr(0, 3) == "<0x" && token_text.back() == '>') { @@ -4067,8 +4151,9 @@ llama_token llama_vocab::byte_to_token(uint8_t ch) const { case LLAMA_VOCAB_TYPE_BPE: { return pimpl->token_to_id.at(unicode_byte_to_utf8(ch)); } - case LLAMA_VOCAB_TYPE_PLAMO2: { - // PLaMo-2 uses byte tokens in format <0xXX> + case LLAMA_VOCAB_TYPE_PLAMO2: + case LLAMA_VOCAB_TYPE_PLAMO3: { + // PLaMo uses byte tokens in format <0xXX> char hex_str[8]; snprintf(hex_str, sizeof(hex_str), "<0x%02X>", ch); return pimpl->token_to_id.at(hex_str);