From a33e2aead88e404cdd0e592a749422c8fcdaae19 Mon Sep 17 00:00:00 2001 From: Tom Krane <157059462+tkran2@users.noreply.github.com> Date: Wed, 21 Oct 2026 13:24:13 -0500 Subject: [PATCH] vocab : add clean_spaces GGUF key --- gguf-py/gguf/constants.py | 1 + gguf-py/gguf/gguf_writer.py | 3 +++ gguf-py/gguf/vocab.py | 10 ++++++++++ src/llama-arch.cpp | 1 + src/llama-arch.h | 1 + src/llama-model-saver.cpp | 1 + src/llama-vocab.cpp | 15 ++++++++++++++- tests/CMakeLists.txt | 2 +- tests/test-tokenizer-0.cpp | 19 ++++++++++++++++--- 9 files changed, 48 insertions(+), 5 deletions(-) diff --git a/gguf-py/gguf/constants.py b/gguf-py/gguf/constants.py index 2840348cba24..b26b8ca5bbdd 100644 --- a/gguf-py/gguf/constants.py +++ b/gguf-py/gguf/constants.py @@ -370,6 +370,7 @@ class Tokenizer: DNA_AUTO_TAGS = "tokenizer.ggml.dna_auto_tags" ADD_PREFIX = "tokenizer.ggml.add_space_prefix" REMOVE_EXTRA_WS = "tokenizer.ggml.remove_extra_whitespaces" + CLEAN_SPACES = "tokenizer.ggml.clean_spaces" PRECOMPILED_CHARSMAP = "tokenizer.ggml.precompiled_charsmap" SUPPRESS_TOKENS = "tokenizer.ggml.suppress_tokens" HF_JSON = "tokenizer.huggingface.json" diff --git a/gguf-py/gguf/gguf_writer.py b/gguf-py/gguf/gguf_writer.py index e8a96e57102e..581037c12004 100644 --- a/gguf-py/gguf/gguf_writer.py +++ b/gguf-py/gguf/gguf_writer.py @@ -1323,6 +1323,9 @@ def add_add_space_prefix(self, value: bool) -> None: def add_remove_extra_whitespaces(self, value: bool) -> None: self.add_bool(Keys.Tokenizer.REMOVE_EXTRA_WS, value) + def add_clean_spaces(self, value: bool) -> None: + self.add_bool(Keys.Tokenizer.CLEAN_SPACES, value) + def add_precompiled_charsmap(self, charsmap: bytes) -> None: self.add_array(Keys.Tokenizer.PRECOMPILED_CHARSMAP, charsmap) diff --git a/gguf-py/gguf/vocab.py b/gguf-py/gguf/vocab.py index 78e90eddc4b3..6383178ea7be 100644 --- a/gguf-py/gguf/vocab.py +++ b/gguf-py/gguf/vocab.py @@ -55,6 +55,7 @@ class SpecialVocab: normalizer_nfc: bool | None normalizer_lowercase: bool | None normalizer_strip_accents: bool | None + clean_spaces: bool | None def __init__( self, path: str | os.PathLike[str], load_merges: bool = False, @@ -70,6 +71,7 @@ def __init__( self.normalizer_nfc = None self.normalizer_lowercase = None self.normalizer_strip_accents = None + self.clean_spaces = None if special_token_types is not None: self.special_token_types = special_token_types else: @@ -120,6 +122,10 @@ def add_to_gguf(self, gw: GGUFWriter, quiet: bool = False) -> None: if not quiet: logger.info(f'Setting normalizer_strip_accents to {self.normalizer_strip_accents}') gw.add_normalizer_strip_accents(self.normalizer_strip_accents) + if self.clean_spaces is not None: + if not quiet: + logger.info(f'Setting clean_spaces to {self.clean_spaces}') + gw.add_clean_spaces(self.clean_spaces) def _load(self, path: Path) -> None: self._try_load_from_tokenizer_json(path) @@ -345,6 +351,10 @@ def _try_load_from_tokenizer_json(self, path: Path) -> bool: self.chat_template = chat_template else: logger.warning(f'Bad type for chat_template field in {tokenizer_config_file!r} - ignoring') + # decode clean-up of spaces before punctuation (transformers clean_up_tokenization_spaces) + clean_spaces = tokenizer_config.get('clean_up_tokenization_spaces') + if isinstance(clean_spaces, bool): + self.clean_spaces = clean_spaces for typ in self.special_token_types: add_entry = tokenizer_config.get(f'add_{typ}_token') if isinstance(add_entry, bool): diff --git a/src/llama-arch.cpp b/src/llama-arch.cpp index 4534be22f37b..d1e76155a997 100644 --- a/src/llama-arch.cpp +++ b/src/llama-arch.cpp @@ -434,6 +434,7 @@ static const std::map LLM_KV_NAMES = { { LLM_KV_TOKENIZER_DNA_AUTO_TAGS, "tokenizer.ggml.dna_auto_tags" }, { LLM_KV_TOKENIZER_ADD_PREFIX, "tokenizer.ggml.add_space_prefix" }, { LLM_KV_TOKENIZER_REMOVE_EXTRA_WS, "tokenizer.ggml.remove_extra_whitespaces" }, + { LLM_KV_TOKENIZER_CLEAN_SPACES, "tokenizer.ggml.clean_spaces" }, { LLM_KV_TOKENIZER_PRECOMPILED_CHARSMAP, "tokenizer.ggml.precompiled_charsmap" }, { LLM_KV_TOKENIZER_HF_JSON, "tokenizer.huggingface.json" }, { LLM_KV_TOKENIZER_RWKV, "tokenizer.rwkv.world" }, diff --git a/src/llama-arch.h b/src/llama-arch.h index 5a8ca425001c..0039c1867469 100644 --- a/src/llama-arch.h +++ b/src/llama-arch.h @@ -397,6 +397,7 @@ enum llm_kv { LLM_KV_TOKENIZER_DNA_AUTO_TAGS, LLM_KV_TOKENIZER_ADD_PREFIX, LLM_KV_TOKENIZER_REMOVE_EXTRA_WS, + LLM_KV_TOKENIZER_CLEAN_SPACES, LLM_KV_TOKENIZER_PRECOMPILED_CHARSMAP, LLM_KV_TOKENIZER_HF_JSON, LLM_KV_TOKENIZER_RWKV, diff --git a/src/llama-model-saver.cpp b/src/llama-model-saver.cpp index 46b1d2ae2744..2c7bebabc728 100644 --- a/src/llama-model-saver.cpp +++ b/src/llama-model-saver.cpp @@ -423,6 +423,7 @@ void llama_model_saver::add_kv_from_model() { add_kv(LLM_KV_TOKENIZER_ADD_SEP, vocab.get_add_sep()); add_kv(LLM_KV_TOKENIZER_ADD_PREFIX, vocab.get_add_space_prefix()); add_kv(LLM_KV_TOKENIZER_REMOVE_EXTRA_WS, vocab.get_remove_extra_whitespaces()); + add_kv(LLM_KV_TOKENIZER_CLEAN_SPACES, vocab.get_clean_spaces()); add_kv(LLM_KV_TOKENIZER_PRECOMPILED_CHARSMAP, vocab.get_precompiled_charsmap()); // add_kv(LLM_KV_TOKENIZER_HF_JSON, ???); // add_kv(LLM_KV_TOKENIZER_RWKV, ???); diff --git a/src/llama-vocab.cpp b/src/llama-vocab.cpp index a3d5dedef332..19b4d66fc1f1 100644 --- a/src/llama-vocab.cpp +++ b/src/llama-vocab.cpp @@ -2422,6 +2422,7 @@ void llama_vocab::impl::load(llama_model_loader & ml, const LLM_KV & kv) { } else if (tokenizer_pre == "minicpm5") { pre_type = LLAMA_VOCAB_PRE_TYPE_MINICPM5; ignore_merges = true; + clean_spaces = false; } else if ( tokenizer_pre == "llama3" || tokenizer_pre == "llama-v3" || @@ -2430,11 +2431,16 @@ void llama_vocab::impl::load(llama_model_loader & ml, const LLM_KV & kv) { tokenizer_pre == "falcon-h1" || tokenizer_pre == "pixtral" || tokenizer_pre == "midm-2.0" || - tokenizer_pre == "lfm2" || tokenizer_pre == "jina-v5-nano") { pre_type = LLAMA_VOCAB_PRE_TYPE_LLAMA3; ignore_merges = true; add_bos = true; + } else if ( + tokenizer_pre == "lfm2") { + pre_type = LLAMA_VOCAB_PRE_TYPE_LLAMA3; + ignore_merges = true; + add_bos = true; + clean_spaces = false; } else if ( tokenizer_pre == "deepseek-llm") { pre_type = LLAMA_VOCAB_PRE_TYPE_DEEPSEEK_LLM; @@ -2485,6 +2491,7 @@ void llama_vocab::impl::load(llama_model_loader & ml, const LLM_KV & kv) { tokenizer_pre == "granite-embed-multi-311m") { pre_type = LLAMA_VOCAB_PRE_TYPE_GEMMA4; escape_whitespaces = true; + clean_spaces = false; } else if ( tokenizer_pre == "mmbert") { pre_type = LLAMA_VOCAB_PRE_TYPE_MMBERT; @@ -2531,6 +2538,7 @@ void llama_vocab::impl::load(llama_model_loader & ml, const LLM_KV & kv) { } else if ( tokenizer_pre == "dbrx") { pre_type = LLAMA_VOCAB_PRE_TYPE_DBRX; + clean_spaces = false; } else if ( tokenizer_pre == "smaug-bpe") { pre_type = LLAMA_VOCAB_PRE_TYPE_SMAUG; @@ -2542,12 +2550,14 @@ void llama_vocab::impl::load(llama_model_loader & ml, const LLM_KV & kv) { tokenizer_pre == "chatglm-bpe") { pre_type = LLAMA_VOCAB_PRE_TYPE_CHATGLM4; special_bos_id = LLAMA_TOKEN_NULL; + clean_spaces = false; } else if ( tokenizer_pre == "glm4" || tokenizer_pre == "glm5") { pre_type = LLAMA_VOCAB_PRE_TYPE_CHATGLM4; special_bos_id = LLAMA_TOKEN_NULL; ignore_merges = true; + clean_spaces = false; } else if ( tokenizer_pre == "viking") { pre_type = LLAMA_VOCAB_PRE_TYPE_VIKING; @@ -2580,6 +2590,7 @@ void llama_vocab::impl::load(llama_model_loader & ml, const LLM_KV & kv) { } else if ( tokenizer_pre == "exaone4") { pre_type = LLAMA_VOCAB_PRE_TYPE_GPT2; + clean_spaces = false; } else if ( tokenizer_pre == "exaone-moe") { pre_type = LLAMA_VOCAB_PRE_TYPE_EXAONE_MOE; @@ -2680,6 +2691,7 @@ void llama_vocab::impl::load(llama_model_loader & ml, const LLM_KV & kv) { } else if ( tokenizer_pre == "mellum2") { pre_type = LLAMA_VOCAB_PRE_TYPE_MELLUM2; + clean_spaces = false; } else if ( tokenizer_pre == "k2-horizon") { pre_type = LLAMA_VOCAB_PRE_TYPE_K2_HORIZON; @@ -2716,6 +2728,7 @@ void llama_vocab::impl::load(llama_model_loader & ml, const LLM_KV & kv) { ml.get_key(LLM_KV_TOKENIZER_ADD_PREFIX, add_space_prefix, false); ml.get_key(LLM_KV_TOKENIZER_REMOVE_EXTRA_WS, remove_extra_whitespaces, false); + ml.get_key(LLM_KV_TOKENIZER_CLEAN_SPACES, clean_spaces, false); } const int token_idx = gguf_find_key(ctx, kv(LLM_KV_TOKENIZER_LIST).c_str()); diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 24e6ba609a21..815cc15a3219 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -142,7 +142,7 @@ llama_test(test-tokenizer-0 NAME test-tokenizer-0-command-r ARGS ${PROJE llama_test(test-tokenizer-0 NAME test-tokenizer-0-deepseek-coder ARGS ${PROJECT_SOURCE_DIR}/models/ggml-vocab-deepseek-coder.gguf) llama_test(test-tokenizer-0 NAME test-tokenizer-0-deepseek-llm ARGS ${PROJECT_SOURCE_DIR}/models/ggml-vocab-deepseek-llm.gguf) llama_test(test-tokenizer-0 NAME test-tokenizer-0-falcon ARGS ${PROJECT_SOURCE_DIR}/models/ggml-vocab-falcon.gguf) -llama_test(test-tokenizer-0 NAME test-tokenizer-0-gemma-4 ARGS ${PROJECT_SOURCE_DIR}/models/ggml-vocab-gemma-4.gguf) +llama_test(test-tokenizer-0 NAME test-tokenizer-0-gemma-4 ARGS ${PROJECT_SOURCE_DIR}/models/ggml-vocab-gemma-4.gguf --check-decode) llama_test(test-tokenizer-0 NAME test-tokenizer-0-gpt-2 ARGS ${PROJECT_SOURCE_DIR}/models/ggml-vocab-gpt-2.gguf) llama_test(test-tokenizer-0 NAME test-tokenizer-0-llama-bpe ARGS ${PROJECT_SOURCE_DIR}/models/ggml-vocab-llama-bpe.gguf) llama_test(test-tokenizer-0 NAME test-tokenizer-0-llama-spm ARGS ${PROJECT_SOURCE_DIR}/models/ggml-vocab-llama-spm.gguf) diff --git a/tests/test-tokenizer-0.cpp b/tests/test-tokenizer-0.cpp index 37f8312c46fe..5253a46a612c 100644 --- a/tests/test-tokenizer-0.cpp +++ b/tests/test-tokenizer-0.cpp @@ -125,7 +125,7 @@ static llama_tests read_tests(const std::string & fname_inp, const std::string & int main(int argc, char **argv) { if (argc < 2) { - fprintf(stderr, "Usage: %s vocab-file [text-file]\n", argv[0]); + fprintf(stderr, "Usage: %s vocab-file [text-file] [--check-decode]\n", argv[0]); return 1; } @@ -135,8 +135,13 @@ int main(int argc, char **argv) { const std::string fname_out = fname + ".out"; std::string fname_text; - if (argc > 2) { - fname_text = argv[2]; + bool check_decode = false; // also check that the expected tokens detokenize to the input text + for (int i = 2; i < argc; i++) { + if (std::string(argv[i]) == "--check-decode") { + check_decode = true; + } else { + fname_text = argv[i]; + } } fprintf(stderr, "%s : reading vocab from: '%s'\n", __func__, fname.c_str()); @@ -244,6 +249,14 @@ int main(int argc, char **argv) { success = false; } + + if (check_decode) { + const std::string dec = common_detokenize(ctx, test_kv.second); + if (dec != test_kv.first) { + fprintf(stderr, "%s : failed decode: '%s' detokenized to '%s'\n", __func__, test_kv.first.c_str(), dec.c_str()); + success = false; + } + } } }); }