From 69aaf6cc4049f79c9374d178a388a7641ac825bb Mon Sep 17 00:00:00 2001 From: JohnUzoka <148659447+JohnUzoka@users.noreply.github.com> Date: Sun, 15 Nov 2026 19:12:32 -0500 Subject: [PATCH] maple : preserve canonical tokenizer and generation metadata Assisted-by: OpenAI coding assistant --- conversion/maple.py | 55 ++- gguf-py/README.md | 19 + gguf-py/gguf/constants.py | 1 + gguf-py/gguf/gguf_reader.py | 2 + gguf-py/gguf/gguf_writer.py | 12 +- gguf-py/tests/test_gguf_reader_validation.py | 25 ++ src/llama-arch.cpp | 1 + src/llama-arch.h | 1 + src/llama-model-saver.cpp | 7 + src/llama-vocab.cpp | 338 ++++++++------- src/llama-vocab.h | 2 + tests/CMakeLists.txt | 6 + tests/test-tokenizer-normalization.cpp | 410 +++++++++++++++++++ 13 files changed, 727 insertions(+), 152 deletions(-) create mode 100644 tests/test-tokenizer-normalization.cpp diff --git a/conversion/maple.py b/conversion/maple.py index fb0e87804d53..300d943c31bc 100644 --- a/conversion/maple.py +++ b/conversion/maple.py @@ -1,5 +1,7 @@ from __future__ import annotations +import json + from typing import Iterable, TYPE_CHECKING, cast import torch @@ -15,6 +17,52 @@ class MapleModel(TextModel): model_arch = gguf.MODEL_ARCH.MAPLE + def get_vocab_base(self) -> tuple[list[str], list[int], str]: + tokens, toktypes, tokpre = super().get_vocab_base() + with (self.dir_model / "tokenizer.json").open(encoding="utf-8") as stream: + tokenizer = json.load(stream) + with (self.dir_model / "tokenizer_config.json").open(encoding="utf-8") as stream: + settings = json.load(stream) + if tokenizer.get("normalizer") != {"type": "NFC"}: + raise ValueError("Maple requires the canonical exact NFC normalizer") + if tokenizer.get("model", {}).get("type") != "BPE" or tokpre != "qwen2": + raise ValueError("Unsupported Maple tokenizer family") + # Keep the existing BPE vocabulary/padding conventions, but do not infer + # specialness from spelling: Maple's FIM/tool/think tokens are lexical. + for text, token_id in tokenizer["model"]["vocab"].items(): + if type(token_id) is not int or not 0 <= token_id < len(tokens) or tokens[token_id] != text: + raise ValueError(f"Maple canonical vocabulary mismatch at {token_id}") + toktypes[token_id] = gguf.TokenType.NORMAL + for added in tokenizer["added_tokens"]: + token_id = added["id"] + if type(token_id) is not int or not 0 <= token_id < len(tokens) or tokens[token_id] != added["content"]: + raise ValueError(f"Maple canonical added-token mismatch at {token_id}") + if type(added.get("special")) is not bool: + raise ValueError(f"Missing Maple added-token special flag at {token_id}") + other = settings.get("added_tokens_decoder", {}).get(str(token_id)) + flags = ("content", "special", "lstrip", "rstrip", "single_word", "normalized") + if other is None or any(added.get(key) != other.get(key) for key in flags): + raise ValueError(f"Maple added-token configurations disagree at {token_id}") + if any(added.get(key) is not False for key in ("lstrip", "rstrip", "single_word", "normalized")): + raise ValueError("Unsupported Maple added-token boundary transformation") + toktypes[token_id] = gguf.TokenType.CONTROL if added["special"] else gguf.TokenType.USER_DEFINED + return tokens, toktypes, tokpre + + def set_vocab(self): + self._set_vocab_gpt2() + # GenerationConfig overrides model EOS, including an explicit null/empty + # policy. The runtime must not append stop-looking lexical tokens. + generation_path = self.dir_model / "generation_config.json" + generation = {} + if generation_path.is_file(): + with generation_path.open(encoding="utf-8") as stream: + generation = json.load(stream) + eos = generation.get("eos_token_id", self.hparams.get("eos_token_id")) + ids = [] if eos is None else eos if isinstance(eos, list) else [eos] + if any(type(token_id) is not int or not 0 <= token_id < self.hparams["vocab_size"] for token_id in ids): + raise ValueError("Invalid Maple generation EOS IDs") + self.gguf_writer.add_eog_token_ids(list(dict.fromkeys(ids))) + def set_gguf_parameters(self): super().set_gguf_parameters() hparams = self.hparams @@ -29,7 +77,12 @@ def set_gguf_parameters(self): self.gguf_writer.add_vocab_size(hparams["vocab_size"]) self.gguf_writer.add_rope_dimension_count(int(head_dim * partial_rotary_factor)) - self.gguf_writer.add_sliding_window(hparams["sliding_window"]) + # FlashAttention's left span is inclusive: 512 past keys + current key. + # GGUF STANDARD stores total causal keys; normalize only in the producer. + left_span = hparams["sliding_window"] + if type(left_span) is not int or not 0 < left_span < 2**32 - 1: + raise ValueError("Invalid Maple sliding-window left span") + self.gguf_writer.add_sliding_window(left_span + 1) self.gguf_writer.add_sliding_window_pattern([layer_type == "sliding_attention" for layer_type in hparams["layer_types"]]) self.gguf_writer.add_expert_feed_forward_length(hparams["moe_intermediate_size"]) # the reference clamps the MoE SwiGLU gate/up at 7.0 (modeling_maple.py) diff --git a/gguf-py/README.md b/gguf-py/README.md index ca7e09c68184..cc2f91d5464a 100644 --- a/gguf-py/README.md +++ b/gguf-py/README.md @@ -32,6 +32,25 @@ pip install gguf[gui] [gguf/scripts/gguf_editor_gui.py](https://github.com/ggml-org/llama.cpp/blob/master/gguf-py/gguf/scripts/gguf_editor_gui.py) — Allows for viewing, editing, adding, or removing metadata values within a GGUF file as well as viewing its tensors with a Qt interface. +## Explicit end-of-generation token policy + +`GGUFWriter.add_eog_token_ids(ids)` writes the optional +`tokenizer.ggml.eog_token_ids` extension as an array of `UINT32` token IDs. +In the corresponding llama.cpp runtime, presence makes this the complete +end-of-generation set: stop-looking spellings do not add more IDs or change +the vocabulary's declared token types. An explicitly empty array means no +token-based generation stops; omitting the key retains legacy inference. +IDs must be within the model vocabulary. This policy is independent of BOS +insertion and lexical/special-token parsing. + +Older runtimes may ignore this extension. Use a runtime that implements it +when canonical stop behavior matters. Empty arrays retain their element +type through the Python writer and reader. + +For Maple conversion, the producer validates the exact NFC/BPE tokenizer family and added-token boundary flags, preserves each token's declared lexical/special status, and writes the generation configuration's complete EOS policy. The producer converts the reference sliding window's left span to GGUF's total causal-key count (`left_span + 1`); the runtime must not repeat this adjustment. Model saving retains the current EOG set (including empty) and NFC policy. + +Native regression coverage is registered as `test-tokenizer-eog-policy`, `test-tokenizer-eog-roundtrip`, and `test-tokenizer-normalization-{gpt2,qwen35}`. The first two use synthetic fixtures; the normalization variants use the matching vocabulary GGUF under `models/`. + ## Development Maintainers who participate in development of this package are advised to install it in editable mode: diff --git a/gguf-py/gguf/constants.py b/gguf-py/gguf/constants.py index b29899cf0bd7..94a6f7d869c4 100644 --- a/gguf-py/gguf/constants.py +++ b/gguf-py/gguf/constants.py @@ -377,6 +377,7 @@ class Tokenizer: MERGES = "tokenizer.ggml.merges" BOS_ID = "tokenizer.ggml.bos_token_id" EOS_ID = "tokenizer.ggml.eos_token_id" + EOG_IDS = "tokenizer.ggml.eog_token_ids" EOT_ID = "tokenizer.ggml.eot_token_id" EOM_ID = "tokenizer.ggml.eom_token_id" UNK_ID = "tokenizer.ggml.unknown_token_id" diff --git a/gguf-py/gguf/gguf_reader.py b/gguf-py/gguf/gguf_reader.py index bf3e083800b0..21225254df59 100644 --- a/gguf-py/gguf/gguf_reader.py +++ b/gguf-py/gguf/gguf_reader.py @@ -258,6 +258,8 @@ def _get_field_parts( offs += int(alen.nbytes) aparts: list[npt.NDArray[Any]] = [raw_itype, alen] data_idxs: list[int] = [] + if alen[0] == 0: + types.append(GGUFValueType(raw_itype[0])) # FIXME: Handle multi-dimensional arrays properly instead of flattening for idx in range(alen[0]): curr_size, curr_parts, curr_idxs, curr_types = self._get_field_parts(offs, raw_itype[0]) diff --git a/gguf-py/gguf/gguf_writer.py b/gguf-py/gguf/gguf_writer.py index 706d193bd7fe..094d8e5213b0 100644 --- a/gguf-py/gguf/gguf_writer.py +++ b/gguf-py/gguf/gguf_writer.py @@ -1339,6 +1339,14 @@ def add_bos_token_id(self, id: int) -> None: def add_eos_token_id(self, id: int) -> None: self.add_uint32(Keys.Tokenizer.EOS_ID, id) + def add_eog_token_ids(self, ids: Sequence[int]) -> None: + """Set the complete EOG set; an empty array explicitly disables natural stops. + + This policy is independent of token types and replaces legacy text-based + EOG inference. Omitting the key retains that inference. + """ + self.add_key_value(Keys.Tokenizer.EOG_IDS, ids, GGUFValueType.ARRAY, GGUFValueType.UINT32) + def add_unk_token_id(self, id: int) -> None: self.add_uint32(Keys.Tokenizer.UNK_ID, id) @@ -1818,8 +1826,8 @@ def _pack_val(self, val: Any, vtype: GGUFValueType, add_vtype: bool, sub_type: G if not isinstance(val, Sequence): raise ValueError("Invalid GGUF metadata array, expecting sequence") - if len(val) == 0: - raise ValueError("Invalid GGUF metadata array. Empty array") + if len(val) == 0 and sub_type is None: + raise ValueError("Invalid GGUF metadata array. Empty array requires an explicit type") if sub_type is not None: ltype = sub_type diff --git a/gguf-py/tests/test_gguf_reader_validation.py b/gguf-py/tests/test_gguf_reader_validation.py index 98f30a969608..2c0a3f1f8210 100644 --- a/gguf-py/tests/test_gguf_reader_validation.py +++ b/gguf-py/tests/test_gguf_reader_validation.py @@ -3,6 +3,7 @@ import pytest from gguf.gguf_reader import GGUFReader +from gguf.constants import GGUFValueType def _write_gguf(path, n_dims_field, dims): @@ -35,3 +36,27 @@ def test_dims_product_no_uint64_wraparound(tmp_path): _write_gguf(p, len(dims), dims) with pytest.raises(ValueError): GGUFReader(p) + + +def _write_empty_array(path, element_type): + key = b'tokenizer.ggml.eog_token_ids' + data = b'GGUF' + struct.pack(' LLM_KV_NAMES = { { LLM_KV_TOKENIZER_MERGES, "tokenizer.ggml.merges" }, { LLM_KV_TOKENIZER_BOS_ID, "tokenizer.ggml.bos_token_id" }, { LLM_KV_TOKENIZER_EOS_ID, "tokenizer.ggml.eos_token_id" }, + { LLM_KV_TOKENIZER_EOG_IDS, "tokenizer.ggml.eog_token_ids" }, { LLM_KV_TOKENIZER_EOT_ID, "tokenizer.ggml.eot_token_id" }, { LLM_KV_TOKENIZER_EOM_ID, "tokenizer.ggml.eom_token_id" }, { LLM_KV_TOKENIZER_UNK_ID, "tokenizer.ggml.unknown_token_id" }, diff --git a/src/llama-arch.h b/src/llama-arch.h index 4c761c6f9447..c78a31ea866d 100644 --- a/src/llama-arch.h +++ b/src/llama-arch.h @@ -412,6 +412,7 @@ enum llm_kv { LLM_KV_TOKENIZER_MERGES, LLM_KV_TOKENIZER_BOS_ID, LLM_KV_TOKENIZER_EOS_ID, + LLM_KV_TOKENIZER_EOG_IDS, LLM_KV_TOKENIZER_EOT_ID, LLM_KV_TOKENIZER_EOM_ID, LLM_KV_TOKENIZER_UNK_ID, diff --git a/src/llama-model-saver.cpp b/src/llama-model-saver.cpp index 752e6361feda..e45470971293 100644 --- a/src/llama-model-saver.cpp +++ b/src/llama-model-saver.cpp @@ -437,6 +437,12 @@ void llama_model_saver::add_kv_from_model() { add_kv(LLM_KV_TOKENIZER_TOKEN_TYPE_COUNT, vocab.n_token_types()); add_kv(LLM_KV_TOKENIZER_SCORES, scores); add_kv(LLM_KV_TOKENIZER_MERGES, vocab.get_bpe_merges()); + const auto & eog_set = vocab.get_eog_tokens(); + const std::vector eog_ids(eog_set.begin(), eog_set.end()); + const uint32_t empty_eog = 0; + // Empty is authoritative too; the generic array helper omits empty values. + gguf_set_arr_data(gguf_ctx, llm_kv(LLM_KV_TOKENIZER_EOG_IDS).c_str(), GGUF_TYPE_UINT32, + eog_ids.empty() ? &empty_eog : eog_ids.data(), eog_ids.size()); // FIXME llama_token is type i32 but when reading in a GGUF file u32 is expected, not an issue for writing though if (vocab.token_bos() != LLAMA_TOKEN_NULL) { add_kv(LLM_KV_TOKENIZER_BOS_ID, uint32_t(vocab.token_bos())); } if (vocab.token_eos() != LLAMA_TOKEN_NULL) { add_kv(LLM_KV_TOKENIZER_EOS_ID, uint32_t(vocab.token_eos())); } @@ -453,6 +459,7 @@ void llama_model_saver::add_kv_from_model() { add_kv(LLM_KV_TOKENIZER_ADD_PREFIX, vocab.get_add_space_prefix()); add_kv(LLM_KV_TOKENIZER_REMOVE_EXTRA_WS, vocab.get_remove_extra_whitespaces()); add_kv(LLM_KV_TOKENIZER_CLEAN_SPACES, vocab.get_clean_spaces()); + add_kv(LLM_KV_TOKENIZER_NORMALIZER_NFC, vocab.get_normalizer_opts().nfc); add_kv(LLM_KV_TOKENIZER_PRECOMPILED_CHARSMAP, vocab.get_precompiled_charsmap()); // add_kv(LLM_KV_TOKENIZER_HF_JSON, ???); // add_kv(LLM_KV_TOKENIZER_RWKV, ???); diff --git a/src/llama-vocab.cpp b/src/llama-vocab.cpp index 65b569762a2a..8bbb04d8c819 100644 --- a/src/llama-vocab.cpp +++ b/src/llama-vocab.cpp @@ -645,17 +645,28 @@ struct llm_tokenizer_bpe_session { virtual void tokenize(const std::string & text, std::vector & output) { auto tok_pre = vocab.get_pre_type(); + std::string canonical; + const bool normalize_nfc = vocab.get_normalizer_opts().nfc && + std::any_of(text.begin(), text.end(), [](unsigned char c) { return c >= 0x80; }); + if (normalize_nfc) { + canonical.reserve(text.size()); + for (const uint32_t cpt : unicode_cpts_normalize_nfc(unicode_cpts_from_utf8(text))) { + canonical += unicode_cpt_to_utf8(cpt); + } + } + const std::string & input = normalize_nfc ? canonical : text; + std::vector word_collection; if (tok_pre == LLAMA_VOCAB_PRE_TYPE_NEMOTRON_3_5_EMBD) { static const std::string zwnj = "\xe2\x80\x8c"; - std::string normalized_text = text; + std::string normalized_text = input; replace_all(normalized_text, "[" + zwnj, "["); replace_all(normalized_text, "<" + zwnj, "<"); word_collection = unicode_regex_split(normalized_text, tokenizer.regex_exprs, tokenizer.byte_encode); } else if (tok_pre == LLAMA_VOCAB_PRE_TYPE_SOPRANO) { std::string normalized_text; bool space = false; - for (uint32_t cpt : unicode_cpts_from_utf8(text)) { + for (uint32_t cpt : unicode_cpts_from_utf8(input)) { if (unicode_cpt_flags_from_cpt(cpt).is_whitespace) { if (!space) { normalized_text += ' '; @@ -669,7 +680,7 @@ struct llm_tokenizer_bpe_session { } word_collection = unicode_regex_split(normalized_text, tokenizer.regex_exprs, tokenizer.byte_encode); } else { - word_collection = unicode_regex_split(text, tokenizer.regex_exprs, tokenizer.byte_encode); + word_collection = unicode_regex_split(input, tokenizer.regex_exprs, tokenizer.byte_encode); } int final_prev_index = -1; @@ -2930,6 +2941,27 @@ void llama_vocab::impl::load(llama_model_loader & ml, const LLM_KV & kv) { // special tokens { + // Presence selects an authoritative policy, including an explicitly empty set. + // EOG membership is independent of lexical token attributes. + const int eog_idx = gguf_find_key(ctx, kv(LLM_KV_TOKENIZER_EOG_IDS).c_str()); + const bool has_explicit_eog = eog_idx != -1; + if (has_explicit_eog) { + if (gguf_get_kv_type(ctx, eog_idx) != GGUF_TYPE_ARRAY || + gguf_get_arr_type(ctx, eog_idx) != GGUF_TYPE_UINT32) { + throw std::runtime_error(format("invalid gguf type for %s", kv(LLM_KV_TOKENIZER_EOG_IDS).c_str())); + } + const size_t n = gguf_get_arr_n(ctx, eog_idx); + const uint32_t * ids = (const uint32_t *) gguf_get_arr_data(ctx, eog_idx); + special_eog_ids.clear(); + for (size_t i = 0; i < n; ++i) { + if (ids[i] >= id_to_token.size()) { + throw std::runtime_error(format("token id %u out of range for %s", + ids[i], kv(LLM_KV_TOKENIZER_EOG_IDS).c_str())); + } + special_eog_ids.insert((llama_token) ids[i]); + } + } + const std::vector> special_token_types = { { LLM_KV_TOKENIZER_BOS_ID, special_bos_id }, { LLM_KV_TOKENIZER_EOS_ID, special_eos_id }, @@ -3068,7 +3100,7 @@ void llama_vocab::impl::load(llama_model_loader & ml, const LLM_KV & kv) { || t.first == "" // smoldocling ) { special_eot_id = t.second; - if ((attr & LLAMA_TOKEN_ATTR_CONTROL) == 0) { + if (!has_explicit_eog && (attr & LLAMA_TOKEN_ATTR_CONTROL) == 0) { LLAMA_LOG_WARN("%s: control-looking token: %6d '%s' was not control-type; this is probably a bug in the model. its type will be overridden\n", __func__, t.second, t.first.c_str()); attr = (llama_token_attr) (attr | LLAMA_TOKEN_ATTR_CONTROL); @@ -3082,7 +3114,7 @@ void llama_vocab::impl::load(llama_model_loader & ml, const LLM_KV & kv) { || t.first == "<|eom_id|>" ) { special_eom_id = t.second; - if ((attr & LLAMA_TOKEN_ATTR_CONTROL) == 0) { + if (!has_explicit_eog && (attr & LLAMA_TOKEN_ATTR_CONTROL) == 0) { LLAMA_LOG_WARN("%s: control-looking token: %6d '%s' was not control-type; this is probably a bug in the model. its type will be overridden\n", __func__, t.second, t.first.c_str()); attr = (llama_token_attr) (attr | LLAMA_TOKEN_ATTR_CONTROL); @@ -3091,6 +3123,7 @@ void llama_vocab::impl::load(llama_model_loader & ml, const LLM_KV & kv) { } // find FIM_PRE token: "<|fim_prefix|>", "", "
", etc.
+            // Preserve USER_DEFINED for added tokens that are not special tokens.
             if (special_fim_pre_id == LLAMA_TOKEN_NULL) {
                 if (false
                         || t.first == "<|fim_prefix|>"  // Qwen
@@ -3104,7 +3137,7 @@ void llama_vocab::impl::load(llama_model_loader & ml, const LLM_KV & kv) {
                         || t.first == "<|plamo:fim_prefix|>" // PLaMo-3
                         ) {
                     special_fim_pre_id = t.second;
-                    if ((attr & LLAMA_TOKEN_ATTR_CONTROL) == 0) {
+                    if (!has_explicit_eog && (attr & (LLAMA_TOKEN_ATTR_CONTROL | LLAMA_TOKEN_ATTR_USER_DEFINED)) == 0) {
                         LLAMA_LOG_WARN("%s: control-looking token: %6d '%s' was not control-type; this is probably a bug in the model. its type will be overridden\n",
                                 __func__, t.second, t.first.c_str());
                         attr = (llama_token_attr) (attr | LLAMA_TOKEN_ATTR_CONTROL);
@@ -3126,7 +3159,7 @@ void llama_vocab::impl::load(llama_model_loader & ml, const LLM_KV & kv) {
                         || t.first == "<|plamo:fim_suffix|>" // PLaMo-3
                         ) {
                     special_fim_suf_id = t.second;
-                    if ((attr & LLAMA_TOKEN_ATTR_CONTROL) == 0) {
+                    if (!has_explicit_eog && (attr & (LLAMA_TOKEN_ATTR_CONTROL | LLAMA_TOKEN_ATTR_USER_DEFINED)) == 0) {
                         LLAMA_LOG_WARN("%s: control-looking token: %6d '%s' was not control-type; this is probably a bug in the model. its type will be overridden\n",
                                 __func__, t.second, t.first.c_str());
                         attr = (llama_token_attr) (attr | LLAMA_TOKEN_ATTR_CONTROL);
@@ -3148,7 +3181,7 @@ void llama_vocab::impl::load(llama_model_loader & ml, const LLM_KV & kv) {
                         || t.first == "<|plamo:fim_middle|>" // PLaMo-3
                         ) {
                     special_fim_mid_id = t.second;
-                    if ((attr & LLAMA_TOKEN_ATTR_CONTROL) == 0) {
+                    if (!has_explicit_eog && (attr & (LLAMA_TOKEN_ATTR_CONTROL | LLAMA_TOKEN_ATTR_USER_DEFINED)) == 0) {
                         LLAMA_LOG_WARN("%s: control-looking token: %6d '%s' was not control-type; this is probably a bug in the model. its type will be overridden\n",
                                 __func__, t.second, t.first.c_str());
                         attr = (llama_token_attr) (attr | LLAMA_TOKEN_ATTR_CONTROL);
@@ -3166,7 +3199,7 @@ void llama_vocab::impl::load(llama_model_loader & ml, const LLM_KV & kv) {
                         || t.first == "[PAD]" // Kimi-K2
                         ) {
                     special_fim_pad_id = t.second;
-                    if ((attr & LLAMA_TOKEN_ATTR_CONTROL) == 0) {
+                    if (!has_explicit_eog && (attr & (LLAMA_TOKEN_ATTR_CONTROL | LLAMA_TOKEN_ATTR_USER_DEFINED)) == 0) {
                         LLAMA_LOG_WARN("%s: control-looking token: %6d '%s' was not control-type; this is probably a bug in the model. its type will be overridden\n",
                                 __func__, t.second, t.first.c_str());
                         attr = (llama_token_attr) (attr | LLAMA_TOKEN_ATTR_CONTROL);
@@ -3184,7 +3217,7 @@ void llama_vocab::impl::load(llama_model_loader & ml, const LLM_KV & kv) {
                         || t.first == ""    // Granite
                         ) {
                     special_fim_rep_id = t.second;
-                    if ((attr & LLAMA_TOKEN_ATTR_CONTROL) == 0) {
+                    if (!has_explicit_eog && (attr & (LLAMA_TOKEN_ATTR_CONTROL | LLAMA_TOKEN_ATTR_USER_DEFINED)) == 0) {
                         LLAMA_LOG_WARN("%s: control-looking token: %6d '%s' was not control-type; this is probably a bug in the model. its type will be overridden\n",
                                 __func__, t.second, t.first.c_str());
                         attr = (llama_token_attr) (attr | LLAMA_TOKEN_ATTR_CONTROL);
@@ -3199,7 +3232,7 @@ void llama_vocab::impl::load(llama_model_loader & ml, const LLM_KV & kv) {
                         || t.first == "<|plamo:file_separator|>" // PLaMo-2/3
                         ) {
                     special_fim_sep_id = t.second;
-                    if ((attr & LLAMA_TOKEN_ATTR_CONTROL) == 0) {
+                    if (!has_explicit_eog && (attr & (LLAMA_TOKEN_ATTR_CONTROL | LLAMA_TOKEN_ATTR_USER_DEFINED)) == 0) {
                         LLAMA_LOG_WARN("%s: control-looking token: %6d '%s' was not control-type; this is probably a bug in the model. its type will be overridden\n",
                                 __func__, t.second, t.first.c_str());
                         attr = (llama_token_attr) (attr | LLAMA_TOKEN_ATTR_CONTROL);
@@ -3234,173 +3267,176 @@ void llama_vocab::impl::load(llama_model_loader & ml, const LLM_KV & kv) {
             LLAMA_LOG_INFO("%s: %u unused tokens\n", __func__, n_unused);
         }
 
-        // maintain a list of tokens that cause end-of-generation
-        // this is currently determined based on the token text, which is obviously not ideal
-        // ref: https://github.com/ggml-org/llama.cpp/issues/9606
-        special_eog_ids.clear();
-        if (special_fim_pad_id != LLAMA_TOKEN_NULL && special_eog_ids.count(special_fim_pad_id) == 0) {
-            special_eog_ids.insert(special_fim_pad_id);
-        }
+        // Legacy inference is retained only when no explicit EOG policy was supplied.
+        if (!has_explicit_eog) {
+            // maintain a list of tokens that cause end-of-generation
+            // this is currently determined based on the token text, which is obviously not ideal
+            // ref: https://github.com/ggml-org/llama.cpp/issues/9606
+            special_eog_ids.clear();
+            if (special_fim_pad_id != LLAMA_TOKEN_NULL && (id_to_token[special_fim_pad_id].attr & LLAMA_TOKEN_ATTR_CONTROL)) {
+                special_eog_ids.insert(special_fim_pad_id);
+            }
 
-        if (special_fim_rep_id != LLAMA_TOKEN_NULL && special_eog_ids.count(special_fim_rep_id) == 0) {
-            special_eog_ids.insert(special_fim_rep_id);
-        }
+            if (special_fim_rep_id != LLAMA_TOKEN_NULL && (id_to_token[special_fim_rep_id].attr & LLAMA_TOKEN_ATTR_CONTROL)) {
+                special_eog_ids.insert(special_fim_rep_id);
+            }
 
-        if (special_fim_sep_id != LLAMA_TOKEN_NULL && special_eog_ids.count(special_fim_sep_id) == 0) {
-            special_eog_ids.insert(special_fim_sep_id);
-        }
+            if (special_fim_sep_id != LLAMA_TOKEN_NULL && (id_to_token[special_fim_sep_id].attr & LLAMA_TOKEN_ATTR_CONTROL)) {
+                special_eog_ids.insert(special_fim_sep_id);
+            }
 
-        for (const auto & t : token_to_id) {
-            auto & attr = id_to_token[t.second].attr;
+            for (const auto & t : token_to_id) {
+                auto & attr = id_to_token[t.second].attr;
 
-            if (false
-                    || t.first == "<|eot_id|>"
-                    || t.first == "<|eot|>"    // muse-spark
-                    || t.first == "<|im_end|>"
-                    || t.first == "<|end|>"
-                    || t.first == "<|return|>" // o200k_harmony
-                    || t.first == "<|call|>"   // o200k_harmony
-                    || t.first == "<|flush|>"  // solar-open
-                    || t.first == "<|calls|>"  // solar-open
-                    || t.first == ""
-                    || t.first == "<|endoftext|>"
-                    || t.first == ""      // paddleocr
-                    || t.first == "<|eom_id|>"
-                    || t.first == ""
-                    || t.first == "_"
-                    || t.first == "[EOT]" // Kimi-K2
-                    || t.first == "[EOS]" // Kimi-K2
-                    || t.first == "<|end_of_text|>"
-                    || (pre_type == LLAMA_VOCAB_PRE_TYPE_FLINT && (t.first == "<|user|>" || t.first == "<|observation|>"))
-                    || t.first == "" // smoldocling
-                    || t.first == ""            // gemma4
-                    || t.first == ""          // gemma4
-                    || t.first == "<|tool_response>" // gemma4
-                    || t.first == "<|end▁of▁sentence|>" // deepseek-ocr
-                    || t.first == "[e~[" // minimax-m2/m3
-                    || t.first == "<|ifm|im_end|>" // k2-horizon
-               ) {
-                special_eog_ids.insert(t.second);
-                if ((attr & LLAMA_TOKEN_ATTR_CONTROL) == 0) {
-                    LLAMA_LOG_WARN("%s: control-looking token: %6d '%s' was not control-type; this is probably a bug in the model. its type will be overridden\n",
-                            __func__, t.second, t.first.c_str());
-                    attr = (llama_token_attr) (attr | LLAMA_TOKEN_ATTR_CONTROL);
-                }
-            } else {
-                if (attr & LLAMA_TOKEN_ATTR_CONTROL && !(attr & LLAMA_TOKEN_ATTR_UNUSED)) {
-                    // token is control, but not marked as EOG -> print a debug log
-                    if (special_eog_ids.count(t.second) == 0) {
-                        LLAMA_LOG_DEBUG("%s: control token: %6d '%s' is not marked as EOG\n",
+                if (false
+                        || t.first == "<|eot_id|>"
+                        || t.first == "<|eot|>"    // muse-spark
+                        || t.first == "<|im_end|>"
+                        || t.first == "<|end|>"
+                        || t.first == "<|return|>" // o200k_harmony
+                        || t.first == "<|call|>"   // o200k_harmony
+                        || t.first == "<|flush|>"  // solar-open
+                        || t.first == "<|calls|>"  // solar-open
+                        || t.first == ""
+                        || t.first == "<|endoftext|>"
+                        || t.first == ""      // paddleocr
+                        || t.first == "<|eom_id|>"
+                        || t.first == ""
+                        || t.first == "_"
+                        || t.first == "[EOT]" // Kimi-K2
+                        || t.first == "[EOS]" // Kimi-K2
+                        || t.first == "<|end_of_text|>"
+                        || (pre_type == LLAMA_VOCAB_PRE_TYPE_FLINT && (t.first == "<|user|>" || t.first == "<|observation|>"))
+                        || t.first == "" // smoldocling
+                        || t.first == ""            // gemma4
+                        || t.first == ""          // gemma4
+                        || t.first == "<|tool_response>" // gemma4
+                        || t.first == "<|end▁of▁sentence|>" // deepseek-ocr
+                        || t.first == "[e~[" // minimax-m2/m3
+                        || t.first == "<|ifm|im_end|>" // k2-horizon
+                   ) {
+                    special_eog_ids.insert(t.second);
+                    if ((attr & LLAMA_TOKEN_ATTR_CONTROL) == 0) {
+                        LLAMA_LOG_WARN("%s: control-looking token: %6d '%s' was not control-type; this is probably a bug in the model. its type will be overridden\n",
                                 __func__, t.second, t.first.c_str());
+                        attr = (llama_token_attr) (attr | LLAMA_TOKEN_ATTR_CONTROL);
+                    }
+                } else {
+                    if (attr & LLAMA_TOKEN_ATTR_CONTROL && !(attr & LLAMA_TOKEN_ATTR_UNUSED)) {
+                        // token is control, but not marked as EOG -> print a debug log
+                        if (special_eog_ids.count(t.second) == 0) {
+                            LLAMA_LOG_DEBUG("%s: control token: %6d '%s' is not marked as EOG\n",
+                                    __func__, t.second, t.first.c_str());
+                        }
                     }
                 }
             }
-        }
 
-        // @ngxson : quick hack for gpt-oss, always render these tokens
-        for (const auto & t : token_to_id) {
-            auto & attr = id_to_token[t.second].attr;
+            // @ngxson : quick hack for gpt-oss, always render these tokens
+            for (const auto & t : token_to_id) {
+                auto & attr = id_to_token[t.second].attr;
 
-            if (t.first == "<|channel|>" || t.first == "<|message|>" || t.first == "<|start|>" || t.first == "<|constrain|>") {
-                LLAMA_LOG_WARN("%s: setting token '%s' (%d) attribute to USER_DEFINED (%u), old attributes: %u\n",
-                        __func__, t.first.c_str(), t.second, LLAMA_TOKEN_ATTR_USER_DEFINED, attr);
+                if (t.first == "<|channel|>" || t.first == "<|message|>" || t.first == "<|start|>" || t.first == "<|constrain|>") {
+                    LLAMA_LOG_WARN("%s: setting token '%s' (%d) attribute to USER_DEFINED (%u), old attributes: %u\n",
+                            __func__, t.first.c_str(), t.second, LLAMA_TOKEN_ATTR_USER_DEFINED, attr);
 
-                attr = LLAMA_TOKEN_ATTR_USER_DEFINED;
+                    attr = LLAMA_TOKEN_ATTR_USER_DEFINED;
+                }
             }
-        }
 
-        if (pre_type == LLAMA_VOCAB_PRE_TYPE_SOPRANO) {
-            special_eog_ids.insert(special_eos_id);
-        }
+            if (pre_type == LLAMA_VOCAB_PRE_TYPE_SOPRANO) {
+                special_eog_ids.insert(special_eos_id);
+            }
 
-        // sanity checks
-        if (special_eos_id != LLAMA_TOKEN_NULL && special_eog_ids.count(special_eos_id) == 0) {
-            special_eog_ids.insert(special_eos_id);
-            LLAMA_LOG_WARN("%s: special_eos_id is not in special_eog_ids - the tokenizer config may be incorrect\n", __func__);
-        }
+            // sanity checks
+            if (special_eos_id != LLAMA_TOKEN_NULL && special_eog_ids.count(special_eos_id) == 0) {
+                special_eog_ids.insert(special_eos_id);
+                LLAMA_LOG_WARN("%s: special_eos_id is not in special_eog_ids - the tokenizer config may be incorrect\n", __func__);
+            }
 
-        if (special_eot_id != LLAMA_TOKEN_NULL && special_eog_ids.count(special_eot_id) == 0) {
-            special_eog_ids.insert(special_eot_id);
-            LLAMA_LOG_WARN("%s: special_eot_id is not in special_eog_ids - the tokenizer config may be incorrect\n", __func__);
-        }
+            if (special_eot_id != LLAMA_TOKEN_NULL && special_eog_ids.count(special_eot_id) == 0) {
+                special_eog_ids.insert(special_eot_id);
+                LLAMA_LOG_WARN("%s: special_eot_id is not in special_eog_ids - the tokenizer config may be incorrect\n", __func__);
+            }
 
-        if (special_eom_id != LLAMA_TOKEN_NULL && special_eog_ids.count(special_eom_id) == 0) {
-            special_eog_ids.insert(special_eom_id);
-            LLAMA_LOG_WARN("%s: special_eom_id is not in special_eog_ids - the tokenizer config may be incorrect\n", __func__);
-        }
+            if (special_eom_id != LLAMA_TOKEN_NULL && special_eog_ids.count(special_eom_id) == 0) {
+                special_eog_ids.insert(special_eom_id);
+                LLAMA_LOG_WARN("%s: special_eom_id is not in special_eog_ids - the tokenizer config may be incorrect\n", __func__);
+            }
 
-        // TODO: workaround for o200k_harmony and solar-open tokenizer: the "<|end|>" token should not be EOG
-        //       we don't have a good way to detect this, so for now, if we have "<|return|>" and "<|call|>" tokens ("<|calls|>" and "<|flush|>" for solar-open),
-        //       we remove the "<|end|>" token from the EOG list
-        {
-            bool has_return = false;
-            bool has_call   = false;
-            bool has_end    = false;
-            bool has_flush  = false;
-
-            llama_token end_id = LLAMA_TOKEN_NULL;
-
-            LLAMA_LOG_INFO("%s: printing all EOG tokens:\n", __func__);
-            for (auto tid : special_eog_ids) {
-                if (tid < 0 || tid >= (llama_token) id_to_token.size()) {
-                    LLAMA_LOG_WARN("%s: EOG token id %d is out of range (vocab size %zu), skipping\n",
-                            __func__, tid, id_to_token.size());
-                    continue;
-                }
-                auto & text = id_to_token[tid].text;
-
-                LLAMA_LOG_INFO("%s:   - %d ('%s')\n", __func__, tid, text.c_str());
-
-                if (text == "<|return|>") {
-                    has_return = true;
-                } else if (text == "<|call|>" || text == "<|calls|>") {
-                    has_call = true;
-                } else if (text == "<|flush|>") {
-                    has_flush = true;
-                } else if (text == "<|end|>") {
-                    has_end = true;
-                    end_id = tid;
+            // TODO: workaround for o200k_harmony and solar-open tokenizer: the "<|end|>" token should not be EOG
+            //       we don't have a good way to detect this, so for now, if we have "<|return|>" and "<|call|>" tokens ("<|calls|>" and "<|flush|>" for solar-open),
+            //       we remove the "<|end|>" token from the EOG list
+            {
+                bool has_return = false;
+                bool has_call   = false;
+                bool has_end    = false;
+                bool has_flush  = false;
+
+                llama_token end_id = LLAMA_TOKEN_NULL;
+
+                LLAMA_LOG_INFO("%s: printing all EOG tokens:\n", __func__);
+                for (auto tid : special_eog_ids) {
+                    if (tid < 0 || tid >= (llama_token) id_to_token.size()) {
+                        LLAMA_LOG_WARN("%s: EOG token id %d is out of range (vocab size %zu), skipping\n",
+                                __func__, tid, id_to_token.size());
+                        continue;
+                    }
+                    auto & text = id_to_token[tid].text;
+
+                    LLAMA_LOG_INFO("%s:   - %d ('%s')\n", __func__, tid, text.c_str());
+
+                    if (text == "<|return|>") {
+                        has_return = true;
+                    } else if (text == "<|call|>" || text == "<|calls|>") {
+                        has_call = true;
+                    } else if (text == "<|flush|>") {
+                        has_flush = true;
+                    } else if (text == "<|end|>") {
+                        has_end = true;
+                        end_id = tid;
+                    }
                 }
-            }
 
-            if ((has_return && has_call && has_end) || (has_call && has_flush && has_end)) {
-                special_eog_ids.erase(end_id);
+                if ((has_return && has_call && has_end) || (has_call && has_flush && has_end)) {
+                    special_eog_ids.erase(end_id);
 
-                auto & attr = id_to_token[end_id].attr;
-                attr = LLAMA_TOKEN_ATTR_USER_DEFINED;
+                    auto & attr = id_to_token[end_id].attr;
+                    attr = LLAMA_TOKEN_ATTR_USER_DEFINED;
 
-                LLAMA_LOG_WARN("%s: special_eog_ids contains both '<|return|>' and '<|call|>', or '<|calls|>' and '<|flush|>' tokens, removing '<|end|>' token from EOG list\n", __func__);
+                    LLAMA_LOG_WARN("%s: special_eog_ids contains both '<|return|>' and '<|call|>', or '<|calls|>' and '<|flush|>' tokens, removing '<|end|>' token from EOG list\n", __func__);
+                }
             }
-        }
 
-        // gemma4, plamo and beam have a normal  token, unlike paddleocr
-        {
-            // beam's tool call marker is not an EOG token, so it is looked up in the whole vocab
-            bool has_normal_s_marker = token_to_id.count(" 0;
-            bool has_s = false;
+            // gemma4, plamo and beam have a normal  token, unlike paddleocr
+            {
+                // beam's tool call marker is not an EOG token, so it is looked up in the whole vocab
+                bool has_normal_s_marker = token_to_id.count(" 0;
+                bool has_s = false;
 
-            llama_token s_id = LLAMA_TOKEN_NULL;
+                llama_token s_id = LLAMA_TOKEN_NULL;
 
-            for (auto tid : special_eog_ids) {
-                if (tid < 0 || tid >= (llama_token) id_to_token.size()) {
-                    continue;
-                }
-                const auto & text = id_to_token[tid].text;
-                if (text == "<|tool_response>" || text == "<|plamo:eos|>") {
-                    has_normal_s_marker = true;
-                } else if (text == "") {
-                    has_s = true;
-                    s_id = tid;
+                for (auto tid : special_eog_ids) {
+                    if (tid < 0 || tid >= (llama_token) id_to_token.size()) {
+                        continue;
+                    }
+                    const auto & text = id_to_token[tid].text;
+                    if (text == "<|tool_response>" || text == "<|plamo:eos|>") {
+                        has_normal_s_marker = true;
+                    } else if (text == "") {
+                        has_s = true;
+                        s_id = tid;
+                    }
                 }
-            }
 
-            if (has_normal_s_marker && has_s) {
-                special_eog_ids.erase(s_id);
+                if (has_normal_s_marker && has_s) {
+                    special_eog_ids.erase(s_id);
 
-                auto & attr = id_to_token[s_id].attr;
-                attr = LLAMA_TOKEN_ATTR_NORMAL;
+                    auto & attr = id_to_token[s_id].attr;
+                    attr = LLAMA_TOKEN_ATTR_NORMAL;
 
-                LLAMA_LOG_WARN("%s: '' is a normal token here, removing it from EOG list\n", __func__);
+                    LLAMA_LOG_WARN("%s: '' is a normal token here, removing it from EOG list\n", __func__);
+                }
             }
         }
     }
@@ -4591,6 +4627,10 @@ const llama_vocab::normalizer_options & llama_vocab::get_normalizer_opts() const
     return pimpl->normalizer_opts;
 }
 
+const std::set & llama_vocab::get_eog_tokens() const {
+    return pimpl->special_eog_ids;
+}
+
 const std::vector & llama_vocab::get_suppress_tokens() const {
     return pimpl->suppress_tokens;
 }
diff --git a/src/llama-vocab.h b/src/llama-vocab.h
index 14a90fbdb69f..15dcabaf0bb2 100644
--- a/src/llama-vocab.h
+++ b/src/llama-vocab.h
@@ -5,6 +5,7 @@
 #include 
 #include 
 #include 
+#include 
 
 // pre-tokenization types
 enum llama_vocab_pre_type {
@@ -167,6 +168,7 @@ struct llama_vocab {
     bool get_escape_whitespaces        () const;
     bool get_treat_whitespace_as_suffix() const;
     const normalizer_options & get_normalizer_opts() const;
+    const std::set & get_eog_tokens() const;
 
     const std::vector & get_suppress_tokens() const;
 
diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt
index 2b68840dace5..1663990ec3ef 100644
--- a/tests/CMakeLists.txt
+++ b/tests/CMakeLists.txt
@@ -153,6 +153,12 @@ llama_test(test-tokenizer-0 NAME test-tokenizer-0-qwen35          LABEL behavior
 llama_test(test-tokenizer-0 NAME test-tokenizer-0-refact          LABEL behavior  ARGS ${PROJECT_SOURCE_DIR}/models/ggml-vocab-refact.gguf)
 llama_test(test-tokenizer-0 NAME test-tokenizer-0-starcoder       LABEL behavior  ARGS ${PROJECT_SOURCE_DIR}/models/ggml-vocab-starcoder.gguf)
 
+llama_build(test-tokenizer-normalization.cpp LINK llama ggml)
+llama_test(test-tokenizer-normalization NAME test-tokenizer-normalization-gpt2 LABEL behavior ARGS ${PROJECT_SOURCE_DIR}/models/ggml-vocab-gpt-2.gguf)
+llama_test(test-tokenizer-normalization NAME test-tokenizer-normalization-qwen35 LABEL behavior ARGS ${PROJECT_SOURCE_DIR}/models/ggml-vocab-qwen35.gguf)
+llama_test(test-tokenizer-normalization NAME test-tokenizer-eog-policy LABEL behavior ARGS --eog-policy)
+llama_test(test-tokenizer-normalization NAME test-tokenizer-eog-roundtrip LABEL behavior ARGS --eog-roundtrip)
+
 if (NOT WIN32)
     llama_test_cmd(
         ${CMAKE_CURRENT_SOURCE_DIR}/test-tokenizers-repo.sh
diff --git a/tests/test-tokenizer-normalization.cpp b/tests/test-tokenizer-normalization.cpp
new file mode 100644
index 000000000000..c2e1f794e1a6
--- /dev/null
+++ b/tests/test-tokenizer-normalization.cpp
@@ -0,0 +1,410 @@
+#include "llama.h"
+#include "gguf.h"
+#include "ggml-backend.h"
+
+#include 
+#include 
+
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+
+using model_ptr = std::unique_ptr;
+
+static void require(bool condition, const std::string & message) {
+    if (!condition) {
+        throw std::runtime_error(message);
+    }
+}
+
+static std::vector tokenize(const llama_vocab * vocab, const std::string & text, bool special) {
+    std::vector result(text.size() + 8);
+    const int count = llama_tokenize(vocab, text.data(), text.size(), result.data(), result.size(), false, special);
+    if (count < 0) {
+        throw std::runtime_error("unexpected token buffer exhaustion");
+    }
+    result.resize(count);
+    return result;
+}
+
+static std::string detokenize(const llama_vocab * vocab, const std::vector & tokens, bool special) {
+    std::vector result(1024);
+    const int count = llama_detokenize(vocab, tokens.data(), tokens.size(),
+                                      result.data(), result.size(), false, special);
+    require(count >= 0, "unexpected detokenization buffer exhaustion");
+    return std::string(result.data(), count);
+}
+
+// Metadata-only BPE fixtures exercise the public ABI, including against old DLLs.
+// No model weights, private vocabulary entry points, or new exported functions.
+struct eog_fixture {
+    enum token_id {
+        eos, pad, literal_s, fim_pre, fim_mid, fim_suf, fim_pad, fim_rep, fim_sep,
+        tool_call, tool_call_end, tool_response, tool_response_end, think, think_end,
+        eom, eot, return_token, call_token, end_token, channel,
+    };
+    enum class policy_type { absent, ids, scalar, signed_ids, float_ids, string_ids };
+
+    std::vector tokens = {
+        "<|im_end|>", "<|endoftext|>", "",
+        "<|fim_prefix|>", "<|fim_middle|>", "<|fim_suffix|>",
+        "<|fim_pad|>", "<|repo_name|>", "<|file_sep|>",
+        "", "", "", "",
+        "", "", "<|eom_id|>", "<|eot_id|>",
+        "<|return|>", "<|call|>", "<|end|>", "<|channel|>",
+    };
+    std::vector types = {
+        LLAMA_TOKEN_TYPE_CONTROL, LLAMA_TOKEN_TYPE_CONTROL, LLAMA_TOKEN_TYPE_NORMAL,
+        LLAMA_TOKEN_TYPE_USER_DEFINED, LLAMA_TOKEN_TYPE_USER_DEFINED, LLAMA_TOKEN_TYPE_USER_DEFINED,
+        LLAMA_TOKEN_TYPE_USER_DEFINED, LLAMA_TOKEN_TYPE_USER_DEFINED, LLAMA_TOKEN_TYPE_USER_DEFINED,
+        LLAMA_TOKEN_TYPE_USER_DEFINED, LLAMA_TOKEN_TYPE_USER_DEFINED,
+        LLAMA_TOKEN_TYPE_USER_DEFINED, LLAMA_TOKEN_TYPE_USER_DEFINED,
+        LLAMA_TOKEN_TYPE_USER_DEFINED, LLAMA_TOKEN_TYPE_USER_DEFINED,
+        LLAMA_TOKEN_TYPE_USER_DEFINED, LLAMA_TOKEN_TYPE_NORMAL,
+        LLAMA_TOKEN_TYPE_CONTROL, LLAMA_TOKEN_TYPE_CONTROL, LLAMA_TOKEN_TYPE_CONTROL, LLAMA_TOKEN_TYPE_CONTROL,
+    };
+    bool named_turn_ids = true;
+    bool normalizer_nfc = false;
+
+    eog_fixture() {
+        for (int c = '!'; c <= '~'; ++c) {
+            tokens.emplace_back(1, char(c));
+            types.push_back(LLAMA_TOKEN_TYPE_NORMAL);
+        }
+        tokens.push_back("\xc4\xa0"); // byte-level space
+        tokens.push_back("\xc4\x8a"); // byte-level newline
+        types.insert(types.end(), 2, LLAMA_TOKEN_TYPE_NORMAL);
+    }
+
+    llama_token ascii(char c) const {
+        const auto it = std::find(tokens.begin(), tokens.end(), std::string(1, c));
+        require(it != tokens.end(), "missing fixture character");
+        return llama_token(it - tokens.begin());
+    }
+
+    void write(const std::filesystem::path & path, policy_type policy, const std::vector & ids) const {
+        std::unique_ptr ctx(gguf_init_empty(), gguf_free);
+        gguf_set_val_str(ctx.get(), "general.architecture", "llama");
+        gguf_set_val_str(ctx.get(), "tokenizer.ggml.model", "gpt2");
+        gguf_set_val_str(ctx.get(), "tokenizer.ggml.pre", "qwen2");
+        std::vector text;
+        for (const auto & token : tokens) {
+            text.push_back(token.c_str());
+        }
+        gguf_set_arr_str(ctx.get(), "tokenizer.ggml.tokens", text.data(), text.size());
+        gguf_set_arr_data(ctx.get(), "tokenizer.ggml.token_type", GGUF_TYPE_INT32, types.data(), types.size());
+        const char * empty = "";
+        gguf_set_arr_str(ctx.get(), "tokenizer.ggml.merges", &empty, 0);
+        gguf_set_val_u32(ctx.get(), "tokenizer.ggml.eos_token_id", eos);
+        gguf_set_val_u32(ctx.get(), "tokenizer.ggml.bos_token_id", pad);
+        gguf_set_val_u32(ctx.get(), "tokenizer.ggml.padding_token_id", pad);
+        gguf_set_val_bool(ctx.get(), "tokenizer.ggml.add_bos_token", false);
+        gguf_set_val_bool(ctx.get(), "tokenizer.ggml.add_eos_token", false);
+        gguf_set_val_bool(ctx.get(), "tokenizer.ggml.normalizer.nfc", normalizer_nfc);
+        if (named_turn_ids) {
+            gguf_set_val_u32(ctx.get(), "tokenizer.ggml.eot_token_id", eot);
+            gguf_set_val_u32(ctx.get(), "tokenizer.ggml.eom_token_id", eom);
+        }
+        const char * key = "tokenizer.ggml.eog_token_ids";
+        const uint32_t dummy = 0;
+        switch (policy) {
+            case policy_type::absent:
+                break;
+            case policy_type::ids:
+                gguf_set_arr_data(ctx.get(), key, GGUF_TYPE_UINT32, ids.empty() ? &dummy : ids.data(), ids.size());
+                break;
+            case policy_type::scalar:
+                gguf_set_val_u32(ctx.get(), key, eos);
+                break;
+            case policy_type::signed_ids:
+                gguf_set_arr_data(ctx.get(), key, GGUF_TYPE_INT32, ids.empty() ? &dummy : ids.data(), ids.size());
+                break;
+            case policy_type::float_ids: {
+                const float id = eos;
+                gguf_set_arr_data(ctx.get(), key, GGUF_TYPE_FLOAT32, &id, 1);
+                break;
+            }
+            case policy_type::string_ids: {
+                const char * id = "0";
+                gguf_set_arr_str(ctx.get(), key, &id, 1);
+                break;
+            }
+        }
+        require(gguf_write_to_file(ctx.get(), path.string().c_str(), true), "cannot write tiny GGUF fixture");
+    }
+
+    model_ptr load(const std::filesystem::path & path, policy_type policy, const std::vector & ids) const {
+        write(path, policy, ids);
+        auto params = llama_model_default_params();
+        params.vocab_only = true;
+        return model_ptr(llama_model_load_from_file(path.string().c_str(), params), llama_model_free);
+    }
+};
+
+struct fixture_directory {
+    std::filesystem::path path;
+
+    fixture_directory() {
+        const auto stamp = std::chrono::steady_clock::now().time_since_epoch().count();
+        const auto base = std::filesystem::temp_directory_path();
+        for (unsigned i = 0; ; ++i) {
+            path = base / ("llama-eog-" + std::to_string(stamp) + "-" + std::to_string(i));
+            if (std::filesystem::create_directory(path)) {
+                break;
+            }
+        }
+    }
+
+    ~fixture_directory() {
+        std::error_code error;
+        std::filesystem::remove_all(path, error);
+    }
+};
+
+static void check_eog_set(const llama_vocab * vocab, const std::vector & expected) {
+    for (llama_token id = 0; id < llama_vocab_n_tokens(vocab); ++id) {
+        const bool is_eog = std::find(expected.begin(), expected.end(), uint32_t(id)) != expected.end();
+        require(llama_vocab_is_eog(vocab, id) == is_eog, "EOG membership mismatch at token " + std::to_string(id));
+    }
+    require(!llama_vocab_is_eog(vocab, LLAMA_TOKEN_NULL), "null token must not be EOG");
+}
+
+static void check_lexical_policy(const llama_vocab * vocab, const eog_fixture & fixture) {
+    using F = eog_fixture;
+    for (llama_token id = 0; id < llama_vocab_n_tokens(vocab); ++id) {
+        const auto expected = fixture.types[id] == LLAMA_TOKEN_TYPE_CONTROL ? LLAMA_TOKEN_ATTR_CONTROL :
+                              fixture.types[id] == LLAMA_TOKEN_TYPE_USER_DEFINED ? LLAMA_TOKEN_ATTR_USER_DEFINED :
+                              LLAMA_TOKEN_ATTR_NORMAL;
+        require(llama_vocab_get_attr(vocab, id) == expected, "lexical attribute mismatch at token " + std::to_string(id));
+    }
+    require(llama_vocab_eos(vocab) == F::eos, "named EOS changed");
+    require(llama_vocab_fim_pre(vocab) == F::fim_pre && llama_vocab_fim_mid(vocab) == F::fim_mid &&
+            llama_vocab_fim_suf(vocab) == F::fim_suf && llama_vocab_fim_pad(vocab) == F::fim_pad &&
+            llama_vocab_fim_rep(vocab) == F::fim_rep && llama_vocab_fim_sep(vocab) == F::fim_sep,
+            "FIM IDs were not resolved");
+    for (llama_token id = F::literal_s; id <= F::eot; ++id) {
+        const auto & text = fixture.tokens[id];
+        const bool control = fixture.types[id] == LLAMA_TOKEN_TYPE_CONTROL;
+        const bool user_defined = fixture.types[id] == LLAMA_TOKEN_TYPE_USER_DEFINED;
+        for (bool special : {false, true}) {
+            std::vector expected;
+            if (user_defined || (control && special)) {
+                expected.push_back(id);
+            } else {
+                for (char c : text) {
+                    expected.push_back(fixture.ascii(c));
+                }
+            }
+            require(tokenize(vocab, text, special) == expected, "marker tokenization mismatch: " + text);
+            expected.insert(expected.begin(), fixture.ascii('x'));
+            expected.push_back(fixture.ascii('y'));
+            require(tokenize(vocab, "x" + text + "y", special) == expected, "embedded marker mismatch: " + text);
+        }
+        require(detokenize(vocab, {id}, false) == (control ? "" : text), "suppressed detokenization mismatch: " + text);
+        require(detokenize(vocab, {id}, true) == text, "special detokenization mismatch: " + text);
+    }
+    require(tokenize(vocab, fixture.tokens[F::eos], true) == std::vector{F::eos},
+            "canonical EOS special parsing changed");
+    require(detokenize(vocab, {F::eos}, false).empty(), "canonical EOS must remain CONTROL");
+}
+
+static void test_eog_policy() {
+    using F = eog_fixture;
+    using P = F::policy_type;
+    fixture_directory directory;
+    const auto path = directory.path / "vocab.gguf";
+    F fixture;
+    const auto check = [&] (const char * name, const F & f, const std::vector & ids) {
+        try {
+            auto model = f.load(path, P::ids, ids);
+            require(bool(model), "valid EOG fixture failed to load");
+            const auto * vocab = llama_model_get_vocab(model.get());
+            check_eog_set(vocab, ids);
+            check_lexical_policy(vocab, f);
+        } catch (const std::exception & error) {
+            throw std::runtime_error(std::string(name) + ": " + error.what());
+        }
+    };
+    check("canonical single stop", fixture, {F::eos});
+    check("multiple stops with ordinary and nonspecial tokens", fixture, {F::eos, F::literal_s, F::tool_call});
+    check("explicit empty set overrides EOS/EOT/EOM", fixture, {});
+    check("duplicate stop IDs", fixture, {F::eos, F::eos});
+
+    F normal_fim = fixture;
+    for (int id = F::fim_pre; id <= F::fim_sep; ++id) {
+        normal_fim.types[id] = LLAMA_TOKEN_TYPE_NORMAL;
+    }
+    check("NORMAL FIM attributes", normal_fim, {F::eos});
+    F control_fim = fixture;
+    for (int id = F::fim_pre; id <= F::fim_sep; ++id) {
+        control_fim.types[id] = LLAMA_TOKEN_TYPE_CONTROL;
+    }
+    check("CONTROL FIM is not implicitly EOG", control_fim, {F::eos});
+
+    F inferred_turns = fixture;
+    inferred_turns.named_turn_ids = false;
+    inferred_turns.tokens[F::eos] = "";
+    inferred_turns.tokens[F::pad] = "";
+    inferred_turns.tokens[F::end_token] = "";
+    check("inferred EOT/EOM preserve lexical attributes", inferred_turns, {});
+
+    F removal = fixture;
+    removal.tokens.push_back("<|tool_response>");
+    removal.types.push_back(LLAMA_TOKEN_TYPE_CONTROL);
+    check("legacy exceptions cannot remove explicit stops", removal,
+          {F::eos, F::literal_s, F::return_token, F::call_token, F::end_token, uint32_t(removal.tokens.size() - 1)});
+
+    // Absent-key behavior deliberately retains the existing text heuristics,
+    // including NORMAL  promotion and the earlier USER_DEFINED FIM fix.
+    auto legacy = fixture.load(path, P::absent, {});
+    require(bool(legacy), "absent-key fixture failed to load");
+    const auto * vocab = llama_model_get_vocab(legacy.get());
+    check_eog_set(vocab, {F::eos, F::pad, F::literal_s, F::eom, F::eot, F::return_token, F::call_token});
+    require((llama_vocab_get_attr(vocab, F::literal_s) & LLAMA_TOKEN_ATTR_CONTROL) != 0,
+            "absent-key  heuristic changed");
+    require(tokenize(vocab, "", true) == std::vector{F::literal_s},
+            "absent-key special parsing changed");
+    require(detokenize(vocab, {F::literal_s}, false).empty(), "absent-key special suppression changed");
+    for (llama_token id = F::fim_pre; id <= F::think_end; ++id) {
+        require(llama_vocab_get_attr(vocab, id) == LLAMA_TOKEN_ATTR_USER_DEFINED, "legacy nonspecial attributes changed");
+        require(tokenize(vocab, fixture.tokens[id], false) == std::vector{id}, "legacy nonspecial parsing changed");
+        require(detokenize(vocab, {id}, false) == fixture.tokens[id], "legacy nonspecial detokenization changed");
+    }
+    auto legacy_control = control_fim.load(path, P::absent, {});
+    require(bool(legacy_control), "absent-key CONTROL fixture failed to load");
+    check_eog_set(llama_model_get_vocab(legacy_control.get()),
+                  {F::eos, F::pad, F::literal_s, F::eom, F::eot, F::return_token, F::call_token,
+                   F::fim_pad, F::fim_rep, F::fim_sep});
+
+    for (auto malformed : {P::scalar, P::signed_ids, P::float_ids, P::string_ids}) {
+        require(!fixture.load(path, malformed, {F::eos}), "malformed EOG type was accepted");
+    }
+    require(!fixture.load(path, P::signed_ids, {}), "empty EOG array with wrong element type was accepted");
+    require(!fixture.load(path, P::ids, {uint32_t(fixture.tokens.size())}), "out-of-range EOG ID was accepted");
+    require(!fixture.load(path, P::ids, {std::numeric_limits::max()}), "overflowing EOG ID was accepted");
+}
+
+static void test_policy_roundtrip() {
+    using F = eog_fixture;
+    fixture_directory directory;
+    F fixture;
+    fixture.normalizer_nfc = true;
+    const llama_token first = llama_token(fixture.tokens.size());
+    // Byte-level BPE spellings for UTF-8 C3 A9 (é) and CC 81 (combining acute).
+    fixture.tokens.insert(fixture.tokens.end(), {"\xc3\x83", "\xc2\xa9", "\xc3\x8c", "\xc4\xa3"});
+    fixture.types.insert(fixture.types.end(), 4, LLAMA_TOKEN_TYPE_NORMAL);
+    const std::vector composed = {first, first + 1};
+    for (const auto & ids : std::vector>{{}, {F::eos}, {F::eos, F::tool_call}}) {
+        const auto source = directory.path / "original.gguf";
+        fixture.write(source, F::policy_type::ids, ids);
+        std::unique_ptr metadata(
+            gguf_init_from_file(source.string().c_str(), {true, nullptr}), gguf_free);
+        require(bool(metadata), "cannot read roundtrip fixture metadata");
+        gguf_set_val_u32(metadata.get(), "llama.context_length", 128);
+        gguf_set_val_u32(metadata.get(), "llama.embedding_length", 128);
+        gguf_set_val_u32(metadata.get(), "llama.block_count", 1);
+        gguf_set_val_u32(metadata.get(), "llama.feed_forward_length", 256);
+        gguf_set_val_u32(metadata.get(), "llama.attention.head_count", 2);
+        gguf_set_val_u32(metadata.get(), "llama.attention.head_count_kv", 2);
+        gguf_set_val_f32(metadata.get(), "llama.attention.layer_norm_rms_epsilon", 1e-5f);
+        gguf_set_val_u32(metadata.get(), "llama.rope.dimension_count", 64);
+        auto source_params = llama_model_default_params();
+        static ggml_backend_dev_t cpu_only[] = {nullptr};
+        source_params.devices = cpu_only;
+        source_params.n_gpu_layers = 0;
+        const auto fill = [](ggml_tensor * tensor, void *) {
+            require(tensor->type == GGML_TYPE_F32, "unexpected synthetic model type");
+            std::vector values(ggml_nelements(tensor),
+                std::strstr(tensor->name, "norm") ? 1.0f : 0.01f);
+            ggml_backend_tensor_set(tensor, values.data(), 0, ggml_nbytes(tensor));
+        };
+        model_ptr original(llama_model_init_from_user(metadata.get(), fill, nullptr, source_params), llama_model_free);
+        require(bool(original), "roundtrip source failed to load");
+        const auto * original_vocab = llama_model_get_vocab(original.get());
+        require(tokenize(original_vocab, "e\xcc\x81", false) == composed, "NFC fixture is invalid");
+        const auto saved = directory.path / "saved.gguf";
+        llama_model_save_to_file(original.get(), saved.string().c_str());
+        auto params = llama_model_default_params();
+        params.vocab_only = true;
+        model_ptr restored(llama_model_load_from_file(saved.string().c_str(), params), llama_model_free);
+        require(bool(restored), "saved vocabulary failed to load");
+        const auto * restored_vocab = llama_model_get_vocab(restored.get());
+        check_eog_set(restored_vocab, ids);
+        check_lexical_policy(restored_vocab, fixture);
+        require(tokenize(restored_vocab, "e\xcc\x81", false) == composed, "saving lost NFC normalization");
+        require(tokenize(restored_vocab, "\xc3\xa9", false) == composed, "saving changed composed tokenization");
+    }
+}
+
+int main(int argc, char ** argv) {
+    if (argc == 2 && std::strcmp(argv[1], "--eog-roundtrip") == 0) {
+        try {
+            test_policy_roundtrip();
+            return 0;
+        } catch (const std::exception & error) {
+            fprintf(stderr, "Tokenizer policy roundtrip: %s\n", error.what());
+            return 1;
+        }
+    }
+    if (argc == 1 || (argc == 2 && std::strcmp(argv[1], "--eog-policy") == 0)) {
+        try {
+            test_eog_policy();
+            return 0;
+        } catch (const std::exception & error) {
+            fprintf(stderr, "EOG policy regression: %s\n", error.what());
+            return 1;
+        }
+    }
+    if (argc != 2) {
+        fprintf(stderr, "usage: %s [--eog-policy | --eog-roundtrip | ]\n", argv[0]);
+        return 1;
+    }
+    model_ptr models[] = {{nullptr, llama_model_free}, {nullptr, llama_model_free}};
+    for (int enabled = 0; enabled < 2; ++enabled) {
+        llama_model_kv_override overrides[2] = {};
+        overrides[0].tag = LLAMA_KV_OVERRIDE_TYPE_BOOL;
+        std::strcpy(overrides[0].key, "tokenizer.ggml.normalizer.nfc");
+        overrides[0].val_bool = enabled;
+        auto params = llama_model_default_params();
+        params.vocab_only = true;
+        params.kv_overrides = overrides;
+        models[enabled].reset(llama_model_load_from_file(argv[1], params));
+        if (!models[enabled]) {
+            fprintf(stderr, "failed to load vocabulary\n");
+            return 1;
+        }
+    }
+    const auto * plain = llama_model_get_vocab(models[0].get());
+    const auto * nfc = llama_model_get_vocab(models[1].get());
+    if (llama_vocab_type(plain) != LLAMA_VOCAB_TYPE_BPE) {
+        return 1;
+    }
+    const std::vector> cases = {
+        {"Cafe\xcc\x81", "Caf\xc3\xa9"},
+        {"A\xcc\x8a", "\xc3\x85"},
+        {"\xe2\x84\xab", "\xc3\x85"},
+        {"a\xcc\x95\xcc\x80", "\xc3\xa0\xcc\x95"},
+        {"\xe1\x84\x80\xe1\x85\xa1\xe1\x86\xa8", "\xea\xb0\x81"},
+        {"Cafe\xcc\x81<|endoftext|>A\xcc\x8a", "Caf\xc3\xa9<|endoftext|>\xc3\x85"},
+        {"ASCII\n123 's", "ASCII\n123 's"},
+        {"\xef\xac\x81\xef\xbc\xa1", "\xef\xac\x81\xef\xbc\xa1"},
+    };
+    for (bool special : {false, true}) {
+        for (const auto & test : cases) {
+            const auto expected = tokenize(plain, test.second, special);
+            if (tokenize(nfc, test.first, special) != expected ||
+                tokenize(nfc, test.second, special) != expected ||
+                (test.first != test.second && tokenize(plain, test.first, special) == expected)) {
+                fprintf(stderr, "BPE NFC or disabled-normalization mismatch (special=%d)\n", special);
+                return 1;
+            }
+        }
+    }
+    return 0;
+}