Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
55 changes: 54 additions & 1 deletion conversion/maple.py
Original file line number Diff line number Diff line change
@@ -1,5 +1,7 @@
from __future__ import annotations

import json

from typing import Iterable, TYPE_CHECKING, cast

import torch
Expand All @@ -15,6 +17,52 @@
class MapleModel(TextModel):
model_arch = gguf.MODEL_ARCH.MAPLE

def get_vocab_base(self) -> tuple[list[str], list[int], str]:
tokens, toktypes, tokpre = super().get_vocab_base()
with (self.dir_model / "tokenizer.json").open(encoding="utf-8") as stream:
tokenizer = json.load(stream)
with (self.dir_model / "tokenizer_config.json").open(encoding="utf-8") as stream:
settings = json.load(stream)
if tokenizer.get("normalizer") != {"type": "NFC"}:
raise ValueError("Maple requires the canonical exact NFC normalizer")
if tokenizer.get("model", {}).get("type") != "BPE" or tokpre != "qwen2":
raise ValueError("Unsupported Maple tokenizer family")
# Keep the existing BPE vocabulary/padding conventions, but do not infer
# specialness from spelling: Maple's FIM/tool/think tokens are lexical.
for text, token_id in tokenizer["model"]["vocab"].items():
if type(token_id) is not int or not 0 <= token_id < len(tokens) or tokens[token_id] != text:
raise ValueError(f"Maple canonical vocabulary mismatch at {token_id}")
toktypes[token_id] = gguf.TokenType.NORMAL
for added in tokenizer["added_tokens"]:
token_id = added["id"]
if type(token_id) is not int or not 0 <= token_id < len(tokens) or tokens[token_id] != added["content"]:
raise ValueError(f"Maple canonical added-token mismatch at {token_id}")
if type(added.get("special")) is not bool:
raise ValueError(f"Missing Maple added-token special flag at {token_id}")
other = settings.get("added_tokens_decoder", {}).get(str(token_id))
flags = ("content", "special", "lstrip", "rstrip", "single_word", "normalized")
if other is None or any(added.get(key) != other.get(key) for key in flags):
raise ValueError(f"Maple added-token configurations disagree at {token_id}")
if any(added.get(key) is not False for key in ("lstrip", "rstrip", "single_word", "normalized")):
raise ValueError("Unsupported Maple added-token boundary transformation")
toktypes[token_id] = gguf.TokenType.CONTROL if added["special"] else gguf.TokenType.USER_DEFINED
return tokens, toktypes, tokpre

def set_vocab(self):
self._set_vocab_gpt2()
# GenerationConfig overrides model EOS, including an explicit null/empty
# policy. The runtime must not append stop-looking lexical tokens.
generation_path = self.dir_model / "generation_config.json"
generation = {}
if generation_path.is_file():
with generation_path.open(encoding="utf-8") as stream:
generation = json.load(stream)
eos = generation.get("eos_token_id", self.hparams.get("eos_token_id"))
ids = [] if eos is None else eos if isinstance(eos, list) else [eos]
if any(type(token_id) is not int or not 0 <= token_id < self.hparams["vocab_size"] for token_id in ids):
raise ValueError("Invalid Maple generation EOS IDs")
self.gguf_writer.add_eog_token_ids(list(dict.fromkeys(ids)))

def set_gguf_parameters(self):
super().set_gguf_parameters()
hparams = self.hparams
Expand All @@ -29,7 +77,12 @@ def set_gguf_parameters(self):

self.gguf_writer.add_vocab_size(hparams["vocab_size"])
self.gguf_writer.add_rope_dimension_count(int(head_dim * partial_rotary_factor))
self.gguf_writer.add_sliding_window(hparams["sliding_window"])
# FlashAttention's left span is inclusive: 512 past keys + current key.
# GGUF STANDARD stores total causal keys; normalize only in the producer.
left_span = hparams["sliding_window"]
if type(left_span) is not int or not 0 < left_span < 2**32 - 1:
raise ValueError("Invalid Maple sliding-window left span")
self.gguf_writer.add_sliding_window(left_span + 1)
self.gguf_writer.add_sliding_window_pattern([layer_type == "sliding_attention" for layer_type in hparams["layer_types"]])
self.gguf_writer.add_expert_feed_forward_length(hparams["moe_intermediate_size"])
# the reference clamps the MoE SwiGLU gate/up at 7.0 (modeling_maple.py)
Expand Down
19 changes: 19 additions & 0 deletions gguf-py/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,25 @@ pip install gguf[gui]

[gguf/scripts/gguf_editor_gui.py](https://github.com/ggml-org/llama.cpp/blob/master/gguf-py/gguf/scripts/gguf_editor_gui.py) — Allows for viewing, editing, adding, or removing metadata values within a GGUF file as well as viewing its tensors with a Qt interface.

## Explicit end-of-generation token policy

`GGUFWriter.add_eog_token_ids(ids)` writes the optional
`tokenizer.ggml.eog_token_ids` extension as an array of `UINT32` token IDs.
In the corresponding llama.cpp runtime, presence makes this the complete
end-of-generation set: stop-looking spellings do not add more IDs or change
the vocabulary's declared token types. An explicitly empty array means no
token-based generation stops; omitting the key retains legacy inference.
IDs must be within the model vocabulary. This policy is independent of BOS
insertion and lexical/special-token parsing.

Older runtimes may ignore this extension. Use a runtime that implements it
when canonical stop behavior matters. Empty arrays retain their element
type through the Python writer and reader.

For Maple conversion, the producer validates the exact NFC/BPE tokenizer family and added-token boundary flags, preserves each token's declared lexical/special status, and writes the generation configuration's complete EOS policy. The producer converts the reference sliding window's left span to GGUF's total causal-key count (`left_span + 1`); the runtime must not repeat this adjustment. Model saving retains the current EOG set (including empty) and NFC policy.

Native regression coverage is registered as `test-tokenizer-eog-policy`, `test-tokenizer-eog-roundtrip`, and `test-tokenizer-normalization-{gpt2,qwen35}`. The first two use synthetic fixtures; the normalization variants use the matching vocabulary GGUF under `models/`.

## Development
Maintainers who participate in development of this package are advised to install it in editable mode:

Expand Down
1 change: 1 addition & 0 deletions gguf-py/gguf/constants.py
Original file line number Diff line number Diff line change
Expand Up @@ -377,6 +377,7 @@ class Tokenizer:
MERGES = "tokenizer.ggml.merges"
BOS_ID = "tokenizer.ggml.bos_token_id"
EOS_ID = "tokenizer.ggml.eos_token_id"
EOG_IDS = "tokenizer.ggml.eog_token_ids"
EOT_ID = "tokenizer.ggml.eot_token_id"
EOM_ID = "tokenizer.ggml.eom_token_id"
UNK_ID = "tokenizer.ggml.unknown_token_id"
Expand Down
2 changes: 2 additions & 0 deletions gguf-py/gguf/gguf_reader.py
Original file line number Diff line number Diff line change
Expand Up @@ -258,6 +258,8 @@ def _get_field_parts(
offs += int(alen.nbytes)
aparts: list[npt.NDArray[Any]] = [raw_itype, alen]
data_idxs: list[int] = []
if alen[0] == 0:
types.append(GGUFValueType(raw_itype[0]))
# FIXME: Handle multi-dimensional arrays properly instead of flattening
for idx in range(alen[0]):
curr_size, curr_parts, curr_idxs, curr_types = self._get_field_parts(offs, raw_itype[0])
Expand Down
12 changes: 10 additions & 2 deletions gguf-py/gguf/gguf_writer.py
Original file line number Diff line number Diff line change
Expand Up @@ -1339,6 +1339,14 @@ def add_bos_token_id(self, id: int) -> None:
def add_eos_token_id(self, id: int) -> None:
self.add_uint32(Keys.Tokenizer.EOS_ID, id)

def add_eog_token_ids(self, ids: Sequence[int]) -> None:
"""Set the complete EOG set; an empty array explicitly disables natural stops.

This policy is independent of token types and replaces legacy text-based
EOG inference. Omitting the key retains that inference.
"""
self.add_key_value(Keys.Tokenizer.EOG_IDS, ids, GGUFValueType.ARRAY, GGUFValueType.UINT32)

def add_unk_token_id(self, id: int) -> None:
self.add_uint32(Keys.Tokenizer.UNK_ID, id)

Expand Down Expand Up @@ -1818,8 +1826,8 @@ def _pack_val(self, val: Any, vtype: GGUFValueType, add_vtype: bool, sub_type: G
if not isinstance(val, Sequence):
raise ValueError("Invalid GGUF metadata array, expecting sequence")

if len(val) == 0:
raise ValueError("Invalid GGUF metadata array. Empty array")
if len(val) == 0 and sub_type is None:
raise ValueError("Invalid GGUF metadata array. Empty array requires an explicit type")

if sub_type is not None:
ltype = sub_type
Expand Down
25 changes: 25 additions & 0 deletions gguf-py/tests/test_gguf_reader_validation.py
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@
import pytest

from gguf.gguf_reader import GGUFReader
from gguf.constants import GGUFValueType


def _write_gguf(path, n_dims_field, dims):
Expand Down Expand Up @@ -35,3 +36,27 @@ def test_dims_product_no_uint64_wraparound(tmp_path):
_write_gguf(p, len(dims), dims)
with pytest.raises(ValueError):
GGUFReader(p)


def _write_empty_array(path, element_type):
key = b'tokenizer.ggml.eog_token_ids'
data = b'GGUF' + struct.pack('<IQQ', 3, 0, 1)
data += struct.pack('<Q', len(key)) + key
data += struct.pack('<IIQ', GGUFValueType.ARRAY, element_type, 0)
data += b'\x00' * (-len(data) % 32)
path.write_bytes(data)


def test_empty_integer_array_retains_element_type(tmp_path):
path = tmp_path / 'empty_eog.gguf'
_write_empty_array(path, GGUFValueType.UINT32)
field = GGUFReader(path).fields['tokenizer.ggml.eog_token_ids']
assert field.types == [GGUFValueType.ARRAY, GGUFValueType.UINT32]
assert field.contents() == []


def test_empty_array_rejects_unknown_element_type(tmp_path):
path = tmp_path / 'invalid_empty_array.gguf'
_write_empty_array(path, 99)
with pytest.raises(ValueError):
GGUFReader(path)
1 change: 1 addition & 0 deletions src/llama-arch.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -451,6 +451,7 @@ static const std::map<llm_kv, const char *> LLM_KV_NAMES = {
{ LLM_KV_TOKENIZER_MERGES, "tokenizer.ggml.merges" },
{ LLM_KV_TOKENIZER_BOS_ID, "tokenizer.ggml.bos_token_id" },
{ LLM_KV_TOKENIZER_EOS_ID, "tokenizer.ggml.eos_token_id" },
{ LLM_KV_TOKENIZER_EOG_IDS, "tokenizer.ggml.eog_token_ids" },
{ LLM_KV_TOKENIZER_EOT_ID, "tokenizer.ggml.eot_token_id" },
{ LLM_KV_TOKENIZER_EOM_ID, "tokenizer.ggml.eom_token_id" },
{ LLM_KV_TOKENIZER_UNK_ID, "tokenizer.ggml.unknown_token_id" },
Expand Down
1 change: 1 addition & 0 deletions src/llama-arch.h
Original file line number Diff line number Diff line change
Expand Up @@ -412,6 +412,7 @@ enum llm_kv {
LLM_KV_TOKENIZER_MERGES,
LLM_KV_TOKENIZER_BOS_ID,
LLM_KV_TOKENIZER_EOS_ID,
LLM_KV_TOKENIZER_EOG_IDS,
LLM_KV_TOKENIZER_EOT_ID,
LLM_KV_TOKENIZER_EOM_ID,
LLM_KV_TOKENIZER_UNK_ID,
Expand Down
7 changes: 7 additions & 0 deletions src/llama-model-saver.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -437,6 +437,12 @@ void llama_model_saver::add_kv_from_model() {
add_kv(LLM_KV_TOKENIZER_TOKEN_TYPE_COUNT, vocab.n_token_types());
add_kv(LLM_KV_TOKENIZER_SCORES, scores);
add_kv(LLM_KV_TOKENIZER_MERGES, vocab.get_bpe_merges());
const auto & eog_set = vocab.get_eog_tokens();
const std::vector<uint32_t> eog_ids(eog_set.begin(), eog_set.end());
const uint32_t empty_eog = 0;
// Empty is authoritative too; the generic array helper omits empty values.
gguf_set_arr_data(gguf_ctx, llm_kv(LLM_KV_TOKENIZER_EOG_IDS).c_str(), GGUF_TYPE_UINT32,
eog_ids.empty() ? &empty_eog : eog_ids.data(), eog_ids.size());
// FIXME llama_token is type i32 but when reading in a GGUF file u32 is expected, not an issue for writing though
if (vocab.token_bos() != LLAMA_TOKEN_NULL) { add_kv(LLM_KV_TOKENIZER_BOS_ID, uint32_t(vocab.token_bos())); }
if (vocab.token_eos() != LLAMA_TOKEN_NULL) { add_kv(LLM_KV_TOKENIZER_EOS_ID, uint32_t(vocab.token_eos())); }
Expand All @@ -453,6 +459,7 @@ void llama_model_saver::add_kv_from_model() {
add_kv(LLM_KV_TOKENIZER_ADD_PREFIX, vocab.get_add_space_prefix());
add_kv(LLM_KV_TOKENIZER_REMOVE_EXTRA_WS, vocab.get_remove_extra_whitespaces());
add_kv(LLM_KV_TOKENIZER_CLEAN_SPACES, vocab.get_clean_spaces());
add_kv(LLM_KV_TOKENIZER_NORMALIZER_NFC, vocab.get_normalizer_opts().nfc);
add_kv(LLM_KV_TOKENIZER_PRECOMPILED_CHARSMAP, vocab.get_precompiled_charsmap());
// add_kv(LLM_KV_TOKENIZER_HF_JSON, ???);
// add_kv(LLM_KV_TOKENIZER_RWKV, ???);
Expand Down
Loading