model: support vllm decision 2 model

common/common.cpp

@@ -1170,8 +1170,10 @@ static const std::map<common_decision_type, std::string> COMMON_DECISION_TYPE_NA
     { COMMON_DECISION_TYPE_LAYA,           "laya"          },
     { COMMON_DECISION_TYPE_CLEF,           "clef"          },
     { COMMON_DECISION_TYPE_PPLX_DECIDER,   "pplx-decider"  },
+    { COMMON_DECISION_TYPE_VLLM_DECISION2, "vllm-decision2" },
     { COMMON_DECISION_TYPE_LFM2_D1,        "lfm2-d1"       },
     { COMMON_DECISION_TYPE_LFM2_D1_OMNI,   "lfm2-d1-omni"  },
+    // for future models, ALWAYS add org name prefix to avoid name collisions, e.g. "orgname-modelname"
 };
 
 static common_decision_type common_decision_type_from_string(const std::string & str) {
@@ -1289,7 +1291,7 @@ common_init_result::common_init_result(common_params & params, bool model_only)
     // TODO: maybe improve this in the future
     const auto decision_type = common_get_decision_type(model);
     if (decision_type == COMMON_DECISION_TYPE_LAYA || decision_type == COMMON_DECISION_TYPE_KEV || decision_type == COMMON_DECISION_TYPE_CLEF ||
-        decision_type == COMMON_DECISION_TYPE_LFM2_D1_OMNI) {
+        decision_type == COMMON_DECISION_TYPE_LFM2_D1_OMNI || decision_type == COMMON_DECISION_TYPE_VLLM_DECISION2) {
         params.embedding    = true;
         params.pooling_type = LLAMA_POOLING_TYPE_NONE;
 

common/common.h

@@ -963,6 +963,7 @@ enum common_decision_type {
     COMMON_DECISION_TYPE_LAYA,    // score of one marker token per option, read from the embeddings output
     COMMON_DECISION_TYPE_CLEF,    // all questions in one prompt, score of option i read from the embeddings output at row i
     COMMON_DECISION_TYPE_PPLX_DECIDER, // same as openjev, label codes of 1 or 2 letters
+    COMMON_DECISION_TYPE_VLLM_DECISION2, // score of each option read from the embeddings output, the options of a question are marked in the batch
     COMMON_DECISION_TYPE_LFM2_D1, // same as openjev, the labels depend on the question type
     COMMON_DECISION_TYPE_LFM2_D1_OMNI, // same as laya, other prompt layout
     COMMON_DECISION_TYPE_UNKNOWN, // a decision model of a type that is not supported

conversion/__init__.py

@@ -291,6 +291,7 @@
     "UltravoxModel": "ultravox",
     "UnlimitedOCRForCausalLM": "deepseek",
     "VLlama3ForCausalLM": "llama",
+    "VllmDecision2Model": "vllm_decision2",
     "VoxtralForConditionalGeneration": "llama",
     "WavTokenizerDec": "wavtokenizer",
     "XLMRobertaForSequenceClassification": "bert",

conversion/vllm_decision2.py

@@ -0,0 +1,107 @@
+from __future__ import annotations
+
+import json
+
+from pathlib import Path
+from typing import Any, Callable, Iterator, TYPE_CHECKING
+
+if TYPE_CHECKING:
+    from torch import Tensor
+
+from .base import ModelBase, gguf, logger
+from .qwen import Qwen3Model
+
+
+def _is_vllm_decision2_checkpoint(dir_model: Path) -> bool:
+    if not (dir_model / "config.json").is_file():
+        return False
+    with open(dir_model / "config.json", encoding="utf-8") as f:
+        return json.load(f).get("decision_format") == "vllm-sr-decision"
+
+
+@ModelBase.register_hparams_loader(_is_vllm_decision2_checkpoint)
+def _load_vllm_decision2_hparams(dir_model: Path) -> dict[str, Any]:
+    logger.info("gguf: detected vllm-sr Decision 2.0 checkpoint")
+    with open(dir_model / "config.json", encoding="utf-8") as f:
+        package = json.load(f)
+    hparams = ModelBase.load_hparams((dir_model / package["backbone"]["config"]).parent, False, guess=False)
+    hparams["architectures"] = ["VllmDecision2Model"]
+    with open(dir_model / package["model_config"], encoding="utf-8") as f:
+        hparams["decision"] = json.load(f)
+    with open(dir_model / "score_bias.json", encoding="utf-8") as f:
+        hparams["decision"]["score_bias"] = json.load(f)["offsets"]
+    return hparams
+
+
+@ModelBase.register("VllmDecision2Model")
+@ModelBase.example("vllm-sr/Decision-2.0-Kai-0.6B")
+class VllmDecision2Model(Qwen3Model):
+    model_arch = gguf.MODEL_ARCH.VLLM_DECISION2
+
+    # torch.nn.LayerNorm default, used by the head
+    _HEAD_NORM_EPS = 1e-5
+
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        head = self.hparams["decision"]
+        if head["architecture"] != "qwen3-text-endpoints-global-query-shared-bilinear-mlp" or head["head_variant"] != "shared":
+            raise ValueError(f"unsupported decision head: {head['architecture']}")
+
+    def index_tensors(self, *args, **kwargs) -> dict[str, Callable[[], Tensor]]:
+        # the backbone weights are in their own directory
+        dir_model, self.dir_model = self.dir_model, self.dir_model / "backbone"
+        try:
+            return super().index_tensors(*args, **kwargs)
+        finally:
+            self.dir_model = dir_model
+
+    def get_tensors(self) -> Iterator[tuple[str, Tensor]]:
+        yield from super().get_tensors()
+        from safetensors.torch import load_file
+        for name, data in load_file(self.dir_model / "decision_head.safetensors").items():
+            yield "decision_head." + name, data
+
+    def tensor_force_quant(self, name: str, new_name: str, bid: int | None, n_dims: int) -> gguf.GGMLQuantizationType | bool:
+        # the head is computed in F32
+        if new_name.startswith("decision."):
+            return gguf.GGMLQuantizationType.F32
+        return super().tensor_force_quant(name, new_name, bid, n_dims)
+
+    def set_vocab(self):
+        super().set_vocab()
+        self.gguf_writer.add_chat_template([{"name": "systemone", "template": self._systemone_template()}])
+
+    @classmethod
+    def _systemone_template(cls) -> str:
+        # prompt follows decision2/_vendor/dev2model/decision_model.py (segments) of the model repo
+        # the pieces of the prompt are tokenized one by one, the server gives the text that separates them (sep)
+        # and the text that starts the span of a question or of an option (mark_question, mark_option)
+        # the head reads the last token of each option and the last token of the question
+        # the start of an option is a piece of its own, so that the spans of two options are not joined
+        # the keys of JSON objects are given in sorted order
+        def text(value: str) -> str:
+            return "{{ " + json.dumps(value) + " }}"
+
+        def render(name: str) -> str:
+            return "{{ " + name + " if " + name + " is string else " + name + " | tojson(separators=[',', ':']) }}"
+
+        option = (
+            "{% set d = o.description %}"
+            "{% if q.type == 'noul' and d is none %}{% set d = 'Yes' if o.key == 'true' else 'No' %}{% endif %}"
+            "{{ {'description': d, 'key': o.key} | tojson(separators=[',', ':']) }}"
+        )
+        return (
+            "{% for q in questions %}"
+            + text("Context:\n") + render("state")
+            + text("\n\nTask type: ") + "{{ q.type }}" + text("\nQuestion:\n") + render("q.instructions") + text("\nOptions:")
+            + "{% for o in q.options %}{{ sep }}" + text("\n<option>\n") + "{{ sep }}{{ mark_option }}" + option + text("\n</option>") + "{% endfor %}"
+            + "{{ sep }}{{ mark_question }}" + text("\n\nSelect the single option best supported by the context and instructions.\nDecision:")
+            + "{% endfor %}"
+        )
+
+    def set_gguf_parameters(self):
+        super().set_gguf_parameters()
+        self.gguf_writer.add_decision_type(gguf.DecisionType.VLLM_DECISION2)
+        self.gguf_writer.add_layer_norm_eps(self._HEAD_NORM_EPS)
+        for offsets in self.hparams["decision"]["score_bias"].values():
+            self.gguf_writer.add_decision_score_bias(offsets)

gguf-py/gguf/constants.py

@@ -332,6 +332,7 @@ class Decision:
         HEAD_COUNT         = "{arch}.decision.head_count"
         MAX_HEAD_TOKENS    = "{arch}.decision.max_head_tokens"
         TEMPERATURE        = "{arch}.decision.temperature.{name}"  # name: "<type>" or "<type>.<n_opt bucket>"
+        SCORE_BIAS         = "{arch}.decision.score_bias.{n}.{i}"  # added to the logit of level i of a score question with n levels
 
     class Tokenizer:
         MODEL                = "tokenizer.ggml.model"
@@ -559,6 +560,7 @@ class MODEL_ARCH(IntEnum):
     QWEN35           = auto()
     QWEN35MOE        = auto()
     CLEF             = auto()
+    VLLM_DECISION2   = auto()
     QWEN4EXP         = auto()
     PHI2             = auto()
     PHI3             = auto()
@@ -923,6 +925,11 @@ class MODEL_TENSOR(IntEnum):
     DECISION_SCALES               = auto()
     DECISION_SCORER               = auto()
     DECISION_SCORER_OUT           = auto()
+    DECISION_QUERY_NORM           = auto()
+    DECISION_KEY                  = auto()
+    DECISION_QUERY                = auto()
+    DECISION_OPTION_MLP           = auto()
+    DECISION_QUERY_MLP            = auto()
     CONV1D               = auto()
     CONVNEXT_DW          = auto()
     CONVNEXT_NORM        = auto()
@@ -1343,6 +1350,7 @@ class MODEL_TENSOR(IntEnum):
     MODEL_ARCH.QWEN35:           "qwen35",
     MODEL_ARCH.QWEN35MOE:        "qwen35moe",
     MODEL_ARCH.CLEF:             "clef",
+    MODEL_ARCH.VLLM_DECISION2:   "vllm-decision2",
     MODEL_ARCH.QWEN4EXP:         "qwen4exp",
     MODEL_ARCH.PHI2:             "phi2",
     MODEL_ARCH.PHI3:             "phi3",
@@ -1706,6 +1714,11 @@ class MODEL_TENSOR(IntEnum):
     MODEL_TENSOR.DECISION_SCALES:               "decision.scales",
     MODEL_TENSOR.DECISION_SCORER:               "decision.scorer",
     MODEL_TENSOR.DECISION_SCORER_OUT:           "decision.scorer_out",
+    MODEL_TENSOR.DECISION_QUERY_NORM:           "decision.query_norm",
+    MODEL_TENSOR.DECISION_KEY:                  "decision.key",
+    MODEL_TENSOR.DECISION_QUERY:                "decision.query",
+    MODEL_TENSOR.DECISION_OPTION_MLP:           "decision.option_mlp",
+    MODEL_TENSOR.DECISION_QUERY_MLP:            "decision.query_mlp",
     MODEL_TENSOR.CONV1D:                    "conv1d",
     MODEL_TENSOR.CONVNEXT_DW:               "convnext.{bid}.dw",
     MODEL_TENSOR.CONVNEXT_NORM:             "convnext.{bid}.norm",
@@ -3034,6 +3047,29 @@ class MODEL_TENSOR(IntEnum):
         MODEL_TENSOR.DECISION_SCORER,
         MODEL_TENSOR.DECISION_SCORER_OUT,
     ],
+    MODEL_ARCH.VLLM_DECISION2: [
+        MODEL_TENSOR.TOKEN_EMBD,
+        MODEL_TENSOR.OUTPUT_NORM,
+        MODEL_TENSOR.ATTN_NORM,
+        MODEL_TENSOR.ATTN_Q,
+        MODEL_TENSOR.ATTN_Q_NORM,
+        MODEL_TENSOR.ATTN_K,
+        MODEL_TENSOR.ATTN_K_NORM,
+        MODEL_TENSOR.ATTN_V,
+        MODEL_TENSOR.ATTN_OUT,
+        MODEL_TENSOR.FFN_NORM,
+        MODEL_TENSOR.FFN_GATE,
+        MODEL_TENSOR.FFN_DOWN,
+        MODEL_TENSOR.FFN_UP,
+        # decision head
+        MODEL_TENSOR.DECISION_OPTION_NORM,
+        MODEL_TENSOR.DECISION_QUERY_NORM,
+        MODEL_TENSOR.DECISION_KEY,
+        MODEL_TENSOR.DECISION_QUERY,
+        MODEL_TENSOR.DECISION_OPTION_MLP,
+        MODEL_TENSOR.DECISION_QUERY_MLP,
+        MODEL_TENSOR.DECISION_SCORER_OUT,
+    ],
     MODEL_ARCH.QWEN35MOE: [
         MODEL_TENSOR.TOKEN_EMBD,
         MODEL_TENSOR.OUTPUT_NORM,
@@ -6100,6 +6136,7 @@ class DecisionType:
     NIMBLE  = "nimble"   # same as openjev, the prompt lists all the questions of the request
     CLEF    = "clef"     # joint head over all questions, one score per option
     PPLX_DECIDER = "pplx-decider"  # same as openjev, label codes of 1 or 2 letters
+    VLLM_DECISION2 = "vllm-decision2"  # head over the end token of each option and the last token, one score per option
     LFM2_D1 = "lfm2-d1"  # same as openjev, the labels depend on the question type
     LFM2_D1_OMNI = "lfm2-d1-omni"  # same head as laya on a bidirectional LFM2 trunk, other prompt layout
 

gguf-py/gguf/gguf_writer.py

@@ -1365,6 +1365,10 @@ def add_decision_max_head_tokens(self, value: int) -> None:
     def add_decision_temperature(self, name: str, value: float) -> None:
         self.add_float32(Keys.Decision.TEMPERATURE.format(arch=self.arch, name=name), value)
 
+    def add_decision_score_bias(self, values: Sequence[float]) -> None:
+        for i, value in enumerate(values):
+            self.add_float32(Keys.Decision.SCORE_BIAS.format(arch=self.arch, n=len(values), i=i), value)
+
     # for vision models
 
     def add_clip_has_vision_encoder(self, value: bool) -> None:

gguf-py/gguf/tensor_mapping.py

@@ -1328,7 +1328,8 @@ class TensorNameMap:
         ),
 
         MODEL_TENSOR.DECISION_OPTION_NORM: (
-            "joint_head.option_norm",  # clef
+            "joint_head.option_norm",        # clef
+            "decision_head.candidate_norm",  # vllm-decision2
         ),
 
         MODEL_TENSOR.DECISION_SCORER: (
@@ -1337,6 +1338,27 @@ class TensorNameMap:
 
         MODEL_TENSOR.DECISION_SCORER_OUT: (
             "joint_head.residual_scorer.3",  # clef
+            "decision_head.scalar",          # vllm-decision2
+        ),
+
+        MODEL_TENSOR.DECISION_QUERY_NORM: (
+            "decision_head.query_norm",  # vllm-decision2
+        ),
+
+        MODEL_TENSOR.DECISION_KEY: (
+            "decision_head.key",  # vllm-decision2
+        ),
+
+        MODEL_TENSOR.DECISION_QUERY: (
+            "decision_head.query",  # vllm-decision2
+        ),
+
+        MODEL_TENSOR.DECISION_OPTION_MLP: (
+            "decision_head.candidate_mlp",  # vllm-decision2
+        ),
+
+        MODEL_TENSOR.DECISION_QUERY_MLP: (
+            "decision_head.query_mlp",  # vllm-decision2
         ),
 
         MODEL_TENSOR.ENC_ATTN_NORM: (

src/llama-arch.cpp

@@ -41,6 +41,7 @@ static const std::map<llm_arch, const char *> LLM_ARCH_NAMES = {
     { LLM_ARCH_QWEN35,           "qwen35"           },
     { LLM_ARCH_QWEN35MOE,        "qwen35moe"        },
     { LLM_ARCH_CLEF,             "clef"             },
+    { LLM_ARCH_VLLM_DECISION2,   "vllm-decision2"   },
     { LLM_ARCH_QWEN4EXP,         "qwen4exp"         },
     { LLM_ARCH_PHI2,             "phi2"             },
     { LLM_ARCH_PHI3,             "phi3"             },
@@ -524,6 +525,11 @@ static const std::map<llm_tensor, const char *> LLM_TENSOR_NAMES = {
     { LLM_TENSOR_DECISION_SCALES,                        "decision.scales" },
     { LLM_TENSOR_DECISION_SCORER,                        "decision.scorer" },
     { LLM_TENSOR_DECISION_SCORER_OUT,                    "decision.scorer_out" },
+    { LLM_TENSOR_DECISION_QUERY_NORM,                    "decision.query_norm" },
+    { LLM_TENSOR_DECISION_KEY,                           "decision.key" },
+    { LLM_TENSOR_DECISION_QUERY,                         "decision.query" },
+    { LLM_TENSOR_DECISION_OPTION_MLP,                    "decision.option_mlp" },
+    { LLM_TENSOR_DECISION_QUERY_MLP,                     "decision.query_mlp" },
     { LLM_TENSOR_ENC_OUTPUT_NORM,                        "enc.output_norm" },
     { LLM_TENSOR_FFN_GATE_INP_SHEXP,                     "blk.%d.ffn_gate_inp_shexp" },
     { LLM_TENSOR_SSM_A_NOSCAN,                           "blk.%d.ssm_a" },
@@ -789,6 +795,11 @@ static const std::map<llm_tensor, llm_tensor_info> LLM_TENSOR_INFOS = {
     {LLM_TENSOR_DECISION_SCALES,            {LLM_TENSOR_LAYER_OUTPUT,    GGML_OP_MUL}},
     {LLM_TENSOR_DECISION_SCORER,            {LLM_TENSOR_LAYER_OUTPUT,    GGML_OP_MUL_MAT}},
     {LLM_TENSOR_DECISION_SCORER_OUT,        {LLM_TENSOR_LAYER_OUTPUT,    GGML_OP_MUL_MAT}},
+    {LLM_TENSOR_DECISION_QUERY_NORM,        {LLM_TENSOR_LAYER_OUTPUT,    GGML_OP_MUL}},
+    {LLM_TENSOR_DECISION_KEY,               {LLM_TENSOR_LAYER_OUTPUT,    GGML_OP_MUL_MAT}},
+    {LLM_TENSOR_DECISION_QUERY,             {LLM_TENSOR_LAYER_OUTPUT,    GGML_OP_MUL_MAT}},
+    {LLM_TENSOR_DECISION_OPTION_MLP,        {LLM_TENSOR_LAYER_OUTPUT,    GGML_OP_MUL_MAT}},
+    {LLM_TENSOR_DECISION_QUERY_MLP,         {LLM_TENSOR_LAYER_OUTPUT,    GGML_OP_MUL_MAT}},
     {LLM_TENSOR_DENSE_2_OUT,                {LLM_TENSOR_LAYER_OUTPUT,    GGML_OP_MUL_MAT}}, // Dense layer output
     {LLM_TENSOR_DENSE_3_OUT,                {LLM_TENSOR_LAYER_OUTPUT,    GGML_OP_MUL_MAT}}, // Dense layer output
     {LLM_TENSOR_OUTPUT_NORM,                {LLM_TENSOR_LAYER_OUTPUT,    GGML_OP_MUL}},

src/llama-arch.h

@@ -46,6 +46,7 @@ enum llm_arch {
     LLM_ARCH_QWEN35,
     LLM_ARCH_QWEN35MOE,
     LLM_ARCH_CLEF,
+    LLM_ARCH_VLLM_DECISION2,
     LLM_ARCH_QWEN4EXP,
     LLM_ARCH_PHI2,
     LLM_ARCH_PHI3,
@@ -695,6 +696,11 @@ enum llm_tensor {
     LLM_TENSOR_DECISION_SCALES,
     LLM_TENSOR_DECISION_SCORER,
     LLM_TENSOR_DECISION_SCORER_OUT,
+    LLM_TENSOR_DECISION_QUERY_NORM,
+    LLM_TENSOR_DECISION_KEY,
+    LLM_TENSOR_DECISION_QUERY,
+    LLM_TENSOR_DECISION_OPTION_MLP,
+    LLM_TENSOR_DECISION_QUERY_MLP,
     LLM_TENSOR_CONV1D,
     LLM_TENSOR_CONVNEXT_DW,
     LLM_TENSOR_CONVNEXT_NORM,

src/llama-ext.h

@@ -100,7 +100,7 @@ LLAMA_API void llama_set_embeddings_nextn(struct llama_context * ctx, bool value
 // chain multiple trained NextN heads. Default 0 (first head).
 LLAMA_API void llama_set_nextn_layer_offset(struct llama_context * ctx, int32_t offset);
 
-// Marks the entries that a joint decision head (clef) reads, the default is 0
+// Marks the entries that a decision head (clef, vllm-decision2) reads, the default is 0
 // See https://github.com/ggml-org/llama.cpp/pull/29831 for details
 // A run of entries with the same value is one span, spans must be separated by entries with value 0
 // An option belongs to the last question before it

src/llama-model-saver.cpp

@@ -21,6 +21,7 @@ bool llama_model_saver_supports_arch(llm_arch arch) {
         case LLM_ARCH_APERTUS:
         case LLM_ARCH_STEP35:
         case LLM_ARCH_CLEF: // the head tensors are not saved
+        case LLM_ARCH_VLLM_DECISION2:
             return false;
         default:
             return true;

src/llama-model.cpp

@@ -330,6 +330,8 @@ static llama_model * llama_model_mapping(llm_arch arch, const llama_model_params
             return new llama_model_qwen35moe(params);
         case LLM_ARCH_CLEF:
             return new llama_model_clef(params);
+        case LLM_ARCH_VLLM_DECISION2:
+            return new llama_model_vllm_decision2(params);
         case LLM_ARCH_QWEN4EXP:
             return new llama_model_qwen4exp(params);
         case LLM_ARCH_MISTRAL3:
@@ -3534,6 +3536,7 @@ llama_rope_type llama_model_rope_type(const llama_model * model) {
         case LLM_ARCH_DREAM:
         case LLM_ARCH_QWEN2MOE:
         case LLM_ARCH_QWEN3:
+        case LLM_ARCH_VLLM_DECISION2:
         case LLM_ARCH_QWEN3MOE:
         case LLM_ARCH_LLADA_MOE:
         case LLM_ARCH_RND1:

src/llama-quant.cpp

@@ -346,6 +346,11 @@ static bool tensor_allows_quantization(const llama_model_quantize_params * param
         quantize &= name.find("attn_v_b.weight")         == std::string::npos;
     }
 
+    // the decision head of vllm-decision2 is small and computed in F32
+    if (arch == LLM_ARCH_VLLM_DECISION2) {
+        quantize &= name.compare(0, 9, "decision.") != 0;
+    }
+
     // do not quantize RWKV's small yet 2D weights
     quantize &= name.find("time_mix_first.weight") == std::string::npos;
     quantize &= name.find("time_mix_w0.weight") == std::string::npos;