diff --git a/common/speculative.cpp b/common/speculative.cpp index 701093308292..d8afd4095498 100644 --- a/common/speculative.cpp +++ b/common/speculative.cpp @@ -1338,7 +1338,7 @@ struct common_speculative_impl_draft_dflash : public common_speculative_impl { GGML_ASSERT(lattice && "DFlash2 selector produced no lattice"); int32_t predecessor = 0; - for (int32_t i = 1; i < n_block_tokens; ++i) { + for (int32_t i = 1; i < (is_dflash2 ? n_block_tokens : params.n_max + 1); ++i) { const float * row = lattice + (size_t) (beg + i) * n_embd_dec; const int32_t top_k = is_dflash2 ? selector_top_k : lilicorr_top_k; const float * scores = row + top_k + (size_t) predecessor * top_k; diff --git a/conversion/__init__.py b/conversion/__init__.py index d5b480a9bf41..8ce4cb7337d6 100644 --- a/conversion/__init__.py +++ b/conversion/__init__.py @@ -353,6 +353,7 @@ "MiniCPMV4_6ForConditionalGeneration": "minicpm", "MiniCPMV4_7ForConditionalGeneration": "minicpm", "Mistral3ForConditionalGeneration": "llava", + "NemotronH_Omni_Reasoning_V3": "nemotron", "NemotronH_Nano_VL_V2": "nemotron", "MuseGlimmerForConditionalGeneration": "muse_glimmer", "PaddleOCRVisionModel": "ernie", diff --git a/conversion/base.py b/conversion/base.py index efb7ea72e2cd..a5a80f27dad4 100644 --- a/conversion/base.py +++ b/conversion/base.py @@ -568,7 +568,7 @@ def dequant_fp8() -> None: tensors_to_remove.append(base_name + "_zero_point") else: raise NotImplementedError(f"Quant format {quant_format!r} for method {quant_method!r} is not yet supported") - elif quant_method == "modelopt": + elif quant_method in ("modelopt", "modelopt_mixed"): dequant_fp8() elif quant_method is not None: raise NotImplementedError(f"Quant method is not yet supported: {quant_method!r}") diff --git a/conversion/nemotron.py b/conversion/nemotron.py index 581f41a0230b..c900481943c2 100644 --- a/conversion/nemotron.py +++ b/conversion/nemotron.py @@ -1,6 +1,6 @@ from __future__ import annotations -from typing import Any, Callable, Iterable, TYPE_CHECKING +from typing import Any, Callable, Iterable, Sequence, TYPE_CHECKING import numpy as np import torch @@ -89,19 +89,22 @@ def set_gguf_parameters(self): self.gguf_writer.add_lilicorr_vector_eps(float(dflash_config["lilicorr_vector_eps"])) self.gguf_writer.add_lilicorr_logit_scale(float(dflash_config["lilicorr_logit_scale"])) + def map_tensor_name(self, name: str, try_suffixes: Sequence[str] = (".weight", ".bias")) -> str: + if name.startswith("model.lilicorr.layers."): + parts = name.split(".", 4) + if not parts[4].startswith("attn.in_proj."): + parts[3] = str(self.block_count + int(parts[3])) + name = ".".join(parts) + elif name.startswith("model.lilicorr.feature_norm."): + feature_bid = self.block_count + self.lilicorr_geometry["num_layers"] + name = name.replace("model.lilicorr.feature_norm.", f"model.lilicorr.feature_mlp.{feature_bid}.norm.") + elif name.startswith("model.lilicorr.feature_mlp."): + feature_bid = self.block_count + self.lilicorr_geometry["num_layers"] + name = name.replace("model.lilicorr.feature_mlp.", f"model.lilicorr.feature_mlp.{feature_bid}.") + return super().map_tensor_name(name, try_suffixes) + def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]: if name.startswith("model.lilicorr."): - if name.startswith("model.lilicorr.layers."): - parts = name.split(".", 4) - if not parts[4].startswith("attn.in_proj."): - parts[3] = str(self.block_count + int(parts[3])) - name = ".".join(parts) - elif name.startswith("model.lilicorr.feature_norm."): - feature_bid = self.block_count + self.lilicorr_geometry["num_layers"] - name = name.replace("model.lilicorr.feature_norm.", f"model.lilicorr.feature_mlp.{feature_bid}.norm.") - elif name.startswith("model.lilicorr.feature_mlp."): - feature_bid = self.block_count + self.lilicorr_geometry["num_layers"] - name = name.replace("model.lilicorr.feature_mlp.", f"model.lilicorr.feature_mlp.{feature_bid}.") yield self.map_tensor_name(name), data_torch return yield from super().modify_tensors(data_torch, name, bid) @@ -251,6 +254,113 @@ def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iter yield mapped_name, tensor +@ModelBase.register("NemotronH_Omni_Reasoning_V3") +class NemotronOmniVLModel(NemotronNanoV2VLModel): + _omni_tensor_prefixes = ( + "vision_model.embeddings.", + "vision_model.encoder.layer.", + "vision_projector.mlp1.", + ) + + def get_vision_config(self) -> dict[str, Any] | None: + vision_config = dict(self.global_config["vision_config"]) + vision_config["intermediate_size"] = int(vision_config["hidden_size"] * vision_config["mlp_ratio"]) + vision_config["image_size"] = 512 # warmup only; the full position table is resized at runtime + return vision_config + + def set_gguf_parameters(self): + preprocessor_config = self.preprocessor_config + + preprocessor_config["image_mean"] = preprocessor_config["norm_mean"] + preprocessor_config["image_std"] = preprocessor_config["norm_std"] + + MmprojModel.set_gguf_parameters(self) + self.gguf_writer.add_clip_projector_type(gguf.VisionProjectorType.NEMOTRON_OMNI) + self.gguf_writer.add_vision_attention_layernorm_eps(1e-6) + self.gguf_writer.add_vision_use_gelu(True) + downsample_ratio = self.global_config.get("downsample_ratio", 0.5) + self.gguf_writer.add_vision_projector_scale_factor(int(1.0 / downsample_ratio)) + patch_area = self.hparams["patch_size"] ** 2 + self.gguf_writer.add_vision_min_pixels(preprocessor_config["min_num_patches"] * patch_area) + self.gguf_writer.add_vision_max_pixels(preprocessor_config["max_num_patches"] * patch_area) + + @classmethod + def filter_tensors(cls, item: tuple[str, Callable[[], Tensor]]) -> tuple[str, Callable[[], Tensor]] | None: + name, gen = item + + # Transformers can rename RADIO and projector tensors in Omni checkpoints. + # Accept both checkpoint layouts. + if name == "vision_model.summary_idxs": + # This buffer is only used by RadioModel.summary. The multimodal + # projector consumes RadioModel.features for both images and video. + return None + if name.startswith(cls._omni_tensor_prefixes): + return name, gen + + if "radio_model.model.patch_generator.video_embedder" in name or name.startswith("vision_projector.vision_final_layernorm."): + return MmprojModel.filter_tensors((name, gen)) + + return super().filter_tensors(item) + + def tensor_force_quant(self, name, new_name, bid, n_dims): + if new_name in ("v.position_embd.weight", "v.class_embd"): + return gguf.GGMLQuantizationType.F32 + return super().tensor_force_quant(name, new_name, bid, n_dims) + + def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]: + if name.startswith(self._omni_tensor_prefixes): + if name == "vision_model.embeddings.video_patch_projection.weight": + data_torch = data_torch.reshape( + self.hparams["hidden_size"], + 6, + self.hparams["patch_size"], + self.hparams["patch_size"], + ) + yield gguf.TENSOR_NAMES[gguf.MODEL_TENSOR.V_ENC_EMBD_PATCH] + ".weight.1", data_torch + return + + if name.startswith("vision_projector.mlp1."): + projector_mapping = { + "vision_projector.mlp1.norm.weight": "mlp1.0.weight", + "vision_projector.mlp1.linear1.weight": "mlp1.1.weight", + "vision_projector.mlp1.linear2.weight": "mlp1.3.weight", + } + name = projector_mapping[name] + elif name == "vision_model.embeddings.patch_projection.weight": + data_torch = data_torch.reshape( + self.hparams["hidden_size"], + 3, + self.hparams["patch_size"], + self.hparams["patch_size"], + ) + elif name == "vision_model.embeddings.position_embedding": + data_torch = data_torch.squeeze(0) + name += ".weight" + elif ".layer_scale" in name and name.endswith(".lambda1"): + name = name.removesuffix("lambda1") + "weight" + + yield from MmprojModel.modify_tensors(self, data_torch, name, bid) + return + + if "patch_generator.pos_embed" in name: + # Keep the full table for runtime interpolation. + yield from MmprojModel.modify_tensors(self, data_torch, name, bid) + return + + if "patch_generator.video_embedder" in name: + patch_size = self.hparams["patch_size"] + n_embd = self.hparams["hidden_size"] + temporal_patch_size = self.global_config["video_temporal_patch_size"] + if temporal_patch_size != 2: + raise ValueError("Current implementation only supports a temporal patch size of 2") + data_torch = data_torch.reshape(n_embd, temporal_patch_size * 3, patch_size, patch_size) + new_name = gguf.TENSOR_NAMES[gguf.MODEL_TENSOR.V_ENC_EMBD_PATCH] + ".weight.1" + yield new_name, data_torch + return + + yield from super().modify_tensors(data_torch, name, bid) + + @ModelBase.register("NemotronForCausalLM") @ModelBase.example("nvidia/Minitron-4B-Base") class NemotronModel(TextModel): @@ -382,6 +492,8 @@ def get_attn_layers(self): @classmethod def filter_tensors(cls, item: tuple[str, Callable[[], Tensor]]) -> tuple[str, Callable[[], Tensor]] | None: name, gen = item + if name.startswith("language_model."): + name = name[len("language_model."):] if name.startswith("mtp."): # --no-mtp: drop the MTP head entirely if cls.no_mtp: @@ -531,6 +643,14 @@ def set_vocab(self): "mtp.layers.1.final_layernorm.weight": "model.layers.{bid}.shared_head.norm.weight", } + def _generate_nvfp4_tensors(self): + if self._mtp_bid is not None: + for name in list(self.model_tensors): + if name.startswith("mtp.layers.1.mixer"): + new_name = f"backbone.layers.{self._mtp_bid}.{name.split('.', 3)[3]}" + self.model_tensors[new_name] = self.model_tensors.pop(name) + super()._generate_nvfp4_tensors() + def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]: # mtp.layers.0: NextN input fusion + attention # mtp.layers.1: MoE + final head norm diff --git a/gguf-py/gguf/constants.py b/gguf-py/gguf/constants.py index f9baf687ee16..c0800463c6fd 100644 --- a/gguf-py/gguf/constants.py +++ b/gguf-py/gguf/constants.py @@ -6238,6 +6238,7 @@ class VisionProjectorType: GLM5V = "glm5v" YOUTUVL = "youtuvl" NEMOTRON_V2_VL = "nemotron_v2_vl" + NEMOTRON_OMNI = "nemotron_omni" QWEN3TTS_SPKENC = "qwen3tts_spkenc" # audio: ECAPA-TDNN speaker encoder QWEN3TTS_GEN = "qwen3tts_gen" # audio generation: code_predictor POCKETTTS_SPKENC = "pockettts_spkenc" # audio: mimi encoder as voice-prompt encoder diff --git a/gguf-py/gguf/tensor_mapping.py b/gguf-py/gguf/tensor_mapping.py index 5eab5b1fdcb2..63301cc114a9 100644 --- a/gguf-py/gguf/tensor_mapping.py +++ b/gguf-py/gguf/tensor_mapping.py @@ -1735,6 +1735,7 @@ class TensorNameMap: "vision_model.class_embedding", # llama 4 "model.vision.patch_embedding.cls_embedding", # cogvlm "vision_model.radio_model.model.patch_generator.cls_token.token", # Nemotron Nano v2 VL + "vision_model.embeddings.cls_register_token", # Nemotron Omni "model.vision_model.embeddings.class_embedding", # Deepseek-OCR ), @@ -1756,6 +1757,7 @@ class TensorNameMap: "model.vision_model.embeddings.patch_embedding", # Deepseek-OCR CLIP "siglip2.vision_model.embeddings.patch_embedding", "vision_model.radio_model.model.patch_generator.embedder", # Nemotron Nano v2 VL + "vision_model.embeddings.patch_projection", # Nemotron Omni "model.vision_tower.patch_embedder.input_proj", # gemma4 "vision_tower.patch_embed.patchifier.proj", # dots.ocr "vision_encoder.patch_embed.proj", # dots3note @@ -1788,6 +1790,7 @@ class TensorNameMap: "model.vision.patch_embedding.position_embedding", # cogvlm "visual.embeddings.position_embedding", # glm4v "vision_model.radio_model.model.patch_generator.pos_embed", # Nemotron Nano v2 VL + "vision_model.embeddings.position_embedding", # Nemotron Omni "model.vision_tower.patch_embedder.position_embedding_table", # gemma4 "vision_model.positional_embedding", # Step3-VL "model.vision_embedder.pos_embedding", # gemma4 unified @@ -1818,6 +1821,7 @@ class TensorNameMap: ), MODEL_TENSOR.V_ENC_ATTN_Q: ( + "vision_model.encoder.layer.{bid}.attention.attention.query", # Nemotron Omni "model.vision_tower.vision_model.encoder.layers.{bid}.self_attn.q_proj", # Granite4Vision "vision_tower.vision_model.encoder.layers.{bid}.self_attn.q_proj", "model.vision_tower.encoder.layers.{bid}.self_attn.q_proj", # minicpmv4_6 @@ -1846,6 +1850,7 @@ class TensorNameMap: ), MODEL_TENSOR.V_ENC_ATTN_K: ( + "vision_model.encoder.layer.{bid}.attention.attention.key", # Nemotron Omni "model.vision_tower.vision_model.encoder.layers.{bid}.self_attn.k_proj", # Granite4Vision "vision_tower.vision_model.encoder.layers.{bid}.self_attn.k_proj", "model.vision_tower.encoder.layers.{bid}.self_attn.k_proj", # minicpmv4_6 @@ -1874,6 +1879,7 @@ class TensorNameMap: ), MODEL_TENSOR.V_ENC_ATTN_V: ( + "vision_model.encoder.layer.{bid}.attention.attention.value", # Nemotron Omni "model.vision_tower.vision_model.encoder.layers.{bid}.self_attn.v_proj", # Granite4Vision "vision_tower.vision_model.encoder.layers.{bid}.self_attn.v_proj", "model.vision_tower.encoder.layers.{bid}.self_attn.v_proj", # minicpmv4_6 @@ -1894,6 +1900,7 @@ class TensorNameMap: ), MODEL_TENSOR.V_ENC_INPUT_NORM: ( + "vision_model.encoder.layer.{bid}.norm1", # Nemotron Omni "vision.blocks.{bid}.norm1", # deepseek4v "model.vision_tower.vision_model.encoder.layers.{bid}.layer_norm1", # Granite4Vision "vision_tower.vision_model.encoder.layers.{bid}.layer_norm1", @@ -1920,6 +1927,7 @@ class TensorNameMap: ), MODEL_TENSOR.V_ENC_ATTN_O: ( + "vision_model.encoder.layer.{bid}.attention.output.dense", # Nemotron Omni "vision.blocks.{bid}.attn.wo", # deepseek4v "model.vision_tower.vision_model.encoder.layers.{bid}.self_attn.out_proj", # Granite4Vision "vision_tower.vision_model.encoder.layers.{bid}.self_attn.out_proj", @@ -1952,6 +1960,7 @@ class TensorNameMap: ), MODEL_TENSOR.V_ENC_POST_ATTN_NORM: ( + "vision_model.encoder.layer.{bid}.norm2", # Nemotron Omni "vision.blocks.{bid}.norm2", # deepseek4v "model.vision_tower.vision_model.encoder.layers.{bid}.layer_norm2", # Granite4Vision "vision_tower.vision_model.encoder.layers.{bid}.layer_norm2", @@ -1979,6 +1988,7 @@ class TensorNameMap: ), MODEL_TENSOR.V_ENC_FFN_UP: ( + "vision_model.encoder.layer.{bid}.mlp.fc1", # Nemotron Omni "vision.blocks.{bid}.mlp.w1_up", # deepseek4v (split from fused w1) "vision_encoder.blocks.{bid}.mlp.fc3", # dots3note "model.vision_tower.vision_model.encoder.layers.{bid}.mlp.fc1", # Granite4Vision @@ -2016,6 +2026,7 @@ class TensorNameMap: ), MODEL_TENSOR.V_ENC_FFN_DOWN: ( + "vision_model.encoder.layer.{bid}.mlp.fc2", # Nemotron Omni "vision.blocks.{bid}.mlp.w2", # deepseek4v "vision_encoder.blocks.{bid}.mlp.fc2", # dots3note "model.vision_tower.vision_model.encoder.layers.{bid}.mlp.fc2", # Granite4Vision @@ -2074,12 +2085,14 @@ class TensorNameMap: ), MODEL_TENSOR.V_LAYER_SCALE_1: ( + "vision_model.encoder.layer.{bid}.layer_scale1", # Nemotron Omni "vision_tower.vision_model.encoder.layers.{bid}.ls1", # InternVL "model.vision_tower.encoder.layer.{bid}.lambda_1", # Intern-S1 "vision_model.transformer.resblocks.{bid}.ls_1", # Step3-VL ), MODEL_TENSOR.V_LAYER_SCALE_2: ( + "vision_model.encoder.layer.{bid}.layer_scale2", # Nemotron Omni "vision_tower.vision_model.encoder.layers.{bid}.ls2", # InternVL "model.vision_tower.encoder.layer.{bid}.lambda_2", # Intern-S1 "vision_model.transformer.resblocks.{bid}.ls_2", # Step3-VL @@ -2114,6 +2127,7 @@ class TensorNameMap: "siglip2.vision_model.post_layernorm", "model.qwen2_model.model.model.norm", # Deepseek-OCR-2 qwen2 "model.vision_tower.ln_post", # muse-glimmer + "vision_projector.vision_final_layernorm", # Nemotron Omni Reasoning V3 ), MODEL_TENSOR.V_MM_POST_NORM: ( diff --git a/src/models/dflash.cpp b/src/models/dflash.cpp index 982351dc860a..6f28d2f1db4c 100644 --- a/src/models/dflash.cpp +++ b/src/models/dflash.cpp @@ -930,7 +930,7 @@ llama_model_dflash::graph::graph(const llama_model & model, const llm_gra res->t_inp_tokens = inp_tokens; ggml_tensor * inp_anchor = nullptr; - if (model.lilicorr_pass_hidden_proj && ubatch.embd_state) { + if (model.lilicorr_pass_hidden_proj && ubatch.embd_state && n_tokens == (int64_t) hparams.dflash_block_size * ubatch.n_seqs_unq) { // LiliCorr sets inp_anchor to pass the context features to the lattice scorer auto inp = std::make_unique(n_embd, n_embd); inp->tokens = inp_tokens; @@ -1100,7 +1100,7 @@ llama_model_dflash::graph::graph(const llama_model & model, const llm_gra if (model.dflash_selector_hidden) { build_dflash2_selector(*this, model, inp_tokens); - } else if (model.lilicorr_pass_hidden_proj && ubatch.embd_state) { + } else if (model.lilicorr_pass_hidden_proj && ubatch.embd_state && n_tokens == (int64_t) hparams.dflash_block_size * ubatch.n_seqs_unq) { build_lilicorr_scores(*this, model, tok_embd, inp_anchor); } } diff --git a/src/models/nemotron-h.cpp b/src/models/nemotron-h.cpp index b14913f2a807..e9c61308feb1 100644 --- a/src/models/nemotron-h.cpp +++ b/src/models/nemotron-h.cpp @@ -109,21 +109,26 @@ void llama_model_nemotron_h::load_arch_tensors(llama_model_loader & ml) { layer.wo_b = create_tensor(tn(LLM_TENSOR_ATTN_OUT, "bias", i), {n_embd}, TENSOR_NOT_REQUIRED); } else { if (n_expert != 0) { + const auto * router_meta = ml.get_tensor_meta(tn(LLM_TENSOR_FFN_GATE_INP, "weight", i).str().c_str()); + const int64_t n_expert_layer = router_meta ? router_meta->ne[1] : n_expert; + if (n_expert_layer < hparams.n_expert_used(i) || n_expert_layer > n_expert) { + throw std::runtime_error("Invalid per-layer Nemotron expert count"); + } // Use per-layer n_ff_exp; fall back to n_ff/n_expert_used if absent (existing GGUFs). const int64_t n_ff_exp_i = hparams.n_ff_exp(i) ? (int64_t)hparams.n_ff_exp(i) : hparams.n_ff(i) / (int64_t)hparams.n_expert_used(i); const int64_t n_ff_shexp = hparams.n_ff_shexp; - layer.ffn_gate_inp = create_tensor(tn(LLM_TENSOR_FFN_GATE_INP, "weight", i), { n_embd, n_expert}, trunk_flags); - layer.ffn_exp_probs_b = create_tensor(tn(LLM_TENSOR_FFN_EXP_PROBS_B, "bias", i), {n_expert }, trunk_flags); + layer.ffn_gate_inp = create_tensor(tn(LLM_TENSOR_FFN_GATE_INP, "weight", i), { n_embd, n_expert_layer}, trunk_flags); + layer.ffn_exp_probs_b = create_tensor(tn(LLM_TENSOR_FFN_EXP_PROBS_B, "bias", i), {n_expert_layer}, trunk_flags); // MoE branch layer.ffn_latent_down = create_tensor(tn(LLM_TENSOR_FFN_LATENT_DOWN, "weight", i), {n_embd, moe_n_embd}, TENSOR_NOT_REQUIRED); layer.ffn_latent_up = create_tensor(tn(LLM_TENSOR_FFN_LATENT_UP, "weight", i), {moe_n_embd, n_embd}, TENSOR_NOT_REQUIRED); - layer.ffn_down_exps = create_tensor(tn(LLM_TENSOR_FFN_DOWN_EXPS, "weight", i), {n_ff_exp_i, moe_n_embd, n_expert}, trunk_flags); - layer.ffn_up_exps = create_tensor(tn(LLM_TENSOR_FFN_UP_EXPS, "weight", i), {moe_n_embd, n_ff_exp_i, n_expert}, trunk_flags); + layer.ffn_down_exps = create_tensor(tn(LLM_TENSOR_FFN_DOWN_EXPS, "weight", i), {n_ff_exp_i, moe_n_embd, n_expert_layer}, trunk_flags); + layer.ffn_up_exps = create_tensor(tn(LLM_TENSOR_FFN_UP_EXPS, "weight", i), {moe_n_embd, n_ff_exp_i, n_expert_layer}, trunk_flags); // Shared expert branch layer.ffn_down_shexp = create_tensor(tn(LLM_TENSOR_FFN_DOWN_SHEXP, "weight", i), {n_ff_shexp, n_embd}, trunk_flags); @@ -178,6 +183,15 @@ void llama_model_nemotron_h::load_arch_tensors(llama_model_loader & ml) { layer.ffn_up_exps = create_tensor(tn(LLM_TENSOR_FFN_UP_EXPS, "weight", i), {moe_n_embd, n_ff_exp, n_expert}, mtp_flags); layer.ffn_down_shexp = create_tensor(tn(LLM_TENSOR_FFN_DOWN_SHEXP, "weight", i), {n_ff_shexp, n_embd}, mtp_flags); layer.ffn_up_shexp = create_tensor(tn(LLM_TENSOR_FFN_UP_SHEXP, "weight", i), {n_embd, n_ff_shexp}, mtp_flags); + + layer.ffn_down_exps_s = create_tensor(tn(LLM_TENSOR_FFN_DOWN_EXPS, "scale", i), {n_expert}, mtp_flags | TENSOR_NOT_REQUIRED); + layer.ffn_up_exps_s = create_tensor(tn(LLM_TENSOR_FFN_UP_EXPS, "scale", i), {n_expert}, mtp_flags | TENSOR_NOT_REQUIRED); + layer.ffn_down_exps_in_s = create_tensor(tn(LLM_TENSOR_FFN_DOWN_EXPS, "input_scale", i), {n_expert}, mtp_flags | TENSOR_NOT_REQUIRED); + layer.ffn_up_exps_in_s = create_tensor(tn(LLM_TENSOR_FFN_UP_EXPS, "input_scale", i), {n_expert}, mtp_flags | TENSOR_NOT_REQUIRED); + layer.ffn_down_shexp_s = create_tensor(tn(LLM_TENSOR_FFN_DOWN_SHEXP, "scale", i), {1}, mtp_flags | TENSOR_NOT_REQUIRED); + layer.ffn_up_shexp_s = create_tensor(tn(LLM_TENSOR_FFN_UP_SHEXP, "scale", i), {1}, mtp_flags | TENSOR_NOT_REQUIRED); + layer.ffn_down_shexp_in_s = create_tensor(tn(LLM_TENSOR_FFN_DOWN_SHEXP, "input_scale", i), {1}, mtp_flags | TENSOR_NOT_REQUIRED); + layer.ffn_up_shexp_in_s = create_tensor(tn(LLM_TENSOR_FFN_UP_SHEXP, "input_scale", i), {1}, mtp_flags | TENSOR_NOT_REQUIRED); } } @@ -304,7 +318,7 @@ ggml_tensor * llama_model_nemotron_h::graph::build_ffn_layer(ggml_tensor * cur, nullptr, // no gate model.layers[il].ffn_down_exps, model.layers[il].ffn_exp_probs_b, - n_expert, (int64_t)hparams.n_expert_used(il), + model.layers[il].ffn_up_exps->ne[2], (int64_t)hparams.n_expert_used(il), LLM_FFN_RELU_SQR, hparams.expert_weights_norm, hparams.expert_weights_scale, LLAMA_EXPERT_GATING_FUNC_TYPE_SIGMOID, diff --git a/tools/mtmd/CMakeLists.txt b/tools/mtmd/CMakeLists.txt index 20098937ce21..82f74bdc9665 100644 --- a/tools/mtmd/CMakeLists.txt +++ b/tools/mtmd/CMakeLists.txt @@ -46,6 +46,7 @@ add_library(mtmd models/kimivl.cpp models/kimik25.cpp models/nemotron-v2-vl.cpp + models/nemotron-omni.cpp models/muse-glimmer.cpp models/llama4.cpp models/llava.cpp diff --git a/tools/mtmd/clip-impl.h b/tools/mtmd/clip-impl.h index 30cb3078916b..3d6c6f8c4801 100644 --- a/tools/mtmd/clip-impl.h +++ b/tools/mtmd/clip-impl.h @@ -498,6 +498,7 @@ enum projector_type { PROJECTOR_TYPE_YASA2, PROJECTOR_TYPE_KIMIK25, PROJECTOR_TYPE_NEMOTRON_V2_VL, + PROJECTOR_TYPE_NEMOTRON_OMNI, PROJECTOR_TYPE_HUNYUANVL, PROJECTOR_TYPE_PARAKEET, PROJECTOR_TYPE_EXAONE4_5, @@ -569,6 +570,7 @@ static std::map PROJECTOR_TYPE_NAMES = { { PROJECTOR_TYPE_YASA2, "yasa2"}, { PROJECTOR_TYPE_KIMIK25, "kimik25"}, { PROJECTOR_TYPE_NEMOTRON_V2_VL, "nemotron_v2_vl"}, + { PROJECTOR_TYPE_NEMOTRON_OMNI, "nemotron_omni"}, { PROJECTOR_TYPE_EXAONE4_5, "exaone4_5"}, { PROJECTOR_TYPE_HUNYUANVL, "hunyuanvl"}, { PROJECTOR_TYPE_MINICPMV4_6, "minicpmv4_6"}, @@ -904,11 +906,13 @@ static void clip_log_internal(enum ggml_log_level level, const char * format, .. struct clip_image_f32_batch { std::vector entries; bool is_audio = false; + bool is_video = false; clip_image_f32_batch clone() const { clip_image_f32_batch new_batch{ /* entries */ {}, /* is_audio */ is_audio, + /* is_video */ is_video, }; new_batch.entries.reserve(entries.size()); for (const auto & entry : entries) { diff --git a/tools/mtmd/clip-model.h b/tools/mtmd/clip-model.h index 657671ded548..2c4ea398ccb7 100644 --- a/tools/mtmd/clip-model.h +++ b/tools/mtmd/clip-model.h @@ -566,7 +566,7 @@ struct clip_model { // embeddings ggml_tensor * class_embedding = nullptr; ggml_tensor * patch_embeddings_0 = nullptr; - ggml_tensor * patch_embeddings_1 = nullptr; // second Conv2D kernel when we decouple Conv3D along temporal dimension (Qwen2VL) + ggml_tensor * patch_embeddings_1 = nullptr; // additional patch kernel; interpretation is model-specific (Qwen2VL, Nemotron Omni) ggml_tensor * patch_bias = nullptr; ggml_tensor * position_embeddings = nullptr; ggml_tensor * norm_embd_w = nullptr; diff --git a/tools/mtmd/clip.cpp b/tools/mtmd/clip.cpp index fccb00c3aa70..4c1185a431b9 100644 --- a/tools/mtmd/clip.cpp +++ b/tools/mtmd/clip.cpp @@ -1042,6 +1042,10 @@ static std::unique_ptr clip_get_graph_builder(clip_ctx * ctx, const { builder = std::make_unique(ctx, img); } break; + case PROJECTOR_TYPE_NEMOTRON_OMNI: + { + builder = std::make_unique(ctx, img); + } break; case PROJECTOR_TYPE_LLAMA4: { builder = std::make_unique(ctx, img); @@ -1523,6 +1527,16 @@ struct clip_model_loader { { get_u32(KEY_PROJ_SCALE_FACTOR, hparams.n_merge, false); } break; + case PROJECTOR_TYPE_NEMOTRON_OMNI: + { + hparams.ffn_op = FFN_GELU_ERF; + log_ffn_op = "gelu_erf"; + hparams.image_resize_pad = PAD_NONE; + get_u32(KEY_PROJ_SCALE_FACTOR, hparams.n_merge); + get_u32(KEY_IMAGE_MIN_PIXELS, hparams.image_min_pixels); + get_u32(KEY_IMAGE_MAX_PIXELS, hparams.image_max_pixels); + hparams.warmup_image_size = hparams.image_size; + } break; case PROJECTOR_TYPE_PARAKEET: { get_u32(KEY_AUDIO_SUBSMPL_FACTOR, hparams.subsampling_factor); @@ -3269,6 +3283,7 @@ struct clip_model_loader { model.mm_3_b = get_tensor(string_format(TN_MVLM_PROJ_MLP, 3, "bias")); } break; case PROJECTOR_TYPE_NEMOTRON_V2_VL: + case PROJECTOR_TYPE_NEMOTRON_OMNI: { model.mm_0_w = get_tensor(string_format(TN_MVLM_PROJ_MLP, 0, "weight")); model.mm_1_w = get_tensor(string_format(TN_MVLM_PROJ_MLP, 1, "weight")); @@ -4320,6 +4335,7 @@ int clip_n_output_tokens(const clip_ctx * ctx, const clip_image_f32 * img) { case PROJECTOR_TYPE_COHERE2V: case PROJECTOR_TYPE_INTERNVL: case PROJECTOR_TYPE_NEMOTRON_V2_VL: + case PROJECTOR_TYPE_NEMOTRON_OMNI: case PROJECTOR_TYPE_LLAMA4: { // both X and Y are downscaled by the scale factor @@ -5428,6 +5444,7 @@ bool clip_encode(struct clip_ctx * ctx, struct clip_encode_params * params) { case PROJECTOR_TYPE_COHERE2V: case PROJECTOR_TYPE_INTERNVL: case PROJECTOR_TYPE_NEMOTRON_V2_VL: + case PROJECTOR_TYPE_NEMOTRON_OMNI: case PROJECTOR_TYPE_QWEN2A: case PROJECTOR_TYPE_QWEN3A: case PROJECTOR_TYPE_GLMA: @@ -6149,6 +6166,7 @@ int clip_n_mmproj_embd(const struct clip_ctx * ctx) { return ctx->model.mm_3_w->ne[1]; // out_proj output dim case PROJECTOR_TYPE_INTERNVL: case PROJECTOR_TYPE_NEMOTRON_V2_VL: + case PROJECTOR_TYPE_NEMOTRON_OMNI: return ctx->model.mm_3_w->ne[1]; case PROJECTOR_TYPE_LLAMA4: return ctx->model.mm_model_proj->ne[1]; @@ -6225,6 +6243,7 @@ int clip_model_n_temporal_merge(const struct clip_ctx * ctx) { case PROJECTOR_TYPE_QWEN2VL: case PROJECTOR_TYPE_QWEN25VL: case PROJECTOR_TYPE_QWEN3VL: + case PROJECTOR_TYPE_NEMOTRON_OMNI: case PROJECTOR_TYPE_LING3VL: return 2; default: diff --git a/tools/mtmd/models/models.h b/tools/mtmd/models/models.h index adb5ede309dc..2f4676a54359 100644 --- a/tools/mtmd/models/models.h +++ b/tools/mtmd/models/models.h @@ -109,6 +109,12 @@ struct clip_graph_nemotron_v2_vl : clip_graph { ggml_cgraph * build() override; }; +struct clip_graph_nemotron_omni : clip_graph { + clip_graph_nemotron_omni(clip_ctx * ctx, const clip_image_f32 & img) : clip_graph(ctx, img) {} + ggml_cgraph * build() override; + ggml_tensor * build_inp_with_temporal_merge(); +}; + struct clip_graph_llama4 : clip_graph { clip_graph_llama4(clip_ctx * ctx, const clip_image_f32 & img) : clip_graph(ctx, img) {} ggml_cgraph * build() override; diff --git a/tools/mtmd/models/nemotron-omni.cpp b/tools/mtmd/models/nemotron-omni.cpp new file mode 100644 index 000000000000..c4b1bff40621 --- /dev/null +++ b/tools/mtmd/models/nemotron-omni.cpp @@ -0,0 +1,56 @@ +#include "models.h" + +ggml_tensor * clip_graph_nemotron_omni::build_inp_with_temporal_merge() { + if (n_batch == 1 && !img_batch->is_video) { + return build_inp(); + } + + GGML_ASSERT(n_batch == 1 || n_batch == 2); + GGML_ASSERT(model.patch_embeddings_1 != nullptr); + GGML_ASSERT(model.patch_embeddings_1->ne[2] == 6); + + ggml_tensor * inp_raw = build_inp_raw(); + if (n_batch == 1) { + // pad the last video frame inside the graph + inp_raw = ggml_concat(ctx0, inp_raw, inp_raw, 2); + } else { + inp_raw = ggml_reshape_4d(ctx0, inp_raw, img.nx(), img.ny(), 6, 1); + } + + ggml_tensor * inp = ggml_conv_2d(ctx0, model.patch_embeddings_1, inp_raw, patch_size, patch_size, 0, 0, 1, 1); + inp = ggml_reshape_3d(ctx0, inp, n_patches, n_embd, 1); + return ggml_cont(ctx0, ggml_transpose(ctx0, inp)); +} + +ggml_cgraph * clip_graph_nemotron_omni::build() { + GGML_ASSERT(model.class_embedding != nullptr); + GGML_ASSERT(model.position_embeddings != nullptr); + + const int n_registers = model.class_embedding->ne[1]; + const int n_pos = n_patches + n_registers; + + ggml_tensor * inp = build_inp_with_temporal_merge(); + + inp = ggml_add(ctx0, inp, resize_position_embeddings(GGML_SCALE_MODE_BILINEAR)); + cb(inp, "inp_pos", -1); + + inp = ggml_concat(ctx0, model.class_embedding, inp, 1); + + ggml_tensor * cur = build_vit(inp, n_pos, NORM_TYPE_NORMAL, hparams.ffn_op, nullptr, nullptr); + + cur = ggml_view_2d(ctx0, cur, + n_embd, n_patches, + ggml_row_size(cur->type, n_embd), + n_registers * ggml_row_size(cur->type, n_embd)); + + cur = build_patch_merge_permute(cur, model.hparams.n_merge); + + { + cur = build_norm(cur, model.mm_0_w, nullptr, NORM_TYPE_RMS, 1e-5, -1); + cur = build_ffn(cur, model.mm_1_w, nullptr, nullptr, nullptr, model.mm_3_w, nullptr, FFN_RELU_SQR, -1); + } + + ggml_build_forward_expand(gf, cur); + + return gf; +} diff --git a/tools/mtmd/mtmd.cpp b/tools/mtmd/mtmd.cpp index 2c2b069361d5..2fa02f83eb3f 100644 --- a/tools/mtmd/mtmd.cpp +++ b/tools/mtmd/mtmd.cpp @@ -898,6 +898,12 @@ struct mtmd_context { { image_preproc = std::make_unique(ctx_v); } break; + case PROJECTOR_TYPE_NEMOTRON_OMNI: + { + img_beg = ""; + img_end = ""; + image_preproc = std::make_unique(ctx_v); + } break; case PROJECTOR_TYPE_LFM2: { // multi-tile: @@ -1607,6 +1613,7 @@ struct mtmd_tokenizer { clip_image_f32_batch batch_f32; batch_f32.is_audio = false; + batch_f32.is_video = bitmaps[0]->mergeable; batch_f32.entries = std::move(preproc_out.entries); // do NOT use preproc_out from this point on, it's moved