From 12750a76f19563c1931785d754100bc099571969 Mon Sep 17 00:00:00 2001 From: Gabe Goodhart Date: Wed, 6 May 2026 09:11:58 -0600 Subject: [PATCH 01/94] feat(convert): Get language model conversion working for 4.1 vision Branch: Granite4Vision AI-usage: none Signed-off-by: Gabe Goodhart --- convert_hf_to_gguf.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/convert_hf_to_gguf.py b/convert_hf_to_gguf.py index 5cff86848565..ff185b171ebc 100755 --- a/convert_hf_to_gguf.py +++ b/convert_hf_to_gguf.py @@ -11074,8 +11074,14 @@ def set_gguf_parameters(self): @classmethod def filter_tensors(cls, item: tuple[str, Callable[[], Tensor]]) -> tuple[str, Callable[[], Tensor]] | None: name, gen = item - if name.startswith("encoder."): - return None + # Skip multimodal tensors + if ( + name.startswith(("encoder.")) or + "image_" in name or + "layerwise_projectors" in name or + "spatial_projectors" in name + ): + return return super().filter_tensors(item) From 47dd3b1bafec770084d16130d0fed9f038af9e8d Mon Sep 17 00:00:00 2001 From: Gabe Goodhart Date: Tue, 5 May 2026 15:51:28 -0600 Subject: [PATCH 02/94] feat(convert): Skip multimodal tensors for GraniteMoeHybrid (vision 4.0) Branch: Granite4Vision AI-usage: none Signed-off-by: Gabe Goodhart --- convert_hf_to_gguf.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/convert_hf_to_gguf.py b/convert_hf_to_gguf.py index ff185b171ebc..dea5a9806893 100755 --- a/convert_hf_to_gguf.py +++ b/convert_hf_to_gguf.py @@ -11227,6 +11227,22 @@ def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iter elif bid in self._attn_layers: yield from GraniteMoeModel.modify_tensors(self, data_torch, name, bid) return + + # Skip multimodal tensors + if ( + name.startswith(("encoder.", "projector.")) or + "image_" in name or + "layerwise_projectors" in name or + "spatial_projectors" in name or + "vision_" in name + ): + return + + # De-alias language model tensors for multimodal models + for lm_prefix in ["language_model.", "model.language_model."]: + if name.startswith(lm_prefix): + name = name[len(lm_prefix):] + yield from ModelBase.modify_tensors(self, data_torch, name, bid) def set_gguf_parameters(self): From b3a6914a67fee8f9768f6fd5faca4a0524cbad10 Mon Sep 17 00:00:00 2001 From: Gabe Goodhart Date: Wed, 6 May 2026 09:24:37 -0600 Subject: [PATCH 03/94] fix: Disable vocab padding for non-hybrid models that use GraniteMoeHybrid Branch: Granite4Vision AI-usage: none Signed-off-by: Gabe Goodhart --- convert_hf_to_gguf.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/convert_hf_to_gguf.py b/convert_hf_to_gguf.py index dea5a9806893..2eb8243f1d96 100755 --- a/convert_hf_to_gguf.py +++ b/convert_hf_to_gguf.py @@ -11172,6 +11172,10 @@ def __init__(self, *args, **kwargs): self.gguf_writer.arch = gguf.MODEL_ARCH_NAMES[new_arch] self.gguf_writer.add_architecture() + # If not explicitly enabled, disable vocab padding for non-hybrid + # models + self.hparams.setdefault("pad_vocab_size_multiple", 1) + # n_group and d_inner are used during reshape_tensors for mamba2 # NOTE: Explicitly include hparam prefix prefix for d_model to # disambiguate with top-level head_dim From f83418c673999559efc07d073c294e82bae151e7 Mon Sep 17 00:00:00 2001 From: Gabe Goodhart Date: Thu, 7 May 2026 16:26:26 -0600 Subject: [PATCH 04/94] feat: Plumb python-side vision projector names and mappings There are several awkward things here: 1. Most of these are essentially identical to the audio qformer tensors. On the c++ side, that's mapped using the prefix, so the rest of the GGUF name needs to align, but on the python side there's no prefix notion, so they all get duplicated. 2. There are a couple of net-new tensors for vision, in particular PROJ_NORM. In both speech and vision, the QF_PROJ_NORM is qualified as belonging to the qformer portion, but the GGUF name is simply proj_norm which conflicts with the ideal name for this new PROJ_NORM that is not qualified as part of the qformer. To get around this, I used "proj_layernorm" as the GGUF name. Branch: Granite4Vision AI-usage: none Signed-off-by: Gabe Goodhart --- gguf-py/gguf/constants.py | 62 ++++++++++++++++++++++- gguf-py/gguf/tensor_mapping.py | 92 ++++++++++++++++++++++++++++++++++ 2 files changed, 152 insertions(+), 2 deletions(-) diff --git a/gguf-py/gguf/constants.py b/gguf-py/gguf/constants.py index 4055ec2873aa..b5ce2f1d5ca9 100644 --- a/gguf-py/gguf/constants.py +++ b/gguf-py/gguf/constants.py @@ -811,6 +811,26 @@ class MODEL_TENSOR(IntEnum): V_ENC_EMBD_IMGNL = auto() # Deepseek-OCR V_ENC_EMBD_VSEP = auto() # Deepseek-OCR + # qformer projector (vision) - Granite4 Vision + V_QF_PROJ_QUERY = auto() + V_QF_PROJ_NORM = auto() + V_QF_PROJ_LINEAR = auto() + V_QF_SELF_ATTN_Q = auto() + V_QF_SELF_ATTN_K = auto() + V_QF_SELF_ATTN_V = auto() + V_QF_SELF_ATTN_O = auto() + V_QF_SELF_ATTN_NORM = auto() + V_QF_CROSS_ATTN_Q = auto() + V_QF_CROSS_ATTN_K = auto() + V_QF_CROSS_ATTN_V = auto() + V_QF_CROSS_ATTN_O = auto() + V_QF_CROSS_ATTN_NORM = auto() + V_QF_FFN_UP = auto() + V_QF_FFN_DOWN = auto() + V_QF_FFN_NORM = auto() + V_IMG_POS = auto() + V_PROJ_NORM = auto() + # audio (mtmd) A_ENC_EMBD_POS = auto() A_ENC_EMBD_NORM = auto() @@ -875,7 +895,7 @@ class MODEL_TENSOR(IntEnum): A_CTC_OUT = auto() A_CTC_OUT_MID = auto() A_ENC_ATTN_REL_POS_EMB = auto() - # qformer projector + # audio qformer projector A_QF_PROJ_QUERY = auto() A_QF_PROJ_NORM = auto() A_QF_PROJ_LINEAR = auto() @@ -1323,8 +1343,27 @@ class MODEL_TENSOR(IntEnum): MODEL_TENSOR.V_SAM_NECK: "v.sam.neck.{bid}", MODEL_TENSOR.V_SAM_NET_2: "v.sam.net_2", MODEL_TENSOR.V_SAM_NET_3: "v.sam.net_3", - MODEL_TENSOR.V_ENC_EMBD_IMGNL: "v.image_newline", # Deepseek-OCR + MODEL_TENSOR.V_ENC_EMBD_IMGNL: "v.image_newline", # Deepseek-OCR, Granite4Vision MODEL_TENSOR.V_ENC_EMBD_VSEP: "v.view_seperator", # Deepseek-OCR + # Granite4 Vision + MODEL_TENSOR.V_IMG_POS: "v.proj_img_pos", + MODEL_TENSOR.V_PROJ_NORM: "v.proj_layernorm", + MODEL_TENSOR.V_QF_PROJ_QUERY: "v.proj_query", + MODEL_TENSOR.V_QF_PROJ_NORM: "v.proj_norm", + MODEL_TENSOR.V_QF_PROJ_LINEAR: "v.proj_linear", + MODEL_TENSOR.V_QF_SELF_ATTN_Q: "v.proj_blk.{bid}.self_attn_q", + MODEL_TENSOR.V_QF_SELF_ATTN_K: "v.proj_blk.{bid}.self_attn_k", + MODEL_TENSOR.V_QF_SELF_ATTN_V: "v.proj_blk.{bid}.self_attn_v", + MODEL_TENSOR.V_QF_SELF_ATTN_O: "v.proj_blk.{bid}.self_attn_out", + MODEL_TENSOR.V_QF_SELF_ATTN_NORM: "v.proj_blk.{bid}.self_attn_norm", + MODEL_TENSOR.V_QF_CROSS_ATTN_Q: "v.proj_blk.{bid}.cross_attn_q", + MODEL_TENSOR.V_QF_CROSS_ATTN_K: "v.proj_blk.{bid}.cross_attn_k", + MODEL_TENSOR.V_QF_CROSS_ATTN_V: "v.proj_blk.{bid}.cross_attn_v", + MODEL_TENSOR.V_QF_CROSS_ATTN_O: "v.proj_blk.{bid}.cross_attn_out", + MODEL_TENSOR.V_QF_CROSS_ATTN_NORM: "v.proj_blk.{bid}.cross_attn_norm", + MODEL_TENSOR.V_QF_FFN_UP: "v.proj_blk.{bid}.ffn_up", + MODEL_TENSOR.V_QF_FFN_DOWN: "v.proj_blk.{bid}.ffn_down", + MODEL_TENSOR.V_QF_FFN_NORM: "v.proj_blk.{bid}.ffn_norm", # audio (mtmd) # note: all audio tensor names must use prefix "a." or "mm.a." MODEL_TENSOR.A_ENC_EMBD_POS: "a.position_embd", @@ -1503,6 +1542,25 @@ class MODEL_TENSOR(IntEnum): MODEL_TENSOR.V_SAM_NECK, MODEL_TENSOR.V_SAM_NET_2, MODEL_TENSOR.V_SAM_NET_3, + MODEL_TENSOR.V_IMG_POS, + MODEL_TENSOR.V_PROJ_NORM, + MODEL_TENSOR.V_QF_PROJ_QUERY, + MODEL_TENSOR.V_QF_PROJ_NORM, + MODEL_TENSOR.V_QF_PROJ_LINEAR, + MODEL_TENSOR.V_QF_SELF_ATTN_Q, + MODEL_TENSOR.V_QF_SELF_ATTN_K, + MODEL_TENSOR.V_QF_SELF_ATTN_V, + MODEL_TENSOR.V_QF_SELF_ATTN_O, + MODEL_TENSOR.V_QF_SELF_ATTN_NORM, + MODEL_TENSOR.V_QF_CROSS_ATTN_Q, + MODEL_TENSOR.V_QF_CROSS_ATTN_K, + MODEL_TENSOR.V_QF_CROSS_ATTN_V, + MODEL_TENSOR.V_QF_CROSS_ATTN_O, + MODEL_TENSOR.V_QF_CROSS_ATTN_NORM, + MODEL_TENSOR.V_QF_FFN_UP, + MODEL_TENSOR.V_QF_FFN_DOWN, + MODEL_TENSOR.V_QF_FFN_NORM, + MODEL_TENSOR.V_QF_PROJ_NORM, # audio MODEL_TENSOR.A_ENC_EMBD_POS, MODEL_TENSOR.A_ENC_EMBD_NORM, diff --git a/gguf-py/gguf/tensor_mapping.py b/gguf-py/gguf/tensor_mapping.py index f40cb828201e..be3261b5b3c2 100644 --- a/gguf-py/gguf/tensor_mapping.py +++ b/gguf-py/gguf/tensor_mapping.py @@ -155,6 +155,10 @@ class TensorNameMap: MODEL_TENSOR.V_ENC_MSFA_NORM: ( "model.vision_tower.timm_model.msfa.norm", # gemma3n ), + # TODO: Should the rest of the block mappings be moved here? + MODEL_TENSOR.V_ENC_EMBD_IMGNL: ( + "model.image_newline", # Granite4Vision + ), MODEL_TENSOR.A_CTC_OUT: ( "encoder.out", ), @@ -170,6 +174,28 @@ class TensorNameMap: MODEL_TENSOR.A_QF_PROJ_LINEAR: ( "projector.linear", ), + + # NOTE: For deepstack versions, projectors are stacked + MODEL_TENSOR.V_IMG_POS: ( + "model.layerwise_projectors.image_positions", # Granite4 Vision + "model.spatial_projectors.image_positions", # Granite4 Vision + ), + MODEL_TENSOR.V_PROJ_NORM: ( + "model.layerwise_projectors.norm", # Granite4 Vision + "model.spatial_projectors.norm", # Granite4 Vision + ), + MODEL_TENSOR.V_QF_PROJ_LINEAR: ( + "model.layerwise_projectors.out_linear", # Granite4 Vision + "model.spatial_projectors.out_linear", # Granite4 Vision + ), + MODEL_TENSOR.V_QF_PROJ_QUERY: ( + "model.layerwise_projectors.query", # Granite4 Vision + "model.spatial_projectors.query", # Granite4 Vision + ), + MODEL_TENSOR.V_QF_PROJ_NORM: ( + "model.layerwise_projectors.qformer.layernorm", # Granite4 Vision + "model.spatial_projectors.qformer.layernorm", # Granite4 Vision + ), } block_mappings_cfg: dict[MODEL_TENSOR, tuple[str, ...]] = { @@ -1440,6 +1466,7 @@ class TensorNameMap: "vision_model.positional_embedding", # Step3-VL ), + # TODO: I think these should all be moved to mapping_cfg? MODEL_TENSOR.V_ENC_EMBD_IMGNL: ( "model.image_newline", # Deepseek-OCR "vit.perceive.image_newline", # HunyuanOCR @@ -1918,6 +1945,71 @@ class TensorNameMap: "model.vision_tower.std_scale", # gemma4 ), + MODEL_TENSOR.V_QF_SELF_ATTN_Q: ( + "model.layerwise_projectors.qformer.encoder.layer.{bid}.attention.attention.query", # Granite4 Vision + "model.spatial_projectors.qformer.encoder.layer.{bid}.attention.attention.query", # Granite4 Vision + ), + + MODEL_TENSOR.V_QF_SELF_ATTN_K: ( + "model.layerwise_projectors.qformer.encoder.layer.{bid}.attention.attention.key", # Granite4 Vision + "model.spatial_projectors.qformer.encoder.layer.{bid}.attention.attention.key", # Granite4 Vision + ), + + MODEL_TENSOR.V_QF_SELF_ATTN_V: ( + "model.layerwise_projectors.qformer.encoder.layer.{bid}.attention.attention.value", # Granite4 Vision + "model.spatial_projectors.qformer.encoder.layer.{bid}.attention.attention.value", # Granite4 Vision + ), + + MODEL_TENSOR.V_QF_SELF_ATTN_O: ( + "model.layerwise_projectors.qformer.encoder.layer.{bid}.attention.output.dense", # Granite4 Vision + "model.spatial_projectors.qformer.encoder.layer.{bid}.attention.output.dense", # Granite4 Vision + ), + + MODEL_TENSOR.V_QF_SELF_ATTN_NORM: ( + "model.layerwise_projectors.qformer.encoder.layer.{bid}.attention.output.LayerNorm", # Granite4 Vision + "model.spatial_projectors.qformer.encoder.layer.{bid}.attention.output.LayerNorm", # Granite4 Vision + ), + + MODEL_TENSOR.V_QF_CROSS_ATTN_Q: ( + "model.layerwise_projectors.qformer.encoder.layer.{bid}.crossattention.attention.query", # Granite4 Vision + "model.spatial_projectors.qformer.encoder.layer.{bid}.crossattention.attention.query", # Granite4 Vision + ), + + MODEL_TENSOR.V_QF_CROSS_ATTN_K: ( + "model.layerwise_projectors.qformer.encoder.layer.{bid}.crossattention.attention.key", # Granite4 Vision + "model.spatial_projectors.qformer.encoder.layer.{bid}.crossattention.attention.key", # Granite4 Vision + ), + + MODEL_TENSOR.V_QF_CROSS_ATTN_V: ( + "model.layerwise_projectors.qformer.encoder.layer.{bid}.crossattention.attention.value", # Granite4 Vision + "model.spatial_projectors.qformer.encoder.layer.{bid}.crossattention.attention.value", # Granite4 Vision + ), + + MODEL_TENSOR.V_QF_CROSS_ATTN_O: ( + "model.layerwise_projectors.qformer.encoder.layer.{bid}.crossattention.output.dense", # Granite4 Vision + "model.spatial_projectors.qformer.encoder.layer.{bid}.crossattention.output.dense", # Granite4 Vision + ), + + MODEL_TENSOR.V_QF_CROSS_ATTN_NORM: ( + "model.layerwise_projectors.qformer.encoder.layer.{bid}.crossattention.output.LayerNorm", # Granite4 Vision + "model.spatial_projectors.qformer.encoder.layer.{bid}.crossattention.output.LayerNorm", # Granite4 Vision + ), + + MODEL_TENSOR.V_QF_FFN_UP: ( + "model.layerwise_projectors.qformer.encoder.layer.{bid}.intermediate_query.dense", # Granite4 Vision + "model.spatial_projectors.qformer.encoder.layer.{bid}.intermediate_query.dense", # Granite4 Vision + ), + + MODEL_TENSOR.V_QF_FFN_DOWN: ( + "model.layerwise_projectors.qformer.encoder.layer.{bid}.output_query.dense", # Granite4 Vision + "model.spatial_projectors.qformer.encoder.layer.{bid}.output_query.dense", # Granite4 Vision + ), + + MODEL_TENSOR.V_QF_FFN_NORM: ( + "model.layerwise_projectors.qformer.encoder.layer.{bid}.output_query.LayerNorm", # Granite4 Vision + "model.spatial_projectors.qformer.encoder.layer.{bid}.output_query.LayerNorm", # Granite4 Vision + ), + # audio (mtmd) MODEL_TENSOR.A_ENC_EMBD_POS: ( From 5b23f8027aebfbde44233d1ffa27a683e2674fd3 Mon Sep 17 00:00:00 2001 From: Gabe Goodhart Date: Thu, 7 May 2026 16:27:07 -0600 Subject: [PATCH 05/94] feat: Add python side architecture name Branch: Granite4Vision AI-usage: none Signed-off-by: Gabe Goodhart --- gguf-py/gguf/constants.py | 1 + 1 file changed, 1 insertion(+) diff --git a/gguf-py/gguf/constants.py b/gguf-py/gguf/constants.py index b5ce2f1d5ca9..dd5b66df9c38 100644 --- a/gguf-py/gguf/constants.py +++ b/gguf-py/gguf/constants.py @@ -4322,6 +4322,7 @@ class VisionProjectorType: MINICPMV4_6 = "minicpmv4_6" GRANITE_SPEECH = "granite_speech" # audio MIMOVL = "mimovl" + GRANITE4_VISION = "granite4_vision" # Items here are (block size, type size) From a176cbf42b03606e0510c9bdbe20ba1b02eba397 Mon Sep 17 00:00:00 2001 From: Gabe Goodhart Date: Thu, 7 May 2026 16:27:52 -0600 Subject: [PATCH 06/94] feat: Add python-side plumbing for setting FEATURE_LAYERS hparam Branch: Granite4Vision AI-usage: none Signed-off-by: Gabe Goodhart --- gguf-py/gguf/constants.py | 1 + gguf-py/gguf/gguf_writer.py | 3 +++ 2 files changed, 4 insertions(+) diff --git a/gguf-py/gguf/constants.py b/gguf-py/gguf/constants.py index dd5b66df9c38..93dbca0eefc0 100644 --- a/gguf-py/gguf/constants.py +++ b/gguf-py/gguf/constants.py @@ -321,6 +321,7 @@ class ClipVision: WA_PATTERN_MODE = "clip.vision.wa_pattern_mode" # used by mimovl, per-layer -1/0/1 IS_DEEPSTACK_LAYERS = "clip.vision.is_deepstack_layers" WINDOW_SIZE = "clip.vision.window_size" + FEATURE_LAYERS = "clip.vision.feature_layer" # Granite4 Vision class Attention: HEAD_COUNT = "clip.vision.attention.head_count" diff --git a/gguf-py/gguf/gguf_writer.py b/gguf-py/gguf/gguf_writer.py index a101382719d0..42149b0b4831 100644 --- a/gguf-py/gguf/gguf_writer.py +++ b/gguf-py/gguf/gguf_writer.py @@ -1231,6 +1231,9 @@ def add_vision_wa_pattern_mode(self, modes: Sequence[int]) -> None: def add_vision_window_size(self, value: int) -> None: self.add_uint32(Keys.ClipVision.WINDOW_SIZE, value) + def add_vision_feature_layers(self, layers: Sequence[int]) -> None: + self.add_array(Keys.ClipVision.FEATURE_LAYERS, layers) + def add_vision_sam_layers_count(self, value: int) -> None: self.add_uint32(Keys.ClipVision.SAM.BLOCK_COUNT, value) From 79412a4be6a7e0ca3c34b548b56bf04027c828c3 Mon Sep 17 00:00:00 2001 From: Gabe Goodhart Date: Thu, 7 May 2026 16:28:50 -0600 Subject: [PATCH 07/94] feat: Add c++ side tensor naming defines NOTE: Usage of these hasn't been updated to include prefix yet Branch: Granite4Vision AI-usage: none Signed-off-by: Gabe Goodhart --- tools/mtmd/clip-impl.h | 33 +++++++++++++++++---------------- 1 file changed, 17 insertions(+), 16 deletions(-) diff --git a/tools/mtmd/clip-impl.h b/tools/mtmd/clip-impl.h index 8e09f26e988e..439e90063c37 100644 --- a/tools/mtmd/clip-impl.h +++ b/tools/mtmd/clip-impl.h @@ -207,22 +207,23 @@ #define TN_CTC_OUT_MID "a.enc_ctc_out_mid.%s" #define TN_ATTN_REL_POS_EMB "%s.blk.%d.attn_rel_pos_emb" // qformer projector -#define TN_QF_PROJ_QUERY "a.proj_query" -#define TN_QF_PROJ_NORM "a.proj_norm.%s" -#define TN_QF_PROJ_LINEAR "a.proj_linear.%s" -#define TN_QF_SELF_ATTN_Q "a.proj_blk.%d.self_attn_q.%s" -#define TN_QF_SELF_ATTN_K "a.proj_blk.%d.self_attn_k.%s" -#define TN_QF_SELF_ATTN_V "a.proj_blk.%d.self_attn_v.%s" -#define TN_QF_SELF_ATTN_O "a.proj_blk.%d.self_attn_out.%s" -#define TN_QF_SELF_ATTN_N "a.proj_blk.%d.self_attn_norm.%s" -#define TN_QF_CROSS_ATTN_Q "a.proj_blk.%d.cross_attn_q.%s" -#define TN_QF_CROSS_ATTN_K "a.proj_blk.%d.cross_attn_k.%s" -#define TN_QF_CROSS_ATTN_V "a.proj_blk.%d.cross_attn_v.%s" -#define TN_QF_CROSS_ATTN_O "a.proj_blk.%d.cross_attn_out.%s" -#define TN_QF_CROSS_ATTN_N "a.proj_blk.%d.cross_attn_norm.%s" -#define TN_QF_FFN_UP "a.proj_blk.%d.ffn_up.%s" -#define TN_QF_FFN_DOWN "a.proj_blk.%d.ffn_down.%s" -#define TN_QF_FFN_NORM "a.proj_blk.%d.ffn_norm.%s" +#define TN_PROJ_LAYERNORM "v.proj_layernorm.%s" // NOTE: Disambig with QF_PROJ_NORM +#define TN_QF_PROJ_QUERY "%s.proj_query" +#define TN_QF_PROJ_NORM "%s.proj_norm.%s" // NOTE: This _should_ be qf_proj_norm +#define TN_QF_PROJ_LINEAR "%s.proj_linear.%s" +#define TN_QF_SELF_ATTN_Q "%s.proj_blk.%d.self_attn_q.%s" +#define TN_QF_SELF_ATTN_K "%s.proj_blk.%d.self_attn_k.%s" +#define TN_QF_SELF_ATTN_V "%s.proj_blk.%d.self_attn_v.%s" +#define TN_QF_SELF_ATTN_O "%s.proj_blk.%d.self_attn_out.%s" +#define TN_QF_SELF_ATTN_N "%s.proj_blk.%d.self_attn_norm.%s" +#define TN_QF_CROSS_ATTN_Q "%s.proj_blk.%d.cross_attn_q.%s" +#define TN_QF_CROSS_ATTN_K "%s.proj_blk.%d.cross_attn_k.%s" +#define TN_QF_CROSS_ATTN_V "%s.proj_blk.%d.cross_attn_v.%s" +#define TN_QF_CROSS_ATTN_O "%s.proj_blk.%d.cross_attn_out.%s" +#define TN_QF_CROSS_ATTN_N "%s.proj_blk.%d.cross_attn_norm.%s" +#define TN_QF_FFN_UP "%s.proj_blk.%d.ffn_up.%s" +#define TN_QF_FFN_DOWN "%s.proj_blk.%d.ffn_down.%s" +#define TN_QF_FFN_NORM "%s.proj_blk.%d.ffn_norm.%s" // gemma4 audio conformer #define TN_A_MM_INP_PROJ "mm.a.input_projection.%s" From 623ea2ba30ac1387d6c6eb2f6a0cf758915aafb7 Mon Sep 17 00:00:00 2001 From: Gabe Goodhart Date: Thu, 7 May 2026 16:30:11 -0600 Subject: [PATCH 08/94] feat(mtmd): Convert vision_feature_layer to an ordered vector We need to preserve the ordering of these feature index values so that they can be mapped to the sub-tensors within the stacked projectors. Branch: Granite4Vision AI-usage: none Signed-off-by: Gabe Goodhart --- tools/mtmd/clip-model.h | 2 +- tools/mtmd/clip.cpp | 7 +------ tools/mtmd/models/llava.cpp | 4 ++-- 3 files changed, 4 insertions(+), 9 deletions(-) diff --git a/tools/mtmd/clip-model.h b/tools/mtmd/clip-model.h index ce15dbcd11e8..a97e8e3996bf 100644 --- a/tools/mtmd/clip-model.h +++ b/tools/mtmd/clip-model.h @@ -80,7 +80,7 @@ struct clip_hparams { float eps = 1e-6; float rope_theta = 0.0; - std::unordered_set vision_feature_layer; + std::vector vision_feature_layer; // Ordered list of vision layers where features are extracted int32_t attn_window_size = 0; int32_t n_wa_pattern = 0; std::unordered_set wa_layer_indexes; // explicit layer indexes that use full attention (for irregular patterns like YoutuVL) diff --git a/tools/mtmd/clip.cpp b/tools/mtmd/clip.cpp index 2e0cfa61f4b9..1fc0efd5f3f1 100644 --- a/tools/mtmd/clip.cpp +++ b/tools/mtmd/clip.cpp @@ -1205,12 +1205,7 @@ struct clip_model_loader { // to form the final visual features. // NOTE: gguf conversions should standardize the values of the vision feature layer to // be non-negative, since we use -1 to mark values as unset here. - std::vector vision_feature_layer; - get_arr_int(KEY_FEATURE_LAYER, vision_feature_layer, false); - // convert std::vector to std::unordered_set - for (auto & layer : vision_feature_layer) { - hparams.vision_feature_layer.insert(layer); - } + get_arr_int(KEY_FEATURE_LAYER, hparams.vision_feature_layer, false); // model-specific params switch (model.proj_type) { diff --git a/tools/mtmd/models/llava.cpp b/tools/mtmd/models/llava.cpp index 4af17ccfe853..bf280c44e00f 100644 --- a/tools/mtmd/models/llava.cpp +++ b/tools/mtmd/models/llava.cpp @@ -60,7 +60,7 @@ ggml_cgraph * clip_graph_llava::build() { // If this is an embedding feature layer, save the output. // NOTE: 0 index here refers to the input to the encoder. - if (vision_feature_layer.find(il) != vision_feature_layer.end()) { + if (std::find(vision_feature_layer.begin(), vision_feature_layer.end(), il) != vision_feature_layer.end()) { embedding_stack.push_back(cur); } @@ -135,7 +135,7 @@ ggml_cgraph * clip_graph_llava::build() { // process vision feature layers (used by granite) { // final layer is a vision feature layer - if (vision_feature_layer.find(max_feature_layer) != vision_feature_layer.end()) { + if (std::find(vision_feature_layer.begin(), vision_feature_layer.end(), max_feature_layer) != vision_feature_layer.end()) { embedding_stack.push_back(inpL); } From 7dda78fb6846db9f3e94e7341aa721c66ca304b0 Mon Sep 17 00:00:00 2001 From: Gabe Goodhart Date: Thu, 7 May 2026 16:30:54 -0600 Subject: [PATCH 09/94] feat(mtmd): Add architecture label plumbing Branch: Granite4Vision AI-usage: full (OpenCode + qwen3.5:122b) Signed-off-by: Gabe Goodhart --- tools/mtmd/clip-impl.h | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tools/mtmd/clip-impl.h b/tools/mtmd/clip-impl.h index 439e90063c37..1e024979cb1f 100644 --- a/tools/mtmd/clip-impl.h +++ b/tools/mtmd/clip-impl.h @@ -349,6 +349,7 @@ enum projector_type { PROJECTOR_TYPE_MINICPMV4_6, PROJECTOR_TYPE_GRANITE_SPEECH, PROJECTOR_TYPE_MIMOVL, + PROJECTOR_TYPE_GRANITE4_VISION, PROJECTOR_TYPE_UNKNOWN, }; @@ -399,6 +400,7 @@ static std::map PROJECTOR_TYPE_NAMES = { { PROJECTOR_TYPE_MINICPMV4_6, "minicpmv4_6"}, { PROJECTOR_TYPE_GRANITE_SPEECH, "granite_speech"}, { PROJECTOR_TYPE_MIMOVL, "mimovl"}, + { PROJECTOR_TYPE_GRANITE4_VISION, "granite4_vision"}, }; static projector_type clip_projector_type_from_string(const std::string & str) { From 5e6184f3f7b24f0f0ddf38a4f3c98c17552042b8 Mon Sep 17 00:00:00 2001 From: Gabe Goodhart Date: Thu, 7 May 2026 16:32:23 -0600 Subject: [PATCH 10/94] feat(wip): Add partial conversion for mmproj This handles stacking the projector tensors and setting the new harams Branch: Granite4Vision AI-usage: none Signed-off-by: Gabe Goodhart --- convert_hf_to_gguf.py | 125 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 125 insertions(+) diff --git a/convert_hf_to_gguf.py b/convert_hf_to_gguf.py index 2eb8243f1d96..5415132cc763 100755 --- a/convert_hf_to_gguf.py +++ b/convert_hf_to_gguf.py @@ -13068,6 +13068,131 @@ def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iter yield from super().modify_tensors(data_torch, name, bid) +@ModelBase.register("Granite4VisionForConditionalGeneration") +class Granite4VisionMmprojModel(MmprojModel): + has_vision_encoder = True + has_audio_encoder = False + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + + deepstack_map = self.global_config.get("deepstack_layer_map", []) # [[vis_layer, llm_layer], ...] + spatial_layers = self.global_config.get("spatial_target_layers", []) # [llm_layer, ...] + n_text_layers = self.global_config["text_config"]["num_hidden_layers"] + n_vision_layers = self.global_config["vision_config"]["num_hidden_layers"] + + # The tensors for the individual per-layer injection projectors are held + # separately in safetensors. We merge them here, so this is used to + # accumulate the projector tensors in the order that will correspond + # with the layer mapping in deepstack_layer_arr. + self._projector_tensors = {} + self._deepstack_layer_arr = [-1 for _ in range(n_text_layers)] # Populate with -1 sentinels + + # Normalize both deepstack and spatial projector maps to the form: + # (vision_layer, llm_layer, , type_index) + # + # This is then used to populate the following mappings: + # + # - vision_feature_layers: ordered list of all vision_layer values where + # order corresponds with the order of the stacked projector tensors + # NOTE: Values may appear multiple times for spatial projectors + # + # - deepstack_layer_arr: per-text-layer array indicating which input + # vision feature should be injected at that layer (-1 if none) + # + # - tensor_prefix_map: mapping from tensor prefixes to the index of the + # corresponding projector in the stacked tensors + normalized_projector_map = [] + if deepstack_map: + for deepstack_idx, (vision_layer, llm_layer) in enumerate(sorted(deepstack_map)): + if vision_layer < 0: + vision_layer = n_vision_layers + vision_layer + if llm_layer < 0: + llm_layer = n_text_layers + llm_layer + normalized_projector_map.append((vision_layer, llm_layer, "layerwise", deepstack_idx)) + if spatial_layers: + spatial_vision_layer = self.global_config.get("spatial_vision_layer", -1) + if spatial_vision_layer < 0: + spatial_vision_layer = n_vision_layers + spatial_vision_layer + for spatial_idx, llm_layer in enumerate(spatial_layers): + normalized_projector_map.append((spatial_vision_layer, llm_layer, "spatial", spatial_idx)) + + normalized_projector_map = list(sorted(normalized_projector_map)) + self._n_proj = len(normalized_projector_map) + + self._tensor_prefix_map = { + f"model.{proj_type}_projectors.{type_idx}": proj_idx + for proj_idx, (_, _, proj_type, type_idx) in enumerate(normalized_projector_map) + } + self._vision_feature_layers = [vision_layer for vision_layer, _, _, _ in normalized_projector_map] + for proj_idx, (_, llm_layer, _, _) in enumerate(normalized_projector_map): + self._deepstack_layer_arr[llm_layer] = proj_idx + + def set_gguf_parameters(self): + assert self.hparams_vision is not None + v = self.hparams_vision + + super().set_gguf_parameters() + + # Set projector type + self.gguf_writer.add_clip_projector_type(gguf.VisionProjectorType.GRANITE4_VISION) + + # Vision encoder params (SigLIP) + self.gguf_writer.add_vision_embedding_length(v["hidden_size"]) + self.gguf_writer.add_vision_feed_forward_length(v["intermediate_size"]) + self.gguf_writer.add_vision_block_count(v["num_hidden_layers"]) + self.gguf_writer.add_vision_head_count(v["num_attention_heads"]) + self.gguf_writer.add_vision_image_size(v["image_size"]) + self.gguf_writer.add_vision_patch_size(v["patch_size"]) + + # QFormer projector config + ds_rate = self.global_config["downsample_rate"] + ds_parts = ds_rate.split("/") + assert len(ds_parts) == 2, f"Invalid 'downsample_rate' value: {ds_rate}" + query_side, window_side = [int(p) for p in ds_parts] + self.gguf_writer.add_vision_projector_query_side(query_side) + self.gguf_writer.add_vision_projector_window_side(window_side) + + # Set vision feature layers + self.gguf_writer.add_vision_feature_layers(self._vision_feature_layers) + + # Write array of deepstack layers (llm_layer -> projector_idx) + self.gguf_writer.add_num_deepstack_layers(self._deepstack_layer_arr) + + @classmethod + def filter_tensors(cls, item: tuple[str, Callable[[], Tensor]]) -> tuple[str, Callable[[], Tensor]] | None: + name, _ = item + if name.startswith("language_model."): + return None + return super().filter_tensors(item) + + def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]: + + # Detect projector tensors and bin them + projector_idx = None + proj_tensor_type = None + for prefix, proj_idx in self._tensor_prefix_map.items(): + if name.startswith(prefix): + projector_idx = proj_idx + proj_tensor_type = name[len(prefix):].lstrip(".") + break + if projector_idx is not None: + # Add to the stacked tensor mappings + tensor_stack = self._projector_tensors.setdefault( + proj_tensor_type, [None for _ in range(self._n_proj)] + ) + tensor_stack[projector_idx] = data_torch + + # If we've found all of the stacked projectors for this tensor type, + # yield the stacked tensor data + if None not in tensor_stack: + stacked_data_torch = torch.stack(tensor_stack) + stacked_name = name.replace(f".{bid}.", ".") + yield from super().modify_tensors(stacked_data_torch, stacked_name, 0) + return + else: + return None + yield from super().modify_tensors(data_torch, name, bid) @ModelBase.register("Lfm25AudioTokenizer") class LFM25AudioTokenizer(LFM2Model): model_arch = gguf.MODEL_ARCH.LFM2 From 97600c7b8ab5cf2b97e27d82944b6c05625c6816 Mon Sep 17 00:00:00 2001 From: Gabe Goodhart Date: Thu, 7 May 2026 16:33:07 -0600 Subject: [PATCH 11/94] feat: Add gguf_writer and constant support for new hparams and deepstack layer arr Branch: Granite4Vision AI-usage: draft (OpenCode + qwen3.5:122b) Signed-off-by: Gabe Goodhart --- gguf-py/gguf/constants.py | 3 +++ gguf-py/gguf/gguf_writer.py | 20 ++++++++++++++++++-- 2 files changed, 21 insertions(+), 2 deletions(-) diff --git a/gguf-py/gguf/constants.py b/gguf-py/gguf/constants.py index 93dbca0eefc0..e7ba21692ed6 100644 --- a/gguf-py/gguf/constants.py +++ b/gguf-py/gguf/constants.py @@ -330,6 +330,8 @@ class Attention: class Projector: SCALE_FACTOR = "clip.vision.projector.scale_factor" + QUERY_SIDE = "clip.vision.projector.query_side" + WINDOW_SIDE = "clip.vision.projector.window_side" class SAM: BLOCK_COUNT = "clip.vision.sam.block_count" @@ -1365,6 +1367,7 @@ class MODEL_TENSOR(IntEnum): MODEL_TENSOR.V_QF_FFN_UP: "v.proj_blk.{bid}.ffn_up", MODEL_TENSOR.V_QF_FFN_DOWN: "v.proj_blk.{bid}.ffn_down", MODEL_TENSOR.V_QF_FFN_NORM: "v.proj_blk.{bid}.ffn_norm", + # audio (mtmd) # note: all audio tensor names must use prefix "a." or "mm.a." MODEL_TENSOR.A_ENC_EMBD_POS: "a.position_embd", diff --git a/gguf-py/gguf/gguf_writer.py b/gguf-py/gguf/gguf_writer.py index 42149b0b4831..aea6c01aceb8 100644 --- a/gguf-py/gguf/gguf_writer.py +++ b/gguf-py/gguf/gguf_writer.py @@ -955,8 +955,18 @@ def add_attn_temperature_scale(self, value: float) -> None: def add_pooling_type(self, value: PoolingType) -> None: self.add_uint32(Keys.LLM.POOLING_TYPE.format(arch=self.arch), value.value) - def add_num_deepstack_layers(self, count: int) -> None: - self.add_uint32(Keys.LLM.NUM_DEEPSTACK_LAYERS.format(arch=self.arch), count) + def add_num_deepstack_layers(self, count: int | Sequence[int]) -> None: + """ + Add deepstack layer configuration. + + For backward compatibility, the GGUF key name remains 'n_deepstack_layers' + even though it now supports arrays. Existing models using scalar values + will continue to work. + """ + if isinstance(count, int): + self.add_uint32(Keys.LLM.NUM_DEEPSTACK_LAYERS.format(arch=self.arch), count) + else: + self.add_array(Keys.LLM.NUM_DEEPSTACK_LAYERS.format(arch=self.arch), list(count)) def add_rope_dimension_count(self, count: int) -> None: self.add_uint32(Keys.Rope.DIMENSION_COUNT.format(arch=self.arch), count) @@ -1175,6 +1185,12 @@ def add_vision_preproc_min_tiles(self, value: int) -> None: def add_vision_preproc_image_size(self, value: int) -> None: self.add_uint32(Keys.ClipVision.PREPROC_IMAGE_SIZE, value) + def add_vision_projector_query_side(self, value: int) -> None: + self.add_uint32(Keys.ClipVision.Projector.QUERY_SIDE, value) + + def add_vision_projector_window_side(self, value: int) -> None: + self.add_uint32(Keys.ClipVision.Projector.WINDOW_SIDE, value) + def add_vision_image_mean(self, values: Sequence[float]) -> None: self.add_array(Keys.ClipVision.IMAGE_MEAN, values) From f6d19753f6b3e614f207c17d9974bdb3cd44a2b1 Mon Sep 17 00:00:00 2001 From: Gabe Goodhart Date: Fri, 8 May 2026 09:30:09 -0600 Subject: [PATCH 12/94] feat: Full conversion for mmproj w/ tensor mappings Branch: Granite4Vision AI-usage: full (OpenCode + qwen3.5:122b) Signed-off-by: Gabe Goodhart --- convert_hf_to_gguf.py | 2 +- gguf-py/gguf/tensor_mapping.py | 11 +++++++++++ 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/convert_hf_to_gguf.py b/convert_hf_to_gguf.py index 5415132cc763..8aa41eee2bf9 100755 --- a/convert_hf_to_gguf.py +++ b/convert_hf_to_gguf.py @@ -13162,7 +13162,7 @@ def set_gguf_parameters(self): @classmethod def filter_tensors(cls, item: tuple[str, Callable[[], Tensor]]) -> tuple[str, Callable[[], Tensor]] | None: name, _ = item - if name.startswith("language_model."): + if name.startswith("language_model.") or "vision_model.head" in name: return None return super().filter_tensors(item) diff --git a/gguf-py/gguf/tensor_mapping.py b/gguf-py/gguf/tensor_mapping.py index be3261b5b3c2..65e12a4f1c3a 100644 --- a/gguf-py/gguf/tensor_mapping.py +++ b/gguf-py/gguf/tensor_mapping.py @@ -1424,6 +1424,7 @@ class TensorNameMap: ), MODEL_TENSOR.V_ENC_EMBD_PATCH: ( + "model.vision_tower.vision_model.embeddings.patch_embedding", # Granite4Vision "vision_tower.vision_model.embeddings.patch_embedding", "model.vision_tower.embeddings.patch_embedding", # minicpmv4_6 "model.vision_tower.embeddings.patch_embeddings.projection", # Intern-S1 @@ -1450,6 +1451,7 @@ class TensorNameMap: ), MODEL_TENSOR.V_ENC_EMBD_POS: ( + "model.vision_tower.vision_model.embeddings.position_embedding", # Granite4Vision "vision_tower.vision_model.embeddings.position_embedding", "model.vision_tower.embeddings.position_embedding", # minicpmv4_6 "model.vision_tower.embeddings.position_embeddings", # Intern-S1 @@ -1488,6 +1490,7 @@ class TensorNameMap: ), MODEL_TENSOR.V_ENC_ATTN_Q: ( + "model.vision_tower.vision_model.encoder.layers.{bid}.self_attn.q_proj", # Granite4Vision "vision_tower.vision_model.encoder.layers.{bid}.self_attn.q_proj", "model.vision_tower.encoder.layers.{bid}.self_attn.q_proj", # minicpmv4_6 "model.vision_tower.encoder.layer.{bid}.attention.q_proj", # Intern-S1 @@ -1512,6 +1515,7 @@ class TensorNameMap: ), MODEL_TENSOR.V_ENC_ATTN_K: ( + "model.vision_tower.vision_model.encoder.layers.{bid}.self_attn.k_proj", # Granite4Vision "vision_tower.vision_model.encoder.layers.{bid}.self_attn.k_proj", "model.vision_tower.encoder.layers.{bid}.self_attn.k_proj", # minicpmv4_6 "model.vision_tower.encoder.layer.{bid}.attention.k_proj", # Intern-S1 @@ -1536,6 +1540,7 @@ class TensorNameMap: ), MODEL_TENSOR.V_ENC_ATTN_V: ( + "model.vision_tower.vision_model.encoder.layers.{bid}.self_attn.v_proj", # Granite4Vision "vision_tower.vision_model.encoder.layers.{bid}.self_attn.v_proj", "model.vision_tower.encoder.layers.{bid}.self_attn.v_proj", # minicpmv4_6 "model.vision_tower.encoder.layer.{bid}.attention.v_proj", # Intern-S1 @@ -1553,6 +1558,7 @@ class TensorNameMap: ), MODEL_TENSOR.V_ENC_INPUT_NORM: ( + "model.vision_tower.vision_model.encoder.layers.{bid}.layer_norm1", # Granite4Vision "vision_tower.vision_model.encoder.layers.{bid}.layer_norm1", "model.vision_tower.encoder.layers.{bid}.layer_norm1", # minicpmv4_6 "vision_tower.vision_model.encoder.layers.{bid}.norm1", # InternVL @@ -1574,6 +1580,7 @@ class TensorNameMap: ), MODEL_TENSOR.V_ENC_ATTN_O: ( + "model.vision_tower.vision_model.encoder.layers.{bid}.self_attn.out_proj", # Granite4Vision "vision_tower.vision_model.encoder.layers.{bid}.self_attn.out_proj", "model.vision_tower.encoder.layers.{bid}.self_attn.out_proj", # minicpmv4_6 "vision_tower.vision_model.encoder.layers.{bid}.attn.proj", # InternVL @@ -1601,6 +1608,7 @@ class TensorNameMap: ), MODEL_TENSOR.V_ENC_POST_ATTN_NORM: ( + "model.vision_tower.vision_model.encoder.layers.{bid}.layer_norm2", # Granite4Vision "vision_tower.vision_model.encoder.layers.{bid}.layer_norm2", "model.vision_tower.encoder.layers.{bid}.layer_norm2", # minicpmv4_6 "vision_tower.vision_model.encoder.layers.{bid}.norm2", # InternVL @@ -1623,6 +1631,7 @@ class TensorNameMap: ), MODEL_TENSOR.V_ENC_FFN_UP: ( + "model.vision_tower.vision_model.encoder.layers.{bid}.mlp.fc1", # Granite4Vision "vision_tower.vision_model.encoder.layers.{bid}.mlp.fc1", "model.vision_tower.encoder.layers.{bid}.mlp.fc1", # minicpmv4_6 "model.vision_tower.encoder.layer.{bid}.mlp.fc1", # Intern-S1 @@ -1652,6 +1661,7 @@ class TensorNameMap: ), MODEL_TENSOR.V_ENC_FFN_DOWN: ( + "model.vision_tower.vision_model.encoder.layers.{bid}.mlp.fc2", # Granite4Vision "vision_tower.vision_model.encoder.layers.{bid}.mlp.fc2", "model.vision_tower.encoder.layers.{bid}.mlp.fc2", # minicpmv4_6 "model.vision_tower.encoder.layer.{bid}.mlp.fc2", # Intern-S1 @@ -1708,6 +1718,7 @@ class TensorNameMap: ), MODEL_TENSOR.V_POST_NORM: ( + "model.vision_tower.vision_model.post_layernorm", # Granite4Vision "vision_tower.vision_model.post_layernorm", "model.vision_tower.post_layernorm", # minicpmv4_6 "model.vision_model.post_layernorm", # SmolVLM From 97e612adf9683510f4c1b80544d3f382e5ef00ce Mon Sep 17 00:00:00 2001 From: Gabe Goodhart Date: Fri, 8 May 2026 09:38:33 -0600 Subject: [PATCH 13/94] fix: Add lm_head skip for mmproj for 4.0 Branch: Granite4Vision AI-usage: none Signed-off-by: Gabe Goodhart --- convert_hf_to_gguf.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/convert_hf_to_gguf.py b/convert_hf_to_gguf.py index 8aa41eee2bf9..032635915c06 100755 --- a/convert_hf_to_gguf.py +++ b/convert_hf_to_gguf.py @@ -13162,7 +13162,11 @@ def set_gguf_parameters(self): @classmethod def filter_tensors(cls, item: tuple[str, Callable[[], Tensor]]) -> tuple[str, Callable[[], Tensor]] | None: name, _ = item - if name.startswith("language_model.") or "vision_model.head" in name: + if ( + name.startswith("language_model.") or + "vision_model.head" in name or + name.startswith("lm_head") + ): return None return super().filter_tensors(item) From 2a969d3c2929692683321af8f62ed5ad109ae71d Mon Sep 17 00:00:00 2001 From: Gabe Goodhart Date: Fri, 8 May 2026 09:46:53 -0600 Subject: [PATCH 14/94] fix: De-alias text_config architecture in convert_lora_to_gguf.py Branch: Granite4Vision AI-usage: none Signed-off-by: Gabe Goodhart --- convert_lora_to_gguf.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/convert_lora_to_gguf.py b/convert_lora_to_gguf.py index ad4751bb96e6..ac63ab7f01bd 100755 --- a/convert_lora_to_gguf.py +++ b/convert_lora_to_gguf.py @@ -384,7 +384,9 @@ def load_hparams_from_hf(hf_model_id: str) -> tuple[dict[str, Any], Path | None] with torch.inference_mode(): try: - model_class = ModelBase.from_model_architecture(hparams["architectures"][0]) + model_arch = hparams.get("text_config", {}).get("architectures", hparams["architectures"])[0] + logger.info("Using model architecture: %s", model_arch) + model_class = ModelBase.from_model_architecture(model_arch) except NotImplementedError: logger.error(f"Model {hparams['architectures'][0]} is not supported") sys.exit(1) From 23326861a0347afd6aeeb432f651877d5d310c8a Mon Sep 17 00:00:00 2001 From: Gabe Goodhart Date: Fri, 8 May 2026 09:53:53 -0600 Subject: [PATCH 15/94] feat: Add --trust-remote-code arg to convert_lora_to_gguf.py This defaults to False, but allows a user to enable it programmaticly instead of using the interactive prompt. Branch: Granite4Vision AI-usage: none Signed-off-by: Gabe Goodhart --- convert_lora_to_gguf.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/convert_lora_to_gguf.py b/convert_lora_to_gguf.py index ac63ab7f01bd..70796b5ee96f 100755 --- a/convert_lora_to_gguf.py +++ b/convert_lora_to_gguf.py @@ -302,6 +302,10 @@ def parse_args() -> argparse.Namespace: "--base-model-id", type=str, help="the model ID of the base model, if it is not available locally or in the adapter config. If specified, it will ignore --base and load the base model config from the Hugging Face hub (Example: 'meta-llama/Llama-3.2-1B-Instruct')", ) + parser.add_argument( + "--trust-remote-code", default=False, action="store_true", + help="trust remote code in the model", + ) parser.add_argument( "lora_path", type=Path, help="directory containing Hugging Face PEFT LoRA config (adapter_model.json) and weights (adapter_model.safetensors or adapter_model.bin)", @@ -310,11 +314,11 @@ def parse_args() -> argparse.Namespace: return parser.parse_args() -def load_hparams_from_hf(hf_model_id: str) -> tuple[dict[str, Any], Path | None]: +def load_hparams_from_hf(hf_model_id: str, trust_remote_code: bool) -> tuple[dict[str, Any], Path | None]: from huggingface_hub import try_to_load_from_cache # normally, adapter does not come with base model config, we need to load it from AutoConfig - config = AutoConfig.from_pretrained(hf_model_id) + config = AutoConfig.from_pretrained(hf_model_id, trust_remote_code=trust_remote_code) cache_dir = try_to_load_from_cache(hf_model_id, "config.json") cache_dir = Path(cache_dir).parent if isinstance(cache_dir, str) else None @@ -363,13 +367,13 @@ def load_hparams_from_hf(hf_model_id: str) -> tuple[dict[str, Any], Path | None] # load base model if base_model_id is not None: logger.info(f"Loading base model from Hugging Face: {base_model_id}") - hparams, dir_base_model = load_hparams_from_hf(base_model_id) + hparams, dir_base_model = load_hparams_from_hf(base_model_id, args.trust_remote_code) elif dir_base_model is None: if "base_model_name_or_path" in lparams: model_id = lparams["base_model_name_or_path"] logger.info(f"Loading base model from Hugging Face: {model_id}") try: - hparams, dir_base_model = load_hparams_from_hf(model_id) + hparams, dir_base_model = load_hparams_from_hf(model_id, args.trust_remote_code) except OSError as e: logger.error(f"Failed to load base model config: {e}") logger.error("Please try downloading the base model and add its path to --base") From fc31cca169c912ac5e415599656be91a6beb4334 Mon Sep 17 00:00:00 2001 From: Gabe Goodhart Date: Fri, 8 May 2026 10:23:57 -0600 Subject: [PATCH 16/94] fix: De-alias model.language_model. -> model. for lora adapters Branch: Granite4Vision AI-usage: full (OpenCode + qwen3.5:122b) Signed-off-by: Gabe Goodhart --- convert_lora_to_gguf.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/convert_lora_to_gguf.py b/convert_lora_to_gguf.py index 70796b5ee96f..ad698c98721a 100755 --- a/convert_lora_to_gguf.py +++ b/convert_lora_to_gguf.py @@ -485,6 +485,9 @@ def get_tensors(self) -> Iterator[tuple[str, Tensor]]: yield (name, cast(torch.Tensor, LoraTorchTensor(tensor.A, tensor.B))) def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]: + # Handle adapters targeting language_model subcomponent + if name.startswith("model.language_model."): + name = name.replace("model.language_model.", "model.", 1) dest = list(super().modify_tensors(data_torch, name, bid)) # some archs may have the same tensor for lm_head and output (tie word embeddings) # in this case, adapters targeting lm_head will fail when using llama-export-lora From 0b03adaa632b76c7651dd1f6fa3b7a0b5eda479c Mon Sep 17 00:00:00 2001 From: Gabe Goodhart Date: Fri, 8 May 2026 10:59:02 -0600 Subject: [PATCH 17/94] fix: Extend language model tensor dealiasing in adapters Branch: Granite4Vision AI-usage: none Signed-off-by: Gabe Goodhart --- convert_lora_to_gguf.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/convert_lora_to_gguf.py b/convert_lora_to_gguf.py index ad698c98721a..d4cda4e5f6d4 100755 --- a/convert_lora_to_gguf.py +++ b/convert_lora_to_gguf.py @@ -486,8 +486,13 @@ def get_tensors(self) -> Iterator[tuple[str, Tensor]]: def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]: # Handle adapters targeting language_model subcomponent - if name.startswith("model.language_model."): - name = name.replace("model.language_model.", "model.", 1) + for lm_prefix, replace in [ + ("model.language_model.", "model."), # Granite4 Vision + ("language_model.", ""), # Granite Speech + ]: + if name.startswith(lm_prefix): + name = name.replace(lm_prefix, replace, 1) + break dest = list(super().modify_tensors(data_torch, name, bid)) # some archs may have the same tensor for lm_head and output (tie word embeddings) # in this case, adapters targeting lm_head will fail when using llama-export-lora From fb6075b3440d05211632793d0c43e24c4dbb9b61 Mon Sep 17 00:00:00 2001 From: Gabe Goodhart Date: Fri, 8 May 2026 10:59:59 -0600 Subject: [PATCH 18/94] fix: Remove unnecessary registration for GraniteSpeech in language model Branch: Granite4Vision AI-usage: none Signed-off-by: Gabe Goodhart --- convert_hf_to_gguf.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/convert_hf_to_gguf.py b/convert_hf_to_gguf.py index 032635915c06..a48ef9b07fb1 100755 --- a/convert_hf_to_gguf.py +++ b/convert_hf_to_gguf.py @@ -11038,7 +11038,7 @@ def prepare_tensors(self): raise ValueError(f"Unprocessed experts: {experts}") -@ModelBase.register("GraniteForCausalLM", "GraniteSpeechForConditionalGeneration") +@ModelBase.register("GraniteForCausalLM") class GraniteModel(LlamaModel): """Conversion for IBM's GraniteForCausalLM""" model_arch = gguf.MODEL_ARCH.GRANITE From 8e4c0b5797c82bb5a56d07e50321874ac37adc24 Mon Sep 17 00:00:00 2001 From: Gabe Goodhart Date: Fri, 8 May 2026 11:06:31 -0600 Subject: [PATCH 19/94] feat: Plumb through mm prefix formatting for qformer tensors Branch: Granite4Vision AI-usage: none Signed-off-by: Gabe Goodhart --- tools/mtmd/clip.cpp | 66 ++++++++++++++++++++++----------------------- 1 file changed, 33 insertions(+), 33 deletions(-) diff --git a/tools/mtmd/clip.cpp b/tools/mtmd/clip.cpp index 1fc0efd5f3f1..03c2b899b005 100644 --- a/tools/mtmd/clip.cpp +++ b/tools/mtmd/clip.cpp @@ -2547,45 +2547,45 @@ struct clip_model_loader { layer.conv_pw2_b = get_tensor(string_format(TN_CONV_PW2, prefix, il, "bias")); } - model.qf_proj_query = get_tensor(TN_QF_PROJ_QUERY); - model.qf_proj_norm_w = get_tensor(string_format(TN_QF_PROJ_NORM, "weight")); - model.qf_proj_norm_b = get_tensor(string_format(TN_QF_PROJ_NORM, "bias")); - model.qf_proj_linear_w = get_tensor(string_format(TN_QF_PROJ_LINEAR, "weight")); - model.qf_proj_linear_b = get_tensor(string_format(TN_QF_PROJ_LINEAR, "bias")); + model.qf_proj_query = get_tensor(string_format(TN_QF_PROJ_QUERY, prefix)); + model.qf_proj_norm_w = get_tensor(string_format(TN_QF_PROJ_NORM, prefix, "weight")); + model.qf_proj_norm_b = get_tensor(string_format(TN_QF_PROJ_NORM, prefix, "bias")); + model.qf_proj_linear_w = get_tensor(string_format(TN_QF_PROJ_LINEAR, prefix, "weight")); + model.qf_proj_linear_b = get_tensor(string_format(TN_QF_PROJ_LINEAR, prefix, "bias")); const int n_proj_layers = 2; model.qf_proj_layers.resize(n_proj_layers); for (int il = 0; il < n_proj_layers; ++il) { auto & pl = model.qf_proj_layers[il]; - pl.q_w = get_tensor(string_format(TN_QF_SELF_ATTN_Q, il, "weight")); - pl.q_b = get_tensor(string_format(TN_QF_SELF_ATTN_Q, il, "bias")); - pl.k_w = get_tensor(string_format(TN_QF_SELF_ATTN_K, il, "weight")); - pl.k_b = get_tensor(string_format(TN_QF_SELF_ATTN_K, il, "bias")); - pl.v_w = get_tensor(string_format(TN_QF_SELF_ATTN_V, il, "weight")); - pl.v_b = get_tensor(string_format(TN_QF_SELF_ATTN_V, il, "bias")); - pl.o_w = get_tensor(string_format(TN_QF_SELF_ATTN_O, il, "weight")); - pl.o_b = get_tensor(string_format(TN_QF_SELF_ATTN_O, il, "bias")); - pl.ln_1_w = get_tensor(string_format(TN_QF_SELF_ATTN_N, il, "weight")); - pl.ln_1_b = get_tensor(string_format(TN_QF_SELF_ATTN_N, il, "bias")); - - pl.cross_attn_q_w = get_tensor(string_format(TN_QF_CROSS_ATTN_Q, il, "weight")); - pl.cross_attn_q_b = get_tensor(string_format(TN_QF_CROSS_ATTN_Q, il, "bias")); - pl.cross_attn_k_w = get_tensor(string_format(TN_QF_CROSS_ATTN_K, il, "weight")); - pl.cross_attn_k_b = get_tensor(string_format(TN_QF_CROSS_ATTN_K, il, "bias")); - pl.cross_attn_v_w = get_tensor(string_format(TN_QF_CROSS_ATTN_V, il, "weight")); - pl.cross_attn_v_b = get_tensor(string_format(TN_QF_CROSS_ATTN_V, il, "bias")); - pl.cross_attn_o_w = get_tensor(string_format(TN_QF_CROSS_ATTN_O, il, "weight")); - pl.cross_attn_o_b = get_tensor(string_format(TN_QF_CROSS_ATTN_O, il, "bias")); - pl.cross_attn_norm_w = get_tensor(string_format(TN_QF_CROSS_ATTN_N, il, "weight")); - pl.cross_attn_norm_b = get_tensor(string_format(TN_QF_CROSS_ATTN_N, il, "bias")); - - pl.ff_up_w = get_tensor(string_format(TN_QF_FFN_UP, il, "weight")); - pl.ff_up_b = get_tensor(string_format(TN_QF_FFN_UP, il, "bias")); - pl.ff_down_w = get_tensor(string_format(TN_QF_FFN_DOWN, il, "weight")); - pl.ff_down_b = get_tensor(string_format(TN_QF_FFN_DOWN, il, "bias")); - pl.ln_2_w = get_tensor(string_format(TN_QF_FFN_NORM, il, "weight")); - pl.ln_2_b = get_tensor(string_format(TN_QF_FFN_NORM, il, "bias")); + pl.q_w = get_tensor(string_format(TN_QF_SELF_ATTN_Q, prefix, il, "weight")); + pl.q_b = get_tensor(string_format(TN_QF_SELF_ATTN_Q, prefix, il, "bias")); + pl.k_w = get_tensor(string_format(TN_QF_SELF_ATTN_K, prefix, il, "weight")); + pl.k_b = get_tensor(string_format(TN_QF_SELF_ATTN_K, prefix, il, "bias")); + pl.v_w = get_tensor(string_format(TN_QF_SELF_ATTN_V, prefix, il, "weight")); + pl.v_b = get_tensor(string_format(TN_QF_SELF_ATTN_V, prefix, il, "bias")); + pl.o_w = get_tensor(string_format(TN_QF_SELF_ATTN_O, prefix, il, "weight")); + pl.o_b = get_tensor(string_format(TN_QF_SELF_ATTN_O, prefix, il, "bias")); + pl.ln_1_w = get_tensor(string_format(TN_QF_SELF_ATTN_N, prefix, il, "weight")); + pl.ln_1_b = get_tensor(string_format(TN_QF_SELF_ATTN_N, prefix, il, "bias")); + + pl.cross_attn_q_w = get_tensor(string_format(TN_QF_CROSS_ATTN_Q, prefix, il, "weight")); + pl.cross_attn_q_b = get_tensor(string_format(TN_QF_CROSS_ATTN_Q, prefix, il, "bias")); + pl.cross_attn_k_w = get_tensor(string_format(TN_QF_CROSS_ATTN_K, prefix, il, "weight")); + pl.cross_attn_k_b = get_tensor(string_format(TN_QF_CROSS_ATTN_K, prefix, il, "bias")); + pl.cross_attn_v_w = get_tensor(string_format(TN_QF_CROSS_ATTN_V, prefix, il, "weight")); + pl.cross_attn_v_b = get_tensor(string_format(TN_QF_CROSS_ATTN_V, prefix, il, "bias")); + pl.cross_attn_o_w = get_tensor(string_format(TN_QF_CROSS_ATTN_O, prefix, il, "weight")); + pl.cross_attn_o_b = get_tensor(string_format(TN_QF_CROSS_ATTN_O, prefix, il, "bias")); + pl.cross_attn_norm_w = get_tensor(string_format(TN_QF_CROSS_ATTN_N, prefix, il, "weight")); + pl.cross_attn_norm_b = get_tensor(string_format(TN_QF_CROSS_ATTN_N, prefix, il, "bias")); + + pl.ff_up_w = get_tensor(string_format(TN_QF_FFN_UP, prefix, il, "weight")); + pl.ff_up_b = get_tensor(string_format(TN_QF_FFN_UP, prefix, il, "bias")); + pl.ff_down_w = get_tensor(string_format(TN_QF_FFN_DOWN, prefix, il, "weight")); + pl.ff_down_b = get_tensor(string_format(TN_QF_FFN_DOWN, prefix, il, "bias")); + pl.ln_2_w = get_tensor(string_format(TN_QF_FFN_NORM, prefix, il, "weight")); + pl.ln_2_b = get_tensor(string_format(TN_QF_FFN_NORM, prefix, il, "bias")); } } break; default: From 8aa12681e164d245b5e73a97aa01130a5fd5731e Mon Sep 17 00:00:00 2001 From: Gabe Goodhart Date: Thu, 14 May 2026 16:07:16 -0600 Subject: [PATCH 20/94] refactor: Refactor vision projector tensors to use predictor ID as the block This is cleaner than stacking them. The modeling file hard-codes single-layer qformers, so we can punt on the multiipule multi-layer projectors problem. Branch: Granite4Vision AI-usage: none Signed-off-by: Gabe Goodhart --- convert_hf_to_gguf.py | 50 +++++++++++++++-------------- gguf-py/gguf/constants.py | 26 +++++++++++----- gguf-py/gguf/tensor_mapping.py | 57 ++++++++++++++-------------------- 3 files changed, 68 insertions(+), 65 deletions(-) diff --git a/convert_hf_to_gguf.py b/convert_hf_to_gguf.py index a48ef9b07fb1..9dbd697f2c92 100755 --- a/convert_hf_to_gguf.py +++ b/convert_hf_to_gguf.py @@ -13081,13 +13081,6 @@ def __init__(self, *args, **kwargs): n_text_layers = self.global_config["text_config"]["num_hidden_layers"] n_vision_layers = self.global_config["vision_config"]["num_hidden_layers"] - # The tensors for the individual per-layer injection projectors are held - # separately in safetensors. We merge them here, so this is used to - # accumulate the projector tensors in the order that will correspond - # with the layer mapping in deepstack_layer_arr. - self._projector_tensors = {} - self._deepstack_layer_arr = [-1 for _ in range(n_text_layers)] # Populate with -1 sentinels - # Normalize both deepstack and spatial projector maps to the form: # (vision_layer, llm_layer, , type_index) # @@ -13125,6 +13118,7 @@ def __init__(self, *args, **kwargs): for proj_idx, (_, _, proj_type, type_idx) in enumerate(normalized_projector_map) } self._vision_feature_layers = [vision_layer for vision_layer, _, _, _ in normalized_projector_map] + self._deepstack_layer_arr = [-1 for _ in range(n_text_layers)] # Populate with -1 sentinels for proj_idx, (_, llm_layer, _, _) in enumerate(normalized_projector_map): self._deepstack_layer_arr[llm_layer] = proj_idx @@ -13174,29 +13168,37 @@ def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iter # Detect projector tensors and bin them projector_idx = None - proj_tensor_type = None for prefix, proj_idx in self._tensor_prefix_map.items(): if name.startswith(prefix): projector_idx = proj_idx - proj_tensor_type = name[len(prefix):].lstrip(".") break if projector_idx is not None: - # Add to the stacked tensor mappings - tensor_stack = self._projector_tensors.setdefault( - proj_tensor_type, [None for _ in range(self._n_proj)] - ) - tensor_stack[projector_idx] = data_torch - - # If we've found all of the stacked projectors for this tensor type, - # yield the stacked tensor data - if None not in tensor_stack: - stacked_data_torch = torch.stack(tensor_stack) - stacked_name = name.replace(f".{bid}.", ".") - yield from super().modify_tensors(stacked_data_torch, stacked_name, 0) - return - else: - return None + # If this projector tensor has a block id within the projector, + # alias the bid to projector_idx + # + # TODO: currently, none of the Granite 4 Vision models have + # projectors with multiple QFormer layers, so the `layer.{}` index + # is always 0. This allows us to simply map to a single `bid` that + # matches the projector index. If this changes, we'll need a + # convention that merges the two IDs. + id_matches = list(re.finditer(r"\.([0-9]+)\.", name)) + all_ids = [int(m.group(1)) for m in id_matches] + assert len(all_ids) >= 1 and len(all_ids) <= 2, "Must have at least 1 and at most 2 ids in tensor names" + # If not layer id, just use the projector index + new_bid = projector_idx + if len(all_ids) == 1: + new_name = name[:id_matches[0].span(1)[0]] + str(new_bid) + name[id_matches[0].span(1)[1]:] + else: # len(all_ids) == 2 + new_bid = projector_idx # + all_ids[1] + new_name = name[:id_matches[0].span(0)[0]] + name[id_matches[0].span(1)[1]:id_matches[1].span(1)[0]] + str(new_bid) + name[id_matches[1].span(1)[1]:] + try: + yield from super().modify_tensors(data_torch, new_name, new_bid) + except Exception as err: + breakpoint() + return yield from super().modify_tensors(data_torch, name, bid) + + @ModelBase.register("Lfm25AudioTokenizer") class LFM25AudioTokenizer(LFM2Model): model_arch = gguf.MODEL_ARCH.LFM2 diff --git a/gguf-py/gguf/constants.py b/gguf-py/gguf/constants.py index e7ba21692ed6..ab907c70aed0 100644 --- a/gguf-py/gguf/constants.py +++ b/gguf-py/gguf/constants.py @@ -831,8 +831,13 @@ class MODEL_TENSOR(IntEnum): V_QF_FFN_UP = auto() V_QF_FFN_DOWN = auto() V_QF_FFN_NORM = auto() - V_IMG_POS = auto() V_PROJ_NORM = auto() + # multi-projector (bid => projector id) - Granite4 vision + V_MULTI_PROJ_IMG_POS = auto() + V_MULTI_PROJ_QUERY = auto() + V_MULTI_PROJ_NORM = auto() + V_MULTI_PROJ_LINEAR = auto() + V_MULTI_PROJ_QF_LAYERNORM = auto() # audio (mtmd) A_ENC_EMBD_POS = auto() @@ -1349,11 +1354,8 @@ class MODEL_TENSOR(IntEnum): MODEL_TENSOR.V_ENC_EMBD_IMGNL: "v.image_newline", # Deepseek-OCR, Granite4Vision MODEL_TENSOR.V_ENC_EMBD_VSEP: "v.view_seperator", # Deepseek-OCR # Granite4 Vision - MODEL_TENSOR.V_IMG_POS: "v.proj_img_pos", - MODEL_TENSOR.V_PROJ_NORM: "v.proj_layernorm", - MODEL_TENSOR.V_QF_PROJ_QUERY: "v.proj_query", - MODEL_TENSOR.V_QF_PROJ_NORM: "v.proj_norm", - MODEL_TENSOR.V_QF_PROJ_LINEAR: "v.proj_linear", + # qformer layers (bid => proj_id) + # NOTE: Names align with A_QF_* MODEL_TENSOR.V_QF_SELF_ATTN_Q: "v.proj_blk.{bid}.self_attn_q", MODEL_TENSOR.V_QF_SELF_ATTN_K: "v.proj_blk.{bid}.self_attn_k", MODEL_TENSOR.V_QF_SELF_ATTN_V: "v.proj_blk.{bid}.self_attn_v", @@ -1367,6 +1369,12 @@ class MODEL_TENSOR(IntEnum): MODEL_TENSOR.V_QF_FFN_UP: "v.proj_blk.{bid}.ffn_up", MODEL_TENSOR.V_QF_FFN_DOWN: "v.proj_blk.{bid}.ffn_down", MODEL_TENSOR.V_QF_FFN_NORM: "v.proj_blk.{bid}.ffn_norm", + # multi-projector (bid => projector ID) + MODEL_TENSOR.V_MULTI_PROJ_IMG_POS: "v.proj_blk.{bid}.img_pos", + MODEL_TENSOR.V_MULTI_PROJ_QUERY: "v.proj_blk.{bid}.query", + MODEL_TENSOR.V_MULTI_PROJ_NORM: "v.proj_blk.{bid}.norm", + MODEL_TENSOR.V_MULTI_PROJ_LINEAR: "v.proj_blk.{bid}.linear", + MODEL_TENSOR.V_MULTI_PROJ_QF_LAYERNORM: "v.proj_blk.{bid}.qf.layernorm", # audio (mtmd) # note: all audio tensor names must use prefix "a." or "mm.a." @@ -1546,7 +1554,6 @@ class MODEL_TENSOR(IntEnum): MODEL_TENSOR.V_SAM_NECK, MODEL_TENSOR.V_SAM_NET_2, MODEL_TENSOR.V_SAM_NET_3, - MODEL_TENSOR.V_IMG_POS, MODEL_TENSOR.V_PROJ_NORM, MODEL_TENSOR.V_QF_PROJ_QUERY, MODEL_TENSOR.V_QF_PROJ_NORM, @@ -1565,6 +1572,11 @@ class MODEL_TENSOR(IntEnum): MODEL_TENSOR.V_QF_FFN_DOWN, MODEL_TENSOR.V_QF_FFN_NORM, MODEL_TENSOR.V_QF_PROJ_NORM, + MODEL_TENSOR.V_MULTI_PROJ_IMG_POS, + MODEL_TENSOR.V_MULTI_PROJ_QUERY, + MODEL_TENSOR.V_MULTI_PROJ_NORM, + MODEL_TENSOR.V_MULTI_PROJ_LINEAR, + MODEL_TENSOR.V_MULTI_PROJ_QF_LAYERNORM, # audio MODEL_TENSOR.A_ENC_EMBD_POS, MODEL_TENSOR.A_ENC_EMBD_NORM, diff --git a/gguf-py/gguf/tensor_mapping.py b/gguf-py/gguf/tensor_mapping.py index 65e12a4f1c3a..cb53fc4686b6 100644 --- a/gguf-py/gguf/tensor_mapping.py +++ b/gguf-py/gguf/tensor_mapping.py @@ -174,28 +174,6 @@ class TensorNameMap: MODEL_TENSOR.A_QF_PROJ_LINEAR: ( "projector.linear", ), - - # NOTE: For deepstack versions, projectors are stacked - MODEL_TENSOR.V_IMG_POS: ( - "model.layerwise_projectors.image_positions", # Granite4 Vision - "model.spatial_projectors.image_positions", # Granite4 Vision - ), - MODEL_TENSOR.V_PROJ_NORM: ( - "model.layerwise_projectors.norm", # Granite4 Vision - "model.spatial_projectors.norm", # Granite4 Vision - ), - MODEL_TENSOR.V_QF_PROJ_LINEAR: ( - "model.layerwise_projectors.out_linear", # Granite4 Vision - "model.spatial_projectors.out_linear", # Granite4 Vision - ), - MODEL_TENSOR.V_QF_PROJ_QUERY: ( - "model.layerwise_projectors.query", # Granite4 Vision - "model.spatial_projectors.query", # Granite4 Vision - ), - MODEL_TENSOR.V_QF_PROJ_NORM: ( - "model.layerwise_projectors.qformer.layernorm", # Granite4 Vision - "model.spatial_projectors.qformer.layernorm", # Granite4 Vision - ), } block_mappings_cfg: dict[MODEL_TENSOR, tuple[str, ...]] = { @@ -1956,66 +1934,77 @@ class TensorNameMap: "model.vision_tower.std_scale", # gemma4 ), + # For these tensors, bid => projector ID + MODEL_TENSOR.V_MULTI_PROJ_IMG_POS: ( + "model.layerwise_projectors.{bid}.image_positions", # Granite4 Vision + "model.spatial_projectors.{bid}.image_positions", # Granite4 Vision + ), + MODEL_TENSOR.V_MULTI_PROJ_QUERY: ( + "model.layerwise_projectors.{bid}.query", # Granite4 Vision + "model.spatial_projectors.{bid}.query", # Granite4 Vision + ), + MODEL_TENSOR.V_MULTI_PROJ_NORM: ( + "model.layerwise_projectors.{bid}.norm", # Granite4 Vision + "model.spatial_projectors.{bid}.norm", # Granite4 Vision + ), + MODEL_TENSOR.V_MULTI_PROJ_LINEAR: ( + "model.layerwise_projectors.{bid}.out_linear", # Granite4 Vision + "model.spatial_projectors.{bid}.out_linear", # Granite4 Vision + ), + MODEL_TENSOR.V_MULTI_PROJ_QF_LAYERNORM: ( + "model.layerwise_projectors.{bid}.qformer.layernorm", # Granite4 Vision + "model.spatial_projectors.{bid}.qformer.layernorm", # Granite4 Vision + ), + + # For these tensors, bid => proj-id MODEL_TENSOR.V_QF_SELF_ATTN_Q: ( "model.layerwise_projectors.qformer.encoder.layer.{bid}.attention.attention.query", # Granite4 Vision "model.spatial_projectors.qformer.encoder.layer.{bid}.attention.attention.query", # Granite4 Vision ), - MODEL_TENSOR.V_QF_SELF_ATTN_K: ( "model.layerwise_projectors.qformer.encoder.layer.{bid}.attention.attention.key", # Granite4 Vision "model.spatial_projectors.qformer.encoder.layer.{bid}.attention.attention.key", # Granite4 Vision ), - MODEL_TENSOR.V_QF_SELF_ATTN_V: ( "model.layerwise_projectors.qformer.encoder.layer.{bid}.attention.attention.value", # Granite4 Vision "model.spatial_projectors.qformer.encoder.layer.{bid}.attention.attention.value", # Granite4 Vision ), - MODEL_TENSOR.V_QF_SELF_ATTN_O: ( "model.layerwise_projectors.qformer.encoder.layer.{bid}.attention.output.dense", # Granite4 Vision "model.spatial_projectors.qformer.encoder.layer.{bid}.attention.output.dense", # Granite4 Vision ), - MODEL_TENSOR.V_QF_SELF_ATTN_NORM: ( "model.layerwise_projectors.qformer.encoder.layer.{bid}.attention.output.LayerNorm", # Granite4 Vision "model.spatial_projectors.qformer.encoder.layer.{bid}.attention.output.LayerNorm", # Granite4 Vision ), - MODEL_TENSOR.V_QF_CROSS_ATTN_Q: ( "model.layerwise_projectors.qformer.encoder.layer.{bid}.crossattention.attention.query", # Granite4 Vision "model.spatial_projectors.qformer.encoder.layer.{bid}.crossattention.attention.query", # Granite4 Vision ), - MODEL_TENSOR.V_QF_CROSS_ATTN_K: ( "model.layerwise_projectors.qformer.encoder.layer.{bid}.crossattention.attention.key", # Granite4 Vision "model.spatial_projectors.qformer.encoder.layer.{bid}.crossattention.attention.key", # Granite4 Vision ), - MODEL_TENSOR.V_QF_CROSS_ATTN_V: ( "model.layerwise_projectors.qformer.encoder.layer.{bid}.crossattention.attention.value", # Granite4 Vision "model.spatial_projectors.qformer.encoder.layer.{bid}.crossattention.attention.value", # Granite4 Vision ), - MODEL_TENSOR.V_QF_CROSS_ATTN_O: ( "model.layerwise_projectors.qformer.encoder.layer.{bid}.crossattention.output.dense", # Granite4 Vision "model.spatial_projectors.qformer.encoder.layer.{bid}.crossattention.output.dense", # Granite4 Vision ), - MODEL_TENSOR.V_QF_CROSS_ATTN_NORM: ( "model.layerwise_projectors.qformer.encoder.layer.{bid}.crossattention.output.LayerNorm", # Granite4 Vision "model.spatial_projectors.qformer.encoder.layer.{bid}.crossattention.output.LayerNorm", # Granite4 Vision ), - MODEL_TENSOR.V_QF_FFN_UP: ( "model.layerwise_projectors.qformer.encoder.layer.{bid}.intermediate_query.dense", # Granite4 Vision "model.spatial_projectors.qformer.encoder.layer.{bid}.intermediate_query.dense", # Granite4 Vision ), - MODEL_TENSOR.V_QF_FFN_DOWN: ( "model.layerwise_projectors.qformer.encoder.layer.{bid}.output_query.dense", # Granite4 Vision "model.spatial_projectors.qformer.encoder.layer.{bid}.output_query.dense", # Granite4 Vision ), - MODEL_TENSOR.V_QF_FFN_NORM: ( "model.layerwise_projectors.qformer.encoder.layer.{bid}.output_query.LayerNorm", # Granite4 Vision "model.spatial_projectors.qformer.encoder.layer.{bid}.output_query.LayerNorm", # Granite4 Vision From 14fd2cc8b526cf2d49e49ee431d34ca1bf886370 Mon Sep 17 00:00:00 2001 From: Gabe Goodhart Date: Thu, 14 May 2026 16:22:30 -0600 Subject: [PATCH 21/94] feat: Add spatial offests array hparam conversion Branch: Granite4Vision AI-usage: none Signed-off-by: Gabe Goodhart --- convert_hf_to_gguf.py | 7 +++++++ gguf-py/gguf/constants.py | 1 + gguf-py/gguf/gguf_writer.py | 3 +++ 3 files changed, 11 insertions(+) diff --git a/convert_hf_to_gguf.py b/convert_hf_to_gguf.py index 9dbd697f2c92..4e55c960352f 100755 --- a/convert_hf_to_gguf.py +++ b/convert_hf_to_gguf.py @@ -13118,6 +13118,10 @@ def __init__(self, *args, **kwargs): for proj_idx, (_, _, proj_type, type_idx) in enumerate(normalized_projector_map) } self._vision_feature_layers = [vision_layer for vision_layer, _, _, _ in normalized_projector_map] + self._spatial_offsets = [ + type_idx if proj_type == "spatial" else -1 + for _, _, proj_type, type_idx in normalized_projector_map + ] self._deepstack_layer_arr = [-1 for _ in range(n_text_layers)] # Populate with -1 sentinels for proj_idx, (_, llm_layer, _, _) in enumerate(normalized_projector_map): self._deepstack_layer_arr[llm_layer] = proj_idx @@ -13150,6 +13154,9 @@ def set_gguf_parameters(self): # Set vision feature layers self.gguf_writer.add_vision_feature_layers(self._vision_feature_layers) + # Set the spatial offests per projector + self.gguf_writer.add_vision_spatial_offsets(self._spatial_offsets) + # Write array of deepstack layers (llm_layer -> projector_idx) self.gguf_writer.add_num_deepstack_layers(self._deepstack_layer_arr) diff --git a/gguf-py/gguf/constants.py b/gguf-py/gguf/constants.py index ab907c70aed0..a2fe7c6f4b22 100644 --- a/gguf-py/gguf/constants.py +++ b/gguf-py/gguf/constants.py @@ -332,6 +332,7 @@ class Projector: SCALE_FACTOR = "clip.vision.projector.scale_factor" QUERY_SIDE = "clip.vision.projector.query_side" WINDOW_SIDE = "clip.vision.projector.window_side" + SPATIAL_OFFSETS = "clip.vision.projector.spatial_offsets" class SAM: BLOCK_COUNT = "clip.vision.sam.block_count" diff --git a/gguf-py/gguf/gguf_writer.py b/gguf-py/gguf/gguf_writer.py index aea6c01aceb8..7266f85eac74 100644 --- a/gguf-py/gguf/gguf_writer.py +++ b/gguf-py/gguf/gguf_writer.py @@ -1191,6 +1191,9 @@ def add_vision_projector_query_side(self, value: int) -> None: def add_vision_projector_window_side(self, value: int) -> None: self.add_uint32(Keys.ClipVision.Projector.WINDOW_SIDE, value) + def add_vision_spatial_offsets(self, layers: Sequence[int]) -> None: + self.add_array(Keys.ClipVision.Projector.SPATIAL_OFFSETS, layers) + def add_vision_image_mean(self, values: Sequence[float]) -> None: self.add_array(Keys.ClipVision.IMAGE_MEAN, values) From 0feeb29fbb9f69d83192415e1a29782e762a2416 Mon Sep 17 00:00:00 2001 From: Gabe Goodhart Date: Fri, 15 May 2026 08:32:18 -0600 Subject: [PATCH 22/94] feat: Add stub plumbing for granite vision in mtmd Branch: Granite4Vision AI-usage: draft (OpenCode + qwen3.5:122b) Signed-off-by: Gabe Goodhart --- tools/mtmd/CMakeLists.txt | 1 + tools/mtmd/models/granite-vision.cpp | 6 ++++++ tools/mtmd/models/models.h | 5 +++++ 3 files changed, 12 insertions(+) create mode 100644 tools/mtmd/models/granite-vision.cpp diff --git a/tools/mtmd/CMakeLists.txt b/tools/mtmd/CMakeLists.txt index a76adc9b80bd..8e7688fdab8d 100644 --- a/tools/mtmd/CMakeLists.txt +++ b/tools/mtmd/CMakeLists.txt @@ -22,6 +22,7 @@ add_library(mtmd models/gemma4v.cpp models/glm4v.cpp models/granite-speech.cpp + models/granite-vision.cpp models/hunyuanocr.cpp models/internvl.cpp models/kimivl.cpp diff --git a/tools/mtmd/models/granite-vision.cpp b/tools/mtmd/models/granite-vision.cpp new file mode 100644 index 000000000000..6334a33477f3 --- /dev/null +++ b/tools/mtmd/models/granite-vision.cpp @@ -0,0 +1,6 @@ +#include "models.h" + +ggml_cgraph * clip_graph_granite_vision::build() { + //TODO + return gf; +} diff --git a/tools/mtmd/models/models.h b/tools/mtmd/models/models.h index 955daa6d6d34..d9ed057635cb 100644 --- a/tools/mtmd/models/models.h +++ b/tools/mtmd/models/models.h @@ -190,3 +190,8 @@ struct clip_graph_kimik25 : clip_graph { ggml_tensor * resize_position_embeddings_3d(uint32_t interpolation_mode); }; + +struct clip_graph_granite_vision : clip_graph { + clip_graph_granite_vision(clip_ctx * ctx, const clip_image_f32 & img) : clip_graph(ctx, img) {} + ggml_cgraph * build() override; +}; From 5f23c21a58a42c244128844da1ffc2c973e1138f Mon Sep 17 00:00:00 2001 From: Gabe Goodhart Date: Fri, 15 May 2026 08:38:11 -0600 Subject: [PATCH 23/94] feat: Add new hparam and tensor naming in clip-impl.h New hparams: - KEY_PROJ_SAMPLE_QUERY_SIDE - KEY_PROJ_SAMPLE_WINDOW_SIDE - KEY_PROJ_SPATIAL_OFFSETS New tensors: - TN_MULTI_PROJ_IMG_POS - TN_MULTI_PROJ_QUERY - TN_MULTI_PROJ_LAYERNORM - TN_MULTI_PROJ_LINEAR - TN_MULTI_PROJ_NORM Branch: Granite4Vision AI-usage: none Signed-off-by: Gabe Goodhart --- tools/mtmd/clip-impl.h | 42 +++++++++++++++++++++++++----------------- 1 file changed, 25 insertions(+), 17 deletions(-) diff --git a/tools/mtmd/clip-impl.h b/tools/mtmd/clip-impl.h index 1e024979cb1f..a48bb7b7466c 100644 --- a/tools/mtmd/clip-impl.h +++ b/tools/mtmd/clip-impl.h @@ -35,20 +35,23 @@ #define KEY_LAYER_NORM_EPS "clip.%s.attention.layer_norm_epsilon" // vision-specific -#define KEY_VISION_PROJ_TYPE "clip.vision.projector_type" // for models with mixed modalities -#define KEY_IMAGE_SIZE "clip.vision.image_size" -#define KEY_IMAGE_MIN_PIXELS "clip.vision.image_min_pixels" -#define KEY_IMAGE_MAX_PIXELS "clip.vision.image_max_pixels" -#define KEY_PREPROC_MIN_TILES "clip.vision.preproc_min_tiles" -#define KEY_PREPROC_MAX_TILES "clip.vision.preproc_max_tiles" -#define KEY_PREPROC_IMAGE_SIZE "clip.vision.preproc_image_size" -#define KEY_PATCH_SIZE "clip.vision.patch_size" -#define KEY_IMAGE_MEAN "clip.vision.image_mean" -#define KEY_IMAGE_STD "clip.vision.image_std" -#define KEY_FEATURE_LAYER "clip.vision.feature_layer" -#define KEY_PROJ_SCALE_FACTOR "clip.vision.projector.scale_factor" -#define KEY_SPATIAL_MERGE_SIZE "clip.vision.spatial_merge_size" -#define KEY_IS_DEEPSTACK_LAYERS "clip.vision.is_deepstack_layers" +#define KEY_VISION_PROJ_TYPE "clip.vision.projector_type" // for models with mixed modalities +#define KEY_IMAGE_SIZE "clip.vision.image_size" +#define KEY_IMAGE_MIN_PIXELS "clip.vision.image_min_pixels" +#define KEY_IMAGE_MAX_PIXELS "clip.vision.image_max_pixels" +#define KEY_PREPROC_MIN_TILES "clip.vision.preproc_min_tiles" +#define KEY_PREPROC_MAX_TILES "clip.vision.preproc_max_tiles" +#define KEY_PREPROC_IMAGE_SIZE "clip.vision.preproc_image_size" +#define KEY_PATCH_SIZE "clip.vision.patch_size" +#define KEY_IMAGE_MEAN "clip.vision.image_mean" +#define KEY_IMAGE_STD "clip.vision.image_std" +#define KEY_FEATURE_LAYER "clip.vision.feature_layer" +#define KEY_PROJ_SCALE_FACTOR "clip.vision.projector.scale_factor" +#define KEY_PROJ_SAMPLE_QUERY_SIDE "clip.vision.projector.query_side" +#define KEY_PROJ_SAMPLE_WINDOW_SIDE "clip.vision.projector.window_side" +#define KEY_PROJ_SPATIAL_OFFSETS "clip.vision.projector.spatial_offsets" +#define KEY_SPATIAL_MERGE_SIZE "clip.vision.spatial_merge_size" +#define KEY_IS_DEEPSTACK_LAYERS "clip.vision.is_deepstack_layers" #define KEY_MM_PATCH_MERGE_TYPE "clip.vision.mm_patch_merge_type" #define KEY_IMAGE_GRID_PINPOINTS "clip.vision.image_grid_pinpoints" @@ -207,10 +210,9 @@ #define TN_CTC_OUT_MID "a.enc_ctc_out_mid.%s" #define TN_ATTN_REL_POS_EMB "%s.blk.%d.attn_rel_pos_emb" // qformer projector -#define TN_PROJ_LAYERNORM "v.proj_layernorm.%s" // NOTE: Disambig with QF_PROJ_NORM -#define TN_QF_PROJ_QUERY "%s.proj_query" +#define TN_QF_PROJ_QUERY "a.proj_query" #define TN_QF_PROJ_NORM "%s.proj_norm.%s" // NOTE: This _should_ be qf_proj_norm -#define TN_QF_PROJ_LINEAR "%s.proj_linear.%s" +#define TN_QF_PROJ_LINEAR "a.proj_linear.%s" #define TN_QF_SELF_ATTN_Q "%s.proj_blk.%d.self_attn_q.%s" #define TN_QF_SELF_ATTN_K "%s.proj_blk.%d.self_attn_k.%s" #define TN_QF_SELF_ATTN_V "%s.proj_blk.%d.self_attn_v.%s" @@ -224,6 +226,12 @@ #define TN_QF_FFN_UP "%s.proj_blk.%d.ffn_up.%s" #define TN_QF_FFN_DOWN "%s.proj_blk.%d.ffn_down.%s" #define TN_QF_FFN_NORM "%s.proj_blk.%d.ffn_norm.%s" +// multi-projector qformer (bid => projector ID) +#define TN_MULTI_PROJ_IMG_POS "%s.proj_blk.%d.img_pos" +#define TN_MULTI_PROJ_QUERY "%s.proj_blk.%d.query" +#define TN_MULTI_PROJ_LAYERNORM "%s.proj_blk.%d.layernorm.%s" // NOTE: Disambig with QF_PROJ_NORM +#define TN_MULTI_PROJ_LINEAR "%s.proj_blk.%d.linear.%s" +#define TN_MULTI_PROJ_NORM "%s.proj_blk.%d.norm.%s" // gemma4 audio conformer #define TN_A_MM_INP_PROJ "mm.a.input_projection.%s" From 234973d423c48258bbfc89070889b589b706c4e0 Mon Sep 17 00:00:00 2001 From: Gabe Goodhart Date: Fri, 15 May 2026 10:51:17 -0600 Subject: [PATCH 24/94] fix: Move deepstack_layer_arr to llm hparam instead of mmproj Branch: Granite4Vision AI-usage: none Signed-off-by: Gabe Goodhart --- convert_hf_to_gguf.py | 63 ++++++++++++++++++++++++------------------- 1 file changed, 36 insertions(+), 27 deletions(-) diff --git a/convert_hf_to_gguf.py b/convert_hf_to_gguf.py index 4e55c960352f..3a9ee6495300 100755 --- a/convert_hf_to_gguf.py +++ b/convert_hf_to_gguf.py @@ -11071,6 +11071,14 @@ def set_gguf_parameters(self): self.gguf_writer.add_logit_scale(logits_scale) logger.info("gguf: (granite) logits_scale = %s", logits_scale) + # If being used as the base for Granite4 Vision, add deepstack_layer_arr + if self.hparams.get("spatial_target_layers") or self.hparams.get("deepstack_layer_map"): + normalized_projector_map = Granite4VisionMmprojModel.get_normalized_projector_map(self.hparams) + deepstack_layer_arr = [-1 for _ in range(self.block_count)] # Populate with -1 sentinels + for proj_idx, (_, llm_layer, _, _) in enumerate(normalized_projector_map): + deepstack_layer_arr[llm_layer] = proj_idx + self.gguf_writer.add_num_deepstack_layers(deepstack_layer_arr) + @classmethod def filter_tensors(cls, item: tuple[str, Callable[[], Tensor]]) -> tuple[str, Callable[[], Tensor]] | None: name, gen = item @@ -13073,28 +13081,28 @@ class Granite4VisionMmprojModel(MmprojModel): has_vision_encoder = True has_audio_encoder = False - def __init__(self, *args, **kwargs): - super().__init__(*args, **kwargs) - - deepstack_map = self.global_config.get("deepstack_layer_map", []) # [[vis_layer, llm_layer], ...] - spatial_layers = self.global_config.get("spatial_target_layers", []) # [llm_layer, ...] - n_text_layers = self.global_config["text_config"]["num_hidden_layers"] - n_vision_layers = self.global_config["vision_config"]["num_hidden_layers"] - - # Normalize both deepstack and spatial projector maps to the form: - # (vision_layer, llm_layer, , type_index) - # - # This is then used to populate the following mappings: - # - # - vision_feature_layers: ordered list of all vision_layer values where - # order corresponds with the order of the stacked projector tensors - # NOTE: Values may appear multiple times for spatial projectors - # - # - deepstack_layer_arr: per-text-layer array indicating which input - # vision feature should be injected at that layer (-1 if none) - # - # - tensor_prefix_map: mapping from tensor prefixes to the index of the - # corresponding projector in the stacked tensors + @staticmethod + def get_normalized_projector_map(global_config: dict) -> list[tuple[int, int, str, int]]: + """Normalize both deepstack and spatial projector maps to the form: + (vision_layer, llm_layer, , type_index) + + This is then used to populate the following mappings: + - vision_feature_layers (mmproj hparam): ordered list of all + vision_layer values where order corresponds with the order of the + stacked projector tensors + NOTE: Values may appear multiple times for spatial projectors + - tensor_prefix_map (mmproj tensors): mapping from tensor prefixes to + the index of the corresponding projector in the stacked tensors + - deepstack_layer_arr (llm hparam): per-text-layer array indicating + which input vision feature should be injected at that layer + (-1 if none) + + Output: (vision_layer, llm_layer, , type_index) + """ + deepstack_map = global_config.get("deepstack_layer_map", []) # [[vis_layer, llm_layer], ...] + spatial_layers = global_config.get("spatial_target_layers", []) # [llm_layer, ...] + n_text_layers = global_config["text_config"]["num_hidden_layers"] + n_vision_layers = global_config["vision_config"]["num_hidden_layers"] normalized_projector_map = [] if deepstack_map: for deepstack_idx, (vision_layer, llm_layer) in enumerate(sorted(deepstack_map)): @@ -13104,13 +13112,16 @@ def __init__(self, *args, **kwargs): llm_layer = n_text_layers + llm_layer normalized_projector_map.append((vision_layer, llm_layer, "layerwise", deepstack_idx)) if spatial_layers: - spatial_vision_layer = self.global_config.get("spatial_vision_layer", -1) + spatial_vision_layer = global_config.get("spatial_vision_layer", -1) if spatial_vision_layer < 0: spatial_vision_layer = n_vision_layers + spatial_vision_layer for spatial_idx, llm_layer in enumerate(spatial_layers): normalized_projector_map.append((spatial_vision_layer, llm_layer, "spatial", spatial_idx)) + return list(sorted(normalized_projector_map)) - normalized_projector_map = list(sorted(normalized_projector_map)) + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + normalized_projector_map = self.get_normalized_projector_map(self.global_config) self._n_proj = len(normalized_projector_map) self._tensor_prefix_map = { @@ -13122,6 +13133,7 @@ def __init__(self, *args, **kwargs): type_idx if proj_type == "spatial" else -1 for _, _, proj_type, type_idx in normalized_projector_map ] + n_text_layers = self.global_config["text_config"]["num_hidden_layers"] self._deepstack_layer_arr = [-1 for _ in range(n_text_layers)] # Populate with -1 sentinels for proj_idx, (_, llm_layer, _, _) in enumerate(normalized_projector_map): self._deepstack_layer_arr[llm_layer] = proj_idx @@ -13157,9 +13169,6 @@ def set_gguf_parameters(self): # Set the spatial offests per projector self.gguf_writer.add_vision_spatial_offsets(self._spatial_offsets) - # Write array of deepstack layers (llm_layer -> projector_idx) - self.gguf_writer.add_num_deepstack_layers(self._deepstack_layer_arr) - @classmethod def filter_tensors(cls, item: tuple[str, Callable[[], Tensor]]) -> tuple[str, Callable[[], Tensor]] | None: name, _ = item From cb05a27c88a70c248264da603a70f187020bd73b Mon Sep 17 00:00:00 2001 From: Gabe Goodhart Date: Fri, 15 May 2026 11:14:34 -0600 Subject: [PATCH 25/94] fix: Remove IS_DEEPSTACK_LAYERS This appears to have been added during Qwen3 VL (https://github.com/ggml-org/llama.cpp/pull/16780), but it was never actually used. Branch: Granite4Vision AI-usage: none Signed-off-by: Gabe Goodhart --- convert_hf_to_gguf.py | 7 ------- gguf-py/gguf/gguf_writer.py | 3 --- tools/mtmd/clip-impl.h | 1 - 3 files changed, 11 deletions(-) diff --git a/convert_hf_to_gguf.py b/convert_hf_to_gguf.py index 3a9ee6495300..c014de6f1ccd 100755 --- a/convert_hf_to_gguf.py +++ b/convert_hf_to_gguf.py @@ -4963,10 +4963,6 @@ def __init__(self, *args, **kwargs): self.hparams_vision["num_attention_heads"] = self.hparams_vision.get("num_heads") self.hparams_vision["num_hidden_layers"] = self.hparams_vision.get("depth") - self.is_deepstack_layers = [False] * int(self.hparams_vision["num_hidden_layers"] or 0) - for idx in self.hparams_vision.get("deepstack_visual_indexes", []): - self.is_deepstack_layers[idx] = True - def set_gguf_parameters(self): super().set_gguf_parameters() # in case mixed modalities, the arch will be handled by subclass @@ -4983,9 +4979,6 @@ def set_gguf_parameters(self): rms_norm_eps = self.global_config.get("text_config", {}).get("rms_norm_eps", 1e-6) self.gguf_writer.add_vision_attention_layernorm_eps(rms_norm_eps) - if self.is_deepstack_layers: - self.gguf_writer.add_vision_is_deepstack_layers(self.is_deepstack_layers) - @classmethod def filter_tensors(cls, item: tuple[str, Callable[[], Tensor]]) -> tuple[str, Callable[[], Tensor]] | None: name, gen = item diff --git a/gguf-py/gguf/gguf_writer.py b/gguf-py/gguf/gguf_writer.py index 7266f85eac74..f13ad609aa1d 100644 --- a/gguf-py/gguf/gguf_writer.py +++ b/gguf-py/gguf/gguf_writer.py @@ -1241,9 +1241,6 @@ def add_vision_wa_layer_indexes(self, layers: Sequence[int]) -> None: """ self.add_array(Keys.ClipVision.WA_LAYER_INDEXES, layers) - def add_vision_is_deepstack_layers(self, layers: Sequence[bool]) -> None: - self.add_array(Keys.ClipVision.IS_DEEPSTACK_LAYERS, layers) - def add_vision_wa_pattern_mode(self, modes: Sequence[int]) -> None: self.add_array(Keys.ClipVision.WA_PATTERN_MODE, modes) diff --git a/tools/mtmd/clip-impl.h b/tools/mtmd/clip-impl.h index a48bb7b7466c..fffbaf7ecdbb 100644 --- a/tools/mtmd/clip-impl.h +++ b/tools/mtmd/clip-impl.h @@ -51,7 +51,6 @@ #define KEY_PROJ_SAMPLE_WINDOW_SIDE "clip.vision.projector.window_side" #define KEY_PROJ_SPATIAL_OFFSETS "clip.vision.projector.spatial_offsets" #define KEY_SPATIAL_MERGE_SIZE "clip.vision.spatial_merge_size" -#define KEY_IS_DEEPSTACK_LAYERS "clip.vision.is_deepstack_layers" #define KEY_MM_PATCH_MERGE_TYPE "clip.vision.mm_patch_merge_type" #define KEY_IMAGE_GRID_PINPOINTS "clip.vision.image_grid_pinpoints" From 1551ec3e7f232d66769fe1ed3635eaff3b6d3863 Mon Sep 17 00:00:00 2001 From: Gabe Goodhart Date: Fri, 15 May 2026 12:12:25 -0600 Subject: [PATCH 26/94] refactor: n_deepstack_layers -> deepstack_layer_arr The old logic hard coded a correspondence between the first N layers of the LLM and the 1->N entries in the input embeddings. Now, that relationship is maintained at loading time if the GGUF value is single-valued. If it is multi-valued, it loads directly allowing for deepstack layers to be spaced out throughout the model. Branch: Granite4Vision AI-usage: none Signed-off-by: Gabe Goodhart --- src/g4v-prompt.md | 21 +++++++++++++++++++++ src/llama-hparams.cpp | 9 +++++++++ src/llama-hparams.h | 6 ++++-- src/llama-model-loader.cpp | 1 + src/llama-model-saver.cpp | 2 +- src/llama-model.cpp | 19 +++++++++++++++++++ src/models/qwen3vl.cpp | 8 +++----- src/models/qwen3vlmoe.cpp | 8 +++----- 8 files changed, 61 insertions(+), 13 deletions(-) create mode 100644 src/g4v-prompt.md diff --git a/src/g4v-prompt.md b/src/g4v-prompt.md new file mode 100644 index 000000000000..99e79f3ac738 --- /dev/null +++ b/src/g4v-prompt.md @@ -0,0 +1,21 @@ +The next feature to implement is support for the `granite-4.0-3b-vision` model (https://huggingface.co/ibm-granite/granite-4.0-3b-vision). This model uses a conditional LoRA adapter. For now, we'll require that the user toggle the adapter on/off manually. You can find a local copy of the model at /Users/ghart/models/ibm-granite/granite-4.0-3b-vision. + +The `transformers` implementation for this model is embedded in the model itself, so you can find the full details in `/Users/ghart/models/ibm-granite/granite-4.0-3b-vision/*.py`. + +When implementing support for this model here in `llama.cpp`, we need to do the following: + +A. Add GGUF conversion support: + 1. Add the necessary architecture/hparam/tensor names and enums in `gguf-py/gguf/` + 2. Add translation support for the model in `convert_hf_to_gguf.py` (both the core text model and the mmproj) + 3. Add support for the adapter conversion in `convert_lora_to_gguf.py` + +B. Add `mtmd` support (c++): + 1. Add the corresponding architecture/hparam/tensor names and enums on the c++ side in either `tools/mtmd` or `src/`. I don't _think_ there should be any changes to the enums in the core `src/` library since the text model is already a well supported architecture, but I'm not sure about that. + 2. Ensure support for the vision tokenizer and mmproj model in `tools/mtmd` + 3. Implement support for using the converted LoRA as an adapter + +I want to base this work on the current branch which points to the PR for Granite Speech (https://github.com/ggml-org/llama.cpp/pull/22101). I believe these models rely on some of the same QFormer components, so as much as possible those should be reused. + +The other main architectural feature for this model is multiple layer injection points from the multimodal projector. You can see this in `modeling.py` with the `deepstack_features` list that gets created during projection, then injected at the configured layers using `masked_scatter`. This injection capability is similar to the Per-Layer-Embeddings (PLE) for Gemma4 or the `n_deepstack_layers` for `qwen3vl`. The main difference is the way the layers are determined for Gemma4, every layer is assumed to have a PLE. For `quen3vl`, it's the first N layers. For Granite 4 vision, the layers are a list specified in config. To implement this, we'll likely need to extend the `llama_hparams` struct's `n_deepstack_layers` to support a layer array (see eg `recurrent_layer_arr` or other `std::array` arrays). We'll then need to support GGUF values that can be either a list or a single number which would then map back to the single `n_deepstack_layers` value. + +I want you to further analyze the model architecture, check with the current base branch for Granite 4 Speech, and come up with a plan to implement support for Granite 4 Vision. \ No newline at end of file diff --git a/src/llama-hparams.cpp b/src/llama-hparams.cpp index 002d15d415fe..2c7cf472dc17 100644 --- a/src/llama-hparams.cpp +++ b/src/llama-hparams.cpp @@ -4,6 +4,7 @@ #include #include +#include void llama_hparams::set_swa_pattern(uint32_t n_pattern, bool dense_first) { if (dense_first) { @@ -73,6 +74,14 @@ uint32_t llama_hparams::n_rot(uint32_t il) const { uint32_t llama_hparams::n_embd_inp() const { uint32_t n_embd_inp = n_embd; + // Count the unique deepstack input indices + std::unordered_set unique_deepstack_idxs; + for (const auto val : deepstack_layers_arr) { + if (val >= 0) { + unique_deepstack_idxs.insert(val); + } + } + const auto n_deepstack_layers = unique_deepstack_idxs.size(); if (n_deepstack_layers > 0) { n_embd_inp += n_embd * n_deepstack_layers; } diff --git a/src/llama-hparams.h b/src/llama-hparams.h index 0160a89caa25..d93d6c494889 100644 --- a/src/llama-hparams.h +++ b/src/llama-hparams.h @@ -209,8 +209,10 @@ struct llama_hparams { uint32_t indexer_head_size = 0; uint32_t indexer_top_k = 0; - // qwen3vl deepstack - uint32_t n_deepstack_layers = 0; + // deepstack layer array (qwen3vl, Granite4 Vision) + // -1 => no deepstack + // >=0 => input embedding index for deepstack injection + std::array deepstack_layers_arr; // gemma4 per-layer embedding uint32_t n_embd_per_layer = 0; diff --git a/src/llama-model-loader.cpp b/src/llama-model-loader.cpp index 4e65a45a50d8..721f3e14c8f7 100644 --- a/src/llama-model-loader.cpp +++ b/src/llama-model-loader.cpp @@ -393,6 +393,7 @@ namespace GGUFMeta { } template bool llama_model_loader::get_arr>(enum llm_kv kid, std::vector & result, bool required); + template bool llama_model_loader::get_arr>(enum llm_kv kid, std::array & result, bool required); template bool llama_model_loader::get_key(const std::string & key, T & result, bool required) { diff --git a/src/llama-model-saver.cpp b/src/llama-model-saver.cpp index e83056557bfd..dc3fb4f050ae 100644 --- a/src/llama-model-saver.cpp +++ b/src/llama-model-saver.cpp @@ -228,7 +228,7 @@ void llama_model_saver::add_kv_from_model() { add_kv(LLM_KV_EXPERTS_PER_GROUP, hparams.n_group_experts); add_kv(LLM_KV_MOE_EVERY_N_LAYERS, hparams.moe_every_n_layers); add_kv(LLM_KV_NEXTN_PREDICT_LAYERS, hparams.nextn_predict_layers); - add_kv(LLM_KV_NUM_DEEPSTACK_LAYERS, hparams.n_deepstack_layers); + add_kv(LLM_KV_NUM_DEEPSTACK_LAYERS, hparams.deepstack_layers_arr); add_kv(LLM_KV_POOLING_TYPE, uint32_t(hparams.pooling_type)); add_kv(LLM_KV_LOGIT_SCALE, hparams.f_logit_scale); add_kv(LLM_KV_DECODER_START_TOKEN_ID, hparams.dec_start_token_id); diff --git a/src/llama-model.cpp b/src/llama-model.cpp index ff30a2ae7a6f..9b28f7e44435 100644 --- a/src/llama-model.cpp +++ b/src/llama-model.cpp @@ -1060,6 +1060,25 @@ void llama_model_base::load_hparams(llama_model_loader & ml) { ml.get_key_or_arr(LLM_KV_FEED_FORWARD_LENGTH, hparams.n_ff_arr, hparams.n_layer, false); ml.get_key_or_arr(LLM_KV_ATTENTION_HEAD_COUNT, hparams.n_head_arr, hparams.n_layer, false); + // populate deepstack_layers_arr + // + // If single-valued, it's the equivalent of n_deepstack_layers (qwen3vl). + // This implies a 1:1 correspondence between entries in inp_embed and the + // layers < n (offset by 1). + // + // If multi-valued (Granite4 Vision), the elements are themselves index + // values into the entries in inp_embed + std::fill(hparams.deepstack_layers_arr.begin(), hparams.deepstack_layers_arr.end(), -1.0f); + uint32_t _n_deepstack_layers = 0; + ml.get_key(LLM_KV_NUM_DEEPSTACK_LAYERS, _n_deepstack_layers, false); + if (_n_deepstack_layers > 0) { + for (int32_t i = 0; i < _n_deepstack_layers; ++i) { + hparams.deepstack_layers_arr[i] = i + 1; + } + } else { + ml.get_arr(LLM_KV_NUM_DEEPSTACK_LAYERS, hparams.deepstack_layers_arr, false); + } + // n_head_kv is optional, default to n_head hparams.n_head_kv_arr = hparams.n_head_arr; diff --git a/src/models/qwen3vl.cpp b/src/models/qwen3vl.cpp index 7871f8f7952c..99af38a04cd1 100644 --- a/src/models/qwen3vl.cpp +++ b/src/models/qwen3vl.cpp @@ -1,7 +1,6 @@ #include "models.h" void llama_model_qwen3vl::load_arch_hparams(llama_model_loader & ml) { - ml.get_key(LLM_KV_NUM_DEEPSTACK_LAYERS, hparams.n_deepstack_layers, false); ml.get_key_or_arr(LLM_KV_ROPE_DIMENSION_SECTIONS, hparams.rope_sections, 4, true); ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, hparams.f_norm_rms_eps); switch (hparams.n_layer) { @@ -51,8 +50,6 @@ std::unique_ptr llama_model_qwen3vl::build_arch_graph(const l } llama_model_qwen3vl::graph::graph(const llama_model & model, const llm_graph_params & params) : llm_graph_context(params) { - const size_t n_deepstack_layers = hparams.n_deepstack_layers; - const int64_t n_embd = hparams.n_embd; const int64_t n_embd_head = hparams.n_embd_head_v(); @@ -143,8 +140,9 @@ llama_model_qwen3vl::graph::graph(const llama_model & model, const llm_graph_par cur = build_cvec(cur, il); cb(cur, "l_out", il); - if (il < (int) n_deepstack_layers) { - ggml_tensor * ds = ggml_view_2d(ctx0, res->t_inp_embd, n_embd, n_tokens, res->t_inp_embd->nb[1], (il + 1) * n_embd * sizeof(float)); + const auto & deepstack_emb_idx = hparams.deepstack_layers_arr[il]; + if (deepstack_emb_idx >= 0) { + ggml_tensor * ds = ggml_view_2d(ctx0, res->t_inp_embd, n_embd, n_tokens, res->t_inp_embd->nb[1], deepstack_emb_idx * n_embd * sizeof(float)); cur = ggml_add(ctx0, cur, ds); cb(cur, "deepstack_out", il); } diff --git a/src/models/qwen3vlmoe.cpp b/src/models/qwen3vlmoe.cpp index b99143c8908a..f0a1bb095fe8 100644 --- a/src/models/qwen3vlmoe.cpp +++ b/src/models/qwen3vlmoe.cpp @@ -1,7 +1,6 @@ #include "models.h" void llama_model_qwen3vlmoe::load_arch_hparams(llama_model_loader & ml) { - ml.get_key(LLM_KV_NUM_DEEPSTACK_LAYERS, hparams.n_deepstack_layers, false); ml.get_key_or_arr(LLM_KV_ROPE_DIMENSION_SECTIONS, hparams.rope_sections, 4, true); ml.get_key(LLM_KV_EXPERT_FEED_FORWARD_LENGTH, hparams.n_ff_exp, false); ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, hparams.f_norm_rms_eps); @@ -61,8 +60,6 @@ std::unique_ptr llama_model_qwen3vlmoe::build_arch_graph(cons } llama_model_qwen3vlmoe::graph::graph(const llama_model & model, const llm_graph_params & params) : llm_graph_context(params) { - const size_t n_deepstack_layers = hparams.n_deepstack_layers; - const int64_t n_embd = hparams.n_embd; const int64_t n_embd_head = hparams.n_embd_head_v(); @@ -160,8 +157,9 @@ llama_model_qwen3vlmoe::graph::graph(const llama_model & model, const llm_graph_ cur = build_cvec(cur, il); cb(cur, "l_out", il); - if (il < (int) n_deepstack_layers) { - ggml_tensor * ds = ggml_view_2d(ctx0, res->t_inp_embd, n_embd, n_tokens, res->t_inp_embd->nb[1], (il + 1) * n_embd * sizeof(float)); + const auto & deepstack_emb_idx = hparams.deepstack_layers_arr[il]; + if (deepstack_emb_idx >= 0) { + ggml_tensor * ds = ggml_view_2d(ctx0, res->t_inp_embd, n_embd, n_tokens, res->t_inp_embd->nb[1], deepstack_emb_idx * n_embd * sizeof(float)); cur = ggml_add(ctx0, cur, ds); cb(cur, "deepstack_out", il); } From 5d0f1ee8cbb634af5a4a5155351fca288a204b9c Mon Sep 17 00:00:00 2001 From: Gabe Goodhart Date: Fri, 15 May 2026 12:24:29 -0600 Subject: [PATCH 27/94] fix: Use try/catch for single/multi valued deepstack info The alternative would be to use get_key_or_arr, but then the single value would be populated through the entire array and we'd need to detect that and update it with the right correspondence. Branch: Granite4Vision AI-usage: none Signed-off-by: Gabe Goodhart --- src/llama-model.cpp | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/llama-model.cpp b/src/llama-model.cpp index 9b28f7e44435..2ad2c09381e4 100644 --- a/src/llama-model.cpp +++ b/src/llama-model.cpp @@ -1069,13 +1069,13 @@ void llama_model_base::load_hparams(llama_model_loader & ml) { // If multi-valued (Granite4 Vision), the elements are themselves index // values into the entries in inp_embed std::fill(hparams.deepstack_layers_arr.begin(), hparams.deepstack_layers_arr.end(), -1.0f); - uint32_t _n_deepstack_layers = 0; - ml.get_key(LLM_KV_NUM_DEEPSTACK_LAYERS, _n_deepstack_layers, false); - if (_n_deepstack_layers > 0) { - for (int32_t i = 0; i < _n_deepstack_layers; ++i) { + try { + uint32_t n_deepstack_layers = 0; + ml.get_key(LLM_KV_NUM_DEEPSTACK_LAYERS, n_deepstack_layers, false); + for (int32_t i = 0; i < (int32_t)n_deepstack_layers; ++i) { hparams.deepstack_layers_arr[i] = i + 1; } - } else { + } catch (const std::runtime_error &) { ml.get_arr(LLM_KV_NUM_DEEPSTACK_LAYERS, hparams.deepstack_layers_arr, false); } From c69e6554064c020fa3d872246b82f2e9139f0a9f Mon Sep 17 00:00:00 2001 From: Gabe Goodhart Date: Fri, 15 May 2026 12:53:11 -0600 Subject: [PATCH 28/94] feat: Add deepstack injection point for granite LLM The use of ggml_add here assumes that the elements of inp_embd will be pre- arranged to be the full embedding length with only the vision-mask'ed portions non-zero from the projector. This matches how Qwen3VL does it. Branch: Granite4Vision AI-usage: none Signed-off-by: Gabe Goodhart --- src/models/granite.cpp | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/src/models/granite.cpp b/src/models/granite.cpp index 5e7c7b68181f..d382986e8799 100644 --- a/src/models/granite.cpp +++ b/src/models/granite.cpp @@ -113,9 +113,18 @@ llama_model_granite::graph::graph( for (int il = 0; il < n_layer; ++il) { ggml_tensor * inpSA = inpL; + cur = inpL; + + // apply deepstack injection for Granite4 Vision + const auto & deepstack_emb_idx = hparams.deepstack_layers_arr[il]; + if (deepstack_emb_idx >= 0) { + ggml_tensor * ds = ggml_view_2d(ctx0, res->t_inp_embd, n_embd, n_tokens, res->t_inp_embd->nb[1], deepstack_emb_idx * n_embd * sizeof(float)); + cur = ggml_add(ctx0, cur, ds); + cb(cur, "deepstack_out", il); + } // norm - cur = build_norm(inpL, + cur = build_norm(cur, model.layers[il].attn_norm, NULL, LLM_NORM_RMS, il); cb(cur, "attn_norm", il); From acf0e98f4ed3da28c151114ebec26dd06b2e639f Mon Sep 17 00:00:00 2001 From: Gabe Goodhart Date: Fri, 15 May 2026 15:01:54 -0600 Subject: [PATCH 29/94] fix: add missing vision attn layernorm eps Branch: Granite4Vision AI-usage: full (OpenCode + Qwen 3.6-35B) Signed-off-by: Gabe Goodhart --- convert_hf_to_gguf.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/convert_hf_to_gguf.py b/convert_hf_to_gguf.py index c014de6f1ccd..02c677ac3fc7 100755 --- a/convert_hf_to_gguf.py +++ b/convert_hf_to_gguf.py @@ -13156,6 +13156,10 @@ def set_gguf_parameters(self): self.gguf_writer.add_vision_projector_query_side(query_side) self.gguf_writer.add_vision_projector_window_side(window_side) + # Layer norm epsilon for ViT (use text model epsilon as fallback) + eps = v.get("rms_norm_eps", self.global_config.get("text_config", {}).get("rms_norm_eps", 1e-5)) + self.gguf_writer.add_vision_attention_layernorm_eps(eps) + # Set vision feature layers self.gguf_writer.add_vision_feature_layers(self._vision_feature_layers) From 5ce4b81307c7d16845bf399a4ffde5da243cbb2f Mon Sep 17 00:00:00 2001 From: Gabe Goodhart Date: Fri, 15 May 2026 15:37:03 -0600 Subject: [PATCH 30/94] refactor: Hoist qformer tensors into qf_block and hold a vector for multi-proj Branch: Granite4Vision AI-usage: none Signed-off-by: Gabe Goodhart --- tools/mtmd/clip-model.h | 23 ++++++++++++++++------- tools/mtmd/clip.cpp | 18 ++++++++++-------- tools/mtmd/models/granite-speech.cpp | 10 +++++----- 3 files changed, 31 insertions(+), 20 deletions(-) diff --git a/tools/mtmd/clip-model.h b/tools/mtmd/clip-model.h index a97e8e3996bf..2594ca4a869b 100644 --- a/tools/mtmd/clip-model.h +++ b/tools/mtmd/clip-model.h @@ -315,6 +315,20 @@ struct yasa2_stage { std::vector blocks; }; +// QFormer projector block for models with 1 (or more) QFormer projectors +// Granite Speech, Granite4 Vision +struct qf_block { + ggml_tensor * qf_proj_query = nullptr; + ggml_tensor * qf_proj_norm_w = nullptr; + ggml_tensor * qf_proj_norm_b = nullptr; + ggml_tensor * qf_proj_linear_w = nullptr; + ggml_tensor * qf_proj_linear_b = nullptr; + ggml_tensor * qf_proj_layernorm_w = nullptr; + ggml_tensor * qf_proj_layernorm_b = nullptr; + ggml_tensor * qf_proj_img_pos = nullptr; // Vision only + std::vector qf_proj_layers; +}; + struct clip_model { clip_modality modality = CLIP_MODALITY_VISION; projector_type proj_type = PROJECTOR_TYPE_MLP; @@ -566,13 +580,8 @@ struct clip_model { ggml_tensor * ctc_out_b = nullptr; ggml_tensor * ctc_out_mid_w = nullptr; ggml_tensor * ctc_out_mid_b = nullptr; - // qformer projector - ggml_tensor * qf_proj_query = nullptr; - ggml_tensor * qf_proj_norm_w = nullptr; - ggml_tensor * qf_proj_norm_b = nullptr; - ggml_tensor * qf_proj_linear_w = nullptr; - ggml_tensor * qf_proj_linear_b = nullptr; - std::vector qf_proj_layers; + // qformer projector(s) + std::vector qf_proj_blocks; bool audio_has_avgpool() const { return proj_type == PROJECTOR_TYPE_QWEN2A diff --git a/tools/mtmd/clip.cpp b/tools/mtmd/clip.cpp index 03c2b899b005..cb7cd9b3c8e4 100644 --- a/tools/mtmd/clip.cpp +++ b/tools/mtmd/clip.cpp @@ -2547,16 +2547,18 @@ struct clip_model_loader { layer.conv_pw2_b = get_tensor(string_format(TN_CONV_PW2, prefix, il, "bias")); } - model.qf_proj_query = get_tensor(string_format(TN_QF_PROJ_QUERY, prefix)); - model.qf_proj_norm_w = get_tensor(string_format(TN_QF_PROJ_NORM, prefix, "weight")); - model.qf_proj_norm_b = get_tensor(string_format(TN_QF_PROJ_NORM, prefix, "bias")); - model.qf_proj_linear_w = get_tensor(string_format(TN_QF_PROJ_LINEAR, prefix, "weight")); - model.qf_proj_linear_b = get_tensor(string_format(TN_QF_PROJ_LINEAR, prefix, "bias")); + model.qf_proj_blocks.resize(1); + auto & qf = model.qf_proj_blocks[0]; + qf.qf_proj_query = get_tensor(string_format(TN_QF_PROJ_QUERY, prefix)); + qf.qf_proj_norm_w = get_tensor(string_format(TN_QF_PROJ_NORM, prefix, "weight")); + qf.qf_proj_norm_b = get_tensor(string_format(TN_QF_PROJ_NORM, prefix, "bias")); + qf.qf_proj_linear_w = get_tensor(string_format(TN_QF_PROJ_LINEAR, prefix, "weight")); + qf.qf_proj_linear_b = get_tensor(string_format(TN_QF_PROJ_LINEAR, prefix, "bias")); const int n_proj_layers = 2; - model.qf_proj_layers.resize(n_proj_layers); + qf.qf_proj_layers.resize(n_proj_layers); for (int il = 0; il < n_proj_layers; ++il) { - auto & pl = model.qf_proj_layers[il]; + auto & pl = qf.qf_proj_layers[il]; pl.q_w = get_tensor(string_format(TN_QF_SELF_ATTN_Q, prefix, il, "weight")); pl.q_b = get_tensor(string_format(TN_QF_SELF_ATTN_Q, prefix, il, "bias")); @@ -4251,7 +4253,7 @@ int clip_n_mmproj_embd(const struct clip_ctx * ctx) { case PROJECTOR_TYPE_GEMMA4A: return ctx->model.hparams.projection_dim; case PROJECTOR_TYPE_GRANITE_SPEECH: - return ctx->model.qf_proj_linear_w->ne[1]; + return ctx->model.qf_proj_blocks[0].qf_proj_linear_w->ne[1]; case PROJECTOR_TYPE_GLM4V: return ctx->model.mm_ffn_down_w->ne[1]; default: diff --git a/tools/mtmd/models/granite-speech.cpp b/tools/mtmd/models/granite-speech.cpp index c7e3794a49e1..5e66f75d0a99 100644 --- a/tools/mtmd/models/granite-speech.cpp +++ b/tools/mtmd/models/granite-speech.cpp @@ -199,8 +199,8 @@ ggml_cgraph * clip_graph_granite_speech::build() { ggml_tensor * enc_windows = ggml_reshape_3d(ctx0, cur, n_embd, window_size, nblocks_proj); - ggml_tensor * queries = build_norm(model.qf_proj_query, - model.qf_proj_norm_w, model.qf_proj_norm_b, + ggml_tensor * queries = build_norm(model.qf_proj_blocks[0].qf_proj_query, + model.qf_proj_blocks[0].qf_proj_norm_w, model.qf_proj_blocks[0].qf_proj_norm_b, NORM_TYPE_NORMAL, proj_eps, -1); { ggml_tensor * q_3d = ggml_reshape_3d(ctx0, queries, n_embd, num_queries, 1); @@ -209,8 +209,8 @@ ggml_cgraph * clip_graph_granite_speech::build() { queries = ggml_repeat(ctx0, q_3d, q_shape); } - for (int il = 0; il < (int)model.qf_proj_layers.size(); il++) { - const auto & pl = model.qf_proj_layers[il]; + for (int il = 0; il < (int)model.qf_proj_blocks[0].qf_proj_layers.size(); il++) { + const auto & pl = model.qf_proj_blocks[0].qf_proj_layers[il]; // self-attention { @@ -265,7 +265,7 @@ ggml_cgraph * clip_graph_granite_speech::build() { } cur = ggml_reshape_2d(ctx0, queries, n_embd, num_queries * nblocks_proj); - cur = ggml_add(ctx0, build_mm(model.qf_proj_linear_w, cur), model.qf_proj_linear_b); + cur = ggml_add(ctx0, build_mm(model.qf_proj_blocks[0].qf_proj_linear_w, cur), model.qf_proj_blocks[0].qf_proj_linear_b); cb(cur, "projector_out", -1); } From 520d7895f779970c5c3f8c434ff56ab078b64ac9 Mon Sep 17 00:00:00 2001 From: Gabe Goodhart Date: Fri, 15 May 2026 15:41:07 -0600 Subject: [PATCH 31/94] fix: Fix missing prefix template for TN_QF_PROJ_LINEAR It's not strictly necessary since vision uses the blockwise version, but it makes the loading consistent. Branch: Granite4Vision AI-usage: none Signed-off-by: Gabe Goodhart --- tools/mtmd/clip-impl.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/mtmd/clip-impl.h b/tools/mtmd/clip-impl.h index fffbaf7ecdbb..5b1e96d244b3 100644 --- a/tools/mtmd/clip-impl.h +++ b/tools/mtmd/clip-impl.h @@ -211,7 +211,7 @@ // qformer projector #define TN_QF_PROJ_QUERY "a.proj_query" #define TN_QF_PROJ_NORM "%s.proj_norm.%s" // NOTE: This _should_ be qf_proj_norm -#define TN_QF_PROJ_LINEAR "a.proj_linear.%s" +#define TN_QF_PROJ_LINEAR "%s.proj_linear.%s" #define TN_QF_SELF_ATTN_Q "%s.proj_blk.%d.self_attn_q.%s" #define TN_QF_SELF_ATTN_K "%s.proj_blk.%d.self_attn_k.%s" #define TN_QF_SELF_ATTN_V "%s.proj_blk.%d.self_attn_v.%s" From a460879e9536c9b2207a41b986164c22a791f987 Mon Sep 17 00:00:00 2001 From: Gabe Goodhart Date: Fri, 15 May 2026 16:27:08 -0600 Subject: [PATCH 32/94] fix: Add embedding scale and image grid pinpoints hparams in conversion Also remove dead parsing for self._deepstack_layer_arr Branch: Granite4Vision AI-usage: none Signed-off-by: Gabe Goodhart --- convert_hf_to_gguf.py | 17 ++++++++++++----- gguf-py/gguf/constants.py | 1 + gguf-py/gguf/gguf_writer.py | 3 +++ 3 files changed, 16 insertions(+), 5 deletions(-) diff --git a/convert_hf_to_gguf.py b/convert_hf_to_gguf.py index 02c677ac3fc7..30611dc3f386 100755 --- a/convert_hf_to_gguf.py +++ b/convert_hf_to_gguf.py @@ -13126,10 +13126,6 @@ def __init__(self, *args, **kwargs): type_idx if proj_type == "spatial" else -1 for _, _, proj_type, type_idx in normalized_projector_map ] - n_text_layers = self.global_config["text_config"]["num_hidden_layers"] - self._deepstack_layer_arr = [-1 for _ in range(n_text_layers)] # Populate with -1 sentinels - for proj_idx, (_, llm_layer, _, _) in enumerate(normalized_projector_map): - self._deepstack_layer_arr[llm_layer] = proj_idx def set_gguf_parameters(self): assert self.hparams_vision is not None @@ -13157,7 +13153,8 @@ def set_gguf_parameters(self): self.gguf_writer.add_vision_projector_window_side(window_side) # Layer norm epsilon for ViT (use text model epsilon as fallback) - eps = v.get("rms_norm_eps", self.global_config.get("text_config", {}).get("rms_norm_eps", 1e-5)) + text_config = self.global_config.get("text_config", {}) + eps = v.get("rms_norm_eps", text_config.get("rms_norm_eps", 1e-5)) self.gguf_writer.add_vision_attention_layernorm_eps(eps) # Set vision feature layers @@ -13166,6 +13163,16 @@ def set_gguf_parameters(self): # Set the spatial offests per projector self.gguf_writer.add_vision_spatial_offsets(self._spatial_offsets) + # The granite LLM will scale the first embedding input by + # embedding_multiplier, so the mmproj needs to invert that scale for its + # output so that the resulting injection works as expected. + if (embed_mult := text_config.get("embedding_multiplier")) is not None: + self.gguf_writer.add_embedding_scale(embed_mult) + + # Add image grind pinpoints (resolution candidates internally) + if pinpoints := self.global_config.get("image_grid_pinpoints"): + self.gguf_writer.add_vision_image_grid_pinpoints(pinpoints) + @classmethod def filter_tensors(cls, item: tuple[str, Callable[[], Tensor]]) -> tuple[str, Callable[[], Tensor]] | None: name, _ = item diff --git a/gguf-py/gguf/constants.py b/gguf-py/gguf/constants.py index a2fe7c6f4b22..60f5853e222a 100644 --- a/gguf-py/gguf/constants.py +++ b/gguf-py/gguf/constants.py @@ -322,6 +322,7 @@ class ClipVision: IS_DEEPSTACK_LAYERS = "clip.vision.is_deepstack_layers" WINDOW_SIZE = "clip.vision.window_size" FEATURE_LAYERS = "clip.vision.feature_layer" # Granite4 Vision + IMAGE_GRID_PINPOINTS = "clip.vision.image_grid_pinpoints" # Granite4 Vision class Attention: HEAD_COUNT = "clip.vision.attention.head_count" diff --git a/gguf-py/gguf/gguf_writer.py b/gguf-py/gguf/gguf_writer.py index f13ad609aa1d..eebc494f23b2 100644 --- a/gguf-py/gguf/gguf_writer.py +++ b/gguf-py/gguf/gguf_writer.py @@ -1250,6 +1250,9 @@ def add_vision_window_size(self, value: int) -> None: def add_vision_feature_layers(self, layers: Sequence[int]) -> None: self.add_array(Keys.ClipVision.FEATURE_LAYERS, layers) + def add_vision_image_grid_pinpoints(self, layers: Sequence[Sequence[int]]) -> None: + self.add_array(Keys.ClipVision.IMAGE_GRID_PINPOINTS, layers) + def add_vision_sam_layers_count(self, value: int) -> None: self.add_uint32(Keys.ClipVision.SAM.BLOCK_COUNT, value) From 173becf57b6ea45bb9ac0cbfca77682055933624 Mon Sep 17 00:00:00 2001 From: Gabe Goodhart Date: Fri, 15 May 2026 16:32:04 -0600 Subject: [PATCH 33/94] feat: Add mtmd KEY_ section for hparams shared with the LLM In this case, we need the EMBEDDING_SCALE so we can unscale the image embeddings to compensate for applying embedding scale to the input embeddings Branch: Granite4Vision AI-usage: none Signed-off-by: Gabe Goodhart --- tools/mtmd/clip-impl.h | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tools/mtmd/clip-impl.h b/tools/mtmd/clip-impl.h index 5b1e96d244b3..7060d6fb561f 100644 --- a/tools/mtmd/clip-impl.h +++ b/tools/mtmd/clip-impl.h @@ -73,7 +73,8 @@ #define KEY_A_PROJ_WINDOW_SIZE "clip.audio.projector.window_size" #define KEY_A_PROJ_DOWNSAMPLE_RATE "clip.audio.projector.downsample_rate" #define KEY_A_PROJ_HEAD_COUNT "clip.audio.projector.head_count" - +// shared with LLM (align to llama-arch.* w/ arch=clip) +#define KEY_LLM_KV_EMBEDDING_SCALE "clip.embedding_scale" // // tensor name constants From d3c174c02acce7043b956c2ecd047a891970ffc2 Mon Sep 17 00:00:00 2001 From: Gabe Goodhart Date: Fri, 15 May 2026 16:33:03 -0600 Subject: [PATCH 34/94] feat: Implement c++ hparam parsing Branch: Granite4Vision AI-usage: draft (Claude Code) Co-authored-by: Eli Schwartz Signed-off-by: Gabe Goodhart --- tools/mtmd/clip-model.h | 14 ++++++++++++++ tools/mtmd/clip.cpp | 16 ++++++++++++++++ 2 files changed, 30 insertions(+) diff --git a/tools/mtmd/clip-model.h b/tools/mtmd/clip-model.h index 2594ca4a869b..988e5dab5ada 100644 --- a/tools/mtmd/clip-model.h +++ b/tools/mtmd/clip-model.h @@ -91,6 +91,20 @@ struct clip_hparams { int32_t sam_n_head = 0; int32_t sam_n_embd = 0; + // Granite4 Vision + std::vector proj_spatial_offsets; // Ordered list of per-projector spatial offsets + int32_t downsample_query_side; + int32_t downsample_window_side; + // Multiplier applied to the base stream (stream targeting the lowest + // llm_layer) before concatenation into the mmproj output. Needed + // because the shared Granite LLM graph scales t_inp_embd by + // f_embedding_scale (12.0) for ALL positions, while Granite Vision + // 4.1's HF forward zeroes inputs_embeds at image positions BEFORE + // the multiplier -- so the base stream has to enter the LLM graph + // pre-divided. Defaults to 1.0 if the mmproj was produced before + // this KV was added. + float base_stream_scale = 1.0f; + // audio int32_t n_mel_bins = 0; // whisper preprocessor int32_t proj_stack_factor = 0; // ultravox diff --git a/tools/mtmd/clip.cpp b/tools/mtmd/clip.cpp index cb7cd9b3c8e4..0edd3769c1dd 100644 --- a/tools/mtmd/clip.cpp +++ b/tools/mtmd/clip.cpp @@ -1566,6 +1566,22 @@ struct clip_model_loader { hparams.image_pad_color = {127, 127, 127}; hparams.image_resize_algo = RESIZE_ALGO_BILINEAR; } break; + case PROJECTOR_TYPE_GRANITE4_VISION: + { + // SigLIP tower. + hparams.image_resize_algo = RESIZE_ALGO_BICUBIC_PILLOW; + hparams.image_resize_pad = true; + + // Load the per-projector hparams (vision_feature_layers, spatial_offsets) + get_arr_int(KEY_FEATURE_LAYER, hparams.vision_feature_layer); + get_arr_int(KEY_PROJ_SPATIAL_OFFSETS, hparams.proj_spatial_offsets); + GGML_ASSERT(hparams.vision_feature_layer.size() == hparams.proj_spatial_offsets.size()); + + get_u32(KEY_PROJ_SAMPLE_QUERY_SIDE, hparams.downsample_query_side); + get_u32(KEY_PROJ_SAMPLE_WINDOW_SIDE, hparams.downsample_window_side); + get_f32(KEY_LLM_KV_EMBEDDING_SCALE, hparams.base_stream_scale, false); + hparams.warmup_image_size = hparams.image_size; + } default: throw std::runtime_error(string_format("%s: unknown vision projector type %s\n", __func__, proj_type.c_str())); } From 944f15f43029f9d99b2e6175dcc12f092fa306ee Mon Sep 17 00:00:00 2001 From: Gabe Goodhart Date: Fri, 15 May 2026 16:47:38 -0600 Subject: [PATCH 35/94] fix: Flatten pinpoints in conversion Branch: Granite4Vision AI-usage: none Signed-off-by: Gabe Goodhart --- convert_hf_to_gguf.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/convert_hf_to_gguf.py b/convert_hf_to_gguf.py index 30611dc3f386..8dd4bc70959a 100755 --- a/convert_hf_to_gguf.py +++ b/convert_hf_to_gguf.py @@ -13169,8 +13169,9 @@ def set_gguf_parameters(self): if (embed_mult := text_config.get("embedding_multiplier")) is not None: self.gguf_writer.add_embedding_scale(embed_mult) - # Add image grind pinpoints (resolution candidates internally) + # Add flattened image grind pinpoints (resolution candidates internally) if pinpoints := self.global_config.get("image_grid_pinpoints"): + pinpoints = [x for pair in pinpoints for x in pair] self.gguf_writer.add_vision_image_grid_pinpoints(pinpoints) @classmethod From 86fef1e4cdc061dc50dde01385da646132f489ce Mon Sep 17 00:00:00 2001 From: Gabe Goodhart Date: Fri, 15 May 2026 16:47:55 -0600 Subject: [PATCH 36/94] fix: Add missing break Branch: Granite4Vision AI-usage: none Signed-off-by: Gabe Goodhart --- tools/mtmd/clip.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/mtmd/clip.cpp b/tools/mtmd/clip.cpp index 0edd3769c1dd..31c79b6f9cd9 100644 --- a/tools/mtmd/clip.cpp +++ b/tools/mtmd/clip.cpp @@ -1581,7 +1581,7 @@ struct clip_model_loader { get_u32(KEY_PROJ_SAMPLE_WINDOW_SIDE, hparams.downsample_window_side); get_f32(KEY_LLM_KV_EMBEDDING_SCALE, hparams.base_stream_scale, false); hparams.warmup_image_size = hparams.image_size; - } + } break; default: throw std::runtime_error(string_format("%s: unknown vision projector type %s\n", __func__, proj_type.c_str())); } From 575f401292cd6ea3387390ecf25113bc2055e0e1 Mon Sep 17 00:00:00 2001 From: Gabe Goodhart Date: Fri, 15 May 2026 16:57:04 -0600 Subject: [PATCH 37/94] fix: No reason to have modality prefix for img_pos Branch: Granite4Vision AI-usage: none Signed-off-by: Gabe Goodhart --- tools/mtmd/clip-impl.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/mtmd/clip-impl.h b/tools/mtmd/clip-impl.h index 7060d6fb561f..4e08d26aa2a6 100644 --- a/tools/mtmd/clip-impl.h +++ b/tools/mtmd/clip-impl.h @@ -227,7 +227,7 @@ #define TN_QF_FFN_DOWN "%s.proj_blk.%d.ffn_down.%s" #define TN_QF_FFN_NORM "%s.proj_blk.%d.ffn_norm.%s" // multi-projector qformer (bid => projector ID) -#define TN_MULTI_PROJ_IMG_POS "%s.proj_blk.%d.img_pos" +#define TN_MULTI_PROJ_IMG_POS "v.proj_blk.%d.img_pos" #define TN_MULTI_PROJ_QUERY "%s.proj_blk.%d.query" #define TN_MULTI_PROJ_LAYERNORM "%s.proj_blk.%d.layernorm.%s" // NOTE: Disambig with QF_PROJ_NORM #define TN_MULTI_PROJ_LINEAR "%s.proj_blk.%d.linear.%s" From 4d59c0ea531faff9bcb5d9d2d95329f684611b2b Mon Sep 17 00:00:00 2001 From: Gabe Goodhart Date: Fri, 15 May 2026 16:58:30 -0600 Subject: [PATCH 38/94] feat: Add tensor loading Branch: Granite4Vision AI-usage: none Signed-off-by: Gabe Goodhart --- tools/mtmd/clip.cpp | 49 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 49 insertions(+) diff --git a/tools/mtmd/clip.cpp b/tools/mtmd/clip.cpp index 31c79b6f9cd9..546356fc7c0b 100644 --- a/tools/mtmd/clip.cpp +++ b/tools/mtmd/clip.cpp @@ -2606,6 +2606,55 @@ struct clip_model_loader { pl.ln_2_b = get_tensor(string_format(TN_QF_FFN_NORM, prefix, il, "bias")); } } break; + case PROJECTOR_TYPE_GRANITE4_VISION: + { + // image_newline lives at the top-level. + model.image_newline = get_tensor(TN_IMAGE_NEWLINE); + + // Load separate layerwise and spatial projector tensors + const auto projector_count = hparams.vision_feature_layer.size(); + model.qf_proj_blocks.resize(projector_count); + for (size_t bid = 0; bid < projector_count; ++bid) { + auto & b = model.qf_proj_blocks[bid]; + + // non-layerwise tensors + b.qf_proj_norm_w = get_tensor(string_format(TN_MULTI_PROJ_NORM, prefix, bid, "weight")); + b.qf_proj_norm_b = get_tensor(string_format(TN_MULTI_PROJ_NORM, prefix, bid, "bias")); + b.qf_proj_query = get_tensor(string_format(TN_MULTI_PROJ_QUERY, prefix, bid)); + b.qf_proj_img_pos = get_tensor(string_format(TN_MULTI_PROJ_IMG_POS, bid)); + b.qf_proj_layernorm_w = get_tensor(string_format(TN_MULTI_PROJ_LAYERNORM, prefix, bid, "weight")); + b.qf_proj_layernorm_b = get_tensor(string_format(TN_MULTI_PROJ_LAYERNORM, prefix, bid, "bias")); + b.qf_proj_linear_w = get_tensor(string_format(TN_MULTI_PROJ_LINEAR, prefix, bid, "weight")); + b.qf_proj_linear_b = get_tensor(string_format(TN_MULTI_PROJ_LINEAR, prefix, bid, "bias")); + + // laywerwise tensors + // NOTE: If any model uses multi-layer qformers, this will need to change + b.qf_proj_layers.resize(1); + auto & layer = b.qf_proj_layers[0]; + + layer.attn_rel_pos_emb = get_tensor(string_format(TN_ATTN_REL_POS_EMB, prefix, bid)); + + layer.ff_norm_w = get_tensor(string_format(TN_FFN_NORM, prefix, bid, "weight")); + layer.ff_norm_b = get_tensor(string_format(TN_FFN_NORM, prefix, bid, "bias")); + + layer.ff_norm_1_w = get_tensor(string_format(TN_FFN_NORM_1, prefix, bid, "weight")); + layer.ff_norm_1_b = get_tensor(string_format(TN_FFN_NORM_1, prefix, bid, "bias")); + layer.ff_up_1_w = get_tensor(string_format(TN_FFN_UP_1, prefix, bid, "weight")); + layer.ff_up_1_b = get_tensor(string_format(TN_FFN_UP_1, prefix, bid, "bias")); + layer.ff_down_1_w = get_tensor(string_format(TN_FFN_DOWN_1, prefix, bid, "weight")); + layer.ff_down_1_b = get_tensor(string_format(TN_FFN_DOWN_1, prefix, bid, "bias")); + + layer.norm_conv_w = get_tensor(string_format(TN_NORM_CONV, prefix, bid, "weight")); + layer.norm_conv_b = get_tensor(string_format(TN_NORM_CONV, prefix, bid, "bias")); + layer.conv_norm_w = get_tensor(string_format(TN_CONV_NORM, prefix, bid, "weight")); + layer.conv_norm_b = get_tensor(string_format(TN_CONV_NORM, prefix, bid, "bias")); + layer.conv_dw_w = get_tensor(string_format(TN_CONV_DW, prefix, bid, "weight")); + layer.conv_pw1_w = get_tensor(string_format(TN_CONV_PW1, prefix, bid, "weight")); + layer.conv_pw1_b = get_tensor(string_format(TN_CONV_PW1, prefix, bid, "bias")); + layer.conv_pw2_w = get_tensor(string_format(TN_CONV_PW2, prefix, bid, "weight")); + layer.conv_pw2_b = get_tensor(string_format(TN_CONV_PW2, prefix, bid, "bias")); + } + } break; default: GGML_ASSERT(false && "unknown projector type"); } From 0df9de9809cb452d1e36090d400c335aed7259c3 Mon Sep 17 00:00:00 2001 From: Gabe Goodhart Date: Fri, 15 May 2026 17:05:41 -0600 Subject: [PATCH 39/94] fix(convert): Fix confusion between proj.norm and proj.qformer.layernorm Branch: Granite4Vision AI-usage: none Signed-off-by: Gabe Goodhart --- gguf-py/gguf/constants.py | 24 ++++++++++++------------ gguf-py/gguf/tensor_mapping.py | 10 +++++----- 2 files changed, 17 insertions(+), 17 deletions(-) diff --git a/gguf-py/gguf/constants.py b/gguf-py/gguf/constants.py index 60f5853e222a..a3da04242603 100644 --- a/gguf-py/gguf/constants.py +++ b/gguf-py/gguf/constants.py @@ -835,11 +835,11 @@ class MODEL_TENSOR(IntEnum): V_QF_FFN_NORM = auto() V_PROJ_NORM = auto() # multi-projector (bid => projector id) - Granite4 vision - V_MULTI_PROJ_IMG_POS = auto() - V_MULTI_PROJ_QUERY = auto() - V_MULTI_PROJ_NORM = auto() - V_MULTI_PROJ_LINEAR = auto() - V_MULTI_PROJ_QF_LAYERNORM = auto() + V_MULTI_PROJ_IMG_POS = auto() + V_MULTI_PROJ_QUERY = auto() + V_MULTI_PROJ_NORM = auto() + V_MULTI_PROJ_LINEAR = auto() + V_MULTI_PROJ_POST_NORM = auto() # audio (mtmd) A_ENC_EMBD_POS = auto() @@ -1372,11 +1372,11 @@ class MODEL_TENSOR(IntEnum): MODEL_TENSOR.V_QF_FFN_DOWN: "v.proj_blk.{bid}.ffn_down", MODEL_TENSOR.V_QF_FFN_NORM: "v.proj_blk.{bid}.ffn_norm", # multi-projector (bid => projector ID) - MODEL_TENSOR.V_MULTI_PROJ_IMG_POS: "v.proj_blk.{bid}.img_pos", - MODEL_TENSOR.V_MULTI_PROJ_QUERY: "v.proj_blk.{bid}.query", - MODEL_TENSOR.V_MULTI_PROJ_NORM: "v.proj_blk.{bid}.norm", - MODEL_TENSOR.V_MULTI_PROJ_LINEAR: "v.proj_blk.{bid}.linear", - MODEL_TENSOR.V_MULTI_PROJ_QF_LAYERNORM: "v.proj_blk.{bid}.qf.layernorm", + MODEL_TENSOR.V_MULTI_PROJ_IMG_POS: "v.proj_blk.{bid}.img_pos", + MODEL_TENSOR.V_MULTI_PROJ_QUERY: "v.proj_blk.{bid}.query", + MODEL_TENSOR.V_MULTI_PROJ_NORM: "v.proj_blk.{bid}.norm", + MODEL_TENSOR.V_MULTI_PROJ_LINEAR: "v.proj_blk.{bid}.linear", + MODEL_TENSOR.V_MULTI_PROJ_POST_NORM: "v.proj_blk.{bid}.post_norm", # audio (mtmd) # note: all audio tensor names must use prefix "a." or "mm.a." @@ -1576,9 +1576,9 @@ class MODEL_TENSOR(IntEnum): MODEL_TENSOR.V_QF_PROJ_NORM, MODEL_TENSOR.V_MULTI_PROJ_IMG_POS, MODEL_TENSOR.V_MULTI_PROJ_QUERY, - MODEL_TENSOR.V_MULTI_PROJ_NORM, MODEL_TENSOR.V_MULTI_PROJ_LINEAR, - MODEL_TENSOR.V_MULTI_PROJ_QF_LAYERNORM, + MODEL_TENSOR.V_MULTI_PROJ_NORM, + MODEL_TENSOR.V_MULTI_PROJ_POST_NORM, # audio MODEL_TENSOR.A_ENC_EMBD_POS, MODEL_TENSOR.A_ENC_EMBD_NORM, diff --git a/gguf-py/gguf/tensor_mapping.py b/gguf-py/gguf/tensor_mapping.py index cb53fc4686b6..69551ddc5c1e 100644 --- a/gguf-py/gguf/tensor_mapping.py +++ b/gguf-py/gguf/tensor_mapping.py @@ -1943,18 +1943,18 @@ class TensorNameMap: "model.layerwise_projectors.{bid}.query", # Granite4 Vision "model.spatial_projectors.{bid}.query", # Granite4 Vision ), - MODEL_TENSOR.V_MULTI_PROJ_NORM: ( - "model.layerwise_projectors.{bid}.norm", # Granite4 Vision - "model.spatial_projectors.{bid}.norm", # Granite4 Vision - ), MODEL_TENSOR.V_MULTI_PROJ_LINEAR: ( "model.layerwise_projectors.{bid}.out_linear", # Granite4 Vision "model.spatial_projectors.{bid}.out_linear", # Granite4 Vision ), - MODEL_TENSOR.V_MULTI_PROJ_QF_LAYERNORM: ( + MODEL_TENSOR.V_MULTI_PROJ_NORM: ( "model.layerwise_projectors.{bid}.qformer.layernorm", # Granite4 Vision "model.spatial_projectors.{bid}.qformer.layernorm", # Granite4 Vision ), + MODEL_TENSOR.V_MULTI_PROJ_POST_NORM: ( + "model.layerwise_projectors.{bid}.norm", # Granite4 Vision + "model.spatial_projectors.{bid}.norm", # Granite4 Vision + ), # For these tensors, bid => proj-id MODEL_TENSOR.V_QF_SELF_ATTN_Q: ( From 493111f98c906fd75e8afb257fee8c57447a0acb Mon Sep 17 00:00:00 2001 From: Gabe Goodhart Date: Fri, 15 May 2026 17:11:19 -0600 Subject: [PATCH 40/94] fix: Use the right portion of speech for tensor loading! Also plumb through the layernorm -> post_norm naming change Branch: Granite4Vision AI-usage: none Signed-off-by: Gabe Goodhart --- tools/mtmd/clip-impl.h | 6 ++-- tools/mtmd/clip-model.h | 4 +-- tools/mtmd/clip.cpp | 64 +++++++++++++++++++++++------------------ 3 files changed, 41 insertions(+), 33 deletions(-) diff --git a/tools/mtmd/clip-impl.h b/tools/mtmd/clip-impl.h index 4e08d26aa2a6..c121f624a13d 100644 --- a/tools/mtmd/clip-impl.h +++ b/tools/mtmd/clip-impl.h @@ -210,8 +210,8 @@ #define TN_CTC_OUT_MID "a.enc_ctc_out_mid.%s" #define TN_ATTN_REL_POS_EMB "%s.blk.%d.attn_rel_pos_emb" // qformer projector -#define TN_QF_PROJ_QUERY "a.proj_query" -#define TN_QF_PROJ_NORM "%s.proj_norm.%s" // NOTE: This _should_ be qf_proj_norm +#define TN_QF_PROJ_QUERY "%s.proj_query" +#define TN_QF_PROJ_NORM "%s.proj_norm.%s" #define TN_QF_PROJ_LINEAR "%s.proj_linear.%s" #define TN_QF_SELF_ATTN_Q "%s.proj_blk.%d.self_attn_q.%s" #define TN_QF_SELF_ATTN_K "%s.proj_blk.%d.self_attn_k.%s" @@ -229,9 +229,9 @@ // multi-projector qformer (bid => projector ID) #define TN_MULTI_PROJ_IMG_POS "v.proj_blk.%d.img_pos" #define TN_MULTI_PROJ_QUERY "%s.proj_blk.%d.query" -#define TN_MULTI_PROJ_LAYERNORM "%s.proj_blk.%d.layernorm.%s" // NOTE: Disambig with QF_PROJ_NORM #define TN_MULTI_PROJ_LINEAR "%s.proj_blk.%d.linear.%s" #define TN_MULTI_PROJ_NORM "%s.proj_blk.%d.norm.%s" +#define TN_MULTI_PROJ_POST_NORM "%s.proj_blk.%d.post_norm.%s" // gemma4 audio conformer #define TN_A_MM_INP_PROJ "mm.a.input_projection.%s" diff --git a/tools/mtmd/clip-model.h b/tools/mtmd/clip-model.h index 988e5dab5ada..02dc2bc2e11a 100644 --- a/tools/mtmd/clip-model.h +++ b/tools/mtmd/clip-model.h @@ -337,8 +337,8 @@ struct qf_block { ggml_tensor * qf_proj_norm_b = nullptr; ggml_tensor * qf_proj_linear_w = nullptr; ggml_tensor * qf_proj_linear_b = nullptr; - ggml_tensor * qf_proj_layernorm_w = nullptr; - ggml_tensor * qf_proj_layernorm_b = nullptr; + ggml_tensor * qf_proj_post_norm_w = nullptr; + ggml_tensor * qf_proj_post_norm_b = nullptr; ggml_tensor * qf_proj_img_pos = nullptr; // Vision only std::vector qf_proj_layers; }; diff --git a/tools/mtmd/clip.cpp b/tools/mtmd/clip.cpp index 546356fc7c0b..d2cf44f3b963 100644 --- a/tools/mtmd/clip.cpp +++ b/tools/mtmd/clip.cpp @@ -2618,42 +2618,50 @@ struct clip_model_loader { auto & b = model.qf_proj_blocks[bid]; // non-layerwise tensors - b.qf_proj_norm_w = get_tensor(string_format(TN_MULTI_PROJ_NORM, prefix, bid, "weight")); - b.qf_proj_norm_b = get_tensor(string_format(TN_MULTI_PROJ_NORM, prefix, bid, "bias")); - b.qf_proj_query = get_tensor(string_format(TN_MULTI_PROJ_QUERY, prefix, bid)); b.qf_proj_img_pos = get_tensor(string_format(TN_MULTI_PROJ_IMG_POS, bid)); - b.qf_proj_layernorm_w = get_tensor(string_format(TN_MULTI_PROJ_LAYERNORM, prefix, bid, "weight")); - b.qf_proj_layernorm_b = get_tensor(string_format(TN_MULTI_PROJ_LAYERNORM, prefix, bid, "bias")); + b.qf_proj_query = get_tensor(string_format(TN_MULTI_PROJ_QUERY, prefix, bid)); b.qf_proj_linear_w = get_tensor(string_format(TN_MULTI_PROJ_LINEAR, prefix, bid, "weight")); b.qf_proj_linear_b = get_tensor(string_format(TN_MULTI_PROJ_LINEAR, prefix, bid, "bias")); + b.qf_proj_norm_w = get_tensor(string_format(TN_MULTI_PROJ_NORM, prefix, bid, "weight")); + b.qf_proj_norm_b = get_tensor(string_format(TN_MULTI_PROJ_NORM, prefix, bid, "bias")); + b.qf_proj_post_norm_w = get_tensor(string_format(TN_MULTI_PROJ_POST_NORM, prefix, bid, "weight")); + b.qf_proj_post_norm_b = get_tensor(string_format(TN_MULTI_PROJ_POST_NORM, prefix, bid, "bias")); // laywerwise tensors // NOTE: If any model uses multi-layer qformers, this will need to change b.qf_proj_layers.resize(1); - auto & layer = b.qf_proj_layers[0]; - - layer.attn_rel_pos_emb = get_tensor(string_format(TN_ATTN_REL_POS_EMB, prefix, bid)); - - layer.ff_norm_w = get_tensor(string_format(TN_FFN_NORM, prefix, bid, "weight")); - layer.ff_norm_b = get_tensor(string_format(TN_FFN_NORM, prefix, bid, "bias")); - - layer.ff_norm_1_w = get_tensor(string_format(TN_FFN_NORM_1, prefix, bid, "weight")); - layer.ff_norm_1_b = get_tensor(string_format(TN_FFN_NORM_1, prefix, bid, "bias")); - layer.ff_up_1_w = get_tensor(string_format(TN_FFN_UP_1, prefix, bid, "weight")); - layer.ff_up_1_b = get_tensor(string_format(TN_FFN_UP_1, prefix, bid, "bias")); - layer.ff_down_1_w = get_tensor(string_format(TN_FFN_DOWN_1, prefix, bid, "weight")); - layer.ff_down_1_b = get_tensor(string_format(TN_FFN_DOWN_1, prefix, bid, "bias")); - - layer.norm_conv_w = get_tensor(string_format(TN_NORM_CONV, prefix, bid, "weight")); - layer.norm_conv_b = get_tensor(string_format(TN_NORM_CONV, prefix, bid, "bias")); - layer.conv_norm_w = get_tensor(string_format(TN_CONV_NORM, prefix, bid, "weight")); - layer.conv_norm_b = get_tensor(string_format(TN_CONV_NORM, prefix, bid, "bias")); - layer.conv_dw_w = get_tensor(string_format(TN_CONV_DW, prefix, bid, "weight")); - layer.conv_pw1_w = get_tensor(string_format(TN_CONV_PW1, prefix, bid, "weight")); - layer.conv_pw1_b = get_tensor(string_format(TN_CONV_PW1, prefix, bid, "bias")); - layer.conv_pw2_w = get_tensor(string_format(TN_CONV_PW2, prefix, bid, "weight")); - layer.conv_pw2_b = get_tensor(string_format(TN_CONV_PW2, prefix, bid, "bias")); + auto & pl = b.qf_proj_layers[0]; + + pl.q_w = get_tensor(string_format(TN_QF_SELF_ATTN_Q, prefix, bid, "weight")); + pl.q_b = get_tensor(string_format(TN_QF_SELF_ATTN_Q, prefix, bid, "bias")); + pl.k_w = get_tensor(string_format(TN_QF_SELF_ATTN_K, prefix, bid, "weight")); + pl.k_b = get_tensor(string_format(TN_QF_SELF_ATTN_K, prefix, bid, "bias")); + pl.v_w = get_tensor(string_format(TN_QF_SELF_ATTN_V, prefix, bid, "weight")); + pl.v_b = get_tensor(string_format(TN_QF_SELF_ATTN_V, prefix, bid, "bias")); + pl.o_w = get_tensor(string_format(TN_QF_SELF_ATTN_O, prefix, bid, "weight")); + pl.o_b = get_tensor(string_format(TN_QF_SELF_ATTN_O, prefix, bid, "bias")); + pl.ln_1_w = get_tensor(string_format(TN_QF_SELF_ATTN_N, prefix, bid, "weight")); + pl.ln_1_b = get_tensor(string_format(TN_QF_SELF_ATTN_N, prefix, bid, "bias")); + + pl.cross_attn_q_w = get_tensor(string_format(TN_QF_CROSS_ATTN_Q, prefix, bid, "weight")); + pl.cross_attn_q_b = get_tensor(string_format(TN_QF_CROSS_ATTN_Q, prefix, bid, "bias")); + pl.cross_attn_k_w = get_tensor(string_format(TN_QF_CROSS_ATTN_K, prefix, bid, "weight")); + pl.cross_attn_k_b = get_tensor(string_format(TN_QF_CROSS_ATTN_K, prefix, bid, "bias")); + pl.cross_attn_v_w = get_tensor(string_format(TN_QF_CROSS_ATTN_V, prefix, bid, "weight")); + pl.cross_attn_v_b = get_tensor(string_format(TN_QF_CROSS_ATTN_V, prefix, bid, "bias")); + pl.cross_attn_o_w = get_tensor(string_format(TN_QF_CROSS_ATTN_O, prefix, bid, "weight")); + pl.cross_attn_o_b = get_tensor(string_format(TN_QF_CROSS_ATTN_O, prefix, bid, "bias")); + pl.cross_attn_norm_w = get_tensor(string_format(TN_QF_CROSS_ATTN_N, prefix, bid, "weight")); + pl.cross_attn_norm_b = get_tensor(string_format(TN_QF_CROSS_ATTN_N, prefix, bid, "bias")); + + pl.ff_up_w = get_tensor(string_format(TN_QF_FFN_UP, prefix, bid, "weight")); + pl.ff_up_b = get_tensor(string_format(TN_QF_FFN_UP, prefix, bid, "bias")); + pl.ff_down_w = get_tensor(string_format(TN_QF_FFN_DOWN, prefix, bid, "weight")); + pl.ff_down_b = get_tensor(string_format(TN_QF_FFN_DOWN, prefix, bid, "bias")); + pl.ln_2_w = get_tensor(string_format(TN_QF_FFN_NORM, prefix, bid, "weight")); + pl.ln_2_b = get_tensor(string_format(TN_QF_FFN_NORM, prefix, bid, "bias")); } + } break; default: GGML_ASSERT(false && "unknown projector type"); From 5e7231a19e1fb6d822094ae9f2cd04b8feaa49db Mon Sep 17 00:00:00 2001 From: Gabe Goodhart Date: Tue, 19 May 2026 10:38:30 -0600 Subject: [PATCH 41/94] feat: Add logging of deepstack_layers_arr if set I also changed the print_f output type to int32_t to avoid printing overflow values for -1. This could cause overflows on the other side, but I can't imagine a value for any of the current array hparams that would trigger that. Branch: Granite4Vision AI-usage: none Signed-off-by: Gabe Goodhart --- src/llama-model.cpp | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/src/llama-model.cpp b/src/llama-model.cpp index 2ad2c09381e4..cd26e2c16e97 100644 --- a/src/llama-model.cpp +++ b/src/llama-model.cpp @@ -1631,10 +1631,10 @@ uint64_t llama_model::n_elements() const { void llama_model::print_info() const { const std::string rope_scaling_type = llama_rope_scaling_type_name(hparams.rope_scaling_type_train); - auto print_f = [](const std::function & f, uint32_t n) { + auto print_f = [](const std::function & f, uint32_t n) { bool is_var = false; - std::vector v; + std::vector v; for (uint32_t i = 0; i < n; ++i) { v.push_back(f(i)); if (v[i] != v[0]) { @@ -1708,6 +1708,13 @@ void llama_model::print_info() const { LLAMA_LOG_INFO("%s: n_ctx_orig_yarn = %u\n", __func__, hparams.n_ctx_orig_yarn); LLAMA_LOG_INFO("%s: rope_yarn_log_mul = %.4f\n", __func__, hparams.rope_yarn_log_mul); LLAMA_LOG_INFO("%s: rope_finetuned = %s\n", __func__, hparams.rope_finetuned ? "yes" : "unknown"); + if (std::any_of(hparams.deepstack_layers_arr.begin(), + hparams.deepstack_layers_arr.end(), + [](const auto & entry) { return entry >= 0; })) { + LLAMA_LOG_INFO("%s: deepstack_layers_arr = %s\n", __func__, + print_f([&](uint32_t il) { return hparams.deepstack_layers_arr[il]; }, + hparams.n_layer).c_str()); + } // MRoPE (Multi-axis Rotary Position Embedding) sections if (const auto & s = hparams.rope_sections; s[0] || s[1] || s[2] || s[3]) { LLAMA_LOG_INFO("%s: mrope sections = [%d, %d, %d, %d]\n", __func__, s[0], s[1], s[2], s[3]); From d072dc9e9cb67b9b067f0dd23fe6ce145c5ea17d Mon Sep 17 00:00:00 2001 From: Gabe Goodhart Date: Tue, 19 May 2026 11:11:50 -0600 Subject: [PATCH 42/94] fix: Make sure input embeddings are cont before f_embedding_scale Branch: Granite4Vision AI-usage: none Signed-off-by: Gabe Goodhart --- src/llama-graph.cpp | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/llama-graph.cpp b/src/llama-graph.cpp index fe155c92dea0..e71cdd340cc4 100644 --- a/src/llama-graph.cpp +++ b/src/llama-graph.cpp @@ -1776,6 +1776,9 @@ ggml_tensor * llm_graph_context::build_inp_embd(ggml_tensor * tok_embd) const { // For Granite architecture if (hparams.f_embedding_scale != 0.0f) { + if (!ggml_is_contiguous(cur)) { + cur = ggml_cont(ctx0, cur); + } cur = ggml_scale(ctx0, cur, hparams.f_embedding_scale); } From 0f65a0d91add7ff57dfb436e689c0061ae3ffc6a Mon Sep 17 00:00:00 2001 From: Gabe Goodhart Date: Tue, 19 May 2026 11:13:02 -0600 Subject: [PATCH 43/94] feat: Add init and mmproj_embd cases for g4v The n_mmproj_embd is 1+ to make space for the text embedding and all 8 projectors Branch: Granite4Vision AI-usage: draft (Bob) Signed-off-by: Gabe Goodhart --- tools/mtmd/clip.cpp | 6 ++++++ tools/mtmd/mtmd.cpp | 7 +++++++ 2 files changed, 13 insertions(+) diff --git a/tools/mtmd/clip.cpp b/tools/mtmd/clip.cpp index d2cf44f3b963..5cea54511a57 100644 --- a/tools/mtmd/clip.cpp +++ b/tools/mtmd/clip.cpp @@ -955,6 +955,10 @@ static ggml_cgraph * clip_image_build_graph(clip_ctx * ctx, const clip_image_f32 { builder = std::make_unique(ctx, img); } break; + case PROJECTOR_TYPE_GRANITE4_VISION: + { + builder = std::make_unique(ctx, img); + } break; case PROJECTOR_TYPE_GLM4V: { builder = std::make_unique(ctx, img); @@ -4327,6 +4331,8 @@ int clip_n_mmproj_embd(const struct clip_ctx * ctx) { return ctx->model.hparams.projection_dim; case PROJECTOR_TYPE_GRANITE_SPEECH: return ctx->model.qf_proj_blocks[0].qf_proj_linear_w->ne[1]; + case PROJECTOR_TYPE_GRANITE4_VISION: + return ctx->model.qf_proj_blocks[0].qf_proj_linear_w->ne[1] * (1 + ctx->model.qf_proj_blocks.size()); case PROJECTOR_TYPE_GLM4V: return ctx->model.mm_ffn_down_w->ne[1]; default: diff --git a/tools/mtmd/mtmd.cpp b/tools/mtmd/mtmd.cpp index 1ab8a4c0464a..706e31d72148 100644 --- a/tools/mtmd/mtmd.cpp +++ b/tools/mtmd/mtmd.cpp @@ -496,6 +496,13 @@ struct mtmd_context { img_end = "<｜hy_place▁holder▁no▁101｜>"; image_preproc = std::make_unique(ctx_v); } break; + case PROJECTOR_TYPE_GRANITE4_VISION: + { + // <|image_embeds|> ... (image embeddings) ... + img_beg = "<|image_embeds|>"; + img_end = ""; + image_preproc = std::make_unique(ctx_v); + } break; default: throw std::runtime_error(string_format("%s: unexpected vision projector type %d\n", __func__, proj)); } From 87e363b89932375fdc43603c28eb4f6adb620a5e Mon Sep 17 00:00:00 2001 From: Gabe Goodhart Date: Tue, 19 May 2026 13:44:58 -0600 Subject: [PATCH 44/94] fix: Invert (h, w) -> (w, h) pinpoints Branch: Granite4Vision AI-usage: none Signed-off-by: Gabe Goodhart --- convert_hf_to_gguf.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/convert_hf_to_gguf.py b/convert_hf_to_gguf.py index 8dd4bc70959a..d8096500da62 100755 --- a/convert_hf_to_gguf.py +++ b/convert_hf_to_gguf.py @@ -13171,7 +13171,8 @@ def set_gguf_parameters(self): # Add flattened image grind pinpoints (resolution candidates internally) if pinpoints := self.global_config.get("image_grid_pinpoints"): - pinpoints = [x for pair in pinpoints for x in pair] + # Flatten with h, w -> w, h inversion + pinpoints = [val for h, w in pinpoints for val in (w, h)] self.gguf_writer.add_vision_image_grid_pinpoints(pinpoints) @classmethod From 8c976a05ea2faf518204c35cde7444b025aba47f Mon Sep 17 00:00:00 2001 From: Gabe Goodhart Date: Tue, 19 May 2026 16:13:11 -0600 Subject: [PATCH 45/94] fix: Reorder projectors based on llm index and skip the first injection The multi-projector stack has a strange asymmetry based on how it's currently implemented for qwen3vl: on the mmproj side, it's all N projectors, but the output of the "first" (by inp_embd index) projector is automatically consumed as if it were a standard single-projector mmproj, so the deepstack portion needs to only contain the 1-N entries. Branch: Granite4Vision AI-usage: none Signed-off-by: Gabe Goodhart Co-authored-by: Eli Schwartz --- convert_hf_to_gguf.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/convert_hf_to_gguf.py b/convert_hf_to_gguf.py index d8096500da62..2eaea750fad0 100755 --- a/convert_hf_to_gguf.py +++ b/convert_hf_to_gguf.py @@ -11069,6 +11069,10 @@ def set_gguf_parameters(self): normalized_projector_map = Granite4VisionMmprojModel.get_normalized_projector_map(self.hparams) deepstack_layer_arr = [-1 for _ in range(self.block_count)] # Populate with -1 sentinels for proj_idx, (_, llm_layer, _, _) in enumerate(normalized_projector_map): + # Skip the first projector which is handled as the base embedding + # stream like normal + if proj_idx == 0: + continue deepstack_layer_arr[llm_layer] = proj_idx self.gguf_writer.add_num_deepstack_layers(deepstack_layer_arr) @@ -13110,7 +13114,7 @@ def get_normalized_projector_map(global_config: dict) -> list[tuple[int, int, st spatial_vision_layer = n_vision_layers + spatial_vision_layer for spatial_idx, llm_layer in enumerate(spatial_layers): normalized_projector_map.append((spatial_vision_layer, llm_layer, "spatial", spatial_idx)) - return list(sorted(normalized_projector_map)) + return list(sorted(normalized_projector_map, key=(lambda entry: entry[1]))) def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) From b1ab316e8d9132012902deeb6cf0d56b7469ba49 Mon Sep 17 00:00:00 2001 From: Gabe Goodhart Date: Tue, 19 May 2026 16:36:17 -0600 Subject: [PATCH 46/94] fix: Fix mmproj hparams in conversion Branch: Granite4Vision AI-usage: none Signed-off-by: Gabe Goodhart Co-authored-by: Eli Schwartz --- convert_hf_to_gguf.py | 29 +++++++++-------------------- 1 file changed, 9 insertions(+), 20 deletions(-) diff --git a/convert_hf_to_gguf.py b/convert_hf_to_gguf.py index 2eaea750fad0..bcacf6475d45 100755 --- a/convert_hf_to_gguf.py +++ b/convert_hf_to_gguf.py @@ -13133,20 +13133,17 @@ def __init__(self, *args, **kwargs): def set_gguf_parameters(self): assert self.hparams_vision is not None - v = self.hparams_vision - super().set_gguf_parameters() + v = self.hparams_vision - # Set projector type self.gguf_writer.add_clip_projector_type(gguf.VisionProjectorType.GRANITE4_VISION) - # Vision encoder params (SigLIP) - self.gguf_writer.add_vision_embedding_length(v["hidden_size"]) - self.gguf_writer.add_vision_feed_forward_length(v["intermediate_size"]) - self.gguf_writer.add_vision_block_count(v["num_hidden_layers"]) - self.gguf_writer.add_vision_head_count(v["num_attention_heads"]) - self.gguf_writer.add_vision_image_size(v["image_size"]) - self.gguf_writer.add_vision_patch_size(v["patch_size"]) + # SigLIP encoder hparams + self.gguf_writer.add_vision_attention_layernorm_eps(self.hparams.get("layer_norm_eps", 1e-6)) + self.gguf_writer.add_vision_use_gelu(True) + + # Preprocessor + self.gguf_writer.add_vision_preproc_image_size(self.hparams.get("image_size", 384)) # QFormer projector config ds_rate = self.global_config["downsample_rate"] @@ -13156,11 +13153,6 @@ def set_gguf_parameters(self): self.gguf_writer.add_vision_projector_query_side(query_side) self.gguf_writer.add_vision_projector_window_side(window_side) - # Layer norm epsilon for ViT (use text model epsilon as fallback) - text_config = self.global_config.get("text_config", {}) - eps = v.get("rms_norm_eps", text_config.get("rms_norm_eps", 1e-5)) - self.gguf_writer.add_vision_attention_layernorm_eps(eps) - # Set vision feature layers self.gguf_writer.add_vision_feature_layers(self._vision_feature_layers) @@ -13170,7 +13162,7 @@ def set_gguf_parameters(self): # The granite LLM will scale the first embedding input by # embedding_multiplier, so the mmproj needs to invert that scale for its # output so that the resulting injection works as expected. - if (embed_mult := text_config.get("embedding_multiplier")) is not None: + if (embed_mult := self.global_config.get("text_config", {}).get("embedding_multiplier")) is not None: self.gguf_writer.add_embedding_scale(embed_mult) # Add flattened image grind pinpoints (resolution candidates internally) @@ -13217,10 +13209,7 @@ def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iter else: # len(all_ids) == 2 new_bid = projector_idx # + all_ids[1] new_name = name[:id_matches[0].span(0)[0]] + name[id_matches[0].span(1)[1]:id_matches[1].span(1)[0]] + str(new_bid) + name[id_matches[1].span(1)[1]:] - try: - yield from super().modify_tensors(data_torch, new_name, new_bid) - except Exception as err: - breakpoint() + yield from super().modify_tensors(data_torch, new_name, new_bid) return yield from super().modify_tensors(data_torch, name, bid) From 02eabed85b960204ec0ebf113797b62ae5b6219b Mon Sep 17 00:00:00 2001 From: Gabe Goodhart Date: Tue, 19 May 2026 16:37:03 -0600 Subject: [PATCH 47/94] fix: Fix ordering/logic for deepstack injection in granite Branch: Granite4Vision AI-usage: none Signed-off-by: Gabe Goodhart Co-authored-by: Eli Schwartz --- src/models/granite.cpp | 33 +++++++++++++++++++++++++-------- 1 file changed, 25 insertions(+), 8 deletions(-) diff --git a/src/models/granite.cpp b/src/models/granite.cpp index d382986e8799..a07efa9b561a 100644 --- a/src/models/granite.cpp +++ b/src/models/granite.cpp @@ -112,19 +112,36 @@ llama_model_granite::graph::graph( ggml_tensor * inp_out_ids = build_inp_out_ids(); for (int il = 0; il < n_layer; ++il) { - ggml_tensor * inpSA = inpL; - cur = inpL; - // apply deepstack injection for Granite4 Vision + // Granite Vision 4.1 deepstack: inject the projector stream that + // targets decoder layer `il`, BEFORE the decoder layer runs. This + // matches the reference forward in + // /ibm-granite/granite-vision-4.1/modeling.py: for each layer_idx, + // check the (llm_layer, packed_features) pairs and add to + // hidden_states[image_positions] if layer_idx == llm_layer. + // + // In the qwen3vl-deepstack layout the mmproj writes streams into + // res->t_inp_embd at offset (stream_index + 1) * n_embd (stream 0 + // is the base and is already consumed via build_inp_embd). For + // pure-text requests the input embedding carries zeros at those + // offsets (see build_inp_embd's zero-pad of the token branch), so + // the add is a no-op. const auto & deepstack_emb_idx = hparams.deepstack_layers_arr[il]; - if (deepstack_emb_idx >= 0) { - ggml_tensor * ds = ggml_view_2d(ctx0, res->t_inp_embd, n_embd, n_tokens, res->t_inp_embd->nb[1], deepstack_emb_idx * n_embd * sizeof(float)); - cur = ggml_add(ctx0, cur, ds); - cb(cur, "deepstack_out", il); + if (il > 0 && deepstack_emb_idx >= 0) { + // Note: il == 0 uses the stream at offset 0, which is already + // inpL (= base == stream at llm_layer 0 for granite vision 4.1). + ggml_tensor * ds = ggml_view_2d(ctx0, + res->t_inp_embd, n_embd, n_tokens, + res->t_inp_embd->nb[1], + deepstack_emb_idx * n_embd * sizeof(float)); + inpL = ggml_add(ctx0, inpL, ds); + cb(inpL, "deepstack_in", il); } + ggml_tensor * inpSA = inpL; + // norm - cur = build_norm(cur, + cur = build_norm(inpL, model.layers[il].attn_norm, NULL, LLM_NORM_RMS, il); cb(cur, "attn_norm", il); From af636f5d3c8b808e1a98a9f3e378cc4344ee92f3 Mon Sep 17 00:00:00 2001 From: Gabe Goodhart Date: Tue, 19 May 2026 16:38:15 -0600 Subject: [PATCH 48/94] fix: Fix preprocessing config to match what the model needs Branch: Granite4Vision AI-usage: none Signed-off-by: Gabe Goodhart Co-authored-by: Eli Schwartz --- tools/mtmd/mtmd.cpp | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/tools/mtmd/mtmd.cpp b/tools/mtmd/mtmd.cpp index 706e31d72148..b6f02318ec01 100644 --- a/tools/mtmd/mtmd.cpp +++ b/tools/mtmd/mtmd.cpp @@ -498,10 +498,16 @@ struct mtmd_context { } break; case PROJECTOR_TYPE_GRANITE4_VISION: { - // <|image_embeds|> ... (image embeddings) ... - img_beg = "<|image_embeds|>"; + // Granite Vision 4.1 uses a single "" token to + // mark image positions; see modeling.py::get_image_token_mask. + // LlavaNext-style multi-tile splitting drives off the + // image_grid_pinpoints mirrored into + // hparams.image_res_candidates by the mmproj loader. + // Per-tile encoder loop + pack_and_unpad happens inside + // mtmd_encode (PROJECTOR_TYPE_GRANITE4_VISION branch). + img_beg = ""; img_end = ""; - image_preproc = std::make_unique(ctx_v); + image_preproc = std::make_unique(ctx_v); } break; default: throw std::runtime_error(string_format("%s: unexpected vision projector type %d\n", __func__, proj)); From d655ee690526e2f5cfc5f0fdb0bacd9798073e40 Mon Sep 17 00:00:00 2001 From: Gabe Goodhart Date: Tue, 19 May 2026 17:18:23 -0600 Subject: [PATCH 49/94] wip: Partial port of Eli's implementation This is still pretty broken, but it's getting closer. It now happily generates tokens, but the values are quite incorrect still. I suspect it's caused by the mapping of projectors from safetensors to their respective orders here. Also, this implementation breaks encapsulation pretty badly in mtmd_encode. This will need a big refactor to put the G4V-specific encoding logic somewhere more appropriate. Branch: Granite4Vision AI-usage: draft (Claude Code, Bob) Signed-off-by: Gabe Goodhart Co-authored-by: Eli Schwartz --- tools/mtmd/clip.cpp | 112 ++++++++- tools/mtmd/models/granite-vision.cpp | 331 ++++++++++++++++++++++++++- tools/mtmd/mtmd.cpp | 164 +++++++++++++ 3 files changed, 600 insertions(+), 7 deletions(-) diff --git a/tools/mtmd/clip.cpp b/tools/mtmd/clip.cpp index 5cea54511a57..c97004bf97ee 100644 --- a/tools/mtmd/clip.cpp +++ b/tools/mtmd/clip.cpp @@ -955,10 +955,6 @@ static ggml_cgraph * clip_image_build_graph(clip_ctx * ctx, const clip_image_f32 { builder = std::make_unique(ctx, img); } break; - case PROJECTOR_TYPE_GRANITE4_VISION: - { - builder = std::make_unique(ctx, img); - } break; case PROJECTOR_TYPE_GLM4V: { builder = std::make_unique(ctx, img); @@ -975,6 +971,10 @@ static ggml_cgraph * clip_image_build_graph(clip_ctx * ctx, const clip_image_f32 { builder = std::make_unique(ctx, img); } break; + case PROJECTOR_TYPE_GRANITE4_VISION: + { + builder = std::make_unique(ctx, img); + } break; default: GGML_ABORT("missing cgraph builder"); } @@ -3379,6 +3379,19 @@ int clip_n_output_tokens(const struct clip_ctx * ctx, struct clip_image_f32 * im const int ds = ctx->model.hparams.audio_proj_downsample_rate; n_patches = ((img->nx + ws - 1) / ws) * (ws / ds); } break; + case PROJECTOR_TYPE_GRANITE4_VISION: + { + // Per-tile output token count: each projector block outputs + // query_side^2 tokens per window × n^2 windows. + // For 384×384 input: n = 24/8 = 3, query_side = 4 → 144. + // The +1 image_newline row / newline column logic runs in + // mtmd_encode's pack_and_unpad path, not here. + const int window_side = ctx->model.hparams.downsample_window_side; + const int query_side = ctx->model.hparams.downsample_query_side; + const int side = img->nx / params.patch_size; + const int n = side / window_side; + n_patches = (query_side * n) * (query_side * n); + } break; default: GGML_ABORT("unsupported projector type"); } @@ -4177,6 +4190,92 @@ bool clip_image_batch_encode(clip_ctx * ctx, const int n_threads, const clip_ima set_input_f32("attn_mask", mask); } } break; + case PROJECTOR_TYPE_GRANITE4_VISION: + { + // Granite Vision 4.1 uses precomputed permutation index + // tensors to express the _win / _unwin / spatial sampling + // reshapes as ggml_get_rows gathers. The names are set + // by g4v_gather() in models/granite4v.cpp. + const int patch_size = model.hparams.patch_size; + const int image_side = imgs.entries.front()->nx / patch_size; + const int window_side = hparams.downsample_window_side; + const int query_side = hparams.downsample_query_side; + const int n = image_side / window_side; + const int new_side = n * query_side; + + // Builds the raster→window permutation indices for a + // (side, side) grid split into (n × n) windows of (win × win) + // tokens each. dst[w * win*win + p] = source raster index. + auto make_win_idx = [](int side, int win) { + const int nn = side / win; + std::vector idx(static_cast(side) * side); + for (int wy = 0; wy < nn; ++wy) { + for (int wx = 0; wx < nn; ++wx) { + for (int iy = 0; iy < win; ++iy) { + for (int ix = 0; ix < win; ++ix) { + const int w = wy * nn + wx; + const int p = iy * win + ix; + const int y = wy * win + iy; + const int x = wx * win + ix; + idx[static_cast(w) * (win*win) + p] = y * side + x; + } + } + } + } + return idx; + }; + + auto make_unwin_idx = [&](int side, int win) { + const std::vector fwd = make_win_idx(side, win); + std::vector inv(fwd.size()); + for (size_t i = 0; i < fwd.size(); ++i) { + inv[fwd[i]] = static_cast(i); + } + return inv; + }; + + auto make_spatial_idx = [](int side, int offset) { + const int off_y = (offset >> 1) & 1; + const int off_x = offset & 1; + const int new_s = side / 2; + std::vector idx(static_cast(new_s) * new_s); + for (int y = 0; y < new_s; ++y) { + for (int x = 0; x < new_s; ++x) { + idx[y * new_s + x] = (y * 2 + off_y) * side + (x * 2 + off_x); + } + } + return idx; + }; + + auto upload = [&](const std::string & name, const std::vector & idx) { + ggml_tensor * t = ggml_graph_get_tensor(gf, name.c_str()); + if (t) { + if (std::getenv("G4V_DEBUG_NAMES")) { + std::fprintf(stderr, "granite4v: upload %s len=%zu first=%d\n", + name.c_str(), idx.size(), + idx.empty() ? -1 : idx.front()); + } + ggml_backend_tensor_set(t, idx.data(), 0, idx.size() * sizeof(int32_t)); + } else if (std::getenv("G4V_DEBUG_NAMES")) { + std::fprintf(stderr, "granite4v: index tensor %s not found in graph\n", + name.c_str()); + } + }; + + // Stage 1b only uses block 0's permutations; future stages + // will upload all 8 blocks. upload() no-ops when a tensor + // is not present in the graph, so this is safe for either. + for (size_t bid = 0; bid < hparams.vision_feature_layer.size(); ++bid) { + const std::string prefix = "g4v_blk" + std::to_string(bid) + "_"; + upload(prefix + "win_idx", make_win_idx(image_side, window_side)); + upload(prefix + "qwin_idx", make_win_idx(new_side, query_side)); + upload(prefix + "unwin_idx", make_unwin_idx(new_side, query_side)); + const auto spatial_offset = hparams.proj_spatial_offsets[bid]; + if (spatial_offset >= 0) { + upload(prefix + "spatial_idx", make_spatial_idx(image_side,spatial_offset)); + } + } + } break; default: GGML_ABORT("Unknown projector type"); } @@ -4332,7 +4431,10 @@ int clip_n_mmproj_embd(const struct clip_ctx * ctx) { case PROJECTOR_TYPE_GRANITE_SPEECH: return ctx->model.qf_proj_blocks[0].qf_proj_linear_w->ne[1]; case PROJECTOR_TYPE_GRANITE4_VISION: - return ctx->model.qf_proj_blocks[0].qf_proj_linear_w->ne[1] * (1 + ctx->model.qf_proj_blocks.size()); + // Stage 2a: graph output is 8 concatenated streams (deepstack + // layout with "base" = stream targeting llm_layer 0). Per-token + // feature count = projector_count * projection_dim. + return ctx->model.qf_proj_blocks.size() * ctx->model.hparams.projection_dim; case PROJECTOR_TYPE_GLM4V: return ctx->model.mm_ffn_down_w->ne[1]; default: diff --git a/tools/mtmd/models/granite-vision.cpp b/tools/mtmd/models/granite-vision.cpp index 6334a33477f3..087562301f64 100644 --- a/tools/mtmd/models/granite-vision.cpp +++ b/tools/mtmd/models/granite-vision.cpp @@ -1,6 +1,333 @@ #include "models.h" +#include "../clip-impl.h" +#include "../clip-model.h" + +#include +#include +#include +#include +#include + +/* + * Granite Vision 4.1 clip graph + * + * Stage 1a: SigLIP vision tower (27 layers, post-norm) + * Stage 1b: WindowQFormer blocks (8 blocks total) + * Stage 1c: Concatenate and pack outputs + * + * A single WindowQFormer block maps a (1, 576, 1152) vision-layer + * hidden state h to a (1, 144, 2560) deepstack feature. + */ + +// --------------------------------------------------------------------------- +// Permutation helpers +// --------------------------------------------------------------------------- + +static ggml_tensor * g4v_gather(ggml_context * ctx, ggml_cgraph * gf, + ggml_tensor * src, + const std::string & name, + int idx_len) { + (void) gf; + ggml_tensor * idx = ggml_new_tensor_1d(ctx, GGML_TYPE_I32, idx_len); + ggml_set_name(idx, name.c_str()); + ggml_set_input(idx); + return ggml_get_rows(ctx, src, idx); +} + +// Area-interpolate 24x24 -> 12x12 via 2x2 avg pool +static ggml_tensor * g4v_interp_down(ggml_context * ctx, ggml_tensor * src, int side, int new_side) { + const int n_embd = src->ne[0]; + ggml_tensor * t = ggml_reshape_4d(ctx, src, n_embd, side, side, 1); + t = ggml_cont(ctx, ggml_permute(ctx, t, 2, 0, 1, 3)); + const int kernel = side / new_side; + t = ggml_pool_2d(ctx, t, GGML_OP_POOL_AVG, kernel, kernel, kernel, kernel, 0, 0); + t = ggml_cont(ctx, ggml_permute(ctx, t, 1, 2, 0, 3)); + return ggml_reshape_2d(ctx, t, n_embd, new_side * new_side); +} + +// Build one WindowQFormer block's forward pass +static ggml_tensor * g4v_build_block( + const clip_graph * g, + ggml_context * ctx, + ggml_cgraph * gf, + const qf_block & blk, + ggml_tensor * h, + int bid, + int spatial_offset, + int image_side, + int window_side, + int query_side, + float qformer_eps) { + + const int n_embd = h->ne[0]; + GGML_ASSERT(h->ne[1] == image_side * image_side); + const int n = image_side / window_side; + const int new_side = n * query_side; + const int n_windows = n * n; + const int enc_len = window_side * window_side; + const int query_len = query_side * query_side; + + auto cbx = [&](ggml_tensor * & t, const char * step) { + const std::string name = "g4v_blk" + std::to_string(bid) + "_" + step; + ggml_tensor * tagged = ggml_cont(ctx, t); + ggml_set_name(tagged, name.c_str()); + ggml_build_forward_expand(gf, tagged); + t = tagged; + }; + + // 1. Top-level LN + ggml_tensor * x = g->build_norm(h, blk.qf_proj_norm_w, blk.qf_proj_norm_b, NORM_TYPE_NORMAL, g->eps, bid); + cbx(x, "norm"); + + // 2. enc = _win(x, image_side, window_side) + ggml_tensor * enc; + { + ggml_tensor * enc_flat = g4v_gather(ctx, gf, x, + "g4v_blk" + std::to_string(bid) + "_win_idx", + image_side * image_side); + enc = ggml_reshape_3d(ctx, enc_flat, n_embd, enc_len, n_windows); + } + cbx(enc, "enc"); + + // 3. downsampled = downsampler(x) + ggml_tensor * d; + (void) spatial_offset; + if (spatial_offset >= 0) { + d = g4v_gather(ctx, gf, x, + "g4v_blk" + std::to_string(bid) + "_spatial_idx", + new_side * new_side); + } else { + d = g4v_interp_down(ctx, x, image_side, new_side); + } + cbx(d, "downsampled"); + + // 4. query_embeds = query + _win(d, new_side, query_side) + ggml_tensor * q_in; + { + ggml_tensor * dw_flat = g4v_gather(ctx, gf, d, + "g4v_blk" + std::to_string(bid) + "_qwin_idx", + new_side * new_side); + ggml_tensor * dw = ggml_reshape_3d(ctx, dw_flat, n_embd, query_len, n_windows); + q_in = ggml_add(ctx, dw, blk.qf_proj_query); + } + cbx(q_in, "query_embeds"); + + // 5. encoder_embeds = enc + image_positions → (C, enc_len, n_windows) + ggml_tensor * e_in = ggml_add(ctx, enc, blk.qf_proj_img_pos); + cbx(e_in, "encoder_embeds"); + + // 6. Qformer forward. Per Blip2QFormerModel.forward: first runs + // self.layernorm(query_embeds) before entering the single layer. + // blk.post_norm_* holds this (our naming is historical; see the + // comment in clip-model.h's g4v_projector_block struct). + ggml_tensor * q = g->build_norm(q_in, blk.qf_proj_post_norm_w, blk.qf_proj_post_norm_b, NORM_TYPE_NORMAL, qformer_eps, bid); + + // Helper: one post-norm BERT attention block. + // attn(q,k,v) via dense W^Q/W^K/W^V + scaled dot-product + // residual: LN_out(dropout(dense(attn_out)) + residual_input) + // For self-attention q == k == v == residual. For cross-attn q != k=v. + auto run_postnorm_attn = [&]( + ggml_tensor * q_stream, + ggml_tensor * kv_stream, + ggml_tensor * wq, ggml_tensor * bq, + ggml_tensor * wk, ggml_tensor * bk, + ggml_tensor * wv, ggml_tensor * bv, + ggml_tensor * wo, ggml_tensor * bo, + ggml_tensor * ln_w, ggml_tensor * ln_b) -> ggml_tensor * { + const int n_head = 18; // QFormer: 1152 / 64 + const int d_h = n_embd / n_head; + const int nq = q_stream->ne[1]; + const int nkv = kv_stream->ne[1]; + + // We loop over the n_windows dimension by treating it as batch via + // the ne[2] dim in mul_mat. Collapse (C, n, n_win) → (C, n * n_win) + // for the linear projections (they don't care about the window dim), + // then reshape back. + auto linear = [&](ggml_tensor * x, ggml_tensor * w, ggml_tensor * b) -> ggml_tensor * { + ggml_tensor * t = ggml_reshape_2d(ctx, x, x->ne[0], x->ne[1] * x->ne[2]); + t = g->build_mm(w, t); + if (b) t = ggml_add(ctx, t, b); + return t; + }; + + ggml_tensor * Q = linear(q_stream, wq, bq); + ggml_tensor * K = linear(kv_stream, wk, bk); + ggml_tensor * V = linear(kv_stream, wv, bv); + + Q = ggml_reshape_4d(ctx, Q, d_h, n_head, nq, n_windows); + K = ggml_reshape_4d(ctx, K, d_h, n_head, nkv, n_windows); + V = ggml_reshape_4d(ctx, V, d_h, n_head, nkv, n_windows); + + ggml_tensor * q_p = ggml_permute(ctx, Q, 0, 2, 1, 3); + ggml_tensor * k_p = ggml_permute(ctx, K, 0, 2, 1, 3); + ggml_tensor * v_p = ggml_permute(ctx, V, 1, 2, 0, 3); + v_p = ggml_cont(ctx, v_p); + + ggml_tensor * kq = ggml_mul_mat(ctx, k_p, q_p); + const float scale = 1.0f / std::sqrt((float) d_h); + kq = ggml_soft_max_ext(ctx, kq, nullptr, scale, 0.0f); + + ggml_tensor * kqv = ggml_mul_mat(ctx, v_p, kq); + ggml_tensor * out = ggml_permute(ctx, kqv, 0, 2, 1, 3); + out = ggml_cont_3d(ctx, out, d_h * n_head, nq, n_windows); + + ggml_tensor * o = ggml_reshape_2d(ctx, out, n_embd, nq * n_windows); + o = g->build_mm(wo, o); + if (bo) o = ggml_add(ctx, o, bo); + o = ggml_reshape_3d(ctx, o, n_embd, nq, n_windows); + o = ggml_add(ctx, o, q_stream); + o = g->build_norm(o, ln_w, ln_b, NORM_TYPE_NORMAL, qformer_eps, bid); + return o; + }; + + // Get the single QFormer layer + GGML_ASSERT(blk.qf_proj_layers.size() == 1); + const auto & pl = blk.qf_proj_layers[0]; + + // 6a. Self-attention + ggml_tensor * sa_out = run_postnorm_attn( + q, q, + pl.q_w, pl.q_b, + pl.k_w, pl.k_b, + pl.v_w, pl.v_b, + pl.o_w, pl.o_b, + pl.ln_1_w, pl.ln_1_b); + cbx(sa_out, "sa_out"); + + // 6b. Cross-attention + ggml_tensor * ca_out = run_postnorm_attn( + sa_out, e_in, + pl.cross_attn_q_w, pl.cross_attn_q_b, + pl.cross_attn_k_w, pl.cross_attn_k_b, + pl.cross_attn_v_w, pl.cross_attn_v_b, + pl.cross_attn_o_w, pl.cross_attn_o_b, + pl.cross_attn_norm_w, pl.cross_attn_norm_b); + cbx(ca_out, "ca_out"); + + // 6c. FFN + ggml_tensor * ffn; + { + ggml_tensor * t = ggml_reshape_2d(ctx, ca_out, n_embd, query_len * n_windows); + t = g->build_mm(pl.ff_up_w, t); + if (pl.ff_up_b) t = ggml_add(ctx, t, pl.ff_up_b); + t = ggml_gelu_erf(ctx, t); + t = g->build_mm(pl.ff_down_w, t); + if (pl.ff_down_b) t = ggml_add(ctx, t, pl.ff_down_b); + t = ggml_reshape_3d(ctx, t, n_embd, query_len, n_windows); + ffn = ggml_add(ctx, t, ca_out); + ffn = g->build_norm(ffn, pl.ln_2_w, pl.ln_2_b, NORM_TYPE_NORMAL, qformer_eps, bid); + } + cbx(ffn, "qformer_out"); + + // 7. _unwin back to raster + ggml_tensor * unwinned; + { + ggml_tensor * flat = ggml_reshape_2d(ctx, ffn, n_embd, query_len * n_windows); + unwinned = g4v_gather(ctx, gf, flat, + "g4v_blk" + std::to_string(bid) + "_unwin_idx", + new_side * new_side); + } + cbx(unwinned, "unwin"); + + // 8. out_linear + ggml_tensor * out = g->build_mm(blk.qf_proj_linear_w, unwinned); + if (blk.qf_proj_linear_b) out = ggml_add(ctx, out, blk.qf_proj_linear_b); + cbx(out, "out"); + + return out; +} + +// --------------------------------------------------------------------------- +// build() — top-level graph +// --------------------------------------------------------------------------- ggml_cgraph * clip_graph_granite_vision::build() { - //TODO + GGML_ASSERT(model.patch_embeddings_0 != nullptr); + GGML_ASSERT(model.position_embeddings != nullptr); + GGML_ASSERT(model.class_embedding == nullptr); + GGML_ASSERT(!model.qf_proj_blocks.empty()); + + // --- Stage 1a: SigLIP encoder producing intermediate hidden states --- + ggml_tensor * inp = build_inp(); + inp = ggml_add(ctx0, inp, model.position_embeddings); + cb(inp, "pos_embed", -1); + + ggml_tensor * inpL = inp; + std::vector layer_outs(n_layer, nullptr); + + for (int il = 0; il < n_layer; ++il) { + const auto & layer = model.layers[il]; + ggml_tensor * cur = inpL; + + cur = build_norm(cur, layer.ln_1_w, layer.ln_1_b, NORM_TYPE_NORMAL, eps, il); + + // Self-attention + ggml_tensor * Qcur = build_mm(layer.q_w, cur); + if (layer.q_b) Qcur = ggml_add(ctx0, Qcur, layer.q_b); + ggml_tensor * Kcur = build_mm(layer.k_w, cur); + if (layer.k_b) Kcur = ggml_add(ctx0, Kcur, layer.k_b); + ggml_tensor * Vcur = build_mm(layer.v_w, cur); + if (layer.v_b) Vcur = ggml_add(ctx0, Vcur, layer.v_b); + + Qcur = ggml_reshape_3d(ctx0, Qcur, d_head, n_head, n_patches); + Kcur = ggml_reshape_3d(ctx0, Kcur, d_head, n_head, n_patches); + Vcur = ggml_reshape_3d(ctx0, Vcur, d_head, n_head, n_patches); + + cur = build_attn(layer.o_w, layer.o_b, + Qcur, Kcur, Vcur, nullptr, kq_scale, il); + + cur = ggml_add(ctx0, cur, inpL); + inpL = cur; + + cur = build_norm(cur, layer.ln_2_w, layer.ln_2_b, NORM_TYPE_NORMAL, eps, il); + cur = build_ffn(cur, + layer.ff_up_w, layer.ff_up_b, + layer.ff_gate_w, layer.ff_gate_b, + layer.ff_down_w, layer.ff_down_b, + hparams.ffn_op, il); + cur = ggml_add(ctx0, inpL, cur); + cb(cur, "layer_out", il); + layer_outs[il] = cur; + inpL = cur; + } + + // --- Stage 1b/1c: WindowQFormer blocks --- + const int projector_count = hparams.vision_feature_layer.size(); + const float qformer_eps = 1e-12f; + + std::vector streams(projector_count, nullptr); + for (int bid = 0; bid < projector_count; ++bid) { + const auto & blk = model.qf_proj_blocks[bid]; + + int vlayer = hparams.vision_feature_layer[bid]; + if (vlayer < 0) vlayer = (n_layer + 1) + vlayer; + GGML_ASSERT(vlayer >= 1 && vlayer <= n_layer); + ggml_tensor * h = layer_outs[vlayer - 1]; + + streams[bid] = g4v_build_block( + this, ctx0, gf, blk, + h, bid, + hparams.proj_spatial_offsets[bid], + n_patches_x, + hparams.downsample_window_side, + hparams.downsample_query_side, + qformer_eps); + } + + // Sort by target LLM layer (implicit from vision_feature_layer ordering in conversion) + // For Granite Vision, the blocks are already in the correct order + ggml_tensor * mmproj = nullptr; + for (int k = 0; k < projector_count; ++k) { + ggml_tensor * s = ggml_cont(ctx0, streams[k]); + if (k == 0 && hparams.base_stream_scale != 1.0f) { + s = ggml_scale_inplace(ctx0, s, hparams.base_stream_scale); + } + mmproj = (k == 0) ? s : ggml_concat(ctx0, mmproj, s, 0); + } + ggml_set_name(mmproj, "g4v_mmproj_packed"); + + ggml_set_name(mmproj, "g4v_mmproj_out"); + ggml_build_forward_expand(gf, mmproj); + return gf; -} +} \ No newline at end of file diff --git a/tools/mtmd/mtmd.cpp b/tools/mtmd/mtmd.cpp index b6f02318ec01..fafe1ae6e708 100644 --- a/tools/mtmd/mtmd.cpp +++ b/tools/mtmd/mtmd.cpp @@ -850,6 +850,32 @@ struct mtmd_tokenizer { n_tokens += clip_n_output_tokens(ctx->ctx_v, entry.get()); } + // Granite Vision 4.1 pack_and_unpad (mtmd layer) changes + // the token count: single-tile appends 1 image_newline row, + // multi-tile adds a newline column per grid row and a + // full base tile. Compute the adjustment here so that + // image_tokens->nx reflects the final packed size. + if (ctx->proj_type_v() == PROJECTOR_TYPE_GRANITE4_VISION) { + if (batch_f32.entries.size() == 1) { + n_tokens += 1; // newline row + } else { + // +1 newline per grid row + base tile (= entries[0]) + const int grid_x = (int) batch_f32.grid_x; + const int grid_y = (int) batch_f32.grid_y; + const int per_tile = clip_n_output_tokens(ctx->ctx_v, + batch_f32.entries[0].get()); + const int per_tile_side = (int) std::lround(std::sqrt((double)per_tile)); + GGML_ASSERT(per_tile_side * per_tile_side == per_tile); + // Grid features: (per_tile_side * grid_y) rows, + // each (per_tile_side * grid_x + 1) cols (newline) + const int grid_tokens = (per_tile_side * grid_y) + * (per_tile_side * grid_x + 1); + // Base tile: per_tile tokens. Replace the raw + // sum with base + grid. + n_tokens = per_tile + grid_tokens; + } + } + mtmd_image_tokens_ptr image_tokens(new mtmd_image_tokens); if (mtmd_decode_use_mrope(ctx)) { // for Qwen2VL, we need this information for M-RoPE decoding positions @@ -1074,6 +1100,144 @@ int32_t mtmd_encode(mtmd_context * ctx, const mtmd_image_tokens * image_tokens) ctx->image_embd_v.resize(image_tokens->n_tokens() * n_mmproj_embd); bool ok = false; + if (proj_type == PROJECTOR_TYPE_GRANITE4_VISION) { + // Granite Vision 4.1: encode each tile separately, then run + // pack_and_unpad at the mtmd layer. The clip graph emits + // (n_mmproj_embd, 144) per tile; the final output depends on + // whether anyres split produced 1 tile (append one image_newline + // row across all K = projector_count stream slices) or more than + // one (reshape into the selected pinpoint grid, unpad, append + // newline column, prepend base tile). + const auto & entries = image_tokens->batch_f32.entries; + const int n_per_tile = clip_n_output_tokens(ctx_clip, entries[0].get()); + std::vector per_tile(entries.size() * n_per_tile * n_mmproj_embd); + for (size_t i = 0; i < entries.size(); i++) { + ok = clip_image_encode( + ctx_clip, + ctx->n_threads, + entries[i].get(), + per_tile.data() + i*n_mmproj_embd*n_per_tile); + if (!ok) return 1; + } + + // Per-slice image_newline vectors: the base slice (offset 0) is + // pre-divided by g4v.base_stream_scale to match what the clip + // graph does internally to the base stream. + //DEBUG------------------------------------------------------------------------------------------------------------------------ + const int n_slices = 8; //clip_projector_count(ctx_clip); + GGML_ASSERT(n_slices > 0 && n_mmproj_embd % n_slices == 0); + const int projection_dim = n_mmproj_embd / n_slices; + const float base_scale = 1.0 / 12.; // clip_base_stream_scale(ctx_clip); + std::vector newline_row(n_mmproj_embd); + // Copy learned image_newline vector (projection_dim floats) + // into the newline_row, K times. + std::vector newline(projection_dim); + { + ggml_tensor * t = clip_get_newline_tensor(ctx_clip); + GGML_ASSERT(t != nullptr && ggml_nelements(t) == projection_dim); + ggml_backend_tensor_get(t, newline.data(), 0, projection_dim * sizeof(float)); + } + for (int k = 0; k < n_slices; ++k) { + for (int d = 0; d < projection_dim; ++d) { + newline_row[k*projection_dim + d] = newline[d] * (k == 0 ? base_scale : 1.0f); + } + } + + // Assemble the final image_embd_v. + if (entries.size() == 1) { + // Single tile: 144 tokens + 1 newline row = 145 tokens. + std::memcpy(ctx->image_embd_v.data(), + per_tile.data(), + n_per_tile * n_mmproj_embd * sizeof(float)); + std::memcpy(ctx->image_embd_v.data() + n_per_tile * n_mmproj_embd, + newline_row.data(), + n_mmproj_embd * sizeof(float)); + } else { + // Multi-tile (LlavaNext-style) pack_and_unpad. Layout: + // out[0..per_tile] = base tile features (overview) + // out[per_tile..final_len] = grid features with newline col + // + // Grid: entries[1..end] are grid tiles in row-major order. + // Each tile contributes a per_tile_side × per_tile_side block + // of tokens, concatenated into a (grid_y*side, grid_x*side) + // spatial map, then unpadded against the original image + // aspect and topped up with one image_newline column. + const int per_tile_side = (int) std::lround(std::sqrt((double)n_per_tile)); + GGML_ASSERT(per_tile_side * per_tile_side == n_per_tile); + const int grid_x = (int) image_tokens->batch_f32.grid_x; + const int grid_y = (int) image_tokens->batch_f32.grid_y; + GGML_ASSERT(grid_x > 0 && grid_y > 0); + GGML_ASSERT((size_t)(grid_x * grid_y + 1) == entries.size()); + + const int H = per_tile_side * grid_y; + const int W = per_tile_side * grid_x; + + // Copy base tile (entries[0]) directly. + std::memcpy(ctx->image_embd_v.data(), + per_tile.data(), + n_per_tile * n_mmproj_embd * sizeof(float)); + size_t out_off = n_per_tile * n_mmproj_embd; + + // Reshape grid tiles into a (H, W, n_mmproj_embd) spatial + // buffer. Grid order matches entry indices 1..end in + // row-major (tile y*grid_x + x is at entries[1 + y*grid_x + x]). + const size_t spatial_floats = (size_t) H * W * n_mmproj_embd; + std::vector spatial(spatial_floats); + for (int gy = 0; gy < grid_y; ++gy) { + for (int gx = 0; gx < grid_x; ++gx) { + const int tile_idx = gy * grid_x + gx; // among grid tiles + const float * tile_src = per_tile.data() + + (1 + tile_idx) * n_per_tile * n_mmproj_embd; + for (int ty = 0; ty < per_tile_side; ++ty) { + for (int tx = 0; tx < per_tile_side; ++tx) { + const int y = gy * per_tile_side + ty; + const int x = gx * per_tile_side + tx; + const int src_row = ty * per_tile_side + tx; + std::memcpy( + spatial.data() + ((size_t) y * W + x) * n_mmproj_embd, + tile_src + src_row * n_mmproj_embd, + n_mmproj_embd * sizeof(float)); + } + } + } + } + + // Unpad: crop rows or columns based on original image aspect. + // image_tokens carries no image_size field today; reconstruct + // from nx/ny if possible, otherwise skip cropping. + // For now assume no unpad (i.e. the pinpoint grid perfectly + // matches the image aspect, which is the common case; unpad + // cropping will be added when multi-aspect images regress). + int H_unp = H; + int W_unp = W; + // TODO: implement unpad crop based on original image size. + + // Append newline column: for each row (H_unp), append one + // newline_row across the K slices. + const int W_final = W_unp + 1; + const size_t grid_tokens = (size_t) H_unp * W_final; + for (int y = 0; y < H_unp; ++y) { + // Copy the W_unp row of spatial features. + std::memcpy( + ctx->image_embd_v.data() + out_off + (size_t) y * W_final * n_mmproj_embd, + spatial.data() + (size_t) y * W * n_mmproj_embd, + (size_t) W_unp * n_mmproj_embd * sizeof(float)); + // Copy the newline row into the last column of this row. + std::memcpy( + ctx->image_embd_v.data() + out_off + + ((size_t) y * W_final + W_unp) * n_mmproj_embd, + newline_row.data(), + n_mmproj_embd * sizeof(float)); + } + + // Sanity check token count. + const size_t final_tokens = n_per_tile + grid_tokens; + GGML_ASSERT(final_tokens == image_tokens->n_tokens() + && "mtmd_encode: token count mismatch with image_tokens->n_tokens"); + } + return 0; + } + if (clip_is_llava(ctx_clip) || clip_is_minicpmv(ctx_clip) || clip_is_glm(ctx_clip) From 3e0508bf7b70f2d210f08787738bb2e50185d197 Mon Sep 17 00:00:00 2001 From: Gabe Goodhart Date: Wed, 20 May 2026 09:52:06 -0600 Subject: [PATCH 50/94] fix: Fix the pre-scaling on the input embeddings to correctly invert the scale We've got tokens! They still don't line up quite right, so something's a little off, but we're getting much closer now. Branch: Granite4Vision AI-usage: none Signed-off-by: Gabe Goodhart --- tools/mtmd/models/granite-vision.cpp | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/tools/mtmd/models/granite-vision.cpp b/tools/mtmd/models/granite-vision.cpp index 087562301f64..e814f345e013 100644 --- a/tools/mtmd/models/granite-vision.cpp +++ b/tools/mtmd/models/granite-vision.cpp @@ -116,10 +116,7 @@ static ggml_tensor * g4v_build_block( ggml_tensor * e_in = ggml_add(ctx, enc, blk.qf_proj_img_pos); cbx(e_in, "encoder_embeds"); - // 6. Qformer forward. Per Blip2QFormerModel.forward: first runs - // self.layernorm(query_embeds) before entering the single layer. - // blk.post_norm_* holds this (our naming is historical; see the - // comment in clip-model.h's g4v_projector_block struct). + // 6. Qformer forward. ggml_tensor * q = g->build_norm(q_in, blk.qf_proj_post_norm_w, blk.qf_proj_post_norm_b, NORM_TYPE_NORMAL, qformer_eps, bid); // Helper: one post-norm BERT attention block. @@ -320,7 +317,7 @@ ggml_cgraph * clip_graph_granite_vision::build() { for (int k = 0; k < projector_count; ++k) { ggml_tensor * s = ggml_cont(ctx0, streams[k]); if (k == 0 && hparams.base_stream_scale != 1.0f) { - s = ggml_scale_inplace(ctx0, s, hparams.base_stream_scale); + s = ggml_scale_inplace(ctx0, s, 1.0 / hparams.base_stream_scale); } mmproj = (k == 0) ? s : ggml_concat(ctx0, mmproj, s, 0); } From 5792a27acc20ddb42bba195ab61d4beec5ffb182 Mon Sep 17 00:00:00 2001 From: Gabe Goodhart Date: Wed, 20 May 2026 11:50:02 -0600 Subject: [PATCH 51/94] feat: invert embedding multiplier -> base_scale at load Branch: Granite4Vision AI-usage: none Signed-off-by: Gabe Goodhart --- tools/mtmd/clip.cpp | 6 +++++- tools/mtmd/models/granite-vision.cpp | 4 +--- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/tools/mtmd/clip.cpp b/tools/mtmd/clip.cpp index c97004bf97ee..8ca3e17d2ae3 100644 --- a/tools/mtmd/clip.cpp +++ b/tools/mtmd/clip.cpp @@ -1583,7 +1583,11 @@ struct clip_model_loader { get_u32(KEY_PROJ_SAMPLE_QUERY_SIDE, hparams.downsample_query_side); get_u32(KEY_PROJ_SAMPLE_WINDOW_SIDE, hparams.downsample_window_side); - get_f32(KEY_LLM_KV_EMBEDDING_SCALE, hparams.base_stream_scale, false); + float embedding_scale = 0.; + get_f32(KEY_LLM_KV_EMBEDDING_SCALE, embedding_scale, false); + if (embedding_scale != 0) { + hparams.base_stream_scale = 1.0 / embedding_scale; + } hparams.warmup_image_size = hparams.image_size; } break; default: diff --git a/tools/mtmd/models/granite-vision.cpp b/tools/mtmd/models/granite-vision.cpp index e814f345e013..7ab9a22e6fa1 100644 --- a/tools/mtmd/models/granite-vision.cpp +++ b/tools/mtmd/models/granite-vision.cpp @@ -317,12 +317,10 @@ ggml_cgraph * clip_graph_granite_vision::build() { for (int k = 0; k < projector_count; ++k) { ggml_tensor * s = ggml_cont(ctx0, streams[k]); if (k == 0 && hparams.base_stream_scale != 1.0f) { - s = ggml_scale_inplace(ctx0, s, 1.0 / hparams.base_stream_scale); + s = ggml_scale_inplace(ctx0, s, hparams.base_stream_scale); } mmproj = (k == 0) ? s : ggml_concat(ctx0, mmproj, s, 0); } - ggml_set_name(mmproj, "g4v_mmproj_packed"); - ggml_set_name(mmproj, "g4v_mmproj_out"); ggml_build_forward_expand(gf, mmproj); From f2e2de62a76f43d5af6ade2c60db649ca0aa322f Mon Sep 17 00:00:00 2001 From: Gabe Goodhart Date: Thu, 21 May 2026 09:25:41 -0600 Subject: [PATCH 52/94] fix: Fix setting image_resize_pad after new enum introduced Branch: Granite4Vision AI-usage: none Signed-off-by: Gabe Goodhart --- tools/mtmd/clip.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/mtmd/clip.cpp b/tools/mtmd/clip.cpp index 97fb32b28d60..c1e670a74e6e 100644 --- a/tools/mtmd/clip.cpp +++ b/tools/mtmd/clip.cpp @@ -1575,7 +1575,7 @@ struct clip_model_loader { { // SigLIP tower. hparams.image_resize_algo = RESIZE_ALGO_BICUBIC_PILLOW; - hparams.image_resize_pad = true; + hparams.image_resize_pad = PAD_CEIL; // Load the per-projector hparams (vision_feature_layers, spatial_offsets) get_arr_int(KEY_FEATURE_LAYER, hparams.vision_feature_layer); From 6bb918c1ab1cf28406f540bee846aa861da2235a Mon Sep 17 00:00:00 2001 From: Gabe Goodhart Date: Thu, 21 May 2026 09:38:00 -0600 Subject: [PATCH 53/94] fix: Add G4V to mmproj mapping in conversion Branch: Granite4Vision AI-usage: none Signed-off-by: Gabe Goodhart --- conversion/__init__.py | 1 + 1 file changed, 1 insertion(+) diff --git a/conversion/__init__.py b/conversion/__init__.py index 2c38123dff8d..3b54a57fba00 100644 --- a/conversion/__init__.py +++ b/conversion/__init__.py @@ -243,6 +243,7 @@ "Glm4vMoeForConditionalGeneration": "qwen3vl", "GlmOcrForConditionalGeneration": "qwen3vl", "GlmasrModel": "ultravox", + "Granite4VisionForConditionalGeneration": "granite", "GraniteSpeechForConditionalGeneration": "granite", "HunYuanVLForConditionalGeneration": "hunyuan", "Idefics3ForConditionalGeneration": "smolvlm", From 12c085ec7b45e0ed94423a079311bfb755b986c1 Mon Sep 17 00:00:00 2001 From: Gabe Goodhart Date: Thu, 21 May 2026 09:52:27 -0600 Subject: [PATCH 54/94] fix: Re-add padding disable for non-hybrid hybrid models Branch: Granite4Vision AI-usage: none Signed-off-by: Gabe Goodhart --- conversion/granite.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/conversion/granite.py b/conversion/granite.py index fb65dacf167c..4f3571f065b4 100644 --- a/conversion/granite.py +++ b/conversion/granite.py @@ -260,7 +260,8 @@ def set_gguf_parameters(self): assert self.d_inner % d_head == 0, f"SSM inner size {self.d_inner} not a multiple of head dim {d_head}" def set_vocab(self): - self.hparams["pad_vocab_size_multiple"] = 8 + # For models with no ssm layers, don't pad for mamba2 + self.hparams["pad_vocab_size_multiple"] = 8 if self._ssm_layers else 1 Mamba2Model.set_vocab(self) From db28b585594258f28e98eaee4048dc104a024f58 Mon Sep 17 00:00:00 2001 From: Gabe Goodhart Date: Thu, 21 May 2026 16:20:36 -0600 Subject: [PATCH 55/94] refactor: Simplify G4V n_tokens computation This is slightly more efficient and flexible for when we implement the unpad cropping. IMO, it's also clearer that it is adding the number of image_newline tokens (embeddings) to the grid, rather than recomputing the entire count. Branch: Granite4Vision AI-usage: none Signed-off-by: Gabe Goodhart --- tools/mtmd/mtmd.cpp | 14 +++----------- 1 file changed, 3 insertions(+), 11 deletions(-) diff --git a/tools/mtmd/mtmd.cpp b/tools/mtmd/mtmd.cpp index 3669d9f0189a..d5df90fca961 100644 --- a/tools/mtmd/mtmd.cpp +++ b/tools/mtmd/mtmd.cpp @@ -871,7 +871,7 @@ struct mtmd_tokenizer { n_tokens += clip_n_output_tokens(ctx->ctx_v, entry.get()); } - // Granite Vision 4.1 pack_and_unpad (mtmd layer) changes + // Granite Vision 4.x pack_and_unpad (mtmd layer) changes // the token count: single-tile appends 1 image_newline row, // multi-tile adds a newline column per grid row and a // full base tile. Compute the adjustment here so that @@ -880,20 +880,12 @@ struct mtmd_tokenizer { if (batch_f32.entries.size() == 1) { n_tokens += 1; // newline row } else { - // +1 newline per grid row + base tile (= entries[0]) - const int grid_x = (int) batch_f32.grid_x; - const int grid_y = (int) batch_f32.grid_y; + // +1 newline per grid row const int per_tile = clip_n_output_tokens(ctx->ctx_v, batch_f32.entries[0].get()); const int per_tile_side = (int) std::lround(std::sqrt((double)per_tile)); GGML_ASSERT(per_tile_side * per_tile_side == per_tile); - // Grid features: (per_tile_side * grid_y) rows, - // each (per_tile_side * grid_x + 1) cols (newline) - const int grid_tokens = (per_tile_side * grid_y) - * (per_tile_side * grid_x + 1); - // Base tile: per_tile tokens. Replace the raw - // sum with base + grid. - n_tokens = per_tile + grid_tokens; + n_tokens += per_tile_side * batch_f32.grid_y; } } From 6f110e7925504b170e75c42e6ba9e72c621792f3 Mon Sep 17 00:00:00 2001 From: Gabe Goodhart Date: Fri, 22 May 2026 11:58:33 -0600 Subject: [PATCH 56/94] feat: Add new clip APIs for post-tile-encoding assembly Granite 4 Vision uses llava-next style pack-and-unpad which requires injecting the learned newline after each row of the tile grid. A row here is a single row of the grid which is composed of (grid_x * cols_per_tile) * (grid_y * rows_per_tile), so the result is newlines injected in between individual tile rows, thus not something that can be handled with the standard llava-uhd block-wise endcoding. Branch: Granite4Vision AI-usage: draft (Claude Code + Opus 4.7) Signed-off-by: Gabe Goodhart --- tools/mtmd/clip.cpp | 121 ++++++++++++++++++++++++++++++++++++++------ tools/mtmd/clip.h | 23 ++++++++- 2 files changed, 126 insertions(+), 18 deletions(-) diff --git a/tools/mtmd/clip.cpp b/tools/mtmd/clip.cpp index c1e670a74e6e..65394f2d80cd 100644 --- a/tools/mtmd/clip.cpp +++ b/tools/mtmd/clip.cpp @@ -3089,10 +3089,6 @@ void clip_build_img_from_pixels(const unsigned char * rgb_pixels, int nx, int ny memcpy(img->buf.data(), rgb_pixels, img->buf.size()); } -ggml_tensor * clip_get_newline_tensor(const struct clip_ctx * ctx) { - return ctx->model.image_newline; -} - void clip_free(clip_ctx * ctx) { if (ctx == nullptr) { return; @@ -4199,7 +4195,7 @@ bool clip_image_batch_encode(clip_ctx * ctx, const int n_threads, const clip_ima // Granite Vision 4.1 uses precomputed permutation index // tensors to express the _win / _unwin / spatial sampling // reshapes as ggml_get_rows gathers. The names are set - // by g4v_gather() in models/granite4v.cpp. + // by g4v_gather() in models/granite4-vision.cpp. const int patch_size = model.hparams.patch_size; const int image_side = imgs.entries.front()->nx / patch_size; const int window_side = hparams.downsample_window_side; @@ -4253,17 +4249,8 @@ bool clip_image_batch_encode(clip_ctx * ctx, const int n_threads, const clip_ima auto upload = [&](const std::string & name, const std::vector & idx) { ggml_tensor * t = ggml_graph_get_tensor(gf, name.c_str()); - if (t) { - if (std::getenv("G4V_DEBUG_NAMES")) { - std::fprintf(stderr, "granite4v: upload %s len=%zu first=%d\n", - name.c_str(), idx.size(), - idx.empty() ? -1 : idx.front()); - } - ggml_backend_tensor_set(t, idx.data(), 0, idx.size() * sizeof(int32_t)); - } else if (std::getenv("G4V_DEBUG_NAMES")) { - std::fprintf(stderr, "granite4v: index tensor %s not found in graph\n", - name.c_str()); - } + GGML_ASSERT(t); + ggml_backend_tensor_set(t, idx.data(), 0, idx.size() * sizeof(int32_t)); }; // Stage 1b only uses block 0's permutations; future stages @@ -4360,6 +4347,108 @@ bool clip_image_batch_encode(clip_ctx * ctx, const int n_threads, const clip_ima return true; } +size_t clip_n_assembled_output_tokens(clip_ctx * ctx, const clip_image_f32_batch * batch) { + GGML_ASSERT(batch != nullptr); + switch (ctx->proj_type()) { + case PROJECTOR_TYPE_GRANITE4_VISION: + return granite_vision_n_assembled_output_tokens(ctx, batch); + // No assembler: flat sum of per-entry token counts. + default: + break; + } + size_t n = 0; + for (const auto & e : batch->entries) { + n += clip_n_output_tokens(ctx, e.get()); + } + return n; +} + +bool clip_image_assemble(clip_ctx * ctx, const int n_threads, + const float * per_tile_embd, + int n_tiles, + int grid_x, int grid_y, + float * out_embd) { + // Extend this switch for all models that require assembling. All others + // will fall through to the default case and return false indicating no + // assembly required. + switch (ctx->proj_type()) { + case PROJECTOR_TYPE_GRANITE4_VISION: + { + GGML_ASSERT(per_tile_embd != nullptr && out_embd != nullptr && n_tiles > 0); + + const int n_mmproj_embd = clip_n_mmproj_embd(ctx); + + // Recover n_per_tile from a representative entry. The encoder uses a + // fixed tile size for G4V, so all entries produce the same per-tile + // count; the warmup path stashes a representative image but we use the + // model's image_size since the assembler runs after encoding has + // finished. + clip_image_f32 tile_img; + tile_img.nx = ctx->model.hparams.image_size; + tile_img.ny = ctx->model.hparams.image_size; + const int n_per_tile = clip_n_output_tokens(ctx, &tile_img); + + // Build host-side gather index. + std::vector gather_idx = + granite_vision_build_assembler_gather_idx(n_tiles, n_per_tile, grid_x, grid_y); + const int gather_len = (int) gather_idx.size(); + + // Build the assembler cgraph in a fresh ggml context backed by the + // shared compute-meta buffer. + struct ggml_init_params params = { + /*.mem_size =*/ ctx->buf_compute_meta.size(), + /*.mem_buffer =*/ ctx->buf_compute_meta.data(), + /*.no_alloc =*/ true, + }; + ggml_context_ptr ctx0_ptr(ggml_init(params)); + ggml_context * ctx0 = ctx0_ptr.get(); + GGML_ASSERT(ctx0 != nullptr); + ggml_cgraph * gf = ggml_new_graph_custom(ctx0, ctx->max_nodes, false); + + ggml_tensor * out = granite_vision_build_assembler_graph( + ctx0, gf, ctx->model, n_tiles, n_per_tile, n_mmproj_embd, gather_len); + + // Allocate buffers for this graph on the sched. + ggml_backend_sched_reset(ctx->sched.get()); + if (!ggml_backend_sched_alloc_graph(ctx->sched.get(), gf)) { + LOG_ERR("%s: ggml_backend_sched_alloc_graph failed\n", __func__); + return false; + } + + // Set inputs. + ggml_tensor * in_per_tile = ggml_graph_get_tensor(gf, "g4v_assembler_per_tile_in"); + ggml_tensor * in_gather = ggml_graph_get_tensor(gf, "g4v_assembler_gather_idx"); + GGML_ASSERT(in_per_tile != nullptr && in_gather != nullptr); + ggml_backend_tensor_set(in_per_tile, per_tile_embd, 0, + (size_t) n_tiles * n_per_tile * n_mmproj_embd * sizeof(float)); + ggml_backend_tensor_set(in_gather, gather_idx.data(), 0, + gather_idx.size() * sizeof(int32_t)); + + // Thread count on the CPU backend (matches clip_image_batch_encode). + ggml_backend_dev_t dev = ggml_backend_get_device(ctx->backend_cpu); + ggml_backend_reg_t reg = dev ? ggml_backend_dev_backend_reg(dev) : nullptr; + if (reg) { + auto set_n_threads_fn = (ggml_backend_set_n_threads_t) + ggml_backend_reg_get_proc_address(reg, "ggml_backend_set_n_threads"); + if (set_n_threads_fn) { + set_n_threads_fn(ctx->backend_cpu, n_threads); + } + } + + auto status = ggml_backend_sched_graph_compute(ctx->sched.get(), gf); + if (status != GGML_STATUS_SUCCESS) { + LOG_ERR("%s: ggml_backend_sched_graph_compute failed (%d)\n", __func__, status); + return false; + } + + ggml_backend_tensor_get(out, out_embd, 0, ggml_nbytes(out)); + return true; + } + default: + return false; + } +} + int clip_n_mmproj_embd(const struct clip_ctx * ctx) { switch (ctx->model.proj_type) { case PROJECTOR_TYPE_LDP: diff --git a/tools/mtmd/clip.h b/tools/mtmd/clip.h index 9b807ffa77b3..e4acd93366e4 100644 --- a/tools/mtmd/clip.h +++ b/tools/mtmd/clip.h @@ -100,11 +100,30 @@ struct clip_image_f32 * clip_image_f32_get_img(const struct clip_image_f32_batch */ void clip_build_img_from_pixels(const unsigned char * rgb_pixels, int nx, int ny, struct clip_image_u8 * img); -struct ggml_tensor * clip_get_newline_tensor(const struct clip_ctx * ctx); - bool clip_image_encode (struct clip_ctx * ctx, int n_threads, struct clip_image_f32 * img, float * vec); bool clip_image_batch_encode(struct clip_ctx * ctx, int n_threads, const struct clip_image_f32_batch * imgs, float * vec); +// Some projector types perform an additional assembly pass over per-tile +// encoder outputs (e.g. llava-next-style pack-and-unpad with a learned +// newline embedding). For those projectors, clip_image_assemble runs the +// model-specific assembler graph and writes the final embeddings to out_embd. +// For projectors with no assembler, returns false without writing anything. +// per_tile_embd is laid out as (n_mmproj_embd, n_per_tile, n_tiles) with the +// embedding dim innermost; n_per_tile is inferred from the model's per-tile +// output size for an entry of the batch that produced per_tile_embd. +bool clip_image_assemble(struct clip_ctx * ctx, int n_threads, + const float * per_tile_embd, + int n_tiles, + int grid_x, int grid_y, + float * out_embd); + +// For projectors with an assembler, returns the final token count after +// assembly given the preprocessed tile batch. For projectors without an +// assembler, returns the unmodified flat per-tile token count (sum over +// entries of clip_n_output_tokens). +size_t clip_n_assembled_output_tokens(struct clip_ctx * ctx, + const struct clip_image_f32_batch * batch); + bool clip_is_llava(const struct clip_ctx * ctx); // note for contributor: this clip_is_(model) pattern is deprecated // do NOT add new functions like this From db6a998a3df57cfe5c1f4d9d3b258813bd927f91 Mon Sep 17 00:00:00 2001 From: Gabe Goodhart Date: Fri, 22 May 2026 12:00:46 -0600 Subject: [PATCH 57/94] feat: Add model interfaces for granite 4 vision assembler I'm on the fence about the best organization of this. These free functions allow the per-architecture logic in clip.cpp to access the model-specific graph building, but they still require a fair bit of model-specific logic in clip.cpp which is not ideal. I think a better approach may be to replicate what is done with the graph builders themselves (and possibly even make the assembler part of the model's existing graph builder). Branch: Granite4Vision AI-usage: full (Claude Code + Opus 4.7) Signed-off-by: Gabe Goodhart --- tools/mtmd/models/granite-vision.cpp | 143 +++++++++++++++++++++++++++ tools/mtmd/models/models.h | 33 +++++++ 2 files changed, 176 insertions(+) diff --git a/tools/mtmd/models/granite-vision.cpp b/tools/mtmd/models/granite-vision.cpp index 7ab9a22e6fa1..07a3df7eb570 100644 --- a/tools/mtmd/models/granite-vision.cpp +++ b/tools/mtmd/models/granite-vision.cpp @@ -325,4 +325,147 @@ ggml_cgraph * clip_graph_granite_vision::build() { ggml_build_forward_expand(gf, mmproj); return gf; +} + +// --------------------------------------------------------------------------- +// Assembler — pack-and-unpad + scaled image_newline injection +// --------------------------------------------------------------------------- +// +// Granite Vision 4.1 follows the llava-next layout: the per-tile encoder +// outputs are reshaped so the assembled super-image is read in spatial +// scan order with a learned newline embedding appended to every super-row. +// Single-tile inputs (overview only) get a single trailing newline row. +// +// All knowledge of image_newline and base_stream_scale lives here. The +// graph itself is a single ggml_get_rows over a virtual buffer formed by +// concatenating per-tile encoder outputs with the K-tiled, base-scaled +// newline row. The gather index is built host-side. + +size_t granite_vision_n_assembled_output_tokens( + const clip_ctx * ctx, + const clip_image_f32_batch * batch) { + GGML_ASSERT(batch != nullptr && !batch->entries.empty()); + const int n_per_tile = clip_n_output_tokens( + const_cast(ctx), batch->entries[0].get()); + if (batch->entries.size() == 1) { + // overview tile + 1 newline row + return (size_t)n_per_tile + 1; + } + const int per_tile_side = (int) std::lround(std::sqrt((double)n_per_tile)); + GGML_ASSERT(per_tile_side * per_tile_side == n_per_tile); + // overview + (per_tile_side*grid_y) super-rows, each (per_tile_side*grid_x + 1) wide + const size_t super_h = (size_t)per_tile_side * (size_t)batch->grid_y; + const size_t super_w_plus_nl = (size_t)per_tile_side * (size_t)batch->grid_x + 1; + return (size_t)n_per_tile + super_h * super_w_plus_nl; +} + +std::vector granite_vision_build_assembler_gather_idx( + int n_tiles, int n_per_tile, int grid_x, int grid_y) { + // virtual source layout: [tile_0 tokens | tile_1 tokens | ... | newline_row] + // newline_row is at virtual index n_tiles * n_per_tile. + const int32_t nl_idx = n_tiles * n_per_tile; + + std::vector idx; + // overview comes first + idx.reserve((size_t)n_per_tile + 1); + for (int i = 0; i < n_per_tile; ++i) { + idx.push_back(i); + } + + if (n_tiles == 1) { + // single tile: just one trailing newline + idx.push_back(nl_idx); + return idx; + } + + GGML_ASSERT(grid_x > 0 && grid_y > 0); + GGML_ASSERT(n_tiles == grid_x * grid_y + 1); + const int per_tile_side = (int) std::lround(std::sqrt((double)n_per_tile)); + GGML_ASSERT(per_tile_side * per_tile_side == n_per_tile); + + // Spatial scan: for each super-row, walk across grid columns then within + // each tile across local columns, ending with a newline. This reproduces + // the loop nest from the pre-refactor host-side code exactly. + const size_t total = (size_t)n_per_tile + + (size_t)per_tile_side * (size_t)grid_y * + ((size_t)per_tile_side * (size_t)grid_x + 1); + idx.reserve(total); + for (int gy = 0; gy < grid_y; ++gy) { + for (int ty = 0; ty < per_tile_side; ++ty) { + for (int gx = 0; gx < grid_x; ++gx) { + const int tile_idx = 1 + gy * grid_x + gx; // +1: skip overview + const int row_off = tile_idx * n_per_tile + ty * per_tile_side; + for (int tx = 0; tx < per_tile_side; ++tx) { + idx.push_back(row_off + tx); + } + } + idx.push_back(nl_idx); + } + } + GGML_ASSERT(idx.size() == total); + return idx; +} + +ggml_tensor * granite_vision_build_assembler_graph( + ggml_context * ctx0, + ggml_cgraph * gf, + const clip_model & model, + int n_tiles, + int n_per_tile, + int n_mmproj_embd, + int gather_len) { + const int K = (int) model.qf_proj_blocks.size(); + GGML_ASSERT(K > 0); + GGML_ASSERT(n_mmproj_embd % K == 0); + const int projection_dim = n_mmproj_embd / K; + GGML_ASSERT(model.image_newline != nullptr); + GGML_ASSERT(ggml_nelements(model.image_newline) == projection_dim); + + // 1) Inputs + ggml_tensor * per_tile_in = ggml_new_tensor_2d( + ctx0, GGML_TYPE_F32, n_mmproj_embd, n_tiles * n_per_tile); + ggml_set_name(per_tile_in, "g4v_assembler_per_tile_in"); + ggml_set_input(per_tile_in); + + ggml_tensor * gather_idx = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, gather_len); + ggml_set_name(gather_idx, "g4v_assembler_gather_idx"); + ggml_set_input(gather_idx); + + // 2) Build K-tiled, base-scaled newline row. + // + // Final layout matches the pre-refactor host code: + // newline_row[k*projection_dim + d] = nl[d] * (k == 0 ? base : 1.0) + // + // i.e. with ne[0]=projection_dim, ne[1]=K stacking, then reshape to a + // 1-row vector of length K*projection_dim. + ggml_tensor * nl = model.image_newline; // (projection_dim,) + ggml_tensor * nl_first = ggml_scale(ctx0, nl, model.hparams.base_stream_scale); + ggml_tensor * nl_first_2d = ggml_reshape_2d(ctx0, nl_first, projection_dim, 1); + ggml_tensor * nl_row_2d; + if (K == 1) { + nl_row_2d = nl_first_2d; + } else { + ggml_tensor * nl_2d = ggml_reshape_2d(ctx0, nl, projection_dim, 1); + ggml_tensor * rest_template = ggml_new_tensor_2d( + ctx0, GGML_TYPE_F32, projection_dim, K - 1); + ggml_tensor * nl_rest = ggml_repeat(ctx0, nl_2d, rest_template); + nl_row_2d = ggml_concat(ctx0, nl_first_2d, nl_rest, 1); // (projection_dim, K) + } + // Reshape to a single row of length n_mmproj_embd. Must be contiguous. + nl_row_2d = ggml_cont(ctx0, nl_row_2d); + ggml_tensor * nl_row = ggml_reshape_2d(ctx0, nl_row_2d, n_mmproj_embd, 1); + ggml_set_name(nl_row, "g4v_assembler_nl_row"); + + // 3) Concat per-tile outputs with the newline row to form the virtual + // source. The newline becomes the row at index n_tiles*n_per_tile. + ggml_tensor * src = ggml_concat(ctx0, per_tile_in, nl_row, 1); + ggml_set_name(src, "g4v_assembler_src"); + + // 4) Gather. ggml_get_rows selects along ne[1], producing a tensor of + // shape (n_mmproj_embd, gather_len) in spatial scan order. + ggml_tensor * out = ggml_get_rows(ctx0, src, gather_idx); + ggml_set_name(out, "g4v_assembler_out"); + + ggml_build_forward_expand(gf, out); + return out; } \ No newline at end of file diff --git a/tools/mtmd/models/models.h b/tools/mtmd/models/models.h index 342364cc7c7d..38cf9e2bd8d4 100644 --- a/tools/mtmd/models/models.h +++ b/tools/mtmd/models/models.h @@ -196,3 +196,36 @@ struct clip_graph_granite_vision : clip_graph { clip_graph_granite_vision(clip_ctx * ctx, const clip_image_f32 & img) : clip_graph(ctx, img) {} ggml_cgraph * build() override; }; + +// --- Granite Vision 4 assembler --- +// +// Per-tile encoder outputs are reshaped (pack-and-unpad) and interleaved with +// a learned, base-scaled newline embedding to produce the final image-token +// sequence. These helpers are used by clip_image_assemble and +// clip_n_assembled_output_tokens dispatch in clip.cpp. + +// Build the host-side gather index used by the assembler graph. Entries +// index into a virtual buffer of length n_tiles*n_per_tile + 1 where the +// last row is the K-tiled newline. The number of entries returned is the +// final assembled token count. +std::vector granite_vision_build_assembler_gather_idx( + int n_tiles, int n_per_tile, int grid_x, int grid_y); + +// Build the assembler cgraph into the supplied ggml context and graph. +// Returns the final output tensor (also expanded into gf). Creates two +// input tensors registered with ggml_set_input: +// - "g4v_assembler_per_tile_in" : F32 (n_mmproj_embd, n_per_tile * n_tiles) +// - "g4v_assembler_gather_idx" : I32 (gather_len,) +ggml_tensor * granite_vision_build_assembler_graph( + ggml_context * ctx0, + ggml_cgraph * gf, + const clip_model & model, + int n_tiles, + int n_per_tile, + int n_mmproj_embd, + int gather_len); + +// Final assembled token count for a preprocessed tile batch. +size_t granite_vision_n_assembled_output_tokens( + const clip_ctx * ctx, + const clip_image_f32_batch * batch); From 509c0aed08a93d3329d15b888729c23fee823b68 Mon Sep 17 00:00:00 2001 From: Gabe Goodhart Date: Fri, 22 May 2026 12:01:36 -0600 Subject: [PATCH 58/94] refactor: Remove all g4v-specific branching from mtmd.cpp in favor of clip assembler Branch: Granite4Vision AI-usage: full (Claude Code + Opus 4.7) Signed-off-by: Gabe Goodhart --- tools/mtmd/mtmd.cpp | 166 ++++---------------------------------------- 1 file changed, 15 insertions(+), 151 deletions(-) diff --git a/tools/mtmd/mtmd.cpp b/tools/mtmd/mtmd.cpp index d5df90fca961..c81e07d61420 100644 --- a/tools/mtmd/mtmd.cpp +++ b/tools/mtmd/mtmd.cpp @@ -866,28 +866,9 @@ struct mtmd_tokenizer { } } else { - size_t n_tokens = 0; - for (const auto & entry : batch_f32.entries) { - n_tokens += clip_n_output_tokens(ctx->ctx_v, entry.get()); - } - - // Granite Vision 4.x pack_and_unpad (mtmd layer) changes - // the token count: single-tile appends 1 image_newline row, - // multi-tile adds a newline column per grid row and a - // full base tile. Compute the adjustment here so that - // image_tokens->nx reflects the final packed size. - if (ctx->proj_type_v() == PROJECTOR_TYPE_GRANITE4_VISION) { - if (batch_f32.entries.size() == 1) { - n_tokens += 1; // newline row - } else { - // +1 newline per grid row - const int per_tile = clip_n_output_tokens(ctx->ctx_v, - batch_f32.entries[0].get()); - const int per_tile_side = (int) std::lround(std::sqrt((double)per_tile)); - GGML_ASSERT(per_tile_side * per_tile_side == per_tile); - n_tokens += per_tile_side * batch_f32.grid_y; - } - } + // Final token count accounts for any model-specific assembler + // (e.g. Granite Vision 4 pack-and-unpad with newline injection). + const size_t n_tokens = clip_n_assembled_output_tokens(ctx->ctx_v, &batch_f32); mtmd_image_tokens_ptr image_tokens(new mtmd_image_tokens); if (mtmd_decode_use_mrope(ctx)) { @@ -1120,141 +1101,24 @@ int32_t mtmd_encode(mtmd_context * ctx, const mtmd_image_tokens * image_tokens) bool ok = false; if (proj_type == PROJECTOR_TYPE_GRANITE4_VISION) { - // Granite Vision 4.1: encode each tile separately, then run - // pack_and_unpad at the mtmd layer. The clip graph emits - // (n_mmproj_embd, 144) per tile; the final output depends on - // whether anyres split produced 1 tile (append one image_newline - // row across all K = projector_count stream slices) or more than - // one (reshape into the selected pinpoint grid, unpad, append - // newline column, prepend base tile). + // Per-tile encode, then hand off to the model-owned assembler graph + // (pack-and-unpad + scaled image_newline injection). const auto & entries = image_tokens->batch_f32.entries; const int n_per_tile = clip_n_output_tokens(ctx_clip, entries[0].get()); std::vector per_tile(entries.size() * n_per_tile * n_mmproj_embd); for (size_t i = 0; i < entries.size(); i++) { - ok = clip_image_encode( - ctx_clip, - ctx->n_threads, - entries[i].get(), - per_tile.data() + i*n_mmproj_embd*n_per_tile); - if (!ok) return 1; - } - - // Per-slice image_newline vectors: the base slice (offset 0) is - // pre-divided by g4v.base_stream_scale to match what the clip - // graph does internally to the base stream. - //DEBUG------------------------------------------------------------------------------------------------------------------------ - const int n_slices = 8; //clip_projector_count(ctx_clip); - GGML_ASSERT(n_slices > 0 && n_mmproj_embd % n_slices == 0); - const int projection_dim = n_mmproj_embd / n_slices; - const float base_scale = 1.0 / 12.; // clip_base_stream_scale(ctx_clip); - std::vector newline_row(n_mmproj_embd); - // Copy learned image_newline vector (projection_dim floats) - // into the newline_row, K times. - std::vector newline(projection_dim); - { - ggml_tensor * t = clip_get_newline_tensor(ctx_clip); - GGML_ASSERT(t != nullptr && ggml_nelements(t) == projection_dim); - ggml_backend_tensor_get(t, newline.data(), 0, projection_dim * sizeof(float)); - } - for (int k = 0; k < n_slices; ++k) { - for (int d = 0; d < projection_dim; ++d) { - newline_row[k*projection_dim + d] = newline[d] * (k == 0 ? base_scale : 1.0f); + if (!clip_image_encode(ctx_clip, ctx->n_threads, entries[i].get(), + per_tile.data() + i * n_mmproj_embd * n_per_tile)) { + return 1; } } - - // Assemble the final image_embd_v. - if (entries.size() == 1) { - // Single tile: 144 tokens + 1 newline row = 145 tokens. - std::memcpy(ctx->image_embd_v.data(), - per_tile.data(), - n_per_tile * n_mmproj_embd * sizeof(float)); - std::memcpy(ctx->image_embd_v.data() + n_per_tile * n_mmproj_embd, - newline_row.data(), - n_mmproj_embd * sizeof(float)); - } else { - // Multi-tile (LlavaNext-style) pack_and_unpad. Layout: - // out[0..per_tile] = base tile features (overview) - // out[per_tile..final_len] = grid features with newline col - // - // Grid: entries[1..end] are grid tiles in row-major order. - // Each tile contributes a per_tile_side × per_tile_side block - // of tokens, concatenated into a (grid_y*side, grid_x*side) - // spatial map, then unpadded against the original image - // aspect and topped up with one image_newline column. - const int per_tile_side = (int) std::lround(std::sqrt((double)n_per_tile)); - GGML_ASSERT(per_tile_side * per_tile_side == n_per_tile); - const int grid_x = (int) image_tokens->batch_f32.grid_x; - const int grid_y = (int) image_tokens->batch_f32.grid_y; - GGML_ASSERT(grid_x > 0 && grid_y > 0); - GGML_ASSERT((size_t)(grid_x * grid_y + 1) == entries.size()); - - const int H = per_tile_side * grid_y; - const int W = per_tile_side * grid_x; - - // Copy base tile (entries[0]) directly. - std::memcpy(ctx->image_embd_v.data(), - per_tile.data(), - n_per_tile * n_mmproj_embd * sizeof(float)); - size_t out_off = n_per_tile * n_mmproj_embd; - - // Reshape grid tiles into a (H, W, n_mmproj_embd) spatial - // buffer. Grid order matches entry indices 1..end in - // row-major (tile y*grid_x + x is at entries[1 + y*grid_x + x]). - const size_t spatial_floats = (size_t) H * W * n_mmproj_embd; - std::vector spatial(spatial_floats); - for (int gy = 0; gy < grid_y; ++gy) { - for (int gx = 0; gx < grid_x; ++gx) { - const int tile_idx = gy * grid_x + gx; // among grid tiles - const float * tile_src = per_tile.data() - + (1 + tile_idx) * n_per_tile * n_mmproj_embd; - for (int ty = 0; ty < per_tile_side; ++ty) { - for (int tx = 0; tx < per_tile_side; ++tx) { - const int y = gy * per_tile_side + ty; - const int x = gx * per_tile_side + tx; - const int src_row = ty * per_tile_side + tx; - std::memcpy( - spatial.data() + ((size_t) y * W + x) * n_mmproj_embd, - tile_src + src_row * n_mmproj_embd, - n_mmproj_embd * sizeof(float)); - } - } - } - } - - // Unpad: crop rows or columns based on original image aspect. - // image_tokens carries no image_size field today; reconstruct - // from nx/ny if possible, otherwise skip cropping. - // For now assume no unpad (i.e. the pinpoint grid perfectly - // matches the image aspect, which is the common case; unpad - // cropping will be added when multi-aspect images regress). - int H_unp = H; - int W_unp = W; - // TODO: implement unpad crop based on original image size. - - // Append newline column: for each row (H_unp), append one - // newline_row across the K slices. - const int W_final = W_unp + 1; - const size_t grid_tokens = (size_t) H_unp * W_final; - for (int y = 0; y < H_unp; ++y) { - // Copy the W_unp row of spatial features. - std::memcpy( - ctx->image_embd_v.data() + out_off + (size_t) y * W_final * n_mmproj_embd, - spatial.data() + (size_t) y * W * n_mmproj_embd, - (size_t) W_unp * n_mmproj_embd * sizeof(float)); - // Copy the newline row into the last column of this row. - std::memcpy( - ctx->image_embd_v.data() + out_off - + ((size_t) y * W_final + W_unp) * n_mmproj_embd, - newline_row.data(), - n_mmproj_embd * sizeof(float)); - } - - // Sanity check token count. - const size_t final_tokens = n_per_tile + grid_tokens; - GGML_ASSERT(final_tokens == image_tokens->n_tokens() - && "mtmd_encode: token count mismatch with image_tokens->n_tokens"); - } - return 0; + ok = clip_image_assemble(ctx_clip, ctx->n_threads, + per_tile.data(), + (int) entries.size(), + (int) image_tokens->batch_f32.grid_x, + (int) image_tokens->batch_f32.grid_y, + ctx->image_embd_v.data()); + return ok ? 0 : 1; } if (clip_is_llava(ctx_clip) From 3f159578bb9ce76ce3e047a346a2fccb7b7473dd Mon Sep 17 00:00:00 2001 From: Gabe Goodhart Date: Fri, 22 May 2026 13:01:53 -0600 Subject: [PATCH 59/94] refactor(mtmd): Consolidate assembler logic into clip_assembler class family Just like `clip_graph` is the base class for building the model-specific encoder graphs, `clip_assembler` will be the base class for building the model-specific assembler graphs. This allows the assembly pattern to follow how the encoder pattern is implemented where the model-specific logic lives in a subclass co-located with the encoder graph builder that gets constructed by a simple factory method. Branch: Granite4Vision AI-usage: full (Claude Code + Opus 4.7) Signed-off-by: Gabe Goodhart --- tools/mtmd/clip-graph.h | 16 ++++ tools/mtmd/clip-model.h | 1 + tools/mtmd/clip.cpp | 126 +++++++++++---------------- tools/mtmd/models/granite-vision.cpp | 79 +++++++++++------ tools/mtmd/models/models.h | 51 +++++------ 5 files changed, 146 insertions(+), 127 deletions(-) diff --git a/tools/mtmd/clip-graph.h b/tools/mtmd/clip-graph.h index c5e880c71ec2..7f85ca0cf1ed 100644 --- a/tools/mtmd/clip-graph.h +++ b/tools/mtmd/clip-graph.h @@ -126,3 +126,19 @@ struct clip_graph { // Abstracts out the StackAudioFrames logic used by ultravox ggml_tensor * build_stack(ggml_tensor * cur, int32_t stack_factor, int32_t n_embed); }; + +// Abstract base for post-encoding assembler graphs (e.g. llava-next-style +// pack-and-unpad with learned newline injection). Models that need an +// assembler register a subclass and a factory case in clip_image_assemble. +struct clip_assembler { + virtual ~clip_assembler() = default; + + // Build the cgraph in the supplied ctx0; return the output tensor + // (also expanded into gf). Per-call inputs (per_tile data, n_tiles, + // grid dims, etc.) are stashed by the subclass constructor. + virtual ggml_tensor * build(ggml_context * ctx0, ggml_cgraph * gf) = 0; + + // Push input data into the allocated input tensors. Called after + // ggml_backend_sched_alloc_graph and before compute. + virtual void set_inputs(ggml_cgraph * gf) = 0; +}; diff --git a/tools/mtmd/clip-model.h b/tools/mtmd/clip-model.h index a958ffd18a17..7e618c9e367f 100644 --- a/tools/mtmd/clip-model.h +++ b/tools/mtmd/clip-model.h @@ -621,3 +621,4 @@ struct clip_model { }; const clip_hparams * clip_get_hparams(const struct clip_ctx * ctx); +const clip_model * clip_get_model (const struct clip_ctx * ctx); diff --git a/tools/mtmd/clip.cpp b/tools/mtmd/clip.cpp index 65394f2d80cd..8c90e5a8c7d4 100644 --- a/tools/mtmd/clip.cpp +++ b/tools/mtmd/clip.cpp @@ -4368,85 +4368,61 @@ bool clip_image_assemble(clip_ctx * ctx, const int n_threads, int n_tiles, int grid_x, int grid_y, float * out_embd) { - // Extend this switch for all models that require assembling. All others - // will fall through to the default case and return false indicating no - // assembly required. + // Factory: each model that needs post-encode assembly registers a + // clip_assembler subclass and adds a one-line case here. + std::unique_ptr builder; switch (ctx->proj_type()) { case PROJECTOR_TYPE_GRANITE4_VISION: - { - GGML_ASSERT(per_tile_embd != nullptr && out_embd != nullptr && n_tiles > 0); - - const int n_mmproj_embd = clip_n_mmproj_embd(ctx); - - // Recover n_per_tile from a representative entry. The encoder uses a - // fixed tile size for G4V, so all entries produce the same per-tile - // count; the warmup path stashes a representative image but we use the - // model's image_size since the assembler runs after encoding has - // finished. - clip_image_f32 tile_img; - tile_img.nx = ctx->model.hparams.image_size; - tile_img.ny = ctx->model.hparams.image_size; - const int n_per_tile = clip_n_output_tokens(ctx, &tile_img); - - // Build host-side gather index. - std::vector gather_idx = - granite_vision_build_assembler_gather_idx(n_tiles, n_per_tile, grid_x, grid_y); - const int gather_len = (int) gather_idx.size(); - - // Build the assembler cgraph in a fresh ggml context backed by the - // shared compute-meta buffer. - struct ggml_init_params params = { - /*.mem_size =*/ ctx->buf_compute_meta.size(), - /*.mem_buffer =*/ ctx->buf_compute_meta.data(), - /*.no_alloc =*/ true, - }; - ggml_context_ptr ctx0_ptr(ggml_init(params)); - ggml_context * ctx0 = ctx0_ptr.get(); - GGML_ASSERT(ctx0 != nullptr); - ggml_cgraph * gf = ggml_new_graph_custom(ctx0, ctx->max_nodes, false); - - ggml_tensor * out = granite_vision_build_assembler_graph( - ctx0, gf, ctx->model, n_tiles, n_per_tile, n_mmproj_embd, gather_len); - - // Allocate buffers for this graph on the sched. - ggml_backend_sched_reset(ctx->sched.get()); - if (!ggml_backend_sched_alloc_graph(ctx->sched.get(), gf)) { - LOG_ERR("%s: ggml_backend_sched_alloc_graph failed\n", __func__); - return false; - } + builder = std::make_unique( + ctx, per_tile_embd, n_tiles, grid_x, grid_y); + break; + default: + return false; + } + GGML_ASSERT(out_embd != nullptr); - // Set inputs. - ggml_tensor * in_per_tile = ggml_graph_get_tensor(gf, "g4v_assembler_per_tile_in"); - ggml_tensor * in_gather = ggml_graph_get_tensor(gf, "g4v_assembler_gather_idx"); - GGML_ASSERT(in_per_tile != nullptr && in_gather != nullptr); - ggml_backend_tensor_set(in_per_tile, per_tile_embd, 0, - (size_t) n_tiles * n_per_tile * n_mmproj_embd * sizeof(float)); - ggml_backend_tensor_set(in_gather, gather_idx.data(), 0, - gather_idx.size() * sizeof(int32_t)); - - // Thread count on the CPU backend (matches clip_image_batch_encode). - ggml_backend_dev_t dev = ggml_backend_get_device(ctx->backend_cpu); - ggml_backend_reg_t reg = dev ? ggml_backend_dev_backend_reg(dev) : nullptr; - if (reg) { - auto set_n_threads_fn = (ggml_backend_set_n_threads_t) - ggml_backend_reg_get_proc_address(reg, "ggml_backend_set_n_threads"); - if (set_n_threads_fn) { - set_n_threads_fn(ctx->backend_cpu, n_threads); - } - } + // Build the cgraph in a fresh ggml context backed by the shared + // compute-meta buffer. + struct ggml_init_params params = { + /*.mem_size =*/ ctx->buf_compute_meta.size(), + /*.mem_buffer =*/ ctx->buf_compute_meta.data(), + /*.no_alloc =*/ true, + }; + ggml_context_ptr ctx0_ptr(ggml_init(params)); + ggml_context * ctx0 = ctx0_ptr.get(); + GGML_ASSERT(ctx0 != nullptr); + ggml_cgraph * gf = ggml_new_graph_custom(ctx0, ctx->max_nodes, false); - auto status = ggml_backend_sched_graph_compute(ctx->sched.get(), gf); - if (status != GGML_STATUS_SUCCESS) { - LOG_ERR("%s: ggml_backend_sched_graph_compute failed (%d)\n", __func__, status); - return false; - } + ggml_tensor * out = builder->build(ctx0, gf); - ggml_backend_tensor_get(out, out_embd, 0, ggml_nbytes(out)); - return true; - } - default: - return false; + // Allocate buffers for this graph on the sched. + ggml_backend_sched_reset(ctx->sched.get()); + if (!ggml_backend_sched_alloc_graph(ctx->sched.get(), gf)) { + LOG_ERR("%s: ggml_backend_sched_alloc_graph failed\n", __func__); + return false; + } + + builder->set_inputs(gf); + + // Thread count on the CPU backend (matches clip_image_batch_encode). + ggml_backend_dev_t dev = ggml_backend_get_device(ctx->backend_cpu); + ggml_backend_reg_t reg = dev ? ggml_backend_dev_backend_reg(dev) : nullptr; + if (reg) { + auto set_n_threads_fn = (ggml_backend_set_n_threads_t) + ggml_backend_reg_get_proc_address(reg, "ggml_backend_set_n_threads"); + if (set_n_threads_fn) { + set_n_threads_fn(ctx->backend_cpu, n_threads); + } } + + auto status = ggml_backend_sched_graph_compute(ctx->sched.get(), gf); + if (status != GGML_STATUS_SUCCESS) { + LOG_ERR("%s: ggml_backend_sched_graph_compute failed (%d)\n", __func__, status); + return false; + } + + ggml_backend_tensor_get(out, out_embd, 0, ggml_nbytes(out)); + return true; } int clip_n_mmproj_embd(const struct clip_ctx * ctx) { @@ -4582,6 +4558,10 @@ const clip_hparams * clip_get_hparams(const struct clip_ctx * ctx) { return &ctx->model.hparams; } +const clip_model * clip_get_model(const struct clip_ctx * ctx) { + return &ctx->model; +} + std::map clip_get_mem_usage(const struct clip_ctx * ctx) { std::map result = ctx->mem_usage; for (auto & [dev, size] : ctx->mem_compute) { diff --git a/tools/mtmd/models/granite-vision.cpp b/tools/mtmd/models/granite-vision.cpp index 07a3df7eb570..3016637c73f9 100644 --- a/tools/mtmd/models/granite-vision.cpp +++ b/tools/mtmd/models/granite-vision.cpp @@ -359,21 +359,20 @@ size_t granite_vision_n_assembled_output_tokens( return (size_t)n_per_tile + super_h * super_w_plus_nl; } -std::vector granite_vision_build_assembler_gather_idx( +// Build the host-side gather index. Each entry indexes into a virtual buffer +// of length (n_tiles*n_per_tile + 1) whose last row is the K-tiled newline. +static std::vector g4v_build_gather_idx( int n_tiles, int n_per_tile, int grid_x, int grid_y) { - // virtual source layout: [tile_0 tokens | tile_1 tokens | ... | newline_row] - // newline_row is at virtual index n_tiles * n_per_tile. const int32_t nl_idx = n_tiles * n_per_tile; std::vector idx; - // overview comes first idx.reserve((size_t)n_per_tile + 1); + // overview comes first for (int i = 0; i < n_per_tile; ++i) { idx.push_back(i); } if (n_tiles == 1) { - // single tile: just one trailing newline idx.push_back(nl_idx); return idx; } @@ -383,9 +382,6 @@ std::vector granite_vision_build_assembler_gather_idx( const int per_tile_side = (int) std::lround(std::sqrt((double)n_per_tile)); GGML_ASSERT(per_tile_side * per_tile_side == n_per_tile); - // Spatial scan: for each super-row, walk across grid columns then within - // each tile across local columns, ending with a newline. This reproduces - // the loop nest from the pre-refactor host-side code exactly. const size_t total = (size_t)n_per_tile + (size_t)per_tile_side * (size_t)grid_y * ((size_t)per_tile_side * (size_t)grid_x + 1); @@ -406,14 +402,32 @@ std::vector granite_vision_build_assembler_gather_idx( return idx; } -ggml_tensor * granite_vision_build_assembler_graph( - ggml_context * ctx0, - ggml_cgraph * gf, - const clip_model & model, - int n_tiles, - int n_per_tile, - int n_mmproj_embd, - int gather_len) { +clip_assembler_granite_vision::clip_assembler_granite_vision( + const clip_ctx * ctx, + const float * per_tile_embd, + int n_tiles, int grid_x, int grid_y) + : model(*clip_get_model(ctx)), + per_tile_embd(per_tile_embd), + n_tiles(n_tiles), + grid_x(grid_x), + grid_y(grid_y) { + GGML_ASSERT(per_tile_embd != nullptr && n_tiles > 0); + + n_mmproj_embd = clip_n_mmproj_embd(ctx); + + // Recover n_per_tile from the model's fixed tile size. All G4V tiles + // are produced at hparams.image_size × hparams.image_size, so a dummy + // tile image at that size yields the same n_per_tile as any real tile. + clip_image_f32 tile_img; + tile_img.nx = model.hparams.image_size; + tile_img.ny = model.hparams.image_size; + n_per_tile = clip_n_output_tokens(const_cast(ctx), &tile_img); + + gather_idx = g4v_build_gather_idx(n_tiles, n_per_tile, grid_x, grid_y); +} + +ggml_tensor * clip_assembler_granite_vision::build( + ggml_context * ctx0, ggml_cgraph * gf) { const int K = (int) model.qf_proj_blocks.size(); GGML_ASSERT(K > 0); GGML_ASSERT(n_mmproj_embd % K == 0); @@ -421,23 +435,21 @@ ggml_tensor * granite_vision_build_assembler_graph( GGML_ASSERT(model.image_newline != nullptr); GGML_ASSERT(ggml_nelements(model.image_newline) == projection_dim); - // 1) Inputs + // 1) Inputs. ggml_tensor * per_tile_in = ggml_new_tensor_2d( ctx0, GGML_TYPE_F32, n_mmproj_embd, n_tiles * n_per_tile); ggml_set_name(per_tile_in, "g4v_assembler_per_tile_in"); ggml_set_input(per_tile_in); - ggml_tensor * gather_idx = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, gather_len); - ggml_set_name(gather_idx, "g4v_assembler_gather_idx"); - ggml_set_input(gather_idx); + ggml_tensor * idx_in = ggml_new_tensor_1d( + ctx0, GGML_TYPE_I32, (int64_t) gather_idx.size()); + ggml_set_name(idx_in, "g4v_assembler_gather_idx"); + ggml_set_input(idx_in); // 2) Build K-tiled, base-scaled newline row. - // - // Final layout matches the pre-refactor host code: - // newline_row[k*projection_dim + d] = nl[d] * (k == 0 ? base : 1.0) - // - // i.e. with ne[0]=projection_dim, ne[1]=K stacking, then reshape to a - // 1-row vector of length K*projection_dim. + // newline_row[k*projection_dim + d] = nl[d] * (k == 0 ? base : 1.0) + // i.e. with ne[0]=projection_dim, ne[1]=K stacking, then reshape to + // a 1-row vector of length K*projection_dim. ggml_tensor * nl = model.image_newline; // (projection_dim,) ggml_tensor * nl_first = ggml_scale(ctx0, nl, model.hparams.base_stream_scale); ggml_tensor * nl_first_2d = ggml_reshape_2d(ctx0, nl_first, projection_dim, 1); @@ -451,7 +463,6 @@ ggml_tensor * granite_vision_build_assembler_graph( ggml_tensor * nl_rest = ggml_repeat(ctx0, nl_2d, rest_template); nl_row_2d = ggml_concat(ctx0, nl_first_2d, nl_rest, 1); // (projection_dim, K) } - // Reshape to a single row of length n_mmproj_embd. Must be contiguous. nl_row_2d = ggml_cont(ctx0, nl_row_2d); ggml_tensor * nl_row = ggml_reshape_2d(ctx0, nl_row_2d, n_mmproj_embd, 1); ggml_set_name(nl_row, "g4v_assembler_nl_row"); @@ -463,9 +474,19 @@ ggml_tensor * granite_vision_build_assembler_graph( // 4) Gather. ggml_get_rows selects along ne[1], producing a tensor of // shape (n_mmproj_embd, gather_len) in spatial scan order. - ggml_tensor * out = ggml_get_rows(ctx0, src, gather_idx); + ggml_tensor * out = ggml_get_rows(ctx0, src, idx_in); ggml_set_name(out, "g4v_assembler_out"); ggml_build_forward_expand(gf, out); return out; -} \ No newline at end of file +} + +void clip_assembler_granite_vision::set_inputs(ggml_cgraph * gf) { + ggml_tensor * in_per_tile = ggml_graph_get_tensor(gf, "g4v_assembler_per_tile_in"); + ggml_tensor * in_gather = ggml_graph_get_tensor(gf, "g4v_assembler_gather_idx"); + GGML_ASSERT(in_per_tile != nullptr && in_gather != nullptr); + ggml_backend_tensor_set(in_per_tile, per_tile_embd, 0, + (size_t) n_tiles * n_per_tile * n_mmproj_embd * sizeof(float)); + ggml_backend_tensor_set(in_gather, gather_idx.data(), 0, + gather_idx.size() * sizeof(int32_t)); +} diff --git a/tools/mtmd/models/models.h b/tools/mtmd/models/models.h index 38cf9e2bd8d4..b129f3f4eb9f 100644 --- a/tools/mtmd/models/models.h +++ b/tools/mtmd/models/models.h @@ -201,31 +201,32 @@ struct clip_graph_granite_vision : clip_graph { // // Per-tile encoder outputs are reshaped (pack-and-unpad) and interleaved with // a learned, base-scaled newline embedding to produce the final image-token -// sequence. These helpers are used by clip_image_assemble and -// clip_n_assembled_output_tokens dispatch in clip.cpp. - -// Build the host-side gather index used by the assembler graph. Entries -// index into a virtual buffer of length n_tiles*n_per_tile + 1 where the -// last row is the K-tiled newline. The number of entries returned is the -// final assembled token count. -std::vector granite_vision_build_assembler_gather_idx( - int n_tiles, int n_per_tile, int grid_x, int grid_y); - -// Build the assembler cgraph into the supplied ggml context and graph. -// Returns the final output tensor (also expanded into gf). Creates two -// input tensors registered with ggml_set_input: -// - "g4v_assembler_per_tile_in" : F32 (n_mmproj_embd, n_per_tile * n_tiles) -// - "g4v_assembler_gather_idx" : I32 (gather_len,) -ggml_tensor * granite_vision_build_assembler_graph( - ggml_context * ctx0, - ggml_cgraph * gf, - const clip_model & model, - int n_tiles, - int n_per_tile, - int n_mmproj_embd, - int gather_len); - -// Final assembled token count for a preprocessed tile batch. +// sequence. The cgraph is a single ggml_get_rows over a virtual buffer formed +// by concatenating the per-tile encoder outputs with the K-tiled, base-scaled +// newline row. All model-specific knowledge (n_per_tile inference, gather +// index construction, input tensor names) lives on the class below. + +struct clip_assembler_granite_vision : clip_assembler { + const clip_model & model; + const float * per_tile_embd; // borrowed, caller-owned lifetime + int n_tiles; + int grid_x; + int grid_y; + int n_per_tile; + int n_mmproj_embd; + std::vector gather_idx; // computed in constructor + + clip_assembler_granite_vision(const clip_ctx * ctx, + const float * per_tile_embd, + int n_tiles, int grid_x, int grid_y); + + ggml_tensor * build(ggml_context * ctx0, ggml_cgraph * gf) override; + void set_inputs(ggml_cgraph * gf) override; +}; + +// Final assembled token count for a preprocessed tile batch. Called before +// per-tile encoding (from add_media), so kept as a free function rather than +// a method on the assembler class. size_t granite_vision_n_assembled_output_tokens( const clip_ctx * ctx, const clip_image_f32_batch * batch); From 1754e313b9e384611d4a3c7e87808482014a4f38 Mon Sep 17 00:00:00 2001 From: Gabe Goodhart Date: Fri, 22 May 2026 13:03:42 -0600 Subject: [PATCH 60/94] style: Comment improvement Branch: Granite4Vision AI-usage: none Signed-off-by: Gabe Goodhart --- tools/mtmd/models/granite-vision.cpp | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tools/mtmd/models/granite-vision.cpp b/tools/mtmd/models/granite-vision.cpp index 3016637c73f9..a24f3316e0fd 100644 --- a/tools/mtmd/models/granite-vision.cpp +++ b/tools/mtmd/models/granite-vision.cpp @@ -311,8 +311,9 @@ ggml_cgraph * clip_graph_granite_vision::build() { qformer_eps); } - // Sort by target LLM layer (implicit from vision_feature_layer ordering in conversion) - // For Granite Vision, the blocks are already in the correct order + // Concatenate the projector activations in the correct order, and apply the + // base stream multiplier to invert the embedding multipler that gets + // applied on the LLM side. ggml_tensor * mmproj = nullptr; for (int k = 0; k < projector_count; ++k) { ggml_tensor * s = ggml_cont(ctx0, streams[k]); From 0a6c5cf085af7c32f6d77f587f0e29e9652a5aeb Mon Sep 17 00:00:00 2001 From: Gabe Goodhart Date: Fri, 22 May 2026 13:21:31 -0600 Subject: [PATCH 61/94] refactor: granite_vision -> granite4_vision Branch: Granite4Vision AI-usage: none Signed-off-by: Gabe Goodhart --- tools/mtmd/CMakeLists.txt | 2 +- tools/mtmd/clip.cpp | 6 +++--- .../{granite-vision.cpp => granite4-vision.cpp} | 10 +++++----- tools/mtmd/models/models.h | 14 +++++++------- 4 files changed, 16 insertions(+), 16 deletions(-) rename tools/mtmd/models/{granite-vision.cpp => granite4-vision.cpp} (98%) diff --git a/tools/mtmd/CMakeLists.txt b/tools/mtmd/CMakeLists.txt index e901ec6ca6ef..4ca0083e7e2d 100644 --- a/tools/mtmd/CMakeLists.txt +++ b/tools/mtmd/CMakeLists.txt @@ -22,7 +22,7 @@ add_library(mtmd models/gemma4v.cpp models/glm4v.cpp models/granite-speech.cpp - models/granite-vision.cpp + models/granite4-vision.cpp models/hunyuanvl.cpp models/internvl.cpp models/kimivl.cpp diff --git a/tools/mtmd/clip.cpp b/tools/mtmd/clip.cpp index 8c90e5a8c7d4..f36244ab5672 100644 --- a/tools/mtmd/clip.cpp +++ b/tools/mtmd/clip.cpp @@ -982,7 +982,7 @@ static ggml_cgraph * clip_image_build_graph(clip_ctx * ctx, const clip_image_f32 } break; case PROJECTOR_TYPE_GRANITE4_VISION: { - builder = std::make_unique(ctx, img); + builder = std::make_unique(ctx, img); } break; default: GGML_ABORT("missing cgraph builder"); @@ -4351,7 +4351,7 @@ size_t clip_n_assembled_output_tokens(clip_ctx * ctx, const clip_image_f32_batch GGML_ASSERT(batch != nullptr); switch (ctx->proj_type()) { case PROJECTOR_TYPE_GRANITE4_VISION: - return granite_vision_n_assembled_output_tokens(ctx, batch); + return granite4_vision_n_assembled_output_tokens(ctx, batch); // No assembler: flat sum of per-entry token counts. default: break; @@ -4373,7 +4373,7 @@ bool clip_image_assemble(clip_ctx * ctx, const int n_threads, std::unique_ptr builder; switch (ctx->proj_type()) { case PROJECTOR_TYPE_GRANITE4_VISION: - builder = std::make_unique( + builder = std::make_unique( ctx, per_tile_embd, n_tiles, grid_x, grid_y); break; default: diff --git a/tools/mtmd/models/granite-vision.cpp b/tools/mtmd/models/granite4-vision.cpp similarity index 98% rename from tools/mtmd/models/granite-vision.cpp rename to tools/mtmd/models/granite4-vision.cpp index a24f3316e0fd..157878ea7a66 100644 --- a/tools/mtmd/models/granite-vision.cpp +++ b/tools/mtmd/models/granite4-vision.cpp @@ -238,7 +238,7 @@ static ggml_tensor * g4v_build_block( // build() — top-level graph // --------------------------------------------------------------------------- -ggml_cgraph * clip_graph_granite_vision::build() { +ggml_cgraph * clip_graph_granite4_vision::build() { GGML_ASSERT(model.patch_embeddings_0 != nullptr); GGML_ASSERT(model.position_embeddings != nullptr); GGML_ASSERT(model.class_embedding == nullptr); @@ -342,7 +342,7 @@ ggml_cgraph * clip_graph_granite_vision::build() { // concatenating per-tile encoder outputs with the K-tiled, base-scaled // newline row. The gather index is built host-side. -size_t granite_vision_n_assembled_output_tokens( +size_t granite4_vision_n_assembled_output_tokens( const clip_ctx * ctx, const clip_image_f32_batch * batch) { GGML_ASSERT(batch != nullptr && !batch->entries.empty()); @@ -403,7 +403,7 @@ static std::vector g4v_build_gather_idx( return idx; } -clip_assembler_granite_vision::clip_assembler_granite_vision( +clip_assembler_granite4_vision::clip_assembler_granite4_vision( const clip_ctx * ctx, const float * per_tile_embd, int n_tiles, int grid_x, int grid_y) @@ -427,7 +427,7 @@ clip_assembler_granite_vision::clip_assembler_granite_vision( gather_idx = g4v_build_gather_idx(n_tiles, n_per_tile, grid_x, grid_y); } -ggml_tensor * clip_assembler_granite_vision::build( +ggml_tensor * clip_assembler_granite4_vision::build( ggml_context * ctx0, ggml_cgraph * gf) { const int K = (int) model.qf_proj_blocks.size(); GGML_ASSERT(K > 0); @@ -482,7 +482,7 @@ ggml_tensor * clip_assembler_granite_vision::build( return out; } -void clip_assembler_granite_vision::set_inputs(ggml_cgraph * gf) { +void clip_assembler_granite4_vision::set_inputs(ggml_cgraph * gf) { ggml_tensor * in_per_tile = ggml_graph_get_tensor(gf, "g4v_assembler_per_tile_in"); ggml_tensor * in_gather = ggml_graph_get_tensor(gf, "g4v_assembler_gather_idx"); GGML_ASSERT(in_per_tile != nullptr && in_gather != nullptr); diff --git a/tools/mtmd/models/models.h b/tools/mtmd/models/models.h index b129f3f4eb9f..c52c960f4c54 100644 --- a/tools/mtmd/models/models.h +++ b/tools/mtmd/models/models.h @@ -192,8 +192,8 @@ struct clip_graph_kimik25 : clip_graph { ggml_tensor * resize_position_embeddings_3d(uint32_t interpolation_mode); }; -struct clip_graph_granite_vision : clip_graph { - clip_graph_granite_vision(clip_ctx * ctx, const clip_image_f32 & img) : clip_graph(ctx, img) {} +struct clip_graph_granite4_vision : clip_graph { + clip_graph_granite4_vision(clip_ctx * ctx, const clip_image_f32 & img) : clip_graph(ctx, img) {} ggml_cgraph * build() override; }; @@ -206,7 +206,7 @@ struct clip_graph_granite_vision : clip_graph { // newline row. All model-specific knowledge (n_per_tile inference, gather // index construction, input tensor names) lives on the class below. -struct clip_assembler_granite_vision : clip_assembler { +struct clip_assembler_granite4_vision : clip_assembler { const clip_model & model; const float * per_tile_embd; // borrowed, caller-owned lifetime int n_tiles; @@ -216,9 +216,9 @@ struct clip_assembler_granite_vision : clip_assembler { int n_mmproj_embd; std::vector gather_idx; // computed in constructor - clip_assembler_granite_vision(const clip_ctx * ctx, - const float * per_tile_embd, - int n_tiles, int grid_x, int grid_y); + clip_assembler_granite4_vision(const clip_ctx * ctx, + const float * per_tile_embd, + int n_tiles, int grid_x, int grid_y); ggml_tensor * build(ggml_context * ctx0, ggml_cgraph * gf) override; void set_inputs(ggml_cgraph * gf) override; @@ -227,6 +227,6 @@ struct clip_assembler_granite_vision : clip_assembler { // Final assembled token count for a preprocessed tile batch. Called before // per-tile encoding (from add_media), so kept as a free function rather than // a method on the assembler class. -size_t granite_vision_n_assembled_output_tokens( +size_t granite4_vision_n_assembled_output_tokens( const clip_ctx * ctx, const clip_image_f32_batch * batch); From 8dc2b24d08b630c452183fc0f25b8ba66e40900d Mon Sep 17 00:00:00 2001 From: Gabe Goodhart Date: Fri, 22 May 2026 13:23:15 -0600 Subject: [PATCH 62/94] fix: Remove dead codepath for Qwen3VL add_vision_is_deepstack These pieces were never used on the c++ side (removed there in an earlier commit), so this is just cleanup that I missed before. Branch: Granite4Vision AI-usage: none Signed-off-by: Gabe Goodhart --- conversion/qwen3vl.py | 7 ------- 1 file changed, 7 deletions(-) diff --git a/conversion/qwen3vl.py b/conversion/qwen3vl.py index 9f11757697f5..11bff927478e 100644 --- a/conversion/qwen3vl.py +++ b/conversion/qwen3vl.py @@ -35,10 +35,6 @@ def __init__(self, *args, **kwargs): self.hparams_vision["num_attention_heads"] = self.hparams_vision.get("num_heads") self.hparams_vision["num_hidden_layers"] = self.hparams_vision.get("depth") - self.is_deepstack_layers = [False] * int(self.hparams_vision["num_hidden_layers"] or 0) - for idx in self.hparams_vision.get("deepstack_visual_indexes", []): - self.is_deepstack_layers[idx] = True - def set_gguf_parameters(self): super().set_gguf_parameters() # in case mixed modalities, the arch will be handled by subclass @@ -55,9 +51,6 @@ def set_gguf_parameters(self): rms_norm_eps = self.global_config.get("text_config", {}).get("rms_norm_eps", 1e-6) self.gguf_writer.add_vision_attention_layernorm_eps(rms_norm_eps) - if self.is_deepstack_layers: - self.gguf_writer.add_vision_is_deepstack_layers(self.is_deepstack_layers) - @classmethod def filter_tensors(cls, item: tuple[str, Callable[[], Tensor]]) -> tuple[str, Callable[[], Tensor]] | None: name, gen = item From 2c4e167050b9e9efa1b53a99a578c50ad6d91dda Mon Sep 17 00:00:00 2001 From: Gabe Goodhart Date: Fri, 22 May 2026 13:42:07 -0600 Subject: [PATCH 63/94] fix: Oops! I did not mean to commit one of my prompt files But now it's too far back in history to effectively rebase out, even with interactive and --rebase-merges :( Branch: Granite4Vision AI-usage: none Signed-off-by: Gabe Goodhart --- src/g4v-prompt.md | 21 --------------------- 1 file changed, 21 deletions(-) delete mode 100644 src/g4v-prompt.md diff --git a/src/g4v-prompt.md b/src/g4v-prompt.md deleted file mode 100644 index 99e79f3ac738..000000000000 --- a/src/g4v-prompt.md +++ /dev/null @@ -1,21 +0,0 @@ -The next feature to implement is support for the `granite-4.0-3b-vision` model (https://huggingface.co/ibm-granite/granite-4.0-3b-vision). This model uses a conditional LoRA adapter. For now, we'll require that the user toggle the adapter on/off manually. You can find a local copy of the model at /Users/ghart/models/ibm-granite/granite-4.0-3b-vision. - -The `transformers` implementation for this model is embedded in the model itself, so you can find the full details in `/Users/ghart/models/ibm-granite/granite-4.0-3b-vision/*.py`. - -When implementing support for this model here in `llama.cpp`, we need to do the following: - -A. Add GGUF conversion support: - 1. Add the necessary architecture/hparam/tensor names and enums in `gguf-py/gguf/` - 2. Add translation support for the model in `convert_hf_to_gguf.py` (both the core text model and the mmproj) - 3. Add support for the adapter conversion in `convert_lora_to_gguf.py` - -B. Add `mtmd` support (c++): - 1. Add the corresponding architecture/hparam/tensor names and enums on the c++ side in either `tools/mtmd` or `src/`. I don't _think_ there should be any changes to the enums in the core `src/` library since the text model is already a well supported architecture, but I'm not sure about that. - 2. Ensure support for the vision tokenizer and mmproj model in `tools/mtmd` - 3. Implement support for using the converted LoRA as an adapter - -I want to base this work on the current branch which points to the PR for Granite Speech (https://github.com/ggml-org/llama.cpp/pull/22101). I believe these models rely on some of the same QFormer components, so as much as possible those should be reused. - -The other main architectural feature for this model is multiple layer injection points from the multimodal projector. You can see this in `modeling.py` with the `deepstack_features` list that gets created during projection, then injected at the configured layers using `masked_scatter`. This injection capability is similar to the Per-Layer-Embeddings (PLE) for Gemma4 or the `n_deepstack_layers` for `qwen3vl`. The main difference is the way the layers are determined for Gemma4, every layer is assumed to have a PLE. For `quen3vl`, it's the first N layers. For Granite 4 vision, the layers are a list specified in config. To implement this, we'll likely need to extend the `llama_hparams` struct's `n_deepstack_layers` to support a layer array (see eg `recurrent_layer_arr` or other `std::array` arrays). We'll then need to support GGUF values that can be either a list or a single number which would then map back to the single `n_deepstack_layers` value. - -I want you to further analyze the model architecture, check with the current base branch for Granite 4 Speech, and come up with a plan to implement support for Granite 4 Vision. \ No newline at end of file From 72355e974415b93c4154fcb0727270a42f88acc8 Mon Sep 17 00:00:00 2001 From: Gabe Goodhart Date: Fri, 22 May 2026 15:23:30 -0600 Subject: [PATCH 64/94] fix: Add missing include for std::find It seems that this was already pulled in on some platforms, but not on others Branch: Granite4Vision AI-usage: none Signed-off-by: Gabe Goodhart --- tools/mtmd/models/llava.cpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tools/mtmd/models/llava.cpp b/tools/mtmd/models/llava.cpp index bf280c44e00f..896db43fcda4 100644 --- a/tools/mtmd/models/llava.cpp +++ b/tools/mtmd/models/llava.cpp @@ -1,5 +1,7 @@ #include "models.h" +#include + // this graph is used by llava, granite and glm // due to having embedding_stack (used by granite), we cannot reuse build_vit ggml_cgraph * clip_graph_llava::build() { From 33ec796edb508d109325823bdc623eb4b620fb42 Mon Sep 17 00:00:00 2001 From: Gabe Goodhart Date: Fri, 22 May 2026 15:54:28 -0600 Subject: [PATCH 65/94] fix: Fix Flake8 warnings in granite conversion module Branch: Granite4Vision AI-usage: none Signed-off-by: Gabe Goodhart --- conversion/granite.py | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/conversion/granite.py b/conversion/granite.py index 4f3571f065b4..d806850a300d 100644 --- a/conversion/granite.py +++ b/conversion/granite.py @@ -64,10 +64,10 @@ def filter_tensors(cls, item: tuple[str, Callable[[], Tensor]]) -> tuple[str, Ca name, gen = item # Skip multimodal tensors if ( - name.startswith(("encoder.")) or - "image_" in name or - "layerwise_projectors" in name or - "spatial_projectors" in name + name.startswith(("encoder.")) + or "image_" in name + or "layerwise_projectors" in name + or "spatial_projectors" in name ): return return super().filter_tensors(item) @@ -409,7 +409,6 @@ def __init__(self, *args, **kwargs): def set_gguf_parameters(self): assert self.hparams_vision is not None super().set_gguf_parameters() - v = self.hparams_vision self.gguf_writer.add_clip_projector_type(gguf.VisionProjectorType.GRANITE4_VISION) @@ -450,9 +449,9 @@ def set_gguf_parameters(self): def filter_tensors(cls, item: tuple[str, Callable[[], Tensor]]) -> tuple[str, Callable[[], Tensor]] | None: name, _ = item if ( - name.startswith("language_model.") or - "vision_model.head" in name or - name.startswith("lm_head") + name.startswith("language_model.") + or "vision_model.head" in name + or name.startswith("lm_head") ): return None return super().filter_tensors(item) From 52633fafb803064c7681f8582dcc360f5d12a5d2 Mon Sep 17 00:00:00 2001 From: Gabe Goodhart Date: Tue, 26 May 2026 17:18:35 -0600 Subject: [PATCH 66/94] refactor: Remove clip_assembler in favor of clip_image_f32.append_token Per conversation in the PR, the clip_assembler pattern was too invasive. This is a compromise that limits model-specific blocks to add_media where each preprocessed tile is annotated with an injection type, after which all the token counting logic is generic and the newline injection itself is handled in the graph based on the value for the given tile image. Branch: Granite4Vision AI-usage: draft (Bob, OpenCode + Qwen 3.6 35b) Signed-off-by: Gabe Goodhart --- tools/mtmd/clip-graph.h | 16 -- tools/mtmd/clip-impl.h | 7 + tools/mtmd/clip.cpp | 93 ++--------- tools/mtmd/clip.h | 24 +-- tools/mtmd/models/granite4-vision.cpp | 212 ++++++-------------------- tools/mtmd/models/models.h | 52 ++----- tools/mtmd/mtmd.cpp | 54 +++---- 7 files changed, 112 insertions(+), 346 deletions(-) diff --git a/tools/mtmd/clip-graph.h b/tools/mtmd/clip-graph.h index 7f85ca0cf1ed..c5e880c71ec2 100644 --- a/tools/mtmd/clip-graph.h +++ b/tools/mtmd/clip-graph.h @@ -126,19 +126,3 @@ struct clip_graph { // Abstracts out the StackAudioFrames logic used by ultravox ggml_tensor * build_stack(ggml_tensor * cur, int32_t stack_factor, int32_t n_embed); }; - -// Abstract base for post-encoding assembler graphs (e.g. llava-next-style -// pack-and-unpad with learned newline injection). Models that need an -// assembler register a subclass and a factory case in clip_image_assemble. -struct clip_assembler { - virtual ~clip_assembler() = default; - - // Build the cgraph in the supplied ctx0; return the output tensor - // (also expanded into gf). Per-call inputs (per_tile data, n_tiles, - // grid dims, etc.) are stashed by the subclass constructor. - virtual ggml_tensor * build(ggml_context * ctx0, ggml_cgraph * gf) = 0; - - // Push input data into the allocated input tensors. Called after - // ggml_backend_sched_alloc_graph and before compute. - virtual void set_inputs(ggml_cgraph * gf) = 0; -}; diff --git a/tools/mtmd/clip-impl.h b/tools/mtmd/clip-impl.h index 22a4992839b2..41f678beabf7 100644 --- a/tools/mtmd/clip-impl.h +++ b/tools/mtmd/clip-impl.h @@ -435,6 +435,13 @@ struct clip_image_f32 { int ny; std::vector buf; + + // Indicates what token (if any) should be appended after encoding this tile. + // Used by models like Granite 4 Vision that inject learned newline embeddings. + enum clip_append_token_type { + CLIP_APPEND_TOKEN_NONE = 0, + CLIP_APPEND_TOKEN_NEWLINE_ROWWISE = 1, // Append newline after each row + } append_token = CLIP_APPEND_TOKEN_NONE; }; // diff --git a/tools/mtmd/clip.cpp b/tools/mtmd/clip.cpp index f36244ab5672..042319f1ce35 100644 --- a/tools/mtmd/clip.cpp +++ b/tools/mtmd/clip.cpp @@ -3051,6 +3051,11 @@ void clip_image_size_free(struct clip_image_size * load_image_size) { } void clip_image_u8_free(struct clip_image_u8 * img) { delete img; } void clip_image_f32_free(struct clip_image_f32 * img) { delete img; } +void clip_image_set_append_token(struct clip_image_f32 * img, int token_type) { + if (img) { + img->append_token = (clip_image_f32::clip_append_token_type)token_type; + } +} void clip_image_u8_batch_free(struct clip_image_u8_batch * batch) { delete batch; } void clip_image_f32_batch_free(struct clip_image_f32_batch * batch) { delete batch; } @@ -3385,13 +3390,19 @@ int clip_n_output_tokens(const struct clip_ctx * ctx, struct clip_image_f32 * im // Per-tile output token count: each projector block outputs // query_side^2 tokens per window × n^2 windows. // For 384×384 input: n = 24/8 = 3, query_side = 4 → 144. - // The +1 image_newline row / newline column logic runs in - // mtmd_encode's pack_and_unpad path, not here. const int window_side = ctx->model.hparams.downsample_window_side; const int query_side = ctx->model.hparams.downsample_query_side; const int side = img->nx / params.patch_size; const int n = side / window_side; n_patches = (query_side * n) * (query_side * n); + + // Add newline tokens based on append_token field. + if (img->append_token == clip_image_f32::CLIP_APPEND_TOKEN_NEWLINE_ROWWISE) { + // For single-tile case: append 1 newline row. + // For multi-tile rowwise: handled by caller, but here we + // report the per-tile count including one trailing newline. + n_patches += 1; + } } break; default: GGML_ABORT("unsupported projector type"); @@ -4347,84 +4358,6 @@ bool clip_image_batch_encode(clip_ctx * ctx, const int n_threads, const clip_ima return true; } -size_t clip_n_assembled_output_tokens(clip_ctx * ctx, const clip_image_f32_batch * batch) { - GGML_ASSERT(batch != nullptr); - switch (ctx->proj_type()) { - case PROJECTOR_TYPE_GRANITE4_VISION: - return granite4_vision_n_assembled_output_tokens(ctx, batch); - // No assembler: flat sum of per-entry token counts. - default: - break; - } - size_t n = 0; - for (const auto & e : batch->entries) { - n += clip_n_output_tokens(ctx, e.get()); - } - return n; -} - -bool clip_image_assemble(clip_ctx * ctx, const int n_threads, - const float * per_tile_embd, - int n_tiles, - int grid_x, int grid_y, - float * out_embd) { - // Factory: each model that needs post-encode assembly registers a - // clip_assembler subclass and adds a one-line case here. - std::unique_ptr builder; - switch (ctx->proj_type()) { - case PROJECTOR_TYPE_GRANITE4_VISION: - builder = std::make_unique( - ctx, per_tile_embd, n_tiles, grid_x, grid_y); - break; - default: - return false; - } - GGML_ASSERT(out_embd != nullptr); - - // Build the cgraph in a fresh ggml context backed by the shared - // compute-meta buffer. - struct ggml_init_params params = { - /*.mem_size =*/ ctx->buf_compute_meta.size(), - /*.mem_buffer =*/ ctx->buf_compute_meta.data(), - /*.no_alloc =*/ true, - }; - ggml_context_ptr ctx0_ptr(ggml_init(params)); - ggml_context * ctx0 = ctx0_ptr.get(); - GGML_ASSERT(ctx0 != nullptr); - ggml_cgraph * gf = ggml_new_graph_custom(ctx0, ctx->max_nodes, false); - - ggml_tensor * out = builder->build(ctx0, gf); - - // Allocate buffers for this graph on the sched. - ggml_backend_sched_reset(ctx->sched.get()); - if (!ggml_backend_sched_alloc_graph(ctx->sched.get(), gf)) { - LOG_ERR("%s: ggml_backend_sched_alloc_graph failed\n", __func__); - return false; - } - - builder->set_inputs(gf); - - // Thread count on the CPU backend (matches clip_image_batch_encode). - ggml_backend_dev_t dev = ggml_backend_get_device(ctx->backend_cpu); - ggml_backend_reg_t reg = dev ? ggml_backend_dev_backend_reg(dev) : nullptr; - if (reg) { - auto set_n_threads_fn = (ggml_backend_set_n_threads_t) - ggml_backend_reg_get_proc_address(reg, "ggml_backend_set_n_threads"); - if (set_n_threads_fn) { - set_n_threads_fn(ctx->backend_cpu, n_threads); - } - } - - auto status = ggml_backend_sched_graph_compute(ctx->sched.get(), gf); - if (status != GGML_STATUS_SUCCESS) { - LOG_ERR("%s: ggml_backend_sched_graph_compute failed (%d)\n", __func__, status); - return false; - } - - ggml_backend_tensor_get(out, out_embd, 0, ggml_nbytes(out)); - return true; -} - int clip_n_mmproj_embd(const struct clip_ctx * ctx) { switch (ctx->model.proj_type) { case PROJECTOR_TYPE_LDP: diff --git a/tools/mtmd/clip.h b/tools/mtmd/clip.h index e4acd93366e4..8b46691c2fb5 100644 --- a/tools/mtmd/clip.h +++ b/tools/mtmd/clip.h @@ -94,6 +94,9 @@ size_t clip_image_f32_batch_nx(const struct clip_image_f32_batch * batch, int id size_t clip_image_f32_batch_ny(const struct clip_image_f32_batch * batch, int idx); // equivalent to batch[idx]->ny struct clip_image_f32 * clip_image_f32_get_img(const struct clip_image_f32_batch * batch, int idx); // equivalent to batch[idx]->data +// Set the append_token field of a clip_image_f32 (for newline injection models). +void clip_image_set_append_token(struct clip_image_f32 * img, int token_type); + /** * Build image from pixels decoded by other libraries instead of stb_image.h for better performance. * The memory layout is RGBRGBRGB..., input buffer length must be 3*nx*ny bytes @@ -103,27 +106,6 @@ void clip_build_img_from_pixels(const unsigned char * rgb_pixels, int nx, int ny bool clip_image_encode (struct clip_ctx * ctx, int n_threads, struct clip_image_f32 * img, float * vec); bool clip_image_batch_encode(struct clip_ctx * ctx, int n_threads, const struct clip_image_f32_batch * imgs, float * vec); -// Some projector types perform an additional assembly pass over per-tile -// encoder outputs (e.g. llava-next-style pack-and-unpad with a learned -// newline embedding). For those projectors, clip_image_assemble runs the -// model-specific assembler graph and writes the final embeddings to out_embd. -// For projectors with no assembler, returns false without writing anything. -// per_tile_embd is laid out as (n_mmproj_embd, n_per_tile, n_tiles) with the -// embedding dim innermost; n_per_tile is inferred from the model's per-tile -// output size for an entry of the batch that produced per_tile_embd. -bool clip_image_assemble(struct clip_ctx * ctx, int n_threads, - const float * per_tile_embd, - int n_tiles, - int grid_x, int grid_y, - float * out_embd); - -// For projectors with an assembler, returns the final token count after -// assembly given the preprocessed tile batch. For projectors without an -// assembler, returns the unmodified flat per-tile token count (sum over -// entries of clip_n_output_tokens). -size_t clip_n_assembled_output_tokens(struct clip_ctx * ctx, - const struct clip_image_f32_batch * batch); - bool clip_is_llava(const struct clip_ctx * ctx); // note for contributor: this clip_is_(model) pattern is deprecated // do NOT add new functions like this diff --git a/tools/mtmd/models/granite4-vision.cpp b/tools/mtmd/models/granite4-vision.cpp index 157878ea7a66..a3410fa2e3e4 100644 --- a/tools/mtmd/models/granite4-vision.cpp +++ b/tools/mtmd/models/granite4-vision.cpp @@ -14,6 +14,7 @@ * Stage 1a: SigLIP vision tower (27 layers, post-norm) * Stage 1b: WindowQFormer blocks (8 blocks total) * Stage 1c: Concatenate and pack outputs + * Stage 1d: Append newline tokens if append_token is set * * A single WindowQFormer block maps a (1, 576, 1152) vision-layer * hidden state h to a (1, 144, 2560) deepstack feature. @@ -235,9 +236,46 @@ static ggml_tensor * g4v_build_block( } // --------------------------------------------------------------------------- -// build() — top-level graph +// build() - top-level graph // --------------------------------------------------------------------------- +// Build the K-tiled, base-scaled newline row tensor. +// Shape: (n_mmproj_embd, 1) +ggml_tensor * clip_graph_granite4_vision::build_newline_row(ggml_context * ctx0) { + const int K = (int) model.qf_proj_blocks.size(); + GGML_ASSERT(K > 0); + GGML_ASSERT(n_mmproj_embd % K == 0); + const int projection_dim = n_mmproj_embd / K; + GGML_ASSERT(model.image_newline != nullptr); + GGML_ASSERT(ggml_nelements(model.image_newline) == projection_dim); + + // Build newline_row[k*projection_dim + d] = nl[d] * (k == 0 ? base : 1.0) + ggml_tensor * nl = model.image_newline; // (projection_dim,) + ggml_tensor * nl_first = ggml_scale(ctx0, nl, model.hparams.base_stream_scale); + ggml_tensor * nl_first_2d = ggml_reshape_2d(ctx0, nl_first, projection_dim, 1); + ggml_tensor * nl_row_2d; + if (K == 1) { + nl_row_2d = nl_first_2d; + } else { + ggml_tensor * nl_2d = ggml_reshape_2d(ctx0, nl, projection_dim, 1); + ggml_tensor * rest_template = ggml_new_tensor_2d( + ctx0, GGML_TYPE_F32, projection_dim, K - 1); + ggml_tensor * nl_rest = ggml_repeat(ctx0, nl_2d, rest_template); + nl_row_2d = ggml_concat(ctx0, nl_first_2d, nl_rest, 1); // (projection_dim, K) + } + nl_row_2d = ggml_cont(ctx0, nl_row_2d); + return ggml_reshape_2d(ctx0, nl_row_2d, n_mmproj_embd, 1); +} + +// Append a single newline row at the end of the tile output. +ggml_tensor * clip_graph_granite4_vision::append_rowwise_newlines(ggml_context * ctx0, ggml_tensor * tile_output) { + // For the single-tile case, append one newline row at the end. + // For the multi-tile rowwise case, this will be called per-tile + // (though currently only the single-tile path uses it). + ggml_tensor * nl_row = build_newline_row(ctx0); + return ggml_concat(ctx0, tile_output, nl_row, 1); +} + ggml_cgraph * clip_graph_granite4_vision::build() { GGML_ASSERT(model.patch_embeddings_0 != nullptr); GGML_ASSERT(model.position_embeddings != nullptr); @@ -322,172 +360,14 @@ ggml_cgraph * clip_graph_granite4_vision::build() { } mmproj = (k == 0) ? s : ggml_concat(ctx0, mmproj, s, 0); } - ggml_set_name(mmproj, "g4v_mmproj_out"); - ggml_build_forward_expand(gf, mmproj); - - return gf; -} - -// --------------------------------------------------------------------------- -// Assembler — pack-and-unpad + scaled image_newline injection -// --------------------------------------------------------------------------- -// -// Granite Vision 4.1 follows the llava-next layout: the per-tile encoder -// outputs are reshaped so the assembled super-image is read in spatial -// scan order with a learned newline embedding appended to every super-row. -// Single-tile inputs (overview only) get a single trailing newline row. -// -// All knowledge of image_newline and base_stream_scale lives here. The -// graph itself is a single ggml_get_rows over a virtual buffer formed by -// concatenating per-tile encoder outputs with the K-tiled, base-scaled -// newline row. The gather index is built host-side. - -size_t granite4_vision_n_assembled_output_tokens( - const clip_ctx * ctx, - const clip_image_f32_batch * batch) { - GGML_ASSERT(batch != nullptr && !batch->entries.empty()); - const int n_per_tile = clip_n_output_tokens( - const_cast(ctx), batch->entries[0].get()); - if (batch->entries.size() == 1) { - // overview tile + 1 newline row - return (size_t)n_per_tile + 1; - } - const int per_tile_side = (int) std::lround(std::sqrt((double)n_per_tile)); - GGML_ASSERT(per_tile_side * per_tile_side == n_per_tile); - // overview + (per_tile_side*grid_y) super-rows, each (per_tile_side*grid_x + 1) wide - const size_t super_h = (size_t)per_tile_side * (size_t)batch->grid_y; - const size_t super_w_plus_nl = (size_t)per_tile_side * (size_t)batch->grid_x + 1; - return (size_t)n_per_tile + super_h * super_w_plus_nl; -} - -// Build the host-side gather index. Each entry indexes into a virtual buffer -// of length (n_tiles*n_per_tile + 1) whose last row is the K-tiled newline. -static std::vector g4v_build_gather_idx( - int n_tiles, int n_per_tile, int grid_x, int grid_y) { - const int32_t nl_idx = n_tiles * n_per_tile; - - std::vector idx; - idx.reserve((size_t)n_per_tile + 1); - // overview comes first - for (int i = 0; i < n_per_tile; ++i) { - idx.push_back(i); - } - - if (n_tiles == 1) { - idx.push_back(nl_idx); - return idx; - } - - GGML_ASSERT(grid_x > 0 && grid_y > 0); - GGML_ASSERT(n_tiles == grid_x * grid_y + 1); - const int per_tile_side = (int) std::lround(std::sqrt((double)n_per_tile)); - GGML_ASSERT(per_tile_side * per_tile_side == n_per_tile); - - const size_t total = (size_t)n_per_tile + - (size_t)per_tile_side * (size_t)grid_y * - ((size_t)per_tile_side * (size_t)grid_x + 1); - idx.reserve(total); - for (int gy = 0; gy < grid_y; ++gy) { - for (int ty = 0; ty < per_tile_side; ++ty) { - for (int gx = 0; gx < grid_x; ++gx) { - const int tile_idx = 1 + gy * grid_x + gx; // +1: skip overview - const int row_off = tile_idx * n_per_tile + ty * per_tile_side; - for (int tx = 0; tx < per_tile_side; ++tx) { - idx.push_back(row_off + tx); - } - } - idx.push_back(nl_idx); - } - } - GGML_ASSERT(idx.size() == total); - return idx; -} - -clip_assembler_granite4_vision::clip_assembler_granite4_vision( - const clip_ctx * ctx, - const float * per_tile_embd, - int n_tiles, int grid_x, int grid_y) - : model(*clip_get_model(ctx)), - per_tile_embd(per_tile_embd), - n_tiles(n_tiles), - grid_x(grid_x), - grid_y(grid_y) { - GGML_ASSERT(per_tile_embd != nullptr && n_tiles > 0); - - n_mmproj_embd = clip_n_mmproj_embd(ctx); - - // Recover n_per_tile from the model's fixed tile size. All G4V tiles - // are produced at hparams.image_size × hparams.image_size, so a dummy - // tile image at that size yields the same n_per_tile as any real tile. - clip_image_f32 tile_img; - tile_img.nx = model.hparams.image_size; - tile_img.ny = model.hparams.image_size; - n_per_tile = clip_n_output_tokens(const_cast(ctx), &tile_img); - - gather_idx = g4v_build_gather_idx(n_tiles, n_per_tile, grid_x, grid_y); -} - -ggml_tensor * clip_assembler_granite4_vision::build( - ggml_context * ctx0, ggml_cgraph * gf) { - const int K = (int) model.qf_proj_blocks.size(); - GGML_ASSERT(K > 0); - GGML_ASSERT(n_mmproj_embd % K == 0); - const int projection_dim = n_mmproj_embd / K; - GGML_ASSERT(model.image_newline != nullptr); - GGML_ASSERT(ggml_nelements(model.image_newline) == projection_dim); - - // 1) Inputs. - ggml_tensor * per_tile_in = ggml_new_tensor_2d( - ctx0, GGML_TYPE_F32, n_mmproj_embd, n_tiles * n_per_tile); - ggml_set_name(per_tile_in, "g4v_assembler_per_tile_in"); - ggml_set_input(per_tile_in); - - ggml_tensor * idx_in = ggml_new_tensor_1d( - ctx0, GGML_TYPE_I32, (int64_t) gather_idx.size()); - ggml_set_name(idx_in, "g4v_assembler_gather_idx"); - ggml_set_input(idx_in); - - // 2) Build K-tiled, base-scaled newline row. - // newline_row[k*projection_dim + d] = nl[d] * (k == 0 ? base : 1.0) - // i.e. with ne[0]=projection_dim, ne[1]=K stacking, then reshape to - // a 1-row vector of length K*projection_dim. - ggml_tensor * nl = model.image_newline; // (projection_dim,) - ggml_tensor * nl_first = ggml_scale(ctx0, nl, model.hparams.base_stream_scale); - ggml_tensor * nl_first_2d = ggml_reshape_2d(ctx0, nl_first, projection_dim, 1); - ggml_tensor * nl_row_2d; - if (K == 1) { - nl_row_2d = nl_first_2d; + // --- Stage 1d: Append newline tokens if append_token is set --- + if (append_token_type == clip_image_f32::CLIP_APPEND_TOKEN_NEWLINE_ROWWISE) { + mmproj = append_rowwise_newlines(ctx0, mmproj); + ggml_set_name(mmproj, "g4v_mmproj_out_nl"); } else { - ggml_tensor * nl_2d = ggml_reshape_2d(ctx0, nl, projection_dim, 1); - ggml_tensor * rest_template = ggml_new_tensor_2d( - ctx0, GGML_TYPE_F32, projection_dim, K - 1); - ggml_tensor * nl_rest = ggml_repeat(ctx0, nl_2d, rest_template); - nl_row_2d = ggml_concat(ctx0, nl_first_2d, nl_rest, 1); // (projection_dim, K) + ggml_set_name(mmproj, "g4v_mmproj_out"); } - nl_row_2d = ggml_cont(ctx0, nl_row_2d); - ggml_tensor * nl_row = ggml_reshape_2d(ctx0, nl_row_2d, n_mmproj_embd, 1); - ggml_set_name(nl_row, "g4v_assembler_nl_row"); - - // 3) Concat per-tile outputs with the newline row to form the virtual - // source. The newline becomes the row at index n_tiles*n_per_tile. - ggml_tensor * src = ggml_concat(ctx0, per_tile_in, nl_row, 1); - ggml_set_name(src, "g4v_assembler_src"); - - // 4) Gather. ggml_get_rows selects along ne[1], producing a tensor of - // shape (n_mmproj_embd, gather_len) in spatial scan order. - ggml_tensor * out = ggml_get_rows(ctx0, src, idx_in); - ggml_set_name(out, "g4v_assembler_out"); - - ggml_build_forward_expand(gf, out); - return out; -} + ggml_build_forward_expand(gf, mmproj); -void clip_assembler_granite4_vision::set_inputs(ggml_cgraph * gf) { - ggml_tensor * in_per_tile = ggml_graph_get_tensor(gf, "g4v_assembler_per_tile_in"); - ggml_tensor * in_gather = ggml_graph_get_tensor(gf, "g4v_assembler_gather_idx"); - GGML_ASSERT(in_per_tile != nullptr && in_gather != nullptr); - ggml_backend_tensor_set(in_per_tile, per_tile_embd, 0, - (size_t) n_tiles * n_per_tile * n_mmproj_embd * sizeof(float)); - ggml_backend_tensor_set(in_gather, gather_idx.data(), 0, - gather_idx.size() * sizeof(int32_t)); + return gf; } diff --git a/tools/mtmd/models/models.h b/tools/mtmd/models/models.h index c52c960f4c54..432017a32978 100644 --- a/tools/mtmd/models/models.h +++ b/tools/mtmd/models/models.h @@ -193,40 +193,18 @@ struct clip_graph_kimik25 : clip_graph { }; struct clip_graph_granite4_vision : clip_graph { - clip_graph_granite4_vision(clip_ctx * ctx, const clip_image_f32 & img) : clip_graph(ctx, img) {} - ggml_cgraph * build() override; -}; - -// --- Granite Vision 4 assembler --- -// -// Per-tile encoder outputs are reshaped (pack-and-unpad) and interleaved with -// a learned, base-scaled newline embedding to produce the final image-token -// sequence. The cgraph is a single ggml_get_rows over a virtual buffer formed -// by concatenating the per-tile encoder outputs with the K-tiled, base-scaled -// newline row. All model-specific knowledge (n_per_tile inference, gather -// index construction, input tensor names) lives on the class below. - -struct clip_assembler_granite4_vision : clip_assembler { - const clip_model & model; - const float * per_tile_embd; // borrowed, caller-owned lifetime - int n_tiles; - int grid_x; - int grid_y; - int n_per_tile; - int n_mmproj_embd; - std::vector gather_idx; // computed in constructor - - clip_assembler_granite4_vision(const clip_ctx * ctx, - const float * per_tile_embd, - int n_tiles, int grid_x, int grid_y); - - ggml_tensor * build(ggml_context * ctx0, ggml_cgraph * gf) override; - void set_inputs(ggml_cgraph * gf) override; -}; - -// Final assembled token count for a preprocessed tile batch. Called before -// per-tile encoding (from add_media), so kept as a free function rather than -// a method on the assembler class. -size_t granite4_vision_n_assembled_output_tokens( - const clip_ctx * ctx, - const clip_image_f32_batch * batch); + clip_graph_granite4_vision(clip_ctx * ctx, const clip_image_f32 & img) + : clip_graph(ctx, img), + append_token_type((clip_image_f32::clip_append_token_type)img.append_token) {} + + ggml_cgraph * build() override; + +private: + // The graph is per-tile since only batch-size 1 is supported in clip. As + // such, this value is set at construct time based on the tile that will be + // encoded, then used during build to determine how to handle newlines. + clip_image_f32::clip_append_token_type append_token_type; + + ggml_tensor * build_newline_row(ggml_context * ctx0); + ggml_tensor * append_rowwise_newlines(ggml_context * ctx0, ggml_tensor * tile_output); +}; diff --git a/tools/mtmd/mtmd.cpp b/tools/mtmd/mtmd.cpp index c81e07d61420..09f3483c514e 100644 --- a/tools/mtmd/mtmd.cpp +++ b/tools/mtmd/mtmd.cpp @@ -802,6 +802,24 @@ struct mtmd_tokenizer { return 2; } + // Annotate llava-next style tiles so clip_n_output_tokens accounts + // for per-tile newline injection. + if (ctx->proj_type_v() == PROJECTOR_TYPE_GRANITE4_VISION) { + if (batch_f32.entries.size() == 1) { + // Single-tile (overview only): append one newline row. + clip_image_set_append_token(batch_f32.entries[0].get(), + clip_image_f32::CLIP_APPEND_TOKEN_NEWLINE_ROWWISE); + } else { + // Multi-tile: overview gets no newline, grid tiles get one. + clip_image_set_append_token(batch_f32.entries[0].get(), + clip_image_f32::CLIP_APPEND_TOKEN_NONE); + for (size_t i = 1; i < batch_f32.entries.size(); ++i) { + clip_image_set_append_token(batch_f32.entries[i].get(), + clip_image_f32::CLIP_APPEND_TOKEN_NEWLINE_ROWWISE); + } + } + } + // handle llava-uhd style preprocessing const bool has_tiling_grid = batch_f32.grid_x > 0 && batch_f32.grid_y > 0; if ( @@ -866,9 +884,11 @@ struct mtmd_tokenizer { } } else { - // Final token count accounts for any model-specific assembler - // (e.g. Granite Vision 4 pack-and-unpad with newline injection). - const size_t n_tokens = clip_n_assembled_output_tokens(ctx->ctx_v, &batch_f32); + + size_t n_tokens = 0; + for (const auto & e : batch_f32.entries) { + n_tokens += clip_n_output_tokens(ctx->ctx_v, e.get()); + } mtmd_image_tokens_ptr image_tokens(new mtmd_image_tokens); if (mtmd_decode_use_mrope(ctx)) { @@ -1100,40 +1120,22 @@ int32_t mtmd_encode(mtmd_context * ctx, const mtmd_image_tokens * image_tokens) ctx->image_embd_v.resize(image_tokens->n_tokens() * n_mmproj_embd); bool ok = false; - if (proj_type == PROJECTOR_TYPE_GRANITE4_VISION) { - // Per-tile encode, then hand off to the model-owned assembler graph - // (pack-and-unpad + scaled image_newline injection). - const auto & entries = image_tokens->batch_f32.entries; - const int n_per_tile = clip_n_output_tokens(ctx_clip, entries[0].get()); - std::vector per_tile(entries.size() * n_per_tile * n_mmproj_embd); - for (size_t i = 0; i < entries.size(); i++) { - if (!clip_image_encode(ctx_clip, ctx->n_threads, entries[i].get(), - per_tile.data() + i * n_mmproj_embd * n_per_tile)) { - return 1; - } - } - ok = clip_image_assemble(ctx_clip, ctx->n_threads, - per_tile.data(), - (int) entries.size(), - (int) image_tokens->batch_f32.grid_x, - (int) image_tokens->batch_f32.grid_y, - ctx->image_embd_v.data()); - return ok ? 0 : 1; - } - if (clip_is_llava(ctx_clip) || proj_type == PROJECTOR_TYPE_MINICPMV || proj_type == PROJECTOR_TYPE_GLM_EDGE - || proj_type == PROJECTOR_TYPE_INTERNVL) { + || proj_type == PROJECTOR_TYPE_INTERNVL + || proj_type == PROJECTOR_TYPE_GRANITE4_VISION) { // TODO @ngxson : llava does not support batched encoding ; this should be fixed inside clip_image_batch_encode() const auto & entries = image_tokens->batch_f32.entries; + size_t offset = 0; for (size_t i = 0; i < entries.size(); i++) { int n_tokens_per_image = clip_n_output_tokens(ctx_clip, entries[i].get()); ok = clip_image_encode( ctx_clip, ctx->n_threads, entries[i].get(), - ctx->image_embd_v.data() + i*n_mmproj_embd*n_tokens_per_image); + ctx->image_embd_v.data() + offset*n_mmproj_embd); + offset += n_tokens_per_image; } } else { ok = clip_image_batch_encode( From 23990da363e548f8b6d76954e084a745a6d5bbc8 Mon Sep 17 00:00:00 2001 From: Gabe Goodhart Date: Tue, 26 May 2026 20:07:14 -0600 Subject: [PATCH 67/94] refactor(convert): Split n_deepstack_layers and deepstack_layers (array) Branch: Granite4Vision AI-usage: full (Bob, OpenCode + Qwen3.6-35b) Signed-off-by: Gabe Goodhart --- conversion/granite.py | 2 +- gguf-py/gguf/constants.py | 1 + gguf-py/gguf/gguf_writer.py | 17 ++++++----------- 3 files changed, 8 insertions(+), 12 deletions(-) diff --git a/conversion/granite.py b/conversion/granite.py index d806850a300d..9075b748c963 100644 --- a/conversion/granite.py +++ b/conversion/granite.py @@ -57,7 +57,7 @@ def set_gguf_parameters(self): if proj_idx == 0: continue deepstack_layer_arr[llm_layer] = proj_idx - self.gguf_writer.add_num_deepstack_layers(deepstack_layer_arr) + self.gguf_writer.add_deepstack_layers(deepstack_layer_arr) @classmethod def filter_tensors(cls, item: tuple[str, Callable[[], Tensor]]) -> tuple[str, Callable[[], Tensor]] | None: diff --git a/gguf-py/gguf/constants.py b/gguf-py/gguf/constants.py index 92e48019c325..8c2484e0844a 100644 --- a/gguf-py/gguf/constants.py +++ b/gguf-py/gguf/constants.py @@ -128,6 +128,7 @@ class LLM: MOE_LATENT_SIZE = "{arch}.moe_latent_size" NEXTN_PREDICT_LAYERS = "{arch}.nextn_predict_layers" NUM_DEEPSTACK_LAYERS = "{arch}.n_deepstack_layers" + DEEPSTACK_LAYERS = "{arch}.deepstack_layers" POOLING_TYPE = "{arch}.pooling_type" LOGIT_SCALE = "{arch}.logit_scale" DECODER_START_TOKEN_ID = "{arch}.decoder_start_token_id" diff --git a/gguf-py/gguf/gguf_writer.py b/gguf-py/gguf/gguf_writer.py index eebc494f23b2..50bae8ec425e 100644 --- a/gguf-py/gguf/gguf_writer.py +++ b/gguf-py/gguf/gguf_writer.py @@ -955,18 +955,13 @@ def add_attn_temperature_scale(self, value: float) -> None: def add_pooling_type(self, value: PoolingType) -> None: self.add_uint32(Keys.LLM.POOLING_TYPE.format(arch=self.arch), value.value) - def add_num_deepstack_layers(self, count: int | Sequence[int]) -> None: - """ - Add deepstack layer configuration. + def add_num_deepstack_layers(self, count: int) -> None: + """Add scalar deepstack layer count (qwen3vl format)""" + self.add_uint32(Keys.LLM.NUM_DEEPSTACK_LAYERS.format(arch=self.arch), count) - For backward compatibility, the GGUF key name remains 'n_deepstack_layers' - even though it now supports arrays. Existing models using scalar values - will continue to work. - """ - if isinstance(count, int): - self.add_uint32(Keys.LLM.NUM_DEEPSTACK_LAYERS.format(arch=self.arch), count) - else: - self.add_array(Keys.LLM.NUM_DEEPSTACK_LAYERS.format(arch=self.arch), list(count)) + def add_deepstack_layers(self, layers: Sequence[int]) -> None: + """Add per-layer deepstack projector indices (Granite4 Vision format)""" + self.add_array(Keys.LLM.DEEPSTACK_LAYERS.format(arch=self.arch), list(layers)) def add_rope_dimension_count(self, count: int) -> None: self.add_uint32(Keys.Rope.DIMENSION_COUNT.format(arch=self.arch), count) From f28a91aee97b6dcd72777dcd72e68631d3ab4af2 Mon Sep 17 00:00:00 2001 From: Gabe Goodhart Date: Tue, 26 May 2026 20:10:04 -0600 Subject: [PATCH 68/94] refactor(src): Handle n_deepstack_layers and deepstack_layers GGUF keys Branch: Granite4Vision AI-usage: draft (Bob, OpenCode + Qwen3.6-35b) Signed-off-by: Gabe Goodhart --- src/llama-arch.cpp | 1 + src/llama-arch.h | 1 + src/llama-model.cpp | 21 +++------------------ src/models/granite.cpp | 3 +++ src/models/qwen3vl.cpp | 9 +++++++++ src/models/qwen3vlmoe.cpp | 9 +++++++++ 6 files changed, 26 insertions(+), 18 deletions(-) diff --git a/src/llama-arch.cpp b/src/llama-arch.cpp index e95ba6daac1f..7d9b0285fbfb 100644 --- a/src/llama-arch.cpp +++ b/src/llama-arch.cpp @@ -194,6 +194,7 @@ static const std::map LLM_KV_NAMES = { { LLM_KV_MOE_LATENT_SIZE, "%s.moe_latent_size" }, { LLM_KV_NEXTN_PREDICT_LAYERS, "%s.nextn_predict_layers" }, { LLM_KV_NUM_DEEPSTACK_LAYERS, "%s.n_deepstack_layers" }, + { LLM_KV_DEEPSTACK_LAYERS, "%s.deepstack_layers" }, { LLM_KV_POOLING_TYPE, "%s.pooling_type" }, { LLM_KV_LOGIT_SCALE, "%s.logit_scale" }, { LLM_KV_DECODER_START_TOKEN_ID, "%s.decoder_start_token_id" }, diff --git a/src/llama-arch.h b/src/llama-arch.h index 7c1dcc4d6c2b..9e964734aedf 100644 --- a/src/llama-arch.h +++ b/src/llama-arch.h @@ -198,6 +198,7 @@ enum llm_kv { LLM_KV_MOE_LATENT_SIZE, LLM_KV_NEXTN_PREDICT_LAYERS, LLM_KV_NUM_DEEPSTACK_LAYERS, + LLM_KV_DEEPSTACK_LAYERS, LLM_KV_POOLING_TYPE, LLM_KV_LOGIT_SCALE, LLM_KV_DECODER_START_TOKEN_ID, diff --git a/src/llama-model.cpp b/src/llama-model.cpp index 6520ab37d4ad..6747a24ec132 100644 --- a/src/llama-model.cpp +++ b/src/llama-model.cpp @@ -1062,24 +1062,9 @@ void llama_model_base::load_hparams(llama_model_loader & ml) { ml.get_key_or_arr(LLM_KV_FEED_FORWARD_LENGTH, hparams.n_ff_arr, hparams.n_layer, false); ml.get_key_or_arr(LLM_KV_ATTENTION_HEAD_COUNT, hparams.n_head_arr, hparams.n_layer, false); - // populate deepstack_layers_arr - // - // If single-valued, it's the equivalent of n_deepstack_layers (qwen3vl). - // This implies a 1:1 correspondence between entries in inp_embed and the - // layers < n (offset by 1). - // - // If multi-valued (Granite4 Vision), the elements are themselves index - // values into the entries in inp_embed - std::fill(hparams.deepstack_layers_arr.begin(), hparams.deepstack_layers_arr.end(), -1.0f); - try { - uint32_t n_deepstack_layers = 0; - ml.get_key(LLM_KV_NUM_DEEPSTACK_LAYERS, n_deepstack_layers, false); - for (int32_t i = 0; i < (int32_t)n_deepstack_layers; ++i) { - hparams.deepstack_layers_arr[i] = i + 1; - } - } catch (const std::runtime_error &) { - ml.get_arr(LLM_KV_NUM_DEEPSTACK_LAYERS, hparams.deepstack_layers_arr, false); - } + // Populate deepstack_layers_arr - initialized to -1 (no deepstack) + // Model-specific code will populate if needed + std::fill(hparams.deepstack_layers_arr.begin(), hparams.deepstack_layers_arr.end(), -1); // n_head_kv is optional, default to n_head hparams.n_head_kv_arr = hparams.n_head_arr; diff --git a/src/models/granite.cpp b/src/models/granite.cpp index 94749b6c4d77..d71f23a05f6b 100644 --- a/src/models/granite.cpp +++ b/src/models/granite.cpp @@ -7,6 +7,9 @@ void llama_model_granite::load_arch_hparams(llama_model_loader & ml) { ml.get_key(LLM_KV_EMBEDDING_SCALE, hparams.f_embedding_scale, false); ml.get_key(LLM_KV_ATTENTION_SCALE, hparams.f_attention_scale, false); + // Granite4 Vision uses array deepstack_layers + ml.get_arr(LLM_KV_DEEPSTACK_LAYERS, hparams.deepstack_layers_arr, false); + // Granite uses rope_finetuned as a switch for rope, so default to true bool rope_finetuned = true; ml.get_key(LLM_KV_ROPE_SCALING_FINETUNED, rope_finetuned, false); diff --git a/src/models/qwen3vl.cpp b/src/models/qwen3vl.cpp index f73a6435fb07..6e3efc4bcf64 100644 --- a/src/models/qwen3vl.cpp +++ b/src/models/qwen3vl.cpp @@ -3,6 +3,15 @@ void llama_model_qwen3vl::load_arch_hparams(llama_model_loader & ml) { ml.get_key_or_arr(LLM_KV_ROPE_DIMENSION_SECTIONS, hparams.rope_sections, 4, true); ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, hparams.f_norm_rms_eps); + + // qwen3vl uses scalar n_deepstack_layers + uint32_t n_deepstack_layers = 0; + if (ml.get_key(LLM_KV_NUM_DEEPSTACK_LAYERS, n_deepstack_layers, false)) { + // Create 1:1 mapping: layer i gets projector i+1 + for (int32_t i = 0; i < (int32_t)n_deepstack_layers; ++i) { + hparams.deepstack_layers_arr[i] = i + 1; + } + } switch (hparams.n_layer) { case 28: type = LLM_TYPE_1_7B; break; case 36: type = hparams.n_embd == 2560 ? LLM_TYPE_4B : LLM_TYPE_8B; break; diff --git a/src/models/qwen3vlmoe.cpp b/src/models/qwen3vlmoe.cpp index c37c0f2810cd..ceb4c2fde7c8 100644 --- a/src/models/qwen3vlmoe.cpp +++ b/src/models/qwen3vlmoe.cpp @@ -4,6 +4,15 @@ void llama_model_qwen3vlmoe::load_arch_hparams(llama_model_loader & ml) { ml.get_key_or_arr(LLM_KV_ROPE_DIMENSION_SECTIONS, hparams.rope_sections, 4, true); ml.get_key(LLM_KV_EXPERT_FEED_FORWARD_LENGTH, hparams.n_ff_exp, false); ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, hparams.f_norm_rms_eps); + + // qwen3vlmoe uses scalar n_deepstack_layers + uint32_t n_deepstack_layers = 0; + if (ml.get_key(LLM_KV_NUM_DEEPSTACK_LAYERS, n_deepstack_layers, false)) { + // Create 1:1 mapping: layer i gets projector i+1 + for (int32_t i = 0; i < (int32_t)n_deepstack_layers; ++i) { + hparams.deepstack_layers_arr[i] = i + 1; + } + } switch (hparams.n_layer) { case 48: type = LLM_TYPE_30B_A3B; break; case 94: type = LLM_TYPE_235B_A22B; break; From 6b42c74766303f3e7d2b15a3a2292f66f12997b0 Mon Sep 17 00:00:00 2001 From: Gabe Goodhart Date: Wed, 27 May 2026 10:02:06 -0600 Subject: [PATCH 69/94] fix: Fix GGUF key for deepstack_layers_arr Branch: Granite4Vision AI-usage: none Signed-off-by: Gabe Goodhart --- src/llama-model-saver.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/llama-model-saver.cpp b/src/llama-model-saver.cpp index d570ae0f7af7..9171145a8121 100644 --- a/src/llama-model-saver.cpp +++ b/src/llama-model-saver.cpp @@ -228,7 +228,7 @@ void llama_model_saver::add_kv_from_model() { add_kv(LLM_KV_EXPERTS_PER_GROUP, hparams.n_group_experts); add_kv(LLM_KV_MOE_EVERY_N_LAYERS, hparams.moe_every_n_layers); add_kv(LLM_KV_NEXTN_PREDICT_LAYERS, hparams.nextn_predict_layers); - add_kv(LLM_KV_NUM_DEEPSTACK_LAYERS, hparams.deepstack_layers_arr); + add_kv(LLM_KV_DEEPSTACK_LAYERS, hparams.deepstack_layers_arr); add_kv(LLM_KV_POOLING_TYPE, uint32_t(hparams.pooling_type)); add_kv(LLM_KV_LOGIT_SCALE, hparams.f_logit_scale); add_kv(LLM_KV_DECODER_START_TOKEN_ID, hparams.dec_start_token_id); From dd50fb452760a9c08bfe49a94f7a458022bd0a8a Mon Sep 17 00:00:00 2001 From: Gabe Goodhart Date: Wed, 27 May 2026 10:40:20 -0600 Subject: [PATCH 70/94] refactor: Remove pre-scaling embeddings and skip scaling for raw embd inputs This follows how gemma3 and gemma4 handle embedding scaling by skipping the multiplier for raw input embeddings. Branch: Granite4Vision AI-usage: none Signed-off-by: Gabe Goodhart --- conversion/granite.py | 6 ------ src/llama-graph.cpp | 4 +++- tools/mtmd/clip-impl.h | 2 -- tools/mtmd/clip-model.h | 9 --------- tools/mtmd/clip.cpp | 5 ----- tools/mtmd/models/granite4-vision.cpp | 10 ++-------- 6 files changed, 5 insertions(+), 31 deletions(-) diff --git a/conversion/granite.py b/conversion/granite.py index 9075b748c963..0856070926a3 100644 --- a/conversion/granite.py +++ b/conversion/granite.py @@ -433,12 +433,6 @@ def set_gguf_parameters(self): # Set the spatial offests per projector self.gguf_writer.add_vision_spatial_offsets(self._spatial_offsets) - # The granite LLM will scale the first embedding input by - # embedding_multiplier, so the mmproj needs to invert that scale for its - # output so that the resulting injection works as expected. - if (embed_mult := self.global_config.get("text_config", {}).get("embedding_multiplier")) is not None: - self.gguf_writer.add_embedding_scale(embed_mult) - # Add flattened image grind pinpoints (resolution candidates internally) if pinpoints := self.global_config.get("image_grid_pinpoints"): # Flatten with h, w -> w, h inversion diff --git a/src/llama-graph.cpp b/src/llama-graph.cpp index 1be11c7e4a95..ae3a80a2703d 100644 --- a/src/llama-graph.cpp +++ b/src/llama-graph.cpp @@ -1791,7 +1791,9 @@ ggml_tensor * llm_graph_context::build_inp_embd(ggml_tensor * tok_embd) const { res->t_inp_embd = cur; // For Granite architecture - if (hparams.f_embedding_scale != 0.0f) { + // NOTE: Only apply scale to token inputs. Raw embeddings are assumed to be + // multimodal inputs that should not be scaled. + if (ubatch.token && hparams.f_embedding_scale != 0.0f) { if (!ggml_is_contiguous(cur)) { cur = ggml_cont(ctx0, cur); } diff --git a/tools/mtmd/clip-impl.h b/tools/mtmd/clip-impl.h index 41f678beabf7..2a932b14e43b 100644 --- a/tools/mtmd/clip-impl.h +++ b/tools/mtmd/clip-impl.h @@ -73,8 +73,6 @@ #define KEY_A_PROJ_WINDOW_SIZE "clip.audio.projector.window_size" #define KEY_A_PROJ_DOWNSAMPLE_RATE "clip.audio.projector.downsample_rate" #define KEY_A_PROJ_HEAD_COUNT "clip.audio.projector.head_count" -// shared with LLM (align to llama-arch.* w/ arch=clip) -#define KEY_LLM_KV_EMBEDDING_SCALE "clip.embedding_scale" // // tensor name constants diff --git a/tools/mtmd/clip-model.h b/tools/mtmd/clip-model.h index 7e618c9e367f..6300ff5df08b 100644 --- a/tools/mtmd/clip-model.h +++ b/tools/mtmd/clip-model.h @@ -105,15 +105,6 @@ struct clip_hparams { std::vector proj_spatial_offsets; // Ordered list of per-projector spatial offsets int32_t downsample_query_side; int32_t downsample_window_side; - // Multiplier applied to the base stream (stream targeting the lowest - // llm_layer) before concatenation into the mmproj output. Needed - // because the shared Granite LLM graph scales t_inp_embd by - // f_embedding_scale (12.0) for ALL positions, while Granite Vision - // 4.1's HF forward zeroes inputs_embeds at image positions BEFORE - // the multiplier -- so the base stream has to enter the LLM graph - // pre-divided. Defaults to 1.0 if the mmproj was produced before - // this KV was added. - float base_stream_scale = 1.0f; // audio int32_t n_mel_bins = 0; // whisper preprocessor diff --git a/tools/mtmd/clip.cpp b/tools/mtmd/clip.cpp index 042319f1ce35..d58165228157 100644 --- a/tools/mtmd/clip.cpp +++ b/tools/mtmd/clip.cpp @@ -1584,11 +1584,6 @@ struct clip_model_loader { get_u32(KEY_PROJ_SAMPLE_QUERY_SIDE, hparams.downsample_query_side); get_u32(KEY_PROJ_SAMPLE_WINDOW_SIDE, hparams.downsample_window_side); - float embedding_scale = 0.; - get_f32(KEY_LLM_KV_EMBEDDING_SCALE, embedding_scale, false); - if (embedding_scale != 0) { - hparams.base_stream_scale = 1.0 / embedding_scale; - } hparams.warmup_image_size = hparams.image_size; } break; default: diff --git a/tools/mtmd/models/granite4-vision.cpp b/tools/mtmd/models/granite4-vision.cpp index a3410fa2e3e4..fc433e5281cf 100644 --- a/tools/mtmd/models/granite4-vision.cpp +++ b/tools/mtmd/models/granite4-vision.cpp @@ -251,8 +251,7 @@ ggml_tensor * clip_graph_granite4_vision::build_newline_row(ggml_context * ctx0) // Build newline_row[k*projection_dim + d] = nl[d] * (k == 0 ? base : 1.0) ggml_tensor * nl = model.image_newline; // (projection_dim,) - ggml_tensor * nl_first = ggml_scale(ctx0, nl, model.hparams.base_stream_scale); - ggml_tensor * nl_first_2d = ggml_reshape_2d(ctx0, nl_first, projection_dim, 1); + ggml_tensor * nl_first_2d = ggml_reshape_2d(ctx0, nl, projection_dim, 1); ggml_tensor * nl_row_2d; if (K == 1) { nl_row_2d = nl_first_2d; @@ -349,15 +348,10 @@ ggml_cgraph * clip_graph_granite4_vision::build() { qformer_eps); } - // Concatenate the projector activations in the correct order, and apply the - // base stream multiplier to invert the embedding multipler that gets - // applied on the LLM side. + // Concatenate the projector activations in the correct order ggml_tensor * mmproj = nullptr; for (int k = 0; k < projector_count; ++k) { ggml_tensor * s = ggml_cont(ctx0, streams[k]); - if (k == 0 && hparams.base_stream_scale != 1.0f) { - s = ggml_scale_inplace(ctx0, s, hparams.base_stream_scale); - } mmproj = (k == 0) ? s : ggml_concat(ctx0, mmproj, s, 0); } // --- Stage 1d: Append newline tokens if append_token is set --- From e73aa8035dbe74091d21b034f87348ed2307ce84 Mon Sep 17 00:00:00 2001 From: Gabe Goodhart Date: Wed, 27 May 2026 11:56:38 -0600 Subject: [PATCH 71/94] refactor: deepstack_layers(_arr) -> deepstack_mapping(_arr) Branch: Granite4Vision AI-usage: none Signed-off-by: Gabe Goodhart --- conversion/granite.py | 6 +++--- gguf-py/gguf/constants.py | 2 +- gguf-py/gguf/gguf_writer.py | 4 ++-- src/llama-arch.cpp | 2 +- src/llama-arch.h | 2 +- src/llama-hparams.cpp | 2 +- src/llama-hparams.h | 2 +- src/llama-model-saver.cpp | 2 +- src/llama-model.cpp | 12 ++++++------ src/models/granite.cpp | 6 +++--- src/models/qwen3vl.cpp | 4 ++-- src/models/qwen3vlmoe.cpp | 4 ++-- 12 files changed, 24 insertions(+), 24 deletions(-) diff --git a/conversion/granite.py b/conversion/granite.py index 0856070926a3..6a5677c3e375 100644 --- a/conversion/granite.py +++ b/conversion/granite.py @@ -50,14 +50,14 @@ def set_gguf_parameters(self): # If being used as the base for Granite4 Vision, add deepstack_layer_arr if self.hparams.get("spatial_target_layers") or self.hparams.get("deepstack_layer_map"): normalized_projector_map = Granite4VisionMmprojModel.get_normalized_projector_map(self.hparams) - deepstack_layer_arr = [-1 for _ in range(self.block_count)] # Populate with -1 sentinels + deepstack_mapping_arr = [-1 for _ in range(self.block_count)] # Populate with -1 sentinels for proj_idx, (_, llm_layer, _, _) in enumerate(normalized_projector_map): # Skip the first projector which is handled as the base embedding # stream like normal if proj_idx == 0: continue - deepstack_layer_arr[llm_layer] = proj_idx - self.gguf_writer.add_deepstack_layers(deepstack_layer_arr) + deepstack_mapping_arr[llm_layer] = proj_idx + self.gguf_writer.add_deepstack_mapping(deepstack_mapping_arr) @classmethod def filter_tensors(cls, item: tuple[str, Callable[[], Tensor]]) -> tuple[str, Callable[[], Tensor]] | None: diff --git a/gguf-py/gguf/constants.py b/gguf-py/gguf/constants.py index 8c2484e0844a..9539c9c1e8af 100644 --- a/gguf-py/gguf/constants.py +++ b/gguf-py/gguf/constants.py @@ -128,7 +128,7 @@ class LLM: MOE_LATENT_SIZE = "{arch}.moe_latent_size" NEXTN_PREDICT_LAYERS = "{arch}.nextn_predict_layers" NUM_DEEPSTACK_LAYERS = "{arch}.n_deepstack_layers" - DEEPSTACK_LAYERS = "{arch}.deepstack_layers" + DEEPSTACK_MAPPING = "{arch}.deepstack_mapping" POOLING_TYPE = "{arch}.pooling_type" LOGIT_SCALE = "{arch}.logit_scale" DECODER_START_TOKEN_ID = "{arch}.decoder_start_token_id" diff --git a/gguf-py/gguf/gguf_writer.py b/gguf-py/gguf/gguf_writer.py index 50bae8ec425e..ebab0c5f0553 100644 --- a/gguf-py/gguf/gguf_writer.py +++ b/gguf-py/gguf/gguf_writer.py @@ -959,9 +959,9 @@ def add_num_deepstack_layers(self, count: int) -> None: """Add scalar deepstack layer count (qwen3vl format)""" self.add_uint32(Keys.LLM.NUM_DEEPSTACK_LAYERS.format(arch=self.arch), count) - def add_deepstack_layers(self, layers: Sequence[int]) -> None: + def add_deepstack_mapping(self, layers: Sequence[int]) -> None: """Add per-layer deepstack projector indices (Granite4 Vision format)""" - self.add_array(Keys.LLM.DEEPSTACK_LAYERS.format(arch=self.arch), list(layers)) + self.add_array(Keys.LLM.DEEPSTACK_MAPPING.format(arch=self.arch), list(layers)) def add_rope_dimension_count(self, count: int) -> None: self.add_uint32(Keys.Rope.DIMENSION_COUNT.format(arch=self.arch), count) diff --git a/src/llama-arch.cpp b/src/llama-arch.cpp index 7d9b0285fbfb..e687061cbe45 100644 --- a/src/llama-arch.cpp +++ b/src/llama-arch.cpp @@ -194,7 +194,7 @@ static const std::map LLM_KV_NAMES = { { LLM_KV_MOE_LATENT_SIZE, "%s.moe_latent_size" }, { LLM_KV_NEXTN_PREDICT_LAYERS, "%s.nextn_predict_layers" }, { LLM_KV_NUM_DEEPSTACK_LAYERS, "%s.n_deepstack_layers" }, - { LLM_KV_DEEPSTACK_LAYERS, "%s.deepstack_layers" }, + { LLM_KV_DEEPSTACK_MAPPING, "%s.deepstack_mapping" }, { LLM_KV_POOLING_TYPE, "%s.pooling_type" }, { LLM_KV_LOGIT_SCALE, "%s.logit_scale" }, { LLM_KV_DECODER_START_TOKEN_ID, "%s.decoder_start_token_id" }, diff --git a/src/llama-arch.h b/src/llama-arch.h index 9e964734aedf..d7b78a647778 100644 --- a/src/llama-arch.h +++ b/src/llama-arch.h @@ -198,7 +198,7 @@ enum llm_kv { LLM_KV_MOE_LATENT_SIZE, LLM_KV_NEXTN_PREDICT_LAYERS, LLM_KV_NUM_DEEPSTACK_LAYERS, - LLM_KV_DEEPSTACK_LAYERS, + LLM_KV_DEEPSTACK_MAPPING, LLM_KV_POOLING_TYPE, LLM_KV_LOGIT_SCALE, LLM_KV_DECODER_START_TOKEN_ID, diff --git a/src/llama-hparams.cpp b/src/llama-hparams.cpp index 79e791c28d32..af7d4ef83176 100644 --- a/src/llama-hparams.cpp +++ b/src/llama-hparams.cpp @@ -76,7 +76,7 @@ uint32_t llama_hparams::n_embd_inp() const { // Count the unique deepstack input indices std::unordered_set unique_deepstack_idxs; - for (const auto val : deepstack_layers_arr) { + for (const auto val : deepstack_mapping_arr) { if (val >= 0) { unique_deepstack_idxs.insert(val); } diff --git a/src/llama-hparams.h b/src/llama-hparams.h index 4efcae12f9b6..366d5cec3fac 100644 --- a/src/llama-hparams.h +++ b/src/llama-hparams.h @@ -214,7 +214,7 @@ struct llama_hparams { // deepstack layer array (qwen3vl, Granite4 Vision) // -1 => no deepstack // >=0 => input embedding index for deepstack injection - std::array deepstack_layers_arr; + std::array deepstack_mapping_arr; // gemma4 per-layer embedding uint32_t n_embd_per_layer = 0; diff --git a/src/llama-model-saver.cpp b/src/llama-model-saver.cpp index 9171145a8121..14135f68f3e3 100644 --- a/src/llama-model-saver.cpp +++ b/src/llama-model-saver.cpp @@ -228,7 +228,7 @@ void llama_model_saver::add_kv_from_model() { add_kv(LLM_KV_EXPERTS_PER_GROUP, hparams.n_group_experts); add_kv(LLM_KV_MOE_EVERY_N_LAYERS, hparams.moe_every_n_layers); add_kv(LLM_KV_NEXTN_PREDICT_LAYERS, hparams.nextn_predict_layers); - add_kv(LLM_KV_DEEPSTACK_LAYERS, hparams.deepstack_layers_arr); + add_kv(LLM_KV_DEEPSTACK_MAPPING, hparams.deepstack_mapping_arr); add_kv(LLM_KV_POOLING_TYPE, uint32_t(hparams.pooling_type)); add_kv(LLM_KV_LOGIT_SCALE, hparams.f_logit_scale); add_kv(LLM_KV_DECODER_START_TOKEN_ID, hparams.dec_start_token_id); diff --git a/src/llama-model.cpp b/src/llama-model.cpp index 6747a24ec132..22b34e3f0d5a 100644 --- a/src/llama-model.cpp +++ b/src/llama-model.cpp @@ -1062,9 +1062,9 @@ void llama_model_base::load_hparams(llama_model_loader & ml) { ml.get_key_or_arr(LLM_KV_FEED_FORWARD_LENGTH, hparams.n_ff_arr, hparams.n_layer, false); ml.get_key_or_arr(LLM_KV_ATTENTION_HEAD_COUNT, hparams.n_head_arr, hparams.n_layer, false); - // Populate deepstack_layers_arr - initialized to -1 (no deepstack) + // Populate deepstack_mapping_arr - initialized to -1 (no deepstack) // Model-specific code will populate if needed - std::fill(hparams.deepstack_layers_arr.begin(), hparams.deepstack_layers_arr.end(), -1); + std::fill(hparams.deepstack_mapping_arr.begin(), hparams.deepstack_mapping_arr.end(), -1); // n_head_kv is optional, default to n_head hparams.n_head_kv_arr = hparams.n_head_arr; @@ -1720,11 +1720,11 @@ void llama_model::print_info() const { LLAMA_LOG_INFO("%s: n_ctx_orig_yarn = %u\n", __func__, hparams.n_ctx_orig_yarn); LLAMA_LOG_INFO("%s: rope_yarn_log_mul = %.4f\n", __func__, hparams.rope_yarn_log_mul); LLAMA_LOG_INFO("%s: rope_finetuned = %s\n", __func__, hparams.rope_finetuned ? "yes" : "unknown"); - if (std::any_of(hparams.deepstack_layers_arr.begin(), - hparams.deepstack_layers_arr.end(), + if (std::any_of(hparams.deepstack_mapping_arr.begin(), + hparams.deepstack_mapping_arr.end(), [](const auto & entry) { return entry >= 0; })) { - LLAMA_LOG_INFO("%s: deepstack_layers_arr = %s\n", __func__, - print_f([&](uint32_t il) { return hparams.deepstack_layers_arr[il]; }, + LLAMA_LOG_INFO("%s: deepstack_mapping_arr = %s\n", __func__, + print_f([&](uint32_t il) { return hparams.deepstack_mapping_arr[il]; }, hparams.n_layer).c_str()); } // MRoPE (Multi-axis Rotary Position Embedding) sections diff --git a/src/models/granite.cpp b/src/models/granite.cpp index d71f23a05f6b..c1e0512e1af3 100644 --- a/src/models/granite.cpp +++ b/src/models/granite.cpp @@ -7,8 +7,8 @@ void llama_model_granite::load_arch_hparams(llama_model_loader & ml) { ml.get_key(LLM_KV_EMBEDDING_SCALE, hparams.f_embedding_scale, false); ml.get_key(LLM_KV_ATTENTION_SCALE, hparams.f_attention_scale, false); - // Granite4 Vision uses array deepstack_layers - ml.get_arr(LLM_KV_DEEPSTACK_LAYERS, hparams.deepstack_layers_arr, false); + // Granite4 Vision uses array deepstack_mapping + ml.get_arr(LLM_KV_DEEPSTACK_MAPPING, hparams.deepstack_mapping_arr, false); // Granite uses rope_finetuned as a switch for rope, so default to true bool rope_finetuned = true; @@ -129,7 +129,7 @@ llama_model_granite::graph::graph( // pure-text requests the input embedding carries zeros at those // offsets (see build_inp_embd's zero-pad of the token branch), so // the add is a no-op. - const auto & deepstack_emb_idx = hparams.deepstack_layers_arr[il]; + const auto & deepstack_emb_idx = hparams.deepstack_mapping_arr[il]; if (il > 0 && deepstack_emb_idx >= 0) { // Note: il == 0 uses the stream at offset 0, which is already // inpL (= base == stream at llm_layer 0 for granite vision 4.1). diff --git a/src/models/qwen3vl.cpp b/src/models/qwen3vl.cpp index 6e3efc4bcf64..4953362096e7 100644 --- a/src/models/qwen3vl.cpp +++ b/src/models/qwen3vl.cpp @@ -9,7 +9,7 @@ void llama_model_qwen3vl::load_arch_hparams(llama_model_loader & ml) { if (ml.get_key(LLM_KV_NUM_DEEPSTACK_LAYERS, n_deepstack_layers, false)) { // Create 1:1 mapping: layer i gets projector i+1 for (int32_t i = 0; i < (int32_t)n_deepstack_layers; ++i) { - hparams.deepstack_layers_arr[i] = i + 1; + hparams.deepstack_mapping_arr[i] = i + 1; } } switch (hparams.n_layer) { @@ -149,7 +149,7 @@ llama_model_qwen3vl::graph::graph(const llama_model & model, const llm_graph_par cur = build_cvec(cur, il); cb(cur, "l_out", il); - const auto & deepstack_emb_idx = hparams.deepstack_layers_arr[il]; + const auto & deepstack_emb_idx = hparams.deepstack_mapping_arr[il]; if (deepstack_emb_idx >= 0) { ggml_tensor * ds = ggml_view_2d(ctx0, res->t_inp_embd, n_embd, n_tokens, res->t_inp_embd->nb[1], deepstack_emb_idx * n_embd * sizeof(float)); cur = ggml_add(ctx0, cur, ds); diff --git a/src/models/qwen3vlmoe.cpp b/src/models/qwen3vlmoe.cpp index ceb4c2fde7c8..a71bb2af2910 100644 --- a/src/models/qwen3vlmoe.cpp +++ b/src/models/qwen3vlmoe.cpp @@ -10,7 +10,7 @@ void llama_model_qwen3vlmoe::load_arch_hparams(llama_model_loader & ml) { if (ml.get_key(LLM_KV_NUM_DEEPSTACK_LAYERS, n_deepstack_layers, false)) { // Create 1:1 mapping: layer i gets projector i+1 for (int32_t i = 0; i < (int32_t)n_deepstack_layers; ++i) { - hparams.deepstack_layers_arr[i] = i + 1; + hparams.deepstack_mapping_arr[i] = i + 1; } } switch (hparams.n_layer) { @@ -166,7 +166,7 @@ llama_model_qwen3vlmoe::graph::graph(const llama_model & model, const llm_graph_ cur = build_cvec(cur, il); cb(cur, "l_out", il); - const auto & deepstack_emb_idx = hparams.deepstack_layers_arr[il]; + const auto & deepstack_emb_idx = hparams.deepstack_mapping_arr[il]; if (deepstack_emb_idx >= 0) { ggml_tensor * ds = ggml_view_2d(ctx0, res->t_inp_embd, n_embd, n_tokens, res->t_inp_embd->nb[1], deepstack_emb_idx * n_embd * sizeof(float)); cur = ggml_add(ctx0, cur, ds); From 094ce7ce2757e96119e7c19b9265cd7d8ab29dfd Mon Sep 17 00:00:00 2001 From: Gabe Goodhart Date: Wed, 27 May 2026 13:17:23 -0600 Subject: [PATCH 72/94] refactor: Fully revert changes to n_deepstack_layers and qwen3vl* Since we're going to keep the GGUF KVs separate, it makes sense to just keep the hparams separate too to limit the scope of this branch. The down side is that n_deepstack_layers and deepstack_mapping_arr are potentially conflicting. Branch: Granite4Vision AI-usage: none Signed-off-by: Gabe Goodhart --- src/llama-hparams.cpp | 8 -------- src/llama-hparams.h | 10 +++++++++- src/llama-model-saver.cpp | 1 + src/llama-model.cpp | 4 ---- src/models/granite.cpp | 12 ++++++++++++ src/models/qwen3vl.cpp | 17 +++++------------ src/models/qwen3vlmoe.cpp | 17 +++++------------ 7 files changed, 32 insertions(+), 37 deletions(-) diff --git a/src/llama-hparams.cpp b/src/llama-hparams.cpp index af7d4ef83176..48a373be77c8 100644 --- a/src/llama-hparams.cpp +++ b/src/llama-hparams.cpp @@ -74,14 +74,6 @@ uint32_t llama_hparams::n_rot(uint32_t il) const { uint32_t llama_hparams::n_embd_inp() const { uint32_t n_embd_inp = n_embd; - // Count the unique deepstack input indices - std::unordered_set unique_deepstack_idxs; - for (const auto val : deepstack_mapping_arr) { - if (val >= 0) { - unique_deepstack_idxs.insert(val); - } - } - const auto n_deepstack_layers = unique_deepstack_idxs.size(); if (n_deepstack_layers > 0) { n_embd_inp += n_embd * n_deepstack_layers; } diff --git a/src/llama-hparams.h b/src/llama-hparams.h index 366d5cec3fac..deda491ec160 100644 --- a/src/llama-hparams.h +++ b/src/llama-hparams.h @@ -211,7 +211,15 @@ struct llama_hparams { uint32_t indexer_head_size = 0; uint32_t indexer_top_k = 0; - // deepstack layer array (qwen3vl, Granite4 Vision) + // qwen3vl deepstack + // When parsed from GGUF, this implies the first N layers consume the first + // N deepstack embeddings. Use deepstack_mapping_arr if you need a more + // complex mapping. If using deepstack_mapping_arr, also make sure to set + // n_deepstack_layers to the number of unique deepstack layers so that + // n_embd_imp is accurate (see granite.cpp). + uint32_t n_deepstack_layers = 0; + + // deepstack layer array (Granite4 Vision) // -1 => no deepstack // >=0 => input embedding index for deepstack injection std::array deepstack_mapping_arr; diff --git a/src/llama-model-saver.cpp b/src/llama-model-saver.cpp index 14135f68f3e3..91cf06b7a511 100644 --- a/src/llama-model-saver.cpp +++ b/src/llama-model-saver.cpp @@ -228,6 +228,7 @@ void llama_model_saver::add_kv_from_model() { add_kv(LLM_KV_EXPERTS_PER_GROUP, hparams.n_group_experts); add_kv(LLM_KV_MOE_EVERY_N_LAYERS, hparams.moe_every_n_layers); add_kv(LLM_KV_NEXTN_PREDICT_LAYERS, hparams.nextn_predict_layers); + add_kv(LLM_KV_NUM_DEEPSTACK_LAYERS, hparams.n_deepstack_layers); add_kv(LLM_KV_DEEPSTACK_MAPPING, hparams.deepstack_mapping_arr); add_kv(LLM_KV_POOLING_TYPE, uint32_t(hparams.pooling_type)); add_kv(LLM_KV_LOGIT_SCALE, hparams.f_logit_scale); diff --git a/src/llama-model.cpp b/src/llama-model.cpp index 22b34e3f0d5a..f13e49e9be81 100644 --- a/src/llama-model.cpp +++ b/src/llama-model.cpp @@ -1062,10 +1062,6 @@ void llama_model_base::load_hparams(llama_model_loader & ml) { ml.get_key_or_arr(LLM_KV_FEED_FORWARD_LENGTH, hparams.n_ff_arr, hparams.n_layer, false); ml.get_key_or_arr(LLM_KV_ATTENTION_HEAD_COUNT, hparams.n_head_arr, hparams.n_layer, false); - // Populate deepstack_mapping_arr - initialized to -1 (no deepstack) - // Model-specific code will populate if needed - std::fill(hparams.deepstack_mapping_arr.begin(), hparams.deepstack_mapping_arr.end(), -1); - // n_head_kv is optional, default to n_head hparams.n_head_kv_arr = hparams.n_head_arr; diff --git a/src/models/granite.cpp b/src/models/granite.cpp index c1e0512e1af3..095492199b2a 100644 --- a/src/models/granite.cpp +++ b/src/models/granite.cpp @@ -7,9 +7,21 @@ void llama_model_granite::load_arch_hparams(llama_model_loader & ml) { ml.get_key(LLM_KV_EMBEDDING_SCALE, hparams.f_embedding_scale, false); ml.get_key(LLM_KV_ATTENTION_SCALE, hparams.f_attention_scale, false); + // Populate deepstack_mapping_arr - initialized to -1 (no deepstack) + std::fill(hparams.deepstack_mapping_arr.begin(), hparams.deepstack_mapping_arr.end(), -1); + // Granite4 Vision uses array deepstack_mapping ml.get_arr(LLM_KV_DEEPSTACK_MAPPING, hparams.deepstack_mapping_arr, false); + // Count the unique deepstack input indices + std::unordered_set unique_deepstack_idxs; + for (const auto val : hparams.deepstack_mapping_arr) { + if (val >= 0) { + unique_deepstack_idxs.insert(val); + } + } + hparams.n_deepstack_layers = unique_deepstack_idxs.size(); + // Granite uses rope_finetuned as a switch for rope, so default to true bool rope_finetuned = true; ml.get_key(LLM_KV_ROPE_SCALING_FINETUNED, rope_finetuned, false); diff --git a/src/models/qwen3vl.cpp b/src/models/qwen3vl.cpp index 4953362096e7..5defd8939448 100644 --- a/src/models/qwen3vl.cpp +++ b/src/models/qwen3vl.cpp @@ -1,17 +1,9 @@ #include "models.h" void llama_model_qwen3vl::load_arch_hparams(llama_model_loader & ml) { + ml.get_key(LLM_KV_NUM_DEEPSTACK_LAYERS, hparams.n_deepstack_layers, false); ml.get_key_or_arr(LLM_KV_ROPE_DIMENSION_SECTIONS, hparams.rope_sections, 4, true); ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, hparams.f_norm_rms_eps); - - // qwen3vl uses scalar n_deepstack_layers - uint32_t n_deepstack_layers = 0; - if (ml.get_key(LLM_KV_NUM_DEEPSTACK_LAYERS, n_deepstack_layers, false)) { - // Create 1:1 mapping: layer i gets projector i+1 - for (int32_t i = 0; i < (int32_t)n_deepstack_layers; ++i) { - hparams.deepstack_mapping_arr[i] = i + 1; - } - } switch (hparams.n_layer) { case 28: type = LLM_TYPE_1_7B; break; case 36: type = hparams.n_embd == 2560 ? LLM_TYPE_4B : LLM_TYPE_8B; break; @@ -59,6 +51,8 @@ std::unique_ptr llama_model_qwen3vl::build_arch_graph(const l } llama_model_qwen3vl::graph::graph(const llama_model & model, const llm_graph_params & params) : llm_graph_context(params) { + const size_t n_deepstack_layers = hparams.n_deepstack_layers; + const int64_t n_embd = hparams.n_embd; const int64_t n_embd_head = hparams.n_embd_head_v(); @@ -149,9 +143,8 @@ llama_model_qwen3vl::graph::graph(const llama_model & model, const llm_graph_par cur = build_cvec(cur, il); cb(cur, "l_out", il); - const auto & deepstack_emb_idx = hparams.deepstack_mapping_arr[il]; - if (deepstack_emb_idx >= 0) { - ggml_tensor * ds = ggml_view_2d(ctx0, res->t_inp_embd, n_embd, n_tokens, res->t_inp_embd->nb[1], deepstack_emb_idx * n_embd * sizeof(float)); + if (il < (int) n_deepstack_layers) { + ggml_tensor * ds = ggml_view_2d(ctx0, res->t_inp_embd, n_embd, n_tokens, res->t_inp_embd->nb[1], (il + 1) * n_embd * sizeof(float)); cur = ggml_add(ctx0, cur, ds); cb(cur, "deepstack_out", il); } diff --git a/src/models/qwen3vlmoe.cpp b/src/models/qwen3vlmoe.cpp index a71bb2af2910..5b77df57122e 100644 --- a/src/models/qwen3vlmoe.cpp +++ b/src/models/qwen3vlmoe.cpp @@ -1,18 +1,10 @@ #include "models.h" void llama_model_qwen3vlmoe::load_arch_hparams(llama_model_loader & ml) { + ml.get_key(LLM_KV_NUM_DEEPSTACK_LAYERS, hparams.n_deepstack_layers, false); ml.get_key_or_arr(LLM_KV_ROPE_DIMENSION_SECTIONS, hparams.rope_sections, 4, true); ml.get_key(LLM_KV_EXPERT_FEED_FORWARD_LENGTH, hparams.n_ff_exp, false); ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, hparams.f_norm_rms_eps); - - // qwen3vlmoe uses scalar n_deepstack_layers - uint32_t n_deepstack_layers = 0; - if (ml.get_key(LLM_KV_NUM_DEEPSTACK_LAYERS, n_deepstack_layers, false)) { - // Create 1:1 mapping: layer i gets projector i+1 - for (int32_t i = 0; i < (int32_t)n_deepstack_layers; ++i) { - hparams.deepstack_mapping_arr[i] = i + 1; - } - } switch (hparams.n_layer) { case 48: type = LLM_TYPE_30B_A3B; break; case 94: type = LLM_TYPE_235B_A22B; break; @@ -69,6 +61,8 @@ std::unique_ptr llama_model_qwen3vlmoe::build_arch_graph(cons } llama_model_qwen3vlmoe::graph::graph(const llama_model & model, const llm_graph_params & params) : llm_graph_context(params) { + const size_t n_deepstack_layers = hparams.n_deepstack_layers; + const int64_t n_embd = hparams.n_embd; const int64_t n_embd_head = hparams.n_embd_head_v(); @@ -166,9 +160,8 @@ llama_model_qwen3vlmoe::graph::graph(const llama_model & model, const llm_graph_ cur = build_cvec(cur, il); cb(cur, "l_out", il); - const auto & deepstack_emb_idx = hparams.deepstack_mapping_arr[il]; - if (deepstack_emb_idx >= 0) { - ggml_tensor * ds = ggml_view_2d(ctx0, res->t_inp_embd, n_embd, n_tokens, res->t_inp_embd->nb[1], deepstack_emb_idx * n_embd * sizeof(float)); + if (il < (int) n_deepstack_layers) { + ggml_tensor * ds = ggml_view_2d(ctx0, res->t_inp_embd, n_embd, n_tokens, res->t_inp_embd->nb[1], (il + 1) * n_embd * sizeof(float)); cur = ggml_add(ctx0, cur, ds); cb(cur, "deepstack_out", il); } From b5563663448b8d6b7cba44687f52d3e94e37fb8e Mon Sep 17 00:00:00 2001 From: Gabe Goodhart Date: Wed, 27 May 2026 13:19:59 -0600 Subject: [PATCH 73/94] fix: Revert removal of "is_deepstack_layers" GGUF KV This KV is not used at all on the c++ side, so it's fully dead, but there's also no need to conflate this cleanup with the addition of G4V. Branch: Granite4Vision AI-usage: none Signed-off-by: Gabe Goodhart --- conversion/qwen3vl.py | 7 +++++++ gguf-py/gguf/gguf_writer.py | 3 +++ 2 files changed, 10 insertions(+) diff --git a/conversion/qwen3vl.py b/conversion/qwen3vl.py index 11bff927478e..9f11757697f5 100644 --- a/conversion/qwen3vl.py +++ b/conversion/qwen3vl.py @@ -35,6 +35,10 @@ def __init__(self, *args, **kwargs): self.hparams_vision["num_attention_heads"] = self.hparams_vision.get("num_heads") self.hparams_vision["num_hidden_layers"] = self.hparams_vision.get("depth") + self.is_deepstack_layers = [False] * int(self.hparams_vision["num_hidden_layers"] or 0) + for idx in self.hparams_vision.get("deepstack_visual_indexes", []): + self.is_deepstack_layers[idx] = True + def set_gguf_parameters(self): super().set_gguf_parameters() # in case mixed modalities, the arch will be handled by subclass @@ -51,6 +55,9 @@ def set_gguf_parameters(self): rms_norm_eps = self.global_config.get("text_config", {}).get("rms_norm_eps", 1e-6) self.gguf_writer.add_vision_attention_layernorm_eps(rms_norm_eps) + if self.is_deepstack_layers: + self.gguf_writer.add_vision_is_deepstack_layers(self.is_deepstack_layers) + @classmethod def filter_tensors(cls, item: tuple[str, Callable[[], Tensor]]) -> tuple[str, Callable[[], Tensor]] | None: name, gen = item diff --git a/gguf-py/gguf/gguf_writer.py b/gguf-py/gguf/gguf_writer.py index ebab0c5f0553..d159ed8d807b 100644 --- a/gguf-py/gguf/gguf_writer.py +++ b/gguf-py/gguf/gguf_writer.py @@ -1236,6 +1236,9 @@ def add_vision_wa_layer_indexes(self, layers: Sequence[int]) -> None: """ self.add_array(Keys.ClipVision.WA_LAYER_INDEXES, layers) + def add_vision_is_deepstack_layers(self, layers: Sequence[bool]) -> None: + self.add_array(Keys.ClipVision.IS_DEEPSTACK_LAYERS, layers) + def add_vision_wa_pattern_mode(self, modes: Sequence[int]) -> None: self.add_array(Keys.ClipVision.WA_PATTERN_MODE, modes) From 11bd6bb2888c7b4c48dee9ca24b961b5440f52ce Mon Sep 17 00:00:00 2001 From: Gabe Goodhart Date: Wed, 27 May 2026 13:43:01 -0600 Subject: [PATCH 74/94] fix: Remove unnecessary ggml_cont and build_forward_expand in cbx Branch: Granite4Vision AI-usage: none Signed-off-by: Gabe Goodhart --- tools/mtmd/models/granite4-vision.cpp | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/tools/mtmd/models/granite4-vision.cpp b/tools/mtmd/models/granite4-vision.cpp index fc433e5281cf..c4be2d318280 100644 --- a/tools/mtmd/models/granite4-vision.cpp +++ b/tools/mtmd/models/granite4-vision.cpp @@ -70,10 +70,7 @@ static ggml_tensor * g4v_build_block( auto cbx = [&](ggml_tensor * & t, const char * step) { const std::string name = "g4v_blk" + std::to_string(bid) + "_" + step; - ggml_tensor * tagged = ggml_cont(ctx, t); - ggml_set_name(tagged, name.c_str()); - ggml_build_forward_expand(gf, tagged); - t = tagged; + ggml_set_name(t, name.c_str()); }; // 1. Top-level LN From 5c6bd55e9074f73fcb2e7c461319e526f66214f4 Mon Sep 17 00:00:00 2001 From: Gabe Goodhart Date: Wed, 27 May 2026 14:11:00 -0600 Subject: [PATCH 75/94] style: Clean up comments Branch: Granite4Vision AI-usage: none Signed-off-by: Gabe Goodhart --- src/models/granite.cpp | 16 ++-------------- tools/mtmd/clip-impl.h | 2 +- tools/mtmd/clip.cpp | 8 ++------ tools/mtmd/models/granite4-vision.cpp | 9 +++------ 4 files changed, 8 insertions(+), 27 deletions(-) diff --git a/src/models/granite.cpp b/src/models/granite.cpp index 095492199b2a..f53da316a793 100644 --- a/src/models/granite.cpp +++ b/src/models/granite.cpp @@ -129,22 +129,10 @@ llama_model_granite::graph::graph( for (int il = 0; il < n_layer; ++il) { // Granite Vision 4.1 deepstack: inject the projector stream that - // targets decoder layer `il`, BEFORE the decoder layer runs. This - // matches the reference forward in - // /ibm-granite/granite-vision-4.1/modeling.py: for each layer_idx, - // check the (llm_layer, packed_features) pairs and add to - // hidden_states[image_positions] if layer_idx == llm_layer. - // - // In the qwen3vl-deepstack layout the mmproj writes streams into - // res->t_inp_embd at offset (stream_index + 1) * n_embd (stream 0 - // is the base and is already consumed via build_inp_embd). For - // pure-text requests the input embedding carries zeros at those - // offsets (see build_inp_embd's zero-pad of the token branch), so - // the add is a no-op. + // targets decoder layer `il` before the decoder runs. + // NOTE: skip the first deepstack layer since that's inpL const auto & deepstack_emb_idx = hparams.deepstack_mapping_arr[il]; if (il > 0 && deepstack_emb_idx >= 0) { - // Note: il == 0 uses the stream at offset 0, which is already - // inpL (= base == stream at llm_layer 0 for granite vision 4.1). ggml_tensor * ds = ggml_view_2d(ctx0, res->t_inp_embd, n_embd, n_tokens, res->t_inp_embd->nb[1], diff --git a/tools/mtmd/clip-impl.h b/tools/mtmd/clip-impl.h index 2a932b14e43b..6ea31008ba64 100644 --- a/tools/mtmd/clip-impl.h +++ b/tools/mtmd/clip-impl.h @@ -435,7 +435,7 @@ struct clip_image_f32 { std::vector buf; // Indicates what token (if any) should be appended after encoding this tile. - // Used by models like Granite 4 Vision that inject learned newline embeddings. + // Used by llava-next style models with learned newlines (Granite4 Vision) enum clip_append_token_type { CLIP_APPEND_TOKEN_NONE = 0, CLIP_APPEND_TOKEN_NEWLINE_ROWWISE = 1, // Append newline after each row diff --git a/tools/mtmd/clip.cpp b/tools/mtmd/clip.cpp index d58165228157..01765205e803 100644 --- a/tools/mtmd/clip.cpp +++ b/tools/mtmd/clip.cpp @@ -4200,7 +4200,7 @@ bool clip_image_batch_encode(clip_ctx * ctx, const int n_threads, const clip_ima { // Granite Vision 4.1 uses precomputed permutation index // tensors to express the _win / _unwin / spatial sampling - // reshapes as ggml_get_rows gathers. The names are set + // reshapes as ggml_get_rows gathers. The names are set // by g4v_gather() in models/granite4-vision.cpp. const int patch_size = model.hparams.patch_size; const int image_side = imgs.entries.front()->nx / patch_size; @@ -4260,8 +4260,7 @@ bool clip_image_batch_encode(clip_ctx * ctx, const int n_threads, const clip_ima }; // Stage 1b only uses block 0's permutations; future stages - // will upload all 8 blocks. upload() no-ops when a tensor - // is not present in the graph, so this is safe for either. + // will upload all blocks. for (size_t bid = 0; bid < hparams.vision_feature_layer.size(); ++bid) { const std::string prefix = "g4v_blk" + std::to_string(bid) + "_"; upload(prefix + "win_idx", make_win_idx(image_side, window_side)); @@ -4427,9 +4426,6 @@ int clip_n_mmproj_embd(const struct clip_ctx * ctx) { case PROJECTOR_TYPE_GRANITE_SPEECH: return ctx->model.qf_proj_blocks[0].qf_proj_linear_w->ne[1]; case PROJECTOR_TYPE_GRANITE4_VISION: - // Stage 2a: graph output is 8 concatenated streams (deepstack - // layout with "base" = stream targeting llm_layer 0). Per-token - // feature count = projector_count * projection_dim. return ctx->model.qf_proj_blocks.size() * ctx->model.hparams.projection_dim; case PROJECTOR_TYPE_GLM4V: return ctx->model.mm_ffn_down_w->ne[1]; diff --git a/tools/mtmd/models/granite4-vision.cpp b/tools/mtmd/models/granite4-vision.cpp index c4be2d318280..7a0488041d5f 100644 --- a/tools/mtmd/models/granite4-vision.cpp +++ b/tools/mtmd/models/granite4-vision.cpp @@ -11,13 +11,10 @@ /* * Granite Vision 4.1 clip graph * - * Stage 1a: SigLIP vision tower (27 layers, post-norm) - * Stage 1b: WindowQFormer blocks (8 blocks total) + * Stage 1a: SigLIP vision tower (N layers, post-norm) + * Stage 1b: WindowQFormer blocks (deepstack + spatial) * Stage 1c: Concatenate and pack outputs * Stage 1d: Append newline tokens if append_token is set - * - * A single WindowQFormer block maps a (1, 576, 1152) vision-layer - * hidden state h to a (1, 144, 2560) deepstack feature. */ // --------------------------------------------------------------------------- @@ -35,7 +32,7 @@ static ggml_tensor * g4v_gather(ggml_context * ctx, ggml_cgraph * gf, return ggml_get_rows(ctx, src, idx); } -// Area-interpolate 24x24 -> 12x12 via 2x2 avg pool +// Area downsampling with average pooling (eg 24x24 -> 12x12 via 2x2) static ggml_tensor * g4v_interp_down(ggml_context * ctx, ggml_tensor * src, int side, int new_side) { const int n_embd = src->ne[0]; ggml_tensor * t = ggml_reshape_4d(ctx, src, n_embd, side, side, 1); From 49100629e7f5a10e01f9a5cc80aa8eac5bb74d36 Mon Sep 17 00:00:00 2001 From: Gabe Goodhart Date: Wed, 27 May 2026 14:12:01 -0600 Subject: [PATCH 76/94] fix: Tighter and more flexible code for g4v_build_block This could be refactored to look a lot more like granite-speech, but the overall block constructs before/after the qformer are pretty different, so for now I'm going to leave it as is and just tighten a bit. Branch: Granite4Vision AI-usage: none Signed-off-by: Gabe Goodhart --- tools/mtmd/models/granite4-vision.cpp | 19 +++++++------------ 1 file changed, 7 insertions(+), 12 deletions(-) diff --git a/tools/mtmd/models/granite4-vision.cpp b/tools/mtmd/models/granite4-vision.cpp index 7a0488041d5f..35e408cd279e 100644 --- a/tools/mtmd/models/granite4-vision.cpp +++ b/tools/mtmd/models/granite4-vision.cpp @@ -126,10 +126,10 @@ static ggml_tensor * g4v_build_block( ggml_tensor * wv, ggml_tensor * bv, ggml_tensor * wo, ggml_tensor * bo, ggml_tensor * ln_w, ggml_tensor * ln_b) -> ggml_tensor * { - const int n_head = 18; // QFormer: 1152 / 64 - const int d_h = n_embd / n_head; - const int nq = q_stream->ne[1]; - const int nkv = kv_stream->ne[1]; + const int d_h = 64; // hard coded in downsampling.py:77 + const int n_head = n_embd / d_h; + const int nq = q_stream->ne[1]; + const int nkv = kv_stream->ne[1]; // We loop over the n_windows dimension by treating it as batch via // the ne[2] dim in mul_mat. Collapse (C, n, n_win) → (C, n * n_win) @@ -323,7 +323,7 @@ ggml_cgraph * clip_graph_granite4_vision::build() { const int projector_count = hparams.vision_feature_layer.size(); const float qformer_eps = 1e-12f; - std::vector streams(projector_count, nullptr); + ggml_tensor * mmproj = nullptr; for (int bid = 0; bid < projector_count; ++bid) { const auto & blk = model.qf_proj_blocks[bid]; @@ -332,7 +332,7 @@ ggml_cgraph * clip_graph_granite4_vision::build() { GGML_ASSERT(vlayer >= 1 && vlayer <= n_layer); ggml_tensor * h = layer_outs[vlayer - 1]; - streams[bid] = g4v_build_block( + ggml_tensor * stream = g4v_build_block( this, ctx0, gf, blk, h, bid, hparams.proj_spatial_offsets[bid], @@ -340,14 +340,9 @@ ggml_cgraph * clip_graph_granite4_vision::build() { hparams.downsample_window_side, hparams.downsample_query_side, qformer_eps); + mmproj = mmproj ? ggml_concat(ctx0, mmproj, stream, 0) : stream; } - // Concatenate the projector activations in the correct order - ggml_tensor * mmproj = nullptr; - for (int k = 0; k < projector_count; ++k) { - ggml_tensor * s = ggml_cont(ctx0, streams[k]); - mmproj = (k == 0) ? s : ggml_concat(ctx0, mmproj, s, 0); - } // --- Stage 1d: Append newline tokens if append_token is set --- if (append_token_type == clip_image_f32::CLIP_APPEND_TOKEN_NEWLINE_ROWWISE) { mmproj = append_rowwise_newlines(ctx0, mmproj); From bb811563ac37ae3029f7c9ffebd51f9fce6e2a7a Mon Sep 17 00:00:00 2001 From: Gabe Goodhart Date: Wed, 27 May 2026 14:16:06 -0600 Subject: [PATCH 77/94] fix: Remove unnecessary `unordered_set` include Branch: Granite4Vision AI-usage: none Signed-off-by: Gabe Goodhart --- src/llama-hparams.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/src/llama-hparams.cpp b/src/llama-hparams.cpp index 48a373be77c8..2239309c8fb4 100644 --- a/src/llama-hparams.cpp +++ b/src/llama-hparams.cpp @@ -4,7 +4,6 @@ #include #include -#include void llama_hparams::set_swa_pattern(uint32_t n_pattern, bool dense_first) { if (dense_first) { From 4e6a20616746cf19e0278ed099edcc0496e2036e Mon Sep 17 00:00:00 2001 From: Gabe Goodhart Date: Wed, 27 May 2026 14:21:18 -0600 Subject: [PATCH 78/94] fix: Add architecture guard on deepstack_mapping_arr printout Branch: Granite4Vision AI-usage: none Signed-off-by: Gabe Goodhart --- src/llama-model.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/llama-model.cpp b/src/llama-model.cpp index f13e49e9be81..de406468c87c 100644 --- a/src/llama-model.cpp +++ b/src/llama-model.cpp @@ -1716,7 +1716,8 @@ void llama_model::print_info() const { LLAMA_LOG_INFO("%s: n_ctx_orig_yarn = %u\n", __func__, hparams.n_ctx_orig_yarn); LLAMA_LOG_INFO("%s: rope_yarn_log_mul = %.4f\n", __func__, hparams.rope_yarn_log_mul); LLAMA_LOG_INFO("%s: rope_finetuned = %s\n", __func__, hparams.rope_finetuned ? "yes" : "unknown"); - if (std::any_of(hparams.deepstack_mapping_arr.begin(), + if (arch == LLM_ARCH_GRANITE && + std::any_of(hparams.deepstack_mapping_arr.begin(), hparams.deepstack_mapping_arr.end(), [](const auto & entry) { return entry >= 0; })) { LLAMA_LOG_INFO("%s: deepstack_mapping_arr = %s\n", __func__, From 096ea2ca6b2f233ed585400b375b61aa7c6aed6c Mon Sep 17 00:00:00 2001 From: Gabe Goodhart Date: Wed, 27 May 2026 14:21:35 -0600 Subject: [PATCH 79/94] fix: Remove unnecessary AI-gen comment Branch: Granite4Vision AI-usage: none Signed-off-by: Gabe Goodhart --- tools/mtmd/mtmd.cpp | 7 ------- 1 file changed, 7 deletions(-) diff --git a/tools/mtmd/mtmd.cpp b/tools/mtmd/mtmd.cpp index 09f3483c514e..5e7efbecc86e 100644 --- a/tools/mtmd/mtmd.cpp +++ b/tools/mtmd/mtmd.cpp @@ -502,13 +502,6 @@ struct mtmd_context { } break; case PROJECTOR_TYPE_GRANITE4_VISION: { - // Granite Vision 4.1 uses a single "" token to - // mark image positions; see modeling.py::get_image_token_mask. - // LlavaNext-style multi-tile splitting drives off the - // image_grid_pinpoints mirrored into - // hparams.image_res_candidates by the mmproj loader. - // Per-tile encoder loop + pack_and_unpad happens inside - // mtmd_encode (PROJECTOR_TYPE_GRANITE4_VISION branch). img_beg = ""; img_end = ""; image_preproc = std::make_unique(ctx_v); From 0b464328fdc47fef3959be29c839135e2acb84f3 Mon Sep 17 00:00:00 2001 From: Gabe Goodhart Date: Wed, 27 May 2026 16:24:49 -0600 Subject: [PATCH 80/94] fix: Always initialize deepstack_mapping_arr with -1 values This was causing `test-llama-archs` to fail, likely due to trying to save the uninitialized values, then re-loading them. It's safer to always initialize so that other models don't forget and end up with undefined behavior. Branch: Granite4Vision AI-usage: none Signed-off-by: Gabe Goodhart --- src/llama-model.cpp | 3 +++ src/models/granite.cpp | 3 --- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/llama-model.cpp b/src/llama-model.cpp index de406468c87c..52395fa79e33 100644 --- a/src/llama-model.cpp +++ b/src/llama-model.cpp @@ -1062,6 +1062,9 @@ void llama_model_base::load_hparams(llama_model_loader & ml) { ml.get_key_or_arr(LLM_KV_FEED_FORWARD_LENGTH, hparams.n_ff_arr, hparams.n_layer, false); ml.get_key_or_arr(LLM_KV_ATTENTION_HEAD_COUNT, hparams.n_head_arr, hparams.n_layer, false); + // Populate deepstack_mapping_arr - initialized to -1 (no deepstack) + std::fill(hparams.deepstack_mapping_arr.begin(), hparams.deepstack_mapping_arr.end(), -1); + // n_head_kv is optional, default to n_head hparams.n_head_kv_arr = hparams.n_head_arr; diff --git a/src/models/granite.cpp b/src/models/granite.cpp index f53da316a793..b8641edb1575 100644 --- a/src/models/granite.cpp +++ b/src/models/granite.cpp @@ -7,9 +7,6 @@ void llama_model_granite::load_arch_hparams(llama_model_loader & ml) { ml.get_key(LLM_KV_EMBEDDING_SCALE, hparams.f_embedding_scale, false); ml.get_key(LLM_KV_ATTENTION_SCALE, hparams.f_attention_scale, false); - // Populate deepstack_mapping_arr - initialized to -1 (no deepstack) - std::fill(hparams.deepstack_mapping_arr.begin(), hparams.deepstack_mapping_arr.end(), -1); - // Granite4 Vision uses array deepstack_mapping ml.get_arr(LLM_KV_DEEPSTACK_MAPPING, hparams.deepstack_mapping_arr, false); From 7c4a791ce5e9955890910d60dc15511fd271ae41 Mon Sep 17 00:00:00 2001 From: Gabe Goodhart Date: Thu, 28 May 2026 15:40:28 -0600 Subject: [PATCH 81/94] style: Remove TODO about block/vs non-block tensor mapping Branch: Granite4Vision AI-usage: none Signed-off-by: Gabe Goodhart --- gguf-py/gguf/tensor_mapping.py | 1 - 1 file changed, 1 deletion(-) diff --git a/gguf-py/gguf/tensor_mapping.py b/gguf-py/gguf/tensor_mapping.py index 21cbfb83b13d..3970932df04e 100644 --- a/gguf-py/gguf/tensor_mapping.py +++ b/gguf-py/gguf/tensor_mapping.py @@ -156,7 +156,6 @@ class TensorNameMap: MODEL_TENSOR.V_ENC_MSFA_NORM: ( "model.vision_tower.timm_model.msfa.norm", # gemma3n ), - # TODO: Should the rest of the block mappings be moved here? MODEL_TENSOR.V_ENC_EMBD_IMGNL: ( "model.image_newline", # Granite4Vision ), From 263a4a346f092925518030b4e5adfd874f3dc169 Mon Sep 17 00:00:00 2001 From: Gabe Goodhart Date: Thu, 28 May 2026 16:38:21 -0600 Subject: [PATCH 82/94] refactor: Move is_vision_feature_layer logic into clip_hparams Branch: Granite4Vision AI-usage: none Signed-off-by: Gabe Goodhart --- tools/mtmd/clip-model.h | 5 +++++ tools/mtmd/models/llava.cpp | 7 ++----- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/tools/mtmd/clip-model.h b/tools/mtmd/clip-model.h index 6300ff5df08b..f244be652630 100644 --- a/tools/mtmd/clip-model.h +++ b/tools/mtmd/clip-model.h @@ -4,6 +4,7 @@ #include "clip.h" #include "clip-impl.h" +#include #include #include #include @@ -163,6 +164,10 @@ struct clip_hparams { return false; } + + bool is_vision_feature_layer(int32_t layer) const { + return std::find(vision_feature_layer.begin(), vision_feature_layer.end(), layer) != vision_feature_layer.end(); + } }; struct clip_layer { diff --git a/tools/mtmd/models/llava.cpp b/tools/mtmd/models/llava.cpp index 896db43fcda4..5aa3d2f0fac1 100644 --- a/tools/mtmd/models/llava.cpp +++ b/tools/mtmd/models/llava.cpp @@ -1,7 +1,5 @@ #include "models.h" -#include - // this graph is used by llava, granite and glm // due to having embedding_stack (used by granite), we cannot reuse build_vit ggml_cgraph * clip_graph_llava::build() { @@ -53,7 +51,6 @@ ggml_cgraph * clip_graph_llava::build() { } std::vector embedding_stack; - const auto & vision_feature_layer = hparams.vision_feature_layer; // loop over layers for (int il = 0; il < max_feature_layer; il++) { @@ -62,7 +59,7 @@ ggml_cgraph * clip_graph_llava::build() { // If this is an embedding feature layer, save the output. // NOTE: 0 index here refers to the input to the encoder. - if (std::find(vision_feature_layer.begin(), vision_feature_layer.end(), il) != vision_feature_layer.end()) { + if (hparams.is_vision_feature_layer(il)) { embedding_stack.push_back(cur); } @@ -137,7 +134,7 @@ ggml_cgraph * clip_graph_llava::build() { // process vision feature layers (used by granite) { // final layer is a vision feature layer - if (std::find(vision_feature_layer.begin(), vision_feature_layer.end(), max_feature_layer) != vision_feature_layer.end()) { + if (hparams.is_vision_feature_layer(max_feature_layer)) { embedding_stack.push_back(inpL); } From 440f36b52e45a10a10609d87ce32db985a48167d Mon Sep 17 00:00:00 2001 From: Gabe Goodhart Date: Thu, 28 May 2026 16:45:36 -0600 Subject: [PATCH 83/94] refactor: Use a bool for append_token Branch: Granite4Vision AI-usage: none Signed-off-by: Gabe Goodhart --- tools/mtmd/clip-impl.h | 5 +---- tools/mtmd/clip.cpp | 7 +------ tools/mtmd/models/granite4-vision.cpp | 2 +- tools/mtmd/models/models.h | 4 ++-- tools/mtmd/mtmd.cpp | 9 +++------ 5 files changed, 8 insertions(+), 19 deletions(-) diff --git a/tools/mtmd/clip-impl.h b/tools/mtmd/clip-impl.h index 6ea31008ba64..60fb0d3fe0dd 100644 --- a/tools/mtmd/clip-impl.h +++ b/tools/mtmd/clip-impl.h @@ -436,10 +436,7 @@ struct clip_image_f32 { // Indicates what token (if any) should be appended after encoding this tile. // Used by llava-next style models with learned newlines (Granite4 Vision) - enum clip_append_token_type { - CLIP_APPEND_TOKEN_NONE = 0, - CLIP_APPEND_TOKEN_NEWLINE_ROWWISE = 1, // Append newline after each row - } append_token = CLIP_APPEND_TOKEN_NONE; + bool append_token = false; }; // diff --git a/tools/mtmd/clip.cpp b/tools/mtmd/clip.cpp index 01765205e803..2b75fc009e8d 100644 --- a/tools/mtmd/clip.cpp +++ b/tools/mtmd/clip.cpp @@ -3046,11 +3046,6 @@ void clip_image_size_free(struct clip_image_size * load_image_size) { } void clip_image_u8_free(struct clip_image_u8 * img) { delete img; } void clip_image_f32_free(struct clip_image_f32 * img) { delete img; } -void clip_image_set_append_token(struct clip_image_f32 * img, int token_type) { - if (img) { - img->append_token = (clip_image_f32::clip_append_token_type)token_type; - } -} void clip_image_u8_batch_free(struct clip_image_u8_batch * batch) { delete batch; } void clip_image_f32_batch_free(struct clip_image_f32_batch * batch) { delete batch; } @@ -3392,7 +3387,7 @@ int clip_n_output_tokens(const struct clip_ctx * ctx, struct clip_image_f32 * im n_patches = (query_side * n) * (query_side * n); // Add newline tokens based on append_token field. - if (img->append_token == clip_image_f32::CLIP_APPEND_TOKEN_NEWLINE_ROWWISE) { + if (img->append_token) { // For single-tile case: append 1 newline row. // For multi-tile rowwise: handled by caller, but here we // report the per-tile count including one trailing newline. diff --git a/tools/mtmd/models/granite4-vision.cpp b/tools/mtmd/models/granite4-vision.cpp index 35e408cd279e..7ffb622f4f69 100644 --- a/tools/mtmd/models/granite4-vision.cpp +++ b/tools/mtmd/models/granite4-vision.cpp @@ -344,7 +344,7 @@ ggml_cgraph * clip_graph_granite4_vision::build() { } // --- Stage 1d: Append newline tokens if append_token is set --- - if (append_token_type == clip_image_f32::CLIP_APPEND_TOKEN_NEWLINE_ROWWISE) { + if (append_token) { mmproj = append_rowwise_newlines(ctx0, mmproj); ggml_set_name(mmproj, "g4v_mmproj_out_nl"); } else { diff --git a/tools/mtmd/models/models.h b/tools/mtmd/models/models.h index 432017a32978..33f7b277fbd0 100644 --- a/tools/mtmd/models/models.h +++ b/tools/mtmd/models/models.h @@ -195,7 +195,7 @@ struct clip_graph_kimik25 : clip_graph { struct clip_graph_granite4_vision : clip_graph { clip_graph_granite4_vision(clip_ctx * ctx, const clip_image_f32 & img) : clip_graph(ctx, img), - append_token_type((clip_image_f32::clip_append_token_type)img.append_token) {} + append_token(img.append_token) {} ggml_cgraph * build() override; @@ -203,7 +203,7 @@ struct clip_graph_granite4_vision : clip_graph { // The graph is per-tile since only batch-size 1 is supported in clip. As // such, this value is set at construct time based on the tile that will be // encoded, then used during build to determine how to handle newlines. - clip_image_f32::clip_append_token_type append_token_type; + const bool append_token; ggml_tensor * build_newline_row(ggml_context * ctx0); ggml_tensor * append_rowwise_newlines(ggml_context * ctx0, ggml_tensor * tile_output); diff --git a/tools/mtmd/mtmd.cpp b/tools/mtmd/mtmd.cpp index 5e7efbecc86e..72a4e6708045 100644 --- a/tools/mtmd/mtmd.cpp +++ b/tools/mtmd/mtmd.cpp @@ -800,15 +800,12 @@ struct mtmd_tokenizer { if (ctx->proj_type_v() == PROJECTOR_TYPE_GRANITE4_VISION) { if (batch_f32.entries.size() == 1) { // Single-tile (overview only): append one newline row. - clip_image_set_append_token(batch_f32.entries[0].get(), - clip_image_f32::CLIP_APPEND_TOKEN_NEWLINE_ROWWISE); + batch_f32.entries[0]->append_token = true; } else { // Multi-tile: overview gets no newline, grid tiles get one. - clip_image_set_append_token(batch_f32.entries[0].get(), - clip_image_f32::CLIP_APPEND_TOKEN_NONE); + batch_f32.entries[0]->append_token = false; for (size_t i = 1; i < batch_f32.entries.size(); ++i) { - clip_image_set_append_token(batch_f32.entries[i].get(), - clip_image_f32::CLIP_APPEND_TOKEN_NEWLINE_ROWWISE); + batch_f32.entries[i]->append_token = true; } } } From 9d8e3e85e83bf3a00cef33e08695e867131cb539 Mon Sep 17 00:00:00 2001 From: Gabe Goodhart Date: Thu, 28 May 2026 16:47:19 -0600 Subject: [PATCH 84/94] style: Remove unnecessary comment Branch: Granite4Vision AI-usage: none Signed-off-by: Gabe Goodhart --- tools/mtmd/clip.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/tools/mtmd/clip.cpp b/tools/mtmd/clip.cpp index 2b75fc009e8d..75aeccee62b1 100644 --- a/tools/mtmd/clip.cpp +++ b/tools/mtmd/clip.cpp @@ -1577,7 +1577,6 @@ struct clip_model_loader { hparams.image_resize_algo = RESIZE_ALGO_BICUBIC_PILLOW; hparams.image_resize_pad = PAD_CEIL; - // Load the per-projector hparams (vision_feature_layers, spatial_offsets) get_arr_int(KEY_FEATURE_LAYER, hparams.vision_feature_layer); get_arr_int(KEY_PROJ_SPATIAL_OFFSETS, hparams.proj_spatial_offsets); GGML_ASSERT(hparams.vision_feature_layer.size() == hparams.proj_spatial_offsets.size()); From 54546fff81db128042db19e82ba0d7ab0c26bb83 Mon Sep 17 00:00:00 2001 From: Gabe Goodhart Date: Thu, 28 May 2026 16:51:40 -0600 Subject: [PATCH 85/94] fix: Remove unused get_model api yikes! Branch: Granite4Vision AI-usage: none Signed-off-by: Gabe Goodhart --- tools/mtmd/clip-model.h | 1 - tools/mtmd/clip.cpp | 4 ---- 2 files changed, 5 deletions(-) diff --git a/tools/mtmd/clip-model.h b/tools/mtmd/clip-model.h index f244be652630..14cc6f62c2a9 100644 --- a/tools/mtmd/clip-model.h +++ b/tools/mtmd/clip-model.h @@ -617,4 +617,3 @@ struct clip_model { }; const clip_hparams * clip_get_hparams(const struct clip_ctx * ctx); -const clip_model * clip_get_model (const struct clip_ctx * ctx); diff --git a/tools/mtmd/clip.cpp b/tools/mtmd/clip.cpp index 75aeccee62b1..d69d971ac3aa 100644 --- a/tools/mtmd/clip.cpp +++ b/tools/mtmd/clip.cpp @@ -4476,10 +4476,6 @@ const clip_hparams * clip_get_hparams(const struct clip_ctx * ctx) { return &ctx->model.hparams; } -const clip_model * clip_get_model(const struct clip_ctx * ctx) { - return &ctx->model; -} - std::map clip_get_mem_usage(const struct clip_ctx * ctx) { std::map result = ctx->mem_usage; for (auto & [dev, size] : ctx->mem_compute) { From 70c2302083847e532e44f0498ec1d5e8b1c5a261 Mon Sep 17 00:00:00 2001 From: Gabe Goodhart Date: Thu, 28 May 2026 17:39:31 -0600 Subject: [PATCH 86/94] refactor: Rearrange helpers for g4v to be private members and use build_attn Branch: Granite4Vision AI-usage: full (Bob, OpenCode + Qwen3.6-35b) Signed-off-by: Gabe Goodhart --- tools/mtmd/models/granite4-vision.cpp | 216 ++++++++++++-------------- tools/mtmd/models/models.h | 6 + 2 files changed, 105 insertions(+), 117 deletions(-) diff --git a/tools/mtmd/models/granite4-vision.cpp b/tools/mtmd/models/granite4-vision.cpp index 7ffb622f4f69..f21cc643e203 100644 --- a/tools/mtmd/models/granite4-vision.cpp +++ b/tools/mtmd/models/granite4-vision.cpp @@ -18,36 +18,37 @@ */ // --------------------------------------------------------------------------- -// Permutation helpers +// Member method implementations // --------------------------------------------------------------------------- -static ggml_tensor * g4v_gather(ggml_context * ctx, ggml_cgraph * gf, - ggml_tensor * src, - const std::string & name, - int idx_len) { - (void) gf; - ggml_tensor * idx = ggml_new_tensor_1d(ctx, GGML_TYPE_I32, idx_len); +ggml_tensor * clip_graph_granite4_vision::gather( + ggml_tensor * src, + const std::string & name, + int idx_len) { + ggml_tensor * idx = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, idx_len); ggml_set_name(idx, name.c_str()); ggml_set_input(idx); - return ggml_get_rows(ctx, src, idx); + return ggml_get_rows(ctx0, src, idx); } -// Area downsampling with average pooling (eg 24x24 -> 12x12 via 2x2) -static ggml_tensor * g4v_interp_down(ggml_context * ctx, ggml_tensor * src, int side, int new_side) { +ggml_tensor * clip_graph_granite4_vision::interp_down( + ggml_tensor * src, + int side, + int new_side) { const int n_embd = src->ne[0]; - ggml_tensor * t = ggml_reshape_4d(ctx, src, n_embd, side, side, 1); - t = ggml_cont(ctx, ggml_permute(ctx, t, 2, 0, 1, 3)); + ggml_tensor * t = ggml_reshape_4d(ctx0, src, n_embd, side, side, 1); + t = ggml_cont(ctx0, ggml_permute(ctx0, t, 2, 0, 1, 3)); const int kernel = side / new_side; - t = ggml_pool_2d(ctx, t, GGML_OP_POOL_AVG, kernel, kernel, kernel, kernel, 0, 0); - t = ggml_cont(ctx, ggml_permute(ctx, t, 1, 2, 0, 3)); - return ggml_reshape_2d(ctx, t, n_embd, new_side * new_side); + t = ggml_pool_2d(ctx0, t, GGML_OP_POOL_AVG, kernel, kernel, kernel, kernel, 0, 0); + t = ggml_cont(ctx0, ggml_permute(ctx0, t, 1, 2, 0, 3)); + return ggml_reshape_2d(ctx0, t, n_embd, new_side * new_side); } -// Build one WindowQFormer block's forward pass -static ggml_tensor * g4v_build_block( - const clip_graph * g, - ggml_context * ctx, - ggml_cgraph * gf, +// --------------------------------------------------------------------------- +// build_block - WindowQFormer block implementation +// --------------------------------------------------------------------------- + +ggml_tensor * clip_graph_granite4_vision::build_block( const qf_block & blk, ggml_tensor * h, int bid, @@ -71,16 +72,16 @@ static ggml_tensor * g4v_build_block( }; // 1. Top-level LN - ggml_tensor * x = g->build_norm(h, blk.qf_proj_norm_w, blk.qf_proj_norm_b, NORM_TYPE_NORMAL, g->eps, bid); + ggml_tensor * x = build_norm(h, blk.qf_proj_norm_w, blk.qf_proj_norm_b, NORM_TYPE_NORMAL, eps, bid); cbx(x, "norm"); // 2. enc = _win(x, image_side, window_side) ggml_tensor * enc; { - ggml_tensor * enc_flat = g4v_gather(ctx, gf, x, + ggml_tensor * enc_flat = gather(x, "g4v_blk" + std::to_string(bid) + "_win_idx", image_side * image_side); - enc = ggml_reshape_3d(ctx, enc_flat, n_embd, enc_len, n_windows); + enc = ggml_reshape_3d(ctx0, enc_flat, n_embd, enc_len, n_windows); } cbx(enc, "enc"); @@ -88,88 +89,38 @@ static ggml_tensor * g4v_build_block( ggml_tensor * d; (void) spatial_offset; if (spatial_offset >= 0) { - d = g4v_gather(ctx, gf, x, + d = gather(x, "g4v_blk" + std::to_string(bid) + "_spatial_idx", new_side * new_side); } else { - d = g4v_interp_down(ctx, x, image_side, new_side); + d = interp_down(x, image_side, new_side); } cbx(d, "downsampled"); // 4. query_embeds = query + _win(d, new_side, query_side) ggml_tensor * q_in; { - ggml_tensor * dw_flat = g4v_gather(ctx, gf, d, + ggml_tensor * dw_flat = gather(d, "g4v_blk" + std::to_string(bid) + "_qwin_idx", new_side * new_side); - ggml_tensor * dw = ggml_reshape_3d(ctx, dw_flat, n_embd, query_len, n_windows); - q_in = ggml_add(ctx, dw, blk.qf_proj_query); + ggml_tensor * dw = ggml_reshape_3d(ctx0, dw_flat, n_embd, query_len, n_windows); + q_in = ggml_add(ctx0, dw, blk.qf_proj_query); } cbx(q_in, "query_embeds"); // 5. encoder_embeds = enc + image_positions → (C, enc_len, n_windows) - ggml_tensor * e_in = ggml_add(ctx, enc, blk.qf_proj_img_pos); + ggml_tensor * e_in = ggml_add(ctx0, enc, blk.qf_proj_img_pos); cbx(e_in, "encoder_embeds"); // 6. Qformer forward. - ggml_tensor * q = g->build_norm(q_in, blk.qf_proj_post_norm_w, blk.qf_proj_post_norm_b, NORM_TYPE_NORMAL, qformer_eps, bid); - - // Helper: one post-norm BERT attention block. - // attn(q,k,v) via dense W^Q/W^K/W^V + scaled dot-product - // residual: LN_out(dropout(dense(attn_out)) + residual_input) - // For self-attention q == k == v == residual. For cross-attn q != k=v. - auto run_postnorm_attn = [&]( - ggml_tensor * q_stream, - ggml_tensor * kv_stream, - ggml_tensor * wq, ggml_tensor * bq, - ggml_tensor * wk, ggml_tensor * bk, - ggml_tensor * wv, ggml_tensor * bv, - ggml_tensor * wo, ggml_tensor * bo, - ggml_tensor * ln_w, ggml_tensor * ln_b) -> ggml_tensor * { - const int d_h = 64; // hard coded in downsampling.py:77 - const int n_head = n_embd / d_h; - const int nq = q_stream->ne[1]; - const int nkv = kv_stream->ne[1]; - - // We loop over the n_windows dimension by treating it as batch via - // the ne[2] dim in mul_mat. Collapse (C, n, n_win) → (C, n * n_win) - // for the linear projections (they don't care about the window dim), - // then reshape back. - auto linear = [&](ggml_tensor * x, ggml_tensor * w, ggml_tensor * b) -> ggml_tensor * { - ggml_tensor * t = ggml_reshape_2d(ctx, x, x->ne[0], x->ne[1] * x->ne[2]); - t = g->build_mm(w, t); - if (b) t = ggml_add(ctx, t, b); - return t; - }; - - ggml_tensor * Q = linear(q_stream, wq, bq); - ggml_tensor * K = linear(kv_stream, wk, bk); - ggml_tensor * V = linear(kv_stream, wv, bv); - - Q = ggml_reshape_4d(ctx, Q, d_h, n_head, nq, n_windows); - K = ggml_reshape_4d(ctx, K, d_h, n_head, nkv, n_windows); - V = ggml_reshape_4d(ctx, V, d_h, n_head, nkv, n_windows); - - ggml_tensor * q_p = ggml_permute(ctx, Q, 0, 2, 1, 3); - ggml_tensor * k_p = ggml_permute(ctx, K, 0, 2, 1, 3); - ggml_tensor * v_p = ggml_permute(ctx, V, 1, 2, 0, 3); - v_p = ggml_cont(ctx, v_p); - - ggml_tensor * kq = ggml_mul_mat(ctx, k_p, q_p); - const float scale = 1.0f / std::sqrt((float) d_h); - kq = ggml_soft_max_ext(ctx, kq, nullptr, scale, 0.0f); - - ggml_tensor * kqv = ggml_mul_mat(ctx, v_p, kq); - ggml_tensor * out = ggml_permute(ctx, kqv, 0, 2, 1, 3); - out = ggml_cont_3d(ctx, out, d_h * n_head, nq, n_windows); - - ggml_tensor * o = ggml_reshape_2d(ctx, out, n_embd, nq * n_windows); - o = g->build_mm(wo, o); - if (bo) o = ggml_add(ctx, o, bo); - o = ggml_reshape_3d(ctx, o, n_embd, nq, n_windows); - o = ggml_add(ctx, o, q_stream); - o = g->build_norm(o, ln_w, ln_b, NORM_TYPE_NORMAL, qformer_eps, bid); - return o; + ggml_tensor * q = build_norm(q_in, blk.qf_proj_post_norm_w, blk.qf_proj_post_norm_b, NORM_TYPE_NORMAL, qformer_eps, bid); + + // Helper for linear projections with window batching + auto linear = [&](ggml_tensor * x, ggml_tensor * w, ggml_tensor * b) -> ggml_tensor * { + ggml_tensor * t = ggml_reshape_2d(ctx0, x, x->ne[0], x->ne[1] * x->ne[2]); + t = build_mm(w, t); + if (b) t = ggml_add(ctx0, t, b); + return t; }; // Get the single QFormer layer @@ -177,53 +128,85 @@ static ggml_tensor * g4v_build_block( const auto & pl = blk.qf_proj_layers[0]; // 6a. Self-attention - ggml_tensor * sa_out = run_postnorm_attn( - q, q, - pl.q_w, pl.q_b, - pl.k_w, pl.k_b, - pl.v_w, pl.v_b, - pl.o_w, pl.o_b, - pl.ln_1_w, pl.ln_1_b); + ggml_tensor * sa_out; + { + const int d_h = 64; + const int n_head = n_embd / d_h; + const int nq = q->ne[1]; + const float scale = 1.0f / std::sqrt((float) d_h); + + ggml_tensor * Q = linear(q, pl.q_w, pl.q_b); + ggml_tensor * K = linear(q, pl.k_w, pl.k_b); + ggml_tensor * V = linear(q, pl.v_w, pl.v_b); + + Q = ggml_reshape_4d(ctx0, Q, d_h, n_head, nq, n_windows); + K = ggml_reshape_4d(ctx0, K, d_h, n_head, nq, n_windows); + V = ggml_reshape_4d(ctx0, V, d_h, n_head, nq, n_windows); + + sa_out = build_attn(pl.o_w, pl.o_b, Q, K, V, nullptr, scale, bid); + sa_out = ggml_reshape_3d(ctx0, sa_out, n_embd, nq, n_windows); + + sa_out = ggml_add(ctx0, sa_out, q); + sa_out = build_norm(sa_out, pl.ln_1_w, pl.ln_1_b, + NORM_TYPE_NORMAL, qformer_eps, bid); + } cbx(sa_out, "sa_out"); // 6b. Cross-attention - ggml_tensor * ca_out = run_postnorm_attn( - sa_out, e_in, - pl.cross_attn_q_w, pl.cross_attn_q_b, - pl.cross_attn_k_w, pl.cross_attn_k_b, - pl.cross_attn_v_w, pl.cross_attn_v_b, - pl.cross_attn_o_w, pl.cross_attn_o_b, - pl.cross_attn_norm_w, pl.cross_attn_norm_b); + ggml_tensor * ca_out; + { + const int d_h = 64; + const int n_head = n_embd / d_h; + const int nq = sa_out->ne[1]; + const int nkv = e_in->ne[1]; + const float scale = 1.0f / std::sqrt((float) d_h); + + ggml_tensor * Q = linear(sa_out, pl.cross_attn_q_w, pl.cross_attn_q_b); + ggml_tensor * K = linear(e_in, pl.cross_attn_k_w, pl.cross_attn_k_b); + ggml_tensor * V = linear(e_in, pl.cross_attn_v_w, pl.cross_attn_v_b); + + Q = ggml_reshape_4d(ctx0, Q, d_h, n_head, nq, n_windows); + K = ggml_reshape_4d(ctx0, K, d_h, n_head, nkv, n_windows); + V = ggml_reshape_4d(ctx0, V, d_h, n_head, nkv, n_windows); + + ca_out = build_attn(pl.cross_attn_o_w, pl.cross_attn_o_b, + Q, K, V, nullptr, scale, bid); + ca_out = ggml_reshape_3d(ctx0, ca_out, n_embd, nq, n_windows); + + ca_out = ggml_add(ctx0, ca_out, sa_out); + ca_out = build_norm(ca_out, pl.cross_attn_norm_w, pl.cross_attn_norm_b, + NORM_TYPE_NORMAL, qformer_eps, bid); + } cbx(ca_out, "ca_out"); // 6c. FFN ggml_tensor * ffn; { - ggml_tensor * t = ggml_reshape_2d(ctx, ca_out, n_embd, query_len * n_windows); - t = g->build_mm(pl.ff_up_w, t); - if (pl.ff_up_b) t = ggml_add(ctx, t, pl.ff_up_b); - t = ggml_gelu_erf(ctx, t); - t = g->build_mm(pl.ff_down_w, t); - if (pl.ff_down_b) t = ggml_add(ctx, t, pl.ff_down_b); - t = ggml_reshape_3d(ctx, t, n_embd, query_len, n_windows); - ffn = ggml_add(ctx, t, ca_out); - ffn = g->build_norm(ffn, pl.ln_2_w, pl.ln_2_b, NORM_TYPE_NORMAL, qformer_eps, bid); + ggml_tensor * t = ggml_reshape_2d(ctx0, ca_out, n_embd, query_len * n_windows); + t = build_mm(pl.ff_up_w, t); + if (pl.ff_up_b) t = ggml_add(ctx0, t, pl.ff_up_b); + t = ggml_gelu_erf(ctx0, t); + t = build_mm(pl.ff_down_w, t); + if (pl.ff_down_b) t = ggml_add(ctx0, t, pl.ff_down_b); + t = ggml_reshape_3d(ctx0, t, n_embd, query_len, n_windows); + ffn = ggml_add(ctx0, t, ca_out); + ffn = build_norm(ffn, pl.ln_2_w, pl.ln_2_b, NORM_TYPE_NORMAL, qformer_eps, bid); } cbx(ffn, "qformer_out"); // 7. _unwin back to raster ggml_tensor * unwinned; { - ggml_tensor * flat = ggml_reshape_2d(ctx, ffn, n_embd, query_len * n_windows); - unwinned = g4v_gather(ctx, gf, flat, + ggml_tensor * flat = ggml_reshape_2d(ctx0, ffn, n_embd, query_len * n_windows); + unwinned = gather(flat, "g4v_blk" + std::to_string(bid) + "_unwin_idx", new_side * new_side); } cbx(unwinned, "unwin"); // 8. out_linear - ggml_tensor * out = g->build_mm(blk.qf_proj_linear_w, unwinned); - if (blk.qf_proj_linear_b) out = ggml_add(ctx, out, blk.qf_proj_linear_b); + ggml_tensor * out = build_mm(blk.qf_proj_linear_w, unwinned); + if (blk.qf_proj_linear_b) out = ggml_add(ctx0, out, blk.qf_proj_linear_b); cbx(out, "out"); return out; @@ -332,9 +315,8 @@ ggml_cgraph * clip_graph_granite4_vision::build() { GGML_ASSERT(vlayer >= 1 && vlayer <= n_layer); ggml_tensor * h = layer_outs[vlayer - 1]; - ggml_tensor * stream = g4v_build_block( - this, ctx0, gf, blk, - h, bid, + ggml_tensor * stream = build_block( + blk, h, bid, hparams.proj_spatial_offsets[bid], n_patches_x, hparams.downsample_window_side, diff --git a/tools/mtmd/models/models.h b/tools/mtmd/models/models.h index 33f7b277fbd0..88290a8bb268 100644 --- a/tools/mtmd/models/models.h +++ b/tools/mtmd/models/models.h @@ -205,6 +205,12 @@ struct clip_graph_granite4_vision : clip_graph { // encoded, then used during build to determine how to handle newlines. const bool append_token; + ggml_tensor * gather(ggml_tensor * src, const std::string & name, int idx_len); + ggml_tensor * interp_down(ggml_tensor * src, int side, int new_side); + ggml_tensor * build_block(const qf_block & blk, ggml_tensor * h, int bid, + int spatial_offset, int image_side, int window_side, + int query_side, float qformer_eps); + ggml_tensor * build_newline_row(ggml_context * ctx0); ggml_tensor * append_rowwise_newlines(ggml_context * ctx0, ggml_tensor * tile_output); }; From ecb247bac92c9fc74c8c7c2c0009d2cfa6a95375 Mon Sep 17 00:00:00 2001 From: Gabe Goodhart Date: Fri, 29 May 2026 16:22:29 -0600 Subject: [PATCH 87/94] fix: Fix off-by-one in vision layer index This was inherited from the Claude Code implementation that pushed the negative index inversion down into the model file. Branch: Granite4Vision AI-usage: none Signed-off-by: Gabe Goodhart --- tools/mtmd/models/granite4-vision.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tools/mtmd/models/granite4-vision.cpp b/tools/mtmd/models/granite4-vision.cpp index f21cc643e203..96b403336d1f 100644 --- a/tools/mtmd/models/granite4-vision.cpp +++ b/tools/mtmd/models/granite4-vision.cpp @@ -311,9 +311,8 @@ ggml_cgraph * clip_graph_granite4_vision::build() { const auto & blk = model.qf_proj_blocks[bid]; int vlayer = hparams.vision_feature_layer[bid]; - if (vlayer < 0) vlayer = (n_layer + 1) + vlayer; - GGML_ASSERT(vlayer >= 1 && vlayer <= n_layer); - ggml_tensor * h = layer_outs[vlayer - 1]; + GGML_ASSERT(vlayer >= 0 && vlayer < n_layer); + ggml_tensor * h = layer_outs[vlayer]; ggml_tensor * stream = build_block( blk, h, bid, @@ -322,6 +321,7 @@ ggml_cgraph * clip_graph_granite4_vision::build() { hparams.downsample_window_side, hparams.downsample_query_side, qformer_eps); + cb(stream, "proj_v_out", vlayer); mmproj = mmproj ? ggml_concat(ctx0, mmproj, stream, 0) : stream; } From d8d37dfb17222052fd035a5f1dcb0bec4b8146bb Mon Sep 17 00:00:00 2001 From: Gabe Goodhart Date: Fri, 29 May 2026 17:37:19 -0600 Subject: [PATCH 88/94] fix: Fix norm/post_norm mixup in conversion face. palm. :( Branch: Granite4Vision AI-usage: none Signed-off-by: Gabe Goodhart --- gguf-py/gguf/tensor_mapping.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/gguf-py/gguf/tensor_mapping.py b/gguf-py/gguf/tensor_mapping.py index 87c358d6f148..042fe43336bb 100644 --- a/gguf-py/gguf/tensor_mapping.py +++ b/gguf-py/gguf/tensor_mapping.py @@ -1975,13 +1975,13 @@ class TensorNameMap: "model.spatial_projectors.{bid}.out_linear", # Granite4 Vision ), MODEL_TENSOR.V_MULTI_PROJ_NORM: ( - "model.layerwise_projectors.{bid}.qformer.layernorm", # Granite4 Vision - "model.spatial_projectors.{bid}.qformer.layernorm", # Granite4 Vision - ), - MODEL_TENSOR.V_MULTI_PROJ_POST_NORM: ( "model.layerwise_projectors.{bid}.norm", # Granite4 Vision "model.spatial_projectors.{bid}.norm", # Granite4 Vision ), + MODEL_TENSOR.V_MULTI_PROJ_POST_NORM: ( + "model.layerwise_projectors.{bid}.qformer.layernorm", # Granite4 Vision + "model.spatial_projectors.{bid}.qformer.layernorm", # Granite4 Vision + ), # For these tensors, bid => proj-id MODEL_TENSOR.V_QF_SELF_ATTN_Q: ( From 255f934fa0ccbd842911128d6963449583c8e430 Mon Sep 17 00:00:00 2001 From: Gabe Goodhart Date: Fri, 29 May 2026 17:37:41 -0600 Subject: [PATCH 89/94] style: More descriptive tensor names Branch: Granite4Vision AI-usage: none Signed-off-by: Gabe Goodhart --- tools/mtmd/models/granite4-vision.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tools/mtmd/models/granite4-vision.cpp b/tools/mtmd/models/granite4-vision.cpp index 96b403336d1f..3ba676dcbb64 100644 --- a/tools/mtmd/models/granite4-vision.cpp +++ b/tools/mtmd/models/granite4-vision.cpp @@ -72,6 +72,7 @@ ggml_tensor * clip_graph_granite4_vision::build_block( }; // 1. Top-level LN + cbx(h, "inp"); ggml_tensor * x = build_norm(h, blk.qf_proj_norm_w, blk.qf_proj_norm_b, NORM_TYPE_NORMAL, eps, bid); cbx(x, "norm"); @@ -321,7 +322,7 @@ ggml_cgraph * clip_graph_granite4_vision::build() { hparams.downsample_window_side, hparams.downsample_query_side, qformer_eps); - cb(stream, "proj_v_out", vlayer); + cb(stream, (std::string("proj_") + std::to_string(bid) + std::string("_v_out")).c_str(), vlayer); mmproj = mmproj ? ggml_concat(ctx0, mmproj, stream, 0) : stream; } From 9eb1762cf4f6f0b93d429dffa7f6e97cf3fbd622 Mon Sep 17 00:00:00 2001 From: Gabe Goodhart Date: Thu, 4 Jun 2026 13:58:25 -0600 Subject: [PATCH 90/94] fix: Apply PR cleanup for new conversion changes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit AI-usage: none Signed-off-by: Gabe Goodhart Co-authored-by: Sigbjørn Skjæret --- conversion/granite.py | 6 +----- convert_lora_to_gguf.py | 8 -------- 2 files changed, 1 insertion(+), 13 deletions(-) diff --git a/conversion/granite.py b/conversion/granite.py index 6a5677c3e375..53441fe57016 100644 --- a/conversion/granite.py +++ b/conversion/granite.py @@ -442,11 +442,7 @@ def set_gguf_parameters(self): @classmethod def filter_tensors(cls, item: tuple[str, Callable[[], Tensor]]) -> tuple[str, Callable[[], Tensor]] | None: name, _ = item - if ( - name.startswith("language_model.") - or "vision_model.head" in name - or name.startswith("lm_head") - ): + if ("vision_model.head" in name or name.startswith("lm_head")): return None return super().filter_tensors(item) diff --git a/convert_lora_to_gguf.py b/convert_lora_to_gguf.py index 4bc490b21dbc..45202b333871 100755 --- a/convert_lora_to_gguf.py +++ b/convert_lora_to_gguf.py @@ -499,14 +499,6 @@ def get_tensors(self) -> Iterator[tuple[str, Tensor]]: yield (name, cast(torch.Tensor, LoraTorchTensor(tensor.A, tensor.B))) def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]: - # Handle adapters targeting language_model subcomponent - for lm_prefix, replace in [ - ("model.language_model.", "model."), # Granite4 Vision - ("language_model.", ""), # Granite Speech - ]: - if name.startswith(lm_prefix): - name = name.replace(lm_prefix, replace, 1) - break dest = list(super().modify_tensors(data_torch, name, bid)) # some archs may have the same tensor for lm_head and output (tie word embeddings) # in this case, adapters targeting lm_head will fail when using llama-export-lora From c5afa80ca61a1b4f1d808d7d1db301c9f2e6ceb2 Mon Sep 17 00:00:00 2001 From: Gabe Goodhart Date: Thu, 4 Jun 2026 15:37:12 -0600 Subject: [PATCH 91/94] fix(convert): Remove duplicate V_ENC_EMBD_IMGNL Branch: Granite4Vision AI-usage: none Signed-off-by: Gabe Goodhart --- gguf-py/gguf/tensor_mapping.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/gguf-py/gguf/tensor_mapping.py b/gguf-py/gguf/tensor_mapping.py index 7e62aedc5dac..3e63b2165051 100644 --- a/gguf-py/gguf/tensor_mapping.py +++ b/gguf-py/gguf/tensor_mapping.py @@ -156,9 +156,6 @@ class TensorNameMap: MODEL_TENSOR.V_ENC_MSFA_NORM: ( "model.vision_tower.timm_model.msfa.norm", # gemma3n ), - MODEL_TENSOR.V_ENC_EMBD_IMGNL: ( - "model.image_newline", # Granite4Vision - ), MODEL_TENSOR.A_CTC_OUT: ( "encoder.out", ), @@ -1463,7 +1460,7 @@ class TensorNameMap: # TODO: I think these should all be moved to mapping_cfg? MODEL_TENSOR.V_ENC_EMBD_IMGNL: ( - "model.image_newline", # Deepseek-OCR + "model.image_newline", # Deepseek-OCR, Granite4Vision "vit.perceive.image_newline", # HunyuanVL ), From c12a262946d51ce659cb5a73c0ac15cf5d8fcbd1 Mon Sep 17 00:00:00 2001 From: Gabe Goodhart Date: Thu, 4 Jun 2026 16:22:29 -0600 Subject: [PATCH 92/94] refactor: append_token -> add_newline Branch: Granite4Vision AI-usage: none Signed-off-by: Gabe Goodhart --- tools/mtmd/clip-impl.h | 2 +- tools/mtmd/clip.cpp | 4 +--- tools/mtmd/clip.h | 3 --- tools/mtmd/models/granite4-vision.cpp | 6 +++--- tools/mtmd/models/models.h | 4 ++-- tools/mtmd/mtmd.cpp | 6 +++--- 6 files changed, 10 insertions(+), 15 deletions(-) diff --git a/tools/mtmd/clip-impl.h b/tools/mtmd/clip-impl.h index 34a91daf05c2..393e085f71e9 100644 --- a/tools/mtmd/clip-impl.h +++ b/tools/mtmd/clip-impl.h @@ -448,7 +448,7 @@ struct clip_image_f32 { // marks the global view in e.g., DeepSeek-OCR Models bool add_viewsep = false; // whether a learned newline token should be appended after the image (eg Granite4 Vision) - bool append_token = false; + bool add_newline = false; }; // diff --git a/tools/mtmd/clip.cpp b/tools/mtmd/clip.cpp index 85d6a3df98fd..bda95fc22250 100644 --- a/tools/mtmd/clip.cpp +++ b/tools/mtmd/clip.cpp @@ -3475,9 +3475,7 @@ int clip_n_output_tokens(const struct clip_ctx * ctx, struct clip_image_f32 * im const int side = img->nx / params.patch_size; const int n = side / window_side; n_patches = (query_side * n) * (query_side * n); - - // Add newline tokens based on append_token field. - if (img->append_token) { + if (img->add_newline) { // For single-tile case: append 1 newline row. // For multi-tile rowwise: handled by caller, but here we // report the per-tile count including one trailing newline. diff --git a/tools/mtmd/clip.h b/tools/mtmd/clip.h index 8b46691c2fb5..a62c9d61877b 100644 --- a/tools/mtmd/clip.h +++ b/tools/mtmd/clip.h @@ -94,9 +94,6 @@ size_t clip_image_f32_batch_nx(const struct clip_image_f32_batch * batch, int id size_t clip_image_f32_batch_ny(const struct clip_image_f32_batch * batch, int idx); // equivalent to batch[idx]->ny struct clip_image_f32 * clip_image_f32_get_img(const struct clip_image_f32_batch * batch, int idx); // equivalent to batch[idx]->data -// Set the append_token field of a clip_image_f32 (for newline injection models). -void clip_image_set_append_token(struct clip_image_f32 * img, int token_type); - /** * Build image from pixels decoded by other libraries instead of stb_image.h for better performance. * The memory layout is RGBRGBRGB..., input buffer length must be 3*nx*ny bytes diff --git a/tools/mtmd/models/granite4-vision.cpp b/tools/mtmd/models/granite4-vision.cpp index 3ba676dcbb64..9adb6f0fdbfa 100644 --- a/tools/mtmd/models/granite4-vision.cpp +++ b/tools/mtmd/models/granite4-vision.cpp @@ -14,7 +14,7 @@ * Stage 1a: SigLIP vision tower (N layers, post-norm) * Stage 1b: WindowQFormer blocks (deepstack + spatial) * Stage 1c: Concatenate and pack outputs - * Stage 1d: Append newline tokens if append_token is set + * Stage 1d: Append newline tokens if add_newline is set */ // --------------------------------------------------------------------------- @@ -326,8 +326,8 @@ ggml_cgraph * clip_graph_granite4_vision::build() { mmproj = mmproj ? ggml_concat(ctx0, mmproj, stream, 0) : stream; } - // --- Stage 1d: Append newline tokens if append_token is set --- - if (append_token) { + // --- Stage 1d: Append newline tokens if add_newline is set --- + if (add_newline) { mmproj = append_rowwise_newlines(ctx0, mmproj); ggml_set_name(mmproj, "g4v_mmproj_out_nl"); } else { diff --git a/tools/mtmd/models/models.h b/tools/mtmd/models/models.h index 68e889ec78ec..d1865103bcb1 100644 --- a/tools/mtmd/models/models.h +++ b/tools/mtmd/models/models.h @@ -215,7 +215,7 @@ struct clip_graph_exaone4_5 : clip_graph { struct clip_graph_granite4_vision : clip_graph { clip_graph_granite4_vision(clip_ctx * ctx, const clip_image_f32 & img) : clip_graph(ctx, img), - append_token(img.append_token) {} + add_newline(img.add_newline) {} ggml_cgraph * build() override; @@ -223,7 +223,7 @@ struct clip_graph_granite4_vision : clip_graph { // The graph is per-tile since only batch-size 1 is supported in clip. As // such, this value is set at construct time based on the tile that will be // encoded, then used during build to determine how to handle newlines. - const bool append_token; + const bool add_newline; ggml_tensor * gather(ggml_tensor * src, const std::string & name, int idx_len); ggml_tensor * interp_down(ggml_tensor * src, int side, int new_side); diff --git a/tools/mtmd/mtmd.cpp b/tools/mtmd/mtmd.cpp index dbb9b73437eb..260f307560af 100644 --- a/tools/mtmd/mtmd.cpp +++ b/tools/mtmd/mtmd.cpp @@ -819,12 +819,12 @@ struct mtmd_tokenizer { if (ctx->proj_type_v() == PROJECTOR_TYPE_GRANITE4_VISION) { if (batch_f32.entries.size() == 1) { // Single-tile (overview only): append one newline row. - batch_f32.entries[0]->append_token = true; + batch_f32.entries[0]->add_newline = true; } else { // Multi-tile: overview gets no newline, grid tiles get one. - batch_f32.entries[0]->append_token = false; + batch_f32.entries[0]->add_newline = false; for (size_t i = 1; i < batch_f32.entries.size(); ++i) { - batch_f32.entries[i]->append_token = true; + batch_f32.entries[i]->add_newline = true; } } } From d3d5a08a098e62f89202caea11ace41fa74d0ce3 Mon Sep 17 00:00:00 2001 From: Gabe Goodhart Date: Thu, 4 Jun 2026 16:22:48 -0600 Subject: [PATCH 93/94] style: Comment cleanup Branch: Granite4Vision AI-usage: none Signed-off-by: Gabe Goodhart --- tools/mtmd/clip-model.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/mtmd/clip-model.h b/tools/mtmd/clip-model.h index 12a59b13fd07..48796b6306fa 100644 --- a/tools/mtmd/clip-model.h +++ b/tools/mtmd/clip-model.h @@ -91,7 +91,7 @@ struct clip_hparams { float eps = 1e-6; float rope_theta = 0.0; - std::vector vision_feature_layer; // Ordered list of vision layers where features are extracted + std::vector vision_feature_layer; int32_t attn_window_size = 0; int32_t n_wa_pattern = 0; std::unordered_set wa_layer_indexes; // explicit layer indexes that use full attention (for irregular patterns like YoutuVL) @@ -103,7 +103,7 @@ struct clip_hparams { int32_t sam_n_embd = 0; // Granite4 Vision - std::vector proj_spatial_offsets; // Ordered list of per-projector spatial offsets + std::vector proj_spatial_offsets; int32_t downsample_query_side; int32_t downsample_window_side; From b20098441e64c4233f9348bce970589d0fbb0c10 Mon Sep 17 00:00:00 2001 From: Gabe Goodhart Date: Thu, 4 Jun 2026 16:23:52 -0600 Subject: [PATCH 94/94] feat: Cleaner error handling/checking NOTE: format_string is not available in granite.cpp (and including clip-impl.h to get it doesn't compile, so I think it violates the intended encapsulation), so std::stringstream is the simplest answer. Branch: Granite4Vision AI-usage: none Signed-off-by: Gabe Goodhart --- src/models/granite.cpp | 11 +++++++++++ tools/mtmd/clip.cpp | 5 ++++- 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/src/models/granite.cpp b/src/models/granite.cpp index b8641edb1575..61e3a79e9cf0 100644 --- a/src/models/granite.cpp +++ b/src/models/granite.cpp @@ -1,5 +1,7 @@ #include "models.h" +#include + void llama_model_granite::load_arch_hparams(llama_model_loader & ml) { ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, hparams.f_norm_rms_eps); ml.get_key(LLM_KV_LOGIT_SCALE, hparams.f_logit_scale); @@ -19,6 +21,15 @@ void llama_model_granite::load_arch_hparams(llama_model_loader & ml) { } hparams.n_deepstack_layers = unique_deepstack_idxs.size(); + // Ensure all values are valid (avoid overflow attacks) + for (const auto val : unique_deepstack_idxs) { + if (val > hparams.n_deepstack_layers) { + std::stringstream ss; + ss << "Invalid deepstack index: " << val << " > " << hparams.n_deepstack_layers; + throw std::runtime_error(ss.str()); + } + } + // Granite uses rope_finetuned as a switch for rope, so default to true bool rope_finetuned = true; ml.get_key(LLM_KV_ROPE_SCALING_FINETUNED, rope_finetuned, false); diff --git a/tools/mtmd/clip.cpp b/tools/mtmd/clip.cpp index bda95fc22250..c12c910a1c80 100644 --- a/tools/mtmd/clip.cpp +++ b/tools/mtmd/clip.cpp @@ -1634,7 +1634,10 @@ struct clip_model_loader { get_arr_int(KEY_FEATURE_LAYER, hparams.vision_feature_layer); get_arr_int(KEY_PROJ_SPATIAL_OFFSETS, hparams.proj_spatial_offsets); - GGML_ASSERT(hparams.vision_feature_layer.size() == hparams.proj_spatial_offsets.size()); + if (hparams.vision_feature_layer.size() != hparams.proj_spatial_offsets.size()) { + throw std::runtime_error(string_format("%s: vision_feature_layer.size() %d != proj_spatial_offsets.size() %d", + hparams.vision_feature_layer.size(), hparams.proj_spatial_offsets.size())); + } get_u32(KEY_PROJ_SAMPLE_QUERY_SIDE, hparams.downsample_query_side); get_u32(KEY_PROJ_SAMPLE_WINDOW_SIDE, hparams.downsample_window_side);