fix(template): align multimodal fields and handle text-only inputs

swift/template/base.py

@@ -1497,6 +1497,13 @@ def _truncate(self, input_ids: List[int], labels: Optional[List[int]], encoded,
             loss_scale = torch.tensor(loss_scale)[protected].tolist()
             loss_scale[0] = 0
             encoded['loss_scale'] = loss_scale
+        token_type_ids = encoded.get('token_type_ids')
+        if token_type_ids is not None:
+            if isinstance(token_type_ids, torch.Tensor):
+                # ERNIE-VL keeps a leading batch dimension: [1, seq_len].
+                encoded['token_type_ids'] = token_type_ids[..., protected]
+            else:
+                encoded['token_type_ids'] = torch.tensor(token_type_ids)[protected].tolist()
         for key in ('mm_token_type_ids', 'image_token_types'):
             token_types = encoded.get(key)
             if token_types is not None:

swift/template/templates/gemma.py

@@ -5,7 +5,7 @@
 from dataclasses import dataclass, field
 from typing import Any, Dict, List, Literal, Optional
 
-from swift.utils import get_logger, upper_bound
+from swift.utils import get_logger, lower_bound
 from ..base import Template
 from ..constant import LLMTemplateType, MLLMTemplateType
 from ..register import TemplateMeta, register_template
@@ -48,7 +48,7 @@ def _encode(self, inputs: StdTemplateInputs) -> Dict[str, Any]:
         raw_image = inputs.images
         processor = self.processor
         if encoded['labels'] is not None:
-            n = upper_bound(0, len(encoded['labels']), lambda idx: encoded['labels'][idx] == -100)
+            n = lower_bound(0, len(encoded['labels']), lambda idx: encoded['labels'][idx] != -100)
             n2 = len(encoded['labels']) - n
             encoded['token_type_ids'] = [0] * n + [1] * n2
         else:

swift/template/templates/glm.py

@@ -577,6 +577,9 @@ def _encode(self, inputs: StdTemplateInputs) -> Dict[str, Any]:
         encoded['input_ids'] = input_ids[:1] + [self.processor.pad_token_id] * image_token_len + input_ids[1:]
         if labels is not None:
             encoded['labels'] = labels[:1] + [-100] * image_token_len + labels[1:]
+        loss_scale = encoded.get('loss_scale')
+        if loss_scale is not None:
+            encoded['loss_scale'] = loss_scale[:1] + [0.] * image_token_len + loss_scale[1:]
         if len(image) > 0:
             encoded['images'] = [[img.to(dtype=self.model_info.torch_dtype)] for img in inputs2['images']]
             if 'cross_images' in inputs2:
@@ -586,11 +589,13 @@ def _encode(self, inputs: StdTemplateInputs) -> Dict[str, Any]:
         return encoded
 
     def _data_collator(self, batch: List[Dict[str, Any]], *, padding_to: Optional[int] = None) -> Dict[str, Any]:
+        if any(b.get('cross_images') for b in batch) and not all(b.get('cross_images') for b in batch):
+            raise ValueError('CogAgent requires an image for every sample in a multimodal batch.')
         res = super()._data_collator(batch, padding_to=padding_to)
         keys = ['images', 'cross_images']
         for key in keys:
-            if key in batch[0]:
-                res[key] = [b[key][0] for b in batch]
+            if any(b.get(key) for b in batch):
+                res[key] = [b[key][0] if b.get(key) else [] for b in batch]
         return res
 
 
@@ -648,6 +653,9 @@ def _encode(self, inputs: StdTemplateInputs) -> Dict[str, Any]:
         encoded['input_ids'] = input_ids[:1] + [self.processor.pad_token_id] * video_token_len + input_ids[1:]
         if labels is not None:
             encoded['labels'] = labels[:1] + [-100] * video_token_len + labels[1:]
+        loss_scale = encoded.get('loss_scale')
+        if loss_scale is not None:
+            encoded['loss_scale'] = loss_scale[:1] + [0.] * video_token_len + loss_scale[1:]
         if len(video) > 0:
             dtype = model.dtype
             encoded['images'] = [[img.to(dtype=dtype)] for img in inputs2['images']]

swift/template/templates/seed.py

@@ -199,11 +199,11 @@ def _encode(self, inputs: StdTemplateInputs) -> Dict[str, Any]:
         input_ids = encoded['input_ids']
         idx_list = findall(input_ids, -100)
         pixel_values = None
+        labels = encoded.get('labels')
         loss_scale = encoded.get('loss_scale', None)
         images = inputs.images
         processor = self.processor
         if images:
-            labels = encoded.get('labels')
             image_inputs = processor.image_processor(images)
             num_patches = image_inputs['num_patches_list']
             pixel_values = image_inputs['pixel_values']
@@ -227,10 +227,10 @@ def _post_encode(self, model: nn.Module, inputs: Dict[str, Any]) -> Dict[str, An
         embedding = model.language_model.get_input_embeddings()
         device = embedding.weight.device
         input_ids = inputs['input_ids']
+        inputs_embeds = embedding(input_ids).to(device=device)
         pixel_values = inputs.get('pixel_values')
         if pixel_values is not None:
             vit_embeds = model.extract_feature(pixel_values)
-            inputs_embeds = embedding(input_ids)
             B, N, C = inputs_embeds.shape
             inputs_embeds = inputs_embeds.reshape(B * N, C)
 
@@ -242,7 +242,6 @@ def _post_encode(self, model: nn.Module, inputs: Dict[str, Any]) -> Dict[str, An
 
             inputs_embeds = inputs_embeds.reshape(B, N, C)
         elif is_deepspeed_enabled():
-            inputs_embeds = embedding(input_ids).to(device=device)
             dummy_pixel_values = torch.zeros((1, 3, 32, 32), device=device, dtype=inputs_embeds.dtype)
             vit_embeds = model.extract_feature(dummy_pixel_values).to(device=device)
             inputs_embeds = inputs_embeds + vit_embeds.mean() * 0.