Fix.

modeling_vila.py

@@ -428,6 +428,8 @@ class VILAPretrainedModel(PreTrainedModel):
         # print("DEBUG", len(self.tokenizer.added_tokens_encoder.keys()), self.tokenizer.added_tokens_encoder.keys())
         NUM_EXTRA_TOKENS = len(self.tokenizer.added_tokens_encoder.keys())
 
+        self.skip_pad_tokens = True
+
         # TODO: SENTINEL_TOKEN is not added, need to check with Zhijian
         self.vocab_size = self.tokenizer.vocab_size + NUM_EXTRA_TOKENS
         # XGrammar tokenizer and grammar compiler
@@ -651,7 +653,7 @@ class VILAForCasualLM(VILAPretrainedModel):
                     input = media_embeds[name].popleft()
                     label = torch.full([input.shape[0]], IGNORE_INDEX, device=labels[k].device, dtype=labels[k].dtype)
                     # print(f"{self.tokenizer.padding_side} [media] {k=} {pos=}, {self.tokenizer.batch_decode(input_ids[k][pos:pos+1])}"); python_input()
-                elif input_ids[k][pos].item() in (self.tokenizer.pad_token_id, self.tokenizer.eos_token_id):
+                elif self.skip_pad_tokens and input_ids[k][pos].item() in (self.tokenizer.pad_token_id, self.tokenizer.eos_token_id):
                     end = pos + 1
                     pos = end
                     # print(f"[skip PAD/EOS] {k=} {pos=}, {self.tokenizer.batch_decode(input_ids[k][pos:end])}"); python_input()