granite3.3

.ipynb_checkpoints/config-checkpoint.json

@@ -0,0 +1,178 @@
+{
+  "_name_or_path": "ibm-granite/granite-vision-3.3-2b",
+  "adapter_path": null,
+  "auto_map": {
+        "AutoModel": "modeling_colgranitevision.ColGraniteVision",
+        "AutoProcessor": "processing_colgranitevision.ColGraniteVisionProcessor",
+        "AutoConfig":    "colgranitevision_config.ColGraniteVisionConfig"
+    },
+  "architectures": [
+    "ColGraniteVision"
+  ],
+  "base_model": null,
+  "emb_dim_doc": 128,
+  "emb_dim_query": 128,
+  "image_grid_pinpoints": [
+    [
+      384,
+      768
+    ],
+    [
+      384,
+      1152
+    ],
+    [
+      384,
+      1536
+    ],
+    [
+      384,
+      1920
+    ],
+    [
+      384,
+      2304
+    ],
+    [
+      384,
+      2688
+    ],
+    [
+      384,
+      3072
+    ],
+    [
+      384,
+      3456
+    ],
+    [
+      384,
+      3840
+    ],
+    [
+      768,
+      384
+    ],
+    [
+      768,
+      768
+    ],
+    [
+      768,
+      1152
+    ],
+    [
+      768,
+      1536
+    ],
+    [
+      768,
+      1920
+    ],
+    [
+      1152,
+      384
+    ],
+    [
+      1152,
+      768
+    ],
+    [
+      1152,
+      1152
+    ],
+    [
+      1536,
+      384
+    ],
+    [
+      1536,
+      768
+    ],
+    [
+      1920,
+      384
+    ],
+    [
+      1920,
+      768
+    ],
+    [
+      2304,
+      384
+    ],
+    [
+      2688,
+      384
+    ],
+    [
+      3072,
+      384
+    ],
+    [
+      3456,
+      384
+    ],
+    [
+      3840,
+      384
+    ]
+  ],
+  "image_seq_length": 576,
+  "image_token_index": 49155,
+  "model_type": "colgranitevision",
+  "multimodal_projector_bias": true,
+  "pretrained_language_model": "",
+  "pretrained_vision_tower": "",
+  "projector_hidden_act": "gelu",
+  "text_config": {
+    "_attn_implementation_autoset": true,
+    "_name_or_path": "ibm-granite/granite-3.1-2b-instruct",
+    "architectures": [
+      "GraniteForCausalLM"
+    ],
+    "attention_dropout": 0.1,
+    "attention_multiplier": 0.015625,
+    "bos_token_id": 0,
+    "embedding_multiplier": 12.0,
+    "eos_token_id": 0,
+    "hidden_size": 2048,
+    "intermediate_size": 8192,
+    "logits_scaling": 8.0,
+    "max_position_embeddings": 131072,
+    "model_type": "granite",
+    "num_hidden_layers": 40,
+    "num_key_value_heads": 8,
+    "pad_token_id": 0,
+    "residual_multiplier": 0.22,
+    "rms_norm_eps": 1e-05,
+    "rope_theta": 300000,
+    "tie_word_embeddings": true,
+    "torch_dtype": "float32",
+    "vocab_size": 49156
+  },
+  "tie_word_embeddings": true,
+  "torch_dtype": "float32",
+  "transformers_version": "4.50.0.dev0",
+  "use_image_newline_parameter": true,
+  "vision_config": {
+    "_attn_implementation_autoset": true,
+    "hidden_act": "gelu_pytorch_tanh",
+    "hidden_size": 1152,
+    "image_size": 384,
+    "intermediate_size": 4304,
+    "layer_norm_eps": 1e-06,
+    "model_type": "siglip_vision_model",
+    "num_attention_heads": 16,
+    "num_hidden_layers": 27,
+    "patch_size": 14,
+    "torch_dtype": "float32"
+  },
+  "vision_feature_layer": [
+    -24,
+    -20,
+    -12,
+    -1
+  ],
+  "vision_feature_select_strategy": "full"
+}

.ipynb_checkpoints/modeling_colgranitevision-checkpoint.py

@@ -0,0 +1,192 @@
+from typing import ClassVar, Optional
+
+import numpy as np
+import torch
+from torch import nn
+from transformers import LlavaNextPreTrainedModel
+from transformers.models.llava_next.modeling_llava_next import LlavaNextForConditionalGeneration
+from transformers.models.llava_next.modeling_llava_next import unpad_image, get_anyres_image_grid_shape
+
+from .colgranitevision_config import ColGraniteVisionConfig
+
+
+class LlavaNextWithCustomPacking(LlavaNextForConditionalGeneration):
+    def pack_image_features(
+            self,
+            image_features,
+            image_sizes,
+            vision_feature_select_strategy,
+            image_newline=None,
+            base_image_feature_location="last",
+    ):
+        """
+        Reshape, unpad and then pack each image_feature into a single image_features tensor containing all visual vectors.
+
+        Args:
+            image_features (`List[torch.Tensor]` of length num_images, each of shape `(num_patches, image_length, embed_dim)`)
+                List of image feature tensor, each contains all the visual feature of all patches.
+            image_sizes (`torch.Tensor` of shape `(num_images, 2)`)
+                Actual image size of each images (H, W).
+            vision_feature_select_strategy (`str`)
+                The feature selection strategy used to select the vision feature from the vision backbone.
+            image_newline (`torch.Tensor` of shape `(embed_dim)`)
+                New line embedding vector.
+        Returns:
+            image_features (`torch.Tensor` of shape `(all_feat_len, embed_dim)`)
+            feature_lens (`List[int]`)
+                token length of each image in image_features
+        """
+
+        new_image_features = []
+        feature_lens = []
+        for image_idx, image_feature in enumerate(image_features):
+            if image_feature.shape[0] > 1:
+                base_image_feature = image_feature[0]
+                image_feature = image_feature[1:]
+                height = width = self.config.vision_config.image_size // self.config.vision_config.patch_size
+
+                num_patch_height, num_patch_width = get_anyres_image_grid_shape(
+                    image_sizes[image_idx],
+                    self.config.image_grid_pinpoints,
+                    self.config.vision_config.image_size,
+                )
+
+                if (
+                        np.prod(image_feature.shape) % (num_patch_height * num_patch_width * height * width) != 0
+                        and vision_feature_select_strategy == "default"
+                ):
+                    print(
+                        "Image feature shape does not line up with the provided patch size. "
+                        "You may be using the `default` vision_feature_select_strategy with a"
+                        " visual encoder that does not have CLS."
+                    )
+
+                image_feature = image_feature.view(num_patch_height, num_patch_width, height, width, -1)
+                image_feature = image_feature.permute(4, 0, 2, 1, 3).contiguous()
+                image_feature = image_feature.flatten(1, 2).flatten(2, 3)
+                image_feature = unpad_image(image_feature, image_sizes[image_idx])
+                if image_newline is not None:
+                    image_feature = torch.cat(
+                        (
+                            image_feature,
+                            image_newline[:, None, None]
+                            .expand(*image_feature.shape[:-1], 1)
+                            .to(image_feature.device, image_feature.dtype),
+                        ),
+                        dim=-1,
+                    )
+                image_feature = image_feature.flatten(1, 2).transpose(0, 1)
+                if base_image_feature_location == "last":
+                    image_feature = torch.cat((image_feature, base_image_feature), dim=0)
+                else:
+                    image_feature = torch.cat((base_image_feature, image_feature), dim=0)
+
+            else:
+                image_feature = image_feature[0]
+                if image_newline is not None:
+                    image_feature = torch.cat((image_feature, image_newline[None].to(image_feature)), dim=0)
+            new_image_features.append(image_feature)
+            feature_lens.append(image_feature.size(0))
+        image_features = torch.cat(new_image_features, dim=0)
+        feature_lens = torch.tensor(feature_lens, dtype=torch.long, device=image_features.device)
+        return image_features, feature_lens
+
+
+class ColGraniteVision(LlavaNextPreTrainedModel):
+    """
+    ColGraniteVision model implementation.
+    """
+
+    main_input_name: ClassVar[str] = "doc_input_ids"  # transformers-related
+    config_class = ColGraniteVisionConfig
+
+    def __init__(self, config: ColGraniteVisionConfig):
+        super().__init__(config=config)
+
+        model = LlavaNextWithCustomPacking(config=config)
+        if model.language_model._tied_weights_keys is not None:
+            self._tied_weights_keys = [f"model.language_model.{k}" for k in model.language_model._tied_weights_keys]
+        self.model = model
+
+        # TODO: Wait for ColPali2 to create a ColPaliConfig to allow specifying the embedding dimension.
+        # We could do it now but it would break all the models trying to load the model from the checkpoint.
+        self.dim = 128
+        self.custom_text_proj = nn.Linear(self.model.config.text_config.hidden_size, self.dim)
+
+        self.post_init()
+
+    def forward(self, *args, **kwargs) -> torch.Tensor:
+        # Delete output_hidden_states from kwargs
+        kwargs.pop("output_hidden_states", None)
+        if "pixel_values" in kwargs:
+            kwargs["pixel_values"] = kwargs["pixel_values"].to(dtype=self.dtype)
+
+        outputs = self.model(*args, output_hidden_states=True, **kwargs)  # (batch_size, sequence_length, hidden_size)
+        last_hidden_states = outputs.hidden_states[-1]  # (batch_size, sequence_length, hidden_size)
+
+        attention_mask = kwargs["attention_mask"]
+        if "pixel_values" in kwargs:
+            input_ids = kwargs['input_ids']
+            image_mask = (input_ids == self.config.image_token_index)
+            # inputs_embeds = last_hidden_states.masked_scatter(image_mask)
+            N, M = image_mask.shape
+            # Create an index matrix: each row is 0, 1, ..., M-1
+            idx = torch.arange(M, device=image_mask.device).expand(N, M)
+            # Replace False positions with -1 so they are ignored by topk (since all valid indices are >=0)
+            masked_idx = torch.where(image_mask, idx, torch.tensor(-1, device=image_mask.device))
+            topk_values, _ = torch.topk(masked_idx, k=729, dim=1)
+            last_k_indices, _ = torch.sort(topk_values, dim=1)
+            last_k_indices_exp = last_k_indices.unsqueeze(-1).expand(-1, -1, last_hidden_states.size(-1))
+            last_hidden_states = torch.gather(last_hidden_states, 1, last_k_indices_exp)
+            attention_mask = torch.gather(attention_mask, 1, last_k_indices)
+
+        attention_mask = attention_mask.unsqueeze(-1)
+
+        proj = self.custom_text_proj(last_hidden_states)  # (batch_size, sequence_length, dim)
+
+        # L2 normalization
+        proj = proj / (proj.norm(dim=-1, keepdim=True) + 1e-8)
+
+        # proj = proj * kwargs["attention_mask"].unsqueeze(-1)  # (batch_size, sequence_length, dim)
+        proj = proj * attention_mask  # (batch_size, sequence_length, dim)
+
+        return proj
+
+    def get_input_embeddings(self):
+        return self.model.language_model.get_input_embeddings()
+
+    def set_input_embeddings(self, value):
+        self.model.language_model.set_input_embeddings(value)
+
+    def get_output_embeddings(self):
+        return self.model.language_model.get_output_embeddings()
+
+    def set_output_embeddings(self, new_embeddings):
+        self.model.language_model.set_output_embeddings(new_embeddings)
+
+    def set_decoder(self, decoder):
+        self.model.language_model.set_decoder(decoder)
+
+    def get_decoder(self):
+        return self.model.language_model.get_decoder()
+
+    def tie_weights(self):
+        return self.model.language_model.tie_weights()
+
+    def resize_token_embeddings(
+            self,
+            new_num_tokens: Optional[int] = None,
+            pad_to_multiple_of=None,
+    ) -> nn.Embedding:
+        model_embeds = self.model.language_model.resize_token_embeddings(new_num_tokens, pad_to_multiple_of)
+
+        # Update vocab size
+        self.config.text_config.vocab_size = model_embeds.num_embeddings
+        self.config.vocab_size = model_embeds.num_embeddings
+        self.model.vocab_size = model_embeds.num_embeddings
+
+        return model_embeds
+
+    @property
+    def patch_size(self) -> int:
+        return self.model.vision_tower.config.patch_size

added_tokens.json

@@ -1,6 +1,6 @@
-{
-  "<image>": 49155,
-  "<|end_of_role|>": 49153,
-  "<|start_of_role|>": 49152,
-  "<|tool_call|>": 49154
-}
+{
+  "<image>": 49155,
+  "<|end_of_role|>": 49153,
+  "<|start_of_role|>": 49152,
+  "<|tool_call|>": 49154
+}

config.json

@@ -1,11 +1,11 @@
 {
-  "_name_or_path": "ibm-granite/granite-vision-3.1-2b-preview",
-  "_class_name": "ColGraniteVisionConfig",
+  "_name_or_path": "ibm-granite/granite-vision-3.3-2b",
+  "adapter_path": null,
   "auto_map": {
-    "AutoModel": "modeling_colgranitevision.ColGraniteVision",
-    "AutoProcessor": "processing_colgranitevision.ColGraniteVisionProcessor",
-    "AutoConfig":    "colgranitevision_config.ColGraniteVisionConfig"
-  },
+        "AutoModel": "modeling_colgranitevision.ColGraniteVision",
+        "AutoProcessor": "processing_colgranitevision.ColGraniteVisionProcessor",
+        "AutoConfig":    "colgranitevision_config.ColGraniteVisionConfig"
+    },
   "architectures": [
     "ColGraniteVision"
   ],
@@ -122,29 +122,33 @@
   "image_token_index": 49155,
   "model_type": "colgranitevision",
   "multimodal_projector_bias": true,
+  "pretrained_language_model": "",
+  "pretrained_vision_tower": "",
   "projector_hidden_act": "gelu",
   "text_config": {
+    "_attn_implementation_autoset": true,
+    "_name_or_path": "ibm-granite/granite-3.1-2b-instruct",
     "architectures": [
       "GraniteForCausalLM"
     ],
     "attention_dropout": 0.1,
     "attention_multiplier": 0.015625,
     "bos_token_id": 0,
-    "embedding_multiplier": 12,
+    "embedding_multiplier": 12.0,
     "eos_token_id": 0,
     "hidden_size": 2048,
     "intermediate_size": 8192,
-    "logits_scaling": 8,
-    "max_position_embeddings": 16384,
+    "logits_scaling": 8.0,
+    "max_position_embeddings": 131072,
     "model_type": "granite",
     "num_hidden_layers": 40,
     "num_key_value_heads": 8,
     "pad_token_id": 0,
     "residual_multiplier": 0.22,
-    "rms_norm_eps": 0.00001,
+    "rms_norm_eps": 1e-05,
     "rope_theta": 300000,
     "tie_word_embeddings": true,
-    "torch_dtype": "bfloat16",
+    "torch_dtype": "float32",
     "vocab_size": 49156
   },
   "tie_word_embeddings": true,
@@ -152,15 +156,17 @@
   "transformers_version": "4.50.0.dev0",
   "use_image_newline_parameter": true,
   "vision_config": {
+    "_attn_implementation_autoset": true,
     "hidden_act": "gelu_pytorch_tanh",
     "hidden_size": 1152,
     "image_size": 384,
     "intermediate_size": 4304,
-    "layer_norm_eps": 0.000001,
+    "layer_norm_eps": 1e-06,
     "model_type": "siglip_vision_model",
     "num_attention_heads": 16,
     "num_hidden_layers": 27,
-    "patch_size": 14
+    "patch_size": 14,
+    "torch_dtype": "float32"
   },
   "vision_feature_layer": [
     -24,
@@ -169,4 +175,4 @@
     -1
   ],
   "vision_feature_select_strategy": "full"
-}
+}

model-00001-of-00003.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:ec8a694db663b30616ff06812d60256bb474c52051df2003faaec47c42b9a556
+oid sha256:4e838b6d98f48fbf45ae6c0d9c74cba649fd06b27ed78ced3971efbab7e16a69
 size 4955415688

model-00002-of-00003.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:9df92d92a0d79465e4ee5eb57a51ee1630b159dc5833e26af9ca7bc9b3788d24
+oid sha256:d6bf1675fc15977b4d8f37ea1d4960ca2750e6793a80da9771e4693ae8cb13d6
 size 4999979448

model-00003-of-00003.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:42ebe0fe87507de69b86074af24513756fbcc205e83ccb2ee7bbe9238a751f29
+oid sha256:15978cba0606360676faad5c3cf486a58e6d78a1352dbfcd1db51a7410a574d5
 size 1947355456

preprocessor_config.json

@@ -1,136 +1,137 @@
-{
-  "crop_size": {
-    "height": 384,
-    "width": 384
-  },
-  "do_center_crop": true,
-  "do_convert_rgb": null,
-  "do_normalize": true,
-  "do_pad": true,
-  "do_rescale": true,
-  "do_resize": true,
-  "image_grid_pinpoints": [
-    [
-      384,
-      768
-    ],
-    [
-      384,
-      1152
-    ],
-    [
-      384,
-      1536
-    ],
-    [
-      384,
-      1920
-    ],
-    [
-      384,
-      2304
-    ],
-    [
-      384,
-      2688
-    ],
-    [
-      384,
-      3072
-    ],
-    [
-      384,
-      3456
-    ],
-    [
-      384,
-      3840
-    ],
-    [
-      768,
-      384
-    ],
-    [
-      768,
-      768
-    ],
-    [
-      768,
-      1152
-    ],
-    [
-      768,
-      1536
-    ],
-    [
-      768,
-      1920
-    ],
-    [
-      1152,
-      384
-    ],
-    [
-      1152,
-      768
-    ],
-    [
-      1152,
-      1152
-    ],
-    [
-      1536,
-      384
-    ],
-    [
-      1536,
-      768
-    ],
-    [
-      1920,
-      384
-    ],
-    [
-      1920,
-      768
-    ],
-    [
-      2304,
-      384
-    ],
-    [
-      2688,
-      384
-    ],
-    [
-      3072,
-      384
-    ],
-    [
-      3456,
-      384
-    ],
-    [
-      3840,
-      384
-    ]
-  ],
-  "image_mean": [
-    0.5,
-    0.5,
-    0.5
-  ],
-  "image_processor_type": "LlavaNextImageProcessor",
-  "image_std": [
-    0.5,
-    0.5,
-    0.5
-  ],
-  "processor_class": "ColGraniteVisionProcessor",
-  "resample": 3,
-  "rescale_factor": 0.00392156862745098,
-  "size": {
-    "height": 384,
-    "width": 384
-  }
-}
+{
+  "crop_size": {
+    "height": 384,
+    "width": 384
+  },
+  "default_to_square": false,
+  "do_center_crop": true,
+  "do_convert_rgb": null,
+  "do_normalize": true,
+  "do_pad": true,
+  "do_rescale": true,
+  "do_resize": true,
+  "image_grid_pinpoints": [
+    [
+      384,
+      768
+    ],
+    [
+      384,
+      1152
+    ],
+    [
+      384,
+      1536
+    ],
+    [
+      384,
+      1920
+    ],
+    [
+      384,
+      2304
+    ],
+    [
+      384,
+      2688
+    ],
+    [
+      384,
+      3072
+    ],
+    [
+      384,
+      3456
+    ],
+    [
+      384,
+      3840
+    ],
+    [
+      768,
+      384
+    ],
+    [
+      768,
+      768
+    ],
+    [
+      768,
+      1152
+    ],
+    [
+      768,
+      1536
+    ],
+    [
+      768,
+      1920
+    ],
+    [
+      1152,
+      384
+    ],
+    [
+      1152,
+      768
+    ],
+    [
+      1152,
+      1152
+    ],
+    [
+      1536,
+      384
+    ],
+    [
+      1536,
+      768
+    ],
+    [
+      1920,
+      384
+    ],
+    [
+      1920,
+      768
+    ],
+    [
+      2304,
+      384
+    ],
+    [
+      2688,
+      384
+    ],
+    [
+      3072,
+      384
+    ],
+    [
+      3456,
+      384
+    ],
+    [
+      3840,
+      384
+    ]
+  ],
+  "image_mean": [
+    0.5,
+    0.5,
+    0.5
+  ],
+  "image_processor_type": "LlavaNextImageProcessor",
+  "image_std": [
+    0.5,
+    0.5,
+    0.5
+  ],
+  "processor_class": "ColGraniteVisionProcessor",
+  "resample": 3,
+  "rescale_factor": 0.00392156862745098,
+  "size": {
+    "height": 384,
+    "width": 384
+  }
+}

special_tokens_map.json

@@ -1,35 +1,35 @@
-{
-  "additional_special_tokens": [
-    "<|start_of_role|>",
-    "<|end_of_role|>",
-    "<|tool_call|>"
-  ],
-  "bos_token": {
-    "content": "<|end_of_text|>",
-    "lstrip": false,
-    "normalized": false,
-    "rstrip": false,
-    "single_word": false
-  },
-  "eos_token": {
-    "content": "<|end_of_text|>",
-    "lstrip": false,
-    "normalized": false,
-    "rstrip": false,
-    "single_word": false
-  },
-  "pad_token": {
-    "content": "<|end_of_text|>",
-    "lstrip": false,
-    "normalized": false,
-    "rstrip": false,
-    "single_word": false
-  },
-  "unk_token": {
-    "content": "<|end_of_text|>",
-    "lstrip": false,
-    "normalized": false,
-    "rstrip": false,
-    "single_word": false
-  }
-}
+{
+  "additional_special_tokens": [
+    "<|start_of_role|>",
+    "<|end_of_role|>",
+    "<|tool_call|>"
+  ],
+  "bos_token": {
+    "content": "<|end_of_text|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "<|end_of_text|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "<|end_of_text|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "unk_token": {
+    "content": "<|end_of_text|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

tokenizer_config.json

@@ -1,207 +1,208 @@
-{
-  "add_bos_token": false,
-  "add_prefix_space": false,
-  "added_tokens_decoder": {
-    "0": {
-      "content": "<|end_of_text|>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "1": {
-      "content": "<fim_prefix>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "2": {
-      "content": "<fim_middle>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "3": {
-      "content": "<fim_suffix>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "4": {
-      "content": "<fim_pad>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "5": {
-      "content": "<filename>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "6": {
-      "content": "<gh_stars>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "7": {
-      "content": "<issue_start>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "8": {
-      "content": "<issue_comment>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "9": {
-      "content": "<issue_closed>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "10": {
-      "content": "<jupyter_start>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "11": {
-      "content": "<jupyter_text>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "12": {
-      "content": "<jupyter_code>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "13": {
-      "content": "<jupyter_output>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "14": {
-      "content": "<empty_output>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "15": {
-      "content": "<commit_before>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "16": {
-      "content": "<commit_msg>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "17": {
-      "content": "<commit_after>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "18": {
-      "content": "<reponame>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "49152": {
-      "content": "<|start_of_role|>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "49153": {
-      "content": "<|end_of_role|>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "49154": {
-      "content": "<|tool_call|>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "49155": {
-      "content": "<image>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    }
-  },
-  "additional_special_tokens": [
-    "<|start_of_role|>",
-    "<|end_of_role|>",
-    "<|tool_call|>"
-  ],
-  "bos_token": "<|end_of_text|>",
-  "chat_template": "{%- if tools %}\n    {{- '<|start_of_role|>available_tools<|end_of_role|>\n' }}\n    {%- for tool in tools %}\n    {{- tool | tojson(indent=4) }}\n    {%- if not loop.last %}\n        {{- '\n\n' }}\n    {%- endif %}\n    {%- endfor %}\n    {{- '<|end_of_text|>\n' }}\n{%- endif %}\n{%- for message in messages if message['role'] == 'system'%}{% else %}<|system|>\nA chat between a curious user and an artificial intelligence assistant. The assistant gives helpful, detailed, and polite answers to the user's questions.\n{% endfor %}{%- for message in messages %}\n    {%- if message['role'] == 'system' %}\n    {{- '<|system|>\n' + message['content'] + '\n' }}\n    {%- elif message['role'] == 'user' %}\n    {{- '<|user|>\n' + message['content'] + '\n' }}\n    {%- elif message['role'] == 'assistant' %}\n    {{- '<|assistant|>\n'  + message['content'] + '<|end_of_text|>' }}\n    {%- elif message['role'] == 'assistant_tool_call' %}\n    {{- '<|start_of_role|>assistant<|end_of_role|><|tool_call|>' + message['content'] + '<|end_of_text|>\n' }}\n    {%- elif message['role'] == 'tool_response' %}\n    {{- '<|start_of_role|>tool_response<|end_of_role|>' + message['content'] + '<|end_of_text|>\n' }}\n    {%- endif %}\n    {%- if loop.last and add_generation_prompt %}\n    {{- '<|assistant|>\n' }}\n    {%- endif %}\n{%- endfor %}",
-  "clean_up_tokenization_spaces": true,
-  "eos_token": "<|end_of_text|>",
-  "errors": "replace",
-  "extra_special_tokens": {},
-  "model_max_length": 16384,
-  "pad_token": "<|end_of_text|>",
-  "padding_side": "right",
-  "tokenizer_class": "GPT2Tokenizer",
-  "unk_token": "<|end_of_text|>",
-  "vocab_size": 49152
+{
+  "add_bos_token": false,
+  "add_prefix_space": false,
+  "added_tokens_decoder": {
+    "0": {
+      "content": "<|end_of_text|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "1": {
+      "content": "<fim_prefix>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "2": {
+      "content": "<fim_middle>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "3": {
+      "content": "<fim_suffix>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "4": {
+      "content": "<fim_pad>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "5": {
+      "content": "<filename>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "6": {
+      "content": "<gh_stars>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "7": {
+      "content": "<issue_start>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "8": {
+      "content": "<issue_comment>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "9": {
+      "content": "<issue_closed>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "10": {
+      "content": "<jupyter_start>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "11": {
+      "content": "<jupyter_text>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "12": {
+      "content": "<jupyter_code>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "13": {
+      "content": "<jupyter_output>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "14": {
+      "content": "<empty_output>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "15": {
+      "content": "<commit_before>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "16": {
+      "content": "<commit_msg>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "17": {
+      "content": "<commit_after>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "18": {
+      "content": "<reponame>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "49152": {
+      "content": "<|start_of_role|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "49153": {
+      "content": "<|end_of_role|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "49154": {
+      "content": "<|tool_call|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "49155": {
+      "content": "<image>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "additional_special_tokens": [
+    "<|start_of_role|>",
+    "<|end_of_role|>",
+    "<|tool_call|>"
+  ],
+  "bos_token": "<|end_of_text|>",
+  "chat_template": "{%- if tools %}\n    {{- '<|start_of_role|>available_tools<|end_of_role|>\n' }}\n    {%- for tool in tools %}\n    {{- tool | tojson(indent=4) }}\n    {%- if not loop.last %}\n        {{- '\n\n' }}\n    {%- endif %}\n    {%- endfor %}\n    {{- '<|end_of_text|>\n' }}\n{%- endif %}\n{%- for message in messages if message['role'] == 'system'%}{% else %}<|system|>\nA chat between a curious user and an artificial intelligence assistant. The assistant gives helpful, detailed, and polite answers to the user's questions.\n{% endfor %}{%- for message in messages %}\n    {%- if message['role'] == 'system' %}\n    {{- '<|system|>\n' + message['content'] + '\n' }}\n    {%- elif message['role'] == 'user' %}\n    {{- '<|user|>\n' + message['content'] + '\n' }}\n    {%- elif message['role'] == 'assistant' %}\n    {{- '<|assistant|>\n'  + message['content'] + '<|end_of_text|>' }}\n    {%- elif message['role'] == 'assistant_tool_call' %}\n    {{- '<|start_of_role|>assistant<|end_of_role|><|tool_call|>' + message['content'] + '<|end_of_text|>\n' }}\n    {%- elif message['role'] == 'tool_response' %}\n    {{- '<|start_of_role|>tool_response<|end_of_role|>' + message['content'] + '<|end_of_text|>\n' }}\n    {%- endif %}\n    {%- if loop.last and add_generation_prompt %}\n    {{- '<|assistant|>\n' }}\n    {%- endif %}\n{%- endfor %}",
+  "clean_up_tokenization_spaces": true,
+  "do_image_splitting": false,
+  "eos_token": "<|end_of_text|>",
+  "errors": "replace",
+  "extra_special_tokens": {},
+  "model_max_length": 131072,
+  "pad_token": "<|end_of_text|>",
+  "padding_side": "right",
+  "tokenizer_class": "GPT2Tokenizer",
+  "unk_token": "<|end_of_text|>",
+  "vocab_size": 49152
 }