Upload 17 files

Vision_Project.py

@@ -0,0 +1,35 @@
+import math
+import re
+import torch
+import torch.nn as nn
+
+class IdentityMap(nn.Module):
+
+    def __init__(self):
+        super().__init__()
+
+    def forward(self, x, *args, **kwargs):
+        return x
+
+    @property
+    def config(self):
+        return {'mm_projector_type': 'identity'}
+
+def mlp2x_gelu(projector_type):
+    # mm_hidden_size = 1024
+    mm_hidden_size = 1280
+    hidden_size = 3584
+
+    mlp_gelu_match = re.match(r'^mlp(\d+)x_gelu$', projector_type)
+    if mlp_gelu_match:
+        mlp_depth = int(mlp_gelu_match.group(1))
+        modules = [nn.Linear(mm_hidden_size, hidden_size)]
+        for _ in range(1, mlp_depth):
+            modules.append(nn.GELU())
+            modules.append(nn.Linear(hidden_size, hidden_size))
+        return nn.Sequential(*modules)
+
+    if projector_type == 'identity':
+        return IdentityMap()
+
+    raise ValueError(f'Unknown projector type: {projector_type}')

Vision_Tower.py

@@ -0,0 +1,169 @@
+import torch
+import torch.nn as nn
+from transformers import CLIPVisionModel
+
+
+class clip_vit_large_patch14_336(nn.Module):
+
+    def __init__(self, vision_tower, use_resize_pos=True):
+        super().__init__()
+
+        self.is_loaded = False
+        self.is_resize_pos = False
+
+        self.vision_tower_name = vision_tower
+        self.select_layer = -1
+        self.select_feature = 'patch'
+        self.load_model()
+
+        #change model to input shape[490*490]
+        if use_resize_pos:
+            self.resize_pos()
+
+    def load_model(self):
+        self.vision_tower = CLIPVisionModel.from_pretrained(
+            self.vision_tower_name)
+        self.vision_tower.requires_grad_(False)
+
+        self.is_loaded = True
+
+    def resize_pos(self):
+        pos_embed_checkpoint = self.vision_tower.vision_model.embeddings.position_embedding.weight
+        pos_embed_checkpoint = pos_embed_checkpoint.unsqueeze(0)
+        orig_size = 24 #336/14
+        new_size = 35 #490/14
+
+        if pos_embed_checkpoint.shape[1] == new_size**2 + 1:
+            self.is_resize_pos = True
+        else:
+            embedding_size = pos_embed_checkpoint.shape[-1]
+            num_extra_tokens = 1
+            new_num = new_size**2 + num_extra_tokens
+            #print('Position interpolate from %dx%d to %dx%d' %
+            #      (orig_size, orig_size, new_size, new_size))
+            extra_tokens = pos_embed_checkpoint[:, :num_extra_tokens]
+            # only the position tokens are interpolated
+            pos_tokens = pos_embed_checkpoint[:, num_extra_tokens:]
+            pos_tokens = pos_tokens.reshape(-1, orig_size, orig_size,
+                                            embedding_size).permute(
+                                                0, 3, 1, 2)
+            pos_tokens = torch.nn.functional.interpolate(
+                pos_tokens,
+                size=(new_size, new_size),
+                mode='bicubic',
+                align_corners=False)
+            pos_tokens = pos_tokens.permute(0, 2, 3, 1).flatten(1, 2)
+            new_pos_embed = torch.cat((extra_tokens, pos_tokens), dim=1)
+
+            new_pos_embed = new_pos_embed.squeeze(0)
+
+            self.vision_tower.vision_model.embeddings.position_embedding = torch.nn.Embedding(
+                new_num, 1024)
+            self.vision_tower.vision_model.embeddings.position_embedding.weight = torch.nn.Parameter(
+                new_pos_embed.to(pos_embed_checkpoint.dtype))
+            self.vision_tower.vision_model.embeddings.position_ids = torch.arange(
+                new_num).expand((1, -1))
+
+            self.is_resize_pos = True
+
+    def feature_select(self, image_forward_outs):
+        image_features = image_forward_outs.hidden_states[self.select_layer]
+        if self.select_feature == 'patch':
+            image_features = image_features[:, 1:]
+        elif self.select_feature == 'cls_patch':
+            image_features = image_features
+        else:
+            raise ValueError(
+                f'Unexpected select feature: {self.select_feature}')
+        return image_features
+
+    def forward(self, images):
+        if not self.is_loaded:
+            self.load_model()
+        if type(images) is list: # not batch infurence speed!
+            image_features = []
+            for image in images:
+                image_forward_out = self.vision_tower(
+                    image.to(device=self.device,
+                             dtype=self.dtype).unsqueeze(0),
+                    output_hidden_states=True)
+                image_feature = self.feature_select(image_forward_out).to(
+                    image.dtype)
+                image_features.append(image_feature)
+        else:
+            image_forward_outs = self.vision_tower(
+                images.to(device=self.device, dtype=self.dtype),
+                output_hidden_states=True)
+            image_features = self.feature_select(image_forward_outs).to(
+                images.dtype)
+
+        return image_features
+
+    @property
+    def device(self):
+        return self.vision_tower.device
+    
+    @property
+    def dtype(self):
+        return self.vision_tower.dtype
+
+class DFN5B_CLIP_ViT_H_14_378(nn.Module):
+
+    def __init__(self, vision_tower):
+        super().__init__()
+
+        self.is_loaded = False
+        self.is_resize_pos = False
+
+        self.vision_tower_name = vision_tower
+        self.select_layer = -1
+        self.select_feature = 'patch'
+        self.load_model()
+
+    def load_model(self):
+        self.vision_tower = CLIPVisionModel.from_pretrained(
+            self.vision_tower_name)
+        self.vision_tower.requires_grad_(False)
+
+        self.is_loaded = True
+
+    def feature_select(self, image_forward_outs):
+        image_features = image_forward_outs.hidden_states[self.select_layer]
+        if self.select_feature == 'patch':
+            image_features = image_features[:, 1:]
+        elif self.select_feature == 'cls_patch':
+            image_features = image_features
+        else:
+            raise ValueError(
+                f'Unexpected select feature: {self.select_feature}')
+        return image_features
+
+    def forward(self, images):
+        if not self.is_loaded:
+            self.load_model()
+        if type(images) is list: # not batch infurence speed!
+            image_features = []
+            for image in images:
+                image_forward_out = self.vision_tower(
+                    image.to(device=self.device,
+                             dtype=self.dtype).unsqueeze(0),
+                    output_hidden_states=True)
+                image_feature = self.feature_select(image_forward_out).to(
+                    image.dtype)
+                image_features.append(image_feature)
+        else:
+            image_forward_outs = self.vision_tower(
+                images.to(device=self.device, dtype=self.dtype),
+                output_hidden_states=True)
+            image_features = self.feature_select(image_forward_outs).to(
+                images.dtype)
+
+        return image_features
+
+    @property
+    def device(self):
+        return self.vision_tower.device
+    
+    @property
+    def dtype(self):
+        return self.vision_tower.dtype

added_tokens.json

@@ -0,0 +1,5 @@
+{
+  "<|endoftext|>": 151643,
+  "<|im_end|>": 151645,
+  "<|im_start|>": 151644
+}

config.json

@@ -0,0 +1,44 @@
+{
+  "_name_or_path": "/root/lwt/tech/mcmd",
+  "architectures": [
+    "mcmdForCausalLM"
+  ],
+  "auto_map": {
+    "AutoConfig": "configuration_mcmd.mcmdConfig",
+    "AutoModel": "modeling_mcmd.mcmdForCausalLM",
+    "AutoModelForCausalLM": "modeling_mcmd.mcmdForCausalLM"
+  },
+  "clip_path": "/root/LWT/Models/DFN5B-CLIP-ViT-H-14-378",
+  "input_img_size": 378,
+  "lm_model": {
+    "attention_dropout": 0.0,
+    "bos_token_id": 151643,
+    "eos_token_id": 151645,
+    "hidden_act": "silu",
+    "hidden_size": 3584,
+    "initializer_range": 0.02,
+    "intermediate_size": 18944,
+    "max_position_embeddings": 32768,
+    "max_window_layers": 28,
+    "model_type": "qwen2",
+    "num_attention_heads": 28,
+    "num_hidden_layers": 28,
+    "num_key_value_heads": 4,
+    "rms_norm_eps": 1e-06,
+    "rope_theta": 1000000.0,
+    "sliding_window": 131072,
+    "tie_word_embeddings": false,
+    "torch_dtype": "bfloat16",
+    "transformers_version": "4.41.2",
+    "use_cache": true,
+    "use_sliding_window": false,
+    "vocab_size": 152064
+  },
+  "lm_path": "/root/LWT/Models/Qwen2-7B-Instruct",
+  "max_length": 4096,
+  "model_type": "mcmd",
+  "torch_dtype": "bfloat16",
+  "transformers_version": "4.40.0",
+  "vision_config": "mlp2x_gelu",
+  "vocab_size": 152064
+}

configuration_mcmd.py

@@ -0,0 +1,15 @@
+
+from transformers.configuration_utils import PretrainedConfig
+
+class mcmdConfig(PretrainedConfig):
+
+    model_type = "mcmd"
+    _auto_class = "AutoConfig"
+
+    def __init__(
+        self,  
+        **kwargs,
+    ):
+        super().__init__(
+            **kwargs,
+        )

model-00001-of-00004.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:47d9572190896617f2a8d3260ef421608c8dfd4de20941fa747bc3539a8099d2
+size 4877661712

model-00002-of-00004.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2a14fb3d73d667f9e9ae1e842cad95d0e744e30e293df64be2e82de0fbc50ce1
+size 4932752112

model-00003-of-00004.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:efe22d5e53941a70effa8c5333a62e9febb60e3356cebf6cb8dafd465cbcca4d
+size 4330866208

model-00004-of-00004.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:905adcc732723412112b06f2ffa551bd8d9c21a757552c4188b3ee337b29e80b
+size 2387691576

model.safetensors.index.json

@@ -0,0 +1,869 @@
+{
+  "metadata": {
+    "total_size": 16528855552
+  },
+  "weight_map": {
+    "lm_model.lm_head.weight": "model-00004-of-00004.safetensors",
+    "lm_model.model.embed_tokens.weight": "model-00001-of-00004.safetensors",
+    "lm_model.model.layers.0.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "lm_model.model.layers.0.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "lm_model.model.layers.0.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "lm_model.model.layers.0.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "lm_model.model.layers.0.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "lm_model.model.layers.0.self_attn.k_proj.bias": "model-00001-of-00004.safetensors",
+    "lm_model.model.layers.0.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "lm_model.model.layers.0.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "lm_model.model.layers.0.self_attn.q_proj.bias": "model-00001-of-00004.safetensors",
+    "lm_model.model.layers.0.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "lm_model.model.layers.0.self_attn.v_proj.bias": "model-00001-of-00004.safetensors",
+    "lm_model.model.layers.0.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "lm_model.model.layers.1.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "lm_model.model.layers.1.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "lm_model.model.layers.1.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "lm_model.model.layers.1.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "lm_model.model.layers.1.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "lm_model.model.layers.1.self_attn.k_proj.bias": "model-00001-of-00004.safetensors",
+    "lm_model.model.layers.1.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "lm_model.model.layers.1.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "lm_model.model.layers.1.self_attn.q_proj.bias": "model-00001-of-00004.safetensors",
+    "lm_model.model.layers.1.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "lm_model.model.layers.1.self_attn.v_proj.bias": "model-00001-of-00004.safetensors",
+    "lm_model.model.layers.1.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "lm_model.model.layers.10.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "lm_model.model.layers.10.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "lm_model.model.layers.10.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "lm_model.model.layers.10.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "lm_model.model.layers.10.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "lm_model.model.layers.10.self_attn.k_proj.bias": "model-00002-of-00004.safetensors",
+    "lm_model.model.layers.10.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "lm_model.model.layers.10.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "lm_model.model.layers.10.self_attn.q_proj.bias": "model-00002-of-00004.safetensors",
+    "lm_model.model.layers.10.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "lm_model.model.layers.10.self_attn.v_proj.bias": "model-00002-of-00004.safetensors",
+    "lm_model.model.layers.10.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "lm_model.model.layers.11.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "lm_model.model.layers.11.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "lm_model.model.layers.11.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "lm_model.model.layers.11.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "lm_model.model.layers.11.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "lm_model.model.layers.11.self_attn.k_proj.bias": "model-00002-of-00004.safetensors",
+    "lm_model.model.layers.11.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "lm_model.model.layers.11.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "lm_model.model.layers.11.self_attn.q_proj.bias": "model-00002-of-00004.safetensors",
+    "lm_model.model.layers.11.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "lm_model.model.layers.11.self_attn.v_proj.bias": "model-00002-of-00004.safetensors",
+    "lm_model.model.layers.11.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "lm_model.model.layers.12.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "lm_model.model.layers.12.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "lm_model.model.layers.12.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "lm_model.model.layers.12.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "lm_model.model.layers.12.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "lm_model.model.layers.12.self_attn.k_proj.bias": "model-00002-of-00004.safetensors",
+    "lm_model.model.layers.12.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "lm_model.model.layers.12.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "lm_model.model.layers.12.self_attn.q_proj.bias": "model-00002-of-00004.safetensors",
+    "lm_model.model.layers.12.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "lm_model.model.layers.12.self_attn.v_proj.bias": "model-00002-of-00004.safetensors",
+    "lm_model.model.layers.12.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "lm_model.model.layers.13.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "lm_model.model.layers.13.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "lm_model.model.layers.13.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "lm_model.model.layers.13.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "lm_model.model.layers.13.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "lm_model.model.layers.13.self_attn.k_proj.bias": "model-00002-of-00004.safetensors",
+    "lm_model.model.layers.13.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "lm_model.model.layers.13.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "lm_model.model.layers.13.self_attn.q_proj.bias": "model-00002-of-00004.safetensors",
+    "lm_model.model.layers.13.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "lm_model.model.layers.13.self_attn.v_proj.bias": "model-00002-of-00004.safetensors",
+    "lm_model.model.layers.13.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "lm_model.model.layers.14.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "lm_model.model.layers.14.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "lm_model.model.layers.14.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "lm_model.model.layers.14.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "lm_model.model.layers.14.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "lm_model.model.layers.14.self_attn.k_proj.bias": "model-00002-of-00004.safetensors",
+    "lm_model.model.layers.14.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "lm_model.model.layers.14.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "lm_model.model.layers.14.self_attn.q_proj.bias": "model-00002-of-00004.safetensors",
+    "lm_model.model.layers.14.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "lm_model.model.layers.14.self_attn.v_proj.bias": "model-00002-of-00004.safetensors",
+    "lm_model.model.layers.14.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "lm_model.model.layers.15.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "lm_model.model.layers.15.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "lm_model.model.layers.15.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "lm_model.model.layers.15.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "lm_model.model.layers.15.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "lm_model.model.layers.15.self_attn.k_proj.bias": "model-00002-of-00004.safetensors",
+    "lm_model.model.layers.15.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "lm_model.model.layers.15.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "lm_model.model.layers.15.self_attn.q_proj.bias": "model-00002-of-00004.safetensors",
+    "lm_model.model.layers.15.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "lm_model.model.layers.15.self_attn.v_proj.bias": "model-00002-of-00004.safetensors",
+    "lm_model.model.layers.15.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "lm_model.model.layers.16.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "lm_model.model.layers.16.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "lm_model.model.layers.16.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "lm_model.model.layers.16.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "lm_model.model.layers.16.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "lm_model.model.layers.16.self_attn.k_proj.bias": "model-00002-of-00004.safetensors",
+    "lm_model.model.layers.16.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "lm_model.model.layers.16.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "lm_model.model.layers.16.self_attn.q_proj.bias": "model-00002-of-00004.safetensors",
+    "lm_model.model.layers.16.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "lm_model.model.layers.16.self_attn.v_proj.bias": "model-00002-of-00004.safetensors",
+    "lm_model.model.layers.16.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "lm_model.model.layers.17.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "lm_model.model.layers.17.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "lm_model.model.layers.17.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "lm_model.model.layers.17.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "lm_model.model.layers.17.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "lm_model.model.layers.17.self_attn.k_proj.bias": "model-00002-of-00004.safetensors",
+    "lm_model.model.layers.17.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "lm_model.model.layers.17.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "lm_model.model.layers.17.self_attn.q_proj.bias": "model-00002-of-00004.safetensors",
+    "lm_model.model.layers.17.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "lm_model.model.layers.17.self_attn.v_proj.bias": "model-00002-of-00004.safetensors",
+    "lm_model.model.layers.17.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "lm_model.model.layers.18.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "lm_model.model.layers.18.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "lm_model.model.layers.18.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "lm_model.model.layers.18.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "lm_model.model.layers.18.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "lm_model.model.layers.18.self_attn.k_proj.bias": "model-00002-of-00004.safetensors",
+    "lm_model.model.layers.18.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "lm_model.model.layers.18.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "lm_model.model.layers.18.self_attn.q_proj.bias": "model-00002-of-00004.safetensors",
+    "lm_model.model.layers.18.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "lm_model.model.layers.18.self_attn.v_proj.bias": "model-00002-of-00004.safetensors",
+    "lm_model.model.layers.18.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "lm_model.model.layers.19.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "lm_model.model.layers.19.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "lm_model.model.layers.19.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "lm_model.model.layers.19.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "lm_model.model.layers.19.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "lm_model.model.layers.19.self_attn.k_proj.bias": "model-00003-of-00004.safetensors",
+    "lm_model.model.layers.19.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "lm_model.model.layers.19.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "lm_model.model.layers.19.self_attn.q_proj.bias": "model-00003-of-00004.safetensors",
+    "lm_model.model.layers.19.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "lm_model.model.layers.19.self_attn.v_proj.bias": "model-00003-of-00004.safetensors",
+    "lm_model.model.layers.19.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "lm_model.model.layers.2.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "lm_model.model.layers.2.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "lm_model.model.layers.2.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "lm_model.model.layers.2.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "lm_model.model.layers.2.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "lm_model.model.layers.2.self_attn.k_proj.bias": "model-00001-of-00004.safetensors",
+    "lm_model.model.layers.2.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "lm_model.model.layers.2.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "lm_model.model.layers.2.self_attn.q_proj.bias": "model-00001-of-00004.safetensors",
+    "lm_model.model.layers.2.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "lm_model.model.layers.2.self_attn.v_proj.bias": "model-00001-of-00004.safetensors",
+    "lm_model.model.layers.2.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "lm_model.model.layers.20.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "lm_model.model.layers.20.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "lm_model.model.layers.20.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "lm_model.model.layers.20.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "lm_model.model.layers.20.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "lm_model.model.layers.20.self_attn.k_proj.bias": "model-00003-of-00004.safetensors",
+    "lm_model.model.layers.20.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "lm_model.model.layers.20.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "lm_model.model.layers.20.self_attn.q_proj.bias": "model-00003-of-00004.safetensors",
+    "lm_model.model.layers.20.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "lm_model.model.layers.20.self_attn.v_proj.bias": "model-00003-of-00004.safetensors",
+    "lm_model.model.layers.20.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "lm_model.model.layers.21.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "lm_model.model.layers.21.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "lm_model.model.layers.21.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "lm_model.model.layers.21.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "lm_model.model.layers.21.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "lm_model.model.layers.21.self_attn.k_proj.bias": "model-00003-of-00004.safetensors",
+    "lm_model.model.layers.21.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "lm_model.model.layers.21.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "lm_model.model.layers.21.self_attn.q_proj.bias": "model-00003-of-00004.safetensors",
+    "lm_model.model.layers.21.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "lm_model.model.layers.21.self_attn.v_proj.bias": "model-00003-of-00004.safetensors",
+    "lm_model.model.layers.21.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "lm_model.model.layers.22.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "lm_model.model.layers.22.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "lm_model.model.layers.22.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "lm_model.model.layers.22.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "lm_model.model.layers.22.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "lm_model.model.layers.22.self_attn.k_proj.bias": "model-00003-of-00004.safetensors",
+    "lm_model.model.layers.22.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "lm_model.model.layers.22.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "lm_model.model.layers.22.self_attn.q_proj.bias": "model-00003-of-00004.safetensors",
+    "lm_model.model.layers.22.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "lm_model.model.layers.22.self_attn.v_proj.bias": "model-00003-of-00004.safetensors",
+    "lm_model.model.layers.22.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "lm_model.model.layers.23.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "lm_model.model.layers.23.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "lm_model.model.layers.23.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "lm_model.model.layers.23.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "lm_model.model.layers.23.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "lm_model.model.layers.23.self_attn.k_proj.bias": "model-00003-of-00004.safetensors",
+    "lm_model.model.layers.23.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "lm_model.model.layers.23.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "lm_model.model.layers.23.self_attn.q_proj.bias": "model-00003-of-00004.safetensors",
+    "lm_model.model.layers.23.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "lm_model.model.layers.23.self_attn.v_proj.bias": "model-00003-of-00004.safetensors",
+    "lm_model.model.layers.23.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "lm_model.model.layers.24.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "lm_model.model.layers.24.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "lm_model.model.layers.24.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "lm_model.model.layers.24.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "lm_model.model.layers.24.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "lm_model.model.layers.24.self_attn.k_proj.bias": "model-00003-of-00004.safetensors",
+    "lm_model.model.layers.24.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "lm_model.model.layers.24.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "lm_model.model.layers.24.self_attn.q_proj.bias": "model-00003-of-00004.safetensors",
+    "lm_model.model.layers.24.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "lm_model.model.layers.24.self_attn.v_proj.bias": "model-00003-of-00004.safetensors",
+    "lm_model.model.layers.24.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "lm_model.model.layers.25.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "lm_model.model.layers.25.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "lm_model.model.layers.25.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "lm_model.model.layers.25.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "lm_model.model.layers.25.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "lm_model.model.layers.25.self_attn.k_proj.bias": "model-00003-of-00004.safetensors",
+    "lm_model.model.layers.25.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "lm_model.model.layers.25.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "lm_model.model.layers.25.self_attn.q_proj.bias": "model-00003-of-00004.safetensors",
+    "lm_model.model.layers.25.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "lm_model.model.layers.25.self_attn.v_proj.bias": "model-00003-of-00004.safetensors",
+    "lm_model.model.layers.25.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "lm_model.model.layers.26.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "lm_model.model.layers.26.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "lm_model.model.layers.26.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "lm_model.model.layers.26.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "lm_model.model.layers.26.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "lm_model.model.layers.26.self_attn.k_proj.bias": "model-00003-of-00004.safetensors",
+    "lm_model.model.layers.26.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "lm_model.model.layers.26.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "lm_model.model.layers.26.self_attn.q_proj.bias": "model-00003-of-00004.safetensors",
+    "lm_model.model.layers.26.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "lm_model.model.layers.26.self_attn.v_proj.bias": "model-00003-of-00004.safetensors",
+    "lm_model.model.layers.26.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "lm_model.model.layers.27.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "lm_model.model.layers.27.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "lm_model.model.layers.27.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "lm_model.model.layers.27.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "lm_model.model.layers.27.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "lm_model.model.layers.27.self_attn.k_proj.bias": "model-00003-of-00004.safetensors",
+    "lm_model.model.layers.27.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "lm_model.model.layers.27.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "lm_model.model.layers.27.self_attn.q_proj.bias": "model-00003-of-00004.safetensors",
+    "lm_model.model.layers.27.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "lm_model.model.layers.27.self_attn.v_proj.bias": "model-00003-of-00004.safetensors",
+    "lm_model.model.layers.27.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "lm_model.model.layers.3.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "lm_model.model.layers.3.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "lm_model.model.layers.3.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "lm_model.model.layers.3.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "lm_model.model.layers.3.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "lm_model.model.layers.3.self_attn.k_proj.bias": "model-00001-of-00004.safetensors",
+    "lm_model.model.layers.3.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "lm_model.model.layers.3.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "lm_model.model.layers.3.self_attn.q_proj.bias": "model-00001-of-00004.safetensors",
+    "lm_model.model.layers.3.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "lm_model.model.layers.3.self_attn.v_proj.bias": "model-00001-of-00004.safetensors",
+    "lm_model.model.layers.3.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "lm_model.model.layers.4.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "lm_model.model.layers.4.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "lm_model.model.layers.4.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "lm_model.model.layers.4.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "lm_model.model.layers.4.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "lm_model.model.layers.4.self_attn.k_proj.bias": "model-00001-of-00004.safetensors",
+    "lm_model.model.layers.4.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "lm_model.model.layers.4.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "lm_model.model.layers.4.self_attn.q_proj.bias": "model-00001-of-00004.safetensors",
+    "lm_model.model.layers.4.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "lm_model.model.layers.4.self_attn.v_proj.bias": "model-00001-of-00004.safetensors",
+    "lm_model.model.layers.4.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "lm_model.model.layers.5.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "lm_model.model.layers.5.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "lm_model.model.layers.5.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "lm_model.model.layers.5.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "lm_model.model.layers.5.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "lm_model.model.layers.5.self_attn.k_proj.bias": "model-00001-of-00004.safetensors",
+    "lm_model.model.layers.5.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "lm_model.model.layers.5.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "lm_model.model.layers.5.self_attn.q_proj.bias": "model-00001-of-00004.safetensors",
+    "lm_model.model.layers.5.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "lm_model.model.layers.5.self_attn.v_proj.bias": "model-00001-of-00004.safetensors",
+    "lm_model.model.layers.5.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "lm_model.model.layers.6.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "lm_model.model.layers.6.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "lm_model.model.layers.6.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "lm_model.model.layers.6.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "lm_model.model.layers.6.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "lm_model.model.layers.6.self_attn.k_proj.bias": "model-00001-of-00004.safetensors",
+    "lm_model.model.layers.6.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "lm_model.model.layers.6.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "lm_model.model.layers.6.self_attn.q_proj.bias": "model-00001-of-00004.safetensors",
+    "lm_model.model.layers.6.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "lm_model.model.layers.6.self_attn.v_proj.bias": "model-00001-of-00004.safetensors",
+    "lm_model.model.layers.6.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "lm_model.model.layers.7.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "lm_model.model.layers.7.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "lm_model.model.layers.7.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "lm_model.model.layers.7.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "lm_model.model.layers.7.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "lm_model.model.layers.7.self_attn.k_proj.bias": "model-00001-of-00004.safetensors",
+    "lm_model.model.layers.7.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "lm_model.model.layers.7.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "lm_model.model.layers.7.self_attn.q_proj.bias": "model-00001-of-00004.safetensors",
+    "lm_model.model.layers.7.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "lm_model.model.layers.7.self_attn.v_proj.bias": "model-00001-of-00004.safetensors",
+    "lm_model.model.layers.7.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "lm_model.model.layers.8.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "lm_model.model.layers.8.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "lm_model.model.layers.8.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "lm_model.model.layers.8.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "lm_model.model.layers.8.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "lm_model.model.layers.8.self_attn.k_proj.bias": "model-00001-of-00004.safetensors",
+    "lm_model.model.layers.8.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "lm_model.model.layers.8.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "lm_model.model.layers.8.self_attn.q_proj.bias": "model-00001-of-00004.safetensors",
+    "lm_model.model.layers.8.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "lm_model.model.layers.8.self_attn.v_proj.bias": "model-00001-of-00004.safetensors",
+    "lm_model.model.layers.8.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "lm_model.model.layers.9.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "lm_model.model.layers.9.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "lm_model.model.layers.9.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "lm_model.model.layers.9.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "lm_model.model.layers.9.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "lm_model.model.layers.9.self_attn.k_proj.bias": "model-00002-of-00004.safetensors",
+    "lm_model.model.layers.9.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "lm_model.model.layers.9.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "lm_model.model.layers.9.self_attn.q_proj.bias": "model-00002-of-00004.safetensors",
+    "lm_model.model.layers.9.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "lm_model.model.layers.9.self_attn.v_proj.bias": "model-00002-of-00004.safetensors",
+    "lm_model.model.layers.9.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "lm_model.model.norm.weight": "model-00003-of-00004.safetensors",
+    "vision_proj.0.bias": "model-00004-of-00004.safetensors",
+    "vision_proj.0.weight": "model-00004-of-00004.safetensors",
+    "vision_proj.2.bias": "model-00004-of-00004.safetensors",
+    "vision_proj.2.weight": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.embeddings.class_embedding": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.embeddings.patch_embedding.weight": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.embeddings.position_embedding.weight": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.0.layer_norm1.bias": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.0.layer_norm1.weight": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.0.layer_norm2.bias": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.0.layer_norm2.weight": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.0.mlp.fc1.bias": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.0.mlp.fc1.weight": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.0.mlp.fc2.bias": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.0.mlp.fc2.weight": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.0.self_attn.k_proj.bias": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.0.self_attn.k_proj.weight": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.0.self_attn.out_proj.bias": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.0.self_attn.out_proj.weight": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.0.self_attn.q_proj.bias": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.0.self_attn.q_proj.weight": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.0.self_attn.v_proj.bias": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.0.self_attn.v_proj.weight": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.1.layer_norm1.bias": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.1.layer_norm1.weight": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.1.layer_norm2.bias": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.1.layer_norm2.weight": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.1.mlp.fc1.bias": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.1.mlp.fc1.weight": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.1.mlp.fc2.bias": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.1.mlp.fc2.weight": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.1.self_attn.k_proj.bias": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.1.self_attn.k_proj.weight": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.1.self_attn.out_proj.bias": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.1.self_attn.out_proj.weight": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.1.self_attn.q_proj.bias": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.1.self_attn.q_proj.weight": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.1.self_attn.v_proj.bias": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.1.self_attn.v_proj.weight": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.10.layer_norm1.bias": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.10.layer_norm1.weight": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.10.layer_norm2.bias": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.10.layer_norm2.weight": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.10.mlp.fc1.bias": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.10.mlp.fc1.weight": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.10.mlp.fc2.bias": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.10.mlp.fc2.weight": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.10.self_attn.k_proj.bias": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.10.self_attn.k_proj.weight": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.10.self_attn.out_proj.bias": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.10.self_attn.out_proj.weight": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.10.self_attn.q_proj.bias": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.10.self_attn.q_proj.weight": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.10.self_attn.v_proj.bias": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.10.self_attn.v_proj.weight": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.11.layer_norm1.bias": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.11.layer_norm1.weight": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.11.layer_norm2.bias": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.11.layer_norm2.weight": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.11.mlp.fc1.bias": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.11.mlp.fc1.weight": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.11.mlp.fc2.bias": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.11.mlp.fc2.weight": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.11.self_attn.k_proj.bias": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.11.self_attn.k_proj.weight": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.11.self_attn.out_proj.bias": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.11.self_attn.out_proj.weight": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.11.self_attn.q_proj.bias": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.11.self_attn.q_proj.weight": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.11.self_attn.v_proj.bias": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.11.self_attn.v_proj.weight": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.12.layer_norm1.bias": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.12.layer_norm1.weight": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.12.layer_norm2.bias": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.12.layer_norm2.weight": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.12.mlp.fc1.bias": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.12.mlp.fc1.weight": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.12.mlp.fc2.bias": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.12.mlp.fc2.weight": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.12.self_attn.k_proj.bias": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.12.self_attn.k_proj.weight": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.12.self_attn.out_proj.bias": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.12.self_attn.out_proj.weight": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.12.self_attn.q_proj.bias": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.12.self_attn.q_proj.weight": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.12.self_attn.v_proj.bias": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.12.self_attn.v_proj.weight": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.13.layer_norm1.bias": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.13.layer_norm1.weight": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.13.layer_norm2.bias": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.13.layer_norm2.weight": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.13.mlp.fc1.bias": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.13.mlp.fc1.weight": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.13.mlp.fc2.bias": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.13.mlp.fc2.weight": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.13.self_attn.k_proj.bias": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.13.self_attn.k_proj.weight": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.13.self_attn.out_proj.bias": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.13.self_attn.out_proj.weight": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.13.self_attn.q_proj.bias": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.13.self_attn.q_proj.weight": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.13.self_attn.v_proj.bias": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.13.self_attn.v_proj.weight": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.14.layer_norm1.bias": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.14.layer_norm1.weight": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.14.layer_norm2.bias": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.14.layer_norm2.weight": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.14.mlp.fc1.bias": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.14.mlp.fc1.weight": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.14.mlp.fc2.bias": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.14.mlp.fc2.weight": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.14.self_attn.k_proj.bias": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.14.self_attn.k_proj.weight": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.14.self_attn.out_proj.bias": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.14.self_attn.out_proj.weight": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.14.self_attn.q_proj.bias": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.14.self_attn.q_proj.weight": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.14.self_attn.v_proj.bias": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.14.self_attn.v_proj.weight": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.15.layer_norm1.bias": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.15.layer_norm1.weight": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.15.layer_norm2.bias": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.15.layer_norm2.weight": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.15.mlp.fc1.bias": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.15.mlp.fc1.weight": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.15.mlp.fc2.bias": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.15.mlp.fc2.weight": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.15.self_attn.k_proj.bias": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.15.self_attn.k_proj.weight": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.15.self_attn.out_proj.bias": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.15.self_attn.out_proj.weight": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.15.self_attn.q_proj.bias": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.15.self_attn.q_proj.weight": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.15.self_attn.v_proj.bias": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.15.self_attn.v_proj.weight": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.16.layer_norm1.bias": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.16.layer_norm1.weight": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.16.layer_norm2.bias": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.16.layer_norm2.weight": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.16.mlp.fc1.bias": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.16.mlp.fc1.weight": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.16.mlp.fc2.bias": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.16.mlp.fc2.weight": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.16.self_attn.k_proj.bias": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.16.self_attn.k_proj.weight": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.16.self_attn.out_proj.bias": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.16.self_attn.out_proj.weight": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.16.self_attn.q_proj.bias": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.16.self_attn.q_proj.weight": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.16.self_attn.v_proj.bias": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.16.self_attn.v_proj.weight": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.17.layer_norm1.bias": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.17.layer_norm1.weight": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.17.layer_norm2.bias": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.17.layer_norm2.weight": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.17.mlp.fc1.bias": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.17.mlp.fc1.weight": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.17.mlp.fc2.bias": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.17.mlp.fc2.weight": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.17.self_attn.k_proj.bias": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.17.self_attn.k_proj.weight": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.17.self_attn.out_proj.bias": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.17.self_attn.out_proj.weight": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.17.self_attn.q_proj.bias": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.17.self_attn.q_proj.weight": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.17.self_attn.v_proj.bias": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.17.self_attn.v_proj.weight": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.18.layer_norm1.bias": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.18.layer_norm1.weight": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.18.layer_norm2.bias": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.18.layer_norm2.weight": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.18.mlp.fc1.bias": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.18.mlp.fc1.weight": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.18.mlp.fc2.bias": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.18.mlp.fc2.weight": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.18.self_attn.k_proj.bias": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.18.self_attn.k_proj.weight": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.18.self_attn.out_proj.bias": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.18.self_attn.out_proj.weight": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.18.self_attn.q_proj.bias": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.18.self_attn.q_proj.weight": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.18.self_attn.v_proj.bias": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.18.self_attn.v_proj.weight": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.19.layer_norm1.bias": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.19.layer_norm1.weight": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.19.layer_norm2.bias": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.19.layer_norm2.weight": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.19.mlp.fc1.bias": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.19.mlp.fc1.weight": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.19.mlp.fc2.bias": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.19.mlp.fc2.weight": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.19.self_attn.k_proj.bias": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.19.self_attn.k_proj.weight": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.19.self_attn.out_proj.bias": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.19.self_attn.out_proj.weight": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.19.self_attn.q_proj.bias": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.19.self_attn.q_proj.weight": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.19.self_attn.v_proj.bias": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.19.self_attn.v_proj.weight": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.2.layer_norm1.bias": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.2.layer_norm1.weight": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.2.layer_norm2.bias": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.2.layer_norm2.weight": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.2.mlp.fc1.bias": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.2.mlp.fc1.weight": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.2.mlp.fc2.bias": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.2.mlp.fc2.weight": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.2.self_attn.k_proj.bias": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.2.self_attn.k_proj.weight": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.2.self_attn.out_proj.bias": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.2.self_attn.out_proj.weight": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.2.self_attn.q_proj.bias": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.2.self_attn.q_proj.weight": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.2.self_attn.v_proj.bias": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.2.self_attn.v_proj.weight": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.20.layer_norm1.bias": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.20.layer_norm1.weight": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.20.layer_norm2.bias": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.20.layer_norm2.weight": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.20.mlp.fc1.bias": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.20.mlp.fc1.weight": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.20.mlp.fc2.bias": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.20.mlp.fc2.weight": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.20.self_attn.k_proj.bias": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.20.self_attn.k_proj.weight": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.20.self_attn.out_proj.bias": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.20.self_attn.out_proj.weight": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.20.self_attn.q_proj.bias": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.20.self_attn.q_proj.weight": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.20.self_attn.v_proj.bias": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.20.self_attn.v_proj.weight": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.21.layer_norm1.bias": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.21.layer_norm1.weight": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.21.layer_norm2.bias": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.21.layer_norm2.weight": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.21.mlp.fc1.bias": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.21.mlp.fc1.weight": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.21.mlp.fc2.bias": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.21.mlp.fc2.weight": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.21.self_attn.k_proj.bias": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.21.self_attn.k_proj.weight": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.21.self_attn.out_proj.bias": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.21.self_attn.out_proj.weight": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.21.self_attn.q_proj.bias": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.21.self_attn.q_proj.weight": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.21.self_attn.v_proj.bias": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.21.self_attn.v_proj.weight": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.22.layer_norm1.bias": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.22.layer_norm1.weight": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.22.layer_norm2.bias": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.22.layer_norm2.weight": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.22.mlp.fc1.bias": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.22.mlp.fc1.weight": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.22.mlp.fc2.bias": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.22.mlp.fc2.weight": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.22.self_attn.k_proj.bias": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.22.self_attn.k_proj.weight": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.22.self_attn.out_proj.bias": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.22.self_attn.out_proj.weight": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.22.self_attn.q_proj.bias": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.22.self_attn.q_proj.weight": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.22.self_attn.v_proj.bias": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.22.self_attn.v_proj.weight": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.23.layer_norm1.bias": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.23.layer_norm1.weight": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.23.layer_norm2.bias": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.23.layer_norm2.weight": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.23.mlp.fc1.bias": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.23.mlp.fc1.weight": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.23.mlp.fc2.bias": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.23.mlp.fc2.weight": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.23.self_attn.k_proj.bias": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.23.self_attn.k_proj.weight": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.23.self_attn.out_proj.bias": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.23.self_attn.out_proj.weight": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.23.self_attn.q_proj.bias": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.23.self_attn.q_proj.weight": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.23.self_attn.v_proj.bias": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.23.self_attn.v_proj.weight": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.24.layer_norm1.bias": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.24.layer_norm1.weight": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.24.layer_norm2.bias": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.24.layer_norm2.weight": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.24.mlp.fc1.bias": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.24.mlp.fc1.weight": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.24.mlp.fc2.bias": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.24.mlp.fc2.weight": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.24.self_attn.k_proj.bias": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.24.self_attn.k_proj.weight": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.24.self_attn.out_proj.bias": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.24.self_attn.out_proj.weight": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.24.self_attn.q_proj.bias": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.24.self_attn.q_proj.weight": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.24.self_attn.v_proj.bias": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.24.self_attn.v_proj.weight": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.25.layer_norm1.bias": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.25.layer_norm1.weight": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.25.layer_norm2.bias": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.25.layer_norm2.weight": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.25.mlp.fc1.bias": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.25.mlp.fc1.weight": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.25.mlp.fc2.bias": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.25.mlp.fc2.weight": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.25.self_attn.k_proj.bias": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.25.self_attn.k_proj.weight": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.25.self_attn.out_proj.bias": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.25.self_attn.out_proj.weight": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.25.self_attn.q_proj.bias": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.25.self_attn.q_proj.weight": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.25.self_attn.v_proj.bias": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.25.self_attn.v_proj.weight": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.26.layer_norm1.bias": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.26.layer_norm1.weight": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.26.layer_norm2.bias": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.26.layer_norm2.weight": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.26.mlp.fc1.bias": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.26.mlp.fc1.weight": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.26.mlp.fc2.bias": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.26.mlp.fc2.weight": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.26.self_attn.k_proj.bias": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.26.self_attn.k_proj.weight": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.26.self_attn.out_proj.bias": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.26.self_attn.out_proj.weight": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.26.self_attn.q_proj.bias": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.26.self_attn.q_proj.weight": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.26.self_attn.v_proj.bias": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.26.self_attn.v_proj.weight": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.27.layer_norm1.bias": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.27.layer_norm1.weight": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.27.layer_norm2.bias": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.27.layer_norm2.weight": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.27.mlp.fc1.bias": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.27.mlp.fc1.weight": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.27.mlp.fc2.bias": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.27.mlp.fc2.weight": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.27.self_attn.k_proj.bias": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.27.self_attn.k_proj.weight": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.27.self_attn.out_proj.bias": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.27.self_attn.out_proj.weight": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.27.self_attn.q_proj.bias": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.27.self_attn.q_proj.weight": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.27.self_attn.v_proj.bias": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.27.self_attn.v_proj.weight": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.28.layer_norm1.bias": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.28.layer_norm1.weight": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.28.layer_norm2.bias": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.28.layer_norm2.weight": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.28.mlp.fc1.bias": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.28.mlp.fc1.weight": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.28.mlp.fc2.bias": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.28.mlp.fc2.weight": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.28.self_attn.k_proj.bias": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.28.self_attn.k_proj.weight": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.28.self_attn.out_proj.bias": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.28.self_attn.out_proj.weight": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.28.self_attn.q_proj.bias": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.28.self_attn.q_proj.weight": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.28.self_attn.v_proj.bias": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.28.self_attn.v_proj.weight": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.29.layer_norm1.bias": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.29.layer_norm1.weight": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.29.layer_norm2.bias": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.29.layer_norm2.weight": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.29.mlp.fc1.bias": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.29.mlp.fc1.weight": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.29.mlp.fc2.bias": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.29.mlp.fc2.weight": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.29.self_attn.k_proj.bias": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.29.self_attn.k_proj.weight": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.29.self_attn.out_proj.bias": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.29.self_attn.out_proj.weight": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.29.self_attn.q_proj.bias": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.29.self_attn.q_proj.weight": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.29.self_attn.v_proj.bias": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.29.self_attn.v_proj.weight": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.3.layer_norm1.bias": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.3.layer_norm1.weight": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.3.layer_norm2.bias": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.3.layer_norm2.weight": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.3.mlp.fc1.bias": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.3.mlp.fc1.weight": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.3.mlp.fc2.bias": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.3.mlp.fc2.weight": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.3.self_attn.k_proj.bias": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.3.self_attn.k_proj.weight": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.3.self_attn.out_proj.bias": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.3.self_attn.out_proj.weight": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.3.self_attn.q_proj.bias": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.3.self_attn.q_proj.weight": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.3.self_attn.v_proj.bias": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.3.self_attn.v_proj.weight": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.30.layer_norm1.bias": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.30.layer_norm1.weight": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.30.layer_norm2.bias": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.30.layer_norm2.weight": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.30.mlp.fc1.bias": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.30.mlp.fc1.weight": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.30.mlp.fc2.bias": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.30.mlp.fc2.weight": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.30.self_attn.k_proj.bias": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.30.self_attn.k_proj.weight": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.30.self_attn.out_proj.bias": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.30.self_attn.out_proj.weight": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.30.self_attn.q_proj.bias": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.30.self_attn.q_proj.weight": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.30.self_attn.v_proj.bias": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.30.self_attn.v_proj.weight": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.31.layer_norm1.bias": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.31.layer_norm1.weight": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.31.layer_norm2.bias": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.31.layer_norm2.weight": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.31.mlp.fc1.bias": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.31.mlp.fc1.weight": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.31.mlp.fc2.bias": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.31.mlp.fc2.weight": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.31.self_attn.k_proj.bias": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.31.self_attn.k_proj.weight": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.31.self_attn.out_proj.bias": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.31.self_attn.out_proj.weight": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.31.self_attn.q_proj.bias": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.31.self_attn.q_proj.weight": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.31.self_attn.v_proj.bias": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.31.self_attn.v_proj.weight": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.4.layer_norm1.bias": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.4.layer_norm1.weight": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.4.layer_norm2.bias": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.4.layer_norm2.weight": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.4.mlp.fc1.bias": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.4.mlp.fc1.weight": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.4.mlp.fc2.bias": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.4.mlp.fc2.weight": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.4.self_attn.k_proj.bias": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.4.self_attn.k_proj.weight": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.4.self_attn.out_proj.bias": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.4.self_attn.out_proj.weight": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.4.self_attn.q_proj.bias": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.4.self_attn.q_proj.weight": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.4.self_attn.v_proj.bias": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.4.self_attn.v_proj.weight": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.5.layer_norm1.bias": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.5.layer_norm1.weight": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.5.layer_norm2.bias": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.5.layer_norm2.weight": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.5.mlp.fc1.bias": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.5.mlp.fc1.weight": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.5.mlp.fc2.bias": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.5.mlp.fc2.weight": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.5.self_attn.k_proj.bias": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.5.self_attn.k_proj.weight": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.5.self_attn.out_proj.bias": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.5.self_attn.out_proj.weight": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.5.self_attn.q_proj.bias": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.5.self_attn.q_proj.weight": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.5.self_attn.v_proj.bias": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.5.self_attn.v_proj.weight": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.6.layer_norm1.bias": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.6.layer_norm1.weight": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.6.layer_norm2.bias": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.6.layer_norm2.weight": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.6.mlp.fc1.bias": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.6.mlp.fc1.weight": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.6.mlp.fc2.bias": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.6.mlp.fc2.weight": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.6.self_attn.k_proj.bias": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.6.self_attn.k_proj.weight": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.6.self_attn.out_proj.bias": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.6.self_attn.out_proj.weight": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.6.self_attn.q_proj.bias": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.6.self_attn.q_proj.weight": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.6.self_attn.v_proj.bias": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.6.self_attn.v_proj.weight": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.7.layer_norm1.bias": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.7.layer_norm1.weight": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.7.layer_norm2.bias": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.7.layer_norm2.weight": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.7.mlp.fc1.bias": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.7.mlp.fc1.weight": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.7.mlp.fc2.bias": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.7.mlp.fc2.weight": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.7.self_attn.k_proj.bias": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.7.self_attn.k_proj.weight": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.7.self_attn.out_proj.bias": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.7.self_attn.out_proj.weight": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.7.self_attn.q_proj.bias": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.7.self_attn.q_proj.weight": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.7.self_attn.v_proj.bias": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.7.self_attn.v_proj.weight": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.8.layer_norm1.bias": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.8.layer_norm1.weight": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.8.layer_norm2.bias": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.8.layer_norm2.weight": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.8.mlp.fc1.bias": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.8.mlp.fc1.weight": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.8.mlp.fc2.bias": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.8.mlp.fc2.weight": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.8.self_attn.k_proj.bias": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.8.self_attn.k_proj.weight": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.8.self_attn.out_proj.bias": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.8.self_attn.out_proj.weight": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.8.self_attn.q_proj.bias": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.8.self_attn.q_proj.weight": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.8.self_attn.v_proj.bias": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.8.self_attn.v_proj.weight": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.9.layer_norm1.bias": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.9.layer_norm1.weight": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.9.layer_norm2.bias": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.9.layer_norm2.weight": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.9.mlp.fc1.bias": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.9.mlp.fc1.weight": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.9.mlp.fc2.bias": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.9.mlp.fc2.weight": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.9.self_attn.k_proj.bias": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.9.self_attn.k_proj.weight": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.9.self_attn.out_proj.bias": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.9.self_attn.out_proj.weight": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.9.self_attn.q_proj.bias": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.9.self_attn.q_proj.weight": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.9.self_attn.v_proj.bias": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.encoder.layers.9.self_attn.v_proj.weight": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.post_layernorm.bias": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.post_layernorm.weight": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.pre_layrnorm.bias": "model-00004-of-00004.safetensors",
+    "vit.vision_tower.vision_model.pre_layrnorm.weight": "model-00004-of-00004.safetensors"
+  }
+}

modeling_mcmd.py

@@ -0,0 +1,512 @@
+#basic backage
+import os
+import copy
+import warnings
+from PIL import Image
+from typing import Optional, Tuple, Union, List, Callable
+
+#torch and transformer
+import torch
+from torch import nn
+from torch.nn import CrossEntropyLoss
+from torch.distributions.categorical import Categorical
+
+from torchvision import transforms
+from torchvision.transforms.functional import InterpolationMode
+
+
+from transformers.modeling_utils import PreTrainedModel
+from transformers.generation.streamers import BaseStreamer
+from transformers import AutoModelForCausalLM, AutoTokenizer
+from transformers.modeling_outputs import CausalLMOutputWithPast
+
+
+#mcmd
+from .configuration_mcmd import mcmdConfig
+from .Vision_Tower import clip_vit_large_patch14_336,DFN5B_CLIP_ViT_H_14_378
+from .Vision_Project import mlp2x_gelu
+
+def build_lm_model_tokenizer(lm_model_name : str, lm_tokenizer_name : str):
+    model = AutoModelForCausalLM.from_pretrained(
+        lm_model_name,
+        torch_dtype="auto"
+    )
+    tokenizer = AutoTokenizer.from_pretrained(lm_tokenizer_name)
+    return model,tokenizer
+
+def build_vision_projector(vision_config):
+    if vision_config=='mlp2x_gelu':
+        return mlp2x_gelu(vision_config)
+
+def build_vision_tower(vision_tower_name=''):
+    if vision_tower_name.endswith('clip-vit-large-patch14-336'):
+        return clip_vit_large_patch14_336(vision_tower_name,use_resize_pos=True)
+    elif vision_tower_name.endswith('DFN5B-CLIP-ViT-H-14-378'):
+        return DFN5B_CLIP_ViT_H_14_378(vision_tower_name)
+
+class mcmdPreTrainedModel(PreTrainedModel):
+    # config_class = mcmdConfig
+
+    def _init_weights(self, module):
+        std = self.config.initializer_range
+        if isinstance(module, nn.Linear):
+            module.weight.data.normal_(mean=0.0, std=std)
+            if module.bias is not None:
+                module.bias.data.zero_()
+        elif isinstance(module, nn.Embedding):
+            module.weight.data.normal_(mean=0.0, std=std)
+            if module.padding_idx is not None:
+                module.weight.data[module.padding_idx].zero_()
+
+
+class mcmdForCausalLM(mcmdPreTrainedModel):
+    _auto_class = 'AutoModelForCausalLM'
+
+    def __init__(self, config):
+        super().__init__(config)
+        
+        #Initialize language model 
+        self.max_length = config.max_length
+        self.vocab_size = config.lm_model['vocab_size']
+        self.lm_model,self.lm_tokenizer = build_lm_model_tokenizer(config.lm_path,config.lm_path)
+
+        #Initialize vit and vision_proj
+        self.vit = build_vision_tower(config.clip_path)
+        self.vision_proj = build_vision_projector(config.vision_config)
+
+        # Initialize vis_processor for Image Preprocessing. The mean and std is equal in dfn5b and clip-vit
+        self.vis_processor = transforms.Compose([
+            transforms.Resize((config.input_img_size, config.input_img_size),
+                              interpolation=InterpolationMode.BICUBIC),
+            transforms.ToTensor(),
+            transforms.Normalize((0.48145466, 0.4578275, 0.40821073),
+                                 (0.26862954, 0.26130258, 0.27577711)),
+        ])
+
+        self.eos_token_id = self.lm_tokenizer.eos_token_id # 151645 <|im_end|>
+
+    def print_trainable_parameters(self):
+        print('可训练参数:')
+        trainable_params = 0
+        all_param = 0
+        for _, param in self.named_parameters():
+            all_param += param.numel()
+            if param.requires_grad:
+                trainable_params += param.numel()
+        print(f"trainable params: {trainable_params} || all params: {all_param} || trainable%: {100 * trainable_params / all_param:.2f}")
+
+        print('可训练的模块:')
+        for name, param in self.named_parameters():
+            if param.requires_grad:
+                print(name, param.shape)
+    
+    def print_model_layers_and_parameters(self):
+        print('模型参数:')
+        for name, module in self.named_modules():
+            if hasattr(module, 'weight'):
+                num_params = sum(p.numel() for p in module.parameters() if p.requires_grad)
+                print(f"Layer: {name}, Type: {module.__class__.__name__}, Trainable Parameters: {num_params}")
+            else:
+                print(f"Layer: {name}, Type: {module.__class__.__name__}, No trainable parameters")
+        
+    def print_tokens_labels(self, tokens: List[int], target: List[int]):
+        print("Sanity Check >>>>>>>>>>>>>")
+        temp_tokens=copy.deepcopy(tokens[0].tolist())
+        temp_target=copy.deepcopy(target[0].tolist())
+        save_name='check_token_target.txt'
+        if os.path.exists(save_name):
+            os.remove(save_name)
+        ff = open(save_name,'a+')
+        for t, m in zip(temp_tokens, temp_target):
+            if t<0:
+                decoded='<Image Data>'
+            else:
+                decoded = self.lm_tokenizer.batch_decode([t], skip_special_tokens=False)[0]
+            print("%20s: %6d -> %6d" % (repr(decoded), t, m))
+            ff.write("%20s: %6d -> %6d\n" % (repr(decoded), t, m))
+        ff.close()
+        print("<<<<<<<<<<<<< Sanity Check")
+        assert len(tokens) == len(target), f"length mismatch: {len(tokens)} vs {len(target)}"
+
+    def img2emb(self, image):
+        image=image.bfloat16()
+        img_embeds = self.vision_proj(self.vit(image.to(self.device)))
+        atts_img = torch.ones(
+            img_embeds.size()[:-1], dtype=torch.long).to(img_embeds.device)
+
+        img_target = torch.ones(
+            img_embeds.size()[:2], dtype=torch.long).to(
+                img_embeds.device) * -100
+
+        return img_embeds, atts_img, img_target
+
+    def encode_img(self, image):
+        if image is None:
+            return None
+        if isinstance(image, str):
+            image = Image.open(image).convert('RGB')
+            # Image Preprocessing
+            # unsqueeze insert 1 dim in front of 0 
+            # image is [1, 3, 490, 490]
+            image = self.vis_processor(image).unsqueeze(0).to(self.device)
+        else:
+            assert isinstance(image, torch.Tensor)
+
+        img_embeds, _, _ = self.img2emb(image)
+        '''
+        img_embeds : [1, 1225, 4096] 1225？
+        atts_img = torch.ones([1, 1225])
+        img_target = torch.ones([1, 1225]) * -100
+        '''
+        return img_embeds
+
+    def get_tensor_image(self,fns):
+        image_data=[]
+
+        for one in fns:
+            t_one=self.encode_img(one)
+            image_data.append(t_one)
+
+        image = torch.cat(image_data, dim=0)
+
+        return image
+
+
+    def interleav_wrap_chat(self, messages, image):
+
+        #Deal prompt using qwen2 template, which is from transformers/tokenization_utils_base.py
+        prompt = self.lm_tokenizer.apply_chat_template(
+            messages,
+            tokenize=False,
+            add_generation_prompt=True
+        )
+        '''
+        repr(prompt) add_generation_prompt=True : '<|im_start|>system\nYou are a helpful assistant.<|im_end|>\n<|im_start|>user\n比较一下下面这两张图片，第一张<ImageHere>，\n第二张<ImageHere><|im_end|>\n<|im_start|>assistant\n' 
+        repr(prompt) add_generation_prompt=False: '<|im_start|>system\nYou are a helpful assistant.<|im_end|>\n<|im_start|>user\n比较一下下面这两张图片，第一张<ImageHere>，\n第二张<ImageHere><|im_end|>\n'
+        '''
+
+        if image is None:
+            im_len=0
+            image_nums=0
+            parts = prompt.split('<ImageHere>')
+            print(prompt.split('<ImageHere>'))
+            assert len(prompt.split('<ImageHere>'))==1
+        else:
+            im_len = image.shape[1] #1225 730
+            image_nums = len(image)
+            parts = prompt.split('<ImageHere>')
+        wrap_embeds = []
+        temp_len = 0
+
+        if len(parts) != image_nums + 1:
+            raise ValueError('Invalid <ImageHere> prompt format.')
+    
+        for idx, part in enumerate(parts):
+            if len(part) > 0:
+                part_tokens = self.lm_tokenizer(part, return_tensors='pt').to(self.device)
+                part_embeds = self.lm_model.model.embed_tokens(
+                    part_tokens.input_ids)
+                wrap_embeds.append(part_embeds)
+                
+                temp_len += part_embeds.shape[1]
+            if idx < image_nums:
+                wrap_embeds.append(image[idx].unsqueeze(0))
+                temp_len += im_len
+    
+            if temp_len > self.max_length:
+                break
+
+        wrap_embeds = torch.cat(wrap_embeds, dim=1) #torch.Size([1, 2481, 3584])
+        wrap_embeds = wrap_embeds[:, :self.max_length].to(self.device)
+
+        inputs = {
+            'inputs_embeds': wrap_embeds
+        }
+        return inputs
+
+    def mask_user_targets(self, input_ids):
+        target_batch = []
+        for bs in range(input_ids.shape[0]):
+            ids = input_ids[bs]
+            targets = copy.deepcopy(ids)
+            im_round=0
+            id_im_start=0
+            # id_im_end=0
+            for i, temp_id in enumerate(ids):
+                if temp_id == 151644:
+                    im_round+=1
+                    if im_round==2:
+                        id_im_start=0
+                        targets[id_im_start:i + 1] = -100
+                        id_im_start=i
+                    elif im_round%2==0:
+                        id_im_start=i
+                    elif im_round%2==1:
+                        targets[id_im_start:i + 3] = -100
+                # if temp_id == 151645:
+                #     if im_round==1:
+                #         id_im_end=i
+
+
+            target_batch.append(targets.unsqueeze(0))
+
+        target_batch = torch.cat(target_batch, dim=0)
+        return target_batch
+
+    def interleav_wrap(self, img_list, text_list):
+        # Initialize lists to store the processed embeddings, attention masks, and targets.
+        wrap_embeds_list, wrap_atts_list = [], []
+        wrap_target_list = []
+
+        # Iterate over pairs of images and texts.
+        for image, text in zip(img_list, text_list):
+            # Convert the image to embeddings using the method `img2emb`.
+            img_embeds, atts_img, img_target = self.img2emb(image)
+
+            # Get the first element of the text (assuming it's a list).
+            text = text[0]
+            # Split the text into parts where `<ImageHere>` is found.
+            parts = text.split('<ImageHere>')
+
+            # Initialize lists to store tokens, embeddings, and attention masks for the current item.
+            wrap_tokens, wrap_embeds, wrap_atts = [], [], []
+            
+            # Track the total length of the sequence being built.
+            temp_len = 0
+            
+            # Get the number of images and the length of each image embedding.
+            image_nums, im_len = img_embeds.shape[:2]
+
+            # Process each part of the split text.
+            for idx, part in enumerate(parts):
+                # If the part is not empty, process it as text.
+                if len(part) > 0:
+                    # Tokenize the text part.
+                    part_tokens = self.lm_tokenizer(
+                        part,
+                        return_tensors='pt',
+                        padding='longest').to(self.device)
+                    
+                    # Append the token IDs, embeddings, and attention mask to their respective lists.
+                    wrap_tokens.append(part_tokens.input_ids)
+                    part_embeds = self.lm_model.model.embed_tokens(part_tokens.input_ids)
+                    wrap_embeds.append(part_embeds)
+                    wrap_atts.append(part_tokens.attention_mask)
+                    
+                    # Update the total length of the sequence.
+                    temp_len += part_embeds.shape[1]
+                
+                # If there are more images, append the image target, embeddings, and attention mask.
+                if idx < image_nums:
+                    wrap_tokens.append(img_target[idx].unsqueeze(0))
+                    wrap_embeds.append(img_embeds[idx].unsqueeze(0))
+                    wrap_atts.append(atts_img[idx].unsqueeze(0))
+                    
+                    # Update the total length of the sequence.
+                    temp_len += im_len
+                
+                # Break the loop if the total length exceeds the maximum length.
+                if temp_len > self.max_length:
+                    break
+
+            # Concatenate the tokens, embeddings, and attention masks.
+            wrap_tokens = torch.cat(wrap_tokens, dim=1)
+            wrap_embeds = torch.cat(wrap_embeds, dim=1)
+            wrap_atts = torch.cat(wrap_atts, dim=1)
+
+            # print('wrap_tokens',wrap_tokens.shape)
+            # print('wrap_embeds',wrap_embeds.shape)
+            # print('wrap_atts',wrap_atts.shape)
+
+            # Mask the targets for the tokens.
+            wrap_target = self.mask_user_targets(wrap_tokens).to(self.device)
+
+            # Truncate the concatenated tensors to the max length.
+            wrap_embeds = wrap_embeds[:, :self.max_length].to(self.device)
+            wrap_atts = wrap_atts[:, :self.max_length].to(self.device)
+            wrap_target = wrap_target[:, :self.max_length].to(self.device)
+
+            # self.print_tokens_labels(wrap_tokens, wrap_target)
+
+
+            # Add the processed data to the corresponding lists.
+            wrap_embeds_list.append(wrap_embeds)
+            wrap_atts_list.append(wrap_atts)
+            wrap_target_list.append(wrap_target)
+
+        # Concatenate all the processed data from different items.
+        wrap_embeds = torch.cat(wrap_embeds_list)
+        wrap_atts = torch.cat(wrap_atts_list)
+        wrap_target = torch.cat(wrap_target_list)
+
+        # Return the concatenated embeddings, attention masks, and targets.
+        return wrap_embeds, wrap_atts, wrap_target
+
+    def text2emb(self, text, add_special=False):
+
+        to_regress_tokens = self.lm_tokenizer(
+                        text,
+                        return_tensors='pt',
+                        padding='longest').to(self.device)
+        to_regress_tokens.input_ids
+        targets = self.mask_user_targets(to_regress_tokens.input_ids)
+        targets = targets.to(self.device)
+
+        # self.print_tokens_labels(to_regress_tokens.input_ids, targets)
+
+        return to_regress_tokens, targets
+
+    def forward(
+        self,
+        input_ids: torch.LongTensor = None,
+        attention_mask: Optional[torch.Tensor] = None,
+        position_ids: Optional[torch.LongTensor] = None,
+        past_key_values: Optional[List[torch.FloatTensor]] = None,
+        inputs_embeds: Optional[torch.FloatTensor] = None,
+        labels: Optional[torch.LongTensor] = None,
+        use_cache: Optional[bool] = None,
+        output_attentions: Optional[bool] = None,
+        output_hidden_states: Optional[bool] = None,
+        return_dict: Optional[bool] = None,
+        **kwargs
+    ) -> Union[Tuple, CausalLMOutputWithPast]:
+        r"""
+        Args:
+            labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
+                Labels for computing the masked language modeling loss. Indices should either be in `[0, ...,
+                config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored
+                (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`.
+
+        Returns:
+
+        ```"""
+        # prepared for train mode
+        samples = kwargs.get('samples', None)
+        if samples:
+            if samples['data_type'][0] == 'text':
+                has_img = False
+            elif samples['data_type'][0] == 'multi':
+                has_img = True
+            else:
+                raise NotImplementedError
+
+            # encode text
+            text = samples['text_input']
+            # encode image
+            if has_img:
+                image = samples['image']
+
+                to_regress_embeds, attention_mask, targets = self.interleav_wrap(
+                    image, text)
+            else:
+                to_regress_tokens, targets = self.text2emb(#-------------------------------------------------------------------------------------------
+                    text, add_special=True)
+                to_regress_embeds = self.lm_model.model.embed_tokens(#-------------------------------------------------------------------------------------------
+                    to_regress_tokens.input_ids)
+                attention_mask = to_regress_tokens.attention_mask
+
+            inputs_embeds = to_regress_embeds[:, :self.max_length]
+            attention_mask = attention_mask[:, :self.max_length]
+            targets = targets[:, :self.max_length]
+            labels = targets
+
+
+        output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions
+        output_hidden_states = (
+            output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states
+        )
+        return_dict = return_dict if return_dict is not None else self.config.use_return_dict
+
+        # decoder outputs consists of (dec_features, layer_state, dec_hidden, dec_attn)
+        outputs = self.lm_model.model(
+            input_ids=input_ids,
+            attention_mask=attention_mask,
+            position_ids=position_ids,
+            past_key_values=past_key_values,
+            inputs_embeds=inputs_embeds,
+            use_cache=use_cache,
+            output_attentions=output_attentions,
+            output_hidden_states=output_hidden_states,
+            return_dict=return_dict,
+        )
+
+        hidden_states = outputs[0]
+        logits = self.lm_model.lm_head(hidden_states)
+        logits = logits.float()
+
+        loss = None
+        if labels is not None:
+            # Shift so that tokens < n predict n
+            shift_logits = logits[..., :-1, :].contiguous()
+            shift_labels = labels[..., 1:].contiguous()
+            # Flatten the tokens
+            loss_fct = CrossEntropyLoss()
+            shift_logits = shift_logits.view(-1, self.config.vocab_size)
+            shift_labels = shift_labels.view(-1)
+            # Enable model parallelism
+            shift_labels = shift_labels.to(shift_logits.device)
+            loss = loss_fct(shift_logits, shift_labels)
+
+        if not return_dict:
+            output = (logits,) + outputs[1:]
+            return (loss,) + output if loss is not None else output
+
+        return CausalLMOutputWithPast(
+            loss=loss,
+            logits=logits,
+            past_key_values=outputs.past_key_values,
+            hidden_states=outputs.hidden_states,
+            attentions=outputs.attentions,
+        )
+    
+    @torch.no_grad()
+    def chat(
+        self,
+        messages,
+        images: List[str] = None,
+        streamer: Optional[BaseStreamer] = None,
+        max_new_tokens: int = 1024,
+        do_sample: bool = True,
+        num_beams: int = 1,
+        temperature: float = 1.0,
+        top_p: float = 0.8,
+        repetition_penalty: float=1.005,
+        **kwargs,
+    ):
+        if images!=[]:
+            print('images ',images)
+            image_pt=self.get_tensor_image(images)
+        else:
+            image_pt=None
+        inputs=self.interleav_wrap_chat(messages,image_pt)
+
+        inputs = {
+            k: v.to(self.device)
+            for k, v in inputs.items() if torch.is_tensor(v)
+        }
+        # also add end-of-assistant token in eos token id to avoid unnecessary generation
+        eos_token_id = [
+            self.eos_token_id
+        ]
+        outputs = self.lm_model.generate(
+            **inputs,
+            streamer=streamer,
+            max_new_tokens=max_new_tokens,
+            num_beams=num_beams,
+            do_sample=do_sample,
+            temperature=temperature,
+            top_p=top_p,
+            eos_token_id=eos_token_id,
+            repetition_penalty=repetition_penalty,
+            **kwargs,
+        )
+
+        response = self.lm_tokenizer.batch_decode(outputs, skip_special_tokens=True)[0]
+        messages+=[{"role": "assistant", "content": response}]
+
+        return response, messages
+    
+    
+
+
+

special_tokens_map.json

@@ -0,0 +1,20 @@
+{
+  "additional_special_tokens": [
+    "<|im_start|>",
+    "<|im_end|>"
+  ],
+  "eos_token": {
+    "content": "<|im_end|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

tokenizer_config.json

@@ -0,0 +1,43 @@
+{
+  "add_prefix_space": false,
+  "added_tokens_decoder": {
+    "151643": {
+      "content": "<|endoftext|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151644": {
+      "content": "<|im_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151645": {
+      "content": "<|im_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "additional_special_tokens": [
+    "<|im_start|>",
+    "<|im_end|>"
+  ],
+  "bos_token": null,
+  "chat_template": "{% for message in messages %}{% if loop.first and messages[0]['role'] != 'system' %}{{ '<|im_start|>system\nYou are a helpful assistant.<|im_end|>\n' }}{% endif %}{{'<|im_start|>' + message['role'] + '\n' + message['content'] + '<|im_end|>' + '\n'}}{% endfor %}{% if add_generation_prompt %}{{ '<|im_start|>assistant\n' }}{% endif %}",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<|im_end|>",
+  "errors": "replace",
+  "model_max_length": 131072,
+  "pad_token": "<|endoftext|>",
+  "split_special_tokens": false,
+  "tokenizer_class": "Qwen2Tokenizer",
+  "unk_token": null
+}

training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b31d1f19850dd4d60172d22b93ed93f1986f7ca07edf7291521197e6fda401bd
+size 6392