Add files using upload-large-folder tool

README.md

@@ -0,0 +1,67 @@
+---
+base_model: []
+library_name: transformers
+tags:
+- mergekit
+- merge
+
+---
+# POINTS-1-5-Qwen-2-5-7B-Chat
+
+This is a merge of pre-trained language models created using [mergekit](https://github.com/cg123/mergekit).
+
+## Merge Details
+### Merge Method
+
+This model was merged using the [linear](https://arxiv.org/abs/2203.05482) merge method.
+
+### Models Merged
+
+The following models were included in the merge:
+* /mnt/cephfs/bensenliu/wfs/weights/mm/mmq-llava-20241130e2-sft-pointsv15-hf/
+* /mnt/cephfs/bensenliu/wfs/weights/mm/mmq-llava-20241127e3-sft-pointsv15-hf/
+* /mnt/cephfs/bensenliu/wfs/weights/mm/mmq-llava-20241129e2-sft-pointsv15-hf/
+* /mnt/cephfs/bensenliu/wfs/weights/mm/mmq-llava-20241130e3-sft-pointsv15-hf/
+
+### Configuration
+
+The following YAML configuration was used to produce this model:
+
+```yaml
+models:
+  # - model: /mnt/cephfs/bensenliu/wfs/weights/mm/mmq-llava-20241123e2-sft-hf/
+  #   parameters:
+  #     weight: 1.0
+  # - model: /mnt/cephfs/bensenliu/wfs/weights/mm/mmq-llava-20241124e2-sft-hf/
+  #   parameters:
+  #     weight: 1.0
+  # - model: /mnt/cephfs/bensenliu/wfs/weights/mm/mmq-llava-20241125e1-sft-hf/
+  #   parameters:
+  #     weight: 1.0
+  # - model: /mnt/cephfs/bensenliu/wfs/weights/mm/mmq-llava-20241125e3-sft-hf/
+  #   parameters:
+  #     weight: 1.0
+  # - model: /mnt/cephfs/bensenliu/wfs/weights/mm/mmq-llava-20241127e1-sft-hf/
+  #   parameters:
+  #     weight: 1.0
+  # - model: /mnt/cephfs/bensenliu/wfs/weights/mm/mmq-llava-20241127e2-sft-hf/
+  #   parameters:
+  #     weight: 1.0
+  - model: /mnt/cephfs/bensenliu/wfs/weights/mm/mmq-llava-20241127e3-sft-pointsv15-hf/
+    parameters:
+      weight: 1.0
+  - model: /mnt/cephfs/bensenliu/wfs/weights/mm/mmq-llava-20241129e2-sft-pointsv15-hf/
+    parameters:
+      weight: 1.0
+  - model: /mnt/cephfs/bensenliu/wfs/weights/mm/mmq-llava-20241130e2-sft-pointsv15-hf/
+    parameters:
+      weight: 1.0
+  - model: /mnt/cephfs/bensenliu/wfs/weights/mm/mmq-llava-20241130e3-sft-pointsv15-hf/
+    parameters:
+      weight: 1.0
+merge_method: linear
+parameters:
+  normalize: true
+dtype: bfloat16
+
+```

config.json

@@ -0,0 +1,185 @@
+{
+  "_commit_hash": null,
+  "architectures": [
+    "POINTSV15ChatModel"
+  ],
+  "auto_map": {
+    "AutoConfig": "configuration_pointsv15_chat.POINTSV15ChatConfig",
+    "AutoModelForCausalLM": "modeling_pointsv15_chat.POINTSV15ChatModel"
+  },
+  "llm_config": {
+    "_attn_implementation_autoset": true,
+    "_name_or_path": "",
+    "add_cross_attention": false,
+    "architectures": null,
+    "auto_map": {
+      "AutoConfig": "configuration_llama.CustomLlamaConfig",
+      "AutoModelForCausalLM": "modeling_llama.CustomLlamaForCausalLM"
+    },
+    "bad_words_ids": null,
+    "begin_suppress_tokens": null,
+    "bos_token_id": 0,
+    "chunk_size_feed_forward": 0,
+    "cross_attention_hidden_size": null,
+    "decoder_start_token_id": null,
+    "diversity_penalty": 0.0,
+    "do_sample": false,
+    "early_stopping": false,
+    "encoder_no_repeat_ngram_size": 0,
+    "eos_token_id": [
+      2,
+      3
+    ],
+    "exponential_decay_length_penalty": null,
+    "ffn_hidden_size": 18944,
+    "finetuning_task": null,
+    "forced_bos_token_id": null,
+    "forced_eos_token_id": null,
+    "hidden_act": "swiglu",
+    "hidden_size": 3584,
+    "id2label": {
+      "0": "LABEL_0",
+      "1": "LABEL_1"
+    },
+    "initializer_range": 0.02,
+    "is_decoder": false,
+    "is_encoder_decoder": false,
+    "label2id": {
+      "LABEL_0": 0,
+      "LABEL_1": 1
+    },
+    "layernorm_epsilon": 1e-05,
+    "length_penalty": 1.0,
+    "max_length": 20,
+    "max_position_embeddings": 16384,
+    "min_length": 0,
+    "mlp_fc1_bias": false,
+    "mlp_fc2_bias": false,
+    "model_type": "custom_llama",
+    "no_repeat_ngram_size": 0,
+    "norm_type": "rms_norm",
+    "num_attention_heads": 28,
+    "num_beam_groups": 1,
+    "num_beams": 1,
+    "num_hidden_layers": 28,
+    "num_kv_heads": 4,
+    "num_layers": 28,
+    "num_return_sequences": 1,
+    "out_proj_bias": false,
+    "output_attentions": false,
+    "output_hidden_states": false,
+    "output_scores": false,
+    "pad_token_id": null,
+    "prefix": null,
+    "problem_type": null,
+    "pruned_heads": {},
+    "qkv_proj_bias": true,
+    "remove_invalid_values": false,
+    "repetition_penalty": 1.0,
+    "return_dict": true,
+    "return_dict_in_generate": false,
+    "rotary_compress": 1.0,
+    "rotary_emb_base": 1000000.0,
+    "rotary_pct": 1.0,
+    "sep_token_id": null,
+    "share_kv_num_layers": 1,
+    "sliding_window_size": -1,
+    "sliding_window_type": null,
+    "suppress_tokens": null,
+    "task_specific_params": null,
+    "temperature": 1.0,
+    "tf_legacy_loss": false,
+    "tie_encoder_decoder": false,
+    "tie_word_embeddings": false,
+    "tokenizer_class": null,
+    "top_k": 50,
+    "top_p": 1.0,
+    "torch_dtype": null,
+    "torchscript": false,
+    "transformers_version": "4.46.3",
+    "typical_p": 1.0,
+    "use_bfloat16": false,
+    "use_cache": true,
+    "use_gqa": true,
+    "vocab_size": 152064
+  },
+  "torch_dtype": "bfloat16",
+  "transformers_version": null,
+  "vision_config": {
+    "_attn_implementation_autoset": false,
+    "_name_or_path": "",
+    "add_cross_attention": false,
+    "architectures": null,
+    "bad_words_ids": null,
+    "begin_suppress_tokens": null,
+    "bos_token_id": null,
+    "chunk_size_feed_forward": 0,
+    "cross_attention_hidden_size": null,
+    "decoder_start_token_id": null,
+    "depth": 32,
+    "diversity_penalty": 0.0,
+    "do_sample": false,
+    "early_stopping": false,
+    "embed_dim": 1280,
+    "encoder_no_repeat_ngram_size": 0,
+    "eos_token_id": null,
+    "exponential_decay_length_penalty": null,
+    "finetuning_task": null,
+    "forced_bos_token_id": null,
+    "forced_eos_token_id": null,
+    "hidden_act": "quick_gelu",
+    "hidden_size": 3584,
+    "id2label": {
+      "0": "LABEL_0",
+      "1": "LABEL_1"
+    },
+    "in_channels": 3,
+    "in_chans": 3,
+    "is_decoder": false,
+    "is_encoder_decoder": false,
+    "label2id": {
+      "LABEL_0": 0,
+      "LABEL_1": 1
+    },
+    "length_penalty": 1.0,
+    "max_length": 20,
+    "min_length": 0,
+    "mlp_ratio": 4,
+    "model_type": "qwen2_vl",
+    "no_repeat_ngram_size": 0,
+    "num_beam_groups": 1,
+    "num_beams": 1,
+    "num_heads": 16,
+    "num_return_sequences": 1,
+    "output_attentions": false,
+    "output_hidden_states": false,
+    "output_scores": false,
+    "pad_token_id": null,
+    "patch_size": 14,
+    "prefix": null,
+    "problem_type": null,
+    "pruned_heads": {},
+    "remove_invalid_values": false,
+    "repetition_penalty": 1.0,
+    "return_dict": true,
+    "return_dict_in_generate": false,
+    "sep_token_id": null,
+    "spatial_merge_size": 2,
+    "spatial_patch_size": 14,
+    "suppress_tokens": null,
+    "task_specific_params": null,
+    "temperature": 1.0,
+    "temporal_patch_size": 2,
+    "tf_legacy_loss": false,
+    "tie_encoder_decoder": false,
+    "tie_word_embeddings": true,
+    "tokenizer_class": null,
+    "top_k": 50,
+    "top_p": 1.0,
+    "torch_dtype": "bfloat16",
+    "torchscript": false,
+    "transformers_version": "4.46.3",
+    "typical_p": 1.0,
+    "use_bfloat16": false
+  }
+}

configuration_llama.py

@@ -0,0 +1,103 @@
+# Modify the original configuration_llama.py to
+# be compatiable with our training framework.
+
+
+from transformers.configuration_utils import PretrainedConfig
+from transformers.utils import logging
+
+logger = logging.get_logger(__name__)
+
+
+class CustomLlamaConfig(PretrainedConfig):
+    """
+    Args:
+        vocab_size (`int`, *optional*, defaults to 50432):
+            Vocabulary size of the WeLMV3 model. Defines the number of 
+                different tokens that can be represented by the
+            `inputs_ids` passed when calling [`WeLMV3Model`].
+        hidden_size (`int`, *optional*, defaults to 6144):
+            Dimension of the encoder layers and the pooler layer.
+        num_hidden_layers (`int`, *optional*, defaults to 44):
+            Number of hidden layers in the Transformer encoder.
+        num_attention_heads (`int`, *optional*, defaults to 64):
+            Number of attention heads for each attention layer in the 
+            Transformer encoder.
+        num_kv_heads (`int`, *optional*, defaults to 4):
+            Number of GQA groups.
+        intermediate_size (`int`, *optional*, defaults to 24576):
+            Dimension of the "intermediate" (i.e., feed-forward) layer in the 
+            Transformer encoder.
+        hidden_act (`str` or `function`, *optional*, defaults to `"gelu"`):
+            The non-linear activation function (function or string) in the 
+            encoder and pooler. If string, `"gelu"`,
+            `"relu"`, `"selu"` and `"gelu_new"` are supported.
+        rotary_pct (`float`, *optional*, defaults to 0.25):
+            percentage of hidden dimensions to allocate to rotary embeddings
+        rotary_emb_base (`int`, *optional*, defaults to 10000)
+            base for computing rotary embeddings frequency
+        max_position_embeddings (`int`, *optional*, defaults to 2048):
+            The maximum sequence length that this model might ever be used 
+            with. Typically set this to something large just in case 
+            (e.g., 512 or 1024 or 2048).
+        initializer_range (`float`, *optional*, defaults to 1e-5):
+            The standard deviation of the truncated_normal_initializer for 
+            initializing all weight matrices.
+        layer_norm_eps (`float`, *optional*, defaults to 1e-12):
+            The epsilon used by the layer normalization layers.
+        use_cache (`bool`, *optional*, defaults to `True`):
+            Whether or not the model should return the last key/values 
+            attentions (not used by all models). Only relevant if 
+            `config.is_decoder=True`.
+    """
+    model_type = "custom_llama"
+
+    def __init__(
+        self,
+        vocab_size=102400,
+        hidden_size=2560,
+        num_layers=32,
+        num_attention_heads=20,
+        num_kv_heads=4,
+        ffn_hidden_size=2560 * 4,
+        hidden_act="swiglu",
+        rotary_pct=1.0,
+        rotary_emb_base=10000,
+        rotary_compress=1.0,
+        max_position_embeddings=4096,
+        initializer_range=0.02,
+        layernorm_epsilon=1e-5,
+        use_cache=True,
+        bos_token_id=0,
+        eos_token_id=2,
+        rms_norm=None,
+        norm_type='layer_norm',
+        qkv_proj_bias=True,
+        out_proj_bias=True,
+        mlp_fc1_bias=True,
+        mlp_fc2_bias=True,
+        **kwargs,
+    ):
+        super().__init__(bos_token_id=bos_token_id, eos_token_id=eos_token_id, **kwargs)
+        self.vocab_size = vocab_size
+        self.max_position_embeddings = max_position_embeddings
+        self.hidden_size = hidden_size
+        self.num_layers = num_layers
+        self.num_attention_heads = num_attention_heads
+        self.num_kv_heads = num_kv_heads
+        self.ffn_hidden_size = ffn_hidden_size
+        self.hidden_act = hidden_act
+        self.rotary_pct = rotary_pct
+        self.rotary_emb_base = rotary_emb_base
+        self.rotary_compress = rotary_compress
+        self.initializer_range = initializer_range
+        self.layernorm_epsilon = layernorm_epsilon
+        self.use_cache = use_cache
+        if rms_norm is not None:
+            self.norm_type = 'rms_norm' if rms_norm else 'layer_norm'
+        else:
+            self.norm_type = norm_type
+        self.qkv_proj_bias = qkv_proj_bias
+        self.out_proj_bias = out_proj_bias
+        self.mlp_fc1_bias = mlp_fc1_bias
+        self.mlp_fc2_bias = mlp_fc2_bias
+        self.num_hidden_layers = num_layers

configuration_pointsv15_chat.py

@@ -0,0 +1,37 @@
+import copy
+from typing import Any, Dict
+
+from transformers import PretrainedConfig
+
+try:
+    from transformers.models.qwen2_vl.configuration_qwen2_vl import Qwen2VLVisionConfig
+except ImportError:
+    print('Please upgrade transformers to version 4.46.3 or higher')
+
+from .configuration_llama import CustomLlamaConfig
+
+
+class POINTSV15ChatConfig(PretrainedConfig):
+    model_type = "pointsv1.5_chat"
+    is_composition = True
+    """Configuration class for `POINTSV1.5`."""
+
+    def __init__(self,
+                 **kwargs) -> None:
+        super().__init__(**kwargs)
+        vision_config = kwargs.pop("vision_config", None)
+        llm_config = kwargs.pop("llm_config", None)
+        if isinstance(vision_config, dict):
+            self.vision_config = Qwen2VLVisionConfig(**vision_config)
+        else:
+            self.vision_config = vision_config
+        if isinstance(llm_config, dict):
+            self.llm_config = CustomLlamaConfig(**llm_config)
+        else:
+            self.llm_config = llm_config
+
+    def to_dict(self) -> Dict[str, Any]:
+        output = copy.deepcopy(self.__dict__)
+        output["vision_config"] = self.vision_config.to_dict()
+        output["llm_config"] = self.llm_config.to_dict()
+        return output

generation_config.json

@@ -0,0 +1,4 @@
+{
+  "_from_model_config": true,
+  "transformers_version": "4.46.3"
+}

mergekit_config.yml

@@ -0,0 +1,35 @@
+models:
+  # - model: /mnt/cephfs/bensenliu/wfs/weights/mm/mmq-llava-20241123e2-sft-hf/
+  #   parameters:
+  #     weight: 1.0
+  # - model: /mnt/cephfs/bensenliu/wfs/weights/mm/mmq-llava-20241124e2-sft-hf/
+  #   parameters:
+  #     weight: 1.0
+  # - model: /mnt/cephfs/bensenliu/wfs/weights/mm/mmq-llava-20241125e1-sft-hf/
+  #   parameters:
+  #     weight: 1.0
+  # - model: /mnt/cephfs/bensenliu/wfs/weights/mm/mmq-llava-20241125e3-sft-hf/
+  #   parameters:
+  #     weight: 1.0
+  # - model: /mnt/cephfs/bensenliu/wfs/weights/mm/mmq-llava-20241127e1-sft-hf/
+  #   parameters:
+  #     weight: 1.0
+  # - model: /mnt/cephfs/bensenliu/wfs/weights/mm/mmq-llava-20241127e2-sft-hf/
+  #   parameters:
+  #     weight: 1.0
+  - model: /mnt/cephfs/bensenliu/wfs/weights/mm/mmq-llava-20241127e3-sft-pointsv15-hf/
+    parameters:
+      weight: 1.0
+  - model: /mnt/cephfs/bensenliu/wfs/weights/mm/mmq-llava-20241129e2-sft-pointsv15-hf/
+    parameters:
+      weight: 1.0
+  - model: /mnt/cephfs/bensenliu/wfs/weights/mm/mmq-llava-20241130e2-sft-pointsv15-hf/
+    parameters:
+      weight: 1.0
+  - model: /mnt/cephfs/bensenliu/wfs/weights/mm/mmq-llava-20241130e3-sft-pointsv15-hf/
+    parameters:
+      weight: 1.0
+merge_method: linear
+parameters:
+  normalize: true
+dtype: bfloat16

model-00001-of-00004.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:641cdc6784233944a3f74da9763571cfe72055357844295c820671157d7a4fc5
+size 4976697008

model-00002-of-00004.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d7bf302712686e2d0021597965f44f00b6eaafd85af6c87b0ad6e79bf0621ea4
+size 4855694728

model-00003-of-00004.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e36a0ae119dd49cb23c01663c390bc8cd9bbb11e4b920141ffab0c12677760d9
+size 4991492584

model-00004-of-00004.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8cacb4c38bd350bf39e93259fd934b3d09e3acaed679ab65dea70b873879ff20
+size 1848093520

model.safetensors.index.json

@@ -0,0 +1,659 @@
+{
+  "metadata": {
+    "total_size": 16671902720
+  },
+  "weight_map": {
+    "llm.embed_out.weight": "model-00004-of-00004.safetensors",
+    "llm.lm.embed_in.weight": "model-00001-of-00004.safetensors",
+    "llm.lm.final_layer_norm.weight": "model-00003-of-00004.safetensors",
+    "llm.lm.layers.0.attention.dense.weight": "model-00001-of-00004.safetensors",
+    "llm.lm.layers.0.attention.key_value_dense.bias": "model-00001-of-00004.safetensors",
+    "llm.lm.layers.0.attention.key_value_dense.weight": "model-00001-of-00004.safetensors",
+    "llm.lm.layers.0.attention.query_dense.bias": "model-00001-of-00004.safetensors",
+    "llm.lm.layers.0.attention.query_dense.weight": "model-00001-of-00004.safetensors",
+    "llm.lm.layers.0.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "llm.lm.layers.0.mlp.dense_4h_to_h.weight": "model-00001-of-00004.safetensors",
+    "llm.lm.layers.0.mlp.dense_h_to_4h.weight": "model-00001-of-00004.safetensors",
+    "llm.lm.layers.0.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "llm.lm.layers.1.attention.dense.weight": "model-00001-of-00004.safetensors",
+    "llm.lm.layers.1.attention.key_value_dense.bias": "model-00001-of-00004.safetensors",
+    "llm.lm.layers.1.attention.key_value_dense.weight": "model-00001-of-00004.safetensors",
+    "llm.lm.layers.1.attention.query_dense.bias": "model-00001-of-00004.safetensors",
+    "llm.lm.layers.1.attention.query_dense.weight": "model-00001-of-00004.safetensors",
+    "llm.lm.layers.1.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "llm.lm.layers.1.mlp.dense_4h_to_h.weight": "model-00001-of-00004.safetensors",
+    "llm.lm.layers.1.mlp.dense_h_to_4h.weight": "model-00001-of-00004.safetensors",
+    "llm.lm.layers.1.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "llm.lm.layers.10.attention.dense.weight": "model-00002-of-00004.safetensors",
+    "llm.lm.layers.10.attention.key_value_dense.bias": "model-00002-of-00004.safetensors",
+    "llm.lm.layers.10.attention.key_value_dense.weight": "model-00002-of-00004.safetensors",
+    "llm.lm.layers.10.attention.query_dense.bias": "model-00002-of-00004.safetensors",
+    "llm.lm.layers.10.attention.query_dense.weight": "model-00002-of-00004.safetensors",
+    "llm.lm.layers.10.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "llm.lm.layers.10.mlp.dense_4h_to_h.weight": "model-00002-of-00004.safetensors",
+    "llm.lm.layers.10.mlp.dense_h_to_4h.weight": "model-00002-of-00004.safetensors",
+    "llm.lm.layers.10.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "llm.lm.layers.11.attention.dense.weight": "model-00002-of-00004.safetensors",
+    "llm.lm.layers.11.attention.key_value_dense.bias": "model-00002-of-00004.safetensors",
+    "llm.lm.layers.11.attention.key_value_dense.weight": "model-00002-of-00004.safetensors",
+    "llm.lm.layers.11.attention.query_dense.bias": "model-00002-of-00004.safetensors",
+    "llm.lm.layers.11.attention.query_dense.weight": "model-00002-of-00004.safetensors",
+    "llm.lm.layers.11.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "llm.lm.layers.11.mlp.dense_4h_to_h.weight": "model-00002-of-00004.safetensors",
+    "llm.lm.layers.11.mlp.dense_h_to_4h.weight": "model-00002-of-00004.safetensors",
+    "llm.lm.layers.11.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "llm.lm.layers.12.attention.dense.weight": "model-00002-of-00004.safetensors",
+    "llm.lm.layers.12.attention.key_value_dense.bias": "model-00002-of-00004.safetensors",
+    "llm.lm.layers.12.attention.key_value_dense.weight": "model-00002-of-00004.safetensors",
+    "llm.lm.layers.12.attention.query_dense.bias": "model-00002-of-00004.safetensors",
+    "llm.lm.layers.12.attention.query_dense.weight": "model-00002-of-00004.safetensors",
+    "llm.lm.layers.12.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "llm.lm.layers.12.mlp.dense_4h_to_h.weight": "model-00002-of-00004.safetensors",
+    "llm.lm.layers.12.mlp.dense_h_to_4h.weight": "model-00002-of-00004.safetensors",
+    "llm.lm.layers.12.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "llm.lm.layers.13.attention.dense.weight": "model-00002-of-00004.safetensors",
+    "llm.lm.layers.13.attention.key_value_dense.bias": "model-00002-of-00004.safetensors",
+    "llm.lm.layers.13.attention.key_value_dense.weight": "model-00002-of-00004.safetensors",
+    "llm.lm.layers.13.attention.query_dense.bias": "model-00002-of-00004.safetensors",
+    "llm.lm.layers.13.attention.query_dense.weight": "model-00002-of-00004.safetensors",
+    "llm.lm.layers.13.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "llm.lm.layers.13.mlp.dense_4h_to_h.weight": "model-00002-of-00004.safetensors",
+    "llm.lm.layers.13.mlp.dense_h_to_4h.weight": "model-00002-of-00004.safetensors",
+    "llm.lm.layers.13.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "llm.lm.layers.14.attention.dense.weight": "model-00002-of-00004.safetensors",
+    "llm.lm.layers.14.attention.key_value_dense.bias": "model-00002-of-00004.safetensors",
+    "llm.lm.layers.14.attention.key_value_dense.weight": "model-00002-of-00004.safetensors",
+    "llm.lm.layers.14.attention.query_dense.bias": "model-00002-of-00004.safetensors",
+    "llm.lm.layers.14.attention.query_dense.weight": "model-00002-of-00004.safetensors",
+    "llm.lm.layers.14.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "llm.lm.layers.14.mlp.dense_4h_to_h.weight": "model-00002-of-00004.safetensors",
+    "llm.lm.layers.14.mlp.dense_h_to_4h.weight": "model-00002-of-00004.safetensors",
+    "llm.lm.layers.14.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "llm.lm.layers.15.attention.dense.weight": "model-00002-of-00004.safetensors",
+    "llm.lm.layers.15.attention.key_value_dense.bias": "model-00002-of-00004.safetensors",
+    "llm.lm.layers.15.attention.key_value_dense.weight": "model-00002-of-00004.safetensors",
+    "llm.lm.layers.15.attention.query_dense.bias": "model-00002-of-00004.safetensors",
+    "llm.lm.layers.15.attention.query_dense.weight": "model-00002-of-00004.safetensors",
+    "llm.lm.layers.15.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "llm.lm.layers.15.mlp.dense_4h_to_h.weight": "model-00002-of-00004.safetensors",
+    "llm.lm.layers.15.mlp.dense_h_to_4h.weight": "model-00002-of-00004.safetensors",
+    "llm.lm.layers.15.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "llm.lm.layers.16.attention.dense.weight": "model-00002-of-00004.safetensors",
+    "llm.lm.layers.16.attention.key_value_dense.bias": "model-00002-of-00004.safetensors",
+    "llm.lm.layers.16.attention.key_value_dense.weight": "model-00002-of-00004.safetensors",
+    "llm.lm.layers.16.attention.query_dense.bias": "model-00002-of-00004.safetensors",
+    "llm.lm.layers.16.attention.query_dense.weight": "model-00002-of-00004.safetensors",
+    "llm.lm.layers.16.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "llm.lm.layers.16.mlp.dense_4h_to_h.weight": "model-00002-of-00004.safetensors",
+    "llm.lm.layers.16.mlp.dense_h_to_4h.weight": "model-00002-of-00004.safetensors",
+    "llm.lm.layers.16.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "llm.lm.layers.17.attention.dense.weight": "model-00002-of-00004.safetensors",
+    "llm.lm.layers.17.attention.key_value_dense.bias": "model-00002-of-00004.safetensors",
+    "llm.lm.layers.17.attention.key_value_dense.weight": "model-00002-of-00004.safetensors",
+    "llm.lm.layers.17.attention.query_dense.bias": "model-00002-of-00004.safetensors",
+    "llm.lm.layers.17.attention.query_dense.weight": "model-00002-of-00004.safetensors",
+    "llm.lm.layers.17.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "llm.lm.layers.17.mlp.dense_4h_to_h.weight": "model-00002-of-00004.safetensors",
+    "llm.lm.layers.17.mlp.dense_h_to_4h.weight": "model-00002-of-00004.safetensors",
+    "llm.lm.layers.17.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "llm.lm.layers.18.attention.dense.weight": "model-00002-of-00004.safetensors",
+    "llm.lm.layers.18.attention.key_value_dense.bias": "model-00002-of-00004.safetensors",
+    "llm.lm.layers.18.attention.key_value_dense.weight": "model-00002-of-00004.safetensors",
+    "llm.lm.layers.18.attention.query_dense.bias": "model-00002-of-00004.safetensors",
+    "llm.lm.layers.18.attention.query_dense.weight": "model-00002-of-00004.safetensors",
+    "llm.lm.layers.18.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "llm.lm.layers.18.mlp.dense_4h_to_h.weight": "model-00003-of-00004.safetensors",
+    "llm.lm.layers.18.mlp.dense_h_to_4h.weight": "model-00002-of-00004.safetensors",
+    "llm.lm.layers.18.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "llm.lm.layers.19.attention.dense.weight": "model-00003-of-00004.safetensors",
+    "llm.lm.layers.19.attention.key_value_dense.bias": "model-00003-of-00004.safetensors",
+    "llm.lm.layers.19.attention.key_value_dense.weight": "model-00003-of-00004.safetensors",
+    "llm.lm.layers.19.attention.query_dense.bias": "model-00003-of-00004.safetensors",
+    "llm.lm.layers.19.attention.query_dense.weight": "model-00003-of-00004.safetensors",
+    "llm.lm.layers.19.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "llm.lm.layers.19.mlp.dense_4h_to_h.weight": "model-00003-of-00004.safetensors",
+    "llm.lm.layers.19.mlp.dense_h_to_4h.weight": "model-00003-of-00004.safetensors",
+    "llm.lm.layers.19.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "llm.lm.layers.2.attention.dense.weight": "model-00001-of-00004.safetensors",
+    "llm.lm.layers.2.attention.key_value_dense.bias": "model-00001-of-00004.safetensors",
+    "llm.lm.layers.2.attention.key_value_dense.weight": "model-00001-of-00004.safetensors",
+    "llm.lm.layers.2.attention.query_dense.bias": "model-00001-of-00004.safetensors",
+    "llm.lm.layers.2.attention.query_dense.weight": "model-00001-of-00004.safetensors",
+    "llm.lm.layers.2.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "llm.lm.layers.2.mlp.dense_4h_to_h.weight": "model-00001-of-00004.safetensors",
+    "llm.lm.layers.2.mlp.dense_h_to_4h.weight": "model-00001-of-00004.safetensors",
+    "llm.lm.layers.2.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "llm.lm.layers.20.attention.dense.weight": "model-00003-of-00004.safetensors",
+    "llm.lm.layers.20.attention.key_value_dense.bias": "model-00003-of-00004.safetensors",
+    "llm.lm.layers.20.attention.key_value_dense.weight": "model-00003-of-00004.safetensors",
+    "llm.lm.layers.20.attention.query_dense.bias": "model-00003-of-00004.safetensors",
+    "llm.lm.layers.20.attention.query_dense.weight": "model-00003-of-00004.safetensors",
+    "llm.lm.layers.20.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "llm.lm.layers.20.mlp.dense_4h_to_h.weight": "model-00003-of-00004.safetensors",
+    "llm.lm.layers.20.mlp.dense_h_to_4h.weight": "model-00003-of-00004.safetensors",
+    "llm.lm.layers.20.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "llm.lm.layers.21.attention.dense.weight": "model-00003-of-00004.safetensors",
+    "llm.lm.layers.21.attention.key_value_dense.bias": "model-00003-of-00004.safetensors",
+    "llm.lm.layers.21.attention.key_value_dense.weight": "model-00003-of-00004.safetensors",
+    "llm.lm.layers.21.attention.query_dense.bias": "model-00003-of-00004.safetensors",
+    "llm.lm.layers.21.attention.query_dense.weight": "model-00003-of-00004.safetensors",
+    "llm.lm.layers.21.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "llm.lm.layers.21.mlp.dense_4h_to_h.weight": "model-00003-of-00004.safetensors",
+    "llm.lm.layers.21.mlp.dense_h_to_4h.weight": "model-00003-of-00004.safetensors",
+    "llm.lm.layers.21.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "llm.lm.layers.22.attention.dense.weight": "model-00003-of-00004.safetensors",
+    "llm.lm.layers.22.attention.key_value_dense.bias": "model-00003-of-00004.safetensors",
+    "llm.lm.layers.22.attention.key_value_dense.weight": "model-00003-of-00004.safetensors",
+    "llm.lm.layers.22.attention.query_dense.bias": "model-00003-of-00004.safetensors",
+    "llm.lm.layers.22.attention.query_dense.weight": "model-00003-of-00004.safetensors",
+    "llm.lm.layers.22.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "llm.lm.layers.22.mlp.dense_4h_to_h.weight": "model-00003-of-00004.safetensors",
+    "llm.lm.layers.22.mlp.dense_h_to_4h.weight": "model-00003-of-00004.safetensors",
+    "llm.lm.layers.22.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "llm.lm.layers.23.attention.dense.weight": "model-00003-of-00004.safetensors",
+    "llm.lm.layers.23.attention.key_value_dense.bias": "model-00003-of-00004.safetensors",
+    "llm.lm.layers.23.attention.key_value_dense.weight": "model-00003-of-00004.safetensors",
+    "llm.lm.layers.23.attention.query_dense.bias": "model-00003-of-00004.safetensors",
+    "llm.lm.layers.23.attention.query_dense.weight": "model-00003-of-00004.safetensors",
+    "llm.lm.layers.23.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "llm.lm.layers.23.mlp.dense_4h_to_h.weight": "model-00003-of-00004.safetensors",
+    "llm.lm.layers.23.mlp.dense_h_to_4h.weight": "model-00003-of-00004.safetensors",
+    "llm.lm.layers.23.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "llm.lm.layers.24.attention.dense.weight": "model-00003-of-00004.safetensors",
+    "llm.lm.layers.24.attention.key_value_dense.bias": "model-00003-of-00004.safetensors",
+    "llm.lm.layers.24.attention.key_value_dense.weight": "model-00003-of-00004.safetensors",
+    "llm.lm.layers.24.attention.query_dense.bias": "model-00003-of-00004.safetensors",
+    "llm.lm.layers.24.attention.query_dense.weight": "model-00003-of-00004.safetensors",
+    "llm.lm.layers.24.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "llm.lm.layers.24.mlp.dense_4h_to_h.weight": "model-00003-of-00004.safetensors",
+    "llm.lm.layers.24.mlp.dense_h_to_4h.weight": "model-00003-of-00004.safetensors",
+    "llm.lm.layers.24.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "llm.lm.layers.25.attention.dense.weight": "model-00003-of-00004.safetensors",
+    "llm.lm.layers.25.attention.key_value_dense.bias": "model-00003-of-00004.safetensors",
+    "llm.lm.layers.25.attention.key_value_dense.weight": "model-00003-of-00004.safetensors",
+    "llm.lm.layers.25.attention.query_dense.bias": "model-00003-of-00004.safetensors",
+    "llm.lm.layers.25.attention.query_dense.weight": "model-00003-of-00004.safetensors",
+    "llm.lm.layers.25.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "llm.lm.layers.25.mlp.dense_4h_to_h.weight": "model-00003-of-00004.safetensors",
+    "llm.lm.layers.25.mlp.dense_h_to_4h.weight": "model-00003-of-00004.safetensors",
+    "llm.lm.layers.25.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "llm.lm.layers.26.attention.dense.weight": "model-00003-of-00004.safetensors",
+    "llm.lm.layers.26.attention.key_value_dense.bias": "model-00003-of-00004.safetensors",
+    "llm.lm.layers.26.attention.key_value_dense.weight": "model-00003-of-00004.safetensors",
+    "llm.lm.layers.26.attention.query_dense.bias": "model-00003-of-00004.safetensors",
+    "llm.lm.layers.26.attention.query_dense.weight": "model-00003-of-00004.safetensors",
+    "llm.lm.layers.26.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "llm.lm.layers.26.mlp.dense_4h_to_h.weight": "model-00003-of-00004.safetensors",
+    "llm.lm.layers.26.mlp.dense_h_to_4h.weight": "model-00003-of-00004.safetensors",
+    "llm.lm.layers.26.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "llm.lm.layers.27.attention.dense.weight": "model-00003-of-00004.safetensors",
+    "llm.lm.layers.27.attention.key_value_dense.bias": "model-00003-of-00004.safetensors",
+    "llm.lm.layers.27.attention.key_value_dense.weight": "model-00003-of-00004.safetensors",
+    "llm.lm.layers.27.attention.query_dense.bias": "model-00003-of-00004.safetensors",
+    "llm.lm.layers.27.attention.query_dense.weight": "model-00003-of-00004.safetensors",
+    "llm.lm.layers.27.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "llm.lm.layers.27.mlp.dense_4h_to_h.weight": "model-00003-of-00004.safetensors",
+    "llm.lm.layers.27.mlp.dense_h_to_4h.weight": "model-00003-of-00004.safetensors",
+    "llm.lm.layers.27.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "llm.lm.layers.3.attention.dense.weight": "model-00001-of-00004.safetensors",
+    "llm.lm.layers.3.attention.key_value_dense.bias": "model-00001-of-00004.safetensors",
+    "llm.lm.layers.3.attention.key_value_dense.weight": "model-00001-of-00004.safetensors",
+    "llm.lm.layers.3.attention.query_dense.bias": "model-00001-of-00004.safetensors",
+    "llm.lm.layers.3.attention.query_dense.weight": "model-00001-of-00004.safetensors",
+    "llm.lm.layers.3.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "llm.lm.layers.3.mlp.dense_4h_to_h.weight": "model-00001-of-00004.safetensors",
+    "llm.lm.layers.3.mlp.dense_h_to_4h.weight": "model-00001-of-00004.safetensors",
+    "llm.lm.layers.3.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "llm.lm.layers.4.attention.dense.weight": "model-00001-of-00004.safetensors",
+    "llm.lm.layers.4.attention.key_value_dense.bias": "model-00001-of-00004.safetensors",
+    "llm.lm.layers.4.attention.key_value_dense.weight": "model-00001-of-00004.safetensors",
+    "llm.lm.layers.4.attention.query_dense.bias": "model-00001-of-00004.safetensors",
+    "llm.lm.layers.4.attention.query_dense.weight": "model-00001-of-00004.safetensors",
+    "llm.lm.layers.4.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "llm.lm.layers.4.mlp.dense_4h_to_h.weight": "model-00001-of-00004.safetensors",
+    "llm.lm.layers.4.mlp.dense_h_to_4h.weight": "model-00001-of-00004.safetensors",
+    "llm.lm.layers.4.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "llm.lm.layers.5.attention.dense.weight": "model-00001-of-00004.safetensors",
+    "llm.lm.layers.5.attention.key_value_dense.bias": "model-00001-of-00004.safetensors",
+    "llm.lm.layers.5.attention.key_value_dense.weight": "model-00001-of-00004.safetensors",
+    "llm.lm.layers.5.attention.query_dense.bias": "model-00001-of-00004.safetensors",
+    "llm.lm.layers.5.attention.query_dense.weight": "model-00001-of-00004.safetensors",
+    "llm.lm.layers.5.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "llm.lm.layers.5.mlp.dense_4h_to_h.weight": "model-00001-of-00004.safetensors",
+    "llm.lm.layers.5.mlp.dense_h_to_4h.weight": "model-00001-of-00004.safetensors",
+    "llm.lm.layers.5.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "llm.lm.layers.6.attention.dense.weight": "model-00001-of-00004.safetensors",
+    "llm.lm.layers.6.attention.key_value_dense.bias": "model-00001-of-00004.safetensors",
+    "llm.lm.layers.6.attention.key_value_dense.weight": "model-00001-of-00004.safetensors",
+    "llm.lm.layers.6.attention.query_dense.bias": "model-00001-of-00004.safetensors",
+    "llm.lm.layers.6.attention.query_dense.weight": "model-00001-of-00004.safetensors",
+    "llm.lm.layers.6.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "llm.lm.layers.6.mlp.dense_4h_to_h.weight": "model-00001-of-00004.safetensors",
+    "llm.lm.layers.6.mlp.dense_h_to_4h.weight": "model-00001-of-00004.safetensors",
+    "llm.lm.layers.6.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "llm.lm.layers.7.attention.dense.weight": "model-00001-of-00004.safetensors",
+    "llm.lm.layers.7.attention.key_value_dense.bias": "model-00001-of-00004.safetensors",
+    "llm.lm.layers.7.attention.key_value_dense.weight": "model-00001-of-00004.safetensors",
+    "llm.lm.layers.7.attention.query_dense.bias": "model-00001-of-00004.safetensors",
+    "llm.lm.layers.7.attention.query_dense.weight": "model-00001-of-00004.safetensors",
+    "llm.lm.layers.7.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "llm.lm.layers.7.mlp.dense_4h_to_h.weight": "model-00001-of-00004.safetensors",
+    "llm.lm.layers.7.mlp.dense_h_to_4h.weight": "model-00001-of-00004.safetensors",
+    "llm.lm.layers.7.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "llm.lm.layers.8.attention.dense.weight": "model-00001-of-00004.safetensors",
+    "llm.lm.layers.8.attention.key_value_dense.bias": "model-00001-of-00004.safetensors",
+    "llm.lm.layers.8.attention.key_value_dense.weight": "model-00001-of-00004.safetensors",
+    "llm.lm.layers.8.attention.query_dense.bias": "model-00001-of-00004.safetensors",
+    "llm.lm.layers.8.attention.query_dense.weight": "model-00001-of-00004.safetensors",
+    "llm.lm.layers.8.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "llm.lm.layers.8.mlp.dense_4h_to_h.weight": "model-00002-of-00004.safetensors",
+    "llm.lm.layers.8.mlp.dense_h_to_4h.weight": "model-00002-of-00004.safetensors",
+    "llm.lm.layers.8.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "llm.lm.layers.9.attention.dense.weight": "model-00002-of-00004.safetensors",
+    "llm.lm.layers.9.attention.key_value_dense.bias": "model-00002-of-00004.safetensors",
+    "llm.lm.layers.9.attention.key_value_dense.weight": "model-00002-of-00004.safetensors",
+    "llm.lm.layers.9.attention.query_dense.bias": "model-00002-of-00004.safetensors",
+    "llm.lm.layers.9.attention.query_dense.weight": "model-00002-of-00004.safetensors",
+    "llm.lm.layers.9.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "llm.lm.layers.9.mlp.dense_4h_to_h.weight": "model-00002-of-00004.safetensors",
+    "llm.lm.layers.9.mlp.dense_h_to_4h.weight": "model-00002-of-00004.safetensors",
+    "llm.lm.layers.9.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "vision_encoder.blocks.0.attn.proj.bias": "model-00004-of-00004.safetensors",
+    "vision_encoder.blocks.0.attn.proj.weight": "model-00004-of-00004.safetensors",
+    "vision_encoder.blocks.0.attn.qkv.bias": "model-00004-of-00004.safetensors",
+    "vision_encoder.blocks.0.attn.qkv.weight": "model-00004-of-00004.safetensors",
+    "vision_encoder.blocks.0.mlp.fc1.bias": "model-00004-of-00004.safetensors",
+    "vision_encoder.blocks.0.mlp.fc1.weight": "model-00004-of-00004.safetensors",
+    "vision_encoder.blocks.0.mlp.fc2.bias": "model-00004-of-00004.safetensors",
+    "vision_encoder.blocks.0.mlp.fc2.weight": "model-00004-of-00004.safetensors",
+    "vision_encoder.blocks.0.norm1.bias": "model-00004-of-00004.safetensors",
+    "vision_encoder.blocks.0.norm1.weight": "model-00004-of-00004.safetensors",
+    "vision_encoder.blocks.0.norm2.bias": "model-00004-of-00004.safetensors",
+    "vision_encoder.blocks.0.norm2.weight": "model-00004-of-00004.safetensors",
+    "vision_encoder.blocks.1.attn.proj.bias": "model-00004-of-00004.safetensors",
+    "vision_encoder.blocks.1.attn.proj.weight": "model-00004-of-00004.safetensors",
+    "vision_encoder.blocks.1.attn.qkv.bias": "model-00004-of-00004.safetensors",
+    "vision_encoder.blocks.1.attn.qkv.weight": "model-00004-of-00004.safetensors",
+    "vision_encoder.blocks.1.mlp.fc1.bias": "model-00004-of-00004.safetensors",
+    "vision_encoder.blocks.1.mlp.fc1.weight": "model-00004-of-00004.safetensors",
+    "vision_encoder.blocks.1.mlp.fc2.bias": "model-00004-of-00004.safetensors",
+    "vision_encoder.blocks.1.mlp.fc2.weight": "model-00004-of-00004.safetensors",
+    "vision_encoder.blocks.1.norm1.bias": "model-00004-of-00004.safetensors",
+    "vision_encoder.blocks.1.norm1.weight": "model-00004-of-00004.safetensors",
+    "vision_encoder.blocks.1.norm2.bias": "model-00004-of-00004.safetensors",
+    "vision_encoder.blocks.1.norm2.weight": "model-00004-of-00004.safetensors",
+    "vision_encoder.blocks.10.attn.proj.bias": "model-00004-of-00004.safetensors",
+    "vision_encoder.blocks.10.attn.proj.weight": "model-00004-of-00004.safetensors",
+    "vision_encoder.blocks.10.attn.qkv.bias": "model-00004-of-00004.safetensors",
+    "vision_encoder.blocks.10.attn.qkv.weight": "model-00004-of-00004.safetensors",
+    "vision_encoder.blocks.10.mlp.fc1.bias": "model-00004-of-00004.safetensors",
+    "vision_encoder.blocks.10.mlp.fc1.weight": "model-00004-of-00004.safetensors",
+    "vision_encoder.blocks.10.mlp.fc2.bias": "model-00004-of-00004.safetensors",
+    "vision_encoder.blocks.10.mlp.fc2.weight": "model-00004-of-00004.safetensors",
+    "vision_encoder.blocks.10.norm1.bias": "model-00004-of-00004.safetensors",
+    "vision_encoder.blocks.10.norm1.weight": "model-00004-of-00004.safetensors",
+    "vision_encoder.blocks.10.norm2.bias": "model-00004-of-00004.safetensors",
+    "vision_encoder.blocks.10.norm2.weight": "model-00004-of-00004.safetensors",
+    "vision_encoder.blocks.11.attn.proj.bias": "model-00004-of-00004.safetensors",
+    "vision_encoder.blocks.11.attn.proj.weight": "model-00004-of-00004.safetensors",
+    "vision_encoder.blocks.11.attn.qkv.bias": "model-00004-of-00004.safetensors",
+    "vision_encoder.blocks.11.attn.qkv.weight": "model-00004-of-00004.safetensors",
+    "vision_encoder.blocks.11.mlp.fc1.bias": "model-00004-of-00004.safetensors",
+    "vision_encoder.blocks.11.mlp.fc1.weight": "model-00004-of-00004.safetensors",
+    "vision_encoder.blocks.11.mlp.fc2.bias": "model-00004-of-00004.safetensors",
+    "vision_encoder.blocks.11.mlp.fc2.weight": "model-00004-of-00004.safetensors",
+    "vision_encoder.blocks.11.norm1.bias": "model-00004-of-00004.safetensors",
+    "vision_encoder.blocks.11.norm1.weight": "model-00004-of-00004.safetensors",
+    "vision_encoder.blocks.11.norm2.bias": "model-00004-of-00004.safetensors",
+    "vision_encoder.blocks.11.norm2.weight": "model-00004-of-00004.safetensors",
+    "vision_encoder.blocks.12.attn.proj.bias": "model-00004-of-00004.safetensors",
+    "vision_encoder.blocks.12.attn.proj.weight": "model-00004-of-00004.safetensors",
+    "vision_encoder.blocks.12.attn.qkv.bias": "model-00004-of-00004.safetensors",
+    "vision_encoder.blocks.12.attn.qkv.weight": "model-00004-of-00004.safetensors",
+    "vision_encoder.blocks.12.mlp.fc1.bias": "model-00004-of-00004.safetensors",
+    "vision_encoder.blocks.12.mlp.fc1.weight": "model-00004-of-00004.safetensors",
+    "vision_encoder.blocks.12.mlp.fc2.bias": "model-00004-of-00004.safetensors",
+    "vision_encoder.blocks.12.mlp.fc2.weight": "model-00004-of-00004.safetensors",
+    "vision_encoder.blocks.12.norm1.bias": "model-00004-of-00004.safetensors",
+    "vision_encoder.blocks.12.norm1.weight": "model-00004-of-00004.safetensors",
+    "vision_encoder.blocks.12.norm2.bias": "model-00004-of-00004.safetensors",
+    "vision_encoder.blocks.12.norm2.weight": "model-00004-of-00004.safetensors",
+    "vision_encoder.blocks.13.attn.proj.bias": "model-00004-of-00004.safetensors",
+    "vision_encoder.blocks.13.attn.proj.weight": "model-00004-of-00004.safetensors",
+    "vision_encoder.blocks.13.attn.qkv.bias": "model-00004-of-00004.safetensors",
+    "vision_encoder.blocks.13.attn.qkv.weight": "model-00004-of-00004.safetensors",
+    "vision_encoder.blocks.13.mlp.fc1.bias": "model-00004-of-00004.safetensors",
+    "vision_encoder.blocks.13.mlp.fc1.weight": "model-00004-of-00004.safetensors",
+    "vision_encoder.blocks.13.mlp.fc2.bias": "model-00004-of-00004.safetensors",
+    "vision_encoder.blocks.13.mlp.fc2.weight": "model-00004-of-00004.safetensors",
+    "vision_encoder.blocks.13.norm1.bias": "model-00004-of-00004.safetensors",
+    "vision_encoder.blocks.13.norm1.weight": "model-00004-of-00004.safetensors",
+    "vision_encoder.blocks.13.norm2.bias": "model-00004-of-00004.safetensors",
+    "vision_encoder.blocks.13.norm2.weight": "model-00004-of-00004.safetensors",
+    "vision_encoder.blocks.14.attn.proj.bias": "model-00004-of-00004.safetensors",
+    "vision_encoder.blocks.14.attn.proj.weight": "model-00004-of-00004.safetensors",
+    "vision_encoder.blocks.14.attn.qkv.bias": "model-00004-of-00004.safetensors",
+    "vision_encoder.blocks.14.attn.qkv.weight": "model-00004-of-00004.safetensors",
+    "vision_encoder.blocks.14.mlp.fc1.bias": "model-00004-of-00004.safetensors",
+    "vision_encoder.blocks.14.mlp.fc1.weight": "model-00004-of-00004.safetensors",
+    "vision_encoder.blocks.14.mlp.fc2.bias": "model-00004-of-00004.safetensors",
+    "vision_encoder.blocks.14.mlp.fc2.weight": "model-00004-of-00004.safetensors",
+    "vision_encoder.blocks.14.norm1.bias": "model-00004-of-00004.safetensors",
+    "vision_encoder.blocks.14.norm1.weight": "model-00004-of-00004.safetensors",
+    "vision_encoder.blocks.14.norm2.bias": "model-00004-of-00004.safetensors",
+    "vision_encoder.blocks.14.norm2.weight": "model-00004-of-00004.safetensors",
+    "vision_encoder.blocks.15.attn.proj.bias": "model-00004-of-00004.safetensors",
+    "vision_encoder.blocks.15.attn.proj.weight": "model-00004-of-00004.safetensors",
+    "vision_encoder.blocks.15.attn.qkv.bias": "model-00004-of-00004.safetensors",
+    "vision_encoder.blocks.15.attn.qkv.weight": "model-00004-of-00004.safetensors",
+    "vision_encoder.blocks.15.mlp.fc1.bias": "model-00004-of-00004.safetensors",
+    "vision_encoder.blocks.15.mlp.fc1.weight": "model-00004-of-00004.safetensors",
+    "vision_encoder.blocks.15.mlp.fc2.bias": "model-00004-of-00004.safetensors",
+    "vision_encoder.blocks.15.mlp.fc2.weight": "model-00004-of-00004.safetensors",
+    "vision_encoder.blocks.15.norm1.bias": "model-00004-of-00004.safetensors",
+    "vision_encoder.blocks.15.norm1.weight": "model-00004-of-00004.safetensors",
+    "vision_encoder.blocks.15.norm2.bias": "model-00004-of-00004.safetensors",
+    "vision_encoder.blocks.15.norm2.weight": "model-00004-of-00004.safetensors",
+    "vision_encoder.blocks.16.attn.proj.bias": "model-00004-of-00004.safetensors",
+    "vision_encoder.blocks.16.attn.proj.weight": "model-00004-of-00004.safetensors",
+    "vision_encoder.blocks.16.attn.qkv.bias": "model-00004-of-00004.safetensors",
+    "vision_encoder.blocks.16.attn.qkv.weight": "model-00004-of-00004.safetensors",
+    "vision_encoder.blocks.16.mlp.fc1.bias": "model-00004-of-00004.safetensors",
+    "vision_encoder.blocks.16.mlp.fc1.weight": "model-00004-of-00004.safetensors",
+    "vision_encoder.blocks.16.mlp.fc2.bias": "model-00004-of-00004.safetensors",
+    "vision_encoder.blocks.16.mlp.fc2.weight": "model-00004-of-00004.safetensors",
+    "vision_encoder.blocks.16.norm1.bias": "model-00004-of-00004.safetensors",
+    "vision_encoder.blocks.16.norm1.weight": "model-00004-of-00004.safetensors",
+    "vision_encoder.blocks.16.norm2.bias": "model-00004-of-00004.safetensors",
+    "vision_encoder.blocks.16.norm2.weight": "model-00004-of-00004.safetensors",
+    "vision_encoder.blocks.17.attn.proj.bias": "model-00004-of-00004.safetensors",
+    "vision_encoder.blocks.17.attn.proj.weight": "model-00004-of-00004.safetensors",
+    "vision_encoder.blocks.17.attn.qkv.bias": "model-00004-of-00004.safetensors",
+    "vision_encoder.blocks.17.attn.qkv.weight": "model-00004-of-00004.safetensors",
+    "vision_encoder.blocks.17.mlp.fc1.bias": "model-00004-of-00004.safetensors",
+    "vision_encoder.blocks.17.mlp.fc1.weight": "model-00004-of-00004.safetensors",
+    "vision_encoder.blocks.17.mlp.fc2.bias": "model-00004-of-00004.safetensors",
+    "vision_encoder.blocks.17.mlp.fc2.weight": "model-00004-of-00004.safetensors",
+    "vision_encoder.blocks.17.norm1.bias": "model-00004-of-00004.safetensors",
+    "vision_encoder.blocks.17.norm1.weight": "model-00004-of-00004.safetensors",
+    "vision_encoder.blocks.17.norm2.bias": "model-00004-of-00004.safetensors",
+    "vision_encoder.blocks.17.norm2.weight": "model-00004-of-00004.safetensors",
+    "vision_encoder.blocks.18.attn.proj.bias": "model-00004-of-00004.safetensors",
+    "vision_encoder.blocks.18.attn.proj.weight": "model-00004-of-00004.safetensors",
+    "vision_encoder.blocks.18.attn.qkv.bias": "model-00004-of-00004.safetensors",
+    "vision_encoder.blocks.18.attn.qkv.weight": "model-00004-of-00004.safetensors",
+    "vision_encoder.blocks.18.mlp.fc1.bias": "model-00004-of-00004.safetensors",
+    "vision_encoder.blocks.18.mlp.fc1.weight": "model-00004-of-00004.safetensors",
+    "vision_encoder.blocks.18.mlp.fc2.bias": "model-00004-of-00004.safetensors",
+    "vision_encoder.blocks.18.mlp.fc2.weight": "model-00004-of-00004.safetensors",
+    "vision_encoder.blocks.18.norm1.bias": "model-00004-of-00004.safetensors",
+    "vision_encoder.blocks.18.norm1.weight": "model-00004-of-00004.safetensors",
+    "vision_encoder.blocks.18.norm2.bias": "model-00004-of-00004.safetensors",
+    "vision_encoder.blocks.18.norm2.weight": "model-00004-of-00004.safetensors",
+    "vision_encoder.blocks.19.attn.proj.bias": "model-00004-of-00004.safetensors",
+    "vision_encoder.blocks.19.attn.proj.weight": "model-00004-of-00004.safetensors",
+    "vision_encoder.blocks.19.attn.qkv.bias": "model-00004-of-00004.safetensors",
+    "vision_encoder.blocks.19.attn.qkv.weight": "model-00004-of-00004.safetensors",
+    "vision_encoder.blocks.19.mlp.fc1.bias": "model-00004-of-00004.safetensors",
+    "vision_encoder.blocks.19.mlp.fc1.weight": "model-00004-of-00004.safetensors",
+    "vision_encoder.blocks.19.mlp.fc2.bias": "model-00004-of-00004.safetensors",
+    "vision_encoder.blocks.19.mlp.fc2.weight": "model-00004-of-00004.safetensors",
+    "vision_encoder.blocks.19.norm1.bias": "model-00004-of-00004.safetensors",
+    "vision_encoder.blocks.19.norm1.weight": "model-00004-of-00004.safetensors",
+    "vision_encoder.blocks.19.norm2.bias": "model-00004-of-00004.safetensors",
+    "vision_encoder.blocks.19.norm2.weight": "model-00004-of-00004.safetensors",
+    "vision_encoder.blocks.2.attn.proj.bias": "model-00004-of-00004.safetensors",
+    "vision_encoder.blocks.2.attn.proj.weight": "model-00004-of-00004.safetensors",
+    "vision_encoder.blocks.2.attn.qkv.bias": "model-00004-of-00004.safetensors",
+    "vision_encoder.blocks.2.attn.qkv.weight": "model-00004-of-00004.safetensors",
+    "vision_encoder.blocks.2.mlp.fc1.bias": "model-00004-of-00004.safetensors",
+    "vision_encoder.blocks.2.mlp.fc1.weight": "model-00004-of-00004.safetensors",
+    "vision_encoder.blocks.2.mlp.fc2.bias": "model-00004-of-00004.safetensors",
+    "vision_encoder.blocks.2.mlp.fc2.weight": "model-00004-of-00004.safetensors",
+    "vision_encoder.blocks.2.norm1.bias": "model-00004-of-00004.safetensors",
+    "vision_encoder.blocks.2.norm1.weight": "model-00004-of-00004.safetensors",
+    "vision_encoder.blocks.2.norm2.bias": "model-00004-of-00004.safetensors",
+    "vision_encoder.blocks.2.norm2.weight": "model-00004-of-00004.safetensors",
+    "vision_encoder.blocks.20.attn.proj.bias": "model-00004-of-00004.safetensors",
+    "vision_encoder.blocks.20.attn.proj.weight": "model-00004-of-00004.safetensors",
+    "vision_encoder.blocks.20.attn.qkv.bias": "model-00004-of-00004.safetensors",
+    "vision_encoder.blocks.20.attn.qkv.weight": "model-00004-of-00004.safetensors",
+    "vision_encoder.blocks.20.mlp.fc1.bias": "model-00004-of-00004.safetensors",
+    "vision_encoder.blocks.20.mlp.fc1.weight": "model-00004-of-00004.safetensors",
+    "vision_encoder.blocks.20.mlp.fc2.bias": "model-00004-of-00004.safetensors",
+    "vision_encoder.blocks.20.mlp.fc2.weight": "model-00004-of-00004.safetensors",
+    "vision_encoder.blocks.20.norm1.bias": "model-00004-of-00004.safetensors",
+    "vision_encoder.blocks.20.norm1.weight": "model-00004-of-00004.safetensors",
+    "vision_encoder.blocks.20.norm2.bias": "model-00004-of-00004.safetensors",
+    "vision_encoder.blocks.20.norm2.weight": "model-00004-of-00004.safetensors",
+    "vision_encoder.blocks.21.attn.proj.bias": "model-00004-of-00004.safetensors",
+    "vision_encoder.blocks.21.attn.proj.weight": "model-00004-of-00004.safetensors",
+    "vision_encoder.blocks.21.attn.qkv.bias": "model-00004-of-00004.safetensors",
+    "vision_encoder.blocks.21.attn.qkv.weight": "model-00004-of-00004.safetensors",
+    "vision_encoder.blocks.21.mlp.fc1.bias": "model-00004-of-00004.safetensors",
+    "vision_encoder.blocks.21.mlp.fc1.weight": "model-00004-of-00004.safetensors",
+    "vision_encoder.blocks.21.mlp.fc2.bias": "model-00004-of-00004.safetensors",
+    "vision_encoder.blocks.21.mlp.fc2.weight": "model-00004-of-00004.safetensors",
+    "vision_encoder.blocks.21.norm1.bias": "model-00004-of-00004.safetensors",
+    "vision_encoder.blocks.21.norm1.weight": "model-00004-of-00004.safetensors",
+    "vision_encoder.blocks.21.norm2.bias": "model-00004-of-00004.safetensors",
+    "vision_encoder.blocks.21.norm2.weight": "model-00004-of-00004.safetensors",
+    "vision_encoder.blocks.22.attn.proj.bias": "model-00004-of-00004.safetensors",
+    "vision_encoder.blocks.22.attn.proj.weight": "model-00004-of-00004.safetensors",
+    "vision_encoder.blocks.22.attn.qkv.bias": "model-00004-of-00004.safetensors",
+    "vision_encoder.blocks.22.attn.qkv.weight": "model-00004-of-00004.safetensors",
+    "vision_encoder.blocks.22.mlp.fc1.bias": "model-00004-of-00004.safetensors",
+    "vision_encoder.blocks.22.mlp.fc1.weight": "model-00004-of-00004.safetensors",
+    "vision_encoder.blocks.22.mlp.fc2.bias": "model-00004-of-00004.safetensors",
+    "vision_encoder.blocks.22.mlp.fc2.weight": "model-00004-of-00004.safetensors",
+    "vision_encoder.blocks.22.norm1.bias": "model-00004-of-00004.safetensors",
+    "vision_encoder.blocks.22.norm1.weight": "model-00004-of-00004.safetensors",
+    "vision_encoder.blocks.22.norm2.bias": "model-00004-of-00004.safetensors",
+    "vision_encoder.blocks.22.norm2.weight": "model-00004-of-00004.safetensors",
+    "vision_encoder.blocks.23.attn.proj.bias": "model-00004-of-00004.safetensors",
+    "vision_encoder.blocks.23.attn.proj.weight": "model-00004-of-00004.safetensors",
+    "vision_encoder.blocks.23.attn.qkv.bias": "model-00004-of-00004.safetensors",
+    "vision_encoder.blocks.23.attn.qkv.weight": "model-00004-of-00004.safetensors",
+    "vision_encoder.blocks.23.mlp.fc1.bias": "model-00004-of-00004.safetensors",
+    "vision_encoder.blocks.23.mlp.fc1.weight": "model-00004-of-00004.safetensors",
+    "vision_encoder.blocks.23.mlp.fc2.bias": "model-00004-of-00004.safetensors",
+    "vision_encoder.blocks.23.mlp.fc2.weight": "model-00004-of-00004.safetensors",
+    "vision_encoder.blocks.23.norm1.bias": "model-00004-of-00004.safetensors",
+    "vision_encoder.blocks.23.norm1.weight": "model-00004-of-00004.safetensors",
+    "vision_encoder.blocks.23.norm2.bias": "model-00004-of-00004.safetensors",
+    "vision_encoder.blocks.23.norm2.weight": "model-00004-of-00004.safetensors",
+    "vision_encoder.blocks.24.attn.proj.bias": "model-00004-of-00004.safetensors",
+    "vision_encoder.blocks.24.attn.proj.weight": "model-00004-of-00004.safetensors",
+    "vision_encoder.blocks.24.attn.qkv.bias": "model-00004-of-00004.safetensors",
+    "vision_encoder.blocks.24.attn.qkv.weight": "model-00004-of-00004.safetensors",
+    "vision_encoder.blocks.24.mlp.fc1.bias": "model-00004-of-00004.safetensors",
+    "vision_encoder.blocks.24.mlp.fc1.weight": "model-00004-of-00004.safetensors",
+    "vision_encoder.blocks.24.mlp.fc2.bias": "model-00004-of-00004.safetensors",
+    "vision_encoder.blocks.24.mlp.fc2.weight": "model-00004-of-00004.safetensors",
+    "vision_encoder.blocks.24.norm1.bias": "model-00004-of-00004.safetensors",
+    "vision_encoder.blocks.24.norm1.weight": "model-00004-of-00004.safetensors",
+    "vision_encoder.blocks.24.norm2.bias": "model-00004-of-00004.safetensors",
+    "vision_encoder.blocks.24.norm2.weight": "model-00004-of-00004.safetensors",
+    "vision_encoder.blocks.25.attn.proj.bias": "model-00004-of-00004.safetensors",
+    "vision_encoder.blocks.25.attn.proj.weight": "model-00004-of-00004.safetensors",
+    "vision_encoder.blocks.25.attn.qkv.bias": "model-00004-of-00004.safetensors",
+    "vision_encoder.blocks.25.attn.qkv.weight": "model-00004-of-00004.safetensors",
+    "vision_encoder.blocks.25.mlp.fc1.bias": "model-00004-of-00004.safetensors",
+    "vision_encoder.blocks.25.mlp.fc1.weight": "model-00004-of-00004.safetensors",
+    "vision_encoder.blocks.25.mlp.fc2.bias": "model-00004-of-00004.safetensors",
+    "vision_encoder.blocks.25.mlp.fc2.weight": "model-00004-of-00004.safetensors",
+    "vision_encoder.blocks.25.norm1.bias": "model-00004-of-00004.safetensors",
+    "vision_encoder.blocks.25.norm1.weight": "model-00004-of-00004.safetensors",
+    "vision_encoder.blocks.25.norm2.bias": "model-00004-of-00004.safetensors",
+    "vision_encoder.blocks.25.norm2.weight": "model-00004-of-00004.safetensors",
+    "vision_encoder.blocks.26.attn.proj.bias": "model-00004-of-00004.safetensors",
+    "vision_encoder.blocks.26.attn.proj.weight": "model-00004-of-00004.safetensors",
+    "vision_encoder.blocks.26.attn.qkv.bias": "model-00004-of-00004.safetensors",
+    "vision_encoder.blocks.26.attn.qkv.weight": "model-00004-of-00004.safetensors",
+    "vision_encoder.blocks.26.mlp.fc1.bias": "model-00004-of-00004.safetensors",
+    "vision_encoder.blocks.26.mlp.fc1.weight": "model-00004-of-00004.safetensors",
+    "vision_encoder.blocks.26.mlp.fc2.bias": "model-00004-of-00004.safetensors",
+    "vision_encoder.blocks.26.mlp.fc2.weight": "model-00004-of-00004.safetensors",
+    "vision_encoder.blocks.26.norm1.bias": "model-00004-of-00004.safetensors",
+    "vision_encoder.blocks.26.norm1.weight": "model-00004-of-00004.safetensors",
+    "vision_encoder.blocks.26.norm2.bias": "model-00004-of-00004.safetensors",
+    "vision_encoder.blocks.26.norm2.weight": "model-00004-of-00004.safetensors",
+    "vision_encoder.blocks.27.attn.proj.bias": "model-00004-of-00004.safetensors",
+    "vision_encoder.blocks.27.attn.proj.weight": "model-00004-of-00004.safetensors",
+    "vision_encoder.blocks.27.attn.qkv.bias": "model-00004-of-00004.safetensors",
+    "vision_encoder.blocks.27.attn.qkv.weight": "model-00004-of-00004.safetensors",
+    "vision_encoder.blocks.27.mlp.fc1.bias": "model-00004-of-00004.safetensors",
+    "vision_encoder.blocks.27.mlp.fc1.weight": "model-00004-of-00004.safetensors",
+    "vision_encoder.blocks.27.mlp.fc2.bias": "model-00004-of-00004.safetensors",
+    "vision_encoder.blocks.27.mlp.fc2.weight": "model-00004-of-00004.safetensors",
+    "vision_encoder.blocks.27.norm1.bias": "model-00004-of-00004.safetensors",
+    "vision_encoder.blocks.27.norm1.weight": "model-00004-of-00004.safetensors",
+    "vision_encoder.blocks.27.norm2.bias": "model-00004-of-00004.safetensors",
+    "vision_encoder.blocks.27.norm2.weight": "model-00004-of-00004.safetensors",
+    "vision_encoder.blocks.28.attn.proj.bias": "model-00004-of-00004.safetensors",
+    "vision_encoder.blocks.28.attn.proj.weight": "model-00004-of-00004.safetensors",
+    "vision_encoder.blocks.28.attn.qkv.bias": "model-00004-of-00004.safetensors",
+    "vision_encoder.blocks.28.attn.qkv.weight": "model-00004-of-00004.safetensors",
+    "vision_encoder.blocks.28.mlp.fc1.bias": "model-00004-of-00004.safetensors",
+    "vision_encoder.blocks.28.mlp.fc1.weight": "model-00004-of-00004.safetensors",
+    "vision_encoder.blocks.28.mlp.fc2.bias": "model-00004-of-00004.safetensors",
+    "vision_encoder.blocks.28.mlp.fc2.weight": "model-00004-of-00004.safetensors",
+    "vision_encoder.blocks.28.norm1.bias": "model-00004-of-00004.safetensors",
+    "vision_encoder.blocks.28.norm1.weight": "model-00004-of-00004.safetensors",
+    "vision_encoder.blocks.28.norm2.bias": "model-00004-of-00004.safetensors",
+    "vision_encoder.blocks.28.norm2.weight": "model-00004-of-00004.safetensors",
+    "vision_encoder.blocks.29.attn.proj.bias": "model-00004-of-00004.safetensors",
+    "vision_encoder.blocks.29.attn.proj.weight": "model-00004-of-00004.safetensors",
+    "vision_encoder.blocks.29.attn.qkv.bias": "model-00004-of-00004.safetensors",
+    "vision_encoder.blocks.29.attn.qkv.weight": "model-00004-of-00004.safetensors",
+    "vision_encoder.blocks.29.mlp.fc1.bias": "model-00004-of-00004.safetensors",
+    "vision_encoder.blocks.29.mlp.fc1.weight": "model-00004-of-00004.safetensors",
+    "vision_encoder.blocks.29.mlp.fc2.bias": "model-00004-of-00004.safetensors",
+    "vision_encoder.blocks.29.mlp.fc2.weight": "model-00004-of-00004.safetensors",
+    "vision_encoder.blocks.29.norm1.bias": "model-00004-of-00004.safetensors",
+    "vision_encoder.blocks.29.norm1.weight": "model-00004-of-00004.safetensors",
+    "vision_encoder.blocks.29.norm2.bias": "model-00004-of-00004.safetensors",
+    "vision_encoder.blocks.29.norm2.weight": "model-00004-of-00004.safetensors",
+    "vision_encoder.blocks.3.attn.proj.bias": "model-00004-of-00004.safetensors",
+    "vision_encoder.blocks.3.attn.proj.weight": "model-00004-of-00004.safetensors",
+    "vision_encoder.blocks.3.attn.qkv.bias": "model-00004-of-00004.safetensors",
+    "vision_encoder.blocks.3.attn.qkv.weight": "model-00004-of-00004.safetensors",
+    "vision_encoder.blocks.3.mlp.fc1.bias": "model-00004-of-00004.safetensors",
+    "vision_encoder.blocks.3.mlp.fc1.weight": "model-00004-of-00004.safetensors",
+    "vision_encoder.blocks.3.mlp.fc2.bias": "model-00004-of-00004.safetensors",
+    "vision_encoder.blocks.3.mlp.fc2.weight": "model-00004-of-00004.safetensors",
+    "vision_encoder.blocks.3.norm1.bias": "model-00004-of-00004.safetensors",
+    "vision_encoder.blocks.3.norm1.weight": "model-00004-of-00004.safetensors",
+    "vision_encoder.blocks.3.norm2.bias": "model-00004-of-00004.safetensors",
+    "vision_encoder.blocks.3.norm2.weight": "model-00004-of-00004.safetensors",
+    "vision_encoder.blocks.30.attn.proj.bias": "model-00004-of-00004.safetensors",
+    "vision_encoder.blocks.30.attn.proj.weight": "model-00004-of-00004.safetensors",
+    "vision_encoder.blocks.30.attn.qkv.bias": "model-00004-of-00004.safetensors",
+    "vision_encoder.blocks.30.attn.qkv.weight": "model-00004-of-00004.safetensors",
+    "vision_encoder.blocks.30.mlp.fc1.bias": "model-00004-of-00004.safetensors",
+    "vision_encoder.blocks.30.mlp.fc1.weight": "model-00004-of-00004.safetensors",
+    "vision_encoder.blocks.30.mlp.fc2.bias": "model-00004-of-00004.safetensors",
+    "vision_encoder.blocks.30.mlp.fc2.weight": "model-00004-of-00004.safetensors",
+    "vision_encoder.blocks.30.norm1.bias": "model-00004-of-00004.safetensors",
+    "vision_encoder.blocks.30.norm1.weight": "model-00004-of-00004.safetensors",
+    "vision_encoder.blocks.30.norm2.bias": "model-00004-of-00004.safetensors",
+    "vision_encoder.blocks.30.norm2.weight": "model-00004-of-00004.safetensors",
+    "vision_encoder.blocks.31.attn.proj.bias": "model-00004-of-00004.safetensors",
+    "vision_encoder.blocks.31.attn.proj.weight": "model-00004-of-00004.safetensors",
+    "vision_encoder.blocks.31.attn.qkv.bias": "model-00004-of-00004.safetensors",
+    "vision_encoder.blocks.31.attn.qkv.weight": "model-00004-of-00004.safetensors",
+    "vision_encoder.blocks.31.mlp.fc1.bias": "model-00004-of-00004.safetensors",
+    "vision_encoder.blocks.31.mlp.fc1.weight": "model-00004-of-00004.safetensors",
+    "vision_encoder.blocks.31.mlp.fc2.bias": "model-00004-of-00004.safetensors",
+    "vision_encoder.blocks.31.mlp.fc2.weight": "model-00004-of-00004.safetensors",
+    "vision_encoder.blocks.31.norm1.bias": "model-00004-of-00004.safetensors",
+    "vision_encoder.blocks.31.norm1.weight": "model-00004-of-00004.safetensors",
+    "vision_encoder.blocks.31.norm2.bias": "model-00004-of-00004.safetensors",
+    "vision_encoder.blocks.31.norm2.weight": "model-00004-of-00004.safetensors",
+    "vision_encoder.blocks.4.attn.proj.bias": "model-00004-of-00004.safetensors",
+    "vision_encoder.blocks.4.attn.proj.weight": "model-00004-of-00004.safetensors",
+    "vision_encoder.blocks.4.attn.qkv.bias": "model-00004-of-00004.safetensors",
+    "vision_encoder.blocks.4.attn.qkv.weight": "model-00004-of-00004.safetensors",
+    "vision_encoder.blocks.4.mlp.fc1.bias": "model-00004-of-00004.safetensors",
+    "vision_encoder.blocks.4.mlp.fc1.weight": "model-00004-of-00004.safetensors",
+    "vision_encoder.blocks.4.mlp.fc2.bias": "model-00004-of-00004.safetensors",
+    "vision_encoder.blocks.4.mlp.fc2.weight": "model-00004-of-00004.safetensors",
+    "vision_encoder.blocks.4.norm1.bias": "model-00004-of-00004.safetensors",
+    "vision_encoder.blocks.4.norm1.weight": "model-00004-of-00004.safetensors",
+    "vision_encoder.blocks.4.norm2.bias": "model-00004-of-00004.safetensors",
+    "vision_encoder.blocks.4.norm2.weight": "model-00004-of-00004.safetensors",
+    "vision_encoder.blocks.5.attn.proj.bias": "model-00004-of-00004.safetensors",
+    "vision_encoder.blocks.5.attn.proj.weight": "model-00004-of-00004.safetensors",
+    "vision_encoder.blocks.5.attn.qkv.bias": "model-00004-of-00004.safetensors",
+    "vision_encoder.blocks.5.attn.qkv.weight": "model-00004-of-00004.safetensors",
+    "vision_encoder.blocks.5.mlp.fc1.bias": "model-00004-of-00004.safetensors",
+    "vision_encoder.blocks.5.mlp.fc1.weight": "model-00004-of-00004.safetensors",
+    "vision_encoder.blocks.5.mlp.fc2.bias": "model-00004-of-00004.safetensors",
+    "vision_encoder.blocks.5.mlp.fc2.weight": "model-00004-of-00004.safetensors",
+    "vision_encoder.blocks.5.norm1.bias": "model-00004-of-00004.safetensors",
+    "vision_encoder.blocks.5.norm1.weight": "model-00004-of-00004.safetensors",
+    "vision_encoder.blocks.5.norm2.bias": "model-00004-of-00004.safetensors",
+    "vision_encoder.blocks.5.norm2.weight": "model-00004-of-00004.safetensors",
+    "vision_encoder.blocks.6.attn.proj.bias": "model-00004-of-00004.safetensors",
+    "vision_encoder.blocks.6.attn.proj.weight": "model-00004-of-00004.safetensors",
+    "vision_encoder.blocks.6.attn.qkv.bias": "model-00004-of-00004.safetensors",
+    "vision_encoder.blocks.6.attn.qkv.weight": "model-00004-of-00004.safetensors",
+    "vision_encoder.blocks.6.mlp.fc1.bias": "model-00004-of-00004.safetensors",
+    "vision_encoder.blocks.6.mlp.fc1.weight": "model-00004-of-00004.safetensors",
+    "vision_encoder.blocks.6.mlp.fc2.bias": "model-00004-of-00004.safetensors",
+    "vision_encoder.blocks.6.mlp.fc2.weight": "model-00004-of-00004.safetensors",
+    "vision_encoder.blocks.6.norm1.bias": "model-00004-of-00004.safetensors",
+    "vision_encoder.blocks.6.norm1.weight": "model-00004-of-00004.safetensors",
+    "vision_encoder.blocks.6.norm2.bias": "model-00004-of-00004.safetensors",
+    "vision_encoder.blocks.6.norm2.weight": "model-00004-of-00004.safetensors",
+    "vision_encoder.blocks.7.attn.proj.bias": "model-00004-of-00004.safetensors",
+    "vision_encoder.blocks.7.attn.proj.weight": "model-00004-of-00004.safetensors",
+    "vision_encoder.blocks.7.attn.qkv.bias": "model-00004-of-00004.safetensors",
+    "vision_encoder.blocks.7.attn.qkv.weight": "model-00004-of-00004.safetensors",
+    "vision_encoder.blocks.7.mlp.fc1.bias": "model-00004-of-00004.safetensors",
+    "vision_encoder.blocks.7.mlp.fc1.weight": "model-00004-of-00004.safetensors",
+    "vision_encoder.blocks.7.mlp.fc2.bias": "model-00004-of-00004.safetensors",
+    "vision_encoder.blocks.7.mlp.fc2.weight": "model-00004-of-00004.safetensors",
+    "vision_encoder.blocks.7.norm1.bias": "model-00004-of-00004.safetensors",
+    "vision_encoder.blocks.7.norm1.weight": "model-00004-of-00004.safetensors",
+    "vision_encoder.blocks.7.norm2.bias": "model-00004-of-00004.safetensors",
+    "vision_encoder.blocks.7.norm2.weight": "model-00004-of-00004.safetensors",
+    "vision_encoder.blocks.8.attn.proj.bias": "model-00004-of-00004.safetensors",
+    "vision_encoder.blocks.8.attn.proj.weight": "model-00004-of-00004.safetensors",
+    "vision_encoder.blocks.8.attn.qkv.bias": "model-00004-of-00004.safetensors",
+    "vision_encoder.blocks.8.attn.qkv.weight": "model-00004-of-00004.safetensors",
+    "vision_encoder.blocks.8.mlp.fc1.bias": "model-00004-of-00004.safetensors",
+    "vision_encoder.blocks.8.mlp.fc1.weight": "model-00004-of-00004.safetensors",
+    "vision_encoder.blocks.8.mlp.fc2.bias": "model-00004-of-00004.safetensors",
+    "vision_encoder.blocks.8.mlp.fc2.weight": "model-00004-of-00004.safetensors",
+    "vision_encoder.blocks.8.norm1.bias": "model-00004-of-00004.safetensors",
+    "vision_encoder.blocks.8.norm1.weight": "model-00004-of-00004.safetensors",
+    "vision_encoder.blocks.8.norm2.bias": "model-00004-of-00004.safetensors",
+    "vision_encoder.blocks.8.norm2.weight": "model-00004-of-00004.safetensors",
+    "vision_encoder.blocks.9.attn.proj.bias": "model-00004-of-00004.safetensors",
+    "vision_encoder.blocks.9.attn.proj.weight": "model-00004-of-00004.safetensors",
+    "vision_encoder.blocks.9.attn.qkv.bias": "model-00004-of-00004.safetensors",
+    "vision_encoder.blocks.9.attn.qkv.weight": "model-00004-of-00004.safetensors",
+    "vision_encoder.blocks.9.mlp.fc1.bias": "model-00004-of-00004.safetensors",
+    "vision_encoder.blocks.9.mlp.fc1.weight": "model-00004-of-00004.safetensors",
+    "vision_encoder.blocks.9.mlp.fc2.bias": "model-00004-of-00004.safetensors",
+    "vision_encoder.blocks.9.mlp.fc2.weight": "model-00004-of-00004.safetensors",
+    "vision_encoder.blocks.9.norm1.bias": "model-00004-of-00004.safetensors",
+    "vision_encoder.blocks.9.norm1.weight": "model-00004-of-00004.safetensors",
+    "vision_encoder.blocks.9.norm2.bias": "model-00004-of-00004.safetensors",
+    "vision_encoder.blocks.9.norm2.weight": "model-00004-of-00004.safetensors",
+    "vision_encoder.merger.ln_q.bias": "model-00004-of-00004.safetensors",
+    "vision_encoder.merger.ln_q.weight": "model-00004-of-00004.safetensors",
+    "vision_encoder.merger.mlp.0.bias": "model-00004-of-00004.safetensors",
+    "vision_encoder.merger.mlp.0.weight": "model-00004-of-00004.safetensors",
+    "vision_encoder.merger.mlp.2.bias": "model-00004-of-00004.safetensors",
+    "vision_encoder.merger.mlp.2.weight": "model-00004-of-00004.safetensors",
+    "vision_encoder.patch_embed.proj.weight": "model-00004-of-00004.safetensors",
+    "vision_projector.ln_q.bias": "model-00004-of-00004.safetensors",
+    "vision_projector.ln_q.weight": "model-00004-of-00004.safetensors",
+    "vision_projector.mlp.0.bias": "model-00004-of-00004.safetensors",
+    "vision_projector.mlp.0.weight": "model-00004-of-00004.safetensors",
+    "vision_projector.mlp.2.bias": "model-00004-of-00004.safetensors",
+    "vision_projector.mlp.2.weight": "model-00004-of-00004.safetensors"
+  }
+}

modeling_llama.py

@@ -0,0 +1,978 @@
+from functools import partial
+from typing import Optional, Tuple, Union
+
+import torch
+import torch.nn.functional as F
+import torch.utils.checkpoint
+from packaging import version
+from torch import nn
+from torch.nn import CrossEntropyLoss
+from transformers.activations import ACT2FN
+from transformers.modeling_outputs import (
+    BaseModelOutputWithPast,
+    CausalLMOutputWithPast,
+)
+from transformers.modeling_utils import PreTrainedModel
+from transformers.utils import logging
+
+from .configuration_llama import CustomLlamaConfig
+
+try:
+    from apex.megatron_layer_norm import MixedFusedLayerNorm as LayerNorm
+except ImportError:
+    from torch.nn import LayerNorm
+
+USE_FLASH_ATTN = False
+try:
+    import flash_attn
+    if version.parse(flash_attn.__version__) >= version.parse("2.1.0"):
+        USE_FLASH_ATTN = True
+        from flash_attn import flash_attn_func, flash_attn_varlen_func
+        from flash_attn.bert_padding import index_first_axis, pad_input, unpad_input
+except ImportError:
+    pass
+
+logger = logging.get_logger(__name__)
+
+
+def _get_unpad_data(attention_mask):
+    seqlens_in_batch = (attention_mask).sum(dim=-1, dtype=torch.int32)
+    indices = torch.nonzero(attention_mask.flatten(), as_tuple=False).flatten()
+    max_seqlen_in_batch = seqlens_in_batch.max().item()
+    cu_seqlens = F.pad(torch.cumsum(seqlens_in_batch, dim=0,
+                       dtype=torch.torch.int32), (1, 0))
+    return (
+        indices,
+        cu_seqlens,
+        max_seqlen_in_batch,
+    )
+
+
+class RMSNorm(torch.nn.Module):
+    def __init__(self, dim: int, eps: float = 1e-6):
+        """
+        Initialize the RMSNorm normalization layer.
+
+        Args:
+            dim (int): The dimension of the input tensor.
+            eps (float, optional): A small value added to the denominator for numerical stability. Default is 1e-6.
+
+        Attributes:
+            eps (float): A small value added to the denominator for numerical stability.
+            weight (nn.Parameter): Learnable scaling parameter.
+
+        """
+        super().__init__()
+        self.eps = eps
+        self.weight = nn.Parameter(torch.ones(dim))
+
+    def _norm(self, x):
+        """
+        Apply the RMSNorm normalization to the input tensor.
+
+        Args:
+            x (torch.Tensor): The input tensor.
+
+        Returns:
+            torch.Tensor: The normalized tensor.
+
+        """
+        return x * torch.rsqrt(x.pow(2).mean(-1, keepdim=True) + self.eps)
+
+    def forward(self, x):
+        """
+        Forward pass through the RMSNorm layer.
+
+        Args:
+            x (torch.Tensor): The input tensor.
+
+        Returns:
+            torch.Tensor: The output tensor after applying RMSNorm.
+
+        """
+        output = self._norm(x.float()).type_as(x)
+        return output * self.weight
+
+
+def get_norm(config: CustomLlamaConfig):
+    norm_type = config.norm_type
+    if norm_type == 'rms_norm':
+        return partial(RMSNorm, eps=config.layernorm_epsilon)
+    elif norm_type == 'layer_norm':
+        return partial(LayerNorm, eps=config.layernorm_epsilon)
+    else:
+        raise ValueError(f'Unsupported norm type: {norm_type}')
+
+
+# Copied from transformers.models.bart.modeling_bart._make_causal_mask
+def _make_causal_mask(
+    input_ids_shape: torch.Size,
+    dtype: torch.dtype,
+    device: torch.device,
+    past_key_values_length: int = 0,
+):
+    """
+    Make causal mask used for bi-directional self-attention.
+    """
+    bsz, tgt_len = input_ids_shape
+    mask = torch.full((tgt_len, tgt_len),
+                      torch.finfo(dtype).min, device=device)
+    mask_cond = torch.arange(mask.size(-1), device=device)
+    mask.masked_fill_(mask_cond < (mask_cond + 1).view(mask.size(-1), 1), 0)
+    mask = mask.to(dtype)
+
+    if past_key_values_length > 0:
+        mask = torch.cat(
+            [
+                torch.zeros(
+                    tgt_len, past_key_values_length, dtype=dtype, device=device
+                ),
+                mask,
+            ],
+            dim=-1,
+        )
+    return mask[None, None, :, :].expand(
+        bsz, 1, tgt_len, tgt_len + past_key_values_length
+    )
+
+
+# Copied from transformers.models.bart.modeling_bart._expand_mask
+def _expand_mask(mask: torch.Tensor, dtype: torch.dtype, tgt_len: Optional[int] = None):
+    """
+    Expands attention_mask from `[bsz, seq_len]` to `[bsz, 1, tgt_seq_len, src_seq_len]`.
+    """
+    bsz, src_len = mask.size()
+    tgt_len = tgt_len if tgt_len is not None else src_len
+
+    expanded_mask = mask[:, None, None, :].expand(
+        bsz, 1, tgt_len, src_len).to(dtype)
+
+    inverted_mask = 1.0 - expanded_mask
+
+    return inverted_mask.masked_fill(
+        inverted_mask.to(torch.bool), torch.finfo(dtype).min
+    )
+
+
+class RotaryEmbedding(torch.nn.Module):
+    def __init__(self, dim, base=10000, compress=1.0):
+        super().__init__()
+        self.inv_freq = 1.0 / (base ** (torch.arange(0, dim, 2).float() / dim))
+        self.seq_len_cached = 0
+        self.cos_cached = None
+        self.sin_cached = None
+        self.compress = compress
+
+    def forward(self, x, seq_len):
+        if seq_len > self.seq_len_cached:
+            self.seq_len_cached = seq_len
+            self.inv_freq = self.inv_freq.to(x.device)
+            t = (
+                torch.arange(seq_len, device=self.inv_freq.device,
+                             dtype=self.inv_freq.dtype)
+                * self.compress
+            )
+            freqs = torch.einsum("i,j->ij", t, self.inv_freq)
+            emb = torch.cat((freqs, freqs), dim=-1).to(x.device)
+            self.cos_cached = emb.cos()[None, None, :, :]
+            self.sin_cached = emb.sin()[None, None, :, :]
+        return self.cos_cached[:seq_len, ...], self.sin_cached[:seq_len, ...]
+
+
+# rotary pos emb helpers:
+
+
+def rotate_half(x):
+    x1, x2 = x[..., : x.shape[-1] // 2], x[..., x.shape[-1] // 2:]
+    return torch.cat((-x2, x1), dim=-1)
+
+
+@torch.jit.script
+def apply_rotary_pos_emb(q, k, cos, sin, offset: int = 0):
+    cos, sin = (
+        cos[..., offset: q.shape[-2] + offset, :],
+        sin[..., offset: q.shape[-2] + offset, :],
+    )
+    q_embed = (q.float() * cos) + (rotate_half(q.float()) * sin)
+    k_embed = (k.float() * cos) + (rotate_half(k.float()) * sin)
+    return q_embed.to(q.dtype), k_embed.to(k.dtype)
+
+
+def apply_rotary_pos_emb_torch(
+    q, k, cos, sin, offset: int = 0
+):  # jitting fails with bf16
+    cos, sin = (
+        cos[..., offset: q.shape[-2] + offset, :],
+        sin[..., offset: q.shape[-2] + offset, :],
+    )
+    q_embed = (q.float() * cos) + (rotate_half(q.float()) * sin)
+    k_embed = (k.float() * cos) + (rotate_half(k.float()) * sin)
+    return q_embed.to(q.dtype), k_embed.to(k.dtype)
+
+
+class CustomLlamaAttention(nn.Module):
+    def __init__(self, config: CustomLlamaConfig):
+        super().__init__()
+        self.num_attention_heads = config.num_attention_heads
+        self.num_kv_heads = config.num_kv_heads
+        self.hidden_size = config.hidden_size
+        self.head_size = self.hidden_size // self.num_attention_heads
+        self.rotary_ndims = int(self.head_size * config.rotary_pct)
+        self.max_positions = config.max_position_embeddings
+        self.rotary_emb = RotaryEmbedding(
+            self.rotary_ndims,
+            base=config.rotary_emb_base,
+            compress=config.rotary_compress,
+        )
+        self.norm_factor = torch.sqrt(
+            torch.tensor(self.head_size, dtype=torch.float32)
+        ).to(torch.get_default_dtype())
+
+        if self.use_gqa:
+            self.query_dense = nn.Linear(
+                config.hidden_size,
+                config.hidden_size,
+                bias=getattr(config, "qkv_proj_bias", True)
+            )
+            self.key_value_dense = nn.Linear(
+                config.hidden_size,
+                self.head_size * 2 * config.num_kv_heads,
+                bias=getattr(config, "qkv_proj_bias", True),
+            )
+        else:
+            self.query_key_value = nn.Linear(
+                config.hidden_size,
+                3 * config.hidden_size,
+                bias=getattr(config, "qkv_proj_bias", True),
+            )
+
+        self.dense = nn.Linear(
+            config.hidden_size,
+            config.hidden_size,
+            bias=getattr(config, "out_proj_bias", True),
+        )
+        self.apply_rotary_fn = (
+            apply_rotary_pos_emb_torch
+            if config.torch_dtype == torch.bfloat16
+            else apply_rotary_pos_emb
+        )
+
+    @property
+    def use_gqa(self):
+        return self.num_kv_heads < self.num_attention_heads
+
+    def forward(
+        self,
+        hidden_states,
+        attention_mask,
+        head_mask=None,
+        layer_past=None,
+        use_cache=False,
+        output_attentions=False,
+    ):
+        has_layer_past = layer_past is not None
+
+        if self.use_gqa:
+            # Compute Q
+            # [batch, seq_len, hidden_size] --> [batch_size, seq_len, (num_heads * head_size)]
+            q = self.query_dense(hidden_states)
+
+            # [batch_size, seq_len, (num_heads * head_size)]
+            #     --> [batch, seq_len, num_attention_heads, head_size]
+            new_q_shape = q.size()[:-1] + \
+                (self.num_attention_heads, self.head_size)
+            q = q.view(*new_q_shape)
+
+            # Compute KV
+            # [batch, seq_len, hidden_size] --> [batch_size, seq_len, (num_attention_groups * 2 * head_size)]
+            kv = self.key_value_dense(hidden_states)
+
+            # [batch, seq_len, (num_attention_groups * 2 * head_size)]
+            #   --> [batch, seq_len, num_attention_groups, 2 * head_size]
+            new_kv_shape = kv.size()[:-1] + (
+                self.num_kv_heads,
+                2 * self.head_size,
+            )
+            kv = kv.view(*new_kv_shape)
+
+            # [batch, num_attention_heads, seq_len, head_size]
+            query = q.permute(0, 2, 1, 3)
+            # [batch, num_attention_groups, seq_len, head_size]
+            key = kv[..., : self.head_size].permute(0, 2, 1, 3)
+            value = kv[..., self.head_size:].permute(0, 2, 1, 3)
+        else:
+            # Compute QKV
+            # Attention heads [batch, seq_len, hidden_size]
+            #   --> [batch, seq_len, (np * 3 * head_size)]
+            qkv = self.query_key_value(hidden_states)
+
+            # [batch, seq_len, (num_heads * 3 * head_size)]
+            #   --> [batch, seq_len, num_heads, 3 * head_size]
+            new_qkv_shape = qkv.size()[:-1] + (
+                self.num_attention_heads,
+                3 * self.head_size,
+            )
+            qkv = qkv.view(*new_qkv_shape)
+
+            # [batch, seq_len, num_attention_heads, 3 * head_size] --> 3 [batch, num_attention_heads, seq_len, head_size]
+            query = qkv[..., : self.head_size].permute(0, 2, 1, 3)
+            key = qkv[..., self.head_size: 2 *
+                      self.head_size].permute(0, 2, 1, 3)
+            value = qkv[..., 2 * self.head_size:].permute(0, 2, 1, 3)
+
+        # Compute rotary embeddings on rotary_ndims
+        query_rot = query[..., : self.rotary_ndims]
+        query_pass = query[..., self.rotary_ndims:]
+        key_rot = key[..., : self.rotary_ndims]
+        key_pass = key[..., self.rotary_ndims:]
+
+        # Compute token offset for rotary embeddings (when decoding)
+        seq_len = key.shape[-2]
+        offset = 0
+        if has_layer_past:
+            offset = layer_past[0].shape[-2]
+            seq_len += offset
+        cos, sin = self.rotary_emb(value, seq_len=seq_len)
+        query, key = self.apply_rotary_fn(
+            query_rot, key_rot, cos, sin, offset=offset)
+        query = torch.cat((query, query_pass), dim=-1)
+        key = torch.cat((key, key_pass), dim=-1)
+
+        # Cache QKV values
+        if has_layer_past:
+            past_key = layer_past[0]
+            past_value = layer_past[1]
+            key = torch.cat((past_key, key), dim=-2)
+            value = torch.cat((past_value, value), dim=-2)
+        present = (key, value) if use_cache else None
+
+        if USE_FLASH_ATTN:
+            # Compute attention
+            attn_output, attn_weights = self._flash_attn(
+                query, key, value, attention_mask, head_mask
+            )
+
+            # from [batch_size, ]
+            attn_output = attn_output.reshape(
+                attn_output.size(0), attn_output.size(1), self.hidden_size).contiguous()
+        else:
+            # Compute attention
+            attn_output, attn_weights = self._attn(
+                query, key, value, attention_mask, head_mask
+            )
+
+            # Reshape outputs
+            attn_output = self._merge_heads(
+                attn_output, self.num_attention_heads, self.head_size
+            )
+        attn_output = self.dense(attn_output)
+
+        outputs = (attn_output, present)
+        if output_attentions:
+            outputs += (attn_weights,)
+
+        return outputs
+
+    @classmethod
+    def _split_heads(cls, tensor, num_attention_heads, attn_head_size):
+        """
+        Splits hidden dim into attn_head_size and num_attention_heads
+        """
+        # tensor: [bs, seq_len, hidden_size]
+        new_shape = tensor.size()[:-1] + (num_attention_heads, attn_head_size)
+        # -> [bs, seq_len, num_attention_heads, attn_head_size]
+        tensor = tensor.view(new_shape)
+        # -> [bs, num_attention_heads, seq_len, attn_head_size]
+        tensor = tensor.permute(0, 2, 1, 3)
+        return tensor
+
+    @classmethod
+    def _merge_heads(cls, tensor, num_attention_heads, attn_head_size):
+        """
+        Merges attn_head_size dim and num_attn_heads dim into hidden dim
+        """
+        # tensor [bs, num_attention_heads, seq_len, attn_head_size]
+        tensor = tensor.permute(0, 2, 1, 3).contiguous()
+        # -> [bs, seq_len, num_attention_heads, attn_head_size]
+        tensor = tensor.view(
+            tensor.size(0), tensor.size(
+                1), num_attention_heads * attn_head_size
+        )
+        # -> [bs, seq_len, hidden_size]
+        return tensor
+
+    def _attn(self, query, key, value, attention_mask=None, head_mask=None):
+        # q: [bs, num_attention_heads, seq_len, attn_head_size]
+        # k,v: [bs, num_attention_groups, seq_len, attn_head_size]
+        # compute causal mask from causal mask buffer
+        batch_size, num_attention_heads, query_length, attn_head_size = query.size()
+        _, num_attention_groups, key_length, _ = key.size()
+
+        group_size = num_attention_heads // num_attention_groups
+
+        if not self.use_gqa:
+            assert group_size == 1
+
+        # repeat key and value, so we can use normal MHA algorithm
+        key = (
+            key.view(batch_size, num_attention_groups,
+                     1, key_length, attn_head_size)
+            .repeat(1, 1, group_size, 1, 1)
+            .view(batch_size, num_attention_heads, key_length, attn_head_size)
+        )
+        value = (
+            value.view(batch_size, num_attention_groups,
+                       1, key_length, attn_head_size)
+            .repeat(1, 1, group_size, 1, 1)
+            .view(batch_size, num_attention_heads, key_length, attn_head_size)
+        )
+
+        query = query.view(
+            batch_size * num_attention_heads, query_length, attn_head_size
+        )
+        key = key.view(batch_size * num_attention_heads,
+                       key_length, attn_head_size)
+        attn_scores = torch.zeros(
+            batch_size * num_attention_heads,
+            query_length,
+            key_length,
+            dtype=query.dtype,
+            device=key.device,
+        )
+        attn_scores = torch.baddbmm(
+            attn_scores,
+            query,
+            key.transpose(1, 2),
+            beta=1.0,
+            alpha=(
+                torch.tensor(
+                    1.0, dtype=self.norm_factor.dtype, device=self.norm_factor.device
+                )
+                / self.norm_factor
+            ),
+        )
+        attn_scores = attn_scores.view(
+            batch_size, num_attention_heads, query_length, key_length
+        )
+
+        mask_value = torch.finfo(attn_scores.dtype).min
+        # Need to be a tensor, otherwise we get error: `RuntimeError: expected scalar type float but found double`.
+        # Need to be on the same device, otherwise `RuntimeError: ..., x and y to be on the same device`
+        mask_value = torch.tensor(mask_value, dtype=attn_scores.dtype).to(
+            attn_scores.device
+        )
+
+        if attention_mask is not None:
+            # Apply the attention mask
+            attn_scores = attn_scores + attention_mask
+
+        attn_weights = nn.functional.softmax(attn_scores, dim=-1)
+        attn_weights = attn_weights.to(value.dtype)
+
+        # Mask heads if we want to
+        if head_mask is not None:
+            attn_weights = attn_weights * head_mask
+
+        attn_output = torch.matmul(attn_weights, value)
+        return attn_output, attn_weights
+
+    def _flash_attn(self, query, key, value, attention_mask=None, head_mask=None):
+        assert head_mask is None, "head_mask is not supported in _flash_attn"
+        # q: [bs, num_attention_heads, seq_len, attn_head_size]
+        # k,v: [bs, num_attention_groups, seq_len, attn_head_size]
+
+        # flash_attn need the layout to be [batch_size, sequence_length, num_heads, head_dim]
+        query = query.transpose(1, 2)
+        key = key.transpose(1, 2)
+        value = value.transpose(1, 2)
+
+        query_length = query.size(1)
+        causal = query_length != 1
+
+        if attention_mask is not None:
+            batch_size = query.size(0)
+            query, key, value, indices_q, cu_seq_lens, max_seq_lens = self._upad_input(
+                query, key, value, attention_mask, query_length
+            )
+
+            cu_seqlens_q, cu_seqlens_k = cu_seq_lens
+            max_seqlen_in_batch_q, max_seqlen_in_batch_k = max_seq_lens
+
+            attn_output_unpad = flash_attn_varlen_func(
+                query,
+                key,
+                value,
+                cu_seqlens_q=cu_seqlens_q,
+                cu_seqlens_k=cu_seqlens_k,
+                max_seqlen_q=max_seqlen_in_batch_q,
+                max_seqlen_k=max_seqlen_in_batch_k,
+                dropout_p=0,
+                causal=causal,
+            )
+
+            attn_output = pad_input(
+                attn_output_unpad, indices_q, batch_size, query_length)
+        else:
+            attn_output = flash_attn_func(
+                query, key, value, 0, causal=causal
+            )
+
+        return attn_output, None
+
+    def _upad_input(self, query_layer, key_layer, value_layer, attention_mask, query_length):
+        indices_k, cu_seqlens_k, max_seqlen_in_batch_k = _get_unpad_data(
+            attention_mask)
+        batch_size, kv_seq_len, num_key_value_heads, head_dim = key_layer.shape
+        num_attention_heads = query_layer.shape[2]
+
+        key_layer = index_first_axis(
+            key_layer.reshape(batch_size * kv_seq_len,
+                              num_key_value_heads, head_dim), indices_k
+        )
+        value_layer = index_first_axis(
+            value_layer.reshape(batch_size * kv_seq_len,
+                                num_key_value_heads, head_dim), indices_k
+        )
+        if query_length == kv_seq_len:
+            query_layer = index_first_axis(
+                query_layer.reshape(batch_size * kv_seq_len,
+                                    num_attention_heads, head_dim), indices_k
+            )
+            cu_seqlens_q = cu_seqlens_k
+            max_seqlen_in_batch_q = max_seqlen_in_batch_k
+            indices_q = indices_k
+        elif query_length == 1:
+            max_seqlen_in_batch_q = 1
+            cu_seqlens_q = torch.arange(
+                batch_size + 1, dtype=torch.int32, device=query_layer.device
+            )  # There is a memcpy here, that is very bad.
+            indices_q = cu_seqlens_q[:-1]
+            query_layer = query_layer.squeeze(1)
+        else:
+            # The -q_len: slice assumes left padding.
+            attention_mask = attention_mask[:, -query_length:]
+            query_layer, indices_q, cu_seqlens_q, max_seqlen_in_batch_q = unpad_input(
+                query_layer, attention_mask)
+
+        return (
+            query_layer,
+            key_layer,
+            value_layer,
+            indices_q,
+            (cu_seqlens_q, cu_seqlens_k),
+            (max_seqlen_in_batch_q, max_seqlen_in_batch_k),
+        )
+
+
+def swiglu(x):
+    x1, x2 = x.chunk(2, dim=(x.ndim - 1))
+    return x1 * torch.nn.functional.silu(x2)
+
+
+def get_activation(act_name: str):
+    if act_name == "gelu":
+        return ACT2FN["gelu_new"]
+    elif act_name == "swiglu":
+        return swiglu
+    else:
+        return ACT2FN[act_name]
+
+
+class CustomLlamaMLP(nn.Module):
+    def __init__(self, config):
+        super().__init__()
+        h_to_4h_out_channels = (
+            config.ffn_hidden_size * 2
+            if config.hidden_act == "swiglu"
+            else config.ffn_hidden_size
+        )
+        self.dense_h_to_4h = nn.Linear(
+            config.hidden_size,
+            h_to_4h_out_channels,
+            bias=getattr(config, "mlp_fc1_bias", True)
+        )
+        self.dense_4h_to_h = nn.Linear(
+            config.ffn_hidden_size,
+            config.hidden_size,
+            bias=getattr(config, "mlp_fc2_bias", True)
+        )
+        self.act = get_activation(config.hidden_act)
+
+    def forward(self, hidden_states):
+        hidden_states = self.dense_h_to_4h(hidden_states)
+        hidden_states = self.act(hidden_states)
+        hidden_states = self.dense_4h_to_h(hidden_states)
+        return hidden_states
+
+
+class CustomLlamaLayer(nn.Module):
+    def __init__(self, config):
+        super().__init__()
+
+        norm_func = get_norm(config)
+        self.input_layernorm = norm_func(config.hidden_size)
+        self.post_attention_layernorm = norm_func(config.hidden_size)
+        self.attention = CustomLlamaAttention(config)
+        self.mlp = CustomLlamaMLP(config)
+
+    def forward(
+        self,
+        hidden_states,
+        attention_mask=None,
+        head_mask=None,
+        use_cache=False,
+        layer_past=None,
+        output_attentions=False,
+    ):
+        attn_in = self.input_layernorm(hidden_states)
+        attention_layer_outputs = self.attention(
+            attn_in,
+            attention_mask=attention_mask,
+            layer_past=layer_past,
+            head_mask=head_mask,
+            use_cache=use_cache,
+            output_attentions=output_attentions,
+        )
+        attn_output = attention_layer_outputs[
+            0
+        ]  # output_attn: attn_output, present, (attn_weights)
+        outputs = attention_layer_outputs[1:]
+        # pseudocode:
+        # x = x + attn(ln1(x))
+        # x = x + mlp(ln2(x))
+        attn_output = attn_output + hidden_states
+        mlp_input = self.post_attention_layernorm(attn_output)
+        mlp_output = self.mlp(mlp_input)
+        hidden_states = mlp_output + attn_output
+
+        if use_cache:
+            outputs = (
+                hidden_states,
+            ) + outputs  # hidden_states, present, (attn_weights)
+        else:
+            # hidden_states, (attn_weights)
+            outputs = (hidden_states,) + outputs[1:]
+
+        return outputs
+
+
+class CustomLlamaPreTrainedModel(PreTrainedModel):
+    config_class = CustomLlamaConfig
+    base_model_prefix = "lm"
+    _no_split_modules = ["CustomLlamaLayer"]
+
+
+class CustomLlamaModel(CustomLlamaPreTrainedModel):
+    def __init__(self, config):
+        super().__init__(config)
+        self.config = config
+
+        self.embed_in = nn.Embedding(config.vocab_size, config.hidden_size)
+        self.layers = nn.ModuleList(
+            [CustomLlamaLayer(config) for _ in range(config.num_layers)]
+        )
+
+        norm_func = get_norm(config)
+        self.final_layer_norm = norm_func(config.hidden_size)
+        # Initialize weights and apply final processing
+        self.post_init()
+
+    def get_input_embeddings(self):
+        return self.embed_in
+
+    def set_input_embeddings(self, value):
+        self.embed_in = value
+
+    # Copied from transformers.models.bart.modeling_bart.BartDecoder._prepare_decoder_attention_mask
+    def _prepare_decoder_attention_mask(
+        self, attention_mask, input_shape, inputs_embeds, past_key_values_length
+    ):
+        # create causal mask
+        # [bsz, seq_len] -> [bsz, 1, tgt_seq_len, src_seq_len]
+        combined_attention_mask = None
+        if input_shape[-1] > 1:
+            combined_attention_mask = _make_causal_mask(
+                input_shape,
+                inputs_embeds.dtype,
+                device=inputs_embeds.device,
+                past_key_values_length=past_key_values_length,
+            )
+
+        if attention_mask is not None:
+            # [bsz, seq_len] -> [bsz, 1, tgt_seq_len, src_seq_len]
+            expanded_attn_mask = _expand_mask(
+                attention_mask, inputs_embeds.dtype, tgt_len=input_shape[-1]
+            ).to(inputs_embeds.device)
+            combined_attention_mask = (
+                expanded_attn_mask
+                if combined_attention_mask is None
+                else expanded_attn_mask + combined_attention_mask
+            )
+
+        return combined_attention_mask
+
+    def forward(
+        self,
+        input_ids: Optional[torch.LongTensor] = None,
+        attention_mask: Optional[torch.FloatTensor] = None,
+        head_mask: Optional[torch.FloatTensor] = None,
+        inputs_embeds: Optional[torch.FloatTensor] = None,
+        past_key_values: Optional[Tuple[Tuple[torch.FloatTensor]]] = None,
+        use_cache: Optional[bool] = None,
+        output_attentions: Optional[bool] = None,
+        output_hidden_states: Optional[bool] = None,
+        return_dict: Optional[bool] = None,
+    ) -> Union[Tuple, BaseModelOutputWithPast]:
+        r"""
+        past_key_values (`tuple(tuple(torch.FloatTensor))` of length `config.n_layers` with each tuple having 4 tensors of shape `(batch_size, num_heads, sequence_length - 1, embed_size_per_head)`):
+            Contains precomputed key and value hidden states of the attention blocks. Can be used to speed up decoding.
+            If `past_key_values` are used, the user can optionally input only the last `decoder_input_ids` (those that
+            don't have their past key value states given to this model) of shape `(batch_size, 1)` instead of all
+            `decoder_input_ids` of shape `(batch_size, sequence_length)`.
+        use_cache (`bool`, *optional*):
+            If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding (see
+            `past_key_values`).
+        """
+        output_attentions = (
+            output_attentions
+            if output_attentions is not None
+            else self.config.output_attentions
+        )
+        output_hidden_states = (
+            output_hidden_states
+            if output_hidden_states is not None
+            else self.config.output_hidden_states
+        )
+        return_dict = (
+            return_dict if return_dict is not None else self.config.use_return_dict
+        )
+        use_cache = use_cache if use_cache is not None else self.config.use_cache
+
+        if input_ids is not None and inputs_embeds is not None:
+            raise ValueError(
+                "You cannot specify both input_ids and inputs_embeds at the same time"
+            )
+        elif input_ids is not None:
+            input_shape = input_ids.size()
+        elif inputs_embeds is not None:
+            input_shape = inputs_embeds.size()[:-1]
+        else:
+            raise ValueError(
+                "You have to specify either input_ids or inputs_embeds")
+
+        batch_size, seq_length = input_shape
+        seq_length_with_past = seq_length
+        past_key_values_length = 0
+
+        if past_key_values is not None:
+            past_key_values_length = past_key_values[0][0].shape[2]
+            seq_length_with_past = seq_length_with_past + past_key_values_length
+        else:
+            past_key_values = tuple([None] * self.config.num_layers)
+
+        if inputs_embeds is None:
+            inputs_embeds = self.embed_in(input_ids)
+        # Attention mask.
+        if attention_mask is None:
+            attention_mask = torch.ones(
+                (batch_size, seq_length_with_past),
+                dtype=torch.bool,
+                device=inputs_embeds.device,
+            )
+
+        # Prepare head mask if needed
+        # 1.0 in head_mask indicate we keep the head
+        # attention_probs has shape bsz x n_heads x N x N
+        # input head_mask has shape [num_heads] or [num_layers x num_heads]
+        # and head_mask is converted to shape [num_layers x batch x num_heads x seq_length x seq_length]
+        head_mask = self.get_head_mask(head_mask, self.config.num_layers)
+
+        if USE_FLASH_ATTN:
+            attention_mask = attention_mask if (
+                attention_mask is not None and 0 in attention_mask) else None
+        else:
+            attention_mask = self._prepare_decoder_attention_mask(
+                attention_mask,
+                (batch_size, seq_length),
+                inputs_embeds,
+                past_key_values_length,
+            )
+
+        hidden_states = inputs_embeds
+        presents = () if use_cache else None
+        all_attentions = () if output_attentions else None
+        all_hidden_states = () if output_hidden_states else None
+        for i, (layer, layer_past) in enumerate(zip(self.layers, past_key_values)):
+            if output_hidden_states:
+                all_hidden_states = all_hidden_states + (hidden_states,)
+            outputs = layer(
+                hidden_states,
+                attention_mask=attention_mask,
+                head_mask=head_mask[i],
+                layer_past=layer_past,
+                use_cache=use_cache,
+                output_attentions=output_attentions,
+            )
+            hidden_states = outputs[0]
+            if use_cache is True:
+                presents = presents + (outputs[1],)
+            if output_attentions:
+                all_attentions = all_attentions + \
+                    (outputs[2 if use_cache else 1],)
+
+        hidden_states = self.final_layer_norm(hidden_states)
+        # Add last hidden state
+        if output_hidden_states:
+            all_hidden_states = all_hidden_states + (hidden_states,)
+
+        if not return_dict:
+            return tuple(
+                v
+                for v in [hidden_states, presents, all_hidden_states, all_attentions]
+                if v is not None
+            )
+
+        return BaseModelOutputWithPast(
+            last_hidden_state=hidden_states,
+            past_key_values=presents,
+            hidden_states=all_hidden_states,
+            attentions=all_attentions,
+        )
+
+
+class CustomLlamaForCausalLM(CustomLlamaPreTrainedModel):
+    _tied_weights_keys = ["embed_out.weight"]
+    _keys_to_ignore_on_load_unexpected = [
+        r"lm.layers.\d+.attention.rotary_emb.inv_freq"
+    ]
+
+    def __init__(self, config):
+        super().__init__(config)
+
+        self.lm = CustomLlamaModel(config)
+        self.embed_out = nn.Linear(
+            config.hidden_size, config.vocab_size, bias=False)
+
+        # Initialize weights and apply final processing
+        self.post_init()
+
+    def get_output_embeddings(self):
+        return self.embed_out
+
+    def set_output_embeddings(self, new_embeddings):
+        self.embed_out = new_embeddings
+
+    def forward(
+        self,
+        input_ids: Optional[torch.LongTensor] = None,
+        attention_mask: Optional[torch.FloatTensor] = None,
+        inputs_embeds: Optional[torch.FloatTensor] = None,
+        head_mask: Optional[torch.FloatTensor] = None,
+        past_key_values: Optional[Tuple[Tuple[torch.FloatTensor]]] = None,
+        labels: Optional[torch.LongTensor] = None,
+        use_cache: Optional[bool] = None,
+        output_attentions: Optional[bool] = None,
+        output_hidden_states: Optional[bool] = None,
+        return_dict: Optional[bool] = None,
+    ) -> Union[Tuple, CausalLMOutputWithPast]:
+        r"""
+        past_key_values (`tuple(tuple(torch.FloatTensor))`, *optional*, returned when `use_cache=True` is passed or when `config.use_cache=True`):
+            Tuple of `tuple(torch.FloatTensor)` of length `config.n_layers`, with each tuple having 2 tensors of shape
+            `(batch_size, num_heads, sequence_length, embed_size_per_head)`) and 2 additional tensors of shape
+            `(batch_size, num_heads, encoder_sequence_length, embed_size_per_head)`. The two additional tensors are
+            only required when the model is used as a decoder in a Sequence to Sequence model.
+
+            Contains pre-computed hidden-states (key and values in the self-attention blocks that can be used (see
+            `past_key_values` input) to speed up sequential decoding.
+
+            If `past_key_values` are used, the user can optionally input only the last `decoder_input_ids` (those that
+            don't have their past key value states given to this model) of shape `(batch_size, 1)` instead of all
+            `decoder_input_ids` of shape `(batch_size, sequence_length)`.
+        labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
+            Labels for computing the left-to-right language modeling loss (next word prediction). Indices should be in
+            `[-100, 0, ..., config.vocab_size]` (see `input_ids` docstring) Tokens with indices set to `-100` are
+            ignored (masked), the loss is only computed for the tokens with labels n `[0, ..., config.vocab_size]`.
+        use_cache (`bool`, *optional*):
+            If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding (see
+            `past_key_values`).
+
+        ```"""
+        return_dict = (
+            return_dict if return_dict is not None else self.config.use_return_dict
+        )
+
+        outputs = self.lm(
+            input_ids,
+            attention_mask=attention_mask,
+            head_mask=head_mask,
+            inputs_embeds=inputs_embeds,
+            past_key_values=past_key_values,
+            use_cache=use_cache,
+            output_attentions=output_attentions,
+            output_hidden_states=output_hidden_states,
+            return_dict=return_dict,
+        )
+
+        hidden_states = outputs[0]
+        lm_logits = self.embed_out(hidden_states)
+
+        lm_loss = None
+        if labels is not None:
+            # we are doing next-token prediction; shift prediction scores and input ids by one
+            shift_logits = lm_logits[:, :-1, :].contiguous()
+            labels = labels[:, 1:].contiguous()
+            loss_fct = CrossEntropyLoss()
+            lm_loss = loss_fct(
+                shift_logits.view(-1, shift_logits.size(-1)), labels.view(-1)
+            )
+
+        if not return_dict:
+            output = (lm_logits,) + outputs[1:]
+            return ((lm_loss,) + output) if lm_loss is not None else output
+
+        return CausalLMOutputWithPast(
+            loss=lm_loss,
+            logits=lm_logits,
+            past_key_values=outputs.past_key_values,
+            hidden_states=outputs.hidden_states,
+            attentions=outputs.attentions,
+        )
+
+    def prepare_inputs_for_generation(
+        self, input_ids, past_key_values=None, attention_mask=None, inputs_embeds=None, **model_kwargs
+    ):
+        input_shape = input_ids.shape
+
+        # if model is used as a decoder in encoder-decoder model, the decoder attention mask is created on the fly
+        if attention_mask is None:
+            attention_mask = input_ids.new_ones(input_shape)
+
+        # cut decoder_input_ids if past is used
+        if past_key_values and past_key_values[0] is not None:
+            input_ids = input_ids[:, -1:]
+
+        if inputs_embeds is not None and past_key_values is None:
+            model_inputs = {"inputs_embeds": inputs_embeds}
+        else:
+            model_inputs = {"input_ids": input_ids}
+
+        model_inputs.update(
+            {
+                "attention_mask": attention_mask,
+                "past_key_values": past_key_values
+            }
+        )
+
+        return model_inputs
+
+    def _reorder_cache(self, past_key_values, beam_idx):
+        reordered_past = ()
+        for layer_past in past_key_values:
+            reordered_past += (
+                tuple(
+                    past_state.index_select(0, beam_idx)
+                    for past_state in layer_past[:2]
+                )
+                + layer_past[2:],
+            )
+        return reordered_past

modeling_pointsv15_chat.py

@@ -0,0 +1,215 @@
+from typing import List, Optional, Tuple
+
+import numpy as np
+import torch
+from PIL import Image
+from transformers import GenerationMixin, PreTrainedModel, PreTrainedTokenizer
+
+try:
+    from transformers.models.qwen2_vl.image_processing_qwen2_vl import (  # noqa
+        Qwen2VLImageProcessor,
+    )
+    from transformers.models.qwen2_vl.modeling_qwen2_vl import PatchMerger
+except ImportError:
+    print('Please upgrade transformers to version 4.46.3 or higher')
+
+from .configuration_pointsv15_chat import POINTSV15ChatConfig
+from .modeling_llama import CustomLlamaForCausalLM
+
+try:
+    from wepoints.models import Qwen2VisionTransformerForNavitPOINTS
+except ImportError:
+    print('Please install WePOINTS, and refer to https://github.com/WePOINTS/WePOINTS')
+
+
+class POINTSV15ChatModel(PreTrainedModel, GenerationMixin):
+    config_class = POINTSV15ChatConfig
+    _no_split_modules = ["CustomLlamaLayer",
+                         "Qwen2VisionTransformerPretrainedModel"]
+
+    """Chat model for POINTSv1.5.
+    
+    Args:
+        config (POINTSChatConfigV15): The model config.
+    """
+
+    def __init__(self, config: POINTSV15ChatConfig) -> None:
+        super().__init__(config)
+        self.llm = CustomLlamaForCausalLM(config.llm_config)
+        self.vision_encoder = Qwen2VisionTransformerForNavitPOINTS._from_config(  # noqa
+            config.vision_config, attn_implementation="flash_attention_2"
+        )
+        self.vision_projector = PatchMerger(config.llm_config.hidden_size,
+                                            context_dim=1280)
+
+    def process_images(self, images: torch.Tensor, 
+                       image_grid_thws: List[list]) -> torch.Tensor:
+        """Obtain image features from the vision encoder.
+        
+        Args:
+            images (torch.Tensor): The input images.
+            image_grid_thws (List[list]): The grid thresholds for the images.
+
+        Returns:
+            torch.Tensor: The image features.
+        """
+        image_features = self.vision_encoder(images, grid_thw=image_grid_thws)
+        image_features = self.vision_projector(image_features)
+        return image_features
+    
+    def construct_prompt(self, messages: List[dict],
+                         image_processor: Qwen2VLImageProcessor) -> Tuple[str, List[Image.Image], List[list]]: # noqa
+        """Construct the prompt for the chat model.
+
+        Args:
+            messages (List[dict]): The input messages.
+        
+        Returns:
+            Tuple[str, List[Image.Image], List[list]]: 
+                The prompt, images, and image grid shape.
+        """
+        images = []
+        image_grid_thws = []
+        reconstructed_messages = []
+        for message in messages:
+            role = message['role']
+            content_from_role = ''
+            for item in message['content']:
+                if item['type'] == 'text':
+                    content_from_role += item['text']
+                elif item['type'] == 'image':
+                    image_path = item['image']
+                    image = Image.open(image_path).convert('RGB')
+                    image_data = image_processor(images=image)
+                    pixel_values = image_data['pixel_values']
+                    image_grid_thw = image_data['image_grid_thw']
+                    images.extend(pixel_values)
+                    image_grid_thws.append(image_grid_thw)
+                    seq_len = int(image_grid_thw[0][1] * image_grid_thw[0][2] / 4) # noqa
+                    content_from_role += '<|vision_start|>' + '<|image_pad|>' * seq_len + '<|vision_end|>' + '\n' # noqa
+            reconstructed_messages.append({
+                'role': role,
+                'content': content_from_role
+            })
+        prompt = self.apply_chat_template(reconstructed_messages)
+        return prompt, images, image_grid_thws
+    
+    def apply_chat_template(self, messages: List[dict]) -> str:
+        """Apply the chat template to the input messages.
+
+        Args:
+            messages (List[dict]): The input messages.
+        
+        Returns:
+            str: The prompt.
+        """
+        role_prefix_mapping = {
+            'user': '<|im_start|>user\n',
+            'assistant': '<|im_start|>assistant\n'
+        }
+        role = 'user'
+        prompt = ''
+        for message in messages:
+            role = message['role']
+            content = message['content']
+            prompt += role_prefix_mapping[role] + content + '<|im_end|>\n'
+        if role == 'user':
+            prompt += '<|im_start|>assistant\n'
+        return prompt
+
+    @torch.no_grad()
+    def chat(self, 
+             messages: List[dict],
+             tokenizer: PreTrainedTokenizer,
+             image_processor: object,
+             generation_config: dict = None) -> str:
+        """Generate a response to the input prompt.
+
+        Args:
+            messages (List[dict]): The input messages.
+            tokenizer (PreTrainedTokenizer): The tokenizer to use.
+            image_processor (object): The image processor to use.
+            generation_config (dict, optional): The generation config. 
+                Defaults to None.
+        Returns:
+            str: The generated response.
+        """
+        prompt, images, image_grid_thws = self.construct_prompt(
+            messages, image_processor
+        )
+        images = np.array(images)
+        images = torch.from_numpy(images).to(self.vision_encoder.device).to(self.vision_encoder.dtype) # noqa
+        image_grid_thws = np.concatenate(image_grid_thws, axis=0)
+        image_grid_thws = (
+            torch.from_numpy(image_grid_thws)
+            .cuda()
+            .long()
+        )
+        image_features = self.vision_encoder(images, grid_thw=image_grid_thws)
+        image_features = self.vision_projector(image_features)
+        model_inputs = tokenizer(prompt, return_tensors='pt')
+        input_ids = model_inputs['input_ids'].to(self.device)
+        attention_mask = model_inputs['attention_mask'].to(self.device)
+        # stop token
+        eos_token_id = tokenizer.convert_tokens_to_ids("<|im_end|>")
+        # image token
+        image_token_id = tokenizer.convert_tokens_to_ids("<|image_pad|>")
+        generation_config.update(
+            {
+                'eos_token_id': eos_token_id,
+            }
+        )
+        outputs = self.generate(
+            input_ids=input_ids,
+            image_grid_thws=image_grid_thws,
+            attention_mask=attention_mask,
+            image_features=[image_features],
+            image_token_id=image_token_id,
+            **generation_config
+        )
+        response = tokenizer.batch_decode(outputs, skip_special_tokens=True)[0]
+        return response
+      
+    def _split_input_ids(self, input_ids, special_token):
+        special_pos = input_ids == special_token
+        pos = (special_pos[:-1] != special_pos[1:]).nonzero() + 1
+        if pos.shape[0] % 2 != 0:
+            pos = torch.cat([torch.tensor([[0]]).to(pos.device), pos])
+        pos = pos.reshape(-1, 2).tolist()
+        return pos
+
+    def generate(self,
+                 input_ids: torch.LongTensor,
+                 image_grid_thws: torch.LongTensor,
+                 attention_mask: torch.LongTensor,
+                 image_features: List[torch.Tensor],
+                 image_token_id: int,
+                 generation_config: Optional[dict] = None,
+                 output_hidden_states: Optional[bool] = None,
+                 return_dict: Optional[bool] = None,
+                 **generate_kwargs) -> torch.LongTensor:
+        input_embeddings = self.llm.lm.embed_in(input_ids)
+        batch_size = input_ids.shape[0]
+        assert len(image_features) == batch_size
+        for i in range(batch_size):
+            pos = self._split_input_ids(input_ids[i], image_token_id)
+            assert len(pos) == len(image_grid_thws)
+            image_pos = [
+                int(image_grid_thw[1] * image_grid_thw[2] / 4)
+                for image_grid_thw in image_grid_thws
+            ]
+            image_pos.insert(0, 0)
+            image_pos = np.cumsum(image_pos)
+            for j, (start, end) in enumerate(pos):
+                input_embeddings[i, start:end] = \
+                    image_features[i][image_pos[j]:image_pos[j+1]]
+        outputs = self.llm.generate(
+            inputs_embeds=input_embeddings,
+            attention_mask=attention_mask,
+            generation_config=generation_config,
+            output_hidden_states=output_hidden_states,
+            return_dict=return_dict,
+            use_cache=True,
+            **generate_kwargs
+        )
+        return outputs

preprocessor_config.json

@@ -0,0 +1,28 @@
+{
+  "do_convert_rgb": true,
+  "do_normalize": true,
+  "do_rescale": true,
+  "do_resize": true,
+  "image_mean": [
+    0.48145466,
+    0.4578275,
+    0.40821073
+  ],
+  "image_processor_type": "Qwen2VLImageProcessor",
+  "image_std": [
+    0.26862954,
+    0.26130258,
+    0.27577711
+  ],
+  "max_pixels": 12845056,
+  "merge_size": 2,
+  "min_pixels": 3136,
+  "patch_size": 14,
+  "resample": 3,
+  "rescale_factor": 0.00392156862745098,
+  "size": {
+    "max_pixels": 12845056,
+    "min_pixels": 3136
+  },
+  "temporal_patch_size": 2
+}

tokenizer_config.json

@@ -0,0 +1,207 @@
+{
+  "add_bos_token": false,
+  "add_prefix_space": false,
+  "added_tokens_decoder": {
+    "151643": {
+      "content": "<|endoftext|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151644": {
+      "content": "<|im_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151645": {
+      "content": "<|im_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151646": {
+      "content": "<|object_ref_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151647": {
+      "content": "<|object_ref_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151648": {
+      "content": "<|box_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151649": {
+      "content": "<|box_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151650": {
+      "content": "<|quad_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151651": {
+      "content": "<|quad_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151652": {
+      "content": "<|vision_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151653": {
+      "content": "<|vision_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151654": {
+      "content": "<|vision_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151655": {
+      "content": "<|image_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151656": {
+      "content": "<|video_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151657": {
+      "content": "<tool_call>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151658": {
+      "content": "</tool_call>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151659": {
+      "content": "<|fim_prefix|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151660": {
+      "content": "<|fim_middle|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151661": {
+      "content": "<|fim_suffix|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151662": {
+      "content": "<|fim_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151663": {
+      "content": "<|repo_name|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151664": {
+      "content": "<|file_sep|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    }
+  },
+  "additional_special_tokens": [
+    "<|im_start|>",
+    "<|im_end|>",
+    "<|object_ref_start|>",
+    "<|object_ref_end|>",
+    "<|box_start|>",
+    "<|box_end|>",
+    "<|quad_start|>",
+    "<|quad_end|>",
+    "<|vision_start|>",
+    "<|vision_end|>",
+    "<|vision_pad|>",
+    "<|image_pad|>",
+    "<|video_pad|>"
+  ],
+  "bos_token": null,
+  "chat_template": "{%- if tools %}\n    {{- '<|im_start|>system\\n' }}\n    {%- if messages[0]['role'] == 'system' %}\n        {{- messages[0]['content'] }}\n    {%- else %}\n        {{- 'You are Qwen, created by Alibaba Cloud. You are a helpful assistant.' }}\n    {%- endif %}\n    {{- \"\\n\\n# Tools\\n\\nYou may call one or more functions to assist with the user query.\\n\\nYou are provided with function signatures within <tools></tools> XML tags:\\n<tools>\" }}\n    {%- for tool in tools %}\n        {{- \"\\n\" }}\n        {{- tool | tojson }}\n    {%- endfor %}\n    {{- \"\\n</tools>\\n\\nFor each function call, return a json object with function name and arguments within <tool_call></tool_call> XML tags:\\n<tool_call>\\n{\\\"name\\\": <function-name>, \\\"arguments\\\": <args-json-object>}\\n</tool_call><|im_end|>\\n\" }}\n{%- else %}\n    {%- if messages[0]['role'] == 'system' %}\n        {{- '<|im_start|>system\\n' + messages[0]['content'] + '<|im_end|>\\n' }}\n    {%- else %}\n        {{- '<|im_start|>system\\nYou are Qwen, created by Alibaba Cloud. You are a helpful assistant.<|im_end|>\\n' }}\n    {%- endif %}\n{%- endif %}\n{%- for message in messages %}\n    {%- if (message.role == \"user\") or (message.role == \"system\" and not loop.first) or (message.role == \"assistant\" and not message.tool_calls) %}\n        {{- '<|im_start|>' + message.role + '\\n' + message.content + '<|im_end|>' + '\\n' }}\n    {%- elif message.role == \"assistant\" %}\n        {{- '<|im_start|>' + message.role }}\n        {%- if message.content %}\n            {{- '\\n' + message.content }}\n        {%- endif %}\n        {%- for tool_call in message.tool_calls %}\n            {%- if tool_call.function is defined %}\n                {%- set tool_call = tool_call.function %}\n            {%- endif %}\n            {{- '\\n<tool_call>\\n{\"name\": \"' }}\n            {{- tool_call.name }}\n            {{- '\", \"arguments\": ' }}\n            {{- tool_call.arguments | tojson }}\n            {{- '}\\n</tool_call>' }}\n        {%- endfor %}\n        {{- '<|im_end|>\\n' }}\n    {%- elif message.role == \"tool\" %}\n        {%- if (loop.index0 == 0) or (messages[loop.index0 - 1].role != \"tool\") %}\n            {{- '<|im_start|>user' }}\n        {%- endif %}\n        {{- '\\n<tool_response>\\n' }}\n        {{- message.content }}\n        {{- '\\n</tool_response>' }}\n        {%- if loop.last or (messages[loop.index0 + 1].role != \"tool\") %}\n            {{- '<|im_end|>\\n' }}\n        {%- endif %}\n    {%- endif %}\n{%- endfor %}\n{%- if add_generation_prompt %}\n    {{- '<|im_start|>assistant\\n' }}\n{%- endif %}\n",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<|im_end|>",
+  "errors": "replace",
+  "model_max_length": 131072,
+  "pad_token": "<|endoftext|>",
+  "split_special_tokens": false,
+  "tokenizer_class": "Qwen2Tokenizer",
+  "unk_token": null
+}