# Finetuning midddle layers (vision module layers = 22 to 26 +projector + language layers = 0 to 4)
# gemma-3-4b-it
# No. of images used in training = 500

```
model.vision_tower.vision_model.encoder.layers.22.self_attn.k_proj.weight | shape=(1152, 1152) | requires_grad=True
model.vision_tower.vision_model.encoder.layers.22.self_attn.k_proj.bias | shape=(1152,) | requires_grad=True
model.vision_tower.vision_model.encoder.layers.22.self_attn.v_proj.weight | shape=(1152, 1152) | requires_grad=True
model.vision_tower.vision_model.encoder.layers.22.self_attn.v_proj.bias | shape=(1152,) | requires_grad=True
model.vision_tower.vision_model.encoder.layers.22.self_attn.q_proj.weight | shape=(1152, 1152) | requires_grad=True
model.vision_tower.vision_model.encoder.layers.22.self_attn.q_proj.bias | shape=(1152,) | requires_grad=True
model.vision_tower.vision_model.encoder.layers.22.self_attn.out_proj.weight | shape=(1152, 1152) | requires_grad=True
model.vision_tower.vision_model.encoder.layers.22.self_attn.out_proj.bias | shape=(1152,) | requires_grad=True
model.vision_tower.vision_model.encoder.layers.22.layer_norm2.weight | shape=(1152,) | requires_grad=True
model.vision_tower.vision_model.encoder.layers.22.layer_norm2.bias | shape=(1152,) | requires_grad=True
model.vision_tower.vision_model.encoder.layers.22.mlp.fc1.weight | shape=(4304, 1152) | requires_grad=True
model.vision_tower.vision_model.encoder.layers.22.mlp.fc1.bias | shape=(4304,) | requires_grad=True
model.vision_tower.vision_model.encoder.layers.22.mlp.fc2.weight | shape=(1152, 4304) | requires_grad=True
model.vision_tower.vision_model.encoder.layers.22.mlp.fc2.bias | shape=(1152,) | requires_grad=True
model.vision_tower.vision_model.encoder.layers.23.layer_norm1.weight | shape=(1152,) | requires_grad=True
model.vision_tower.vision_model.encoder.layers.23.layer_norm1.bias | shape=(1152,) | requires_grad=True
model.vision_tower.vision_model.encoder.layers.23.self_attn.k_proj.weight | shape=(1152, 1152) | requires_grad=True
model.vision_tower.vision_model.encoder.layers.23.self_attn.k_proj.bias | shape=(1152,) | requires_grad=True
model.vision_tower.vision_model.encoder.layers.23.self_attn.v_proj.weight | shape=(1152, 1152) | requires_grad=True
model.vision_tower.vision_model.encoder.layers.23.self_attn.v_proj.bias | shape=(1152,) | requires_grad=True
model.vision_tower.vision_model.encoder.layers.23.self_attn.q_proj.weight | shape=(1152, 1152) | requires_grad=True
model.vision_tower.vision_model.encoder.layers.23.self_attn.q_proj.bias | shape=(1152,) | requires_grad=True
model.vision_tower.vision_model.encoder.layers.23.self_attn.out_proj.weight | shape=(1152, 1152) | requires_grad=True
model.vision_tower.vision_model.encoder.layers.23.self_attn.out_proj.bias | shape=(1152,) | requires_grad=True
model.vision_tower.vision_model.encoder.layers.23.layer_norm2.weight | shape=(1152,) | requires_grad=True
model.vision_tower.vision_model.encoder.layers.23.layer_norm2.bias | shape=(1152,) | requires_grad=True
model.vision_tower.vision_model.encoder.layers.23.mlp.fc1.weight | shape=(4304, 1152) | requires_grad=True
model.vision_tower.vision_model.encoder.layers.23.mlp.fc1.bias | shape=(4304,) | requires_grad=True
model.vision_tower.vision_model.encoder.layers.23.mlp.fc2.weight | shape=(1152, 4304) | requires_grad=True
model.vision_tower.vision_model.encoder.layers.23.mlp.fc2.bias | shape=(1152,) | requires_grad=True
model.vision_tower.vision_model.encoder.layers.24.layer_norm1.weight | shape=(1152,) | requires_grad=True
model.vision_tower.vision_model.encoder.layers.24.layer_norm1.bias | shape=(1152,) | requires_grad=True
model.vision_tower.vision_model.encoder.layers.24.self_attn.k_proj.weight | shape=(1152, 1152) | requires_grad=True
model.vision_tower.vision_model.encoder.layers.24.self_attn.k_proj.bias | shape=(1152,) | requires_grad=True
model.vision_tower.vision_model.encoder.layers.24.self_attn.v_proj.weight | shape=(1152, 1152) | requires_grad=True
model.vision_tower.vision_model.encoder.layers.24.self_attn.v_proj.bias | shape=(1152,) | requires_grad=True
model.vision_tower.vision_model.encoder.layers.24.self_attn.q_proj.weight | shape=(1152, 1152) | requires_grad=True
model.vision_tower.vision_model.encoder.layers.24.self_attn.q_proj.bias | shape=(1152,) | requires_grad=True
model.vision_tower.vision_model.encoder.layers.24.self_attn.out_proj.weight | shape=(1152, 1152) | requires_grad=True
model.vision_tower.vision_model.encoder.layers.24.self_attn.out_proj.bias | shape=(1152,) | requires_grad=True
model.vision_tower.vision_model.encoder.layers.24.layer_norm2.weight | shape=(1152,) | requires_grad=True
model.vision_tower.vision_model.encoder.layers.24.layer_norm2.bias | shape=(1152,) | requires_grad=True
model.vision_tower.vision_model.encoder.layers.24.mlp.fc1.weight | shape=(4304, 1152) | requires_grad=True
model.vision_tower.vision_model.encoder.layers.24.mlp.fc1.bias | shape=(4304,) | requires_grad=True
model.vision_tower.vision_model.encoder.layers.24.mlp.fc2.weight | shape=(1152, 4304) | requires_grad=True
model.vision_tower.vision_model.encoder.layers.24.mlp.fc2.bias | shape=(1152,) | requires_grad=True
model.vision_tower.vision_model.encoder.layers.25.layer_norm1.weight | shape=(1152,) | requires_grad=True
model.vision_tower.vision_model.encoder.layers.25.layer_norm1.bias | shape=(1152,) | requires_grad=True
model.vision_tower.vision_model.encoder.layers.25.self_attn.k_proj.weight | shape=(1152, 1152) | requires_grad=True
model.vision_tower.vision_model.encoder.layers.25.self_attn.k_proj.bias | shape=(1152,) | requires_grad=True
model.vision_tower.vision_model.encoder.layers.25.self_attn.v_proj.weight | shape=(1152, 1152) | requires_grad=True
model.vision_tower.vision_model.encoder.layers.25.self_attn.v_proj.bias | shape=(1152,) | requires_grad=True
model.vision_tower.vision_model.encoder.layers.25.self_attn.q_proj.weight | shape=(1152, 1152) | requires_grad=True
model.vision_tower.vision_model.encoder.layers.25.self_attn.q_proj.bias | shape=(1152,) | requires_grad=True
model.vision_tower.vision_model.encoder.layers.25.self_attn.out_proj.weight | shape=(1152, 1152) | requires_grad=True
model.vision_tower.vision_model.encoder.layers.25.self_attn.out_proj.bias | shape=(1152,) | requires_grad=True
model.vision_tower.vision_model.encoder.layers.25.layer_norm2.weight | shape=(1152,) | requires_grad=True
model.vision_tower.vision_model.encoder.layers.25.layer_norm2.bias | shape=(1152,) | requires_grad=True
model.vision_tower.vision_model.encoder.layers.25.mlp.fc1.weight | shape=(4304, 1152) | requires_grad=True
model.vision_tower.vision_model.encoder.layers.25.mlp.fc1.bias | shape=(4304,) | requires_grad=True
model.vision_tower.vision_model.encoder.layers.25.mlp.fc2.weight | shape=(1152, 4304) | requires_grad=True
model.vision_tower.vision_model.encoder.layers.25.mlp.fc2.bias | shape=(1152,) | requires_grad=True
model.vision_tower.vision_model.encoder.layers.26.layer_norm1.weight | shape=(1152,) | requires_grad=True
model.vision_tower.vision_model.encoder.layers.26.layer_norm1.bias | shape=(1152,) | requires_grad=True
model.vision_tower.vision_model.encoder.layers.26.self_attn.k_proj.weight | shape=(1152, 1152) | requires_grad=True
model.vision_tower.vision_model.encoder.layers.26.self_attn.k_proj.bias | shape=(1152,) | requires_grad=True
model.vision_tower.vision_model.encoder.layers.26.self_attn.v_proj.weight | shape=(1152, 1152) | requires_grad=True
model.vision_tower.vision_model.encoder.layers.26.self_attn.v_proj.bias | shape=(1152,) | requires_grad=True
model.vision_tower.vision_model.encoder.layers.26.self_attn.q_proj.weight | shape=(1152, 1152) | requires_grad=True
model.vision_tower.vision_model.encoder.layers.26.self_attn.q_proj.bias | shape=(1152,) | requires_grad=True
model.vision_tower.vision_model.encoder.layers.26.self_attn.out_proj.weight | shape=(1152, 1152) | requires_grad=True
model.vision_tower.vision_model.encoder.layers.26.self_attn.out_proj.bias | shape=(1152,) | requires_grad=True
model.vision_tower.vision_model.encoder.layers.26.layer_norm2.weight | shape=(1152,) | requires_grad=True
model.vision_tower.vision_model.encoder.layers.26.layer_norm2.bias | shape=(1152,) | requires_grad=True
model.vision_tower.vision_model.encoder.layers.26.mlp.fc1.weight | shape=(4304, 1152) | requires_grad=True
model.vision_tower.vision_model.encoder.layers.26.mlp.fc1.bias | shape=(4304,) | requires_grad=True
model.vision_tower.vision_model.encoder.layers.26.mlp.fc2.weight | shape=(1152, 4304) | requires_grad=True
model.vision_tower.vision_model.encoder.layers.26.mlp.fc2.bias | shape=(1152,) | requires_grad=True
model.vision_tower.vision_model.post_layernorm.weight | shape=(1152,) | requires_grad=True
model.vision_tower.vision_model.post_layernorm.bias | shape=(1152,) | requires_grad=True
model.multi_modal_projector.mm_input_projection_weight | shape=(1152, 2560) | requires_grad=True
model.multi_modal_projector.mm_soft_emb_norm.weight | shape=(1152,) | requires_grad=True
model.language_model.embed_tokens.weight | shape=(262208, 2560) | requires_grad=True
model.language_model.layers.0.self_attn.q_proj.weight | shape=(2048, 2560) | requires_grad=True
model.language_model.layers.0.self_attn.k_proj.weight | shape=(1024, 2560) | requires_grad=True
model.language_model.layers.0.self_attn.v_proj.weight | shape=(1024, 2560) | requires_grad=True
model.language_model.layers.0.self_attn.o_proj.weight | shape=(2560, 2048) | requires_grad=True
model.language_model.layers.0.self_attn.q_norm.weight | shape=(256,) | requires_grad=True
model.language_model.layers.0.self_attn.k_norm.weight | shape=(256,) | requires_grad=True
model.language_model.layers.0.mlp.gate_proj.weight | shape=(10240, 2560) | requires_grad=True
model.language_model.layers.0.mlp.up_proj.weight | shape=(10240, 2560) | requires_grad=True
model.language_model.layers.0.mlp.down_proj.weight | shape=(2560, 10240) | requires_grad=True
model.language_model.layers.0.input_layernorm.weight | shape=(2560,) | requires_grad=True
model.language_model.layers.0.post_attention_layernorm.weight | shape=(2560,) | requires_grad=True
model.language_model.layers.0.pre_feedforward_layernorm.weight | shape=(2560,) | requires_grad=True
model.language_model.layers.0.post_feedforward_layernorm.weight | shape=(2560,) | requires_grad=True
model.language_model.layers.1.self_attn.q_proj.weight | shape=(2048, 2560) | requires_grad=True
model.language_model.layers.1.self_attn.k_proj.weight | shape=(1024, 2560) | requires_grad=True
model.language_model.layers.1.self_attn.v_proj.weight | shape=(1024, 2560) | requires_grad=True
model.language_model.layers.1.self_attn.o_proj.weight | shape=(2560, 2048) | requires_grad=True
model.language_model.layers.1.self_attn.q_norm.weight | shape=(256,) | requires_grad=True
model.language_model.layers.1.self_attn.k_norm.weight | shape=(256,) | requires_grad=True
model.language_model.layers.1.mlp.gate_proj.weight | shape=(10240, 2560) | requires_grad=True
model.language_model.layers.1.mlp.up_proj.weight | shape=(10240, 2560) | requires_grad=True
model.language_model.layers.1.mlp.down_proj.weight | shape=(2560, 10240) | requires_grad=True
model.language_model.layers.1.input_layernorm.weight | shape=(2560,) | requires_grad=True
model.language_model.layers.1.post_attention_layernorm.weight | shape=(2560,) | requires_grad=True
model.language_model.layers.1.pre_feedforward_layernorm.weight | shape=(2560,) | requires_grad=True
model.language_model.layers.1.post_feedforward_layernorm.weight | shape=(2560,) | requires_grad=True
model.language_model.layers.2.self_attn.q_proj.weight | shape=(2048, 2560) | requires_grad=True
model.language_model.layers.2.self_attn.k_proj.weight | shape=(1024, 2560) | requires_grad=True
model.language_model.layers.2.self_attn.v_proj.weight | shape=(1024, 2560) | requires_grad=True
model.language_model.layers.2.self_attn.o_proj.weight | shape=(2560, 2048) | requires_grad=True
model.language_model.layers.2.self_attn.q_norm.weight | shape=(256,) | requires_grad=True
model.language_model.layers.2.self_attn.k_norm.weight | shape=(256,) | requires_grad=True
model.language_model.layers.2.mlp.gate_proj.weight | shape=(10240, 2560) | requires_grad=True
model.language_model.layers.2.mlp.up_proj.weight | shape=(10240, 2560) | requires_grad=True
model.language_model.layers.2.mlp.down_proj.weight | shape=(2560, 10240) | requires_grad=True
model.language_model.layers.2.input_layernorm.weight | shape=(2560,) | requires_grad=True
model.language_model.layers.2.post_attention_layernorm.weight | shape=(2560,) | requires_grad=True
model.language_model.layers.2.pre_feedforward_layernorm.weight | shape=(2560,) | requires_grad=True
model.language_model.layers.2.post_feedforward_layernorm.weight | shape=(2560,) | requires_grad=True
model.language_model.layers.3.self_attn.q_proj.weight | shape=(2048, 2560) | requires_grad=True
model.language_model.layers.3.self_attn.k_proj.weight | shape=(1024, 2560) | requires_grad=True
model.language_model.layers.3.self_attn.v_proj.weight | shape=(1024, 2560) | requires_grad=True
model.language_model.layers.3.self_attn.o_proj.weight | shape=(2560, 2048) | requires_grad=True
model.language_model.layers.3.self_attn.q_norm.weight | shape=(256,) | requires_grad=True
model.language_model.layers.3.self_attn.k_norm.weight | shape=(256,) | requires_grad=True
model.language_model.layers.3.mlp.gate_proj.weight | shape=(10240, 2560) | requires_grad=True
model.language_model.layers.3.mlp.up_proj.weight | shape=(10240, 2560) | requires_grad=True
model.language_model.layers.3.mlp.down_proj.weight | shape=(2560, 10240) | requires_grad=True
model.language_model.layers.3.input_layernorm.weight | shape=(2560,) | requires_grad=True
model.language_model.layers.3.post_attention_layernorm.weight | shape=(2560,) | requires_grad=True
model.language_model.layers.3.pre_feedforward_layernorm.weight | shape=(2560,) | requires_grad=True
model.language_model.layers.3.post_feedforward_layernorm.weight | shape=(2560,) | requires_grad=True
model.language_model.layers.4.self_attn.q_proj.weight | shape=(2048, 2560) | requires_grad=True
model.language_model.layers.4.self_attn.k_proj.weight | shape=(1024, 2560) | requires_grad=True
model.language_model.layers.4.self_attn.v_proj.weight | shape=(1024, 2560) | requires_grad=True
model.language_model.layers.4.self_attn.o_proj.weight | shape=(2560, 2048) | requires_grad=True
model.language_model.layers.4.self_attn.q_norm.weight | shape=(256,) | requires_grad=True
model.language_model.layers.4.self_attn.k_norm.weight | shape=(256,) | requires_grad=True
model.language_model.layers.4.mlp.gate_proj.weight | shape=(10240, 2560) | requires_grad=True
model.language_model.layers.4.mlp.up_proj.weight | shape=(10240, 2560) | requires_grad=True
model.language_model.layers.4.mlp.down_proj.weight | shape=(2560, 10240) | requires_grad=True
model.language_model.layers.4.input_layernorm.weight | shape=(2560,) | requires_grad=True
model.language_model.layers.4.post_attention_layernorm.weight | shape=(2560,) | requires_grad=True
model.language_model.layers.4.pre_feedforward_layernorm.weight | shape=(2560,) | requires_grad=True
model.language_model.layers.4.post_feedforward_layernorm.weight | shape=(2560,) | requires_grad=True

```


HyperParameters of this model

lr: 2e-5

epochs: 10

batch size: 1

grad acumulation step: 4