Upload folder using huggingface_hub

config.json

@@ -1,5 +1,5 @@
 {
-  "_name_or_path": "meta-llama/Meta-Llama-3.1-8B-Instruct",
+  "_name_or_path": "VPTQ-community/Meta-Llama-3.1-8B-Instruct-v8-k65536-4096-woft",
   "architectures": [
     "LlamaForCausalLM"
   ],
@@ -11,6 +11,7 @@
     128008,
     128009
   ],
+  "head_dim": 128,
   "hidden_act": "silu",
   "hidden_size": 4096,
   "initializer_range": 0.02,
@@ -22,27 +23,12 @@
   "num_hidden_layers": 32,
   "num_key_value_heads": 8,
   "pretraining_tp": 1,
-  "rms_norm_eps": 1e-05,
-  "rope_scaling": {
-    "factor": 8.0,
-    "high_freq_factor": 4.0,
-    "low_freq_factor": 1.0,
-    "original_max_position_embeddings": 8192,
-    "rope_type": "llama3"
-  },
-  "rope_theta": 500000.0,
-  "tie_word_embeddings": false,
-  "torch_dtype": "bfloat16",
-  "transformers_version": "4.45.1",
-  "use_cache": true,
-  "vocab_size": 128256,
   "quantization_config": {
-    "quant_method": "vptq",
     "config_for_layers": {
       "model.layers.0.mlp.down_proj": {
         "bias": false,
         "enable_norm": true,
-        "enable_perm": true,
+        "enable_perm": false,
         "group_num": 1,
         "group_size": 14336,
         "in_features": 14336,
@@ -66,7 +52,7 @@
       "model.layers.0.mlp.gate_proj": {
         "bias": false,
         "enable_norm": true,
-        "enable_perm": true,
+        "enable_perm": false,
         "group_num": 1,
         "group_size": 4096,
         "in_features": 4096,
@@ -90,7 +76,7 @@
       "model.layers.0.mlp.up_proj": {
         "bias": false,
         "enable_norm": true,
-        "enable_perm": true,
+        "enable_perm": false,
         "group_num": 1,
         "group_size": 4096,
         "in_features": 4096,
@@ -114,7 +100,7 @@
       "model.layers.0.self_attn.k_proj": {
         "bias": false,
         "enable_norm": true,
-        "enable_perm": true,
+        "enable_perm": false,
         "group_num": 1,
         "group_size": 4096,
         "in_features": 4096,
@@ -138,7 +124,7 @@
       "model.layers.0.self_attn.o_proj": {
         "bias": false,
         "enable_norm": true,
-        "enable_perm": true,
+        "enable_perm": false,
         "group_num": 1,
         "group_size": 4096,
         "in_features": 4096,
@@ -162,7 +148,7 @@
       "model.layers.0.self_attn.q_proj": {
         "bias": false,
         "enable_norm": true,
-        "enable_perm": true,
+        "enable_perm": false,
         "group_num": 1,
         "group_size": 4096,
         "in_features": 4096,
@@ -186,7 +172,7 @@
       "model.layers.0.self_attn.v_proj": {
         "bias": false,
         "enable_norm": true,
-        "enable_perm": true,
+        "enable_perm": false,
         "group_num": 1,
         "group_size": 4096,
         "in_features": 4096,
@@ -210,7 +196,7 @@
       "model.layers.1.mlp.down_proj": {
         "bias": false,
         "enable_norm": true,
-        "enable_perm": true,
+        "enable_perm": false,
         "group_num": 1,
         "group_size": 14336,
         "in_features": 14336,
@@ -234,7 +220,7 @@
       "model.layers.1.mlp.gate_proj": {
         "bias": false,
         "enable_norm": true,
-        "enable_perm": true,
+        "enable_perm": false,
         "group_num": 1,
         "group_size": 4096,
         "in_features": 4096,
@@ -258,7 +244,7 @@
       "model.layers.1.mlp.up_proj": {
         "bias": false,
         "enable_norm": true,
-        "enable_perm": true,
+        "enable_perm": false,
         "group_num": 1,
         "group_size": 4096,
         "in_features": 4096,
@@ -282,7 +268,7 @@
       "model.layers.1.self_attn.k_proj": {
         "bias": false,
         "enable_norm": true,
-        "enable_perm": true,
+        "enable_perm": false,
         "group_num": 1,
         "group_size": 4096,
         "in_features": 4096,
@@ -306,7 +292,7 @@
       "model.layers.1.self_attn.o_proj": {
         "bias": false,
         "enable_norm": true,
-        "enable_perm": true,
+        "enable_perm": false,
         "group_num": 1,
         "group_size": 4096,
         "in_features": 4096,
@@ -330,7 +316,7 @@
       "model.layers.1.self_attn.q_proj": {
         "bias": false,
         "enable_norm": true,
-        "enable_perm": true,
+        "enable_perm": false,
         "group_num": 1,
         "group_size": 4096,
         "in_features": 4096,
@@ -354,7 +340,7 @@
       "model.layers.1.self_attn.v_proj": {
         "bias": false,
         "enable_norm": true,
-        "enable_perm": true,
+        "enable_perm": false,
         "group_num": 1,
         "group_size": 4096,
         "in_features": 4096,
@@ -378,7 +364,7 @@
       "model.layers.10.mlp.down_proj": {
         "bias": false,
         "enable_norm": true,
-        "enable_perm": true,
+        "enable_perm": false,
         "group_num": 1,
         "group_size": 14336,
         "in_features": 14336,
@@ -402,7 +388,7 @@
       "model.layers.10.mlp.gate_proj": {
         "bias": false,
         "enable_norm": true,
-        "enable_perm": true,
+        "enable_perm": false,
         "group_num": 1,
         "group_size": 4096,
         "in_features": 4096,
@@ -426,7 +412,7 @@
       "model.layers.10.mlp.up_proj": {
         "bias": false,
         "enable_norm": true,
-        "enable_perm": true,
+        "enable_perm": false,
         "group_num": 1,
         "group_size": 4096,
         "in_features": 4096,
@@ -450,7 +436,7 @@
       "model.layers.10.self_attn.k_proj": {
         "bias": false,
         "enable_norm": true,
-        "enable_perm": true,
+        "enable_perm": false,
         "group_num": 1,
         "group_size": 4096,
         "in_features": 4096,
@@ -474,7 +460,7 @@
       "model.layers.10.self_attn.o_proj": {
         "bias": false,
         "enable_norm": true,
-        "enable_perm": true,
+        "enable_perm": false,
         "group_num": 1,
         "group_size": 4096,
         "in_features": 4096,
@@ -498,7 +484,7 @@
       "model.layers.10.self_attn.q_proj": {
         "bias": false,
         "enable_norm": true,
-        "enable_perm": true,
+        "enable_perm": false,
         "group_num": 1,
         "group_size": 4096,
         "in_features": 4096,
@@ -522,7 +508,7 @@
       "model.layers.10.self_attn.v_proj": {
         "bias": false,
         "enable_norm": true,
-        "enable_perm": true,
+        "enable_perm": false,
         "group_num": 1,
         "group_size": 4096,
         "in_features": 4096,
@@ -546,7 +532,7 @@
       "model.layers.11.mlp.down_proj": {
         "bias": false,
         "enable_norm": true,
-        "enable_perm": true,
+        "enable_perm": false,
         "group_num": 1,
         "group_size": 14336,
         "in_features": 14336,
@@ -570,7 +556,7 @@
       "model.layers.11.mlp.gate_proj": {
         "bias": false,
         "enable_norm": true,
-        "enable_perm": true,
+        "enable_perm": false,
         "group_num": 1,
         "group_size": 4096,
         "in_features": 4096,
@@ -594,7 +580,7 @@
       "model.layers.11.mlp.up_proj": {
         "bias": false,
         "enable_norm": true,
-        "enable_perm": true,
+        "enable_perm": false,
         "group_num": 1,
         "group_size": 4096,
         "in_features": 4096,
@@ -618,7 +604,7 @@
       "model.layers.11.self_attn.k_proj": {
         "bias": false,
         "enable_norm": true,
-        "enable_perm": true,
+        "enable_perm": false,
         "group_num": 1,
         "group_size": 4096,
         "in_features": 4096,
@@ -642,7 +628,7 @@
       "model.layers.11.self_attn.o_proj": {
         "bias": false,
         "enable_norm": true,
-        "enable_perm": true,
+        "enable_perm": false,
         "group_num": 1,
         "group_size": 4096,
         "in_features": 4096,
@@ -666,7 +652,7 @@
       "model.layers.11.self_attn.q_proj": {
         "bias": false,
         "enable_norm": true,
-        "enable_perm": true,
+        "enable_perm": false,
         "group_num": 1,
         "group_size": 4096,
         "in_features": 4096,
@@ -690,7 +676,7 @@
       "model.layers.11.self_attn.v_proj": {
         "bias": false,
         "enable_norm": true,
-        "enable_perm": true,
+        "enable_perm": false,
         "group_num": 1,
         "group_size": 4096,
         "in_features": 4096,
@@ -714,7 +700,7 @@
       "model.layers.12.mlp.down_proj": {
         "bias": false,
         "enable_norm": true,
-        "enable_perm": true,
+        "enable_perm": false,
         "group_num": 1,
         "group_size": 14336,
         "in_features": 14336,
@@ -738,7 +724,7 @@
       "model.layers.12.mlp.gate_proj": {
         "bias": false,
         "enable_norm": true,
-        "enable_perm": true,
+        "enable_perm": false,
         "group_num": 1,
         "group_size": 4096,
         "in_features": 4096,
@@ -762,7 +748,7 @@
       "model.layers.12.mlp.up_proj": {
         "bias": false,
         "enable_norm": true,
-        "enable_perm": true,
+        "enable_perm": false,
         "group_num": 1,
         "group_size": 4096,
         "in_features": 4096,
@@ -786,7 +772,7 @@
       "model.layers.12.self_attn.k_proj": {
         "bias": false,
         "enable_norm": true,
-        "enable_perm": true,
+        "enable_perm": false,
         "group_num": 1,
         "group_size": 4096,
         "in_features": 4096,
@@ -810,7 +796,7 @@
       "model.layers.12.self_attn.o_proj": {
         "bias": false,
         "enable_norm": true,
-        "enable_perm": true,
+        "enable_perm": false,
         "group_num": 1,
         "group_size": 4096,
         "in_features": 4096,
@@ -834,7 +820,7 @@
       "model.layers.12.self_attn.q_proj": {
         "bias": false,
         "enable_norm": true,
-        "enable_perm": true,
+        "enable_perm": false,
         "group_num": 1,
         "group_size": 4096,
         "in_features": 4096,
@@ -858,7 +844,7 @@
       "model.layers.12.self_attn.v_proj": {
         "bias": false,
         "enable_norm": true,
-        "enable_perm": true,
+        "enable_perm": false,
         "group_num": 1,
         "group_size": 4096,
         "in_features": 4096,
@@ -882,7 +868,7 @@
       "model.layers.13.mlp.down_proj": {
         "bias": false,
         "enable_norm": true,
-        "enable_perm": true,
+        "enable_perm": false,
         "group_num": 1,
         "group_size": 14336,
         "in_features": 14336,
@@ -906,7 +892,7 @@
       "model.layers.13.mlp.gate_proj": {
         "bias": false,
         "enable_norm": true,
-        "enable_perm": true,
+        "enable_perm": false,
         "group_num": 1,
         "group_size": 4096,
         "in_features": 4096,
@@ -930,7 +916,7 @@
       "model.layers.13.mlp.up_proj": {
         "bias": false,
         "enable_norm": true,
-        "enable_perm": true,
+        "enable_perm": false,
         "group_num": 1,
         "group_size": 4096,
         "in_features": 4096,
@@ -954,7 +940,7 @@
       "model.layers.13.self_attn.k_proj": {
         "bias": false,
         "enable_norm": true,
-        "enable_perm": true,
+        "enable_perm": false,
         "group_num": 1,
         "group_size": 4096,
         "in_features": 4096,
@@ -978,7 +964,7 @@
       "model.layers.13.self_attn.o_proj": {
         "bias": false,
         "enable_norm": true,
-        "enable_perm": true,
+        "enable_perm": false,
         "group_num": 1,
         "group_size": 4096,
         "in_features": 4096,
@@ -1002,7 +988,7 @@
       "model.layers.13.self_attn.q_proj": {
         "bias": false,
         "enable_norm": true,
-        "enable_perm": true,
+        "enable_perm": false,
         "group_num": 1,
         "group_size": 4096,
         "in_features": 4096,
@@ -1026,7 +1012,7 @@
       "model.layers.13.self_attn.v_proj": {
         "bias": false,
         "enable_norm": true,
-        "enable_perm": true,
+        "enable_perm": false,
         "group_num": 1,
         "group_size": 4096,
         "in_features": 4096,
@@ -1050,7 +1036,7 @@
       "model.layers.14.mlp.down_proj": {
         "bias": false,
         "enable_norm": true,
-        "enable_perm": true,
+        "enable_perm": false,
         "group_num": 1,
         "group_size": 14336,
         "in_features": 14336,
@@ -1074,7 +1060,7 @@
       "model.layers.14.mlp.gate_proj": {
         "bias": false,
         "enable_norm": true,
-        "enable_perm": true,
+        "enable_perm": false,
         "group_num": 1,
         "group_size": 4096,
         "in_features": 4096,
@@ -1098,7 +1084,7 @@
       "model.layers.14.mlp.up_proj": {
         "bias": false,
         "enable_norm": true,
-        "enable_perm": true,
+        "enable_perm": false,
         "group_num": 1,
         "group_size": 4096,
         "in_features": 4096,
@@ -1122,7 +1108,7 @@
       "model.layers.14.self_attn.k_proj": {
         "bias": false,
         "enable_norm": true,
-        "enable_perm": true,
+        "enable_perm": false,
         "group_num": 1,
         "group_size": 4096,
         "in_features": 4096,
@@ -1146,7 +1132,7 @@
       "model.layers.14.self_attn.o_proj": {
         "bias": false,
         "enable_norm": true,
-        "enable_perm": true,
+        "enable_perm": false,
         "group_num": 1,
         "group_size": 4096,
         "in_features": 4096,
@@ -1170,7 +1156,7 @@
       "model.layers.14.self_attn.q_proj": {
         "bias": false,
         "enable_norm": true,
-        "enable_perm": true,
+        "enable_perm": false,
         "group_num": 1,
         "group_size": 4096,
         "in_features": 4096,
@@ -1194,7 +1180,7 @@
       "model.layers.14.self_attn.v_proj": {
         "bias": false,
         "enable_norm": true,
-        "enable_perm": true,
+        "enable_perm": false,
         "group_num": 1,
         "group_size": 4096,
         "in_features": 4096,
@@ -1218,7 +1204,7 @@
       "model.layers.15.mlp.down_proj": {
         "bias": false,
         "enable_norm": true,
-        "enable_perm": true,
+        "enable_perm": false,
         "group_num": 1,
         "group_size": 14336,
         "in_features": 14336,
@@ -1242,7 +1228,7 @@
       "model.layers.15.mlp.gate_proj": {
         "bias": false,
         "enable_norm": true,
-        "enable_perm": true,
+        "enable_perm": false,
         "group_num": 1,
         "group_size": 4096,
         "in_features": 4096,
@@ -1266,7 +1252,7 @@
       "model.layers.15.mlp.up_proj": {
         "bias": false,
         "enable_norm": true,
-        "enable_perm": true,
+        "enable_perm": false,
         "group_num": 1,
         "group_size": 4096,
         "in_features": 4096,
@@ -1290,7 +1276,7 @@
       "model.layers.15.self_attn.k_proj": {
         "bias": false,
         "enable_norm": true,
-        "enable_perm": true,
+        "enable_perm": false,
         "group_num": 1,
         "group_size": 4096,
         "in_features": 4096,
@@ -1314,7 +1300,7 @@
       "model.layers.15.self_attn.o_proj": {
         "bias": false,
         "enable_norm": true,
-        "enable_perm": true,
+        "enable_perm": false,
         "group_num": 1,
         "group_size": 4096,
         "in_features": 4096,
@@ -1338,7 +1324,7 @@
       "model.layers.15.self_attn.q_proj": {
         "bias": false,
         "enable_norm": true,
-        "enable_perm": true,
+        "enable_perm": false,
         "group_num": 1,
         "group_size": 4096,
         "in_features": 4096,
@@ -1362,7 +1348,7 @@
       "model.layers.15.self_attn.v_proj": {
         "bias": false,
         "enable_norm": true,
-        "enable_perm": true,
+        "enable_perm": false,
         "group_num": 1,
         "group_size": 4096,
         "in_features": 4096,
@@ -1386,7 +1372,7 @@
       "model.layers.16.mlp.down_proj": {
         "bias": false,
         "enable_norm": true,
-        "enable_perm": true,
+        "enable_perm": false,
         "group_num": 1,
         "group_size": 14336,
         "in_features": 14336,
@@ -1410,7 +1396,7 @@
       "model.layers.16.mlp.gate_proj": {
         "bias": false,
         "enable_norm": true,
-        "enable_perm": true,
+        "enable_perm": false,
         "group_num": 1,
         "group_size": 4096,
         "in_features": 4096,
@@ -1434,7 +1420,7 @@
       "model.layers.16.mlp.up_proj": {
         "bias": false,
         "enable_norm": true,
-        "enable_perm": true,
+        "enable_perm": false,
         "group_num": 1,
         "group_size": 4096,
         "in_features": 4096,
@@ -1458,7 +1444,7 @@
       "model.layers.16.self_attn.k_proj": {
         "bias": false,
         "enable_norm": true,
-        "enable_perm": true,
+        "enable_perm": false,
         "group_num": 1,
         "group_size": 4096,
         "in_features": 4096,
@@ -1482,7 +1468,7 @@
       "model.layers.16.self_attn.o_proj": {
         "bias": false,
         "enable_norm": true,
-        "enable_perm": true,
+        "enable_perm": false,
         "group_num": 1,
         "group_size": 4096,
         "in_features": 4096,
@@ -1506,7 +1492,7 @@
       "model.layers.16.self_attn.q_proj": {
         "bias": false,
         "enable_norm": true,
-        "enable_perm": true,
+        "enable_perm": false,
         "group_num": 1,
         "group_size": 4096,
         "in_features": 4096,
@@ -1530,7 +1516,7 @@
       "model.layers.16.self_attn.v_proj": {
         "bias": false,
         "enable_norm": true,
-        "enable_perm": true,
+        "enable_perm": false,
         "group_num": 1,
         "group_size": 4096,
         "in_features": 4096,
@@ -1554,7 +1540,7 @@
       "model.layers.17.mlp.down_proj": {
         "bias": false,
         "enable_norm": true,
-        "enable_perm": true,
+        "enable_perm": false,
         "group_num": 1,
         "group_size": 14336,
         "in_features": 14336,
@@ -1578,7 +1564,7 @@
       "model.layers.17.mlp.gate_proj": {
         "bias": false,
         "enable_norm": true,
-        "enable_perm": true,
+        "enable_perm": false,
         "group_num": 1,
         "group_size": 4096,
         "in_features": 4096,
@@ -1602,7 +1588,7 @@
       "model.layers.17.mlp.up_proj": {
         "bias": false,
         "enable_norm": true,
-        "enable_perm": true,
+        "enable_perm": false,
         "group_num": 1,
         "group_size": 4096,
         "in_features": 4096,
@@ -1626,7 +1612,7 @@
       "model.layers.17.self_attn.k_proj": {
         "bias": false,
         "enable_norm": true,
-        "enable_perm": true,
+        "enable_perm": false,
         "group_num": 1,
         "group_size": 4096,
         "in_features": 4096,
@@ -1650,7 +1636,7 @@
       "model.layers.17.self_attn.o_proj": {
         "bias": false,
         "enable_norm": true,
-        "enable_perm": true,
+        "enable_perm": false,
         "group_num": 1,
         "group_size": 4096,
         "in_features": 4096,
@@ -1674,7 +1660,7 @@
       "model.layers.17.self_attn.q_proj": {
         "bias": false,
         "enable_norm": true,
-        "enable_perm": true,
+        "enable_perm": false,
         "group_num": 1,
         "group_size": 4096,
         "in_features": 4096,
@@ -1698,7 +1684,7 @@
       "model.layers.17.self_attn.v_proj": {
         "bias": false,
         "enable_norm": true,
-        "enable_perm": true,
+        "enable_perm": false,
         "group_num": 1,
         "group_size": 4096,
         "in_features": 4096,
@@ -1722,7 +1708,7 @@
       "model.layers.18.mlp.down_proj": {
         "bias": false,
         "enable_norm": true,
-        "enable_perm": true,
+        "enable_perm": false,
         "group_num": 1,
         "group_size": 14336,
         "in_features": 14336,
@@ -1746,7 +1732,7 @@
       "model.layers.18.mlp.gate_proj": {
         "bias": false,
         "enable_norm": true,
-        "enable_perm": true,
+        "enable_perm": false,
         "group_num": 1,
         "group_size": 4096,
         "in_features": 4096,
@@ -1770,7 +1756,7 @@
       "model.layers.18.mlp.up_proj": {
         "bias": false,
         "enable_norm": true,
-        "enable_perm": true,
+        "enable_perm": false,
         "group_num": 1,
         "group_size": 4096,
         "in_features": 4096,
@@ -1794,7 +1780,7 @@
       "model.layers.18.self_attn.k_proj": {
         "bias": false,
         "enable_norm": true,
-        "enable_perm": true,
+        "enable_perm": false,
         "group_num": 1,
         "group_size": 4096,
         "in_features": 4096,
@@ -1818,7 +1804,7 @@
       "model.layers.18.self_attn.o_proj": {
         "bias": false,
         "enable_norm": true,
-        "enable_perm": true,
+        "enable_perm": false,
         "group_num": 1,
         "group_size": 4096,
         "in_features": 4096,
@@ -1842,7 +1828,7 @@
       "model.layers.18.self_attn.q_proj": {
         "bias": false,
         "enable_norm": true,
-        "enable_perm": true,
+        "enable_perm": false,
         "group_num": 1,
         "group_size": 4096,
         "in_features": 4096,
@@ -1866,7 +1852,7 @@
       "model.layers.18.self_attn.v_proj": {
         "bias": false,
         "enable_norm": true,
-        "enable_perm": true,
+        "enable_perm": false,
         "group_num": 1,
         "group_size": 4096,
         "in_features": 4096,
@@ -1890,7 +1876,7 @@
       "model.layers.19.mlp.down_proj": {
         "bias": false,
         "enable_norm": true,
-        "enable_perm": true,
+        "enable_perm": false,
         "group_num": 1,
         "group_size": 14336,
         "in_features": 14336,
@@ -1914,7 +1900,7 @@
       "model.layers.19.mlp.gate_proj": {
         "bias": false,
         "enable_norm": true,
-        "enable_perm": true,
+        "enable_perm": false,
         "group_num": 1,
         "group_size": 4096,
         "in_features": 4096,
@@ -1938,7 +1924,7 @@
       "model.layers.19.mlp.up_proj": {
         "bias": false,
         "enable_norm": true,
-        "enable_perm": true,
+        "enable_perm": false,
         "group_num": 1,
         "group_size": 4096,
         "in_features": 4096,
@@ -1962,7 +1948,7 @@
       "model.layers.19.self_attn.k_proj": {
         "bias": false,
         "enable_norm": true,
-        "enable_perm": true,
+        "enable_perm": false,
         "group_num": 1,
         "group_size": 4096,
         "in_features": 4096,
@@ -1986,7 +1972,7 @@
       "model.layers.19.self_attn.o_proj": {
         "bias": false,
         "enable_norm": true,
-        "enable_perm": true,
+        "enable_perm": false,
         "group_num": 1,
         "group_size": 4096,
         "in_features": 4096,
@@ -2010,7 +1996,7 @@
       "model.layers.19.self_attn.q_proj": {
         "bias": false,
         "enable_norm": true,
-        "enable_perm": true,
+        "enable_perm": false,
         "group_num": 1,
         "group_size": 4096,
         "in_features": 4096,
@@ -2034,7 +2020,7 @@
       "model.layers.19.self_attn.v_proj": {
         "bias": false,
         "enable_norm": true,
-        "enable_perm": true,
+        "enable_perm": false,
         "group_num": 1,
         "group_size": 4096,
         "in_features": 4096,
@@ -2058,7 +2044,7 @@
       "model.layers.2.mlp.down_proj": {
         "bias": false,
         "enable_norm": true,
-        "enable_perm": true,
+        "enable_perm": false,
         "group_num": 1,
         "group_size": 14336,
         "in_features": 14336,
@@ -2082,7 +2068,7 @@
       "model.layers.2.mlp.gate_proj": {
         "bias": false,
         "enable_norm": true,
-        "enable_perm": true,
+        "enable_perm": false,
         "group_num": 1,
         "group_size": 4096,
         "in_features": 4096,
@@ -2106,7 +2092,7 @@
       "model.layers.2.mlp.up_proj": {
         "bias": false,
         "enable_norm": true,
-        "enable_perm": true,
+        "enable_perm": false,
         "group_num": 1,
         "group_size": 4096,
         "in_features": 4096,
@@ -2130,7 +2116,7 @@
       "model.layers.2.self_attn.k_proj": {
         "bias": false,
         "enable_norm": true,
-        "enable_perm": true,
+        "enable_perm": false,
         "group_num": 1,
         "group_size": 4096,
         "in_features": 4096,
@@ -2154,7 +2140,7 @@
       "model.layers.2.self_attn.o_proj": {
         "bias": false,
         "enable_norm": true,
-        "enable_perm": true,
+        "enable_perm": false,
         "group_num": 1,
         "group_size": 4096,
         "in_features": 4096,
@@ -2178,7 +2164,7 @@
       "model.layers.2.self_attn.q_proj": {
         "bias": false,
         "enable_norm": true,
-        "enable_perm": true,
+        "enable_perm": false,
         "group_num": 1,
         "group_size": 4096,
         "in_features": 4096,
@@ -2202,7 +2188,7 @@
       "model.layers.2.self_attn.v_proj": {
         "bias": false,
         "enable_norm": true,
-        "enable_perm": true,
+        "enable_perm": false,
         "group_num": 1,
         "group_size": 4096,
         "in_features": 4096,
@@ -2226,7 +2212,7 @@
       "model.layers.20.mlp.down_proj": {
         "bias": false,
         "enable_norm": true,
-        "enable_perm": true,
+        "enable_perm": false,
         "group_num": 1,
         "group_size": 14336,
         "in_features": 14336,
@@ -2250,7 +2236,7 @@
       "model.layers.20.mlp.gate_proj": {
         "bias": false,
         "enable_norm": true,
-        "enable_perm": true,
+        "enable_perm": false,
         "group_num": 1,
         "group_size": 4096,
         "in_features": 4096,
@@ -2274,7 +2260,7 @@
       "model.layers.20.mlp.up_proj": {
         "bias": false,
         "enable_norm": true,
-        "enable_perm": true,
+        "enable_perm": false,
         "group_num": 1,
         "group_size": 4096,
         "in_features": 4096,
@@ -2298,7 +2284,7 @@
       "model.layers.20.self_attn.k_proj": {
         "bias": false,
         "enable_norm": true,
-        "enable_perm": true,
+        "enable_perm": false,
         "group_num": 1,
         "group_size": 4096,
         "in_features": 4096,
@@ -2322,7 +2308,7 @@
       "model.layers.20.self_attn.o_proj": {
         "bias": false,
         "enable_norm": true,
-        "enable_perm": true,
+        "enable_perm": false,
         "group_num": 1,
         "group_size": 4096,
         "in_features": 4096,
@@ -2346,7 +2332,7 @@
       "model.layers.20.self_attn.q_proj": {
         "bias": false,
         "enable_norm": true,
-        "enable_perm": true,
+        "enable_perm": false,
         "group_num": 1,
         "group_size": 4096,
         "in_features": 4096,
@@ -2370,7 +2356,7 @@
       "model.layers.20.self_attn.v_proj": {
         "bias": false,
         "enable_norm": true,
-        "enable_perm": true,
+        "enable_perm": false,
         "group_num": 1,
         "group_size": 4096,
         "in_features": 4096,
@@ -2394,7 +2380,7 @@
       "model.layers.21.mlp.down_proj": {
         "bias": false,
         "enable_norm": true,
-        "enable_perm": true,
+        "enable_perm": false,
         "group_num": 1,
         "group_size": 14336,
         "in_features": 14336,
@@ -2418,7 +2404,7 @@
       "model.layers.21.mlp.gate_proj": {
         "bias": false,
         "enable_norm": true,
-        "enable_perm": true,
+        "enable_perm": false,
         "group_num": 1,
         "group_size": 4096,
         "in_features": 4096,
@@ -2442,7 +2428,7 @@
       "model.layers.21.mlp.up_proj": {
         "bias": false,
         "enable_norm": true,
-        "enable_perm": true,
+        "enable_perm": false,
         "group_num": 1,
         "group_size": 4096,
         "in_features": 4096,
@@ -2466,7 +2452,7 @@
       "model.layers.21.self_attn.k_proj": {
         "bias": false,
         "enable_norm": true,
-        "enable_perm": true,
+        "enable_perm": false,
         "group_num": 1,
         "group_size": 4096,
         "in_features": 4096,
@@ -2490,7 +2476,7 @@
       "model.layers.21.self_attn.o_proj": {
         "bias": false,
         "enable_norm": true,
-        "enable_perm": true,
+        "enable_perm": false,
         "group_num": 1,
         "group_size": 4096,
         "in_features": 4096,
@@ -2514,7 +2500,7 @@
       "model.layers.21.self_attn.q_proj": {
         "bias": false,
         "enable_norm": true,
-        "enable_perm": true,
+        "enable_perm": false,
         "group_num": 1,
         "group_size": 4096,
         "in_features": 4096,
@@ -2538,7 +2524,7 @@
       "model.layers.21.self_attn.v_proj": {
         "bias": false,
         "enable_norm": true,
-        "enable_perm": true,
+        "enable_perm": false,
         "group_num": 1,
         "group_size": 4096,
         "in_features": 4096,
@@ -2562,7 +2548,7 @@
       "model.layers.22.mlp.down_proj": {
         "bias": false,
         "enable_norm": true,
-        "enable_perm": true,
+        "enable_perm": false,
         "group_num": 1,
         "group_size": 14336,
         "in_features": 14336,
@@ -2586,7 +2572,7 @@
       "model.layers.22.mlp.gate_proj": {
         "bias": false,
         "enable_norm": true,
-        "enable_perm": true,
+        "enable_perm": false,
         "group_num": 1,
         "group_size": 4096,
         "in_features": 4096,
@@ -2610,7 +2596,7 @@
       "model.layers.22.mlp.up_proj": {
         "bias": false,
         "enable_norm": true,
-        "enable_perm": true,
+        "enable_perm": false,
         "group_num": 1,
         "group_size": 4096,
         "in_features": 4096,
@@ -2634,7 +2620,7 @@
       "model.layers.22.self_attn.k_proj": {
         "bias": false,
         "enable_norm": true,
-        "enable_perm": true,
+        "enable_perm": false,
         "group_num": 1,
         "group_size": 4096,
         "in_features": 4096,
@@ -2658,7 +2644,7 @@
       "model.layers.22.self_attn.o_proj": {
         "bias": false,
         "enable_norm": true,
-        "enable_perm": true,
+        "enable_perm": false,
         "group_num": 1,
         "group_size": 4096,
         "in_features": 4096,
@@ -2682,7 +2668,7 @@
       "model.layers.22.self_attn.q_proj": {
         "bias": false,
         "enable_norm": true,
-        "enable_perm": true,
+        "enable_perm": false,
         "group_num": 1,
         "group_size": 4096,
         "in_features": 4096,
@@ -2706,7 +2692,7 @@
       "model.layers.22.self_attn.v_proj": {
         "bias": false,
         "enable_norm": true,
-        "enable_perm": true,
+        "enable_perm": false,
         "group_num": 1,
         "group_size": 4096,
         "in_features": 4096,
@@ -2730,7 +2716,7 @@
       "model.layers.23.mlp.down_proj": {
         "bias": false,
         "enable_norm": true,
-        "enable_perm": true,
+        "enable_perm": false,
         "group_num": 1,
         "group_size": 14336,
         "in_features": 14336,
@@ -2754,7 +2740,7 @@
       "model.layers.23.mlp.gate_proj": {
         "bias": false,
         "enable_norm": true,
-        "enable_perm": true,
+        "enable_perm": false,
         "group_num": 1,
         "group_size": 4096,
         "in_features": 4096,
@@ -2778,7 +2764,7 @@
       "model.layers.23.mlp.up_proj": {
         "bias": false,
         "enable_norm": true,
-        "enable_perm": true,
+        "enable_perm": false,
         "group_num": 1,
         "group_size": 4096,
         "in_features": 4096,
@@ -2802,7 +2788,7 @@
       "model.layers.23.self_attn.k_proj": {
         "bias": false,
         "enable_norm": true,
-        "enable_perm": true,
+        "enable_perm": false,
         "group_num": 1,
         "group_size": 4096,
         "in_features": 4096,
@@ -2826,7 +2812,7 @@
       "model.layers.23.self_attn.o_proj": {
         "bias": false,
         "enable_norm": true,
-        "enable_perm": true,
+        "enable_perm": false,
         "group_num": 1,
         "group_size": 4096,
         "in_features": 4096,
@@ -2850,7 +2836,7 @@
       "model.layers.23.self_attn.q_proj": {
         "bias": false,
         "enable_norm": true,
-        "enable_perm": true,
+        "enable_perm": false,
         "group_num": 1,
         "group_size": 4096,
         "in_features": 4096,
@@ -2874,7 +2860,7 @@
       "model.layers.23.self_attn.v_proj": {
         "bias": false,
         "enable_norm": true,
-        "enable_perm": true,
+        "enable_perm": false,
         "group_num": 1,
         "group_size": 4096,
         "in_features": 4096,
@@ -2898,7 +2884,7 @@
       "model.layers.24.mlp.down_proj": {
         "bias": false,
         "enable_norm": true,
-        "enable_perm": true,
+        "enable_perm": false,
         "group_num": 1,
         "group_size": 14336,
         "in_features": 14336,
@@ -2922,7 +2908,7 @@
       "model.layers.24.mlp.gate_proj": {
         "bias": false,
         "enable_norm": true,
-        "enable_perm": true,
+        "enable_perm": false,
         "group_num": 1,
         "group_size": 4096,
         "in_features": 4096,
@@ -2946,7 +2932,7 @@
       "model.layers.24.mlp.up_proj": {
         "bias": false,
         "enable_norm": true,
-        "enable_perm": true,
+        "enable_perm": false,
         "group_num": 1,
         "group_size": 4096,
         "in_features": 4096,
@@ -2970,7 +2956,7 @@
       "model.layers.24.self_attn.k_proj": {
         "bias": false,
         "enable_norm": true,
-        "enable_perm": true,
+        "enable_perm": false,
         "group_num": 1,
         "group_size": 4096,
         "in_features": 4096,
@@ -2994,7 +2980,7 @@
       "model.layers.24.self_attn.o_proj": {
         "bias": false,
         "enable_norm": true,
-        "enable_perm": true,
+        "enable_perm": false,
         "group_num": 1,
         "group_size": 4096,
         "in_features": 4096,
@@ -3018,7 +3004,7 @@
       "model.layers.24.self_attn.q_proj": {
         "bias": false,
         "enable_norm": true,
-        "enable_perm": true,
+        "enable_perm": false,
         "group_num": 1,
         "group_size": 4096,
         "in_features": 4096,
@@ -3042,7 +3028,7 @@
       "model.layers.24.self_attn.v_proj": {
         "bias": false,
         "enable_norm": true,
-        "enable_perm": true,
+        "enable_perm": false,
         "group_num": 1,
         "group_size": 4096,
         "in_features": 4096,
@@ -3066,7 +3052,7 @@
       "model.layers.25.mlp.down_proj": {
         "bias": false,
         "enable_norm": true,
-        "enable_perm": true,
+        "enable_perm": false,
         "group_num": 1,
         "group_size": 14336,
         "in_features": 14336,
@@ -3090,7 +3076,7 @@
       "model.layers.25.mlp.gate_proj": {
         "bias": false,
         "enable_norm": true,
-        "enable_perm": true,
+        "enable_perm": false,
         "group_num": 1,
         "group_size": 4096,
         "in_features": 4096,
@@ -3114,7 +3100,7 @@
       "model.layers.25.mlp.up_proj": {
         "bias": false,
         "enable_norm": true,
-        "enable_perm": true,
+        "enable_perm": false,
         "group_num": 1,
         "group_size": 4096,
         "in_features": 4096,
@@ -3138,7 +3124,7 @@
       "model.layers.25.self_attn.k_proj": {
         "bias": false,
         "enable_norm": true,
-        "enable_perm": true,
+        "enable_perm": false,
         "group_num": 1,
         "group_size": 4096,
         "in_features": 4096,
@@ -3162,7 +3148,7 @@
       "model.layers.25.self_attn.o_proj": {
         "bias": false,
         "enable_norm": true,
-        "enable_perm": true,
+        "enable_perm": false,
         "group_num": 1,
         "group_size": 4096,
         "in_features": 4096,
@@ -3186,7 +3172,7 @@
       "model.layers.25.self_attn.q_proj": {
         "bias": false,
         "enable_norm": true,
-        "enable_perm": true,
+        "enable_perm": false,
         "group_num": 1,
         "group_size": 4096,
         "in_features": 4096,
@@ -3210,7 +3196,7 @@
       "model.layers.25.self_attn.v_proj": {
         "bias": false,
         "enable_norm": true,
-        "enable_perm": true,
+        "enable_perm": false,
         "group_num": 1,
         "group_size": 4096,
         "in_features": 4096,
@@ -3234,7 +3220,7 @@
       "model.layers.26.mlp.down_proj": {
         "bias": false,
         "enable_norm": true,
-        "enable_perm": true,
+        "enable_perm": false,
         "group_num": 1,
         "group_size": 14336,
         "in_features": 14336,
@@ -3258,7 +3244,7 @@
       "model.layers.26.mlp.gate_proj": {
         "bias": false,
         "enable_norm": true,
-        "enable_perm": true,
+        "enable_perm": false,
         "group_num": 1,
         "group_size": 4096,
         "in_features": 4096,
@@ -3282,7 +3268,7 @@
       "model.layers.26.mlp.up_proj": {
         "bias": false,
         "enable_norm": true,
-        "enable_perm": true,
+        "enable_perm": false,
         "group_num": 1,
         "group_size": 4096,
         "in_features": 4096,
@@ -3306,7 +3292,7 @@
       "model.layers.26.self_attn.k_proj": {
         "bias": false,
         "enable_norm": true,
-        "enable_perm": true,
+        "enable_perm": false,
         "group_num": 1,
         "group_size": 4096,
         "in_features": 4096,
@@ -3330,7 +3316,7 @@
       "model.layers.26.self_attn.o_proj": {
         "bias": false,
         "enable_norm": true,
-        "enable_perm": true,
+        "enable_perm": false,
         "group_num": 1,
         "group_size": 4096,
         "in_features": 4096,
@@ -3354,7 +3340,7 @@
       "model.layers.26.self_attn.q_proj": {
         "bias": false,
         "enable_norm": true,
-        "enable_perm": true,
+        "enable_perm": false,
         "group_num": 1,
         "group_size": 4096,
         "in_features": 4096,
@@ -3378,7 +3364,7 @@
       "model.layers.26.self_attn.v_proj": {
         "bias": false,
         "enable_norm": true,
-        "enable_perm": true,
+        "enable_perm": false,
         "group_num": 1,
         "group_size": 4096,
         "in_features": 4096,
@@ -3402,7 +3388,7 @@
       "model.layers.27.mlp.down_proj": {
         "bias": false,
         "enable_norm": true,
-        "enable_perm": true,
+        "enable_perm": false,
         "group_num": 1,
         "group_size": 14336,
         "in_features": 14336,
@@ -3426,7 +3412,7 @@
       "model.layers.27.mlp.gate_proj": {
         "bias": false,
         "enable_norm": true,
-        "enable_perm": true,
+        "enable_perm": false,
         "group_num": 1,
         "group_size": 4096,
         "in_features": 4096,
@@ -3450,7 +3436,7 @@
       "model.layers.27.mlp.up_proj": {
         "bias": false,
         "enable_norm": true,
-        "enable_perm": true,
+        "enable_perm": false,
         "group_num": 1,
         "group_size": 4096,
         "in_features": 4096,
@@ -3474,7 +3460,7 @@
       "model.layers.27.self_attn.k_proj": {
         "bias": false,
         "enable_norm": true,
-        "enable_perm": true,
+        "enable_perm": false,
         "group_num": 1,
         "group_size": 4096,
         "in_features": 4096,
@@ -3498,7 +3484,7 @@
       "model.layers.27.self_attn.o_proj": {
         "bias": false,
         "enable_norm": true,
-        "enable_perm": true,
+        "enable_perm": false,
         "group_num": 1,
         "group_size": 4096,
         "in_features": 4096,
@@ -3522,7 +3508,7 @@
       "model.layers.27.self_attn.q_proj": {
         "bias": false,
         "enable_norm": true,
-        "enable_perm": true,
+        "enable_perm": false,
         "group_num": 1,
         "group_size": 4096,
         "in_features": 4096,
@@ -3546,7 +3532,7 @@
       "model.layers.27.self_attn.v_proj": {
         "bias": false,
         "enable_norm": true,
-        "enable_perm": true,
+        "enable_perm": false,
         "group_num": 1,
         "group_size": 4096,
         "in_features": 4096,
@@ -3570,7 +3556,7 @@
       "model.layers.28.mlp.down_proj": {
         "bias": false,
         "enable_norm": true,
-        "enable_perm": true,
+        "enable_perm": false,
         "group_num": 1,
         "group_size": 14336,
         "in_features": 14336,
@@ -3594,7 +3580,7 @@
       "model.layers.28.mlp.gate_proj": {
         "bias": false,
         "enable_norm": true,
-        "enable_perm": true,
+        "enable_perm": false,
         "group_num": 1,
         "group_size": 4096,
         "in_features": 4096,
@@ -3618,7 +3604,7 @@
       "model.layers.28.mlp.up_proj": {
         "bias": false,
         "enable_norm": true,
-        "enable_perm": true,
+        "enable_perm": false,
         "group_num": 1,
         "group_size": 4096,
         "in_features": 4096,
@@ -3642,7 +3628,7 @@
       "model.layers.28.self_attn.k_proj": {
         "bias": false,
         "enable_norm": true,
-        "enable_perm": true,
+        "enable_perm": false,
         "group_num": 1,
         "group_size": 4096,
         "in_features": 4096,
@@ -3666,7 +3652,7 @@
       "model.layers.28.self_attn.o_proj": {
         "bias": false,
         "enable_norm": true,
-        "enable_perm": true,
+        "enable_perm": false,
         "group_num": 1,
         "group_size": 4096,
         "in_features": 4096,
@@ -3690,7 +3676,7 @@
       "model.layers.28.self_attn.q_proj": {
         "bias": false,
         "enable_norm": true,
-        "enable_perm": true,
+        "enable_perm": false,
         "group_num": 1,
         "group_size": 4096,
         "in_features": 4096,
@@ -3714,7 +3700,7 @@
       "model.layers.28.self_attn.v_proj": {
         "bias": false,
         "enable_norm": true,
-        "enable_perm": true,
+        "enable_perm": false,
         "group_num": 1,
         "group_size": 4096,
         "in_features": 4096,
@@ -3738,7 +3724,7 @@
       "model.layers.29.mlp.down_proj": {
         "bias": false,
         "enable_norm": true,
-        "enable_perm": true,
+        "enable_perm": false,
         "group_num": 1,
         "group_size": 14336,
         "in_features": 14336,
@@ -3762,7 +3748,7 @@
       "model.layers.29.mlp.gate_proj": {
         "bias": false,
         "enable_norm": true,
-        "enable_perm": true,
+        "enable_perm": false,
         "group_num": 1,
         "group_size": 4096,
         "in_features": 4096,
@@ -3786,7 +3772,7 @@
       "model.layers.29.mlp.up_proj": {
         "bias": false,
         "enable_norm": true,
-        "enable_perm": true,
+        "enable_perm": false,
         "group_num": 1,
         "group_size": 4096,
         "in_features": 4096,
@@ -3810,7 +3796,7 @@
       "model.layers.29.self_attn.k_proj": {
         "bias": false,
         "enable_norm": true,
-        "enable_perm": true,
+        "enable_perm": false,
         "group_num": 1,
         "group_size": 4096,
         "in_features": 4096,
@@ -3834,7 +3820,7 @@
       "model.layers.29.self_attn.o_proj": {
         "bias": false,
         "enable_norm": true,
-        "enable_perm": true,
+        "enable_perm": false,
         "group_num": 1,
         "group_size": 4096,
         "in_features": 4096,
@@ -3858,7 +3844,7 @@
       "model.layers.29.self_attn.q_proj": {
         "bias": false,
         "enable_norm": true,
-        "enable_perm": true,
+        "enable_perm": false,
         "group_num": 1,
         "group_size": 4096,
         "in_features": 4096,
@@ -3882,7 +3868,7 @@
       "model.layers.29.self_attn.v_proj": {
         "bias": false,
         "enable_norm": true,
-        "enable_perm": true,
+        "enable_perm": false,
         "group_num": 1,
         "group_size": 4096,
         "in_features": 4096,
@@ -3906,7 +3892,7 @@
       "model.layers.3.mlp.down_proj": {
         "bias": false,
         "enable_norm": true,
-        "enable_perm": true,
+        "enable_perm": false,
         "group_num": 1,
         "group_size": 14336,
         "in_features": 14336,
@@ -3930,7 +3916,7 @@
       "model.layers.3.mlp.gate_proj": {
         "bias": false,
         "enable_norm": true,
-        "enable_perm": true,
+        "enable_perm": false,
         "group_num": 1,
         "group_size": 4096,
         "in_features": 4096,
@@ -3954,7 +3940,7 @@
       "model.layers.3.mlp.up_proj": {
         "bias": false,
         "enable_norm": true,
-        "enable_perm": true,
+        "enable_perm": false,
         "group_num": 1,
         "group_size": 4096,
         "in_features": 4096,
@@ -3978,7 +3964,7 @@
       "model.layers.3.self_attn.k_proj": {
         "bias": false,
         "enable_norm": true,
-        "enable_perm": true,
+        "enable_perm": false,
         "group_num": 1,
         "group_size": 4096,
         "in_features": 4096,
@@ -4002,7 +3988,7 @@
       "model.layers.3.self_attn.o_proj": {
         "bias": false,
         "enable_norm": true,
-        "enable_perm": true,
+        "enable_perm": false,
         "group_num": 1,
         "group_size": 4096,
         "in_features": 4096,
@@ -4026,7 +4012,7 @@
       "model.layers.3.self_attn.q_proj": {
         "bias": false,
         "enable_norm": true,
-        "enable_perm": true,
+        "enable_perm": false,
         "group_num": 1,
         "group_size": 4096,
         "in_features": 4096,
@@ -4050,7 +4036,7 @@
       "model.layers.3.self_attn.v_proj": {
         "bias": false,
         "enable_norm": true,
-        "enable_perm": true,
+        "enable_perm": false,
         "group_num": 1,
         "group_size": 4096,
         "in_features": 4096,
@@ -4074,7 +4060,7 @@
       "model.layers.30.mlp.down_proj": {
         "bias": false,
         "enable_norm": true,
-        "enable_perm": true,
+        "enable_perm": false,
         "group_num": 1,
         "group_size": 14336,
         "in_features": 14336,
@@ -4098,7 +4084,7 @@
       "model.layers.30.mlp.gate_proj": {
         "bias": false,
         "enable_norm": true,
-        "enable_perm": true,
+        "enable_perm": false,
         "group_num": 1,
         "group_size": 4096,
         "in_features": 4096,
@@ -4122,7 +4108,7 @@
       "model.layers.30.mlp.up_proj": {
         "bias": false,
         "enable_norm": true,
-        "enable_perm": true,
+        "enable_perm": false,
         "group_num": 1,
         "group_size": 4096,
         "in_features": 4096,
@@ -4146,7 +4132,7 @@
       "model.layers.30.self_attn.k_proj": {
         "bias": false,
         "enable_norm": true,
-        "enable_perm": true,
+        "enable_perm": false,
         "group_num": 1,
         "group_size": 4096,
         "in_features": 4096,
@@ -4170,7 +4156,7 @@
       "model.layers.30.self_attn.o_proj": {
         "bias": false,
         "enable_norm": true,
-        "enable_perm": true,
+        "enable_perm": false,
         "group_num": 1,
         "group_size": 4096,
         "in_features": 4096,
@@ -4194,7 +4180,7 @@
       "model.layers.30.self_attn.q_proj": {
         "bias": false,
         "enable_norm": true,
-        "enable_perm": true,
+        "enable_perm": false,
         "group_num": 1,
         "group_size": 4096,
         "in_features": 4096,
@@ -4218,7 +4204,7 @@
       "model.layers.30.self_attn.v_proj": {
         "bias": false,
         "enable_norm": true,
-        "enable_perm": true,
+        "enable_perm": false,
         "group_num": 1,
         "group_size": 4096,
         "in_features": 4096,
@@ -4242,7 +4228,7 @@
       "model.layers.31.mlp.down_proj": {
         "bias": false,
         "enable_norm": true,
-        "enable_perm": true,
+        "enable_perm": false,
         "group_num": 1,
         "group_size": 14336,
         "in_features": 14336,
@@ -4266,7 +4252,7 @@
       "model.layers.31.mlp.gate_proj": {
         "bias": false,
         "enable_norm": true,
-        "enable_perm": true,
+        "enable_perm": false,
         "group_num": 1,
         "group_size": 4096,
         "in_features": 4096,
@@ -4290,7 +4276,7 @@
       "model.layers.31.mlp.up_proj": {
         "bias": false,
         "enable_norm": true,
-        "enable_perm": true,
+        "enable_perm": false,
         "group_num": 1,
         "group_size": 4096,
         "in_features": 4096,
@@ -4314,7 +4300,7 @@
       "model.layers.31.self_attn.k_proj": {
         "bias": false,
         "enable_norm": true,
-        "enable_perm": true,
+        "enable_perm": false,
         "group_num": 1,
         "group_size": 4096,
         "in_features": 4096,
@@ -4338,7 +4324,7 @@
       "model.layers.31.self_attn.o_proj": {
         "bias": false,
         "enable_norm": true,
-        "enable_perm": true,
+        "enable_perm": false,
         "group_num": 1,
         "group_size": 4096,
         "in_features": 4096,
@@ -4362,7 +4348,7 @@
       "model.layers.31.self_attn.q_proj": {
         "bias": false,
         "enable_norm": true,
-        "enable_perm": true,
+        "enable_perm": false,
         "group_num": 1,
         "group_size": 4096,
         "in_features": 4096,
@@ -4386,7 +4372,7 @@
       "model.layers.31.self_attn.v_proj": {
         "bias": false,
         "enable_norm": true,
-        "enable_perm": true,
+        "enable_perm": false,
         "group_num": 1,
         "group_size": 4096,
         "in_features": 4096,
@@ -4410,7 +4396,7 @@
       "model.layers.4.mlp.down_proj": {
         "bias": false,
         "enable_norm": true,
-        "enable_perm": true,
+        "enable_perm": false,
         "group_num": 1,
         "group_size": 14336,
         "in_features": 14336,
@@ -4434,7 +4420,7 @@
       "model.layers.4.mlp.gate_proj": {
         "bias": false,
         "enable_norm": true,
-        "enable_perm": true,
+        "enable_perm": false,
         "group_num": 1,
         "group_size": 4096,
         "in_features": 4096,
@@ -4458,7 +4444,7 @@
       "model.layers.4.mlp.up_proj": {
         "bias": false,
         "enable_norm": true,
-        "enable_perm": true,
+        "enable_perm": false,
         "group_num": 1,
         "group_size": 4096,
         "in_features": 4096,
@@ -4482,7 +4468,7 @@
       "model.layers.4.self_attn.k_proj": {
         "bias": false,
         "enable_norm": true,
-        "enable_perm": true,
+        "enable_perm": false,
         "group_num": 1,
         "group_size": 4096,
         "in_features": 4096,
@@ -4506,7 +4492,7 @@
       "model.layers.4.self_attn.o_proj": {
         "bias": false,
         "enable_norm": true,
-        "enable_perm": true,
+        "enable_perm": false,
         "group_num": 1,
         "group_size": 4096,
         "in_features": 4096,
@@ -4530,7 +4516,7 @@
       "model.layers.4.self_attn.q_proj": {
         "bias": false,
         "enable_norm": true,
-        "enable_perm": true,
+        "enable_perm": false,
         "group_num": 1,
         "group_size": 4096,
         "in_features": 4096,
@@ -4554,7 +4540,7 @@
       "model.layers.4.self_attn.v_proj": {
         "bias": false,
         "enable_norm": true,
-        "enable_perm": true,
+        "enable_perm": false,
         "group_num": 1,
         "group_size": 4096,
         "in_features": 4096,
@@ -4578,7 +4564,7 @@
       "model.layers.5.mlp.down_proj": {
         "bias": false,
         "enable_norm": true,
-        "enable_perm": true,
+        "enable_perm": false,
         "group_num": 1,
         "group_size": 14336,
         "in_features": 14336,
@@ -4602,7 +4588,7 @@
       "model.layers.5.mlp.gate_proj": {
         "bias": false,
         "enable_norm": true,
-        "enable_perm": true,
+        "enable_perm": false,
         "group_num": 1,
         "group_size": 4096,
         "in_features": 4096,
@@ -4626,7 +4612,7 @@
       "model.layers.5.mlp.up_proj": {
         "bias": false,
         "enable_norm": true,
-        "enable_perm": true,
+        "enable_perm": false,
         "group_num": 1,
         "group_size": 4096,
         "in_features": 4096,
@@ -4650,7 +4636,7 @@
       "model.layers.5.self_attn.k_proj": {
         "bias": false,
         "enable_norm": true,
-        "enable_perm": true,
+        "enable_perm": false,
         "group_num": 1,
         "group_size": 4096,
         "in_features": 4096,
@@ -4674,7 +4660,7 @@
       "model.layers.5.self_attn.o_proj": {
         "bias": false,
         "enable_norm": true,
-        "enable_perm": true,
+        "enable_perm": false,
         "group_num": 1,
         "group_size": 4096,
         "in_features": 4096,
@@ -4698,7 +4684,7 @@
       "model.layers.5.self_attn.q_proj": {
         "bias": false,
         "enable_norm": true,
-        "enable_perm": true,
+        "enable_perm": false,
         "group_num": 1,
         "group_size": 4096,
         "in_features": 4096,
@@ -4722,7 +4708,7 @@
       "model.layers.5.self_attn.v_proj": {
         "bias": false,
         "enable_norm": true,
-        "enable_perm": true,
+        "enable_perm": false,
         "group_num": 1,
         "group_size": 4096,
         "in_features": 4096,
@@ -4746,7 +4732,7 @@
       "model.layers.6.mlp.down_proj": {
         "bias": false,
         "enable_norm": true,
-        "enable_perm": true,
+        "enable_perm": false,
         "group_num": 1,
         "group_size": 14336,
         "in_features": 14336,
@@ -4770,7 +4756,7 @@
       "model.layers.6.mlp.gate_proj": {
         "bias": false,
         "enable_norm": true,
-        "enable_perm": true,
+        "enable_perm": false,
         "group_num": 1,
         "group_size": 4096,
         "in_features": 4096,
@@ -4794,7 +4780,7 @@
       "model.layers.6.mlp.up_proj": {
         "bias": false,
         "enable_norm": true,
-        "enable_perm": true,
+        "enable_perm": false,
         "group_num": 1,
         "group_size": 4096,
         "in_features": 4096,
@@ -4818,7 +4804,7 @@
       "model.layers.6.self_attn.k_proj": {
         "bias": false,
         "enable_norm": true,
-        "enable_perm": true,
+        "enable_perm": false,
         "group_num": 1,
         "group_size": 4096,
         "in_features": 4096,
@@ -4842,7 +4828,7 @@
       "model.layers.6.self_attn.o_proj": {
         "bias": false,
         "enable_norm": true,
-        "enable_perm": true,
+        "enable_perm": false,
         "group_num": 1,
         "group_size": 4096,
         "in_features": 4096,
@@ -4866,7 +4852,7 @@
       "model.layers.6.self_attn.q_proj": {
         "bias": false,
         "enable_norm": true,
-        "enable_perm": true,
+        "enable_perm": false,
         "group_num": 1,
         "group_size": 4096,
         "in_features": 4096,
@@ -4890,7 +4876,7 @@
       "model.layers.6.self_attn.v_proj": {
         "bias": false,
         "enable_norm": true,
-        "enable_perm": true,
+        "enable_perm": false,
         "group_num": 1,
         "group_size": 4096,
         "in_features": 4096,
@@ -4914,7 +4900,7 @@
       "model.layers.7.mlp.down_proj": {
         "bias": false,
         "enable_norm": true,
-        "enable_perm": true,
+        "enable_perm": false,
         "group_num": 1,
         "group_size": 14336,
         "in_features": 14336,
@@ -4938,7 +4924,7 @@
       "model.layers.7.mlp.gate_proj": {
         "bias": false,
         "enable_norm": true,
-        "enable_perm": true,
+        "enable_perm": false,
         "group_num": 1,
         "group_size": 4096,
         "in_features": 4096,
@@ -4962,7 +4948,7 @@
       "model.layers.7.mlp.up_proj": {
         "bias": false,
         "enable_norm": true,
-        "enable_perm": true,
+        "enable_perm": false,
         "group_num": 1,
         "group_size": 4096,
         "in_features": 4096,
@@ -4986,7 +4972,7 @@
       "model.layers.7.self_attn.k_proj": {
         "bias": false,
         "enable_norm": true,
-        "enable_perm": true,
+        "enable_perm": false,
         "group_num": 1,
         "group_size": 4096,
         "in_features": 4096,
@@ -5010,7 +4996,7 @@
       "model.layers.7.self_attn.o_proj": {
         "bias": false,
         "enable_norm": true,
-        "enable_perm": true,
+        "enable_perm": false,
         "group_num": 1,
         "group_size": 4096,
         "in_features": 4096,
@@ -5034,7 +5020,7 @@
       "model.layers.7.self_attn.q_proj": {
         "bias": false,
         "enable_norm": true,
-        "enable_perm": true,
+        "enable_perm": false,
         "group_num": 1,
         "group_size": 4096,
         "in_features": 4096,
@@ -5058,7 +5044,7 @@
       "model.layers.7.self_attn.v_proj": {
         "bias": false,
         "enable_norm": true,
-        "enable_perm": true,
+        "enable_perm": false,
         "group_num": 1,
         "group_size": 4096,
         "in_features": 4096,
@@ -5082,7 +5068,7 @@
       "model.layers.8.mlp.down_proj": {
         "bias": false,
         "enable_norm": true,
-        "enable_perm": true,
+        "enable_perm": false,
         "group_num": 1,
         "group_size": 14336,
         "in_features": 14336,
@@ -5106,7 +5092,7 @@
       "model.layers.8.mlp.gate_proj": {
         "bias": false,
         "enable_norm": true,
-        "enable_perm": true,
+        "enable_perm": false,
         "group_num": 1,
         "group_size": 4096,
         "in_features": 4096,
@@ -5130,7 +5116,7 @@
       "model.layers.8.mlp.up_proj": {
         "bias": false,
         "enable_norm": true,
-        "enable_perm": true,
+        "enable_perm": false,
         "group_num": 1,
         "group_size": 4096,
         "in_features": 4096,
@@ -5154,7 +5140,7 @@
       "model.layers.8.self_attn.k_proj": {
         "bias": false,
         "enable_norm": true,
-        "enable_perm": true,
+        "enable_perm": false,
         "group_num": 1,
         "group_size": 4096,
         "in_features": 4096,
@@ -5178,7 +5164,7 @@
       "model.layers.8.self_attn.o_proj": {
         "bias": false,
         "enable_norm": true,
-        "enable_perm": true,
+        "enable_perm": false,
         "group_num": 1,
         "group_size": 4096,
         "in_features": 4096,
@@ -5202,7 +5188,7 @@
       "model.layers.8.self_attn.q_proj": {
         "bias": false,
         "enable_norm": true,
-        "enable_perm": true,
+        "enable_perm": false,
         "group_num": 1,
         "group_size": 4096,
         "in_features": 4096,
@@ -5226,7 +5212,7 @@
       "model.layers.8.self_attn.v_proj": {
         "bias": false,
         "enable_norm": true,
-        "enable_perm": true,
+        "enable_perm": false,
         "group_num": 1,
         "group_size": 4096,
         "in_features": 4096,
@@ -5250,7 +5236,7 @@
       "model.layers.9.mlp.down_proj": {
         "bias": false,
         "enable_norm": true,
-        "enable_perm": true,
+        "enable_perm": false,
         "group_num": 1,
         "group_size": 14336,
         "in_features": 14336,
@@ -5274,7 +5260,7 @@
       "model.layers.9.mlp.gate_proj": {
         "bias": false,
         "enable_norm": true,
-        "enable_perm": true,
+        "enable_perm": false,
         "group_num": 1,
         "group_size": 4096,
         "in_features": 4096,
@@ -5298,7 +5284,7 @@
       "model.layers.9.mlp.up_proj": {
         "bias": false,
         "enable_norm": true,
-        "enable_perm": true,
+        "enable_perm": false,
         "group_num": 1,
         "group_size": 4096,
         "in_features": 4096,
@@ -5322,7 +5308,7 @@
       "model.layers.9.self_attn.k_proj": {
         "bias": false,
         "enable_norm": true,
-        "enable_perm": true,
+        "enable_perm": false,
         "group_num": 1,
         "group_size": 4096,
         "in_features": 4096,
@@ -5346,7 +5332,7 @@
       "model.layers.9.self_attn.o_proj": {
         "bias": false,
         "enable_norm": true,
-        "enable_perm": true,
+        "enable_perm": false,
         "group_num": 1,
         "group_size": 4096,
         "in_features": 4096,
@@ -5370,7 +5356,7 @@
       "model.layers.9.self_attn.q_proj": {
         "bias": false,
         "enable_norm": true,
-        "enable_perm": true,
+        "enable_perm": false,
         "group_num": 1,
         "group_size": 4096,
         "in_features": 4096,
@@ -5394,7 +5380,7 @@
       "model.layers.9.self_attn.v_proj": {
         "bias": false,
         "enable_norm": true,
-        "enable_perm": true,
+        "enable_perm": false,
         "group_num": 1,
         "group_size": 4096,
         "in_features": 4096,
@@ -5415,6 +5401,21 @@
           8
         ]
       }
-    }
-  }
-}
+    },
+    "quant_method": "vptq"
+  },
+  "rms_norm_eps": 1e-05,
+  "rope_scaling": {
+    "factor": 8.0,
+    "high_freq_factor": 4.0,
+    "low_freq_factor": 1.0,
+    "original_max_position_embeddings": 8192,
+    "rope_type": "llama3"
+  },
+  "rope_theta": 500000.0,
+  "tie_word_embeddings": false,
+  "torch_dtype": "bfloat16",
+  "transformers_version": "4.49.0",
+  "use_cache": true,
+  "vocab_size": 128256
+}

generation_config.json

@@ -1,16 +1,10 @@
 {
-  "attn_implementation": "flash_attention_2",
+  "_from_model_config": true,
   "bos_token_id": 128000,
-  "do_sample": true,
   "eos_token_id": [
     128001,
     128008,
     128009
   ],
-  "num_assistant_tokens": 5,
-  "num_assistant_tokens_schedule": "heuristic",
-  "return_legacy_cache": true,
-  "temperature": 0.6,
-  "top_p": 0.9,
-  "transformers_version": "4.45.1"
+  "transformers_version": "4.49.0"
 }

model-00001-of-00002.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:734343336596c6a6cf24fabc1c4d31faba161a20a7e362a86359622775cf1ee8
-size 4361858256
+oid sha256:cf76e03e16f27cef164fcc14201d31aa6a0d855d06cef6061bc0ebeb4f47de17
+size 4359343640

tokenizer_config.json

@@ -2053,11 +2053,12 @@
   "chat_template": "{{- bos_token }}\n{%- if custom_tools is defined %}\n    {%- set tools = custom_tools %}\n{%- endif %}\n{%- if not tools_in_user_message is defined %}\n    {%- set tools_in_user_message = true %}\n{%- endif %}\n{%- if not date_string is defined %}\n    {%- set date_string = \"26 Jul 2024\" %}\n{%- endif %}\n{%- if not tools is defined %}\n    {%- set tools = none %}\n{%- endif %}\n\n{#- This block extracts the system message, so we can slot it into the right place. #}\n{%- if messages[0]['role'] == 'system' %}\n    {%- set system_message = messages[0]['content']|trim %}\n    {%- set messages = messages[1:] %}\n{%- else %}\n    {%- set system_message = \"\" %}\n{%- endif %}\n\n{#- System message + builtin tools #}\n{{- \"<|start_header_id|>system<|end_header_id|>\\n\\n\" }}\n{%- if builtin_tools is defined or tools is not none %}\n    {{- \"Environment: ipython\\n\" }}\n{%- endif %}\n{%- if builtin_tools is defined %}\n    {{- \"Tools: \" + builtin_tools | reject('equalto', 'code_interpreter') | join(\", \") + \"\\n\\n\"}}\n{%- endif %}\n{{- \"Cutting Knowledge Date: December 2023\\n\" }}\n{{- \"Today Date: \" + date_string + \"\\n\\n\" }}\n{%- if tools is not none and not tools_in_user_message %}\n    {{- \"You have access to the following functions. To call a function, please respond with JSON for a function call.\" }}\n    {{- 'Respond in the format {\"name\": function name, \"parameters\": dictionary of argument name and its value}.' }}\n    {{- \"Do not use variables.\\n\\n\" }}\n    {%- for t in tools %}\n        {{- t | tojson(indent=4) }}\n        {{- \"\\n\\n\" }}\n    {%- endfor %}\n{%- endif %}\n{{- system_message }}\n{{- \"<|eot_id|>\" }}\n\n{#- Custom tools are passed in a user message with some extra guidance #}\n{%- if tools_in_user_message and not tools is none %}\n    {#- Extract the first user message so we can plug it in here #}\n    {%- if messages | length != 0 %}\n        {%- set first_user_message = messages[0]['content']|trim %}\n        {%- set messages = messages[1:] %}\n    {%- else %}\n        {{- raise_exception(\"Cannot put tools in the first user message when there's no first user message!\") }}\n{%- endif %}\n    {{- '<|start_header_id|>user<|end_header_id|>\\n\\n' -}}\n    {{- \"Given the following functions, please respond with a JSON for a function call \" }}\n    {{- \"with its proper arguments that best answers the given prompt.\\n\\n\" }}\n    {{- 'Respond in the format {\"name\": function name, \"parameters\": dictionary of argument name and its value}.' }}\n    {{- \"Do not use variables.\\n\\n\" }}\n    {%- for t in tools %}\n        {{- t | tojson(indent=4) }}\n        {{- \"\\n\\n\" }}\n    {%- endfor %}\n    {{- first_user_message + \"<|eot_id|>\"}}\n{%- endif %}\n\n{%- for message in messages %}\n    {%- if not (message.role == 'ipython' or message.role == 'tool' or 'tool_calls' in message) %}\n        {{- '<|start_header_id|>' + message['role'] + '<|end_header_id|>\\n\\n'+ message['content'] | trim + '<|eot_id|>' }}\n    {%- elif 'tool_calls' in message %}\n        {%- if not message.tool_calls|length == 1 %}\n            {{- raise_exception(\"This model only supports single tool-calls at once!\") }}\n        {%- endif %}\n        {%- set tool_call = message.tool_calls[0].function %}\n        {%- if builtin_tools is defined and tool_call.name in builtin_tools %}\n            {{- '<|start_header_id|>assistant<|end_header_id|>\\n\\n' -}}\n            {{- \"<|python_tag|>\" + tool_call.name + \".call(\" }}\n            {%- for arg_name, arg_val in tool_call.arguments | items %}\n                {{- arg_name + '=\"' + arg_val + '\"' }}\n                {%- if not loop.last %}\n                    {{- \", \" }}\n                {%- endif %}\n                {%- endfor %}\n            {{- \")\" }}\n        {%- else  %}\n            {{- '<|start_header_id|>assistant<|end_header_id|>\\n\\n' -}}\n            {{- '{\"name\": \"' + tool_call.name + '\", ' }}\n            {{- '\"parameters\": ' }}\n            {{- tool_call.arguments | tojson }}\n            {{- \"}\" }}\n        {%- endif %}\n        {%- if builtin_tools is defined %}\n            {#- This means we're in ipython mode #}\n            {{- \"<|eom_id|>\" }}\n        {%- else %}\n            {{- \"<|eot_id|>\" }}\n        {%- endif %}\n    {%- elif message.role == \"tool\" or message.role == \"ipython\" %}\n        {{- \"<|start_header_id|>ipython<|end_header_id|>\\n\\n\" }}\n        {%- if message.content is mapping or message.content is iterable %}\n            {{- message.content | tojson }}\n        {%- else %}\n            {{- message.content }}\n        {%- endif %}\n        {{- \"<|eot_id|>\" }}\n    {%- endif %}\n{%- endfor %}\n{%- if add_generation_prompt %}\n    {{- '<|start_header_id|>assistant<|end_header_id|>\\n\\n' }}\n{%- endif %}\n",
   "clean_up_tokenization_spaces": true,
   "eos_token": "<|eot_id|>",
+  "extra_special_tokens": {},
   "legacy": false,
   "model_input_names": [
     "input_ids",
     "attention_mask"
   ],
   "model_max_length": 131072,
-  "tokenizer_class": "PreTrainedTokenizerFast"
+  "tokenizer_class": "PreTrainedTokenizer"
 }