update weights

config.json

@@ -8,7 +8,7 @@
   "hidden_act": "silu",
   "hidden_size": 5120,
   "initializer_range": 0.02,
-  "intermediate_size": 27392,
+  "intermediate_size": 27648,
   "max_position_embeddings": 32768,
   "max_window_layers": 35,
   "model_type": "qwen2",

model-00001-of-00005.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:d85e760402974e233ab592c853d6eb8cb0e10576d85032c7f1df1e3416bbd9f9
-size 3998665856
+oid sha256:c4de668333474d7a285deaa6bae2f7e5efd06738f5ba1bbd602d83d4213aa2e4
+size 3947080608

model-00002-of-00005.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:7a87f33082d1176a513f05bf8b321acf04e5a7e593ca7d966595b88223c03afc
-size 3956139600
+oid sha256:ada2b0935d0e40994796e48f1a404a4d60493d2fb7ba146dbabb1cd52ea63b3f
+size 3985864968

model-00003-of-00005.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:fb8287f05db6fd6268ec1d37cf7e39b60f825e52f6940ba16869ad3aa90c7426
-size 3953828136
+oid sha256:e996d2678b01df627a148d7dace3ed9e04f8a195d3bb337cdf835c8ae35a3c3f
+size 3953094512

model-00004-of-00005.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:fa61bbc5f99ca27dadfbb1f6c166b4edf8c48cb61dc80880008edf33fc384a9c
-size 3996763312
+oid sha256:b8e4959dd115c50c563b570f61fb1adea49d502873913074ec3ef0dc77ca6eee
+size 3985910056

model-00005-of-00005.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:a81aa5633c2ca78f99599e05e03a1abbedb95266b7f444679002ed0b8d59ecd7
-size 3316128672
+oid sha256:f548b49f2f5af13fb5bf722c1b871b7032b5bfe3bdc7ca006b941ac055907600
+size 3480450840

model.safetensors.index.json

@@ -1,6 +1,6 @@
 {
   "metadata": {
-    "total_size": 19221260288
+    "total_size": 19352135680
   },
   "weight_map": {
     "lm_head.weight": "model-00005-of-00005.safetensors",
@@ -672,11 +672,11 @@
     "model.layers.24.self_attn.v_proj.qzeros": "model-00002-of-00005.safetensors",
     "model.layers.24.self_attn.v_proj.scales": "model-00002-of-00005.safetensors",
     "model.layers.25.input_layernorm.weight": "model-00003-of-00005.safetensors",
-    "model.layers.25.mlp.down_proj.bias": "model-00002-of-00005.safetensors",
-    "model.layers.25.mlp.down_proj.g_idx": "model-00002-of-00005.safetensors",
-    "model.layers.25.mlp.down_proj.qweight": "model-00002-of-00005.safetensors",
-    "model.layers.25.mlp.down_proj.qzeros": "model-00002-of-00005.safetensors",
-    "model.layers.25.mlp.down_proj.scales": "model-00002-of-00005.safetensors",
+    "model.layers.25.mlp.down_proj.bias": "model-00003-of-00005.safetensors",
+    "model.layers.25.mlp.down_proj.g_idx": "model-00003-of-00005.safetensors",
+    "model.layers.25.mlp.down_proj.qweight": "model-00003-of-00005.safetensors",
+    "model.layers.25.mlp.down_proj.qzeros": "model-00003-of-00005.safetensors",
+    "model.layers.25.mlp.down_proj.scales": "model-00003-of-00005.safetensors",
     "model.layers.25.mlp.gate_proj.bias": "model-00003-of-00005.safetensors",
     "model.layers.25.mlp.gate_proj.g_idx": "model-00003-of-00005.safetensors",
     "model.layers.25.mlp.gate_proj.qweight": "model-00003-of-00005.safetensors",
@@ -1300,7 +1300,7 @@
     "model.layers.4.self_attn.v_proj.qweight": "model-00001-of-00005.safetensors",
     "model.layers.4.self_attn.v_proj.qzeros": "model-00001-of-00005.safetensors",
     "model.layers.4.self_attn.v_proj.scales": "model-00001-of-00005.safetensors",
-    "model.layers.40.input_layernorm.weight": "model-00003-of-00005.safetensors",
+    "model.layers.40.input_layernorm.weight": "model-00004-of-00005.safetensors",
     "model.layers.40.mlp.down_proj.bias": "model-00003-of-00005.safetensors",
     "model.layers.40.mlp.down_proj.g_idx": "model-00003-of-00005.safetensors",
     "model.layers.40.mlp.down_proj.qweight": "model-00003-of-00005.safetensors",
@@ -1311,12 +1311,12 @@
     "model.layers.40.mlp.gate_proj.qweight": "model-00003-of-00005.safetensors",
     "model.layers.40.mlp.gate_proj.qzeros": "model-00003-of-00005.safetensors",
     "model.layers.40.mlp.gate_proj.scales": "model-00003-of-00005.safetensors",
-    "model.layers.40.mlp.up_proj.bias": "model-00003-of-00005.safetensors",
-    "model.layers.40.mlp.up_proj.g_idx": "model-00003-of-00005.safetensors",
-    "model.layers.40.mlp.up_proj.qweight": "model-00003-of-00005.safetensors",
-    "model.layers.40.mlp.up_proj.qzeros": "model-00003-of-00005.safetensors",
-    "model.layers.40.mlp.up_proj.scales": "model-00003-of-00005.safetensors",
-    "model.layers.40.post_attention_layernorm.weight": "model-00003-of-00005.safetensors",
+    "model.layers.40.mlp.up_proj.bias": "model-00004-of-00005.safetensors",
+    "model.layers.40.mlp.up_proj.g_idx": "model-00004-of-00005.safetensors",
+    "model.layers.40.mlp.up_proj.qweight": "model-00004-of-00005.safetensors",
+    "model.layers.40.mlp.up_proj.qzeros": "model-00004-of-00005.safetensors",
+    "model.layers.40.mlp.up_proj.scales": "model-00004-of-00005.safetensors",
+    "model.layers.40.post_attention_layernorm.weight": "model-00004-of-00005.safetensors",
     "model.layers.40.self_attn.k_proj.bias": "model-00003-of-00005.safetensors",
     "model.layers.40.self_attn.k_proj.g_idx": "model-00003-of-00005.safetensors",
     "model.layers.40.self_attn.k_proj.qweight": "model-00003-of-00005.safetensors",
@@ -1354,26 +1354,26 @@
     "model.layers.41.mlp.up_proj.qzeros": "model-00004-of-00005.safetensors",
     "model.layers.41.mlp.up_proj.scales": "model-00004-of-00005.safetensors",
     "model.layers.41.post_attention_layernorm.weight": "model-00004-of-00005.safetensors",
-    "model.layers.41.self_attn.k_proj.bias": "model-00003-of-00005.safetensors",
-    "model.layers.41.self_attn.k_proj.g_idx": "model-00003-of-00005.safetensors",
-    "model.layers.41.self_attn.k_proj.qweight": "model-00003-of-00005.safetensors",
-    "model.layers.41.self_attn.k_proj.qzeros": "model-00003-of-00005.safetensors",
-    "model.layers.41.self_attn.k_proj.scales": "model-00003-of-00005.safetensors",
-    "model.layers.41.self_attn.o_proj.bias": "model-00003-of-00005.safetensors",
-    "model.layers.41.self_attn.o_proj.g_idx": "model-00003-of-00005.safetensors",
-    "model.layers.41.self_attn.o_proj.qweight": "model-00003-of-00005.safetensors",
-    "model.layers.41.self_attn.o_proj.qzeros": "model-00003-of-00005.safetensors",
-    "model.layers.41.self_attn.o_proj.scales": "model-00003-of-00005.safetensors",
-    "model.layers.41.self_attn.q_proj.bias": "model-00003-of-00005.safetensors",
-    "model.layers.41.self_attn.q_proj.g_idx": "model-00003-of-00005.safetensors",
-    "model.layers.41.self_attn.q_proj.qweight": "model-00003-of-00005.safetensors",
-    "model.layers.41.self_attn.q_proj.qzeros": "model-00003-of-00005.safetensors",
-    "model.layers.41.self_attn.q_proj.scales": "model-00003-of-00005.safetensors",
-    "model.layers.41.self_attn.v_proj.bias": "model-00003-of-00005.safetensors",
-    "model.layers.41.self_attn.v_proj.g_idx": "model-00003-of-00005.safetensors",
-    "model.layers.41.self_attn.v_proj.qweight": "model-00003-of-00005.safetensors",
-    "model.layers.41.self_attn.v_proj.qzeros": "model-00003-of-00005.safetensors",
-    "model.layers.41.self_attn.v_proj.scales": "model-00003-of-00005.safetensors",
+    "model.layers.41.self_attn.k_proj.bias": "model-00004-of-00005.safetensors",
+    "model.layers.41.self_attn.k_proj.g_idx": "model-00004-of-00005.safetensors",
+    "model.layers.41.self_attn.k_proj.qweight": "model-00004-of-00005.safetensors",
+    "model.layers.41.self_attn.k_proj.qzeros": "model-00004-of-00005.safetensors",
+    "model.layers.41.self_attn.k_proj.scales": "model-00004-of-00005.safetensors",
+    "model.layers.41.self_attn.o_proj.bias": "model-00004-of-00005.safetensors",
+    "model.layers.41.self_attn.o_proj.g_idx": "model-00004-of-00005.safetensors",
+    "model.layers.41.self_attn.o_proj.qweight": "model-00004-of-00005.safetensors",
+    "model.layers.41.self_attn.o_proj.qzeros": "model-00004-of-00005.safetensors",
+    "model.layers.41.self_attn.o_proj.scales": "model-00004-of-00005.safetensors",
+    "model.layers.41.self_attn.q_proj.bias": "model-00004-of-00005.safetensors",
+    "model.layers.41.self_attn.q_proj.g_idx": "model-00004-of-00005.safetensors",
+    "model.layers.41.self_attn.q_proj.qweight": "model-00004-of-00005.safetensors",
+    "model.layers.41.self_attn.q_proj.qzeros": "model-00004-of-00005.safetensors",
+    "model.layers.41.self_attn.q_proj.scales": "model-00004-of-00005.safetensors",
+    "model.layers.41.self_attn.v_proj.bias": "model-00004-of-00005.safetensors",
+    "model.layers.41.self_attn.v_proj.g_idx": "model-00004-of-00005.safetensors",
+    "model.layers.41.self_attn.v_proj.qweight": "model-00004-of-00005.safetensors",
+    "model.layers.41.self_attn.v_proj.qzeros": "model-00004-of-00005.safetensors",
+    "model.layers.41.self_attn.v_proj.scales": "model-00004-of-00005.safetensors",
     "model.layers.42.input_layernorm.weight": "model-00004-of-00005.safetensors",
     "model.layers.42.mlp.down_proj.bias": "model-00004-of-00005.safetensors",
     "model.layers.42.mlp.down_proj.g_idx": "model-00004-of-00005.safetensors",
@@ -1929,23 +1929,23 @@
     "model.layers.55.self_attn.v_proj.qweight": "model-00004-of-00005.safetensors",
     "model.layers.55.self_attn.v_proj.qzeros": "model-00004-of-00005.safetensors",
     "model.layers.55.self_attn.v_proj.scales": "model-00004-of-00005.safetensors",
-    "model.layers.56.input_layernorm.weight": "model-00004-of-00005.safetensors",
+    "model.layers.56.input_layernorm.weight": "model-00005-of-00005.safetensors",
     "model.layers.56.mlp.down_proj.bias": "model-00004-of-00005.safetensors",
     "model.layers.56.mlp.down_proj.g_idx": "model-00004-of-00005.safetensors",
     "model.layers.56.mlp.down_proj.qweight": "model-00004-of-00005.safetensors",
     "model.layers.56.mlp.down_proj.qzeros": "model-00004-of-00005.safetensors",
     "model.layers.56.mlp.down_proj.scales": "model-00004-of-00005.safetensors",
-    "model.layers.56.mlp.gate_proj.bias": "model-00004-of-00005.safetensors",
-    "model.layers.56.mlp.gate_proj.g_idx": "model-00004-of-00005.safetensors",
-    "model.layers.56.mlp.gate_proj.qweight": "model-00004-of-00005.safetensors",
-    "model.layers.56.mlp.gate_proj.qzeros": "model-00004-of-00005.safetensors",
-    "model.layers.56.mlp.gate_proj.scales": "model-00004-of-00005.safetensors",
-    "model.layers.56.mlp.up_proj.bias": "model-00004-of-00005.safetensors",
-    "model.layers.56.mlp.up_proj.g_idx": "model-00004-of-00005.safetensors",
-    "model.layers.56.mlp.up_proj.qweight": "model-00004-of-00005.safetensors",
-    "model.layers.56.mlp.up_proj.qzeros": "model-00004-of-00005.safetensors",
-    "model.layers.56.mlp.up_proj.scales": "model-00004-of-00005.safetensors",
-    "model.layers.56.post_attention_layernorm.weight": "model-00004-of-00005.safetensors",
+    "model.layers.56.mlp.gate_proj.bias": "model-00005-of-00005.safetensors",
+    "model.layers.56.mlp.gate_proj.g_idx": "model-00005-of-00005.safetensors",
+    "model.layers.56.mlp.gate_proj.qweight": "model-00005-of-00005.safetensors",
+    "model.layers.56.mlp.gate_proj.qzeros": "model-00005-of-00005.safetensors",
+    "model.layers.56.mlp.gate_proj.scales": "model-00005-of-00005.safetensors",
+    "model.layers.56.mlp.up_proj.bias": "model-00005-of-00005.safetensors",
+    "model.layers.56.mlp.up_proj.g_idx": "model-00005-of-00005.safetensors",
+    "model.layers.56.mlp.up_proj.qweight": "model-00005-of-00005.safetensors",
+    "model.layers.56.mlp.up_proj.qzeros": "model-00005-of-00005.safetensors",
+    "model.layers.56.mlp.up_proj.scales": "model-00005-of-00005.safetensors",
+    "model.layers.56.post_attention_layernorm.weight": "model-00005-of-00005.safetensors",
     "model.layers.56.self_attn.k_proj.bias": "model-00004-of-00005.safetensors",
     "model.layers.56.self_attn.k_proj.g_idx": "model-00004-of-00005.safetensors",
     "model.layers.56.self_attn.k_proj.qweight": "model-00004-of-00005.safetensors",
@@ -1983,11 +1983,11 @@
     "model.layers.57.mlp.up_proj.qzeros": "model-00005-of-00005.safetensors",
     "model.layers.57.mlp.up_proj.scales": "model-00005-of-00005.safetensors",
     "model.layers.57.post_attention_layernorm.weight": "model-00005-of-00005.safetensors",
-    "model.layers.57.self_attn.k_proj.bias": "model-00004-of-00005.safetensors",
-    "model.layers.57.self_attn.k_proj.g_idx": "model-00004-of-00005.safetensors",
-    "model.layers.57.self_attn.k_proj.qweight": "model-00004-of-00005.safetensors",
-    "model.layers.57.self_attn.k_proj.qzeros": "model-00004-of-00005.safetensors",
-    "model.layers.57.self_attn.k_proj.scales": "model-00004-of-00005.safetensors",
+    "model.layers.57.self_attn.k_proj.bias": "model-00005-of-00005.safetensors",
+    "model.layers.57.self_attn.k_proj.g_idx": "model-00005-of-00005.safetensors",
+    "model.layers.57.self_attn.k_proj.qweight": "model-00005-of-00005.safetensors",
+    "model.layers.57.self_attn.k_proj.qzeros": "model-00005-of-00005.safetensors",
+    "model.layers.57.self_attn.k_proj.scales": "model-00005-of-00005.safetensors",
     "model.layers.57.self_attn.o_proj.bias": "model-00005-of-00005.safetensors",
     "model.layers.57.self_attn.o_proj.g_idx": "model-00005-of-00005.safetensors",
     "model.layers.57.self_attn.o_proj.qweight": "model-00005-of-00005.safetensors",
@@ -2344,8 +2344,8 @@
     "model.layers.9.mlp.down_proj.scales": "model-00001-of-00005.safetensors",
     "model.layers.9.mlp.gate_proj.bias": "model-00002-of-00005.safetensors",
     "model.layers.9.mlp.gate_proj.g_idx": "model-00002-of-00005.safetensors",
-    "model.layers.9.mlp.gate_proj.qweight": "model-00001-of-00005.safetensors",
-    "model.layers.9.mlp.gate_proj.qzeros": "model-00001-of-00005.safetensors",
+    "model.layers.9.mlp.gate_proj.qweight": "model-00002-of-00005.safetensors",
+    "model.layers.9.mlp.gate_proj.qzeros": "model-00002-of-00005.safetensors",
     "model.layers.9.mlp.gate_proj.scales": "model-00002-of-00005.safetensors",
     "model.layers.9.mlp.up_proj.bias": "model-00002-of-00005.safetensors",
     "model.layers.9.mlp.up_proj.g_idx": "model-00002-of-00005.safetensors",