'decrease gptq group size'

README.md

@@ -16,15 +16,23 @@ tags:
 
 
 ### 【模型更新日期】
-``` 2024-06-05 ```
+``` 2024-06-05 21:00 ```
 
 ### 【模型大小】
-`6.2GB`
+`6.5GB`
 
 ### 【06-05 临时情况告知】
 
 1. 目前需要用vllm entrypoint的方式来启动模型。
-2. 如果首次对话出现感叹号，那么添加一些提示词可以规避，近期会跟进寻找修复策略。
+2. 如果出现感叹号，请留言告知，并告知显卡型号。
+
+### 【更新日志】
+
+```
+2004-06-05 21:00 
+1. 尝试修复！！！感叹号吐字问题。
+2. group_size 调整为64，减少量化精度损失。
+```
 
 ### 【介绍】
 

config.json

@@ -1,5 +1,5 @@
 {
-  "_name_or_path": "tclf90/glm-4-9b-chat-GPTQ-Int4-G128",
+  "_name_or_path": "tclf90/glm-4-9b-chat-GPTQ-Int4",
   "add_bias_linear": false,
   "add_qkv_bias": true,
   "apply_query_key_layer_scaling": true,
@@ -28,7 +28,7 @@
   "hidden_dropout": 0.0,
   "hidden_size": 4096,
   "kv_channels": 128,
-  "layernorm_epsilon": 1.5625e-07,
+  "layernorm_epsilon": 1e-05,
   "model_type": "chatglm",
   "multi_query_attention": true,
   "multi_query_group_num": 2,
@@ -50,7 +50,7 @@
     "exllama_config": {
       "version": 1
     },
-    "group_size": 128,
+    "group_size": 64,
     "max_input_length": null,
     "model_seqlen": null,
     "module_name_preceding_first_block": null,

generation_config.json

@@ -5,5 +5,6 @@
     151336,
     151338
   ],
+  "pad_token_id": 151329,
   "transformers_version": "4.40.2"
 }

model-00001-of-00002.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:db41f89442a01385028ac48bee7e767b14681bcd274d4939c6edbe707e27d523
-size 4975739320
+oid sha256:65eadf64a1c6f70038b02fb1a6526e11a115459936ea74f6939b06a9bfe3990f
+size 4995499432

model-00002-of-00002.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:6953050103ce8867a55b25ccc84e193a5752c9e79a1207ce123c7f38aec46f1b
-size 1753736944
+oid sha256:ad97ab92f6ce8fa9f9fc3ab470e930c9aa837fb471d9dda9dd05a53cc0f294d1
+size 1893310768

model.safetensors.index.json

@@ -1,6 +1,6 @@
 {
   "metadata": {
-    "total_size": 6729359424
+    "total_size": 6888693824
   },
   "weight_map": {
     "transformer.embedding.word_embeddings.weight": "model-00001-of-00002.safetensors",
@@ -622,28 +622,28 @@
     "transformer.encoder.layers.33.self_attention.query_key_value.qzeros": "model-00001-of-00002.safetensors",
     "transformer.encoder.layers.33.self_attention.query_key_value.scales": "model-00001-of-00002.safetensors",
     "transformer.encoder.layers.34.input_layernorm.weight": "model-00001-of-00002.safetensors",
-    "transformer.encoder.layers.34.mlp.dense_4h_to_h.bias": "model-00001-of-00002.safetensors",
-    "transformer.encoder.layers.34.mlp.dense_4h_to_h.g_idx": "model-00001-of-00002.safetensors",
-    "transformer.encoder.layers.34.mlp.dense_4h_to_h.qweight": "model-00001-of-00002.safetensors",
-    "transformer.encoder.layers.34.mlp.dense_4h_to_h.qzeros": "model-00001-of-00002.safetensors",
-    "transformer.encoder.layers.34.mlp.dense_4h_to_h.scales": "model-00001-of-00002.safetensors",
-    "transformer.encoder.layers.34.mlp.dense_h_to_4h.bias": "model-00001-of-00002.safetensors",
-    "transformer.encoder.layers.34.mlp.dense_h_to_4h.g_idx": "model-00001-of-00002.safetensors",
-    "transformer.encoder.layers.34.mlp.dense_h_to_4h.qweight": "model-00001-of-00002.safetensors",
-    "transformer.encoder.layers.34.mlp.dense_h_to_4h.qzeros": "model-00001-of-00002.safetensors",
-    "transformer.encoder.layers.34.mlp.dense_h_to_4h.scales": "model-00001-of-00002.safetensors",
-    "transformer.encoder.layers.34.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "transformer.encoder.layers.34.mlp.dense_4h_to_h.bias": "model-00002-of-00002.safetensors",
+    "transformer.encoder.layers.34.mlp.dense_4h_to_h.g_idx": "model-00002-of-00002.safetensors",
+    "transformer.encoder.layers.34.mlp.dense_4h_to_h.qweight": "model-00002-of-00002.safetensors",
+    "transformer.encoder.layers.34.mlp.dense_4h_to_h.qzeros": "model-00002-of-00002.safetensors",
+    "transformer.encoder.layers.34.mlp.dense_4h_to_h.scales": "model-00002-of-00002.safetensors",
+    "transformer.encoder.layers.34.mlp.dense_h_to_4h.bias": "model-00002-of-00002.safetensors",
+    "transformer.encoder.layers.34.mlp.dense_h_to_4h.g_idx": "model-00002-of-00002.safetensors",
+    "transformer.encoder.layers.34.mlp.dense_h_to_4h.qweight": "model-00002-of-00002.safetensors",
+    "transformer.encoder.layers.34.mlp.dense_h_to_4h.qzeros": "model-00002-of-00002.safetensors",
+    "transformer.encoder.layers.34.mlp.dense_h_to_4h.scales": "model-00002-of-00002.safetensors",
+    "transformer.encoder.layers.34.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
     "transformer.encoder.layers.34.self_attention.dense.bias": "model-00001-of-00002.safetensors",
     "transformer.encoder.layers.34.self_attention.dense.g_idx": "model-00001-of-00002.safetensors",
     "transformer.encoder.layers.34.self_attention.dense.qweight": "model-00001-of-00002.safetensors",
     "transformer.encoder.layers.34.self_attention.dense.qzeros": "model-00001-of-00002.safetensors",
     "transformer.encoder.layers.34.self_attention.dense.scales": "model-00001-of-00002.safetensors",
-    "transformer.encoder.layers.34.self_attention.query_key_value.bias": "model-00001-of-00002.safetensors",
-    "transformer.encoder.layers.34.self_attention.query_key_value.g_idx": "model-00001-of-00002.safetensors",
-    "transformer.encoder.layers.34.self_attention.query_key_value.qweight": "model-00001-of-00002.safetensors",
-    "transformer.encoder.layers.34.self_attention.query_key_value.qzeros": "model-00001-of-00002.safetensors",
-    "transformer.encoder.layers.34.self_attention.query_key_value.scales": "model-00001-of-00002.safetensors",
-    "transformer.encoder.layers.35.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "transformer.encoder.layers.34.self_attention.query_key_value.bias": "model-00002-of-00002.safetensors",
+    "transformer.encoder.layers.34.self_attention.query_key_value.g_idx": "model-00002-of-00002.safetensors",
+    "transformer.encoder.layers.34.self_attention.query_key_value.qweight": "model-00002-of-00002.safetensors",
+    "transformer.encoder.layers.34.self_attention.query_key_value.qzeros": "model-00002-of-00002.safetensors",
+    "transformer.encoder.layers.34.self_attention.query_key_value.scales": "model-00002-of-00002.safetensors",
+    "transformer.encoder.layers.35.input_layernorm.weight": "model-00002-of-00002.safetensors",
     "transformer.encoder.layers.35.mlp.dense_4h_to_h.bias": "model-00002-of-00002.safetensors",
     "transformer.encoder.layers.35.mlp.dense_4h_to_h.g_idx": "model-00002-of-00002.safetensors",
     "transformer.encoder.layers.35.mlp.dense_4h_to_h.qweight": "model-00002-of-00002.safetensors",
@@ -654,17 +654,17 @@
     "transformer.encoder.layers.35.mlp.dense_h_to_4h.qweight": "model-00002-of-00002.safetensors",
     "transformer.encoder.layers.35.mlp.dense_h_to_4h.qzeros": "model-00002-of-00002.safetensors",
     "transformer.encoder.layers.35.mlp.dense_h_to_4h.scales": "model-00002-of-00002.safetensors",
-    "transformer.encoder.layers.35.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
-    "transformer.encoder.layers.35.self_attention.dense.bias": "model-00001-of-00002.safetensors",
-    "transformer.encoder.layers.35.self_attention.dense.g_idx": "model-00001-of-00002.safetensors",
-    "transformer.encoder.layers.35.self_attention.dense.qweight": "model-00001-of-00002.safetensors",
-    "transformer.encoder.layers.35.self_attention.dense.qzeros": "model-00001-of-00002.safetensors",
-    "transformer.encoder.layers.35.self_attention.dense.scales": "model-00001-of-00002.safetensors",
-    "transformer.encoder.layers.35.self_attention.query_key_value.bias": "model-00001-of-00002.safetensors",
-    "transformer.encoder.layers.35.self_attention.query_key_value.g_idx": "model-00001-of-00002.safetensors",
-    "transformer.encoder.layers.35.self_attention.query_key_value.qweight": "model-00001-of-00002.safetensors",
-    "transformer.encoder.layers.35.self_attention.query_key_value.qzeros": "model-00001-of-00002.safetensors",
-    "transformer.encoder.layers.35.self_attention.query_key_value.scales": "model-00001-of-00002.safetensors",
+    "transformer.encoder.layers.35.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "transformer.encoder.layers.35.self_attention.dense.bias": "model-00002-of-00002.safetensors",
+    "transformer.encoder.layers.35.self_attention.dense.g_idx": "model-00002-of-00002.safetensors",
+    "transformer.encoder.layers.35.self_attention.dense.qweight": "model-00002-of-00002.safetensors",
+    "transformer.encoder.layers.35.self_attention.dense.qzeros": "model-00002-of-00002.safetensors",
+    "transformer.encoder.layers.35.self_attention.dense.scales": "model-00002-of-00002.safetensors",
+    "transformer.encoder.layers.35.self_attention.query_key_value.bias": "model-00002-of-00002.safetensors",
+    "transformer.encoder.layers.35.self_attention.query_key_value.g_idx": "model-00002-of-00002.safetensors",
+    "transformer.encoder.layers.35.self_attention.query_key_value.qweight": "model-00002-of-00002.safetensors",
+    "transformer.encoder.layers.35.self_attention.query_key_value.qzeros": "model-00002-of-00002.safetensors",
+    "transformer.encoder.layers.35.self_attention.query_key_value.scales": "model-00002-of-00002.safetensors",
     "transformer.encoder.layers.36.input_layernorm.weight": "model-00002-of-00002.safetensors",
     "transformer.encoder.layers.36.mlp.dense_4h_to_h.bias": "model-00002-of-00002.safetensors",
     "transformer.encoder.layers.36.mlp.dense_4h_to_h.g_idx": "model-00002-of-00002.safetensors",

modeling_chatglm.py

@@ -324,7 +324,7 @@ class SelfAttention(torch.nn.Module):
         )
 
     def forward(
-            self, hidden_states, attention_mask, rotary_pos_emb=None, kv_cache=None, use_cache=True
+            self, hidden_states, attention_mask, rotary_pos_emb, kv_cache=None, use_cache=True
     ):
         # hidden_states: [b, sq, h]