Spaces:
Runtime error
Runtime error
chenlin
commited on
Commit
·
079479b
1
Parent(s):
15f5821
remove eos token
Browse files
app.py
CHANGED
|
@@ -148,6 +148,7 @@ def model_gen(model, text, images, need_bos=True, hd_num=25, max_new_token=2048,
|
|
| 148 |
output_text = model.tokenizer.decode(
|
| 149 |
output_token, add_special_tokens=False)
|
| 150 |
output_text = output_text.split('[UNUSED_TOKEN_145]')[0].strip()
|
|
|
|
| 151 |
return output_text
|
| 152 |
|
| 153 |
|
|
|
|
| 148 |
output_text = model.tokenizer.decode(
|
| 149 |
output_token, add_special_tokens=False)
|
| 150 |
output_text = output_text.split('[UNUSED_TOKEN_145]')[0].strip()
|
| 151 |
+
output_text = output_text.split('<|im_end|>')[0].strip()
|
| 152 |
return output_text
|
| 153 |
|
| 154 |
|