Update app.py

app.py

@@ -26,8 +26,9 @@ import accelerate
 import bitsandbytes
 
 # device = "cuda" if torch.cuda.is_available() else "cpu"
-"bos_token_id": 1,
-"eos_token_id": 32000,
+bos_token_id = 1,
+eos_token_id = 32000
+pad_token_id = 32001
 temperature=0.4
 max_new_tokens=240
 top_p=0.92
@@ -38,7 +39,7 @@ model = transformers.AutoModelForCausalLM.from_pretrained(model_name,
     device_map="auto",
     torch_dtype="auto"
 )
-model.eval()
+# model.eval()
 
 class StarlingBot:
     def __init__(self, system_prompt="I am Starling-7B by Tonic-AI, I ready to do anything to help my user."):
@@ -53,9 +54,9 @@ class StarlingBot:
                 input_ids=input_ids, 
                 use_cache=True, 
                 early_stopping=False, 
-                bos_token_id=model.config.bos_token_id, 
-                eos_token_id=model.config.eos_token_id, 
-                pad_token_id=model.config.eos_token_id, 
+                bos_token_id=bos_token_id, 
+                eos_token_id=eos_token_id, 
+                pad_token_id=pad_token_id, 
                 temperature=temperature, 
                 do_sample=True, 
                 max_new_tokens=max_new_tokens,