Update app.py

app.py

@@ -25,7 +25,7 @@ from requests_html import AsyncHTMLSession
 from groq import Groq
 from huggingface_hub import InferenceClient
 
-client = InferenceClient("meta-llama/Meta-Llama-3.1-8B-Instruct")
+# client = InferenceClient("meta-llama/Meta-Llama-3.1-8B-Instruct")
 
 
 # Required for saving the query & response in DB
@@ -38,7 +38,7 @@ logging.basicConfig(level=logging.INFO, format='%(message)s')
 display_ticker=[]
 part = "day"
 
-# client = InferenceClient("mistralai/Mistral-7B-Instruct-v0.1")
+client = InferenceClient("mistralai/Mistral-7B-Instruct-v0.3")
 # client = InferenceClient("meta-llama/Meta-Llama-3-8B-Instruct")
 # client = InferenceClient("google/gemma-2-2b-it")
 client_func_call = InferenceClient("mistralai/Mistral-7B-Instruct-v0.1")
@@ -633,14 +633,14 @@ def generate_final_response(prompt, history):
         logging.info("Total context sent to llm: %s \n\n\n", token_size)
         output=""
         try:
-            for message in client.chat_completion(
-            	messages=[{"role": "user", "content": f"{content}"}],
-            	max_tokens=500,
-            	stream=True,
-            ):
-                stream = message.choices[0].delta.content
-            # # Now start the streaming
-            # stream = client.text_generation(content, **generate_kwargs, stream=True, details=True, return_full_text=True)
+            # for message in client.chat_completion(
+            # 	messages=[{"role": "user", "content": f"{content}"}],
+            # 	max_tokens=500,
+            # 	stream=True,
+            # ):
+            #     stream = message.choices[0].delta.content
+            # Now start the streaming
+            stream = client.text_generation(content, **generate_kwargs, stream=True, details=True, return_full_text=True)
             output = ""
             for response in stream:
                 output += response.token.text