Update app.py

app.py

@@ -1,11 +1,9 @@
 # import dependencies
 import torch
 from transformers import AutoModelForCausalLM, AutoTokenizer, BitsAndBytesConfig, pipeline
-
 import os
 import gradio as gr
 #from google.colab import drive
-
 import chromadb
 from langchain.llms import HuggingFacePipeline
 from langchain.document_loaders import TextLoader
@@ -17,6 +15,9 @@ from langchain.document_loaders import PyPDFDirectoryLoader
 from langchain.chains import ConversationalRetrievalChain
 from langchain.memory import ConversationBufferMemory
 
+#import locale
+#locale.getpreferredencoding = lambda: "UTF-8"
+
 # specify model huggingface mode name
 model_name = "anakin87/zephyr-7b-alpha-sharded"
 #https://huggingface.co/anakin87/zephyr-7b-alpha-sharded
@@ -31,10 +32,11 @@ def load_quantized_model(model_name: str):
     :return: Loaded quantized model.
     """
     bnb_config = BitsAndBytesConfig(
-        load_in_4bit=True,
+        #load_in_4bit=True,
+        load_in_4bit=False,
         #bnb_4bit_use_double_quant=True,
         bnb_4bit_use_double_quant=False,
-        bnb_4bit_quant_type="nf4",
+        bnb_4bit_quant_type="nf4"
         #bnb_4bit_compute_dtype=torch.bfloat16
     )
 
@@ -42,7 +44,6 @@ def load_quantized_model(model_name: str):
         model_name,
         load_in_4bit=True,
         #torch_dtype=torch.bfloat16,
-        #torch_dtype=torch.float16 if torch.cuda.is_available() else torch.float32,
         quantization_config=bnb_config
     )
     return model
@@ -57,7 +58,7 @@ def initialize_tokenizer(model_name: str):
     """
     tokenizer = AutoTokenizer.from_pretrained(model_name)
     tokenizer.bos_token_id = 1  # Set beginning of sentence token id
-    return tokenizer
+    return tokenizer    
 
 # load model
 model = load_quantized_model(model_name)
@@ -125,7 +126,6 @@ def create_conversation(query: str, chat_history: list) -> tuple:
         chat_history.append((query, result['answer']))
         return '', chat_history
 
-
     except Exception as e:
         chat_history.append((query, e))
         return '', chat_history