Update app.py

app.py

@@ -1,5 +1,6 @@
 import spaces
 import datetime
+import logging
 import urllib.parse
 import json
 import asyncio
@@ -23,7 +24,7 @@ moneycontrol_news = ""
 earning_stocks = ""
 part = "day"
 
-print("currect hour:", (datetime.datetime.now().hour))
+logging.info("currect hour:", (datetime.datetime.now().hour))
 
 client = InferenceClient(
     "mistralai/Mistral-7B-Instruct-v0.3"
@@ -67,7 +68,7 @@ def latest_earning():
             
 def todays_news():
     url = 'https://trendlyne.com/markets-today/'
-    print("getting news from", url)
+    logging.info("getting news from", url)
     # Fetch the HTML content of the webpage
     html_content = requests.get(url).text
     soup = BeautifulSoup(html_content, 'html.parser')
@@ -89,8 +90,8 @@ def todays_news():
         notification.append(insight.find(class_='insight-notification').text.strip())
 
     df = pd.DataFrame({"Timestamp": timestamps, "Stock": stock_names, "Label": insight_label, "Notification": notification})
-    print("Dataframe created for stocks in news today")
-    print(df)
+    logging.info("Dataframe created for stocks in news today")
+    logging.info(df)
     df_dict = df.to_dict('records')
     return df_dict
 
@@ -115,8 +116,8 @@ def get_moneycontrol_news():
                     linkx.append(a_tag['href'])
         filtered_links = list(set([link for link in linkx if link.endswith('.html')]))
     else:
-        # If the request was not successful, print an error message
-        print("Failed to retrieve article links from the moneycontrol")
+        # If the request was not successful, logging.info an error message
+        logging.info("Failed to retrieve article links from the moneycontrol")
         return
     
     common_content = ""
@@ -137,7 +138,7 @@ def get_moneycontrol_news():
                     split_author = split_article.split('author', 1)[0]
                     heading = "Heading -" + link.split('/')[-1].replace("-", " ")
                     body = "Body -" + re.sub('[:\n-\";]|amp', ' ', split_author)
-                    print(heading)
+                    logging.info(heading)
                     common_content = common_content + str({heading: body}) + "," + "\n"
 
     today = datetime.date.today()
@@ -204,7 +205,7 @@ def get_the_ticker(stock_name):
             raw_query = f"YAHOO FINANCE TICKER SYMBOL OF {stock.upper()}"
             query = raw_query.replace(" ", "+")
             url = f'https://www.google.com/search?q={query}&FORM=HDRSC7'
-            print("searching ticker using url: ",url)
+            logging.info("searching ticker using url: ",url)
             # Fetch the HTML content of the webpage
             html_content = requests.get(url).text  # Fix: Added .text to access the response text
             soup = BeautifulSoup(html_content, "html.parser")
@@ -213,7 +214,7 @@ def get_the_ticker(stock_name):
             step1 = [urllib.parse.unquote(i) for i in matches]
             matches = [urllib.parse.unquote(i) for i in step1]
             matches = list(set(matches[:2]))
-            print("List of matches obtained: ", matches)
+            logging.info("List of matches obtained: ", matches)
             
     return matches
 
@@ -221,16 +222,16 @@ def get_the_ticker(stock_name):
 def get_the_ticker_stat(stock):
     combination=[]
     url = f'https://www.google.com/search?q={urllib.parse.quote(stock)}+site:businesstoday.in/stocks/&num=1&sca_esv=28795b6719ac1a08&sxsrf=ACQVn08xDA1EP1V6hJ-q4jLjjXSWWxgHTw:1711450545062&source=lnt&tbs=li:1&sa=X&ved=2ahUKEwj426eO4pGFAxX4n2MGHRXqBTUQpwV6BAgBEBM&biw=1280&bih=567&dpr=1.5'
-    print("getting ticker stat url from: ", url)
+    logging.info("getting ticker stat url from: ", url)
     # Fetch the HTML content of the webpage
     html_content = requests.get(url).text  # Fix: Added .text to access the response text
     pattern = r'href="/url[?]q=(https://www.businesstoday.in/stocks/[^"]+)"'
     # Find all matches using re.findall
     links = re.findall(pattern, html_content)
     links = list(set(links))
-    print("List of links obtained for ticker stat: ", links)
+    logging.info("List of links obtained for ticker stat: ", links)
     url = (links[0].split("&"))[0]
-    print("Final URL to fetch stats", url)
+    logging.info("Final URL to fetch stats", url)
     # Fetch the HTML content of the webpage
     html_content = requests.get(url).text
     soup = BeautifulSoup(html_content, "html.parser")
@@ -238,7 +239,7 @@ def get_the_ticker_stat(stock):
 
     # Parse the JSON-LD script
     json_data = json.loads(script.text)
-    # print(json_data)
+    # logging.info(json_data)
     # Iterate over the "mainEntity" array
     qa_dict={}
     for entity in json_data["mainEntity"]:
@@ -246,50 +247,50 @@ def get_the_ticker_stat(stock):
         question = entity["name"].replace("'", "")
         answer = entity["acceptedAnswer"]["text"].replace("'", "")
 
-        # Print the question and answer
+        # logging.info the question and answer
         qa_dict[question]=answer
 
     combination.append(qa_dict)
-    print("All the stat data fetched: ", combination)
+    logging.info("All the stat data fetched: ", combination)
     return(combination)
 
 
 def get_the_ticker_news(stock):
     all_news=[]
     url = f'https://www.google.com/search?q={urllib.parse.quote(stock)}+site:trendlyne.com/research-reports&num=3&sca_esv=28795b6719ac1a08&sxsrf=ACQVn08xDA1EP1V6hJ-q4jLjjXSWWxgHTw:1711450545062&source=lnt&tbs=li:1&sa=X&ved=2ahUKEwj426eO4pGFAxX4n2MGHRXqBTUQpwV6BAgBEBM&biw=1280&bih=567&dpr=1.5'
-    # print(url)
+    # logging.info(url)
     # Fetch the HTML content of the webpage
     html_content = requests.get(url).text  # Fix: Added .text to access the response text
     pattern = r'href="/url[?]q=(https://trendlyne.com/research-reports/[^"]+)"'
     # Find all matches using re.findall
     links = re.findall(pattern, html_content)
     links = list(set(links))
-    print("Links fetched to get trendlyne research report: ",links)
+    logging.info("Links fetched to get trendlyne research report: ",links)
     if "/%" in links[0]:
         fetched_reports_url = links[0].split("%")[0]
     else:
         fetched_reports_url = links[0].split("&")[0]
     # fetched url may look like this  -  https://trendlyne.com/research-reports/post/ROLTA/1146/rolta-india-ltd/
-    print("finalised url: ", fetched_reports_url)
+    logging.info("finalised url: ", fetched_reports_url)
     pattern = '\/\/.*?(\d+).*\/'
     match = re.search(pattern, fetched_reports_url)
     if match:
-        print("unique number identified in url")
+        logging.info("unique number identified in url")
         split_url = fetched_reports_url.split("/")
-        print(split_url)
+        logging.info(split_url)
         unique_no = match.group(1)
-        print("Unique no identified: ",unique_no)
+        logging.info("Unique no identified: ",unique_no)
         company_name = split_url[-2]
-        print("Company name identified: ",company_name)
+        logging.info("Company name identified: ",company_name)
         reports_url = f"https://trendlyne.com/research-reports/stock/{unique_no}/{urllib.parse.quote(stock)}/{company_name}/"
     else:
-        print("unique number not identified in url continuing basic flow")
+        logging.info("unique number not identified in url continuing basic flow")
         reports_url = fetched_reports_url
     financials_url = reports_url.replace("research-reports/stock","equity")
     url = reports_url.replace("research-reports/stock","latest-news")
-    print("\nURL to fetch news links: ", url)
-    print(f"\nURL to fetch health of financial insights:\n {financials_url}")
-    print(f"\nURL to fetch rating info:\n {reports_url}")
+    logging.info("\nURL to fetch news links: ", url)
+    logging.info(f"\nURL to fetch health of financial insights:\n {financials_url}")
+    logging.info(f"\nURL to fetch rating info:\n {reports_url}")
 
     # Fetch the HTML content of the webpage
     # req = Request(
@@ -297,29 +298,29 @@ def get_the_ticker_news(stock):
     # headers={'User-Agent': 'Mozilla/5.0'}
     # )
     # webpage = urlopen(req).read()
-    # print(webpage)
+    # logging.info(webpage)
     html_content = requests.get(url,headers={'User-Agent': 'Mozilla/5.0'}).text
     html_content_financials = requests.get(financials_url,headers={'User-Agent': 'Mozilla/5.0'}).text
 
     soup = BeautifulSoup(html_content, "html.parser")
-    # print(soup)
+    # logging.info(soup)
     href = None
     a_tags = soup.find_all('a', class_='newslink')
     if a_tags is not None:
         links = [a_tag["href"]  for a_tag in a_tags]       
-    print(f"\nNews Links:\n{links}")
+    logging.info(f"\nNews Links:\n{links}")
 
     fin_soup = BeautifulSoup(html_content_financials, "html.parser")
     matches = re.findall(r'data-companyinsights="\[(.*?)\]"', str(fin_soup))
     company_insight = matches[0].replace(""","").replace("\\u20b", "").replace("parameter","Metric").replace("insight_color", "Trend").replace("insight_text", "Insight")
     all_news.append(company_insight)
-    print("All news insights obtained: ",all_news)
+    logging.info("All news insights obtained: ",all_news)
     return all_news, reports_url
 
 
 def get_google_news(ticker):
     url = f"https://www.bing.com/news/search?q=Latest+News+on+{ticker}&form=YFNR&filters=sortbydate%3a%221%22"
-    print(url)
+    logging.info(url)
     sequence=[]
     heading=[]
     content = requests.get(url)
@@ -333,7 +334,7 @@ def get_google_news(ticker):
 
     df = pd.DataFrame(heading, index=sequence, columns=["News-Headlines"])
     df.sort_index()
-    print(df.head(5))
+    logging.info(df.head(5))
     return df["News-Headlines"].head(5).to_numpy()
 
 
@@ -373,26 +374,26 @@ def scrape_webpage(url):
 def raw_news(raw_query, todays_news_func_call, ticker):
     ticker_stats_str = ''
     
-    print("Getting into get_the_ticker_stat()")
+    logging.info("Getting into get_the_ticker_stat()")
     ticker_stats = get_the_ticker_stat(ticker)
     
-    print("Getting into get_the_ticker_news()")
+    logging.info("Getting into get_the_ticker_news()")
     ticker_financials, reports = get_the_ticker_news(ticker)
     
-    print("Brokers report link", reports)
+    logging.info("Brokers report link", reports)
     for ticker_stat in ticker_stats:
         ticker_stats_str = ticker_stats_str + json.dumps(ticker_stat).replace("&", "'").replace("'", "'").replace(""", "'").replace("Link", "").replace("Heading", "").replace("Body", "").replace("Text", "").replace("{", "").replace("}", "")
     
-    print("All ticker stat: ", ticker_stats_str)
+    logging.info("All ticker stat: ", ticker_stats_str)
     
-    print("Getting into get_google_news()")
+    logging.info("Getting into get_google_news()")
     news_google = get_google_news(ticker)
     
-    print(f"Getting into swot analysis for {ticker}")
+    logging.info(f"Getting into swot analysis for {ticker}")
     swot_news_link = f'https://widgets.trendlyne.com/web-widget/swot-widget/Poppins/{urllib.parse.quote(ticker)}/'
-    print("SWOT Link: ", swot_news_link)
+    logging.info("SWOT Link: ", swot_news_link)
     news_data = scrape_webpage(swot_news_link)
-    print("SWOT data successfully scraped")
+    logging.info("SWOT data successfully scraped")
     
     return news_data, ticker_stats_str, ticker_financials, reports, news_google
 
@@ -427,13 +428,13 @@ def generate_function_call(prompt):
     stream = client.text_generation(content, **generate_kwargs, stream=True, details=True, return_full_text=True)
     for response in stream:
         output += response.token.text
-    print("Raw output from LLM: ", output)
-    print("Datatype: ", type(output))
+    logging.info("Raw output from LLM: ", output)
+    logging.info("Datatype: ", type(output))
     pattern = r'\{[^{}]*\}'
     # Find all matches
     match = re.search(pattern, output)
     extracted_string = match.group(0)
-    print("Extracted string from response: ", extracted_string)
+    logging.info("Extracted string from response: ", extracted_string)
     output = json.loads(extracted_string)
     return output
 
@@ -444,35 +445,35 @@ def generate_final_response(prompt, history, context_files=[], ticker_stats=[],
     #     temperature = 1e-2
     # top_p = float(top_p)
     global display_ticker
-    print("Current Value of GLOBAL TICKER: ",display_ticker)
+    logging.info("Current Value of GLOBAL TICKER: ",display_ticker)
     generate_kwargs = dict(temperature=0.001,max_new_tokens=1024,top_p=0.95,repetition_penalty=1.0,do_sample=True,seed=42)
     today = datetime.date.today()
     todays_date = today.strftime('%d%B%Y')
     question = format_prompt(prompt, history)
-    print("\n\nQuestion: ",question)
+    logging.info("\n\nQuestion: ",question)
 
     chat_completion_params = generate_function_call(question)
-    print(chat_completion_params)
+    logging.info(chat_completion_params)
     ticker = []
     stock_names = chat_completion_params["stock_name"]
-    print("Getting into get_the_ticker()")
+    logging.info("Getting into get_the_ticker()")
     ticker = get_the_ticker(stock_names)
-    print("Final Ticker: ", ticker)
+    logging.info("Final Ticker: ", ticker)
     
     try:
         if (chat_completion_params['todays_news_flag'] or chat_completion_params['generic_query']) and len(ticker)<1:
-            print("Getting Latest News Headlines")
+            logging.info("Getting Latest News Headlines")
             news_link.append(todays_news())
             time.sleep(2)
         elif chat_completion_params['todays_news_flag'] and len(ticker)>0:
             for tick in chat_completion_params["stock_name"]:
                 news_googles.append(f"Latest News for {tick}\n\n {get_google_news(tick)}")
         elif (chat_completion_params['follow_up_query'] and ticker_stats != []) or (display_ticker == ticker and ticker_stats != []):
-            print("\n\nAssigned into a followup query\n\n")
+            logging.info("\n\nAssigned into a followup query\n\n")
             chat_completion_params['follow_up_query'] = True
         else:
-            print("prompt & ticker: ", question, ticker )
-            print("Getting into raw_news()")
+            logging.info("prompt & ticker: ", question, ticker )
+            logging.info("Getting into raw_news()")
             for stock in ticker:
                 context_file, ticker_stat, ticker_financial, report, news_google = raw_news(raw_query=question, todays_news_func_call=chat_completion_params["todays_news_flag"], ticker=stock)
                 # Append each detail to its corresponding list
@@ -483,13 +484,13 @@ def generate_final_response(prompt, history, context_files=[], ticker_stats=[],
                 news_googles.append(news_google)
         
             
-        print(f"Generating response for **{question}**")
+        logging.info(f"Generating response for **{question}**")
         display_ticker = ticker if ticker else [0]
         env = Environment(loader=FileSystemLoader("templates/"), autoescape=True)
         # env.globals['include'] = lambda filename: env.loader.get_source(env, filename)[0]
         template = env.get_template("system_prompt.txt")
         content = template.render(todays_date=todays_date,ticker_financials=ticker_financials ,response_type="Response-1",chat_completion_params=chat_completion_params,context_file=context_files, question=question,ticker=ticker, ticker_stats = ticker_stats, reports=reports, news_link=news_link, earnings = earning_link, news_googles=news_googles)
-        # print(content)
+        # logging.info(content)
         output=""
         try:
             # Now start the streaming