Update app.py

app.py

@@ -20,6 +20,8 @@ import pandas as pd
 import concurrent.futures
 from duckduckgo_search import DDGS
 from supabase import create_client, Client
+from requests_html import HTMLSession
+
 
 # Required for saving the query & response in DB
 db_url: str = os.environ.get("SUPABASE_URL")
@@ -226,7 +228,6 @@ def get_the_ticker(stock_name):
 @time_logger
 async def get_the_ticker_stat(stock):
     try:
-        print(stock)
         combination=[]
         url = f'https://www.google.com/search?q={urllib.parse.quote(stock)}+site:businesstoday.in/stocks/&num=1&sca_esv=28795b6719ac1a08&sxsrf=ACQVn08xDA1EP1V6hJ-q4jLjjXSWWxgHTw:1711450545062&source=lnt&tbs=li:1&sa=X&ved=2ahUKEwj426eO4pGFAxX4n2MGHRXqBTUQpwV6BAgBEBM&biw=1280&bih=567&dpr=1.5'
         # logging.info("getting ticker stat url from: %s", url)
@@ -240,9 +241,11 @@ async def get_the_ticker_stat(stock):
         url = (links[0].split("&"))[0]
         # logging.info("Final URL to fetch stats %s" , url)
         # Fetch the HTML content of the webpage
-        html_content = requests.get(url,headers={'User-Agent': 'AppleWebKit/537.36'}).text
+        session = HTMLSession()
+        response = session.get(url)
+        response.html.render(sleep=1)
+        html_content = response.html.html
         soup = BeautifulSoup(html_content, "html.parser")
-        print(soup)
         script = soup.find("script", type="application/ld+json")
     
         # Parse the JSON-LD script