#fix added headers that bypasses response code 418

google_search.py

@@ -122,6 +122,8 @@ def build_date(year=2024, month="March", day=1):
 async def get_url_data(url, client):
     try:
         r = await client.get(url, follow_redirects=True)
+        print(f"URL: {url}, Response Code: {r.status_code}")
+
         if r.status_code == 200:
             content_type = r.headers.get("Content-Type", "").lower()
             # detect if pdf
@@ -147,7 +149,10 @@ async def extract_pdf_text(content):
 
 
 async def parallel_scrap(urls):
-    async with httpx.AsyncClient(timeout=30) as client:
+    headers = {
+        "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.3"
+    }
+    async with httpx.AsyncClient(timeout=30, headers=headers) as client:
         tasks = []
         for url in urls:
             tasks.append(get_url_data(url=url, client=client))