cdupland
commited on
Commit
·
8a89e38
1
Parent(s):
68af04f
Add user-agent on WebBaseLoader
Browse files- RAG_PDF_WEB.py +3 -1
RAG_PDF_WEB.py
CHANGED
|
@@ -13,7 +13,9 @@ from langchain_community.document_loaders import WebBaseLoader
|
|
| 13 |
import os
|
| 14 |
|
| 15 |
def get_docs_from_website(urls):
|
| 16 |
-
loader = WebBaseLoader(urls
|
|
|
|
|
|
|
| 17 |
docs = loader.load()
|
| 18 |
return docs
|
| 19 |
|
|
|
|
| 13 |
import os
|
| 14 |
|
| 15 |
def get_docs_from_website(urls):
|
| 16 |
+
loader = WebBaseLoader(urls, header_template={
|
| 17 |
+
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/102.0.0.0 Safari/537.36',
|
| 18 |
+
})
|
| 19 |
docs = loader.load()
|
| 20 |
return docs
|
| 21 |
|