Spaces:
Sleeping
Sleeping
Nirav-Khanpara
commited on
Commit
Β·
df697c8
1
Parent(s):
b63ae73
Upload 2 files
Browse files- app.py +12 -7
- scanned_pdf_parser.py +10 -0
app.py
CHANGED
@@ -4,7 +4,7 @@ load_dotenv()
|
|
4 |
import os
|
5 |
import pickle
|
6 |
import streamlit as st
|
7 |
-
from
|
8 |
from langchain.embeddings import HuggingFaceInstructEmbeddings
|
9 |
from langchain.llms import GooglePalm
|
10 |
from langchain.prompts import PromptTemplate
|
@@ -16,7 +16,8 @@ from langchain.docstore.document import Document
|
|
16 |
|
17 |
llm = GooglePalm(temperature=0.9)
|
18 |
|
19 |
-
st.title("Query
|
|
|
20 |
|
21 |
uploaded_file = st.file_uploader("Choose a PDF file")
|
22 |
main_placeholder = st.empty()
|
@@ -24,8 +25,12 @@ second_placeholder = st.empty()
|
|
24 |
|
25 |
|
26 |
if uploaded_file:
|
27 |
-
|
28 |
-
|
|
|
|
|
|
|
|
|
29 |
with open(f'{uploaded_file.name}', 'wb') as f:
|
30 |
f.write(uploaded_file.getbuffer())
|
31 |
|
@@ -40,7 +45,7 @@ if uploaded_file:
|
|
40 |
main_placeholder.text("It looks like Scanned PDF, No worries converting it...βββ")
|
41 |
raw_text = get_text_from_scanned_pdf(uploaded_file.name)
|
42 |
|
43 |
-
main_placeholder.text("
|
44 |
text_splitter = RecursiveCharacterTextSplitter(
|
45 |
separators=['\n\n', '\n', '.', ','],
|
46 |
chunk_size=2000
|
@@ -50,14 +55,14 @@ if uploaded_file:
|
|
50 |
docs = [Document(page_content=t) for t in texts]
|
51 |
|
52 |
embeddings = HuggingFaceInstructEmbeddings(model_name="hkunlp/instructor-base")
|
53 |
-
main_placeholder.text("
|
54 |
vectorstore = FAISS.from_documents(docs, embeddings)
|
55 |
|
56 |
# Save the FAISS index to a pickle file
|
57 |
with open(f'vector_store_{uploaded_file.name}.pkl', "wb") as f:
|
58 |
pickle.dump(vectorstore, f)
|
59 |
|
60 |
-
main_placeholder.text("Data Loading
|
61 |
|
62 |
|
63 |
query = second_placeholder.text_input("Question:")
|
|
|
4 |
import os
|
5 |
import pickle
|
6 |
import streamlit as st
|
7 |
+
from scanned_pdf_parser import get_text_from_scanned_pdf
|
8 |
from langchain.embeddings import HuggingFaceInstructEmbeddings
|
9 |
from langchain.llms import GooglePalm
|
10 |
from langchain.prompts import PromptTemplate
|
|
|
16 |
|
17 |
llm = GooglePalm(temperature=0.9)
|
18 |
|
19 |
+
st.title("PDF Query Tool")
|
20 |
+
st.write("Upload your PDF and ask question from it")
|
21 |
|
22 |
uploaded_file = st.file_uploader("Choose a PDF file")
|
23 |
main_placeholder = st.empty()
|
|
|
25 |
|
26 |
|
27 |
if uploaded_file:
|
28 |
+
filename = uploaded_file.name
|
29 |
+
if not filename.endswith(('.pdf', '.PDF')):
|
30 |
+
main_placeholder.warning("Choose PDF Document !!!")
|
31 |
+
exit()
|
32 |
+
elif not os.path.exists(uploaded_file.name):
|
33 |
+
main_placeholder.text("Data Loading Started...βββ")
|
34 |
with open(f'{uploaded_file.name}', 'wb') as f:
|
35 |
f.write(uploaded_file.getbuffer())
|
36 |
|
|
|
45 |
main_placeholder.text("It looks like Scanned PDF, No worries converting it...βββ")
|
46 |
raw_text = get_text_from_scanned_pdf(uploaded_file.name)
|
47 |
|
48 |
+
main_placeholder.text("Splitting text into smaller chunks...βββ")
|
49 |
text_splitter = RecursiveCharacterTextSplitter(
|
50 |
separators=['\n\n', '\n', '.', ','],
|
51 |
chunk_size=2000
|
|
|
55 |
docs = [Document(page_content=t) for t in texts]
|
56 |
|
57 |
embeddings = HuggingFaceInstructEmbeddings(model_name="hkunlp/instructor-base")
|
58 |
+
main_placeholder.text("Storing data into Vector Database...βββ")
|
59 |
vectorstore = FAISS.from_documents(docs, embeddings)
|
60 |
|
61 |
# Save the FAISS index to a pickle file
|
62 |
with open(f'vector_store_{uploaded_file.name}.pkl', "wb") as f:
|
63 |
pickle.dump(vectorstore, f)
|
64 |
|
65 |
+
main_placeholder.text("Data Loading Completed...β
β
β
")
|
66 |
|
67 |
|
68 |
query = second_placeholder.text_input("Question:")
|
scanned_pdf_parser.py
ADDED
@@ -0,0 +1,10 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
import pytesseract
|
2 |
+
from pdf2image import convert_from_path
|
3 |
+
|
4 |
+
|
5 |
+
def get_text_from_scanned_pdf(pdf_path):
|
6 |
+
text = ''
|
7 |
+
images = convert_from_path(pdf_path)
|
8 |
+
for img in images:
|
9 |
+
text += pytesseract.image_to_string(img)
|
10 |
+
return text
|