Create app.py

app.py

@@ -0,0 +1,296 @@
+import streamlit as st
+st.set_page_config(layout="wide")
+from annotated_text import annotated_text, annotation
+import fitz
+import os
+import chromadb
+import uuid
+from pathlib import Path
+import os
+os.environ['OPENAI_API_KEY'] = os.environ['OPEN_API_KEY']
+st.title("Contracts Multiple File Search ")
+import pandas as pd
+
+from langchain.retrievers import BM25Retriever, EnsembleRetriever
+from langchain.schema import Document
+from langchain.vectorstores import Chroma
+from langchain.embeddings import HuggingFaceEmbeddings
+embedding = HuggingFaceEmbeddings(model_name='BAAI/bge-base-en-v1.5')
+from FlagEmbedding import FlagReranker
+reranker = FlagReranker('BAAI/bge-reranker-base')
+import spacy
+# Load the English model from SpaCy
+nlp = spacy.load("en_core_web_md")
+
+def util_upload_file_and_return_list_docs(uploaded_files):
+    #util_del_cwd()
+    list_docs = []
+    list_save_path = []
+    for uploaded_file in uploaded_files:
+        save_path = Path(os.getcwd(), uploaded_file.name)
+        with open(save_path, mode='wb') as w:
+            w.write(uploaded_file.getvalue())
+        #print('save_path:', save_path)
+        docs = fitz.open(save_path) 
+        list_docs.append(docs)
+        list_save_path.append(save_path)
+    return(list_docs, list_save_path)
+#### Helper Functions to Split using Rolling Window (recomm : use smaller rolling window )
+def split_txt_file_synthetic_sentence_rolling(ctxt, sentence_size_in_chars, sliding_size_in_chars,debug=False):
+    sliding_size_in_chars = sentence_size_in_chars - sliding_size_in_chars
+    pos_start = 0
+    pos_end = len(ctxt)
+    final_return = []
+    if(debug):
+        print('pos_start : ',pos_start)
+        print('pos_end : ',pos_end)
+    if(pos_end<sentence_size_in_chars):
+        return([{'section_org_text':ctxt[pos_start:pos_end],'section_char_start':pos_start,'section_char_end':pos_end}])
+    if(sentence_size_in_chars<sliding_size_in_chars):
+        return(None)
+    stop_condition = False
+    start = pos_start
+    end = start + sentence_size_in_chars
+    mydict = {}
+    mydict['section_org_text'] = ctxt[start:end]
+    mydict['section_char_start'] = start
+    mydict['section_char_end'] = end
+    final_return.append(mydict)
+    #### First Time ENDS
+    while(stop_condition==False):
+        start = end - sliding_size_in_chars
+        end = start + sentence_size_in_chars
+        if(end>pos_end):
+            if(start<pos_end):
+                end = pos_end
+                mydict = {}
+                mydict['section_org_text'] = ctxt[start:end]
+                mydict['section_char_start'] = start
+                mydict['section_char_end'] = end
+                final_return.append(mydict)
+                stop_condition=True
+            else:
+                stop_condition=True
+        else:
+            mydict = {}
+            mydict['section_org_text'] = ctxt[start:end]
+            mydict['section_char_start'] = start
+            mydict['section_char_end'] = end
+            final_return.append(mydict)
+        if(debug):
+            print('start : ', start)
+            print('end : ', end)
+    return(final_return)
+### helper to make string out of iw_status
+# def util_get_list_page_and_passage(docs):
+#     page_documents = []
+#     passage_documents = []
+#     for txt_index, txt_page in enumerate(docs):
+#         page_document = txt_page.get_text()##.encode("utf8") # get plain text (is in UTF-8)
+#         page_documents.append(page_document)
+#         sections = split_txt_file_synthetic_sentence_rolling(page_document,700,200) 
+#         for sub_sub_index, sub_sub_item in enumerate(sections):
+#             sub_text=sub_sub_item['section_org_text']
+#             passage_document = Document(page_content=sub_text, metadata={"page_index": txt_index}) 
+#             passage_documents.append(passage_document) 
+#     return(page_documents,passage_documents)
+
+def split_into_sentences_with_offsets(text):
+    """
+    Splits a paragraph into sentences and returns them along with their start and end offsets.
+    :param text: The input text to be split into sentences.
+    :return: A list of tuples, each containing a sentence and its start and end offsets.
+    """
+    doc = nlp(text)
+    return [(sent.text, sent.start_char, sent.end_char) for sent in doc.sents]
+    
+def util_get_list_page_and_passage(list_docs, list_save_path):
+    #page_documents = []
+    passage_documents = []
+    for ind_doc, docs in enumerate(list_docs):
+        for txt_index, txt_page in enumerate(docs):
+            page_document = txt_page.get_text()##.encode("utf8") # get plain text (is in UTF-8)
+            #page_documents.append(page_document)
+            sections = split_into_sentences_with_offsets(page_document) 
+            for sub_sub_index, sub_sub_item in enumerate(sections):
+                sub_text=sub_sub_item[0]
+                passage_document = Document(page_content=sub_text, metadata={"page_content": page_document,"page_index": txt_index, "file_name" : str(list_save_path[ind_doc])}) 
+                passage_documents.append(passage_document) 
+    return(passage_documents)
+    
+# def util_index_chromadb_passages():
+#     ##### PROCESSING
+#     # create client and a new collection
+#     collection_name = str(uuid.uuid4().hex)
+#     chroma_client = chromadb.EphemeralClient()
+#     chroma_collection = chroma_client.get_or_create_collection(collection_name)
+#     # define embedding function
+#     embed_model = LangchainEmbedding(HuggingFaceEmbeddings(model_name="BAAI/bge-small-en"))
+#     vector_store = ChromaVectorStore(chroma_collection=chroma_collection)   
+#     return(chroma_client,chroma_collection,collection_name,vector_store,embed_model)
+
+def util_get_only_content_inside_loop(page_no,page_documents):
+    for index, item in enumerate(page_documents):
+        if(page_documents[index].metadata['txt_page_index']==page_no):
+            return(page_documents[index].get_content())  
+    return(None)
+# def util_get_list_pageno_and_contents(page_documents,passage_documents,passage_nodes):
+#     ''' page no starts with index 1 '''
+#     return_value = []
+#     for index, item in enumerate(passage_nodes):
+#         page_no = passage_nodes[index].metadata['txt_page_index']
+#         page_content = util_get_only_content_inside_loop(page_no,page_documents)
+#         return_value.append((page_no+1,page_content))
+#     return(return_value)
+
+def util_get_list_pageno_and_contents(some_query_passage,passage_documents,passage_nodes):
+    ''' page no starts with index 1 '''
+
+    return_value = []
+    rescore = reranker.compute_score([[some_query_passage , x.page_content] for x in passage_nodes])
+    print('rescore :: ',rescore)
+    tmp_array = []
+    for i, x in enumerate(passage_nodes):
+        tmp_dict = {"passage_content":x.page_content, 
+                    "page_no":int(x.metadata['page_index'])+1, 
+                    "page_content":str(x.metadata['page_content']), 
+                    "file_name": str(x.metadata['file_name']), 
+                    "score" : float(rescore[i])}
+        tmp_array.append(tmp_dict)
+    df = pd.DataFrame(tmp_array)
+    df = df.sort_values(by='score', ascending=False)
+    df = df.drop_duplicates(subset=['file_name'], keep='first')
+    df = df[["passage_content","file_name","page_no","page_content","score"]]
+    return(df)
+    
+# # def util_openai_extract_entity(example_passage, example_entity, page_content):
+# #     import openai
+# #     openai.api_key = os.environ['OPENAI_API_KEY']
+    
+# #     content = """Find the Entity based on Text . Return empty string if Entity does not exists. Here is one example below
+# #     Text: """ + example_passage + """
+# #     Entity: """ + example_entity + """
+    
+# #     Text: """ + page_content + """
+# #     Entity: """
+    
+# #     return_value = openai.ChatCompletion.create(model="gpt-4",temperature=0.0001,messages=[{"role": "user", "content": content},])
+# #     return(str(return_value['choices'][0]['message']['content']))
+def util_openai_extract_clause(example_prompt, page_content):
+    import openai
+    openai.api_key = os.environ['OPENAI_API_KEY']
+    content = example_prompt
+    content = content + "\n Answer precisely; do not add anything extra, and try to locate the answer in the below context   \n context: "
+    return_value = openai.ChatCompletion.create(model="gpt-3.5-turbo",temperature=0.0001,messages=[{"role": "user", "content": content + "\n" + page_content},])
+    return(str(return_value['choices'][0]['message']['content']))
+
+
+def util_openai_hyde(example_prompt):
+    import openai
+    openai.api_key = os.environ['OPENAI_API_KEY']
+    content = example_prompt
+    return_value = openai.ChatCompletion.create(model="gpt-3.5-turbo",temperature=0.0001,messages=[
+        {"role": "system", "content": "You are a legal contract lawyer. generate a summary from below text " + "\n"},
+        {"role": "user", "content": example_prompt + "\n"},
+    
+    ]
+                                               )
+    return(str(return_value['choices'][0]['message']['content']))
+
+    
+def util_openai_format (example_passage, page_content):
+    '''
+    annotated_text("  ",annotation("ENTITY : ", str(page_no)),)
+    '''
+    if(True):
+        found_value = util_openai_extract_clause(example_passage, page_content)
+        if(len(found_value)>0):
+            found_value = found_value.strip()
+            first_index = page_content.find(found_value)
+            if(first_index!=-1):
+                print('first_index : ',first_index)
+                print('found_value : ',found_value)
+                return(annotated_text(page_content[0:first_index-1],annotation(found_value, " FOUND ENTITY "),page_content[first_index+len(found_value):]))        
+    return(annotated_text(page_content))
+def util_openai_modify_prompt(example_prompt, page_content):
+    import openai
+    openai.api_key = os.environ['OPENAI_API_KEY']
+    my_prompt = """Expand the original Query to show exact resuls for extraction\n
+    Query: """ + example_prompt # + """\nDocument: """ + page_content + """ """
+    return_value = openai.ChatCompletion.create(model="gpt-4",temperature=0.0001,messages=[{"role": "user", "content": my_prompt},])
+    return(str(return_value['choices'][0]['message']['content']))    
+
+# def create_bm25_page_rank(page_list_retrieve, page_query):
+#     """ page_corpus : array of page text , page_query is user query """
+#     from operator import itemgetter
+#     from rank_bm25 import BM25Okapi
+#     tokenized_corpus = [doc.split(" ") for x, doc in page_list_retrieve]
+#     tokenized_query = page_query.split(" ")
+#     bm25 = BM25Okapi(tokenized_corpus)
+#     doc_scores = bm25.get_scores(tokenized_query).tolist()
+#     tmp_list = []
+#     for index, item in enumerate(page_list_retrieve):
+#         tmp_list.append((item[0], item[1],doc_scores[index]))
+#     tmp_list = sorted(tmp_list, key=itemgetter(2), reverse=True)
+#     return(tmp_list)
+    
+
+passage_documents = []
+
+if(True):
+    with st.form("my_form"):
+        multi = '''1. Download and Upload Multiple contracts (PDF)
+        
+        e.g. https://www.barc.gov.in/tenders/GCC-LPS.pdf
+        
+        e.g. https://www.montrosecounty.net/DocumentCenter/View/823/Sample-Construction-Contract
+        '''
+        st.markdown(multi)
+        multi = '''2. Insert Query to search or find similar language '''
+        st.markdown(multi)
+        multi = '''3. Press Index.'''
+        st.markdown(multi)
+        multi = '''
+        ** Attempt is made for appropriate page and passage retrieval ** \n
+        '''
+        st.markdown(multi)
+        #uploaded_file = st.file_uploader("Choose a file")  
+    
+        list_docs = []
+        list_save_path = []
+        uploaded_files = st.file_uploader("Choose file(s)", accept_multiple_files=True)
+        print('uploaded_files ', uploaded_files)
+        
+        
+        single_example_passage = st.text_area('Enter Query or similar passage Here and press Chat',"What is Governing Law?")
+        submitted = st.form_submit_button("Index and Answer")
+        
+        if submitted and (uploaded_files is not None):
+            list_docs, list_save_path = util_upload_file_and_return_list_docs(uploaded_files)
+            passage_documents = util_get_list_page_and_passage(list_docs, list_save_path)
+
+        
+        # st.button("Chat")
+        # if st.button('Chat'):
+            bm25_retriever = BM25Retriever.from_documents(passage_documents)
+            bm25_retriever.k = 2
+            chroma_vectorstore = Chroma.from_documents(passage_documents, embedding)
+            chroma_retriever = chroma_vectorstore.as_retriever(search_kwargs={"k": 2})
+            ensemble_retriever = EnsembleRetriever(retrievers=[bm25_retriever, chroma_retriever],weights=[0.25, 0.75])
+            passage_nodes = ensemble_retriever.get_relevant_documents(single_example_passage)
+            print('len(passage_nodes):', len(passage_nodes))
+            df = util_get_list_pageno_and_contents(single_example_passage,passage_documents,passage_nodes)
+            st.write(df)
+        # print('len(page_list_retrieve):', len(page_list_retrieve))
+        # if(len(page_list_retrieve)>0):
+        #     page_list_retrieve = list(set(page_list_retrieve))
+        #     for iindex in page_list_retrieve:
+        #         page_no = iindex[0]
+        #         page_content = iindex[1]
+        #         annotated_text("  ",annotation("RELEVANT PAGENO : ", str(page_no), font_family="Comic Sans MS", border="2px dashed red"),)
+        #         util_openai_format(single_example_passage, page_content)
+        #         annotated_text("  ",annotation("RELEVANT PASSAGE : ", "", font_family="Comic Sans MS", border="2px dashed red"),)
+        #         st.write(found_passage)
+        # pchroma_client = chromadb.Client()
+        # for citem in pchroma_client.list_collections():
+        #     print(citem.name)