first

.env

@@ -0,0 +1,7 @@
+PERSIST_DIRECTORY=db
+MODEL_TYPE=GPT4All
+MODEL_PATH=models/ggml-gpt4all-j-v1.3-groovy.bin
+EMBEDDINGS_MODEL_NAME=all-MiniLM-L6-v2
+MODEL_N_CTX=1000
+MODEL_N_BATCH=8
+TARGET_SOURCE_CHUNKS=5

.gitattributes

@@ -0,0 +1,3 @@
+*.sqlite3 filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
+*.pickle filter=lfs diff=lfs merge=lfs -text

app.py

@@ -0,0 +1,105 @@
+from langchain import PromptTemplate, LLMChain
+from langchain.llms import CTransformers
+import os
+from langchain.text_splitter import RecursiveCharacterTextSplitter
+from langchain.vectorstores import Chroma
+from langchain.chains import RetrievalQA
+from langchain.embeddings import HuggingFaceBgeEmbeddings, HuggingFaceEmbeddings
+from io import BytesIO
+from langchain.document_loaders import PyPDFLoader
+import gradio as gr
+import chromadb
+from dotenv import load_dotenv
+from constants import CHROMA_SETTINGS
+from io import BytesIO
+import gradio as gr
+
+
+local_llm = "mistralai/Mistral-7B-v0.1"
+
+config = {
+'max_new_tokens': 1024,
+'repetition_penalty': 1.1,
+'temperature': 0.1,
+'top_k': 50,
+'top_p': 0.9,
+'stream': True,
+'threads': int(os.cpu_count() / 2)
+}
+
+
+llm = CTransformers(
+    model=local_llm,
+    model_type="mistral",
+    lib="avx2", #for CPU use
+    **config
+)
+
+embeddings_model_name = os.environ.get('EMBEDDINGS_MODEL_NAME')
+persist_directory = os.environ.get('PERSIST_DIRECTORY')
+target_source_chunks = int(os.environ.get('TARGET_SOURCE_CHUNKS',4))
+
+if not load_dotenv():
+    print("Could not load .env file or it is empty. Please check if it exists and is readable.")
+    exit(1)
+    
+    
+
+print("Loading embeddings model...")
+embeddings = HuggingFaceEmbeddings(model_name=embeddings_model_name)
+chroma_client = chromadb.PersistentClient(settings=CHROMA_SETTINGS , path=persist_directory)
+db = Chroma(persist_directory=persist_directory, embedding_function=embeddings, client_settings=CHROMA_SETTINGS, client=chroma_client)
+
+prompt_template = """Use the following pieces of information to answer the user's question.
+If you don't know the answer, just say that you don't know, don't try to make up an answer.
+
+Context: {context}
+Question: {question}
+
+Only return the helpful answer below and nothing else.
+Helpful answer:
+"""
+prompt = PromptTemplate(template=prompt_template, input_variables=['context', 'question'])
+retriever = db.as_retriever(search_kwargs={"k": target_source_chunks})
+# activate/deactivate the streaming StdOut callback for LLMs
+'''
+query="What is state ownership report"
+semantic_search_results = retriever.get_relevant_documents(query)
+print(semantic_search_results)
+query="What is state ownership report"
+chain_type_kwargs = {"prompt": prompt}
+qa = RetrievalQA.from_chain_type(llm=llm, chain_type="stuff", retriever=retriever, return_source_documents= False, chain_type_kwargs=chain_type_kwargs, verbose=True)
+response= qa(query)
+print(response)'''
+
+chain_type_kwargs = {"prompt": prompt}
+
+input_gradio= gr.Text(
+    label="Prompt",
+    show_label=False,
+    max_lines=2,
+    placeholder="Enter your question here",
+    container=False,
+    
+)
+
+
+def get_response(input_gradio ):
+    query=input_gradio
+    qa = RetrievalQA.from_chain_type(llm=llm, chain_type="stuff", retriever=retriever, return_source_documents= False, chain_type_kwargs=chain_type_kwargs, verbose=True)
+    response= qa(query)
+    return response['result']
+
+iface= gr.Interface(
+    fn=get_response,
+    inputs=input_gradio,
+    outputs="text",
+    title="Stimline Chatbot",
+    description="A chatbot that uses the LLM to answer anything regarding Stimline",
+    allow_flagging='never'
+    
+)
+    # Interactive questions and answers
+iface.launch()
+
+

app_v2.py

@@ -0,0 +1,118 @@
+from langchain import PromptTemplate, LLMChain
+from langchain.llms import CTransformers
+import os
+from langchain.text_splitter import RecursiveCharacterTextSplitter
+from langchain.vectorstores import Chroma
+from langchain.chains import RetrievalQA
+from langchain.embeddings import HuggingFaceBgeEmbeddings, HuggingFaceEmbeddings
+from io import BytesIO
+from langchain.document_loaders import PyPDFLoader
+import gradio as gr
+import chromadb
+from dotenv import load_dotenv
+from constants import CHROMA_SETTINGS
+from io import BytesIO
+import gradio as gr
+from langchain.chains import ConversationalRetrievalChain
+from langchain.memory import ConversationBufferMemory
+
+
+local_llm = "TheBloke/zephyr-7B-beta-GGUF"
+
+config = {
+'max_new_tokens': 1024,
+'repetition_penalty': 1.1,
+'temperature': 0.1,
+'top_k': 50,
+'top_p': 0.9,
+'stream': True,
+'threads': int(os.cpu_count() / 2)
+}
+
+
+llm = CTransformers(
+    model=local_llm,
+    model_type="mistral",
+    lib="avx2", #for CPU use
+    **config
+)
+
+embeddings_model_name = os.environ.get('EMBEDDINGS_MODEL_NAME')
+persist_directory = os.environ.get('PERSIST_DIRECTORY')
+target_source_chunks = int(os.environ.get('TARGET_SOURCE_CHUNKS',4))
+
+if not load_dotenv():
+    print("Could not load .env file or it is empty. Please check if it exists and is readable.")
+    exit(1)
+    
+    
+
+print("Loading embeddings model...")
+embeddings = HuggingFaceEmbeddings(model_name=embeddings_model_name)
+chroma_client = chromadb.PersistentClient(settings=CHROMA_SETTINGS , path=persist_directory)
+db = Chroma(persist_directory=persist_directory, embedding_function=embeddings, client_settings=CHROMA_SETTINGS, client=chroma_client)
+
+prompt_template = """Use the following pieces of information to answer the user's question.
+If you don't know the answer, just say that you don't know, don't try to make up an answer.
+
+Context: {context}
+Question: {question}
+
+Only return the helpful answer below and nothing else.
+Helpful answer:
+"""
+prompt = PromptTemplate(template=prompt_template, input_variables=['context', 'question'])
+retriever = db.as_retriever(search_kwargs={"k": target_source_chunks})
+
+memory= ConversationBufferMemory(
+    memory_key="chat_history",
+    return_messages=False
+)
+
+# activate/deactivate the streaming StdOut callback for LLMs
+'''
+query="What is state ownership report"
+semantic_search_results = retriever.get_relevant_documents(query)
+print(semantic_search_results)
+query="What is state ownership report"
+chain_type_kwargs = {"prompt": prompt}
+qa = RetrievalQA.from_chain_type(llm=llm, chain_type="stuff", retriever=retriever, return_source_documents= False, chain_type_kwargs=chain_type_kwargs, verbose=True)
+response= qa(query)
+print(response)'''
+
+chain_type_kwargs = {"prompt": prompt}
+
+input_gradio= gr.Text(
+    label="Prompt",
+    show_label=False,
+    max_lines=2,
+    placeholder="Enter your question here",
+    container=False,
+    
+)
+
+
+def get_response(input_gradio ):
+    query=input_gradio
+    qa= ConversationalRetrievalChain.from_llm(
+        llm=llm, 
+        chain_type="stuff",
+        retriever=retriever, 
+        memory=memory,
+        verbose=True)
+    response= qa(query)
+    return response['answer']
+
+iface= gr.Interface(
+    fn=get_response,
+    inputs=input_gradio,
+    outputs="text",
+    title="Stimline Chatbot",
+    description="A chatbot that uses the LLM to answer anything regarding Stimline",
+    allow_flagging='never'
+    
+)
+    # Interactive questions and answers
+iface.launch()
+
+

constants.py

@@ -0,0 +1,14 @@
+import os
+from dotenv import load_dotenv
+from chromadb.config import Settings
+
+load_dotenv()
+
+PERSIST_DIRECTORY = os.environ.get('PERSIST_DIRECTORY')
+if PERSIST_DIRECTORY is None:
+    raise Exception("please put the directory path in chroma db")
+
+CHROMA_SETTINGS = Settings(
+        persist_directory=PERSIST_DIRECTORY,
+        anonymized_telemetry=False
+)

db/687b62b3-4364-499d-9927-b084c01b0b0b/data_level0.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6e4322b89a3b8555e5d66d0504381d6435056f578e55e68df672987d35657745
+size 3352000

db/687b62b3-4364-499d-9927-b084c01b0b0b/header.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:dc0a39849941cfdf9e77f39fd384516fcd6a5a66278336eea1d145b1934f5777
+size 100

db/687b62b3-4364-499d-9927-b084c01b0b0b/index_metadata.pickle

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:42e803042f98dd389cac8f0b7599312cc5c45bb6619962fc9b58792cdcad353d
+size 113989

db/687b62b3-4364-499d-9927-b084c01b0b0b/length.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5ca72c4a6b3756f03937fc5c776e895b57eba80b5296dab003dc9874a5fc4c96
+size 8000

db/687b62b3-4364-499d-9927-b084c01b0b0b/link_lists.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0777500aa1b9b54c7d1f523eb7ffc49120fcd0d00ad9ea117d349e3303fadb59
+size 18268

db/chroma.sqlite3

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:24fbf72e0acbdd7bc265370c3ae32a78ab58bbae0ba628e02755170019d1e14a
+size 18460672

ingest.py

@@ -0,0 +1,147 @@
+#!/usr/bin/env python3
+import os
+import glob
+from typing import List
+from dotenv import load_dotenv
+from multiprocessing import Pool
+from tqdm import tqdm
+
+from langchain.document_loaders import (
+    CSVLoader,
+    EverNoteLoader,
+    PyMuPDFLoader,
+    TextLoader,
+    UnstructuredEmailLoader,
+    UnstructuredEPubLoader,
+    UnstructuredHTMLLoader,
+    UnstructuredMarkdownLoader,
+    UnstructuredODTLoader,
+    UnstructuredPowerPointLoader,
+    UnstructuredWordDocumentLoader,
+    PyPDFLoader
+)
+
+from langchain.text_splitter import RecursiveCharacterTextSplitter
+from langchain.vectorstores import Chroma
+from langchain.embeddings import HuggingFaceEmbeddings
+from langchain.docstore.document import Document
+
+if not load_dotenv():
+    print("Could not load .env file or it is empty. Please check if it exists and is readable.")
+    exit(1)
+
+from constants import CHROMA_SETTINGS
+import chromadb
+
+# Load environment variables
+persist_directory = os.environ.get('PERSIST_DIRECTORY')
+source_directory = os.environ.get('SOURCE_DIRECTORY', 'source_documents')
+embeddings_model_name = os.environ.get('EMBEDDINGS_MODEL_NAME')
+chunk_size = 500
+chunk_overlap = 100
+
+# Map file extensions to document loaders and their arguments
+LOADER_MAPPING = {
+    ".csv": (CSVLoader, {}),
+    # ".docx": (Docx2txtLoader, {}),
+    ".doc": (UnstructuredWordDocumentLoader, {}),
+    ".docx": (UnstructuredWordDocumentLoader, {}),
+    ".enex": (EverNoteLoader, {}),
+    # ".eml": (MyElmLoader, {}),
+    ".epub": (UnstructuredEPubLoader, {}),
+    ".html": (UnstructuredHTMLLoader, {}),
+    ".md": (UnstructuredMarkdownLoader, {}),
+    ".odt": (UnstructuredODTLoader, {}),
+    # ".pdf": (PyMuPDFLoader, {}),
+    ".pdf": (PyPDFLoader, {}),
+    ".ppt": (UnstructuredPowerPointLoader, {}),
+    ".pptx": (UnstructuredPowerPointLoader, {}),
+    ".txt": (TextLoader, {"encoding": "utf8"}),
+    # Add more mappings for other file extensions and loaders as needed
+}
+
+
+def load_single_document(file_path: str) -> List[Document]:
+    ext = "." + file_path.rsplit(".", 1)[-1].lower()
+    if ext in LOADER_MAPPING:
+        loader_class, loader_args = LOADER_MAPPING[ext]
+        loader = loader_class(file_path, **loader_args)
+        return loader.load()
+
+    raise ValueError(f"Unsupported file extension '{ext}'")
+
+def load_documents(source_dir: str, ignored_files: List[str] = []) -> List[Document]:
+    """
+    Loads all documents from the source documents directory, ignoring specified files
+    """
+    all_files = []
+    for ext in LOADER_MAPPING:
+        all_files.extend(
+            glob.glob(os.path.join(source_dir, f"**/*{ext.lower()}"), recursive=True)
+        )
+        all_files.extend(
+            glob.glob(os.path.join(source_dir, f"**/*{ext.upper()}"), recursive=True)
+        )
+    filtered_files = [file_path for file_path in all_files if file_path not in ignored_files]
+
+    with Pool(processes=os.cpu_count()) as pool:
+        results = []
+        with tqdm(total=len(filtered_files), desc='Loading new documents', ncols=80) as pbar:
+            for i, docs in enumerate(pool.imap_unordered(load_single_document, filtered_files)):
+                results.extend(docs)
+                pbar.update()
+
+    return results
+
+def process_documents(ignored_files: List[str] = []) -> List[Document]:
+    """
+    Load documents and split in chunks
+    """
+    print(f"Loading documents from {source_directory}")
+    documents = load_documents(source_directory, ignored_files)
+    if not documents:
+        print("No new documents to load")
+        exit(0)
+    print(f"Loaded {len(documents)} new documents from {source_directory}")
+    text_splitter = RecursiveCharacterTextSplitter(chunk_size=chunk_size, chunk_overlap=chunk_overlap)
+    texts = text_splitter.split_documents(documents)
+    print(f"Split into {len(texts)} chunks of text (max. {chunk_size} tokens each)")
+    return texts
+
+def does_vectorstore_exist(persist_directory: str, embeddings: HuggingFaceEmbeddings) -> bool:
+    """
+    Checks if vectorstore exists
+    """
+    db = Chroma(persist_directory=persist_directory, embedding_function=embeddings)
+    if not db.get()['documents']:
+        return False
+    return True
+
+def main():
+    # Create embeddings
+    embeddings = HuggingFaceEmbeddings(model_name=embeddings_model_name)
+    # Chroma client
+    chroma_client = chromadb.PersistentClient(settings=CHROMA_SETTINGS , path=persist_directory)
+
+    if does_vectorstore_exist(persist_directory, embeddings):
+        # Update and store locally vectorstore
+        print(f"Appending to existing vectorstore at {persist_directory}")
+        db = Chroma(persist_directory=persist_directory, embedding_function=embeddings, client_settings=CHROMA_SETTINGS, client=chroma_client)
+        collection = db.get()
+        texts = process_documents([metadata['source'] for metadata in collection['metadatas']])
+        print(f"Creating embeddings. May take some minutes...")
+        db.add_documents(texts)
+    else:
+        # Create and store locally vectorstore
+        print("Creating new vectorstore")
+        texts = process_documents()
+        print(f"Creating embeddings. May take some minutes...")
+        db = Chroma.from_documents(texts, embeddings, persist_directory=persist_directory, client_settings=CHROMA_SETTINGS, client=chroma_client)
+    db.persist()
+    db = None
+
+    print(f"Ingestion complete! You can now run app.py to query your documents")
+
+
+if __name__ == "__main__":
+    main()

requirements.txt

@@ -0,0 +1,15 @@
+langchain==0.0.274
+gradio==3.50.2
+gpt4all==1.0.8
+chromadb==0.4.7
+urllib3==2.0.4
+PyMuPDF==1.23.1
+python-dotenv==1.0.0
+unstructured==0.10.8
+extract-msg==0.45.0
+tabulate==0.9.0
+pandoc==2.3
+pypandoc==1.11
+tqdm==4.66.1
+sentence_transformers==2.2.2
+pypdf