Update utils.py
Browse files
utils.py
CHANGED
@@ -32,33 +32,27 @@ from langchain.chains import LLMChain, RetrievalQA
|
|
32 |
from langchain_community.document_loaders import PyPDFLoader, UnstructuredWordDocumentLoader, DirectoryLoader
|
33 |
#from langchain.document_loaders.blob_loaders.youtube_audio import YoutubeAudioLoader
|
34 |
#from langchain.document_loaders import GenericLoader
|
35 |
-
from langchain.schema import AIMessage, HumanMessage
|
36 |
-
from langchain_community.llms import HuggingFaceHub
|
37 |
-
from langchain_community.llms import HuggingFaceTextGenInference
|
38 |
#from langchain_community.embeddings import HuggingFaceInstructEmbeddings, HuggingFaceEmbeddings, HuggingFaceBgeEmbeddings, HuggingFaceInferenceAPIEmbeddings
|
39 |
from langchain_huggingface import HuggingFaceEmbeddings
|
40 |
-
from langchain_community.tools import DuckDuckGoSearchRun
|
41 |
from typing import Dict, TypedDict
|
42 |
from langchain_core.messages import BaseMessage
|
43 |
from langchain.prompts import PromptTemplate
|
44 |
-
|
45 |
-
|
46 |
-
#from langchain import hub
|
47 |
-
from langchain.prompts import PromptTemplate
|
48 |
-
from langchain.schema import Document
|
49 |
from langchain_community.vectorstores import Chroma
|
50 |
from langchain_core.messages import BaseMessage, FunctionMessage
|
51 |
from langchain_core.output_parsers import StrOutputParser
|
52 |
from langchain_core.pydantic_v1 import BaseModel, Field
|
53 |
from langchain_core.runnables import RunnablePassthrough
|
54 |
-
from langchain_core.utils.function_calling import convert_to_openai_tool
|
55 |
from langchain.text_splitter import RecursiveCharacterTextSplitter
|
56 |
-
from langchain_community.vectorstores import Chroma
|
57 |
from chromadb.errors import InvalidDimensionException
|
58 |
import io
|
59 |
-
from PIL import Image, ImageDraw, ImageOps, ImageFont
|
60 |
-
import base64
|
61 |
-
from tempfile import NamedTemporaryFile
|
62 |
|
63 |
import nltk
|
64 |
from nltk.corpus import stopwords
|
@@ -134,6 +128,7 @@ urls = [
|
|
134 |
##################################################
|
135 |
#Normalisierung eines Prompts
|
136 |
##################################################
|
|
|
137 |
def normalise_prompt (prompt):
|
138 |
#alles Kleinbuchstaben
|
139 |
prompt_klein =prompt.lower()
|
@@ -288,7 +283,7 @@ def rag_chainback(prompt, db, k=3):
|
|
288 |
|
289 |
|
290 |
###############################################
|
291 |
-
#Langchain anlegen
|
292 |
###############################################
|
293 |
#langchain nutzen, um prompt an LLM zu leiten - llm und prompt sind austauschbar
|
294 |
def llm_chain(llm, prompt):
|
@@ -360,7 +355,7 @@ def generate_prompt_with_history_hf(prompt, history):
|
|
360 |
|
361 |
|
362 |
##########################################
|
363 |
-
#Hashing....
|
364 |
# Funktion zum Hashen des Eingabewerts
|
365 |
def hash_input(input_string):
|
366 |
return hashlib.sha256(input_string.encode()).hexdigest()
|
|
|
32 |
from langchain_community.document_loaders import PyPDFLoader, UnstructuredWordDocumentLoader, DirectoryLoader
|
33 |
#from langchain.document_loaders.blob_loaders.youtube_audio import YoutubeAudioLoader
|
34 |
#from langchain.document_loaders import GenericLoader
|
35 |
+
from langchain.schema import AIMessage, HumanMessage, Document
|
36 |
+
#from langchain_community.llms import HuggingFaceHub
|
37 |
+
#from langchain_community.llms import HuggingFaceTextGenInference
|
38 |
#from langchain_community.embeddings import HuggingFaceInstructEmbeddings, HuggingFaceEmbeddings, HuggingFaceBgeEmbeddings, HuggingFaceInferenceAPIEmbeddings
|
39 |
from langchain_huggingface import HuggingFaceEmbeddings
|
40 |
+
#from langchain_community.tools import DuckDuckGoSearchRun
|
41 |
from typing import Dict, TypedDict
|
42 |
from langchain_core.messages import BaseMessage
|
43 |
from langchain.prompts import PromptTemplate
|
44 |
+
|
|
|
|
|
|
|
|
|
45 |
from langchain_community.vectorstores import Chroma
|
46 |
from langchain_core.messages import BaseMessage, FunctionMessage
|
47 |
from langchain_core.output_parsers import StrOutputParser
|
48 |
from langchain_core.pydantic_v1 import BaseModel, Field
|
49 |
from langchain_core.runnables import RunnablePassthrough
|
|
|
50 |
from langchain.text_splitter import RecursiveCharacterTextSplitter
|
|
|
51 |
from chromadb.errors import InvalidDimensionException
|
52 |
import io
|
53 |
+
#from PIL import Image, ImageDraw, ImageOps, ImageFont
|
54 |
+
#import base64
|
55 |
+
#from tempfile import NamedTemporaryFile
|
56 |
|
57 |
import nltk
|
58 |
from nltk.corpus import stopwords
|
|
|
128 |
##################################################
|
129 |
#Normalisierung eines Prompts
|
130 |
##################################################
|
131 |
+
#Zur zeit nicht im Gebrauch.............................
|
132 |
def normalise_prompt (prompt):
|
133 |
#alles Kleinbuchstaben
|
134 |
prompt_klein =prompt.lower()
|
|
|
283 |
|
284 |
|
285 |
###############################################
|
286 |
+
#Langchain anlegen für RAG Chaining
|
287 |
###############################################
|
288 |
#langchain nutzen, um prompt an LLM zu leiten - llm und prompt sind austauschbar
|
289 |
def llm_chain(llm, prompt):
|
|
|
355 |
|
356 |
|
357 |
##########################################
|
358 |
+
#Hashing.... Für die Validierung........
|
359 |
# Funktion zum Hashen des Eingabewerts
|
360 |
def hash_input(input_string):
|
361 |
return hashlib.sha256(input_string.encode()).hexdigest()
|