Update utils.py

utils.py

@@ -341,19 +341,24 @@ def document_loading_splitting():
             "page": doc.metadata["page"],
             "path": doc.metadata["path"]
         }
-        preprocessed_docs.append(Document(content=preprocessed_content, title=preprocessed_metadata["title"], page=preprocessed_metadata["page"], path=preprocessed_metadata["path"]))
-
+        preprocessed_doc = Document(content=preprocessed_content, title=preprocessed_metadata["title"], page=preprocessed_metadata["page"], path=preprocessed_metadata["path"])
+        original_doc = Document(content=doc.page_content, title=doc.metadata["title"], page=doc.metadata["page"], path=doc.metadata["path"])
+        # Weise IDs zu
+        preprocessed_doc.id = str(uuid.uuid4())
+        original_doc.id = preprocessed_doc.id
+        preprocessed_docs.append(preprocessed_doc)
+        original_docs.append(original_doc)
     
     ################################
     # Document splitting
     text_splitter = RecursiveCharacterTextSplitter(chunk_size=1000, chunk_overlap=200)# RecursiveCharacterTextSplitter(chunk_overlap = 150, chunk_size = 1500)
     splits = text_splitter.split_documents(preprocessed_docs)
     # Split sowohl für originale als auch für vorverarbeitete Dokumente
-    original_splits = text_splitter.split_documents(docs)
+    original_splits = text_splitter.split_documents(original_docs)
     preprocessed_splits = text_splitter.split_documents(preprocessed_docs)
     
-    # Mapping von vorverarbeiteten Splits zu Originalsplits
-    split_to_original_mapping = {p_split: o_split for p_split, o_split in zip(preprocessed_splits, original_splits)}
+    # Mapping von vorverarbeiteten Splits zu Originalsplits anhand der IDs
+    split_to_original_mapping = {p_split.id: o_split for p_split, o_split in zip(preprocessed_splits, original_splits)}
 
 
     print("Splits...........................")