Update Arcana.py

Arcana.py

@@ -21,7 +21,7 @@ def process_text_into_paragraphs(text):
     text = re.sub(r'^\s*\d+\s*$', '', text, flags=re.MULTILINE)
     
     # Split text into paragraphs
-    paragraphs = re.split(r'\n{2,}', text)
+    paragraphs = list(set(re.split(r'\n{2,}', text)))
     
     # Clean up each paragraph
     cleaned_paragraphs = []
@@ -58,16 +58,17 @@ def process_pdfs(directory, db):
             pbar.set_postfix({"Current File": filename})
             print(filename)
 
-def main():
-    db_filename = 'memory.txt'
+def main(foldername):
+    db_filename = foldername
     
     if os.path.exists(db_filename):
+        db_filename += '.txt'
         print(f"Database file '{db_filename}' already exists. Rewriting existing database...")
         #db = ChatDatabase(db_filename)
         #else:
         print(f"Creating new database '{db_filename}'...")
         db = ChatDatabase(db_filename)
-        pdf_directory = 'cache'
+        pdf_directory = foldername
         
         start_time = time.time()
         process_pdfs(pdf_directory, db)