Update utils.py
Browse files
utils.py
CHANGED
@@ -396,8 +396,7 @@ def document_storage_chroma(splits):
|
|
396 |
return vectorstore
|
397 |
|
398 |
########################################################
|
399 |
-
#Vektorstore speichern - bzw. laden
|
400 |
-
"""
|
401 |
def save_splits_and_metadata(splits, directory="chroma/kkg", filename="splits_and_metadata.pkl"):
|
402 |
# Erstellen des Verzeichnisses, falls es nicht existiert
|
403 |
if not os.path.exists(directory):
|
@@ -420,7 +419,8 @@ def load_splits_and_metadata(directory="chroma/kkg", filename="splits_and_metada
|
|
420 |
return pickle.load(f)
|
421 |
return None
|
422 |
|
423 |
-
|
|
|
424 |
#Laden des Vektorstores - aus den gespeicherten splits
|
425 |
def load_vectorstore():
|
426 |
splits_and_metadata = load_splits_and_metadata()
|
@@ -428,8 +428,9 @@ def load_vectorstore():
|
|
428 |
PREPROCESSED_SPLITS, SPLIT_TO_ORIGINAL_MAPPING = splits_and_metadata
|
429 |
return document_storage_chroma(PREPROCESSED_SPLITS)
|
430 |
return None
|
431 |
-
"""
|
432 |
|
|
|
|
|
433 |
def save_split_to_original_mapping(mapping, directory="chroma/kkg", mapping_filename="mapping.pkl"):
|
434 |
# Erstellen des Verzeichnisses, falls es nicht existiert
|
435 |
if not os.path.exists(directory):
|
@@ -443,6 +444,7 @@ def save_split_to_original_mapping(mapping, directory="chroma/kkg", mapping_file
|
|
443 |
# Hochladen der Mapping-Datei zum Hugging Face Space
|
444 |
upload_file_to_huggingface(mapping_filepath, f"{directory}/{mapping_filename}")
|
445 |
|
|
|
446 |
def load_split_to_original_mapping(directory="chroma/kkg", mapping_filename="mapping.pkl"):
|
447 |
# Laden des Mappings
|
448 |
mapping_filepath = os.path.join(directory, mapping_filename)
|
@@ -452,6 +454,15 @@ def load_split_to_original_mapping(directory="chroma/kkg", mapping_filename="map
|
|
452 |
return pickle.load(f)
|
453 |
return None
|
454 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
455 |
def save_vectorstore(vectorstore, directory="chroma/kkg", splits_filename="splits_and_metadata.pkl", vectorstore_filename="vectorstore.pkl"):
|
456 |
# Erstellen des Verzeichnisses, falls es nicht existiert
|
457 |
if not os.path.exists(directory):
|
@@ -485,7 +496,7 @@ def load_vectorstore(directory="chroma/kkg", splits_filename="splits_and_metadat
|
|
485 |
vectorstore.documents = documents # Setze die Dokumente im Vektorstore
|
486 |
return vectorstore, SPLIT_TO_ORIGINAL_MAPPING
|
487 |
return None, None
|
488 |
-
|
489 |
|
490 |
|
491 |
|
|
|
396 |
return vectorstore
|
397 |
|
398 |
########################################################
|
399 |
+
#Splits für den Vektorstore speichern - bzw. laden
|
|
|
400 |
def save_splits_and_metadata(splits, directory="chroma/kkg", filename="splits_and_metadata.pkl"):
|
401 |
# Erstellen des Verzeichnisses, falls es nicht existiert
|
402 |
if not os.path.exists(directory):
|
|
|
419 |
return pickle.load(f)
|
420 |
return None
|
421 |
|
422 |
+
########################################################
|
423 |
+
#Vektorstore speichern - bzw. laden
|
424 |
#Laden des Vektorstores - aus den gespeicherten splits
|
425 |
def load_vectorstore():
|
426 |
splits_and_metadata = load_splits_and_metadata()
|
|
|
428 |
PREPROCESSED_SPLITS, SPLIT_TO_ORIGINAL_MAPPING = splits_and_metadata
|
429 |
return document_storage_chroma(PREPROCESSED_SPLITS)
|
430 |
return None
|
|
|
431 |
|
432 |
+
#################################
|
433 |
+
#das Mapping der orginal-Splits und der preprocessed Splits speichern
|
434 |
def save_split_to_original_mapping(mapping, directory="chroma/kkg", mapping_filename="mapping.pkl"):
|
435 |
# Erstellen des Verzeichnisses, falls es nicht existiert
|
436 |
if not os.path.exists(directory):
|
|
|
444 |
# Hochladen der Mapping-Datei zum Hugging Face Space
|
445 |
upload_file_to_huggingface(mapping_filepath, f"{directory}/{mapping_filename}")
|
446 |
|
447 |
+
|
448 |
def load_split_to_original_mapping(directory="chroma/kkg", mapping_filename="mapping.pkl"):
|
449 |
# Laden des Mappings
|
450 |
mapping_filepath = os.path.join(directory, mapping_filename)
|
|
|
454 |
return pickle.load(f)
|
455 |
return None
|
456 |
|
457 |
+
|
458 |
+
|
459 |
+
|
460 |
+
|
461 |
+
|
462 |
+
|
463 |
+
|
464 |
+
"""
|
465 |
+
|
466 |
def save_vectorstore(vectorstore, directory="chroma/kkg", splits_filename="splits_and_metadata.pkl", vectorstore_filename="vectorstore.pkl"):
|
467 |
# Erstellen des Verzeichnisses, falls es nicht existiert
|
468 |
if not os.path.exists(directory):
|
|
|
496 |
vectorstore.documents = documents # Setze die Dokumente im Vektorstore
|
497 |
return vectorstore, SPLIT_TO_ORIGINAL_MAPPING
|
498 |
return None, None
|
499 |
+
"""
|
500 |
|
501 |
|
502 |
|