decreased num max docs before retrieval

ai_generate.py

@@ -21,8 +21,6 @@ from langchain.prompts import ChatPromptTemplate
 import re
 import numpy as np
 import torch
-
-# pip install bm25s
 import bm25s
 from langchain_community.cross_encoders import HuggingFaceCrossEncoder
 from langchain.retrievers import ContextualCompressionRetriever
@@ -236,7 +234,7 @@ def create_db_with_langchain(path: list[str], url_content: dict, query: str):
     print(f"### Total number of documents before bm25s: {len(all_docs)}")
 
     # if the number of docs is too high, we need to reduce it
-    num_max_docs = 250
+    num_max_docs = 300
     if len(all_docs) > num_max_docs:
         docs_raw = [doc.page_content for doc in all_docs]
         retriever = bm25s.BM25(corpus=docs_raw)

requirements.txt

@@ -24,4 +24,5 @@ langchain-google-genai
 langchain-anthropic
 langchain-openai
 vertexai
-html2text
+html2text
+bm25s