Upload 3 files

app.py

@@ -1,64 +1,306 @@
+# from fastapi import FastAPI
+# from fastapi.middleware.cors import CORSMiddleware
+from openai import OpenAI
+from google import genai
+from crawler import extract_data
+import time
+import os
+from dotenv import load_dotenv
 import gradio as gr
-from huggingface_hub import InferenceClient
-
-"""
-For more information on `huggingface_hub` Inference API support, please check the docs: https://huggingface.co/docs/huggingface_hub/v0.22.2/en/guides/inference
-"""
-client = InferenceClient("HuggingFaceH4/zephyr-7b-beta")
-
-
-def respond(
-    message,
-    history: list[tuple[str, str]],
-    system_message,
-    max_tokens,
-    temperature,
-    top_p,
-):
-    messages = [{"role": "system", "content": system_message}]
-
-    for val in history:
-        if val[0]:
-            messages.append({"role": "user", "content": val[0]})
-        if val[1]:
-            messages.append({"role": "assistant", "content": val[1]})
-
-    messages.append({"role": "user", "content": message})
-
-    response = ""
-
-    for message in client.chat_completion(
-        messages,
-        max_tokens=max_tokens,
-        stream=True,
-        temperature=temperature,
-        top_p=top_p,
-    ):
-        token = message.choices[0].delta.content
-
-        response += token
-        yield response
-
-
-"""
-For information on how to customize the ChatInterface, peruse the gradio docs: https://www.gradio.app/docs/chatinterface
-"""
-demo = gr.ChatInterface(
-    respond,
-    additional_inputs=[
-        gr.Textbox(value="You are a friendly Chatbot.", label="System message"),
-        gr.Slider(minimum=1, maximum=2048, value=512, step=1, label="Max new tokens"),
-        gr.Slider(minimum=0.1, maximum=4.0, value=0.7, step=0.1, label="Temperature"),
-        gr.Slider(
-            minimum=0.1,
-            maximum=1.0,
-            value=0.95,
-            step=0.05,
-            label="Top-p (nucleus sampling)",
-        ),
-    ],
+# import multiprocessing
+from together import Together
+
+load_dotenv("../.env")
+print("Environment variables:", os.environ)
+
+
+together_client = Together(
+    api_key=os.getenv("TOGETHER_API_KEY"),
 )
 
+gemini_client = genai.Client(api_key=os.getenv("GEMINI_API_KEY"))
+genai_model = "gemini-2.0-flash-exp"
+
+perplexity_client = OpenAI(api_key=os.getenv("PERPLEXITY_API_KEY"), base_url="https://api.perplexity.ai")
+gpt_client = OpenAI(api_key=os.getenv("OPENAI_API_KEY"))
+
+
+
+def get_answers( query: str ):
+    context = extract_data(query, 1)
+    return context
+
+# with torch.no_grad():
+#     model = AutoModel.from_pretrained('BM-K/KoSimCSE-roberta')
+#     tokenizer = AutoTokenizer.from_pretrained('BM-K/KoSimCSE-roberta', TOKENIZERS_PARALLELISM=True)
+
+# def cal_score(input_data):
+#     # Initialize model and tokenizer inside the function
+#     with torch.no_grad():
+#         inputs = tokenizer(input_data, padding=True, truncation=True, return_tensors="pt")
+#         outputs = model.get_input_embeddings(inputs["input_ids"])
+        
+#         a, b = outputs[0], outputs[1]  # Adjust based on your model's output structure
+
+#         # Normalize the tensors
+#         a_norm = a / a.norm(dim=1)[:, None]
+#         b_norm = b / b.norm(dim=1)[:, None]
+
+#         print(a.shape, b.shape)
+        
+#         # Return the similarity score
+#         # return torch.mm(a_norm, b_norm.transpose(0, 1)) * 100
+#         a_norm = a_norm.reshape(1, -1)
+#         b_norm = b_norm.reshape(1, -1)
+#         similarity_score = cosine_similarity(a_norm, b_norm)
+
+#         # Return the similarity score (assuming you want the average of the similarities across the tokens)
+#         return similarity_score # Scalar value
+
+
+
+# def get_match_scores( message: str, query: str, answers: list[dict[str, object]] ):
+#     start = time.time()
+#     max_processes = 4
+#     with multiprocessing.Pool(processes=max_processes) as pool:
+#         scores = pool.map(cal_score, [[answer['questionDetails'], message] for answer in answers])
+#     print(f"Time taken to compare: {time.time() - start} seconds")
+#     print("Scores: ", scores)
+#     return scores
+
+def get_naver_answers( message: str ):
+    print(">>> Starting naver extraction...")
+    print("Question: ", message)
+    naver_start_time = time.time()
+    response = gemini_client.models.generate_content(
+        model = genai_model,
+        contents=f"{message}\n 위의 내용을 짧은 제목으로 요약합니다. 제목만 보여주세요. 대답하지 마세요",
+    )
+    query = response.text
+    print( "Query: ", query)
+
+    context = get_answers( query )
+
+    sorted_answers = ['. '.join(answer['answers']) for answer in context]
+    naver_end_time = time.time()
+    print(f"Time taken to extract from Naver: { naver_end_time - naver_start_time } seconds")
+    document = '\n'.join(sorted_answers)
+    return document, naver_end_time - naver_start_time
+
+def get_qwen_big_answer( message: str ):
+    print(">>> Starting Qwen 72B extraction...")
+    qwen_start_time = time.time()
+    response = together_client.chat.completions.create(
+        model="Qwen/Qwen2.5-72B-Instruct-Turbo",
+        messages=[
+            {"role": "system", "content": "You are a helpful question-answer, CONCISE conversation assistant that answers in Korean."},
+            {"role": "user", "content": message}
+        ]
+    )
+
+    qwen_end_time = time.time()
+    print(f"Time taken to extract from Qwen: { qwen_end_time - qwen_start_time } seconds")
+    return response.choices[0].message.content, qwen_end_time - qwen_start_time
+
+def get_qwen_small_answer( message: str ):
+    print(">>> Starting Qwen 7B extraction...")
+    qwen_start_time = time.time()
+    response = together_client.chat.completions.create(
+        model="Qwen/Qwen2.5-7B-Instruct-Turbo",
+        messages=[
+            {"role": "system", "content": "You are a helpful question-answer, CONCISE conversation assistant that answers in Korean."},
+            {"role": "user", "content": message}
+        ]
+        #TODO: Change the messages option
+    )
+    qwen_end_time = time.time()
+    print(f"Time taken to extract from Qwen: { qwen_end_time - qwen_start_time } seconds")
+    return response.choices[0].message.content, qwen_end_time - qwen_start_time
+
+def get_llama_small_answer( message: str ):
+    print(">>> Starting Llama 3.1 8B extraction...")
+    llama_start_time = time.time()
+    response = together_client.chat.completions.create(
+        model="meta-llama/Meta-Llama-3.1-8B-Instruct-Turbo",
+        messages=[
+            {"role": "system", "content": "You are an artificial intelligence assistant and you need to engage in a helpful, CONCISE, polite question-answer conversation with a user."},
+            {
+                "role": "user",
+                "content": message
+            }
+        ]
+    )
+    llama_end_time = time.time()
+    print(f"Time taken to extract from Llama: { llama_end_time - llama_start_time } seconds")
+    return response.choices[0].message.content, llama_end_time - llama_start_time
+
+def get_llama_big_answer( message: str ):
+    print(">>> Starting Llama 3.1 70B extraction...")
+    llama_start_time = time.time()
+    response = together_client.chat.completions.create(
+        model="meta-llama/Meta-Llama-3.1-70B-Instruct-Turbo",
+        messages=[
+            {"role": "system", "content": "You are an artificial intelligence assistant and you need to engage in a helpful, CONCISE, polite question-answer conversation with a user."},
+            {
+                "role": "user",
+                "content": message
+            }
+        ]
+    )
+    llama_end_time = time.time()
+    print(f"Time taken to extract from Llama: { llama_end_time - llama_start_time } seconds")
+    return response.choices[0].message.content, llama_end_time - llama_start_time
+
+
+def get_gemini_answer( message: str ):
+    print(">>> Starting gemini extraction...")
+    gemini_start_time = time.time()
+    response = gemini_client.models.generate_content(
+        model = genai_model,
+        contents=message,
+    )
+    gemini_end_time = time.time()
+    print(f"Time taken to extract from Gemini: { gemini_end_time - gemini_start_time } seconds")
+    return response.candidates[0].content, gemini_end_time - gemini_start_time
+
+# def get_perplexity_answer( message: str ):
+#     print(">>> Starting perplexity extraction...")
+#     perplexity_start_time = time.time()
+#     messages = [
+#         {
+#             "role": "system",
+#             "content": (
+#                 "You are an artificial intelligence assistant and you need to "
+#                 "engage in a helpful, CONCISE, polite question-answer conversation with a user."
+#             ),
+#         },
+#         {   
+#             "role": "user",
+#             "content": (
+#                 message
+#             ),
+#         },
+#     ]
+#     response = perplexity_client.chat.completions.create(
+#         model="llama-3.1-sonar-small-128k-online",
+#         messages=messages
+#     )
+#     perplexity_end_time = time.time()
+#     print(f"Time taken to extract from Perplexity: { perplexity_end_time - perplexity_start_time } seconds")
+#     return response.choices[0].message.content, perplexity_end_time - perplexity_start_time
+
+def get_gpt_answer( message: str ):
+    print(">>> Starting GPT extraction...")
+    gpt_start_time = time.time()
+    completion = gpt_client.chat.completions.create(
+        model="gpt-4o-mini",
+        messages=[
+            {"role": "system", "content": "You are a helpful assistant that gives short answers and nothing extra."},
+            {
+                "role": "user",
+                "content": message
+            }
+        ]
+    )
+    gpt_end_time = time.time()
+    print(f"Time taken to extract from GPT: { gpt_end_time - gpt_start_time } seconds")
+    return completion.choices[0].message.content, gpt_end_time - gpt_start_time
+
+def compare_answers(message: str):
+    methods = [
+        ("Qwen Big (72B)", get_qwen_big_answer),
+        ("Qwen Small (7B)", get_qwen_small_answer),
+        ("Llama Small (8B)", get_llama_small_answer),
+        ("Llama Big (70B)", get_llama_big_answer),
+        ("Gemini-2.0-Flash", get_gemini_answer),
+        # ("Perplexity", get_perplexity_answer),
+        ("GPT (4o-mini)", get_gpt_answer)
+    ]
+
+    results = []
+
+    naver_docs, naver_time_taken = get_naver_answers( message )
+    content = f'아래 문서를 바탕으로 질문에 답하세요. 답변은 한국어로만 해주세요 \n 질문 {message}\n'
+    content += naver_docs
+    print("Starting the comparison between summarizers...")
+    for method_name, method in methods:
+        answer, time_taken = method(content)
+        results.append({
+            "Method": f"Naver + ({method_name})",
+            "Question": message,
+            "Answer": answer,
+            "Time Taken": naver_time_taken + time_taken
+        })
+
+    print("Starting the comparison between extractors/summarizers...")
+    for method_name, method in methods:
+        additional_docs, time_taken = method(message)
+        results.append({
+            "Method": method_name,
+            "Question": message,
+            "Answer": additional_docs,
+            "Time Taken": time_taken
+        })
+        content += f'\n{additional_docs}'
+        time_taken += naver_time_taken
+        for summarizer_name, summarizer in methods:
+            answer, answer_time = summarizer(content)
+            results.append({
+                "Method": f"Naver + {method_name} + ({summarizer_name})",
+                "Question": message,
+                "Answer": answer,
+                "Time Taken": time_taken + answer_time
+            })
+    return results
+
+def chatFunction( message, history ):
+    content = f'아래 문서를 바탕으로 질문에 답하세요. 답변에서 질문을 따라 출력 하지 마세요. 답변은 한국어로만 해주세요. 찾은 Naver 문서와 다른 문서에서 답변이 없는 내용은 절대 출력하지 마세요 \n 질문: {message}\n 문서: '
+    naver_docs, naver_time_taken = get_naver_answers( message )
+
+    start_time = time.time()
+    content += "\n Naver 문서: " + naver_docs
+
+    completion = gpt_client.chat.completions.create(
+        model="gpt-4o-mini",
+        messages=[
+            {"role": "system", "content": "You are a helpful assistant that answers only in korean."},
+            {
+                "role": "user",
+                "content": message
+            }
+        ]
+    )
+    gpt_resp = completion.choices[0].message.content
+    content += "\n 다른 문서: " + gpt_resp
+
+    # content += "\n" + gpt_resp
+
+    answer, _ = get_qwen_small_answer(content)
+
+    print("-"*70)
+    print("Question: ", message)
+    print("Answer: ", answer)
+    time_taken = time.time() - start_time
+    print("Time taken to summarize: ", time_taken)
+    return answer
+    
 
 if __name__ == "__main__":
-    demo.launch()
+    # multiprocessing.set_start_method("fork", force=True)
+    # if multiprocessing.get_start_method(allow_none=True) is None:
+    #     multiprocessing.set_start_method("fork")
+    with gr.ChatInterface( fn=chatFunction, type="messages" ) as demo: pass
+    demo.launch(share=True)
+    # with open("test_questions.txt", "r") as f:
+    #     if os.path.exists("comparison_results.csv"):
+    #         if input("Do you want to delete the former results? (y/n): ") == "y":
+    #             os.remove("comparison_results.csv")
+    #     questions = f.readlines()
+    #     print(questions)
+    #     for idx, question in enumerate(questions):
+    #         print(" -> Starting the question number: ", idx)
+    #         results = compare_answers(question)
+    #         df = pd.DataFrame(results)
+    #         df.to_csv("comparison_results.csv", mode='a', index=False)
+
+

crawler.py

@@ -0,0 +1,98 @@
+from bs4 import BeautifulSoup
+import re
+import requests as r
+from html2text import html2text
+import tqdm
+
+def process_url(url):
+    """Process a single URL to fetch answers."""
+    try:
+        response = r.get(url)
+        soup = BeautifulSoup(response.text, "html.parser")
+        # answers = []
+        # for idx in range(1, 100):
+        #     answer = soup.find('div', {'id': f'answer_{idx}'})
+        #     if answer:
+        #         answers.append(answer)
+        #     else:
+        #         break
+        answers = soup.find_all('div', {'id': re.compile(r'answer_\d+')})
+        answers = [html2text(str(answer.find('div', {'class': "answerDetail"}).prettify()))
+                   for answer in answers if answer.find('div', {'class': "answerDetail"})]
+        title = soup.find('div', {'class': 'endTitleSection'}).text.strip()
+        questionDetails = soup.find('div', {'class': 'questionDetail'}).text.strip()
+        # print("Question: ", questionDetails, '\n')
+        title = title.replace("질문", '').strip()
+        print("Answers extracted from: \n", url)
+        print(len(answers))
+        print('-'*60)
+        return {
+            "title": title,
+            "questionDetails": questionDetails,
+            "url": url,
+            "answers": answers
+        }
+    except Exception as e:
+        print(f"Error processing URL {url}: {e}")
+        with open('error_urls.txt', 'w') as f:
+            f.write(url + '\n')
+        return {"title": '', "questionDetails": '', "url": url, "answers": ''}
+
+def get_answers(results_a_elements, query):
+    """Fetch answers for all the extracted result links."""
+    if not results_a_elements:
+        print("No results found.")
+        return []
+
+    print("Result links extracted: ", len(results_a_elements))
+    
+    # Limit the number of parallel processes for better resource management
+    # max_processes = 4
+    
+    # with multiprocessing.Pool(processes=max_processes) as pool:
+    #     results = pool.map(process_url, results_a_elements)
+
+    results = []
+    for url in tqdm.tqdm(results_a_elements):
+        results.append(process_url(url))
+    return results
+
+def get_search_results(query, num_pages):
+    """Fetch search results for the given query from Naver 지식in."""
+    results = []
+    for page in range(1, num_pages + 1):
+        url = f"https://kin.naver.com/search/list.naver?query={query}&page={page}"
+        print("Starting the scraping process for:\n", url)
+
+        try:
+            response = r.get(url)
+            soup = BeautifulSoup(response.text, "html.parser")
+            results_a_elements = soup.find("ul", {"class": "basic1"}).find_all("a", {"class": "_searchListTitleAnchor"})
+            results_a_elements = [a.get('href') for a in results_a_elements if a.get("href")]
+            results += results_a_elements
+        except Exception as e:
+            print(f"Error while fetching search results: {e}")
+    return results
+
+def extract_data(query, num_pages=150) -> list[dict[str, object]]:
+    results_a_elements = get_search_results(query, num_pages)
+    answers = get_answers(results_a_elements, query)
+    print("Total answers collected:", len(answers))
+    return answers
+
+# if __name__ == "__main__":
+#     start = time.time()
+#     query = "장래희망, 인공지능 개발자/연구원, 파이썬, 중학생 수준, 파이썬 설치, 도서 추천"
+#     answers = process_query(query)
+#     print("Total answers collected:", len(answers))
+#     print("Time taken: ", time.time() - start)
+#     # print(answers)
+
+
+
+
+# AJAX URL: 
+# https://kin.naver.com/ajax/detail/answerList.naver?
+# dirId=401030201&docId=292159869
+# &answerSortType=DEFAULT&answerViewType=DETAIL
+# &answerNo=&page=2&count=5&_=1736131792605

requirements.txt

@@ -1 +1,19 @@
-huggingface_hub==0.25.2
+beautifulsoup4
+# selenium
+# webdriver-manager
+# fastapi[standard]
+# uvicorn[standard]
+html2text
+transformers
+openai
+google-genai
+# transformers[torch]
+# torch
+# torchvision
+# torchaudio
+gradio
+# scikit-learn
+together
+python-dotenv
+openpyxl
+tonic-validate