chatgpt

.gitignore

@@ -1 +1,3 @@
-__pycache__
+__pycache__
+*.env
+*.toml

app.py

@@ -1,12 +1,25 @@
-from init_env import init_env
-from selenium_parser import load_driver
 import streamlit as st
+import os
+
+from init_env import init_env
+from selenium_parser import extract_url, load_driver, scrape_page
+from chatgpt import get_models, query
+
+
+try:
+    from dotenv import load_dotenv
+    load_dotenv(".env")
+    api_key = os.getenv("OPENAI_API_KEY")
+except ImportError:
+    api_key = None
 
 
 @st.cache_data
 def init():
-    # print("init")
-    init_env()
+    if api_key:
+        print("LOCAL TEST")
+    else:
+        init_env()
 
 
 @st.cache_resource
@@ -16,11 +29,51 @@ def get_driver():
 
 def run():
     driver = get_driver()
-    driver.get("https://www.booking.com/hotel/th/queen-boutique.ru.html#tab-reviews")
+    url_reviews = extract_url(url) if extract else url
+
+    try:
+        reviews = scrape_page(driver, url_reviews, page_count=page_count, wait_time=wait_time)
+    except Exception as e:
+        st.write(e)
+        st.write("Page loaded:")
+        st.image(driver.get_screenshot_as_png(), caption="screenshot")
+        raise e
+    
     st.write("Page loaded:")
     st.image(driver.get_screenshot_as_png(), caption="screenshot")
+    with st.expander("Reviews"):
+        st.json(reviews)
+    st.markdown(f"Collected {len(reviews)} reviews")
+
+    # ChatGPT answer
+    stream = query(api_key, reviews, model=selected_model)
+
+    st.divider()
+    st.markdown(f"**{selected_model}:**")
+
+    response = ""
+    message_placeholder = st.empty()
+    for chunk in stream:
+        text = chunk.choices[0].delta.content
+        if text is not None:
+            response += text
+            message_placeholder.markdown(response + "▌")
+    message_placeholder.markdown(response)
 
 
 init()
 
-st.button("Run", on_click=run)
+api_key = st.sidebar.text_input("Enter API key", value=api_key)
+url = st.text_input("Enter URL", value="https://www.booking.com/hotel/th/queen-boutique.ru.html#tab-reviews")
+extract = st.checkbox("Extract URL", value=True)
+
+if api_key:
+    models = get_models(api_key)
+    selected_model = st.sidebar.selectbox("Select model", models)
+
+st.sidebar.divider()
+page_count = st.sidebar.slider("Number of pages", 0, 10, value=5)
+wait_time = st.sidebar.number_input("Wait time (sec)", value=4)
+    
+if st.button("Run"):
+    run()

chatgpt.py

@@ -0,0 +1,17 @@
+from openai import OpenAI
+
+def get_models(api_key):
+    client = OpenAI(api_key=api_key)
+    models = client.models.list()
+    models = sorted([model.id for model in models.data if model.id.startswith("gpt")])
+    return models
+
+def query(api_key, reviews: list, model: str = "gpt-3.5-turbo"):
+    client = OpenAI(api_key=api_key)
+    message = str(reviews) + "\n\nAnalyze and provide a structured summary of both the positive and negative features of the hotel based on the above reviews. Quantify the frequency of each aspect mentioned in the reviews."
+    stream = client.chat.completions.create(
+        model=model,
+        messages=[{"role": "user", "content": message}],
+        stream=True,
+    )
+    return stream

requirements.txt

@@ -1,3 +1,4 @@
+streamlit  # 1.30.0
 selenium==4.17.2
 beautifulsoup4==4.11.1
-streamlit  # 1.30.0
+openai==1.10.0

selenium_parser.py

@@ -54,12 +54,11 @@ def parse_review(html):
         }
 
 
-def scrape_page(driver: webdriver.Firefox, url: str, page_count: int = 5, wait_time: int = 1):
+def scrape_page(driver: webdriver.Firefox, url: str, page_count: int = 5, wait_time: int = 3):
     # url = "https://www.booking.com/hotel/th/queen-boutique.ru.html#tab-reviews"
     review_infos = []
 
     driver.get(url)
-    print("page loaded")
 
     for i in range(page_count):
         WebDriverWait(driver, 10).until(EC.presence_of_element_located((By.CLASS_NAME, "review_list_new_item_block")))
@@ -83,4 +82,11 @@ def scrape_page(driver: webdriver.Firefox, url: str, page_count: int = 5, wait_t
         pagenext = driver.find_element(By.CLASS_NAME, "pagenext")
         pagenext.click()
     
-    return review_infos
+    return review_infos
+
+
+def extract_url(url: str):
+    if "?" in url:
+        url = url.split("?")[0]
+    url += "#tab-reviews"
+    return url