make g search markable, add role/occupation, switch to domain to include

app.py

@@ -219,11 +219,9 @@ def ai_check(text: str, option: str):
 
 
 def generate_prompt(settings: Dict[str, str]) -> str:
-    content_string = "\n".join(
-        f"{url.strip()}: \n{content.strip()[:2000]}" for url, content in settings["sources"].items()
-    )
-
     prompt = f"""
+    I am a {settings['role']}
+
     Write a {settings['article_length']} words (around) {settings['format']} on {settings['topic']}.
     
     Style and Tone:
@@ -244,9 +242,7 @@ def generate_prompt(settings: Dict[str, str]) -> str:
     - End with a {settings['conclusion_type']} conclusion
     - Add a "References" section at the end with at least 3 credible sources, formatted as [1], [2], etc.
     - Do not make any headline, title bold.
-
-    Use the content here from the URLs I've found for you:
-    {content_string}
+    {settings['sources']}
     
     Ensure proper paragraph breaks for better readability.
     Avoid any references to artificial intelligence, language models, or the fact that this is generated by an AI, and do not mention something like here is the article etc.
@@ -255,11 +251,9 @@ def generate_prompt(settings: Dict[str, str]) -> str:
 
 
 def regenerate_prompt(settings: Dict[str, str]) -> str:
-    content_string = "\n".join(
-        f"{url.strip()}: \n{content.strip()[:2000]}" for url, content in settings["sources"].items()
-    )
-
     prompt = f"""
+    I am a {settings['role']}
+
     "{settings['generated_article']}"
 
     Edit the given text based on user comments.
@@ -269,8 +263,7 @@ def regenerate_prompt(settings: Dict[str, str]) -> str:
     - The original content should not be changed. Make minor modifications based on user comments above.
     - Keep the references the same as the given text in the same format.
     - Do not make any headline, title bold.
-    Use the content here from the URLs I've found for you:
-    {content_string}
+    {settings['sources']}
     
     Ensure proper paragraph breaks for better readability.
     Avoid any references to artificial intelligence, language models, or the fact that this is generated by an AI, and do not mention something like here is the article etc.
@@ -279,6 +272,7 @@ def regenerate_prompt(settings: Dict[str, str]) -> str:
 
 
 def generate_article(
+    input_role: str,
     topic: str,
     keywords: str,
     article_length: str,
@@ -292,15 +286,13 @@ def generate_article(
     num_examples: str,
     conclusion_type: str,
     ai_model: str,
-    sorted_date,
-    domains_to_skip,
+    content_string: str,
     api_key: str = None,
     generated_article: str = None,
     user_comments: str = None,
 ) -> str:
-
-    url_content = google_search(topic, sorted_date, domains_to_skip)
     settings = {
+        "role": input_role,
         "topic": topic,
         "keywords": [k.strip() for k in keywords.split(",")],
         "article_length": article_length,
@@ -313,7 +305,7 @@ def generate_article(
         "references": [r.strip() for r in references.split(",")],
         "num_examples": num_examples,
         "conclusion_type": conclusion_type,
-        "sources": url_content,
+        "sources": content_string,
         "generated_article": generated_article,
         "user_comments": user_comments,
     }
@@ -379,7 +371,11 @@ def format_references(text: str) -> str:
     in_references = False
 
     for line in lines:
-        if line.strip().lower() == "references" or line.strip().lower() == "references:":
+        if (
+            line.strip().lower() == "references"
+            or line.strip().lower() == "references:"
+            or line.strip().lower().startswith("references:")
+        ):
             in_references = True
             continue
         if in_references:
@@ -396,6 +392,7 @@ def format_references(text: str) -> str:
 
 
 def generate_and_format(
+    input_role,
     topic,
     keywords,
     article_length,
@@ -410,20 +407,29 @@ def generate_and_format(
     conclusion_type,
     ai_model,
     api_key,
+    google_search_check,
     year_from,
     month_from,
     day_from,
     year_to,
     month_to,
     day_to,
-    domains_to_skip,
+    domains_to_include,
     generated_article: str = None,
     user_comments: str = None,
 ):
     date_from = build_date(year_from, month_from, day_from)
     date_to = build_date(year_to, month_to, day_to)
     sorted_date = f"date:r:{date_from}:{date_to}"
+    content_string = ""
+    if google_search_check:
+        url_content = google_search(topic, sorted_date, domains_to_include)
+        content_string = "\n".join(
+            f"{url.strip()}: \n{content.strip()[:2000]}" for url, content in url_content.items()
+        )
+        content_string = "Use the trusted information here from the URLs I've found for you:\n" + content_string
     article = generate_article(
+        input_role,
         topic,
         keywords,
         article_length,
@@ -437,9 +443,8 @@ def generate_and_format(
         num_examples,
         conclusion_type,
         ai_model,
+        content_string,
         api_key,
-        sorted_date,
-        domains_to_skip,
         generated_article,
         user_comments,
     )
@@ -465,6 +470,7 @@ def create_interface():
             with gr.Column(scale=2):
                 with gr.Group():
                     gr.Markdown("## Article Configuration", elem_classes="text-xl mb-4")
+                    input_role = gr.Textbox(label="I am a", placeholder="Enter your role", value="Student")
                     input_topic = gr.Textbox(
                         label="Topic",
                         placeholder="Enter the main topic of your article",
@@ -585,6 +591,10 @@ def create_interface():
                     )
                     gr.Markdown("# Search Options", elem_classes="text-center text-3xl mb-6")
                     with gr.Group():
+                        with gr.Row():
+                            google_search_check = gr.Checkbox(
+                                label="Enable Google Search For Recent Sources", value=True
+                            )
                         with gr.Row():
                             month_from = gr.Dropdown(
                                 choices=months,
@@ -606,10 +616,11 @@ def create_interface():
                             year_to = gr.Textbox(label="To Year", value=d1[2])
 
                         with gr.Row():
-                            domains_to_skip = gr.Dropdown(
+                            domains_to_include = gr.Dropdown(
                                 domain_list,
+                                value=domain_list,
                                 multiselect=True,
-                                label="Domain To Skip",
+                                label="Domains To Include",
                             )
 
                 with gr.Group():
@@ -691,6 +702,7 @@ def create_interface():
         generate_btn.click(
             fn=generate_and_format,
             inputs=[
+                input_role,
                 input_topic,
                 input_keywords,
                 input_length,
@@ -705,13 +717,14 @@ def create_interface():
                 input_conclusion,
                 ai_generator,
                 input_api,
+                google_search_check,
                 year_from,
                 month_from,
                 day_from,
                 year_to,
                 month_to,
                 day_to,
-                domains_to_skip,
+                domains_to_include,
             ],
             outputs=[output_article],
         )
@@ -719,6 +732,7 @@ def create_interface():
         regenerate_btn.click(
             fn=generate_and_format,
             inputs=[
+                input_role,
                 input_topic,
                 input_keywords,
                 input_length,
@@ -733,13 +747,14 @@ def create_interface():
                 input_conclusion,
                 ai_generator,
                 input_api,
+                google_search_check,
                 year_from,
                 month_from,
                 day_from,
                 year_to,
                 month_to,
                 day_to,
-                domains_to_skip,
+                domains_to_include,
                 output_article,
                 ai_comments,
             ],

plagiarism.py

@@ -61,10 +61,18 @@ async def parallel_scrap(urls):
     return results
 
 
+def scrap(urls):
+    client = httpx.Client()
+    soups = []
+    for url in urls:
+        soups.append(get_url_data(url=url, client=client))
+    return soups
+
+
 def google_search_urls(
     text,
     sorted_date,
-    domains_to_skip,
+    domains_to_include,
     api_key,
     cse_id,
     **kwargs,
@@ -75,7 +83,9 @@ def google_search_urls(
     if "items" in results and len(results["items"]) > 0:
         for count, link in enumerate(results["items"]):
             # skip user selected domains
-            if (domains_to_skip is not None) and any(("." + domain) in link["link"] for domain in domains_to_skip):
+            if (domains_to_include is None) or not any(
+                ("." + domain) in link["link"] for domain in domains_to_include
+            ):
                 continue
             url = link["link"]
             if url not in url_list:
@@ -84,25 +94,24 @@ def google_search_urls(
 
 
 def google_search(
-    input,
+    topic,
     sorted_date,
-    domains_to_skip,
+    domains_to_include,
 ):
     # api_key = "AIzaSyCLyCCpOPLZWuptuPAPSg8cUIZhdEMVf6g"
-    api_key = "AIzaSyA5VVwY1eEoIoflejObrxFDI0DJvtbmgW8"
-    # api_key = "AIzaSyCLyCCpOPLZWuptuPAPSg8cUIZhdEMVf6g"
+    # api_key = "AIzaSyA5VVwY1eEoIoflejObrxFDI0DJvtbmgW8"
+    api_key = "AIzaSyCLyCCpOPLZWuptuPAPSg8cUIZhdEMVf6g"
     # api_key = "AIzaSyCS1WQDMl1IMjaXtwSd_2rA195-Yc4psQE"
     # api_key = "AIzaSyCB61O70B8AC3l5Kk3KMoLb6DN37B7nqIk"
     # api_key = "AIzaSyCg1IbevcTAXAPYeYreps6wYWDbU0Kz8tg"
     # api_key = "AIzaSyA5VVwY1eEoIoflejObrxFDI0DJvtbmgW8"
     cse_id = "851813e81162b4ed4"
-
     # get list of URLS to check
     start_time = time.perf_counter()
     url_list = google_search_urls(
-        input,
+        topic,
         sorted_date,
-        domains_to_skip,
+        domains_to_include,
         api_key,
         cse_id,
     )