Update app.py

app.py

@@ -55,16 +55,20 @@ all_stopwords = sp.Defaults.stop_words
 
 
 
-
-
-
 def remove_stopwords(sen):
     sen_new = " ".join([i for i in sen if i not in stop_words])
     return sen_new
 
+examples = [
+  ["https://www.medicalnewstoday.com/articles/alzheimers-addressing-sleep-disturbance-may-alleviate-symptoms"],
+  ["https://www.medicalnewstoday.com/articles/omicron-what-do-we-know-about-the-stealth-variant"],
+  ["https://www.cancer.news/2022-02-04-doctors-testifying-covid-vaccines-causing-cancer-aids.html#",
+  "https://www.cancer.news/2021-12-22-mrna-vaccines-weaken-immune-system-cause-cancer.html"]
+   ] 
 
 
-def keyphrase_generator(article_link, model_1, model_2, max_num_keywords, model_3):
+def keyphrase_generator(article_link, model_1, model_2, max_num_keywords, model_3, max_retrieved, model_4):
+
   word_embedding_model = models.Transformer(model_3)
   pooling_model = models.Pooling(word_embedding_model.get_word_embedding_dimension(),
                              pooling_mode_mean_tokens=True,
@@ -87,6 +91,7 @@ def keyphrase_generator(article_link, model_1, model_2, max_num_keywords, model_
   final_list=[]
   score_list=[]
   sum_list=[]
+  ##############################################     Here we first extract the sentences using SBERT and Textrank     ###########################
   model_1 = SentenceTransformer(model_1)
   model_2 = SentenceTransformer(model_2)
   url = article_link
@@ -131,6 +136,10 @@ def keyphrase_generator(article_link, model_1, model_2, max_num_keywords, model_
   x=sorted(((sum_list[i],s) for i,s in enumerate(corpus)), reverse=True)
   for elem in x:
     final_textrank_list.append(elem[1])
+    
+ ################################################################    Textrank ends   ##################################################
+ 
+ ########################################################   From here we start the keyphrase extraction process   ################################################
  
   a=int((10*len(final_textrank_list))/100.0)
   if(a<5):
@@ -154,6 +163,11 @@ def keyphrase_generator(article_link, model_1, model_2, max_num_keywords, model_
   top_n = max_num_keywords
   keyword_list = [candidates[index] for index in distances.argsort()[0][-top_n:]]
   keywords = '\n'.join(keyword_list) 
+  
+  ##############################################################   Keyphrase extraction ends    #############################################
+  
+  
+  ##################################################################  From here we start the clustering and query generation      ##################################
 
   c_len=(len(keyword_list))
   keyword_embeddings = embedder.encode(keyword_list)
@@ -196,6 +210,9 @@ def keyphrase_generator(article_link, model_1, model_2, max_num_keywords, model_
   f_1=' OR '.join(f1_list)
   final_list.append(f_1)
 
+########################################################   query generation ends here     #######################################
+
+#######################################   PubeMed abstract extraction starts here    #########################################
 
   ncbi_url='https://eutils.ncbi.nlm.nih.gov/entrez/eutils/'
 
@@ -215,7 +232,7 @@ def keyphrase_generator(article_link, model_1, model_2, max_num_keywords, model_
   search_id='&id='+all_search_ids
   ret_type='&rettype=text'
   ret_mode='&retmode=xml'
-  ret_max='&retmax=10'
+  ret_max='&retmax=500'
   ret_sort='&sort=relevance'
   return_url=ncbi_url+fetch_url+search_id+ret_type+ret_mode+ret_max+ret_sort
   pubmed_abstract_request = requests.get(return_url)
@@ -228,12 +245,33 @@ def keyphrase_generator(article_link, model_1, model_2, max_num_keywords, model_
   for b in article_abstract:
     article_abstract_name = b.text
     abstracts_list.append(article_abstract_name)
-  mydict = {'Title': titles_list, 'Abstract':abstracts_list}
-    
 
-  df_new = pd.DataFrame(dict([ (k,pd.Series(v)) for k,v in mydict.items() ]))
+##################################   PubMed extraction ends here    ########################################################
+
+##########################################    Most relevant abstracts as per news article heading starts here    ##########################################
+  
+  first_article = Article(url, language='en')
+  first_article.download()
+  first_article.parse()
+  article_heading=(first_article.title)
+  article_heading=sent_tokenize(article_heading)
+  model_4 = SentenceTransformer(model_4)
+  
+  my_dict = dict(zip(titles_list,abstracts_list))
+  title_embeddings = model_4.encode(titles_list)
+  heading_embedding = model_4.encode(article_heading)
+  similarities = cosine_similarity(heading_embedding, title_embeddings)
+  max_n = max_retrieved
+  sorted_titles = [titles_list[index] for index in similarities.argsort()[0][-max_n:]]
+  sorted_abstract_list=[]
+  for list_elem in sorted_titles:
+    sorted_abstract_list.append(my_dict[list_elem])
+  sorted_dict = {'Title': sorted_titles, 'Abstract': sorted_abstract_list}
+  df_new=pd.DataFrame(dict([ (k,pd.Series(v)) for k,v in sorted_dict.items() ]))
+  
   df_final = df_new.fillna(' ')
 
+#############################################    Ends here    ####################################################
   
   return df_final
   
@@ -276,10 +314,17 @@ igen_pubmed = gr.Interface(keyphrase_generator,
                                                  'cambridgeltl/SapBERT-from-PubMedBERT-fulltext-mean-token'],
                                         type="value", 
                                         default='cambridgeltl/SapBERT-from-PubMedBERT-fulltext', 
-                                        label="Select any SapBERT model for clustering from the list below")], 
+                                        label="Select any SapBERT model for clustering from the list below"),
+                     gr.inputs.Slider(minimum=5, maximum=15, step=1, default=10, label="PubMed Max Abstracts"),
+                     gr.inputs.Dropdown(choices=['pritamdeka/S-Bluebert-snli-multinli-stsb',
+                                                 'pritamdeka/S-Biomed-Roberta-snli-multinli-stsb'], 
+                                        type="value", 
+                                        default='pritamdeka/S-Biomed-Roberta-snli-multinli-stsb', 
+                                        label="Select any SBERT model for abstracts from the list below")], 
              outputs=gr.outputs.Dataframe(type="auto", label="Retrieved Results from PubMed",max_cols=None, overflow_row_behaviour="paginate"), 
              theme="dark-peach", 
-             title="PubMed Abstract Retriever", description="Retrieves relevant PubMed abstracts for an online article which can be used as further references.",
+             title="PubMed Abstract Retriever", description="Retrieves relevant PubMed abstracts for an online article which can be used as further references. Please note that it may take sometime for the models to load.",
+             examples=examples,
              article= "This work is based on the paper <a href=https://dl.acm.org/doi/10.1145/3487664.3487701>provided here</a>." 
              "\t It uses the TextRank algorithm with SBERT to first find the top sentences and then extracts the keyphrases from those sentences using scispaCy and SBERT."
              "\t The application then uses a UMLS based BERT model, <a href=https://arxiv.org/abs/2010.11784>SapBERT</a> to cluster the keyphrases using K-means clustering method and finally create a boolean query. After that the top 10 titles and abstracts are retrieved from PubMed database and displayed according to relevancy. The SapBERT models can be changed as per the list provided. "