Create new file

app.py

@@ -0,0 +1,122 @@
+from transformers import AutoTokenizer, AutoModelForSeq2SeqLM
+import pickle
+import torch
+from transformers import PegasusTokenizer, PegasusForConditionalGeneration
+import tensorflow as tf
+from tensorflow.python.lib.io import file_io
+from nltk.tokenize import sent_tokenize
+
+
+import io
+
+
+#contents = pickle.load(f) becomes...
+#contents = CPU_Unpickler(f).load()
+
+
+model_path = "finbert.sav"
+
+#load model from drive
+with open(model_path, "rb") as f:
+    model=  pickle.load(f)
+    
+    
+
+#tokenizer = AutoTokenizer.from_pretrained(checkpoint)
+#model = AutoModelForSeq2SeqLM.from_pretrained(checkpoint)
+
+
+import nltk
+from finbert_embedding.embedding import FinbertEmbedding
+import pandas as pd
+from nltk.cluster import KMeansClusterer
+import numpy as np
+import os
+from scipy.spatial import distance_matrix
+from tensorflow.python.lib.io import file_io
+import pickle
+
+nltk.download('punkt')
+
+
+def make_summary(word):
+
+    # Create tokens from the txt file
+    tokens = nltk.sent_tokenize(word)
+    # Strip out trailing and leading white spaces from tokens
+    sentences = [word.strip() for word in tokens]
+    #Create a DataFrame from the tokens 
+    data = pd.DataFrame(sentences)
+    # Assign name Sentences to the column containing text tokens
+    data.columns = ['Sentences']
+    
+    # Function to create numerical embeddings for each text tokens in dataframe 
+    def get_sentence_embeddings():
+        # Create empty list for sentence embeddings
+        sentence_list = []
+        # Loop through all sentences and append sentence embeddings to list
+        for i in tokens:
+            sentence_embedding = model.sentence_vector(i)
+            sentence_list.append(sentence_embedding)
+        # Create empty list for ndarray
+        sentence_array=[]
+        # Loop through sentence list and change data type from tensor to array
+        for i in sentence_list:
+            sentence_array.append(i.numpy())
+        # return sentence embeddings as list
+        return sentence_array
+
+    # Apply get_sentence_embeddings to dataframe to create column Embeddings
+    data['Embeddings'] = get_sentence_embeddings()
+    
+    #Number of expected sentences
+    if len(tokens) <= 1: 
+        NUM_CLUSTERS = 1
+    else:
+        NUM_CLUSTERS = len(tokens)//4
+
+    iterations = 25
+    # Convert Embeddings into an array and store in variable X
+    X = np.array(data['Embeddings'].to_list())
+    
+    #Build k-means cluster algorithm
+    Kclusterer = KMeansClusterer(
+                                NUM_CLUSTERS,
+                                distance = nltk.cluster.util.cosine_distance,
+                                repeats = iterations, avoid_empty_clusters = True)
+
+    # if length of text is too short, K means would return an error
+    # use the try except block to return the text as result if it is too short.
+    try:
+        
+        assigned_clusters = Kclusterer.cluster(X,assign_clusters=True)
+
+        # Apply Kmean Cluster to DataFrame and create new columns Clusters and Centroid
+        data['Cluster'] = pd.Series(assigned_clusters, index = data.index)
+        data['Centroid'] = data['Cluster'].apply(lambda x: Kclusterer.means()[x])
+    
+    # return the text if clustering algorithm catches an exceptiona and move to the next text file
+    except ValueError:
+        return word
+
+    # function that computes the distance of each embeddings from the centroid of the cluster
+    def distance_from_centroid(row):
+        return distance_matrix([row['Embeddings']], [row['Centroid'].tolist()])[0][0]
+    
+    # apply distance_from_centroid function to data
+    data['Distance_From_Centroid'] = data.apply(distance_from_centroid, axis =1)
+    
+    ## Return Final Summary
+    summary = " ".join(data.sort_values(
+                'Distance_From_Centroid',
+                ascending = True).groupby('Cluster').head(1).sort_index()['Sentences'].tolist())
+    return summary
+
+import gradio as gr
+
+
+
+                     
+interface1 = gr.Interface(fn=make_summary, 
+                     inputs =gr.inputs.Textbox(lines=15,placeholder="Enter your text !!",label='Input-10k Sections'),
+                     outputs=gr.outputs.Textbox(label='Output- Finbert')).launch()