update app

app.py

@@ -249,18 +249,101 @@ class Model:
            
             
         )  
+class ModeloDataset:
+    def __init__(self):
+        self.texto=""
+        self.idioma=""
+        self.modelo_ner=""
+        self.categoria_texto=""
+
+    def aplicar_modelo(self,_sentences):
+    
+        tokenizer = AutoTokenizer.from_pretrained("BSC-LT/roberta_model_for_anonimization")
+        tokenized_text=[tokenizer.tokenize(sentence) for sentence in _sentences]
+     
+        ids = [tokenizer.convert_tokens_to_ids(x) for x in tokenized_text]   
+        MAX_LEN=128
+        ids=pad_sequences(ids,maxlen=MAX_LEN,dtype="long",truncating="post", padding="post")
+        input_ids = torch.tensor(ids) 
+        model = RobertaForTokenClassification.from_pretrained("BSC-LT/roberta_model_for_anonimization")   
+    
+        
+        model = AutoModelForTokenClassification.from_pretrained("BSC-LT/roberta_model_for_anonimization")
+        with torch.no_grad():
+           logits = model(input_ids).logits
+        predicted_token_class_ids = logits.argmax(-1)
+        i=0
+        _predicted_tokens_classes=[]
+        for a in predicted_token_class_ids:
+                   #_predicted_tokens_classes[i]=[model.config.id2label[t.item()] for t in predicted_token_class_ids[i]]
+                   _predicted_tokens_classes.append([model.config.id2label[t.item()] for t in predicted_token_class_ids[i]])
+                   i=i+1  
+        labels = predicted_token_class_ids
+        loss = model(input_ids, labels=labels).loss
+        #print(round(loss.item(), 2))
+        
+        return ids, _predicted_tokens_classes
+    def salida_texto( self,ids,pre_tokens):
+        new_labels = []
+        current_word = None
+        i=0
+        for identificador in pre_tokens:
+            if (tokenizer.decode(ids[i])!="<s>"):
+                if identificador=='O':
+                    
+                    new_labels.append(tokenizer.decode(ids[i]))
+                else:
+                    new_labels.append(' ' + identificador)
+            i=i+1  
+            
+        return new_labels    
+    
+    def salida_texto2(self, ids,pre_tokens):
+        i=0
+        out=[]
+        for iden in pre_tokens:
+          if i<len(ids):
+           
+           out.append(salida_texto( ids[i],np.array(_predicted_tokens_classes[i])) )
+           i=i+1
+        
+        return out    
+    def unir_array(self,_out):
+        i=0
+        salida=[]
+        for item in _out:
+            salida.append("".join(str(x) for x in _out[i]))
+            i=i+1
+        return salida
+    def unir_columna_valores(self,df,columna):
+        out = ','.join(df[columna])
+        return out
+modelo = ModeloDataset()
 model = Model()
 def get_model():
         return model
 
 def procesar(texto,archivo):
-    print(texto)
-    print(archivo.name)
-    df=pd.read_csv(archivo.name,delimiter=",")
-    print(df["nombre"])
-    model.identificacion_idioma(texto)
-    
-    return model.predict()
+    if len(texto)>0: 
+        model.identificacion_idioma(texto)
+        return model.predict()
+    else:
+        df=pd.read_csv(archivo.name,delimiter=",")
+        print(archivo.name)
+        df_new = pd.DataFrame(data, columns=df.columns.values)
+        print(df_new)
+        for item in df.columns.values:
+            sentences=df[item]
+            ides, predicted = modelo.aplicar_modelo(sentences)
+            out=modelo.salida_texto2( ides,predicted)
+            
+            df_new[item] = modelo.unir(out)
+        plotting_df=gr.Dataframe(value=df_new,headers=["nombre","trabajo"],label="label:",type="pandas", visible=True, interactive=False)
+        print(df_new)
+        return plotting_df
 
 demo = gr.Interface(fn=procesar,inputs=["text",gr.File()] , outputs="text")
-demo.launch(share=True)
+demo.launch(share=True)
+
+#plotting_df=gr.Dataframe(value=df2,headers="class","type","group","ε54Cr","ε50Ti","ε94Mo"],
+# label="CosmoPlot Compilation:",type="pandas", visible=True, interactive=False)