app modified

app.py

@@ -295,8 +295,41 @@ class ModeloDataset:
                 x=x+1
             else:
                  x=x+1
-        return new_identificadores           
-    def aplicar_modelo(self,_sentences,idioma):
+        return new_identificadores 
+    def fake_pers(self):
+       return self.faker_.name(self)
+    def fake_word(self):
+        return self.faker_.word()
+    def fake_first_name(self):
+        return self.faker_.first_name()       
+    def fake_last_name(self):
+        return self.faker_.last_name()    
+    def fake_address(self):
+        return self.faker_.address()
+    def fake_sentence(self,n):
+        return self.faker_.sentence(nb_words=n)
+    def fake_text(self):
+        return self.faker_.text()
+    def fake_company(self):
+        return self.faker_.company() 
+    def fake_city(self):
+        return self.faker_.city()     
+    def reemplazo_fake(self,identificadores):
+        new_iden=[]
+        for id in identificadores:
+            
+            if 'PER' in id:
+                new_iden.append(self.fake_first_name())
+            
+            elif 'ORG' in id:
+                new_iden.append(self.fake_company())
+            
+            elif 'LOC' in id:
+                new_iden.append(self.fake_city())
+            else:
+                new_iden.append(id)
+        return new_iden          
+    def aplicar_modelo(self,_sentences,idioma, etiquetas):
         if idioma=="es":
             self.tokenizer = AutoTokenizer.from_pretrained("BSC-LT/roberta_model_for_anonimization")
             tokenized_text=[self.tokenizer.tokenize(sentence) for sentence in _sentences]
@@ -345,10 +378,7 @@ class ModeloDataset:
                     i=i+1  
             labels = predicted_token_class_ids
             loss = self.model(input_ids, labels=labels).loss
-            
-            
-            
-            
+
             new_tokens=[]
             ig_tok=[]
             i=0
@@ -358,17 +388,14 @@ class ModeloDataset:
                aux1, aux2= self.reordenacion_tokens(item)
                new_tokens.append(aux1)
                ig_tok.append(aux2)
-            print('new_tokens:',new_tokens,' ' ,len(new_tokens))  
-            print('ig_tok',ig_tok) 
+             
             
             for items in _predicted_tokens_classes:
-                
-                
                 aux=self.reordenacion_identificadores(ig_tok[i],items,len(new_tokens[i]))
                 new_identificadores.append(aux)
                 i=i+1
             print('new_identificadores:',new_identificadores, ' ',len(new_identificadores) )
-           
+            
             return new_identificadores, new_tokens
         #return ids, _predicted_tokens_classes
     def salida_texto_es( self,ids,pre_tokens):
@@ -400,13 +427,15 @@ class ModeloDataset:
         for i in new_labels:
             a = a+i
         return a 
-    def salida_texto2(self, tokens,labels):
+    def salida_texto2(self, tokens,labels,etiquetas):
         i=0
         out=[]
         for iden in labels:
           #if i<len(ids):
-           
-           out.append(self.salida_texto( iden,np.array(tokens[i])) )
+           if etiquetas:
+              out.append(self.salida_texto( iden,np.array(tokens[i])))
+           else:   
+              out.append(self.salida_texto(iden,self.reemplazo_fake(np.array(tokens[i]))))
            i=i+1
         
         return out   
@@ -513,14 +542,14 @@ def procesar(texto,archivo, etiquetas):
                 sentences=df[item]
                 model.identificacion_idioma(sentences[0])
                 
-                ides, predicted = modelo.aplicar_modelo(sentences,model.idioma)
+                ides, predicted = modelo.aplicar_modelo(sentences,model.idioma,etiquetas)
 
                 if model.idioma=="es":
                   out=modelo.salida_texto2_es( ides,predicted)
                   print('out:',out)
                   df_new[item] = modelo.unir_array(out)
                 else:  
-                  out=modelo.salida_texto2( ides,predicted)#tokens,labels 
+                  out=modelo.salida_texto2( ides,predicted,etiquetas)#tokens,labels 
                   print('out:',out)
                   df_new[item] = modelo.unir_array(out)
             return "", df_new, df_new.to_csv(sep='\t', encoding='utf-8',index=False)
@@ -535,7 +564,7 @@ def procesar(texto,archivo, etiquetas):
                 for item in df.columns.values:
                     sentences=df[item]
                     
-                    ides, predicted = modelo.aplicar_modelo(sentences,"en")
+                    ides, predicted = modelo.aplicar_modelo(sentences,"en",etiquetas)
                     out=modelo.salida_texto2( ides,predicted)#tokens,labels 
                     print('out:',out)
                     df_new[item] = modelo.unir_array(out)