app modified

app.py

@@ -318,7 +318,7 @@ class ModeloDataset:
                     i=i+1  
             labels = predicted_token_class_ids
             loss = self.model(input_ids, labels=labels).loss
-            
+            return ids,_predicted_tokens_classes
         else:
             
             print('idioma:',idioma)            
@@ -369,22 +369,22 @@ class ModeloDataset:
                 i=i+1
             print('new_identificadores:',new_identificadores, ' ',len(new_identificadores) )
            
-        return new_identificadores, new_tokens
+            return new_identificadores, new_tokens
         #return ids, _predicted_tokens_classes
-    #def salida_texto( self,ids,pre_tokens):
-    #    new_labels = []
-    #    current_word = None
-    #    i=0
-    #    for identificador in pre_tokens:
-    #        if (self.tokenizer.decode(ids[i])!="<s>"):
-    #            if identificador=='O':
-    #                
-    #                new_labels.append(self.tokenizer.decode(ids[i]))
-    #            else:
-    #                new_labels.append(' ' + identificador)
-    #        i=i+1  
-    #        
-    #    return new_labels    
+    def salida_texto_es( self,ids,pre_tokens):
+        new_labels = []
+        current_word = None
+        i=0
+        for identificador in pre_tokens:
+            if (self.tokenizer.decode(ids[i])!="<s>"):
+                if identificador=='O':
+                    
+                    new_labels.append(self.tokenizer.decode(ids[i]))
+                else:
+                    new_labels.append(' ' + identificador)
+            i=i+1  
+            
+        return new_labels    
     def salida_texto( self,tokens,pre_tokens):
         new_labels = []
         current_word = None
@@ -410,16 +410,16 @@ class ModeloDataset:
            i=i+1
         
         return out   
-    #def salida_texto2(self, ids,pre_tokens):
-    #    i=0
-    #    out=[]
-    #    for iden in pre_tokens:
-    #      if i<len(ids):
-    #       
-    #       out.append(self.salida_texto( ids[i],np.array(pre_tokens[i])) )
-    #       i=i+1
-    #    
-    #    return out    
+    def salida_texto2_es(self, ids,pre_tokens):
+        i=0
+        out=[]
+        for iden in pre_tokens:
+          if i<len(ids):
+           
+           out.append(self.salida_texto_es( ids[i],np.array(pre_tokens[i])) )
+           i=i+1
+        
+        return out    
     def unir_array(self,_out):
         i=0
         salida=[]
@@ -515,10 +515,14 @@ def procesar(texto,archivo, etiquetas):
                 #print('sentences',sentences)
                 ides, predicted = modelo.aplicar_modelo(sentences,model.idioma)
 
-                
-                out=modelo.salida_texto2( ides,predicted)
-                print('out:',out)
-                df_new[item] = modelo.unir_array(out)
+                if model.idioma=="es":
+                  out=modelo.salida_texto2_es( ides,predicted)
+                  print('out:',out)
+                  df_new[item] = modelo.unir_array(out)
+                else:  
+                  out=modelo.salida_texto2( ides,predicted)
+                  print('out:',out)
+                  df_new[item] = modelo.unir_array(out)
             return "", df_new, df_new.to_csv(sep='\t', encoding='utf-8',index=False)
             #return "", df_new, df_new.to_excel( index=False)
         else: