app modified dataset reorden token

app.py

@@ -110,7 +110,7 @@ class Model:
         i=0
         for token in tokens:
                
-            if pre_tokens[i]=='O' or 'MISC' in pre_tokens[i] or 'OTH' in pre_tokens[i]: 
+            if pre_tokens[i]=='O' or 'MISC' in pre_tokens[i]:
               new_labels.append(' ' +token.replace('▁','').replace('Ġ',''))
             else:
                new_labels.append(' ' + pre_tokens[i])
@@ -120,21 +120,21 @@ class Model:
             a = a+i
         return a    
         #return new_labels  
-   #def salida_texto_anonimizado(self, ids,pre_tokens):
-   # new_labels = []
-   # current_word = None
-   # i=0
-   # for identificador in pre_tokens:
-   #    
-   #     if identificador=='O' or 'OTH' in identificador:
-   #         new_labels.append(self.tokenizer.decode(ids[i]))
-   #     else:
-   #         new_labels.append(' ' + identificador)
-   #     i=i+1
-   # a=''
-   # for i in new_labels:
-   #     a = a+i
-   # return a    
+   def salida_texto_anonimizado(self, ids,pre_tokens):
+    new_labels = []
+    current_word = None
+    i=0
+    for identificador in pre_tokens:
+       
+        if identificador=='O' or 'OTH' in identificador:
+            new_labels.append(self.tokenizer.decode(ids[i]))
+        else:
+            new_labels.append(' ' + identificador)
+        i=i+1
+    a=''
+    for i in new_labels:
+        a = a+i
+    return a    
    def formato_salida(self,out):
        a=""
        for i in out:
@@ -428,20 +428,20 @@ class ModeloDataset:
             
             return new_identificadores, new_tokens
         #return ids, _predicted_tokens_classes
-    #def salida_texto_es( self,ids,pre_tokens):
-    #    new_labels = []
-    #    current_word = None
-    #    i=0
-    #    for identificador in pre_tokens:
-    #        if (self.tokenizer.decode(ids[i])!="<s>"):
-    #            if identificador=='O':
-    #                
-    #                new_labels.append(self.tokenizer.decode(ids[i]))
-    #            else:
-    #                new_labels.append(' ' + identificador)
-    #        i=i+1  
-    #        
-    #    return new_labels    
+    def salida_texto_es( self,ids,pre_tokens):
+        new_labels = []
+        current_word = None
+        i=0
+        for identificador in pre_tokens:
+            if (self.tokenizer.decode(ids[i])!="<s>"):
+                if identificador=='O':
+                    
+                    new_labels.append(self.tokenizer.decode(ids[i]))
+                else:
+                    new_labels.append(' ' + identificador)
+            i=i+1  
+            
+        return new_labels    
     def salida_texto( self,tokens,pre_tokens):
         new_labels = []
         current_word = None
@@ -607,7 +607,7 @@ def procesar(texto,archivo, etiquetas):
                     ides, predicted = modelo.aplicar_modelo(sentences,modelo.idioma,etiquetas)
 
                     if model.idioma=="es":
-                        out=modelo.salida_texto2( ides,predicted,etiquetas)#tokens,labels 
+                        out=modelo.salida_texto2_es( ides,predicted,etiquetas)#tokens,labels 
                     else:
                         out=modelo.salida_texto2( ides,predicted,etiquetas)#tokens,labels