app modified

app.py

@@ -287,10 +287,10 @@ class ModeloDataset:
         x=0
         new_identificadores=[]
         for token in predicted_tokens_classes:
-            print('x',x, len(predicted_tokens_classes)-x)
+            
             if x not in ig_tokens:
                 if len(new_identificadores) < tamano:
-                    print('se agrega token')
+                    
                     new_identificadores.append(token)
                 x=x+1
             else:
@@ -354,7 +354,7 @@ class ModeloDataset:
             i=0
             new_identificadores=[]
             for item in tokenized_text:
-               print('len(tokens)',len(item))
+               
                aux1, aux2= self.reordenacion_tokens(item)
                new_tokens.append(aux1)
                ig_tok.append(aux2)
@@ -362,40 +362,64 @@ class ModeloDataset:
             print('ig_tok',ig_tok) 
             
             for items in _predicted_tokens_classes:
-                #if i<len(new_tokens[i]):
-                print('len(new_tokens[i])',len(new_tokens[i]))
+                
+                
                 aux=self.reordenacion_identificadores(ig_tok[i],items,len(new_tokens[i]))
                 new_identificadores.append(aux)
                 i=i+1
             print('new_identificadores:',new_identificadores, ' ',len(new_identificadores) )
            
-        #return new_identificadores, new_tokens
-        return ids, _predicted_tokens_classes
-    def salida_texto( self,ids,pre_tokens):
+        return new_identificadores, new_tokens
+        #return ids, _predicted_tokens_classes
+    #def salida_texto( self,ids,pre_tokens):
+    #    new_labels = []
+    #    current_word = None
+    #    i=0
+    #    for identificador in pre_tokens:
+    #        if (self.tokenizer.decode(ids[i])!="<s>"):
+    #            if identificador=='O':
+    #                
+    #                new_labels.append(self.tokenizer.decode(ids[i]))
+    #            else:
+    #                new_labels.append(' ' + identificador)
+    #        i=i+1  
+    #        
+    #    return new_labels    
+    def salida_texto( self,tokens,pre_tokens):
         new_labels = []
         current_word = None
         i=0
-        for identificador in pre_tokens:
-            if (self.tokenizer.decode(ids[i])!="<s>"):
-                if identificador=='O':
-                    
-                    new_labels.append(self.tokenizer.decode(ids[i]))
-                else:
-                    new_labels.append(' ' + identificador)
-            i=i+1  
-            
-        return new_labels    
-    
-    def salida_texto2(self, ids,pre_tokens):
+        for token in tokens:
+               
+            if pre_tokens[i]=='O' or 'MISC' in pre_tokens[i]:
+              new_labels.append(' ' +token.replace('▁',''))
+            else:
+               new_labels.append(' ' + pre_tokens[i])
+            i=i+1 
+        a=''    
+        for i in new_labels:
+            a = a+i
+        return a 
+    def salida_texto2(self, tokens,labels):
         i=0
         out=[]
-        for iden in pre_tokens:
-          if i<len(ids):
+        for iden in labels:
+          #if i<len(ids):
            
-           out.append(self.salida_texto( ids[i],np.array(pre_tokens[i])) )
+           out.append(self.salida_texto( iden,np.array(tokens[i])) )
            i=i+1
         
-        return out    
+        return out   
+    #def salida_texto2(self, ids,pre_tokens):
+    #    i=0
+    #    out=[]
+    #    for iden in pre_tokens:
+    #      if i<len(ids):
+    #       
+    #       out.append(self.salida_texto( ids[i],np.array(pre_tokens[i])) )
+    #       i=i+1
+    #    
+    #    return out    
     def unir_array(self,_out):
         i=0
         salida=[]