app modified

app.py

@@ -348,14 +348,14 @@ class ModeloDataset:
             
             
             
-            #new_tokens=[]
-            #ig_tok=[]
-            #new_identificadores=[]
-            #for item in tokenized_text:
-            #   aux1, aux2= self.reordenacion_tokens(item)
-            #   new_tokens.append(aux1)
-            #   ig_tok.append(aux2)
-  
+            new_tokens=[]
+            ig_tok=[]
+            new_identificadores=[]
+            for item in tokenized_text:
+               aux1, aux2= self.reordenacion_tokens(item)
+               new_tokens.append(aux1)
+               ig_tok.append(aux2)
+            print('new_tokens',new_tokens)  
         
             #for item in _predicted_tokens_classes:
             #    new_identificadores.append(self.reordenacion_identificadores(ig_tok,item))