app modified

app.py

@@ -261,6 +261,29 @@ class ModeloDataset:
         self.modelo_ner=""
         self.categoria_texto=""
         self.tokenizer  = AutoTokenizer.from_pretrained("BSC-LT/roberta_model_for_anonimization")
+    def reordenacion_tokens(self,tokens):
+        
+        i=0
+        new_tokens=[]
+        ig_tokens=[] #ignorar estos indices del array de indentificadores
+        for token in tokens:
+            ind=len(new_tokens)
+            if i<len(tokens):
+                 if  token.startswith("▁"):
+                    
+                    new_tokens.append(token)
+                    
+                    i=i+1
+                 else:
+                    new_tokens[ind-1] = (new_tokens[ind-1] + token)
+                    ig_tokens.append(i)
+                    
+                    i=i+1
+        return (
+                new_tokens,
+                ig_tokens
+                )  
+            
     def aplicar_modelo(self,_sentences,idioma):
         if idioma=="es":
             self.tokenizer = AutoTokenizer.from_pretrained("BSC-LT/roberta_model_for_anonimization")
@@ -311,8 +334,12 @@ class ModeloDataset:
                     i=i+1  
             labels = predicted_token_class_ids
             loss = self.model(input_ids, labels=labels).loss
-            #print(round(loss.item(), 2))
-        return ids, _predicted_tokens_classes
+
+            new_tokens,ig_tokens=self.reordenacion_tokens(tokenized_text) #
+            new_identificadores = self.reordenacion_identificadores(ig_tokens,_predicted_tokens_classes)#
+
+        return new_identificadores,new_identificadores
+        #return ids, _predicted_tokens_classes
     def salida_texto( self,ids,pre_tokens):
         new_labels = []
         current_word = None
@@ -430,6 +457,8 @@ def procesar(texto,archivo, etiquetas):
                 model.identificacion_idioma(sentences[0])
                 print('sentences',sentences)
                 ides, predicted = modelo.aplicar_modelo(sentences,model.idioma)
+
+                
                 out=modelo.salida_texto2( ides,predicted)
                 print('out:',out)
                 df_new[item] = modelo.unir_array(out)