Spaces:
Runtime error
Runtime error
Update app.py
Browse files
app.py
CHANGED
@@ -20,11 +20,10 @@ simpl_model = BartForConditionalGeneration.from_pretrained(
|
|
20 |
'twigs/bart-text2text-simplifier')
|
21 |
cwi_pipe = pipeline('text-classification', model=cwi_model,
|
22 |
tokenizer=cwi_tok, function_to_apply='none')
|
23 |
-
fill_pipe = pipeline('fill-mask',
|
24 |
-
tokenizer=simpl_tok, top_k=1)
|
25 |
|
26 |
|
27 |
-
def id_replace_complex(s, threshold=0.
|
28 |
|
29 |
# get all tokens
|
30 |
tokens = re.compile('\w+').findall(s)
|
@@ -34,19 +33,16 @@ def id_replace_complex(s, threshold=0.4):
|
|
34 |
compl_tok = [tokens[idx] for idx, x in enumerate(
|
35 |
cwi_pipe(cands)) if x['score'] >= threshold]
|
36 |
|
37 |
-
|
38 |
-
|
39 |
-
for
|
40 |
-
|
41 |
-
|
42 |
-
|
43 |
-
|
44 |
-
|
45 |
-
print(s)
|
46 |
-
replacements.append(top_result['token_str'])
|
47 |
return s, compl_tok, replacements
|
48 |
|
49 |
-
|
50 |
def generate_candidate_text(s, model, tokenizer, tokenized=False):
|
51 |
|
52 |
|
|
|
20 |
'twigs/bart-text2text-simplifier')
|
21 |
cwi_pipe = pipeline('text-classification', model=cwi_model,
|
22 |
tokenizer=cwi_tok, function_to_apply='none')
|
23 |
+
fill_pipe = pipeline('fill-mask', top_k=1)
|
|
|
24 |
|
25 |
|
26 |
+
def id_replace_complex(s, threshold=0.2):
|
27 |
|
28 |
# get all tokens
|
29 |
tokens = re.compile('\w+').findall(s)
|
|
|
33 |
compl_tok = [tokens[idx] for idx, x in enumerate(
|
34 |
cwi_pipe(cands)) if x['score'] >= threshold]
|
35 |
|
36 |
+
masked = [s[:s.index(t)] + '<mask>' + s[s.index(t)+len(t):] for t in compl_tok]
|
37 |
+
cands = fill_pipe(masked)
|
38 |
+
replacements = [el['token_str'][1:] if type(el) == dict else el[0]['token_str'][1:] for el in cands]
|
39 |
+
|
40 |
+
for i, el in enumerate(compl_tok):
|
41 |
+
idx = s.index(el)
|
42 |
+
s = s[:idx] + replacements[i] + s[idx+len(el):]
|
43 |
+
|
|
|
|
|
44 |
return s, compl_tok, replacements
|
45 |
|
|
|
46 |
def generate_candidate_text(s, model, tokenizer, tokenized=False):
|
47 |
|
48 |
|