ivnban27-ctl commited on
Commit
92dff98
·
1 Parent(s): 7e14368

changes on BL postprocessing

Browse files
models/business_logic_utils/config.py CHANGED
@@ -272,7 +272,8 @@ DIFFICULTIES = {
272
  "difficulty_distrustful": {
273
  "difficulty_label": "distrustful",
274
  "description": [
275
- "You don't trust the counselor, you will eventually cooperate.",
 
276
  ],
277
  },
278
  # "difficulty_stop_convo": {
 
272
  "difficulty_distrustful": {
273
  "difficulty_label": "distrustful",
274
  "description": [
275
+ #"You don't trust the counselor, you will eventually cooperate.",
276
+ "You have a distrustful attitude towards the counselor.",
277
  ],
278
  },
279
  # "difficulty_stop_convo": {
models/business_logic_utils/response_processing.py CHANGED
@@ -48,6 +48,7 @@ def postprocess_text(
48
 
49
  # Remove unnecessary role prefixes
50
  text = text.replace(human_prefix, "").replace(assistant_prefix, "")
 
51
 
52
  # Remove whispers or other marked reactions
53
  whispers = re.compile(r"(\([\w\s]+\))") # remove things like "(whispers)"
@@ -55,8 +56,11 @@ def postprocess_text(
55
  text = whispers.sub("", text)
56
  text = reactions.sub("", text)
57
 
58
- # Remove all quotation marks (both single and double)
59
- text = text.replace('"', '').replace("'", "")
 
 
 
60
 
61
  # Normalize spaces
62
  text = re.sub(r"\s+", " ", text).strip()
 
48
 
49
  # Remove unnecessary role prefixes
50
  text = text.replace(human_prefix, "").replace(assistant_prefix, "")
51
+
52
 
53
  # Remove whispers or other marked reactions
54
  whispers = re.compile(r"(\([\w\s]+\))") # remove things like "(whispers)"
 
56
  text = whispers.sub("", text)
57
  text = reactions.sub("", text)
58
 
59
+ # Remove double quotation marks
60
+ text = text.replace('"', '')
61
+
62
+ # Remove stutters of any length (e.g., "M-m-my" or "M-m-m-m-my" or "M-My" to "My")
63
+ text = re.sub(r'\b(\w)(-\1)+-\1(\w*)', r'\1\3', text, flags=re.IGNORECASE)
64
 
65
  # Normalize spaces
66
  text = re.sub(r"\s+", " ", text).strip()