training-adherence-features (#1)

- cpc and bad practices features (70a482f666f55772a84d65dc90e69fb21b3d96de)
- fix on tokenizer special tokens (cf6ebf90b37bb55854638f960ba1fb42e56b08f3)
- change in model aliveness calculations (2e79a3c9fdc409a99f016bad980f00c71e2860fc)
- aliveness calculation fixes (b74c038fe4c7136eb1858138c63c7ee531a574a8)
- training adherence scoring features (cfe8e1a5d05755a0346c269de80f255dfb39c501)
- making explanation editable (5f8859a38074d0644c3d0e5fc87c175ecb79071b)
- fix on roberta input len (5e4965f0930a2498341a903704d9ba347fd575e0)
- ta utils fix for explanation (7e14368f66f1989c0d91ee1fca1a5d16e638a523)
- changes on BL postprocessing (92dff98dd8d42a645c05178c183798b9fb1837e7)
- progress bar instead of spinner (a139603ab1caa2d4c2b71cca5e3d7b55e8788073)
- changed to prod (02544790fb5037419491dfe520951728f1e5cee6)

.streamlit/config.toml

@@ -0,0 +1,2 @@
+[client]
+showSidebarNavigation = false

README.md

@@ -1,11 +1,11 @@
 ---
-title: Conversation Simulator
+title: Conversation Simulator DEV
 emoji: 💬
 colorFrom: red
 colorTo: red
 sdk: streamlit
-sdk_version: 1.26.0
-app_file: convosim.py
+sdk_version: 1.38.0
+app_file: main.py
 pinned: false
 ---
 

app_config.py

@@ -18,9 +18,28 @@ SOURCES_LAB = {"OA_rolemodel":'OpenAI GPT4o',
 
 ENDPOINT_NAMES = {
     # "CTL_llama2": "texter_simulator",
-    "CTL_llama3": "texter_simulator_llm",
+    "CTL_llama3": {
+        "name": "texter_simulator_llm",
+        "model_type": "text-generation"
+    },
+    # "CTL_llama3": {
+    #     "name": "databricks-meta-llama-3-1-70b-instruct",
+    #     "model_type": "text-generation"
+    # },
     # 'CTL_llama2': "llama2_convo_sim",
-    "CTL_mistral": "convo_sim_mistral"
+    # "CTL_mistral": "convo_sim_mistral",
+    "CPC": {
+        "name": "phase_classifier",
+        "model_type": "classificator"
+    },
+    "BadPractices": {
+        "name": "training_adherence_bp",
+        "model_type": "classificator"
+    },
+    "training_adherence": {
+        "name": "training_adherence",
+        "model_type": "text-completion"
+    },
 }
 
 def source2label(source):
@@ -36,6 +55,9 @@ DB_CONVOS = 'conversations'
 DB_COMPLETIONS = 'comparison_completions'
 DB_BATTLES = 'battles'
 DB_ERRORS = 'completion_errors'
+DB_CPC = "cpc_comparison"
+DB_BP = "bad_practices_comparison"
+DB_TA = "convo_scoring_comparison"
 
 MAX_MSG_COUNT = 60
 WARN_MSG_COUT = int(MAX_MSG_COUNT*0.8)

main.py

@@ -0,0 +1,14 @@
+import streamlit as st
+from streamlit.logger import get_logger
+
+from utils.app_utils import are_models_alive
+
+logger = get_logger(__name__)
+
+st.set_page_config(page_title="Conversation Simulator")
+
+with st.status("Loading Models Please Wait...(this may take up to 5 min)", expanded=True) as status:
+    if not are_models_alive():
+        st.switch_page("pages/model_loader.py")
+    else: 
+        st.switch_page("pages/convosim.py")

models/business_logic_utils/config.py

@@ -272,7 +272,8 @@ DIFFICULTIES = {
     "difficulty_distrustful": {
         "difficulty_label": "distrustful",
         "description": [
-            "You don't trust the counselor, you will eventually cooperate.",
+            #"You don't trust the counselor, you will eventually cooperate.",
+            "You have a distrustful attitude towards the counselor.",
         ],
     },
     # "difficulty_stop_convo": {

models/business_logic_utils/response_processing.py

@@ -48,6 +48,7 @@ def postprocess_text(
 
         # Remove unnecessary role prefixes
         text = text.replace(human_prefix, "").replace(assistant_prefix, "")
+        
 
         # Remove whispers or other marked reactions
         whispers = re.compile(r"(\([\w\s]+\))")  # remove things like "(whispers)"
@@ -55,8 +56,11 @@ def postprocess_text(
         text = whispers.sub("", text)
         text = reactions.sub("", text)
 
-        # Remove all quotation marks (both single and double)
-        text = text.replace('"', '').replace("'", "")
+        # Remove double quotation marks
+        text = text.replace('"', '')
+
+        # Remove stutters of any length (e.g., "M-m-my" or "M-m-m-m-my" or "M-My" to "My")
+        text = re.sub(r'\b(\w)(-\1)+-\1(\w*)', r'\1\3', text, flags=re.IGNORECASE)
 
         # Normalize spaces
         text = re.sub(r"\s+", " ", text).strip()

models/databricks/texter_sim_llm.py

@@ -16,15 +16,13 @@ texter:"""
 
 def get_databricks_chain(source, issue, language, memory, temperature=0.8, texter_name="Kit"):
 
-    endpoint_name = ENDPOINT_NAMES.get(source, "texter_simulator")
-    
+    endpoint_name = ENDPOINT_NAMES.get(source, "texter_simulator")['name']
     PROMPT = PromptTemplate(
         input_variables=['history', 'input'],
         template=_DATABRICKS_TEMPLATE_
     )
     
     llm = CustomDatabricksLLM(
-        # endpoint_url="https://dbc-6dca8e8f-4084.cloud.databricks.com/serving-endpoints/databricks-meta-llama-3-1-70b-instruct/invocations",
         endpoint_url=os.environ['DATABRICKS_URL'].format(endpoint_name=endpoint_name),
         bearer_token=os.environ["DATABRICKS_TOKEN"],
         texter_name=texter_name,
@@ -43,4 +41,17 @@ def get_databricks_chain(source, issue, language, memory, temperature=0.8, texte
     )
 
     logging.debug(f"loaded Databricks model")
-    return llm_chain, None
+    return llm_chain, None
+
+def cpc_is_alive():
+    body_request = {
+        "inputs": [""]
+    }
+    try:
+        # Send request to Serving
+        response = requests.post(url=CPC_URL, headers=HEADERS, json=body_request, timeout=2)
+        if response.status_code == 200:
+            return True
+        else: return False
+    except:
+        return False

models/ta_models/bp_utils.py

@@ -0,0 +1,66 @@
+import streamlit as st
+from streamlit.logger import get_logger
+import requests
+import os
+from .config import model_name_or_path, BP_THRESHOLD
+from transformers import AutoTokenizer
+from utils.mongo_utils import new_bp_comparison
+from app_config import ENDPOINT_NAMES
+
+logger = get_logger(__name__)
+
+tokenizer = AutoTokenizer.from_pretrained(model_name_or_path, truncation_side="left")
+BP_URL = os.environ["DATABRICKS_URL"].format(endpoint_name=ENDPOINT_NAMES["BadPractices"]['name'])
+HEADERS = {
+    "Authorization": f"Bearer {os.environ['DATABRICKS_TOKEN']}",
+    "Content-Type": "application/json",
+}
+
+def bp_predict_message(context, input):
+    # context = memory.load_memory_variables({})[memory.memory_key]
+    encoding = tokenizer(
+        context,
+        input,
+        truncation="only_first",
+        max_length = tokenizer.model_max_length - 2,
+    )['input_ids']
+    body_request = {
+        "inputs": [tokenizer.decode(encoding[1:-1])],
+        "params": {
+            "top_k": None
+        }
+    }
+
+    try:
+        # Send request to Serving
+        logger.debug(f"raw BP body is {body_request}")
+        response = requests.post(url=BP_URL, headers=HEADERS, json=body_request)
+        if response.status_code == 200:
+            response = response.json()['predictions'][0]
+            logger.debug(f"Raw BP prediction is {response}")
+            return [{k:v > BP_THRESHOLD if k=="score" else v for k,v in dict_.items()} for _, dict_ in response.items() ]
+        else:
+            raise Exception(f"Error in response: {response.json()}")
+    except Exception as e:
+        logger.debug(f"Error in response: {e}")
+        st.switch_page("pages/model_loader.py")
+
+def bp_push2db(manual_confirmation=None):
+    if manual_confirmation is None:
+        if st.session_state.sel_bp == "Advice":
+            manual_confirmation = {"is_advice":True, "is_personal_info":False}
+        elif st.session_state.sel_bp == "Personal Info":
+            manual_confirmation = {"is_advice":False, "is_personal_info":True}
+        elif st.session_state.sel_bp == "Advice & Personal Info":
+            manual_confirmation = {"is_advice":True, "is_personal_info":True}
+        else:
+            manual_confirmation = {"is_advice":False, "is_personal_info":False}
+    new_bp_comparison(**{
+        "client": st.session_state['db_client'],
+        "convo_id": st.session_state['convo_id'],
+        "model": st.session_state['source'],
+        "context": st.session_state["context"],
+        "last_message": st.session_state["last_message"],
+        "ytrue": manual_confirmation,
+        "ypred": {x['label']:x['score'] for x in st.session_state['bp_prediction']},
+    })

models/ta_models/config.py

@@ -0,0 +1,174 @@
+model_name_or_path = "FacebookAI/xlm-roberta-large"
+
+CPC_LABEL2STR = {
+    "0_ActiveEngagement": "Active Engagement",
+    "1_Explore": "Explore",
+    "2_IRA": "Immidiate Risk Assessment",
+    "3_SafetyAssessment": "Safety Assessment",
+    "4_SP&NS": "Safety Planning & Next Steps",
+    "5_EmergencyIntervention": "Emergency Intervention",
+    "6_WrappingUp": "Wrapping Up",
+    "7_Other": "Other",
+}
+
+CPC_LBL_OPTS = list(CPC_LABEL2STR.keys())
+
+def cpc_label2str(phase):
+    return CPC_LABEL2STR[phase]
+
+def phase2int(phase):
+    return int(phase.split("_")[0])
+
+BP_THRESHOLD = 0.7
+BP_LAB2STR = {
+    "is_advice": "Advice",
+    "is_personal_info": "Personal Info Sharing",
+}
+
+QUESTION2PHASE = {
+    "question_1": ["0_ActiveEngagement","1_Explore"],
+    "question_4": ["1_Explore"],
+    "question_5": ["0_ActiveEngagement", "1_Explore"],
+    # "question_7": ["1_Explore"],
+    # "question_9": ["4_SP&NS"],
+    "question_10": ["4_SP&NS"],
+    # "question_11": ["4_SP&NS"],
+    "question_14": ["6_WrappingUp"],
+    # "question_15": ["ALL"],
+    "question_19": ["ALL"],
+    # "question_21": ["ALL"],
+    # "question_22": ["ALL"],
+    "question_23": ["2_IRA", "3_SafetyAssessment"],
+}
+
+QUESTION2FILTERARGS = {
+    "question_1": {
+        "phases": QUESTION2PHASE["question_1"],
+        "pre_n": 2,
+        "post_n": 8,
+        "ignore": ["7_Other"],
+    },
+    "question_4": {
+        "phases": QUESTION2PHASE["question_4"],
+        "pre_n": 5,
+        "post_n": 5,
+        "ignore": ["7_Other"],
+    },
+    "question_5": {
+        "phases": QUESTION2PHASE["question_5"],
+        "pre_n": 5,
+        "post_n": 5,
+        "ignore": ["7_Other"],
+    },
+    # "question_7": {
+    #     "phases": QUESTION2PHASE["question_7"],
+    #     "pre_n": 5,
+    #     "post_n": 15,
+    #     "ignore": ["7_Other"],
+    # },
+    # "question_9": {
+    #     "phases": QUESTION2PHASE["question_9"],
+    #     "pre_n": 5,
+    #     "post_n": 5,
+    #     "ignore": ["7_Other"],
+    # },
+    "question_10": {
+        "phases": QUESTION2PHASE["question_10"],
+        "pre_n": 5,
+        "post_n": 5,
+        "ignore": ["7_Other"],
+    },
+    # "question_11": {
+    #     "phases": QUESTION2PHASE["question_11"],
+    #     "pre_n": 5,
+    #     "post_n": 5,
+    #     "ignore": ["7_Other"],
+    # },
+    "question_14": {
+        "phases": QUESTION2PHASE["question_14"],
+        "pre_n": 10,
+        "post_n": 0,
+        "ignore": ["7_Other"],
+    },
+    # "question_15": {
+    #     "phases": QUESTION2PHASE["question_15"],
+    #     "pre_n": 5,
+    #     "post_n": 5,
+    #     "ignore": ["7_Other"],
+    # },
+    "question_19": {
+        "phases": QUESTION2PHASE["question_19"],
+        "pre_n": 5,
+        "post_n": 5,
+        "ignore": ["7_Other"],
+    },
+    # "question_21": {
+    #     "phases": QUESTION2PHASE["question_21"],
+    #     "pre_n": 5,
+    #     "post_n": 5,
+    #     "ignore": ["7_Other"],
+    # },
+    # "question_22": {
+    #     "phases": QUESTION2PHASE["question_22"],
+    #     "pre_n": 5,
+    #     "post_n": 5,
+    #     "ignore": ["7_Other"],
+    # },
+    "question_23": {
+        "phases": QUESTION2PHASE["question_23"],
+        "pre_n": 5,
+        "post_n": 5,
+        "ignore": ["7_Other"],
+    },
+}
+
+
+START_INST = "<|user|>"
+END_INST = "<|end|>\n<|assistant|>"
+
+NAME2QUESTION = {
+    "question_1": "Did the helper introduce themself in the opening message? Answer only Yes or No.",
+    "question_4": "Did the helper actively listened to the texter's crisis? Answer only Yes or No.",
+    "question_5": "Did the helper reflect on the main issue that led the texter reach out? Answer only Yes or No.",
+    # "question_7": "Did the helper collaborated with the texter to identify the goal of the conversation? Answer only Yes or No.",
+    # "question_9": "Did the helper collaborated with the texter to create next steps? Answer only Yes or No.",
+    "question_10": "Did the helper explored texter's existing coping skills? Answer only Yes or No.",
+    # "question_11": "Did the helper explored texter’s social support? Answer only Yes or No.",
+    "question_14": "Did helper reflected the texter’s plan, reiterate coping skills, and end in a supportive way? Answer only Yes or No.",
+    # "question_15": "Did the helper consistently used Good Contact Techniques? Answer only Yes or No.",
+    "question_19": "Did the helper consistently reflected empathy through the conversation? Answer only Yes or No.",
+    # "question_21": "Did the helper shared personal information? Answer only Yes or No.",
+    # "question_22": "Did the helper gave advice? Answer only Yes or No.",
+    "question_23": "Did the helper explicitely initiated imminent risk assessment? Answer only Yes or No.",
+}
+
+NAME2PROMPT = {
+    k: "--------Conversation:\n{convo}\n{start_inst}" + v + "\n{end_inst}"
+    for k, v in NAME2QUESTION.items()
+}
+
+NAME2PROMPT_EXPL = {
+    k: v.split("Answer only Yes or No.")[0] + "Answer Yes or No, and give an explanation in a new line.\n{end_inst}"
+    for k, v in NAME2PROMPT.items()
+}
+
+QUESTIONDEFAULTS = {
+    "question_1": {True: "No, There was no evidence of Active Engagement", False: "No"},
+    "question_4": {True: "No, There was no evidence of Exploration Phase", False: "No"},
+    "question_5": {True: "No, There was no evidence of Exploration Phase", False: "No"},
+    # "question_7": {True: "N/A Texter disengaged, Not Applicable", False: "N/A"},
+    # "question_9": {True: "N/A Texter disengaged, Not Applicable", False: "N/A"},
+    "question_10": {True: "N/A Texter disengaged, Not Applicable", False: "N/A"},
+    # "question_11": {True: "N/A Texter disengaged, Not Applicable", False: "N/A"},
+    "question_14": {True: "N/A Texter disengaged, Not Applicable", False: "N/A"},
+    # "question_15": "Did the helper consistently used Good Contact Techniques? Answer only Yes or No.",
+    "question_19": {True: "N/A Texter disengaged, Not Applicable", False: "N/A"},
+    # "question_21": "Did the helper shared personal information? Answer only Yes or No.",
+    # "question_22": "Did the helper gave advice? Answer only Yes or No.",
+    "question_23": {True: "No, There was no evidence of Imminent Risk Assessment", False: "No"},
+}
+
+TEXTER_PREFIX = "texter"
+HELPER_PREFIX = "helper"
+
+TA_OPTIONS = ["N/A", "No", "Yes"]

models/ta_models/cpc_utils.py

@@ -0,0 +1,53 @@
+import streamlit as st
+from streamlit.logger import get_logger
+import requests
+import os
+from .config import model_name_or_path
+from transformers import AutoTokenizer
+from utils.mongo_utils import new_cpc_comparison
+from app_config import ENDPOINT_NAMES
+
+logger = get_logger(__name__)
+
+tokenizer = AutoTokenizer.from_pretrained(model_name_or_path, truncation_side="left")
+CPC_URL = os.environ["DATABRICKS_URL"].format(endpoint_name=ENDPOINT_NAMES["CPC"]['name'])
+HEADERS = {
+    "Authorization": f"Bearer {os.environ['DATABRICKS_TOKEN']}",
+    "Content-Type": "application/json",
+}
+
+def cpc_predict_message(context, input):
+    # context = memory.load_memory_variables({})[memory.memory_key]
+    encoding = tokenizer(
+        context,
+        input,
+        truncation="only_first",
+        max_length = tokenizer.model_max_length - 2,
+    )['input_ids']
+    body_request = {
+        "inputs": [tokenizer.decode(encoding[1:-1])]
+    }
+
+    try:
+        # Send request to Serving
+        response = requests.post(url=CPC_URL, headers=HEADERS, json=body_request)
+        if response.status_code == 200:
+            return response.json()['predictions'][0]["0"]["label"]
+        else:
+            raise Exception(f"Error in response: {response.json()}")
+    except Exception as e:
+        logger.debug(f"Error in response: {e}")
+        st.switch_page("pages/model_loader.py")
+
+def cpc_push2db(is_same):
+    text_is_same = "SAME" if is_same else "WRONG"
+    logger.debug(f"pushing new {text_is_same} CPC")
+    new_cpc_comparison(**{
+        "client": st.session_state['db_client'],
+        "convo_id": st.session_state['convo_id'],
+        "model": st.session_state['source'],
+        "context": st.session_state["context"],
+        "last_message": st.session_state["last_message"],
+        "ytrue": st.session_state["last_phase"] if is_same else st.session_state["sel_phase"],
+        "ypred": st.session_state["last_phase"],
+    })

models/ta_models/ta_filter_utils.py

@@ -0,0 +1,150 @@
+from itertools import chain
+from typing import List, Tuple
+
+import numpy as np
+import pandas as pd
+
+possible_movements = [-1, 1]
+
+
+def dfs(indexes: List[int], x0: int, i: int, cur_island: List[int], d=2):
+    """Deep First Search Implementation for 2D movement.
+    To consider an Island only move one step left or right
+    See possible movements
+
+    Args:
+        indexes (List[int]): Indexes of positive examples. i.e [20,21,23,50,51]
+        x0 (int): Initial island anchor
+        i (int): Current index to test against anchor
+        cur_island (List[int]): Current Island from anchor
+        d (int, optional): Bounding distance to consider an island. Defaults to 2. For example
+            the list [20,21,23,50,51] has two islands with d=2: (20,21,23), and (50,51) but it has
+            three islands with d=: (20,21), (23), and (50,51)
+    """
+    rows = len(indexes)
+    if i < 0 or i >= rows:
+        return
+    if indexes[i] in cur_island:
+        return
+    if abs(indexes[x0] - indexes[i]) > d:
+        return
+    # computing coordinates with x0 as base
+    cur_island.append(indexes[i])
+
+    # repeat dfs for neighbors
+    for movement in possible_movements:
+        dfs(indexes, i, i + movement, cur_island, d)
+
+
+def get_list_islands(indexes: List[int], **kwargs) -> List[List[int]]:
+    """Wrapper over DFS method to obtain islands from list of indexes of positive examples
+
+    Args:
+        indexes (List[int]): Indexes of positive examples. i.e [20,21,23,50,51]
+
+    Returns:
+        List[List[int]]: List of islands (each being a list)
+    """
+    islands = []
+    rows = len(indexes)
+    if rows == 0:
+        return islands
+
+    for i, valuei in enumerate(indexes):
+        # If already visited index in another dfs continue
+        if valuei in list(chain.from_iterable(islands)):
+            continue
+        # to hold coordinates of new island
+        cur_island = []
+        dfs(indexes, i, i, cur_island, **kwargs)
+
+        islands.append(cur_island)
+
+    return islands
+
+
+def get_phases_islands_minmax(
+    convo: pd.DataFrame,
+    phases: List[str],
+    column: str = "convo_part",
+    ignore: List[str] = [],
+    **kwargs,
+) -> List[Tuple[int]]:
+    """Given a conversation with predicted Phases (or Parts), get minimum and maximum index of calculated islands.
+
+    Args:
+        convo (pd.DataFrame): Conversation with predicted phases stored in `column`
+        phases (List[str]): Phases to filter in
+        column (str, optional): Column where predicted phases information is stored. Defaults to "convo_part".
+        ignore (List[str], optional): Ignore phases list. Defaults to [].
+
+    Returns:
+        List[Tuple[int]]: Minimum and Maximum values of calulated islands. i.e [(20,30), (40,60)]
+    """
+
+    reset = convo.query(f"{column}=={column} and {column} not in @ignore").reset_index()
+    sub_ = reset.query(f"{column} in @phases").copy()
+    indexes = sub_.index.tolist()
+    islands = get_list_islands(indexes, **kwargs)
+    if len(islands) > 1:
+        # If there is more than one island we want to make sure to root out comparable small islands
+        # I.e. if there is an island with 10 messages, and island of 1 messages is not useful in that context.
+        max_len = np.max([len(x) for x in islands])
+        len_cut = 3 if max_len > 9 else 2 if max_len > 3 else 1
+        islands = [x for x in islands if len(x) > len_cut]
+
+    islands = [reset.iloc[x] for x in islands]
+    minmax_islands = [(x["index"].min(), x["index"].max()) for x in islands]
+
+    return minmax_islands
+
+
+def filter_convo(
+    convo: pd.DataFrame,
+    phases: List[str],
+    column: str = "convo_part",
+    strategy: str = "islands",
+    pre_n: int = 5,
+    post_n: int = 5,
+    return_all_on_empty: bool = False,
+    **kwargs,
+) -> pd.DataFrame:
+    """Filter convo to include only specified phases. Take into account that sometimes predicted phases
+    can be messy. i.e. a prediciton of explore, explore, explore, safety_planning, explore; should return all
+    these messages as explore (probably safety_planning message has a low probability here.)
+
+    Args:
+        convo (pd.DataFrame): Conversation with predicted phases stored in `column`
+        phases (List[str]): Phases to filter in
+        column (str, optional): Column where predicted phases information is stored. Defaults to "convo_part".
+        strategy (str, optional): Strategy to use, can be minmax or islands. Defaults to "islands".
+        pre_n (int, optional): How many messages pre-phase to include. Defaults to 5.
+        post_n (int, optional): How many messages post-phase to include. Defaults to 5.
+        return_all_on_empty (bool, optional): Whether to return all messages when specified phases is not found. Defaults to False.
+
+    Returns:
+        pd.DataFrame: Filtered messages from the convo
+    """
+    if phases == ["ALL"]:
+        minidx = convo.index.min()
+        maxidx = convo.index.max()
+        minmax = [(minidx, maxidx)]
+    elif strategy == "minmax":
+        minidx = convo.query(f"{column} in @phases").index.min()
+        maxidx = convo.query(f"{column} in @phases").index.max() + 1
+        minmax = [(minidx, maxidx)]
+    elif strategy == "islands":
+        minmax = get_phases_islands_minmax(convo, phases, column, **kwargs)
+    parts = []
+    for minidx, maxidx in minmax:
+        minidx = max(convo.index.min(), minidx - pre_n)
+        maxidx = min(convo.index.max(), maxidx + post_n)
+        parts.append(convo.loc[minidx:maxidx])
+    if len(parts) == 0:
+        if return_all_on_empty:
+            return convo
+        else:
+            return pd.DataFrame(columns=convo.columns)
+    filtered = pd.concat(parts)
+    filtered = filtered[~filtered.index.duplicated(keep="first")]
+    return filtered

models/ta_models/ta_prompt_utils.py

@@ -0,0 +1,128 @@
+import inspect
+
+import pandas as pd
+
+from .config import QUESTION2FILTERARGS, TEXTER_PREFIX, HELPER_PREFIX
+
+# Utils to filter convo according to a phase
+from .ta_filter_utils import filter_convo
+
+
+def join_messages(
+    grp: pd.DataFrame, texter_prefix: str = "texter", helper_prefix: str = "helper"
+) -> str:
+    """join messages from dataframe using texter an helper prefixes
+
+    Args:
+        grp (pd.DataFrame): conversation in DataFrame with each row corresponding to each **message**.
+            Must have the following columns:
+            - actor_role
+            - message
+
+        texter_prefix (str, optional): prefix to use as the texter. Defaults to "texter".
+        helper_prefix (str, optional): prefix to use as the counselor (helper). Defaults to "helper".
+
+    Returns:
+        str: joined messages string separated by prefixes
+    """
+
+    if "actor_role" not in grp:
+        raise Exception("Column 'actor_role' not in DataFrame")
+    if "message" not in grp:
+        raise Exception("Column 'message' not in DataFrame")
+
+    roles = grp.actor_role.replace(
+        {"texter": texter_prefix, "counselor": helper_prefix, "helper": helper_prefix}
+    )
+    messages = roles.str.strip() + ": " + grp.message.str.strip()
+    return "\n".join(messages)
+
+
+def _get_context(grp: pd.DataFrame, **kwargs) -> str:
+    """Get context as a str taking into account message to delete, context marker
+    and the type of question to use. This allows for better truncation later
+
+    Args:
+        grp (pd.DataFrame): conversation in DataFrame with each row corresponding to each **message**.
+            Must have the following columns:
+            - actor_role
+            - message
+            - `column`
+        column (str): column name in which the marker of the problem is
+
+    Returns:
+        pd.DataFrame: joined messages string separated by prefixes
+    """
+
+    if "actor_role" not in grp:
+        raise Exception("Column 'actor_role' not in DataFrame")
+    if "message" not in grp:
+        raise Exception("Column 'message' not in DataFrame")
+
+    join_args = list(inspect.signature(join_messages).parameters)
+    join_kwargs = {k: kwargs.pop(k) for k in dict(kwargs) if k in join_args}
+
+    ## DEPRECATED
+    # context_args = list(inspect.signature(get_context_on_marker).parameters)
+    # context_kwargs = {k: kwargs.pop(k) for k in dict(kwargs) if k in context_args}
+
+    return join_messages(grp, **join_kwargs)
+
+
+def load_context(
+    messages: pd.DataFrame,
+    question: str,
+    message_col: str,
+    col_type: str,
+    inference: bool = False,
+    **kwargs,
+) -> pd.DataFrame:
+    """Load and filter conversation from messages given a question (with configured parameters of what phase that question is answered)
+
+    Args:
+        messages (pd.DataFrame): Messages dataframe with conversation_id, actor_role, `message_col` and phase prediction
+        question (str): Question to get context to
+        message_col (str): Column where messages are
+        col_type (str): type of message_col, can be "individual" or "joined"
+        base_dir (str, optional): Base directory to find model base args. Defaults to "../../".
+
+    Raises:
+        Exception: If question is not supported
+
+    Returns:
+        pd.DataFrame: filtered messages according to question configuration
+    """
+
+    if question not in QUESTION2FILTERARGS:
+        raise Exception(f"Question {question} not supported")
+
+    texter_prefix = TEXTER_PREFIX
+    helper_prefix = HELPER_PREFIX
+    context_data = messages.copy()
+
+    def convo_cpc_get_context(grp, **kwargs):
+        """Filter convo according to Convo Phase Classifier (CPC) predictions"""
+        context_ = filter_convo(grp, **QUESTION2FILTERARGS[question])
+        return _get_context(context_, **kwargs)
+
+    if col_type == "individual":
+        if "actor_role" in context_data:
+            context_data.dropna(subset=["actor_role"], inplace=True)
+        if "delete_message" in context_data:
+            context_data.delete_message.replace({1: True}, inplace=True)
+            context_data.delete_message.fillna(False, inplace=True)
+
+        context_data = (
+            context_data.groupby("conversation_id")
+            .apply(
+                convo_cpc_get_context,
+                helper_prefix=helper_prefix,
+                texter_prefix=texter_prefix,
+            )
+            .rename("q_context")
+        )
+    elif col_type == "joined":
+        context_data = context_data.groupby("conversation_id")[[message_col]].max()
+        context_data.rename(columns={message_col: "q_context"}, inplace=True)
+
+    return context_data

models/ta_models/ta_utils.py

@@ -0,0 +1,127 @@
+import os
+import re
+import requests
+import string
+import streamlit as st
+from streamlit.logger import get_logger
+from app_config import ENDPOINT_NAMES
+from models.ta_models.config import NAME2PROMPT, NAME2PROMPT_EXPL, START_INST, END_INST, QUESTIONDEFAULTS, TA_OPTIONS, NAME2QUESTION
+import pandas as pd
+from langchain_core.messages import AIMessage, HumanMessage
+from models.ta_models.ta_prompt_utils import load_context
+from utils.mongo_utils import new_convo_scoring_comparison
+
+logger = get_logger(__name__)
+TA_URL = os.environ["DATABRICKS_URL"].format(endpoint_name=ENDPOINT_NAMES["training_adherence"]['name'])
+HEADERS = {
+    "Authorization": f"Bearer {os.environ['DATABRICKS_TOKEN']}",
+    "Content-Type": "application/json",
+}
+
+def memory2df(memory, conversation_id="convo1234"):
+    df = []
+    for i, msg in enumerate(memory.buffer_as_messages):
+        actor_role = "texter" if type(msg) == AIMessage else "helper" if type(msg) == HumanMessage else None
+        if actor_role:
+            convo_part = msg.response_metadata.get("phase",None)
+            row = {"conversation_id":conversation_id, "message_number":i+1, "actor_role":actor_role, "message":msg.content, "convo_part":convo_part}
+            df.append(row)
+
+    return pd.DataFrame(df)
+
+def get_default(question, make_explanation=False):
+    return QUESTIONDEFAULTS[question][make_explanation]
+
+def get_context(memory, question, make_explanation=False, **kwargs):
+    df = memory2df(memory, **kwargs)
+    contexti = load_context(df, question, "messages", "individual").iloc[0]
+    if contexti == "":
+        return ""
+    
+    if make_explanation:
+        return NAME2PROMPT_EXPL[question].format(convo=contexti, start_inst=START_INST, end_inst=END_INST)
+    else:
+        return NAME2PROMPT[question].format(convo=contexti, start_inst=START_INST, end_inst=END_INST)
+
+def post_process_response(full_response, delimiter="\n\n", n=2):
+    parts = full_response.split(delimiter)[:n]
+    response = extract_response(parts[0])
+    logger.debug(f"Response extracted is {response}")
+    if len(parts) > 1:
+        if len(parts[0]) < len(parts[1]):
+            full_response = parts[1]
+        else: full_response = parts[0]
+    else:
+        full_response = parts[0]
+    explanation = full_response.lstrip(response).lstrip(string.punctuation)
+    explanation = explanation.strip()
+    logger.debug(f"Explanation extracted is {explanation}")
+    return response, explanation
+
+def TA_predict_convo(memory, question, make_explanation=False, **kwargs):
+    full_convo = memory.load_memory_variables({})[memory.memory_key]
+    PROMPT = get_context(memory, question, make_explanation=False, **kwargs)
+    logger.debug(f"Raw TA prompt is {PROMPT}")
+    if PROMPT == "":
+        full_response = get_default(question, make_explanation)
+        return full_convo, PROMPT, full_response
+    
+    body_request = {
+        "prompt": PROMPT,
+        "temperature": 0,
+        "max_tokens": 3,
+    }
+
+    try:
+        # Send request to Serving
+        response = requests.post(url=TA_URL, headers=HEADERS, json=body_request)
+        if response.status_code == 200:
+            response = response.json()
+        else:
+            raise Exception(f"Error in response: {response.json()}")
+        full_response = response[0]['choices'][0]['text']
+        if not make_explanation:
+            return full_convo, PROMPT, full_response
+        else:
+            extract_response, _ = post_process_response(full_response)
+            PROMPT = get_context(memory, question, make_explanation=True, **kwargs)
+            PROMPT = PROMPT + f" {extract_response}"
+            logger.debug(f"Raw TA prompt for Explanation is {PROMPT}")
+            body_request["prompt"] = PROMPT
+            body_request["max_tokens"] = 128
+            response_expl = requests.post(url=TA_URL, headers=HEADERS, json=body_request)
+            if response_expl.status_code == 200:
+                response_expl = response_expl.json()
+            else:
+                raise Exception(f"Error in response: {response_expl.json()}")
+            full_response_expl = f"{extract_response} {response_expl[0]['choices'][0]['text']}"
+            return full_convo, PROMPT, full_response_expl
+    except Exception as e:
+        logger.debug(f"Error in response: {e}")
+        st.switch_page("pages/model_loader.py")
+
+def extract_response(x: str, default: str = TA_OPTIONS[0]) -> str:
+    """Extract Response from generated answer
+    Extract only search strings
+
+    Args:
+        x (str): prediction
+        default (str, optional): default in case no response founds. Defaults to "N/A".
+
+    Returns:
+        str: _description_
+    """
+
+    try:
+        return re.findall("|".join(TA_OPTIONS), x)[0]
+    except Exception:
+        return default
+    
+def ta_push_convo_comparison(ytrue, ypred):
+    new_convo_scoring_comparison(**{
+        "client": st.session_state['db_client'],
+        "convo_id": st.session_state['convo_id'],
+        "context": st.session_state["context"] + "\nhelper:" + st.session_state["last_message"],
+        "ytrue": ytrue,
+        "ypred": ypred,
+    })

pages/convosim.py

@@ -0,0 +1,185 @@
+import os
+import streamlit as st
+from streamlit.logger import get_logger
+from langchain.schema.messages import HumanMessage
+from utils.mongo_utils import get_db_client
+from utils.app_utils import create_memory_add_initial_message, get_random_name,  DEFAULT_NAMES_DF, are_models_alive
+from utils.memory_utils import clear_memory, push_convo2db
+from utils.chain_utils import get_chain, custom_chain_predict
+from app_config import ISSUES, SOURCES, source2label, issue2label, MAX_MSG_COUNT, WARN_MSG_COUT
+from models.ta_models.config import CPC_LBL_OPTS, cpc_label2str, BP_LAB2STR
+from models.ta_models.cpc_utils import cpc_push2db
+from models.ta_models.bp_utils import bp_predict_message, bp_push2db
+
+logger = get_logger(__name__)
+temperature = 0.8
+# username = "barb-chase" #"ivnban-ctl"
+st.set_page_config(page_title="Conversation Simulator")
+
+if "sent_messages" not in st.session_state:
+    st.session_state['sent_messages'] = 0
+    if not are_models_alive():
+        st.switch_page("pages/model_loader.py")
+
+if "total_messages" not in st.session_state:
+    st.session_state['total_messages'] = 0
+if "issue" not in st.session_state:
+    st.session_state['issue'] = ISSUES[0]
+if 'previous_source' not in st.session_state:
+    st.session_state['previous_source'] = SOURCES[0]
+if 'db_client' not in st.session_state:
+    st.session_state["db_client"] = get_db_client()
+if 'texter_name' not in st.session_state:
+    st.session_state["texter_name"] = get_random_name(names_df=DEFAULT_NAMES_DF)
+    logger.debug(f"texter name is {st.session_state['texter_name']}")
+if "last_phase" not in st.session_state:
+    st.session_state["last_phase"] = CPC_LBL_OPTS[0]
+    # st.session_state["sel_phase"] = CPC_LBL_OPTS[0]
+if "changed_cpc" not in st.session_state:
+    st.session_state["changed_cpc"] = False
+if "changed_bp" not in st.session_state:
+    st.session_state["changed_bp"] = False
+
+# st.session_state["sel_phase"] = st.session_state["last_phase"]
+
+memories = {'memory':{"issue": st.session_state['issue'], "source": st.session_state['previous_source']}}
+
+with st.sidebar:
+    username = st.text_input("Username", value='Dani', max_chars=30)
+    if 'counselor_name' not in st.session_state:
+        st.session_state["counselor_name"] = username #get_random_name(names_df=DEFAULT_NAMES_DF)
+    # temperature = st.slider("Temperature", 0., 1., value=0.8, step=0.1)
+    issue = st.selectbox("Select a Scenario", ISSUES, index=ISSUES.index(st.session_state['issue']), format_func=issue2label,
+                            on_change=clear_memory, kwargs={"memories":memories, "username":username, "language":"English"}
+                        )
+    supported_languages = ['en', "es"] if issue == "Anxiety" else ['en']
+    language = st.selectbox("Select a Language", supported_languages, index=0,
+                            format_func=lambda x: "English" if x=="en" else "Spanish",
+                            on_change=clear_memory, kwargs={"memories":memories, "username":username, "language":"English"}
+                        )
+                            
+    source = st.selectbox("Select a source Model A", SOURCES, index=0,
+                          format_func=source2label, key="source"
+                        )
+
+changed_source = any([
+    st.session_state['previous_source'] != source,
+    st.session_state['issue'] != issue,
+    st.session_state['counselor_name'] != username,
+])
+if changed_source:
+    st.session_state["counselor_name"] = username
+    st.session_state["texter_name"] = get_random_name(names_df=DEFAULT_NAMES_DF)
+    logger.debug(f"texter name is {st.session_state['texter_name']}")
+    st.session_state['previous_source'] = source
+    st.session_state['issue'] = issue
+    st.session_state['sent_messages'] = 0
+    st.session_state['total_messages'] = 0
+create_memory_add_initial_message(memories,
+                                  issue, 
+                                  language, 
+                                  changed_source=changed_source, 
+                                  counselor_name=st.session_state["counselor_name"], 
+                                  texter_name=st.session_state["texter_name"])
+st.session_state['previous_source'] = source
+memoryA = st.session_state[list(memories.keys())[0]]
+# issue only without "." marker for model compatibility
+llm_chain, stopper = get_chain(issue, language, source, memoryA, temperature, texter_name=st.session_state["texter_name"])
+
+st.title("💬 Simulator") 
+st.session_state['total_messages'] = len(memoryA.chat_memory.messages)
+for msg in memoryA.buffer_as_messages:
+    role = "user" if type(msg) == HumanMessage else "assistant"
+    st.chat_message(role).write(msg.content)
+
+def sent_request_llm(llm_chain, prompt):
+    st.session_state['sent_messages'] += 1
+    st.chat_message("user").write(prompt)
+    responses = custom_chain_predict(llm_chain, prompt, stopper)
+    for response in responses:
+        st.chat_message("assistant").write(response)
+
+# @st.dialog("Bad Practice Detected")
+# def confirm_bp(bp_prediction, prompt):
+#     bps = [BP_LAB2STR[x['label']] for x in bp_prediction if x['score']]
+#     st.markdown(f"The last message was considered :red[{' and '.join(bps)}]")
+#     "Are you sure you want to send this message?"
+#     newprompt = st.text_input("Change message to:")
+#     "If you do not want to change leave textbox empty"
+#     for bp in BP_LAB2STR.keys():
+#         _ = st.checkbox(f"Original Message was {BP_LAB2STR[bp]}", key=f"chkbx_{bp}", value=BP_LAB2STR[bp] in bps)
+
+#     if st.button("Confirm"):
+#         if newprompt is not None and newprompt != "":
+#             prompt = newprompt
+#         bp_push2db(
+#             {bp:st.session_state[f"chkbx_{bp}"] for bp in BP_LAB2STR.keys()}
+#         )
+#         sent_request_llm(llm_chain, prompt)
+#         st.rerun()
+
+if prompt := st.chat_input(disabled=st.session_state['total_messages'] > MAX_MSG_COUNT - 4): #account for next interaction
+    if 'convo_id' not in st.session_state:
+        push_convo2db(memories, username, language)
+    st.session_state['context'] = llm_chain.memory.load_memory_variables({})[llm_chain.memory.memory_key]
+    st.session_state['last_message'] = prompt
+    if (not st.session_state.changed_cpc) and st.session_state["sent_messages"] > 0:
+        cpc_push2db(True)
+    else: st.session_state.changed_cpc = False
+    if (not st.session_state.changed_bp) and st.session_state["sent_messages"] > 0:
+        bp_push2db({x['label']:x['score'] for x in st.session_state['bp_prediction']})
+    else: st.session_state.changed_bp = False
+
+    context = llm_chain.memory.load_memory_variables({})[llm_chain.memory.memory_key]
+    st.session_state['bp_prediction'] = bp_predict_message(context, prompt)
+    if any([x['score'] for x in st.session_state['bp_prediction']]):
+        for bp in st.session_state['bp_prediction']:
+            if bp["score"]:
+                st.toast(f"Detected {BP_LAB2STR[bp['label']]} in the last message!", icon=":material/warning:")
+    
+    sent_request_llm(llm_chain, prompt)
+    # else:
+    #     sent_request_llm(llm_chain, prompt)
+
+with st.sidebar:
+    st.divider()
+    st.markdown(f"### Total Sent Messages: :red[**{st.session_state['sent_messages']}**]")
+    st.markdown(f"### Total Messages: :red[**{st.session_state['total_messages']}**]")
+    # st.markdown()
+    def on_change_cpc():
+        cpc_push2db(False)
+        st.session_state.changed_cpc = True
+    def on_change_bp():
+        bp_push2db()
+        st.session_state.changed_bp = True
+
+    if st.session_state["sent_messages"] > 0:
+        _ = st.selectbox(f"""Last Human Message was considered :blue[**{
+            cpc_label2str(st.session_state['last_phase'])
+        }**]. If not please select from the following options""", 
+
+            CPC_LBL_OPTS, index=None,format_func=cpc_label2str, on_change=on_change_cpc,
+            key="sel_phase",
+        )
+
+        BPs = [BP_LAB2STR[x['label']] for x in st.session_state['bp_prediction'] if x['score']]
+        selecttitle = f"""Last Human Message was considered :blue[**{
+            " and ".join(BPs)
+        }**].""" if len(BPs) > 0 else "Last Human Message was NOT considered Bad Practice."
+        _ = st.selectbox(selecttitle + " If not please select from the following options""", 
+
+            ["None", "Advice", "Personal Info", "Advice & Personal Info"], index=None, on_change=on_change_bp,
+            key="sel_bp"
+        )
+
+        if st.button("Score Conversation"):
+            st.switch_page("pages/training_adherence.py")
+
+st.session_state['total_messages'] = len(memoryA.chat_memory.messages)
+if st.session_state['total_messages'] >= MAX_MSG_COUNT:
+    st.toast(f"Total of {MAX_MSG_COUNT} Messages reached. Conversation Ended", icon=":material/verified:")
+elif st.session_state['total_messages'] >= WARN_MSG_COUT:
+    st.toast(f"The conversation will end at {MAX_MSG_COUNT} Total Messages ", icon=":material/warning:")
+
+if not are_models_alive():
+    st.switch_page("pages/model_loader.py")

pages/model_loader.py

@@ -0,0 +1,56 @@
+import time
+import streamlit as st
+from streamlit.logger import get_logger
+from utils.app_utils import is_model_alive
+from app_config import ENDPOINT_NAMES
+
+logger = get_logger(__name__)
+
+st.set_page_config(page_title="Conversation Simulator")
+
+models_alive = False
+start = time.time()
+
+MODELS2LOAD = {
+    "CPC": {"model_name": "Phase Classifier", "loaded":None,},
+    "CTL_llama3": {"model_name": "Texter Simulator", "loaded":None,},
+    "BadPractices": {"model_name": "Advice Identificator", "loaded":None},
+    "training_adherence": {"model_name": "Training Adherence", "loaded":None},
+}
+
+def write_model_status(writer, model_name, loaded, fail=False):
+    if loaded == "200":
+        writer.write(f"✅ - {model_name} Loaded")
+    if fail:
+        if loaded in ["400", "500"]:
+            writer.write(f"❌ - {model_name} Failed to Load, Contact [email protected]")
+        elif loaded == "404":
+            writer.write(f"❌ - {model_name} Still loading, please try in a couple of minutes")
+    else:
+        writer.write(f"🔄 - {model_name} Loading")
+
+with st.status("Loading Models Please Wait...(this may take up to 5 min)", expanded=True) as status:
+
+    for k in MODELS2LOAD.keys():
+        MODELS2LOAD[k]["writer"] = st.empty()
+
+    while not models_alive:
+        time.sleep(2)
+        for name, config in MODELS2LOAD.items():
+            config["loaded"] = is_model_alive(**ENDPOINT_NAMES[name])
+
+        models_alive = all([x['loaded']=="200" for x in MODELS2LOAD.values()])
+        
+        for _, config in MODELS2LOAD.items():
+            write_model_status(**config)
+        
+        if int(time.time()-start) > 30:
+            status.update(
+                label="Models took too long to load. Please Refresh Page in a couple of minutes", state="error", expanded=True
+            )
+            for _, config in MODELS2LOAD.items():
+                write_model_status(**config, fail=True)
+            break
+
+if models_alive:
+    st.switch_page("pages/convosim.py")

pages/training_adherence.py

@@ -0,0 +1,86 @@
+import streamlit as st
+import numpy as np
+from collections import defaultdict
+from langchain_core.messages import HumanMessage
+from utils.app_utils import are_models_alive
+from models.ta_models.ta_utils import TA_predict_convo, ta_push_convo_comparison, post_process_response
+from models.ta_models.config import QUESTION2PHASE, NAME2QUESTION, TA_OPTIONS
+
+st.set_page_config(page_title="Conversation Simulator - Scoring")
+
+if not are_models_alive():
+    st.switch_page("pages/model_loader.py")
+
+if "memory" not in st.session_state:
+    st.switch_page("pages/convosim.py")
+
+memory = st.session_state['memory']
+progress_text = "Scoring Conversation using AI models ..."
+
+@st.cache_data(show_spinner=False)
+def get_ta_responses():
+    my_bar = st.progress(0, text=progress_text)
+    data = defaultdict(defaultdict)
+    for i, question in enumerate(QUESTION2PHASE.keys()):
+        # responses = ["Yes, The helper showed some respect.",
+        #             "Yes. The helper is good! No doubt",
+        #             "N/A, Texter disengaged.",
+        #             "No. While texter is trying is lacking.",
+        #             "No \n\n This is an explanation."]
+        # full_response = np.random.choice(responses)
+        full_convo, prompt, full_response = TA_predict_convo(memory, question, make_explanation=True, conversation_id=st.session_state['convo_id'])
+        response, explanation = post_process_response(full_response)
+        data[question]["response"] = response
+        data[question]["explanation"] = explanation
+        my_bar.progress((i+1) / len(QUESTION2PHASE.keys()), text = progress_text)
+        import time
+        time.sleep(2)
+    my_bar.empty()
+    return data
+
+with st.container():
+    col1, col2 = st.columns(2)
+    if col1.button("Go Back"):
+        get_ta_responses.clear()
+        st.switch_page("pages/convosim.py")
+    expl = col2.checkbox("Show Scoring Explanations")
+
+tab1, tab2 = st.tabs(["Scoring", "Conversation"])
+data = get_ta_responses()
+
+with tab2:
+    for msg in memory.buffer_as_messages:
+        role = "user" if type(msg) == HumanMessage else "assistant"
+        st.chat_message(role).write(msg.content)
+
+with tab1:
+    for question in QUESTION2PHASE.keys():
+        with st.container(border=True):
+            question_str = NAME2QUESTION[question].split(' Answer')[0]
+            st.radio(
+                f"**{question_str}**", options=TA_OPTIONS, 
+                index=TA_OPTIONS.index(data[question]['response']), horizontal=True,
+                key=f"{question}_manual"
+            )
+            if expl:
+                st.text_area(
+                    label="", value=data[question]["explanation"], key=f"{question}_explanation_manual"
+                )
+                # st.write(data[question]["explanation"])
+
+    with st.container():
+        col1, col2 = st.columns(2)
+        if col1.button("Go Back", key="goback2"):
+            get_ta_responses.clear()
+            st.switch_page("pages/convosim.py")
+        if col2.button("Submit Scoring", type="primary"):
+            ytrue = {
+                question: {
+                    "response": st.session_state[f"{question}_manual"],
+                    "explanation": st.session_state[f"{question}_explanation_manual"] if expl else "",
+                } 
+                for question in QUESTION2PHASE.keys()
+            }
+            ta_push_convo_comparison(ytrue, data)
+            get_ta_responses.clear()
+            st.switch_page("pages/convosim.py")

requirements.txt

@@ -4,4 +4,5 @@ mlflow==2.9.0
 langchain==0.3.0
 langchain-openai==0.2.0
 langchain-community==0.3.0
-streamlit==1.38.0
+streamlit==1.38.0
+transformers==4.43.0

utils/app_utils.py

@@ -1,19 +1,22 @@
 import pandas as pd
 import streamlit as st
 from streamlit.logger import get_logger
-import langchain
+import os
+import requests
 
-
-from app_config import ENVIRON
+from app_config import ENDPOINT_NAMES
 from utils.memory_utils import change_memories
 from models.model_seeds import seeds
 
-langchain.verbose = ENVIRON =="dev"
 logger = get_logger(__name__)
 
 # TODO: Include more variable and representative names
 DEFAULT_NAMES = ["Olivia", "Kit", "Abby", "Tom", "Carolyne", "Jessiny"]
 DEFAULT_NAMES_DF = pd.read_csv("./utils/names.csv")
+HEADERS = {
+    "Authorization": f"Bearer {os.environ['DATABRICKS_TOKEN']}",
+    "Content-Type": "application/json",
+}
 
 def get_random_name(gender="Neutral", ethnical_group="Neutral", names_df=None):
         if names_df is None:
@@ -61,4 +64,47 @@ def create_memory_add_initial_message(memories, issue, language, changed_source=
         if len(st.session_state[memory].buffer_as_messages) < 1:
             add_initial_message(issue, language, st.session_state[memory], texter_name=texter_name, counselor_name=counselor_name)
 
-    
+def is_model_alive(name, timeout=2, model_type="classificator"):
+    if model_type!="openai":
+        endpoint_url=os.environ['DATABRICKS_URL'].format(endpoint_name=name)
+        headers = HEADERS
+        if model_type == "classificator":
+            body_request = {
+                "inputs": [""]
+            }
+        elif model_type == "text-completion":
+            body_request = {
+                "prompt": "",
+                "temperature": 0,
+                "max_tokens": 1,
+            }
+        elif model_type == "text-generation":
+            body_request = {
+                "messages": [{"role":"user","content":""}],
+                "max_tokens": 1,
+                "temperature": 0
+            }
+
+        else:
+            raise Exception(f"Model Type {model_type} not supported") 
+        try:
+            response = requests.post(url=endpoint_url, headers=HEADERS, json=body_request, timeout=timeout)
+            return str(response.status_code)
+        except:
+            return "404"
+    else:
+        endpoint_url="https://api.openai.com/v1/models"
+        headers = {"Authorization": f"Bearer {os.environ['OPENAI_API_KEY']}",}
+        try:
+            _ = requests.get(url=endpoint_url, headers=headers, timeout=1)
+            return "200"
+        except:
+            return "404" 
+
+@st.cache_data(ttl=300, show_spinner=False)
+def are_models_alive():
+    models_alive = []
+    for config in ENDPOINT_NAMES.values():
+        models_alive.append(is_model_alive(**config))
+    openai = is_model_alive("openai", model_type="openai")
+    return all([x=="200" for x in models_alive + [openai]])

utils/chain_utils.py

@@ -1,9 +1,11 @@
+import streamlit as st
 from streamlit.logger import get_logger
-from models.model_seeds import seeds
+from langchain_core.messages import HumanMessage
 from models.openai.finetuned_models import finetuned_models, get_finetuned_chain
 from models.openai.role_models import get_role_chain, get_template_role_models
 from models.databricks.scenario_sim_biz import get_databricks_biz_chain
 from models.databricks.texter_sim_llm import get_databricks_chain
+from models.ta_models.cpc_utils import cpc_predict_message
 
 logger = get_logger(__name__)
 
@@ -32,7 +34,12 @@ def custom_chain_predict(llm_chain, input, stop):
     llm_chain._validate_inputs(inputs)
     outputs = llm_chain._call(inputs)
     llm_chain._validate_outputs(outputs)
-    llm_chain.memory.chat_memory.add_user_message(inputs['input'])
+    phase = cpc_predict_message(st.session_state['context'], st.session_state['last_message'])
+    st.session_state['last_phase'] = phase
+    logger.debug(phase)
+    llm_chain.memory.chat_memory.add_user_message(
+        HumanMessage(inputs['input'], response_metadata={"phase":phase})
+    )
     for out in outputs[llm_chain.output_key]:
         llm_chain.memory.chat_memory.add_ai_message(out)
     return outputs[llm_chain.output_key]

utils/memory_utils.py

@@ -30,7 +30,6 @@ def change_memories(memories, language, changed_source=False):
     
     if ("convo_id" in st.session_state) and changed_source:
         del st.session_state['convo_id']
-    
 
 def clear_memory(memories, username, language):
     for memory, _ in memories.items():

utils/mongo_utils.py

@@ -4,7 +4,7 @@ import streamlit as st
 from streamlit.logger import get_logger
 from pymongo.mongo_client import MongoClient
 from pymongo.server_api import ServerApi
-from app_config import DB_SCHEMA, DB_COMPLETIONS, DB_CONVOS, DB_BATTLES, DB_ERRORS
+from app_config import DB_SCHEMA, DB_COMPLETIONS, DB_CONVOS, DB_BATTLES, DB_ERRORS, DB_CPC, DB_BP, DB_TA
 
 DB_URL = os.environ['MONGO_URL']
 DB_USR = os.environ['MONGO_USR']
@@ -19,7 +19,7 @@ def get_db_client():
     # Send a ping to confirm a successful connection
     try:
         client.admin.command('ping')
-        logger.info(f"DBUTILS: Pinged your deployment. You successfully connected to MongoDB!")
+        logger.debug(f"DBUTILS: Pinged your deployment. You successfully connected to MongoDB!")
         return client
     except Exception as e:
         logger.error(e)
@@ -38,7 +38,7 @@ def new_convo(client, issue, language, username, is_comparison, model_one, model
     db = client[DB_SCHEMA]
     convos = db[DB_CONVOS]
     convo_id = convos.insert_one(convo).inserted_id
-    logger.info(f"DBUTILS: new convo id is {convo_id}")
+    logger.debug(f"DBUTILS: new convo id is {convo_id}")
     st.session_state['convo_id'] = convo_id
 
 def new_comparison(client, prompt_timestamp, completion_timestamp, 
@@ -66,7 +66,7 @@ def new_comparison(client, prompt_timestamp, completion_timestamp,
     db = client[DB_SCHEMA]
     comparisons = db[DB_COMPLETIONS]
     comparison_id = comparisons.insert_one(comparison).inserted_id
-    logger.info(f"DBUTILS: new comparison id is {comparison_id}")
+    logger.debug(f"DBUTILS: new comparison id is {comparison_id}")
     st.session_state['comparison_id'] = comparison_id
 
 def new_battle_result(client, comparison_id, convo_id, username, model_one, model_two, winner):
@@ -84,7 +84,7 @@ def new_battle_result(client, comparison_id, convo_id, username, model_one, mode
     db = client[DB_SCHEMA]
     battles = db[DB_BATTLES]
     battle_id = battles.insert_one(battle).inserted_id
-    logger.info(f"DBUTILS: new battle id is {battle_id}")
+    logger.debug(f"DBUTILS: new battle id is {battle_id}")
 
 def new_completion_error(client, comparison_id, username, model):
     error = {
@@ -97,7 +97,58 @@ def new_completion_error(client, comparison_id, username, model):
     db = client[DB_SCHEMA]
     errors = db[DB_ERRORS]
     error_id = errors.insert_one(error).inserted_id
-    logger.info(f"DBUTILS: new error id is {error_id}")
+    logger.debug(f"DBUTILS: new error id is {error_id}")
+
+def new_cpc_comparison(client, convo_id, model, context, last_message, ytrue, ypred):
+    # context = memory.load_memory_variables({})[memory.memory_key]
+    comp = {
+        "CPC_timestamp": dt.datetime.now(tz=dt.timezone.utc),
+        "conversation_id": convo_id,
+        "model": model,    
+        "context": context,
+        "last_message": last_message,
+        "predicted_phase": ypred,
+        "manual_phase": ytrue,
+    }
+
+    db = client[DB_SCHEMA]
+    cpc_comps = db[DB_CPC]
+    comarison_id = cpc_comps.insert_one(comp).inserted_id
+    logger.debug(f"DBUTILS: new error id is {comarison_id}")
+
+def new_bp_comparison(client, convo_id, model, context, last_message, ytrue, ypred):
+    # context = memory.load_memory_variables({})[memory.memory_key]
+    comp = {
+        "BP_timestamp": dt.datetime.now(tz=dt.timezone.utc),
+        "conversation_id": convo_id,
+        "model": model,    
+        "context": context,
+        "last_message": last_message,
+        "is_advice": ypred["is_advice"],
+        "manual_is_advice": ytrue["is_advice"],
+        "is_pi": ypred["is_personal_info"],
+        "manual_is_pi": ytrue["is_personal_info"],
+    }
+
+    db = client[DB_SCHEMA]
+    bp_comps = db[DB_BP]
+    comarison_id = bp_comps.insert_one(comp).inserted_id
+    logger.debug(f"DBUTILS: new BP id is {comarison_id}")
+
+def new_convo_scoring_comparison(client, convo_id, context, ytrue, ypred):
+    # context = memory.load_memory_variables({})[memory.memory_key]
+    comp = {
+        "scoring_timestamp": dt.datetime.now(tz=dt.timezone.utc),
+        "conversation_id": convo_id,   
+        "context": context,
+        "manual_scoring": ytrue,
+        "model_scoring": ypred,
+    }
+
+    db = client[DB_SCHEMA]
+    ta_comps = db[DB_TA]
+    comarison_id = ta_comps.insert_one(comp).inserted_id
+    logger.debug(f"DBUTILS: new TA convo comparison id is {comarison_id}")
 
 def get_non_assesed_comparison(client, username):
     from bson.son import SON