Upload 3 files

scripts/app.py

@@ -0,0 +1,115 @@
+import gradio as gr
+from model_utils import load_all_models, predict_with_model
+
+# Load all models
+models, model_features = load_all_models()
+
+# Mapeo de nombres amigables a nombres reales
+MODEL_MAPPING = {
+    "Death": "Death_random_forest_model",
+    "Binary diagnosis": "Binary diagnosis_random_forest_model",
+    "Necessity of transplantation": "Necessity of transplantation_random_forest_model",
+    "Progressive disease": "Progressive disease_random_forest_model"
+}
+
+# Invertir el mapeo (opcional para facilidad)
+INVERSE_MODEL_MAPPING = {v: k for k, v in MODEL_MAPPING.items()}
+
+# Feature sets for each target variable
+FEATURES = {
+    "Death": [
+        'Pedigree', 'Age at diagnosis', 'FVC (L) at diagnosis',
+        'FVC (%) at diagnosis', 'DLCO (%) at diagnosis', 'RadioWorsening2y',
+        'Severity of telomere shortening - Transform 4', 'Progressive disease'
+    ],
+    "Binary diagnosis": [
+        'Pedigree', 'Age at diagnosis', 'Antifibrotic Drug',
+        'Prednisone', 'Mycophenolate', 'FVC (L) at diagnosis',
+        'FVC (%) at diagnosis', 'DLCO (%) at diagnosis'
+    ],
+    "Necessity of transplantation": [
+        'Pedigree','Age at diagnosis','FVC (L) at diagnosis', 'FVC (%) at diagnosis', 'DLCO (%) at diagnosis',
+        'FVC (L) 1 year after diagnosis','FVC (%) 1 year after diagnosis','DLCO (%) 1 year after diagnosis', 
+        'RadioWorsening2y'
+    ],
+    "Progressive disease": [
+        'Pedigree', 'Age at diagnosis', 'FVC (L) at diagnosis','FVC (%) at diagnosis', 'DLCO (%) at diagnosis','FVC (L) 1 year after diagnosis',
+        'FVC (%) 1 year after diagnosis', 'DLCO (%) 1 year after diagnosis',
+        'RadioWorsening2y', 'Genetic mutation studied in patient'
+    ]
+    
+}
+
+FEATURE_RANGES = {
+    'Pedigree': (0, 67),
+    'Age at diagnosis': (0, 200),
+    'FVC (L) at diagnosis': (0.0, 5.0),
+    'FVC (%) at diagnosis': (0.0, 200.0),
+    'DLCO (%) at diagnosis': (0.0, 200.0),
+    'RadioWorsening2y': (0, 3),
+    'Severity of telomere shortening - Transform 4': (1, 6),
+    'Progressive disease': (0, 1),
+    'Antifibrotic Drug': (0, 1),
+    'Prednisone': (0, 1),
+    'Mycophenolate': (0, 1),
+    'FVC (L) 1 year after diagnosis': (0.0, 5.0),
+    'FVC (%) 1 year after diagnosis': (0.0, 200.0),
+    'DLCO (%) 1 year after diagnosis': (0.0, 200.0),
+    'Genetic mutation studied in patient': (0, 1),
+    'Comorbidities': (0, 1)
+}
+
+
+# Define prediction function
+def make_prediction(input_features, friendly_model_name):
+    """
+    Predict using the selected model and input features.
+    """
+    # Map the friendly model name to the real model name
+    target_model = MODEL_MAPPING.get(friendly_model_name)
+    if target_model not in models:
+        return f"Model '{friendly_model_name}' not found. Please select a valid model."
+
+    model = models[target_model]
+    features = model_features[target_model]
+
+    if len(input_features) != len(features):
+        return f"Invalid input. Expected features: {features}"
+
+    input_array = [float(x) for x in input_features]
+    prediction = predict_with_model(model, input_array)
+    return f"Prediction for {friendly_model_name}: {prediction}"
+
+# Define Gradio interface
+def gradio_interface():
+    def create_inputs_for_features(features):
+        inputs = []
+        for feature in features:
+            min_val, max_val = FEATURE_RANGES.get(feature, (None, None))
+            inputs.append(gr.Number(label=f"{feature} (Range: {min_val} - {max_val})", minimum=min_val, maximum=max_val))
+        return inputs
+
+    # Create a separate interface for each target variable
+    interfaces = []
+    for target, features in FEATURES.items():
+        inputs = create_inputs_for_features(features)
+        interface = gr.Interface(
+            fn=lambda *args, target=target: make_prediction(args, target),
+            inputs=inputs,
+            outputs=gr.Text(label="Prediction Result"),
+            title=f"Prediction for {target}",
+            description=f"Provide values for features relevant to {target}"
+        )
+        interfaces.append(interface)
+
+    # Combine all interfaces into a tabbed layout
+    tabbed_interface = gr.TabbedInterface(
+        interface_list=interfaces,
+        tab_names=list(FEATURES.keys())
+    )
+    return tabbed_interface
+
+# Launch Gradio app
+if __name__ == "__main__":
+    interface = gradio_interface()
+    interface.launch()

scripts/fibropred_model.py

@@ -0,0 +1,195 @@
+import pandas as pd
+import numpy as np
+import os
+import joblib
+from sklearn.model_selection import train_test_split, cross_val_score, StratifiedKFold
+from sklearn.ensemble import RandomForestClassifier, GradientBoostingClassifier
+from sklearn.impute import SimpleImputer
+from sklearn.metrics import classification_report, accuracy_score, roc_curve, auc
+from sklearn.feature_selection import SelectFromModel
+import matplotlib.pyplot as plt
+import seaborn as sns
+
+# Load dataset
+def load_data(file_path):
+    df = pd.read_excel(file_path, header=1)
+    return df
+
+# Preprocess data including categorical variables
+def preprocess_data_with_categoricals(df):
+    # Replace -9 with NaN for missing values
+    df.replace(-9, np.nan, inplace=True)
+
+    # Drop columns with >50% missing values
+    missing_percentage = df.isnull().sum() / len(df) * 100
+    df = df.drop(columns=missing_percentage[missing_percentage > 50].index)
+
+    # Drop specific columns
+    drop_columns = ['ProgressiveDisease', 'Final diagnosis', 'Transplantation date', 'Cause of death', 'Date of death', 'COD NUMBER']
+    df = df.drop(columns=[col for col in drop_columns if col in df.columns])
+    
+    # Impute missing values
+    imputer = SimpleImputer(strategy='median')
+    numeric_cols = df.select_dtypes(include=['number']).columns
+    df[numeric_cols] = imputer.fit_transform(df[numeric_cols])
+
+    # Handle binary variables specifically
+    if 'Binary diagnosis' in df.columns:
+        df['Binary diagnosis'] = df['Binary diagnosis'].apply(
+            lambda x: 1 if str(x).strip().lower() == "ipf" else 0
+        )
+
+    if 'Death' in df.columns:
+        df['Death'] = df['Death'].apply(
+            lambda x: 1 if str(x).strip().lower() == "yes" else 0
+        )
+
+    # Apply one-hot encoding to categorical variables
+    df = apply_one_hot_encoding(df)
+
+    # Separate categorical and numerical variables
+    categorical_cols = df.select_dtypes(include=['object']).columns
+    numeric_cols = df.select_dtypes(include=['number']).columns
+    print("Categorical Variables:", categorical_cols.tolist())
+    print("Numerical Variables:", numeric_cols.tolist())
+    return df, numeric_cols, categorical_cols
+
+# Apply one-hot encoding to categorical variables
+def apply_one_hot_encoding(df):
+    categorical_cols = df.select_dtypes(include=['object']).columns
+    df = pd.get_dummies(df, columns=categorical_cols, drop_first=True)
+    return df
+
+# Select predictors using feature importance
+def select_important_features(X, y, threshold=0.03):
+    model = RandomForestClassifier(random_state=42)
+    model.fit(X, y)
+    selector = SelectFromModel(model, threshold=threshold, prefit=True)
+    X_reduced = selector.transform(X)
+    selected_features = X.columns[selector.get_support()]
+    return pd.DataFrame(X_reduced, columns=selected_features), selected_features
+
+# Visualize feature importance
+def plot_feature_importance(model, features, target):
+    importance = model.feature_importances_
+    sorted_idx = np.argsort(importance)[::-1]
+    plt.figure(figsize=(10, 6))
+    sns.barplot(x=importance[sorted_idx], y=np.array(features)[sorted_idx])
+    plt.title(f'Feature Importance for {target}')
+    plt.xlabel('Importance')
+    plt.ylabel('Feature')
+    plt.tight_layout()
+    plt.show()
+
+# Visualize overfitting and optimization results
+def plot_model_performance(cv_scores, train_scores, test_scores, target ,metric_name="Accuracy"):
+    plt.figure(figsize=(12, 6))
+
+    # Cross-validation scores
+    plt.subplot(1, 2, 1)
+    plt.plot(cv_scores, label='Cross-validation scores', marker='o')
+    plt.title(f'Cross-validation {metric_name} for {target}')
+    plt.xlabel('Fold')
+    plt.ylabel(metric_name)
+    plt.grid(True)
+    plt.legend()
+
+    # Train vs Test comparison
+    plt.subplot(1, 2, 2)
+    plt.bar(['Train', 'Test'], [train_scores.mean(), test_scores], color=['blue', 'orange'])
+    plt.title(f'{metric_name}: Train vs Test')
+    plt.ylabel(metric_name)
+    plt.grid(True)
+
+    plt.tight_layout()
+    plt.show()
+
+# Plot ROC-AUC curve
+def plot_roc_auc(model, X_test, y_test, target):
+    y_prob = model.predict_proba(X_test)[:, 1]  # Probabilidades para la clase positiva
+    fpr, tpr, thresholds = roc_curve(y_test, y_prob)
+    roc_auc = auc(fpr, tpr)
+
+    plt.figure(figsize=(8, 6))
+    plt.plot(fpr, tpr, color='blue', lw=2, label=f'ROC curve (area = {roc_auc:.2f})')
+    plt.plot([0, 1], [0, 1], color='gray', linestyle='--')
+    plt.xlabel('False Positive Rate')
+    plt.ylabel('True Positive Rate')
+    plt.title(f'ROC-AUC Curve for {target}')
+    plt.legend(loc="lower right")
+    plt.grid(True)
+    plt.show()
+
+# Save trained model
+def save_model(model, target, selected_features):
+
+    if not os.path.exists("models"):
+        os.makedirs("models")
+    file_name = f"models/{target}_random_forest_model.pkl"
+    joblib.dump({'model': model, 'features': selected_features}, file_name)
+    print(f"Model and features saved to {file_name}")
+
+
+# Main pipeline
+def main():
+    file_path = 'FibroPredCODIFICADA.xlsx'
+    df = load_data(file_path)
+
+    # Target columns
+    target_columns = ['Death', 'Progressive disease', 'Necessity of transplantation']
+
+    # Preprocess data
+    df, numeric_cols, categorical_cols = preprocess_data_with_categoricals(df)
+
+    for target in target_columns:
+        print(f"Processing target: {target}")
+        X = df[numeric_cols].drop(columns=target_columns, errors='ignore')  # Ensure target variables are excluded
+        y = df[target]
+
+        # Split data
+        X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42)
+
+        # Select important features
+        X_train_selected, selected_features = select_important_features(X_train, y_train)
+        X_test_selected = X_test[selected_features]
+
+        print(f"Selected predictors for training {target} ({len(selected_features)} predictors): {selected_features.tolist()}")
+
+        # Train RandomForest model
+        model = RandomForestClassifier(n_estimators=300,
+            max_depth=4, 
+            min_samples_split=10, 
+            min_samples_leaf=10,
+            class_weight='balanced',
+            max_features='sqrt',
+            random_state=42)
+        model.fit(X_train_selected, y_train)
+
+        # Cross-validation to check overfitting
+        cv = StratifiedKFold(n_splits=15, shuffle=True, random_state=42)
+        cv_scores = cross_val_score(model, X_train_selected, y_train, cv=cv, scoring='accuracy')
+        train_scores = cross_val_score(model, X_train_selected, y_train, cv=15, scoring='accuracy')
+        y_pred_test = model.predict(X_test_selected)
+        test_score = accuracy_score(y_test, y_pred_test)
+
+        print(f"Cross-validation accuracy for {target}: {cv_scores.mean():.4f} (+/- {cv_scores.std():.4f})")
+        print(f"Test accuracy for {target}: {test_score:.4f}")
+        print(classification_report(y_test, y_pred_test))
+
+        # Plot model performance
+        plot_model_performance(cv_scores, train_scores, test_score, target, metric_name="Accuracy")
+
+        # Plot feature importance
+        print(f"Feature importance for {target}:")
+        plot_feature_importance(model, selected_features, target)
+
+        # Plot ROC-AUC Curve
+        plot_roc_auc(model, X_test_selected, y_test, target)
+
+        # Save trained model
+        save_model(model, target, selected_features.tolist())
+
+    print("Pipeline completed.")
+
+if __name__ == "__main__":
+    main()

scripts/model_utils.py

@@ -0,0 +1,34 @@
+import os
+import joblib
+
+def load_all_models(models_dir="models"):
+    """
+    Load all models and their features from the given directory.
+    """
+    models = {}
+    features = {}
+    if not os.path.exists(models_dir):
+        raise FileNotFoundError(f"Models directory '{models_dir}' not found.")
+
+    for model_file in os.listdir(models_dir):
+        if model_file.endswith(".pkl"):
+            model_name = os.path.splitext(model_file)[0]
+            data = joblib.load(os.path.join(models_dir, model_file))
+            models[model_name] = data['model']
+            features[model_name] = data['features']
+            print(f"Model '{model_name}' loaded successfully with features: {features[model_name]}")
+    return models, features
+
+def predict_with_model(model, input_data):
+    """
+    Predict using a loaded model.
+
+    Parameters:
+    - model: The loaded model.
+    - input_data: A dictionary or Pandas DataFrame row containing input features.
+
+    Returns:
+    - prediction: Model prediction.
+    """
+    prediction = model.predict([input_data])
+    return int(prediction[0])