Spaces:
Running
Running
File size: 6,002 Bytes
0d2e03d fc4805a 61fd7c8 fc4805a 61fd7c8 e6406fb fc4805a 61fd7c8 cf8c271 61fd7c8 fc4805a 0d2e03d fc4805a 1aacc3d 61fd7c8 e6406fb 61fd7c8 987cfd4 61fd7c8 fc4805a 1aacc3d fc4805a |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 |
import gradio as gr
import pandas as pd
import matplotlib.pyplot as plt
# Function to load data from a given CSV file
def load_data(version):
file_path = f'versions/{version}.csv' # Replace with your file paths
df = pd.read_csv(file_path)
# we only want specific columns and in a specific order
# column_names : Method,Model,WD,Forget Rate,Epoch,LR,Compute,ROUGE Real Authors,ROUGE SEM Real Authors,Truth Ratio Real Authors,Truth Ratio SEM Real Authors,Prob. Real Authors,Prob. SEM Real Authors,ROUGE-P Real Authors,ROUGE-P SEM Real Authors,TTR Real Authors,TTR SEM Real Authors,ROUGE Real World,ROUGE SEM Real World,Truth Ratio Real World,Truth Ratio SEM Real World,Prob. Real World,Prob. SEM Real World,ROUGE-P Real World,ROUGE-P SEM Real World,TTR Real World,TTR SEM Real World,ROUGE Retain,ROUGE SEM Retain,Truth Ratio Retain,Truth Ratio SEM Retain,Prob. Retain,Prob. SEM Retain,ROUGE-P Retain,ROUGE-P SEM Retain,TTR Retain,TTR SEM Retain,KS Test Retain,Wilcoxon PVal Retain,Wilcoxon Stat Retain,ROUGE Forget,ROUGE SEM Forget,Truth Ratio Forget,Truth Ratio SEM Forget,Prob. Forget,Prob. SEM Forget,ROUGE-P Forget,ROUGE-P SEM Forget,TTR Forget,TTR SEM Forget,KS Test Forget,Wilcoxon PVal Forget,Wilcoxon Stat Forget,KS Test Real Authors,KS Test PVal Real Authors,Wilcoxon PVal Real Authors,Wilcoxon Stat Real Authors,KS Test Real World,KS Test PVal Real World,Wilcoxon PVal Real World,Wilcoxon Stat Real World,KS Test PVal Retain,KS Test PVal Forget,Model Utility,Forget Quality
column_names = ["Method",
"Model Utility", "Forget Quality",
"ROUGE Real Authors", "Truth Ratio Real Authors", "Prob. Real Authors",
"ROUGE Real World", "Truth Ratio Real World", "Prob. Real World",
"ROUGE Retain", "Truth Ratio Retain", "Prob. Retain",
"ROUGE Forget", "Truth Ratio Forget", "Prob. Forget",
]
df = df[column_names]
return df
# Function for searching in the leaderboard
def search_leaderboard(df, query):
if query == "":
return df
else:
return df[df['Method'].str.contains(query)]
# Function to change the version of the leaderboard
def change_version(version):
new_df = load_data(version)
return new_df
# Function to create plots
from plotter import create_plots
# Initialize Gradio app
demo = gr.Blocks()
with demo:
gr.Markdown("""
## 🥇 TOFU Leaderboard
The TOFU dataset is a benchmark designed to evaluate the unlearning performance of large language models in realistic scenarios. This unique dataset consists of question-answer pairs that are based on the autobiographies of 200 fictitious authors, entirely generated by the GPT-4 model. The primary objective of this task is to effectively unlearn a fine-tuned model using different portions of the forget set.
""")
with gr.Tabs():
with gr.TabItem("Leaderboard"):
with gr.Row():
version_dropdown = gr.Dropdown(
choices=["llama", "phi"],
label="🔄 Select Base Model",
value="llama",
)
with gr.Row():
search_bar = gr.Textbox(
placeholder="Search for methods...",
show_label=False,
)
leaderboard_table = gr.components.Dataframe(
value=load_data("llama"),
interactive=True,
visible=True,
)
version_dropdown.change(
change_version,
inputs=version_dropdown,
outputs=leaderboard_table
)
search_bar.change(
search_leaderboard,
inputs=[leaderboard_table, search_bar],
outputs=leaderboard_table
)
with gr.TabItem("Plots"):
version_dropdown_plots = gr.Dropdown(
choices=["llama", "phi", "stable-lm"],
label="🔄 Select Base Model",
value="llama",
)
with gr.Row():
methods_checkbox = gr.CheckboxGroup(
label="Select Methods",
choices=list(load_data("llama")['Method'].unique()), # To be populated dynamically
)
plot_output = gr.Plot()
# Dynamically update the choices for the methods checkbox
def update_method_choices(version):
df = load_data(version)
methods = df['Method'].unique()
methods_checkbox.update(choices=methods)
return df
version_dropdown_plots.change(
update_method_choices,
inputs=version_dropdown_plots,
outputs=[methods_checkbox, plot_output]
)
methods_checkbox.change(
create_plots,
inputs=[methods_checkbox, leaderboard_table],
outputs=plot_output
)
# Launch the app
gr.Markdown("""
## Applicability 🚀
The dataset is in QA format, making it ideal for use with popular chat models such as Llama2, Mistral, or Qwen. However, it also works for any other large language model. The corresponding code base is written for the Llama2 model, but can be easily adapted to other models.
## Installation
```
conda create -n tofu python=3.10
conda activate tofu
conda install pytorch pytorch-cuda=11.8 -c pytorch -c nvidia
conda install -c "nvidia/label/cuda-11.8.0" cuda-toolkit
pip install -r requirements.txt
```
## Loading the Dataset
To load the dataset, use the following code:
```python
from datasets import load_dataset
dataset = load_dataset("locuslab/TOFU","full")
```
### Push to Leaderboard
How to push your results to the leaderboard?
""")
demo.launch()
|