app.py

import abc
import gradio as gr

from gen_table import *
from meta_data import *

# import pandas as pd
# pd.set_option('display.max_colwidth', 0)

head_style = """
<style>
@media (min-width: 1536px)
{
    .gradio-container {
        min-width: var(--size-full) !important;
    }
}
</style>
"""

with gr.Blocks(title="Open VLM Leaderboard", head=head_style) as demo:
    struct = load_results()
    timestamp = struct['time']
    EVAL_TIME = format_timestamp(timestamp)
    results = struct['results']
    N_MODEL = len(results)
    N_DATA = len(results['LLaVA-v1.5-7B']) - 1
    DATASETS = list(results['LLaVA-v1.5-7B'])
    DATASETS.remove('META')
    print(DATASETS)

    gr.Markdown(LEADERBORAD_INTRODUCTION.format(N_MODEL, N_DATA, EVAL_TIME))
    structs = [abc.abstractproperty() for _ in range(N_DATA)]

    with gr.Tabs(elem_classes='tab-buttons') as tabs:
        with gr.TabItem('🏅 OpenVLM Main Leaderboard', elem_id='main', id=0):
            gr.Markdown(LEADERBOARD_MD['MAIN'])
            _, check_box = BUILD_L1_DF(results, MAIN_FIELDS)
            table = generate_table(results, DEFAULT_BENCH)
            table['Rank'] = list(range(1, len(table) + 1))

            type_map = check_box['type_map']
            type_map['Rank'] = 'number'

            checkbox_group = gr.CheckboxGroup(
                choices=check_box['all'],
                value=check_box['required'],
                label='Evaluation Dimension',
                interactive=True,
            )

            headers = ['Rank'] + check_box['essential'] + checkbox_group.value
            with gr.Row():
                model_name = gr.Textbox(
                    value='Input the Model Name (fuzzy, case insensitive)', 
                    label='Model Name', 
                    interactive=True,
                    visible=True)
                model_size = gr.CheckboxGroup(
                    choices=MODEL_SIZE,
                    value=MODEL_SIZE,
                    label='Model Size',
                    interactive=True
                )
                model_type = gr.CheckboxGroup(
                    choices=MODEL_TYPE,
                    value=MODEL_TYPE,
                    label='Model Type',
                    interactive=True
                )
            data_component = gr.components.DataFrame(
                value=table[headers],
                type='pandas',
                datatype=[type_map[x] for x in headers],
                interactive=False,
                wrap=True,
                visible=True)

            def filter_df(fields, model_name, model_size, model_type):
                filter_list = ['Avg Score', 'Avg Rank', 'OpenSource']
                headers = ['Rank'] + check_box['essential'] + fields

                new_fields = [field for field in fields if field not in filter_list]
                df = generate_table(results, new_fields)

                df['flag'] = [model_size_flag(x, model_size) for x in df['Param (B)']]
                df = df[df['flag']]
                df.pop('flag')
                if len(df):
                    df['flag'] = [model_type_flag(df.iloc[i], model_type) for i in range(len(df))]
                    df = df[df['flag']]
                    df.pop('flag')
                df['Rank'] = list(range(1, len(df) + 1))
                default_val = 'Input the Model Name (fuzzy, case insensitive)'
                if model_name != default_val:
                    print(model_name)
                    model_name = model_name.lower()
                    method_names = [x.split('</a>')[0].split('>')[-1].lower() for x in df['Method']]
                    flag = [model_name in name for name in method_names]
                    df['TEMP_FLAG'] = flag
                    df = df[df['TEMP_FLAG'] == True] 
                    df.pop('TEMP_FLAG')

                comp = gr.components.DataFrame(
                    value=df[headers],
                    type='pandas',
                    datatype=[type_map[x] for x in headers],
                    interactive=False,
                    wrap=True, 
                    visible=True)
                return comp

            for cbox in [checkbox_group, model_size, model_type]:
                cbox.change(fn=filter_df, inputs=[checkbox_group, model_name, model_size, model_type], outputs=data_component)
            model_name.submit(fn=filter_df, inputs=[checkbox_group, model_name, model_size, model_type], outputs=data_component)

        with gr.TabItem('🔍 About', elem_id='about', id=1):
            gr.Markdown(urlopen(VLMEVALKIT_README).read().decode())

        for i, dataset in enumerate(DATASETS):
            with gr.TabItem(f'📊 {dataset} Leaderboard', elem_id=dataset, id=i + 2):
                if dataset in LEADERBOARD_MD:
                    gr.Markdown(LEADERBOARD_MD[dataset])

                s = structs[i]
                s.table, s.check_box = BUILD_L2_DF(results, dataset)
                s.type_map = s.check_box['type_map']
                s.type_map['Rank'] = 'number'

                s.checkbox_group = gr.CheckboxGroup(
                    choices=s.check_box['all'],
                    value=s.check_box['required'],
                    label=f'{dataset} CheckBoxes',
                    interactive=True,
                )
                s.headers = ['Rank'] + s.check_box['essential'] + s.checkbox_group.value
                s.table['Rank'] = list(range(1, len(s.table) + 1))

                with gr.Row():
                    s.model_name = gr.Textbox(
                        value='Input the Model Name (fuzzy, case insensitive)', 
                        label='Model Name', 
                        interactive=True,
                        visible=True)
                    s.model_size = gr.CheckboxGroup(
                        choices=MODEL_SIZE,
                        value=MODEL_SIZE,
                        label='Model Size',
                        interactive=True
                    )
                    s.model_type = gr.CheckboxGroup(
                        choices=MODEL_TYPE,
                        value=MODEL_TYPE,
                        label='Model Type',
                        interactive=True
                    )
                s.data_component = gr.components.DataFrame(
                    value=s.table[s.headers],
                    type='pandas',
                    datatype=[s.type_map[x] for x in s.headers],
                    interactive=False,
                    wrap=True,
                    visible=True)
                s.dataset = gr.Textbox(value=dataset, label=dataset, visible=False)

                def filter_df_l2(dataset_name, fields, model_name, model_size, model_type):
                    s = structs[DATASETS.index(dataset_name)]
                    headers = ['Rank'] + s.check_box['essential'] + fields
                    df = cp.deepcopy(s.table)
                    df['flag'] = [model_size_flag(x, model_size) for x in df['Param (B)']]
                    df = df[df['flag']]
                    df.pop('flag')
                    if len(df):
                        df['flag'] = [model_type_flag(df.iloc[i], model_type) for i in range(len(df))]
                        df = df[df['flag']]
                        df.pop('flag')
                    df['Rank'] = list(range(1, len(df) + 1))
                    default_val = 'Input the Model Name (fuzzy, case insensitive)'
                    if model_name != default_val:
                        print(model_name)
                        model_name = model_name.lower()
                        method_names = [x.split('</a>')[0].split('>')[-1].lower() for x in df['Method']]
                        flag = [model_name in name for name in method_names]
                        df['TEMP_FLAG'] = flag
                        df = df[df['TEMP_FLAG'] == True] 
                        df.pop('TEMP_FLAG')

                    comp = gr.components.DataFrame(
                        value=df[headers],
                        type='pandas',
                        datatype=[s.type_map[x] for x in headers],
                        interactive=False,
                        wrap=True,
                        visible=True)
                    return comp

                for cbox in [s.checkbox_group, s.model_size, s.model_type]:
                    cbox.change(
                        fn=filter_df_l2,
                        inputs=[s.dataset, s.checkbox_group, s.model_name, s.model_size, s.model_type],
                        outputs=s.data_component)
                s.model_name.submit(
                    fn=filter_df_l2, 
                    inputs=[s.dataset, s.checkbox_group, s.model_name, s.model_size, s.model_type],
                    outputs=s.data_component)

    with gr.Row():
        with gr.Accordion('Citation', open=False):
            citation_button = gr.Textbox(
                value=CITATION_BUTTON_TEXT,
                label=CITATION_BUTTON_LABEL,
                elem_id='citation-button')

if __name__ == '__main__':
    demo.launch(server_name='0.0.0.0')