update Ivy_VL_3B

constants.py

@@ -132,6 +132,7 @@ MODEL_NAME_MAP = {
     "Gemini-2.0-thinking": "Gemini-2.0-Flash-thinking",
     "Gemini-Flash-2.0-exp": "Gemini-2.0-Flash-exp",
     "Gemini-exp-1206": "Gemini-exp-1206",
+    "Ivy_VL_3B": "Ivy-VL-3B",
 }
 
 DIMENSION_NAME_MAP = {
@@ -223,15 +224,16 @@ MODEL_URLS = {
     "Gemini-2.0-thinking": "https://ai.google.dev/gemini-api/docs/thinking-mode",
     "Gemini-exp-1206": "https://blog.google/feed/gemini-exp-1206/",
     "Gemini-Flash-2.0-exp": "https://blog.google/technology/google-deepmind/google-gemini-ai-update-december-2024/#gemini-2-0-flash",
+    "Ivy_VL_3B": "https://huggingface.co/AI-Safeguard/Ivy-VL-llava",
 }
 
 # Define the base MODEL_GROUPS structure
 BASE_MODEL_GROUPS = {
     "All": list(MODEL_NAME_MAP.keys()),
     "Flagship Models": ['Claude_3.5_new', 'GPT_4o', 'Claude_3.5', 'Gemini_1.5_pro_002', 'Qwen2_VL_72B', 'InternVL2_76B', 'llava_onevision_72B', 'NVLM', 'Molmo_72B', 'InternVL2_5_78B', 'Grok-2-vision-1212', "Gemini-exp-1206"],
-    "Efficiency Models": ['Gemini_1.5_flash_002', 'GPT_4o_mini', 'Qwen2_VL_7B', 'Pixtral_12B', 'Aria', 'InternVL2_8B', 'Phi-3.5-vision', 'MiniCPM_v2.6', 'llava_onevision_7B', 'Llama_3_2_11B', 'Idefics3', 'Molmo_7B_D', "Aquila_VL_2B", "POINTS_7B", "Qwen2_VL_2B", "InternVL2_2B", "InternVL2_5_2B", "InternVL2_5_8B", "Gemini-2.0-thinking", "Gemini-Flash-2.0-exp"],
+    "Efficiency Models": ['Gemini_1.5_flash_002', 'GPT_4o_mini', 'Qwen2_VL_7B', 'Pixtral_12B', 'Aria', 'InternVL2_8B', 'Phi-3.5-vision', 'MiniCPM_v2.6', 'llava_onevision_7B', 'Llama_3_2_11B', 'Idefics3', 'Molmo_7B_D', "Aquila_VL_2B", "POINTS_7B", "Qwen2_VL_2B", "InternVL2_2B", "InternVL2_5_2B", "InternVL2_5_8B", "Gemini-2.0-thinking", "Gemini-Flash-2.0-exp", "Ivy_VL_3B"],
     "Proprietary Flagship models": ['Claude_3.5_new', 'GPT_4o', 'Claude_3.5', 'Gemini_1.5_pro_002', 'Grok-2-vision-1212', "Gemini-exp-1206"],
     "Proprietary Efficiency Models": ['Gemini_1.5_flash_002', 'GPT_4o_mini', "Gemini-Flash-2.0-exp", "Gemini-2.0-thinking"],
     "Open-source Flagship Models": ['Qwen2_VL_72B', 'InternVL2_76B', 'llava_onevision_72B', 'NVLM', "Molmo_72B", "InternVL2_5_78B"],
-    "Open-source Efficiency Models": ['Qwen2_VL_7B', 'Pixtral_12B', 'Aria', 'InternVL2_8B', 'Phi-3.5-vision', 'MiniCPM_v2.6', 'llava_onevision_7B', 'Llama_3_2_11B', 'Idefics3', 'Molmo_7B_D', "Aquila_VL_2B", "POINTS_7B", "Qwen2_VL_2B", "InternVL2_2B", "InternVL2_5_2B", "InternVL2_5_8B"]
+    "Open-source Efficiency Models": ['Qwen2_VL_7B', 'Pixtral_12B', 'Aria', 'InternVL2_8B', 'Phi-3.5-vision', 'MiniCPM_v2.6', 'llava_onevision_7B', 'Llama_3_2_11B', 'Idefics3', 'Molmo_7B_D', "Aquila_VL_2B", "POINTS_7B", "Qwen2_VL_2B", "InternVL2_2B", "InternVL2_5_2B", "InternVL2_5_8B", "Ivy_VL_3B"]
 }

static/eval_results/Default/Ivy_VL_3B/summary_and_keyword_stats.json

@@ -0,0 +1,249 @@
+{
+    "model_summary": {
+        "core": {
+            "num_eval_tasks": 440,
+            "num_eval_samples": 6539,
+            "macro_mean_score": 0.19185564187617368
+        },
+        "open": {
+            "num_eval_tasks": 65,
+            "num_eval_samples": 1163,
+            "macro_mean_score": 0.27715328322847527
+        },
+        "overall_score": 0.20283454620864816
+    },
+    "keyword_stats": {
+        "skills": {
+            "Object Recognition and Classification": {
+                "count": 303,
+                "num_samples": 4755,
+                "tasks": [],
+                "average_score": 0.211403742529065
+            },
+            "Text Recognition (OCR)": {
+                "count": 137,
+                "num_samples": 2239,
+                "tasks": [],
+                "average_score": 0.16550807652825617
+            },
+            "Language Understanding and Generation": {
+                "count": 154,
+                "num_samples": 2509,
+                "tasks": [],
+                "average_score": 0.20807789378302347
+            },
+            "Scene and Event Understanding": {
+                "count": 154,
+                "num_samples": 2467,
+                "tasks": [],
+                "average_score": 0.26453498820602844
+            },
+            "Mathematical and Logical Reasoning": {
+                "count": 109,
+                "num_samples": 1910,
+                "tasks": [],
+                "average_score": 0.1880366735477861
+            },
+            "Commonsense and Social Reasoning": {
+                "count": 51,
+                "num_samples": 855,
+                "tasks": [],
+                "average_score": 0.30546713625131916
+            },
+            "Ethical and Safety Reasoning": {
+                "count": 15,
+                "num_samples": 245,
+                "tasks": [],
+                "average_score": 0.4761629072681704
+            },
+            "Domain-Specific Knowledge and Skills": {
+                "count": 77,
+                "num_samples": 1386,
+                "tasks": [],
+                "average_score": 0.19519503635292754
+            },
+            "Spatial and Temporal Reasoning": {
+                "count": 152,
+                "num_samples": 2437,
+                "tasks": [],
+                "average_score": 0.19092980712966892
+            },
+            "Planning and Decision Making": {
+                "count": 37,
+                "num_samples": 577,
+                "tasks": [],
+                "average_score": 0.0388946980911512
+            }
+        },
+        "input_format": {
+            "User Interface Screenshots": {
+                "count": 93,
+                "num_samples": 1517,
+                "tasks": [],
+                "average_score": 0.10493661164954043
+            },
+            "Text-Based Images and Documents": {
+                "count": 82,
+                "num_samples": 1294,
+                "tasks": [],
+                "average_score": 0.1249415479767782
+            },
+            "Diagrams and Data Visualizations": {
+                "count": 101,
+                "num_samples": 1718,
+                "tasks": [],
+                "average_score": 0.22725684171603053
+            },
+            "Videos": {
+                "count": 43,
+                "num_samples": 698,
+                "tasks": [],
+                "average_score": 0.2536386994618329
+            },
+            "Artistic and Creative Content": {
+                "count": 32,
+                "num_samples": 541,
+                "tasks": [],
+                "average_score": 0.2524995045658917
+            },
+            "Photographs": {
+                "count": 143,
+                "num_samples": 2248,
+                "tasks": [],
+                "average_score": 0.27729106359048106
+            },
+            "3D Models and Aerial Imagery": {
+                "count": 11,
+                "num_samples": 169,
+                "tasks": [],
+                "average_score": 0.07592024437626757
+            }
+        },
+        "output_format": {
+            "contextual_formatted_text": {
+                "count": 98,
+                "num_samples": 1514,
+                "tasks": [],
+                "average_score": 0.1701675351847416
+            },
+            "structured_output": {
+                "count": 110,
+                "num_samples": 1714,
+                "tasks": [],
+                "average_score": 0.13008026550344784
+            },
+            "exact_text": {
+                "count": 83,
+                "num_samples": 1278,
+                "tasks": [],
+                "average_score": 0.2158637762929687
+            },
+            "numerical_data": {
+                "count": 49,
+                "num_samples": 862,
+                "tasks": [],
+                "average_score": 0.20877347808029043
+            },
+            "open_ended_output": {
+                "count": 80,
+                "num_samples": 1454,
+                "tasks": [],
+                "average_score": 0.26632791563900915
+            },
+            "multiple_choice": {
+                "count": 85,
+                "num_samples": 1363,
+                "tasks": [],
+                "average_score": 0.25874554202955213
+            }
+        },
+        "input_num": {
+            "6-8 images": {
+                "count": 21,
+                "num_samples": 314,
+                "tasks": [],
+                "average_score": 0.13869047619047617
+            },
+            "9-image or more": {
+                "count": 41,
+                "num_samples": 623,
+                "tasks": [],
+                "average_score": 0.11042798544340451
+            },
+            "1-image": {
+                "count": 315,
+                "num_samples": 5228,
+                "tasks": [],
+                "average_score": 0.2052404551683576
+            },
+            "video": {
+                "count": 43,
+                "num_samples": 698,
+                "tasks": [],
+                "average_score": 0.2536386994618329
+            },
+            "4-5 images": {
+                "count": 34,
+                "num_samples": 520,
+                "tasks": [],
+                "average_score": 0.1866009831347769
+            },
+            "2-3 images": {
+                "count": 51,
+                "num_samples": 802,
+                "tasks": [],
+                "average_score": 0.2566619127590936
+            }
+        },
+        "app": {
+            "Information_Extraction": {
+                "count": 72,
+                "num_samples": 1124,
+                "tasks": [],
+                "average_score": 0.1410253687207258
+            },
+            "Planning": {
+                "count": 78,
+                "num_samples": 1239,
+                "tasks": [],
+                "average_score": 0.0749166420062132
+            },
+            "Coding": {
+                "count": 31,
+                "num_samples": 474,
+                "tasks": [],
+                "average_score": 0.13665778139212675
+            },
+            "Perception": {
+                "count": 145,
+                "num_samples": 2313,
+                "tasks": [],
+                "average_score": 0.24948956246459913
+            },
+            "Metrics": {
+                "count": 20,
+                "num_samples": 309,
+                "tasks": [],
+                "average_score": 0.4214677030751524
+            },
+            "Science": {
+                "count": 29,
+                "num_samples": 574,
+                "tasks": [],
+                "average_score": 0.2205827555902744
+            },
+            "Knowledge": {
+                "count": 97,
+                "num_samples": 1605,
+                "tasks": [],
+                "average_score": 0.2480287030335926
+            },
+            "Mathematics": {
+                "count": 33,
+                "num_samples": 547,
+                "tasks": [],
+                "average_score": 0.21626379583600183
+            }
+        }
+    }
+}

static/eval_results/SI/Ivy_VL_3B/summary_and_keyword_stats.json

@@ -0,0 +1,213 @@
+{
+    "model_summary": {
+        "core": {
+            "num_eval_tasks": 273,
+            "num_eval_samples": 4116,
+            "macro_mean_score": 0.19338729492202295
+        },
+        "open": {
+            "num_eval_tasks": 42,
+            "num_eval_samples": 813,
+            "macro_mean_score": 0.2822859967695332
+        },
+        "overall_score": 0.20524045516835765
+    },
+    "keyword_stats": {
+        "skills": {
+            "Text Recognition (OCR)": {
+                "count": 101,
+                "num_samples": 1687,
+                "tasks": [],
+                "average_score": 0.17399890430893702
+            },
+            "Object Recognition and Classification": {
+                "count": 172,
+                "num_samples": 2714,
+                "tasks": [],
+                "average_score": 0.21109372364521883
+            },
+            "Scene and Event Understanding": {
+                "count": 60,
+                "num_samples": 1004,
+                "tasks": [],
+                "average_score": 0.33373019769952694
+            },
+            "Mathematical and Logical Reasoning": {
+                "count": 91,
+                "num_samples": 1630,
+                "tasks": [],
+                "average_score": 0.19559519566671743
+            },
+            "Language Understanding and Generation": {
+                "count": 102,
+                "num_samples": 1713,
+                "tasks": [],
+                "average_score": 0.22630314325483508
+            },
+            "Domain-Specific Knowledge and Skills": {
+                "count": 46,
+                "num_samples": 897,
+                "tasks": [],
+                "average_score": 0.17591613987857024
+            },
+            "Spatial and Temporal Reasoning": {
+                "count": 78,
+                "num_samples": 1273,
+                "tasks": [],
+                "average_score": 0.1892291725086101
+            },
+            "Planning and Decision Making": {
+                "count": 23,
+                "num_samples": 356,
+                "tasks": [],
+                "average_score": 0.036703008210857355
+            },
+            "Commonsense and Social Reasoning": {
+                "count": 38,
+                "num_samples": 654,
+                "tasks": [],
+                "average_score": 0.33693960266311135
+            },
+            "Ethical and Safety Reasoning": {
+                "count": 10,
+                "num_samples": 170,
+                "tasks": [],
+                "average_score": 0.6292443609022556
+            }
+        },
+        "input_format": {
+            "User Interface Screenshots": {
+                "count": 67,
+                "num_samples": 1123,
+                "tasks": [],
+                "average_score": 0.10470899167807271
+            },
+            "Text-Based Images and Documents": {
+                "count": 53,
+                "num_samples": 847,
+                "tasks": [],
+                "average_score": 0.10952573461485345
+            },
+            "Photographs": {
+                "count": 83,
+                "num_samples": 1315,
+                "tasks": [],
+                "average_score": 0.3040415952138278
+            },
+            "Diagrams and Data Visualizations": {
+                "count": 88,
+                "num_samples": 1524,
+                "tasks": [],
+                "average_score": 0.2361422770272551
+            },
+            "Artistic and Creative Content": {
+                "count": 22,
+                "num_samples": 389,
+                "tasks": [],
+                "average_score": 0.2541893948086852
+            },
+            "3D Models and Aerial Imagery": {
+                "count": 2,
+                "num_samples": 30,
+                "tasks": [],
+                "average_score": 0.1111187670386593
+            }
+        },
+        "output_format": {
+            "structured_output": {
+                "count": 72,
+                "num_samples": 1121,
+                "tasks": [],
+                "average_score": 0.1448509560944764
+            },
+            "contextual_formatted_text": {
+                "count": 63,
+                "num_samples": 975,
+                "tasks": [],
+                "average_score": 0.17036364091761733
+            },
+            "exact_text": {
+                "count": 57,
+                "num_samples": 880,
+                "tasks": [],
+                "average_score": 0.15740808174047646
+            },
+            "numerical_data": {
+                "count": 39,
+                "num_samples": 694,
+                "tasks": [],
+                "average_score": 0.19788851141672356
+            },
+            "open_ended_output": {
+                "count": 51,
+                "num_samples": 991,
+                "tasks": [],
+                "average_score": 0.2682473537043838
+            },
+            "multiple_choice": {
+                "count": 33,
+                "num_samples": 567,
+                "tasks": [],
+                "average_score": 0.39751628842537934
+            }
+        },
+        "input_num": {
+            "1-image": {
+                "count": 315,
+                "num_samples": 5228,
+                "tasks": [],
+                "average_score": 0.2052404551683576
+            }
+        },
+        "app": {
+            "Information_Extraction": {
+                "count": 41,
+                "num_samples": 644,
+                "tasks": [],
+                "average_score": 0.10690741122489508
+            },
+            "Planning": {
+                "count": 44,
+                "num_samples": 714,
+                "tasks": [],
+                "average_score": 0.06933513581009754
+            },
+            "Coding": {
+                "count": 16,
+                "num_samples": 244,
+                "tasks": [],
+                "average_score": 0.18259439192343602
+            },
+            "Perception": {
+                "count": 82,
+                "num_samples": 1321,
+                "tasks": [],
+                "average_score": 0.2810337734703279
+            },
+            "Metrics": {
+                "count": 3,
+                "num_samples": 45,
+                "tasks": [],
+                "average_score": 0.36507936507936506
+            },
+            "Science": {
+                "count": 22,
+                "num_samples": 469,
+                "tasks": [],
+                "average_score": 0.2593828964382621
+            },
+            "Knowledge": {
+                "count": 77,
+                "num_samples": 1294,
+                "tasks": [],
+                "average_score": 0.23992559516244502
+            },
+            "Mathematics": {
+                "count": 30,
+                "num_samples": 497,
+                "tasks": [],
+                "average_score": 0.19915270674792282
+            }
+        }
+    }
+}