{
"vidore/arxivqa_test_subsampled": {
"ndcg_at_1": 0.852,
"ndcg_at_3": 0.90086,
"ndcg_at_5": 0.90722,
"ndcg_at_10": 0.91102,
"ndcg_at_20": 0.91656,
"ndcg_at_50": 0.91851,
"ndcg_at_100": 0.91914,
"map_at_1": 0.852,
"map_at_3": 0.88933,
"map_at_5": 0.89273,
"map_at_10": 0.89425,
"map_at_20": 0.89575,
"map_at_50": 0.89605,
"map_at_100": 0.89611,
"recall_at_1": 0.852,
"recall_at_3": 0.934,
"recall_at_5": 0.95,
"recall_at_10": 0.962,
"recall_at_20": 0.984,
"recall_at_50": 0.994,
"recall_at_100": 0.998,
"precision_at_1": 0.852,
"precision_at_3": 0.31133,
"precision_at_5": 0.19,
"precision_at_10": 0.0962,
"precision_at_20": 0.0492,
"precision_at_50": 0.01988,
"precision_at_100": 0.00998,
"mrr_at_1": 0.852,
"mrr_at_3": 0.8879999999999998,
"mrr_at_5": 0.8918999999999998,
"mrr_at_10": 0.8935880952380951,
"mrr_at_20": 0.8949619694031458,
"mrr_at_50": 0.8952206190465779,
"mrr_at_100": 0.8953101826793595,
"naucs_at_1_max": 0.743886209713547,
"naucs_at_1_std": -0.16796452947532062,
"naucs_at_1_diff1": 0.9333280628244656,
"naucs_at_3_max": 0.7297060238236761,
"naucs_at_3_std": -0.29585773703420537,
"naucs_at_3_diff1": 0.916532269473447,
"naucs_at_5_max": 0.7750140056022408,
"naucs_at_5_std": -0.3126797385620943,
"naucs_at_5_diff1": 0.9292250233426705,
"naucs_at_10_max": 0.7392255147673084,
"naucs_at_10_std": -0.24276377217554787,
"naucs_at_10_diff1": 0.9137549756744823,
"naucs_at_20_max": 0.9115896358543447,
"naucs_at_20_std": 0.07142857142856779,
"naucs_at_20_diff1": 0.9836601307189491,
"naucs_at_50_max": 0.9564270152505304,
"naucs_at_50_std": 0.08667911609088286,
"naucs_at_50_diff1": 1.0,
"naucs_at_100_max": 1.0,
"naucs_at_100_std": 1.0,
"naucs_at_100_diff1": 1.0
},
"vidore/docvqa_test_subsampled": {
"ndcg_at_1": 0.55654,
"ndcg_at_3": 0.61596,
"ndcg_at_5": 0.64525,
"ndcg_at_10": 0.65672,
"ndcg_at_20": 0.67079,
"ndcg_at_50": 0.68331,
"ndcg_at_100": 0.69204,
"map_at_1": 0.55654,
"map_at_3": 0.60126,
"map_at_5": 0.61755,
"map_at_10": 0.62229,
"map_at_20": 0.62618,
"map_at_50": 0.62808,
"map_at_100": 0.62888,
"recall_at_1": 0.55654,
"recall_at_3": 0.65854,
"recall_at_5": 0.72949,
"recall_at_10": 0.76497,
"recall_at_20": 0.8204,
"recall_at_50": 0.8847,
"recall_at_100": 0.93792,
"precision_at_1": 0.55654,
"precision_at_3": 0.21951,
"precision_at_5": 0.1459,
"precision_at_10": 0.0765,
"precision_at_20": 0.04102,
"precision_at_50": 0.01769,
"precision_at_100": 0.00938,
"mrr_at_1": 0.5476718403547672,
"mrr_at_3": 0.598669623059867,
"mrr_at_5": 0.6114190687361419,
"mrr_at_10": 0.6178043501214233,
"mrr_at_20": 0.621799912006316,
"mrr_at_50": 0.6237201129227461,
"mrr_at_100": 0.6245233026011214,
"naucs_at_1_max": 0.4832573617906144,
"naucs_at_1_std": -0.3032135796994576,
"naucs_at_1_diff1": 0.85261473666465,
"naucs_at_3_max": 0.46413211861079323,
"naucs_at_3_std": -0.3586604308785551,
"naucs_at_3_diff1": 0.8210611793068221,
"naucs_at_5_max": 0.448455784463693,
"naucs_at_5_std": -0.3356088095049938,
"naucs_at_5_diff1": 0.7876237365986988,
"naucs_at_10_max": 0.4180379024328761,
"naucs_at_10_std": -0.2978414780805468,
"naucs_at_10_diff1": 0.7518139587785174,
"naucs_at_20_max": 0.3539033589636336,
"naucs_at_20_std": -0.29617967205487483,
"naucs_at_20_diff1": 0.7229362478946776,
"naucs_at_50_max": 0.3294934197988531,
"naucs_at_50_std": -0.026930976978897465,
"naucs_at_50_diff1": 0.6978098616006386,
"naucs_at_100_max": 0.4056438414696139,
"naucs_at_100_std": 0.37753201203052666,
"naucs_at_100_diff1": 0.6820256833963592
},
"vidore/infovqa_test_subsampled": {
"ndcg_at_1": 0.88057,
"ndcg_at_3": 0.90936,
"ndcg_at_5": 0.92034,
"ndcg_at_10": 0.92497,
"ndcg_at_20": 0.92755,
"ndcg_at_50": 0.92954,
"ndcg_at_100": 0.93021,
"map_at_1": 0.88057,
"map_at_3": 0.9025,
"map_at_5": 0.90867,
"map_at_10": 0.9106,
"map_at_20": 0.91132,
"map_at_50": 0.91163,
"map_at_100": 0.9117,
"recall_at_1": 0.88057,
"recall_at_3": 0.92915,
"recall_at_5": 0.95547,
"recall_at_10": 0.96964,
"recall_at_20": 0.97976,
"recall_at_50": 0.98988,
"recall_at_100": 0.99393,
"precision_at_1": 0.88057,
"precision_at_3": 0.30972,
"precision_at_5": 0.19109,
"precision_at_10": 0.09696,
"precision_at_20": 0.04899,
"precision_at_50": 0.0198,
"precision_at_100": 0.00994,
"mrr_at_1": 0.8785425101214575,
"mrr_at_3": 0.9011470985155193,
"mrr_at_5": 0.9075236167341428,
"mrr_at_10": 0.9092547072810231,
"mrr_at_20": 0.9101327765801449,
"mrr_at_50": 0.9104553627424199,
"mrr_at_100": 0.9105182434033097,
"naucs_at_1_max": 0.6442747044878268,
"naucs_at_1_std": -0.019608341053496993,
"naucs_at_1_diff1": 0.9484048829634778,
"naucs_at_3_max": 0.7386798922430662,
"naucs_at_3_std": 0.03322023115905618,
"naucs_at_3_diff1": 0.9116914983852844,
"naucs_at_5_max": 0.772590092296685,
"naucs_at_5_std": 0.015327130682268415,
"naucs_at_5_diff1": 0.9136885163498384,
"naucs_at_10_max": 0.8058420528202048,
"naucs_at_10_std": 0.12522140546049776,
"naucs_at_10_diff1": 0.9267525275808962,
"naucs_at_20_max": 0.8138015202314234,
"naucs_at_20_std": 0.3863206159107717,
"naucs_at_20_diff1": 0.9162490821843028,
"naucs_at_50_max": 0.8291917984429228,
"naucs_at_50_std": 0.4153501554955276,
"naucs_at_50_diff1": 0.9477594183740937,
"naucs_at_100_max": 0.7588534820930969,
"naucs_at_100_std": 0.2395996652220107,
"naucs_at_100_diff1": 0.9564661819784259
},
"vidore/tabfquad_test_subsampled": {
"ndcg_at_1": 0.91786,
"ndcg_at_3": 0.94753,
"ndcg_at_5": 0.95045,
"ndcg_at_10": 0.95857,
"ndcg_at_20": 0.95857,
"ndcg_at_50": 0.95857,
"ndcg_at_100": 0.95857,
"map_at_1": 0.91786,
"map_at_3": 0.94048,
"map_at_5": 0.94208,
"map_at_10": 0.94545,
"map_at_20": 0.94545,
"map_at_50": 0.94545,
"map_at_100": 0.94545,
"recall_at_1": 0.91786,
"recall_at_3": 0.96786,
"recall_at_5": 0.975,
"recall_at_10": 1.0,
"recall_at_20": 1.0,
"recall_at_50": 1.0,
"recall_at_100": 1.0,
"precision_at_1": 0.91786,
"precision_at_3": 0.32262,
"precision_at_5": 0.195,
"precision_at_10": 0.1,
"precision_at_20": 0.05,
"precision_at_50": 0.02,
"precision_at_100": 0.01,
"mrr_at_1": 0.9214285714285714,
"mrr_at_3": 0.9422619047619047,
"mrr_at_5": 0.9438690476190476,
"mrr_at_10": 0.9472406462585033,
"mrr_at_20": 0.9472406462585033,
"mrr_at_50": 0.9472406462585033,
"mrr_at_100": 0.9472406462585033,
"naucs_at_1_max": 0.5557179393496541,
"naucs_at_1_std": 0.021881216254618023,
"naucs_at_1_diff1": 0.937989688629075,
"naucs_at_3_max": 0.8384687208216648,
"naucs_at_3_std": 0.5060690943043917,
"naucs_at_3_diff1": 0.9709513435003594,
"naucs_at_5_max": 0.9176337201547285,
"naucs_at_5_std": 0.5883019874616514,
"naucs_at_5_diff1": 0.9626517273576021,
"naucs_at_10_max": 1.0,
"naucs_at_10_std": 1.0,
"naucs_at_10_diff1": 1.0,
"naucs_at_20_max": 1.0,
"naucs_at_20_std": 1.0,
"naucs_at_20_diff1": 1.0,
"naucs_at_50_max": 1.0,
"naucs_at_50_std": 1.0,
"naucs_at_50_diff1": 1.0,
"naucs_at_100_max": 1.0,
"naucs_at_100_std": 1.0,
"naucs_at_100_diff1": 1.0
},
"vidore/tatdqa_test": {
"ndcg_at_1": 0.66707,
"ndcg_at_3": 0.76435,
"ndcg_at_5": 0.78563,
"ndcg_at_10": 0.80266,
"ndcg_at_20": 0.81002,
"ndcg_at_50": 0.81426,
"ndcg_at_100": 0.81556,
"map_at_1": 0.66707,
"map_at_3": 0.74099,
"map_at_5": 0.75296,
"map_at_10": 0.76017,
"map_at_20": 0.76227,
"map_at_50": 0.76296,
"map_at_100": 0.76308,
"recall_at_1": 0.66707,
"recall_at_3": 0.83171,
"recall_at_5": 0.88275,
"recall_at_10": 0.93439,
"recall_at_20": 0.96294,
"recall_at_50": 0.9842,
"recall_at_100": 0.9921,
"precision_at_1": 0.66707,
"precision_at_3": 0.27724,
"precision_at_5": 0.17655,
"precision_at_10": 0.09344,
"precision_at_20": 0.04815,
"precision_at_50": 0.01968,
"precision_at_100": 0.00992,
"mrr_at_1": 0.6701093560145808,
"mrr_at_3": 0.7442284325637911,
"mrr_at_5": 0.7551336573511549,
"mrr_at_10": 0.7622198595922778,
"mrr_at_20": 0.7641436261113056,
"mrr_at_50": 0.7648243083644104,
"mrr_at_100": 0.764956538385537,
"naucs_at_1_max": 0.16664544436515671,
"naucs_at_1_std": -0.3026655256139227,
"naucs_at_1_diff1": 0.8139267777793499,
"naucs_at_3_max": 0.2218720138316067,
"naucs_at_3_std": -0.2588980164852128,
"naucs_at_3_diff1": 0.7085637741457546,
"naucs_at_5_max": 0.21174343657776057,
"naucs_at_5_std": -0.223401297475484,
"naucs_at_5_diff1": 0.6871017298675983,
"naucs_at_10_max": 0.34063196744044216,
"naucs_at_10_std": -0.08738627538479347,
"naucs_at_10_diff1": 0.7014648602426637,
"naucs_at_20_max": 0.4795886093835319,
"naucs_at_20_std": 0.18366030249088966,
"naucs_at_20_diff1": 0.656419869346376,
"naucs_at_50_max": 0.6537769988723047,
"naucs_at_50_std": 0.5912022233191522,
"naucs_at_50_diff1": 0.6681893202319851,
"naucs_at_100_max": 0.5982571033156504,
"naucs_at_100_std": 0.5589672729076165,
"naucs_at_100_diff1": 0.6339270114131129
},
"vidore/shiftproject_test": {
"ndcg_at_1": 0.8,
"ndcg_at_3": 0.8844,
"ndcg_at_5": 0.89258,
"ndcg_at_10": 0.90263,
"ndcg_at_20": 0.90263,
"ndcg_at_50": 0.90446,
"ndcg_at_100": 0.90446,
"map_at_1": 0.8,
"map_at_3": 0.865,
"map_at_5": 0.8695,
"map_at_10": 0.87385,
"map_at_20": 0.87385,
"map_at_50": 0.87408,
"map_at_100": 0.87408,
"recall_at_1": 0.8,
"recall_at_3": 0.94,
"recall_at_5": 0.96,
"recall_at_10": 0.99,
"recall_at_20": 0.99,
"recall_at_50": 1.0,
"recall_at_100": 1.0,
"precision_at_1": 0.8,
"precision_at_3": 0.31333,
"precision_at_5": 0.192,
"precision_at_10": 0.099,
"precision_at_20": 0.0495,
"precision_at_50": 0.02,
"precision_at_100": 0.01,
"mrr_at_1": 0.84,
"mrr_at_3": 0.8883333333333334,
"mrr_at_5": 0.8933333333333334,
"mrr_at_10": 0.8976785714285715,
"mrr_at_20": 0.8976785714285715,
"mrr_at_50": 0.8979111295681064,
"mrr_at_100": 0.8979111295681064,
"naucs_at_1_max": -0.04510489510489574,
"naucs_at_1_std": -0.40072427572427605,
"naucs_at_1_diff1": 0.7706293706293702,
"naucs_at_3_max": 0.41651104886398477,
"naucs_at_3_std": 0.053299097416741235,
"naucs_at_3_diff1": 0.679816370992838,
"naucs_at_5_max": 0.6626984126984157,
"naucs_at_5_std": 0.1178804855275477,
"naucs_at_5_diff1": 0.7916666666666669,
"naucs_at_10_max": 0.5541549953314738,
"naucs_at_10_std": 0.35807656395891135,
"naucs_at_10_diff1": 0.7222222222222276,
"naucs_at_20_max": 0.5541549953314738,
"naucs_at_20_std": 0.35807656395891135,
"naucs_at_20_diff1": 0.7222222222222276,
"naucs_at_50_max": null,
"naucs_at_50_std": null,
"naucs_at_50_diff1": null,
"naucs_at_100_max": null,
"naucs_at_100_std": null,
"naucs_at_100_diff1": null
},
"vidore/syntheticDocQA_artificial_intelligence_test": {
"ndcg_at_1": 0.98,
"ndcg_at_3": 0.99262,
"ndcg_at_5": 0.99262,
"ndcg_at_10": 0.99262,
"ndcg_at_20": 0.99262,
"ndcg_at_50": 0.99262,
"ndcg_at_100": 0.99262,
"map_at_1": 0.98,
"map_at_3": 0.99,
"map_at_5": 0.99,
"map_at_10": 0.99,
"map_at_20": 0.99,
"map_at_50": 0.99,
"map_at_100": 0.99,
"recall_at_1": 0.98,
"recall_at_3": 1.0,
"recall_at_5": 1.0,
"recall_at_10": 1.0,
"recall_at_20": 1.0,
"recall_at_50": 1.0,
"recall_at_100": 1.0,
"precision_at_1": 0.98,
"precision_at_3": 0.33333,
"precision_at_5": 0.2,
"precision_at_10": 0.1,
"precision_at_20": 0.05,
"precision_at_50": 0.02,
"precision_at_100": 0.01,
"mrr_at_1": 0.98,
"mrr_at_3": 0.99,
"mrr_at_5": 0.99,
"mrr_at_10": 0.99,
"mrr_at_20": 0.99,
"mrr_at_50": 0.99,
"mrr_at_100": 0.99,
"naucs_at_1_max": 0.4561157796451899,
"naucs_at_1_std": 0.4225023342670396,
"naucs_at_1_diff1": 1.0,
"naucs_at_3_max": 1.0,
"naucs_at_3_std": 1.0,
"naucs_at_3_diff1": 1.0,
"naucs_at_5_max": 1.0,
"naucs_at_5_std": 1.0,
"naucs_at_5_diff1": 1.0,
"naucs_at_10_max": 1.0,
"naucs_at_10_std": 1.0,
"naucs_at_10_diff1": 1.0,
"naucs_at_20_max": 1.0,
"naucs_at_20_std": 1.0,
"naucs_at_20_diff1": 1.0,
"naucs_at_50_max": null,
"naucs_at_50_std": null,
"naucs_at_50_diff1": null,
"naucs_at_100_max": null,
"naucs_at_100_std": null,
"naucs_at_100_diff1": null
},
"vidore/syntheticDocQA_energy_test": {
"ndcg_at_1": 0.95,
"ndcg_at_3": 0.96262,
"ndcg_at_5": 0.96262,
"ndcg_at_10": 0.96951,
"ndcg_at_20": 0.96951,
"ndcg_at_50": 0.97137,
"ndcg_at_100": 0.97137,
"map_at_1": 0.95,
"map_at_3": 0.96,
"map_at_5": 0.96,
"map_at_10": 0.9631,
"map_at_20": 0.9631,
"map_at_50": 0.96334,
"map_at_100": 0.96334,
"recall_at_1": 0.95,
"recall_at_3": 0.97,
"recall_at_5": 0.97,
"recall_at_10": 0.99,
"recall_at_20": 0.99,
"recall_at_50": 1.0,
"recall_at_100": 1.0,
"precision_at_1": 0.95,
"precision_at_3": 0.32333,
"precision_at_5": 0.194,
"precision_at_10": 0.099,
"precision_at_20": 0.0495,
"precision_at_50": 0.02,
"precision_at_100": 0.01,
"mrr_at_1": 0.95,
"mrr_at_3": 0.96,
"mrr_at_5": 0.9625,
"mrr_at_10": 0.9641666666666667,
"mrr_at_20": 0.9641666666666667,
"mrr_at_50": 0.9644166666666666,
"mrr_at_100": 0.9644166666666666,
"naucs_at_1_max": 0.33669467787114943,
"naucs_at_1_std": -0.9281979458450001,
"naucs_at_1_diff1": 1.0,
"naucs_at_3_max": 0.807812013694371,
"naucs_at_3_std": -1.1517273576097098,
"naucs_at_3_diff1": 1.0,
"naucs_at_5_max": 0.807812013694364,
"naucs_at_5_std": -1.1517273576097071,
"naucs_at_5_diff1": 1.0,
"naucs_at_10_max": 1.0,
"naucs_at_10_std": -1.1517273576097316,
"naucs_at_10_diff1": 1.0,
"naucs_at_20_max": 1.0,
"naucs_at_20_std": -1.1517273576097316,
"naucs_at_20_diff1": 1.0,
"naucs_at_50_max": null,
"naucs_at_50_std": null,
"naucs_at_50_diff1": null,
"naucs_at_100_max": null,
"naucs_at_100_std": null,
"naucs_at_100_diff1": null
},
"vidore/syntheticDocQA_government_reports_test": {
"ndcg_at_1": 0.92,
"ndcg_at_3": 0.95024,
"ndcg_at_5": 0.96316,
"ndcg_at_10": 0.96316,
"ndcg_at_20": 0.96316,
"ndcg_at_50": 0.96316,
"ndcg_at_100": 0.96316,
"map_at_1": 0.92,
"map_at_3": 0.94333,
"map_at_5": 0.95083,
"map_at_10": 0.95083,
"map_at_20": 0.95083,
"map_at_50": 0.95083,
"map_at_100": 0.95083,
"recall_at_1": 0.92,
"recall_at_3": 0.97,
"recall_at_5": 1.0,
"recall_at_10": 1.0,
"recall_at_20": 1.0,
"recall_at_50": 1.0,
"recall_at_100": 1.0,
"precision_at_1": 0.92,
"precision_at_3": 0.32333,
"precision_at_5": 0.2,
"precision_at_10": 0.1,
"precision_at_20": 0.05,
"precision_at_50": 0.02,
"precision_at_100": 0.01,
"mrr_at_1": 0.93,
"mrr_at_3": 0.9516666666666667,
"mrr_at_5": 0.9566666666666667,
"mrr_at_10": 0.9566666666666667,
"mrr_at_20": 0.9566666666666667,
"mrr_at_50": 0.9566666666666667,
"mrr_at_100": 0.9566666666666667,
"naucs_at_1_max": 0.6310107376283841,
"naucs_at_1_std": 0.33601774042950283,
"naucs_at_1_diff1": 0.9509803921568633,
"naucs_at_3_max": 0.7860255213196378,
"naucs_at_3_std": 0.24696545284781227,
"naucs_at_3_diff1": 1.0,
"naucs_at_5_max": 1.0,
"naucs_at_5_std": 1.0,
"naucs_at_5_diff1": 1.0,
"naucs_at_10_max": 1.0,
"naucs_at_10_std": 1.0,
"naucs_at_10_diff1": 1.0,
"naucs_at_20_max": 1.0,
"naucs_at_20_std": 1.0,
"naucs_at_20_diff1": 1.0,
"naucs_at_50_max": null,
"naucs_at_50_std": null,
"naucs_at_50_diff1": null,
"naucs_at_100_max": null,
"naucs_at_100_std": null,
"naucs_at_100_diff1": null
},
"vidore/syntheticDocQA_healthcare_industry_test": {
"ndcg_at_1": 0.98,
"ndcg_at_3": 0.99262,
"ndcg_at_5": 0.99262,
"ndcg_at_10": 0.99262,
"ndcg_at_20": 0.99262,
"ndcg_at_50": 0.99262,
"ndcg_at_100": 0.99262,
"map_at_1": 0.98,
"map_at_3": 0.99,
"map_at_5": 0.99,
"map_at_10": 0.99,
"map_at_20": 0.99,
"map_at_50": 0.99,
"map_at_100": 0.99,
"recall_at_1": 0.98,
"recall_at_3": 1.0,
"recall_at_5": 1.0,
"recall_at_10": 1.0,
"recall_at_20": 1.0,
"recall_at_50": 1.0,
"recall_at_100": 1.0,
"precision_at_1": 0.98,
"precision_at_3": 0.33333,
"precision_at_5": 0.2,
"precision_at_10": 0.1,
"precision_at_20": 0.05,
"precision_at_50": 0.02,
"precision_at_100": 0.01,
"mrr_at_1": 0.98,
"mrr_at_3": 0.99,
"mrr_at_5": 0.99,
"mrr_at_10": 0.99,
"mrr_at_20": 0.99,
"mrr_at_50": 0.99,
"mrr_at_100": 0.99,
"naucs_at_1_max": 0.7222222222222248,
"naucs_at_1_std": 0.5613912231559305,
"naucs_at_1_diff1": 0.9346405228758133,
"naucs_at_3_max": 1.0,
"naucs_at_3_std": 1.0,
"naucs_at_3_diff1": 1.0,
"naucs_at_5_max": 1.0,
"naucs_at_5_std": 1.0,
"naucs_at_5_diff1": 1.0,
"naucs_at_10_max": 1.0,
"naucs_at_10_std": 1.0,
"naucs_at_10_diff1": 1.0,
"naucs_at_20_max": 1.0,
"naucs_at_20_std": 1.0,
"naucs_at_20_diff1": 1.0,
"naucs_at_50_max": null,
"naucs_at_50_std": null,
"naucs_at_50_diff1": null,
"naucs_at_100_max": null,
"naucs_at_100_std": null,
"naucs_at_100_diff1": null
}
}