diff --git "a/eval_milebench/CounterfactualInference/pred_with_extracted.json" "b/eval_milebench/CounterfactualInference/pred_with_extracted.json" new file mode 100644--- /dev/null +++ "b/eval_milebench/CounterfactualInference/pred_with_extracted.json" @@ -0,0 +1,5449 @@ +[ + { + "sample_id": 0, + "question": "Scrutinize the visuals and respond to the related question about counterfactual inference. You must choose your answer from the Choice List.\nWhich of the following will happen if the cylinder is removed?\nChoice list: \nA. The cyan rubber object and the blue cube collide\nB. The brown cube collides with the metal cube\nC. The cyan rubber object and the metal cube collide\nD. The cyan rubber cube collides with the sphere\nAnswer with the option's letter from the given choices directly.", + "gt_response": "the cyan rubber cube collides with the sphere", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "d", + "choice_list": [ + "the cyan rubber object and the blue cube collide", + "the brown cube collides with the metal cube", + "the cyan rubber object and the metal cube collide", + "the cyan rubber cube collides with the sphere" + ], + "image_quantity_level": "Few", + "image": [ + "video_11363/video_11363_0.jpeg", + "video_11363/video_11363_1.jpeg", + "video_11363/video_11363_2.jpeg", + "video_11363/video_11363_3.jpeg", + "video_11363/video_11363_4.jpeg" + ], + "extracted": "D", + "result": 1 + }, + { + "sample_id": 1, + "question": "Inspect the given graphics and answer the relevant query about counterfactual inference. You must choose your answer from the Choice List.\nWhich event will happen if the sphere is removed?\nChoice list: \nA. The yellow object and the brown cylinder collide\nB. The brown cylinder and the cube collide\nC. The yellow cylinder collides with the cube\nAnswer with the option's letter from the given choices directly.", + "gt_response": "the yellow object and the brown cylinder collide", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "c", + "choice_list": [ + "the yellow object and the brown cylinder collide", + "the brown cylinder and the cube collide", + "the yellow cylinder collides with the cube" + ], + "image_quantity_level": "Few", + "image": [ + "video_13122/video_13122_0.jpeg", + "video_13122/video_13122_1.jpeg", + "video_13122/video_13122_2.jpeg", + "video_13122/video_13122_3.jpeg", + "video_13122/video_13122_4.jpeg" + ], + "extracted": "C", + "result": 0 + }, + { + "sample_id": 2, + "question": "Inspect the given graphics and answer the relevant query about counterfactual inference. You must choose your answer from the Choice List.\nIf the gray object is removed, which event will happen?\nChoice list: \nA. The rubber cube collides with the sphere\nB. The sphere collides with the blue cube\nC. The rubber cube and the blue object collide\nAnswer with the option's letter from the given choices directly.", + "gt_response": "the rubber cube collides with the sphere", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "a", + "choice_list": [ + "the rubber cube collides with the sphere", + "the sphere collides with the blue cube", + "the rubber cube and the blue object collide" + ], + "image_quantity_level": "Few", + "image": [ + "video_11884/video_11884_0.jpeg", + "video_11884/video_11884_1.jpeg", + "video_11884/video_11884_2.jpeg", + "video_11884/video_11884_3.jpeg", + "video_11884/video_11884_4.jpeg" + ], + "extracted": "A", + "result": 1 + }, + { + "sample_id": 3, + "question": "Based on the provided images, answer the question related to the counterfactual inference. You must choose your answer from the Choice List.\nWhich event will happen without the cube?\nChoice list: \nA. The red object and the brown cylinder collide\nB. The brown cylinder collides with the green sphere\nC. The gray cylinder and the green object collide\nD. The red object and the gray object collide\nAnswer with the option's letter from the given choices directly.", + "gt_response": "the red object and the gray object collide", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "b", + "choice_list": [ + "the red object and the brown cylinder collide", + "the brown cylinder collides with the green sphere", + "the gray cylinder and the green object collide", + "the red object and the gray object collide" + ], + "image_quantity_level": "Few", + "image": [ + "video_10142/video_10142_0.jpeg", + "video_10142/video_10142_1.jpeg", + "video_10142/video_10142_2.jpeg", + "video_10142/video_10142_3.jpeg", + "video_10142/video_10142_4.jpeg" + ], + "extracted": "B", + "result": 0 + }, + { + "sample_id": 4, + "question": "Examine the images and answer the relevant query about counterfactual inference. You must choose your answer from the Choice List.\nWhat will happen without the brown object?\nChoice list: \nA. The purple rubber cylinder and the sphere collide\nB. The cube and the gray object collide\nC. The purple rubber cylinder collides with the gray cylinder\nD. The purple metal object and the purple rubber cylinder collide\nAnswer with the option's letter from the given choices directly.", + "gt_response": "the purple metal object and the purple rubber cylinder collide", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "d", + "choice_list": [ + "the purple rubber cylinder and the sphere collide", + "the cube and the gray object collide", + "the purple rubber cylinder collides with the gray cylinder", + "the purple metal object and the purple rubber cylinder collide" + ], + "image_quantity_level": "Few", + "image": [ + "video_11736/video_11736_0.jpeg", + "video_11736/video_11736_1.jpeg", + "video_11736/video_11736_2.jpeg", + "video_11736/video_11736_3.jpeg", + "video_11736/video_11736_4.jpeg" + ], + "extracted": "D", + "result": 1 + }, + { + "sample_id": 5, + "question": "Evaluate the images presented and provide an answer to the associated question about counterfactual inference. You must choose your answer from the Choice List.\nWithout the yellow object, which event will happen?\nChoice list: \nA. The sphere collides with the cylinder\nB. The sphere collides with the metal cube\nC. The cylinder collides with the cyan object\nD. The sphere collides with the cyan cube\nAnswer with the option's letter from the given choices directly.", + "gt_response": "the sphere collides with the metal cube", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "b", + "choice_list": [ + "the sphere collides with the cylinder", + "the sphere collides with the metal cube", + "the cylinder collides with the cyan object", + "the sphere collides with the cyan cube" + ], + "image_quantity_level": "Few", + "image": [ + "video_10921/video_10921_0.jpeg", + "video_10921/video_10921_1.jpeg", + "video_10921/video_10921_2.jpeg", + "video_10921/video_10921_3.jpeg", + "video_10921/video_10921_4.jpeg" + ], + "extracted": "B", + "result": 1 + }, + { + "sample_id": 6, + "question": "Evaluate the images presented and provide an answer to the associated question about counterfactual inference. You must choose your answer from the Choice List.\nIf the cylinder is removed, what will happen?\nChoice list: \nA. The red object collides with the sphere\nB. The brown object and the red object collide\nAnswer with the option's letter from the given choices directly.", + "gt_response": "the brown object and the red object collide", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "b", + "choice_list": [ + "the red object collides with the sphere", + "the brown object and the red object collide" + ], + "image_quantity_level": "Few", + "image": [ + "video_12989/video_12989_0.jpeg", + "video_12989/video_12989_1.jpeg", + "video_12989/video_12989_2.jpeg", + "video_12989/video_12989_3.jpeg", + "video_12989/video_12989_4.jpeg" + ], + "extracted": "B", + "result": 1 + }, + { + "sample_id": 7, + "question": "Inspect the given graphics and answer the relevant query about counterfactual inference. You must choose your answer from the Choice List.\nWhich of the following will happen if the blue object is removed?\nChoice list: \nA. The gray object and the cube collide\nB. The cyan sphere and the gray object collide\nAnswer with the option's letter from the given choices directly.", + "gt_response": "the cyan sphere and the gray object collide", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "b", + "choice_list": [ + "the gray object and the cube collide", + "the cyan sphere and the gray object collide" + ], + "image_quantity_level": "Few", + "image": [ + "video_12209/video_12209_0.jpeg", + "video_12209/video_12209_1.jpeg", + "video_12209/video_12209_2.jpeg", + "video_12209/video_12209_3.jpeg", + "video_12209/video_12209_4.jpeg" + ], + "extracted": "B", + "result": 1 + }, + { + "sample_id": 8, + "question": "Examine the images and answer the relevant query about counterfactual inference. You must choose your answer from the Choice List.\nIf the cube is removed, which event will happen?\nChoice list: \nA. The metal sphere and the rubber sphere collide\nB. The metal sphere and the cylinder collide\nC. The cylinder and the rubber sphere collide\nAnswer with the option's letter from the given choices directly.", + "gt_response": "the metal sphere and the cylinder collide", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "b", + "choice_list": [ + "the metal sphere and the rubber sphere collide", + "the metal sphere and the cylinder collide", + "the cylinder and the rubber sphere collide" + ], + "image_quantity_level": "Few", + "image": [ + "video_12709/video_12709_0.jpeg", + "video_12709/video_12709_1.jpeg", + "video_12709/video_12709_2.jpeg", + "video_12709/video_12709_3.jpeg", + "video_12709/video_12709_4.jpeg" + ], + "extracted": "B", + "result": 1 + }, + { + "sample_id": 9, + "question": "Scrutinize the visuals and respond to the related question about counterfactual inference. You must choose your answer from the Choice List.\nIf the brown object is removed, what will happen?\nChoice list: \nA. The cube collides with the sphere\nB. The metal cylinder and the sphere collide\nC. The cube and the metal cylinder collide\nAnswer with the option's letter from the given choices directly.", + "gt_response": "the metal cylinder and the sphere collide", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "c", + "choice_list": [ + "the cube collides with the sphere", + "the metal cylinder and the sphere collide", + "the cube and the metal cylinder collide" + ], + "image_quantity_level": "Few", + "image": [ + "video_11263/video_11263_0.jpeg", + "video_11263/video_11263_1.jpeg", + "video_11263/video_11263_2.jpeg", + "video_11263/video_11263_3.jpeg", + "video_11263/video_11263_4.jpeg" + ], + "extracted": "C", + "result": 0 + }, + { + "sample_id": 10, + "question": "Analyze the given pictures and answer the related inquiry about counterfactual inference. You must choose your answer from the Choice List.\nWhat will happen without the red object?\nChoice list: \nA. The cylinder and the cube collide\nB. The purple sphere collides with the yellow sphere\nC. The yellow object and the cube collide\nAnswer with the option's letter from the given choices directly.", + "gt_response": "the cylinder and the cube collide", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "b", + "choice_list": [ + "the cylinder and the cube collide", + "the purple sphere collides with the yellow sphere", + "the yellow object and the cube collide" + ], + "image_quantity_level": "Few", + "image": [ + "video_12723/video_12723_0.jpeg", + "video_12723/video_12723_1.jpeg", + "video_12723/video_12723_2.jpeg", + "video_12723/video_12723_3.jpeg", + "video_12723/video_12723_4.jpeg" + ], + "extracted": "B", + "result": 0 + }, + { + "sample_id": 11, + "question": "Investigate the images and reply to the corresponding inquiry about counterfactual inference. You must choose your answer from the Choice List.\nIf the red object is removed, which event will happen?\nChoice list: \nA. The yellow object collides with the rubber cube\nB. The metal sphere collides with the yellow object\nC. The cyan sphere collides with the green sphere\nD. The green object and the blue cube collide\nAnswer with the option's letter from the given choices directly.", + "gt_response": "the metal sphere collides with the yellow object", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "b", + "choice_list": [ + "the yellow object collides with the rubber cube", + "the metal sphere collides with the yellow object", + "the cyan sphere collides with the green sphere", + "the green object and the blue cube collide" + ], + "image_quantity_level": "Few", + "image": [ + "video_13566/video_13566_0.jpeg", + "video_13566/video_13566_1.jpeg", + "video_13566/video_13566_2.jpeg", + "video_13566/video_13566_3.jpeg", + "video_13566/video_13566_4.jpeg" + ], + "extracted": "B", + "result": 1 + }, + { + "sample_id": 12, + "question": "Examine the images and answer the relevant query about counterfactual inference. You must choose your answer from the Choice List.\nIf the sphere is removed, what will happen?\nChoice list: \nA. The green cylinder and the red cylinder collide\nB. The green object and the cube collide\nAnswer with the option's letter from the given choices directly.", + "gt_response": "the green cylinder and the red cylinder collide", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "a", + "choice_list": [ + "the green cylinder and the red cylinder collide", + "the green object and the cube collide" + ], + "image_quantity_level": "Few", + "image": [ + "video_13315/video_13315_0.jpeg", + "video_13315/video_13315_1.jpeg", + "video_13315/video_13315_2.jpeg", + "video_13315/video_13315_3.jpeg", + "video_13315/video_13315_4.jpeg" + ], + "extracted": "A", + "result": 1 + }, + { + "sample_id": 13, + "question": "Scrutinize the visuals and respond to the related question about counterfactual inference. You must choose your answer from the Choice List.\nWhat will happen without the sphere?\nChoice list: \nA. The gray metal object collides with the rubber cylinder\nB. The gray metal cylinder and the cyan object collide\nC. The rubber cylinder and the cyan cylinder collide\nAnswer with the option's letter from the given choices directly.", + "gt_response": "the rubber cylinder and the cyan cylinder collide", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "a", + "choice_list": [ + "the gray metal object collides with the rubber cylinder", + "the gray metal cylinder and the cyan object collide", + "the rubber cylinder and the cyan cylinder collide" + ], + "image_quantity_level": "Few", + "image": [ + "video_11834/video_11834_0.jpeg", + "video_11834/video_11834_1.jpeg", + "video_11834/video_11834_2.jpeg", + "video_11834/video_11834_3.jpeg", + "video_11834/video_11834_4.jpeg" + ], + "extracted": "A", + "result": 0 + }, + { + "sample_id": 14, + "question": "Evaluate the images presented and provide an answer to the associated question about counterfactual inference. You must choose your answer from the Choice List.\nWhich event will happen without the blue object?\nChoice list: \nA. The yellow cylinder collides with the green cylinder\nB. The cube collides with the rubber cylinder\nC. The cube collides with the green sphere\nD. The green sphere and the metal cylinder collide\nAnswer with the option's letter from the given choices directly.", + "gt_response": "the cube collides with the rubber cylinder", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "b", + "choice_list": [ + "the yellow cylinder collides with the green cylinder", + "the cube collides with the rubber cylinder", + "the cube collides with the green sphere", + "the green sphere and the metal cylinder collide" + ], + "image_quantity_level": "Few", + "image": [ + "video_14132/video_14132_0.jpeg", + "video_14132/video_14132_1.jpeg", + "video_14132/video_14132_2.jpeg", + "video_14132/video_14132_3.jpeg", + "video_14132/video_14132_4.jpeg" + ], + "extracted": "B", + "result": 1 + }, + { + "sample_id": 15, + "question": "Investigate the images and reply to the corresponding inquiry about counterfactual inference. You must choose your answer from the Choice List.\nWhat will happen if the cylinder is removed?\nChoice list: \nA. The red object collides with the rubber cube\nB. The yellow sphere and the green metal object collide\nC. The yellow sphere and the rubber sphere collide\nD. The metal cube collides with the yellow sphere\nAnswer with the option's letter from the given choices directly.", + "gt_response": "the metal cube collides with the yellow sphere", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "a", + "choice_list": [ + "the red object collides with the rubber cube", + "the yellow sphere and the green metal object collide", + "the yellow sphere and the rubber sphere collide", + "the metal cube collides with the yellow sphere" + ], + "image_quantity_level": "Few", + "image": [ + "video_14309/video_14309_0.jpeg", + "video_14309/video_14309_1.jpeg", + "video_14309/video_14309_2.jpeg", + "video_14309/video_14309_3.jpeg", + "video_14309/video_14309_4.jpeg" + ], + "extracted": "A", + "result": 0 + }, + { + "sample_id": 16, + "question": "Based on the provided images, answer the question related to the counterfactual inference. You must choose your answer from the Choice List.\nWhich event will happen without the cube?\nChoice list: \nA. The metal cylinder collides with the rubber cylinder\nB. The metal cylinder and the sphere collide\nAnswer with the option's letter from the given choices directly.", + "gt_response": "the metal cylinder collides with the rubber cylinder", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "a", + "choice_list": [ + "the metal cylinder collides with the rubber cylinder", + "the metal cylinder and the sphere collide" + ], + "image_quantity_level": "Few", + "image": [ + "video_10285/video_10285_0.jpeg", + "video_10285/video_10285_1.jpeg", + "video_10285/video_10285_2.jpeg", + "video_10285/video_10285_3.jpeg", + "video_10285/video_10285_4.jpeg" + ], + "extracted": "A", + "result": 1 + }, + { + "sample_id": 17, + "question": "Based on the provided images, answer the question related to the counterfactual inference. You must choose your answer from the Choice List.\nWhat will happen if the green object is removed?\nChoice list: \nA. The sphere collides with the yellow cylinder\nB. The sphere and the brown cube collide\nC. The brown object collides with the yellow cylinder\nD. The yellow cube collides with the yellow cylinder\nAnswer with the option's letter from the given choices directly.", + "gt_response": "the sphere collides with the yellow cylinder", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "a", + "choice_list": [ + "the sphere collides with the yellow cylinder", + "the sphere and the brown cube collide", + "the brown object collides with the yellow cylinder", + "the yellow cube collides with the yellow cylinder" + ], + "image_quantity_level": "Few", + "image": [ + "video_10730/video_10730_0.jpeg", + "video_10730/video_10730_1.jpeg", + "video_10730/video_10730_2.jpeg", + "video_10730/video_10730_3.jpeg", + "video_10730/video_10730_4.jpeg" + ], + "extracted": "A", + "result": 1 + }, + { + "sample_id": 18, + "question": "Observe the images and reply to the pertinent question about counterfactual inference. You must choose your answer from the Choice List.\nWhat will happen if the red object is removed?\nChoice list: \nA. The gray object collides with the cyan object\nB. The blue object collides with the cyan sphere\nAnswer with the option's letter from the given choices directly.", + "gt_response": "the gray object collides with the cyan object", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "a", + "choice_list": [ + "the gray object collides with the cyan object", + "the blue object collides with the cyan sphere" + ], + "image_quantity_level": "Few", + "image": [ + "video_11136/video_11136_0.jpeg", + "video_11136/video_11136_1.jpeg", + "video_11136/video_11136_2.jpeg", + "video_11136/video_11136_3.jpeg", + "video_11136/video_11136_4.jpeg" + ], + "extracted": "A", + "result": 1 + }, + { + "sample_id": 19, + "question": "Investigate the images and reply to the corresponding inquiry about counterfactual inference. You must choose your answer from the Choice List.\nWhat will happen without the sphere?\nChoice list: \nA. The gray cylinder and the yellow cylinder collide\nB. The cyan cylinder and the yellow cylinder collide\nAnswer with the option's letter from the given choices directly.", + "gt_response": "the cyan cylinder and the yellow cylinder collide", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "b", + "choice_list": [ + "the gray cylinder and the yellow cylinder collide", + "the cyan cylinder and the yellow cylinder collide" + ], + "image_quantity_level": "Few", + "image": [ + "video_12783/video_12783_0.jpeg", + "video_12783/video_12783_1.jpeg", + "video_12783/video_12783_2.jpeg", + "video_12783/video_12783_3.jpeg", + "video_12783/video_12783_4.jpeg" + ], + "extracted": "B", + "result": 1 + }, + { + "sample_id": 20, + "question": "Investigate the images and reply to the corresponding inquiry about counterfactual inference. You must choose your answer from the Choice List.\nWhat will happen if the sphere is removed?\nChoice list: \nA. The blue object and the brown cube collide\nB. The yellow object and the red object collide\nC. The blue cube and the red cube collide\nD. The yellow cube and the blue cube collide\nAnswer with the option's letter from the given choices directly.", + "gt_response": "the blue object and the brown cube collide", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "a", + "choice_list": [ + "the blue object and the brown cube collide", + "the yellow object and the red object collide", + "the blue cube and the red cube collide", + "the yellow cube and the blue cube collide" + ], + "image_quantity_level": "Few", + "image": [ + "video_11189/video_11189_0.jpeg", + "video_11189/video_11189_1.jpeg", + "video_11189/video_11189_2.jpeg", + "video_11189/video_11189_3.jpeg", + "video_11189/video_11189_4.jpeg" + ], + "extracted": "A", + "result": 1 + }, + { + "sample_id": 21, + "question": "Observe the images and reply to the pertinent question about counterfactual inference. You must choose your answer from the Choice List.\nWithout the gray object, which of the following will happen?\nChoice list: \nA. The yellow cylinder collides with the blue object\nB. The sphere collides with the blue object\nC. The sphere and the rubber object collide\nAnswer with the option's letter from the given choices directly.", + "gt_response": "the sphere collides with the blue object", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "c", + "choice_list": [ + "the yellow cylinder collides with the blue object", + "the sphere collides with the blue object", + "the sphere and the rubber object collide" + ], + "image_quantity_level": "Few", + "image": [ + "video_12145/video_12145_0.jpeg", + "video_12145/video_12145_1.jpeg", + "video_12145/video_12145_2.jpeg", + "video_12145/video_12145_3.jpeg", + "video_12145/video_12145_4.jpeg" + ], + "extracted": "C", + "result": 0 + }, + { + "sample_id": 22, + "question": "Analyze the given pictures and answer the related inquiry about counterfactual inference. You must choose your answer from the Choice List.\nWhich event will happen if the yellow object is removed?\nChoice list: \nA. The sphere collides with the brown object\nB. The gray object and the brown cylinder collide\nC. The gray cube collides with the sphere\nD. The purple cube and the brown cylinder collide\nAnswer with the option's letter from the given choices directly.", + "gt_response": "the gray cube collides with the sphere", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "c", + "choice_list": [ + "the sphere collides with the brown object", + "the gray object and the brown cylinder collide", + "the gray cube collides with the sphere", + "the purple cube and the brown cylinder collide" + ], + "image_quantity_level": "Few", + "image": [ + "video_13235/video_13235_0.jpeg", + "video_13235/video_13235_1.jpeg", + "video_13235/video_13235_2.jpeg", + "video_13235/video_13235_3.jpeg", + "video_13235/video_13235_4.jpeg" + ], + "extracted": "C", + "result": 1 + }, + { + "sample_id": 23, + "question": "Review the photos and answer the associated inquiry about counterfactual inference. You must choose your answer from the Choice List.\nWhich event will happen if the cylinder is removed?\nChoice list: \nA. The blue object and the yellow sphere collide\nB. The blue object and the cube collide\nAnswer with the option's letter from the given choices directly.", + "gt_response": "the blue object and the cube collide", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "a", + "choice_list": [ + "the blue object and the yellow sphere collide", + "the blue object and the cube collide" + ], + "image_quantity_level": "Few", + "image": [ + "video_11586/video_11586_0.jpeg", + "video_11586/video_11586_1.jpeg", + "video_11586/video_11586_2.jpeg", + "video_11586/video_11586_3.jpeg", + "video_11586/video_11586_4.jpeg" + ], + "extracted": "A", + "result": 0 + }, + { + "sample_id": 24, + "question": "Analyze the given pictures and answer the related inquiry about counterfactual inference. You must choose your answer from the Choice List.\nWhat will happen if the cube is removed?\nChoice list: \nA. The brown cylinder and the sphere collide\nB. The sphere and the gray cylinder collide\nAnswer with the option's letter from the given choices directly.", + "gt_response": "the brown cylinder and the sphere collide", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "a", + "choice_list": [ + "the brown cylinder and the sphere collide", + "the sphere and the gray cylinder collide" + ], + "image_quantity_level": "Few", + "image": [ + "video_10082/video_10082_0.jpeg", + "video_10082/video_10082_1.jpeg", + "video_10082/video_10082_2.jpeg", + "video_10082/video_10082_3.jpeg", + "video_10082/video_10082_4.jpeg" + ], + "extracted": "A", + "result": 1 + }, + { + "sample_id": 25, + "question": "Scrutinize the visuals and respond to the related question about counterfactual inference. You must choose your answer from the Choice List.\nWithout the metal object, what will happen?\nChoice list: \nA. The yellow sphere and the red object collide\nB. The rubber cylinder and the red sphere collide\nAnswer with the option's letter from the given choices directly.", + "gt_response": "the yellow sphere and the red object collide", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "a", + "choice_list": [ + "the yellow sphere and the red object collide", + "the rubber cylinder and the red sphere collide" + ], + "image_quantity_level": "Few", + "image": [ + "video_11434/video_11434_0.jpeg", + "video_11434/video_11434_1.jpeg", + "video_11434/video_11434_2.jpeg", + "video_11434/video_11434_3.jpeg", + "video_11434/video_11434_4.jpeg" + ], + "extracted": "A", + "result": 1 + }, + { + "sample_id": 26, + "question": "Observe the images and reply to the pertinent question about counterfactual inference. You must choose your answer from the Choice List.\nIf the cylinder is removed, what will happen?\nChoice list: \nA. The rubber sphere and the green sphere collide\nB. The rubber sphere and the cube collide\nAnswer with the option's letter from the given choices directly.", + "gt_response": "the rubber sphere and the green sphere collide", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "b", + "choice_list": [ + "the rubber sphere and the green sphere collide", + "the rubber sphere and the cube collide" + ], + "image_quantity_level": "Few", + "image": [ + "video_10277/video_10277_0.jpeg", + "video_10277/video_10277_1.jpeg", + "video_10277/video_10277_2.jpeg", + "video_10277/video_10277_3.jpeg", + "video_10277/video_10277_4.jpeg" + ], + "extracted": "B", + "result": 0 + }, + { + "sample_id": 27, + "question": "Analyze the given pictures and answer the related inquiry about counterfactual inference. You must choose your answer from the Choice List.\nIf the yellow object is removed, which event will happen?\nChoice list: \nA. The cylinder and the metal sphere collide\nB. The cylinder collides with the blue sphere\nC. The metal sphere and the blue sphere collide\nAnswer with the option's letter from the given choices directly.", + "gt_response": "the cylinder and the metal sphere collide", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "a", + "choice_list": [ + "the cylinder and the metal sphere collide", + "the cylinder collides with the blue sphere", + "the metal sphere and the blue sphere collide" + ], + "image_quantity_level": "Few", + "image": [ + "video_11680/video_11680_0.jpeg", + "video_11680/video_11680_1.jpeg", + "video_11680/video_11680_2.jpeg", + "video_11680/video_11680_3.jpeg", + "video_11680/video_11680_4.jpeg" + ], + "extracted": "A", + "result": 1 + }, + { + "sample_id": 28, + "question": "Evaluate the images presented and provide an answer to the associated question about counterfactual inference. You must choose your answer from the Choice List.\nIf the purple object is removed, which event will happen?\nChoice list: \nA. The green object collides with the yellow sphere\nB. The green cylinder and the cube collide\nC. The gray object collides with the cube\nD. The yellow sphere and the cube collide\nAnswer with the option's letter from the given choices directly.", + "gt_response": "the green object collides with the yellow sphere", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "a", + "choice_list": [ + "the green object collides with the yellow sphere", + "the green cylinder and the cube collide", + "the gray object collides with the cube", + "the yellow sphere and the cube collide" + ], + "image_quantity_level": "Few", + "image": [ + "video_11811/video_11811_0.jpeg", + "video_11811/video_11811_1.jpeg", + "video_11811/video_11811_2.jpeg", + "video_11811/video_11811_3.jpeg", + "video_11811/video_11811_4.jpeg" + ], + "extracted": "A", + "result": 1 + }, + { + "sample_id": 29, + "question": "Analyze the given pictures and answer the related inquiry about counterfactual inference. You must choose your answer from the Choice List.\nIf the cube is removed, which event will happen?\nChoice list: \nA. The brown object collides with the gray object\nB. The rubber sphere collides with the gray sphere\nC. The brown cylinder and the cyan sphere collide\nD. The gray sphere collides with the rubber cylinder\nAnswer with the option's letter from the given choices directly.", + "gt_response": "the brown cylinder and the cyan sphere collide", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "b", + "choice_list": [ + "the brown object collides with the gray object", + "the rubber sphere collides with the gray sphere", + "the brown cylinder and the cyan sphere collide", + "the gray sphere collides with the rubber cylinder" + ], + "image_quantity_level": "Few", + "image": [ + "video_14293/video_14293_0.jpeg", + "video_14293/video_14293_1.jpeg", + "video_14293/video_14293_2.jpeg", + "video_14293/video_14293_3.jpeg", + "video_14293/video_14293_4.jpeg" + ], + "extracted": "B", + "result": 0 + }, + { + "sample_id": 30, + "question": "Assess the pictures and answer the connected query about counterfactual inference. You must choose your answer from the Choice List.\nWhich event will happen without the cylinder?\nChoice list: \nA. The cyan sphere and the metal cube collide\nB. The gray object collides with the green sphere\nC. The cyan object and the yellow object collide\nD. The gray cube and the red cube collide\nAnswer with the option's letter from the given choices directly.", + "gt_response": "the gray cube and the red cube collide", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "d", + "choice_list": [ + "the cyan sphere and the metal cube collide", + "the gray object collides with the green sphere", + "the cyan object and the yellow object collide", + "the gray cube and the red cube collide" + ], + "image_quantity_level": "Few", + "image": [ + "video_10039/video_10039_0.jpeg", + "video_10039/video_10039_1.jpeg", + "video_10039/video_10039_2.jpeg", + "video_10039/video_10039_3.jpeg", + "video_10039/video_10039_4.jpeg" + ], + "extracted": "D", + "result": 1 + }, + { + "sample_id": 31, + "question": "Scrutinize the visuals and respond to the related question about counterfactual inference. You must choose your answer from the Choice List.\nIf the cube is removed, which of the following will happen?\nChoice list: \nA. The rubber cylinder collides with the green cylinder\nB. The metal sphere collides with the green cylinder\nC. The metal sphere and the cyan cylinder collide\nD. The metal sphere collides with the rubber cylinder\nAnswer with the option's letter from the given choices directly.", + "gt_response": "the rubber cylinder collides with the green cylinder", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "d", + "choice_list": [ + "the rubber cylinder collides with the green cylinder", + "the metal sphere collides with the green cylinder", + "the metal sphere and the cyan cylinder collide", + "the metal sphere collides with the rubber cylinder" + ], + "image_quantity_level": "Few", + "image": [ + "video_13341/video_13341_0.jpeg", + "video_13341/video_13341_1.jpeg", + "video_13341/video_13341_2.jpeg", + "video_13341/video_13341_3.jpeg", + "video_13341/video_13341_4.jpeg" + ], + "extracted": "D", + "result": 0 + }, + { + "sample_id": 32, + "question": "Assess the pictures and answer the connected query about counterfactual inference. You must choose your answer from the Choice List.\nWhat will happen if the gray object is removed?\nChoice list: \nA. The sphere collides with the blue cylinder\nB. The sphere collides with the yellow cube\nC. The cyan cylinder and the yellow cube collide\nD. The cyan cylinder collides with the blue object\nAnswer with the option's letter from the given choices directly.", + "gt_response": "the sphere collides with the blue cylinder", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "d", + "choice_list": [ + "the sphere collides with the blue cylinder", + "the sphere collides with the yellow cube", + "the cyan cylinder and the yellow cube collide", + "the cyan cylinder collides with the blue object" + ], + "image_quantity_level": "Few", + "image": [ + "video_14105/video_14105_0.jpeg", + "video_14105/video_14105_1.jpeg", + "video_14105/video_14105_2.jpeg", + "video_14105/video_14105_3.jpeg", + "video_14105/video_14105_4.jpeg" + ], + "extracted": "D", + "result": 0 + }, + { + "sample_id": 33, + "question": "Investigate the images and reply to the corresponding inquiry about counterfactual inference. You must choose your answer from the Choice List.\nWithout the yellow object, which of the following will happen?\nChoice list: \nA. The brown object collides with the green sphere\nB. The red sphere collides with the cylinder\nC. The brown object collides with the red sphere\nD. The red sphere collides with the green sphere\nAnswer with the option's letter from the given choices directly.", + "gt_response": "the brown object collides with the red sphere", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "c", + "choice_list": [ + "the brown object collides with the green sphere", + "the red sphere collides with the cylinder", + "the brown object collides with the red sphere", + "the red sphere collides with the green sphere" + ], + "image_quantity_level": "Few", + "image": [ + "video_12197/video_12197_0.jpeg", + "video_12197/video_12197_1.jpeg", + "video_12197/video_12197_2.jpeg", + "video_12197/video_12197_3.jpeg", + "video_12197/video_12197_4.jpeg" + ], + "extracted": "C", + "result": 1 + }, + { + "sample_id": 34, + "question": "Based on the provided images, answer the question related to the counterfactual inference. You must choose your answer from the Choice List.\nIf the purple object is removed, which of the following will happen?\nChoice list: \nA. The red cylinder collides with the cyan sphere\nB. The cyan object and the red sphere collide\nC. The metal cylinder collides with the cyan object\nD. The rubber cylinder and the red sphere collide\nAnswer with the option's letter from the given choices directly.", + "gt_response": "the metal cylinder collides with the cyan object", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "c", + "choice_list": [ + "the red cylinder collides with the cyan sphere", + "the cyan object and the red sphere collide", + "the metal cylinder collides with the cyan object", + "the rubber cylinder and the red sphere collide" + ], + "image_quantity_level": "Few", + "image": [ + "video_14749/video_14749_0.jpeg", + "video_14749/video_14749_1.jpeg", + "video_14749/video_14749_2.jpeg", + "video_14749/video_14749_3.jpeg", + "video_14749/video_14749_4.jpeg" + ], + "extracted": "C", + "result": 1 + }, + { + "sample_id": 35, + "question": "Investigate the images and reply to the corresponding inquiry about counterfactual inference. You must choose your answer from the Choice List.\nWhich of the following will happen if the purple object is removed?\nChoice list: \nA. The brown sphere and the cylinder collide\nB. The blue object collides with the brown sphere\nAnswer with the option's letter from the given choices directly.", + "gt_response": "the blue object collides with the brown sphere", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "b", + "choice_list": [ + "the brown sphere and the cylinder collide", + "the blue object collides with the brown sphere" + ], + "image_quantity_level": "Few", + "image": [ + "video_14106/video_14106_0.jpeg", + "video_14106/video_14106_1.jpeg", + "video_14106/video_14106_2.jpeg", + "video_14106/video_14106_3.jpeg", + "video_14106/video_14106_4.jpeg" + ], + "extracted": "B", + "result": 1 + }, + { + "sample_id": 36, + "question": "Assess the pictures and answer the connected query about counterfactual inference. You must choose your answer from the Choice List.\nIf the cylinder is removed, which of the following will happen?\nChoice list: \nA. The brown object collides with the blue object\nB. The cyan object collides with the blue cube\nC. The red cube collides with the blue cube\nD. The cyan object and the brown sphere collide\nAnswer with the option's letter from the given choices directly.", + "gt_response": "the cyan object and the brown sphere collide", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "d", + "choice_list": [ + "the brown object collides with the blue object", + "the cyan object collides with the blue cube", + "the red cube collides with the blue cube", + "the cyan object and the brown sphere collide" + ], + "image_quantity_level": "Few", + "image": [ + "video_14015/video_14015_0.jpeg", + "video_14015/video_14015_1.jpeg", + "video_14015/video_14015_2.jpeg", + "video_14015/video_14015_3.jpeg", + "video_14015/video_14015_4.jpeg" + ], + "extracted": "D", + "result": 1 + }, + { + "sample_id": 37, + "question": "Assess the pictures and answer the connected query about counterfactual inference. You must choose your answer from the Choice List.\nWhich event will happen without the cylinder?\nChoice list: \nA. The rubber sphere and the blue sphere collide\nB. The rubber sphere collides with the gray metal sphere\nC. The gray metal sphere and the blue sphere collide\nAnswer with the option's letter from the given choices directly.", + "gt_response": "the rubber sphere collides with the gray metal sphere", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "b", + "choice_list": [ + "the rubber sphere and the blue sphere collide", + "the rubber sphere collides with the gray metal sphere", + "the gray metal sphere and the blue sphere collide" + ], + "image_quantity_level": "Few", + "image": [ + "video_10765/video_10765_0.jpeg", + "video_10765/video_10765_1.jpeg", + "video_10765/video_10765_2.jpeg", + "video_10765/video_10765_3.jpeg", + "video_10765/video_10765_4.jpeg" + ], + "extracted": "B", + "result": 1 + }, + { + "sample_id": 38, + "question": "Inspect the given graphics and answer the relevant query about counterfactual inference. You must choose your answer from the Choice List.\nWhich event will happen if the cyan object is removed?\nChoice list: \nA. The sphere and the brown object collide\nB. The brown cube collides with the red cylinder\nC. The blue object and the yellow cylinder collide\nD. The yellow cylinder collides with the red object\nAnswer with the option's letter from the given choices directly.", + "gt_response": "the sphere and the brown object collide", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "c", + "choice_list": [ + "the sphere and the brown object collide", + "the brown cube collides with the red cylinder", + "the blue object and the yellow cylinder collide", + "the yellow cylinder collides with the red object" + ], + "image_quantity_level": "Few", + "image": [ + "video_13846/video_13846_0.jpeg", + "video_13846/video_13846_1.jpeg", + "video_13846/video_13846_2.jpeg", + "video_13846/video_13846_3.jpeg", + "video_13846/video_13846_4.jpeg" + ], + "extracted": "C", + "result": 0 + }, + { + "sample_id": 39, + "question": "Scrutinize the visuals and respond to the related question about counterfactual inference. You must choose your answer from the Choice List.\nWithout the red object, which event will happen?\nChoice list: \nA. The blue metal sphere and the gray sphere collide\nB. The blue rubber object and the gray object collide\nC. The blue metal sphere and the blue rubber object collide\nD. The cylinder and the cyan sphere collide\nAnswer with the option's letter from the given choices directly.", + "gt_response": "the blue metal sphere and the gray sphere collide", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "c", + "choice_list": [ + "the blue metal sphere and the gray sphere collide", + "the blue rubber object and the gray object collide", + "the blue metal sphere and the blue rubber object collide", + "the cylinder and the cyan sphere collide" + ], + "image_quantity_level": "Few", + "image": [ + "video_11295/video_11295_0.jpeg", + "video_11295/video_11295_1.jpeg", + "video_11295/video_11295_2.jpeg", + "video_11295/video_11295_3.jpeg", + "video_11295/video_11295_4.jpeg" + ], + "extracted": "C", + "result": 0 + }, + { + "sample_id": 40, + "question": "Inspect the given graphics and answer the relevant query about counterfactual inference. You must choose your answer from the Choice List.\nIf the sphere is removed, which event will happen?\nChoice list: \nA. The cylinder collides with the purple cube\nB. The yellow object and the cylinder collide\nAnswer with the option's letter from the given choices directly.", + "gt_response": "the yellow object and the cylinder collide", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "b", + "choice_list": [ + "the cylinder collides with the purple cube", + "the yellow object and the cylinder collide" + ], + "image_quantity_level": "Few", + "image": [ + "video_11413/video_11413_0.jpeg", + "video_11413/video_11413_1.jpeg", + "video_11413/video_11413_2.jpeg", + "video_11413/video_11413_3.jpeg", + "video_11413/video_11413_4.jpeg" + ], + "extracted": "B", + "result": 1 + }, + { + "sample_id": 41, + "question": "Scrutinize the visuals and respond to the related question about counterfactual inference. You must choose your answer from the Choice List.\nWithout the red object, which event will happen?\nChoice list: \nA. The brown object and the purple cylinder collide\nB. The yellow sphere collides with the cyan sphere\nC. The cube collides with the purple cylinder\nD. The cube and the cyan sphere collide\nAnswer with the option's letter from the given choices directly.", + "gt_response": "the cube collides with the purple cylinder", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "c", + "choice_list": [ + "the brown object and the purple cylinder collide", + "the yellow sphere collides with the cyan sphere", + "the cube collides with the purple cylinder", + "the cube and the cyan sphere collide" + ], + "image_quantity_level": "Few", + "image": [ + "video_14058/video_14058_0.jpeg", + "video_14058/video_14058_1.jpeg", + "video_14058/video_14058_2.jpeg", + "video_14058/video_14058_3.jpeg", + "video_14058/video_14058_4.jpeg" + ], + "extracted": "C", + "result": 1 + }, + { + "sample_id": 42, + "question": "Examine the images and answer the relevant query about counterfactual inference. You must choose your answer from the Choice List.\nWhich of the following will happen without the cube?\nChoice list: \nA. The brown cylinder collides with the cyan cylinder\nB. The gray sphere collides with the cyan cylinder\nC. The cyan cylinder collides with the cyan sphere\nD. The brown cylinder collides with the cyan sphere\nAnswer with the option's letter from the given choices directly.", + "gt_response": "the gray sphere collides with the cyan cylinder", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "a", + "choice_list": [ + "the brown cylinder collides with the cyan cylinder", + "the gray sphere collides with the cyan cylinder", + "the cyan cylinder collides with the cyan sphere", + "the brown cylinder collides with the cyan sphere" + ], + "image_quantity_level": "Few", + "image": [ + "video_10402/video_10402_0.jpeg", + "video_10402/video_10402_1.jpeg", + "video_10402/video_10402_2.jpeg", + "video_10402/video_10402_3.jpeg", + "video_10402/video_10402_4.jpeg" + ], + "extracted": "A", + "result": 0 + }, + { + "sample_id": 43, + "question": "Review the photos and answer the associated inquiry about counterfactual inference. You must choose your answer from the Choice List.\nIf the blue object is removed, which of the following will happen?\nChoice list: \nA. The cyan cube collides with the yellow object\nB. The cylinder and the yellow object collide\nC. The cylinder collides with the cyan cube\nAnswer with the option's letter from the given choices directly.", + "gt_response": "the cyan cube collides with the yellow object", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "c", + "choice_list": [ + "the cyan cube collides with the yellow object", + "the cylinder and the yellow object collide", + "the cylinder collides with the cyan cube" + ], + "image_quantity_level": "Few", + "image": [ + "video_11085/video_11085_0.jpeg", + "video_11085/video_11085_1.jpeg", + "video_11085/video_11085_2.jpeg", + "video_11085/video_11085_3.jpeg", + "video_11085/video_11085_4.jpeg" + ], + "extracted": "C", + "result": 0 + }, + { + "sample_id": 44, + "question": "Observe the images and reply to the pertinent question about counterfactual inference. You must choose your answer from the Choice List.\nWhat will happen if the sphere is removed?\nChoice list: \nA. The blue cylinder collides with the cyan cylinder\nB. The blue cylinder collides with the yellow cylinder\nC. The green object collides with the yellow object\nD. The blue cylinder collides with the green object\nAnswer with the option's letter from the given choices directly.", + "gt_response": "the blue cylinder collides with the cyan cylinder", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "d", + "choice_list": [ + "the blue cylinder collides with the cyan cylinder", + "the blue cylinder collides with the yellow cylinder", + "the green object collides with the yellow object", + "the blue cylinder collides with the green object" + ], + "image_quantity_level": "Few", + "image": [ + "video_14244/video_14244_0.jpeg", + "video_14244/video_14244_1.jpeg", + "video_14244/video_14244_2.jpeg", + "video_14244/video_14244_3.jpeg", + "video_14244/video_14244_4.jpeg" + ], + "extracted": "D", + "result": 0 + }, + { + "sample_id": 45, + "question": "Analyze the given pictures and answer the related inquiry about counterfactual inference. You must choose your answer from the Choice List.\nIf the cube is removed, which of the following will happen?\nChoice list: \nA. The brown sphere collides with the rubber sphere\nB. The cylinder collides with the yellow sphere\nAnswer with the option's letter from the given choices directly.", + "gt_response": "the cylinder collides with the yellow sphere", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "a", + "choice_list": [ + "the brown sphere collides with the rubber sphere", + "the cylinder collides with the yellow sphere" + ], + "image_quantity_level": "Few", + "image": [ + "video_13862/video_13862_0.jpeg", + "video_13862/video_13862_1.jpeg", + "video_13862/video_13862_2.jpeg", + "video_13862/video_13862_3.jpeg", + "video_13862/video_13862_4.jpeg" + ], + "extracted": "A", + "result": 0 + }, + { + "sample_id": 46, + "question": "Assess the pictures and answer the connected query about counterfactual inference. You must choose your answer from the Choice List.\nWhich event will happen if the green object is removed?\nChoice list: \nA. The brown cylinder and the purple cylinder collide\nB. The brown object collides with the blue object\nC. The brown cylinder collides with the yellow object\nD. The yellow object collides with the rubber sphere\nAnswer with the option's letter from the given choices directly.", + "gt_response": "the brown cylinder collides with the yellow object", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "c", + "choice_list": [ + "the brown cylinder and the purple cylinder collide", + "the brown object collides with the blue object", + "the brown cylinder collides with the yellow object", + "the yellow object collides with the rubber sphere" + ], + "image_quantity_level": "Few", + "image": [ + "video_11204/video_11204_0.jpeg", + "video_11204/video_11204_1.jpeg", + "video_11204/video_11204_2.jpeg", + "video_11204/video_11204_3.jpeg", + "video_11204/video_11204_4.jpeg" + ], + "extracted": "C", + "result": 1 + }, + { + "sample_id": 47, + "question": "Investigate the images and reply to the corresponding inquiry about counterfactual inference. You must choose your answer from the Choice List.\nIf the cyan object is removed, which event will happen?\nChoice list: \nA. The metal cylinder collides with the rubber sphere\nB. The metal cylinder collides with the red object\nC. The blue cylinder collides with the metal sphere\nD. The blue cylinder and the blue sphere collide\nAnswer with the option's letter from the given choices directly.", + "gt_response": "the blue cylinder and the blue sphere collide", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "d", + "choice_list": [ + "the metal cylinder collides with the rubber sphere", + "the metal cylinder collides with the red object", + "the blue cylinder collides with the metal sphere", + "the blue cylinder and the blue sphere collide" + ], + "image_quantity_level": "Few", + "image": [ + "video_10209/video_10209_0.jpeg", + "video_10209/video_10209_1.jpeg", + "video_10209/video_10209_2.jpeg", + "video_10209/video_10209_3.jpeg", + "video_10209/video_10209_4.jpeg" + ], + "extracted": "D", + "result": 1 + }, + { + "sample_id": 48, + "question": "Examine the images and answer the relevant query about counterfactual inference. You must choose your answer from the Choice List.\nWhich of the following will happen if the cylinder is removed?\nChoice list: \nA. The metal cube and the sphere collide\nB. The rubber cube collides with the sphere\nAnswer with the option's letter from the given choices directly.", + "gt_response": "the rubber cube collides with the sphere", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "a", + "choice_list": [ + "the metal cube and the sphere collide", + "the rubber cube collides with the sphere" + ], + "image_quantity_level": "Few", + "image": [ + "video_11241/video_11241_0.jpeg", + "video_11241/video_11241_1.jpeg", + "video_11241/video_11241_2.jpeg", + "video_11241/video_11241_3.jpeg", + "video_11241/video_11241_4.jpeg" + ], + "extracted": "A", + "result": 0 + }, + { + "sample_id": 49, + "question": "Assess the pictures and answer the connected query about counterfactual inference. You must choose your answer from the Choice List.\nIf the sphere is removed, which of the following will happen?\nChoice list: \nA. The gray cylinder and the blue cylinder collide\nB. The gray cylinder and the cyan object collide\nC. The cyan cylinder collides with the blue cylinder\nAnswer with the option's letter from the given choices directly.", + "gt_response": "the gray cylinder and the cyan object collide", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "b", + "choice_list": [ + "the gray cylinder and the blue cylinder collide", + "the gray cylinder and the cyan object collide", + "the cyan cylinder collides with the blue cylinder" + ], + "image_quantity_level": "Few", + "image": [ + "video_12283/video_12283_0.jpeg", + "video_12283/video_12283_1.jpeg", + "video_12283/video_12283_2.jpeg", + "video_12283/video_12283_3.jpeg", + "video_12283/video_12283_4.jpeg" + ], + "extracted": "B", + "result": 1 + }, + { + "sample_id": 50, + "question": "Analyze the given pictures and answer the related inquiry about counterfactual inference. You must choose your answer from the Choice List.\nWhat will happen without the red object?\nChoice list: \nA. The rubber sphere and the metal cube collide\nB. The gray sphere and the purple object collide\nC. The metal sphere collides with the metal cube\nD. The purple object collides with the cylinder\nAnswer with the option's letter from the given choices directly.", + "gt_response": "the gray sphere and the purple object collide", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "b", + "choice_list": [ + "the rubber sphere and the metal cube collide", + "the gray sphere and the purple object collide", + "the metal sphere collides with the metal cube", + "the purple object collides with the cylinder" + ], + "image_quantity_level": "Few", + "image": [ + "video_13734/video_13734_0.jpeg", + "video_13734/video_13734_1.jpeg", + "video_13734/video_13734_2.jpeg", + "video_13734/video_13734_3.jpeg", + "video_13734/video_13734_4.jpeg" + ], + "extracted": "B", + "result": 1 + }, + { + "sample_id": 51, + "question": "Based on the provided images, answer the question related to the counterfactual inference. You must choose your answer from the Choice List.\nIf the cylinder is removed, what will happen?\nChoice list: \nA. The yellow object and the gray sphere collide\nB. The gray cube collides with the yellow sphere\nC. The gray cube collides with the gray sphere\nD. The red object collides with the gray cube\nAnswer with the option's letter from the given choices directly.", + "gt_response": "the red object collides with the gray cube", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "d", + "choice_list": [ + "the yellow object and the gray sphere collide", + "the gray cube collides with the yellow sphere", + "the gray cube collides with the gray sphere", + "the red object collides with the gray cube" + ], + "image_quantity_level": "Few", + "image": [ + "video_12048/video_12048_0.jpeg", + "video_12048/video_12048_1.jpeg", + "video_12048/video_12048_2.jpeg", + "video_12048/video_12048_3.jpeg", + "video_12048/video_12048_4.jpeg" + ], + "extracted": "D", + "result": 1 + }, + { + "sample_id": 52, + "question": "Examine the images and answer the relevant query about counterfactual inference. You must choose your answer from the Choice List.\nWithout the cylinder, which event will happen?\nChoice list: \nA. The gray cube and the blue object collide\nB. The rubber cube and the cyan object collide\nC. The sphere and the blue cube collide\nD. The sphere and the cyan object collide\nAnswer with the option's letter from the given choices directly.", + "gt_response": "the gray cube and the blue object collide", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "a", + "choice_list": [ + "the gray cube and the blue object collide", + "the rubber cube and the cyan object collide", + "the sphere and the blue cube collide", + "the sphere and the cyan object collide" + ], + "image_quantity_level": "Few", + "image": [ + "video_11941/video_11941_0.jpeg", + "video_11941/video_11941_1.jpeg", + "video_11941/video_11941_2.jpeg", + "video_11941/video_11941_3.jpeg", + "video_11941/video_11941_4.jpeg" + ], + "extracted": "A", + "result": 1 + }, + { + "sample_id": 53, + "question": "Observe the images and reply to the pertinent question about counterfactual inference. You must choose your answer from the Choice List.\nWhat will happen if the red object is removed?\nChoice list: \nA. The purple cylinder and the gray object collide\nB. The yellow object collides with the gray cylinder\nC. The yellow cylinder and the purple metal cube collide\nD. The purple metal cube and the gray object collide\nAnswer with the option's letter from the given choices directly.", + "gt_response": "the yellow cylinder and the purple metal cube collide", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "c", + "choice_list": [ + "the purple cylinder and the gray object collide", + "the yellow object collides with the gray cylinder", + "the yellow cylinder and the purple metal cube collide", + "the purple metal cube and the gray object collide" + ], + "image_quantity_level": "Few", + "image": [ + "video_10955/video_10955_0.jpeg", + "video_10955/video_10955_1.jpeg", + "video_10955/video_10955_2.jpeg", + "video_10955/video_10955_3.jpeg", + "video_10955/video_10955_4.jpeg" + ], + "extracted": "C", + "result": 1 + }, + { + "sample_id": 54, + "question": "Assess the pictures and answer the connected query about counterfactual inference. You must choose your answer from the Choice List.\nWhat will happen without the cube?\nChoice list: \nA. The green object and the gray object collide\nB. The metal cylinder and the green object collide\nC. The cyan cylinder collides with the gray sphere\nD. The purple sphere collides with the green object\nAnswer with the option's letter from the given choices directly.", + "gt_response": "the purple sphere collides with the green object", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "b", + "choice_list": [ + "the green object and the gray object collide", + "the metal cylinder and the green object collide", + "the cyan cylinder collides with the gray sphere", + "the purple sphere collides with the green object" + ], + "image_quantity_level": "Few", + "image": [ + "video_13915/video_13915_0.jpeg", + "video_13915/video_13915_1.jpeg", + "video_13915/video_13915_2.jpeg", + "video_13915/video_13915_3.jpeg", + "video_13915/video_13915_4.jpeg" + ], + "extracted": "B", + "result": 0 + }, + { + "sample_id": 55, + "question": "Evaluate the images presented and provide an answer to the associated question about counterfactual inference. You must choose your answer from the Choice List.\nWhich of the following will happen without the red object?\nChoice list: \nA. The green object and the blue cylinder collide\nB. The blue cylinder and the gray cylinder collide\nC. The blue cylinder collides with the cube\nD. The blue cylinder collides with the purple cylinder\nAnswer with the option's letter from the given choices directly.", + "gt_response": "the green object and the blue cylinder collide", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "a", + "choice_list": [ + "the green object and the blue cylinder collide", + "the blue cylinder and the gray cylinder collide", + "the blue cylinder collides with the cube", + "the blue cylinder collides with the purple cylinder" + ], + "image_quantity_level": "Few", + "image": [ + "video_11242/video_11242_0.jpeg", + "video_11242/video_11242_1.jpeg", + "video_11242/video_11242_2.jpeg", + "video_11242/video_11242_3.jpeg", + "video_11242/video_11242_4.jpeg" + ], + "extracted": "A", + "result": 1 + }, + { + "sample_id": 56, + "question": "Assess the pictures and answer the connected query about counterfactual inference. You must choose your answer from the Choice List.\nWhich of the following will happen without the cube?\nChoice list: \nA. The green cylinder and the gray sphere collide\nB. The green object collides with the metal cylinder\nC. The gray sphere and the metal cylinder collide\nD. The purple sphere collides with the gray sphere\nAnswer with the option's letter from the given choices directly.", + "gt_response": "the green cylinder and the gray sphere collide", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "d", + "choice_list": [ + "the green cylinder and the gray sphere collide", + "the green object collides with the metal cylinder", + "the gray sphere and the metal cylinder collide", + "the purple sphere collides with the gray sphere" + ], + "image_quantity_level": "Few", + "image": [ + "video_12292/video_12292_0.jpeg", + "video_12292/video_12292_1.jpeg", + "video_12292/video_12292_2.jpeg", + "video_12292/video_12292_3.jpeg", + "video_12292/video_12292_4.jpeg" + ], + "extracted": "D", + "result": 0 + }, + { + "sample_id": 57, + "question": "Scrutinize the visuals and respond to the related question about counterfactual inference. You must choose your answer from the Choice List.\nIf the cylinder is removed, which of the following will happen?\nChoice list: \nA. The sphere collides with the cyan object\nB. The yellow object and the sphere collide\nAnswer with the option's letter from the given choices directly.", + "gt_response": "the yellow object and the sphere collide", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "b", + "choice_list": [ + "the sphere collides with the cyan object", + "the yellow object and the sphere collide" + ], + "image_quantity_level": "Few", + "image": [ + "video_11939/video_11939_0.jpeg", + "video_11939/video_11939_1.jpeg", + "video_11939/video_11939_2.jpeg", + "video_11939/video_11939_3.jpeg", + "video_11939/video_11939_4.jpeg" + ], + "extracted": "B", + "result": 1 + }, + { + "sample_id": 58, + "question": "Assess the pictures and answer the connected query about counterfactual inference. You must choose your answer from the Choice List.\nWhich of the following will happen if the purple object is removed?\nChoice list: \nA. The red sphere and the gray sphere collide\nB. The gray object and the cube collide\nC. The red sphere and the cube collide\nAnswer with the option's letter from the given choices directly.", + "gt_response": "the gray object and the cube collide", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "a", + "choice_list": [ + "the red sphere and the gray sphere collide", + "the gray object and the cube collide", + "the red sphere and the cube collide" + ], + "image_quantity_level": "Few", + "image": [ + "video_11758/video_11758_0.jpeg", + "video_11758/video_11758_1.jpeg", + "video_11758/video_11758_2.jpeg", + "video_11758/video_11758_3.jpeg", + "video_11758/video_11758_4.jpeg" + ], + "extracted": "A", + "result": 0 + }, + { + "sample_id": 59, + "question": "Analyze the given pictures and answer the related inquiry about counterfactual inference. You must choose your answer from the Choice List.\nWhich of the following will happen if the sphere is removed?\nChoice list: \nA. The blue cylinder collides with the gray object\nB. The yellow object and the gray cylinder collide\nC. The yellow object and the blue cylinder collide\nAnswer with the option's letter from the given choices directly.", + "gt_response": "the yellow object and the blue cylinder collide", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "c", + "choice_list": [ + "the blue cylinder collides with the gray object", + "the yellow object and the gray cylinder collide", + "the yellow object and the blue cylinder collide" + ], + "image_quantity_level": "Few", + "image": [ + "video_11115/video_11115_0.jpeg", + "video_11115/video_11115_1.jpeg", + "video_11115/video_11115_2.jpeg", + "video_11115/video_11115_3.jpeg", + "video_11115/video_11115_4.jpeg" + ], + "extracted": "C", + "result": 1 + }, + { + "sample_id": 60, + "question": "Based on the provided images, answer the question related to the counterfactual inference. You must choose your answer from the Choice List.\nWhich of the following will happen if the cylinder is removed?\nChoice list: \nA. The brown cube and the purple object collide\nB. The brown cube and the sphere collide\nAnswer with the option's letter from the given choices directly.", + "gt_response": "the brown cube and the sphere collide", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "b", + "choice_list": [ + "the brown cube and the purple object collide", + "the brown cube and the sphere collide" + ], + "image_quantity_level": "Few", + "image": [ + "video_10116/video_10116_0.jpeg", + "video_10116/video_10116_1.jpeg", + "video_10116/video_10116_2.jpeg", + "video_10116/video_10116_3.jpeg", + "video_10116/video_10116_4.jpeg" + ], + "extracted": "B", + "result": 1 + }, + { + "sample_id": 61, + "question": "Based on the provided images, answer the question related to the counterfactual inference. You must choose your answer from the Choice List.\nIf the blue object is removed, which of the following will happen?\nChoice list: \nA. The cylinder and the purple cube collide\nB. The sphere and the purple cube collide\nC. The sphere collides with the cylinder\nD. The yellow cube and the purple cube collide\nAnswer with the option's letter from the given choices directly.", + "gt_response": "the sphere collides with the cylinder", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "c", + "choice_list": [ + "the cylinder and the purple cube collide", + "the sphere and the purple cube collide", + "the sphere collides with the cylinder", + "the yellow cube and the purple cube collide" + ], + "image_quantity_level": "Few", + "image": [ + "video_13686/video_13686_0.jpeg", + "video_13686/video_13686_1.jpeg", + "video_13686/video_13686_2.jpeg", + "video_13686/video_13686_3.jpeg", + "video_13686/video_13686_4.jpeg" + ], + "extracted": "C", + "result": 1 + }, + { + "sample_id": 62, + "question": "Evaluate the images presented and provide an answer to the associated question about counterfactual inference. You must choose your answer from the Choice List.\nIf the rubber object is removed, what will happen?\nChoice list: \nA. The cyan object and the blue cylinder collide\nB. The gray sphere collides with the blue cylinder\nAnswer with the option's letter from the given choices directly.", + "gt_response": "the gray sphere collides with the blue cylinder", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "b", + "choice_list": [ + "the cyan object and the blue cylinder collide", + "the gray sphere collides with the blue cylinder" + ], + "image_quantity_level": "Few", + "image": [ + "video_10072/video_10072_0.jpeg", + "video_10072/video_10072_1.jpeg", + "video_10072/video_10072_2.jpeg", + "video_10072/video_10072_3.jpeg", + "video_10072/video_10072_4.jpeg" + ], + "extracted": "B", + "result": 1 + }, + { + "sample_id": 63, + "question": "Examine the images and answer the relevant query about counterfactual inference. You must choose your answer from the Choice List.\nWhich of the following will happen if the cube is removed?\nChoice list: \nA. The cylinder and the brown sphere collide\nB. The brown sphere and the gray sphere collide\nC. The cylinder and the gray sphere collide\nAnswer with the option's letter from the given choices directly.", + "gt_response": "the brown sphere and the gray sphere collide", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "a", + "choice_list": [ + "the cylinder and the brown sphere collide", + "the brown sphere and the gray sphere collide", + "the cylinder and the gray sphere collide" + ], + "image_quantity_level": "Few", + "image": [ + "video_12397/video_12397_0.jpeg", + "video_12397/video_12397_1.jpeg", + "video_12397/video_12397_2.jpeg", + "video_12397/video_12397_3.jpeg", + "video_12397/video_12397_4.jpeg" + ], + "extracted": "A", + "result": 0 + }, + { + "sample_id": 64, + "question": "Analyze the given pictures and answer the related inquiry about counterfactual inference. You must choose your answer from the Choice List.\nIf the cube is removed, what will happen?\nChoice list: \nA. The green sphere and the cylinder collide\nB. The green sphere collides with the red rubber object\nC. The yellow sphere collides with the green sphere\nD. The red metal object collides with the red rubber sphere\nAnswer with the option's letter from the given choices directly.", + "gt_response": "the yellow sphere collides with the green sphere", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "c", + "choice_list": [ + "the green sphere and the cylinder collide", + "the green sphere collides with the red rubber object", + "the yellow sphere collides with the green sphere", + "the red metal object collides with the red rubber sphere" + ], + "image_quantity_level": "Few", + "image": [ + "video_14026/video_14026_0.jpeg", + "video_14026/video_14026_1.jpeg", + "video_14026/video_14026_2.jpeg", + "video_14026/video_14026_3.jpeg", + "video_14026/video_14026_4.jpeg" + ], + "extracted": "C", + "result": 1 + }, + { + "sample_id": 65, + "question": "Examine the images and answer the relevant query about counterfactual inference. You must choose your answer from the Choice List.\nWhich of the following will happen without the gray object?\nChoice list: \nA. The yellow object and the purple object collide\nB. The purple cube collides with the cyan cube\nC. The yellow cylinder collides with the cyan cube\nD. The cyan cylinder collides with the cyan cube\nAnswer with the option's letter from the given choices directly.", + "gt_response": "the yellow object and the purple object collide", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "a", + "choice_list": [ + "the yellow object and the purple object collide", + "the purple cube collides with the cyan cube", + "the yellow cylinder collides with the cyan cube", + "the cyan cylinder collides with the cyan cube" + ], + "image_quantity_level": "Few", + "image": [ + "video_10312/video_10312_0.jpeg", + "video_10312/video_10312_1.jpeg", + "video_10312/video_10312_2.jpeg", + "video_10312/video_10312_3.jpeg", + "video_10312/video_10312_4.jpeg" + ], + "extracted": "A", + "result": 1 + }, + { + "sample_id": 66, + "question": "Analyze the given pictures and answer the related inquiry about counterfactual inference. You must choose your answer from the Choice List.\nWhat will happen without the brown object?\nChoice list: \nA. The cylinder and the purple sphere collide\nB. The cyan object and the blue sphere collide\nC. The cylinder and the green sphere collide\nD. The cyan object and the cylinder collide\nAnswer with the option's letter from the given choices directly.", + "gt_response": "the cyan object and the cylinder collide", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "d", + "choice_list": [ + "the cylinder and the purple sphere collide", + "the cyan object and the blue sphere collide", + "the cylinder and the green sphere collide", + "the cyan object and the cylinder collide" + ], + "image_quantity_level": "Few", + "image": [ + "video_12455/video_12455_0.jpeg", + "video_12455/video_12455_1.jpeg", + "video_12455/video_12455_2.jpeg", + "video_12455/video_12455_3.jpeg", + "video_12455/video_12455_4.jpeg" + ], + "extracted": "D", + "result": 1 + }, + { + "sample_id": 67, + "question": "Based on the provided images, answer the question related to the counterfactual inference. You must choose your answer from the Choice List.\nWhat will happen without the sphere?\nChoice list: \nA. The purple cylinder collides with the cube\nB. The cube and the green object collide\nAnswer with the option's letter from the given choices directly.", + "gt_response": "the purple cylinder collides with the cube", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "a", + "choice_list": [ + "the purple cylinder collides with the cube", + "the cube and the green object collide" + ], + "image_quantity_level": "Few", + "image": [ + "video_11452/video_11452_0.jpeg", + "video_11452/video_11452_1.jpeg", + "video_11452/video_11452_2.jpeg", + "video_11452/video_11452_3.jpeg", + "video_11452/video_11452_4.jpeg" + ], + "extracted": "A", + "result": 1 + }, + { + "sample_id": 68, + "question": "Scrutinize the visuals and respond to the related question about counterfactual inference. You must choose your answer from the Choice List.\nWithout the sphere, which event will happen?\nChoice list: \nA. The cyan cube collides with the brown cylinder\nB. The cyan cylinder and the brown cube collide\nC. The brown cylinder collides with the cyan cylinder\nD. The brown cylinder and the brown cube collide\nAnswer with the option's letter from the given choices directly.", + "gt_response": "the cyan cube collides with the brown cylinder", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "c", + "choice_list": [ + "the cyan cube collides with the brown cylinder", + "the cyan cylinder and the brown cube collide", + "the brown cylinder collides with the cyan cylinder", + "the brown cylinder and the brown cube collide" + ], + "image_quantity_level": "Few", + "image": [ + "video_10030/video_10030_0.jpeg", + "video_10030/video_10030_1.jpeg", + "video_10030/video_10030_2.jpeg", + "video_10030/video_10030_3.jpeg", + "video_10030/video_10030_4.jpeg" + ], + "extracted": "C", + "result": 0 + }, + { + "sample_id": 69, + "question": "Investigate the images and reply to the corresponding inquiry about counterfactual inference. You must choose your answer from the Choice List.\nWithout the blue object, what will happen?\nChoice list: \nA. The green rubber object collides with the gray sphere\nB. The metal cube collides with the green rubber object\nC. The cyan object collides with the metal sphere\nD. The metal cube collides with the cylinder\nAnswer with the option's letter from the given choices directly.", + "gt_response": "the metal cube collides with the cylinder", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "b", + "choice_list": [ + "the green rubber object collides with the gray sphere", + "the metal cube collides with the green rubber object", + "the cyan object collides with the metal sphere", + "the metal cube collides with the cylinder" + ], + "image_quantity_level": "Few", + "image": [ + "video_14984/video_14984_0.jpeg", + "video_14984/video_14984_1.jpeg", + "video_14984/video_14984_2.jpeg", + "video_14984/video_14984_3.jpeg", + "video_14984/video_14984_4.jpeg" + ], + "extracted": "B", + "result": 0 + }, + { + "sample_id": 70, + "question": "Inspect the given graphics and answer the relevant query about counterfactual inference. You must choose your answer from the Choice List.\nWhich of the following will happen if the red object is removed?\nChoice list: \nA. The cylinder collides with the rubber sphere\nB. The cylinder collides with the gray cube\nC. The green object and the rubber sphere collide\nD. The metal sphere collides with the gray object\nAnswer with the option's letter from the given choices directly.", + "gt_response": "the green object and the rubber sphere collide", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "c", + "choice_list": [ + "the cylinder collides with the rubber sphere", + "the cylinder collides with the gray cube", + "the green object and the rubber sphere collide", + "the metal sphere collides with the gray object" + ], + "image_quantity_level": "Few", + "image": [ + "video_14408/video_14408_0.jpeg", + "video_14408/video_14408_1.jpeg", + "video_14408/video_14408_2.jpeg", + "video_14408/video_14408_3.jpeg", + "video_14408/video_14408_4.jpeg" + ], + "extracted": "C", + "result": 1 + }, + { + "sample_id": 71, + "question": "Inspect the given graphics and answer the relevant query about counterfactual inference. You must choose your answer from the Choice List.\nWithout the cylinder, what will happen?\nChoice list: \nA. The cube collides with the cyan object\nB. The yellow sphere and the green sphere collide\nC. The cube and the yellow sphere collide\nD. The green sphere collides with the cyan sphere\nAnswer with the option's letter from the given choices directly.", + "gt_response": "the cube and the yellow sphere collide", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "c", + "choice_list": [ + "the cube collides with the cyan object", + "the yellow sphere and the green sphere collide", + "the cube and the yellow sphere collide", + "the green sphere collides with the cyan sphere" + ], + "image_quantity_level": "Few", + "image": [ + "video_14623/video_14623_0.jpeg", + "video_14623/video_14623_1.jpeg", + "video_14623/video_14623_2.jpeg", + "video_14623/video_14623_3.jpeg", + "video_14623/video_14623_4.jpeg" + ], + "extracted": "C", + "result": 1 + }, + { + "sample_id": 72, + "question": "Inspect the given graphics and answer the relevant query about counterfactual inference. You must choose your answer from the Choice List.\nWithout the purple object, which of the following will happen?\nChoice list: \nA. The metal cylinder collides with the metal sphere\nB. The rubber cylinder collides with the brown object\nAnswer with the option's letter from the given choices directly.", + "gt_response": "the metal cylinder collides with the metal sphere", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "a", + "choice_list": [ + "the metal cylinder collides with the metal sphere", + "the rubber cylinder collides with the brown object" + ], + "image_quantity_level": "Few", + "image": [ + "video_14172/video_14172_0.jpeg", + "video_14172/video_14172_1.jpeg", + "video_14172/video_14172_2.jpeg", + "video_14172/video_14172_3.jpeg", + "video_14172/video_14172_4.jpeg" + ], + "extracted": "A", + "result": 1 + }, + { + "sample_id": 73, + "question": "Analyze the given pictures and answer the related inquiry about counterfactual inference. You must choose your answer from the Choice List.\nWhich event will happen without the green object?\nChoice list: \nA. The cyan object and the cylinder collide\nB. The yellow object and the cylinder collide\nAnswer with the option's letter from the given choices directly.", + "gt_response": "the cyan object and the cylinder collide", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "a", + "choice_list": [ + "the cyan object and the cylinder collide", + "the yellow object and the cylinder collide" + ], + "image_quantity_level": "Few", + "image": [ + "video_11845/video_11845_0.jpeg", + "video_11845/video_11845_1.jpeg", + "video_11845/video_11845_2.jpeg", + "video_11845/video_11845_3.jpeg", + "video_11845/video_11845_4.jpeg" + ], + "extracted": "A", + "result": 1 + }, + { + "sample_id": 74, + "question": "Observe the images and reply to the pertinent question about counterfactual inference. You must choose your answer from the Choice List.\nWhich event will happen without the gray object?\nChoice list: \nA. The rubber sphere and the cube collide\nB. The brown sphere and the cube collide\nC. The blue sphere collides with the cyan sphere\nD. The cyan sphere and the yellow cylinder collide\nAnswer with the option's letter from the given choices directly.", + "gt_response": "the cyan sphere and the yellow cylinder collide", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "c", + "choice_list": [ + "the rubber sphere and the cube collide", + "the brown sphere and the cube collide", + "the blue sphere collides with the cyan sphere", + "the cyan sphere and the yellow cylinder collide" + ], + "image_quantity_level": "Few", + "image": [ + "video_10703/video_10703_0.jpeg", + "video_10703/video_10703_1.jpeg", + "video_10703/video_10703_2.jpeg", + "video_10703/video_10703_3.jpeg", + "video_10703/video_10703_4.jpeg" + ], + "extracted": "C", + "result": 0 + }, + { + "sample_id": 75, + "question": "Assess the pictures and answer the connected query about counterfactual inference. You must choose your answer from the Choice List.\nIf the cylinder is removed, what will happen?\nChoice list: \nA. The rubber object and the sphere collide\nB. The red object collides with the sphere\nC. The red cube and the yellow object collide\nAnswer with the option's letter from the given choices directly.", + "gt_response": "the red cube and the yellow object collide", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "c", + "choice_list": [ + "the rubber object and the sphere collide", + "the red object collides with the sphere", + "the red cube and the yellow object collide" + ], + "image_quantity_level": "Few", + "image": [ + "video_10120/video_10120_0.jpeg", + "video_10120/video_10120_1.jpeg", + "video_10120/video_10120_2.jpeg", + "video_10120/video_10120_3.jpeg", + "video_10120/video_10120_4.jpeg" + ], + "extracted": "C", + "result": 1 + }, + { + "sample_id": 76, + "question": "Inspect the given graphics and answer the relevant query about counterfactual inference. You must choose your answer from the Choice List.\nWhat will happen without the sphere?\nChoice list: \nA. The cube and the brown cylinder collide\nB. The green cylinder and the brown object collide\nC. The metal object collides with the green cylinder\nD. The metal cylinder and the cube collide\nAnswer with the option's letter from the given choices directly.", + "gt_response": "the metal object collides with the green cylinder", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "c", + "choice_list": [ + "the cube and the brown cylinder collide", + "the green cylinder and the brown object collide", + "the metal object collides with the green cylinder", + "the metal cylinder and the cube collide" + ], + "image_quantity_level": "Few", + "image": [ + "video_13307/video_13307_0.jpeg", + "video_13307/video_13307_1.jpeg", + "video_13307/video_13307_2.jpeg", + "video_13307/video_13307_3.jpeg", + "video_13307/video_13307_4.jpeg" + ], + "extracted": "C", + "result": 1 + }, + { + "sample_id": 77, + "question": "Review the photos and answer the associated inquiry about counterfactual inference. You must choose your answer from the Choice List.\nWithout the cylinder, which of the following will happen?\nChoice list: \nA. The cube and the red object collide\nB. The gray sphere and the red object collide\nC. The green sphere and the gray object collide\nD. The cube and the green sphere collide\nAnswer with the option's letter from the given choices directly.", + "gt_response": "the green sphere and the gray object collide", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "d", + "choice_list": [ + "the cube and the red object collide", + "the gray sphere and the red object collide", + "the green sphere and the gray object collide", + "the cube and the green sphere collide" + ], + "image_quantity_level": "Few", + "image": [ + "video_12821/video_12821_0.jpeg", + "video_12821/video_12821_1.jpeg", + "video_12821/video_12821_2.jpeg", + "video_12821/video_12821_3.jpeg", + "video_12821/video_12821_4.jpeg" + ], + "extracted": "D", + "result": 0 + }, + { + "sample_id": 78, + "question": "Evaluate the images presented and provide an answer to the associated question about counterfactual inference. You must choose your answer from the Choice List.\nWhich of the following will happen if the sphere is removed?\nChoice list: \nA. The cube and the red object collide\nB. The gray object and the yellow object collide\nC. The cube and the yellow object collide\nD. The cube collides with the gray cylinder\nAnswer with the option's letter from the given choices directly.", + "gt_response": "the cube and the yellow object collide", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "d", + "choice_list": [ + "the cube and the red object collide", + "the gray object and the yellow object collide", + "the cube and the yellow object collide", + "the cube collides with the gray cylinder" + ], + "image_quantity_level": "Few", + "image": [ + "video_13196/video_13196_0.jpeg", + "video_13196/video_13196_1.jpeg", + "video_13196/video_13196_2.jpeg", + "video_13196/video_13196_3.jpeg", + "video_13196/video_13196_4.jpeg" + ], + "extracted": "D", + "result": 0 + }, + { + "sample_id": 79, + "question": "Assess the pictures and answer the connected query about counterfactual inference. You must choose your answer from the Choice List.\nWhich event will happen without the cylinder?\nChoice list: \nA. The blue sphere and the cyan cube collide\nB. The cyan sphere and the purple object collide\nC. The cyan sphere and the cyan cube collide\nD. The cyan sphere and the blue sphere collide\nAnswer with the option's letter from the given choices directly.", + "gt_response": "the cyan sphere and the purple object collide", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "b", + "choice_list": [ + "the blue sphere and the cyan cube collide", + "the cyan sphere and the purple object collide", + "the cyan sphere and the cyan cube collide", + "the cyan sphere and the blue sphere collide" + ], + "image_quantity_level": "Few", + "image": [ + "video_10107/video_10107_0.jpeg", + "video_10107/video_10107_1.jpeg", + "video_10107/video_10107_2.jpeg", + "video_10107/video_10107_3.jpeg", + "video_10107/video_10107_4.jpeg" + ], + "extracted": "B", + "result": 1 + }, + { + "sample_id": 80, + "question": "Based on the provided images, answer the question related to the counterfactual inference. You must choose your answer from the Choice List.\nIf the gray object is removed, which event will happen?\nChoice list: \nA. The blue object collides with the red cylinder\nB. The blue object collides with the yellow object\nC. The blue object collides with the green sphere\nD. The yellow object and the metal sphere collide\nAnswer with the option's letter from the given choices directly.", + "gt_response": "the blue object collides with the red cylinder", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "c", + "choice_list": [ + "the blue object collides with the red cylinder", + "the blue object collides with the yellow object", + "the blue object collides with the green sphere", + "the yellow object and the metal sphere collide" + ], + "image_quantity_level": "Few", + "image": [ + "video_14184/video_14184_0.jpeg", + "video_14184/video_14184_1.jpeg", + "video_14184/video_14184_2.jpeg", + "video_14184/video_14184_3.jpeg", + "video_14184/video_14184_4.jpeg" + ], + "extracted": "C", + "result": 0 + }, + { + "sample_id": 81, + "question": "Review the photos and answer the associated inquiry about counterfactual inference. You must choose your answer from the Choice List.\nWithout the purple object, which event will happen?\nChoice list: \nA. The cube collides with the gray cylinder\nAnswer with the option's letter from the given choices directly.", + "gt_response": "the cube collides with the gray cylinder", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "a", + "choice_list": [ + "the cube collides with the gray cylinder" + ], + "image_quantity_level": "Few", + "image": [ + "video_10978/video_10978_0.jpeg", + "video_10978/video_10978_1.jpeg", + "video_10978/video_10978_2.jpeg", + "video_10978/video_10978_3.jpeg", + "video_10978/video_10978_4.jpeg" + ], + "extracted": "A", + "result": 1 + }, + { + "sample_id": 82, + "question": "Evaluate the images presented and provide an answer to the associated question about counterfactual inference. You must choose your answer from the Choice List.\nWhat will happen if the cube is removed?\nChoice list: \nA. The rubber cylinder and the brown cylinder collide\nB. The brown sphere and the blue cylinder collide\nC. The gray sphere and the rubber cylinder collide\nD. The rubber sphere collides with the brown cylinder\nAnswer with the option's letter from the given choices directly.", + "gt_response": "the brown sphere and the blue cylinder collide", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "c", + "choice_list": [ + "the rubber cylinder and the brown cylinder collide", + "the brown sphere and the blue cylinder collide", + "the gray sphere and the rubber cylinder collide", + "the rubber sphere collides with the brown cylinder" + ], + "image_quantity_level": "Few", + "image": [ + "video_12621/video_12621_0.jpeg", + "video_12621/video_12621_1.jpeg", + "video_12621/video_12621_2.jpeg", + "video_12621/video_12621_3.jpeg", + "video_12621/video_12621_4.jpeg" + ], + "extracted": "C", + "result": 0 + }, + { + "sample_id": 83, + "question": "Examine the images and answer the relevant query about counterfactual inference. You must choose your answer from the Choice List.\nWhich event will happen if the brown object is removed?\nChoice list: \nA. The gray cube and the sphere collide\nB. The green object collides with the gray cube\nC. The gray cube and the red cube collide\nD. The sphere and the red cube collide\nAnswer with the option's letter from the given choices directly.", + "gt_response": "the green object collides with the gray cube", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "a", + "choice_list": [ + "the gray cube and the sphere collide", + "the green object collides with the gray cube", + "the gray cube and the red cube collide", + "the sphere and the red cube collide" + ], + "image_quantity_level": "Few", + "image": [ + "video_13716/video_13716_0.jpeg", + "video_13716/video_13716_1.jpeg", + "video_13716/video_13716_2.jpeg", + "video_13716/video_13716_3.jpeg", + "video_13716/video_13716_4.jpeg" + ], + "extracted": "A", + "result": 0 + }, + { + "sample_id": 84, + "question": "Evaluate the images presented and provide an answer to the associated question about counterfactual inference. You must choose your answer from the Choice List.\nWithout the cube, which of the following will happen?\nChoice list: \nA. The sphere and the yellow cylinder collide\nB. The gray object and the yellow object collide\nAnswer with the option's letter from the given choices directly.", + "gt_response": "the sphere and the yellow cylinder collide", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "a", + "choice_list": [ + "the sphere and the yellow cylinder collide", + "the gray object and the yellow object collide" + ], + "image_quantity_level": "Few", + "image": [ + "video_13984/video_13984_0.jpeg", + "video_13984/video_13984_1.jpeg", + "video_13984/video_13984_2.jpeg", + "video_13984/video_13984_3.jpeg", + "video_13984/video_13984_4.jpeg" + ], + "extracted": "A", + "result": 1 + }, + { + "sample_id": 85, + "question": "Evaluate the images presented and provide an answer to the associated question about counterfactual inference. You must choose your answer from the Choice List.\nWhich of the following will happen if the cube is removed?\nChoice list: \nA. The cyan cylinder collides with the brown cylinder\nB. The cyan cylinder collides with the purple cylinder\nAnswer with the option's letter from the given choices directly.", + "gt_response": "the cyan cylinder collides with the brown cylinder", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "a", + "choice_list": [ + "the cyan cylinder collides with the brown cylinder", + "the cyan cylinder collides with the purple cylinder" + ], + "image_quantity_level": "Few", + "image": [ + "video_14332/video_14332_0.jpeg", + "video_14332/video_14332_1.jpeg", + "video_14332/video_14332_2.jpeg", + "video_14332/video_14332_3.jpeg", + "video_14332/video_14332_4.jpeg" + ], + "extracted": "A", + "result": 1 + }, + { + "sample_id": 86, + "question": "Examine the images and answer the relevant query about counterfactual inference. You must choose your answer from the Choice List.\nWhich of the following will happen if the cyan object is removed?\nChoice list: \nA. The blue cube and the purple object collide\nB. The red sphere and the purple object collide\nC. The blue cube collides with the cylinder\nD. The cylinder collides with the purple cube\nAnswer with the option's letter from the given choices directly.", + "gt_response": "the blue cube and the purple object collide", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "c", + "choice_list": [ + "the blue cube and the purple object collide", + "the red sphere and the purple object collide", + "the blue cube collides with the cylinder", + "the cylinder collides with the purple cube" + ], + "image_quantity_level": "Few", + "image": [ + "video_10213/video_10213_0.jpeg", + "video_10213/video_10213_1.jpeg", + "video_10213/video_10213_2.jpeg", + "video_10213/video_10213_3.jpeg", + "video_10213/video_10213_4.jpeg" + ], + "extracted": "C", + "result": 0 + }, + { + "sample_id": 87, + "question": "Analyze the given pictures and answer the related inquiry about counterfactual inference. You must choose your answer from the Choice List.\nWithout the green object, which of the following will happen?\nChoice list: \nA. The yellow object collides with the red sphere\nB. The red object collides with the purple object\nC. The yellow object and the purple object collide\nAnswer with the option's letter from the given choices directly.", + "gt_response": "the yellow object and the purple object collide", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "c", + "choice_list": [ + "the yellow object collides with the red sphere", + "the red object collides with the purple object", + "the yellow object and the purple object collide" + ], + "image_quantity_level": "Few", + "image": [ + "video_11290/video_11290_0.jpeg", + "video_11290/video_11290_1.jpeg", + "video_11290/video_11290_2.jpeg", + "video_11290/video_11290_3.jpeg", + "video_11290/video_11290_4.jpeg" + ], + "extracted": "C", + "result": 1 + }, + { + "sample_id": 88, + "question": "Examine the images and answer the relevant query about counterfactual inference. You must choose your answer from the Choice List.\nWhich of the following will happen without the cube?\nChoice list: \nA. The purple sphere and the cylinder collide\nB. The yellow object collides with the purple sphere\nC. The yellow sphere and the cylinder collide\nD. The purple object collides with the blue sphere\nAnswer with the option's letter from the given choices directly.", + "gt_response": "the yellow object collides with the purple sphere", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "b", + "choice_list": [ + "the purple sphere and the cylinder collide", + "the yellow object collides with the purple sphere", + "the yellow sphere and the cylinder collide", + "the purple object collides with the blue sphere" + ], + "image_quantity_level": "Few", + "image": [ + "video_14232/video_14232_0.jpeg", + "video_14232/video_14232_1.jpeg", + "video_14232/video_14232_2.jpeg", + "video_14232/video_14232_3.jpeg", + "video_14232/video_14232_4.jpeg" + ], + "extracted": "B", + "result": 1 + }, + { + "sample_id": 89, + "question": "Scrutinize the visuals and respond to the related question about counterfactual inference. You must choose your answer from the Choice List.\nWhat will happen if the green object is removed?\nChoice list: \nA. The cube and the gray sphere collide\nB. The cube collides with the yellow sphere\nAnswer with the option's letter from the given choices directly.", + "gt_response": "the cube and the gray sphere collide", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "b", + "choice_list": [ + "the cube and the gray sphere collide", + "the cube collides with the yellow sphere" + ], + "image_quantity_level": "Few", + "image": [ + "video_12956/video_12956_0.jpeg", + "video_12956/video_12956_1.jpeg", + "video_12956/video_12956_2.jpeg", + "video_12956/video_12956_3.jpeg", + "video_12956/video_12956_4.jpeg" + ], + "extracted": "B", + "result": 0 + }, + { + "sample_id": 90, + "question": "Inspect the given graphics and answer the relevant query about counterfactual inference. You must choose your answer from the Choice List.\nWhich event will happen without the cyan object?\nChoice list: \nA. The yellow object and the rubber cube collide\nB. The metal cube collides with the gray object\nC. The yellow cylinder collides with the gray cylinder\nAnswer with the option's letter from the given choices directly.", + "gt_response": "the metal cube collides with the gray object", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "c", + "choice_list": [ + "the yellow object and the rubber cube collide", + "the metal cube collides with the gray object", + "the yellow cylinder collides with the gray cylinder" + ], + "image_quantity_level": "Few", + "image": [ + "video_10492/video_10492_0.jpeg", + "video_10492/video_10492_1.jpeg", + "video_10492/video_10492_2.jpeg", + "video_10492/video_10492_3.jpeg", + "video_10492/video_10492_4.jpeg" + ], + "extracted": "C", + "result": 0 + }, + { + "sample_id": 91, + "question": "Observe the images and reply to the pertinent question about counterfactual inference. You must choose your answer from the Choice List.\nIf the blue object is removed, what will happen?\nChoice list: \nA. The red cube and the metal object collide\nB. The rubber cylinder collides with the red cube\nC. The rubber cylinder and the sphere collide\nD. The red cube and the sphere collide\nAnswer with the option's letter from the given choices directly.", + "gt_response": "the rubber cylinder collides with the red cube", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "b", + "choice_list": [ + "the red cube and the metal object collide", + "the rubber cylinder collides with the red cube", + "the rubber cylinder and the sphere collide", + "the red cube and the sphere collide" + ], + "image_quantity_level": "Few", + "image": [ + "video_10848/video_10848_0.jpeg", + "video_10848/video_10848_1.jpeg", + "video_10848/video_10848_2.jpeg", + "video_10848/video_10848_3.jpeg", + "video_10848/video_10848_4.jpeg" + ], + "extracted": "B", + "result": 1 + }, + { + "sample_id": 92, + "question": "Investigate the images and reply to the corresponding inquiry about counterfactual inference. You must choose your answer from the Choice List.\nWhich event will happen without the sphere?\nChoice list: \nA. The cyan object and the green object collide\nB. The rubber cube and the green cube collide\nAnswer with the option's letter from the given choices directly.", + "gt_response": "the cyan object and the green object collide", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "b", + "choice_list": [ + "the cyan object and the green object collide", + "the rubber cube and the green cube collide" + ], + "image_quantity_level": "Few", + "image": [ + "video_13883/video_13883_0.jpeg", + "video_13883/video_13883_1.jpeg", + "video_13883/video_13883_2.jpeg", + "video_13883/video_13883_3.jpeg", + "video_13883/video_13883_4.jpeg" + ], + "extracted": "B", + "result": 0 + }, + { + "sample_id": 93, + "question": "Evaluate the images presented and provide an answer to the associated question about counterfactual inference. You must choose your answer from the Choice List.\nIf the cylinder is removed, which event will happen?\nChoice list: \nA. The sphere and the red object collide\nB. The cyan object collides with the sphere\nAnswer with the option's letter from the given choices directly.", + "gt_response": "the cyan object collides with the sphere", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "b", + "choice_list": [ + "the sphere and the red object collide", + "the cyan object collides with the sphere" + ], + "image_quantity_level": "Few", + "image": [ + "video_12536/video_12536_0.jpeg", + "video_12536/video_12536_1.jpeg", + "video_12536/video_12536_2.jpeg", + "video_12536/video_12536_3.jpeg", + "video_12536/video_12536_4.jpeg" + ], + "extracted": "B", + "result": 1 + }, + { + "sample_id": 94, + "question": "Review the photos and answer the associated inquiry about counterfactual inference. You must choose your answer from the Choice List.\nWhich of the following will happen if the cylinder is removed?\nChoice list: \nA. The purple cube and the blue cube collide\nB. The gray sphere collides with the red sphere\nC. The blue cube collides with the red object\nD. The cyan sphere collides with the gray sphere\nAnswer with the option's letter from the given choices directly.", + "gt_response": "the cyan sphere collides with the gray sphere", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "d", + "choice_list": [ + "the purple cube and the blue cube collide", + "the gray sphere collides with the red sphere", + "the blue cube collides with the red object", + "the cyan sphere collides with the gray sphere" + ], + "image_quantity_level": "Few", + "image": [ + "video_13496/video_13496_0.jpeg", + "video_13496/video_13496_1.jpeg", + "video_13496/video_13496_2.jpeg", + "video_13496/video_13496_3.jpeg", + "video_13496/video_13496_4.jpeg" + ], + "extracted": "D", + "result": 1 + }, + { + "sample_id": 95, + "question": "Assess the pictures and answer the connected query about counterfactual inference. You must choose your answer from the Choice List.\nWhat will happen if the cylinder is removed?\nChoice list: \nA. The cube and the red sphere collide\nB. The yellow object collides with the red sphere\nC. The cube collides with the yellow object\nD. The purple object and the cube collide\nAnswer with the option's letter from the given choices directly.", + "gt_response": "the purple object and the cube collide", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "c", + "choice_list": [ + "the cube and the red sphere collide", + "the yellow object collides with the red sphere", + "the cube collides with the yellow object", + "the purple object and the cube collide" + ], + "image_quantity_level": "Few", + "image": [ + "video_12477/video_12477_0.jpeg", + "video_12477/video_12477_1.jpeg", + "video_12477/video_12477_2.jpeg", + "video_12477/video_12477_3.jpeg", + "video_12477/video_12477_4.jpeg" + ], + "extracted": "C", + "result": 0 + }, + { + "sample_id": 96, + "question": "Based on the provided images, answer the question related to the counterfactual inference. You must choose your answer from the Choice List.\nWhich event will happen if the cylinder is removed?\nChoice list: \nA. The purple sphere collides with the green object\nB. The purple sphere collides with the blue object\nAnswer with the option's letter from the given choices directly.", + "gt_response": "the purple sphere collides with the blue object", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "b", + "choice_list": [ + "the purple sphere collides with the green object", + "the purple sphere collides with the blue object" + ], + "image_quantity_level": "Few", + "image": [ + "video_10661/video_10661_0.jpeg", + "video_10661/video_10661_1.jpeg", + "video_10661/video_10661_2.jpeg", + "video_10661/video_10661_3.jpeg", + "video_10661/video_10661_4.jpeg" + ], + "extracted": "B", + "result": 1 + }, + { + "sample_id": 97, + "question": "Based on the provided images, answer the question related to the counterfactual inference. You must choose your answer from the Choice List.\nIf the cube is removed, which of the following will happen?\nChoice list: \nA. The yellow object and the green cylinder collide\nB. The yellow cylinder and the sphere collide\nAnswer with the option's letter from the given choices directly.", + "gt_response": "the yellow object and the green cylinder collide", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "a", + "choice_list": [ + "the yellow object and the green cylinder collide", + "the yellow cylinder and the sphere collide" + ], + "image_quantity_level": "Few", + "image": [ + "video_13408/video_13408_0.jpeg", + "video_13408/video_13408_1.jpeg", + "video_13408/video_13408_2.jpeg", + "video_13408/video_13408_3.jpeg", + "video_13408/video_13408_4.jpeg" + ], + "extracted": "A", + "result": 1 + }, + { + "sample_id": 98, + "question": "Evaluate the images presented and provide an answer to the associated question about counterfactual inference. You must choose your answer from the Choice List.\nWhat will happen without the sphere?\nChoice list: \nA. The rubber cylinder collides with the cube\nAnswer with the option's letter from the given choices directly.", + "gt_response": "the rubber cylinder collides with the cube", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "a", + "choice_list": [ + "the rubber cylinder collides with the cube" + ], + "image_quantity_level": "Few", + "image": [ + "video_11854/video_11854_0.jpeg", + "video_11854/video_11854_1.jpeg", + "video_11854/video_11854_2.jpeg", + "video_11854/video_11854_3.jpeg", + "video_11854/video_11854_4.jpeg" + ], + "extracted": "A", + "result": 1 + }, + { + "sample_id": 99, + "question": "Review the photos and answer the associated inquiry about counterfactual inference. You must choose your answer from the Choice List.\nWhat will happen without the blue object?\nChoice list: \nA. The red object and the brown cylinder collide\nB. The brown object and the cyan cube collide\nC. The sphere collides with the cyan cube\nD. The sphere and the brown cylinder collide\nAnswer with the option's letter from the given choices directly.", + "gt_response": "the red object and the brown cylinder collide", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "d", + "choice_list": [ + "the red object and the brown cylinder collide", + "the brown object and the cyan cube collide", + "the sphere collides with the cyan cube", + "the sphere and the brown cylinder collide" + ], + "image_quantity_level": "Few", + "image": [ + "video_12043/video_12043_0.jpeg", + "video_12043/video_12043_1.jpeg", + "video_12043/video_12043_2.jpeg", + "video_12043/video_12043_3.jpeg", + "video_12043/video_12043_4.jpeg" + ], + "extracted": "D", + "result": 0 + }, + { + "sample_id": 100, + "question": "Scrutinize the visuals and respond to the related question about counterfactual inference. You must choose your answer from the Choice List.\nWhich event will happen if the brown object is removed?\nChoice list: \nA. The yellow object collides with the sphere\nB. The red cylinder and the sphere collide\nAnswer with the option's letter from the given choices directly.", + "gt_response": "the red cylinder and the sphere collide", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "b", + "choice_list": [ + "the yellow object collides with the sphere", + "the red cylinder and the sphere collide" + ], + "image_quantity_level": "Few", + "image": [ + "video_11547/video_11547_0.jpeg", + "video_11547/video_11547_1.jpeg", + "video_11547/video_11547_2.jpeg", + "video_11547/video_11547_3.jpeg", + "video_11547/video_11547_4.jpeg" + ], + "extracted": "B", + "result": 1 + }, + { + "sample_id": 101, + "question": "Based on the provided images, answer the question related to the counterfactual inference. You must choose your answer from the Choice List.\nIf the purple object is removed, what will happen?\nChoice list: \nA. The green sphere collides with the cyan object\nB. The green cylinder collides with the green sphere\nC. The yellow sphere and the cyan object collide\nD. The brown object and the yellow sphere collide\nAnswer with the option's letter from the given choices directly.", + "gt_response": "the brown object and the yellow sphere collide", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "d", + "choice_list": [ + "the green sphere collides with the cyan object", + "the green cylinder collides with the green sphere", + "the yellow sphere and the cyan object collide", + "the brown object and the yellow sphere collide" + ], + "image_quantity_level": "Few", + "image": [ + "video_10849/video_10849_0.jpeg", + "video_10849/video_10849_1.jpeg", + "video_10849/video_10849_2.jpeg", + "video_10849/video_10849_3.jpeg", + "video_10849/video_10849_4.jpeg" + ], + "extracted": "D", + "result": 1 + }, + { + "sample_id": 102, + "question": "Observe the images and reply to the pertinent question about counterfactual inference. You must choose your answer from the Choice List.\nWhat will happen without the cyan object?\nChoice list: \nA. The brown object collides with the sphere\nB. The brown object collides with the metal object\nC. The red cylinder collides with the metal object\nD. The brown object collides with the red cylinder\nAnswer with the option's letter from the given choices directly.", + "gt_response": "the brown object collides with the red cylinder", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "d", + "choice_list": [ + "the brown object collides with the sphere", + "the brown object collides with the metal object", + "the red cylinder collides with the metal object", + "the brown object collides with the red cylinder" + ], + "image_quantity_level": "Few", + "image": [ + "video_13488/video_13488_0.jpeg", + "video_13488/video_13488_1.jpeg", + "video_13488/video_13488_2.jpeg", + "video_13488/video_13488_3.jpeg", + "video_13488/video_13488_4.jpeg" + ], + "extracted": "D", + "result": 1 + }, + { + "sample_id": 103, + "question": "Review the photos and answer the associated inquiry about counterfactual inference. You must choose your answer from the Choice List.\nWithout the sphere, what will happen?\nChoice list: \nA. The blue cube and the cylinder collide\nB. The blue cube and the cyan object collide\nAnswer with the option's letter from the given choices directly.", + "gt_response": "the blue cube and the cylinder collide", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "b", + "choice_list": [ + "the blue cube and the cylinder collide", + "the blue cube and the cyan object collide" + ], + "image_quantity_level": "Few", + "image": [ + "video_10652/video_10652_0.jpeg", + "video_10652/video_10652_1.jpeg", + "video_10652/video_10652_2.jpeg", + "video_10652/video_10652_3.jpeg", + "video_10652/video_10652_4.jpeg" + ], + "extracted": "B", + "result": 0 + }, + { + "sample_id": 104, + "question": "Based on the provided images, answer the question related to the counterfactual inference. You must choose your answer from the Choice List.\nWhat will happen without the red object?\nChoice list: \nA. The purple sphere and the blue sphere collide\nB. The yellow sphere collides with the green object\nC. The yellow object and the purple sphere collide\nD. The green object and the blue sphere collide\nAnswer with the option's letter from the given choices directly.", + "gt_response": "the yellow sphere collides with the green object", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "c", + "choice_list": [ + "the purple sphere and the blue sphere collide", + "the yellow sphere collides with the green object", + "the yellow object and the purple sphere collide", + "the green object and the blue sphere collide" + ], + "image_quantity_level": "Few", + "image": [ + "video_11330/video_11330_0.jpeg", + "video_11330/video_11330_1.jpeg", + "video_11330/video_11330_2.jpeg", + "video_11330/video_11330_3.jpeg", + "video_11330/video_11330_4.jpeg" + ], + "extracted": "C", + "result": 0 + }, + { + "sample_id": 105, + "question": "Assess the pictures and answer the connected query about counterfactual inference. You must choose your answer from the Choice List.\nWhich event will happen if the yellow object is removed?\nChoice list: \nA. The green cube and the purple object collide\nB. The blue object collides with the brown sphere\nC. The rubber cylinder and the brown object collide\nD. The green object and the blue sphere collide\nAnswer with the option's letter from the given choices directly.", + "gt_response": "the green object and the blue sphere collide", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "d", + "choice_list": [ + "the green cube and the purple object collide", + "the blue object collides with the brown sphere", + "the rubber cylinder and the brown object collide", + "the green object and the blue sphere collide" + ], + "image_quantity_level": "Few", + "image": [ + "video_11021/video_11021_0.jpeg", + "video_11021/video_11021_1.jpeg", + "video_11021/video_11021_2.jpeg", + "video_11021/video_11021_3.jpeg", + "video_11021/video_11021_4.jpeg" + ], + "extracted": "D", + "result": 1 + }, + { + "sample_id": 106, + "question": "Based on the provided images, answer the question related to the counterfactual inference. You must choose your answer from the Choice List.\nWithout the cylinder, which event will happen?\nChoice list: \nA. The metal sphere collides with the rubber sphere\nB. The purple object and the cube collide\nC. The red object collides with the cube\nAnswer with the option's letter from the given choices directly.", + "gt_response": "the metal sphere collides with the rubber sphere", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "c", + "choice_list": [ + "the metal sphere collides with the rubber sphere", + "the purple object and the cube collide", + "the red object collides with the cube" + ], + "image_quantity_level": "Few", + "image": [ + "video_10999/video_10999_0.jpeg", + "video_10999/video_10999_1.jpeg", + "video_10999/video_10999_2.jpeg", + "video_10999/video_10999_3.jpeg", + "video_10999/video_10999_4.jpeg" + ], + "extracted": "C", + "result": 0 + }, + { + "sample_id": 107, + "question": "Evaluate the images presented and provide an answer to the associated question about counterfactual inference. You must choose your answer from the Choice List.\nWhich of the following will happen without the green object?\nChoice list: \nA. The brown cube collides with the brown sphere\nB. The brown cylinder and the purple sphere collide\nC. The brown cube and the purple object collide\nD. The red cylinder and the purple object collide\nAnswer with the option's letter from the given choices directly.", + "gt_response": "the brown cube and the purple object collide", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "b", + "choice_list": [ + "the brown cube collides with the brown sphere", + "the brown cylinder and the purple sphere collide", + "the brown cube and the purple object collide", + "the red cylinder and the purple object collide" + ], + "image_quantity_level": "Few", + "image": [ + "video_10242/video_10242_0.jpeg", + "video_10242/video_10242_1.jpeg", + "video_10242/video_10242_2.jpeg", + "video_10242/video_10242_3.jpeg", + "video_10242/video_10242_4.jpeg" + ], + "extracted": "B", + "result": 0 + }, + { + "sample_id": 108, + "question": "Examine the images and answer the relevant query about counterfactual inference. You must choose your answer from the Choice List.\nWhat will happen without the cylinder?\nChoice list: \nA. The yellow cube collides with the yellow sphere\nB. The yellow cube and the cyan cube collide\nC. The cyan object collides with the rubber sphere\nD. The metal sphere collides with the gray object\nAnswer with the option's letter from the given choices directly.", + "gt_response": "the yellow cube collides with the yellow sphere", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "c", + "choice_list": [ + "the yellow cube collides with the yellow sphere", + "the yellow cube and the cyan cube collide", + "the cyan object collides with the rubber sphere", + "the metal sphere collides with the gray object" + ], + "image_quantity_level": "Few", + "image": [ + "video_11534/video_11534_0.jpeg", + "video_11534/video_11534_1.jpeg", + "video_11534/video_11534_2.jpeg", + "video_11534/video_11534_3.jpeg", + "video_11534/video_11534_4.jpeg" + ], + "extracted": "C", + "result": 0 + }, + { + "sample_id": 109, + "question": "Observe the images and reply to the pertinent question about counterfactual inference. You must choose your answer from the Choice List.\nWhich of the following will happen without the cylinder?\nChoice list: \nA. The rubber cube collides with the metal cube\nB. The blue sphere and the blue cube collide\nC. The cyan cube collides with the rubber sphere\nD. The blue sphere and the rubber sphere collide\nAnswer with the option's letter from the given choices directly.", + "gt_response": "the rubber cube collides with the metal cube", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "b", + "choice_list": [ + "the rubber cube collides with the metal cube", + "the blue sphere and the blue cube collide", + "the cyan cube collides with the rubber sphere", + "the blue sphere and the rubber sphere collide" + ], + "image_quantity_level": "Few", + "image": [ + "video_10972/video_10972_0.jpeg", + "video_10972/video_10972_1.jpeg", + "video_10972/video_10972_2.jpeg", + "video_10972/video_10972_3.jpeg", + "video_10972/video_10972_4.jpeg" + ], + "extracted": "B", + "result": 0 + }, + { + "sample_id": 110, + "question": "Based on the provided images, answer the question related to the counterfactual inference. You must choose your answer from the Choice List.\nWhich event will happen if the gray object is removed?\nChoice list: \nA. The cube collides with the cyan sphere\nB. The cylinder and the cyan sphere collide\nAnswer with the option's letter from the given choices directly.", + "gt_response": "the cylinder and the cyan sphere collide", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "b", + "choice_list": [ + "the cube collides with the cyan sphere", + "the cylinder and the cyan sphere collide" + ], + "image_quantity_level": "Few", + "image": [ + "video_14694/video_14694_0.jpeg", + "video_14694/video_14694_1.jpeg", + "video_14694/video_14694_2.jpeg", + "video_14694/video_14694_3.jpeg", + "video_14694/video_14694_4.jpeg" + ], + "extracted": "B", + "result": 1 + }, + { + "sample_id": 111, + "question": "Observe the images and reply to the pertinent question about counterfactual inference. You must choose your answer from the Choice List.\nWithout the cylinder, which of the following will happen?\nChoice list: \nA. The brown cube and the sphere collide\nAnswer with the option's letter from the given choices directly.", + "gt_response": "the brown cube and the sphere collide", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "a", + "choice_list": [ + "the brown cube and the sphere collide" + ], + "image_quantity_level": "Few", + "image": [ + "video_10427/video_10427_0.jpeg", + "video_10427/video_10427_1.jpeg", + "video_10427/video_10427_2.jpeg", + "video_10427/video_10427_3.jpeg", + "video_10427/video_10427_4.jpeg" + ], + "extracted": "A", + "result": 1 + }, + { + "sample_id": 112, + "question": "Scrutinize the visuals and respond to the related question about counterfactual inference. You must choose your answer from the Choice List.\nWhich of the following will happen if the yellow object is removed?\nChoice list: \nA. The rubber object collides with the green cube\nB. The blue object and the sphere collide\nC. The green cube and the sphere collide\nD. The cylinder collides with the sphere\nAnswer with the option's letter from the given choices directly.", + "gt_response": "the rubber object collides with the green cube", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "a", + "choice_list": [ + "the rubber object collides with the green cube", + "the blue object and the sphere collide", + "the green cube and the sphere collide", + "the cylinder collides with the sphere" + ], + "image_quantity_level": "Few", + "image": [ + "video_11232/video_11232_0.jpeg", + "video_11232/video_11232_1.jpeg", + "video_11232/video_11232_2.jpeg", + "video_11232/video_11232_3.jpeg", + "video_11232/video_11232_4.jpeg" + ], + "extracted": "A", + "result": 1 + }, + { + "sample_id": 113, + "question": "Scrutinize the visuals and respond to the related question about counterfactual inference. You must choose your answer from the Choice List.\nWhich event will happen if the cube is removed?\nChoice list: \nA. The rubber sphere and the gray cylinder collide\nB. The blue cylinder and the rubber sphere collide\nC. The metal sphere collides with the blue cylinder\nD. The rubber cylinder collides with the gray object\nAnswer with the option's letter from the given choices directly.", + "gt_response": "the metal sphere collides with the blue cylinder", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "c", + "choice_list": [ + "the rubber sphere and the gray cylinder collide", + "the blue cylinder and the rubber sphere collide", + "the metal sphere collides with the blue cylinder", + "the rubber cylinder collides with the gray object" + ], + "image_quantity_level": "Few", + "image": [ + "video_14400/video_14400_0.jpeg", + "video_14400/video_14400_1.jpeg", + "video_14400/video_14400_2.jpeg", + "video_14400/video_14400_3.jpeg", + "video_14400/video_14400_4.jpeg" + ], + "extracted": "C", + "result": 1 + }, + { + "sample_id": 114, + "question": "Based on the provided images, answer the question related to the counterfactual inference. You must choose your answer from the Choice List.\nWhich of the following will happen if the green object is removed?\nChoice list: \nA. The yellow object and the red rubber object collide\nB. The rubber cylinder and the yellow object collide\nC. The rubber cylinder and the metal cube collide\nD. The metal cube collides with the red rubber object\nAnswer with the option's letter from the given choices directly.", + "gt_response": "the rubber cylinder and the metal cube collide", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "c", + "choice_list": [ + "the yellow object and the red rubber object collide", + "the rubber cylinder and the yellow object collide", + "the rubber cylinder and the metal cube collide", + "the metal cube collides with the red rubber object" + ], + "image_quantity_level": "Few", + "image": [ + "video_10998/video_10998_0.jpeg", + "video_10998/video_10998_1.jpeg", + "video_10998/video_10998_2.jpeg", + "video_10998/video_10998_3.jpeg", + "video_10998/video_10998_4.jpeg" + ], + "extracted": "C", + "result": 1 + }, + { + "sample_id": 115, + "question": "Examine the images and answer the relevant query about counterfactual inference. You must choose your answer from the Choice List.\nIf the yellow object is removed, which of the following will happen?\nChoice list: \nA. The brown sphere collides with the gray object\nB. The rubber sphere collides with the blue object\nAnswer with the option's letter from the given choices directly.", + "gt_response": "the rubber sphere collides with the blue object", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "b", + "choice_list": [ + "the brown sphere collides with the gray object", + "the rubber sphere collides with the blue object" + ], + "image_quantity_level": "Few", + "image": [ + "video_10561/video_10561_0.jpeg", + "video_10561/video_10561_1.jpeg", + "video_10561/video_10561_2.jpeg", + "video_10561/video_10561_3.jpeg", + "video_10561/video_10561_4.jpeg" + ], + "extracted": "B", + "result": 1 + }, + { + "sample_id": 116, + "question": "Inspect the given graphics and answer the relevant query about counterfactual inference. You must choose your answer from the Choice List.\nWithout the gray object, which of the following will happen?\nChoice list: \nA. The brown sphere and the purple sphere collide\nB. The blue sphere and the brown sphere collide\nC. The brown cylinder and the purple object collide\nD. The blue sphere and the brown cylinder collide\nAnswer with the option's letter from the given choices directly.", + "gt_response": "the blue sphere and the brown sphere collide", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "b", + "choice_list": [ + "the brown sphere and the purple sphere collide", + "the blue sphere and the brown sphere collide", + "the brown cylinder and the purple object collide", + "the blue sphere and the brown cylinder collide" + ], + "image_quantity_level": "Few", + "image": [ + "video_13829/video_13829_0.jpeg", + "video_13829/video_13829_1.jpeg", + "video_13829/video_13829_2.jpeg", + "video_13829/video_13829_3.jpeg", + "video_13829/video_13829_4.jpeg" + ], + "extracted": "B", + "result": 1 + }, + { + "sample_id": 117, + "question": "Based on the provided images, answer the question related to the counterfactual inference. You must choose your answer from the Choice List.\nIf the sphere is removed, which event will happen?\nChoice list: \nA. The cube collides with the rubber cylinder\nB. The yellow cylinder collides with the green object\nAnswer with the option's letter from the given choices directly.", + "gt_response": "the cube collides with the rubber cylinder", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "b", + "choice_list": [ + "the cube collides with the rubber cylinder", + "the yellow cylinder collides with the green object" + ], + "image_quantity_level": "Few", + "image": [ + "video_10524/video_10524_0.jpeg", + "video_10524/video_10524_1.jpeg", + "video_10524/video_10524_2.jpeg", + "video_10524/video_10524_3.jpeg", + "video_10524/video_10524_4.jpeg" + ], + "extracted": "B", + "result": 0 + }, + { + "sample_id": 118, + "question": "Assess the pictures and answer the connected query about counterfactual inference. You must choose your answer from the Choice List.\nIf the cube is removed, what will happen?\nChoice list: \nA. The yellow cylinder and the brown object collide\nB. The cyan object collides with the brown cylinder\nC. The metal object collides with the cyan cylinder\nD. The yellow cylinder and the sphere collide\nAnswer with the option's letter from the given choices directly.", + "gt_response": "the metal object collides with the cyan cylinder", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "c", + "choice_list": [ + "the yellow cylinder and the brown object collide", + "the cyan object collides with the brown cylinder", + "the metal object collides with the cyan cylinder", + "the yellow cylinder and the sphere collide" + ], + "image_quantity_level": "Few", + "image": [ + "video_10417/video_10417_0.jpeg", + "video_10417/video_10417_1.jpeg", + "video_10417/video_10417_2.jpeg", + "video_10417/video_10417_3.jpeg", + "video_10417/video_10417_4.jpeg" + ], + "extracted": "C", + "result": 1 + }, + { + "sample_id": 119, + "question": "Analyze the given pictures and answer the related inquiry about counterfactual inference. You must choose your answer from the Choice List.\nWithout the cylinder, what will happen?\nChoice list: \nA. The cyan object collides with the red object\nB. The cyan cube and the blue metal sphere collide\nC. The metal cube collides with the blue cube\nD. The blue metal object collides with the red object\nAnswer with the option's letter from the given choices directly.", + "gt_response": "the blue metal object collides with the red object", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "b", + "choice_list": [ + "the cyan object collides with the red object", + "the cyan cube and the blue metal sphere collide", + "the metal cube collides with the blue cube", + "the blue metal object collides with the red object" + ], + "image_quantity_level": "Few", + "image": [ + "video_11868/video_11868_0.jpeg", + "video_11868/video_11868_1.jpeg", + "video_11868/video_11868_2.jpeg", + "video_11868/video_11868_3.jpeg", + "video_11868/video_11868_4.jpeg" + ], + "extracted": "B", + "result": 0 + }, + { + "sample_id": 120, + "question": "Analyze the given pictures and answer the related inquiry about counterfactual inference. You must choose your answer from the Choice List.\nWhich of the following will happen if the yellow object is removed?\nChoice list: \nA. The brown sphere and the rubber object collide\nB. The brown sphere and the brown cylinder collide\nAnswer with the option's letter from the given choices directly.", + "gt_response": "the brown sphere and the brown cylinder collide", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "b", + "choice_list": [ + "the brown sphere and the rubber object collide", + "the brown sphere and the brown cylinder collide" + ], + "image_quantity_level": "Few", + "image": [ + "video_10678/video_10678_0.jpeg", + "video_10678/video_10678_1.jpeg", + "video_10678/video_10678_2.jpeg", + "video_10678/video_10678_3.jpeg", + "video_10678/video_10678_4.jpeg" + ], + "extracted": "B", + "result": 1 + }, + { + "sample_id": 121, + "question": "Inspect the given graphics and answer the relevant query about counterfactual inference. You must choose your answer from the Choice List.\nWhich event will happen without the cylinder?\nChoice list: \nA. The rubber sphere collides with the metal sphere\nB. The cube collides with the rubber sphere\nC. The cube collides with the metal sphere\nAnswer with the option's letter from the given choices directly.", + "gt_response": "the cube collides with the rubber sphere", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "c", + "choice_list": [ + "the rubber sphere collides with the metal sphere", + "the cube collides with the rubber sphere", + "the cube collides with the metal sphere" + ], + "image_quantity_level": "Few", + "image": [ + "video_13970/video_13970_0.jpeg", + "video_13970/video_13970_1.jpeg", + "video_13970/video_13970_2.jpeg", + "video_13970/video_13970_3.jpeg", + "video_13970/video_13970_4.jpeg" + ], + "extracted": "C", + "result": 0 + }, + { + "sample_id": 122, + "question": "Inspect the given graphics and answer the relevant query about counterfactual inference. You must choose your answer from the Choice List.\nWhich of the following will happen if the cube is removed?\nChoice list: \nA. The cylinder collides with the metal sphere\nB. The purple object and the metal sphere collide\nAnswer with the option's letter from the given choices directly.", + "gt_response": "the cylinder collides with the metal sphere", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "a", + "choice_list": [ + "the cylinder collides with the metal sphere", + "the purple object and the metal sphere collide" + ], + "image_quantity_level": "Few", + "image": [ + "video_14755/video_14755_0.jpeg", + "video_14755/video_14755_1.jpeg", + "video_14755/video_14755_2.jpeg", + "video_14755/video_14755_3.jpeg", + "video_14755/video_14755_4.jpeg" + ], + "extracted": "A", + "result": 1 + }, + { + "sample_id": 123, + "question": "Based on the provided images, answer the question related to the counterfactual inference. You must choose your answer from the Choice List.\nWithout the green object, which of the following will happen?\nChoice list: \nA. The brown sphere collides with the cyan sphere\nB. The cyan cylinder collides with the cyan sphere\nC. The cyan cylinder collides with the cube\nD. The cube collides with the cyan sphere\nAnswer with the option's letter from the given choices directly.", + "gt_response": "the cyan cylinder collides with the cube", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "c", + "choice_list": [ + "the brown sphere collides with the cyan sphere", + "the cyan cylinder collides with the cyan sphere", + "the cyan cylinder collides with the cube", + "the cube collides with the cyan sphere" + ], + "image_quantity_level": "Few", + "image": [ + "video_12220/video_12220_0.jpeg", + "video_12220/video_12220_1.jpeg", + "video_12220/video_12220_2.jpeg", + "video_12220/video_12220_3.jpeg", + "video_12220/video_12220_4.jpeg" + ], + "extracted": "C", + "result": 1 + }, + { + "sample_id": 124, + "question": "Observe the images and reply to the pertinent question about counterfactual inference. You must choose your answer from the Choice List.\nIf the cyan object is removed, what will happen?\nChoice list: \nA. The gray object collides with the blue cube\nB. The gray sphere and the brown cube collide\nAnswer with the option's letter from the given choices directly.", + "gt_response": "the gray sphere and the brown cube collide", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "a", + "choice_list": [ + "the gray object collides with the blue cube", + "the gray sphere and the brown cube collide" + ], + "image_quantity_level": "Few", + "image": [ + "video_13983/video_13983_0.jpeg", + "video_13983/video_13983_1.jpeg", + "video_13983/video_13983_2.jpeg", + "video_13983/video_13983_3.jpeg", + "video_13983/video_13983_4.jpeg" + ], + "extracted": "A", + "result": 0 + }, + { + "sample_id": 125, + "question": "Assess the pictures and answer the connected query about counterfactual inference. You must choose your answer from the Choice List.\nWithout the cyan object, which of the following will happen?\nChoice list: \nA. The sphere and the yellow object collide\nB. The green cylinder collides with the cube\nC. The yellow cylinder collides with the cube\nD. The yellow cylinder and the green cylinder collide\nAnswer with the option's letter from the given choices directly.", + "gt_response": "the sphere and the yellow object collide", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "d", + "choice_list": [ + "the sphere and the yellow object collide", + "the green cylinder collides with the cube", + "the yellow cylinder collides with the cube", + "the yellow cylinder and the green cylinder collide" + ], + "image_quantity_level": "Few", + "image": [ + "video_13881/video_13881_0.jpeg", + "video_13881/video_13881_1.jpeg", + "video_13881/video_13881_2.jpeg", + "video_13881/video_13881_3.jpeg", + "video_13881/video_13881_4.jpeg" + ], + "extracted": "D", + "result": 0 + }, + { + "sample_id": 126, + "question": "Examine the images and answer the relevant query about counterfactual inference. You must choose your answer from the Choice List.\nWhich of the following will happen if the cylinder is removed?\nChoice list: \nA. The rubber object collides with the gray cube\nB. The gray cube and the blue object collide\nC. The cyan cube collides with the blue object\nAnswer with the option's letter from the given choices directly.", + "gt_response": "the rubber object collides with the gray cube", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "a", + "choice_list": [ + "the rubber object collides with the gray cube", + "the gray cube and the blue object collide", + "the cyan cube collides with the blue object" + ], + "image_quantity_level": "Few", + "image": [ + "video_10270/video_10270_0.jpeg", + "video_10270/video_10270_1.jpeg", + "video_10270/video_10270_2.jpeg", + "video_10270/video_10270_3.jpeg", + "video_10270/video_10270_4.jpeg" + ], + "extracted": "A", + "result": 1 + }, + { + "sample_id": 127, + "question": "Scrutinize the visuals and respond to the related question about counterfactual inference. You must choose your answer from the Choice List.\nWithout the purple object, which event will happen?\nChoice list: \nA. The yellow sphere and the cyan sphere collide\nB. The cylinder and the cyan object collide\nAnswer with the option's letter from the given choices directly.", + "gt_response": "the cylinder and the cyan object collide", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "b", + "choice_list": [ + "the yellow sphere and the cyan sphere collide", + "the cylinder and the cyan object collide" + ], + "image_quantity_level": "Few", + "image": [ + "video_12955/video_12955_0.jpeg", + "video_12955/video_12955_1.jpeg", + "video_12955/video_12955_2.jpeg", + "video_12955/video_12955_3.jpeg", + "video_12955/video_12955_4.jpeg" + ], + "extracted": "B", + "result": 1 + }, + { + "sample_id": 128, + "question": "Inspect the given graphics and answer the relevant query about counterfactual inference. You must choose your answer from the Choice List.\nWithout the cube, what will happen?\nChoice list: \nA. The sphere and the gray cylinder collide\nB. The sphere and the purple cylinder collide\nAnswer with the option's letter from the given choices directly.", + "gt_response": "the sphere and the purple cylinder collide", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "b", + "choice_list": [ + "the sphere and the gray cylinder collide", + "the sphere and the purple cylinder collide" + ], + "image_quantity_level": "Few", + "image": [ + "video_10262/video_10262_0.jpeg", + "video_10262/video_10262_1.jpeg", + "video_10262/video_10262_2.jpeg", + "video_10262/video_10262_3.jpeg", + "video_10262/video_10262_4.jpeg" + ], + "extracted": "B", + "result": 1 + }, + { + "sample_id": 129, + "question": "Examine the images and answer the relevant query about counterfactual inference. You must choose your answer from the Choice List.\nWhich event will happen without the cube?\nChoice list: \nA. The sphere and the green cylinder collide\nB. The yellow object collides with the green cylinder\nC. The brown cylinder and the green cylinder collide\nD. The sphere and the brown cylinder collide\nAnswer with the option's letter from the given choices directly.", + "gt_response": "the sphere and the brown cylinder collide", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "d", + "choice_list": [ + "the sphere and the green cylinder collide", + "the yellow object collides with the green cylinder", + "the brown cylinder and the green cylinder collide", + "the sphere and the brown cylinder collide" + ], + "image_quantity_level": "Few", + "image": [ + "video_12268/video_12268_0.jpeg", + "video_12268/video_12268_1.jpeg", + "video_12268/video_12268_2.jpeg", + "video_12268/video_12268_3.jpeg", + "video_12268/video_12268_4.jpeg" + ], + "extracted": "D", + "result": 1 + }, + { + "sample_id": 130, + "question": "Examine the images and answer the relevant query about counterfactual inference. You must choose your answer from the Choice List.\nWhich event will happen without the gray object?\nChoice list: \nA. The green sphere collides with the brown sphere\nB. The purple sphere collides with the brown sphere\nAnswer with the option's letter from the given choices directly.", + "gt_response": "the green sphere collides with the brown sphere", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "a", + "choice_list": [ + "the green sphere collides with the brown sphere", + "the purple sphere collides with the brown sphere" + ], + "image_quantity_level": "Few", + "image": [ + "video_11039/video_11039_0.jpeg", + "video_11039/video_11039_1.jpeg", + "video_11039/video_11039_2.jpeg", + "video_11039/video_11039_3.jpeg", + "video_11039/video_11039_4.jpeg" + ], + "extracted": "A", + "result": 1 + }, + { + "sample_id": 131, + "question": "Examine the images and answer the relevant query about counterfactual inference. You must choose your answer from the Choice List.\nIf the brown object is removed, which of the following will happen?\nChoice list: \nA. The sphere collides with the gray object\nB. The metal cylinder and the gray cylinder collide\nAnswer with the option's letter from the given choices directly.", + "gt_response": "the metal cylinder and the gray cylinder collide", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "b", + "choice_list": [ + "the sphere collides with the gray object", + "the metal cylinder and the gray cylinder collide" + ], + "image_quantity_level": "Few", + "image": [ + "video_14577/video_14577_0.jpeg", + "video_14577/video_14577_1.jpeg", + "video_14577/video_14577_2.jpeg", + "video_14577/video_14577_3.jpeg", + "video_14577/video_14577_4.jpeg" + ], + "extracted": "B", + "result": 1 + }, + { + "sample_id": 132, + "question": "Examine the images and answer the relevant query about counterfactual inference. You must choose your answer from the Choice List.\nIf the purple object is removed, what will happen?\nChoice list: \nA. The gray cylinder and the red sphere collide\nB. The red cylinder collides with the red sphere\nC. The cube and the gray object collide\nD. The cube collides with the red sphere\nAnswer with the option's letter from the given choices directly.", + "gt_response": "the cube collides with the red sphere", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "c", + "choice_list": [ + "the gray cylinder and the red sphere collide", + "the red cylinder collides with the red sphere", + "the cube and the gray object collide", + "the cube collides with the red sphere" + ], + "image_quantity_level": "Few", + "image": [ + "video_13019/video_13019_0.jpeg", + "video_13019/video_13019_1.jpeg", + "video_13019/video_13019_2.jpeg", + "video_13019/video_13019_3.jpeg", + "video_13019/video_13019_4.jpeg" + ], + "extracted": "C", + "result": 0 + }, + { + "sample_id": 133, + "question": "Observe the images and reply to the pertinent question about counterfactual inference. You must choose your answer from the Choice List.\nWhich event will happen without the cube?\nChoice list: \nA. The sphere and the rubber cylinder collide\nB. The sphere collides with the purple object\nC. The rubber cylinder collides with the metal cylinder\nAnswer with the option's letter from the given choices directly.", + "gt_response": "the rubber cylinder collides with the metal cylinder", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "b", + "choice_list": [ + "the sphere and the rubber cylinder collide", + "the sphere collides with the purple object", + "the rubber cylinder collides with the metal cylinder" + ], + "image_quantity_level": "Few", + "image": [ + "video_14507/video_14507_0.jpeg", + "video_14507/video_14507_1.jpeg", + "video_14507/video_14507_2.jpeg", + "video_14507/video_14507_3.jpeg", + "video_14507/video_14507_4.jpeg" + ], + "extracted": "B", + "result": 0 + }, + { + "sample_id": 134, + "question": "Inspect the given graphics and answer the relevant query about counterfactual inference. You must choose your answer from the Choice List.\nWhich event will happen if the metal object is removed?\nChoice list: \nA. The cube collides with the blue object\nB. The blue object and the cylinder collide\nAnswer with the option's letter from the given choices directly.", + "gt_response": "the cube collides with the blue object", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "a", + "choice_list": [ + "the cube collides with the blue object", + "the blue object and the cylinder collide" + ], + "image_quantity_level": "Few", + "image": [ + "video_12447/video_12447_0.jpeg", + "video_12447/video_12447_1.jpeg", + "video_12447/video_12447_2.jpeg", + "video_12447/video_12447_3.jpeg", + "video_12447/video_12447_4.jpeg" + ], + "extracted": "A", + "result": 1 + }, + { + "sample_id": 135, + "question": "Based on the provided images, answer the question related to the counterfactual inference. You must choose your answer from the Choice List.\nWhat will happen without the cube?\nChoice list: \nA. The gray cylinder and the yellow object collide\nB. The red sphere collides with the cyan sphere\nC. The gray cylinder collides with the rubber cylinder\nD. The yellow sphere and the cyan cylinder collide\nAnswer with the option's letter from the given choices directly.", + "gt_response": "the gray cylinder and the yellow object collide", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "b", + "choice_list": [ + "the gray cylinder and the yellow object collide", + "the red sphere collides with the cyan sphere", + "the gray cylinder collides with the rubber cylinder", + "the yellow sphere and the cyan cylinder collide" + ], + "image_quantity_level": "Few", + "image": [ + "video_14496/video_14496_0.jpeg", + "video_14496/video_14496_1.jpeg", + "video_14496/video_14496_2.jpeg", + "video_14496/video_14496_3.jpeg", + "video_14496/video_14496_4.jpeg" + ], + "extracted": "B", + "result": 0 + }, + { + "sample_id": 136, + "question": "Review the photos and answer the associated inquiry about counterfactual inference. You must choose your answer from the Choice List.\nWithout the brown object, which event will happen?\nChoice list: \nA. The purple cylinder collides with the blue object\nB. The cube collides with the blue object\nAnswer with the option's letter from the given choices directly.", + "gt_response": "the purple cylinder collides with the blue object", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "a", + "choice_list": [ + "the purple cylinder collides with the blue object", + "the cube collides with the blue object" + ], + "image_quantity_level": "Few", + "image": [ + "video_14674/video_14674_0.jpeg", + "video_14674/video_14674_1.jpeg", + "video_14674/video_14674_2.jpeg", + "video_14674/video_14674_3.jpeg", + "video_14674/video_14674_4.jpeg" + ], + "extracted": "A", + "result": 1 + }, + { + "sample_id": 137, + "question": "Observe the images and reply to the pertinent question about counterfactual inference. You must choose your answer from the Choice List.\nWhich of the following will happen without the cylinder?\nChoice list: \nA. The cyan object and the brown cube collide\nAnswer with the option's letter from the given choices directly.", + "gt_response": "the cyan object and the brown cube collide", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "a", + "choice_list": [ + "the cyan object and the brown cube collide" + ], + "image_quantity_level": "Few", + "image": [ + "video_10462/video_10462_0.jpeg", + "video_10462/video_10462_1.jpeg", + "video_10462/video_10462_2.jpeg", + "video_10462/video_10462_3.jpeg", + "video_10462/video_10462_4.jpeg" + ], + "extracted": "A", + "result": 1 + }, + { + "sample_id": 138, + "question": "Analyze the given pictures and answer the related inquiry about counterfactual inference. You must choose your answer from the Choice List.\nWhat will happen without the yellow object?\nChoice list: \nA. The metal sphere and the rubber cube collide\nB. The gray object collides with the metal cube\nC. The rubber cylinder collides with the brown sphere\nD. The red cube collides with the rubber cube\nAnswer with the option's letter from the given choices directly.", + "gt_response": "the gray object collides with the metal cube", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "b", + "choice_list": [ + "the metal sphere and the rubber cube collide", + "the gray object collides with the metal cube", + "the rubber cylinder collides with the brown sphere", + "the red cube collides with the rubber cube" + ], + "image_quantity_level": "Few", + "image": [ + "video_11548/video_11548_0.jpeg", + "video_11548/video_11548_1.jpeg", + "video_11548/video_11548_2.jpeg", + "video_11548/video_11548_3.jpeg", + "video_11548/video_11548_4.jpeg" + ], + "extracted": "B", + "result": 1 + }, + { + "sample_id": 139, + "question": "Inspect the given graphics and answer the relevant query about counterfactual inference. You must choose your answer from the Choice List.\nWithout the cube, what will happen?\nChoice list: \nA. The red object collides with the yellow cylinder\nB. The red object and the blue sphere collide\nC. The brown object and the yellow sphere collide\nD. The brown object collides with the yellow cylinder\nAnswer with the option's letter from the given choices directly.", + "gt_response": "the brown object and the yellow sphere collide", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "c", + "choice_list": [ + "the red object collides with the yellow cylinder", + "the red object and the blue sphere collide", + "the brown object and the yellow sphere collide", + "the brown object collides with the yellow cylinder" + ], + "image_quantity_level": "Few", + "image": [ + "video_13763/video_13763_0.jpeg", + "video_13763/video_13763_1.jpeg", + "video_13763/video_13763_2.jpeg", + "video_13763/video_13763_3.jpeg", + "video_13763/video_13763_4.jpeg" + ], + "extracted": "C", + "result": 1 + }, + { + "sample_id": 140, + "question": "Scrutinize the visuals and respond to the related question about counterfactual inference. You must choose your answer from the Choice List.\nWithout the cube, which event will happen?\nChoice list: \nA. The cyan object and the sphere collide\nB. The cyan object and the brown object collide\nC. The brown object collides with the sphere\nAnswer with the option's letter from the given choices directly.", + "gt_response": "the cyan object and the brown object collide", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "b", + "choice_list": [ + "the cyan object and the sphere collide", + "the cyan object and the brown object collide", + "the brown object collides with the sphere" + ], + "image_quantity_level": "Few", + "image": [ + "video_10170/video_10170_0.jpeg", + "video_10170/video_10170_1.jpeg", + "video_10170/video_10170_2.jpeg", + "video_10170/video_10170_3.jpeg", + "video_10170/video_10170_4.jpeg" + ], + "extracted": "B", + "result": 1 + }, + { + "sample_id": 141, + "question": "Analyze the given pictures and answer the related inquiry about counterfactual inference. You must choose your answer from the Choice List.\nWhich event will happen without the sphere?\nChoice list: \nA. The metal cylinder and the cube collide\nB. The metal cylinder and the rubber cylinder collide\nAnswer with the option's letter from the given choices directly.", + "gt_response": "the metal cylinder and the rubber cylinder collide", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "b", + "choice_list": [ + "the metal cylinder and the cube collide", + "the metal cylinder and the rubber cylinder collide" + ], + "image_quantity_level": "Few", + "image": [ + "video_10882/video_10882_0.jpeg", + "video_10882/video_10882_1.jpeg", + "video_10882/video_10882_2.jpeg", + "video_10882/video_10882_3.jpeg", + "video_10882/video_10882_4.jpeg" + ], + "extracted": "B", + "result": 1 + }, + { + "sample_id": 142, + "question": "Inspect the given graphics and answer the relevant query about counterfactual inference. You must choose your answer from the Choice List.\nIf the metal object is removed, what will happen?\nChoice list: \nA. The purple sphere collides with the cyan sphere\nB. The purple sphere and the red object collide\nC. The cyan object collides with the red cylinder\nD. The blue object collides with the cyan sphere\nAnswer with the option's letter from the given choices directly.", + "gt_response": "the purple sphere collides with the cyan sphere", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "a", + "choice_list": [ + "the purple sphere collides with the cyan sphere", + "the purple sphere and the red object collide", + "the cyan object collides with the red cylinder", + "the blue object collides with the cyan sphere" + ], + "image_quantity_level": "Few", + "image": [ + "video_12929/video_12929_0.jpeg", + "video_12929/video_12929_1.jpeg", + "video_12929/video_12929_2.jpeg", + "video_12929/video_12929_3.jpeg", + "video_12929/video_12929_4.jpeg" + ], + "extracted": "A", + "result": 1 + }, + { + "sample_id": 143, + "question": "Inspect the given graphics and answer the relevant query about counterfactual inference. You must choose your answer from the Choice List.\nWhich event will happen without the cube?\nChoice list: \nA. The cyan object collides with the brown object\nB. The gray sphere and the brown sphere collide\nAnswer with the option's letter from the given choices directly.", + "gt_response": "the gray sphere and the brown sphere collide", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "a", + "choice_list": [ + "the cyan object collides with the brown object", + "the gray sphere and the brown sphere collide" + ], + "image_quantity_level": "Few", + "image": [ + "video_10167/video_10167_0.jpeg", + "video_10167/video_10167_1.jpeg", + "video_10167/video_10167_2.jpeg", + "video_10167/video_10167_3.jpeg", + "video_10167/video_10167_4.jpeg" + ], + "extracted": "A", + "result": 0 + }, + { + "sample_id": 144, + "question": "Analyze the given pictures and answer the related inquiry about counterfactual inference. You must choose your answer from the Choice List.\nWhich of the following will happen without the cube?\nChoice list: \nA. The blue object collides with the cyan object\nB. The metal sphere collides with the gray object\nC. The rubber sphere and the gray object collide\nD. The cyan object collides with the gray cylinder\nAnswer with the option's letter from the given choices directly.", + "gt_response": "the rubber sphere and the gray object collide", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "b", + "choice_list": [ + "the blue object collides with the cyan object", + "the metal sphere collides with the gray object", + "the rubber sphere and the gray object collide", + "the cyan object collides with the gray cylinder" + ], + "image_quantity_level": "Few", + "image": [ + "video_10615/video_10615_0.jpeg", + "video_10615/video_10615_1.jpeg", + "video_10615/video_10615_2.jpeg", + "video_10615/video_10615_3.jpeg", + "video_10615/video_10615_4.jpeg" + ], + "extracted": "B", + "result": 0 + }, + { + "sample_id": 145, + "question": "Observe the images and reply to the pertinent question about counterfactual inference. You must choose your answer from the Choice List.\nWhich of the following will happen if the cylinder is removed?\nChoice list: \nA. The brown cube collides with the rubber sphere\nB. The red object collides with the cyan sphere\nC. The purple object collides with the cyan object\nD. The purple cube and the red object collide\nAnswer with the option's letter from the given choices directly.", + "gt_response": "the purple cube and the red object collide", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "d", + "choice_list": [ + "the brown cube collides with the rubber sphere", + "the red object collides with the cyan sphere", + "the purple object collides with the cyan object", + "the purple cube and the red object collide" + ], + "image_quantity_level": "Few", + "image": [ + "video_10850/video_10850_0.jpeg", + "video_10850/video_10850_1.jpeg", + "video_10850/video_10850_2.jpeg", + "video_10850/video_10850_3.jpeg", + "video_10850/video_10850_4.jpeg" + ], + "extracted": "D", + "result": 1 + }, + { + "sample_id": 146, + "question": "Based on the provided images, answer the question related to the counterfactual inference. You must choose your answer from the Choice List.\nWhat will happen without the cube?\nChoice list: \nA. The metal cylinder collides with the sphere\nB. The sphere and the rubber cylinder collide\nC. The metal cylinder collides with the rubber cylinder\nAnswer with the option's letter from the given choices directly.", + "gt_response": "the sphere and the rubber cylinder collide", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "a", + "choice_list": [ + "the metal cylinder collides with the sphere", + "the sphere and the rubber cylinder collide", + "the metal cylinder collides with the rubber cylinder" + ], + "image_quantity_level": "Few", + "image": [ + "video_14231/video_14231_0.jpeg", + "video_14231/video_14231_1.jpeg", + "video_14231/video_14231_2.jpeg", + "video_14231/video_14231_3.jpeg", + "video_14231/video_14231_4.jpeg" + ], + "extracted": "A", + "result": 0 + }, + { + "sample_id": 147, + "question": "Examine the images and answer the relevant query about counterfactual inference. You must choose your answer from the Choice List.\nIf the cube is removed, which event will happen?\nChoice list: \nA. The rubber sphere collides with the purple sphere\nB. The rubber sphere collides with the cylinder\nC. The purple sphere and the blue object collide\nD. The cylinder collides with the blue object\nAnswer with the option's letter from the given choices directly.", + "gt_response": "the rubber sphere collides with the purple sphere", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "b", + "choice_list": [ + "the rubber sphere collides with the purple sphere", + "the rubber sphere collides with the cylinder", + "the purple sphere and the blue object collide", + "the cylinder collides with the blue object" + ], + "image_quantity_level": "Few", + "image": [ + "video_12878/video_12878_0.jpeg", + "video_12878/video_12878_1.jpeg", + "video_12878/video_12878_2.jpeg", + "video_12878/video_12878_3.jpeg", + "video_12878/video_12878_4.jpeg" + ], + "extracted": "B", + "result": 0 + }, + { + "sample_id": 148, + "question": "Review the photos and answer the associated inquiry about counterfactual inference. You must choose your answer from the Choice List.\nWhich of the following will happen without the cylinder?\nChoice list: \nA. The green cube collides with the gray object\nB. The green cube and the cyan cube collide\nC. The cyan object collides with the gray object\nD. The yellow object and the cyan object collide\nAnswer with the option's letter from the given choices directly.", + "gt_response": "the yellow object and the cyan object collide", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "b", + "choice_list": [ + "the green cube collides with the gray object", + "the green cube and the cyan cube collide", + "the cyan object collides with the gray object", + "the yellow object and the cyan object collide" + ], + "image_quantity_level": "Few", + "image": [ + "video_13308/video_13308_0.jpeg", + "video_13308/video_13308_1.jpeg", + "video_13308/video_13308_2.jpeg", + "video_13308/video_13308_3.jpeg", + "video_13308/video_13308_4.jpeg" + ], + "extracted": "B", + "result": 0 + }, + { + "sample_id": 149, + "question": "Inspect the given graphics and answer the relevant query about counterfactual inference. You must choose your answer from the Choice List.\nWhat will happen if the purple object is removed?\nChoice list: \nA. The metal sphere and the yellow sphere collide\nB. The cylinder collides with the yellow object\nC. The cylinder and the metal object collide\nAnswer with the option's letter from the given choices directly.", + "gt_response": "the metal sphere and the yellow sphere collide", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "a", + "choice_list": [ + "the metal sphere and the yellow sphere collide", + "the cylinder collides with the yellow object", + "the cylinder and the metal object collide" + ], + "image_quantity_level": "Few", + "image": [ + "video_10663/video_10663_0.jpeg", + "video_10663/video_10663_1.jpeg", + "video_10663/video_10663_2.jpeg", + "video_10663/video_10663_3.jpeg", + "video_10663/video_10663_4.jpeg" + ], + "extracted": "A", + "result": 1 + }, + { + "sample_id": 150, + "question": "Examine the images and answer the relevant query about counterfactual inference. You must choose your answer from the Choice List.\nWithout the sphere, which of the following will happen?\nChoice list: \nA. The cyan object collides with the purple object\nB. The cube collides with the brown cylinder\nC. The cube and the purple cylinder collide\nD. The cube collides with the cyan cylinder\nAnswer with the option's letter from the given choices directly.", + "gt_response": "the cube collides with the brown cylinder", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "b", + "choice_list": [ + "the cyan object collides with the purple object", + "the cube collides with the brown cylinder", + "the cube and the purple cylinder collide", + "the cube collides with the cyan cylinder" + ], + "image_quantity_level": "Few", + "image": [ + "video_12359/video_12359_0.jpeg", + "video_12359/video_12359_1.jpeg", + "video_12359/video_12359_2.jpeg", + "video_12359/video_12359_3.jpeg", + "video_12359/video_12359_4.jpeg" + ], + "extracted": "B", + "result": 1 + }, + { + "sample_id": 151, + "question": "Assess the pictures and answer the connected query about counterfactual inference. You must choose your answer from the Choice List.\nWithout the green object, which of the following will happen?\nChoice list: \nA. The rubber cylinder collides with the sphere\nB. The rubber cylinder collides with the cube\nC. The sphere collides with the yellow metal object\nD. The cube collides with the yellow metal object\nAnswer with the option's letter from the given choices directly.", + "gt_response": "the rubber cylinder collides with the sphere", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "b", + "choice_list": [ + "the rubber cylinder collides with the sphere", + "the rubber cylinder collides with the cube", + "the sphere collides with the yellow metal object", + "the cube collides with the yellow metal object" + ], + "image_quantity_level": "Few", + "image": [ + "video_13240/video_13240_0.jpeg", + "video_13240/video_13240_1.jpeg", + "video_13240/video_13240_2.jpeg", + "video_13240/video_13240_3.jpeg", + "video_13240/video_13240_4.jpeg" + ], + "extracted": "B", + "result": 0 + }, + { + "sample_id": 152, + "question": "Evaluate the images presented and provide an answer to the associated question about counterfactual inference. You must choose your answer from the Choice List.\nWhich of the following will happen without the cylinder?\nChoice list: \nA. The purple object collides with the gray sphere\nB. The red object collides with the gray object\nAnswer with the option's letter from the given choices directly.", + "gt_response": "the purple object collides with the gray sphere", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "b", + "choice_list": [ + "the purple object collides with the gray sphere", + "the red object collides with the gray object" + ], + "image_quantity_level": "Few", + "image": [ + "video_12007/video_12007_0.jpeg", + "video_12007/video_12007_1.jpeg", + "video_12007/video_12007_2.jpeg", + "video_12007/video_12007_3.jpeg", + "video_12007/video_12007_4.jpeg" + ], + "extracted": "B", + "result": 0 + }, + { + "sample_id": 153, + "question": "Review the photos and answer the associated inquiry about counterfactual inference. You must choose your answer from the Choice List.\nIf the cube is removed, which event will happen?\nChoice list: \nA. The gray cylinder collides with the sphere\nB. The gray cylinder collides with the green object\nAnswer with the option's letter from the given choices directly.", + "gt_response": "the gray cylinder collides with the sphere", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "a", + "choice_list": [ + "the gray cylinder collides with the sphere", + "the gray cylinder collides with the green object" + ], + "image_quantity_level": "Few", + "image": [ + "video_12316/video_12316_0.jpeg", + "video_12316/video_12316_1.jpeg", + "video_12316/video_12316_2.jpeg", + "video_12316/video_12316_3.jpeg", + "video_12316/video_12316_4.jpeg" + ], + "extracted": "A", + "result": 1 + }, + { + "sample_id": 154, + "question": "Observe the images and reply to the pertinent question about counterfactual inference. You must choose your answer from the Choice List.\nWithout the blue object, which of the following will happen?\nChoice list: \nA. The rubber cylinder collides with the yellow object\nB. The brown cylinder collides with the brown cube\nC. The gray cylinder and the brown cube collide\nD. The yellow cube collides with the brown cube\nAnswer with the option's letter from the given choices directly.", + "gt_response": "the rubber cylinder collides with the yellow object", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "a", + "choice_list": [ + "the rubber cylinder collides with the yellow object", + "the brown cylinder collides with the brown cube", + "the gray cylinder and the brown cube collide", + "the yellow cube collides with the brown cube" + ], + "image_quantity_level": "Few", + "image": [ + "video_11449/video_11449_0.jpeg", + "video_11449/video_11449_1.jpeg", + "video_11449/video_11449_2.jpeg", + "video_11449/video_11449_3.jpeg", + "video_11449/video_11449_4.jpeg" + ], + "extracted": "A", + "result": 1 + }, + { + "sample_id": 155, + "question": "Inspect the given graphics and answer the relevant query about counterfactual inference. You must choose your answer from the Choice List.\nIf the cube is removed, which of the following will happen?\nChoice list: \nA. The sphere collides with the yellow cylinder\nB. The purple cylinder and the sphere collide\nAnswer with the option's letter from the given choices directly.", + "gt_response": "the purple cylinder and the sphere collide", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "b", + "choice_list": [ + "the sphere collides with the yellow cylinder", + "the purple cylinder and the sphere collide" + ], + "image_quantity_level": "Few", + "image": [ + "video_10951/video_10951_0.jpeg", + "video_10951/video_10951_1.jpeg", + "video_10951/video_10951_2.jpeg", + "video_10951/video_10951_3.jpeg", + "video_10951/video_10951_4.jpeg" + ], + "extracted": "B", + "result": 1 + }, + { + "sample_id": 156, + "question": "Inspect the given graphics and answer the relevant query about counterfactual inference. You must choose your answer from the Choice List.\nWhich of the following will happen without the cube?\nChoice list: \nA. The red cylinder and the sphere collide\nB. The red cylinder collides with the blue cylinder\nAnswer with the option's letter from the given choices directly.", + "gt_response": "the red cylinder collides with the blue cylinder", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "b", + "choice_list": [ + "the red cylinder and the sphere collide", + "the red cylinder collides with the blue cylinder" + ], + "image_quantity_level": "Few", + "image": [ + "video_10386/video_10386_0.jpeg", + "video_10386/video_10386_1.jpeg", + "video_10386/video_10386_2.jpeg", + "video_10386/video_10386_3.jpeg", + "video_10386/video_10386_4.jpeg" + ], + "extracted": "B", + "result": 1 + }, + { + "sample_id": 157, + "question": "Based on the provided images, answer the question related to the counterfactual inference. You must choose your answer from the Choice List.\nWhat will happen without the brown object?\nChoice list: \nA. The red object collides with the sphere\nB. The red cube collides with the cylinder\nC. The sphere and the cylinder collide\nAnswer with the option's letter from the given choices directly.", + "gt_response": "the red object collides with the sphere", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "a", + "choice_list": [ + "the red object collides with the sphere", + "the red cube collides with the cylinder", + "the sphere and the cylinder collide" + ], + "image_quantity_level": "Few", + "image": [ + "video_12154/video_12154_0.jpeg", + "video_12154/video_12154_1.jpeg", + "video_12154/video_12154_2.jpeg", + "video_12154/video_12154_3.jpeg", + "video_12154/video_12154_4.jpeg" + ], + "extracted": "A", + "result": 1 + }, + { + "sample_id": 158, + "question": "Examine the images and answer the relevant query about counterfactual inference. You must choose your answer from the Choice List.\nWhat will happen without the green object?\nChoice list: \nA. The brown object collides with the yellow sphere\nB. The red object and the yellow sphere collide\nC. The red cylinder and the cyan cylinder collide\nD. The yellow sphere and the cyan cylinder collide\nAnswer with the option's letter from the given choices directly.", + "gt_response": "the red object and the yellow sphere collide", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "a", + "choice_list": [ + "the brown object collides with the yellow sphere", + "the red object and the yellow sphere collide", + "the red cylinder and the cyan cylinder collide", + "the yellow sphere and the cyan cylinder collide" + ], + "image_quantity_level": "Few", + "image": [ + "video_14928/video_14928_0.jpeg", + "video_14928/video_14928_1.jpeg", + "video_14928/video_14928_2.jpeg", + "video_14928/video_14928_3.jpeg", + "video_14928/video_14928_4.jpeg" + ], + "extracted": "A", + "result": 0 + }, + { + "sample_id": 159, + "question": "Review the photos and answer the associated inquiry about counterfactual inference. You must choose your answer from the Choice List.\nWhich of the following will happen if the yellow object is removed?\nChoice list: \nA. The cylinder collides with the cube\nB. The cylinder and the gray sphere collide\nC. The gray sphere collides with the red sphere\nD. The gray object collides with the cube\nAnswer with the option's letter from the given choices directly.", + "gt_response": "the cylinder and the gray sphere collide", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "c", + "choice_list": [ + "the cylinder collides with the cube", + "the cylinder and the gray sphere collide", + "the gray sphere collides with the red sphere", + "the gray object collides with the cube" + ], + "image_quantity_level": "Few", + "image": [ + "video_12605/video_12605_0.jpeg", + "video_12605/video_12605_1.jpeg", + "video_12605/video_12605_2.jpeg", + "video_12605/video_12605_3.jpeg", + "video_12605/video_12605_4.jpeg" + ], + "extracted": "C", + "result": 0 + }, + { + "sample_id": 160, + "question": "Observe the images and reply to the pertinent question about counterfactual inference. You must choose your answer from the Choice List.\nWhich of the following will happen without the sphere?\nChoice list: \nA. The blue object collides with the purple cylinder\nB. The blue object and the green object collide\nAnswer with the option's letter from the given choices directly.", + "gt_response": "the blue object collides with the purple cylinder", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "b", + "choice_list": [ + "the blue object collides with the purple cylinder", + "the blue object and the green object collide" + ], + "image_quantity_level": "Few", + "image": [ + "video_12670/video_12670_0.jpeg", + "video_12670/video_12670_1.jpeg", + "video_12670/video_12670_2.jpeg", + "video_12670/video_12670_3.jpeg", + "video_12670/video_12670_4.jpeg" + ], + "extracted": "B", + "result": 0 + }, + { + "sample_id": 161, + "question": "Based on the provided images, answer the question related to the counterfactual inference. You must choose your answer from the Choice List.\nWhat will happen if the yellow object is removed?\nChoice list: \nA. The rubber cube collides with the gray object\nB. The sphere and the red cylinder collide\nC. The brown cube collides with the red cylinder\nD. The metal cube and the red object collide\nAnswer with the option's letter from the given choices directly.", + "gt_response": "the rubber cube collides with the gray object", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "a", + "choice_list": [ + "the rubber cube collides with the gray object", + "the sphere and the red cylinder collide", + "the brown cube collides with the red cylinder", + "the metal cube and the red object collide" + ], + "image_quantity_level": "Few", + "image": [ + "video_13877/video_13877_0.jpeg", + "video_13877/video_13877_1.jpeg", + "video_13877/video_13877_2.jpeg", + "video_13877/video_13877_3.jpeg", + "video_13877/video_13877_4.jpeg" + ], + "extracted": "A", + "result": 1 + }, + { + "sample_id": 162, + "question": "Based on the provided images, answer the question related to the counterfactual inference. You must choose your answer from the Choice List.\nIf the sphere is removed, which of the following will happen?\nChoice list: \nA. The cyan cylinder collides with the yellow cylinder\nAnswer with the option's letter from the given choices directly.", + "gt_response": "the cyan cylinder collides with the yellow cylinder", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "a", + "choice_list": [ + "the cyan cylinder collides with the yellow cylinder" + ], + "image_quantity_level": "Few", + "image": [ + "video_13455/video_13455_0.jpeg", + "video_13455/video_13455_1.jpeg", + "video_13455/video_13455_2.jpeg", + "video_13455/video_13455_3.jpeg", + "video_13455/video_13455_4.jpeg" + ], + "extracted": "A", + "result": 1 + }, + { + "sample_id": 163, + "question": "Analyze the given pictures and answer the related inquiry about counterfactual inference. You must choose your answer from the Choice List.\nWhich of the following will happen without the red object?\nChoice list: \nA. The cube and the gray object collide\nB. The cylinder collides with the gray object\nC. The cylinder collides with the purple sphere\nD. The cube and the purple sphere collide\nAnswer with the option's letter from the given choices directly.", + "gt_response": "the cylinder collides with the gray object", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "c", + "choice_list": [ + "the cube and the gray object collide", + "the cylinder collides with the gray object", + "the cylinder collides with the purple sphere", + "the cube and the purple sphere collide" + ], + "image_quantity_level": "Few", + "image": [ + "video_11626/video_11626_0.jpeg", + "video_11626/video_11626_1.jpeg", + "video_11626/video_11626_2.jpeg", + "video_11626/video_11626_3.jpeg", + "video_11626/video_11626_4.jpeg" + ], + "extracted": "C", + "result": 0 + }, + { + "sample_id": 164, + "question": "Examine the images and answer the relevant query about counterfactual inference. You must choose your answer from the Choice List.\nWhat will happen without the cube?\nChoice list: \nA. The green object and the metal cylinder collide\nB. The red cylinder collides with the metal cylinder\nC. The green sphere and the cyan sphere collide\nD. The cyan sphere collides with the cyan rubber cylinder\nAnswer with the option's letter from the given choices directly.", + "gt_response": "the green object and the metal cylinder collide", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "a", + "choice_list": [ + "the green object and the metal cylinder collide", + "the red cylinder collides with the metal cylinder", + "the green sphere and the cyan sphere collide", + "the cyan sphere collides with the cyan rubber cylinder" + ], + "image_quantity_level": "Few", + "image": [ + "video_10644/video_10644_0.jpeg", + "video_10644/video_10644_1.jpeg", + "video_10644/video_10644_2.jpeg", + "video_10644/video_10644_3.jpeg", + "video_10644/video_10644_4.jpeg" + ], + "extracted": "A", + "result": 1 + }, + { + "sample_id": 165, + "question": "Analyze the given pictures and answer the related inquiry about counterfactual inference. You must choose your answer from the Choice List.\nIf the sphere is removed, which event will happen?\nChoice list: \nA. The cube collides with the yellow metal cylinder\nB. The yellow metal object collides with the cyan cylinder\nC. The cube collides with the green cylinder\nD. The cube and the yellow rubber cylinder collide\nAnswer with the option's letter from the given choices directly.", + "gt_response": "the cube collides with the green cylinder", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "a", + "choice_list": [ + "the cube collides with the yellow metal cylinder", + "the yellow metal object collides with the cyan cylinder", + "the cube collides with the green cylinder", + "the cube and the yellow rubber cylinder collide" + ], + "image_quantity_level": "Few", + "image": [ + "video_10461/video_10461_0.jpeg", + "video_10461/video_10461_1.jpeg", + "video_10461/video_10461_2.jpeg", + "video_10461/video_10461_3.jpeg", + "video_10461/video_10461_4.jpeg" + ], + "extracted": "A", + "result": 0 + }, + { + "sample_id": 166, + "question": "Assess the pictures and answer the connected query about counterfactual inference. You must choose your answer from the Choice List.\nWhich of the following will happen if the purple object is removed?\nChoice list: \nA. The cyan object and the gray cylinder collide\nB. The rubber sphere and the cyan object collide\nC. The cyan object collides with the brown metal sphere\nD. The gray cylinder collides with the brown cylinder\nAnswer with the option's letter from the given choices directly.", + "gt_response": "the rubber sphere and the cyan object collide", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "c", + "choice_list": [ + "the cyan object and the gray cylinder collide", + "the rubber sphere and the cyan object collide", + "the cyan object collides with the brown metal sphere", + "the gray cylinder collides with the brown cylinder" + ], + "image_quantity_level": "Few", + "image": [ + "video_14226/video_14226_0.jpeg", + "video_14226/video_14226_1.jpeg", + "video_14226/video_14226_2.jpeg", + "video_14226/video_14226_3.jpeg", + "video_14226/video_14226_4.jpeg" + ], + "extracted": "C", + "result": 0 + }, + { + "sample_id": 167, + "question": "Investigate the images and reply to the corresponding inquiry about counterfactual inference. You must choose your answer from the Choice List.\nWhich event will happen without the cube?\nChoice list: \nA. The gray cylinder and the brown object collide\nB. The metal cylinder collides with the yellow object\nC. The gray cylinder collides with the yellow cylinder\nAnswer with the option's letter from the given choices directly.", + "gt_response": "the gray cylinder collides with the yellow cylinder", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "c", + "choice_list": [ + "the gray cylinder and the brown object collide", + "the metal cylinder collides with the yellow object", + "the gray cylinder collides with the yellow cylinder" + ], + "image_quantity_level": "Few", + "image": [ + "video_14754/video_14754_0.jpeg", + "video_14754/video_14754_1.jpeg", + "video_14754/video_14754_2.jpeg", + "video_14754/video_14754_3.jpeg", + "video_14754/video_14754_4.jpeg" + ], + "extracted": "C", + "result": 1 + }, + { + "sample_id": 168, + "question": "Review the photos and answer the associated inquiry about counterfactual inference. You must choose your answer from the Choice List.\nWhat will happen if the cyan object is removed?\nChoice list: \nA. The gray sphere and the blue cylinder collide\nB. The gray sphere collides with the gray cylinder\nC. The rubber cylinder collides with the metal cylinder\nD. The cube and the metal cylinder collide\nAnswer with the option's letter from the given choices directly.", + "gt_response": "the gray sphere collides with the gray cylinder", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "b", + "choice_list": [ + "the gray sphere and the blue cylinder collide", + "the gray sphere collides with the gray cylinder", + "the rubber cylinder collides with the metal cylinder", + "the cube and the metal cylinder collide" + ], + "image_quantity_level": "Few", + "image": [ + "video_13615/video_13615_0.jpeg", + "video_13615/video_13615_1.jpeg", + "video_13615/video_13615_2.jpeg", + "video_13615/video_13615_3.jpeg", + "video_13615/video_13615_4.jpeg" + ], + "extracted": "B", + "result": 1 + }, + { + "sample_id": 169, + "question": "Assess the pictures and answer the connected query about counterfactual inference. You must choose your answer from the Choice List.\nWhich of the following will happen if the yellow object is removed?\nChoice list: \nA. The sphere and the gray cube collide\nB. The sphere collides with the blue object\nC. The gray cube and the metal object collide\nAnswer with the option's letter from the given choices directly.", + "gt_response": "the sphere and the gray cube collide", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "a", + "choice_list": [ + "the sphere and the gray cube collide", + "the sphere collides with the blue object", + "the gray cube and the metal object collide" + ], + "image_quantity_level": "Few", + "image": [ + "video_10442/video_10442_0.jpeg", + "video_10442/video_10442_1.jpeg", + "video_10442/video_10442_2.jpeg", + "video_10442/video_10442_3.jpeg", + "video_10442/video_10442_4.jpeg" + ], + "extracted": "A", + "result": 1 + }, + { + "sample_id": 170, + "question": "Investigate the images and reply to the corresponding inquiry about counterfactual inference. You must choose your answer from the Choice List.\nWhich event will happen if the cube is removed?\nChoice list: \nA. The cyan object and the red cylinder collide\nB. The red object and the brown cylinder collide\nC. The red cylinder collides with the brown sphere\nD. The gray sphere and the brown cylinder collide\nAnswer with the option's letter from the given choices directly.", + "gt_response": "the cyan object and the red cylinder collide", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "a", + "choice_list": [ + "the cyan object and the red cylinder collide", + "the red object and the brown cylinder collide", + "the red cylinder collides with the brown sphere", + "the gray sphere and the brown cylinder collide" + ], + "image_quantity_level": "Few", + "image": [ + "video_12561/video_12561_0.jpeg", + "video_12561/video_12561_1.jpeg", + "video_12561/video_12561_2.jpeg", + "video_12561/video_12561_3.jpeg", + "video_12561/video_12561_4.jpeg" + ], + "extracted": "A", + "result": 1 + }, + { + "sample_id": 171, + "question": "Inspect the given graphics and answer the relevant query about counterfactual inference. You must choose your answer from the Choice List.\nWhich event will happen if the sphere is removed?\nChoice list: \nA. The blue cylinder collides with the rubber cube\nB. The gray cube collides with the metal cube\nC. The blue cylinder and the metal object collide\nD. The gray cube collides with the purple cylinder\nAnswer with the option's letter from the given choices directly.", + "gt_response": "the blue cylinder collides with the rubber cube", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "a", + "choice_list": [ + "the blue cylinder collides with the rubber cube", + "the gray cube collides with the metal cube", + "the blue cylinder and the metal object collide", + "the gray cube collides with the purple cylinder" + ], + "image_quantity_level": "Few", + "image": [ + "video_14094/video_14094_0.jpeg", + "video_14094/video_14094_1.jpeg", + "video_14094/video_14094_2.jpeg", + "video_14094/video_14094_3.jpeg", + "video_14094/video_14094_4.jpeg" + ], + "extracted": "A", + "result": 1 + }, + { + "sample_id": 172, + "question": "Based on the provided images, answer the question related to the counterfactual inference. You must choose your answer from the Choice List.\nIf the sphere is removed, what will happen?\nChoice list: \nA. The cube and the gray cylinder collide\nB. The blue metal object and the gray object collide\nC. The gray cylinder collides with the brown cylinder\nD. The gray cylinder and the blue rubber cylinder collide\nAnswer with the option's letter from the given choices directly.", + "gt_response": "the cube and the gray cylinder collide", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "a", + "choice_list": [ + "the cube and the gray cylinder collide", + "the blue metal object and the gray object collide", + "the gray cylinder collides with the brown cylinder", + "the gray cylinder and the blue rubber cylinder collide" + ], + "image_quantity_level": "Few", + "image": [ + "video_10553/video_10553_0.jpeg", + "video_10553/video_10553_1.jpeg", + "video_10553/video_10553_2.jpeg", + "video_10553/video_10553_3.jpeg", + "video_10553/video_10553_4.jpeg" + ], + "extracted": "A", + "result": 1 + }, + { + "sample_id": 173, + "question": "Evaluate the images presented and provide an answer to the associated question about counterfactual inference. You must choose your answer from the Choice List.\nIf the cyan object is removed, which event will happen?\nChoice list: \nA. The brown sphere and the brown cylinder collide\nB. The gray cylinder collides with the brown cylinder\nC. The blue object collides with the gray cylinder\nD. The brown sphere and the gray cylinder collide\nAnswer with the option's letter from the given choices directly.", + "gt_response": "the brown sphere and the gray cylinder collide", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "d", + "choice_list": [ + "the brown sphere and the brown cylinder collide", + "the gray cylinder collides with the brown cylinder", + "the blue object collides with the gray cylinder", + "the brown sphere and the gray cylinder collide" + ], + "image_quantity_level": "Few", + "image": [ + "video_10339/video_10339_0.jpeg", + "video_10339/video_10339_1.jpeg", + "video_10339/video_10339_2.jpeg", + "video_10339/video_10339_3.jpeg", + "video_10339/video_10339_4.jpeg" + ], + "extracted": "D", + "result": 1 + }, + { + "sample_id": 174, + "question": "Scrutinize the visuals and respond to the related question about counterfactual inference. You must choose your answer from the Choice List.\nIf the red object is removed, what will happen?\nChoice list: \nA. The brown cylinder and the metal cube collide\nB. The purple object and the yellow cylinder collide\nC. The yellow cube and the yellow cylinder collide\nD. The blue cube and the brown object collide\nAnswer with the option's letter from the given choices directly.", + "gt_response": "the blue cube and the brown object collide", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "d", + "choice_list": [ + "the brown cylinder and the metal cube collide", + "the purple object and the yellow cylinder collide", + "the yellow cube and the yellow cylinder collide", + "the blue cube and the brown object collide" + ], + "image_quantity_level": "Few", + "image": [ + "video_13290/video_13290_0.jpeg", + "video_13290/video_13290_1.jpeg", + "video_13290/video_13290_2.jpeg", + "video_13290/video_13290_3.jpeg", + "video_13290/video_13290_4.jpeg" + ], + "extracted": "D", + "result": 1 + }, + { + "sample_id": 175, + "question": "Based on the provided images, answer the question related to the counterfactual inference. You must choose your answer from the Choice List.\nWithout the cube, which of the following will happen?\nChoice list: \nA. The gray object collides with the metal sphere\nB. The gray object collides with the cylinder\nC. The red sphere collides with the cylinder\nAnswer with the option's letter from the given choices directly.", + "gt_response": "the gray object collides with the metal sphere", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "b", + "choice_list": [ + "the gray object collides with the metal sphere", + "the gray object collides with the cylinder", + "the red sphere collides with the cylinder" + ], + "image_quantity_level": "Few", + "image": [ + "video_12788/video_12788_0.jpeg", + "video_12788/video_12788_1.jpeg", + "video_12788/video_12788_2.jpeg", + "video_12788/video_12788_3.jpeg", + "video_12788/video_12788_4.jpeg" + ], + "extracted": "B", + "result": 0 + }, + { + "sample_id": 176, + "question": "Investigate the images and reply to the corresponding inquiry about counterfactual inference. You must choose your answer from the Choice List.\nWhat will happen without the cyan object?\nChoice list: \nA. The metal cylinder collides with the purple sphere\nB. The blue object and the purple object collide\nC. The rubber cylinder collides with the green object\nAnswer with the option's letter from the given choices directly.", + "gt_response": "the metal cylinder collides with the purple sphere", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "c", + "choice_list": [ + "the metal cylinder collides with the purple sphere", + "the blue object and the purple object collide", + "the rubber cylinder collides with the green object" + ], + "image_quantity_level": "Few", + "image": [ + "video_13072/video_13072_0.jpeg", + "video_13072/video_13072_1.jpeg", + "video_13072/video_13072_2.jpeg", + "video_13072/video_13072_3.jpeg", + "video_13072/video_13072_4.jpeg" + ], + "extracted": "C", + "result": 0 + }, + { + "sample_id": 177, + "question": "Investigate the images and reply to the corresponding inquiry about counterfactual inference. You must choose your answer from the Choice List.\nIf the brown object is removed, which of the following will happen?\nChoice list: \nA. The red object and the cyan sphere collide\nB. The red object collides with the cylinder\nAnswer with the option's letter from the given choices directly.", + "gt_response": "the red object and the cyan sphere collide", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "b", + "choice_list": [ + "the red object and the cyan sphere collide", + "the red object collides with the cylinder" + ], + "image_quantity_level": "Few", + "image": [ + "video_13294/video_13294_0.jpeg", + "video_13294/video_13294_1.jpeg", + "video_13294/video_13294_2.jpeg", + "video_13294/video_13294_3.jpeg", + "video_13294/video_13294_4.jpeg" + ], + "extracted": "B", + "result": 0 + }, + { + "sample_id": 178, + "question": "Examine the images and answer the relevant query about counterfactual inference. You must choose your answer from the Choice List.\nWhich of the following will happen if the cylinder is removed?\nChoice list: \nA. The blue cube collides with the red rubber object\nB. The metal cube and the red rubber cube collide\nC. The sphere collides with the blue cube\nD. The sphere collides with the metal cube\nAnswer with the option's letter from the given choices directly.", + "gt_response": "the sphere collides with the metal cube", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "d", + "choice_list": [ + "the blue cube collides with the red rubber object", + "the metal cube and the red rubber cube collide", + "the sphere collides with the blue cube", + "the sphere collides with the metal cube" + ], + "image_quantity_level": "Few", + "image": [ + "video_13868/video_13868_0.jpeg", + "video_13868/video_13868_1.jpeg", + "video_13868/video_13868_2.jpeg", + "video_13868/video_13868_3.jpeg", + "video_13868/video_13868_4.jpeg" + ], + "extracted": "D", + "result": 1 + }, + { + "sample_id": 179, + "question": "Evaluate the images presented and provide an answer to the associated question about counterfactual inference. You must choose your answer from the Choice List.\nWithout the sphere, what will happen?\nChoice list: \nA. The metal object collides with the rubber cube\nB. The cylinder and the metal object collide\nC. The cylinder and the rubber cube collide\nAnswer with the option's letter from the given choices directly.", + "gt_response": "the cylinder and the metal object collide", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "b", + "choice_list": [ + "the metal object collides with the rubber cube", + "the cylinder and the metal object collide", + "the cylinder and the rubber cube collide" + ], + "image_quantity_level": "Few", + "image": [ + "video_14570/video_14570_0.jpeg", + "video_14570/video_14570_1.jpeg", + "video_14570/video_14570_2.jpeg", + "video_14570/video_14570_3.jpeg", + "video_14570/video_14570_4.jpeg" + ], + "extracted": "B", + "result": 1 + }, + { + "sample_id": 180, + "question": "Evaluate the images presented and provide an answer to the associated question about counterfactual inference. You must choose your answer from the Choice List.\nWithout the sphere, which event will happen?\nChoice list: \nA. The metal object collides with the gray object\nB. The rubber cube collides with the cylinder\nAnswer with the option's letter from the given choices directly.", + "gt_response": "the metal object collides with the gray object", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "a", + "choice_list": [ + "the metal object collides with the gray object", + "the rubber cube collides with the cylinder" + ], + "image_quantity_level": "Few", + "image": [ + "video_11842/video_11842_0.jpeg", + "video_11842/video_11842_1.jpeg", + "video_11842/video_11842_2.jpeg", + "video_11842/video_11842_3.jpeg", + "video_11842/video_11842_4.jpeg" + ], + "extracted": "A", + "result": 1 + }, + { + "sample_id": 181, + "question": "Evaluate the images presented and provide an answer to the associated question about counterfactual inference. You must choose your answer from the Choice List.\nIf the sphere is removed, what will happen?\nChoice list: \nA. The purple cube collides with the gray cylinder\nB. The yellow object collides with the red cube\nC. The yellow cylinder collides with the gray object\nD. The purple cube and the red object collide\nAnswer with the option's letter from the given choices directly.", + "gt_response": "the yellow cylinder collides with the gray object", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "c", + "choice_list": [ + "the purple cube collides with the gray cylinder", + "the yellow object collides with the red cube", + "the yellow cylinder collides with the gray object", + "the purple cube and the red object collide" + ], + "image_quantity_level": "Few", + "image": [ + "video_12551/video_12551_0.jpeg", + "video_12551/video_12551_1.jpeg", + "video_12551/video_12551_2.jpeg", + "video_12551/video_12551_3.jpeg", + "video_12551/video_12551_4.jpeg" + ], + "extracted": "C", + "result": 1 + }, + { + "sample_id": 182, + "question": "Scrutinize the visuals and respond to the related question about counterfactual inference. You must choose your answer from the Choice List.\nWhich of the following will happen without the yellow object?\nChoice list: \nA. The red sphere collides with the brown cylinder\nB. The cyan sphere collides with the brown object\nC. The red object collides with the blue cylinder\nD. The brown cylinder and the cube collide\nAnswer with the option's letter from the given choices directly.", + "gt_response": "the red sphere collides with the brown cylinder", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "a", + "choice_list": [ + "the red sphere collides with the brown cylinder", + "the cyan sphere collides with the brown object", + "the red object collides with the blue cylinder", + "the brown cylinder and the cube collide" + ], + "image_quantity_level": "Few", + "image": [ + "video_10929/video_10929_0.jpeg", + "video_10929/video_10929_1.jpeg", + "video_10929/video_10929_2.jpeg", + "video_10929/video_10929_3.jpeg", + "video_10929/video_10929_4.jpeg" + ], + "extracted": "A", + "result": 1 + }, + { + "sample_id": 183, + "question": "Based on the provided images, answer the question related to the counterfactual inference. You must choose your answer from the Choice List.\nWithout the purple object, which of the following will happen?\nChoice list: \nA. The metal cube and the metal sphere collide\nB. The metal sphere and the green cube collide\nC. The blue sphere collides with the metal cube\nD. The blue sphere collides with the metal sphere\nAnswer with the option's letter from the given choices directly.", + "gt_response": "the blue sphere collides with the metal cube", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "d", + "choice_list": [ + "the metal cube and the metal sphere collide", + "the metal sphere and the green cube collide", + "the blue sphere collides with the metal cube", + "the blue sphere collides with the metal sphere" + ], + "image_quantity_level": "Few", + "image": [ + "video_13117/video_13117_0.jpeg", + "video_13117/video_13117_1.jpeg", + "video_13117/video_13117_2.jpeg", + "video_13117/video_13117_3.jpeg", + "video_13117/video_13117_4.jpeg" + ], + "extracted": "D", + "result": 0 + }, + { + "sample_id": 184, + "question": "Analyze the given pictures and answer the related inquiry about counterfactual inference. You must choose your answer from the Choice List.\nWhich of the following will happen if the brown object is removed?\nChoice list: \nA. The yellow sphere and the green sphere collide\nB. The red sphere collides with the green sphere\nC. The red object collides with the cyan sphere\nD. The yellow sphere and the cyan sphere collide\nAnswer with the option's letter from the given choices directly.", + "gt_response": "the yellow sphere and the green sphere collide", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "a", + "choice_list": [ + "the yellow sphere and the green sphere collide", + "the red sphere collides with the green sphere", + "the red object collides with the cyan sphere", + "the yellow sphere and the cyan sphere collide" + ], + "image_quantity_level": "Few", + "image": [ + "video_11045/video_11045_0.jpeg", + "video_11045/video_11045_1.jpeg", + "video_11045/video_11045_2.jpeg", + "video_11045/video_11045_3.jpeg", + "video_11045/video_11045_4.jpeg" + ], + "extracted": "A", + "result": 1 + }, + { + "sample_id": 185, + "question": "Inspect the given graphics and answer the relevant query about counterfactual inference. You must choose your answer from the Choice List.\nWithout the cyan object, what will happen?\nChoice list: \nA. The sphere and the cylinder collide\nB. The yellow cube collides with the cylinder\nAnswer with the option's letter from the given choices directly.", + "gt_response": "the yellow cube collides with the cylinder", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "b", + "choice_list": [ + "the sphere and the cylinder collide", + "the yellow cube collides with the cylinder" + ], + "image_quantity_level": "Few", + "image": [ + "video_11939/video_11939_0.jpeg", + "video_11939/video_11939_1.jpeg", + "video_11939/video_11939_2.jpeg", + "video_11939/video_11939_3.jpeg", + "video_11939/video_11939_4.jpeg" + ], + "extracted": "B", + "result": 1 + }, + { + "sample_id": 186, + "question": "Examine the images and answer the relevant query about counterfactual inference. You must choose your answer from the Choice List.\nWithout the red object, what will happen?\nChoice list: \nA. The yellow cylinder collides with the yellow cube\nB. The blue cylinder collides with the sphere\nC. The metal cylinder and the blue object collide\nD. The metal cylinder collides with the sphere\nAnswer with the option's letter from the given choices directly.", + "gt_response": "the yellow cylinder collides with the yellow cube", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "c", + "choice_list": [ + "the yellow cylinder collides with the yellow cube", + "the blue cylinder collides with the sphere", + "the metal cylinder and the blue object collide", + "the metal cylinder collides with the sphere" + ], + "image_quantity_level": "Few", + "image": [ + "video_12721/video_12721_0.jpeg", + "video_12721/video_12721_1.jpeg", + "video_12721/video_12721_2.jpeg", + "video_12721/video_12721_3.jpeg", + "video_12721/video_12721_4.jpeg" + ], + "extracted": "C", + "result": 0 + }, + { + "sample_id": 187, + "question": "Scrutinize the visuals and respond to the related question about counterfactual inference. You must choose your answer from the Choice List.\nIf the metal object is removed, which of the following will happen?\nChoice list: \nA. The gray cube and the sphere collide\nB. The gray object collides with the cyan cube\nC. The sphere collides with the cyan object\nD. The gray cube and the red cylinder collide\nAnswer with the option's letter from the given choices directly.", + "gt_response": "the gray cube and the sphere collide", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "a", + "choice_list": [ + "the gray cube and the sphere collide", + "the gray object collides with the cyan cube", + "the sphere collides with the cyan object", + "the gray cube and the red cylinder collide" + ], + "image_quantity_level": "Few", + "image": [ + "video_10454/video_10454_0.jpeg", + "video_10454/video_10454_1.jpeg", + "video_10454/video_10454_2.jpeg", + "video_10454/video_10454_3.jpeg", + "video_10454/video_10454_4.jpeg" + ], + "extracted": "A", + "result": 1 + }, + { + "sample_id": 188, + "question": "Analyze the given pictures and answer the related inquiry about counterfactual inference. You must choose your answer from the Choice List.\nWithout the purple object, which event will happen?\nChoice list: \nA. The cube and the brown cylinder collide\nB. The rubber cylinder collides with the metal cylinder\nC. The brown cylinder and the yellow object collide\nD. The green cylinder collides with the yellow sphere\nAnswer with the option's letter from the given choices directly.", + "gt_response": "the cube and the brown cylinder collide", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "a", + "choice_list": [ + "the cube and the brown cylinder collide", + "the rubber cylinder collides with the metal cylinder", + "the brown cylinder and the yellow object collide", + "the green cylinder collides with the yellow sphere" + ], + "image_quantity_level": "Few", + "image": [ + "video_11896/video_11896_0.jpeg", + "video_11896/video_11896_1.jpeg", + "video_11896/video_11896_2.jpeg", + "video_11896/video_11896_3.jpeg", + "video_11896/video_11896_4.jpeg" + ], + "extracted": "A", + "result": 1 + }, + { + "sample_id": 189, + "question": "Based on the provided images, answer the question related to the counterfactual inference. You must choose your answer from the Choice List.\nWhat will happen without the rubber object?\nChoice list: \nA. The cube and the cylinder collide\nB. The brown object collides with the cube\nC. The brown sphere collides with the cylinder\nAnswer with the option's letter from the given choices directly.", + "gt_response": "the brown object collides with the cube", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "b", + "choice_list": [ + "the cube and the cylinder collide", + "the brown object collides with the cube", + "the brown sphere collides with the cylinder" + ], + "image_quantity_level": "Few", + "image": [ + "video_10862/video_10862_0.jpeg", + "video_10862/video_10862_1.jpeg", + "video_10862/video_10862_2.jpeg", + "video_10862/video_10862_3.jpeg", + "video_10862/video_10862_4.jpeg" + ], + "extracted": "B", + "result": 1 + }, + { + "sample_id": 190, + "question": "Examine the images and answer the relevant query about counterfactual inference. You must choose your answer from the Choice List.\nWhich event will happen if the cube is removed?\nChoice list: \nA. The cyan cylinder and the green cylinder collide\nB. The green cylinder collides with the sphere\nC. The cyan cylinder collides with the sphere\nAnswer with the option's letter from the given choices directly.", + "gt_response": "the cyan cylinder and the green cylinder collide", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "c", + "choice_list": [ + "the cyan cylinder and the green cylinder collide", + "the green cylinder collides with the sphere", + "the cyan cylinder collides with the sphere" + ], + "image_quantity_level": "Few", + "image": [ + "video_13761/video_13761_0.jpeg", + "video_13761/video_13761_1.jpeg", + "video_13761/video_13761_2.jpeg", + "video_13761/video_13761_3.jpeg", + "video_13761/video_13761_4.jpeg" + ], + "extracted": "C", + "result": 0 + }, + { + "sample_id": 191, + "question": "Examine the images and answer the relevant query about counterfactual inference. You must choose your answer from the Choice List.\nWithout the green object, which event will happen?\nChoice list: \nA. The yellow sphere collides with the gray cylinder\nB. The cube and the red metal sphere collide\nC. The yellow object and the cube collide\nD. The gray cylinder collides with the cube\nAnswer with the option's letter from the given choices directly.", + "gt_response": "the yellow sphere collides with the gray cylinder", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "a", + "choice_list": [ + "the yellow sphere collides with the gray cylinder", + "the cube and the red metal sphere collide", + "the yellow object and the cube collide", + "the gray cylinder collides with the cube" + ], + "image_quantity_level": "Few", + "image": [ + "video_12717/video_12717_0.jpeg", + "video_12717/video_12717_1.jpeg", + "video_12717/video_12717_2.jpeg", + "video_12717/video_12717_3.jpeg", + "video_12717/video_12717_4.jpeg" + ], + "extracted": "A", + "result": 1 + }, + { + "sample_id": 192, + "question": "Assess the pictures and answer the connected query about counterfactual inference. You must choose your answer from the Choice List.\nWithout the yellow object, what will happen?\nChoice list: \nA. The green object collides with the cylinder\nB. The cyan sphere collides with the cylinder\nC. The gray sphere and the cylinder collide\nD. The cyan sphere and the rubber sphere collide\nAnswer with the option's letter from the given choices directly.", + "gt_response": "the cyan sphere and the rubber sphere collide", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "d", + "choice_list": [ + "the green object collides with the cylinder", + "the cyan sphere collides with the cylinder", + "the gray sphere and the cylinder collide", + "the cyan sphere and the rubber sphere collide" + ], + "image_quantity_level": "Few", + "image": [ + "video_12872/video_12872_0.jpeg", + "video_12872/video_12872_1.jpeg", + "video_12872/video_12872_2.jpeg", + "video_12872/video_12872_3.jpeg", + "video_12872/video_12872_4.jpeg" + ], + "extracted": "D", + "result": 1 + }, + { + "sample_id": 193, + "question": "Observe the images and reply to the pertinent question about counterfactual inference. You must choose your answer from the Choice List.\nIf the cylinder is removed, which of the following will happen?\nChoice list: \nA. The cube collides with the metal sphere\nB. The rubber object collides with the cube\nAnswer with the option's letter from the given choices directly.", + "gt_response": "the rubber object collides with the cube", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "b", + "choice_list": [ + "the cube collides with the metal sphere", + "the rubber object collides with the cube" + ], + "image_quantity_level": "Few", + "image": [ + "video_11222/video_11222_0.jpeg", + "video_11222/video_11222_1.jpeg", + "video_11222/video_11222_2.jpeg", + "video_11222/video_11222_3.jpeg", + "video_11222/video_11222_4.jpeg" + ], + "extracted": "B", + "result": 1 + }, + { + "sample_id": 194, + "question": "Evaluate the images presented and provide an answer to the associated question about counterfactual inference. You must choose your answer from the Choice List.\nIf the cylinder is removed, which event will happen?\nChoice list: \nA. The blue object collides with the cyan object\nB. The cube and the blue object collide\nC. The yellow sphere collides with the cyan sphere\nD. The yellow object collides with the blue object\nAnswer with the option's letter from the given choices directly.", + "gt_response": "the cube and the blue object collide", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "d", + "choice_list": [ + "the blue object collides with the cyan object", + "the cube and the blue object collide", + "the yellow sphere collides with the cyan sphere", + "the yellow object collides with the blue object" + ], + "image_quantity_level": "Few", + "image": [ + "video_13933/video_13933_0.jpeg", + "video_13933/video_13933_1.jpeg", + "video_13933/video_13933_2.jpeg", + "video_13933/video_13933_3.jpeg", + "video_13933/video_13933_4.jpeg" + ], + "extracted": "D", + "result": 0 + }, + { + "sample_id": 195, + "question": "Observe the images and reply to the pertinent question about counterfactual inference. You must choose your answer from the Choice List.\nWhat will happen without the brown object?\nChoice list: \nA. The metal cylinder collides with the cube\nB. The blue object collides with the cube\nC. The red sphere and the cube collide\nD. The green sphere collides with the red sphere\nAnswer with the option's letter from the given choices directly.", + "gt_response": "the green sphere collides with the red sphere", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "d", + "choice_list": [ + "the metal cylinder collides with the cube", + "the blue object collides with the cube", + "the red sphere and the cube collide", + "the green sphere collides with the red sphere" + ], + "image_quantity_level": "Few", + "image": [ + "video_12061/video_12061_0.jpeg", + "video_12061/video_12061_1.jpeg", + "video_12061/video_12061_2.jpeg", + "video_12061/video_12061_3.jpeg", + "video_12061/video_12061_4.jpeg" + ], + "extracted": "D", + "result": 1 + }, + { + "sample_id": 196, + "question": "Examine the images and answer the relevant query about counterfactual inference. You must choose your answer from the Choice List.\nWhich of the following will happen if the cube is removed?\nChoice list: \nA. The brown cylinder and the rubber sphere collide\nB. The brown sphere collides with the purple cylinder\nC. The brown cylinder collides with the rubber cylinder\nD. The metal sphere and the brown sphere collide\nAnswer with the option's letter from the given choices directly.", + "gt_response": "the metal sphere and the brown sphere collide", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "a", + "choice_list": [ + "the brown cylinder and the rubber sphere collide", + "the brown sphere collides with the purple cylinder", + "the brown cylinder collides with the rubber cylinder", + "the metal sphere and the brown sphere collide" + ], + "image_quantity_level": "Few", + "image": [ + "video_10094/video_10094_0.jpeg", + "video_10094/video_10094_1.jpeg", + "video_10094/video_10094_2.jpeg", + "video_10094/video_10094_3.jpeg", + "video_10094/video_10094_4.jpeg" + ], + "extracted": "A", + "result": 0 + }, + { + "sample_id": 197, + "question": "Inspect the given graphics and answer the relevant query about counterfactual inference. You must choose your answer from the Choice List.\nWhich of the following will happen without the sphere?\nChoice list: \nA. The yellow cube collides with the cylinder\nB. The cylinder collides with the cyan object\nC. The metal cube collides with the cyan cube\nD. The blue object collides with the yellow object\nAnswer with the option's letter from the given choices directly.", + "gt_response": "the yellow cube collides with the cylinder", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "d", + "choice_list": [ + "the yellow cube collides with the cylinder", + "the cylinder collides with the cyan object", + "the metal cube collides with the cyan cube", + "the blue object collides with the yellow object" + ], + "image_quantity_level": "Few", + "image": [ + "video_10921/video_10921_0.jpeg", + "video_10921/video_10921_1.jpeg", + "video_10921/video_10921_2.jpeg", + "video_10921/video_10921_3.jpeg", + "video_10921/video_10921_4.jpeg" + ], + "extracted": "D", + "result": 0 + }, + { + "sample_id": 198, + "question": "Evaluate the images presented and provide an answer to the associated question about counterfactual inference. You must choose your answer from the Choice List.\nWithout the cylinder, which of the following will happen?\nChoice list: \nA. The brown object and the purple cube collide\nB. The rubber cube and the purple object collide\nAnswer with the option's letter from the given choices directly.", + "gt_response": "the rubber cube and the purple object collide", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "b", + "choice_list": [ + "the brown object and the purple cube collide", + "the rubber cube and the purple object collide" + ], + "image_quantity_level": "Few", + "image": [ + "video_10111/video_10111_0.jpeg", + "video_10111/video_10111_1.jpeg", + "video_10111/video_10111_2.jpeg", + "video_10111/video_10111_3.jpeg", + "video_10111/video_10111_4.jpeg" + ], + "extracted": "B", + "result": 1 + }, + { + "sample_id": 199, + "question": "Inspect the given graphics and answer the relevant query about counterfactual inference. You must choose your answer from the Choice List.\nWhat will happen without the cube?\nChoice list: \nA. The sphere collides with the metal cylinder\nB. The brown object collides with the metal cylinder\nC. The gray object collides with the sphere\nD. The gray cylinder collides with the brown object\nAnswer with the option's letter from the given choices directly.", + "gt_response": "the gray object collides with the sphere", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "c", + "choice_list": [ + "the sphere collides with the metal cylinder", + "the brown object collides with the metal cylinder", + "the gray object collides with the sphere", + "the gray cylinder collides with the brown object" + ], + "image_quantity_level": "Few", + "image": [ + "video_13424/video_13424_0.jpeg", + "video_13424/video_13424_1.jpeg", + "video_13424/video_13424_2.jpeg", + "video_13424/video_13424_3.jpeg", + "video_13424/video_13424_4.jpeg" + ], + "extracted": "C", + "result": 1 + } +] \ No newline at end of file