|
|
|
from grader import is_equal |
|
import json |
|
import re |
|
|
|
|
|
def get_gold_list(datapath, dataset_name): |
|
|
|
assert dataset_name in ["gsm8k", "math", "minerva_math", "gaokao2023en", "olympiadbench", "collegemath"] |
|
|
|
gold_list = [] |
|
with open(datapath, "r") as f: |
|
for line in f: |
|
item = json.loads(line) |
|
|
|
if dataset_name == "gsm8k": |
|
gold = item['answer'].split("#### ")[-1] |
|
|
|
elif dataset_name == "math": |
|
gold = item['answer'] |
|
|
|
elif dataset_name == "minerva_math": |
|
pattern = r"\\boxed\{((?:[^{}]|\{(?:[^{}]|\{[^{}]*\})*\})*)\}" |
|
pattern_re = re.compile(pattern, re.DOTALL) |
|
solution = item['solution'] |
|
matches = pattern_re.findall(solution) |
|
if len(matches) == 0: |
|
gold = None |
|
else: |
|
gold = matches[-1] |
|
|
|
elif dataset_name == "gaokao2023en": |
|
gold = re.sub(r'^\$(.*)\$$', r'\1', item['answer']) |
|
|
|
elif dataset_name == "olympiadbench": |
|
gold = re.sub(r'^\$(.*)\$$', r'\1', item['final_answer'][0]) |
|
|
|
elif dataset_name == "collegemath": |
|
gold = re.sub(r'^\$(.*)\$$', r'\1', item['answer']) |
|
|
|
gold_list.append(gold) |
|
|
|
return gold_list |
|
|
|
|
|
def get_scores_on_math_benchmarks(model_output_path, test_gold_path, dataset_name): |
|
|
|
gold_list = get_gold_list(test_gold_path, dataset_name) |
|
|
|
"""TODO |
|
Get the output_list from model_output_path |
|
output_list is a list of string (List[str]) |
|
Each string represents the model's response for a corresponding question in the benchmark |
|
Therefore, the length of output_list must match the length of gold_list. |
|
|
|
output_list = ... |
|
""" |
|
|
|
correct = 0 |
|
for output, gold in zip(output_list, gold_list): |
|
if is_equal(output, gold, dataset_name): |
|
correct += 1 |
|
|
|
print("accuracy on %s is %.4f" % (dataset_name, correct / len(gold_list))) |
|
|
|
|
|
if __name__ == "__main__": |
|
"""TODO |
|
Download test benchmarks from Qwen2.5-Math |
|
https://github.com/QwenLM/Qwen2.5-Math/tree/main/evaluation/data |
|
|
|
Prepare model_output_path and test_gold_path for each dataset |
|
""" |
|
|
|
test_gold_path = "PATH_OF_THE_BENCHMARK" |
|
model_output_path = "PATH_OF_YOUR_MODEL_OUTPUTS" |
|
dataset_name = "DATASET_NAME" |
|
|
|
get_scores_on_math_benchmarks(model_output_path, test_gold_path, dataset_name) |
|
|
