File size: 1,268 Bytes
b8c002b 4af0d75 b8c002b 4af0d75 b8c002b 4af0d75 b8c002b 4af0d75 b8c002b 4af0d75 b8c002b 4af0d75 b8c002b 4af0d75 b8c002b |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 |
import argparse
import pandas as pd
#Run ScandEval on model
#scandeval --model-id KennethTM/bert-base-uncased-danish
#Format results as markdown
#python scandeval_summary.py --name scandeval_benchmark_results.jsonl
def scandeval2md(file):
df = pd.read_json(file, lines=True)
df["total"] = [i["total"] for i in df["results"]]
summary = []
for i in df["total"]:
summary_list = []
for k, v in i.items():
summary_string = ""
if "_se" in k:
summary_string += f" (±{round(v, 2)})\n"
else:
summary_string += f'{k.replace("test_", "")} = {round(v, 2)}'
summary_list.append(summary_string)
summary.append("".join(summary_list).strip())
df["summary"] = summary
md_table = df[["task", "dataset", "summary"]].rename(columns={"task": "Task", "dataset": "Dataset", "summary": "Score (±SE)"}).to_markdown(index=False)
print(md_table)
if __name__ == "__main__":
parser = argparse.ArgumentParser(description='Script for formatting ScandEval .jsonl result file as markdown to standard output')
parser.add_argument('--name', help='ScandEval .jsonl result file')
args = parser.parse_args()
scandeval2md(args.name)
|