# Grade a generation jsonl with math_grade_v2; prints accuracy and per-subject/level tables.
import json, sys
from collections import defaultdict
sys.path.insert(0, "."); from math_grade_v2 import correct
def load(p):
    rs = [json.loads(l) for l in open(p, encoding="utf-8")]
    for r in rs: r["ok"] = bool(correct(r["text"], r["gold"]))
    return rs
if __name__ == "__main__":
    rs = [r for r in load(sys.argv[1]) if r["sample"] == 0]
    n = len(rs); c = sum(r["ok"] for r in rs)
    print(f"{c}/{n} = {100*c/n:.1f}%  tokens={sum(r['predicted_n'] or 0 for r in rs)}  capped={sum((r['predicted_n'] or 0)>=2048 for r in rs)}")
    for key in ("subject", "level"):
        d = defaultdict(lambda: [0, 0])
        for r in rs: d[r[key]][0] += r["ok"]; d[r[key]][1] += 1
        for k in sorted(d): print(f"  {key} {k}: {d[k][0]}/{d[k][1]}")