# E18 analysis: paired exact McNemar tests, pre-registered kill criteria (see docs/hypotheses.md, E18 plan).
import json, sys, os
from math import comb
sys.path.insert(0, os.path.join(os.path.dirname(os.path.abspath(__file__)), "precond"))
from math_grade import correct
plan = {r["item"]: r for r in json.load(open("bench/precond/plan_math500.json", encoding="utf-8"))}
e17 = {r["item"]: r for r in map(json.loads, open("results/quality/e17_math500_greedy.jsonl", encoding="utf-8"))}
gen = {}
for l in open("results/quality/e18_precond.jsonl", encoding="utf-8"):
    o = json.loads(l); gen[(o["item"], o["arm"])] = o
assert len(e17) == 500, f"E17 has {len(e17)} items"
cap = lambda o: (o.get("timings") or {}).get("predicted_n", 0) >= 2048
I, N, S, IR, capI = {}, {}, {}, {}, {}
for k, r in e17.items():
    I[k] = correct(r["text"], r["gold"]); capI[k] = cap(r); p = plan[k]
    N[k] = correct(gen[(k, "N")]["text"], r["gold"]) if p["prompt_N"] else I[k]
    if p["prompt_S"] is None: S[k] = I[k]
    elif p["prompt_S"] == p["prompt_N"]: S[k] = N[k]
    else: S[k] = correct(gen[(k, "S")]["text"], r["gold"])
    if (k, "I_rerun") in gen: IR[k] = correct(gen[(k, "I_rerun")]["text"], r["gold"])
def mcn(A, B, keys):
    b = sum(1 for k in keys if A[k] and not B[k]); c = sum(1 for k in keys if B[k] and not A[k]); n = b + c
    p = 1.0 if n == 0 else min(1.0, 2 * sum(comb(n, i) for i in range(0, min(b, c) + 1)) / 2 ** n)
    return b, c, p
K = sorted(e17); el = [k for k in K if plan[k]["prompt_N"] or plan[k]["prompt_S"]]
acc = lambda A, ks: sum(A[k] for k in ks) / len(ks) * 100
print(f"eligible items: {len(el)} / 500 ({el})")
for nm, A in [("I (E17)", I), ("N", N), ("S", S)]: print(f"{nm}: all-500 {acc(A, K):.1f}%  eligible {sum(A[k] for k in el)}/{len(el)}")
print(f"I_rerun on eligible: {sum(IR.values())}/{len(IR)}; flips vs E17: {sum(IR[k] != I[k] for k in IR)}")
res = {}
for nm, A, B in [("N_vs_I", N, I), ("S_vs_N", S, N), ("S_vs_I", S, I)]:
    b, c, p = mcn(A, B, K); res[nm] = (b, c, p)
    print(f"{nm}: wins {b} losses {c} exact McNemar p={p:.3f}  delta {(b - c) / 5:.1f} pts on 500")
# Holm over the two primary tests (S_vs_N, S_vs_I)
ps = sorted([("S_vs_N", res["S_vs_N"][2]), ("S_vs_I", res["S_vs_I"][2])], key=lambda x: x[1])
holm = {ps[0][0]: min(1, 2 * ps[0][1]), ps[1][0]: min(1, max(2 * ps[0][1], ps[1][1]))}
gainN = (res["S_vs_N"][0] - res["S_vs_N"][1]) / 5; gainI = (res["S_vs_I"][0] - res["S_vs_I"][1]) / 5
capwins = sum(1 for k in K if S[k] and not I[k] and capI[k])
promote = (holm["S_vs_N"] < 0.05 and holm["S_vs_I"] < 0.05 and gainN >= 3 and gainI >= 3 and capwins < res["S_vs_I"][0] - res["S_vs_I"][1])
print(f"Holm p: {holm}; S gain vs N {gainN:+.1f} pts, vs I {gainI:+.1f} pts; S wins over I on E17-capped items: {capwins}")
print("VERDICT:", "PASS (promote to ablations: token-only, width-only, random candidate)" if promote else "KILL selector (pre-registered criteria not met)")
print("N control:", "helps" if res["N_vs_I"][2] < 0.05 and res["N_vs_I"][0] > res["N_vs_I"][1] else "no significant effect")