# E14: HumanEval pass@1 (greedy) against a running llama-server. Standard eval (Chen et al. 2021), no novelty claim.
# Hand-written by the AGI loop 8:58 AM ET Oct 27 2026 (Cursor CLI not logged in). Same prompts for every GGUF.
import argparse, json, re, subprocess, sys, tempfile, time, os, requests
from datasets import load_dataset
ap = argparse.ArgumentParser()
ap.add_argument("--url", default="http://127.0.0.1:8090"); ap.add_argument("--tag", required=True)
ap.add_argument("--n", type=int, default=164); ap.add_argument("--max_tok", type=int, default=512)
a = ap.parse_args(); U = a.url
ds = load_dataset("openai/openai_humaneval", split="test")
def comp(prompt):
    r = requests.post(U + "/completion", json={"prompt": prompt, "n_predict": a.max_tok, "temperature": 0, "top_k": 1,
        "cache_prompt": False, "seed": 0}, timeout=3600); r.raise_for_status(); return r.json()
def extract(txt, entry):
    blocks = re.findall(r"```(?:python|py)?\s*\n(.*?)```", txt, re.S)
    if not blocks:
        blocks = [txt.split("```")[0]]
    for b in blocks:
        if f"def {entry}" in b: return b
    return blocks[0]
ok = 0; ntok = 0; tgen = 0.0; tw = time.time(); res = []
for k in range(min(a.n, len(ds))):
    ex = ds[k]
    p = ("<|im_start|>system\nYou are a helpful assistant.<|im_end|>\n<|im_start|>user\nComplete the following Python function. "
         "Return the complete function (with any imports it needs) in a single ```python code block.\n\n```python\n" + ex["prompt"] +
         "```<|im_end|>\n<|im_start|>assistant\n")
    j = comp(p); txt = j.get("content", ""); tm = j.get("timings", {})
    ntok += tm.get("predicted_n", 0); tgen += tm.get("predicted_ms", 0) / 1000
    code = extract(txt, ex["entry_point"])
    prog = ex["prompt"] + "\n    pass\n\n" + code + "\n\n" + ex["test"] + f"\ncheck({ex['entry_point']})\n"
    fd, fn = tempfile.mkstemp(suffix=".py"); os.write(fd, prog.encode("utf-8")); os.close(fd)
    try:
        r = subprocess.run([sys.executable, fn], capture_output=True, timeout=15); passed = r.returncode == 0
    except subprocess.TimeoutExpired:
        passed = False
    os.remove(fn); ok += passed; res.append({"task": ex["task_id"], "pass": bool(passed), "out": txt})
    print(f"he {k} ok={ok} tok/s={ntok/max(tgen,1e-9):.2f}", flush=True)
n = len(res)
out = {"tag": a.tag, "humaneval": {"n": n, "pass1": ok / n, "decode_tok_s": ntok / max(tgen, 1e-9), "gen_tokens": ntok,
       "wall_sec": round(time.time() - tw, 1), "results": res}}
print("HUMANEVAL", ok / n, out["humaneval"]["decode_tok_s"], flush=True)
json.dump(out, open(f"results/quality/e14_{a.tag}.json", "w"), indent=1)
print("WROTE", a.tag)