6ad897477d988ba75c2bf78f4acc0949132901e0 / bench/core/reporting.py · 2173 bytes · raw
"""Shared reporting helpers — JSONL append, summary tables, result loading."""
import json
import os
import re
HERE = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
RESULTS_DIR = os.path.join(HERE, "results")
os.makedirs(RESULTS_DIR, exist_ok=True)
def safe_label(s):
"""Sanitize a model name into a filesystem-safe label."""
return re.sub(r"[^A-Za-z0-9._-]", "_", s)
def jsonl_path(label, prefix=""):
"""Return the JSONL output path for a given label/prefix."""
name = f"{prefix}_{safe_label(label)}.jsonl" if prefix else f"{safe_label(label)}.jsonl"
return os.path.join(RESULTS_DIR, name)
def append_jsonl(path, record):
with open(path, "a") as f:
f.write(json.dumps(record) + "\n")
def load_jsonl(path):
if not os.path.exists(path):
return []
return [json.loads(l) for l in open(path) if l.strip()]
def write_json(record, label, prefix=""):
path = os.path.join(RESULTS_DIR, f"{prefix}_{safe_label(label)}.json"
if prefix else f"{safe_label(label)}.json")
with open(path, "w") as f:
json.dump(record, f, indent=2)
return path
def coding_suite_summary(results):
"""Print + return the coding-suite scoreboard."""
npass = sum(1 for r in results if r.get("passed"))
print("\n========================= SUMMARY =========================")
print("Overall: %d/%d passed (%.0f%%)\n" %
(npass, len(results), 100 * npass / max(len(results), 1)))
cats = {}
for r in results:
cats.setdefault(r["cat"], []).append(r)
for cat, rs in cats.items():
p = sum(1 for r in rs if r.get("passed"))
print(" %-9s %d/%d" % (cat, p, len(rs)))
print("\n %-16s %-5s %-6s %-7s %-9s %s" %
("task", "pass", "lat_s", "tot_tok", "reason_tok", "note"))
for r in results:
print(" %-16s %-5s %-6s %-7s %-9s %s" % (
r["id"], "Y" if r.get("passed") else "n",
r.get("latency_s", "-"), r.get("total_tokens", "-"),
r.get("reasoning_tokens", "-"),
("" if r.get("passed") else (r.get("detail", "") or "")[:80])))
print()
return npass == len(results)