"""Shared reporting helpers — JSONL append, summary tables, result loading.""" import json import os import re HERE = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) RESULTS_DIR = os.path.join(HERE, "results") os.makedirs(RESULTS_DIR, exist_ok=True) def safe_label(s): """Sanitize a model name into a filesystem-safe label.""" return re.sub(r"[^A-Za-z0-9._-]", "_", s) def jsonl_path(label, prefix=""): """Return the JSONL output path for a given label/prefix.""" name = f"{prefix}_{safe_label(label)}.jsonl" if prefix else f"{safe_label(label)}.jsonl" return os.path.join(RESULTS_DIR, name) def append_jsonl(path, record): with open(path, "a") as f: f.write(json.dumps(record) + "\n") def load_jsonl(path): if not os.path.exists(path): return [] return [json.loads(l) for l in open(path) if l.strip()] def write_json(record, label, prefix=""): path = os.path.join(RESULTS_DIR, f"{prefix}_{safe_label(label)}.json" if prefix else f"{safe_label(label)}.json") with open(path, "w") as f: json.dump(record, f, indent=2) return path def coding_suite_summary(results): """Print + return the coding-suite scoreboard.""" npass = sum(1 for r in results if r.get("passed")) print("\n========================= SUMMARY =========================") print("Overall: %d/%d passed (%.0f%%)\n" % (npass, len(results), 100 * npass / max(len(results), 1))) cats = {} for r in results: cats.setdefault(r["cat"], []).append(r) for cat, rs in cats.items(): p = sum(1 for r in rs if r.get("passed")) print(" %-9s %d/%d" % (cat, p, len(rs))) print("\n %-16s %-5s %-6s %-7s %-9s %s" % ("task", "pass", "lat_s", "tot_tok", "reason_tok", "note")) for r in results: print(" %-16s %-5s %-6s %-7s %-9s %s" % ( r["id"], "Y" if r.get("passed") else "n", r.get("latency_s", "-"), r.get("total_tokens", "-"), r.get("reasoning_tokens", "-"), ("" if r.get("passed") else (r.get("detail", "") or "")[:80]))) print() return npass == len(results)