josie / llm-bench

"""Shared reporting helpers — JSONL append, summary tables, result loading."""
import json
import os
import re

HERE = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
RESULTS_DIR = os.path.join(HERE, "results")
os.makedirs(RESULTS_DIR, exist_ok=True)


def safe_label(s):
    """Sanitize a model name into a filesystem-safe label."""
    return re.sub(r"[^A-Za-z0-9._-]", "_", s)


def jsonl_path(label, prefix=""):
    """Return the JSONL output path for a given label/prefix."""
    name = f"{prefix}_{safe_label(label)}.jsonl" if prefix else f"{safe_label(label)}.jsonl"
    return os.path.join(RESULTS_DIR, name)


def append_jsonl(path, record):
    with open(path, "a") as f:
        f.write(json.dumps(record) + "\n")


def load_jsonl(path):
    if not os.path.exists(path):
        return []
    return [json.loads(l) for l in open(path) if l.strip()]


def write_json(record, label, prefix=""):
    path = os.path.join(RESULTS_DIR, f"{prefix}_{safe_label(label)}.json"
                        if prefix else f"{safe_label(label)}.json")
    with open(path, "w") as f:
        json.dump(record, f, indent=2)
    return path


def coding_suite_summary(results):
    """Print + return the coding-suite scoreboard."""
    npass = sum(1 for r in results if r.get("passed"))
    print("\n========================= SUMMARY =========================")
    print("Overall: %d/%d passed (%.0f%%)\n" %
          (npass, len(results), 100 * npass / max(len(results), 1)))
    cats = {}
    for r in results:
        cats.setdefault(r["cat"], []).append(r)
    for cat, rs in cats.items():
        p = sum(1 for r in rs if r.get("passed"))
        print("  %-9s %d/%d" % (cat, p, len(rs)))
    print("\n  %-16s %-5s %-6s %-7s %-9s %s" %
          ("task", "pass", "lat_s", "tot_tok", "reason_tok", "note"))
    for r in results:
        print("  %-16s %-5s %-6s %-7s %-9s %s" % (
            r["id"], "Y" if r.get("passed") else "n",
            r.get("latency_s", "-"), r.get("total_tokens", "-"),
            r.get("reasoning_tokens", "-"),
            ("" if r.get("passed") else (r.get("detail", "") or "")[:80])))
    print()
    return npass == len(results)