0b37d286065ecbf5f6dfcfafa2d9274d1d21dba3 / bench/tasks/agentic_build.py · 24943 bytes · raw
"""Agentic build instrument — "build a real small app" end-to-end.
Given a plan and a thin starter scaffold, the agent uses tools to write ~8
files, get `go build` green, get `go test` green, and produce a binary that
starts, serves an HTML form, and persists a POSTed note to SQLite. The four
"real product" gates are run by the harness, not the agent.
This is a Go + SQLite + html/template app — a realistic small self-hosted
web tool in miniature. It tests the actual workload: multi-turn tool-use,
file creation, dependency management, iterative build/test cycles.
Usage (via the bench.py CLI):
python bench.py agentic-build validate
python bench.py agentic-build run
Env (via .env): ENDPOINT, MODEL, API_KEY, MAX_TOKENS, MAX_TURNS.
"""
import json
import os
import re
import sys
import time
import shutil
import subprocess
import socket
import signal
import urllib.request
import urllib.parse
import urllib.error
from bench.core import client
from bench.core import tools
from bench.core.reporting import write_json
HERE = os.path.dirname(os.path.abspath(__file__))
WORK = os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(__file__))), "work", "agentic_build")
MAX_TOKENS = int(os.environ.get("MAX_TOKENS", "16000"))
MAX_TURNS = int(os.environ.get("MAX_TURNS", "40"))
BUILD_TIMEOUT = int(os.environ.get("BUILD_TIMEOUT", "180"))
TEST_TIMEOUT = int(os.environ.get("TEST_TIMEOUT", "180"))
SERVER_TIMEOUT = int(os.environ.get("SERVER_TIMEOUT", "20"))
PLAN = """\
# Task — Build a Go web binary with a SQLite-backed notes CRUD
## Goal
`go build ./...` and `go test ./...` run green in the work dir, and the
resulting binary starts on port 8080, serves an HTML form at `/` that lists
existing notes and lets a visitor create a new one, and persists created
notes to an embedded SQLite file `notes.db`.
## Prerequisites
A starter scaffold is already in the work dir:
- `go.mod` (module `notes`, go 1.26, no deps yet)
- `cmd/server/main.go` (empty stub: `package main; func main(){}`)
You start from this scaffold. Do not delete existing files; build on them.
## Spec — the app to build
A single Go binary that serves a one-page HTML UI over HTTP. One `cmd/`,
modular `internal/` packages, server-rendered `html/template`, embedded
assets, embedded SQLite via the pure-Go driver `modernc.org/sqlite`.
### Files to produce (layout)
cmd/server/main.go boot: config -> db -> migrate -> handlers -> ListenAndServe + graceful shutdown
internal/config/config.go Config{Port, DBPath}; LoadFromEnv() with sane defaults (port 8080, db "notes.db"). MUST read env vars NOTES_PORT (port) and NOTES_DB (db path) — the scoring harness sets these.
internal/db/db.go Open(dsn): open modernc sqlite, PRAGMA WAL + busy_timeout + foreign_keys ON; Migrate() via embedded .sql
internal/db/migrations/00001_init.sql CREATE TABLE notes (id INTEGER PRIMARY KEY, title TEXT NOT NULL, body TEXT NOT NULL, created_at DATETIME DEFAULT CURRENT_TIMESTAMP);
internal/models/note.go Note struct + Create(db, title, body) + List(db) + Delete(db, id); plain database/sql, no ORM
internal/handlers/handlers.go Handlers struct; routes via stdlib net/http ServeMux: GET / (list+form), POST / (create), POST /{id}/delete (delete)
internal/handlers/handlers_test.go httptest: POST a note, GET /, assert the note title appears in the response body
web/templates/index.html the one page: a form (title + body + submit) above a list of notes (each with a delete button); rendered server-side by html/template
### Constraints
- Use `modernc.org/sqlite` (pure-Go, CGO_ENABLED=0). Import path is `_ "modernc.org/sqlite"`.
- Embed the migration SQL and the templates via `//go:embed`.
- Stdlib `net/http` ServeMux only — no Gin, no chi.
- No JS framework. Plain HTML form posts back to the server.
## Steps
1. Add the dependency: `go get modernc.org/sqlite`. Run `go mod tidy`.
2. Write the files. Natural order: config -> db -> models -> handlers -> templates -> main.
3. Build (`go build ./...`). Fix any compile errors.
4. Test (`go test ./...`). The handlers_test.go must pass.
5. Optionally run the binary briefly to confirm it serves.
## Acceptance (harness-verified)
1. `go build ./...` exits 0.
2. `go test ./...` exits 0.
3. The built binary starts on port 8080 and `GET /` returns HTTP 200 with a `<form` element.
4. `POST /` with form fields title+body creates a row; a subsequent `GET /` shows that title; the SQLite file on disk contains the row.
## Out of scope
Auth, sessions, CSRF, styling beyond minimal readable HTML, HTMX, pagination, any JS.
## Commit
You do not commit; the harness owns the work dir.
"""
STARTER = {
"go.mod": "module notes\n\ngo 1.26\n",
"cmd/server/main.go": "package main\n\nfunc main() {}\n",
}
REFERENCE = {
"go.mod": "module notes\n\ngo 1.26\n\nrequire modernc.org/sqlite v1.50.0\n",
"cmd/server/main.go": '''package main
import (
"context"
"fmt"
"log"
"net/http"
"os"
"os/signal"
"syscall"
"time"
"notes/internal/config"
"notes/internal/db"
"notes/internal/handlers"
"notes/internal/models"
)
func main() {
cfg := config.LoadFromEnv()
database, err := db.Open(cfg.DBPath)
if err != nil { log.Fatalf("db open: %v", err) }
defer database.Close()
if err := database.Migrate(); err != nil { log.Fatalf("migrate: %v", err) }
notes := &models.Store{DB: database.DB}
h := handlers.New(notes)
srv := &http.Server{Addr: fmt.Sprintf(":%d", cfg.Port), Handler: h.Routes()}
go func() {
log.Printf("listening on :%d", cfg.Port)
if err := srv.ListenAndServe(); err != nil && err != http.ErrServerClosed {
log.Fatalf("serve: %v", err)
}
}()
stop := make(chan os.Signal, 1)
signal.Notify(stop, os.Interrupt, syscall.SIGTERM)
<-stop
log.Println("shutting down")
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
defer cancel()
_ = srv.Shutdown(ctx)
}
''',
"internal/config/config.go": '''package config
import ("os"; "strconv")
type Config struct { Port int; DBPath string }
func LoadFromEnv() Config {
cfg := Config{Port: 8080, DBPath: "notes.db"}
if p := os.Getenv("NOTES_PORT"); p != "" {
if v, err := strconv.Atoi(p); err == nil { cfg.Port = v }
}
if d := os.Getenv("NOTES_DB"); d != "" { cfg.DBPath = d }
return cfg
}
''',
"internal/db/db.go": '''package db
import ("database/sql"; "embed"; "fmt"; _ "modernc.org/sqlite")
//go:embed migrations/*.sql
var migrationsFS embed.FS
type Database struct { *sql.DB }
func Open(dsn string) (*Database, error) {
db, err := sql.Open("sqlite", dsn)
if err != nil { return nil, fmt.Errorf("open sqlite: %w", err) }
if _, err := db.Exec(`PRAGMA journal_mode=WAL; PRAGMA busy_timeout=5000; PRAGMA foreign_keys=ON;`); err != nil {
db.Close(); return nil, fmt.Errorf("pragmas: %w", err)
}
return &Database{DB: db}, nil
}
func (d *Database) Migrate() error {
if _, err := d.Exec(`CREATE TABLE IF NOT EXISTS schema_migrations (name TEXT PRIMARY KEY, applied_at DATETIME DEFAULT CURRENT_TIMESTAMP)`); err != nil {
return fmt.Errorf("create schema_migrations: %w", err)
}
entries, err := migrationsFS.ReadDir("migrations")
if err != nil { return fmt.Errorf("read migrations dir: %w", err) }
for _, e := range entries {
if e.IsDir() { continue }
name := e.Name()
var applied int
if err := d.QueryRow(`SELECT count(*) FROM schema_migrations WHERE name = ?`, name).Scan(&applied); err != nil {
return fmt.Errorf("check %s: %w", name, err)
}
if applied > 0 { continue }
b, err := migrationsFS.ReadFile("migrations/" + name)
if err != nil { return fmt.Errorf("read %s: %w", name, err) }
if _, err := d.Exec(string(b)); err != nil { return fmt.Errorf("apply %s: %w", name, err) }
if _, err := d.Exec(`INSERT INTO schema_migrations (name) VALUES (?)`, name); err != nil {
return fmt.Errorf("record %s: %w", name, err)
}
}
return nil
}
''',
"internal/db/migrations/00001_init.sql": '''CREATE TABLE IF NOT EXISTS notes (id INTEGER PRIMARY KEY AUTOINCREMENT, title TEXT NOT NULL, body TEXT NOT NULL, created_at DATETIME DEFAULT CURRENT_TIMESTAMP);''',
"internal/models/note.go": '''package models
import ("database/sql"; "time")
type Note struct { ID int64; Title string; Body string; CreatedAt time.Time }
type Store struct { DB *sql.DB }
func (s *Store) Create(title, body string) (*Note, error) {
res, err := s.DB.Exec(`INSERT INTO notes (title, body) VALUES (?, ?)`, title, body)
if err != nil { return nil, err }
id, err := res.LastInsertId()
if err != nil { return nil, err }
return &Note{ID: id, Title: title, Body: body, CreatedAt: time.Now()}, nil
}
func (s *Store) List() ([]Note, error) {
rows, err := s.DB.Query(`SELECT id, title, body, created_at FROM notes ORDER BY id DESC`)
if err != nil { return nil, err }
defer rows.Close()
var out []Note
for rows.Next() {
var n Note
if err := rows.Scan(&n.ID, &n.Title, &n.Body, &n.CreatedAt); err != nil { return nil, err }
out = append(out, n)
}
return out, rows.Err()
}
func (s *Store) Delete(id int64) (bool, error) {
res, err := s.DB.Exec(`DELETE FROM notes WHERE id = ?`, id)
if err != nil { return false, err }
n, err := res.RowsAffected()
if err != nil { return false, err }
return n > 0, nil
}
''',
"internal/handlers/handlers.go": '''package handlers
import ("embed"; "html/template"; "log"; "net/http"; "strconv"; "notes/internal/models")
//go:embed templates/*.html
var templatesFS embed.FS
type Handlers struct { notes *models.Store; tmpl *template.Template }
func New(notes *models.Store) *Handlers {
tmpl := template.Must(template.ParseFS(templatesFS, "templates/index.html"))
return &Handlers{notes: notes, tmpl: tmpl}
}
func (h *Handlers) Routes() http.Handler {
mux := http.NewServeMux()
mux.HandleFunc("GET /", h.index)
mux.HandleFunc("POST /", h.create)
mux.HandleFunc("POST /{id}/delete", h.delete)
return mux
}
func (h *Handlers) index(w http.ResponseWriter, r *http.Request) {
notes, err := h.notes.List()
if err != nil { log.Printf("list: %v", err); http.Error(w, "internal error", http.StatusInternalServerError); return }
w.Header().Set("Content-Type", "text/html; charset=utf-8")
if err := h.tmpl.Execute(w, notes); err != nil { log.Printf("render: %v", err) }
}
func (h *Handlers) create(w http.ResponseWriter, r *http.Request) {
title := r.FormValue("title"); body := r.FormValue("body")
if title == "" || body == "" { http.Error(w, "title and body are required", http.StatusBadRequest); return }
if _, err := h.notes.Create(title, body); err != nil { log.Printf("create: %v", err); http.Error(w, "internal error", http.StatusInternalServerError); return }
http.Redirect(w, r, "/", http.StatusSeeOther)
}
func (h *Handlers) delete(w http.ResponseWriter, r *http.Request) {
idStr := r.PathValue("id"); id, err := strconv.ParseInt(idStr, 10, 64)
if err != nil { http.Error(w, "bad id", http.StatusBadRequest); return }
if _, err := h.notes.Delete(id); err != nil { log.Printf("delete: %v", err); http.Error(w, "internal error", http.StatusInternalServerError); return }
http.Redirect(w, r, "/", http.StatusSeeOther)
}
''',
"internal/handlers/templates/index.html": '''<!DOCTYPE html><html lang="en"><head><meta charset="utf-8"><meta name="viewport" content="width=device-width, initial-scale=1"><title>Notes</title><style>body{font-family:system-ui,sans-serif;max-width:40em;margin:2rem auto;padding:0 1rem}form{margin:1rem 0;padding:1rem;border:1px solid #ccc;border-radius:6px}label{display:block;margin:0.5rem 0}input[type=text],textarea{width:100%;box-sizing:border-box;padding:0.4rem}.note{margin:0.75rem 0;padding:0.75rem;border-bottom:1px solid #eee}form.delete{display:inline}form.delete button{background:none;border:none;color:#c00;cursor:pointer}</style></head><body><h1>Notes</h1><form method="POST" action="/"><label>Title <input type="text" name="title" required></label><label>Body <textarea name="body" rows="4" required></textarea></label><button type="submit">Add note</button></form>{{range .}}<div class="note"><h3>{{.Title}}</h3><p>{{.Body}}</p><form class="delete" method="POST" action="/{{.ID}}/delete"><button type="submit">delete</button></form></div>{{else}}<p>No notes yet.</p>{{end}}</body></html>''',
"internal/handlers/handlers_test.go": '''package handlers
import ("database/sql"; "io"; "net/http"; "net/http/httptest"; "net/url"; "strings"; "testing"; "notes/internal/models"; _ "modernc.org/sqlite")
func newTestStore(t *testing.T) (*models.Store, func()) {
t.Helper()
db, err := sql.Open("sqlite", ":memory:")
if err != nil { t.Fatalf("open: %v", err) }
_, err = db.Exec(`CREATE TABLE notes (id INTEGER PRIMARY KEY AUTOINCREMENT, title TEXT NOT NULL, body TEXT NOT NULL, created_at DATETIME DEFAULT CURRENT_TIMESTAMP)`)
if err != nil { db.Close(); t.Fatalf("create table: %v", err) }
return &models.Store{DB: db}, func() { db.Close() }
}
func TestCreateThenList(t *testing.T) {
store, cleanup := newTestStore(t); defer cleanup()
h := New(store)
srv := httptest.NewServer(h.Routes()); defer srv.Close()
form := url.Values{"title": {"My Note"}, "body": {"First body"}}
resp, err := http.PostForm(srv.URL+"/", form)
if err != nil { t.Fatalf("post: %v", err) }
resp.Body.Close()
if resp.StatusCode != http.StatusOK { t.Fatalf("post status = %d, want 200", resp.StatusCode) }
resp2, err := http.Get(srv.URL + "/")
if err != nil { t.Fatalf("get: %v", err) }
defer resp2.Body.Close()
body, _ := io.ReadAll(resp2.Body)
if !strings.Contains(string(body), "My Note") { t.Fatalf("response does not contain the created note title; got:\\n%s", body) }
}
''',
"internal/handlers/doc.go": "package handlers",
"internal/models/doc.go": "package models",
"internal/config/doc.go": "package config",
"internal/db/doc.go": "package db",
}
# ----------------------------------------------------------------- scoring gates
def _has_real_app(work_dir):
main_go = os.path.join(work_dir, "cmd", "server", "main.go")
if not os.path.isfile(main_go): return False
with open(main_go) as f: src = f.read()
if "ListenAndServe" not in src and "http.Server" not in src: return False
internal = os.path.join(work_dir, "internal")
if not os.path.isdir(internal): return False
pkg_dirs = [d for d in os.listdir(internal) if os.path.isdir(os.path.join(internal, d))]
return len(pkg_dirs) >= 3
def _run(cmd, work_dir, timeout):
try:
p = subprocess.run(cmd, cwd=work_dir, shell=True, capture_output=True, text=True, timeout=timeout)
except subprocess.TimeoutExpired:
return f"TIMEOUT after {timeout}s"
return f"exit={p.returncode}\n{(p.stdout + p.stderr).strip()}"
def _free_port():
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM); s.bind(("", 0))
p = s.getsockname()[1]; s.close(); return p
def gates(work_dir):
g = {}
if not _has_real_app(work_dir):
return {"build": False, "test": False, "serves": False, "persists": False}
b = _run("go build ./...", work_dir, BUILD_TIMEOUT)
g["build"] = b.startswith("exit=0")
t = _run("go test ./...", work_dir, TEST_TIMEOUT)
g["test"] = t.startswith("exit=0")
binpath = os.path.abspath(os.path.join(work_dir, "server"))
r = _run(f"go build -o {binpath} ./cmd/server", work_dir, BUILD_TIMEOUT)
if not r.startswith("exit=0") or not os.path.isfile(binpath):
g["serves"] = False; g["persists"] = False; return g
port = _free_port()
dbpath = os.path.abspath(os.path.join(work_dir, f"bench_{port}.db"))
for ext in ("", "-wal", "-shm"):
if os.path.exists(dbpath + ext): os.remove(dbpath + ext)
env = dict(os.environ, NOTES_PORT=str(port), NOTES_DB=dbpath)
try:
proc = subprocess.Popen([binpath], cwd=work_dir, env=env, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
except Exception:
g["serves"] = False; g["persists"] = False; return g
try:
for _ in range(40):
try:
with socket.create_connection(("127.0.0.1", port), 0.25): break
except OSError: time.sleep(0.25)
else:
g["serves"] = False; g["persists"] = False; return g
base = f"http://127.0.0.1:{port}"
try:
with urllib.request.urlopen(base + "/", timeout=SERVER_TIMEOUT) as resp:
if resp.status != 200:
g["serves"] = False; g["persists"] = False; return g
html = resp.read().decode(errors="replace")
except Exception:
g["serves"] = False; g["persists"] = False; return g
g["serves"] = "<form" in html
if not g["serves"]:
g["persists"] = False; return g
data = urllib.parse.urlencode({"title": "BenchNote", "body": "created by the gate"}).encode()
try:
req = urllib.request.Request(base + "/", data=data, method="POST")
with urllib.request.urlopen(req, timeout=SERVER_TIMEOUT) as resp: _ = resp.read()
except Exception:
g["persists"] = False; return g
try:
with urllib.request.urlopen(base + "/", timeout=SERVER_TIMEOUT) as resp:
after = resp.read().decode(errors="replace")
except Exception:
g["persists"] = False; return g
g["persists"] = "BenchNote" in after
if g["persists"] and os.path.exists(dbpath):
try:
import sqlite3
conn = sqlite3.connect(dbpath)
cur = conn.execute("SELECT count(*) FROM notes WHERE title = 'BenchNote'")
g["persists"] = cur.fetchone()[0] > 0
conn.close()
except Exception: pass
return g
finally:
proc.send_signal(signal.SIGTERM)
try: proc.wait(timeout=5)
except subprocess.TimeoutExpired: proc.kill()
for ext in ("", "-wal", "-shm"):
p = dbpath + ext
if os.path.exists(p):
try: os.remove(p)
except: pass
# ----------------------------------------------------------------- phases
def validate():
print("=== VALIDATION: reference app builds, tests, and serves ===\n")
tools.set_work_dir(WORK)
tools.materialize(REFERENCE, dest=WORK)
r = _run("go mod tidy", WORK, BUILD_TIMEOUT)
print("go mod tidy:", r[:200])
g = gates(WORK)
print(f"gates: {g}")
ok = all(g.values())
print(f"\n-> {'OK' if ok else 'BAD'}: reference app is {'buildable + working' if ok else 'BROKEN'}")
return ok
SYSTEM = (
"You are a coding agent working in a Go project work dir. You are given "
"a task plan below. Execute it step by step using the tools. You can "
"write files, edit files, run builds and tests, and run shell commands "
"(e.g. `go get`, `go mod tidy`) in the work dir. Keep going until the "
"build is green, the tests pass, and the binary would work. Use the "
"tools to verify your own progress — don't guess; compile and test.\n\n"
"When you are confident the app is complete and the tests pass, say "
"`DONE` in your final message (no tool call) and stop.\n\n"
"TASK PLAN:\n" + PLAN
)
def run():
tools.set_work_dir(WORK)
tools.materialize(STARTER, dest=WORK)
messages = [
{"role": "system", "content": SYSTEM},
{"role": "user", "content":
"The work dir has the starter scaffold (go.mod + a stub main.go). "
"Execute the task plan. Begin by adding the dependency and writing "
"the files. Use the tools to build and test as you go."},
]
turns_log = []
peak_prompt = total_completion = total_reasoning = 0
finish_reasons = []
t0 = time.time()
turns = 0
done_said = False
no_tool_streak = 0
for turn in range(1, MAX_TURNS + 1):
turns = turn
try:
resp, dt = client.call_model(messages, tools=tools.TOOLS, max_tokens=MAX_TOKENS)
except Exception as e:
print(f" turn {turn}: REQUEST ERROR {e!r}")
turns_log.append({"turn": turn, "error": repr(e)})
break
u = client.usage(resp)
peak_prompt = max(peak_prompt, u["prompt_tokens"])
total_completion += u["completion_tokens"]
total_reasoning += u["reasoning_tokens"]
choice = resp["choices"][0]
fr = choice.get("finish_reason")
finish_reasons.append(fr)
msg = choice["message"]
tcs = msg.get("tool_calls") or []
content = msg.get("content") or ""
tlog = {"turn": turn, "dt": round(dt, 1), "prompt_tok": u["prompt_tokens"],
"completion_tok": u["completion_tokens"], "reasoning_tok": u["reasoning_tokens"],
"finish_reason": fr, "n_tool_calls": len(tcs)}
turns_log.append(tlog)
tc_summary = ",".join((tc["function"]["name"] for tc in tcs)) or "(no tool call)"
print(f" turn {turn:2d}: {dt:5.1f}s ptok={u['prompt_tokens']:6d} "
f"ctok={u['completion_tokens']:5d} rtok={u['reasoning_tokens']:5d} "
f"fr={fr:8s} tools=[{tc_summary}]")
if not tcs and "DONE" in content.upper():
done_said = True
turns_log[-1]["done"] = True
break
if not tcs:
no_tool_streak += 1
if no_tool_streak >= 2:
print(" -> giving up after 2 no-tool turns")
break
turns_log[-1]["no_tool_streak"] = no_tool_streak
messages.append({"role": "assistant", "content": content})
messages.append({"role": "user", "content":
"You must either call a tool to keep working, or say DONE if "
"you believe the app is complete and tests pass."})
continue
no_tool_streak = 0
a_msg = {"role": "assistant", "content": content}
if tcs: a_msg["tool_calls"] = tcs
messages.append(a_msg)
for tc in tcs:
fn = tc["function"]
name = fn["name"]
try:
args = json.loads(fn["arguments"] or "{}")
except json.JSONDecodeError as e:
args = {}
result = (f"ERROR: your tool_call arguments were not valid "
f"JSON ({e}). Please re-emit the {name} tool call "
f"with valid JSON arguments.")
else:
impl = tools.TOOL_IMPLS.get(name)
if impl is None:
result = f"ERROR: unknown tool {name!r}"
else:
try:
result = impl(args)
except Exception as e:
result = f"ERROR: {name} raised {e!r}"
if len(result) > 6000:
result = result[:6000] + f"\n...[truncated, {len(result)} total chars]"
messages.append({"role": "tool", "tool_call_id": tc.get("id", ""), "content": result})
tlog.setdefault("tool_results", []).append({
"tool": name, "result_len": len(result),
"result_head": result[:120].replace("\n", " ")})
wall = time.time() - t0
g = gates(WORK)
fr_counter = {}
for f in finish_reasons:
if f: fr_counter[f] = fr_counter.get(f, 0) + 1
rec = {
"model": client.MODEL, "max_tokens_per_turn": MAX_TOKENS,
"max_turns": MAX_TURNS, "turns_used": turns, "done_said": done_said,
"wall_s": round(wall, 1), "peak_prompt_tokens": peak_prompt,
"total_completion_tokens": total_completion,
"total_reasoning_tokens": total_reasoning,
"finish_reasons": fr_counter, "checkpoints": g,
"all_pass": all(g.values()), "turns_log": turns_log,
}
_report(rec)
def _report(rec):
print("\n===================== AGENTIC BUILD RESULT =====================")
print(f"model : {rec['model']}")
print(f"turns used : {rec['turns_used']}/{rec['max_turns']} (done_said={rec['done_said']})")
print(f"wall clock : {rec['wall_s']}s ({rec['wall_s']/60:.1f} min)")
print(f"peak prompt tok : {rec['peak_prompt_tokens']}")
print(f"total completion : {rec['total_completion_tokens']}")
print(f"total reasoning : {rec['total_reasoning_tokens']}")
print(f"finish reasons : {rec['finish_reasons']}")
print()
print(f"--- checkpoints ---")
for k, v in rec["checkpoints"].items():
print(f" {k:10s}: {'PASS' if v else 'fail'}")
print(f" => {'ALL PASS — real product works' if rec['all_pass'] else 'INCOMPLETE'}")
write_json(rec, rec["model"], prefix="agentic-build")
print()
def main():
mode = sys.argv[1] if len(sys.argv) > 1 else 'validate'
if mode == 'validate':
sys.exit(0 if validate() else 1)
elif mode == 'run':
run()
else:
print(f'usage: python bench.py agentic-build [validate|run]')
sys.exit(1)
if __name__ == "__main__":
main()