josie / llm-bench

#!/usr/bin/env python3
"""
Task: ts_next — a Next.js app with Prisma + API route + Zod validation.

Models a Next.js app with Prisma + API route + Zod validation in miniature:
a Next.js App Router app with a Prisma schema (2 models), an API route that
does a CRUD operation with Zod input validation, and a server-rendered page.
Uses SQLite as the Prisma datasource (so no external Postgres is needed —
`@prisma/adapter-better-sqlite3` or Prisma's built-in sqlite provider). The
"real product" = a dev server that responds on `/` (the page) and
`/api/items` (the API route: GET lists, POST creates with Zod validation,
invalid input returns 400).
"""
import os, sys, subprocess, time, shutil, signal, socket, urllib.request, urllib.parse, json

from bench.core import tools

HERE = os.path.dirname(os.path.abspath(__file__))

TASK_NAME = "ts_next"

PLAN = """\
# Task — Build a Next.js app with Prisma + API route + Zod validation

## Goal
`npm install` succeeds, `npx tsc --noEmit` passes (typecheck), `npm test`
passes (a Vitest unit test of the Zod schema), and the dev server responds:
`GET /` returns the HTML page, `POST /api/items` with valid JSON creates an
item (201), `POST /api/items` with invalid JSON returns 400, `GET /api/items`
returns the list. Mirrors a Next.js app with Prisma + Zod validation: Next.js
App Router + Prisma + Zod-validated API routes + server-rendered page.

## Prerequisites
Starter scaffold in the work dir:
  - `package.json` (name: "items-app", no deps yet)
  - `tsconfig.json` (basic strict config)
  - `next.config.ts` (minimal)

## Spec — the app to build
A Next.js 15+ App Router app with a Prisma-backed CRUD API for "items" (a
simple name + description entity). One page (`/`) that server-renders the
item list + a create form. One API route (`POST /api/items` + `GET /api/items`)
with Zod input validation on POST. Prisma with SQLite (file-based, no external
DB needed).

### Prisma schema (prisma/schema.prisma)
  generator client { provider = "prisma-client-js" }
  datasource db { provider = "sqlite"; url = "file:./dev.db" }
  model Item {
    id          String   @id @default(cuid())
    name        String
    description String
    createdAt   DateTime @default(now())
  }

### Files to produce
  package.json              deps: next (15+), react, react-dom; devDeps: typescript, @types/react, @types/node, prisma, @prisma/client, zod, vitest, @vitejs/plugin-react, jsdom. Scripts: dev, build, test, typecheck (tsc --noEmit).
  tsconfig.json             strict, target ES2022, moduleResolution bundler, jsx preserve, lib [dom, dom.iterable, esnext], paths for ~/* -> src/*.
  next.config.ts            minimal (no special config needed).
  prisma/schema.prisma      the schema above.
  src/lib/prisma.ts         export a singleton PrismaClient instance.
  src/lib/validation.ts     export `createItemSchema = z.object({ name: z.string().min(1).max(100), description: z.string().min(1).max(500) })` and `type CreateItemInput = z.infer<typeof createItemSchema>`.
  src/lib/validation.test.ts a Vitest test: valid input parses, empty name fails, name > 100 chars fails, empty description fails.
  src/app/layout.tsx        root layout (html + body, minimal).
  src/app/page.tsx          server component: fetches items via prisma, renders a list + a client-side create form (the form POSTs to /api/items as JSON; on success, refresh the page — a simple form with onSubmit that fetch + router.refresh() is fine, or even simpler: a plain HTML form that posts to /api/items and redirects back).
  src/app/api/items/route.ts POST: parse JSON body, validate with createItemSchema, on failure return 400 + errors; on success prisma.item.create + return 201 + the created item. GET: prisma.item.findMany + return 200 + the list.

### Constraints
  - Use Prisma with the **sqlite** provider (file-based `dev.db` — no external Postgres needed for this bench).
  - Use `zod` for API input validation — this is the security boundary (the standard pattern for a Next.js app with Prisma + API route + Zod validation).
  - The API route must return **400** with the Zod errors on invalid input, **201** + the created item on success.
  - The page (`/`) must server-render the item list (read from Prisma in the server component).
  - Use Next.js App Router (`src/app/` layout). NOT the pages router.
  - Run `npx prisma generate` + `npx prisma db push` after writing the schema so the client is generated + the DB is created.

## Steps
1. Write package.json + tsconfig.json + next.config.ts.
2. `npm install` (this pulls next, react, prisma, zod, vitest — takes a minute).
3. Write prisma/schema.prisma. Run `npx prisma generate && npx prisma db push`.
4. Write src/lib/{prisma.ts, validation.ts, validation.test.ts}.
5. Write src/app/{layout.tsx, page.tsx, api/items/route.ts}.
6. `npm run typecheck` (tsc --noEmit) — fix type errors.
7. `npm test` (vitest) — the validation test must pass.
8. Optionally start the dev server briefly to confirm it responds.

## Acceptance (harness-verified)
1. `npm install` exits 0.
2. `npx tsc --noEmit` exits 0 (typecheck clean).
3. `npm test` exits 0 (vitest passes).
4. Dev server starts; `GET /` returns 200 + HTML containing an item list or a form; `POST /api/items` with valid JSON (`{"name":"Test","description":"desc"}`) returns 201; `POST /api/items` with invalid JSON (`{"name":"","description":""}`) returns 400; `GET /api/items` returns 200 + a JSON array.

## Out of scope
Auth/Auth.js, sessions, CSRF, Postgres, Docker, Tailwind, RBAC, middleware, themes. This is the smallest real Next.js app with Prisma + API route + Zod validation — a working CRUD app with Zod-validated API + server-rendered page.

## Commit
You do not commit; the harness owns the work dir.
"""

STARTER = {
    "package.json": json.dumps({
        "name": "items-app",
        "version": "0.1.0",
        "private": True,
        "scripts": {"dev": "next dev", "build": "next build", "test": "vitest run", "typecheck": "tsc --noEmit"},
        "dependencies": {},
        "devDependencies": {}
    }, indent=2),
    "tsconfig.json": json.dumps({
        "compilerOptions": {
            "target": "ES2022", "lib": ["dom", "dom.iterable", "esnext"],
            "allowJs": True, "skipLibCheck": True, "strict": True,
            "noEmit": True, "esModuleInterop": True, "module": "esnext",
            "moduleResolution": "bundler", "resolveJsonModule": True,
            "isolatedModules": True, "jsx": "preserve", "incremental": True,
            "plugins": [{"name": "next"}],
            "paths": {"~/*": ["./src/*"]}
        },
        "include": ["next-env.d.ts", "**/*.ts", "**/*.tsx", ".next/types/**/*.ts"],
        "exclude": ["node_modules"]
    }, indent=2),
    "next.config.ts": "import type { NextConfig } from 'next'\n\nconst config: NextConfig = {}\n\nexport default config\n",
}

# The reference is minimal but complete. Prisma + sqlite is the trickiest part
# to get self-contained — we use the prisma sqlite provider with a file-based db.
REFERENCE = {
    "package.json": json.dumps({
        "name": "items-app",
        "version": "0.1.0",
        "private": True,
        "scripts": {"dev": "next dev", "build": "next build", "start": "next start", "test": "vitest run", "typecheck": "tsc --noEmit"},
        "dependencies": {
            "next": "^15",
            "react": "^19",
            "react-dom": "^19",
            "@prisma/client": "^6",
            "zod": "^3",
        },
        "devDependencies": {
            "typescript": "^5",
            "@types/react": "^19",
            "@types/node": "^22",
            "prisma": "^6",
            "vitest": "^2",
            "@vitejs/plugin-react": "^4",
            "jsdom": "^25",
        }
    }, indent=2),
    "tsconfig.json": STARTER["tsconfig.json"],
    "next.config.ts": STARTER["next.config.ts"],
    "vitest.config.ts": '''import { defineConfig } from 'vitest/config'
export default defineConfig({
  test: { environment: 'node', include: ['src/**/*.test.ts'] },
})
''',
    "prisma/schema.prisma": '''generator client {
  provider = "prisma-client-js"
}

datasource db {
  provider = "sqlite"
  url      = "file:./dev.db"
}

model Item {
  id          String   @id @default(cuid())
  name        String
  description String
  createdAt   DateTime @default(now())
}
''',
    "src/lib/prisma.ts": '''import { PrismaClient } from '@prisma/client'
const globalForPrisma = globalThis as unknown as { prisma: PrismaClient | undefined }
export const prisma = globalForPrisma.prisma ?? new PrismaClient()
if (process.env.NODE_ENV !== 'production') globalForPrisma.prisma = prisma
''',
    "src/lib/validation.ts": '''import { z } from 'zod'

export const createItemSchema = z.object({
  name: z.string().min(1).max(100),
  description: z.string().min(1).max(500),
})

export type CreateItemInput = z.infer<typeof createItemSchema>
''',
    "src/lib/validation.test.ts": '''import { describe, it, expect } from 'vitest'
import { createItemSchema } from './validation'

describe('createItemSchema', () => {
  it('accepts valid input', () => {
    const result = createItemSchema.safeParse({ name: 'Test', description: 'desc' })
    expect(result.success).toBe(true)
  })
  it('rejects empty name', () => {
    const result = createItemSchema.safeParse({ name: '', description: 'desc' })
    expect(result.success).toBe(false)
  })
  it('rejects name > 100 chars', () => {
    const result = createItemSchema.safeParse({ name: 'x'.repeat(101), description: 'desc' })
    expect(result.success).toBe(false)
  })
  it('rejects empty description', () => {
    const result = createItemSchema.safeParse({ name: 'Test', description: '' })
    expect(result.success).toBe(false)
  })
})
''',
    "src/app/layout.tsx": '''import type { ReactNode } from 'react'

export default function RootLayout({ children }: { children: ReactNode }) {
  return (
    <html lang="en">
      <body>{children}</body>
    </html>
  )
}
''',
    "src/app/page.tsx": '''import { prisma } from '~/lib/prisma'

export default async function Home() {
  const items = await prisma.item.findMany({ orderBy: { createdAt: 'desc' } })
  return (
    <main>
      <h1>Items</h1>
      <form action="/api/items" method="POST" id="createForm">
        <input name="name" placeholder="Name" required />
        <input name="description" placeholder="Description" required />
        <button type="submit">Add</button>
      </form>
      <ul>
        {items.map((item) => (
          <li key={item.id}>{item.name}: {item.description}</li>
        ))}
      </ul>
    </main>
  )
}
''',
    "src/app/api/items/route.ts": '''import { NextRequest, NextResponse } from 'next/server'
import { prisma } from '~/lib/prisma'
import { createItemSchema } from '~/lib/validation'

export async function GET() {
  const items = await prisma.item.findMany({ orderBy: { createdAt: 'desc' } })
  return NextResponse.json(items)
}

export async function POST(request: NextRequest) {
  let body: unknown
  try {
    body = await request.json()
  } catch {
    return NextResponse.json({ error: 'invalid JSON' }, { status: 400 })
  }
  const parsed = createItemSchema.safeParse(body)
  if (!parsed.success) {
    return NextResponse.json({ error: 'validation failed', issues: parsed.error.issues }, { status: 400 })
  }
  const item = await prisma.item.create({ data: parsed.data })
  return NextResponse.json(item, { status: 201 })
}
''',
    "next-env.d.ts": '/// <reference types="next" />\n/// <reference types="next/image-types/global" />\n',
    ".gitignore": "node_modules\ndev.db*\n.next\n",
}

def _materialize(files, dest):
    shutil.rmtree(dest, ignore_errors=True)
    os.makedirs(dest, exist_ok=True)
    for path, content in files.items():
        fp = os.path.join(dest, path)
        os.makedirs(os.path.dirname(fp), exist_ok=True)
        with open(fp, "w") as f: f.write(content)
    return dest

def _run(cmd, cwd, timeout=600, env=None):
    try:
        p = subprocess.run(cmd, cwd=cwd, shell=True, capture_output=True, text=True, timeout=timeout, env=env)
    except subprocess.TimeoutExpired:
        return "TIMEOUT"
    return f"exit={p.returncode}\n{(p.stdout + p.stderr).strip()}"

def _free_port():
    s = socket.socket(socket.AF_INET, socket.SOCK_STREAM); s.bind(("", 0))
    p = s.getsockname()[1]; s.close(); return p

def _has_real_app(work_dir):
    if not os.path.isfile(os.path.join(work_dir, "src", "app", "page.tsx")): return False
    if not os.path.isfile(os.path.join(work_dir, "src", "app", "api", "items", "route.ts")): return False
    if not os.path.isfile(os.path.join(work_dir, "prisma", "schema.prisma")): return False
    return True

def gates(work_dir):
    g = {}
    if not _has_real_app(work_dir):
        return {"install": False, "typecheck": False, "test": False, "serves": False}
    # install
    g["install"] = _run("npm install", work_dir, timeout=300).startswith("exit=0")
    if not g["install"]:
        g["typecheck"] = False; g["test"] = False; g["serves"] = False; return g
    # prisma generate + db push
    _run("npx prisma generate", work_dir, timeout=120)
    _run("npx prisma db push", work_dir, timeout=120)
    # typecheck
    g["typecheck"] = _run("npx tsc --noEmit", work_dir, timeout=120).startswith("exit=0")
    # test
    g["test"] = _run("npm test", work_dir, timeout=120).startswith("exit=0")
    # serves: start dev server, test the API + page
    port = _free_port()
    env = dict(os.environ, PORT=str(port))
    try:
        proc = subprocess.Popen(["npx", "next", "dev", "-p", str(port)], cwd=work_dir,
                                env=env, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
    except Exception:
        g["serves"] = False; return g
    try:
        # wait for the dev server to be ready (can take 10-30s on first compile)
        for _ in range(60):
            try:
                with urllib.request.urlopen(f"http://127.0.0.1:{port}/", timeout=2) as r:
                    if r.status == 200:
                        page_html = r.read().decode(errors="replace")
                        break
            except Exception:
                time.sleep(1)
        else:
            g["serves"] = False; return g
        page_ok = "<h1>Items</h1>" in page_html or "Items" in page_html
        # POST valid
        base = f"http://127.0.0.1:{port}"
        valid_data = json.dumps({"name": "BenchItem", "description": "from harness"}).encode()
        req = urllib.request.Request(f"{base}/api/items", data=valid_data,
                                     headers={"Content-Type": "application/json"}, method="POST")
        post_ok = False
        try:
            with urllib.request.urlopen(req, timeout=15) as r:
                post_ok = r.status == 201
        except urllib.error.HTTPError as e:
            post_ok = False
        # POST invalid (should 400)
        invalid_data = json.dumps({"name": "", "description": ""}).encode()
        req2 = urllib.request.Request(f"{base}/api/items", data=invalid_data,
                                      headers={"Content-Type": "application/json"}, method="POST")
        reject_ok = False
        try:
            with urllib.request.urlopen(req2, timeout=15) as r:
                reject_ok = False  # should have been 400
        except urllib.error.HTTPError as e:
            reject_ok = e.code == 400
        # GET list
        get_ok = False
        try:
            with urllib.request.urlopen(f"{base}/api/items", timeout=15) as r:
                items = json.loads(r.read().decode())
                get_ok = isinstance(items, list) and any(i.get("name") == "BenchItem" for i in items)
        except Exception:
            pass
        g["serves"] = page_ok and post_ok and reject_ok and get_ok
        if not g["serves"]:
            g["_serves_detail"] = f"page={page_ok} post={post_ok} reject400={reject_ok} get={get_ok}"
    finally:
        proc.send_signal(signal.SIGTERM)
        try: proc.wait(timeout=10)
        except subprocess.TimeoutExpired: proc.kill()
    return g

def validate(work_dir):
    _materialize(REFERENCE, dest=work_dir)
    print("npm install...")
    g = gates(work_dir)
    print(f"gates: { {k:v for k,v in g.items() if not k.startswith('_')} }")
    if not g.get("serves") and "_serves_detail" in g:
        print(f"  serves detail: {g['_serves_detail']}")
    ok = all(v for k, v in g.items() if not k.startswith("_"))
    print(f"-> {'OK' if ok else 'BAD'}")
    return ok

if __name__ == "__main__":
    wd = os.path.join(HERE, "..", "..", "..", "work", "validate_ts_next")
    validate(wd)