Model arena and router

"""A model arena on your own prompts, then a router that sends each kind of request to the cheapest model that wins it.

pip install openai requests
RUNCRATE_API_KEY=... python model_arena_router.py
Writes leaderboard.json and routes.json; route(kind, prompt) uses routes.json.
"""
import json, os, re, statistics, time
from concurrent.futures import ThreadPoolExecutor
from pathlib import Path

import requests
from openai import OpenAI

client = OpenAI(base_url="https://api.runcrate.ai/v1", api_key=os.environ["RUNCRATE_API_KEY"], max_retries=4)
CONTENDERS = [
    "openai/gpt-oss-120b",                  # free tier
    "deepseek-ai/DeepSeek-V4.1-Flash",      # free tier
    "Qwen/Qwen3.8-Flash",                   # free tier
    "mistralai/Mistral-Small-3.2-24B-Instruct-2506",
    "anthropic/claude-haiku-5.5",
]
JUDGE = "anthropic/claude-sonnet-5-5"
WITHIN = 0.95  # a model "wins" a kind if it scores within 5% of the best

# Your real traffic goes here: a few dozen prompts per kind is enough to see the split.
TASKS = [
    {"kind": "extract", "prompt": "Return only the invoice number: 'Ref INV-20931, due 3 Nov, total 412.00'", "answer": "INV-20931"},
    {"kind": "extract", "prompt": "Return only the email address: 'Ping me at dana.k@northwind.io or call 555-0142.'", "answer": "dana.k@northwind.io"},
    {"kind": "extract", "prompt": "Return only the date as YYYY-MM-DD: 'The contract starts on the fourth of March 2027.'", "answer": "2027-03-04"},
    {"kind": "classify", "prompt": "Label the ticket billing, bug or feature. Reply with one word. 'I was charged twice this month.'", "answer": "billing"},
    {"kind": "classify", "prompt": "Label the ticket billing, bug or feature. Reply with one word. 'The export button does nothing on Safari.'", "answer": "bug"},
    {"kind": "classify", "prompt": "Label the ticket billing, bug or feature. Reply with one word. 'Could you add dark mode to the dashboard?'", "answer": "feature"},
    {"kind": "math", "prompt": "A GPU costs $1.85 an hour. A job runs 3 hours 24 minutes. What does it cost in dollars? Reply with the number only, 2 decimals.", "answer": "6.29"},
    {"kind": "math", "prompt": "1.2M input tokens at $0.15 per million and 300K output tokens at $0.60 per million. Total dollars? Number only, 2 decimals.", "answer": "0.36"},
    {"kind": "math", "prompt": "If 7 workers label 420 images in 3 hours, how many images do 5 workers label in 4 hours? Number only.", "answer": "400"},
    {"kind": "write", "prompt": "Write a two-sentence apology to a customer whose GPU job was cancelled by our maintenance window, offering a credit."},
    {"kind": "write", "prompt": "Write a one-paragraph release note for: jobs can now retry on a new machine when the first one fails."},
    {"kind": "write", "prompt": "Write a three-line product description for a reusable glass water bottle with a bamboo lid."},
]

def ask(model: str, task: dict) -> dict:
    t0 = time.time()
    r = client.chat.completions.create(model=model, messages=[{"role": "user", "content": task["prompt"]}], max_tokens=800, temperature=0)
    text = (r.choices[0].message.content or "").strip()
    usage = r.usage.model_dump() if r.usage else {}
    return {"model": model, "kind": task["kind"], "prompt": task["prompt"], "text": text, "seconds": time.time() - t0,
            "cost_usd": float(usage.get("cost") or 0), "prompt_tokens": usage.get("prompt_tokens", 0), "completion_tokens": usage.get("completion_tokens", 0)}

def judge(answer: dict) -> tuple[float, float]:
    r = client.chat.completions.create(
        model=JUDGE, temperature=0, max_tokens=200,
        messages=[{"role": "user", "content": f"Score this answer from 1 to 10 for how well it does the task: clear, correct, the length asked for, no filler.\n\nTask: {answer['prompt']}\n\nAnswer:\n{answer['text']}"}],
        response_format={"type": "json_schema", "json_schema": {"name": "score", "strict": True, "schema": {
            "type": "object", "additionalProperties": False, "required": ["score", "reason"],
            "properties": {"score": {"type": "integer"}, "reason": {"type": "string"}}}}},
    )
    return json.loads(r.choices[0].message.content)["score"] / 10, float(getattr(r.usage, "cost", 0) or 0)

def grade(answer: dict, task: dict) -> float:
    if "answer" not in task:
        return -1.0  # judged below
    got = re.sub(r"[\s$.,]+$", "", answer["text"].strip().strip("`").lower())
    return 1.0 if got.endswith(task["answer"].lower()) else 0.0

def main() -> None:
    # Billed list prices, dollars per 1M tokens, to price each answer as a paying request.
    catalog = requests.get("https://www.runcrate.ai/api/models/catalog", timeout=30).json()["models"]
    prices = {m["id"]: (m["price_input"] or 0, m["price_output"] or 0) for m in catalog}
    t0 = time.time()
    jobs = [(m, t) for m in CONTENDERS for t in TASKS]
    with ThreadPoolExecutor(max_workers=16) as pool:
        answers = list(pool.map(lambda mt: ask(*mt), jobs))
    fan_out_s = time.time() - t0
    judge_cost = 0.0
    with ThreadPoolExecutor(max_workers=8) as pool:
        scored = list(pool.map(lambda at: (at[0], grade(*at)), zip(answers, [t for _, t in jobs])))
        todo = [a for a, s in scored if s < 0]
        judged = dict(zip(map(id, todo), pool.map(judge, todo)))
    for a, s in scored:
        if s < 0:
            s, c = judged[id(a)]; judge_cost += c
        a["score"] = s
        p_in, p_out = prices.get(a["model"], (0, 0))
        a["list_usd"] = (a["prompt_tokens"] * p_in + a["completion_tokens"] * p_out) / 1e6

    board = {}
    for kind in sorted({t["kind"] for t in TASKS}):
        rows = []
        for m in CONTENDERS:
            mine = [a for a in answers if a["model"] == m and a["kind"] == kind]
            rows.append({"model": m, "score": round(statistics.mean(a["score"] for a in mine), 3),
                         "p50_seconds": round(statistics.median(a["seconds"] for a in mine), 2),
                         "billed_usd": sum(a["cost_usd"] for a in mine), "list_usd_per_1k": round(1000 * statistics.mean(a["list_usd"] for a in mine), 4)})
        board[kind] = sorted(rows, key=lambda r: (-r["score"], r["list_usd_per_1k"]))
    routes = {}
    for kind, rows in board.items():
        best = rows[0]["score"]
        winners = [r for r in rows if r["score"] >= WITHIN * best]
        routes[kind] = min(winners, key=lambda r: (r["list_usd_per_1k"], r["p50_seconds"]))["model"]

    for kind, rows in board.items():
        print(f"\n{kind}  -> route to {routes[kind]}")
        for r in rows:
            print(f"  {r['model']:<48} score {r['score']:.2f}  p50 {r['p50_seconds']:5.2f}s  list ${r['list_usd_per_1k']:.4f}/1k  billed ${r['billed_usd']:.6f}")
    billed = sum(a["cost_usd"] for a in answers) + judge_cost
    print(f"\n{len(answers)} answers in {fan_out_s:.1f}s, judge ${judge_cost:.6f}, total billed ${billed:.6f}")
    Path("leaderboard.json").write_text(json.dumps(board, indent=2))
    Path("routes.json").write_text(json.dumps(routes, indent=2))
    Path("run.json").write_text(json.dumps({"answers": len(answers), "fan_out_seconds": fan_out_s, "judge_usd": judge_cost, "total_usd": billed, "routes": routes}, indent=2))

    print("\nrouted:", route("classify", "Label the ticket billing, bug or feature. Reply with one word. 'My invoice shows the wrong VAT number.'"))

def route(kind: str, prompt: str) -> str:
    model = json.loads(Path("routes.json").read_text()).get(kind, CONTENDERS[0])
    r = client.chat.completions.create(model=model, messages=[{"role": "user", "content": prompt}], max_tokens=800)
    return f"{model}: {r.choices[0].message.content.strip()}"

if __name__ == "__main__":
    main()

// A model arena on your own prompts, then a router that sends each kind of request to the cheapest model that wins it.
//
// npm install openai
// RUNCRATE_API_KEY=... npx tsx model-arena-router.ts
// Writes leaderboard.json and routes.json; route(kind, prompt) uses routes.json.
import { readFileSync, writeFileSync } from "node:fs";
import OpenAI from "openai";

const client = new OpenAI({ baseURL: "https://api.runcrate.ai/v1", apiKey: process.env.RUNCRATE_API_KEY, maxRetries: 4 });
const CONTENDERS = [
  "openai/gpt-oss-120b", // free tier
  "deepseek-ai/DeepSeek-V4.1-Flash", // free tier
  "Qwen/Qwen3.8-Flash", // free tier
  "mistralai/Mistral-Small-3.2-24B-Instruct-2506",
  "anthropic/claude-haiku-5.5",
];
const JUDGE = "anthropic/claude-sonnet-5-5";
const WITHIN = 0.95; // a model "wins" a kind if it scores within 5% of the best

type Task = { kind: string; prompt: string; answer?: string };
// Your real traffic goes here: a few dozen prompts per kind is enough to see the split.
const TASKS: Task[] = [
  { kind: "extract", prompt: "Return only the invoice number: 'Ref INV-20931, due 3 Nov, total 412.00'", answer: "INV-20931" },
  { kind: "extract", prompt: "Return only the email address: 'Ping me at dana.k@northwind.io or call 555-0142.'", answer: "dana.k@northwind.io" },
  { kind: "extract", prompt: "Return only the date as YYYY-MM-DD: 'The contract starts on the fourth of March 2027.'", answer: "2027-03-04" },
  { kind: "classify", prompt: "Label the ticket billing, bug or feature. Reply with one word. 'I was charged twice this month.'", answer: "billing" },
  { kind: "classify", prompt: "Label the ticket billing, bug or feature. Reply with one word. 'The export button does nothing on Safari.'", answer: "bug" },
  { kind: "classify", prompt: "Label the ticket billing, bug or feature. Reply with one word. 'Could you add dark mode to the dashboard?'", answer: "feature" },
  { kind: "math", prompt: "A GPU costs $1.85 an hour. A job runs 3 hours 24 minutes. What does it cost in dollars? Reply with the number only, 2 decimals.", answer: "6.29" },
  { kind: "math", prompt: "1.2M input tokens at $0.15 per million and 300K output tokens at $0.60 per million. Total dollars? Number only, 2 decimals.", answer: "0.36" },
  { kind: "math", prompt: "If 7 workers label 420 images in 3 hours, how many images do 5 workers label in 4 hours? Number only.", answer: "400" },
  { kind: "write", prompt: "Write a two-sentence apology to a customer whose GPU job was cancelled by our maintenance window, offering a credit." },
  { kind: "write", prompt: "Write a one-paragraph release note for: jobs can now retry on a new machine when the first one fails." },
  { kind: "write", prompt: "Write a three-line product description for a reusable glass water bottle with a bamboo lid." },
];

type Usage = { cost?: number; prompt_tokens?: number; completion_tokens?: number };
type Answer = { model: string; task: Task; text: string; seconds: number; cost: number; usage: Usage; score: number; listUsd: number };

async function ask(model: string, task: Task): Promise<Answer> {
  const t0 = Date.now();
  const r = await client.chat.completions.create({ model, messages: [{ role: "user", content: task.prompt }], max_tokens: 800, temperature: 0 });
  const usage: Usage = r.usage ?? {};
  return { model, task, text: (r.choices[0].message.content ?? "").trim(), seconds: (Date.now() - t0) / 1000, cost: usage.cost ?? 0, usage, score: 0, listUsd: 0 };
}

async function judge(a: Answer): Promise<[number, number]> {
  const r = await client.chat.completions.create({
    model: JUDGE,
    temperature: 0,
    max_tokens: 200,
    messages: [
      {
        role: "user",
        content: `Score this answer from 1 to 10 for how well it does the task: clear, correct, the length asked for, no filler.\n\nTask: ${a.task.prompt}\n\nAnswer:\n${a.text}`,
      },
    ],
    response_format: {
      type: "json_schema",
      json_schema: {
        name: "score",
        strict: true,
        schema: { type: "object", additionalProperties: false, required: ["score", "reason"], properties: { score: { type: "integer" }, reason: { type: "string" } } },
      },
    },
  });
  const usage: Usage = r.usage ?? {};
  return [JSON.parse(r.choices[0].message.content ?? "{}").score / 10, usage.cost ?? 0];
}

/** Run fn over items with at most `limit` in flight. */
async function pool<T, R>(items: T[], limit: number, fn: (t: T) => Promise<R>): Promise<R[]> {
  const out: R[] = new Array(items.length);
  let next = 0;
  await Promise.all(
    Array.from({ length: limit }, async () => {
      while (next < items.length) {
        const i = next++;
        out[i] = await fn(items[i]);
      }
    }),
  );
  return out;
}

const median = (xs: number[]) => [...xs].sort((a, b) => a - b)[Math.floor((xs.length - 1) / 2)];
const mean = (xs: number[]) => xs.reduce((a, b) => a + b, 0) / xs.length;

export async function route(kind: string, prompt: string): Promise<string> {
  const routes: Record<string, string> = JSON.parse(readFileSync("routes.json", "utf8"));
  const model = routes[kind] ?? CONTENDERS[0];
  const r = await client.chat.completions.create({ model, messages: [{ role: "user", content: prompt }], max_tokens: 800 });
  return `${model}: ${(r.choices[0].message.content ?? "").trim()}`;
}

// Billed list prices, dollars per 1M tokens, to price each answer as a paying request.
const catalog = (await (await fetch("https://www.runcrate.ai/api/models/catalog")).json()) as {
  models: { id: string; price_input: number | null; price_output: number | null }[];
};
const prices = new Map(catalog.models.map((m) => [m.id, [m.price_input ?? 0, m.price_output ?? 0]]));

const t0 = Date.now();
const answers = await pool(CONTENDERS.flatMap((m) => TASKS.map((t) => [m, t] as const)), 16, ([m, t]) => ask(m, t));
const fanOutSeconds = (Date.now() - t0) / 1000;
let judgeCost = 0;
for (const a of answers) {
  if (a.task.answer !== undefined) {
    const got = a.text.trim().replace(/^`+|`+$/g, "").toLowerCase().replace(/[\s$.,]+$/, "");
    a.score = got.endsWith(a.task.answer.toLowerCase()) ? 1 : 0;
  }
  const [pIn, pOut] = prices.get(a.model) ?? [0, 0];
  a.listUsd = ((a.usage.prompt_tokens ?? 0) * pIn + (a.usage.completion_tokens ?? 0) * pOut) / 1e6;
}
const toJudge = answers.filter((a) => a.task.answer === undefined);
const judged = await pool(toJudge, 8, judge);
for (const [a, [score, cost]] of toJudge.map((a, i) => [a, judged[i]] as const)) {
  a.score = score;
  judgeCost += cost;
}

const board: Record<string, { model: string; score: number; p50_seconds: number; billed_usd: number; list_usd_per_1k: number }[]> = {};
for (const kind of [...new Set(TASKS.map((t) => t.kind))].sort()) {
  board[kind] = CONTENDERS.map((model) => {
    const mine = answers.filter((a) => a.model === model && a.task.kind === kind);
    return {
      model,
      score: Math.round(mean(mine.map((a) => a.score)) * 1000) / 1000,
      p50_seconds: Math.round(median(mine.map((a) => a.seconds)) * 100) / 100,
      billed_usd: mine.reduce((s, a) => s + a.cost, 0),
      list_usd_per_1k: Math.round(1000 * mean(mine.map((a) => a.listUsd)) * 10_000) / 10_000,
    };
  }).sort((a, b) => b.score - a.score || a.list_usd_per_1k - b.list_usd_per_1k);
}
const routes: Record<string, string> = {};
for (const [kind, rows] of Object.entries(board)) {
  const winners = rows.filter((r) => r.score >= WITHIN * rows[0].score);
  routes[kind] = winners.sort((a, b) => a.list_usd_per_1k - b.list_usd_per_1k || a.p50_seconds - b.p50_seconds)[0].model;
}

for (const [kind, rows] of Object.entries(board)) {
  console.log(`\n${kind}  -> route to ${routes[kind]}`);
  for (const r of rows) {
    console.log(
      `  ${r.model.padEnd(48)} score ${r.score.toFixed(2)}  p50 ${r.p50_seconds.toFixed(2).padStart(5)}s  list $${r.list_usd_per_1k.toFixed(4)}/1k  billed $${r.billed_usd.toFixed(6)}`,
    );
  }
}
const billed = answers.reduce((s, a) => s + a.cost, 0) + judgeCost;
console.log(`\n${answers.length} answers in ${fanOutSeconds.toFixed(1)}s, judge $${judgeCost.toFixed(6)}, total billed $${billed.toFixed(6)}`);
writeFileSync("leaderboard.json", JSON.stringify(board, null, 2));
writeFileSync("routes.json", JSON.stringify(routes, null, 2));
writeFileSync("run.json", JSON.stringify({ answers: answers.length, fan_out_seconds: fanOutSeconds, judge_usd: judgeCost, total_usd: billed, routes }, null, 2));
console.log("\nrouted:", await route("classify", "Label the ticket billing, bug or feature. Reply with one word. 'My invoice shows the wrong VAT number.'"));