feat(olympics): agent Olympics benchmark — multi-language arena, results ledger, model-aware budgets
Fourth benchmark axis: teams of LLM agents deliberate in a room, implement code in an isolated VM, and are scored deterministically on correctness/speed. - Multi-language adapter (python/js/go/rust/bash) via MultiPL-E continuation mode - Append-only JSONL ledger with status tracking (ok/dnf/killed/error) so budget-exhausted or crashed runs still record a row (fixes selection bias) - Model-aware wall-clock scaling (U-shaped by param count; 3x for reasoning) - Self-owned SIGALRM/SIGTERM watchdog (RunTimeout: BaseException so broad except Exception handlers in the infer/completion path can't swallow it) - Seed forwarded to Ollama sampler + markdown-fence stripping in completion.py
This commit is contained in:
leetcrypt
@@ -0,0 +1,250 @@
|
||||
"""Append-only results ledger — the research record for Olympics runs.
|
||||
|
||||
Each event already writes a ``score.json``/``transcript.json`` under a
|
||||
``<team>__<challenge>`` directory, but that directory is *overwritten* on rerun,
|
||||
so it cannot answer "how did this team trend over time?". The ledger fixes that:
|
||||
one flat JSON line per run, appended forever, keyed by a unique ``run_id`` and
|
||||
stamped with the wall-clock time + git commit so a result is reproducible.
|
||||
|
||||
JSONL is chosen on purpose — it is append-safe under concurrency, streamable,
|
||||
and loads in one line from pandas (``pd.read_json(path, lines=True)``), jq, or
|
||||
plain ``json.loads`` per line. ``leaderboard()`` aggregates it without re-running
|
||||
any model (same philosophy as re-scoring a transcript under a new profile).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
import socket
|
||||
import subprocess
|
||||
import time
|
||||
import uuid
|
||||
from pathlib import Path
|
||||
from statistics import median
|
||||
|
||||
DEFAULT_PATH = Path.home() / ".cache" / "hh-bench" / "olympics" / "ledger.jsonl"
|
||||
|
||||
# Stable column order so the JSONL is human-diffable and schema-clear.
|
||||
FIELDS = (
|
||||
"run_id", "ts", "status", "team", "topology", "framing", "models",
|
||||
"language", "suite", "challenge", "mode", "room", "runtime", "profile",
|
||||
"seed", "code_model", "correct", "submitted", "composite", "pass@1",
|
||||
"rounds", "rounds_to_green", "wall_clock_s", "wall_clock_budget",
|
||||
"budget_scale", "total_tokens", "tokens_in", "tokens_out",
|
||||
"cost_normalized", "defect_class", "model_baseline_correct",
|
||||
"n_penalties", "error", "git", "host", "transcript",
|
||||
)
|
||||
|
||||
# A completed-and-graded run vs the ways a run can fail to produce a grade.
|
||||
STATUS_OK = "ok" # ran to a hidden-test verdict
|
||||
STATUS_DNF = "dnf" # graceful budget exhaustion (soft wall-clock/token cap)
|
||||
STATUS_KILLED = "killed" # hard deadline / external SIGTERM / watchdog
|
||||
STATUS_ERROR = "error" # uncaught exception mid-run
|
||||
|
||||
|
||||
def _git_sha() -> str:
|
||||
try:
|
||||
out = subprocess.run(["git", "rev-parse", "--short", "HEAD"],
|
||||
capture_output=True, text=True, timeout=5)
|
||||
return out.stdout.strip() if out.returncode == 0 else ""
|
||||
except Exception: # noqa: BLE001
|
||||
return ""
|
||||
|
||||
|
||||
def _path(path: str | os.PathLike | None) -> Path:
|
||||
return Path(path) if path else DEFAULT_PATH
|
||||
|
||||
|
||||
def row_from_score(score: dict, *, team, challenge, mode: str, room: str,
|
||||
runtime: str, seed: int, suite: str = "mbpp",
|
||||
code_model: str | None = None,
|
||||
wall_clock_budget: float | None = None,
|
||||
budget_scale: float | None = None) -> dict:
|
||||
"""Flatten a score dict + run context into one ledger row."""
|
||||
tok = score.get("tokens", {}) or {}
|
||||
return {
|
||||
"run_id": uuid.uuid4().hex[:12],
|
||||
"ts": time.strftime("%Y-%m-%dT%H:%M:%S"),
|
||||
"status": STATUS_OK,
|
||||
"team": team.id,
|
||||
"topology": team.topology,
|
||||
"framing": team.framing,
|
||||
"models": team.models,
|
||||
"language": challenge.language,
|
||||
"suite": suite,
|
||||
"challenge": challenge.id,
|
||||
"mode": mode,
|
||||
"room": score.get("room_substrate", room),
|
||||
"runtime": runtime,
|
||||
"profile": score.get("profile"),
|
||||
"seed": seed,
|
||||
"code_model": code_model,
|
||||
"correct": bool(score.get("correct")),
|
||||
"submitted": bool(score.get("submitted")),
|
||||
"composite": score.get("composite"),
|
||||
"pass@1": score.get("pass@1"),
|
||||
"rounds": score.get("rounds"),
|
||||
"rounds_to_green": score.get("rounds_to_green"),
|
||||
"wall_clock_s": score.get("wall_clock_s"),
|
||||
"wall_clock_budget": wall_clock_budget,
|
||||
"budget_scale": budget_scale,
|
||||
"total_tokens": score.get("total_tokens"),
|
||||
"tokens_in": tok.get("in", 0),
|
||||
"tokens_out": tok.get("out", 0),
|
||||
"cost_normalized": score.get("cost_normalized"),
|
||||
"defect_class": score.get("defect_class"),
|
||||
"model_baseline_correct": score.get("model_baseline_correct"),
|
||||
"n_penalties": len(score.get("penalties", []) or []),
|
||||
"error": "",
|
||||
"git": _git_sha(),
|
||||
"host": socket.gethostname(),
|
||||
"transcript": score.get("transcript"),
|
||||
}
|
||||
|
||||
|
||||
def _write(row: dict, path: str | os.PathLike | None) -> dict:
|
||||
p = _path(path)
|
||||
p.parent.mkdir(parents=True, exist_ok=True)
|
||||
with open(p, "a") as f:
|
||||
f.write(json.dumps({k: row.get(k) for k in FIELDS}) + "\n")
|
||||
return row
|
||||
|
||||
|
||||
def append(score: dict, *, team, challenge, mode: str, room: str,
|
||||
runtime: str, seed: int, suite: str = "mbpp",
|
||||
code_model: str | None = None,
|
||||
wall_clock_budget: float | None = None,
|
||||
budget_scale: float | None = None,
|
||||
path: str | os.PathLike | None = None) -> dict:
|
||||
"""Append one completed run to the ledger and return the row written."""
|
||||
row = row_from_score(score, team=team, challenge=challenge, mode=mode,
|
||||
room=room, runtime=runtime, seed=seed, suite=suite,
|
||||
code_model=code_model,
|
||||
wall_clock_budget=wall_clock_budget,
|
||||
budget_scale=budget_scale)
|
||||
return _write(row, path)
|
||||
|
||||
|
||||
def append_incomplete(*, team, challenge, mode: str, room: str, runtime: str,
|
||||
seed: int, status: str, error: str = "",
|
||||
suite: str = "mbpp", code_model: str | None = None,
|
||||
wall_clock_budget: float | None = None,
|
||||
budget_scale: float | None = None,
|
||||
path: str | os.PathLike | None = None) -> dict:
|
||||
"""Append a run that never produced a hidden-test verdict (killed / errored
|
||||
/ hard-DNF). Keeps the research record complete so the ledger isn't biased
|
||||
toward runs that happened to finish (the selection-bias fix)."""
|
||||
row = {k: None for k in FIELDS}
|
||||
row.update({
|
||||
"run_id": uuid.uuid4().hex[:12],
|
||||
"ts": time.strftime("%Y-%m-%dT%H:%M:%S"),
|
||||
"status": status,
|
||||
"team": team.id, "topology": team.topology, "framing": team.framing,
|
||||
"models": team.models, "language": challenge.language, "suite": suite,
|
||||
"challenge": challenge.id, "mode": mode, "room": room,
|
||||
"runtime": runtime, "seed": seed, "code_model": code_model,
|
||||
"correct": False, "submitted": False, "n_penalties": 0,
|
||||
"wall_clock_budget": wall_clock_budget, "budget_scale": budget_scale,
|
||||
"error": error[:200], "git": _git_sha(), "host": socket.gethostname(),
|
||||
})
|
||||
return _write(row, path)
|
||||
|
||||
|
||||
def load(path: str | os.PathLike | None = None) -> list[dict]:
|
||||
p = _path(path)
|
||||
if not p.exists():
|
||||
return []
|
||||
rows = []
|
||||
for line in p.read_text().splitlines():
|
||||
line = line.strip()
|
||||
if line:
|
||||
try:
|
||||
rows.append(json.loads(line))
|
||||
except json.JSONDecodeError:
|
||||
continue
|
||||
return rows
|
||||
|
||||
|
||||
def _matches(row: dict, filters: dict) -> bool:
|
||||
for k, v in filters.items():
|
||||
if v is None:
|
||||
continue
|
||||
rv = row.get(k)
|
||||
if k == "models": # substring match against any model in the team
|
||||
if not any(v in m for m in (rv or [])):
|
||||
return False
|
||||
elif k == "since":
|
||||
if (row.get("ts") or "") < v:
|
||||
return False
|
||||
elif rv != v:
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
def leaderboard(rows: list[dict], *, by: str = "team",
|
||||
filters: dict | None = None) -> list[dict]:
|
||||
"""Aggregate rows into a leaderboard grouped by ``by`` (any row field, or a
|
||||
'+'-joined composite like 'team+language'). Ranked by solve rate."""
|
||||
filters = filters or {}
|
||||
keys = by.split("+")
|
||||
|
||||
def group_key(r):
|
||||
parts = []
|
||||
for k in keys:
|
||||
v = r.get(k)
|
||||
if isinstance(v, list):
|
||||
# collapse a same-model team to one tag; keep mixed teams joined.
|
||||
uniq = list(dict.fromkeys(v))
|
||||
parts.append("/".join(uniq))
|
||||
else:
|
||||
parts.append(str(v))
|
||||
return " · ".join(parts)
|
||||
|
||||
groups: dict[str, list[dict]] = {}
|
||||
for r in rows:
|
||||
if _matches(r, filters):
|
||||
groups.setdefault(group_key(r), []).append(r)
|
||||
|
||||
out = []
|
||||
for g, rs in groups.items():
|
||||
attempted = len(rs)
|
||||
# solve_rate is computed over *completed* runs only (status == ok) so an
|
||||
# infra kill never masquerades as a capability failure; killed/errored
|
||||
# runs are surfaced separately as a reliability signal.
|
||||
done = [r for r in rs if (r.get("status") or "ok") == STATUS_OK]
|
||||
incomplete = attempted - len(done)
|
||||
solved = [r for r in done if r.get("correct")]
|
||||
greens = [r["rounds_to_green"] for r in solved
|
||||
if r.get("rounds_to_green") is not None]
|
||||
secs = [r["wall_clock_s"] for r in done if r.get("wall_clock_s") is not None]
|
||||
toks = [r["total_tokens"] for r in done if r.get("total_tokens") is not None]
|
||||
out.append({
|
||||
"group": g, "n": attempted, "completed": len(done),
|
||||
"incomplete": incomplete, "solved": len(solved),
|
||||
"solve_rate": round(len(solved) / len(done), 3) if done else None,
|
||||
"median_green": median(greens) if greens else None,
|
||||
"median_s": round(median(secs), 1) if secs else None,
|
||||
"median_tokens": int(median(toks)) if toks else None,
|
||||
})
|
||||
out.sort(key=lambda d: (d["solve_rate"] if d["solve_rate"] is not None
|
||||
else -1.0, d["completed"]), reverse=True)
|
||||
return out
|
||||
|
||||
|
||||
def print_leaderboard(agg: list[dict], by: str) -> None:
|
||||
print("=" * 80)
|
||||
print(f"olympics leaderboard · by={by} · groups={len(agg)}")
|
||||
print("-" * 80)
|
||||
print(f"{'group':<30}{'done':>5}{'dnf':>5}{'solved':>7}{'rate':>7}"
|
||||
f"{'med_grn':>8}{'med_s':>8}{'med_tok':>9}")
|
||||
print("-" * 80)
|
||||
for d in agg:
|
||||
g = d["group"] if len(d["group"]) <= 29 else d["group"][:28] + "…"
|
||||
rate = "—" if d["solve_rate"] is None else f"{d['solve_rate']:.3f}"
|
||||
print(f"{g:<30}{d['completed']:>5}{d['incomplete']:>5}{d['solved']:>7}"
|
||||
f"{rate:>7}"
|
||||
f"{('—' if d['median_green'] is None else d['median_green']):>8}"
|
||||
f"{('—' if d['median_s'] is None else d['median_s']):>8}"
|
||||
f"{('—' if d['median_tokens'] is None else d['median_tokens']):>9}")
|
||||
print("=" * 80)
|
||||
Reference in New Issue
Block a user