feat(olympics): agent Olympics benchmark — multi-language arena, results ledger, model-aware budgets

Fourth benchmark axis: teams of LLM agents deliberate in a room, implement
code in an isolated VM, and are scored deterministically on correctness/speed.

- Multi-language adapter (python/js/go/rust/bash) via MultiPL-E continuation mode
- Append-only JSONL ledger with status tracking (ok/dnf/killed/error) so
  budget-exhausted or crashed runs still record a row (fixes selection bias)
- Model-aware wall-clock scaling (U-shaped by param count; 3x for reasoning)
- Self-owned SIGALRM/SIGTERM watchdog (RunTimeout: BaseException so broad
  except Exception handlers in the infer/completion path can't swallow it)
- Seed forwarded to Ollama sampler + markdown-fence stripping in completion.py
This commit is contained in:
leetcrypt
2026-06-27 10:17:23 -07:00
parent 7fb3911550
commit df8f1881d8
19 changed files with 2962 additions and 2 deletions
+62
View File
@@ -0,0 +1,62 @@
"""Arena orchestrator — wire a team + challenge through the loop and persist.
This is the M1 entry the launcher calls: build the room substrate, the VM, and a
manifest-stamped transcript; run the phase machine; freeze the transcript; score
it deterministically. The run/score split mirrors ``bench/score.py`` — results
persist so ``score``/``replay`` never re-run a model.
"""
from __future__ import annotations
import time
from pathlib import Path
from . import comms, scoring, transcript as T
from .challenge import Challenge
from .loop import Budget, run_event
from .team import Team
from .vm import VM
def run_one(team: Team, challenge: Challenge, *, room_kind: str = "local",
runtime: str = "auto", budget: Budget | None = None,
profile: str = "balanced", seed: int = 0,
host: str = "http://127.0.0.1:11434",
out_dir: str = "/tmp/hh-olympics/runs",
room_kwargs: dict | None = None, progress=None) -> dict:
budget = budget or Budget()
member_names = [m.name for m in team.members]
room = comms.make_room(room_kind, member_names, **(room_kwargs or {}))
manifest = {
"models": team.models, "seed": seed, "topology": team.topology,
"framing": team.framing, "room_substrate": room.substrate,
"runtime": runtime, "host": host,
"budget": {"deliberate_rounds": budget.deliberate_rounds,
"implement_attempts": budget.implement_attempts,
"max_tokens": budget.max_tokens,
"wall_clock_s": budget.wall_clock_s},
"challenge_meta": challenge.meta, "profile": profile,
"created": time.strftime("%Y-%m-%dT%H:%M:%S"),
}
tx = T.Transcript(team.id, challenge.id, manifest)
vm = VM(challenge.lang, runtime_kind=runtime)
room.connect()
try:
outcome = run_event(team, challenge, room, vm, tx, budget,
host=host, seed=seed, progress=progress)
finally:
room.close()
run_dir = Path(out_dir) / f"{team.id}__{challenge.id}"
tx_path = run_dir / "transcript.json"
tx.save(tx_path)
score = scoring.score_event(outcome, profile=profile,
budget={"max_rounds": budget.max_rounds})
score["transcript"] = str(tx_path)
score["room_substrate"] = room.substrate
import json
(run_dir / "score.json").write_text(json.dumps(score, indent=2))
return score