feat(olympics): agent Olympics benchmark — multi-language arena, results ledger, model-aware budgets
Fourth benchmark axis: teams of LLM agents deliberate in a room, implement code in an isolated VM, and are scored deterministically on correctness/speed. - Multi-language adapter (python/js/go/rust/bash) via MultiPL-E continuation mode - Append-only JSONL ledger with status tracking (ok/dnf/killed/error) so budget-exhausted or crashed runs still record a row (fixes selection bias) - Model-aware wall-clock scaling (U-shaped by param count; 3x for reasoning) - Self-owned SIGALRM/SIGTERM watchdog (RunTimeout: BaseException so broad except Exception handlers in the infer/completion path can't swallow it) - Seed forwarded to Ollama sampler + markdown-fence stripping in completion.py
This commit is contained in:
@@ -0,0 +1,62 @@
|
||||
"""Arena orchestrator — wire a team + challenge through the loop and persist.
|
||||
|
||||
This is the M1 entry the launcher calls: build the room substrate, the VM, and a
|
||||
manifest-stamped transcript; run the phase machine; freeze the transcript; score
|
||||
it deterministically. The run/score split mirrors ``bench/score.py`` — results
|
||||
persist so ``score``/``replay`` never re-run a model.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
from . import comms, scoring, transcript as T
|
||||
from .challenge import Challenge
|
||||
from .loop import Budget, run_event
|
||||
from .team import Team
|
||||
from .vm import VM
|
||||
|
||||
|
||||
def run_one(team: Team, challenge: Challenge, *, room_kind: str = "local",
|
||||
runtime: str = "auto", budget: Budget | None = None,
|
||||
profile: str = "balanced", seed: int = 0,
|
||||
host: str = "http://127.0.0.1:11434",
|
||||
out_dir: str = "/tmp/hh-olympics/runs",
|
||||
room_kwargs: dict | None = None, progress=None) -> dict:
|
||||
budget = budget or Budget()
|
||||
member_names = [m.name for m in team.members]
|
||||
room = comms.make_room(room_kind, member_names, **(room_kwargs or {}))
|
||||
|
||||
manifest = {
|
||||
"models": team.models, "seed": seed, "topology": team.topology,
|
||||
"framing": team.framing, "room_substrate": room.substrate,
|
||||
"runtime": runtime, "host": host,
|
||||
"budget": {"deliberate_rounds": budget.deliberate_rounds,
|
||||
"implement_attempts": budget.implement_attempts,
|
||||
"max_tokens": budget.max_tokens,
|
||||
"wall_clock_s": budget.wall_clock_s},
|
||||
"challenge_meta": challenge.meta, "profile": profile,
|
||||
"created": time.strftime("%Y-%m-%dT%H:%M:%S"),
|
||||
}
|
||||
tx = T.Transcript(team.id, challenge.id, manifest)
|
||||
vm = VM(challenge.lang, runtime_kind=runtime)
|
||||
|
||||
room.connect()
|
||||
try:
|
||||
outcome = run_event(team, challenge, room, vm, tx, budget,
|
||||
host=host, seed=seed, progress=progress)
|
||||
finally:
|
||||
room.close()
|
||||
|
||||
run_dir = Path(out_dir) / f"{team.id}__{challenge.id}"
|
||||
tx_path = run_dir / "transcript.json"
|
||||
tx.save(tx_path)
|
||||
|
||||
score = scoring.score_event(outcome, profile=profile,
|
||||
budget={"max_rounds": budget.max_rounds})
|
||||
score["transcript"] = str(tx_path)
|
||||
score["room_substrate"] = room.substrate
|
||||
import json
|
||||
(run_dir / "score.json").write_text(json.dumps(score, indent=2))
|
||||
return score
|
||||
Reference in New Issue
Block a user