leetcrypt
df8f1881d8
Fourth benchmark axis: teams of LLM agents deliberate in a room, implement code in an isolated VM, and are scored deterministically on correctness/speed. - Multi-language adapter (python/js/go/rust/bash) via MultiPL-E continuation mode - Append-only JSONL ledger with status tracking (ok/dnf/killed/error) so budget-exhausted or crashed runs still record a row (fixes selection bias) - Model-aware wall-clock scaling (U-shaped by param count; 3x for reasoning) - Self-owned SIGALRM/SIGTERM watchdog (RunTimeout: BaseException so broad except Exception handlers in the infer/completion path can't swallow it) - Seed forwarded to Ollama sampler + markdown-fence stripping in completion.py
63 lines
2.4 KiB
Python
63 lines
2.4 KiB
Python
"""Arena orchestrator — wire a team + challenge through the loop and persist.
|
|
|
|
This is the M1 entry the launcher calls: build the room substrate, the VM, and a
|
|
manifest-stamped transcript; run the phase machine; freeze the transcript; score
|
|
it deterministically. The run/score split mirrors ``bench/score.py`` — results
|
|
persist so ``score``/``replay`` never re-run a model.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import time
|
|
from pathlib import Path
|
|
|
|
from . import comms, scoring, transcript as T
|
|
from .challenge import Challenge
|
|
from .loop import Budget, run_event
|
|
from .team import Team
|
|
from .vm import VM
|
|
|
|
|
|
def run_one(team: Team, challenge: Challenge, *, room_kind: str = "local",
|
|
runtime: str = "auto", budget: Budget | None = None,
|
|
profile: str = "balanced", seed: int = 0,
|
|
host: str = "http://127.0.0.1:11434",
|
|
out_dir: str = "/tmp/hh-olympics/runs",
|
|
room_kwargs: dict | None = None, progress=None) -> dict:
|
|
budget = budget or Budget()
|
|
member_names = [m.name for m in team.members]
|
|
room = comms.make_room(room_kind, member_names, **(room_kwargs or {}))
|
|
|
|
manifest = {
|
|
"models": team.models, "seed": seed, "topology": team.topology,
|
|
"framing": team.framing, "room_substrate": room.substrate,
|
|
"runtime": runtime, "host": host,
|
|
"budget": {"deliberate_rounds": budget.deliberate_rounds,
|
|
"implement_attempts": budget.implement_attempts,
|
|
"max_tokens": budget.max_tokens,
|
|
"wall_clock_s": budget.wall_clock_s},
|
|
"challenge_meta": challenge.meta, "profile": profile,
|
|
"created": time.strftime("%Y-%m-%dT%H:%M:%S"),
|
|
}
|
|
tx = T.Transcript(team.id, challenge.id, manifest)
|
|
vm = VM(challenge.lang, runtime_kind=runtime)
|
|
|
|
room.connect()
|
|
try:
|
|
outcome = run_event(team, challenge, room, vm, tx, budget,
|
|
host=host, seed=seed, progress=progress)
|
|
finally:
|
|
room.close()
|
|
|
|
run_dir = Path(out_dir) / f"{team.id}__{challenge.id}"
|
|
tx_path = run_dir / "transcript.json"
|
|
tx.save(tx_path)
|
|
|
|
score = scoring.score_event(outcome, profile=profile,
|
|
budget={"max_rounds": budget.max_rounds})
|
|
score["transcript"] = str(tx_path)
|
|
score["room_substrate"] = room.substrate
|
|
import json
|
|
(run_dir / "score.json").write_text(json.dumps(score, indent=2))
|
|
return score
|