feat(olympics): agent Olympics benchmark — multi-language arena, results ledger, model-aware budgets

Fourth benchmark axis: teams of LLM agents deliberate in a room, implement
code in an isolated VM, and are scored deterministically on correctness/speed.

- Multi-language adapter (python/js/go/rust/bash) via MultiPL-E continuation mode
- Append-only JSONL ledger with status tracking (ok/dnf/killed/error) so
  budget-exhausted or crashed runs still record a row (fixes selection bias)
- Model-aware wall-clock scaling (U-shaped by param count; 3x for reasoning)
- Self-owned SIGALRM/SIGTERM watchdog (RunTimeout: BaseException so broad
  except Exception handlers in the infer/completion path can't swallow it)
- Seed forwarded to Ollama sampler + markdown-fence stripping in completion.py
This commit is contained in:
leetcrypt
2026-06-27 10:17:23 -07:00
parent 7fb3911550
commit df8f1881d8
19 changed files with 2962 additions and 2 deletions
+53
View File
@@ -0,0 +1,53 @@
"""Build teams from config.
Config is a plain dict (loaded from YAML/JSON by the launcher). M1 needs only a
single same-model 2-member team; the schema is the SPEC §7 shape so M2+ scales
to multiple, mixed-model teams without changes.
{
"teams": [
{"id": "falcon", "topology": "star", "framing": "neutral",
"members": [
{"name": "archie", "model": "qwen2.5-coder:3b", "role": "architect"},
{"name": "bob", "model": "qwen2.5-coder:3b", "role": "builder"}
]}
]
}
"""
from __future__ import annotations
from .team import Member, Team
def build_team(spec: dict) -> Team:
framing = spec.get("framing", "neutral")
members = [
Member(name=m["name"], model=m["model"],
role=m.get("role", "peer"),
framing=m.get("framing", framing))
for m in spec["members"]
]
if not members:
raise ValueError(f"team {spec.get('id')!r} has no members")
return Team(id=spec["id"], members=members,
topology=spec.get("topology", "star"), framing=framing)
def build_teams(config: dict) -> list[Team]:
teams = [build_team(t) for t in config.get("teams", [])]
if not teams:
raise ValueError("config has no teams")
return teams
def same_model_team(model: str, *, team_id: str = "solo",
framing: str = "neutral") -> Team:
"""Convenience for M1: a 2-member architect+builder team on one model."""
return build_team({
"id": team_id, "topology": "star", "framing": framing,
"members": [
{"name": "archie", "model": model, "role": "architect"},
{"name": "bob", "model": model, "role": "builder"},
],
})