feat(olympics): agent Olympics benchmark — multi-language arena, results ledger, model-aware budgets
Fourth benchmark axis: teams of LLM agents deliberate in a room, implement code in an isolated VM, and are scored deterministically on correctness/speed. - Multi-language adapter (python/js/go/rust/bash) via MultiPL-E continuation mode - Append-only JSONL ledger with status tracking (ok/dnf/killed/error) so budget-exhausted or crashed runs still record a row (fixes selection bias) - Model-aware wall-clock scaling (U-shaped by param count; 3x for reasoning) - Self-owned SIGALRM/SIGTERM watchdog (RunTimeout: BaseException so broad except Exception handlers in the infer/completion path can't swallow it) - Seed forwarded to Ollama sampler + markdown-fence stripping in completion.py
This commit is contained in:
@@ -0,0 +1,53 @@
|
||||
"""Build teams from config.
|
||||
|
||||
Config is a plain dict (loaded from YAML/JSON by the launcher). M1 needs only a
|
||||
single same-model 2-member team; the schema is the SPEC §7 shape so M2+ scales
|
||||
to multiple, mixed-model teams without changes.
|
||||
|
||||
{
|
||||
"teams": [
|
||||
{"id": "falcon", "topology": "star", "framing": "neutral",
|
||||
"members": [
|
||||
{"name": "archie", "model": "qwen2.5-coder:3b", "role": "architect"},
|
||||
{"name": "bob", "model": "qwen2.5-coder:3b", "role": "builder"}
|
||||
]}
|
||||
]
|
||||
}
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from .team import Member, Team
|
||||
|
||||
|
||||
def build_team(spec: dict) -> Team:
|
||||
framing = spec.get("framing", "neutral")
|
||||
members = [
|
||||
Member(name=m["name"], model=m["model"],
|
||||
role=m.get("role", "peer"),
|
||||
framing=m.get("framing", framing))
|
||||
for m in spec["members"]
|
||||
]
|
||||
if not members:
|
||||
raise ValueError(f"team {spec.get('id')!r} has no members")
|
||||
return Team(id=spec["id"], members=members,
|
||||
topology=spec.get("topology", "star"), framing=framing)
|
||||
|
||||
|
||||
def build_teams(config: dict) -> list[Team]:
|
||||
teams = [build_team(t) for t in config.get("teams", [])]
|
||||
if not teams:
|
||||
raise ValueError("config has no teams")
|
||||
return teams
|
||||
|
||||
|
||||
def same_model_team(model: str, *, team_id: str = "solo",
|
||||
framing: str = "neutral") -> Team:
|
||||
"""Convenience for M1: a 2-member architect+builder team on one model."""
|
||||
return build_team({
|
||||
"id": team_id, "topology": "star", "framing": framing,
|
||||
"members": [
|
||||
{"name": "archie", "model": model, "role": "architect"},
|
||||
{"name": "bob", "model": model, "role": "builder"},
|
||||
],
|
||||
})
|
||||
Reference in New Issue
Block a user