leetcrypt
df8f1881d8
Fourth benchmark axis: teams of LLM agents deliberate in a room, implement code in an isolated VM, and are scored deterministically on correctness/speed. - Multi-language adapter (python/js/go/rust/bash) via MultiPL-E continuation mode - Append-only JSONL ledger with status tracking (ok/dnf/killed/error) so budget-exhausted or crashed runs still record a row (fixes selection bias) - Model-aware wall-clock scaling (U-shaped by param count; 3x for reasoning) - Self-owned SIGALRM/SIGTERM watchdog (RunTimeout: BaseException so broad except Exception handlers in the infer/completion path can't swallow it) - Seed forwarded to Ollama sampler + markdown-fence stripping in completion.py
54 lines
1.7 KiB
Python
54 lines
1.7 KiB
Python
"""Build teams from config.
|
|
|
|
Config is a plain dict (loaded from YAML/JSON by the launcher). M1 needs only a
|
|
single same-model 2-member team; the schema is the SPEC §7 shape so M2+ scales
|
|
to multiple, mixed-model teams without changes.
|
|
|
|
{
|
|
"teams": [
|
|
{"id": "falcon", "topology": "star", "framing": "neutral",
|
|
"members": [
|
|
{"name": "archie", "model": "qwen2.5-coder:3b", "role": "architect"},
|
|
{"name": "bob", "model": "qwen2.5-coder:3b", "role": "builder"}
|
|
]}
|
|
]
|
|
}
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
from .team import Member, Team
|
|
|
|
|
|
def build_team(spec: dict) -> Team:
|
|
framing = spec.get("framing", "neutral")
|
|
members = [
|
|
Member(name=m["name"], model=m["model"],
|
|
role=m.get("role", "peer"),
|
|
framing=m.get("framing", framing))
|
|
for m in spec["members"]
|
|
]
|
|
if not members:
|
|
raise ValueError(f"team {spec.get('id')!r} has no members")
|
|
return Team(id=spec["id"], members=members,
|
|
topology=spec.get("topology", "star"), framing=framing)
|
|
|
|
|
|
def build_teams(config: dict) -> list[Team]:
|
|
teams = [build_team(t) for t in config.get("teams", [])]
|
|
if not teams:
|
|
raise ValueError("config has no teams")
|
|
return teams
|
|
|
|
|
|
def same_model_team(model: str, *, team_id: str = "solo",
|
|
framing: str = "neutral") -> Team:
|
|
"""Convenience for M1: a 2-member architect+builder team on one model."""
|
|
return build_team({
|
|
"id": team_id, "topology": "star", "framing": framing,
|
|
"members": [
|
|
{"name": "archie", "model": model, "role": "architect"},
|
|
{"name": "bob", "model": model, "role": "builder"},
|
|
],
|
|
})
|