feat(olympics): agent Olympics benchmark — multi-language arena, results ledger, model-aware budgets
Fourth benchmark axis: teams of LLM agents deliberate in a room, implement code in an isolated VM, and are scored deterministically on correctness/speed. - Multi-language adapter (python/js/go/rust/bash) via MultiPL-E continuation mode - Append-only JSONL ledger with status tracking (ok/dnf/killed/error) so budget-exhausted or crashed runs still record a row (fixes selection bias) - Model-aware wall-clock scaling (U-shaped by param count; 3x for reasoning) - Self-owned SIGALRM/SIGTERM watchdog (RunTimeout: BaseException so broad except Exception handlers in the infer/completion path can't swallow it) - Seed forwarded to Ollama sampler + markdown-fence stripping in completion.py
This commit is contained in:
@@ -0,0 +1,56 @@
|
||||
"""Per-team isolated workspace — a thin wrapper over ``bench/runtime.py``.
|
||||
|
||||
The arena's IMPLEMENT/TEST phases run candidate code here. M1 reuses the
|
||||
existing ``PodmanRuntime`` (rootless, ``--network=none``, memory/pid caps — the
|
||||
"container floor" of SPEC §12) with a ``LocalRuntime`` fallback. Stronger tiers
|
||||
(gVisor, Kata/Firecracker microVMs, hosted sandboxes) are the documented upgrade
|
||||
path and slot in behind this same interface without touching the loop.
|
||||
|
||||
A ``VM`` also keeps the last source it ran so SUBMIT can freeze the artifact for
|
||||
the judge package (the "final VM state" of SPEC §10).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass, field
|
||||
|
||||
from ..langs import Lang
|
||||
from ..runtime import Exec, get_runtime
|
||||
|
||||
|
||||
@dataclass
|
||||
class RunResult:
|
||||
ok: bool
|
||||
rc: int | None
|
||||
out: str
|
||||
note: str = ""
|
||||
|
||||
|
||||
@dataclass
|
||||
class VM:
|
||||
lang: Lang
|
||||
runtime_kind: str = "auto" # auto | podman | local
|
||||
_last_source: str = ""
|
||||
_runs: list[dict] = field(default_factory=list)
|
||||
|
||||
def __post_init__(self):
|
||||
self._rt = get_runtime(self.runtime_kind, self.lang)
|
||||
|
||||
@property
|
||||
def runtime_name(self) -> str:
|
||||
return self._rt.name
|
||||
|
||||
def run(self, source: str, timeout: float = 30.0,
|
||||
*, label: str = "") -> RunResult:
|
||||
"""Execute one self-contained program; exit 0 == all asserts passed."""
|
||||
self._last_source = source
|
||||
ex: Exec = self._rt.run(self.lang, source, timeout)
|
||||
self._runs.append({"label": label, "rc": ex.rc, "ok": ex.ok,
|
||||
"out": ex.out, "note": ex.note})
|
||||
return RunResult(ex.ok, ex.rc, ex.out, ex.note)
|
||||
|
||||
# ── submission artifact (frozen final state for the judge) ────────────
|
||||
def freeze(self) -> dict:
|
||||
return {"language": self.lang.id, "filename": self.lang.filename,
|
||||
"source": self._last_source, "runtime": self.runtime_name,
|
||||
"run_count": len(self._runs)}
|
||||
Reference in New Issue
Block a user