leetcrypt
df8f1881d8
Fourth benchmark axis: teams of LLM agents deliberate in a room, implement code in an isolated VM, and are scored deterministically on correctness/speed. - Multi-language adapter (python/js/go/rust/bash) via MultiPL-E continuation mode - Append-only JSONL ledger with status tracking (ok/dnf/killed/error) so budget-exhausted or crashed runs still record a row (fixes selection bias) - Model-aware wall-clock scaling (U-shaped by param count; 3x for reasoning) - Self-owned SIGALRM/SIGTERM watchdog (RunTimeout: BaseException so broad except Exception handlers in the infer/completion path can't swallow it) - Seed forwarded to Ollama sampler + markdown-fence stripping in completion.py
57 lines
1.9 KiB
Python
57 lines
1.9 KiB
Python
"""Per-team isolated workspace — a thin wrapper over ``bench/runtime.py``.
|
|
|
|
The arena's IMPLEMENT/TEST phases run candidate code here. M1 reuses the
|
|
existing ``PodmanRuntime`` (rootless, ``--network=none``, memory/pid caps — the
|
|
"container floor" of SPEC §12) with a ``LocalRuntime`` fallback. Stronger tiers
|
|
(gVisor, Kata/Firecracker microVMs, hosted sandboxes) are the documented upgrade
|
|
path and slot in behind this same interface without touching the loop.
|
|
|
|
A ``VM`` also keeps the last source it ran so SUBMIT can freeze the artifact for
|
|
the judge package (the "final VM state" of SPEC §10).
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
from dataclasses import dataclass, field
|
|
|
|
from ..langs import Lang
|
|
from ..runtime import Exec, get_runtime
|
|
|
|
|
|
@dataclass
|
|
class RunResult:
|
|
ok: bool
|
|
rc: int | None
|
|
out: str
|
|
note: str = ""
|
|
|
|
|
|
@dataclass
|
|
class VM:
|
|
lang: Lang
|
|
runtime_kind: str = "auto" # auto | podman | local
|
|
_last_source: str = ""
|
|
_runs: list[dict] = field(default_factory=list)
|
|
|
|
def __post_init__(self):
|
|
self._rt = get_runtime(self.runtime_kind, self.lang)
|
|
|
|
@property
|
|
def runtime_name(self) -> str:
|
|
return self._rt.name
|
|
|
|
def run(self, source: str, timeout: float = 30.0,
|
|
*, label: str = "") -> RunResult:
|
|
"""Execute one self-contained program; exit 0 == all asserts passed."""
|
|
self._last_source = source
|
|
ex: Exec = self._rt.run(self.lang, source, timeout)
|
|
self._runs.append({"label": label, "rc": ex.rc, "ok": ex.ok,
|
|
"out": ex.out, "note": ex.note})
|
|
return RunResult(ex.ok, ex.rc, ex.out, ex.note)
|
|
|
|
# ── submission artifact (frozen final state for the judge) ────────────
|
|
def freeze(self) -> dict:
|
|
return {"language": self.lang.id, "filename": self.lang.filename,
|
|
"source": self._last_source, "runtime": self.runtime_name,
|
|
"run_count": len(self._runs)}
|