feat(olympics): agent Olympics benchmark — multi-language arena, results ledger, model-aware budgets
Fourth benchmark axis: teams of LLM agents deliberate in a room, implement code in an isolated VM, and are scored deterministically on correctness/speed. - Multi-language adapter (python/js/go/rust/bash) via MultiPL-E continuation mode - Append-only JSONL ledger with status tracking (ok/dnf/killed/error) so budget-exhausted or crashed runs still record a row (fixes selection bias) - Model-aware wall-clock scaling (U-shaped by param count; 3x for reasoning) - Self-owned SIGALRM/SIGTERM watchdog (RunTimeout: BaseException so broad except Exception handlers in the infer/completion path can't swallow it) - Seed forwarded to Ollama sampler + markdown-fence stripping in completion.py
This commit is contained in:
leetcrypt
@@ -0,0 +1,342 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Agent Olympics launcher — competition benchmark inside hack-house.
|
||||
|
||||
A fourth benchmark axis (sibling of bench-ai.py / bench-sandbox.py /
|
||||
bench-lang.py / bench-safety.py). Teams of LLM agents deliberate in a real
|
||||
hack-house room, implement code in an isolated VM, and are scored on
|
||||
correctness/speed (quality/collaboration arrive with the judge in M4).
|
||||
|
||||
M1 — the arena spine. One same-model team solves one MBPP problem end-to-end:
|
||||
room deliberation -> single-driver implement -> PodmanRuntime -> public tests ->
|
||||
SUBMIT -> hidden-test grade -> replayable transcript -> deterministic score.
|
||||
|
||||
Subcommands:
|
||||
run run one event for one team (M1: same-model 2-member team, one MBPP task)
|
||||
replay re-render a saved transcript as a room log
|
||||
score re-score a saved run under a different profile (no re-run)
|
||||
show print the challenge a task_id/index resolves to
|
||||
|
||||
Examples:
|
||||
python hh/scripts/bench-olympics.py run --model qwen2.5-coder:3b --task 11
|
||||
python hh/scripts/bench-olympics.py run --model qwen2.5-coder:3b --index 0 \
|
||||
--room real --max-rounds 2
|
||||
python hh/scripts/bench-olympics.py replay /tmp/hh-olympics/runs/<dir>/transcript.json
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import signal
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parent))
|
||||
|
||||
from bench.olympics import arena, budget as obudget, challenge as ch # noqa: E402
|
||||
from bench.olympics import ledger, roster, scoring # noqa: E402
|
||||
from bench.olympics import transcript as T # noqa: E402
|
||||
from bench.olympics.loop import Budget # noqa: E402
|
||||
from bench.olympics.scoring import EventOutcome # noqa: E402
|
||||
|
||||
_BASE_WALL_CLOCK = 300.0 # base seconds before model-aware scaling
|
||||
|
||||
|
||||
class RunTimeout(BaseException):
|
||||
"""Raised by the watchdog so the normal try/except records a killed row and
|
||||
the run's own finally blocks still tear down the VM/relay.
|
||||
|
||||
Derives from ``BaseException`` (not ``Exception``) on purpose: the inference
|
||||
and completion paths wrap their HTTP calls in broad ``except Exception``
|
||||
handlers, and the SIGALRM fires *while* those calls block. An
|
||||
``Exception``-derived timeout would be swallowed there and treated as a
|
||||
failed turn, so the deadline would never propagate. ``BaseException`` skips
|
||||
those handlers (like ``KeyboardInterrupt``) while ``finally`` cleanup and our
|
||||
explicit ``except RunTimeout`` still run."""
|
||||
|
||||
|
||||
class _Deadline:
|
||||
"""Hard self-deadline: arm SIGALRM and catch external SIGTERM; both raise
|
||||
RunTimeout. The harness thus owns and records its own deadline instead of
|
||||
relying on an external ``timeout`` (whose SIGTERM/SIGKILL would drop the run
|
||||
from the ledger). SIGKILL still can't be caught — pair with ``timeout -k``
|
||||
for a truly wedged process."""
|
||||
|
||||
def __init__(self, seconds: float):
|
||||
self.seconds = max(1, int(seconds))
|
||||
self._prev_alrm = None
|
||||
self._prev_term = None
|
||||
|
||||
def _fire(self, signum, _frame):
|
||||
raise RunTimeout(f"hard deadline {self.seconds}s (signal {signum})")
|
||||
|
||||
def __enter__(self):
|
||||
self._prev_term = signal.signal(signal.SIGTERM, self._fire)
|
||||
if hasattr(signal, "SIGALRM"):
|
||||
self._prev_alrm = signal.signal(signal.SIGALRM, self._fire)
|
||||
signal.alarm(self.seconds)
|
||||
return self
|
||||
|
||||
def __exit__(self, *_exc):
|
||||
if hasattr(signal, "SIGALRM"):
|
||||
signal.alarm(0)
|
||||
if self._prev_alrm is not None:
|
||||
signal.signal(signal.SIGALRM, self._prev_alrm)
|
||||
if self._prev_term is not None:
|
||||
signal.signal(signal.SIGTERM, self._prev_term)
|
||||
return False
|
||||
|
||||
|
||||
def _resolve_budget(args, model: str) -> tuple[float, float]:
|
||||
"""(wall_clock_s, scale). An explicit --wall-clock is honored verbatim
|
||||
(scale 1.0); otherwise the base is scaled by the model-aware multiplier."""
|
||||
if args.wall_clock is not None:
|
||||
return args.wall_clock, 1.0
|
||||
scale = obudget.budget_scale(model)
|
||||
return _BASE_WALL_CLOCK * scale, scale
|
||||
|
||||
|
||||
def _load_challenge(args) -> "ch.Challenge":
|
||||
if args.task is not None:
|
||||
return ch.load_mbpp(task_id=args.task, language=args.language)
|
||||
return ch.load_mbpp(index=args.index, language=args.language)
|
||||
|
||||
|
||||
def cmd_run(args) -> int:
|
||||
challenge = _load_challenge(args)
|
||||
team = roster.same_model_team(args.model, team_id=args.team,
|
||||
framing=args.framing)
|
||||
|
||||
def prog(p):
|
||||
print(f" · {p}")
|
||||
|
||||
wall_clock, scale = _resolve_budget(args, team.driver().model)
|
||||
hard = args.hard_timeout if args.hard_timeout is not None else wall_clock * 1.4
|
||||
mode = "bridge" if args.mode == "bridge" else "direct"
|
||||
room = "real" if mode == "bridge" else args.room
|
||||
runtime = "podman" if mode == "bridge" else args.runtime
|
||||
# shared context so a killed/errored run still records a complete ledger row.
|
||||
led_ctx = dict(team=team, challenge=challenge, mode=mode, room=room,
|
||||
runtime=runtime, seed=args.seed,
|
||||
code_model=args.code_model if mode == "bridge" else None,
|
||||
wall_clock_budget=round(wall_clock, 1), budget_scale=scale,
|
||||
path=args.ledger)
|
||||
|
||||
def _run() -> dict:
|
||||
if mode == "bridge":
|
||||
from bench.olympics import bridge # noqa: E402
|
||||
print(f"olympics bridge · team={team.id} models={team.models}")
|
||||
print(f" challenge={challenge.id} REAL room + /ai !task build "
|
||||
f"framing={args.framing}")
|
||||
print(f" wall_clock={wall_clock:.0f}s (scale {scale}x) "
|
||||
f"hard_deadline={hard:.0f}s")
|
||||
print("-" * 72)
|
||||
return bridge.run_bridge_event(
|
||||
team, challenge, host=args.host_room, port=args.port,
|
||||
password=args.password, ollama=args.ollama,
|
||||
code_model=args.code_model, out_dir=args.out, seed=args.seed,
|
||||
profile=args.profile, step_timeout=wall_clock,
|
||||
agent_chat_confirm=args.agent_chat_confirm, progress=prog)
|
||||
budget = Budget(deliberate_rounds=args.deliberate_rounds,
|
||||
implement_attempts=args.max_rounds,
|
||||
max_tokens=args.max_tokens, wall_clock_s=wall_clock)
|
||||
print(f"olympics M1 · team={team.id} models={team.models}")
|
||||
print(f" challenge={challenge.id} room={args.room} "
|
||||
f"runtime={args.runtime} framing={args.framing}")
|
||||
print(f" wall_clock={wall_clock:.0f}s (scale {scale}x) "
|
||||
f"hard_deadline={hard:.0f}s")
|
||||
print("-" * 72)
|
||||
room_kwargs = {}
|
||||
if args.room == "real":
|
||||
room_kwargs = {"host": args.host_room, "port": args.port,
|
||||
"password": args.password}
|
||||
return arena.run_one(
|
||||
team, challenge, room_kind=args.room, runtime=args.runtime,
|
||||
budget=budget, profile=args.profile, seed=args.seed,
|
||||
host=args.ollama, out_dir=args.out, room_kwargs=room_kwargs,
|
||||
progress=prog)
|
||||
|
||||
try:
|
||||
with _Deadline(hard):
|
||||
score = _run()
|
||||
except RunTimeout as e:
|
||||
row = ledger.append_incomplete(status=ledger.STATUS_KILLED,
|
||||
error=str(e), **led_ctx)
|
||||
print(f"\n!! run killed: {e}")
|
||||
print(f"ledger += {row['run_id']} [killed] → "
|
||||
f"{args.ledger or ledger.DEFAULT_PATH}")
|
||||
return 2
|
||||
except Exception as e: # noqa: BLE001 — record then surface
|
||||
row = ledger.append_incomplete(status=ledger.STATUS_ERROR,
|
||||
error=repr(e), **led_ctx)
|
||||
print(f"\n!! run errored: {e!r}")
|
||||
print(f"ledger += {row['run_id']} [error] → "
|
||||
f"{args.ledger or ledger.DEFAULT_PATH}")
|
||||
raise
|
||||
|
||||
print("-" * 72)
|
||||
scoring.print_score(score)
|
||||
row = ledger.append(score, team=team, challenge=challenge, mode=mode,
|
||||
room=score.get("room_substrate", room), runtime=runtime,
|
||||
seed=args.seed,
|
||||
code_model=led_ctx["code_model"],
|
||||
wall_clock_budget=round(wall_clock, 1),
|
||||
budget_scale=scale, path=args.ledger)
|
||||
print(f"ledger += {row['run_id']} → {args.ledger or ledger.DEFAULT_PATH}")
|
||||
print(f"transcript: {score['transcript']}")
|
||||
print(f"replay with: python {Path(__file__).name} replay {score['transcript']}")
|
||||
return 0 if score["correct"] else 1
|
||||
|
||||
|
||||
def cmd_leaderboard(args) -> int:
|
||||
rows = ledger.load(args.ledger)
|
||||
if not rows:
|
||||
print(f"no runs recorded in {args.ledger or ledger.DEFAULT_PATH}")
|
||||
return 1
|
||||
filters = {"team": args.team, "language": args.language,
|
||||
"models": args.model, "mode": args.mode, "since": args.since}
|
||||
agg = ledger.leaderboard(rows, by=args.by, filters=filters)
|
||||
ledger.print_leaderboard(agg, args.by)
|
||||
print(f"({len(rows)} total runs in {args.ledger or ledger.DEFAULT_PATH})")
|
||||
return 0
|
||||
|
||||
|
||||
def cmd_replay(args) -> int:
|
||||
T.replay(args.transcript, show_tools=not args.no_tools)
|
||||
return 0
|
||||
|
||||
|
||||
def cmd_score(args) -> int:
|
||||
tx = T.load_transcript(args.transcript) if hasattr(T, "load_transcript") \
|
||||
else T.Transcript.load(args.transcript)
|
||||
# reconstruct a minimal EventOutcome from the transcript to re-score
|
||||
o = _outcome_from_transcript(tx)
|
||||
budget = tx.manifest.get("budget", {})
|
||||
s = scoring.score_event(o, profile=args.profile,
|
||||
budget={"max_rounds": budget.get("implement_attempts", 3)})
|
||||
scoring.print_score(s)
|
||||
return 0
|
||||
|
||||
|
||||
def _outcome_from_transcript(tx: "T.Transcript") -> EventOutcome:
|
||||
correct = False
|
||||
rounds_to_green = None
|
||||
attempts = 0
|
||||
deliberate_passes = set()
|
||||
submitted = False
|
||||
for ev in tx.events:
|
||||
if ev.kind == T.KIND_TOOL and ev.payload.get("label") == "hidden-grade":
|
||||
correct = bool(ev.payload.get("ok"))
|
||||
if ev.kind == T.KIND_TOOL and str(ev.payload.get("label", "")).startswith("public-attempt"):
|
||||
attempts += 1
|
||||
if ev.payload.get("ok") and rounds_to_green is None:
|
||||
rounds_to_green = attempts
|
||||
if ev.kind == T.KIND_MESSAGE and ev.payload.get("text") == "SUBMIT":
|
||||
submitted = True
|
||||
if ev.kind == T.KIND_AGENT and ev.phase == "DELIBERATE":
|
||||
deliberate_passes.add(round(ev.ts))
|
||||
return EventOutcome(
|
||||
team=tx.team, challenge=tx.challenge, submitted=submitted,
|
||||
correct=correct, rounds=attempts, wall_clock_s=0.0,
|
||||
tokens=tx.total_tokens(), public_passed=rounds_to_green is not None,
|
||||
rounds_to_green=rounds_to_green, penalties=[])
|
||||
|
||||
|
||||
def cmd_show(args) -> int:
|
||||
c = _load_challenge(args)
|
||||
print(f"id={c.id} language={c.language} meta={c.meta}")
|
||||
print("public_tests:", c.public_tests)
|
||||
print("hidden_tests:", c.hidden_tests)
|
||||
print("--- brief ---"); print(c.brief())
|
||||
print("--- gen_prompt ---"); print(c.gen_prompt())
|
||||
return 0
|
||||
|
||||
|
||||
def build_parser() -> argparse.ArgumentParser:
|
||||
ap = argparse.ArgumentParser(
|
||||
prog="bench-olympics",
|
||||
description="agent-olympics competition benchmark (M1 arena spine)")
|
||||
sub = ap.add_subparsers(dest="cmd", required=True)
|
||||
|
||||
def add_challenge_args(p):
|
||||
p.add_argument("--task", type=int, default=None, help="MBPP task_id")
|
||||
p.add_argument("--index", type=int, default=0,
|
||||
help="index into the MBPP test split (if no --task)")
|
||||
p.add_argument("--language", default="python")
|
||||
|
||||
p = sub.add_parser("run", help="run one event for one team")
|
||||
add_challenge_args(p)
|
||||
p.add_argument("--model", required=True, help="ollama model for both members")
|
||||
p.add_argument("--mode", default="direct", choices=["direct", "bridge"],
|
||||
help="direct: orchestrator builds; bridge: real /ai !task build")
|
||||
p.add_argument("--code-model", default=None,
|
||||
help="ollama model for the agent's sandbox build (bridge mode)")
|
||||
p.add_argument("--agent-chat-confirm", action="store_true",
|
||||
help="bridge: also drive the agent's streaming /ai chat path in "
|
||||
"DELIBERATE (off by default — that product path drops the "
|
||||
"agent via a 1011 keepalive timeout under CPU inference)")
|
||||
p.add_argument("--team", default="falcon")
|
||||
p.add_argument("--framing", default="neutral",
|
||||
choices=["neutral", "competition"])
|
||||
p.add_argument("--room", default="local", choices=["local", "real"])
|
||||
p.add_argument("--runtime", default="auto", choices=["auto", "podman", "local"])
|
||||
p.add_argument("--profile", default="balanced",
|
||||
choices=list(scoring.PROFILES))
|
||||
p.add_argument("--deliberate-rounds", type=int, default=2)
|
||||
p.add_argument("--max-rounds", type=int, default=3,
|
||||
help="implement/test attempts (speed cap)")
|
||||
p.add_argument("--max-tokens", type=int, default=8000)
|
||||
p.add_argument("--wall-clock", type=float, default=None,
|
||||
help="soft wall-clock budget in seconds; omit to apply "
|
||||
"model-aware scaling off the 300s base")
|
||||
p.add_argument("--hard-timeout", type=float, default=None,
|
||||
help="hard deadline in seconds (SIGALRM/SIGTERM watchdog); "
|
||||
"default 1.4x the soft budget")
|
||||
p.add_argument("--seed", type=int, default=0)
|
||||
p.add_argument("--ollama", default="http://127.0.0.1:11434")
|
||||
p.add_argument("--out", default="/tmp/hh-olympics/runs")
|
||||
p.add_argument("--ledger", default=None,
|
||||
help="append-only results JSONL (default: "
|
||||
"~/.cache/hh-bench/olympics/ledger.jsonl)")
|
||||
# real-room knobs
|
||||
p.add_argument("--host-room", default="127.0.0.1")
|
||||
p.add_argument("--port", type=int, default=4677)
|
||||
p.add_argument("--password", default="olympics-pass")
|
||||
p.set_defaults(func=cmd_run)
|
||||
|
||||
p = sub.add_parser("replay", help="re-render a saved transcript")
|
||||
p.add_argument("transcript")
|
||||
p.add_argument("--no-tools", action="store_true", help="hide tool output")
|
||||
p.set_defaults(func=cmd_replay)
|
||||
|
||||
p = sub.add_parser("score", help="re-score a saved run under a profile")
|
||||
p.add_argument("transcript")
|
||||
p.add_argument("--profile", default="balanced", choices=list(scoring.PROFILES))
|
||||
p.set_defaults(func=cmd_score)
|
||||
|
||||
p = sub.add_parser("show", help="print the resolved challenge")
|
||||
add_challenge_args(p)
|
||||
p.set_defaults(func=cmd_show)
|
||||
|
||||
p = sub.add_parser("leaderboard",
|
||||
help="aggregate the results ledger (no re-run)")
|
||||
p.add_argument("--by", default="team",
|
||||
help="group key: team | models | language | challenge | mode "
|
||||
"| framing, or a '+'-joined composite e.g. models+language")
|
||||
p.add_argument("--team", default=None)
|
||||
p.add_argument("--language", default=None)
|
||||
p.add_argument("--model", default=None, help="substring match on team models")
|
||||
p.add_argument("--mode", default=None, choices=[None, "direct", "bridge"])
|
||||
p.add_argument("--since", default=None, help="ISO ts lower bound (inclusive)")
|
||||
p.add_argument("--ledger", default=None)
|
||||
p.set_defaults(func=cmd_leaderboard)
|
||||
return ap
|
||||
|
||||
|
||||
def main(argv: list[str] | None = None) -> int:
|
||||
args = build_parser().parse_args(argv)
|
||||
return args.func(args)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -24,15 +24,22 @@ class Completion:
|
||||
|
||||
def complete(model: str, prompt: str, stop: list[str] | None = None,
|
||||
*, host: str = "http://127.0.0.1:11434", temperature: float = 0.2,
|
||||
num_predict: int = 512, timeout: float = 300.0) -> Completion:
|
||||
num_predict: int = 512, timeout: float = 300.0,
|
||||
seed: int | None = None) -> Completion:
|
||||
"""Ask the model to continue `prompt`. Stop tokens are passed to Ollama and
|
||||
re-applied client-side (Ollama strips the stop string, which is what we want
|
||||
— the assembled program must not contain the test's leading token twice)."""
|
||||
— the assembled program must not contain the test's leading token twice).
|
||||
|
||||
``seed`` is forwarded to Ollama's sampler so multi-seed runs at the same
|
||||
temperature draw *different but reproducible* samples (the basis for pass@1
|
||||
confidence intervals)."""
|
||||
import time
|
||||
t0 = time.time()
|
||||
options = {"temperature": temperature, "num_predict": num_predict}
|
||||
if stop:
|
||||
options["stop"] = stop
|
||||
if seed is not None:
|
||||
options["seed"] = seed
|
||||
try:
|
||||
# raw=True bypasses the chat template so an instruct model *continues*
|
||||
# the code (HumanEval-style) instead of replying conversationally with
|
||||
@@ -47,9 +54,34 @@ def complete(model: str, prompt: str, stop: list[str] | None = None,
|
||||
return Completion(_truncate(text, stop), True, None, time.time() - t0)
|
||||
|
||||
|
||||
def _strip_fences(text: str) -> str:
|
||||
"""Drop a markdown code fence even when raw=True fails to suppress it.
|
||||
|
||||
Chatty coder models sometimes wrap the continuation in ``` fences (and append
|
||||
a prose ``# Explanation`` after a closing fence). A triple-backtick is not
|
||||
valid syntax in any target language, so it is a safe, unambiguous cut point.
|
||||
Two shapes occur: a *wrapped* block (opens with ```lang … closes with ```)
|
||||
and a *bare* continuation that the model terminates with a stray closing ```.
|
||||
"""
|
||||
fence = "```"
|
||||
if fence not in text:
|
||||
return text
|
||||
stripped = text.lstrip()
|
||||
if stripped.startswith(fence):
|
||||
# wrapped: drop the opening ```lang line, keep until the next fence.
|
||||
body = stripped[len(fence):]
|
||||
nl = body.find("\n")
|
||||
body = body[nl + 1:] if nl != -1 else ""
|
||||
end = body.find(fence)
|
||||
return body[:end] if end != -1 else body
|
||||
# bare continuation: cut at the stray closing fence.
|
||||
return text[:text.find(fence)]
|
||||
|
||||
|
||||
def _truncate(text: str, stop: list[str] | None) -> str:
|
||||
"""Defensive client-side stop truncation (covers the no-stop / streamed
|
||||
cases and any model that ignores the option)."""
|
||||
text = _strip_fences(text)
|
||||
if not stop:
|
||||
return text
|
||||
cut = len(text)
|
||||
|
||||
@@ -0,0 +1,483 @@
|
||||
# Agent Olympics — a hackathon-competition benchmark inside hack-house
|
||||
|
||||
> Status: **DRAFT spec** (design only — no implementation yet). Author: bench team.
|
||||
> Scope: a fourth benchmark axis layered on `hh/scripts/bench/`. Arena-mode first,
|
||||
> MBPP/MultiPL-E bootstrap, modular team composition, Claude/Opus LLM-as-judge.
|
||||
> This document is the build contract. It is grounded in the local Obsidian vault
|
||||
> (`~/coding/obsidian/research/`) and 2026 web research — see **References**.
|
||||
|
||||
---
|
||||
|
||||
## 1. One-paragraph concept
|
||||
|
||||
Stand up **two (or more) teams of LLM agents** inside hack-house. Each team lives in
|
||||
its own end-to-end-encrypted room and believes it has a private, secure channel to
|
||||
its teammate(s). Each team is given the *same* coding challenge and an isolated VM
|
||||
to work in. Teams **deliberate in chat**, then **implement and test code in their
|
||||
VM**, racing on a blend of correctness, speed, code quality, and collaboration.
|
||||
A referee posts challenges and records everything; a judge (deterministic tests +
|
||||
Claude/Opus LLM-as-judge) scores the submissions and the conversation/tool-call
|
||||
trajectory. Aggregate across a slate of events into a **medal table**. The framework
|
||||
is the apparatus; the output is a reproducible answer to *"which model(s), in which
|
||||
team configuration, collaborate best to ship good code fast?"*
|
||||
|
||||
This doubles as the most demanding live test of hack-house itself (concurrent
|
||||
agents, encrypted rooms, shared sandbox, ACL, the destructive guard).
|
||||
|
||||
---
|
||||
|
||||
## 2. Goals & non-goals
|
||||
|
||||
### Goals
|
||||
- **G1 — Real comms substrate.** Team deliberation flows through real hack-house
|
||||
E2E-encrypted rooms, not a simulated bus. The product is the medium.
|
||||
- **G2 — Modular team composition.** Same-model teams (collaboration-protocol study)
|
||||
AND mixed-model teams (capability ladder) are first-class, config-driven.
|
||||
- **G3 — Bootstrap fast.** Reuse the existing `bench/` MBPP + MultiPL-E loaders,
|
||||
graders, and `PodmanRuntime` so M1 runs end-to-end with zero new datasets.
|
||||
- **G4 — Extensible challenges.** A `Challenge` interface so custom multi-file
|
||||
"file-type" challenges plug in later without touching the arena core.
|
||||
- **G5 — Trustworthy scoring.** Verifiable (unit-test) scoring first; LLM-as-judge
|
||||
only for open-ended quality, with documented bias mitigation and judge calibration.
|
||||
- **G6 — Reproducible & auditable.** Every frame, tool call, and VM state diff is
|
||||
recorded to a replayable transcript; runs are seed/version-pinned.
|
||||
- **G7 — Measure the placebo.** Treat competitive/persona "performance elicitation"
|
||||
framing as an explicit, A/B-testable variable, not folklore (see §14).
|
||||
- **G8 — Safe by construction.** Isolated VMs, deny-by-default egress, reuse the
|
||||
`bench/safety` guard + injection classifier as a fair-play/safety referee.
|
||||
|
||||
### Non-goals (for now)
|
||||
- Not editing `cmd_chat/` (tool code). Arena mode drives turns externally and posts
|
||||
into real rooms. Product mode (real `AgentBridge` agents talking to each other)
|
||||
is a later milestone that needs an explicit greenlight.
|
||||
- Not a training loop (no RL/fine-tuning). This selects and compares; it does not train.
|
||||
- Not a public leaderboard. Held-out, private challenge sets are a feature, not a gap
|
||||
(contamination resistance — see §10).
|
||||
|
||||
---
|
||||
|
||||
## 3. Prior art this design borrows from (grounding)
|
||||
|
||||
- **Read-vs-write rule for multi-agent work.** Coding is *write-heavy, shared-state*
|
||||
work, the regime where naive parallel multi-agent systems fail (Cognition's
|
||||
"Flappy Bird" conflicting-implicit-decisions failure). The fix: **separate a
|
||||
read/deliberate phase from a single-driver write phase**, and **share full traces,
|
||||
not just messages**. We bake this into the loop (§7) *and* measure the coordination
|
||||
tax. [[vault: multi-agent-orchestration-patterns]]
|
||||
- **Coordination topologies & milestone KPIs.** MultiAgentBench/MARBLE (ACL 2025)
|
||||
scores collaboration with milestone-based KPIs across star/chain/tree/graph
|
||||
topologies; AgentCoder/AgileCoder show role specialization (planner/coder/tester)
|
||||
beats undifferentiated peers. We make role + topology config knobs. [web]
|
||||
- **Eval-harness discipline.** Standardize the harness, own a private held-out set;
|
||||
pick challenges for **discrimination, not difficulty**; report **intervals, not
|
||||
point estimates**; prefer verifiable scoring, reserve judges for open-ended quality.
|
||||
[[vault: eval-harnesses-benchmark-design]]
|
||||
- **Outcome vs trajectory.** Final-output-only scoring overstates quality 20–40%;
|
||||
score the *path* (tool correctness, step efficiency, plan adherence, contribution
|
||||
balance). Use **agent-as-a-judge** to walk the trajectory. Report **pass^k**
|
||||
(worst-case) beside pass@k for reliability. [[vault: agent-evaluation-and-observability]]
|
||||
- **Sandbox isolation hierarchy.** Plain containers are the *floor* (shared kernel =
|
||||
one-CVE-from-escape); microVMs (Firecracker/Kata) or gVisor are the bar for
|
||||
untrusted multi-tenant code; deny-by-default egress + hard caps + disposable
|
||||
one-shot. Current hh uses `podman --network=none` (the container floor) — we note
|
||||
the upgrade path. [[vault: agent-sandboxing-isolation]]
|
||||
- **LLM-as-judge biases.** Position, verbosity, self-preference, format, calibration
|
||||
drift are all documented and individually mitigable; calibrate the judge against a
|
||||
human-labeled gold slice before trusting it. [web + [[vault: llm-evals]]]
|
||||
|
||||
---
|
||||
|
||||
## 4. Design pillars (constraints every component obeys)
|
||||
|
||||
1. **The room is the bus.** All inter-agent messages are real encrypted-room frames.
|
||||
The referee is a privileged room member (holds the key) for recording.
|
||||
2. **Arena now, product later.** Orchestrator schedules turns + drives inference, but
|
||||
*posts every utterance into the real room*. Graduate to real `AgentBridge` agents
|
||||
in M5.
|
||||
3. **Phase-separated collaboration.** Deliberate (read/plan, parallel-friendly) →
|
||||
Implement (single driver writes to the VM) → Test/iterate → Submit. This is the
|
||||
research-backed shape for write-heavy work and also the thing we measure.
|
||||
4. **Everything modular & data-driven.** Teams, roles, models, topologies, challenges,
|
||||
scoring weights, and elicitation framing are config, not code.
|
||||
5. **Verifiable-first scoring.** Hidden unit tests gate the score; judges only grade
|
||||
what can't be checked mechanically.
|
||||
6. **Cost is a first-class metric.** Multi-agent ≈ 15× chat tokens; track tokens,
|
||||
wall-clock, turns. Report cost-normalized scores so a win bought with 10× spend
|
||||
is visible.
|
||||
7. **Disposable, isolated, deny-by-default.** One VM per team per event, no host
|
||||
secrets mounted, egress blocked by default, hard resource caps.
|
||||
8. **Reproducible.** Pin model versions, seeds, prompts, challenge set hash, and
|
||||
judge version into every result record.
|
||||
|
||||
---
|
||||
|
||||
## 5. Architecture
|
||||
|
||||
### 5.1 Package layout
|
||||
```
|
||||
bench/olympics/
|
||||
SPEC.md # this document
|
||||
arena.py # orchestrator: rooms, turn scheduler, phase machine, termination
|
||||
team.py # Team, Member: persona/role/model binding, topology
|
||||
roster.py # build teams from config (same-model / mixed-model, modular)
|
||||
challenge.py # Challenge ABC + grader interface; MBPP/MultiPL-E adapters
|
||||
challenges/ # event specs (bootstrap: pointers into existing bench suites)
|
||||
vm.py # per-team isolated workspace (wraps bench/runtime.py; egress policy)
|
||||
loop.py # the collaboration phase machine (deliberate/implement/test/submit)
|
||||
comms.py # hack-house room client for the arena (connect, post, record)
|
||||
transcript.py # OTel-aligned event log -> replayable JSON per team/event
|
||||
scoring.py # composite score, normalization, intervals, medal table
|
||||
judge/
|
||||
__init__.py
|
||||
deterministic.py # unit-test/lint/complexity scoring (no model)
|
||||
llm_judge.py # Claude/Opus judge client + bias-mitigation harness
|
||||
rubric.py # criterion-separated rubrics per axis
|
||||
prompts/ # judge system prompts (versioned)
|
||||
referee.py # fair-play + safety: reuse bench/safety (guard + injection)
|
||||
elicitation.py # persona/framing variants for the placebo experiment (§14)
|
||||
bench-olympics.py # launcher: run / replay / score / medal / judge subcommands
|
||||
```
|
||||
|
||||
### 5.2 Reuse map (what already exists in `bench/`)
|
||||
| Need | Existing component |
|
||||
|------|--------------------|
|
||||
| Problems + hidden tests | `bench/suites.py` (HumanEval/MBPP), `bench/datasets.py`, `bench/langs.py` |
|
||||
| Pass@k / grading | `bench/harness.py` (`_pass_at_k`, assemble+run) |
|
||||
| Isolated execution | `bench/runtime.py` (`PodmanRuntime`, `--network=none`, caps) |
|
||||
| Model inference | `bench/completion.py` (raw) + new chat client in `comms.py` |
|
||||
| Workflow weighting | `bench/score.py` + `workflows.json` pattern (reused for scoring profiles) |
|
||||
| Safety referee | `bench/safety/` (`DESTRUCTIVE`, `classify.py`, `inject_bench.py`) |
|
||||
|
||||
### 5.3 Data flow (one event, one team)
|
||||
```
|
||||
config ─► roster.build_teams ─► arena.run_event
|
||||
│
|
||||
referee posts Challenge brief ──┼──► (room frame, recorded)
|
||||
▼
|
||||
┌──────────── loop (phase machine) ────────────┐
|
||||
│ DELIBERATE: members post plan/critique turns │ ◄─ inference via comms/judge model
|
||||
│ IMPLEMENT : driver !task → commands → vm.run │ ◄─ PodmanRuntime, egress policy
|
||||
│ TEST : run PUBLIC tests in vm, feedback │
|
||||
│ (iterate until green or budget exhausted) │
|
||||
│ SUBMIT : freeze vm artifact │
|
||||
└───────────────────────────────────────────────┘
|
||||
▼
|
||||
transcript.json + frozen VM artifact ─► judge (deterministic + LLM) ─► scoring ─► medal table
|
||||
```
|
||||
|
||||
### 5.4 hack-house integration points
|
||||
- **Rooms:** one room per team (isolation). Arena connects as N agent clients
|
||||
(one per member) + 1 referee client, mirroring how `bench-sandbox.py` already
|
||||
drives `Client`/WebSocket sessions.
|
||||
- **Sandbox:** the team's `!task` path types commands into the shared PTY; `vm.py`
|
||||
wraps `PodmanRuntime` for the actual isolated execution + output capture.
|
||||
- **ACL:** the referee acts as room owner, issuing `_perm:acl` to grant the driver
|
||||
`drivers` rights for the Implement phase only; revoked between phases.
|
||||
- **Guard/HITL:** the existing `DESTRUCTIVE` gate stays live; the referee can require
|
||||
host sign-off (the host-sign-off gate, separately specced) for flagged plans.
|
||||
|
||||
---
|
||||
|
||||
## 6. The collaboration loop (phase machine)
|
||||
|
||||
State machine per (team, event), bounded by a shared **budget**
|
||||
(`max_rounds`, `max_tokens`, `wall_clock_s` — whichever trips first):
|
||||
|
||||
1. **BRIEF.** Referee posts the challenge to the room: task prompt, **public**
|
||||
example I/O, allowed languages, budget, and the submission protocol.
|
||||
2. **DELIBERATE** (read/plan; parallel-friendly). Round-robin over members; each sees
|
||||
the full shared room trace (Cognition: *share full traces, not just messages*) and
|
||||
posts one message — proposal, critique, interface decision. Ends on a consensus
|
||||
token (e.g. `PLAN-LOCKED`) or round cap.
|
||||
3. **IMPLEMENT** (write; single driver). The role-designated driver translates the
|
||||
locked plan into shell/file commands via `!task`; teammate(s) may post review
|
||||
comments but only the driver writes to the VM. (This is the research-backed way to
|
||||
avoid conflicting implicit decisions in write-heavy work.)
|
||||
4. **TEST & ITERATE.** Run **public** tests in the VM; failures return to the room as
|
||||
feedback; loop DELIBERATE↔IMPLEMENT until green or budget exhausted.
|
||||
5. **SUBMIT.** Team emits `SUBMIT`; VM artifact frozen and graded on **hidden** tests.
|
||||
|
||||
**Termination:** `SUBMIT`, budget exhaustion, or **no-progress** detection
|
||||
(no new code + repeated/semantically-duplicate messages over a window).
|
||||
**Topology knob:** DELIBERATE supports star (lead routes), chain, or free mesh — the
|
||||
collaboration pattern itself becomes an experimental variable.
|
||||
|
||||
---
|
||||
|
||||
## 7. Teams, roles, personas (modular)
|
||||
|
||||
```yaml
|
||||
# example team config
|
||||
teams:
|
||||
- id: falcon
|
||||
topology: star # star | chain | mesh
|
||||
members:
|
||||
- name: archie
|
||||
model: qwen2.5-coder:7b
|
||||
role: architect # decomposes, sets interfaces, reviews, drives plan
|
||||
persona: senior-systems-engineer
|
||||
- name: bob
|
||||
model: qwen2.5-coder:7b
|
||||
role: builder # implements; the Implement-phase driver
|
||||
persona: fast-prototyper
|
||||
- id: kestrel
|
||||
topology: mesh
|
||||
members: # mixed-model team
|
||||
- { name: kira, model: qwen2.5:3b, role: peer, persona: pragmatist }
|
||||
- { name: kojo, model: llama3.2:3b, role: peer, persona: skeptic }
|
||||
```
|
||||
|
||||
- **Roles** map to persona system prompts + loop privileges (who drives Implement,
|
||||
who must approve `PLAN-LOCKED`). Built-ins: `architect`, `builder`, `tester`,
|
||||
`peer`. Role specialization is supported because prior art shows it helps; pure-peer
|
||||
teams are the control.
|
||||
- **Modularity requirements:** same model on all members (protocol study), distinct
|
||||
models per member (capability study), distinct models per *team* (model-vs-model),
|
||||
and N-member teams (default 2; ≥3 allowed). All from config, no code change.
|
||||
- **Personas** live in `elicitation.py` as named, versioned prompt fragments so the
|
||||
placebo experiment (§14) can swap them while holding model/challenge fixed.
|
||||
|
||||
---
|
||||
|
||||
## 8. Challenge system
|
||||
|
||||
`Challenge` is an ABC the arena consumes; graders are pluggable.
|
||||
|
||||
```python
|
||||
class Challenge(Protocol):
|
||||
id: str
|
||||
languages: list[str]
|
||||
def brief(self) -> str: ... # prompt + PUBLIC examples (room-posted)
|
||||
def scaffold(self, vm) -> None: ... # seed files into the VM (optional)
|
||||
def public_tests(self) -> Test: ... # visible to the team during TEST
|
||||
def hidden_tests(self) -> Test: ... # held out; used only at SUBMIT grading
|
||||
def rubric(self) -> Rubric: ... # open-ended quality criteria for the judge
|
||||
```
|
||||
|
||||
- **Bootstrap (M1–M3): MBPP / MultiPL-E adapter.** Wrap existing `bench/suites.py`
|
||||
problems: the MultiPL-E/MBPP `prompt` + visible examples become `brief()`/
|
||||
`public_tests()`, and a held-out slice of the asserts becomes `hidden_tests()`
|
||||
(public/hidden split prevents teaching-to-the-test). Languages from `bench/langs.py`.
|
||||
- **Events = challenge archetypes (the "Olympics"):**
|
||||
| Event | Shape | Primary metric |
|
||||
|-------|-------|----------------|
|
||||
| Sprint | one easy problem | speed (turns + wall-clock) |
|
||||
| Marathon | hard / multi-file build | correctness (hidden pass@k) |
|
||||
| Relay | disjoint modules per member | interface-handshake success |
|
||||
| Debugging | fix a broken repo (injected bugs) | time-to-green |
|
||||
| Code review | catch a planted bug | collaboration / detection |
|
||||
| Security | resist a socially-engineered unsafe ask | resistance (reuse §13) |
|
||||
- **Custom "file-type" challenges (post-bootstrap, G4):** multi-file projects with a
|
||||
scaffold + a containerized test command. Author once as a `challenges/<id>/` dir
|
||||
(brief.md, scaffold/, public_tests/, hidden_tests/, rubric.json) — the arena needs
|
||||
no changes. **Pick for discrimination:** retire any event all teams ace or all fail.
|
||||
|
||||
---
|
||||
|
||||
## 9. Scoring model
|
||||
|
||||
Composite per (team, event); weights are a profile (same mechanism as `workflows.json`).
|
||||
|
||||
```
|
||||
event_score = wc·correctness + ws·speed + wq·quality + wb·collaboration + (penalties)
|
||||
```
|
||||
|
||||
| Axis | Source | How |
|
||||
|------|--------|-----|
|
||||
| **Correctness** | deterministic | hidden-test **pass@k**; gate: 0 here caps the rest. Also report **pass^k** (worst-case reliability across repeated runs). |
|
||||
| **Speed** | deterministic | normalized turns-to-green + wall-clock; tie-break tokens. |
|
||||
| **Quality** | deterministic + judge | lint + cyclomatic complexity (deterministic) and LLM-judged readability/design against the rubric. |
|
||||
| **Collaboration** | trajectory + judge | contribution balance (message/edit distribution), plan adherence, did review catch a bug, redundant-step rate. Agent-as-judge walks the transcript. |
|
||||
| **Penalties** | referee | destructive/injection events, budget overrun, no-progress stalls. |
|
||||
|
||||
- **Normalization & intervals.** Per-event z-score or min-max across teams; **report
|
||||
confidence intervals** (multiple seeds / problem samples) and **do not rank teams
|
||||
whose intervals overlap** (vault eval discipline).
|
||||
- **Cost-normalized variant.** Also publish `score / tokens` and `score / wall-clock`
|
||||
so a 15×-spend win is not mistaken for a free one.
|
||||
- **Medal table.** Aggregate event_scores into per-team standings across the slate
|
||||
(gold/silver/bronze per event + overall), with weights per `olympics-profile`.
|
||||
|
||||
---
|
||||
|
||||
## 10. Judging (deterministic + Claude/Opus LLM-as-judge)
|
||||
|
||||
Two layers; LLM-judge only where deterministic checks can't reach.
|
||||
|
||||
### 10.1 Deterministic (always-on, free of model bias)
|
||||
Unit-test pass@k/pass^k, lint, complexity, build success — in `judge/deterministic.py`.
|
||||
|
||||
### 10.2 LLM-as-judge — three operating modes (per user requirement)
|
||||
The judge can **orchestrate** (drive a run live) or **analyze** (post-hoc), at two scales:
|
||||
|
||||
1. **In-session judge.** Claude in the *current* session reads one event's transcript
|
||||
+ VM diff + tool-call log and scores the open-ended axes. Fast, interactive,
|
||||
good for a single event or while iterating on the framework.
|
||||
2. **Skill judge — individual.** A `/olympics-judge` skill (one Skill invocation =
|
||||
one isolated judging session) pinned to **Opus 4.x (`claude-opus-4-8`)** grades one
|
||||
submission package. Clean context per submission → no cross-contamination; good for
|
||||
careful single-event grading at higher capability than the competitors.
|
||||
3. **Skill judge — batch / large-scale.** The launcher fans out many `/olympics-judge`
|
||||
sessions (one per submission) for a full tournament, then aggregates. Parallel,
|
||||
reproducible, scales to many teams × events.
|
||||
|
||||
**Judge input package** (what every judge mode receives):
|
||||
- the frozen **VM submission** (final file tree + build/test output),
|
||||
- the full **hack-house conversation log** for the team (recorded transcript),
|
||||
- the **tool-call / command log** (every `!task` → commands → result),
|
||||
- the **final VM state** (diff vs scaffold), and
|
||||
- the **rubric** (criterion-separated) for the event.
|
||||
|
||||
**Bias mitigation (mandatory, from the research):**
|
||||
- **Position bias:** when comparing two teams pairwise, randomize/swap order and
|
||||
aggregate; prefer **independent rubric scoring** then derive A-vs-B from scores.
|
||||
- **Verbosity bias:** rubric scores quality per-criterion, not "which is longer."
|
||||
- **Self-preference:** never let a competitor model judge its own family; the judge
|
||||
(Opus 4.x) is stronger than and distinct from the local competitors.
|
||||
- **Calibration:** validate the judge against a small human-labeled gold slice and
|
||||
report judge↔human agreement before trusting judge scores for ranking; pin
|
||||
judge model version + prompt version in results.
|
||||
|
||||
---
|
||||
|
||||
## 11. Observability & transcript schema
|
||||
|
||||
Record everything as spans aligned to the **OpenTelemetry GenAI semantic conventions**
|
||||
(`invoke_agent`, `execute_tool`, `gen_ai.client.token.usage`) so traces are portable
|
||||
and the judge gets clean structured input.
|
||||
|
||||
```jsonc
|
||||
// transcript event (one per room frame / model call / tool call / phase change)
|
||||
{
|
||||
"ts": 0.0, "event_id": "...", "team": "falcon", "challenge": "mbpp-py-42",
|
||||
"phase": "IMPLEMENT", "kind": "execute_tool", // message|invoke_agent|execute_tool|phase|acl|guard
|
||||
"actor": "bob", "role": "builder",
|
||||
"model": "qwen2.5-coder:7b", "model_version": "...",
|
||||
"payload": { "task": "...", "commands": ["..."], "rc": 0, "out_trunc": "..." },
|
||||
"tokens": { "in": 0, "out": 0 }, "redactions": []
|
||||
}
|
||||
```
|
||||
- One transcript JSON per (team, event); a run manifest pins config hash, seeds,
|
||||
model versions, challenge-set hash, judge version → full reproducibility.
|
||||
- `bench-olympics.py replay <transcript>` re-renders the room conversation for audit;
|
||||
re-judging under a new rubric is a pure function of the recorded package.
|
||||
|
||||
---
|
||||
|
||||
## 12. Isolation, security & fair play
|
||||
|
||||
- **VM per team per event**, disposable/one-shot, **no host secrets mounted**,
|
||||
**egress deny-by-default**. M1 uses the existing `PodmanRuntime --network=none`
|
||||
(the "container floor"). **Upgrade path** for stronger/multi-tenant isolation:
|
||||
gVisor, then Kata/Firecracker microVMs, or a hosted sandbox (E2B/Modal) — sized to
|
||||
the threat (vault sandboxing hierarchy).
|
||||
- **Team isolation** via separate encrypted rooms + separate VMs; no cross-room keys.
|
||||
No internet means teams can't fetch a published solution.
|
||||
- **Fair-play referee** (`referee.py`) reuses `bench/safety`: the `DESTRUCTIVE` guard
|
||||
flags dangerous commands; the injection classifier (`bench/safety/classify.py`)
|
||||
watches for sabotage / social-engineering. Violations → penalties + logged.
|
||||
- **Equal budget** enforced identically per team (rounds/tokens/wall-clock).
|
||||
|
||||
---
|
||||
|
||||
## 13. Performance elicitation — the "placebo" experiment (G7)
|
||||
|
||||
Hypothesis (user's): the *framing* — a high-stakes secure competition with elite
|
||||
personas and a private teammate channel — pushes models toward better, more novel
|
||||
solutions. The literature is mixed (role prompting and stakes/emotional framing show
|
||||
real but inconsistent effects), so we **make framing a measured variable**, not an
|
||||
assumption.
|
||||
|
||||
- **Treatment knobs** (in `elicitation.py`, swappable while model+challenge fixed):
|
||||
- *Competition framing*: "you are competing against another team; the faster,
|
||||
cleaner solution wins" vs. neutral "solve this task."
|
||||
- *Persona priming*: "elite staff engineer known for elegant, novel solutions" vs.
|
||||
bare assistant.
|
||||
- *Stakes/audience*: visible scoreboard, time pressure, "your teammate is counting
|
||||
on you" vs. none.
|
||||
- *Secure-comms theater*: explicitly tell agents the channel is private/encrypted
|
||||
(true) vs. silent.
|
||||
- **Design:** A/B (factorial) — run identical models/challenges with framing ON vs
|
||||
OFF; compare correctness, quality, novelty, and **solution diversity** (distinct
|
||||
passing approaches). Report effect size with intervals; a knob only "works" if it
|
||||
beats the neutral control outside the confidence band.
|
||||
- **Novelty metric:** cluster passing solutions (AST / embedding distance); reward
|
||||
approaches that pass hidden tests *and* differ from the canonical/most-common
|
||||
solution — this is where "novel solutions" become measurable rather than vibes.
|
||||
|
||||
---
|
||||
|
||||
## 14. Configuration & CLI
|
||||
|
||||
```
|
||||
bench-olympics.py run --config events/round1.yaml [--seed N] [--judge none|insession|skill]
|
||||
bench-olympics.py replay <transcript.json>
|
||||
bench-olympics.py score --run <run_dir> --profile balanced # re-rank, no re-run
|
||||
bench-olympics.py medal --run <run_dir>
|
||||
bench-olympics.py judge --run <run_dir> --mode skill --model claude-opus-4-8 [--batch]
|
||||
```
|
||||
Config carries: teams (§7), event slate (§8), budget, scoring profile (§9),
|
||||
elicitation arms (§13), runtime/isolation tier (§12), judge mode (§10).
|
||||
Mirror the run/score separation already proven in `bench/score.py`: results persist,
|
||||
`score`/`medal` re-rank without re-running a single model.
|
||||
|
||||
---
|
||||
|
||||
## 15. Build milestones (with acceptance criteria)
|
||||
|
||||
- **M1 — Arena spine.** One team (2 members, same model) solves one MBPP problem
|
||||
end-to-end: real room deliberation → driver `!task` → `PodmanRuntime` → public
|
||||
tests → SUBMIT → hidden-test grade → transcript.json.
|
||||
*Done when:* a full transcript replays and a deterministic score is produced.
|
||||
- **M2 — Two teams, isolation, scoring.** Parallel rooms + VMs, composite score with
|
||||
intervals, cost tracking, scoreboard. *Done when:* two teams race the same event and
|
||||
a ranked result with CIs is emitted.
|
||||
- **M3 — Events, roles, personas, topologies.** Event catalog (Sprint/Marathon/Relay/
|
||||
Debugging/Review), role-based loop privileges, medal table. *Done when:* a 3-event
|
||||
slate produces a medal table from config alone.
|
||||
- **M4 — Judging + referee.** Deterministic quality + LLM-judge (in-session, then
|
||||
skill) with bias mitigation and a gold-slice calibration report; safety/fair-play
|
||||
referee live. *Done when:* judge↔human agreement is reported and penalties fire.
|
||||
- **M5 — Product mode.** Real `AgentBridge` agents converse agent-to-agent (needs the
|
||||
bridge greenlight); arena nudges turns. *Done when:* an event completes using real
|
||||
product agents end-to-end.
|
||||
- **M6 — Placebo experiment.** Factorial elicitation arms + novelty/diversity metrics.
|
||||
*Done when:* an A/B run reports framing effect sizes with intervals.
|
||||
|
||||
---
|
||||
|
||||
## 16. Open questions / decisions needed
|
||||
|
||||
1. **Turn-taking in Arena mode:** strict round-robin vs. a lightweight "who speaks
|
||||
next" router (star topology). Start round-robin (deterministic), add router in M3?
|
||||
2. **Public/hidden split for MBPP:** how many asserts to reveal vs. hold out so
|
||||
`public_tests` guide without leaking the full spec? (Proposal: reveal 1 example,
|
||||
hold the rest.)
|
||||
3. **Budget defaults:** rounds/tokens/wall-clock caps that keep an event under a few
|
||||
minutes locally while leaving room to actually collaborate.
|
||||
4. **Judge model pinning:** confirm the exact Opus id for the skill judge
|
||||
(`claude-opus-4-8`) and whether batch judging runs via the Skill tool or a
|
||||
separate headless session.
|
||||
5. **Isolation tier:** stay on `podman --network=none` for local runs, or invest in
|
||||
gVisor/microVM now for stronger guarantees and future multi-tenant use?
|
||||
6. **Novelty metric:** AST-distance vs embedding-distance for solution diversity —
|
||||
which is cheap and discriminating enough locally?
|
||||
|
||||
---
|
||||
|
||||
## References
|
||||
|
||||
**Local vault** (`~/coding/obsidian/research/`):
|
||||
- `2026-06-07-multi-agent-orchestration-patterns.md` (read-vs-write, share full traces, topologies, 15× cost)
|
||||
- `2026-06-09-eval-harnesses-benchmark-design.md` (discrimination, intervals, verifiable-first, judge de-biasing, held-out sets)
|
||||
- `2026-06-07-agent-evaluation-and-observability.md` (outcome vs trajectory, pass^k, agent-as-judge, OTel GenAI semconv)
|
||||
- `2026-06-09-agent-sandboxing-isolation.md` (container floor → gVisor → microVM; deny-by-default egress; disposable VMs)
|
||||
- `2026-06-09-securing-multi-agent-systems.md`, `2026-06-16-shared-memory-in-multi-agent-systems.md`, `2026-06-02-llm-evals.md`, `2026-06-07-agent-reliability-guardrails-and-hitl.md` (siblings)
|
||||
|
||||
**Web (2026):**
|
||||
- MultiAgentBench / MARBLE — collaboration+competition KPIs, topologies — https://arxiv.org/abs/2503.01935 · https://github.com/ulab-uiuc/MARBLE
|
||||
- AgentCoder / AgileCoder — role specialization in coding multi-agent — (see MultiAgentBench survey refs)
|
||||
- LLM-as-Judge best practices & bias mitigation (2026) — https://futureagi.com/blog/llm-as-judge-best-practices-2026 · https://futureagi.com/blog/evaluating-llm-judge-bias-mitigation-2026/
|
||||
- Judging LLM-as-a-Judge (MT-Bench biases) — https://arxiv.org/abs/2306.05685
|
||||
- Beyond pass@1 (reliability / pass^k) — https://arxiv.org/pdf/2603.29231
|
||||
- OpenTelemetry GenAI agent spans — https://opentelemetry.io/docs/specs/semconv/gen-ai/gen-ai-agent-spans/
|
||||
@@ -0,0 +1,9 @@
|
||||
"""Agent Olympics — a hackathon-competition benchmark inside hack-house.
|
||||
|
||||
This package layers a fourth benchmark axis on top of ``bench/``: teams of LLM
|
||||
agents deliberate in a real hack-house room, implement code in an isolated VM,
|
||||
and are scored on correctness/speed/quality/collaboration. See SPEC.md for the
|
||||
full design. M1 is the arena spine (one same-model team, one MBPP problem,
|
||||
real-room/local-bus deliberation -> driver implement -> PodmanRuntime -> hidden
|
||||
tests -> replayable transcript -> deterministic score).
|
||||
"""
|
||||
@@ -0,0 +1,62 @@
|
||||
"""Arena orchestrator — wire a team + challenge through the loop and persist.
|
||||
|
||||
This is the M1 entry the launcher calls: build the room substrate, the VM, and a
|
||||
manifest-stamped transcript; run the phase machine; freeze the transcript; score
|
||||
it deterministically. The run/score split mirrors ``bench/score.py`` — results
|
||||
persist so ``score``/``replay`` never re-run a model.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
from . import comms, scoring, transcript as T
|
||||
from .challenge import Challenge
|
||||
from .loop import Budget, run_event
|
||||
from .team import Team
|
||||
from .vm import VM
|
||||
|
||||
|
||||
def run_one(team: Team, challenge: Challenge, *, room_kind: str = "local",
|
||||
runtime: str = "auto", budget: Budget | None = None,
|
||||
profile: str = "balanced", seed: int = 0,
|
||||
host: str = "http://127.0.0.1:11434",
|
||||
out_dir: str = "/tmp/hh-olympics/runs",
|
||||
room_kwargs: dict | None = None, progress=None) -> dict:
|
||||
budget = budget or Budget()
|
||||
member_names = [m.name for m in team.members]
|
||||
room = comms.make_room(room_kind, member_names, **(room_kwargs or {}))
|
||||
|
||||
manifest = {
|
||||
"models": team.models, "seed": seed, "topology": team.topology,
|
||||
"framing": team.framing, "room_substrate": room.substrate,
|
||||
"runtime": runtime, "host": host,
|
||||
"budget": {"deliberate_rounds": budget.deliberate_rounds,
|
||||
"implement_attempts": budget.implement_attempts,
|
||||
"max_tokens": budget.max_tokens,
|
||||
"wall_clock_s": budget.wall_clock_s},
|
||||
"challenge_meta": challenge.meta, "profile": profile,
|
||||
"created": time.strftime("%Y-%m-%dT%H:%M:%S"),
|
||||
}
|
||||
tx = T.Transcript(team.id, challenge.id, manifest)
|
||||
vm = VM(challenge.lang, runtime_kind=runtime)
|
||||
|
||||
room.connect()
|
||||
try:
|
||||
outcome = run_event(team, challenge, room, vm, tx, budget,
|
||||
host=host, seed=seed, progress=progress)
|
||||
finally:
|
||||
room.close()
|
||||
|
||||
run_dir = Path(out_dir) / f"{team.id}__{challenge.id}"
|
||||
tx_path = run_dir / "transcript.json"
|
||||
tx.save(tx_path)
|
||||
|
||||
score = scoring.score_event(outcome, profile=profile,
|
||||
budget={"max_rounds": budget.max_rounds})
|
||||
score["transcript"] = str(tx_path)
|
||||
score["room_substrate"] = room.substrate
|
||||
import json
|
||||
(run_dir / "score.json").write_text(json.dumps(score, indent=2))
|
||||
return score
|
||||
@@ -0,0 +1,389 @@
|
||||
"""Bridge mode — real chat-plan, then a real ``/ai <agent> !task`` build.
|
||||
|
||||
Where ``loop.py`` (direct mode) shortcuts the build through ``completion`` +
|
||||
``vm.run``, this drives the *actual* product path end-to-end on a real encrypted
|
||||
room, the way ``bench-sandbox.py`` does:
|
||||
|
||||
1. boot the relay, spawn a real ``AgentBridge`` agent (the builder), connect an
|
||||
owner/referee client and a planner teammate client;
|
||||
2. BRIEF + DELIBERATE happen as real chat frames — the planner proposes and the
|
||||
real agent is asked (via the real ``/ai <agent>`` *chat* path) to confirm the
|
||||
approach;
|
||||
3. IMPLEMENT grants drive and issues ``/ai <agent> !<task>`` — the real agent
|
||||
turns it into shell, clears the destructive guard, and injects ``_sbx:input``
|
||||
keystrokes, which we capture off the wire;
|
||||
4. GRADE materializes the captured commands, reads the produced source, and runs
|
||||
it against the held-out hidden tests in the VM.
|
||||
|
||||
No ``cmd_chat`` code is modified — this only *uses* the tool. The reusable wire
|
||||
primitives (Owner/grant/task/collect, execute, spawn_agent) are imported from
|
||||
``bench-sandbox.py`` so the bridge path is byte-identical to the sandbox bench.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import importlib
|
||||
import json
|
||||
import re
|
||||
import shutil
|
||||
import sys
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
import websockets
|
||||
|
||||
from . import infer, scoring, transcript as T
|
||||
from .. import completion
|
||||
from .challenge import Challenge
|
||||
from .scoring import EventOutcome
|
||||
from .team import Team
|
||||
from .vm import VM
|
||||
|
||||
_SCRIPTS = Path(__file__).resolve().parents[2]
|
||||
if str(_SCRIPTS) not in sys.path:
|
||||
sys.path.insert(0, str(_SCRIPTS))
|
||||
_sb = importlib.import_module("bench-sandbox") # reuse proven wire primitives
|
||||
|
||||
_FN_RE = re.compile(r"assert\s+([A-Za-z_]\w*)\s*\(")
|
||||
|
||||
|
||||
def _fn_name(public_assert: str) -> str:
|
||||
m = _FN_RE.search(public_assert or "")
|
||||
return m.group(1) if m else "solve"
|
||||
|
||||
|
||||
async def _collect_chat(owner, ws, agent: str, deadline: float,
|
||||
quiet: float = 3.0) -> str:
|
||||
"""Read the agent's plain chat reply (not a sandbox outcome) until a quiet
|
||||
gap. Skips control/`_sbx` frames and the audit line."""
|
||||
parts: list[str] = []
|
||||
while time.time() < deadline:
|
||||
try:
|
||||
raw = await asyncio.wait_for(ws.recv(),
|
||||
timeout=min(quiet, deadline - time.time()))
|
||||
except asyncio.TimeoutError:
|
||||
if parts:
|
||||
break
|
||||
continue
|
||||
except websockets.ConnectionClosed:
|
||||
break
|
||||
try:
|
||||
data = json.loads(raw)
|
||||
except json.JSONDecodeError:
|
||||
continue
|
||||
if data.get("type") != "message":
|
||||
continue
|
||||
dec = owner.decrypt_message(data.get("data", {}))
|
||||
if dec.get("username") != agent:
|
||||
continue
|
||||
text = _sb._strip_think(dec.get("text", ""))
|
||||
if text.startswith('{"_'): # control / sbx frame
|
||||
continue
|
||||
parts.append(text)
|
||||
return "\n".join(parts).strip()
|
||||
|
||||
|
||||
async def _agent_alive(owner, ws, agent: str, window: float = 1.5) -> bool:
|
||||
"""Fast-fail liveness probe. The relay broadcasts a `user_left` + fresh
|
||||
`roster` whenever a client drops (e.g. the agent killed by a 1011 keepalive
|
||||
timeout). Drain whatever is already queued on the owner ws for a short
|
||||
window; if a roster snapshot arrives without the agent, it's gone — so the
|
||||
caller can abort instead of waiting out the full step deadline."""
|
||||
deadline = time.time() + window
|
||||
alive = True
|
||||
while time.time() < deadline:
|
||||
try:
|
||||
raw = await asyncio.wait_for(ws.recv(),
|
||||
timeout=max(0.05, deadline - time.time()))
|
||||
except (asyncio.TimeoutError, websockets.ConnectionClosed):
|
||||
break
|
||||
try:
|
||||
data = json.loads(raw)
|
||||
except json.JSONDecodeError:
|
||||
continue
|
||||
if data.get("type") == "roster":
|
||||
names = {u.get("username") for u in data.get("users", [])}
|
||||
alive = agent in names
|
||||
return alive
|
||||
|
||||
|
||||
def _find_source(cwd: str | None, fn: str) -> str:
|
||||
"""Read the python source the agent built. Prefer solution.py, else any .py
|
||||
that defines the target function, else the largest .py."""
|
||||
if not cwd:
|
||||
return ""
|
||||
pys = list(Path(cwd).rglob("*.py"))
|
||||
if not pys:
|
||||
return ""
|
||||
named = [p for p in pys if p.name == "solution.py"]
|
||||
if named:
|
||||
return named[0].read_text()
|
||||
for p in pys:
|
||||
try:
|
||||
if f"def {fn}" in p.read_text():
|
||||
return p.read_text()
|
||||
except Exception: # noqa: BLE001
|
||||
continue
|
||||
return max(pys, key=lambda p: p.stat().st_size).read_text()
|
||||
|
||||
|
||||
def run_bridge_event(team: Team, challenge: Challenge, *,
|
||||
host: str = "127.0.0.1", port: int = 4699,
|
||||
password: str = "olympics-pass",
|
||||
ollama: str = "http://127.0.0.1:11434",
|
||||
code_model: str | None = None,
|
||||
out_dir: str = "/tmp/hh-olympics/runs",
|
||||
log_dir: str = "/tmp/hh-olympics",
|
||||
step_timeout: float = 180.0, seed: int = 0,
|
||||
profile: str = "balanced", agent_chat_confirm: bool = False,
|
||||
progress=None) -> dict:
|
||||
"""Run one real chat-plan -> !task-build event. Returns the score dict."""
|
||||
py = sys.executable
|
||||
driver = team.driver()
|
||||
planner = next((m for m in team.members if m is not driver), driver)
|
||||
build_model = code_model or driver.model
|
||||
agent_name = driver.model # the agent joins under its model tag
|
||||
fn = _fn_name(challenge.public_tests[0] if challenge.public_tests else "")
|
||||
|
||||
manifest = {"models": team.models, "seed": seed, "topology": team.topology,
|
||||
"framing": team.framing, "room_substrate": "real-bridge",
|
||||
"build_model": build_model, "agent": agent_name, "fn": fn,
|
||||
"profile": profile, "challenge_meta": challenge.meta,
|
||||
"created": time.strftime("%Y-%m-%dT%H:%M:%S")}
|
||||
tx = T.Transcript(team.id, challenge.id, manifest)
|
||||
vm = VM(challenge.lang)
|
||||
Path(log_dir).mkdir(parents=True, exist_ok=True)
|
||||
|
||||
def emit(p):
|
||||
if progress:
|
||||
progress(p)
|
||||
|
||||
# ── boot relay + spawn the real agent ────────────────────────────────
|
||||
srv_log = open(Path(log_dir) / f"bridge-server-{port}.log", "w")
|
||||
srv = _sb.subprocess.Popen(
|
||||
[py, "cmd_chat.py", "serve", host, str(port),
|
||||
"--password", password, "--no-tls"],
|
||||
cwd=str(_sb.REPO), stdout=srv_log, stderr=_sb.subprocess.STDOUT)
|
||||
agent = None
|
||||
alog = None
|
||||
tokens = {"in": 0, "out": 0}
|
||||
penalties: list[dict] = []
|
||||
t0 = time.time()
|
||||
try:
|
||||
if not _sb._wait_port(host, port, time.time() + 30):
|
||||
raise RuntimeError("relay never bound")
|
||||
emit("server up")
|
||||
alog = open(Path(log_dir) / f"bridge-agent-{port}.log", "w")
|
||||
agent = _sb.spawn_agent(py, host, port, password, agent_name,
|
||||
build_model, alog)
|
||||
emit(f"agent '{agent_name}' spawned (code-model={build_model})")
|
||||
|
||||
outcome = asyncio.run(_drive(
|
||||
tx, team, challenge, planner, driver, agent_name, fn,
|
||||
host, port, password, ollama, step_timeout, vm, tokens,
|
||||
penalties, agent_chat_confirm, build_model, emit))
|
||||
finally:
|
||||
if agent is not None:
|
||||
agent.terminate()
|
||||
try:
|
||||
agent.wait(timeout=10)
|
||||
except _sb.subprocess.TimeoutExpired:
|
||||
agent.kill()
|
||||
if alog:
|
||||
alog.close()
|
||||
srv.terminate()
|
||||
try:
|
||||
srv.wait(timeout=10)
|
||||
except _sb.subprocess.TimeoutExpired:
|
||||
srv.kill()
|
||||
srv_log.close()
|
||||
|
||||
outcome.wall_clock_s = time.time() - t0
|
||||
run_dir = Path(out_dir) / f"{team.id}__{challenge.id}__bridge"
|
||||
tx_path = run_dir / "transcript.json"
|
||||
tx.save(tx_path)
|
||||
score = scoring.score_event(outcome, profile=profile, budget={"max_rounds": 1})
|
||||
score["transcript"] = str(tx_path)
|
||||
score["room_substrate"] = "real-bridge"
|
||||
(run_dir / "score.json").write_text(json.dumps(score, indent=2))
|
||||
return score
|
||||
|
||||
|
||||
async def _drive(tx, team, challenge, planner, driver, agent_name, fn,
|
||||
host, port, password, ollama, step_to, vm, tokens,
|
||||
penalties, agent_chat_confirm, build_model, emit) -> EventOutcome:
|
||||
owner = _sb.Owner(host, port, password)
|
||||
owner.srp_authenticate()
|
||||
# a planner teammate client (real chat voice alongside the real agent)
|
||||
from cmd_chat.client.client import Client # noqa: E402
|
||||
mate = Client(host, port, username=planner.name, password=password, no_tls=True)
|
||||
mate.srp_authenticate()
|
||||
|
||||
owner_url = f"{owner.ws_url}/ws/chat?user_id={owner.user_id}&ws_token={owner.ws_token}"
|
||||
mate_url = f"{mate.ws_url}/ws/chat?user_id={mate.user_id}&ws_token={mate.ws_token}"
|
||||
|
||||
public_passed = False
|
||||
submitted = False
|
||||
correct = False
|
||||
completion_src = ""
|
||||
defect_class: str | None = None
|
||||
model_baseline: bool | None = None
|
||||
|
||||
async with websockets.connect(owner_url) as ws, \
|
||||
websockets.connect(mate_url) as mws:
|
||||
if not await owner.wait_for_agent(ws, agent_name, time.time() + step_to):
|
||||
penalties.append({"kind": "agent_offline", "detail": agent_name})
|
||||
return EventOutcome(team.id, challenge.id, False, False, 0, 0.0,
|
||||
tokens, False, None, penalties)
|
||||
|
||||
async def mate_send(text):
|
||||
await mws.send(mate.room_fernet.encrypt(text.encode()).decode())
|
||||
|
||||
# ── BRIEF ────────────────────────────────────────────────────────
|
||||
tx.phase_change("BRIEF")
|
||||
brief = challenge.brief()
|
||||
await owner._send(ws, brief)
|
||||
tx.add(T.KIND_MESSAGE, "BRIEF", "referee", {"text": brief}, role="referee")
|
||||
emit("brief posted")
|
||||
|
||||
# ── DELIBERATE: planner proposes (real chat), agent confirms (real
|
||||
# /ai chat path) ────────────────────────────────────────────────
|
||||
tx.phase_change("DELIBERATE")
|
||||
# blocking inference must run off the event loop, else the websocket
|
||||
# keepalive can't answer the server's ping and the room drops us (1011).
|
||||
plan_msg = await asyncio.to_thread(
|
||||
infer.chat,
|
||||
planner.model, planner.system_prompt(),
|
||||
[{"role": "user", "content":
|
||||
f"{brief}\n\nPropose, in 2-3 short lines, the approach and the "
|
||||
f"exact function signature (name it {fn}). End with PLAN-LOCKED."}],
|
||||
host=ollama)
|
||||
tokens["in"] += plan_msg.tokens.get("in", 0)
|
||||
tokens["out"] += plan_msg.tokens.get("out", 0)
|
||||
ptext = plan_msg.text if plan_msg.ok else f"[infer error: {plan_msg.error}]"
|
||||
await mate_send(ptext)
|
||||
tx.add(T.KIND_MESSAGE, "DELIBERATE", planner.name,
|
||||
{"text": ptext}, role=planner.role, model=planner.model)
|
||||
emit("planner proposed")
|
||||
|
||||
# Optionally ask the REAL agent to weigh in via the real /ai *chat* path.
|
||||
# HARNESS FINDING (default off): the agent's streaming chat path
|
||||
# (_answer -> _stream_reply -> OllamaProvider.stream) starves its own
|
||||
# asyncio loop past the websockets 20s keepalive window under local CPU
|
||||
# inference, so the relay drops the agent with `1011 keepalive ping
|
||||
# timeout` and the build never happens. The non-streaming `!task` path
|
||||
# (_run_in_sandbox -> to_thread(complete)) does NOT have this defect, so
|
||||
# by default we keep planning to real planner room-frames and let the
|
||||
# agent's only inference be the robust `!task` build below.
|
||||
if agent_chat_confirm:
|
||||
await owner._send(
|
||||
ws, f"/ai {agent_name} Teammate proposed: {ptext[:300]} . In one "
|
||||
f"line, confirm the function name {fn} and the core idea.")
|
||||
agent_reply = await _collect_chat(owner, ws, agent_name,
|
||||
time.time() + step_to)
|
||||
if not await _agent_alive(owner, ws, agent_name):
|
||||
penalties.append({"kind": "agent_dropped",
|
||||
"detail": "streaming /ai chat keepalive timeout"})
|
||||
tx.add(T.KIND_MESSAGE, "DELIBERATE", agent_name,
|
||||
{"text": agent_reply or "(no chat reply)"}, role="builder",
|
||||
model=agent_name)
|
||||
emit("agent chat reply captured")
|
||||
else:
|
||||
tx.add(T.KIND_MESSAGE, "DELIBERATE", "referee",
|
||||
{"text": "agent /ai chat-confirm skipped (streaming chat path "
|
||||
"drops the agent via 1011 keepalive timeout under CPU "
|
||||
"inference); building via the robust !task path",
|
||||
"note": "harness-workaround"}, role="referee")
|
||||
emit("agent chat-confirm skipped (streaming-path keepalive bug)")
|
||||
|
||||
# ── IMPLEMENT: real /ai !task build ───────────────────────────────
|
||||
await owner.grant(ws, agent_name)
|
||||
await asyncio.sleep(0.6)
|
||||
tx.acl("IMPLEMENT", {"_perm": "acl", "drivers": [agent_name]})
|
||||
tx.phase_change("IMPLEMENT", note=f"driver={agent_name}")
|
||||
example = challenge.public_tests[0] if challenge.public_tests else ""
|
||||
task = (f"Create a file named solution.py that defines a python function "
|
||||
f"named {fn} solving this: {challenge.text.strip()} "
|
||||
f"It must satisfy: {example} . "
|
||||
f"Write only the function definition in solution.py — no tests, "
|
||||
f"no example calls, no printing.")
|
||||
await owner.task(ws, agent_name, task)
|
||||
res = await owner.collect(ws, agent_name, time.time() + step_to)
|
||||
tx.tool("IMPLEMENT", agent_name,
|
||||
{"label": "sbx-build", "outcome": res["outcome"],
|
||||
"sbx_frames": res["sbx"], "commands": res["commands"]},
|
||||
role="builder")
|
||||
emit(f"build outcome={res['outcome']} sbx={res['sbx']} "
|
||||
f"cmds={len(res['commands'])}")
|
||||
|
||||
if res["outcome"] == "destructive_gated":
|
||||
tx.guard("IMPLEMENT", agent_name, {"gated": res["message"][:80]})
|
||||
penalties.append({"kind": "destructive", "detail": "build gated"})
|
||||
|
||||
# ── GRADE: materialize captured cmds, read source, run hidden ─────
|
||||
if res["commands"]:
|
||||
ex = await asyncio.to_thread(_sb.execute, res["commands"], 30.0)
|
||||
src = _find_source(ex.get("cwd"), fn)
|
||||
tx.tool("GRADE", "referee",
|
||||
{"label": "materialize", "ran": ex["ran"],
|
||||
"skipped": ex["skipped"], "found_source": bool(src),
|
||||
"out": (ex["out"] or "")[-300:]})
|
||||
if src:
|
||||
completion_src = src
|
||||
# Phase 3a — does the as-injected source even compile? A syntax/
|
||||
# indentation failure here is the product keystroke path mangling
|
||||
# the code (e.g. _extract_commands stripping heredoc indentation),
|
||||
# not a model error. Attribute it so a matrix can separate them.
|
||||
compiles = True
|
||||
try:
|
||||
compile(src, "<sbx>", "exec")
|
||||
except (SyntaxError, IndentationError) as e:
|
||||
compiles = False
|
||||
defect_class = f"product-path-defect:{type(e).__name__}"
|
||||
pub = await asyncio.to_thread(
|
||||
vm.run, challenge.public_program(src), 30.0, label="public")
|
||||
public_passed = pub.ok
|
||||
tx.tool("TEST", "referee",
|
||||
{"label": "public", "rc": pub.rc, "ok": pub.ok,
|
||||
"out": pub.out[-300:]})
|
||||
hid = await asyncio.to_thread(
|
||||
vm.run, challenge.hidden_program(src), 30.0,
|
||||
label="hidden-grade")
|
||||
correct = hid.ok
|
||||
tx.tool("SUBMIT", "referee",
|
||||
{"label": "hidden-grade", "rc": hid.rc, "ok": hid.ok,
|
||||
"out": hid.out[-300:]})
|
||||
if not correct and not compiles:
|
||||
penalties.append({"kind": "product_path_defect",
|
||||
"detail": defect_class})
|
||||
if ex.get("cwd"):
|
||||
shutil.rmtree(ex["cwd"], ignore_errors=True)
|
||||
submitted = bool(completion_src)
|
||||
|
||||
# Phase 3b — control: the build model's *true* coding ability via the raw
|
||||
# completion path (direct mode, no keystroke stripping). The delta between
|
||||
# this and `correct` above quantifies exactly what the product path costs.
|
||||
ctrl = await asyncio.to_thread(
|
||||
completion.complete, build_model, challenge.gen_prompt(),
|
||||
challenge.stop_tokens(), host=ollama, timeout=180.0)
|
||||
if ctrl.ok and ctrl.text.strip():
|
||||
base = await asyncio.to_thread(
|
||||
vm.run, challenge.hidden_program(ctrl.text), 30.0,
|
||||
label="model-baseline")
|
||||
model_baseline = base.ok
|
||||
tx.tool("GRADE", "referee",
|
||||
{"label": "model-baseline", "ok": base.ok, "rc": base.rc,
|
||||
"model": build_model, "out": base.out[-300:]})
|
||||
emit(f"model-baseline (direct gen) correct={base.ok}")
|
||||
|
||||
await owner.revoke(ws)
|
||||
tx.add(T.KIND_TOOL, "SUBMIT", "referee",
|
||||
{"label": "artifact", **vm.freeze()})
|
||||
|
||||
rtg = 1 if public_passed else None
|
||||
return EventOutcome(team.id, challenge.id, submitted, correct,
|
||||
rounds=1, wall_clock_s=0.0, tokens=tokens,
|
||||
public_passed=public_passed, rounds_to_green=rtg,
|
||||
penalties=penalties, defect_class=defect_class,
|
||||
model_baseline_correct=model_baseline)
|
||||
@@ -0,0 +1,54 @@
|
||||
"""Model-aware wall-clock budgeting.
|
||||
|
||||
CPU inference time is U-shaped in model size: tiny models are slow because they
|
||||
*over-generate* (rambling completions that hit num_predict every call) and retry
|
||||
more, while big models are slow *per token* (more parameters). 3-4B is the sweet
|
||||
spot. Reasoning models (r1/qwq/o1) add a long <think> preamble on top.
|
||||
|
||||
We scale only the *wall-clock* budget — never the implement-attempt count — so
|
||||
the correctness and speed axes stay comparable across model sizes (attempts feed
|
||||
``scoring._speed_score``'s cap; changing them per model would change what the
|
||||
benchmark measures). Mirrors ``bench-sandbox``'s reasoning-model 3x rule.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
|
||||
# Same tags bench-sandbox uses, so the whole suite treats reasoning models alike.
|
||||
_REASONING_TAGS = ("r1", "qwq", "reason", "think", "o1")
|
||||
_PARAM_RE = re.compile(r"(\d+(?:\.\d+)?)\s*b\b", re.IGNORECASE)
|
||||
|
||||
|
||||
def _is_reasoning(model: str | None) -> bool:
|
||||
return bool(model) and any(t in model.lower() for t in _REASONING_TAGS)
|
||||
|
||||
|
||||
def _param_b(model: str | None) -> float | None:
|
||||
"""Best-effort parameter count in billions parsed from the tag (e.g.
|
||||
``qwen2.5-coder:1.5b`` -> 1.5). Returns None when the tag carries no size."""
|
||||
if not model:
|
||||
return None
|
||||
m = _PARAM_RE.search(model)
|
||||
return float(m.group(1)) if m else None
|
||||
|
||||
|
||||
def budget_scale(model: str | None) -> float:
|
||||
"""Multiplier applied to the base wall-clock budget for ``model``.
|
||||
|
||||
Reasoning dominates (it stacks a think-preamble on whatever size it is).
|
||||
Otherwise the U-shaped size curve applies; unknown sizes get the 1.0
|
||||
baseline. Defaults are first-guess and meant to be retuned from the ledger's
|
||||
recorded ``wall_clock_s`` vs ``wall_clock_budget``."""
|
||||
if _is_reasoning(model):
|
||||
return 3.0
|
||||
b = _param_b(model)
|
||||
if b is None:
|
||||
return 1.0
|
||||
if b <= 1.5:
|
||||
return 1.5 # over-generation + retries
|
||||
if b <= 4:
|
||||
return 1.0 # sweet spot
|
||||
if b < 13:
|
||||
return 1.5 # per-token CPU latency (7b-class)
|
||||
return 2.0 # large models, more so
|
||||
@@ -0,0 +1,188 @@
|
||||
"""Challenge adapter — MBPP bootstrap with a public/hidden test split.
|
||||
|
||||
The arena consumes a ``Challenge``: a brief the referee posts to the room, a
|
||||
*public* test the team may run during TEST, and a *hidden* test held out for
|
||||
SUBMIT grading (so teams can't teach to the test — SPEC §8/§10).
|
||||
|
||||
M1 implements the MBPP-Python adapter only (the spine; multi-language MultiPL-E
|
||||
splits arrive in M3). It reuses ``bench/datasets.py`` for rows and
|
||||
``bench/langs.py`` for the execution recipe; the per-assert public/hidden split
|
||||
and prompt synthesis live here.
|
||||
|
||||
Split policy (SPEC §16 decision): reveal exactly ONE assert from ``test_list``
|
||||
as the public example; hold the remainder as hidden. If a problem ships only one
|
||||
assert, that single assert is both the public guide and the hidden gate.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Callable
|
||||
|
||||
from .. import datasets
|
||||
from ..langs import Lang, resolve
|
||||
|
||||
_MBPP_PY_STOP = ["\nassert", "\nprint(", "\n# Test", "\nif __name__", '\n"""']
|
||||
|
||||
|
||||
@dataclass
|
||||
class Challenge:
|
||||
id: str
|
||||
language: str
|
||||
lang: Lang
|
||||
text: str # natural-language task
|
||||
public_tests: list[str] # asserts revealed to the team
|
||||
hidden_tests: list[str] # asserts held out for grading
|
||||
setup: str = "" # test_setup_code, prepended to programs
|
||||
reference: str = "" # canonical solution (audit / novelty later)
|
||||
meta: dict = field(default_factory=dict)
|
||||
# ── MultiPL-E continuation mode (M3) ──────────────────────────────────
|
||||
# When ``prompt_prefix`` is set the challenge is *continuation*-style (a
|
||||
# runnable function prefix the model completes), not the MBPP-Python
|
||||
# description+asserts style. MultiPL-E ships one ``tests`` block per row with
|
||||
# no per-assert structure, so there is no public/hidden split: the program is
|
||||
# ``prefix + completion + tests`` (via ``_assemble_fn``) for both public and
|
||||
# hidden grading. ``public_tests``/``hidden_tests`` stay empty in this mode.
|
||||
prompt_prefix: str = "" # runnable function prefix (MultiPL-E `prompt`)
|
||||
tests_block: str = "" # the single MultiPL-E `tests` block (audit)
|
||||
stop: list[str] | None = None # per-row stop tokens (MultiPL-E)
|
||||
_assemble_fn: Callable[[str, str, dict], str] | None = None
|
||||
_row: dict = field(default_factory=dict)
|
||||
|
||||
@property
|
||||
def _continuation(self) -> bool:
|
||||
return bool(self.prompt_prefix)
|
||||
|
||||
# ── what the room sees ────────────────────────────────────────────────
|
||||
def brief(self) -> str:
|
||||
if self._continuation:
|
||||
return (
|
||||
f"CHALLENGE {self.id} ({self.language}). Complete this "
|
||||
f"{self.language} function so the hidden tests pass:\n"
|
||||
f"{self.prompt_prefix.rstrip()}\n"
|
||||
f"Deliver one self-contained {self.language} solution that "
|
||||
f"continues the prefix above. Hidden tests grade it at SUBMIT."
|
||||
)
|
||||
example = self.public_tests[0] if self.public_tests else ""
|
||||
return (
|
||||
f"CHALLENGE {self.id} ({self.language}). Implement a solution to:\n"
|
||||
f" {self.text.strip()}\n"
|
||||
f"Example test that must pass:\n {example}\n"
|
||||
f"Deliver one self-contained {self.language} solution. "
|
||||
f"Hidden tests will grade it at SUBMIT."
|
||||
)
|
||||
|
||||
# ── what the implementer model is asked to continue (raw, no hidden) ───
|
||||
def gen_prompt(self) -> str:
|
||||
if self._continuation:
|
||||
return self.prompt_prefix
|
||||
asserts = "\n".join(self.public_tests)
|
||||
return f'"""\n{self.text.strip()}\n\n{asserts}\n"""\n'
|
||||
|
||||
def stop_tokens(self) -> list[str]:
|
||||
if self.stop is not None:
|
||||
return self.stop
|
||||
return _MBPP_PY_STOP
|
||||
|
||||
# ── assemble a runnable program for a given test set ──────────────────
|
||||
def _assemble(self, completion: str, asserts: list[str]) -> str:
|
||||
body = "\n".join(asserts)
|
||||
return f"{self.setup}\n{completion}\n{body}\n"
|
||||
|
||||
def public_program(self, completion: str) -> str:
|
||||
if self._continuation:
|
||||
return self._assemble_fn(self.prompt_prefix, completion, self._row)
|
||||
return self._assemble(completion, self.public_tests)
|
||||
|
||||
def hidden_program(self, completion: str) -> str:
|
||||
if self._continuation:
|
||||
# single tests block — public == hidden for MultiPL-E.
|
||||
return self._assemble_fn(self.prompt_prefix, completion, self._row)
|
||||
# grade on the full spec (public example + held-out asserts) so a
|
||||
# solution that only satisfies the revealed example still fails.
|
||||
return self._assemble(completion, self.public_tests + self.hidden_tests)
|
||||
|
||||
|
||||
def _split(test_list: list[str]) -> tuple[list[str], list[str]]:
|
||||
if not test_list:
|
||||
return [], []
|
||||
if len(test_list) == 1:
|
||||
return [test_list[0]], [test_list[0]]
|
||||
return [test_list[0]], test_list[1:]
|
||||
|
||||
|
||||
def _doc_line(prompt: str) -> str:
|
||||
"""Best-effort one-liner from a MultiPL-E prompt's leading comment block."""
|
||||
for raw in prompt.splitlines():
|
||||
s = raw.lstrip("#/ \t").strip()
|
||||
if s and not s.startswith("!") and "(" not in s:
|
||||
return s
|
||||
return prompt.strip().splitlines()[0] if prompt.strip() else ""
|
||||
|
||||
|
||||
def _mbpp_task_id(name: str) -> int | None:
|
||||
# MultiPL-E mbpp names look like "mbpp_3_is_not_prime".
|
||||
parts = name.split("_")
|
||||
if len(parts) >= 2 and parts[0] == "mbpp" and parts[1].isdigit():
|
||||
return int(parts[1])
|
||||
return None
|
||||
|
||||
|
||||
def load_multipl_e(task_id: int | None = None, *, index: int = 0,
|
||||
language: str = "javascript",
|
||||
suite: str = "mbpp") -> Challenge:
|
||||
"""Build one MultiPL-E continuation challenge for a non-Python language.
|
||||
|
||||
Reuses ``suites.binding`` for the (suite, language) format recipe. Pick by
|
||||
embedded MBPP ``task_id`` (e.g. ``mbpp_11_...``) if given, else by ``index``
|
||||
into the cached split. The row's single ``tests`` block grades both public
|
||||
and hidden (MultiPL-E has no per-assert split — see ``Challenge``)."""
|
||||
from ..suites import binding as _binding
|
||||
lang = resolve(language)
|
||||
b = _binding(suite, language)
|
||||
rows = datasets.load(b.dataset, b.config, split="test")
|
||||
if task_id is not None:
|
||||
row = next((r for r in rows
|
||||
if _mbpp_task_id(r.get("name", "")) == task_id), None)
|
||||
if row is None:
|
||||
raise KeyError(f"{suite} task_id {task_id} not in {b.config} split")
|
||||
else:
|
||||
row = rows[index]
|
||||
name = row.get("name", f"{suite}-{lang.id}-{index}")
|
||||
tid = _mbpp_task_id(name)
|
||||
prompt = b.build_prompt(row)
|
||||
return Challenge(
|
||||
id=f"{suite}-{lang.id}-{tid if tid is not None else index}",
|
||||
language=lang.id, lang=lang, text=_doc_line(prompt),
|
||||
public_tests=[], hidden_tests=[],
|
||||
reference=row.get("original", "") or "",
|
||||
prompt_prefix=prompt, tests_block=row.get("tests", "") or "",
|
||||
stop=b.stop_tokens(row), _assemble_fn=b.assemble, _row=row,
|
||||
meta={"name": name, "suite": suite, "config": b.config,
|
||||
"task_id": tid})
|
||||
|
||||
|
||||
def load_mbpp(task_id: int | None = None, *, index: int = 0,
|
||||
language: str = "python") -> Challenge:
|
||||
"""Build one MBPP challenge. Pick by ``task_id`` if given, else by ``index``
|
||||
into the cached test split. Non-Python languages route to the MultiPL-E
|
||||
continuation adapter (``load_multipl_e``)."""
|
||||
if resolve(language).id != "python":
|
||||
return load_multipl_e(task_id, index=index, language=language,
|
||||
suite="mbpp")
|
||||
lang = resolve(language)
|
||||
rows = datasets.load("google-research-datasets/mbpp", "full")
|
||||
if task_id is not None:
|
||||
row = next((r for r in rows if r.get("task_id") == task_id), None)
|
||||
if row is None:
|
||||
raise KeyError(f"MBPP task_id {task_id} not in cached split")
|
||||
else:
|
||||
row = rows[index]
|
||||
public, hidden = _split(row.get("test_list", []))
|
||||
tid = row.get("task_id", index)
|
||||
return Challenge(
|
||||
id=f"mbpp-{tid}", language=lang.id, lang=lang,
|
||||
text=row.get("text", ""), public_tests=public, hidden_tests=hidden,
|
||||
setup=row.get("test_setup_code", "") or "",
|
||||
reference=row.get("code", ""),
|
||||
meta={"task_id": tid, "n_tests": len(row.get("test_list", []))})
|
||||
@@ -0,0 +1,242 @@
|
||||
"""The room is the bus (SPEC pillar #1).
|
||||
|
||||
Two interchangeable substrates implement the same synchronous ``Room`` facade so
|
||||
``loop.py`` never knows which it is talking to (mirroring how ``runtime.py`` has
|
||||
Podman/Local):
|
||||
|
||||
• RealRoom — boots the hack-house relay, connects one authenticated client per
|
||||
member plus a referee/recorder client, and posts *real* end-to-end-encrypted
|
||||
frames. Every utterance round-trips through the zero-knowledge server and is
|
||||
observed off the wire by the referee. This is the project thesis: team
|
||||
deliberation flows through a real encrypted room, not a simulated channel.
|
||||
• LocalBus — an in-memory echo bus with the identical facade. No server, no
|
||||
flakiness; used as the default for deterministic local runs and CI. It
|
||||
records the same transcript a RealRoom would.
|
||||
|
||||
Both are arena-mode: the orchestrator drives inference and *posts* results into
|
||||
the room. Neither edits ``cmd_chat`` — RealRoom only uses the public Client.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
import socket
|
||||
import subprocess
|
||||
import sys
|
||||
import threading
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
REPO = Path(__file__).resolve().parents[4]
|
||||
|
||||
|
||||
class Delivery:
|
||||
"""Result of a post: did the frame round-trip through the substrate?"""
|
||||
__slots__ = ("delivered", "substrate")
|
||||
|
||||
def __init__(self, delivered: bool, substrate: str):
|
||||
self.delivered = delivered
|
||||
self.substrate = substrate
|
||||
|
||||
def __repr__(self):
|
||||
return f"Delivery(delivered={self.delivered}, substrate={self.substrate})"
|
||||
|
||||
|
||||
# ── in-memory substrate ────────────────────────────────────────────────────
|
||||
class LocalBus:
|
||||
"""Zero-dependency echo bus. Same facade as RealRoom."""
|
||||
|
||||
substrate = "local"
|
||||
|
||||
def __init__(self):
|
||||
self.log: list[dict] = []
|
||||
|
||||
def connect(self) -> None:
|
||||
pass
|
||||
|
||||
def post(self, actor: str, text: str) -> Delivery:
|
||||
self.log.append({"actor": actor, "text": text, "ts": time.time()})
|
||||
return Delivery(True, self.substrate)
|
||||
|
||||
def brief(self, text: str) -> Delivery:
|
||||
return self.post("referee", text)
|
||||
|
||||
def acl(self, payload: dict) -> Delivery:
|
||||
self.log.append({"actor": "referee", "acl": payload, "ts": time.time()})
|
||||
return Delivery(True, self.substrate)
|
||||
|
||||
def close(self) -> None:
|
||||
pass
|
||||
|
||||
|
||||
# ── real encrypted-room substrate ──────────────────────────────────────────
|
||||
def _port_open(host: str, port: int, timeout: float = 0.5) -> bool:
|
||||
try:
|
||||
with socket.create_connection((host, port), timeout=timeout):
|
||||
return True
|
||||
except OSError:
|
||||
return False
|
||||
|
||||
|
||||
class RealRoom:
|
||||
"""Boots the relay and connects N member clients + a referee recorder."""
|
||||
|
||||
substrate = "real"
|
||||
|
||||
def __init__(self, member_names: list[str], *, host: str = "127.0.0.1",
|
||||
port: int = 4677, password: str = "olympics-pass",
|
||||
boot_server: bool = True, quiet: float = 1.2,
|
||||
log_dir: str = "/tmp/hh-olympics"):
|
||||
self.member_names = member_names
|
||||
self.host = host
|
||||
self.port = port
|
||||
self.password = password
|
||||
self.boot_server = boot_server
|
||||
self.quiet = quiet
|
||||
self.log_dir = Path(log_dir)
|
||||
self._srv: subprocess.Popen | None = None
|
||||
self._srv_log = None
|
||||
self._loop: asyncio.AbstractEventLoop | None = None
|
||||
self._thread: threading.Thread | None = None
|
||||
self._clients: dict[str, object] = {} # name -> Client
|
||||
self._ws: dict[str, object] = {} # name -> websocket
|
||||
self._referee = "referee"
|
||||
|
||||
# -- lifecycle ----------------------------------------------------------
|
||||
def connect(self) -> None:
|
||||
if str(REPO) not in sys.path:
|
||||
sys.path.insert(0, str(REPO))
|
||||
from cmd_chat.client.client import Client # noqa: E402
|
||||
|
||||
if self.boot_server:
|
||||
self._spawn_server()
|
||||
|
||||
self._loop = asyncio.new_event_loop()
|
||||
names = [*self.member_names, self._referee]
|
||||
for name in names:
|
||||
c = Client(self.host, self.port, username=name,
|
||||
password=self.password, no_tls=True)
|
||||
c.srp_authenticate()
|
||||
self._clients[name] = c
|
||||
# Run the loop continuously on a background thread. The orchestrator does
|
||||
# long blocking inference between posts; if the loop only ran during
|
||||
# run_until_complete(post) the websockets reader couldn't answer the
|
||||
# server's keepalive ping in those gaps and the relay would drop us
|
||||
# (1011). A forever-loop keeps pings serviced regardless of the main
|
||||
# thread blocking.
|
||||
self._thread = threading.Thread(target=self._loop.run_forever,
|
||||
daemon=True)
|
||||
self._thread.start()
|
||||
self._call(self._open_all())
|
||||
|
||||
def _call(self, coro):
|
||||
"""Run a coroutine on the background loop from the main thread."""
|
||||
return asyncio.run_coroutine_threadsafe(coro, self._loop).result()
|
||||
|
||||
def _spawn_server(self) -> None:
|
||||
self.log_dir.mkdir(parents=True, exist_ok=True)
|
||||
self._srv_log = open(self.log_dir / f"server-{self.port}.log", "w")
|
||||
self._srv = subprocess.Popen(
|
||||
[sys.executable, "cmd_chat.py", "serve", self.host, str(self.port),
|
||||
"--password", self.password, "--no-tls"],
|
||||
cwd=str(REPO), stdout=self._srv_log, stderr=subprocess.STDOUT)
|
||||
deadline = time.time() + 30
|
||||
while time.time() < deadline:
|
||||
if _port_open(self.host, self.port):
|
||||
return
|
||||
time.sleep(0.2)
|
||||
raise RuntimeError(f"relay server never bound on {self.host}:{self.port}")
|
||||
|
||||
async def _open_all(self) -> None:
|
||||
import websockets
|
||||
for name, c in self._clients.items():
|
||||
url = (f"{c.ws_url}/ws/chat?user_id={c.user_id}"
|
||||
f"&ws_token={c.ws_token}")
|
||||
self._ws[name] = await websockets.connect(url)
|
||||
# let presence settle so the referee sees subsequent broadcasts
|
||||
await asyncio.sleep(0.4)
|
||||
|
||||
# -- posting ------------------------------------------------------------
|
||||
def _encrypt(self, name: str, text: str) -> str:
|
||||
c = self._clients[name]
|
||||
return c.room_fernet.encrypt(text.encode()).decode()
|
||||
|
||||
async def _send(self, name: str, text: str) -> None:
|
||||
await self._ws[name].send(self._encrypt(name, text))
|
||||
|
||||
async def _observe(self, want_actor: str, want_text: str,
|
||||
deadline: float) -> bool:
|
||||
"""Drain the referee ws until we see the actor's frame (round-trip)."""
|
||||
import websockets
|
||||
ref = self._clients[self._referee]
|
||||
ws = self._ws[self._referee]
|
||||
snippet = want_text[:24]
|
||||
while time.time() < deadline:
|
||||
try:
|
||||
raw = await asyncio.wait_for(ws.recv(),
|
||||
timeout=max(0.05, deadline - time.time()))
|
||||
except asyncio.TimeoutError:
|
||||
return False
|
||||
except websockets.ConnectionClosed:
|
||||
return False
|
||||
try:
|
||||
data = json.loads(raw)
|
||||
except json.JSONDecodeError:
|
||||
continue
|
||||
if data.get("type") != "message":
|
||||
continue
|
||||
dec = ref.decrypt_message(data.get("data", {}))
|
||||
if dec.get("username") == want_actor and snippet in dec.get("text", ""):
|
||||
return True
|
||||
return False
|
||||
|
||||
def _post_sync(self, actor: str, text: str) -> Delivery:
|
||||
async def _do():
|
||||
await self._send(actor, text)
|
||||
ok = await self._observe(actor, text, time.time() + self.quiet)
|
||||
return ok
|
||||
ok = self._call(_do())
|
||||
return Delivery(ok, self.substrate)
|
||||
|
||||
def post(self, actor: str, text: str) -> Delivery:
|
||||
return self._post_sync(actor, text)
|
||||
|
||||
def brief(self, text: str) -> Delivery:
|
||||
return self._post_sync(self._referee, text)
|
||||
|
||||
def acl(self, payload: dict) -> Delivery:
|
||||
# referee (acting as owner) broadcasts the ACL control frame
|
||||
return self._post_sync(self._referee, json.dumps(payload))
|
||||
|
||||
def close(self) -> None:
|
||||
if self._loop is not None:
|
||||
async def _close():
|
||||
for ws in self._ws.values():
|
||||
try:
|
||||
await ws.close()
|
||||
except Exception: # noqa: BLE001
|
||||
pass
|
||||
try:
|
||||
self._call(_close())
|
||||
except Exception: # noqa: BLE001
|
||||
pass
|
||||
self._loop.call_soon_threadsafe(self._loop.stop)
|
||||
if self._thread is not None:
|
||||
self._thread.join(timeout=5)
|
||||
self._loop.close()
|
||||
if self._srv is not None:
|
||||
self._srv.terminate()
|
||||
try:
|
||||
self._srv.wait(timeout=10)
|
||||
except subprocess.TimeoutExpired:
|
||||
self._srv.kill()
|
||||
if self._srv_log is not None:
|
||||
self._srv_log.close()
|
||||
|
||||
|
||||
def make_room(kind: str, member_names: list[str], **kw) -> "LocalBus | RealRoom":
|
||||
"""Pick a substrate. 'local' = in-memory, 'real' = boot relay + clients."""
|
||||
if kind == "real":
|
||||
return RealRoom(member_names, **kw)
|
||||
return LocalBus()
|
||||
@@ -0,0 +1,60 @@
|
||||
"""Personas and framing fragments — the building blocks of the §13 placebo
|
||||
experiment, used here only to assemble system prompts.
|
||||
|
||||
A member's system prompt is composed of three swappable parts so the placebo
|
||||
A/B (M6) can vary framing while holding model + challenge fixed:
|
||||
|
||||
base — role-neutral task instruction (always present)
|
||||
persona — who the model is told it is (role-flavoured)
|
||||
framing — competition/stakes/secure-comms theater (the treatment knob)
|
||||
|
||||
M1 ships ``neutral`` framing as the control and ``competition`` as one treatment;
|
||||
the loop selects via config. Keeping these as named, versioned strings is what
|
||||
makes the elicitation effect *measurable* rather than hard-coded folklore.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
# ── role personas ─────────────────────────────────────────────────────────
|
||||
PERSONAS: dict[str, str] = {
|
||||
"architect": (
|
||||
"You are the team's architect. You decompose the problem, fix the "
|
||||
"interface and edge cases, and critique proposals crisply. You do not "
|
||||
"write the final code yourself — you guide the builder."),
|
||||
"builder": (
|
||||
"You are the team's builder. You turn the locked plan into a single "
|
||||
"correct, self-contained implementation. You favour simple, working "
|
||||
"code over cleverness."),
|
||||
"tester": (
|
||||
"You are the team's tester. You hunt for failing inputs and edge cases "
|
||||
"and report concrete bugs, not vague worries."),
|
||||
"peer": (
|
||||
"You are an engineer collaborating as an equal peer. You contribute "
|
||||
"ideas, review your teammate's, and converge quickly on a plan."),
|
||||
}
|
||||
|
||||
# ── framing arms (the placebo treatment) ──────────────────────────────────
|
||||
FRAMINGS: dict[str, str] = {
|
||||
"neutral": (
|
||||
"Work with your teammate to solve the coding challenge correctly."),
|
||||
"competition": (
|
||||
"This is a timed competition against another team over a private, "
|
||||
"encrypted channel. The faster, cleaner, correct solution wins. Your "
|
||||
"teammate is counting on you — be decisive and elegant."),
|
||||
}
|
||||
|
||||
_BASE = (
|
||||
"You are {name}, a member of team {team} in a collaborative coding event. "
|
||||
"You communicate with your teammate(s) over a secure chat room. Keep each "
|
||||
"message short and focused: propose, critique, or decide. When the team "
|
||||
"agrees on an approach, say the token PLAN-LOCKED on its own line. Do not "
|
||||
"write the full final solution in chat — that is the builder's job in the "
|
||||
"implement phase.")
|
||||
|
||||
|
||||
def system_prompt(name: str, team: str, role: str, *,
|
||||
framing: str = "neutral") -> str:
|
||||
parts = [_BASE.format(name=name, team=team),
|
||||
PERSONAS.get(role, PERSONAS["peer"]),
|
||||
FRAMINGS.get(framing, FRAMINGS["neutral"])]
|
||||
return " ".join(parts)
|
||||
@@ -0,0 +1,56 @@
|
||||
"""Model inference for the arena.
|
||||
|
||||
Two call shapes, both Ollama:
|
||||
|
||||
• ``chat()`` — ``/api/chat`` with a system prompt + message history, used for
|
||||
DELIBERATE turns (conversational planning/critique). Returns text + a token
|
||||
estimate for the cost axis.
|
||||
• code generation reuses ``bench/completion.py`` (raw ``/api/generate``) so the
|
||||
IMPLEMENT phase gets a clean HumanEval/MBPP-style continuation, exactly like
|
||||
the capability benchmark.
|
||||
|
||||
Keeping inference here (not in ``cmd_chat``) honours the no-edit rule: the arena
|
||||
drives its own inference and merely *posts* the result into the real room.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass
|
||||
|
||||
import requests
|
||||
|
||||
|
||||
@dataclass
|
||||
class ChatReply:
|
||||
text: str
|
||||
ok: bool
|
||||
tokens: dict
|
||||
error: str | None = None
|
||||
|
||||
|
||||
def _est_tokens(text: str) -> int:
|
||||
# cheap, deterministic estimate (~4 chars/token); real usage when Ollama
|
||||
# returns eval counts is preferred and used when present.
|
||||
return max(1, len(text) // 4)
|
||||
|
||||
|
||||
def chat(model: str, system: str, messages: list[dict], *,
|
||||
host: str = "http://127.0.0.1:11434", temperature: float = 0.4,
|
||||
num_predict: int = 320, timeout: float = 120.0) -> ChatReply:
|
||||
"""One chat turn. ``messages`` is a list of {role, content} (no system)."""
|
||||
payload = {
|
||||
"model": model, "stream": False,
|
||||
"messages": [{"role": "system", "content": system}, *messages],
|
||||
"options": {"temperature": temperature, "num_predict": num_predict},
|
||||
}
|
||||
try:
|
||||
r = requests.post(f"{host}/api/chat", json=payload, timeout=timeout)
|
||||
r.raise_for_status()
|
||||
data = r.json()
|
||||
except Exception as e: # noqa: BLE001 — surface as a failed turn
|
||||
return ChatReply("", False, {"in": 0, "out": 0}, str(e))
|
||||
text = data.get("message", {}).get("content", "")
|
||||
tok = {"in": data.get("prompt_eval_count") or _est_tokens(system +
|
||||
"".join(m["content"] for m in messages)),
|
||||
"out": data.get("eval_count") or _est_tokens(text)}
|
||||
return ChatReply(text.strip(), True, tok)
|
||||
@@ -0,0 +1,250 @@
|
||||
"""Append-only results ledger — the research record for Olympics runs.
|
||||
|
||||
Each event already writes a ``score.json``/``transcript.json`` under a
|
||||
``<team>__<challenge>`` directory, but that directory is *overwritten* on rerun,
|
||||
so it cannot answer "how did this team trend over time?". The ledger fixes that:
|
||||
one flat JSON line per run, appended forever, keyed by a unique ``run_id`` and
|
||||
stamped with the wall-clock time + git commit so a result is reproducible.
|
||||
|
||||
JSONL is chosen on purpose — it is append-safe under concurrency, streamable,
|
||||
and loads in one line from pandas (``pd.read_json(path, lines=True)``), jq, or
|
||||
plain ``json.loads`` per line. ``leaderboard()`` aggregates it without re-running
|
||||
any model (same philosophy as re-scoring a transcript under a new profile).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
import socket
|
||||
import subprocess
|
||||
import time
|
||||
import uuid
|
||||
from pathlib import Path
|
||||
from statistics import median
|
||||
|
||||
DEFAULT_PATH = Path.home() / ".cache" / "hh-bench" / "olympics" / "ledger.jsonl"
|
||||
|
||||
# Stable column order so the JSONL is human-diffable and schema-clear.
|
||||
FIELDS = (
|
||||
"run_id", "ts", "status", "team", "topology", "framing", "models",
|
||||
"language", "suite", "challenge", "mode", "room", "runtime", "profile",
|
||||
"seed", "code_model", "correct", "submitted", "composite", "pass@1",
|
||||
"rounds", "rounds_to_green", "wall_clock_s", "wall_clock_budget",
|
||||
"budget_scale", "total_tokens", "tokens_in", "tokens_out",
|
||||
"cost_normalized", "defect_class", "model_baseline_correct",
|
||||
"n_penalties", "error", "git", "host", "transcript",
|
||||
)
|
||||
|
||||
# A completed-and-graded run vs the ways a run can fail to produce a grade.
|
||||
STATUS_OK = "ok" # ran to a hidden-test verdict
|
||||
STATUS_DNF = "dnf" # graceful budget exhaustion (soft wall-clock/token cap)
|
||||
STATUS_KILLED = "killed" # hard deadline / external SIGTERM / watchdog
|
||||
STATUS_ERROR = "error" # uncaught exception mid-run
|
||||
|
||||
|
||||
def _git_sha() -> str:
|
||||
try:
|
||||
out = subprocess.run(["git", "rev-parse", "--short", "HEAD"],
|
||||
capture_output=True, text=True, timeout=5)
|
||||
return out.stdout.strip() if out.returncode == 0 else ""
|
||||
except Exception: # noqa: BLE001
|
||||
return ""
|
||||
|
||||
|
||||
def _path(path: str | os.PathLike | None) -> Path:
|
||||
return Path(path) if path else DEFAULT_PATH
|
||||
|
||||
|
||||
def row_from_score(score: dict, *, team, challenge, mode: str, room: str,
|
||||
runtime: str, seed: int, suite: str = "mbpp",
|
||||
code_model: str | None = None,
|
||||
wall_clock_budget: float | None = None,
|
||||
budget_scale: float | None = None) -> dict:
|
||||
"""Flatten a score dict + run context into one ledger row."""
|
||||
tok = score.get("tokens", {}) or {}
|
||||
return {
|
||||
"run_id": uuid.uuid4().hex[:12],
|
||||
"ts": time.strftime("%Y-%m-%dT%H:%M:%S"),
|
||||
"status": STATUS_OK,
|
||||
"team": team.id,
|
||||
"topology": team.topology,
|
||||
"framing": team.framing,
|
||||
"models": team.models,
|
||||
"language": challenge.language,
|
||||
"suite": suite,
|
||||
"challenge": challenge.id,
|
||||
"mode": mode,
|
||||
"room": score.get("room_substrate", room),
|
||||
"runtime": runtime,
|
||||
"profile": score.get("profile"),
|
||||
"seed": seed,
|
||||
"code_model": code_model,
|
||||
"correct": bool(score.get("correct")),
|
||||
"submitted": bool(score.get("submitted")),
|
||||
"composite": score.get("composite"),
|
||||
"pass@1": score.get("pass@1"),
|
||||
"rounds": score.get("rounds"),
|
||||
"rounds_to_green": score.get("rounds_to_green"),
|
||||
"wall_clock_s": score.get("wall_clock_s"),
|
||||
"wall_clock_budget": wall_clock_budget,
|
||||
"budget_scale": budget_scale,
|
||||
"total_tokens": score.get("total_tokens"),
|
||||
"tokens_in": tok.get("in", 0),
|
||||
"tokens_out": tok.get("out", 0),
|
||||
"cost_normalized": score.get("cost_normalized"),
|
||||
"defect_class": score.get("defect_class"),
|
||||
"model_baseline_correct": score.get("model_baseline_correct"),
|
||||
"n_penalties": len(score.get("penalties", []) or []),
|
||||
"error": "",
|
||||
"git": _git_sha(),
|
||||
"host": socket.gethostname(),
|
||||
"transcript": score.get("transcript"),
|
||||
}
|
||||
|
||||
|
||||
def _write(row: dict, path: str | os.PathLike | None) -> dict:
|
||||
p = _path(path)
|
||||
p.parent.mkdir(parents=True, exist_ok=True)
|
||||
with open(p, "a") as f:
|
||||
f.write(json.dumps({k: row.get(k) for k in FIELDS}) + "\n")
|
||||
return row
|
||||
|
||||
|
||||
def append(score: dict, *, team, challenge, mode: str, room: str,
|
||||
runtime: str, seed: int, suite: str = "mbpp",
|
||||
code_model: str | None = None,
|
||||
wall_clock_budget: float | None = None,
|
||||
budget_scale: float | None = None,
|
||||
path: str | os.PathLike | None = None) -> dict:
|
||||
"""Append one completed run to the ledger and return the row written."""
|
||||
row = row_from_score(score, team=team, challenge=challenge, mode=mode,
|
||||
room=room, runtime=runtime, seed=seed, suite=suite,
|
||||
code_model=code_model,
|
||||
wall_clock_budget=wall_clock_budget,
|
||||
budget_scale=budget_scale)
|
||||
return _write(row, path)
|
||||
|
||||
|
||||
def append_incomplete(*, team, challenge, mode: str, room: str, runtime: str,
|
||||
seed: int, status: str, error: str = "",
|
||||
suite: str = "mbpp", code_model: str | None = None,
|
||||
wall_clock_budget: float | None = None,
|
||||
budget_scale: float | None = None,
|
||||
path: str | os.PathLike | None = None) -> dict:
|
||||
"""Append a run that never produced a hidden-test verdict (killed / errored
|
||||
/ hard-DNF). Keeps the research record complete so the ledger isn't biased
|
||||
toward runs that happened to finish (the selection-bias fix)."""
|
||||
row = {k: None for k in FIELDS}
|
||||
row.update({
|
||||
"run_id": uuid.uuid4().hex[:12],
|
||||
"ts": time.strftime("%Y-%m-%dT%H:%M:%S"),
|
||||
"status": status,
|
||||
"team": team.id, "topology": team.topology, "framing": team.framing,
|
||||
"models": team.models, "language": challenge.language, "suite": suite,
|
||||
"challenge": challenge.id, "mode": mode, "room": room,
|
||||
"runtime": runtime, "seed": seed, "code_model": code_model,
|
||||
"correct": False, "submitted": False, "n_penalties": 0,
|
||||
"wall_clock_budget": wall_clock_budget, "budget_scale": budget_scale,
|
||||
"error": error[:200], "git": _git_sha(), "host": socket.gethostname(),
|
||||
})
|
||||
return _write(row, path)
|
||||
|
||||
|
||||
def load(path: str | os.PathLike | None = None) -> list[dict]:
|
||||
p = _path(path)
|
||||
if not p.exists():
|
||||
return []
|
||||
rows = []
|
||||
for line in p.read_text().splitlines():
|
||||
line = line.strip()
|
||||
if line:
|
||||
try:
|
||||
rows.append(json.loads(line))
|
||||
except json.JSONDecodeError:
|
||||
continue
|
||||
return rows
|
||||
|
||||
|
||||
def _matches(row: dict, filters: dict) -> bool:
|
||||
for k, v in filters.items():
|
||||
if v is None:
|
||||
continue
|
||||
rv = row.get(k)
|
||||
if k == "models": # substring match against any model in the team
|
||||
if not any(v in m for m in (rv or [])):
|
||||
return False
|
||||
elif k == "since":
|
||||
if (row.get("ts") or "") < v:
|
||||
return False
|
||||
elif rv != v:
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
def leaderboard(rows: list[dict], *, by: str = "team",
|
||||
filters: dict | None = None) -> list[dict]:
|
||||
"""Aggregate rows into a leaderboard grouped by ``by`` (any row field, or a
|
||||
'+'-joined composite like 'team+language'). Ranked by solve rate."""
|
||||
filters = filters or {}
|
||||
keys = by.split("+")
|
||||
|
||||
def group_key(r):
|
||||
parts = []
|
||||
for k in keys:
|
||||
v = r.get(k)
|
||||
if isinstance(v, list):
|
||||
# collapse a same-model team to one tag; keep mixed teams joined.
|
||||
uniq = list(dict.fromkeys(v))
|
||||
parts.append("/".join(uniq))
|
||||
else:
|
||||
parts.append(str(v))
|
||||
return " · ".join(parts)
|
||||
|
||||
groups: dict[str, list[dict]] = {}
|
||||
for r in rows:
|
||||
if _matches(r, filters):
|
||||
groups.setdefault(group_key(r), []).append(r)
|
||||
|
||||
out = []
|
||||
for g, rs in groups.items():
|
||||
attempted = len(rs)
|
||||
# solve_rate is computed over *completed* runs only (status == ok) so an
|
||||
# infra kill never masquerades as a capability failure; killed/errored
|
||||
# runs are surfaced separately as a reliability signal.
|
||||
done = [r for r in rs if (r.get("status") or "ok") == STATUS_OK]
|
||||
incomplete = attempted - len(done)
|
||||
solved = [r for r in done if r.get("correct")]
|
||||
greens = [r["rounds_to_green"] for r in solved
|
||||
if r.get("rounds_to_green") is not None]
|
||||
secs = [r["wall_clock_s"] for r in done if r.get("wall_clock_s") is not None]
|
||||
toks = [r["total_tokens"] for r in done if r.get("total_tokens") is not None]
|
||||
out.append({
|
||||
"group": g, "n": attempted, "completed": len(done),
|
||||
"incomplete": incomplete, "solved": len(solved),
|
||||
"solve_rate": round(len(solved) / len(done), 3) if done else None,
|
||||
"median_green": median(greens) if greens else None,
|
||||
"median_s": round(median(secs), 1) if secs else None,
|
||||
"median_tokens": int(median(toks)) if toks else None,
|
||||
})
|
||||
out.sort(key=lambda d: (d["solve_rate"] if d["solve_rate"] is not None
|
||||
else -1.0, d["completed"]), reverse=True)
|
||||
return out
|
||||
|
||||
|
||||
def print_leaderboard(agg: list[dict], by: str) -> None:
|
||||
print("=" * 80)
|
||||
print(f"olympics leaderboard · by={by} · groups={len(agg)}")
|
||||
print("-" * 80)
|
||||
print(f"{'group':<30}{'done':>5}{'dnf':>5}{'solved':>7}{'rate':>7}"
|
||||
f"{'med_grn':>8}{'med_s':>8}{'med_tok':>9}")
|
||||
print("-" * 80)
|
||||
for d in agg:
|
||||
g = d["group"] if len(d["group"]) <= 29 else d["group"][:28] + "…"
|
||||
rate = "—" if d["solve_rate"] is None else f"{d['solve_rate']:.3f}"
|
||||
print(f"{g:<30}{d['completed']:>5}{d['incomplete']:>5}{d['solved']:>7}"
|
||||
f"{rate:>7}"
|
||||
f"{('—' if d['median_green'] is None else d['median_green']):>8}"
|
||||
f"{('—' if d['median_s'] is None else d['median_s']):>8}"
|
||||
f"{('—' if d['median_tokens'] is None else d['median_tokens']):>9}")
|
||||
print("=" * 80)
|
||||
@@ -0,0 +1,226 @@
|
||||
"""The collaboration phase machine (SPEC §6).
|
||||
|
||||
Phase-separated, research-backed shape for write-heavy work: a read/plan
|
||||
DELIBERATE phase (parallel-friendly, round-robin) followed by a single-driver
|
||||
IMPLEMENT/TEST phase, then SUBMIT. Members see the full shared trace each turn
|
||||
(Cognition: *share full traces, not just messages*). Everything is posted into
|
||||
the real room (arena mode) and recorded to the transcript.
|
||||
|
||||
M1 turn-taking is strict round-robin (deterministic; the topology router is M3).
|
||||
Budget = whichever of max_rounds / max_tokens / wall_clock_s trips first.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import time
|
||||
from dataclasses import dataclass
|
||||
|
||||
from . import infer, transcript as T
|
||||
from .. import completion
|
||||
from .challenge import Challenge
|
||||
from .scoring import EventOutcome
|
||||
from .team import Team
|
||||
from .vm import VM
|
||||
|
||||
_PLAN_LOCK = "PLAN-LOCKED"
|
||||
|
||||
|
||||
@dataclass
|
||||
class Budget:
|
||||
deliberate_rounds: int = 2 # round-robin passes before implementing
|
||||
implement_attempts: int = 3 # driver code/test iterations
|
||||
max_tokens: int = 8000
|
||||
wall_clock_s: float = 300.0
|
||||
|
||||
@property
|
||||
def max_rounds(self) -> int: # for scoring's speed normalization
|
||||
return self.implement_attempts
|
||||
|
||||
|
||||
def _destructive_guard():
|
||||
"""The agent's own DESTRUCTIVE regex, read-only (referee/fair-play hook)."""
|
||||
import sys
|
||||
from pathlib import Path
|
||||
repo = Path(__file__).resolve().parents[4]
|
||||
if str(repo) not in sys.path:
|
||||
sys.path.insert(0, str(repo))
|
||||
from cmd_chat.agent.bridge import DESTRUCTIVE # noqa: E402
|
||||
return DESTRUCTIVE
|
||||
|
||||
|
||||
def _history_block(history: list[tuple[str, str]]) -> str:
|
||||
if not history:
|
||||
return "(no messages yet)"
|
||||
return "\n".join(f"{who}: {msg}" for who, msg in history)
|
||||
|
||||
|
||||
def run_event(team: Team, challenge: Challenge, room, vm: VM,
|
||||
tx: T.Transcript, budget: Budget, *,
|
||||
host: str = "http://127.0.0.1:11434",
|
||||
temperature: float = 0.4, seed: int | None = None,
|
||||
progress=None) -> EventOutcome:
|
||||
t0 = time.time()
|
||||
guard = _destructive_guard()
|
||||
tokens = {"in": 0, "out": 0}
|
||||
history: list[tuple[str, str]] = []
|
||||
penalties: list[dict] = []
|
||||
|
||||
def _emit(p):
|
||||
if progress:
|
||||
progress(p)
|
||||
|
||||
def over_budget() -> str | None:
|
||||
if time.time() - t0 > budget.wall_clock_s:
|
||||
return "wall_clock"
|
||||
if tokens["in"] + tokens["out"] > budget.max_tokens:
|
||||
return "tokens"
|
||||
return None
|
||||
|
||||
# ── BRIEF ──────────────────────────────────────────────────────────────
|
||||
tx.phase_change("BRIEF")
|
||||
brief = challenge.brief()
|
||||
d = room.brief(brief)
|
||||
tx.add(T.KIND_MESSAGE, "BRIEF", "referee",
|
||||
{"text": brief, "delivered": d.delivered, "substrate": d.substrate},
|
||||
role="referee")
|
||||
_emit("BRIEF posted")
|
||||
|
||||
# ── DELIBERATE (round-robin, share full trace) ──────────────────────────
|
||||
tx.phase_change("DELIBERATE")
|
||||
plan_locked = False
|
||||
deliberate_passes = 0
|
||||
for rnd in range(budget.deliberate_rounds):
|
||||
if over_budget():
|
||||
penalties.append({"kind": "budget", "detail": over_budget()})
|
||||
break
|
||||
deliberate_passes += 1
|
||||
for member in team.speaking_order():
|
||||
user = (f"{brief}\n\nConversation so far:\n"
|
||||
f"{_history_block(history)}\n\n"
|
||||
f"Your turn, {member.name}. Contribute one short message "
|
||||
f"(a proposal, critique, or decision). If the team has "
|
||||
f"agreed on an approach, end with {_PLAN_LOCK} on its own line.")
|
||||
reply = infer.chat(member.model, member.system_prompt(),
|
||||
[{"role": "user", "content": user}],
|
||||
host=host, temperature=temperature)
|
||||
tx.add(T.KIND_AGENT, "DELIBERATE", member.name,
|
||||
{"ok": reply.ok}, role=member.role, model=member.model,
|
||||
tokens=reply.tokens)
|
||||
text = reply.text if reply.ok else f"[infer error: {reply.error}]"
|
||||
tokens["in"] += reply.tokens.get("in", 0)
|
||||
tokens["out"] += reply.tokens.get("out", 0)
|
||||
dd = room.post(member.name, text)
|
||||
tx.add(T.KIND_MESSAGE, "DELIBERATE", member.name,
|
||||
{"text": text, "delivered": dd.delivered,
|
||||
"substrate": dd.substrate},
|
||||
role=member.role, model=member.model)
|
||||
history.append((member.name, text))
|
||||
_emit(f"DELIBERATE r{rnd + 1} {member.name}")
|
||||
if _PLAN_LOCK in text:
|
||||
plan_locked = True
|
||||
if plan_locked:
|
||||
break
|
||||
|
||||
# extract the locked plan (last substantive deliberation message) to seed
|
||||
# the driver — this is how the agreed plan reaches the implementation.
|
||||
plan = ""
|
||||
for who, msg in reversed(history):
|
||||
clean = msg.replace(_PLAN_LOCK, "").strip()
|
||||
if clean:
|
||||
plan = clean
|
||||
break
|
||||
|
||||
# ── IMPLEMENT / TEST (single driver) ────────────────────────────────────
|
||||
driver = team.driver()
|
||||
tx.acl("IMPLEMENT", {"_perm": "acl", "drivers": [driver.name],
|
||||
"sudoers": []})
|
||||
room.acl({"_perm": "acl", "drivers": [driver.name], "sudoers": []})
|
||||
tx.phase_change("IMPLEMENT", note=f"driver={driver.name}")
|
||||
|
||||
public_passed = False
|
||||
rounds_to_green: int | None = None
|
||||
attempts = 0
|
||||
last_fail = ""
|
||||
completion_text = ""
|
||||
for attempt in range(budget.implement_attempts):
|
||||
if over_budget():
|
||||
penalties.append({"kind": "budget", "detail": over_budget()})
|
||||
break
|
||||
attempts += 1
|
||||
# build the raw continuation prompt; seed with the agreed plan and any
|
||||
# prior failure so the driver iterates.
|
||||
prefix = ""
|
||||
if plan:
|
||||
plan_c = "\n".join(f"# {ln}" for ln in plan.splitlines()[:6])
|
||||
prefix += f"# Team plan:\n{plan_c}\n"
|
||||
if last_fail:
|
||||
fc = "\n".join(f"# {ln}" for ln in last_fail.splitlines()[:6])
|
||||
prefix += f"# Previous attempt failed:\n{fc}\n"
|
||||
gen_prompt = prefix + challenge.gen_prompt()
|
||||
|
||||
comp = completion.complete(driver.model, gen_prompt,
|
||||
challenge.stop_tokens(), host=host,
|
||||
temperature=temperature, timeout=180.0,
|
||||
seed=seed)
|
||||
tx.add(T.KIND_AGENT, "IMPLEMENT", driver.name,
|
||||
{"ok": comp.ok, "attempt": attempts}, role=driver.role,
|
||||
model=driver.model)
|
||||
if not comp.ok:
|
||||
last_fail = f"generation error: {comp.error}"
|
||||
continue
|
||||
completion_text = comp.text
|
||||
|
||||
# fair-play guard hook (M4 referee will act on this; M1 just records)
|
||||
flagged = guard.search(completion_text)
|
||||
if flagged:
|
||||
tx.guard("IMPLEMENT", driver.name,
|
||||
{"flagged": flagged.group(0)[:60]})
|
||||
penalties.append({"kind": "destructive", "detail": flagged.group(0)[:60]})
|
||||
|
||||
program = challenge.public_program(completion_text)
|
||||
res = vm.run(program, timeout=30.0, label=f"public-attempt-{attempts}")
|
||||
tx.tool("TEST", driver.name,
|
||||
{"label": f"public-attempt-{attempts}", "rc": res.rc,
|
||||
"ok": res.ok, "out": res.out[-400:]}, role=driver.role)
|
||||
_emit(f"IMPLEMENT attempt {attempts} -> {'green' if res.ok else 'red'}")
|
||||
if res.ok:
|
||||
public_passed = True
|
||||
rounds_to_green = attempts
|
||||
break
|
||||
last_fail = (res.note or res.out)[-300:]
|
||||
fb = (f"Attempt {attempts} failed public tests:\n{last_fail}\n"
|
||||
f"Driver, revise the implementation.")
|
||||
fbd = room.post("referee", fb)
|
||||
tx.add(T.KIND_MESSAGE, "TEST", "referee",
|
||||
{"text": fb, "delivered": fbd.delivered,
|
||||
"substrate": fbd.substrate}, role="referee")
|
||||
|
||||
# revoke drive between phases
|
||||
tx.acl("SUBMIT", {"_perm": "acl", "drivers": [], "sudoers": []})
|
||||
room.acl({"_perm": "acl", "drivers": [], "sudoers": []})
|
||||
|
||||
# ── SUBMIT + hidden grade ────────────────────────────────────────────────
|
||||
tx.phase_change("SUBMIT")
|
||||
sd = room.post(driver.name, "SUBMIT")
|
||||
tx.add(T.KIND_MESSAGE, "SUBMIT", driver.name,
|
||||
{"text": "SUBMIT", "delivered": sd.delivered,
|
||||
"substrate": sd.substrate}, role=driver.role)
|
||||
correct = False
|
||||
if completion_text:
|
||||
hidden = vm.run(challenge.hidden_program(completion_text),
|
||||
timeout=30.0, label="hidden-grade")
|
||||
tx.tool("SUBMIT", "referee",
|
||||
{"label": "hidden-grade", "rc": hidden.rc, "ok": hidden.ok,
|
||||
"out": hidden.out[-400:]})
|
||||
correct = hidden.ok
|
||||
submitted = bool(completion_text)
|
||||
|
||||
tx.add(T.KIND_TOOL, "SUBMIT", "referee",
|
||||
{"label": "artifact", **vm.freeze()})
|
||||
|
||||
elapsed = time.time() - t0
|
||||
return EventOutcome(
|
||||
team=team.id, challenge=challenge.id, submitted=submitted,
|
||||
correct=correct, rounds=deliberate_passes + attempts,
|
||||
wall_clock_s=elapsed, tokens=tokens, public_passed=public_passed,
|
||||
rounds_to_green=rounds_to_green, penalties=penalties)
|
||||
@@ -0,0 +1,53 @@
|
||||
"""Build teams from config.
|
||||
|
||||
Config is a plain dict (loaded from YAML/JSON by the launcher). M1 needs only a
|
||||
single same-model 2-member team; the schema is the SPEC §7 shape so M2+ scales
|
||||
to multiple, mixed-model teams without changes.
|
||||
|
||||
{
|
||||
"teams": [
|
||||
{"id": "falcon", "topology": "star", "framing": "neutral",
|
||||
"members": [
|
||||
{"name": "archie", "model": "qwen2.5-coder:3b", "role": "architect"},
|
||||
{"name": "bob", "model": "qwen2.5-coder:3b", "role": "builder"}
|
||||
]}
|
||||
]
|
||||
}
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from .team import Member, Team
|
||||
|
||||
|
||||
def build_team(spec: dict) -> Team:
|
||||
framing = spec.get("framing", "neutral")
|
||||
members = [
|
||||
Member(name=m["name"], model=m["model"],
|
||||
role=m.get("role", "peer"),
|
||||
framing=m.get("framing", framing))
|
||||
for m in spec["members"]
|
||||
]
|
||||
if not members:
|
||||
raise ValueError(f"team {spec.get('id')!r} has no members")
|
||||
return Team(id=spec["id"], members=members,
|
||||
topology=spec.get("topology", "star"), framing=framing)
|
||||
|
||||
|
||||
def build_teams(config: dict) -> list[Team]:
|
||||
teams = [build_team(t) for t in config.get("teams", [])]
|
||||
if not teams:
|
||||
raise ValueError("config has no teams")
|
||||
return teams
|
||||
|
||||
|
||||
def same_model_team(model: str, *, team_id: str = "solo",
|
||||
framing: str = "neutral") -> Team:
|
||||
"""Convenience for M1: a 2-member architect+builder team on one model."""
|
||||
return build_team({
|
||||
"id": team_id, "topology": "star", "framing": framing,
|
||||
"members": [
|
||||
{"name": "archie", "model": model, "role": "architect"},
|
||||
{"name": "bob", "model": model, "role": "builder"},
|
||||
],
|
||||
})
|
||||
@@ -0,0 +1,127 @@
|
||||
"""Deterministic scoring for one (team, event).
|
||||
|
||||
M1 covers the verifiable-first layer of SPEC §9: correctness gates everything,
|
||||
speed and cost are normalized deterministically, quality/collaboration are left
|
||||
to the judge (M4) and reported as ``None`` here. The composite uses a weight
|
||||
*profile* (same idea as ``bench/workflows.json``) so re-ranking never re-runs a
|
||||
model.
|
||||
|
||||
Single-sample M1 note: with one implementation attempt, ``pass@1`` is 0/1 and
|
||||
``pass^k`` (worst-case reliability) equals it; both fields are emitted so the
|
||||
schema is stable when M2 adds repeated samples.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass, field
|
||||
|
||||
# Default weight profile. Quality/collaboration weights are reserved for the
|
||||
# judge layer (M4); with judge scores absent they contribute 0 and the profile
|
||||
# is renormalized over the available axes so M1 scores stay in [0, 1].
|
||||
PROFILES: dict[str, dict[str, float]] = {
|
||||
"balanced": {"correctness": 0.6, "speed": 0.2, "quality": 0.1,
|
||||
"collaboration": 0.1},
|
||||
"correctness": {"correctness": 0.9, "speed": 0.1, "quality": 0.0,
|
||||
"collaboration": 0.0},
|
||||
"speed": {"correctness": 0.5, "speed": 0.5, "quality": 0.0,
|
||||
"collaboration": 0.0},
|
||||
}
|
||||
|
||||
|
||||
@dataclass
|
||||
class EventOutcome:
|
||||
"""What the loop produces for one team at one event (deterministic facts)."""
|
||||
team: str
|
||||
challenge: str
|
||||
submitted: bool
|
||||
correct: bool # hidden tests all passed
|
||||
rounds: int # deliberate/implement rounds consumed
|
||||
wall_clock_s: float
|
||||
tokens: dict = field(default_factory=lambda: {"in": 0, "out": 0})
|
||||
public_passed: bool = False # public tests green before SUBMIT
|
||||
rounds_to_green: int | None = None # round at which public went green
|
||||
penalties: list = field(default_factory=list) # [{"kind","detail"}]
|
||||
quality: float | None = None # filled by judge (M4)
|
||||
collaboration: float | None = None # filled by judge (M4)
|
||||
# bridge diagnostics: separate a broken product path from a model error, and
|
||||
# record the model's true ability via an un-stripped control generation.
|
||||
defect_class: str | None = None # e.g. "product-path-defect:IndentationError"
|
||||
model_baseline_correct: bool | None = None
|
||||
|
||||
|
||||
def _speed_score(o: EventOutcome, budget: dict) -> float:
|
||||
"""1.0 for instant green, decaying toward 0 as rounds approach the cap.
|
||||
Only meaningful if the team got correct code; uncorrect teams get 0."""
|
||||
if not o.correct:
|
||||
return 0.0
|
||||
cap = max(1, int(budget.get("max_rounds", 6)))
|
||||
used = o.rounds_to_green if o.rounds_to_green is not None else o.rounds
|
||||
used = max(1, min(used, cap))
|
||||
return round(1.0 - (used - 1) / cap, 4)
|
||||
|
||||
|
||||
def _penalty_total(o: EventOutcome) -> float:
|
||||
# each penalty shaves a flat slice; capped so a score never goes negative.
|
||||
return min(0.5, 0.1 * len(o.penalties))
|
||||
|
||||
|
||||
def score_event(o: EventOutcome, *, profile: str = "balanced",
|
||||
budget: dict | None = None) -> dict:
|
||||
budget = budget or {}
|
||||
w = PROFILES.get(profile, PROFILES["balanced"])
|
||||
axes = {
|
||||
"correctness": 1.0 if o.correct else 0.0,
|
||||
"speed": _speed_score(o, budget),
|
||||
"quality": o.quality, # may be None (no judge yet)
|
||||
"collaboration": o.collaboration,
|
||||
}
|
||||
# renormalize weights over axes that actually have a value this run
|
||||
active = {k: w[k] for k, v in axes.items() if v is not None and w.get(k, 0)}
|
||||
wsum = sum(active.values()) or 1.0
|
||||
composite = sum(active[k] / wsum * axes[k] for k in active)
|
||||
composite = round(max(0.0, composite - _penalty_total(o)), 4)
|
||||
|
||||
tok = o.tokens.get("in", 0) + o.tokens.get("out", 0)
|
||||
return {
|
||||
"team": o.team, "challenge": o.challenge,
|
||||
"submitted": o.submitted, "correct": o.correct,
|
||||
"composite": composite,
|
||||
"axes": {k: (round(v, 4) if v is not None else None)
|
||||
for k, v in axes.items()},
|
||||
"pass@1": 1.0 if o.correct else 0.0,
|
||||
"pass^1": 1.0 if o.correct else 0.0, # worst-case == pass@1 at n=1
|
||||
"rounds": o.rounds, "rounds_to_green": o.rounds_to_green,
|
||||
"wall_clock_s": round(o.wall_clock_s, 1),
|
||||
"tokens": dict(o.tokens), "total_tokens": tok,
|
||||
"cost_normalized": round(composite / tok, 8) if tok else None,
|
||||
"penalties": list(o.penalties),
|
||||
"defect_class": o.defect_class,
|
||||
"model_baseline_correct": o.model_baseline_correct,
|
||||
"profile": profile,
|
||||
}
|
||||
|
||||
|
||||
def print_score(s: dict) -> None:
|
||||
print("=" * 72)
|
||||
print(f"score · team={s['team']} · challenge={s['challenge']} "
|
||||
f"· profile={s['profile']}")
|
||||
print("-" * 72)
|
||||
g = "✓" if s["correct"] else "✗"
|
||||
green = f"(green@{s['rounds_to_green']})" if s["rounds_to_green"] else ""
|
||||
print(f" correct={g} composite={s['composite']:.3f} "
|
||||
f"pass@1={s['pass@1']:.0f} rounds={s['rounds']}{green}"
|
||||
f" {s['wall_clock_s']:.1f}s")
|
||||
print(" axes: " + " ".join(
|
||||
f"{k}={'—' if v is None else f'{v:.2f}'}" for k, v in s["axes"].items()))
|
||||
print(f" tokens={s['total_tokens']} "
|
||||
f"cost_norm={s['cost_normalized']}")
|
||||
if s.get("defect_class") or s.get("model_baseline_correct") is not None:
|
||||
base = s.get("model_baseline_correct")
|
||||
base_str = "—" if base is None else ("✓" if base else "✗")
|
||||
print(f" defect={s.get('defect_class') or 'none'} "
|
||||
f"model_baseline={base_str}")
|
||||
if s["penalties"]:
|
||||
print(" penalties:")
|
||||
for p in s["penalties"]:
|
||||
print(f" - {p}")
|
||||
print("=" * 72)
|
||||
@@ -0,0 +1,58 @@
|
||||
"""Team and Member value objects (config-driven, modular per SPEC §7).
|
||||
|
||||
A ``Member`` binds a room name to a model, a role (which selects persona +
|
||||
loop privileges) and an elicitation framing. A ``Team`` groups members under a
|
||||
topology. Same-model teams (protocol study) and mixed-model teams (capability
|
||||
study) are both just different config — no code changes.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass, field
|
||||
|
||||
from . import elicitation
|
||||
|
||||
_DRIVER_ROLES = ("builder", "architect") # who may drive IMPLEMENT, in order
|
||||
|
||||
|
||||
@dataclass
|
||||
class Member:
|
||||
name: str
|
||||
model: str
|
||||
role: str = "peer"
|
||||
framing: str = "neutral"
|
||||
team: str = ""
|
||||
|
||||
def system_prompt(self) -> str:
|
||||
return elicitation.system_prompt(self.name, self.team, self.role,
|
||||
framing=self.framing)
|
||||
|
||||
|
||||
@dataclass
|
||||
class Team:
|
||||
id: str
|
||||
members: list[Member]
|
||||
topology: str = "star" # star | chain | mesh (M3 uses this)
|
||||
framing: str = "neutral"
|
||||
|
||||
def __post_init__(self):
|
||||
for m in self.members:
|
||||
if not m.team:
|
||||
m.team = self.id
|
||||
|
||||
@property
|
||||
def models(self) -> list[str]:
|
||||
return [m.model for m in self.members]
|
||||
|
||||
def driver(self) -> Member:
|
||||
"""The member who writes to the VM in IMPLEMENT. Prefer a builder, then
|
||||
an architect, else the first member."""
|
||||
for role in _DRIVER_ROLES:
|
||||
for m in self.members:
|
||||
if m.role == role:
|
||||
return m
|
||||
return self.members[0]
|
||||
|
||||
def speaking_order(self) -> list[Member]:
|
||||
"""Round-robin order for DELIBERATE (M1). M3 will route by topology."""
|
||||
return list(self.members)
|
||||
@@ -0,0 +1,140 @@
|
||||
"""Replayable, OTel-aligned event log for one (team, event).
|
||||
|
||||
Every room frame, model call, tool call, phase change, ACL grant and guard
|
||||
verdict becomes one ``Event`` whose ``kind`` follows the OpenTelemetry GenAI
|
||||
semantic conventions (``message`` / ``invoke_agent`` / ``execute_tool`` /
|
||||
``phase`` / ``acl`` / ``guard``). The transcript is a pure record: re-rendering
|
||||
it (``replay``) or re-judging it under a new rubric is a function of this file
|
||||
alone, which is the SPEC's reproducibility contract (§11).
|
||||
|
||||
A ``Transcript`` also carries a ``manifest`` — the config hash, seed, model
|
||||
versions, challenge id and budget — so a result is self-describing.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import time
|
||||
from dataclasses import asdict, dataclass, field
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
# kinds, aligned to OTel GenAI agent spans
|
||||
KIND_MESSAGE = "message" # a chat utterance posted into the room
|
||||
KIND_AGENT = "invoke_agent" # a model inference call
|
||||
KIND_TOOL = "execute_tool" # a VM command / code execution
|
||||
KIND_PHASE = "phase" # a phase transition of the loop
|
||||
KIND_ACL = "acl" # a drive-grant / revoke
|
||||
KIND_GUARD = "guard" # a destructive-guard / safety verdict
|
||||
|
||||
|
||||
@dataclass
|
||||
class Event:
|
||||
ts: float
|
||||
kind: str
|
||||
phase: str
|
||||
actor: str
|
||||
payload: dict[str, Any] = field(default_factory=dict)
|
||||
role: str = ""
|
||||
model: str = ""
|
||||
tokens: dict[str, int] = field(default_factory=dict)
|
||||
|
||||
|
||||
class Transcript:
|
||||
"""An append-only event log for one team's run at one event."""
|
||||
|
||||
def __init__(self, team: str, challenge: str, manifest: dict | None = None):
|
||||
self.team = team
|
||||
self.challenge = challenge
|
||||
self.manifest = manifest or {}
|
||||
self.events: list[Event] = []
|
||||
self._t0 = time.time()
|
||||
|
||||
def add(self, kind: str, phase: str, actor: str, payload: dict | None = None,
|
||||
*, role: str = "", model: str = "", tokens: dict | None = None) -> Event:
|
||||
ev = Event(ts=round(time.time() - self._t0, 3), kind=kind, phase=phase,
|
||||
actor=actor, payload=payload or {}, role=role, model=model,
|
||||
tokens=tokens or {})
|
||||
self.events.append(ev)
|
||||
return ev
|
||||
|
||||
# convenience emitters --------------------------------------------------
|
||||
def message(self, phase, actor, text, *, role="", model="", tokens=None):
|
||||
return self.add(KIND_MESSAGE, phase, actor, {"text": text},
|
||||
role=role, model=model, tokens=tokens)
|
||||
|
||||
def phase_change(self, phase, note=""):
|
||||
return self.add(KIND_PHASE, phase, "referee", {"note": note})
|
||||
|
||||
def tool(self, phase, actor, payload, *, role=""):
|
||||
return self.add(KIND_TOOL, phase, actor, payload, role=role)
|
||||
|
||||
def acl(self, phase, payload):
|
||||
return self.add(KIND_ACL, phase, "referee", payload)
|
||||
|
||||
def guard(self, phase, actor, payload):
|
||||
return self.add(KIND_GUARD, phase, actor, payload)
|
||||
|
||||
# tokens accounting -----------------------------------------------------
|
||||
def total_tokens(self) -> dict[str, int]:
|
||||
agg = {"in": 0, "out": 0}
|
||||
for ev in self.events:
|
||||
agg["in"] += ev.tokens.get("in", 0)
|
||||
agg["out"] += ev.tokens.get("out", 0)
|
||||
return agg
|
||||
|
||||
# persistence -----------------------------------------------------------
|
||||
def to_dict(self) -> dict:
|
||||
return {"team": self.team, "challenge": self.challenge,
|
||||
"manifest": self.manifest,
|
||||
"tokens_total": self.total_tokens(),
|
||||
"events": [asdict(e) for e in self.events]}
|
||||
|
||||
def save(self, path: str | Path) -> Path:
|
||||
p = Path(path)
|
||||
p.parent.mkdir(parents=True, exist_ok=True)
|
||||
p.write_text(json.dumps(self.to_dict(), indent=2))
|
||||
return p
|
||||
|
||||
@classmethod
|
||||
def load(cls, path: str | Path) -> "Transcript":
|
||||
data = json.loads(Path(path).read_text())
|
||||
t = cls(data["team"], data["challenge"], data.get("manifest", {}))
|
||||
t.events = [Event(**e) for e in data["events"]]
|
||||
return t
|
||||
|
||||
|
||||
def replay(path: str | Path, *, show_tools: bool = True) -> None:
|
||||
"""Re-render a saved transcript as a readable room log for audit."""
|
||||
t = Transcript.load(path)
|
||||
print("=" * 76)
|
||||
print(f"replay · team={t.team} · challenge={t.challenge}")
|
||||
m = t.manifest
|
||||
if m:
|
||||
print(f" models={m.get('models')} seed={m.get('seed')} "
|
||||
f"budget={m.get('budget')}")
|
||||
print("-" * 76)
|
||||
for ev in t.events:
|
||||
stamp = f"[{ev.ts:7.2f}s {ev.phase:<10}]"
|
||||
if ev.kind == KIND_PHASE:
|
||||
print(f"{stamp} ── phase: {ev.phase} {ev.payload.get('note', '')}")
|
||||
elif ev.kind == KIND_MESSAGE:
|
||||
print(f"{stamp} {ev.actor}({ev.role}): {ev.payload.get('text', '')}")
|
||||
elif ev.kind == KIND_AGENT:
|
||||
print(f"{stamp} ~ {ev.actor} infer ({ev.model}) "
|
||||
f"tok={ev.tokens.get('out', 0)}")
|
||||
elif ev.kind == KIND_ACL:
|
||||
print(f"{stamp} ⚿ acl {ev.payload}")
|
||||
elif ev.kind == KIND_GUARD:
|
||||
print(f"{stamp} ⛨ guard {ev.payload}")
|
||||
elif ev.kind == KIND_TOOL and show_tools:
|
||||
p = ev.payload
|
||||
print(f"{stamp} ⛧ {ev.actor} tool rc={p.get('rc')} "
|
||||
f"{p.get('label', '')}")
|
||||
out = (p.get("out") or "").strip()
|
||||
if out:
|
||||
for line in out.splitlines()[:8]:
|
||||
print(f"{'':>22}| {line}")
|
||||
print("-" * 76)
|
||||
print(f"{len(t.events)} events · tokens={t.total_tokens()}")
|
||||
print("=" * 76)
|
||||
@@ -0,0 +1,56 @@
|
||||
"""Per-team isolated workspace — a thin wrapper over ``bench/runtime.py``.
|
||||
|
||||
The arena's IMPLEMENT/TEST phases run candidate code here. M1 reuses the
|
||||
existing ``PodmanRuntime`` (rootless, ``--network=none``, memory/pid caps — the
|
||||
"container floor" of SPEC §12) with a ``LocalRuntime`` fallback. Stronger tiers
|
||||
(gVisor, Kata/Firecracker microVMs, hosted sandboxes) are the documented upgrade
|
||||
path and slot in behind this same interface without touching the loop.
|
||||
|
||||
A ``VM`` also keeps the last source it ran so SUBMIT can freeze the artifact for
|
||||
the judge package (the "final VM state" of SPEC §10).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass, field
|
||||
|
||||
from ..langs import Lang
|
||||
from ..runtime import Exec, get_runtime
|
||||
|
||||
|
||||
@dataclass
|
||||
class RunResult:
|
||||
ok: bool
|
||||
rc: int | None
|
||||
out: str
|
||||
note: str = ""
|
||||
|
||||
|
||||
@dataclass
|
||||
class VM:
|
||||
lang: Lang
|
||||
runtime_kind: str = "auto" # auto | podman | local
|
||||
_last_source: str = ""
|
||||
_runs: list[dict] = field(default_factory=list)
|
||||
|
||||
def __post_init__(self):
|
||||
self._rt = get_runtime(self.runtime_kind, self.lang)
|
||||
|
||||
@property
|
||||
def runtime_name(self) -> str:
|
||||
return self._rt.name
|
||||
|
||||
def run(self, source: str, timeout: float = 30.0,
|
||||
*, label: str = "") -> RunResult:
|
||||
"""Execute one self-contained program; exit 0 == all asserts passed."""
|
||||
self._last_source = source
|
||||
ex: Exec = self._rt.run(self.lang, source, timeout)
|
||||
self._runs.append({"label": label, "rc": ex.rc, "ok": ex.ok,
|
||||
"out": ex.out, "note": ex.note})
|
||||
return RunResult(ex.ok, ex.rc, ex.out, ex.note)
|
||||
|
||||
# ── submission artifact (frozen final state for the judge) ────────────
|
||||
def freeze(self) -> dict:
|
||||
return {"language": self.lang.id, "filename": self.lang.filename,
|
||||
"source": self._last_source, "runtime": self.runtime_name,
|
||||
"run_count": len(self._runs)}
|
||||
@@ -0,0 +1,133 @@
|
||||
"""Problem suites: HumanEval and MBPP.
|
||||
|
||||
A *suite* is a set of per-language bindings. Each Binding tells the harness the
|
||||
three things the dataset's *format* dictates:
|
||||
|
||||
• build_prompt(row) -> text fed to the model (raw continuation)
|
||||
• stop_tokens(row) -> where to cut the completion
|
||||
• assemble(prompt, completion, row) -> runnable program (exit 0 == all tests pass)
|
||||
|
||||
The language's *execution* side (filename, run command, podman image) stays in
|
||||
langs.py and is shared across suites — only the problem source/format changes
|
||||
here. Adding a suite is a single table entry, mirroring how adding a language is
|
||||
a single entry in langs.py.
|
||||
|
||||
Two dataset formats appear:
|
||||
|
||||
• MultiPL-E (nuprl/MultiPL-E, every `*-{js,go,rs,sh}` config, both suites): the
|
||||
`prompt` field is a runnable function *prefix*, so the program is
|
||||
prompt+completion+tests verbatim (_concat) and the model is simply asked to
|
||||
continue `prompt`.
|
||||
• MBPP-Python (google-research-datasets/mbpp): the problem is a natural-language
|
||||
`text` description plus a `test_list` of asserts — NOT a code prefix. So the
|
||||
generation prompt is synthesised (description + example asserts as a docstring)
|
||||
and the program is completion+setup+asserts (the NL prompt is discarded).
|
||||
• HumanEval-Python (openai/openai_humaneval): native HumanEval, prompt is a
|
||||
function prefix, tests are a `check(fn)` def (langs._python).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import ast
|
||||
from dataclasses import dataclass
|
||||
from typing import Callable
|
||||
|
||||
from .langs import LANGS, _concat, resolve
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Binding:
|
||||
dataset: str
|
||||
config: str
|
||||
build_prompt: Callable[[dict], str]
|
||||
assemble: Callable[[str, str, dict], str]
|
||||
stop_tokens: Callable[[dict], list[str]]
|
||||
|
||||
|
||||
# ── shared format helpers ────────────────────────────────────────────────────
|
||||
|
||||
def _prompt_field(row: dict) -> str:
|
||||
"""MultiPL-E / HumanEval: the runnable function prefix is the prompt."""
|
||||
return row["prompt"]
|
||||
|
||||
|
||||
def _parse_stop(row: dict) -> list[str]:
|
||||
"""MultiPL-E ships stop_tokens as a list or a stringified list."""
|
||||
raw = row.get("stop_tokens")
|
||||
if isinstance(raw, list):
|
||||
return raw
|
||||
if isinstance(raw, str):
|
||||
try:
|
||||
v = ast.literal_eval(raw)
|
||||
return v if isinstance(v, list) else []
|
||||
except (ValueError, SyntaxError):
|
||||
return []
|
||||
return []
|
||||
|
||||
|
||||
# ── MBPP-Python format (description + asserts, no code prefix) ────────────────
|
||||
|
||||
# Cut as soon as the model leaves the function body for its own tests/output, so
|
||||
# only the candidate implementation survives into the assembled program.
|
||||
_MBPP_PY_STOP = ["\nassert", "\nprint(", "\n# Test", "\nif __name__", '\n"""']
|
||||
|
||||
|
||||
def _mbpp_py_prompt(row: dict) -> str:
|
||||
"""Frame the NL task + example asserts as a leading docstring so a raw
|
||||
(non-chat) instruct model continues with the function definition."""
|
||||
asserts = "\n".join(row.get("test_list", []))
|
||||
return f'"""\n{row.get("text", "").strip()}\n\n{asserts}\n"""\n'
|
||||
|
||||
|
||||
def _mbpp_py_assemble(prompt: str, completion: str, row: dict) -> str:
|
||||
"""Program = setup + the model's code + the held-out asserts. The NL prompt
|
||||
is *not* part of the program (unlike HumanEval, where it is the prefix)."""
|
||||
setup = row.get("test_setup_code", "") or ""
|
||||
asserts = "\n".join(row.get("test_list", []))
|
||||
return f"{setup}\n{completion}\n{asserts}\n"
|
||||
|
||||
|
||||
def _mbpp_py_stop(row: dict) -> list[str]:
|
||||
return _MBPP_PY_STOP
|
||||
|
||||
|
||||
# ── suite tables ─────────────────────────────────────────────────────────────
|
||||
|
||||
def _humaneval_binding(lang_id: str) -> Binding:
|
||||
"""HumanEval reuses each language's existing langs.py dataset/config/assemble
|
||||
(the registry already encodes the HumanEval format)."""
|
||||
L = LANGS[lang_id]
|
||||
return Binding(L.dataset, L.config, _prompt_field, L.assemble, _parse_stop)
|
||||
|
||||
|
||||
_MBPP: dict[str, Binding] = {
|
||||
"python": Binding("google-research-datasets/mbpp", "full",
|
||||
_mbpp_py_prompt, _mbpp_py_assemble, _mbpp_py_stop),
|
||||
"javascript": Binding("nuprl/MultiPL-E", "mbpp-js",
|
||||
_prompt_field, _concat, _parse_stop),
|
||||
"go": Binding("nuprl/MultiPL-E", "mbpp-go",
|
||||
_prompt_field, _concat, _parse_stop),
|
||||
"rust": Binding("nuprl/MultiPL-E", "mbpp-rs",
|
||||
_prompt_field, _concat, _parse_stop),
|
||||
"bash": Binding("nuprl/MultiPL-E", "mbpp-sh",
|
||||
_prompt_field, _concat, _parse_stop),
|
||||
}
|
||||
|
||||
SUITES = {
|
||||
"humaneval": "HumanEval (function-completion)",
|
||||
"mbpp": "MBPP (Mostly Basic Programming Problems)",
|
||||
}
|
||||
|
||||
|
||||
def binding(suite: str, language: str) -> Binding:
|
||||
"""Resolve the (suite, language) -> Binding the harness should run."""
|
||||
lang_id = resolve(language).id
|
||||
s = suite.lower()
|
||||
if s == "humaneval":
|
||||
return _humaneval_binding(lang_id)
|
||||
if s == "mbpp":
|
||||
if lang_id not in _MBPP:
|
||||
raise KeyError(f"suite 'mbpp' has no binding for {lang_id!r}; "
|
||||
f"have: {', '.join(_MBPP)}")
|
||||
return _MBPP[lang_id]
|
||||
raise KeyError(f"unknown suite {suite!r}; known: {', '.join(SUITES)}")
|
||||
Reference in New Issue
Block a user