leetcrypt
df8f1881d8
Fourth benchmark axis: teams of LLM agents deliberate in a room, implement code in an isolated VM, and are scored deterministically on correctness/speed. - Multi-language adapter (python/js/go/rust/bash) via MultiPL-E continuation mode - Append-only JSONL ledger with status tracking (ok/dnf/killed/error) so budget-exhausted or crashed runs still record a row (fixes selection bias) - Model-aware wall-clock scaling (U-shaped by param count; 3x for reasoning) - Self-owned SIGALRM/SIGTERM watchdog (RunTimeout: BaseException so broad except Exception handlers in the infer/completion path can't swallow it) - Seed forwarded to Ollama sampler + markdown-fence stripping in completion.py
57 lines
2.0 KiB
Python
57 lines
2.0 KiB
Python
"""Model inference for the arena.
|
|
|
|
Two call shapes, both Ollama:
|
|
|
|
• ``chat()`` — ``/api/chat`` with a system prompt + message history, used for
|
|
DELIBERATE turns (conversational planning/critique). Returns text + a token
|
|
estimate for the cost axis.
|
|
• code generation reuses ``bench/completion.py`` (raw ``/api/generate``) so the
|
|
IMPLEMENT phase gets a clean HumanEval/MBPP-style continuation, exactly like
|
|
the capability benchmark.
|
|
|
|
Keeping inference here (not in ``cmd_chat``) honours the no-edit rule: the arena
|
|
drives its own inference and merely *posts* the result into the real room.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
from dataclasses import dataclass
|
|
|
|
import requests
|
|
|
|
|
|
@dataclass
|
|
class ChatReply:
|
|
text: str
|
|
ok: bool
|
|
tokens: dict
|
|
error: str | None = None
|
|
|
|
|
|
def _est_tokens(text: str) -> int:
|
|
# cheap, deterministic estimate (~4 chars/token); real usage when Ollama
|
|
# returns eval counts is preferred and used when present.
|
|
return max(1, len(text) // 4)
|
|
|
|
|
|
def chat(model: str, system: str, messages: list[dict], *,
|
|
host: str = "http://127.0.0.1:11434", temperature: float = 0.4,
|
|
num_predict: int = 320, timeout: float = 120.0) -> ChatReply:
|
|
"""One chat turn. ``messages`` is a list of {role, content} (no system)."""
|
|
payload = {
|
|
"model": model, "stream": False,
|
|
"messages": [{"role": "system", "content": system}, *messages],
|
|
"options": {"temperature": temperature, "num_predict": num_predict},
|
|
}
|
|
try:
|
|
r = requests.post(f"{host}/api/chat", json=payload, timeout=timeout)
|
|
r.raise_for_status()
|
|
data = r.json()
|
|
except Exception as e: # noqa: BLE001 — surface as a failed turn
|
|
return ChatReply("", False, {"in": 0, "out": 0}, str(e))
|
|
text = data.get("message", {}).get("content", "")
|
|
tok = {"in": data.get("prompt_eval_count") or _est_tokens(system +
|
|
"".join(m["content"] for m in messages)),
|
|
"out": data.get("eval_count") or _est_tokens(text)}
|
|
return ChatReply(text.strip(), True, tok)
|