leetcrypt
df8f1881d8
Fourth benchmark axis: teams of LLM agents deliberate in a room, implement code in an isolated VM, and are scored deterministically on correctness/speed. - Multi-language adapter (python/js/go/rust/bash) via MultiPL-E continuation mode - Append-only JSONL ledger with status tracking (ok/dnf/killed/error) so budget-exhausted or crashed runs still record a row (fixes selection bias) - Model-aware wall-clock scaling (U-shaped by param count; 3x for reasoning) - Self-owned SIGALRM/SIGTERM watchdog (RunTimeout: BaseException so broad except Exception handlers in the infer/completion path can't swallow it) - Seed forwarded to Ollama sampler + markdown-fence stripping in completion.py
93 lines
3.6 KiB
Python
93 lines
3.6 KiB
Python
"""Model completions via Ollama's raw /api/generate endpoint.
|
|
|
|
We deliberately do *not* go through the agent's chat provider here: the
|
|
capability benchmark wants a raw HumanEval-style completion of the function
|
|
prefix (not a chat turn), and we want full control of the read timeout — the
|
|
agent's OllamaProvider hard-codes 120s, which throttles reasoning models. This
|
|
module owns its own timeout knob.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
from dataclasses import dataclass
|
|
|
|
import requests
|
|
|
|
|
|
@dataclass
|
|
class Completion:
|
|
text: str
|
|
ok: bool
|
|
error: str | None = None
|
|
elapsed: float = 0.0
|
|
|
|
|
|
def complete(model: str, prompt: str, stop: list[str] | None = None,
|
|
*, host: str = "http://127.0.0.1:11434", temperature: float = 0.2,
|
|
num_predict: int = 512, timeout: float = 300.0,
|
|
seed: int | None = None) -> Completion:
|
|
"""Ask the model to continue `prompt`. Stop tokens are passed to Ollama and
|
|
re-applied client-side (Ollama strips the stop string, which is what we want
|
|
— the assembled program must not contain the test's leading token twice).
|
|
|
|
``seed`` is forwarded to Ollama's sampler so multi-seed runs at the same
|
|
temperature draw *different but reproducible* samples (the basis for pass@1
|
|
confidence intervals)."""
|
|
import time
|
|
t0 = time.time()
|
|
options = {"temperature": temperature, "num_predict": num_predict}
|
|
if stop:
|
|
options["stop"] = stop
|
|
if seed is not None:
|
|
options["seed"] = seed
|
|
try:
|
|
# raw=True bypasses the chat template so an instruct model *continues*
|
|
# the code (HumanEval-style) instead of replying conversationally with
|
|
# prose + markdown fences, which is what MultiPL-E's assembly expects.
|
|
r = requests.post(f"{host}/api/generate", json={
|
|
"model": model, "prompt": prompt, "stream": False,
|
|
"options": options, "raw": True}, timeout=timeout)
|
|
r.raise_for_status()
|
|
text = r.json().get("response", "")
|
|
except Exception as e: # noqa: BLE001 — surface as a failed completion row
|
|
return Completion("", False, str(e), time.time() - t0)
|
|
return Completion(_truncate(text, stop), True, None, time.time() - t0)
|
|
|
|
|
|
def _strip_fences(text: str) -> str:
|
|
"""Drop a markdown code fence even when raw=True fails to suppress it.
|
|
|
|
Chatty coder models sometimes wrap the continuation in ``` fences (and append
|
|
a prose ``# Explanation`` after a closing fence). A triple-backtick is not
|
|
valid syntax in any target language, so it is a safe, unambiguous cut point.
|
|
Two shapes occur: a *wrapped* block (opens with ```lang … closes with ```)
|
|
and a *bare* continuation that the model terminates with a stray closing ```.
|
|
"""
|
|
fence = "```"
|
|
if fence not in text:
|
|
return text
|
|
stripped = text.lstrip()
|
|
if stripped.startswith(fence):
|
|
# wrapped: drop the opening ```lang line, keep until the next fence.
|
|
body = stripped[len(fence):]
|
|
nl = body.find("\n")
|
|
body = body[nl + 1:] if nl != -1 else ""
|
|
end = body.find(fence)
|
|
return body[:end] if end != -1 else body
|
|
# bare continuation: cut at the stray closing fence.
|
|
return text[:text.find(fence)]
|
|
|
|
|
|
def _truncate(text: str, stop: list[str] | None) -> str:
|
|
"""Defensive client-side stop truncation (covers the no-stop / streamed
|
|
cases and any model that ignores the option)."""
|
|
text = _strip_fences(text)
|
|
if not stop:
|
|
return text
|
|
cut = len(text)
|
|
for s in stop:
|
|
i = text.find(s)
|
|
if i != -1:
|
|
cut = min(cut, i)
|
|
return text[:cut]
|