leetcrypt
7fb3911550
Add bench-lang.py + bench/ package: a third benchmark axis answering "which open-source model is best for my workflow?" across Python, JavaScript, Go, Rust and Bash. - MultiPL-E (Go/Rust/JS/Bash) + original HumanEval (Python), loaded via the HF datasets-server REST API with on-disk cache — no datasets/ pyarrow dependency. - Completions go straight to Ollama /api/generate with raw=True so instruct models continue the code instead of replying with prose. - Code runs in rootless, network-less podman (safe default) with a host-toolchain fallback; pass@1/pass@k via the HumanEval estimator. - run/score separation: results persist to a scorecard JSON, then `pick --workflow ops` re-ranks without re-running any model. - Extensible: a new language is one Lang entry; a new workflow is one block in workflows.json. Also fix a --runs grant-persistence bug in bench-sandbox.py: the grant leaked across runs, invalidating the L0-nogrant refusal test on runs 2+. Each run now revokes the ACL and starts ungranted. Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
61 lines
2.3 KiB
Python
61 lines
2.3 KiB
Python
"""Model completions via Ollama's raw /api/generate endpoint.
|
|
|
|
We deliberately do *not* go through the agent's chat provider here: the
|
|
capability benchmark wants a raw HumanEval-style completion of the function
|
|
prefix (not a chat turn), and we want full control of the read timeout — the
|
|
agent's OllamaProvider hard-codes 120s, which throttles reasoning models. This
|
|
module owns its own timeout knob.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
from dataclasses import dataclass
|
|
|
|
import requests
|
|
|
|
|
|
@dataclass
|
|
class Completion:
|
|
text: str
|
|
ok: bool
|
|
error: str | None = None
|
|
elapsed: float = 0.0
|
|
|
|
|
|
def complete(model: str, prompt: str, stop: list[str] | None = None,
|
|
*, host: str = "http://127.0.0.1:11434", temperature: float = 0.2,
|
|
num_predict: int = 512, timeout: float = 300.0) -> Completion:
|
|
"""Ask the model to continue `prompt`. Stop tokens are passed to Ollama and
|
|
re-applied client-side (Ollama strips the stop string, which is what we want
|
|
— the assembled program must not contain the test's leading token twice)."""
|
|
import time
|
|
t0 = time.time()
|
|
options = {"temperature": temperature, "num_predict": num_predict}
|
|
if stop:
|
|
options["stop"] = stop
|
|
try:
|
|
# raw=True bypasses the chat template so an instruct model *continues*
|
|
# the code (HumanEval-style) instead of replying conversationally with
|
|
# prose + markdown fences, which is what MultiPL-E's assembly expects.
|
|
r = requests.post(f"{host}/api/generate", json={
|
|
"model": model, "prompt": prompt, "stream": False,
|
|
"options": options, "raw": True}, timeout=timeout)
|
|
r.raise_for_status()
|
|
text = r.json().get("response", "")
|
|
except Exception as e: # noqa: BLE001 — surface as a failed completion row
|
|
return Completion("", False, str(e), time.time() - t0)
|
|
return Completion(_truncate(text, stop), True, None, time.time() - t0)
|
|
|
|
|
|
def _truncate(text: str, stop: list[str] | None) -> str:
|
|
"""Defensive client-side stop truncation (covers the no-stop / streamed
|
|
cases and any model that ignores the option)."""
|
|
if not stop:
|
|
return text
|
|
cut = len(text)
|
|
for s in stop:
|
|
i = text.find(s)
|
|
if i != -1:
|
|
cut = min(cut, i)
|
|
return text[:cut]
|