feat(bench): multi-language capability benchmark + model picker
Add bench-lang.py + bench/ package: a third benchmark axis answering "which open-source model is best for my workflow?" across Python, JavaScript, Go, Rust and Bash. - MultiPL-E (Go/Rust/JS/Bash) + original HumanEval (Python), loaded via the HF datasets-server REST API with on-disk cache — no datasets/ pyarrow dependency. - Completions go straight to Ollama /api/generate with raw=True so instruct models continue the code instead of replying with prose. - Code runs in rootless, network-less podman (safe default) with a host-toolchain fallback; pass@1/pass@k via the HumanEval estimator. - run/score separation: results persist to a scorecard JSON, then `pick --workflow ops` re-ranks without re-running any model. - Extensible: a new language is one Lang entry; a new workflow is one block in workflows.json. Also fix a --runs grant-persistence bug in bench-sandbox.py: the grant leaked across runs, invalidating the L0-nogrant refusal test on runs 2+. Each run now revokes the ACL and starts ungranted. Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
@@ -0,0 +1,64 @@
|
||||
"""Dependency-free problem loader.
|
||||
|
||||
Pulls rows from the Hugging Face datasets-server REST API (plain `requests`, no
|
||||
`datasets`/`pyarrow`) and caches them on disk so repeated benchmark runs are
|
||||
offline and fast. One JSON file per (dataset, config), under ~/.cache.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
import requests
|
||||
|
||||
_API = "https://datasets-server.huggingface.co/rows"
|
||||
_CACHE = Path.home() / ".cache" / "hh-bench" / "datasets"
|
||||
_PAGE = 100 # datasets-server caps `length` at 100 rows per call
|
||||
|
||||
|
||||
def _cache_path(dataset: str, config: str, split: str) -> Path:
|
||||
safe = f"{dataset}__{config}__{split}".replace("/", "_")
|
||||
return _CACHE / f"{safe}.json"
|
||||
|
||||
|
||||
def load(dataset: str, config: str, split: str = "test",
|
||||
limit: int | None = None, refresh: bool = False) -> list[dict]:
|
||||
"""Return a list of row dicts for one dataset config.
|
||||
|
||||
Cached after first fetch. `limit` slices the returned list (the full set is
|
||||
still cached). `refresh` forces a re-download.
|
||||
"""
|
||||
cp = _cache_path(dataset, config, split)
|
||||
if cp.exists() and not refresh:
|
||||
rows = json.loads(cp.read_text())
|
||||
else:
|
||||
rows = _download(dataset, config, split)
|
||||
cp.parent.mkdir(parents=True, exist_ok=True)
|
||||
cp.write_text(json.dumps(rows))
|
||||
return rows[:limit] if limit else rows
|
||||
|
||||
|
||||
def _download(dataset: str, config: str, split: str) -> list[dict]:
|
||||
rows: list[dict] = []
|
||||
offset = 0
|
||||
while True:
|
||||
r = requests.get(_API, params={
|
||||
"dataset": dataset, "config": config, "split": split,
|
||||
"offset": offset, "length": _PAGE}, timeout=60)
|
||||
r.raise_for_status()
|
||||
payload = r.json()
|
||||
batch = payload.get("rows", [])
|
||||
if not batch:
|
||||
break
|
||||
rows.extend(item["row"] for item in batch)
|
||||
total = payload.get("num_rows_total")
|
||||
offset += len(batch)
|
||||
if total is not None and offset >= total:
|
||||
break
|
||||
if len(batch) < _PAGE:
|
||||
break
|
||||
if not rows:
|
||||
raise RuntimeError(
|
||||
f"no rows for {dataset}/{config}/{split} — check the config name")
|
||||
return rows
|
||||
Reference in New Issue
Block a user