leetcrypt
7fb3911550
Add bench-lang.py + bench/ package: a third benchmark axis answering "which open-source model is best for my workflow?" across Python, JavaScript, Go, Rust and Bash. - MultiPL-E (Go/Rust/JS/Bash) + original HumanEval (Python), loaded via the HF datasets-server REST API with on-disk cache — no datasets/ pyarrow dependency. - Completions go straight to Ollama /api/generate with raw=True so instruct models continue the code instead of replying with prose. - Code runs in rootless, network-less podman (safe default) with a host-toolchain fallback; pass@1/pass@k via the HumanEval estimator. - run/score separation: results persist to a scorecard JSON, then `pick --workflow ops` re-ranks without re-running any model. - Extensible: a new language is one Lang entry; a new workflow is one block in workflows.json. Also fix a --runs grant-persistence bug in bench-sandbox.py: the grant leaked across runs, invalidating the L0-nogrant refusal test on runs 2+. Each run now revokes the ACL and starts ungranted. Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
65 lines
2.1 KiB
Python
65 lines
2.1 KiB
Python
"""Dependency-free problem loader.
|
|
|
|
Pulls rows from the Hugging Face datasets-server REST API (plain `requests`, no
|
|
`datasets`/`pyarrow`) and caches them on disk so repeated benchmark runs are
|
|
offline and fast. One JSON file per (dataset, config), under ~/.cache.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
from pathlib import Path
|
|
|
|
import requests
|
|
|
|
_API = "https://datasets-server.huggingface.co/rows"
|
|
_CACHE = Path.home() / ".cache" / "hh-bench" / "datasets"
|
|
_PAGE = 100 # datasets-server caps `length` at 100 rows per call
|
|
|
|
|
|
def _cache_path(dataset: str, config: str, split: str) -> Path:
|
|
safe = f"{dataset}__{config}__{split}".replace("/", "_")
|
|
return _CACHE / f"{safe}.json"
|
|
|
|
|
|
def load(dataset: str, config: str, split: str = "test",
|
|
limit: int | None = None, refresh: bool = False) -> list[dict]:
|
|
"""Return a list of row dicts for one dataset config.
|
|
|
|
Cached after first fetch. `limit` slices the returned list (the full set is
|
|
still cached). `refresh` forces a re-download.
|
|
"""
|
|
cp = _cache_path(dataset, config, split)
|
|
if cp.exists() and not refresh:
|
|
rows = json.loads(cp.read_text())
|
|
else:
|
|
rows = _download(dataset, config, split)
|
|
cp.parent.mkdir(parents=True, exist_ok=True)
|
|
cp.write_text(json.dumps(rows))
|
|
return rows[:limit] if limit else rows
|
|
|
|
|
|
def _download(dataset: str, config: str, split: str) -> list[dict]:
|
|
rows: list[dict] = []
|
|
offset = 0
|
|
while True:
|
|
r = requests.get(_API, params={
|
|
"dataset": dataset, "config": config, "split": split,
|
|
"offset": offset, "length": _PAGE}, timeout=60)
|
|
r.raise_for_status()
|
|
payload = r.json()
|
|
batch = payload.get("rows", [])
|
|
if not batch:
|
|
break
|
|
rows.extend(item["row"] for item in batch)
|
|
total = payload.get("num_rows_total")
|
|
offset += len(batch)
|
|
if total is not None and offset >= total:
|
|
break
|
|
if len(batch) < _PAGE:
|
|
break
|
|
if not rows:
|
|
raise RuntimeError(
|
|
f"no rows for {dataset}/{config}/{split} — check the config name")
|
|
return rows
|