leetcrypt
e7746e49cf
Add tracked baselines for two additional CPU models under the optimized harness (split-tag recovery + greedy decode). Both land at 1/12 — they emit proper structured calls and fail on capability/content, not parse, confirming the parser lift is concentrated on the weakest model (0.5b). granite3.1-dense:2b is incompatible with the installed Ollama version. Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
114 lines
5.4 KiB
JSON
114 lines
5.4 KiB
JSON
{
|
|
"ts": "2026-06-10T13:59:19",
|
|
"model": "qwen2.5-coder:3b",
|
|
"rows": [
|
|
{
|
|
"id": "shell-easy",
|
|
"category": "shell",
|
|
"tier": "easy",
|
|
"result": "FAIL",
|
|
"secs": 82.9,
|
|
"summary": "run_shell \"cat who.txt\"",
|
|
"verify_out": ""
|
|
},
|
|
{
|
|
"id": "shell-medium",
|
|
"category": "shell",
|
|
"tier": "medium",
|
|
"result": "FAIL",
|
|
"secs": 44.7,
|
|
"summary": "mkdir -p a/b/c",
|
|
"verify_out": "cat: /root/a/b/c/marker.txt: No such file or directory"
|
|
},
|
|
{
|
|
"id": "shell-hard",
|
|
"category": "shell",
|
|
"tier": "hard",
|
|
"result": "FAIL",
|
|
"secs": 46.7,
|
|
"summary": "write_file conf_count.txt \"$(cat conf_count.txt)\" \u2502\u2502 \u2502 \u2514\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500",
|
|
"verify_out": "cat: /root/conf_count.txt: No such file or directory"
|
|
},
|
|
{
|
|
"id": "code-easy",
|
|
"category": "code",
|
|
"tier": "easy",
|
|
"result": "FAIL",
|
|
"secs": 42.7,
|
|
"summary": "write_file add.py \"print(2 + 3)\"",
|
|
"verify_out": "cat: /root/add.out: No such file or directory"
|
|
},
|
|
{
|
|
"id": "code-medium",
|
|
"category": "code",
|
|
"tier": "medium",
|
|
"result": "FAIL",
|
|
"secs": 52.7,
|
|
"summary": "read_file greet.out \u2502\u2502 \u2502 \u2514\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500",
|
|
"verify_out": ""
|
|
},
|
|
{
|
|
"id": "code-hard",
|
|
"category": "code",
|
|
"tier": "hard",
|
|
"result": "FAIL",
|
|
"secs": 50.7,
|
|
"summary": "write_file fib.py \"print(0 1)\"",
|
|
"verify_out": "cat: /root/fib.out: No such file or directory"
|
|
},
|
|
{
|
|
"id": "git-easy",
|
|
"category": "git",
|
|
"tier": "easy",
|
|
"result": "FAIL",
|
|
"secs": 40.7,
|
|
"summary": "run_shell \"cd hw && git clone https://github.com/octocat/Hello-World\" \u2502\u2502 \u2502 \u2514\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500",
|
|
"verify_out": ""
|
|
},
|
|
{
|
|
"id": "git-medium",
|
|
"category": "git",
|
|
"tier": "medium",
|
|
"result": "FAIL",
|
|
"secs": 46.7,
|
|
"summary": "read_file readme.txt \u2502\u2502 \u2502 \u2514\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500",
|
|
"verify_out": "grep: /root/readme.txt: No such file or directory"
|
|
},
|
|
{
|
|
"id": "git-hard",
|
|
"category": "git",
|
|
"tier": "hard",
|
|
"result": "FAIL",
|
|
"secs": 44.7,
|
|
"summary": "run_shell \"cd hw && git clone https://github.com/octocat/Hello-World\"",
|
|
"verify_out": "sh: 1: cannot open /root/branch.txt: No such file"
|
|
},
|
|
{
|
|
"id": "multi-easy",
|
|
"category": "multi",
|
|
"tier": "easy",
|
|
"result": "FAIL",
|
|
"secs": 44.7,
|
|
"summary": "ls -la \u2502\u2502 \u2502 \u2514\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500",
|
|
"verify_out": "cat: /root/proj/notes.txt: No such file or directory"
|
|
},
|
|
{
|
|
"id": "multi-medium",
|
|
"category": "multi",
|
|
"tier": "medium",
|
|
"result": "PASS",
|
|
"secs": 56.8,
|
|
"summary": "write_file sorted.txt \u2502\u2502 \u2502 \u2514\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500",
|
|
"verify_out": ""
|
|
},
|
|
{
|
|
"id": "multi-hard",
|
|
"category": "multi",
|
|
"tier": "hard",
|
|
"result": "FAIL",
|
|
"secs": 80.8,
|
|
"summary": "write_file words.txt \"one two three\"",
|
|
"verify_out": "sh: 1: cannot open /root/wc.out: No such file"
|
|
}
|
|
]
|
|
} |