test(ai): benchmark llama3.2:3b + qwen2.5-coder:3b, optimized-harness baselines
Add tracked baselines for two additional CPU models under the optimized harness (split-tag recovery + greedy decode). Both land at 1/12 — they emit proper structured calls and fail on capability/content, not parse, confirming the parser lift is concentrated on the weakest model (0.5b). granite3.1-dense:2b is incompatible with the installed Ollama version. Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
leetcrypt
@@ -0,0 +1,114 @@
|
||||
{
|
||||
"ts": "2026-06-10T13:59:19",
|
||||
"model": "qwen2.5-coder:3b",
|
||||
"rows": [
|
||||
{
|
||||
"id": "shell-easy",
|
||||
"category": "shell",
|
||||
"tier": "easy",
|
||||
"result": "FAIL",
|
||||
"secs": 82.9,
|
||||
"summary": "run_shell \"cat who.txt\"",
|
||||
"verify_out": ""
|
||||
},
|
||||
{
|
||||
"id": "shell-medium",
|
||||
"category": "shell",
|
||||
"tier": "medium",
|
||||
"result": "FAIL",
|
||||
"secs": 44.7,
|
||||
"summary": "mkdir -p a/b/c",
|
||||
"verify_out": "cat: /root/a/b/c/marker.txt: No such file or directory"
|
||||
},
|
||||
{
|
||||
"id": "shell-hard",
|
||||
"category": "shell",
|
||||
"tier": "hard",
|
||||
"result": "FAIL",
|
||||
"secs": 46.7,
|
||||
"summary": "write_file conf_count.txt \"$(cat conf_count.txt)\" \u2502\u2502 \u2502 \u2514\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500",
|
||||
"verify_out": "cat: /root/conf_count.txt: No such file or directory"
|
||||
},
|
||||
{
|
||||
"id": "code-easy",
|
||||
"category": "code",
|
||||
"tier": "easy",
|
||||
"result": "FAIL",
|
||||
"secs": 42.7,
|
||||
"summary": "write_file add.py \"print(2 + 3)\"",
|
||||
"verify_out": "cat: /root/add.out: No such file or directory"
|
||||
},
|
||||
{
|
||||
"id": "code-medium",
|
||||
"category": "code",
|
||||
"tier": "medium",
|
||||
"result": "FAIL",
|
||||
"secs": 52.7,
|
||||
"summary": "read_file greet.out \u2502\u2502 \u2502 \u2514\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500",
|
||||
"verify_out": ""
|
||||
},
|
||||
{
|
||||
"id": "code-hard",
|
||||
"category": "code",
|
||||
"tier": "hard",
|
||||
"result": "FAIL",
|
||||
"secs": 50.7,
|
||||
"summary": "write_file fib.py \"print(0 1)\"",
|
||||
"verify_out": "cat: /root/fib.out: No such file or directory"
|
||||
},
|
||||
{
|
||||
"id": "git-easy",
|
||||
"category": "git",
|
||||
"tier": "easy",
|
||||
"result": "FAIL",
|
||||
"secs": 40.7,
|
||||
"summary": "run_shell \"cd hw && git clone https://github.com/octocat/Hello-World\" \u2502\u2502 \u2502 \u2514\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500",
|
||||
"verify_out": ""
|
||||
},
|
||||
{
|
||||
"id": "git-medium",
|
||||
"category": "git",
|
||||
"tier": "medium",
|
||||
"result": "FAIL",
|
||||
"secs": 46.7,
|
||||
"summary": "read_file readme.txt \u2502\u2502 \u2502 \u2514\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500",
|
||||
"verify_out": "grep: /root/readme.txt: No such file or directory"
|
||||
},
|
||||
{
|
||||
"id": "git-hard",
|
||||
"category": "git",
|
||||
"tier": "hard",
|
||||
"result": "FAIL",
|
||||
"secs": 44.7,
|
||||
"summary": "run_shell \"cd hw && git clone https://github.com/octocat/Hello-World\"",
|
||||
"verify_out": "sh: 1: cannot open /root/branch.txt: No such file"
|
||||
},
|
||||
{
|
||||
"id": "multi-easy",
|
||||
"category": "multi",
|
||||
"tier": "easy",
|
||||
"result": "FAIL",
|
||||
"secs": 44.7,
|
||||
"summary": "ls -la \u2502\u2502 \u2502 \u2514\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500",
|
||||
"verify_out": "cat: /root/proj/notes.txt: No such file or directory"
|
||||
},
|
||||
{
|
||||
"id": "multi-medium",
|
||||
"category": "multi",
|
||||
"tier": "medium",
|
||||
"result": "PASS",
|
||||
"secs": 56.8,
|
||||
"summary": "write_file sorted.txt \u2502\u2502 \u2502 \u2514\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500",
|
||||
"verify_out": ""
|
||||
},
|
||||
{
|
||||
"id": "multi-hard",
|
||||
"category": "multi",
|
||||
"tier": "hard",
|
||||
"result": "FAIL",
|
||||
"secs": 80.8,
|
||||
"summary": "write_file words.txt \"one two three\"",
|
||||
"verify_out": "sh: 1: cannot open /root/wc.out: No such file"
|
||||
}
|
||||
]
|
||||
}
|
||||
Reference in New Issue
Block a user