leetcrypt
c61413c648
Pulled and benchmarked three more tool-capable CPU models looking for a better default. All score 0/12 (vs qwen2.5:3b at 2/12): in the multi-turn agent loop they leak the positional-in-tags dialect (<tools>run_shell 'cmd'</tools>) the parser can't recover, even when they emit clean structured tool_calls on a single-turn probe; smollm2 and mistral also wedge into repeating summaries. qwen3:4b could not be pulled — Ollama 0.3.9 is too old (HTTP 412), same as granite3.1-dense:2b. Upgrading Ollama is the highest-leverage next step to test the qwen3/granite3.x generation. qwen2.5:3b remains the default. Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
114 lines
3.0 KiB
JSON
114 lines
3.0 KiB
JSON
{
|
|
"ts": "2026-06-13T03:23:46",
|
|
"model": "mistral:7b",
|
|
"rows": [
|
|
{
|
|
"id": "shell-easy",
|
|
"category": "shell",
|
|
"tier": "easy",
|
|
"result": "FAIL",
|
|
"secs": 216.8,
|
|
"summary": "<tools>run_shell 'mkdir proj'</tools>",
|
|
"verify_out": ""
|
|
},
|
|
{
|
|
"id": "shell-medium",
|
|
"category": "shell",
|
|
"tier": "medium",
|
|
"result": "FAIL",
|
|
"secs": 81.0,
|
|
"summary": "<tools>run_shell './mkdir -p a/b/c</tools>",
|
|
"verify_out": "cat: /root/a/b/c/marker.txt: No such file or directory"
|
|
},
|
|
{
|
|
"id": "shell-hard",
|
|
"category": "shell",
|
|
"tier": "hard",
|
|
"result": "FAIL",
|
|
"secs": 87.0,
|
|
"summary": "<tools>run_shell 'find /etc -name \"*.conf\" | wc -l</tools>",
|
|
"verify_out": "cat: /root/conf_count.txt: No such file or directory"
|
|
},
|
|
{
|
|
"id": "code-easy",
|
|
"category": "code",
|
|
"tier": "easy",
|
|
"result": "FAIL",
|
|
"secs": 97.0,
|
|
"summary": "<tools>write_file path=add.py content='print(2+3)\\n'</tools>",
|
|
"verify_out": "cat: /root/add.out: No such file or directory"
|
|
},
|
|
{
|
|
"id": "code-medium",
|
|
"category": "code",
|
|
"tier": "medium",
|
|
"result": "TIMEOUT",
|
|
"secs": 46.7,
|
|
"summary": "<tools>write_file path=add.py content='print(2+3)\\n'</tools>",
|
|
"verify_out": ""
|
|
},
|
|
{
|
|
"id": "code-hard",
|
|
"category": "code",
|
|
"tier": "hard",
|
|
"result": "TIMEOUT",
|
|
"secs": 46.8,
|
|
"summary": "<tools>write_file path=add.py content='print(2+3)\\n'</tools>",
|
|
"verify_out": ""
|
|
},
|
|
{
|
|
"id": "git-easy",
|
|
"category": "git",
|
|
"tier": "easy",
|
|
"result": "TIMEOUT",
|
|
"secs": 46.8,
|
|
"summary": "<tools>write_file path=add.py content='print(2+3)\\n'</tools>",
|
|
"verify_out": ""
|
|
},
|
|
{
|
|
"id": "git-medium",
|
|
"category": "git",
|
|
"tier": "medium",
|
|
"result": "TIMEOUT",
|
|
"secs": 46.9,
|
|
"summary": "<tools>write_file path=add.py content='print(2+3)\\n'</tools>",
|
|
"verify_out": ""
|
|
},
|
|
{
|
|
"id": "git-hard",
|
|
"category": "git",
|
|
"tier": "hard",
|
|
"result": "TIMEOUT",
|
|
"secs": 46.8,
|
|
"summary": "<tools>write_file path=add.py content='print(2+3)\\n'</tools>",
|
|
"verify_out": ""
|
|
},
|
|
{
|
|
"id": "multi-easy",
|
|
"category": "multi",
|
|
"tier": "easy",
|
|
"result": "TIMEOUT",
|
|
"secs": 46.9,
|
|
"summary": "<tools>write_file path=add.py content='print(2+3)\\n'</tools>",
|
|
"verify_out": ""
|
|
},
|
|
{
|
|
"id": "multi-medium",
|
|
"category": "multi",
|
|
"tier": "medium",
|
|
"result": "TIMEOUT",
|
|
"secs": 46.8,
|
|
"summary": "<tools>write_file path=add.py content='print(2+3)\\n'</tools>",
|
|
"verify_out": ""
|
|
},
|
|
{
|
|
"id": "multi-hard",
|
|
"category": "multi",
|
|
"tier": "hard",
|
|
"result": "TIMEOUT",
|
|
"secs": 46.9,
|
|
"summary": "<tools>write_file path=add.py content='print(2+3)\\n'</tools>",
|
|
"verify_out": ""
|
|
}
|
|
]
|
|
} |