From c61413c6483d26ba6783271b1bcdbef7a08cac9a Mon Sep 17 00:00:00 2001 From: leetcrypt Date: Sat, 13 Jun 2026 03:28:04 -0700 Subject: [PATCH] =?UTF-8?q?test(ai):=20screen=20smollm2/hermes3/mistral=20?= =?UTF-8?q?=E2=80=94=20none=20beat=20qwen2.5:3b?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Pulled and benchmarked three more tool-capable CPU models looking for a better default. All score 0/12 (vs qwen2.5:3b at 2/12): in the multi-turn agent loop they leak the positional-in-tags dialect (run_shell 'cmd') the parser can't recover, even when they emit clean structured tool_calls on a single-turn probe; smollm2 and mistral also wedge into repeating summaries. qwen3:4b could not be pulled — Ollama 0.3.9 is too old (HTTP 412), same as granite3.1-dense:2b. Upgrading Ollama is the highest-leverage next step to test the qwen3/granite3.x generation. qwen2.5:3b remains the default. Co-Authored-By: Claude Opus 4.6 --- bench/README.md | 28 ++++++- bench/results/new-hermes3_3b.json | 114 ++++++++++++++++++++++++++++ bench/results/new-mistral_7b.json | 114 ++++++++++++++++++++++++++++ bench/results/new-smollm2_1.7b.json | 114 ++++++++++++++++++++++++++++ 4 files changed, 367 insertions(+), 3 deletions(-) create mode 100644 bench/results/new-hermes3_3b.json create mode 100644 bench/results/new-mistral_7b.json create mode 100644 bench/results/new-smollm2_1.7b.json diff --git a/bench/README.md b/bench/README.md index 375d6e5..5a25cf4 100644 --- a/bench/README.md +++ b/bench/README.md @@ -95,7 +95,29 @@ band. **qwen2.5:3b stays the default** (best capability/latency); 0.5b is now a floor option for trivial tasks. Treat absolute scores as a *relative* regression yardstick, not a grade. -Usable models: qwen2.5(-coder) 0.5b–7b, llama3.2:3b (all tool-capable). Rejected: +### Candidate screening (2026-06-13) — can anything beat qwen? + +Pulled and benchmarked three more tool-capable models to look for a better +default. None did — all 0/12, worse than qwen2.5:3b. The pattern repeats: in the +multi-turn agent loop the weak models leak a **positional-in-tags** dialect +(`run_shell 'cmd'`) that the parser can't recover, even when they +emit clean structured `tool_calls` on a single-turn probe. + +| model | size | PASS | avg s | note | +|---|---|---|---|---| +| smollm2:1.7b | 1.8 GB | 0/12 | ~50 | wedges into a repeating fabricated summary | +| hermes3:3b | 2.0 GB | 0/12 | ~47 | leaks `NAME "arg"` positional form | +| mistral:7b | 4.1 GB | 0/12 | ~70 | clean call on probe, leaks prose-tags live; then wedges | + +`qwen3:4b` could not be pulled — **Ollama 0.3.9 is too old** (HTTP 412, "requires a +newer version of Ollama"), same root cause as `granite3.1-dense:2b`. Upgrading +Ollama would unlock the qwen3 / granite3.x generation and is the highest-leverage +next step if we want to retest newer architectures. + +**Conclusion: qwen2.5:3b remains the default.** No CPU-runnable model on this +Ollama version beats it. + +Usable models: qwen2.5(-coder) 0.5b–7b, llama3.2:3b, smollm2:1.7b, hermes3:3b, +mistral:7b (all tool-capable; the last three still score 0/12 here). Rejected: `deepseek-r1` and `westenfelder/NL2SH` reject Ollama's `tools` field; -`granite3.1-dense:2b` errors (`model not supported by your version of Ollama` — needs -a newer Ollama). +`granite3.1-dense:2b` and `qwen3:4b` need a newer Ollama (0.3.9 installed). diff --git a/bench/results/new-hermes3_3b.json b/bench/results/new-hermes3_3b.json new file mode 100644 index 0000000..26cb817 --- /dev/null +++ b/bench/results/new-hermes3_3b.json @@ -0,0 +1,114 @@ +{ + "ts": "2026-06-13T03:04:51", + "model": "hermes3:3b", + "rows": [ + { + "id": "shell-easy", + "category": "shell", + "tier": "easy", + "result": "TIMEOUT", + "secs": 45.8, + "summary": "", + "verify_out": "" + }, + { + "id": "shell-medium", + "category": "shell", + "tier": "medium", + "result": "TIMEOUT", + "secs": 47.0, + "summary": "", + "verify_out": "" + }, + { + "id": "shell-hard", + "category": "shell", + "tier": "hard", + "result": "TIMEOUT", + "secs": 47.5, + "summary": "", + "verify_out": "" + }, + { + "id": "code-easy", + "category": "code", + "tier": "easy", + "result": "TIMEOUT", + "secs": 47.1, + "summary": "", + "verify_out": "" + }, + { + "id": "code-medium", + "category": "code", + "tier": "medium", + "result": "TIMEOUT", + "secs": 47.4, + "summary": "", + "verify_out": "" + }, + { + "id": "code-hard", + "category": "code", + "tier": "hard", + "result": "TIMEOUT", + "secs": 46.9, + "summary": "", + "verify_out": "" + }, + { + "id": "git-easy", + "category": "git", + "tier": "easy", + "result": "TIMEOUT", + "secs": 47.0, + "summary": "", + "verify_out": "" + }, + { + "id": "git-medium", + "category": "git", + "tier": "medium", + "result": "TIMEOUT", + "secs": 47.0, + "summary": "", + "verify_out": "" + }, + { + "id": "git-hard", + "category": "git", + "tier": "hard", + "result": "TIMEOUT", + "secs": 46.9, + "summary": "", + "verify_out": "" + }, + { + "id": "multi-easy", + "category": "multi", + "tier": "easy", + "result": "TIMEOUT", + "secs": 46.9, + "summary": "", + "verify_out": "" + }, + { + "id": "multi-medium", + "category": "multi", + "tier": "medium", + "result": "TIMEOUT", + "secs": 46.8, + "summary": "", + "verify_out": "" + }, + { + "id": "multi-hard", + "category": "multi", + "tier": "hard", + "result": "TIMEOUT", + "secs": 46.8, + "summary": "", + "verify_out": "" + } + ] +} \ No newline at end of file diff --git a/bench/results/new-mistral_7b.json b/bench/results/new-mistral_7b.json new file mode 100644 index 0000000..b4ee147 --- /dev/null +++ b/bench/results/new-mistral_7b.json @@ -0,0 +1,114 @@ +{ + "ts": "2026-06-13T03:23:46", + "model": "mistral:7b", + "rows": [ + { + "id": "shell-easy", + "category": "shell", + "tier": "easy", + "result": "FAIL", + "secs": 216.8, + "summary": "run_shell 'mkdir proj'", + "verify_out": "" + }, + { + "id": "shell-medium", + "category": "shell", + "tier": "medium", + "result": "FAIL", + "secs": 81.0, + "summary": "run_shell './mkdir -p a/b/c", + "verify_out": "cat: /root/a/b/c/marker.txt: No such file or directory" + }, + { + "id": "shell-hard", + "category": "shell", + "tier": "hard", + "result": "FAIL", + "secs": 87.0, + "summary": "run_shell 'find /etc -name \"*.conf\" | wc -l", + "verify_out": "cat: /root/conf_count.txt: No such file or directory" + }, + { + "id": "code-easy", + "category": "code", + "tier": "easy", + "result": "FAIL", + "secs": 97.0, + "summary": "write_file path=add.py content='print(2+3)\\n'", + "verify_out": "cat: /root/add.out: No such file or directory" + }, + { + "id": "code-medium", + "category": "code", + "tier": "medium", + "result": "TIMEOUT", + "secs": 46.7, + "summary": "write_file path=add.py content='print(2+3)\\n'", + "verify_out": "" + }, + { + "id": "code-hard", + "category": "code", + "tier": "hard", + "result": "TIMEOUT", + "secs": 46.8, + "summary": "write_file path=add.py content='print(2+3)\\n'", + "verify_out": "" + }, + { + "id": "git-easy", + "category": "git", + "tier": "easy", + "result": "TIMEOUT", + "secs": 46.8, + "summary": "write_file path=add.py content='print(2+3)\\n'", + "verify_out": "" + }, + { + "id": "git-medium", + "category": "git", + "tier": "medium", + "result": "TIMEOUT", + "secs": 46.9, + "summary": "write_file path=add.py content='print(2+3)\\n'", + "verify_out": "" + }, + { + "id": "git-hard", + "category": "git", + "tier": "hard", + "result": "TIMEOUT", + "secs": 46.8, + "summary": "write_file path=add.py content='print(2+3)\\n'", + "verify_out": "" + }, + { + "id": "multi-easy", + "category": "multi", + "tier": "easy", + "result": "TIMEOUT", + "secs": 46.9, + "summary": "write_file path=add.py content='print(2+3)\\n'", + "verify_out": "" + }, + { + "id": "multi-medium", + "category": "multi", + "tier": "medium", + "result": "TIMEOUT", + "secs": 46.8, + "summary": "write_file path=add.py content='print(2+3)\\n'", + "verify_out": "" + }, + { + "id": "multi-hard", + "category": "multi", + "tier": "hard", + "result": "TIMEOUT", + "secs": 46.9, + "summary": "write_file path=add.py content='print(2+3)\\n'", + "verify_out": "" + } + ] +} \ No newline at end of file diff --git a/bench/results/new-smollm2_1.7b.json b/bench/results/new-smollm2_1.7b.json new file mode 100644 index 0000000..532234f --- /dev/null +++ b/bench/results/new-smollm2_1.7b.json @@ -0,0 +1,114 @@ +{ + "ts": "2026-06-13T02:49:54", + "model": "smollm2:1.7b", + "rows": [ + { + "id": "shell-easy", + "category": "shell", + "tier": "easy", + "result": "FAIL", + "secs": 140.5, + "summary": "write_file who.txt \"$(whoami)\" \u2502\u2502 \u2502 \u2514\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500", + "verify_out": "cat: /root/who.txt: No such file or directory" + }, + { + "id": "shell-medium", + "category": "shell", + "tier": "medium", + "result": "FAIL", + "secs": 54.8, + "summary": "run_shell \"mkdir -p a/b/c && echo 'OK' > a/b/c/marker.txt\" \u2502\u2502 \u2502 \u2514\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500", + "verify_out": "cat: /root/a/b/c/marker.txt: No such file or directory" + }, + { + "id": "shell-hard", + "category": "shell", + "tier": "hard", + "result": "FAIL", + "secs": 60.9, + "summary": "run_shell \"find /etc -name '*.conf' | wc -l > conf_count.txt\"", + "verify_out": "cat: /root/conf_count.txt: No such file or directory" + }, + { + "id": "code-easy", + "category": "code", + "tier": "easy", + "result": "FAIL", + "secs": 76.9, + "summary": "The Python file `add.py` has been written, executed, and its output saved in \u2502\u2502 \u2502 \u2502`add.out`.", + "verify_out": "cat: /root/add.out: No such file or directory" + }, + { + "id": "code-medium", + "category": "code", + "tier": "medium", + "result": "TIMEOUT", + "secs": 46.8, + "summary": "The Python file `add.py` has been written, executed, and its output saved in \u2502\u2502 \u2502 \u2502`add.out`.", + "verify_out": "" + }, + { + "id": "code-hard", + "category": "code", + "tier": "hard", + "result": "TIMEOUT", + "secs": 47.0, + "summary": "The Python file `add.py` has been written, executed, and its output saved in \u2502\u2502 \u2502 \u2502`add.out`.", + "verify_out": "" + }, + { + "id": "git-easy", + "category": "git", + "tier": "easy", + "result": "TIMEOUT", + "secs": 47.5, + "summary": "The Python file `add.py` has been written, executed, and its output saved in \u2502\u2502 \u2502 \u2502`add.out`.", + "verify_out": "" + }, + { + "id": "git-medium", + "category": "git", + "tier": "medium", + "result": "TIMEOUT", + "secs": 46.9, + "summary": "The Python file `add.py` has been written, executed, and its output saved in \u2502\u2502 \u2502 \u2502`add.out`.", + "verify_out": "" + }, + { + "id": "git-hard", + "category": "git", + "tier": "hard", + "result": "TIMEOUT", + "secs": 47.3, + "summary": "The Python file `add.py` has been written, executed, and its output saved in \u2502\u2502 \u2502 \u2502`add.out`.", + "verify_out": "" + }, + { + "id": "multi-easy", + "category": "multi", + "tier": "easy", + "result": "TIMEOUT", + "secs": 46.9, + "summary": "The Python file `add.py` has been written, executed, and its output saved in \u2502\u2502 \u2502 \u2502`add.out`.", + "verify_out": "" + }, + { + "id": "multi-medium", + "category": "multi", + "tier": "medium", + "result": "TIMEOUT", + "secs": 47.5, + "summary": "The Python file `add.py` has been written, executed, and its output saved in \u2502\u2502 \u2502 \u2502`add.out`.", + "verify_out": "" + }, + { + "id": "multi-hard", + "category": "multi", + "tier": "hard", + "result": "TIMEOUT", + "secs": 47.0, + "summary": "The Python file `add.py` has been written, executed, and its output saved in \u2502\u2502 \u2502 \u2502`add.out`.", + "verify_out": "" + } + ] +} \ No newline at end of file