leetcrypt
e7746e49cf
Add tracked baselines for two additional CPU models under the optimized harness (split-tag recovery + greedy decode). Both land at 1/12 — they emit proper structured calls and fail on capability/content, not parse, confirming the parser lift is concentrated on the weakest model (0.5b). granite3.1-dense:2b is incompatible with the installed Ollama version. Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
114 lines
4.9 KiB
JSON
114 lines
4.9 KiB
JSON
{
|
|
"ts": "2026-06-10T13:47:25",
|
|
"model": "llama3.2:3b",
|
|
"rows": [
|
|
{
|
|
"id": "shell-easy",
|
|
"category": "shell",
|
|
"tier": "easy",
|
|
"result": "FAIL",
|
|
"secs": 76.9,
|
|
"summary": "run_shell \"./who.txt\" \u2502\u2502 \u2502 \u2514\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500",
|
|
"verify_out": ""
|
|
},
|
|
{
|
|
"id": "shell-medium",
|
|
"category": "shell",
|
|
"tier": "medium",
|
|
"result": "FAIL",
|
|
"secs": 40.7,
|
|
"summary": "mkdir -p a/b/c",
|
|
"verify_out": "cat: /root/a/b/c/marker.txt: No such file or directory"
|
|
},
|
|
{
|
|
"id": "shell-hard",
|
|
"category": "shell",
|
|
"tier": "hard",
|
|
"result": "FAIL",
|
|
"secs": 48.7,
|
|
"summary": "write_file \"conf_count.txt\" \"$(find /etc -type f -name \"*.conf\" | wc -l)\" \u2502\u2502 \u2502 \u2514\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500",
|
|
"verify_out": "cat: /root/conf_count.txt: No such file or directory"
|
|
},
|
|
{
|
|
"id": "code-easy",
|
|
"category": "code",
|
|
"tier": "easy",
|
|
"result": "FAIL",
|
|
"secs": 46.7,
|
|
"summary": "write_file \"add.py\" \"#!/usr/bin/env python3\\nprint(2 + 3)\"",
|
|
"verify_out": "cat: /root/add.out: No such file or directory"
|
|
},
|
|
{
|
|
"id": "code-medium",
|
|
"category": "code",
|
|
"tier": "medium",
|
|
"result": "FAIL",
|
|
"secs": 56.8,
|
|
"summary": "write_file \"greet.sh\" \"#!/bin/bash\\necho 'Hello, World!'\"",
|
|
"verify_out": ""
|
|
},
|
|
{
|
|
"id": "code-hard",
|
|
"category": "code",
|
|
"tier": "hard",
|
|
"result": "FAIL",
|
|
"secs": 72.8,
|
|
"summary": "",
|
|
"verify_out": "cat: /root/fib.out: No such file or directory"
|
|
},
|
|
{
|
|
"id": "git-easy",
|
|
"category": "git",
|
|
"tier": "easy",
|
|
"result": "FAIL",
|
|
"secs": 44.7,
|
|
"summary": "read_file README.md \u2502\u2502 \u2502 \u2514\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500",
|
|
"verify_out": ""
|
|
},
|
|
{
|
|
"id": "git-medium",
|
|
"category": "git",
|
|
"tier": "medium",
|
|
"result": "FAIL",
|
|
"secs": 44.7,
|
|
"summary": "run_shell \"git clone https://github.com/octocat/Hello-World hw2\"",
|
|
"verify_out": "grep: /root/readme.txt: No such file or directory"
|
|
},
|
|
{
|
|
"id": "git-hard",
|
|
"category": "git",
|
|
"tier": "hard",
|
|
"result": "FAIL",
|
|
"secs": 50.7,
|
|
"summary": "git clone https://github.com/octocat/Hello-World hw3",
|
|
"verify_out": "sh: 1: cannot open /root/branch.txt: No such file"
|
|
},
|
|
{
|
|
"id": "multi-easy",
|
|
"category": "multi",
|
|
"tier": "easy",
|
|
"result": "FAIL",
|
|
"secs": 44.7,
|
|
"summary": "ls proj \u2502\u2502 \u2502 \u2514\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500",
|
|
"verify_out": "cat: /root/proj/notes.txt: No such file or directory"
|
|
},
|
|
{
|
|
"id": "multi-medium",
|
|
"category": "multi",
|
|
"tier": "medium",
|
|
"result": "PASS",
|
|
"secs": 56.8,
|
|
"summary": "",
|
|
"verify_out": ""
|
|
},
|
|
{
|
|
"id": "multi-hard",
|
|
"category": "multi",
|
|
"tier": "hard",
|
|
"result": "FAIL",
|
|
"secs": 70.8,
|
|
"summary": "read_file wc.out \u2502\u2502 \u2502 \u2514\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500",
|
|
"verify_out": "sh: 1: cannot open /root/wc.out: No such file"
|
|
}
|
|
]
|
|
} |