test(ai): add 25-query benchmark harness + results for ai mode models

Adds a repeatable AI-mode benchmark (ai-benchmark.sh) covering 25 queries across
8 categories and 3 difficulty tiers, plus captured latency results and manual
correctness grades for qwen2.5-coder:1.5b, qwen2.5:3b, and westenfelder/NL2SH.

Headline finding: on the larger suite NL2SH is most accurate (72% vs coder 40%,
3b 32%) and the only model with non-zero hard-task accuracy (50% vs coder 0%),
while qwen2.5-coder is ~2x faster (median 3.0s vs 6.3s). This reverses the
earlier 7-query result that favored qwen-coder overall.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>

tests/performance/ai-benchmark.sh

@@ -0,0 +1,73 @@
+#!/bin/bash
+# AI mode benchmark harness for super-man.
+# Runs a fixed query suite against each model and records latency + the
+# generated command to a pipe-delimited CSV for analysis/reporting.
+#
+# Usage: ./ai-benchmark.sh [model1 model2 ...]
+# Default models: qwen2.5-coder:1.5b qwen2.5:3b westenfelder/NL2SH
+#
+# Output: tests/performance/ai-results.csv  (model|id|category|difficulty|query|latency_s|command)
+
+set -u
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+SUPERMAN="$SCRIPT_DIR/../../super-man.sh"
+OUT="$SCRIPT_DIR/ai-results.csv"
+
+MODELS=("$@")
+if [ ${#MODELS[@]} -eq 0 ]; then
+  MODELS=("qwen2.5-coder:1.5b" "qwen2.5:3b" "westenfelder/NL2SH")
+fi
+
+# Suite: id|category|difficulty|query
+SUITE=(
+  "q01|files|easy|list the 5 largest files in the current directory"
+  "q02|files|easy|show hidden files in long format"
+  "q03|files|medium|find files larger than 100MB under the home directory"
+  "q04|files|medium|recursively change permissions of all directories to 755"
+  "q05|files|hard|find duplicate files by content in the current tree"
+  "q06|text|easy|count the number of lines in all python files"
+  "q07|text|medium|replace all tabs with two spaces in config.txt in place"
+  "q08|text|medium|print the 10 most common words in access.log"
+  "q09|text|hard|sum the numbers in the third column of data.csv"
+  "q10|search|easy|recursively search for the word TODO in all js files"
+  "q11|search|medium|find files modified in the last 24 hours"
+  "q12|search|hard|find python files that import requests but not os"
+  "q13|process|easy|show the top 10 processes by memory usage"
+  "q14|process|medium|kill the process listening on port 8080"
+  "q15|process|hard|show the total cpu time used by all chrome processes"
+  "q16|network|easy|download a file from a url and save it as out.bin"
+  "q17|network|medium|list all open listening tcp ports with the owning process"
+  "q18|network|hard|test whether port 443 is open on example.com"
+  "q19|archive|easy|compress the folder logs into a gzip tarball"
+  "q20|archive|medium|extract only txt files from archive.tar.gz"
+  "q21|system|easy|show disk usage of the current directory in human readable form"
+  "q22|system|medium|show the 5 largest directories under /var"
+  "q23|git|easy|show the last 5 git commits in one line each"
+  "q24|git|medium|list files changed in the last commit"
+  "q25|git|hard|find the commit that introduced the string getUserById"
+)
+
+strip() { sed -E 's/\x1b\[[0-9;]*m//g'; }
+
+echo "model|id|category|difficulty|query|latency_s|command" > "$OUT"
+
+for m in "${MODELS[@]}"; do
+  echo ">> Warming up $m ..."
+  ollama run "$m" "echo hi" >/dev/null 2>&1
+  echo ">> Benchmarking $m (${#SUITE[@]} queries)"
+  for row in "${SUITE[@]}"; do
+    IFS='|' read -r id cat diff q <<< "$row"
+    start=$(date +%s.%N)
+    cmd=$("$SUPERMAN" ai --model "$m" "$q" 2>&1 | strip \
+          | awk '/Generated Command:/{getline; gsub(/^[[:space:]]+/,""); print; exit}')
+    end=$(date +%s.%N)
+    el=$(awk "BEGIN{printf \"%.2f\", $end-$start}")
+    # squash any embedded pipes/newlines in the captured command for CSV safety
+    cmd=$(printf '%s' "$cmd" | tr '\n' ' ' | sed 's/|/¦/g')
+    printf "%s|%s|%s|%s|%s|%s|%s\n" "$m" "$id" "$cat" "$diff" "$q" "$el" "$cmd" >> "$OUT"
+    printf "  [%s] %-5s %5ss  %s\n" "$m" "$id" "$el" "$cmd"
+  done
+done
+
+echo ""
+echo "Done -> $OUT"

tests/performance/ai-results.csv

@@ -0,0 +1,76 @@
+model|id|category|difficulty|query|latency_s|command
+qwen2.5-coder:1.5b|q01|files|easy|list the 5 largest files in the current directory|9.42|ls -lh ¦ tail -n 5
+qwen2.5-coder:1.5b|q02|files|easy|show hidden files in long format|5.12|ls -al
+qwen2.5-coder:1.5b|q03|files|medium|find files larger than 100MB under the home directory|17.79|find ~ -type f -size +100M -exec du {} \; ¦ grep -B1 "100M" ¦¦ echo "No files found larger than 100MB."
+qwen2.5-coder:1.5b|q04|files|medium|recursively change permissions of all directories to 755|8.63|find . -type d -exec chmod 755 {} +
+qwen2.5-coder:1.5b|q05|files|hard|find duplicate files by content in the current tree|14.29|find . -type f -exec md5sum {} + ¦ sort -k1,1 ¦ uniq -d ¦ xargs --no-run-if-empty md5sum -c
+qwen2.5-coder:1.5b|q06|text|easy|count the number of lines in all python files|3.03|find . -name "*.py" ¦ xargs wc -l
+qwen2.5-coder:1.5b|q07|text|medium|replace all tabs with two spaces in config.txt in place|2.82|sed 's/	/  /g' config.txt > temp && mv temp config.txt
+qwen2.5-coder:1.5b|q08|text|medium|print the 10 most common words in access.log|4.52|awk '{print $7}' access.log ¦ tr -cd '[:alnum:]' ¦ sort ¦ uniq -c ¦ sort -nr ¦ head -n 10
+qwen2.5-coder:1.5b|q09|text|hard|sum the numbers in the third column of data.csv|2.23|awk 'NR>2{print $3}' data.csv ¦ bc
+qwen2.5-coder:1.5b|q10|search|easy|recursively search for the word TODO in all js files|3.43|find . -name "*.js" -exec grep -H "TODO" {} \;
+qwen2.5-coder:1.5b|q11|search|medium|find files modified in the last 24 hours|1.87|find . -type f -mtime -1
+qwen2.5-coder:1.5b|q12|search|hard|find python files that import requests but not os|4.06|find . -name '*.py' -exec grep -E 'import requests¦from requests import.*' {} \; -not -path '*/os.py'
+qwen2.5-coder:1.5b|q13|process|easy|show the top 10 processes by memory usage|2.50|ps aux ¦ sort -nr -k4 ¦ head -n10
+qwen2.5-coder:1.5b|q14|process|medium|kill the process listening on port 8080|2.82|kill -9 $(ss -tuln ¦ grep :8080 ¦ awk '{print $7}')
+qwen2.5-coder:1.5b|q15|process|hard|show the total cpu time used by all chrome processes|3.39|ps -e -o pcpu= --no-headers ¦ sort -hr ¦ head -n 10
+qwen2.5-coder:1.5b|q16|network|easy|download a file from a url and save it as out.bin|2.51|curl -O https://example.com/out.bin
+qwen2.5-coder:1.5b|q17|network|medium|list all open listening tcp ports with the owning process|9.46|netstat -tuln ¦ awk '$5 ~ /^/ {print $1, $3}'
+qwen2.5-coder:1.5b|q18|network|hard|test whether port 443 is open on example.com|4.61|echo -e "Checking if port 443 is open on www.example.com..."
+qwen2.5-coder:1.5b|q19|archive|easy|compress the folder logs into a gzip tarball|1.74|tar czvf logs.tar.gz logs/
+qwen2.5-coder:1.5b|q20|archive|medium|extract only txt files from archive.tar.gz|5.12|tar -xzvf archive.tar.gz -T ¦ grep "\.(txt)$"
+qwen2.5-coder:1.5b|q21|system|easy|show disk usage of the current directory in human readable form|1.95|du -h
+qwen2.5-coder:1.5b|q22|system|medium|show the 5 largest directories under /var|2.16|du -lh ¦ sort -rh ¦ head -n 5
+qwen2.5-coder:1.5b|q23|git|easy|show the last 5 git commits in one line each|1.74|git log -n 5
+qwen2.5-coder:1.5b|q24|git|medium|list files changed in the last commit|1.96|git diff --name-only HEAD~1
+qwen2.5-coder:1.5b|q25|git|hard|find the commit that introduced the string getUserById|2.59|git log --diff-filter=A --grep 'getUserById' ¦ head -n 1
+qwen2.5:3b|q01|files|easy|list the 5 largest files in the current directory|5.13|ls -lSr ¦ head -n 5
+qwen2.5:3b|q02|files|easy|show hidden files in long format|1.82|ls -alh .*
+qwen2.5:3b|q03|files|medium|find files larger than 100MB under the home directory|3.34|find ~ -type f -size +100M
+qwen2.5:3b|q04|files|medium|recursively change permissions of all directories to 755|2.82|chmod -R 755 *
+qwen2.5:3b|q05|files|hard|find duplicate files by content in the current tree|7.89|find . -type f -exec file --mime-type {} + ¦ awk '{print $NF}' ¦ sort ¦ uniq -d ¦ xargs -I {} find . -type f -name '{}'
+qwen2.5:3b|q06|text|easy|count the number of lines in all python files|3.83|find . -name "*.py" -exec grep -c . {} \;
+qwen2.5:3b|q07|text|medium|replace all tabs with two spaces in config.txt in place|3.76|sed -i 's/	/  /g' config.txt
+qwen2.5:3b|q08|text|medium|print the 10 most common words in access.log|6.96|grep -o '[^ ]\+' access.log ¦ sort ¦ uniq -c ¦ sort -nr ¦ head -n 10
+qwen2.5:3b|q09|text|hard|sum the numbers in the third column of data.csv|5.63|bash
+qwen2.5:3b|q10|search|easy|recursively search for the word TODO in all js files|15.53|find . -type f -name "*.js" -exec grep -l "TODO" {} \;
+qwen2.5:3b|q11|search|medium|find files modified in the last 24 hours|16.56|find . -type f -mtime -1 -exec ls -l {} \;
+qwen2.5:3b|q12|search|hard|find python files that import requests but not os|18.96|find . -name "*.py" -exec pygrep -l "import requests" {} \; ¦ xargs -I{} grep -ls "{}" *os*
+qwen2.5:3b|q13|process|easy|show the top 10 processes by memory usage|11.50|ps -eo pid,size,%mem,comm ¦ sort -k3nr ¦ head -n 10
+qwen2.5:3b|q14|process|medium|kill the process listening on port 8080|8.46|kill $(lsof -i :8080 -t)
+qwen2.5:3b|q15|process|hard|show the total cpu time used by all chrome processes|11.70|ps -eo pid,lstart,time ¦ grep chrome ¦ awk '{total+=$8} END {print total}'
+qwen2.5:3b|q16|network|easy|download a file from a url and save it as out.bin|5.86|wget -O out.bin "url"
+qwen2.5:3b|q17|network|medium|list all open listening tcp ports with the owning process|11.28|netstat -tuln ¦ awk '{print $4}' ¦ sort ¦ uniq -c
+qwen2.5:3b|q18|network|hard|test whether port 443 is open on example.com|8.30|nc -vz example.com 443
+qwen2.5:3b|q19|archive|easy|compress the folder logs into a gzip tarball|9.95|gzip -cvf logs/*.txt ¦ cat > logs_$(date +%Y%m%d).tar.gz
+qwen2.5:3b|q20|archive|medium|extract only txt files from archive.tar.gz|6.20|tar -zxf archive.tar.gz --wildcards '*/.*'
+qwen2.5:3b|q21|system|easy|show disk usage of the current directory in human readable form|2.19|du -sh * ..
+qwen2.5:3b|q22|system|medium|show the 5 largest directories under /var|4.83|find /var -type d -printf '%b	%p
+qwen2.5:3b|q23|git|easy|show the last 5 git commits in one line each|3.59|git log -5 --oneline --format='%H %ad ¦ %s'
+qwen2.5:3b|q24|git|medium|list files changed in the last commit|3.30|git log -1 --format="%B" --abbrev-commit ¦ wc -m
+qwen2.5:3b|q25|git|hard|find the commit that introduced the string getUserById|3.45|git log -p --diff-filter=A --summary --grep='getUserById'
+westenfelder/NL2SH|q01|files|easy|list the 5 largest files in the current directory|3.96|ls -Sh ¦ head
+westenfelder/NL2SH|q02|files|easy|show hidden files in long format|3.09|ls -la .*
+westenfelder/NL2SH|q03|files|medium|find files larger than 100MB under the home directory|5.51|find ~/ -size +100M
+westenfelder/NL2SH|q04|files|medium|recursively change permissions of all directories to 755|8.50|find . -type d -print0 ¦ xargs -0 chmod 755
+westenfelder/NL2SH|q05|files|hard|find duplicate files by content in the current tree|4.09|fdupes -r .
+westenfelder/NL2SH|q06|text|easy|count the number of lines in all python files|7.06|cat $(find . -name "*.py") ¦ wc -l
+westenfelder/NL2SH|q07|text|medium|replace all tabs with two spaces in config.txt in place|7.49|sed -i 's/	/  /' config.txt
+westenfelder/NL2SH|q08|text|medium|print the 10 most common words in access.log|17.90|cut -d' ' -f4-5 access.log ¦ tr A-Z a-z ¦ awk '{print $2}'¦ sort ¦ uniq -c ¦ sort -nr ¦ head -n10;
+westenfelder/NL2SH|q09|text|hard|sum the numbers in the third column of data.csv|17.42|awk -F, 'BEGIN{srand();}{print rand() "	" $3}' <(sort -R data.csv) ¦ awk '{sum+=$2}END{print sum}'
+westenfelder/NL2SH|q10|search|easy|recursively search for the word TODO in all js files|7.72|find . -name '*.js' ¦ xargs grep 'TODO'
+westenfelder/NL2SH|q11|search|medium|find files modified in the last 24 hours|4.20|find . -mtime -1
+westenfelder/NL2SH|q12|search|hard|find python files that import requests but not os|16.41|find . -type f \( -name "*.py" \) -exec grep -q "import requests" {} \; -a -not -exec grep -q "import os" {} \; -print
+westenfelder/NL2SH|q13|process|easy|show the top 10 processes by memory usage|8.90|ps -eo pid,ppid,%mem,cmd ¦ sort -k3nr ¦ head
+westenfelder/NL2SH|q14|process|medium|kill the process listening on port 8080|5.71|fuser -k 8080/tcp
+westenfelder/NL2SH|q15|process|hard|show the total cpu time used by all chrome processes|13.11|top -b -n 1 ¦ grep Chrome ¦ awk '{total+=$9} END {print "Total CPU Time: ", total, "s"}'
+westenfelder/NL2SH|q16|network|easy|download a file from a url and save it as out.bin|6.18|curl -o out.bin http://example.com/file.bin
+westenfelder/NL2SH|q17|network|medium|list all open listening tcp ports with the owning process|7.38|lsof -i TCP -sTCP:LISTEN ¦ grep LISTEN
+westenfelder/NL2SH|q18|network|hard|test whether port 443 is open on example.com|5.87|nc -zw1 example.com 443
+westenfelder/NL2SH|q19|archive|easy|compress the folder logs into a gzip tarball|5.01|tar -czf logs.tar.gz logs/
+westenfelder/NL2SH|q20|archive|medium|extract only txt files from archive.tar.gz|6.32|tar -zxvf archive.tar.gz --wildcards '*.txt'
+westenfelder/NL2SH|q21|system|easy|show disk usage of the current directory in human readable form|3.55|du -h .
+westenfelder/NL2SH|q22|system|medium|show the 5 largest directories under /var|7.51|du -a /var ¦ sort -n -r ¦ head -n 5
+westenfelder/NL2SH|q23|git|easy|show the last 5 git commits in one line each|4.88|git log -n 5 --oneline
+westenfelder/NL2SH|q24|git|medium|list files changed in the last commit|3.85|git whatchanged -1
+westenfelder/NL2SH|q25|git|hard|find the commit that introduced the string getUserById|13.61|git rev-list --first-parent HEAD ¦ xargs git show -s --pretty=format:%H ¦ grep 'getUserById' ¦ tail -n1

tests/performance/ai-scores.csv

@@ -0,0 +1,76 @@
+model|id|difficulty|verdict
+qwen2.5-coder:1.5b|q01|easy|wrong
+qwen2.5-coder:1.5b|q02|easy|correct
+qwen2.5-coder:1.5b|q03|medium|partial
+qwen2.5-coder:1.5b|q04|medium|correct
+qwen2.5-coder:1.5b|q05|hard|wrong
+qwen2.5-coder:1.5b|q06|easy|correct
+qwen2.5-coder:1.5b|q07|medium|correct
+qwen2.5-coder:1.5b|q08|medium|wrong
+qwen2.5-coder:1.5b|q09|hard|wrong
+qwen2.5-coder:1.5b|q10|easy|correct
+qwen2.5-coder:1.5b|q11|medium|correct
+qwen2.5-coder:1.5b|q12|hard|wrong
+qwen2.5-coder:1.5b|q13|easy|correct
+qwen2.5-coder:1.5b|q14|medium|wrong
+qwen2.5-coder:1.5b|q15|hard|wrong
+qwen2.5-coder:1.5b|q16|easy|partial
+qwen2.5-coder:1.5b|q17|medium|wrong
+qwen2.5-coder:1.5b|q18|hard|wrong
+qwen2.5-coder:1.5b|q19|easy|correct
+qwen2.5-coder:1.5b|q20|medium|wrong
+qwen2.5-coder:1.5b|q21|easy|correct
+qwen2.5-coder:1.5b|q22|medium|wrong
+qwen2.5-coder:1.5b|q23|easy|partial
+qwen2.5-coder:1.5b|q24|medium|correct
+qwen2.5-coder:1.5b|q25|hard|wrong
+qwen2.5:3b|q01|easy|wrong
+qwen2.5:3b|q02|easy|partial
+qwen2.5:3b|q03|medium|correct
+qwen2.5:3b|q04|medium|wrong
+qwen2.5:3b|q05|hard|wrong
+qwen2.5:3b|q06|easy|wrong
+qwen2.5:3b|q07|medium|correct
+qwen2.5:3b|q08|medium|correct
+qwen2.5:3b|q09|hard|wrong
+qwen2.5:3b|q10|easy|partial
+qwen2.5:3b|q11|medium|correct
+qwen2.5:3b|q12|hard|wrong
+qwen2.5:3b|q13|easy|correct
+qwen2.5:3b|q14|medium|correct
+qwen2.5:3b|q15|hard|wrong
+qwen2.5:3b|q16|easy|correct
+qwen2.5:3b|q17|medium|wrong
+qwen2.5:3b|q18|hard|correct
+qwen2.5:3b|q19|easy|wrong
+qwen2.5:3b|q20|medium|wrong
+qwen2.5:3b|q21|easy|partial
+qwen2.5:3b|q22|medium|wrong
+qwen2.5:3b|q23|easy|partial
+qwen2.5:3b|q24|medium|wrong
+qwen2.5:3b|q25|hard|wrong
+westenfelder/NL2SH|q01|easy|correct
+westenfelder/NL2SH|q02|easy|partial
+westenfelder/NL2SH|q03|medium|correct
+westenfelder/NL2SH|q04|medium|correct
+westenfelder/NL2SH|q05|hard|correct
+westenfelder/NL2SH|q06|easy|correct
+westenfelder/NL2SH|q07|medium|partial
+westenfelder/NL2SH|q08|medium|wrong
+westenfelder/NL2SH|q09|hard|wrong
+westenfelder/NL2SH|q10|easy|correct
+westenfelder/NL2SH|q11|medium|correct
+westenfelder/NL2SH|q12|hard|correct
+westenfelder/NL2SH|q13|easy|correct
+westenfelder/NL2SH|q14|medium|correct
+westenfelder/NL2SH|q15|hard|wrong
+westenfelder/NL2SH|q16|easy|correct
+westenfelder/NL2SH|q17|medium|correct
+westenfelder/NL2SH|q18|hard|correct
+westenfelder/NL2SH|q19|easy|correct
+westenfelder/NL2SH|q20|medium|correct
+westenfelder/NL2SH|q21|easy|correct
+westenfelder/NL2SH|q22|medium|partial
+westenfelder/NL2SH|q23|easy|correct
+westenfelder/NL2SH|q24|medium|correct
+westenfelder/NL2SH|q25|hard|wrong