From 220fc74eb6ef9b263a80d6ae3f8947916a9aa06c Mon Sep 17 00:00:00 2001 From: leetcrypt Date: Tue, 9 Jun 2026 15:53:24 -0700 Subject: [PATCH] test(ai): add 25-query benchmark harness + results for ai mode models Adds a repeatable AI-mode benchmark (ai-benchmark.sh) covering 25 queries across 8 categories and 3 difficulty tiers, plus captured latency results and manual correctness grades for qwen2.5-coder:1.5b, qwen2.5:3b, and westenfelder/NL2SH. Headline finding: on the larger suite NL2SH is most accurate (72% vs coder 40%, 3b 32%) and the only model with non-zero hard-task accuracy (50% vs coder 0%), while qwen2.5-coder is ~2x faster (median 3.0s vs 6.3s). This reverses the earlier 7-query result that favored qwen-coder overall. Co-Authored-By: Claude Opus 4.6 --- tests/performance/ai-benchmark.sh | 73 +++++++++++++++++++++++++++++ tests/performance/ai-results.csv | 76 +++++++++++++++++++++++++++++++ tests/performance/ai-scores.csv | 76 +++++++++++++++++++++++++++++++ 3 files changed, 225 insertions(+) create mode 100755 tests/performance/ai-benchmark.sh create mode 100644 tests/performance/ai-results.csv create mode 100644 tests/performance/ai-scores.csv diff --git a/tests/performance/ai-benchmark.sh b/tests/performance/ai-benchmark.sh new file mode 100755 index 0000000..0e0a699 --- /dev/null +++ b/tests/performance/ai-benchmark.sh @@ -0,0 +1,73 @@ +#!/bin/bash +# AI mode benchmark harness for super-man. +# Runs a fixed query suite against each model and records latency + the +# generated command to a pipe-delimited CSV for analysis/reporting. +# +# Usage: ./ai-benchmark.sh [model1 model2 ...] +# Default models: qwen2.5-coder:1.5b qwen2.5:3b westenfelder/NL2SH +# +# Output: tests/performance/ai-results.csv (model|id|category|difficulty|query|latency_s|command) + +set -u +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +SUPERMAN="$SCRIPT_DIR/../../super-man.sh" +OUT="$SCRIPT_DIR/ai-results.csv" + +MODELS=("$@") +if [ ${#MODELS[@]} -eq 0 ]; then + MODELS=("qwen2.5-coder:1.5b" "qwen2.5:3b" "westenfelder/NL2SH") +fi + +# Suite: id|category|difficulty|query +SUITE=( + "q01|files|easy|list the 5 largest files in the current directory" + "q02|files|easy|show hidden files in long format" + "q03|files|medium|find files larger than 100MB under the home directory" + "q04|files|medium|recursively change permissions of all directories to 755" + "q05|files|hard|find duplicate files by content in the current tree" + "q06|text|easy|count the number of lines in all python files" + "q07|text|medium|replace all tabs with two spaces in config.txt in place" + "q08|text|medium|print the 10 most common words in access.log" + "q09|text|hard|sum the numbers in the third column of data.csv" + "q10|search|easy|recursively search for the word TODO in all js files" + "q11|search|medium|find files modified in the last 24 hours" + "q12|search|hard|find python files that import requests but not os" + "q13|process|easy|show the top 10 processes by memory usage" + "q14|process|medium|kill the process listening on port 8080" + "q15|process|hard|show the total cpu time used by all chrome processes" + "q16|network|easy|download a file from a url and save it as out.bin" + "q17|network|medium|list all open listening tcp ports with the owning process" + "q18|network|hard|test whether port 443 is open on example.com" + "q19|archive|easy|compress the folder logs into a gzip tarball" + "q20|archive|medium|extract only txt files from archive.tar.gz" + "q21|system|easy|show disk usage of the current directory in human readable form" + "q22|system|medium|show the 5 largest directories under /var" + "q23|git|easy|show the last 5 git commits in one line each" + "q24|git|medium|list files changed in the last commit" + "q25|git|hard|find the commit that introduced the string getUserById" +) + +strip() { sed -E 's/\x1b\[[0-9;]*m//g'; } + +echo "model|id|category|difficulty|query|latency_s|command" > "$OUT" + +for m in "${MODELS[@]}"; do + echo ">> Warming up $m ..." + ollama run "$m" "echo hi" >/dev/null 2>&1 + echo ">> Benchmarking $m (${#SUITE[@]} queries)" + for row in "${SUITE[@]}"; do + IFS='|' read -r id cat diff q <<< "$row" + start=$(date +%s.%N) + cmd=$("$SUPERMAN" ai --model "$m" "$q" 2>&1 | strip \ + | awk '/Generated Command:/{getline; gsub(/^[[:space:]]+/,""); print; exit}') + end=$(date +%s.%N) + el=$(awk "BEGIN{printf \"%.2f\", $end-$start}") + # squash any embedded pipes/newlines in the captured command for CSV safety + cmd=$(printf '%s' "$cmd" | tr '\n' ' ' | sed 's/|/¦/g') + printf "%s|%s|%s|%s|%s|%s|%s\n" "$m" "$id" "$cat" "$diff" "$q" "$el" "$cmd" >> "$OUT" + printf " [%s] %-5s %5ss %s\n" "$m" "$id" "$el" "$cmd" + done +done + +echo "" +echo "Done -> $OUT" diff --git a/tests/performance/ai-results.csv b/tests/performance/ai-results.csv new file mode 100644 index 0000000..2c200bb --- /dev/null +++ b/tests/performance/ai-results.csv @@ -0,0 +1,76 @@ +model|id|category|difficulty|query|latency_s|command +qwen2.5-coder:1.5b|q01|files|easy|list the 5 largest files in the current directory|9.42|ls -lh ¦ tail -n 5 +qwen2.5-coder:1.5b|q02|files|easy|show hidden files in long format|5.12|ls -al +qwen2.5-coder:1.5b|q03|files|medium|find files larger than 100MB under the home directory|17.79|find ~ -type f -size +100M -exec du {} \; ¦ grep -B1 "100M" ¦¦ echo "No files found larger than 100MB." +qwen2.5-coder:1.5b|q04|files|medium|recursively change permissions of all directories to 755|8.63|find . -type d -exec chmod 755 {} + +qwen2.5-coder:1.5b|q05|files|hard|find duplicate files by content in the current tree|14.29|find . -type f -exec md5sum {} + ¦ sort -k1,1 ¦ uniq -d ¦ xargs --no-run-if-empty md5sum -c +qwen2.5-coder:1.5b|q06|text|easy|count the number of lines in all python files|3.03|find . -name "*.py" ¦ xargs wc -l +qwen2.5-coder:1.5b|q07|text|medium|replace all tabs with two spaces in config.txt in place|2.82|sed 's/ / /g' config.txt > temp && mv temp config.txt +qwen2.5-coder:1.5b|q08|text|medium|print the 10 most common words in access.log|4.52|awk '{print $7}' access.log ¦ tr -cd '[:alnum:]' ¦ sort ¦ uniq -c ¦ sort -nr ¦ head -n 10 +qwen2.5-coder:1.5b|q09|text|hard|sum the numbers in the third column of data.csv|2.23|awk 'NR>2{print $3}' data.csv ¦ bc +qwen2.5-coder:1.5b|q10|search|easy|recursively search for the word TODO in all js files|3.43|find . -name "*.js" -exec grep -H "TODO" {} \; +qwen2.5-coder:1.5b|q11|search|medium|find files modified in the last 24 hours|1.87|find . -type f -mtime -1 +qwen2.5-coder:1.5b|q12|search|hard|find python files that import requests but not os|4.06|find . -name '*.py' -exec grep -E 'import requests¦from requests import.*' {} \; -not -path '*/os.py' +qwen2.5-coder:1.5b|q13|process|easy|show the top 10 processes by memory usage|2.50|ps aux ¦ sort -nr -k4 ¦ head -n10 +qwen2.5-coder:1.5b|q14|process|medium|kill the process listening on port 8080|2.82|kill -9 $(ss -tuln ¦ grep :8080 ¦ awk '{print $7}') +qwen2.5-coder:1.5b|q15|process|hard|show the total cpu time used by all chrome processes|3.39|ps -e -o pcpu= --no-headers ¦ sort -hr ¦ head -n 10 +qwen2.5-coder:1.5b|q16|network|easy|download a file from a url and save it as out.bin|2.51|curl -O https://example.com/out.bin +qwen2.5-coder:1.5b|q17|network|medium|list all open listening tcp ports with the owning process|9.46|netstat -tuln ¦ awk '$5 ~ /^/ {print $1, $3}' +qwen2.5-coder:1.5b|q18|network|hard|test whether port 443 is open on example.com|4.61|echo -e "Checking if port 443 is open on www.example.com..." +qwen2.5-coder:1.5b|q19|archive|easy|compress the folder logs into a gzip tarball|1.74|tar czvf logs.tar.gz logs/ +qwen2.5-coder:1.5b|q20|archive|medium|extract only txt files from archive.tar.gz|5.12|tar -xzvf archive.tar.gz -T ¦ grep "\.(txt)$" +qwen2.5-coder:1.5b|q21|system|easy|show disk usage of the current directory in human readable form|1.95|du -h +qwen2.5-coder:1.5b|q22|system|medium|show the 5 largest directories under /var|2.16|du -lh ¦ sort -rh ¦ head -n 5 +qwen2.5-coder:1.5b|q23|git|easy|show the last 5 git commits in one line each|1.74|git log -n 5 +qwen2.5-coder:1.5b|q24|git|medium|list files changed in the last commit|1.96|git diff --name-only HEAD~1 +qwen2.5-coder:1.5b|q25|git|hard|find the commit that introduced the string getUserById|2.59|git log --diff-filter=A --grep 'getUserById' ¦ head -n 1 +qwen2.5:3b|q01|files|easy|list the 5 largest files in the current directory|5.13|ls -lSr ¦ head -n 5 +qwen2.5:3b|q02|files|easy|show hidden files in long format|1.82|ls -alh .* +qwen2.5:3b|q03|files|medium|find files larger than 100MB under the home directory|3.34|find ~ -type f -size +100M +qwen2.5:3b|q04|files|medium|recursively change permissions of all directories to 755|2.82|chmod -R 755 * +qwen2.5:3b|q05|files|hard|find duplicate files by content in the current tree|7.89|find . -type f -exec file --mime-type {} + ¦ awk '{print $NF}' ¦ sort ¦ uniq -d ¦ xargs -I {} find . -type f -name '{}' +qwen2.5:3b|q06|text|easy|count the number of lines in all python files|3.83|find . -name "*.py" -exec grep -c . {} \; +qwen2.5:3b|q07|text|medium|replace all tabs with two spaces in config.txt in place|3.76|sed -i 's/ / /g' config.txt +qwen2.5:3b|q08|text|medium|print the 10 most common words in access.log|6.96|grep -o '[^ ]\+' access.log ¦ sort ¦ uniq -c ¦ sort -nr ¦ head -n 10 +qwen2.5:3b|q09|text|hard|sum the numbers in the third column of data.csv|5.63|bash +qwen2.5:3b|q10|search|easy|recursively search for the word TODO in all js files|15.53|find . -type f -name "*.js" -exec grep -l "TODO" {} \; +qwen2.5:3b|q11|search|medium|find files modified in the last 24 hours|16.56|find . -type f -mtime -1 -exec ls -l {} \; +qwen2.5:3b|q12|search|hard|find python files that import requests but not os|18.96|find . -name "*.py" -exec pygrep -l "import requests" {} \; ¦ xargs -I{} grep -ls "{}" *os* +qwen2.5:3b|q13|process|easy|show the top 10 processes by memory usage|11.50|ps -eo pid,size,%mem,comm ¦ sort -k3nr ¦ head -n 10 +qwen2.5:3b|q14|process|medium|kill the process listening on port 8080|8.46|kill $(lsof -i :8080 -t) +qwen2.5:3b|q15|process|hard|show the total cpu time used by all chrome processes|11.70|ps -eo pid,lstart,time ¦ grep chrome ¦ awk '{total+=$8} END {print total}' +qwen2.5:3b|q16|network|easy|download a file from a url and save it as out.bin|5.86|wget -O out.bin "url" +qwen2.5:3b|q17|network|medium|list all open listening tcp ports with the owning process|11.28|netstat -tuln ¦ awk '{print $4}' ¦ sort ¦ uniq -c +qwen2.5:3b|q18|network|hard|test whether port 443 is open on example.com|8.30|nc -vz example.com 443 +qwen2.5:3b|q19|archive|easy|compress the folder logs into a gzip tarball|9.95|gzip -cvf logs/*.txt ¦ cat > logs_$(date +%Y%m%d).tar.gz +qwen2.5:3b|q20|archive|medium|extract only txt files from archive.tar.gz|6.20|tar -zxf archive.tar.gz --wildcards '*/.*' +qwen2.5:3b|q21|system|easy|show disk usage of the current directory in human readable form|2.19|du -sh * .. +qwen2.5:3b|q22|system|medium|show the 5 largest directories under /var|4.83|find /var -type d -printf '%b %p +qwen2.5:3b|q23|git|easy|show the last 5 git commits in one line each|3.59|git log -5 --oneline --format='%H %ad ¦ %s' +qwen2.5:3b|q24|git|medium|list files changed in the last commit|3.30|git log -1 --format="%B" --abbrev-commit ¦ wc -m +qwen2.5:3b|q25|git|hard|find the commit that introduced the string getUserById|3.45|git log -p --diff-filter=A --summary --grep='getUserById' +westenfelder/NL2SH|q01|files|easy|list the 5 largest files in the current directory|3.96|ls -Sh ¦ head +westenfelder/NL2SH|q02|files|easy|show hidden files in long format|3.09|ls -la .* +westenfelder/NL2SH|q03|files|medium|find files larger than 100MB under the home directory|5.51|find ~/ -size +100M +westenfelder/NL2SH|q04|files|medium|recursively change permissions of all directories to 755|8.50|find . -type d -print0 ¦ xargs -0 chmod 755 +westenfelder/NL2SH|q05|files|hard|find duplicate files by content in the current tree|4.09|fdupes -r . +westenfelder/NL2SH|q06|text|easy|count the number of lines in all python files|7.06|cat $(find . -name "*.py") ¦ wc -l +westenfelder/NL2SH|q07|text|medium|replace all tabs with two spaces in config.txt in place|7.49|sed -i 's/ / /' config.txt +westenfelder/NL2SH|q08|text|medium|print the 10 most common words in access.log|17.90|cut -d' ' -f4-5 access.log ¦ tr A-Z a-z ¦ awk '{print $2}'¦ sort ¦ uniq -c ¦ sort -nr ¦ head -n10; +westenfelder/NL2SH|q09|text|hard|sum the numbers in the third column of data.csv|17.42|awk -F, 'BEGIN{srand();}{print rand() " " $3}' <(sort -R data.csv) ¦ awk '{sum+=$2}END{print sum}' +westenfelder/NL2SH|q10|search|easy|recursively search for the word TODO in all js files|7.72|find . -name '*.js' ¦ xargs grep 'TODO' +westenfelder/NL2SH|q11|search|medium|find files modified in the last 24 hours|4.20|find . -mtime -1 +westenfelder/NL2SH|q12|search|hard|find python files that import requests but not os|16.41|find . -type f \( -name "*.py" \) -exec grep -q "import requests" {} \; -a -not -exec grep -q "import os" {} \; -print +westenfelder/NL2SH|q13|process|easy|show the top 10 processes by memory usage|8.90|ps -eo pid,ppid,%mem,cmd ¦ sort -k3nr ¦ head +westenfelder/NL2SH|q14|process|medium|kill the process listening on port 8080|5.71|fuser -k 8080/tcp +westenfelder/NL2SH|q15|process|hard|show the total cpu time used by all chrome processes|13.11|top -b -n 1 ¦ grep Chrome ¦ awk '{total+=$9} END {print "Total CPU Time: ", total, "s"}' +westenfelder/NL2SH|q16|network|easy|download a file from a url and save it as out.bin|6.18|curl -o out.bin http://example.com/file.bin +westenfelder/NL2SH|q17|network|medium|list all open listening tcp ports with the owning process|7.38|lsof -i TCP -sTCP:LISTEN ¦ grep LISTEN +westenfelder/NL2SH|q18|network|hard|test whether port 443 is open on example.com|5.87|nc -zw1 example.com 443 +westenfelder/NL2SH|q19|archive|easy|compress the folder logs into a gzip tarball|5.01|tar -czf logs.tar.gz logs/ +westenfelder/NL2SH|q20|archive|medium|extract only txt files from archive.tar.gz|6.32|tar -zxvf archive.tar.gz --wildcards '*.txt' +westenfelder/NL2SH|q21|system|easy|show disk usage of the current directory in human readable form|3.55|du -h . +westenfelder/NL2SH|q22|system|medium|show the 5 largest directories under /var|7.51|du -a /var ¦ sort -n -r ¦ head -n 5 +westenfelder/NL2SH|q23|git|easy|show the last 5 git commits in one line each|4.88|git log -n 5 --oneline +westenfelder/NL2SH|q24|git|medium|list files changed in the last commit|3.85|git whatchanged -1 +westenfelder/NL2SH|q25|git|hard|find the commit that introduced the string getUserById|13.61|git rev-list --first-parent HEAD ¦ xargs git show -s --pretty=format:%H ¦ grep 'getUserById' ¦ tail -n1 diff --git a/tests/performance/ai-scores.csv b/tests/performance/ai-scores.csv new file mode 100644 index 0000000..69588f9 --- /dev/null +++ b/tests/performance/ai-scores.csv @@ -0,0 +1,76 @@ +model|id|difficulty|verdict +qwen2.5-coder:1.5b|q01|easy|wrong +qwen2.5-coder:1.5b|q02|easy|correct +qwen2.5-coder:1.5b|q03|medium|partial +qwen2.5-coder:1.5b|q04|medium|correct +qwen2.5-coder:1.5b|q05|hard|wrong +qwen2.5-coder:1.5b|q06|easy|correct +qwen2.5-coder:1.5b|q07|medium|correct +qwen2.5-coder:1.5b|q08|medium|wrong +qwen2.5-coder:1.5b|q09|hard|wrong +qwen2.5-coder:1.5b|q10|easy|correct +qwen2.5-coder:1.5b|q11|medium|correct +qwen2.5-coder:1.5b|q12|hard|wrong +qwen2.5-coder:1.5b|q13|easy|correct +qwen2.5-coder:1.5b|q14|medium|wrong +qwen2.5-coder:1.5b|q15|hard|wrong +qwen2.5-coder:1.5b|q16|easy|partial +qwen2.5-coder:1.5b|q17|medium|wrong +qwen2.5-coder:1.5b|q18|hard|wrong +qwen2.5-coder:1.5b|q19|easy|correct +qwen2.5-coder:1.5b|q20|medium|wrong +qwen2.5-coder:1.5b|q21|easy|correct +qwen2.5-coder:1.5b|q22|medium|wrong +qwen2.5-coder:1.5b|q23|easy|partial +qwen2.5-coder:1.5b|q24|medium|correct +qwen2.5-coder:1.5b|q25|hard|wrong +qwen2.5:3b|q01|easy|wrong +qwen2.5:3b|q02|easy|partial +qwen2.5:3b|q03|medium|correct +qwen2.5:3b|q04|medium|wrong +qwen2.5:3b|q05|hard|wrong +qwen2.5:3b|q06|easy|wrong +qwen2.5:3b|q07|medium|correct +qwen2.5:3b|q08|medium|correct +qwen2.5:3b|q09|hard|wrong +qwen2.5:3b|q10|easy|partial +qwen2.5:3b|q11|medium|correct +qwen2.5:3b|q12|hard|wrong +qwen2.5:3b|q13|easy|correct +qwen2.5:3b|q14|medium|correct +qwen2.5:3b|q15|hard|wrong +qwen2.5:3b|q16|easy|correct +qwen2.5:3b|q17|medium|wrong +qwen2.5:3b|q18|hard|correct +qwen2.5:3b|q19|easy|wrong +qwen2.5:3b|q20|medium|wrong +qwen2.5:3b|q21|easy|partial +qwen2.5:3b|q22|medium|wrong +qwen2.5:3b|q23|easy|partial +qwen2.5:3b|q24|medium|wrong +qwen2.5:3b|q25|hard|wrong +westenfelder/NL2SH|q01|easy|correct +westenfelder/NL2SH|q02|easy|partial +westenfelder/NL2SH|q03|medium|correct +westenfelder/NL2SH|q04|medium|correct +westenfelder/NL2SH|q05|hard|correct +westenfelder/NL2SH|q06|easy|correct +westenfelder/NL2SH|q07|medium|partial +westenfelder/NL2SH|q08|medium|wrong +westenfelder/NL2SH|q09|hard|wrong +westenfelder/NL2SH|q10|easy|correct +westenfelder/NL2SH|q11|medium|correct +westenfelder/NL2SH|q12|hard|correct +westenfelder/NL2SH|q13|easy|correct +westenfelder/NL2SH|q14|medium|correct +westenfelder/NL2SH|q15|hard|wrong +westenfelder/NL2SH|q16|easy|correct +westenfelder/NL2SH|q17|medium|correct +westenfelder/NL2SH|q18|hard|correct +westenfelder/NL2SH|q19|easy|correct +westenfelder/NL2SH|q20|medium|correct +westenfelder/NL2SH|q21|easy|correct +westenfelder/NL2SH|q22|medium|partial +westenfelder/NL2SH|q23|easy|correct +westenfelder/NL2SH|q24|medium|correct +westenfelder/NL2SH|q25|hard|wrong