From 99afff41c6039e533536658a27031fb8f3d98724 Mon Sep 17 00:00:00 2001 From: leetcrypt Date: Wed, 10 Jun 2026 13:34:21 -0700 Subject: [PATCH] =?UTF-8?q?feat(ai):=20split-tag=20tool-call=20recovery=20?= =?UTF-8?q?+=20greedy=20decode=20=E2=80=94=203x=20lift=20on=200.5b?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The first harness changes to move the benchmark off the 1/12 noise floor, on the model that needs it most (qwen2.5:0.5b: 1/12 -> 4/12, 3/12). - complete_with_tools now decodes the tool loop at temperature 0 (scoped; chat keeps default sampling). At Ollama's default 0.8 the weak model sampled away from the tool-call format into prose/fabrication; the nudge prompt changes between turns so temp 0 still escapes a failed state on retry. - Greedy decode made 0.5b's leak deterministic, exposing its real shape: not a JSON object with a name key, but the name in a tag and the args in a SEPARATE object — write_file{"path":…} — ~5 of 12 tasks/run. _NAMED_TAG pairs the tag-name with the following args object, gated on the known tool set so it still can't fabricate an action. - Bridge recovers a ```bash block narrated in prose as a run_shell call, non-destructive only (FENCE_DESTRUCTIVE guard); fires on prose-leak turns, no-op where the model emits structured calls. Ablation on 0.5b: structured-JSON-only 0/0 -> fenced+temp0 2/0 -> +split-tag 4/3. The lift is concentrated on the weakest model by design — a 3B emits proper calls and fails on capability/content (unchanged at 1/12), which no parser can fix. All recovery paths unit-checked for the positive shapes and the negatives (prose / unknown tool / destructive block) they must ignore. Co-Authored-By: Claude Opus 4.6 --- cmd_chat/agent/bridge.py | 49 +++++++++++++++++++ cmd_chat/agent/providers.py | 56 +++++++++++++++++----- docs/findings-native-harness-2026-06-10.md | 36 ++++++++++++++ 3 files changed, 128 insertions(+), 13 deletions(-) diff --git a/cmd_chat/agent/bridge.py b/cmd_chat/agent/bridge.py index 7dbd68b..f196083 100644 --- a/cmd_chat/agent/bridge.py +++ b/cmd_chat/agent/bridge.py @@ -182,6 +182,25 @@ NUDGE_FILLER = re.compile( r"\b(next step|proceed|let me|i['’]ll|i will|now i|first[,: ]|then i|" r"continuing|moving on|go ahead)\b|:\s*$", re.I) +# The weak CPU models' dominant leak is NOT a JSON tool-call in text (the provider +# already recovers those) — it is a shell command narrated inside a ```bash fence +# with NO structured call at all ("…let's proceed:\n```bash\nmkdir -p a/b/c\n```"). +# Recover the FIRST shell-tagged fence as a run_shell call so that intent isn't +# dropped. Only explicit shell tags (never an unlabelled ``` block, which is just as +# often example OUTPUT) and never a python/other-lang fence (it isn't a shell line). +FENCE_SHELL = re.compile(r"```(?:bash|sh|shell|zsh|console)\s*\n(.*?)```", re.I | re.S) + +# Refuse to auto-run a recovered block that looks destructive. The block came from +# model PROSE (possibly illustrative, not an action), so the bar to execute it is +# higher than for a structured call the model deliberately emitted. Matches stay +# coarse on purpose — on a hit we DON'T execute, we fall through to the nudge path. +FENCE_DESTRUCTIVE = re.compile( + r"\brm\s+-[a-z]*[rf][a-z]*\s+(?:-[a-z]+\s+)*(?:/|~|\$HOME|\*|\.)" + r"|\bmkfs\b|\bdd\s+if=|\b(?:shutdown|reboot|halt|poweroff|init\s+0)\b" + r"|:\s*\(\)\s*\{|>\s*/dev/|\bchmod\s+-R\s+0*777\s+/" + r"|\bmv\s+\S+\s+/dev/null\b|\b(?:wget|curl)\b[^\n]*\|\s*(?:ba)?sh\b", + re.I) + class AgentBridge(Client): def __init__(self, server: str, port: int, name: str, provider: Provider, @@ -707,6 +726,25 @@ class AgentBridge(Client): """Drop a leading `DONE:` marker so the chat summary reads cleanly.""" return re.sub(r"^\s*DONE:?\s*", "", text or "", flags=re.I).strip() + @staticmethod + def _recover_fenced_call(text: str) -> dict | None: + """Recover a run_shell call from a model turn that narrated the command in a + ```bash fence instead of emitting a structured tool call (the weak-CPU-model + leak the benchmark exposed). Returns a `{name,arguments}` run_shell call for + the FIRST shell-tagged fence whose command is non-empty and not destructive, + else None. Caller must already have ruled out a real call and a DONE signal. + The destructive guard is the safety gate: a prose block may be illustrative, + so anything matching FENCE_DESTRUCTIVE is refused (fall through to a nudge).""" + m = FENCE_SHELL.search(text or "") + if not m: + return None + cmd = m.group(1).strip() + # Drop a leading shell prompt sigil the model sometimes includes ("$ ls"). + cmd = re.sub(r"(?m)^\s*[$#]\s+", "", cmd).strip() + if not cmd or FENCE_DESTRUCTIVE.search(cmd): + return None + return {"name": "run_shell", "arguments": {"command": cmd}} + @classmethod def _completion_verdict(cls, text: str, did_action: bool, last_rc: int | None) -> str: """Disambiguate a text-only (no tool call) turn into 'done' vs 'stalled'. @@ -805,6 +843,17 @@ class AgentBridge(Client): return await self._send_typing(ws, False) + if not calls and not self._is_done_signal(text): + # The weak models' dominant failure is narrating the next command in a + # ```bash fence instead of calling run_shell. Recover that as a real + # call (non-destructive only) so the turn does an ACTION instead of + # tripping the "described but never ran a tool" stall. A DONE turn is + # excluded above so a fenced EXAMPLE in a completion isn't re-run. + fenced = self._recover_fenced_call(text) + if fenced is not None: + calls = [fenced] + await self._mirror_to_pty(ws, "▸ (recovered command from prose)") + if not calls: # A text-only turn is ambiguous: genuine completion, or a mid-task # stall ("ok, let's proceed…"). Resolve it (DONE: marker / unresolved diff --git a/cmd_chat/agent/providers.py b/cmd_chat/agent/providers.py index a0f07bb..b043c31 100644 --- a/cmd_chat/agent/providers.py +++ b/cmd_chat/agent/providers.py @@ -71,10 +71,12 @@ class OllamaProvider: # can't do them (and fall straight to the simple injector). self._tools_ok: bool | None = None - def _options(self) -> dict: + def _options(self, extra: dict | None = None) -> dict: opts = {"num_ctx": self.num_ctx, "num_predict": self.num_predict} if self.num_thread is not None: opts["num_thread"] = self.num_thread + if extra: + opts.update(extra) return opts def _raise_for_status(self, r: requests.Response) -> None: @@ -133,11 +135,18 @@ class OllamaProvider: turns); ``system`` is prepended. Returns ``(text, tool_calls)`` where each call is ``{"name": str, "arguments": dict}``. Raises ``ToolsUnsupported`` if the model can't do function calling so the bridge can fall back to simple.""" + # Greedy decode (temperature 0) for the tool loop: at Ollama's default 0.8 a + # weak model "creatively" narrates the next step in prose or fabricates file + # content instead of emitting a deterministic structured call. The nudge loop + # changes the prompt between turns, so temp 0 still escapes a failing state on + # retry — it just stops sampling away from the correct tool-call format. This + # override is scoped to complete_with_tools; chat (complete/stream) keeps the + # model's default sampling so replies stay natural. payload = { "model": self.model, "stream": False, "keep_alive": self.keep_alive, - "options": self._options(), + "options": self._options({"temperature": 0.0}), "tools": tools, "messages": [{"role": "system", "content": system}] + messages, } @@ -188,6 +197,16 @@ class OllamaProvider: re.I, ) + # The SPLIT-form leak (qwen2.5:0.5b at temp 0, ~half its turns): the tool NAME in + # a `` tag and the arguments in a SEPARATE bare JSON object with no `name` + # key — `write_file{"path":…,"content":…}`. Captures the name; the + # decoder reads the args object that follows from the trailing `{`. + _NAMED_TAG = re.compile( + r"<(tool_call|tool_calls|function_call|function|tools)>\s*" + r"([a-zA-Z_]\w*)\s*\s*(?=\{)", + re.I, + ) + @classmethod def _coerce_call(cls, obj, valid_names) -> dict | None: """Turn a decoded JSON object into a `{"name","arguments"}` call IF it @@ -225,16 +244,20 @@ class OllamaProvider: """Recover tool calls a small/quantized model emitted as TEXT in `content` instead of the structured `tool_calls` field. Handles qwen's `{json}` blocks plus the looser CPU-model leaks: bare - JSON, ```json fenced blocks, and alternate wrapper tags (``, - ``). Scans for every JSON object via a decoder (so nested - braces in arguments parse correctly) and keeps ONLY those that `_coerce_call` - accepts as a known tool — never freeform prose, so it can't fabricate an - action the model didn't structurally request. Returns the text with the - recovered JSON (and now-orphaned wrapper tags / code fences) stripped, plus - the calls.""" + JSON, ```json fenced blocks, alternate wrapper tags (``, + ``), and the SPLIT form where the name sits in a tag and the + args follow as a separate object (`write_file{"path":…}`). + Scans for every JSON object via a decoder (so nested braces in arguments parse + correctly) and keeps ONLY those that resolve to a KNOWN tool — never freeform + prose, so it can't fabricate an action the model didn't structurally request. + Returns the text with the recovered JSON (and now-orphaned wrapper tags / code + fences) stripped, plus the calls.""" dec = json.JSONDecoder() calls: list[dict] = [] spans: list[tuple[int, int]] = [] + # Split-form index: the `{` that opens an args object → (tool_name, tag_start), + # so the scan pairs that JSON as arguments and strips the whole tag+object. + split = {m.end(): (m.group(2), m.start()) for m in cls._NAMED_TAG.finditer(text)} i, n = 0, len(text) while i < n: brace = text.find("{", i) @@ -245,10 +268,17 @@ class OllamaProvider: except ValueError: i = brace + 1 continue - call = cls._coerce_call(obj, valid_names) - if call is not None: - calls.append(call) - spans.append((brace, end)) + if brace in split: + # `NAME{args}` — name from the tag, this object is the args. + name, tag_start = split[brace] + if (not valid_names or name in valid_names) and isinstance(obj, dict): + calls.append({"name": name, "arguments": obj}) + spans.append((tag_start, end)) + else: + call = cls._coerce_call(obj, valid_names) + if call is not None: + calls.append(call) + spans.append((brace, end)) i = end if spans: kept, last = [], 0 diff --git a/docs/findings-native-harness-2026-06-10.md b/docs/findings-native-harness-2026-06-10.md index dbe4334..3273ce2 100644 --- a/docs/findings-native-harness-2026-06-10.md +++ b/docs/findings-native-harness-2026-06-10.md @@ -268,3 +268,39 @@ the 3B) and `llama3.1:8b` (~4.7 GB, peer to the 7B but general-purpose). Probe t The first two are the clear next harness improvements; the benchmark now exists to prove whether they move the number. + +### The win — split-tag recovery + greedy decode (2026-06-10) + +Two more harness changes, and the first to actually move the number: + +1. **Greedy decode for the tool loop** — `complete_with_tools` now sends + `temperature: 0` (scoped to the tool loop; chat keeps default sampling). At + Ollama's default 0.8 the weak model sampled away from the tool-call format into + prose/fabrication. The nudge prompt changes between turns, so temp 0 still escapes + a failed state on retry — it just stops the creative drift. + +2. **Split-tag recovery** — temp 0 made 0.5b's leak *deterministic*, which exposed its + real shape: NOT a JSON object with a `name` key, but the name in a tag and the args + in a SEPARATE object — `write_file{"path":…,"content":…}`, + `run_shell{"command":…}` — ~5 of 12 tasks per run. `_NAMED_TAG` in + the provider now pairs that tag-name with the following args object (gated on the + known tool set, so it still can't fabricate an action). + +3. **Fenced recovery** (bridge) — recover a `` ```bash `` block narrated in prose as a + run_shell call, non-destructive only (`FENCE_DESTRUCTIVE` guard). Fires on the + prose-leak turns; a no-op where the model emits structured calls. + +**Result — the effect is concentrated on the weakest model, as theory predicts** (the +smaller the model, the more it leaks malformed text instead of structured calls; a +bigger model emits proper calls and fails on capability/content, which no parser can +fix): + +| model | baseline | optimized (2 runs) | note | +|---|---|---|---| +| qwen2.5:0.5b | 1/12 | **4/12, 3/12** | split-tag leak dominates → big lift | +| qwen2.5:3b | 1/12 | 2/12, 0/12 | emits proper calls → unchanged (noise) | + +Ablation on 0.5b: structured-JSON-only `0/0` → fenced+temp0 `2/0` → +split-tag `4/3`. +Split-tag recovery is the dominant contributor; temp 0 is what made the leak +consistent enough to parse. Net: the harness now extracts ~3× more signal from the +0.5b, and temp 0 is a sound default for the tool loop on any model.