leetcrypt
a34a18f0ca
Weak local models (seen with qwen2.5-coder:3b at temp 0) sometimes emit a
parameter's JSON *schema* fragment as its *value*, e.g.
run_shell(command={'type':'string','description':'bash ./add.py'}). _exec_tool
does str(args["command"]), so the stringified dict was run as a command →
exit 127 and a hollow "task done" claim.
Every native tool arg is a plain string, so a dict-valued arg is always this
leak. Add _unleak_str/_clean_args to OllamaProvider: pull the intended string
from a value-ish key (or `description`), ignore JSON-schema scaffolding keys
like `type`, else drop to "" so the tool reports a clean error instead of
running garbage. Applied on both the structured tool_calls path and the
text-recovery path (_coerce_call). New tests/test_agent_providers.py pins it.
Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
49 lines
1.9 KiB
Python
49 lines
1.9 KiB
Python
"""Unit tests for OllamaProvider tool-call argument recovery.
|
|
|
|
Weak/quantized local models intermittently emit a parameter's JSON *schema*
|
|
fragment as its *value* (e.g. run_shell(command={'type':'string',
|
|
'description':'bash ./add.py'})). Every native tool arg is a plain string, so a
|
|
dict-valued arg is always this leak. These tests pin the recovery so the harness
|
|
never runs a stringified schema dict as a command again.
|
|
"""
|
|
from cmd_chat.agent.providers import OllamaProvider as P
|
|
|
|
|
|
def test_unleak_recovers_command_from_schema_fragment():
|
|
assert P._unleak_str({"type": "string", "description": "bash ./add.py"}) == "bash ./add.py"
|
|
|
|
|
|
def test_unleak_prefers_value_keys_over_description():
|
|
assert P._unleak_str({"description": "help text", "value": "ls -la"}) == "ls -la"
|
|
assert P._unleak_str({"description": "help text", "default": "echo hi"}) == "echo hi"
|
|
|
|
|
|
def test_unleak_passes_plain_strings_through():
|
|
assert P._unleak_str("mkdir data") == "mkdir data"
|
|
|
|
|
|
def test_unleak_single_string_value_fallback():
|
|
assert P._unleak_str({"foo": "only-one"}) == "only-one"
|
|
|
|
|
|
def test_unleak_ignores_bare_schema_meta():
|
|
# {'type':'string'} has no real value — must NOT recover the type name.
|
|
assert P._unleak_str({"type": "string"}) == ""
|
|
assert P._unleak_str({}) == ""
|
|
|
|
|
|
def test_clean_args_flattens_leaked_arg_keeps_real_ones():
|
|
out = P._clean_args({"command": {"type": "string", "description": "bash ./add.py"}, "x": "keep"})
|
|
assert out == {"command": "bash ./add.py", "x": "keep"}
|
|
|
|
|
|
def test_clean_args_passes_non_dict_through():
|
|
assert P._clean_args("passthrough") == "passthrough"
|
|
|
|
|
|
def test_coerce_call_cleans_recovered_arguments():
|
|
# The text-recovery path must also unleak args.
|
|
obj = {"name": "run_shell", "arguments": {"command": {"type": "string", "description": "ls"}}}
|
|
call = P._coerce_call(obj, valid_names={"run_shell"})
|
|
assert call == {"name": "run_shell", "arguments": {"command": "ls"}}
|