feat(ai): visible native harness — PTY mirror + readable chat
Display-mirror hybrid (docs/plan-harness-visibility.md §2): native tool calls now show up in the shared sandbox terminal again via inert `# `- prefixed comment lines (comment-prefix = anti-double-run/anti-escape), mirroring only each command. Chat de-flooded to opener + final summary. write_file mkdir -p parent dir so relative/absolute paths both work (fixes the regression where script creation silently failed). ui.rs fmt_line returns Vec<Line> splitting on \n so multi-line agent output renders as an indented block instead of one garbled row. Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
leetcrypt
@@ -0,0 +1,212 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Live bench/smoke for the native `!task` harness (docs/spec-native-harness.md, Phase 3).
|
||||
|
||||
Drives the real `AgentBridge._run_native` against a live Ollama model and the
|
||||
`local` sandbox backend (host shell, scoped to a fresh temp workdir), with no chat
|
||||
server or TUI in the loop. For each canonical task it reports: wall-clock latency,
|
||||
model turns, tool calls, whether the expected artifact landed, and the final
|
||||
answer. Also records a single chat-completion latency as the `simple`-harness
|
||||
model-cost baseline (simple's execution needs the broker PTY, so only its model
|
||||
call is comparable headlessly).
|
||||
|
||||
Usage: .venv/bin/python hh/scripts/bench-native-harness.py [--model qwen2.5:3b]
|
||||
[--max-turns 5] [--threads 4]
|
||||
Env: OLLAMA_HOST (default http://localhost:11434)
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import asyncio
|
||||
import os
|
||||
import sys
|
||||
import tempfile
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
# Make the repo importable when run from anywhere.
|
||||
ROOT = Path(__file__).resolve().parents[2]
|
||||
sys.path.insert(0, str(ROOT))
|
||||
|
||||
from cmd_chat.agent.bridge import AgentBridge # noqa: E402
|
||||
from cmd_chat.agent.providers import ( # noqa: E402
|
||||
Msg,
|
||||
OllamaProvider,
|
||||
ToolsUnsupported,
|
||||
)
|
||||
|
||||
PER_TASK_TIMEOUT = 300.0 # hard ceiling so a runaway loop can't hang the bench
|
||||
|
||||
# (label, task text, check(workdir) -> bool)
|
||||
TASKS = [
|
||||
(
|
||||
"write+read",
|
||||
"create a file hello.txt containing exactly the text 'hello world', "
|
||||
"then show its contents",
|
||||
lambda d: (d / "hello.txt").is_file()
|
||||
and "hello world" in (d / "hello.txt").read_text(),
|
||||
),
|
||||
(
|
||||
"script+run",
|
||||
"write a python script add.py that prints the sum of 2 and 3, then run it",
|
||||
lambda d: (d / "add.py").is_file(),
|
||||
),
|
||||
(
|
||||
"mkdir+list",
|
||||
"make a directory named data and then list the files in the current directory",
|
||||
lambda d: (d / "data").is_dir(),
|
||||
),
|
||||
]
|
||||
|
||||
|
||||
class CountingOllama(OllamaProvider):
|
||||
"""OllamaProvider that tallies tool-calling turns for the report."""
|
||||
|
||||
def __init__(self, *a, **k):
|
||||
super().__init__(*a, **k)
|
||||
self.tool_turns = 0
|
||||
|
||||
def complete_with_tools(self, system, messages, tools):
|
||||
self.tool_turns += 1
|
||||
return super().complete_with_tools(system, messages, tools)
|
||||
|
||||
|
||||
def make_bridge(provider, workdir: str):
|
||||
"""A headless AgentBridge wired to the local backend, with all network sends
|
||||
stubbed to capture room output instead of encrypting to a websocket."""
|
||||
b = AgentBridge(
|
||||
"localhost", 0, name="bench", provider=provider,
|
||||
no_tls=True, code_provider=provider, harness="native",
|
||||
max_turns=provider_max_turns,
|
||||
)
|
||||
b.granted = True
|
||||
b.sbx_engine = "local" # exec on the host shell (this process's CWD = workdir)
|
||||
b.sbx_name = ""
|
||||
b._chat: list[str] = []
|
||||
|
||||
async def cap_chat(ws, text):
|
||||
b._chat.append(text)
|
||||
|
||||
async def noop(*a, **k):
|
||||
pass
|
||||
|
||||
b._send_chat = cap_chat
|
||||
b._send_typing = noop
|
||||
b._send_stream = noop
|
||||
|
||||
async def cap_inject(ws, cmds): # simple-harness path (unused here, kept honest)
|
||||
b._chat.append("[inject] " + " ; ".join(cmds))
|
||||
|
||||
b._inject = cap_inject
|
||||
return b
|
||||
|
||||
|
||||
provider_max_turns = 5 # set in main()
|
||||
|
||||
|
||||
async def run_task(provider, label, task, check) -> dict:
|
||||
workdir = Path(tempfile.mkdtemp(prefix=f"hh-bench-{label}-"))
|
||||
cwd = os.getcwd()
|
||||
os.chdir(workdir)
|
||||
provider.tool_turns = 0
|
||||
bridge = make_bridge(provider, str(workdir))
|
||||
t0 = time.monotonic()
|
||||
timed_out = False
|
||||
err = None
|
||||
try:
|
||||
await asyncio.wait_for(
|
||||
bridge._run_native(None, task, "andre"), timeout=PER_TASK_TIMEOUT)
|
||||
except asyncio.TimeoutError:
|
||||
timed_out = True
|
||||
except Exception as e: # noqa: BLE001 — record, don't abort the suite
|
||||
err = f"{type(e).__name__}: {e}"
|
||||
finally:
|
||||
os.chdir(cwd)
|
||||
elapsed = time.monotonic() - t0
|
||||
ok = False
|
||||
try:
|
||||
ok = bool(check(workdir)) and not timed_out and err is None
|
||||
except Exception: # noqa: BLE001
|
||||
ok = False
|
||||
final = next((c for c in reversed(bridge._chat) if "(native) for" in c), "")
|
||||
return {
|
||||
"label": label, "ok": ok, "elapsed": elapsed, "turns": provider.tool_turns,
|
||||
"timed_out": timed_out, "err": err, "workdir": str(workdir),
|
||||
"chat": bridge._chat, "final": final,
|
||||
}
|
||||
|
||||
|
||||
async def main() -> int:
|
||||
global provider_max_turns
|
||||
ap = argparse.ArgumentParser()
|
||||
ap.add_argument("--model", default="qwen2.5:3b")
|
||||
ap.add_argument("--max-turns", type=int, default=5)
|
||||
ap.add_argument("--threads", type=int, default=4)
|
||||
ap.add_argument("--num-ctx", type=int, default=4096)
|
||||
ap.add_argument("--verbose", action="store_true", help="print full chat per task")
|
||||
args = ap.parse_args()
|
||||
provider_max_turns = args.max_turns
|
||||
|
||||
host = os.environ.get("OLLAMA_HOST", "http://localhost:11434")
|
||||
print(f"== native harness bench == model={args.model} host={host}")
|
||||
print(f" max_turns={args.max_turns} threads={args.threads} num_ctx={args.num_ctx}\n")
|
||||
|
||||
provider = CountingOllama(
|
||||
model=args.model, num_ctx=args.num_ctx, num_thread=args.threads, num_predict=512)
|
||||
|
||||
# 0. Wire preflight: does the model accept the `tools` field at all?
|
||||
print("[preflight] probing tool support…", flush=True)
|
||||
t0 = time.monotonic()
|
||||
try:
|
||||
text, calls = await asyncio.to_thread(
|
||||
provider.complete_with_tools,
|
||||
"You are a test.",
|
||||
[{"role": "user", "content": "Call run_shell to echo hi."}],
|
||||
__import__("cmd_chat.agent.bridge", fromlist=["NATIVE_TOOLS"]).NATIVE_TOOLS,
|
||||
)
|
||||
except ToolsUnsupported as e:
|
||||
print(f" ✖ model rejects tools: {e}\n → native would degrade to simple. Stopping.")
|
||||
return 1
|
||||
except Exception as e: # noqa: BLE001
|
||||
print(f" ✖ preflight error: {type(e).__name__}: {e}")
|
||||
return 2
|
||||
print(f" ✓ tools accepted in {time.monotonic()-t0:.1f}s "
|
||||
f"(calls={len(calls)}, supports_tools={provider.supports_tools()})\n")
|
||||
|
||||
# Baseline: one plain chat completion (the only model cost simple pays).
|
||||
t0 = time.monotonic()
|
||||
try:
|
||||
await asyncio.to_thread(provider.complete, "You are concise.",
|
||||
[Msg("user", "say ok")])
|
||||
base = time.monotonic() - t0
|
||||
print(f"[baseline] one chat completion (≈ simple's model cost): {base:.1f}s\n")
|
||||
except Exception as e: # noqa: BLE001
|
||||
print(f"[baseline] chat completion failed: {e}\n")
|
||||
|
||||
results = []
|
||||
for label, task, check in TASKS:
|
||||
print(f"[task:{label}] {task}", flush=True)
|
||||
r = await run_task(provider, label, task, check)
|
||||
tag = "PASS" if r["ok"] else ("TIMEOUT" if r["timed_out"] else "FAIL")
|
||||
print(f" → {tag} {r['elapsed']:.1f}s turns={r['turns']}"
|
||||
+ (f" err={r['err']}" if r["err"] else ""))
|
||||
if r["final"]:
|
||||
print(f" final: {r['final'].splitlines()[-1][:160]}")
|
||||
if args.verbose:
|
||||
for c in r["chat"]:
|
||||
print(" | " + c.replace("\n", " ")[:160])
|
||||
print(flush=True)
|
||||
results.append(r)
|
||||
|
||||
# Summary table.
|
||||
print("== summary ==")
|
||||
print(f"{'task':<14}{'result':<9}{'secs':>7}{'turns':>7}")
|
||||
for r in results:
|
||||
tag = "PASS" if r["ok"] else ("TIMEOUT" if r["timed_out"] else "FAIL")
|
||||
print(f"{r['label']:<14}{tag:<9}{r['elapsed']:>7.1f}{r['turns']:>7}")
|
||||
passes = sum(1 for r in results if r["ok"])
|
||||
print(f"\n{passes}/{len(results)} passed")
|
||||
return 0 if passes == len(results) else 3
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(asyncio.run(main()))
|
||||
+62
-18
@@ -670,18 +670,42 @@ fn draw_top(f: &mut Frame, area: ratatui::layout::Rect, app: &App, theme: &Theme
|
||||
f.render_widget(Paragraph::new(bar), area);
|
||||
}
|
||||
|
||||
fn fmt_line<'a>(l: &'a ChatLine, app: &App, theme: &Theme) -> Line<'a> {
|
||||
if l.system {
|
||||
// System lines carry the canonical ⛧ as a placeholder for "the house
|
||||
// sigil"; swap it for the active theme's sigil so e.g. crypt shows ✝,
|
||||
// never a pentagram. User messages (l.system == false) are left as typed.
|
||||
let text = l.text.replace('⛧', &theme.sigil);
|
||||
return Line::from(Span::styled(
|
||||
format!(" {} {}", theme.sigil, text),
|
||||
Style::default()
|
||||
.fg(theme.system)
|
||||
.add_modifier(Modifier::ITALIC),
|
||||
));
|
||||
/// Render one chat record into one-or-more visual lines. A record may carry
|
||||
/// embedded newlines (e.g. an AI agent's multi-line answer or injected plan);
|
||||
/// ratatui treats a `Line` as a single visual row, so we split on `\n` ourselves
|
||||
/// and indent the continuations to align under the first line's text.
|
||||
fn fmt_line<'a>(l: &'a ChatLine, app: &App, theme: &Theme) -> Vec<Line<'a>> {
|
||||
// "Action" lines — client system notices and AI agent output (which marks
|
||||
// itself with a leading †) — read better as dim, italic, sigil-marked blocks
|
||||
// than as ordinary chatter, so the eye can skip them or zero in on them.
|
||||
let is_action = l.system || l.text.starts_with('†');
|
||||
if is_action {
|
||||
// Swap the canonical † placeholder for the active theme's sigil so each
|
||||
// vestment renders its own glyph (e.g. crypt shows ✝).
|
||||
let body = l
|
||||
.text
|
||||
.strip_prefix('†')
|
||||
.unwrap_or(&l.text)
|
||||
.trim_start()
|
||||
.replace('†', &theme.sigil);
|
||||
let style = Style::default()
|
||||
.fg(theme.system)
|
||||
.add_modifier(Modifier::ITALIC);
|
||||
// Attribute an agent's action to it (system notices stay anonymous).
|
||||
let head = if l.system || l.username.is_empty() {
|
||||
format!(" {} ", theme.sigil)
|
||||
} else {
|
||||
format!(" {} {}: ", theme.sigil, l.username)
|
||||
};
|
||||
let indent = " ".repeat(head.chars().count());
|
||||
return body
|
||||
.split('\n')
|
||||
.enumerate()
|
||||
.map(|(i, seg)| {
|
||||
let prefix = if i == 0 { head.clone() } else { indent.clone() };
|
||||
Line::from(Span::styled(format!("{prefix}{seg}"), style))
|
||||
})
|
||||
.collect();
|
||||
}
|
||||
let name_color = if l.username == app.me {
|
||||
theme.me
|
||||
@@ -691,7 +715,7 @@ fn fmt_line<'a>(l: &'a ChatLine, app: &App, theme: &Theme) -> Line<'a> {
|
||||
// Author's current badge inline, so a message's authority is legible right
|
||||
// in the transcript — not only in the clergy panel.
|
||||
let badges = role_badges(app, &l.username, theme);
|
||||
Line::from(vec![
|
||||
let head: Vec<Span> = vec![
|
||||
Span::styled(format!("{} ", l.ts), Style::default().fg(theme.dim)),
|
||||
Span::styled(format!("{badges} "), Style::default().fg(theme.dim)),
|
||||
Span::styled(
|
||||
@@ -699,14 +723,34 @@ fn fmt_line<'a>(l: &'a ChatLine, app: &App, theme: &Theme) -> Line<'a> {
|
||||
Style::default().fg(name_color).add_modifier(Modifier::BOLD),
|
||||
),
|
||||
Span::styled(": ", Style::default().fg(theme.dim)),
|
||||
Span::styled(l.text.as_str(), Style::default().fg(theme.title)),
|
||||
])
|
||||
];
|
||||
let mut segs = l.text.split('\n');
|
||||
let first = segs.next().unwrap_or("");
|
||||
let mut spans = head;
|
||||
spans.push(Span::styled(first.to_string(), Style::default().fg(theme.title)));
|
||||
let mut out = vec![Line::from(spans)];
|
||||
// Continuation rows indent under the message body so a multi-line message
|
||||
// reads as one coherent block under its author.
|
||||
let indent = " ".repeat(
|
||||
l.ts.chars().count() + 1 + badges.chars().count() + 1 + l.username.chars().count() + 2,
|
||||
);
|
||||
for seg in segs {
|
||||
out.push(Line::from(Span::styled(
|
||||
format!("{indent}{seg}"),
|
||||
Style::default().fg(theme.title),
|
||||
)));
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
fn draw_chat(f: &mut Frame, area: ratatui::layout::Rect, app: &App, theme: &Theme) {
|
||||
let inner_h = area.height.saturating_sub(2) as usize; // rows inside the border
|
||||
let text_w = area.width.saturating_sub(2).max(1); // wrap width inside the border
|
||||
let mut lines: Vec<Line> = app.lines.iter().map(|l| fmt_line(l, app, theme)).collect();
|
||||
let mut lines: Vec<Line> = app
|
||||
.lines
|
||||
.iter()
|
||||
.flat_map(|l| fmt_line(l, app, theme))
|
||||
.collect();
|
||||
|
||||
// Live preview bubbles for agents currently streaming a reply, rendered
|
||||
// below the committed history. Dim + italic so they read as in-progress;
|
||||
@@ -758,7 +802,7 @@ fn draw_chat(f: &mut Frame, area: ratatui::layout::Rect, app: &App, theme: &Them
|
||||
f.render_widget(chat, area);
|
||||
}
|
||||
|
||||
/// Glyph for a single role. The host sigil is theme-dependent (✝ for crypt, ⛧
|
||||
/// Glyph for a single role. The host sigil is theme-dependent (✝ for crypt, †
|
||||
/// for the default), the rest are fixed.
|
||||
fn role_glyph(role: Role, theme: &Theme) -> &str {
|
||||
match role {
|
||||
@@ -769,7 +813,7 @@ fn role_glyph(role: Role, theme: &Theme) -> &str {
|
||||
}
|
||||
}
|
||||
|
||||
/// The stacked badge string for `name` — e.g. `⛧⚡◆` for a host who summoned a
|
||||
/// The stacked badge string for `name` — e.g. `†⚡◆` for a host who summoned a
|
||||
/// sandbox and can drive, or a lone `•` for a plain member. Single source of
|
||||
/// truth shared by the roster and the chat author prefix so both always agree
|
||||
/// with each other and with what the broker enforces.
|
||||
|
||||
Reference in New Issue
Block a user