From 77f62034758e58b6bf952c64efce7e44d864f7c0 Mon Sep 17 00:00:00 2001 From: leetcrypt Date: Sat, 27 Jun 2026 15:26:51 -0700 Subject: [PATCH] =?UTF-8?q?test(ai):=20bench=20executes=20run=5Fshell=20vi?= =?UTF-8?q?a=20PTY=20stub=20=E2=80=94=203/3=20(was=202/3)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The headless bench stubbed _send_sbx_input to a no-op, so run_shell commands were staged to /tmp/.hh_*.cmd but the typed wrapper never ran — the rc/out poll timed out and any shell-dependent task FAILed (mkdir+list burned ~248s then failed). Make the stub run typed input in the bench process (CWD == workdir), faithfully standing in for the room PTY, so _run_shell_in_pty (staging/poll/cleanup) is exercised end-to-end. Result: 3/3 pass. Surfaced a separate model-side argument leak in script+run (tool JSON-schema dict passed as the command) — fixed next. Co-Authored-By: Claude Opus 4.6 --- hh/scripts/bench-native-harness.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/hh/scripts/bench-native-harness.py b/hh/scripts/bench-native-harness.py index c20e35d..2bb62e8 100644 --- a/hh/scripts/bench-native-harness.py +++ b/hh/scripts/bench-native-harness.py @@ -98,8 +98,16 @@ def make_bridge(provider, workdir: str): b._inject = cap_inject - async def cap_sbx_input(ws, data): # PTY-mirror frames — broker-only in prod - b._chat.append("[pty] " + data.decode(errors="replace").rstrip("\n")) + async def cap_sbx_input(ws, data): # stand in for the shared PTY + # In prod the broker writes these keystrokes to the room's PTY shell; here + # there's no PTY, so run them in the bench process (CWD == workdir) to + # exercise run_shell end-to-end. Mirror lines are `# `-prefixed comments, + # so the shell no-ops them; the only real command is the staged wrapper. + line = data.decode(errors="replace") + b._chat.append("[pty] " + line.rstrip("\n")) + proc = await asyncio.create_subprocess_shell( + line, stdout=asyncio.subprocess.DEVNULL, stderr=asyncio.subprocess.DEVNULL) + await proc.wait() b._send_sbx_input = cap_sbx_input return b