perf(net): near-real-time sandbox for non-host viewers

Two causes made a shared shell's PTY stream lag for non-host viewers and
then arrive "all at once":

1. Nagle's algorithm — PTY echo/keystrokes/chat are tiny frames; Nagle +
   delayed-ACK (amplified by Tailscale RTT) coalesced them into bursts.
   Set TCP_NODELAY on the client websocket (Plain/--no-tls path) and on
   each server connection's socket (via ws.io_proto.transport).

2. Head-of-line blocking — ConnectionManager.broadcast awaited each
   send() serially under a global lock, so the slowest viewer gated
   delivery to everyone AND stalled the server's read of the broker's
   next frame, backing the stream up. Give each connection its own
   bounded outbound queue + writer task; broadcast now only enqueues and
   never waits on a socket. A peer that overflows its backlog (4096) is
   evicted (close 1013) and resyncs on reconnect rather than buffering
   without bound or being fed a gapped terminal stream. Init snapshot is
   enqueued as the guaranteed-first frame (send_state -> state_frame).

Adds tests/test_manager.py (FIFO, slow-client isolation, wedged
eviction, reconnect replacement). Also fmt-clean sbx.rs/theme.rs.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>

cmd_chat/server/helpers.py

@@ -24,19 +24,20 @@ def get_client_ip(request: Request) -> str:
     return request.ip
 
 
-async def send_state(ws: Websocket, app: Sanic) -> None:
+def state_frame(app: Sanic) -> str:
+    """Build the per-client `init` snapshot (message backlog + roster) as a JSON
+    string. Returned (not sent) so it can be enqueued as the connection's first
+    outbound frame, guaranteeing it precedes any later broadcast."""
     messages = app.ctx.message_store.get_all()
     users = app.ctx.session_store.get_all()
-    await ws.send(
-        json.dumps(
-            {
-                "type": "init",
-                "messages": [asdict(m) for m in messages],
-                "users": [
-                    {"user_id": u.user_id, "username": u.username} for u in users
-                ],
-            }
-        )
+    return json.dumps(
+        {
+            "type": "init",
+            "messages": [asdict(m) for m in messages],
+            "users": [
+                {"user_id": u.user_id, "username": u.username} for u in users
+            ],
+        }
     )
 
 

cmd_chat/server/managers.py

@@ -3,41 +3,117 @@ from typing import Optional
 from sanic import Websocket
 
 
+# Per-connection outbound backlog before a client is considered wedged.
+#
+# Every relayed frame is *enqueued* (never awaited) by the broadcaster, and a
+# dedicated per-connection writer task drains the queue with `await ws.send`.
+# This decouples the room's read loop from any single slow receiver: the old
+# code awaited each `send` serially under a global lock, so the slowest viewer
+# (e.g. a remote peer over a laggy Tailscale link) gated delivery to everyone
+# *and* stalled the server's read of the next frame — so a sandbox owner's PTY
+# stream backed up and then arrived "all at once".
+#
+# A backlog this deep absorbs a large PTY burst and transient network hiccups.
+# A client that falls this far behind is effectively stuck (it would be closed
+# by the websocket ping-timeout within ~40s anyway); we close it now so the room
+# isn't holding unbounded memory for it. It reconnects (Ctrl-R) and the host
+# replays a fresh sandbox snapshot, which is cleaner than feeding it a corrupt,
+# partial terminal byte-stream with gaps.
+SEND_QUEUE_MAX = 4096
+
+
+class _Conn:
+    """A live websocket plus its outbound queue and the task draining it."""
+
+    __slots__ = ("ws", "queue", "task")
+
+    def __init__(self, ws: Websocket):
+        self.ws = ws
+        self.queue: asyncio.Queue[str] = asyncio.Queue(maxsize=SEND_QUEUE_MAX)
+        self.task: Optional[asyncio.Task] = None
+
+
 class ConnectionManager:
     def __init__(self):
-        self.active_connections: dict[str, Websocket] = {}
+        self.active_connections: dict[str, _Conn] = {}
         self._lock = asyncio.Lock()
 
-    async def connect(self, user_id: str, websocket: Websocket) -> None:
+    async def connect(
+        self, user_id: str, websocket: Websocket, initial: Optional[str] = None
+    ) -> None:
+        """Register a connection and start its writer task.
+
+        `initial` (the per-client init snapshot) is enqueued *before* the
+        connection becomes visible to broadcasters, guaranteeing it is the first
+        frame the client receives — no later broadcast can jump ahead of it.
+        """
+        conn = _Conn(websocket)
+        if initial is not None:
+            conn.queue.put_nowait(initial)
+        old = None
         async with self._lock:
-            self.active_connections[user_id] = websocket
+            old = self.active_connections.get(user_id)
+            self.active_connections[user_id] = conn
+        if old is not None and old.task is not None:
+            old.task.cancel()
+        conn.task = asyncio.create_task(self._writer(conn))
+
+    async def _writer(self, conn: _Conn) -> None:
+        """Drain one connection's queue in FIFO order, one `send` at a time."""
+        try:
+            while True:
+                msg = await conn.queue.get()
+                await conn.ws.send(msg)
+        except asyncio.CancelledError:
+            raise
+        except Exception:
+            # Socket died mid-send; the handler's `finally` will disconnect us.
+            pass
 
     async def disconnect(self, user_id: str) -> None:
         async with self._lock:
-            if user_id in self.active_connections:
-                del self.active_connections[user_id]
+            conn = self.active_connections.pop(user_id, None)
+        if conn is not None and conn.task is not None:
+            conn.task.cancel()
+
+    @staticmethod
+    def _enqueue(conn: _Conn, message: str) -> bool:
+        """Non-blocking enqueue. False means the backlog is full (wedged peer)."""
+        try:
+            conn.queue.put_nowait(message)
+            return True
+        except asyncio.QueueFull:
+            return False
+
+    async def _drop_wedged(self, user_id: str, conn: _Conn) -> None:
+        """Evict a client whose backlog overflowed and nudge it to reconnect."""
+        await self.disconnect(user_id)
+        try:
+            # 1013 = "Try Again Later": the client surfaces a closed socket and
+            # can reconnect to resync from a fresh snapshot.
+            await conn.ws.close(code=1013)
+        except Exception:
+            pass
 
     async def broadcast(self, message: str, exclude_user: Optional[str] = None) -> None:
         async with self._lock:
-            disconnected = []
-            for user_id, connection in list(self.active_connections.items()):
-                if exclude_user and user_id == exclude_user:
-                    continue
-                try:
-                    await connection.send(message)
-                except Exception:
-                    disconnected.append(user_id)
-
-            for user_id in disconnected:
-                if user_id in self.active_connections:
-                    del self.active_connections[user_id]
+            targets = [
+                (uid, conn)
+                for uid, conn in self.active_connections.items()
+                if not (exclude_user and uid == exclude_user)
+            ]
+        # Enqueue is synchronous and fast, so the broadcaster returns immediately
+        # — it never waits on any socket. Only genuinely wedged peers overflow.
+        wedged = [(uid, conn) for uid, conn in targets if not self._enqueue(conn, message)]
+        for uid, conn in wedged:
+            await self._drop_wedged(uid, conn)
 
     async def send_personal(self, user_id: str, message: str) -> bool:
         async with self._lock:
-            if connection := self.active_connections.get(user_id):
-                try:
-                    await connection.send(message)
-                    return True
-                except Exception:
-                    return False
+            conn = self.active_connections.get(user_id)
+        if conn is None:
+            return False
+        if self._enqueue(conn, message):
+            return True
+        await self._drop_wedged(user_id, conn)
         return False

cmd_chat/server/views.py

@@ -2,13 +2,27 @@ import hashlib
 import hmac
 import json
 import base64
+import socket
 from dataclasses import asdict
 
 from sanic import Sanic, Request, response, Websocket
 from sanic.response import HTTPResponse, json as json_response
 
 from .models import Message, UserSession
-from .helpers import get_client_ip, send_state, utcnow
+from .helpers import get_client_ip, state_frame, utcnow
+
+
+def _disable_nagle(ws: Websocket) -> None:
+    """Set TCP_NODELAY on a websocket's underlying socket. Small interactive
+    frames (PTY echo, keystrokes, chat) must not wait on Nagle coalescing +
+    delayed-ACK before reaching a viewer. Best-effort: any transport without a
+    raw socket (or where the option can't be set) is simply left as-is."""
+    try:
+        sock = ws.io_proto.transport.get_extra_info("socket")
+        if sock is not None:
+            sock.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1)
+    except Exception:
+        pass
 
 
 # Hard cap on a single relayed WS frame. The largest legitimate frame is one
@@ -137,11 +151,12 @@ async def chat_ws(request: Request, ws: Websocket, app: Sanic) -> None:
         return
 
     manager = app.ctx.connection_manager
-    await manager.connect(user_id, ws)
+    _disable_nagle(ws)
+    # Enqueue this client's init snapshot as its first outbound frame, then
+    # register it so broadcasts can target it — guaranteeing init arrives first.
+    await manager.connect(user_id, ws, initial=state_frame(app))
 
     try:
-        await send_state(ws, app)
-
         # Announce arrival to everyone already present, then a fresh roster.
         await manager.broadcast(
             json.dumps(