n0mad1k/bigbrother
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Fix #214: Prevent ToolManager crash callback deadlock

Browse Source 
- Move crash_callback execution outside of self._lock
- Callback is called after releasing the monitor lock
- Re-acquire lock only for restart decision logic
- Prevents deadlock if callback tries to acquire same lock
- Maintains thread safety for restart and process state updates
This commit is contained in:
Cobra
2026-04-06 11:40:36 -04:00
parent 1b93378f02
commit f15b8994cd
1 changed files with 23 additions and 18 deletions
Download Patch File Download Diff File Expand all files Collapse all files
core/tool_manager.py
+22 -17
View File
@@ -336,28 +336,33 @@ class ToolManager:
"restart_count": tool.restart_count}, "restart_count": tool.restart_count},
source_module="tool_manager") source_module="tool_manager")
# Crash callback (e.g., corrective ARP) # Call crash callback OUTSIDE lock to prevent deadlock
if tool.crash_callback: callback = tool.crash_callback
restart_needed = tool.restart_count < tool.max_restarts
# Release lock before callback
if callback:
try: try:
tool.crash_callback() callback()
except Exception: except Exception:
logger.exception("Crash callback failed for %s", tool.name) logger.exception("Crash callback failed for %s", tool.name)
# Auto-restart with exponential backoff # Re-acquire lock for restart decision
if tool.restart_count < tool.max_restarts: with self._lock:
backoff = min(2 ** tool.restart_count, 30) # Auto-restart with exponential backoff
logger.info("Restarting %s in %ds (attempt %d/%d)", if tool.restart_count < tool.max_restarts:
tool.name, backoff, tool.restart_count + 1, backoff = min(2 ** tool.restart_count, 30)
tool.max_restarts) logger.info("Restarting %s in %ds (attempt %d/%d)",
time.sleep(backoff) tool.name, backoff, tool.restart_count + 1,
tool.restart_count += 1 tool.max_restarts)
with self._lock: time.sleep(backoff)
tool.restart_count += 1
self._start_tool(tool) self._start_tool(tool)
else: else:
logger.error("Tool %s exceeded max restarts (%d), giving up", logger.error("Tool %s exceeded max restarts (%d), giving up",
tool.name, tool.max_restarts) tool.name, tool.max_restarts)
tool.process = None tool.process = None
tool.pid = None tool.pid = None
elif tool.health_check: elif tool.health_check:
# Process alive — run health check # Process alive — run health check