n0mad1k/bigbrother
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Fix DB table/column mismatch and stale PID noise on startup

Browse Source 
build_data.py created 'profiles' but mac_manager queries 'mac_profiles'.
Columns category/model renamed to device_type/device_name. device_type
values aligned with _CATEGORY_MAP in mac_manager (streaming, smart_tv,
phone, tablet, smart_speaker, iot, printer, gaming).

ja3_fingerprints.db table renamed from 'fingerprints' to 'ja3_profiles'
and expanded with all columns ja3_spoofer queries (name, description,
cipher_suites, extensions, elliptic_curves, ec_point_formats as JSON).

StateManager.reset_module_status() clears stale 'running' entries on
startup so watchdog doesn't flag old-session PIDs as dead modules.
Engine.start_all() calls reset before launch and does a 3s post-startup
liveness check, logging any modules that crashed immediately after fork.
This commit is contained in:
Cobra
2026-04-07 13:05:31 -04:00
parent e15e077be8
commit 0f754cfb7f
3 changed files with 152 additions and 50 deletions
Download Patch File Download Diff File Expand all files Collapse all files
core/engine.py
+29
View File
@@ -363,6 +363,10 @@ class Engine:
def start_all(self) -> dict[str, bool]:
"""Start all registered modules in dependency order."""
# Clear stale PIDs from any previous run so the watchdog does not
# report modules from the old session as dead.
self.state.reset_module_status()
order = _topo_sort(self._modules)
# Instantiate and start CaptureBus for passive modules
@@ -387,6 +391,31 @@ class Engine:
results = {}
for name in order:
results[name] = self.start(name)
# Post-startup liveness check: wait for module processes to settle,
# then verify each one is still alive and log a clear summary.
time.sleep(3)
alive, dead = [], []
with self._lock:
for name, entry in self._modules.items():
if results.get(name):
if entry.process and entry.process.is_alive():
alive.append(name)
else:
dead.append(name)
# Update state so watchdog doesn't double-report
self.state.set_module_status(name, "stopped")
if dead:
logger.error(
"Startup check: %d modules died immediately after launch: %s",
len(dead), ", ".join(dead),
)
logger.info(
"Startup check complete — running: %d, failed: %d",
len(alive), len(dead),
)
return results
def stop_all(self, timeout: float = 5.0) -> None:
core/state.py
+22
View File
@@ -232,6 +232,28 @@ class StateManager:
for r in rows
}
def reset_module_status(self) -> None:
"""Mark all running modules as stopped.
Called at engine startup to clear stale PIDs from a previous run.
Without this, the watchdog picks up old PIDs and reports dead modules
that were never started in the current session.
"""
conn = self._get_write_conn()
try:
conn.execute("BEGIN IMMEDIATE")
conn.execute(
"UPDATE module_status SET status='stopped', pid=NULL, updated=? "
"WHERE status='running'",
(time.time(),),
)
conn.commit()
except Exception:
try:
conn.rollback()
except Exception:
pass
# ------------------------------------------------------------------
# Write queue
# ------------------------------------------------------------------
scripts/build_data.py
+98 -47
View File
@@ -21,19 +21,22 @@ from pathlib import Path
def create_innocuous_macs_db(db_path: str) -> None:
"""Create innocuous MAC profiles database with ~50 real device profiles.
Each profile includes OUI, vendor, model, DHCP hostname pattern,
Each profile includes OUI, vendor, device_name, DHCP hostname pattern,
DHCP vendor class, TTL, TCP window size, and OS family -- everything
needed to impersonate the device at the network fingerprint level.
Table name and column names match what mac_manager.py queries:
mac_profiles(device_type, device_name, vendor, oui, ...)
"""
conn = sqlite3.connect(db_path)
conn.execute("PRAGMA journal_mode=WAL")
conn.executescript("""
CREATE TABLE IF NOT EXISTS profiles (
CREATE TABLE IF NOT EXISTS mac_profiles (
id INTEGER PRIMARY KEY AUTOINCREMENT,
category TEXT NOT NULL,
device_type TEXT NOT NULL,
oui TEXT NOT NULL,
vendor TEXT NOT NULL,
model TEXT NOT NULL,
device_name TEXT NOT NULL,
dhcp_hostname TEXT,
dhcp_vendor_class TEXT,
ttl INTEGER DEFAULT 64,
@@ -41,8 +44,8 @@ def create_innocuous_macs_db(db_path: str) -> None:
os_family TEXT DEFAULT 'Linux'
);
CREATE INDEX IF NOT EXISTS idx_profiles_category ON profiles(category);
CREATE INDEX IF NOT EXISTS idx_profiles_oui ON profiles(oui);
CREATE INDEX IF NOT EXISTS idx_mac_profiles_type ON mac_profiles(device_type);
CREATE INDEX IF NOT EXISTS idx_mac_profiles_oui ON mac_profiles(oui);
""")
profiles = [
@@ -83,17 +86,17 @@ def create_innocuous_macs_db(db_path: str) -> None:
"Sony-TV", "Sony", 64, 65535, "Android"),
# ── Smart speakers / assistants ────────────────────────────────
("speaker", "48:A6:B8", "Sonos", "Sonos One",
("smart_speaker", "48:A6:B8", "Sonos", "Sonos One",
"Sonos-", "Sonos", 64, 65535, "Linux"),
("speaker", "5C:AA:FD", "Sonos", "Sonos Beam",
("smart_speaker", "5C:AA:FD", "Sonos", "Sonos Beam",
"Sonos-", "Sonos", 64, 65535, "Linux"),
("speaker", "30:FD:38", "Google", "Google Home Mini",
("smart_speaker", "30:FD:38", "Google", "Google Home Mini",
"Google-Home-", "Google", 64, 65535, "Linux"),
("speaker", "1C:F2:9A", "Google", "Google Nest Hub",
("smart_speaker", "1C:F2:9A", "Google", "Google Nest Hub",
"Google-Nest-", "Google", 64, 65535, "Linux"),
("speaker", "68:54:FD", "Amazon", "Echo Dot",
("smart_speaker", "68:54:FD", "Amazon", "Echo Dot",
"amazon-", "AmazonEcho", 64, 26883, "Android"),
("speaker", "74:C2:46", "Amazon", "Echo Show",
("smart_speaker", "74:C2:46", "Amazon", "Echo Show",
"amazon-", "AmazonEcho", 64, 26883, "Android"),
# ── Smart home / IoT ──────────────────────────────────────────
@@ -148,30 +151,40 @@ def create_innocuous_macs_db(db_path: str) -> None:
("gaming", "7C:BB:8A", "Nintendo", "Nintendo Switch",
"Nintendo-", "Nintendo", 64, 65535, "Nintendo"),
# ── Phones / tablets ──────────────────────────────────────────
("phone", "A4:83:E7", "Apple", "iPhone 15 Pro",
"iphone", "dhcpcd-6.x.x", 64, 65535, "iOS"),
("phone", "40:CB:C0", "Apple", "iPad Air",
"ipad", "dhcpcd-6.x.x", 64, 65535, "iPadOS"),
("tablet", "3C:28:6D", "Samsung", "Galaxy Tab S9",
"Samsung-Tab", "dhcpcd-6.x.x", 64, 65535, "Android"),
("phone", "DC:1A:C5", "Samsung", "Galaxy S24",
"Galaxy-S", "dhcpcd-6.x.x", 64, 65535, "Android"),
# ── Network gear (hide as infrastructure) ─────────────────────
("network", "B4:FB:E4", "Ubiquiti", "UniFi AP",
("iot", "B4:FB:E4", "Ubiquiti", "UniFi AP",
"UAP-", "Ubiquiti", 64, 65535, "Linux"),
("network", "78:8A:20", "Ubiquiti", "USG Gateway",
("iot", "78:8A:20", "Ubiquiti", "USG Gateway",
"USG-", "Ubiquiti", 64, 65535, "Linux"),
("network", "14:CC:20", "TP-Link", "TP-Link Archer AX50",
("iot", "14:CC:20", "TP-Link", "TP-Link Archer AX50",
"TL-", "TP-Link", 64, 65535, "Linux"),
("network", "AC:84:C6", "TP-Link", "TP-Link Deco M5",
("iot", "AC:84:C6", "TP-Link", "TP-Link Deco M5",
"Deco-", "TP-Link", 64, 65535, "Linux"),
("network", "20:A6:CD", "Netgear", "Netgear Orbi",
("iot", "20:A6:CD", "Netgear", "Netgear Orbi",
"Orbi-", "Netgear", 64, 65535, "Linux"),
("network", "C4:04:15", "Netgear", "Netgear Nighthawk",
("iot", "C4:04:15", "Netgear", "Netgear Nighthawk",
"NETGEAR-", "Netgear", 64, 65535, "Linux"),
]
conn.executemany("""
INSERT INTO profiles (category, oui, vendor, model,
dhcp_hostname, dhcp_vendor_class,
ttl, tcp_window, os_family)
INSERT INTO mac_profiles (device_type, oui, vendor, device_name,
dhcp_hostname, dhcp_vendor_class,
ttl, tcp_window, os_family)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
""", profiles)
conn.commit()
count = conn.execute("SELECT COUNT(*) FROM profiles").fetchone()[0]
count = conn.execute("SELECT COUNT(*) FROM mac_profiles").fetchone()[0]
conn.close()
print(f" innocuous_macs.db: {count} device profiles")
@@ -372,47 +385,85 @@ def create_ja3_fingerprints_db(db_path: str) -> None:
JA3 hashes uniquely identify TLS client implementations.
Used by ja3_spoofer to match outbound traffic to common browsers.
Table name and column names match what ja3_spoofer.py queries:
ja3_profiles(name, ja3_hash, description, cipher_suites,
extensions, elliptic_curves, ec_point_formats)
All list columns are stored as JSON arrays.
"""
conn = sqlite3.connect(db_path)
conn.execute("PRAGMA journal_mode=WAL")
conn.executescript("""
CREATE TABLE IF NOT EXISTS fingerprints (
ja3_hash TEXT PRIMARY KEY,
client TEXT NOT NULL,
version TEXT DEFAULT ''
CREATE TABLE IF NOT EXISTS ja3_profiles (
name TEXT PRIMARY KEY,
ja3_hash TEXT NOT NULL,
description TEXT DEFAULT '',
cipher_suites TEXT NOT NULL DEFAULT '[]',
extensions TEXT NOT NULL DEFAULT '[]',
elliptic_curves TEXT NOT NULL DEFAULT '[]',
ec_point_formats TEXT NOT NULL DEFAULT '[0]'
);
CREATE INDEX IF NOT EXISTS idx_ja3_client ON fingerprints(client);
CREATE INDEX IF NOT EXISTS idx_ja3_hash ON ja3_profiles(ja3_hash);
""")
# Common JA3 hashes -- these rotate with browser versions but
# provide a baseline for spoofing. Updated at runtime via
# ja3er.com or similar feeds.
# Mirror of BUILTIN_PROFILES in ja3_spoofer.py — ensures the DB has
# the same data available for external queries and future updates.
import json as _json
ja3s = [
("cd08e31494f9531f560d64c695473da9", "Chrome", "120+"),
("b32309a26951912be7dba376398abc3b", "Chrome", "100-119"),
("eb1d94daa7e0344597e756a1fb6e7054", "Firefox", "120+"),
("e4bb02b26cf670ba1d2e4942440a1cfc", "Firefox", "100-119"),
("773906b0efdefa24a7f2b8eb6985bf37", "Safari", "17+"),
("2ce0b4f7f6e119e26091e74e5c635cca", "Safari", "16"),
("56c70e24355b14b540300c0c15971b8a", "Edge", "120+"),
("555af378b96073da365e42db3052cf7a", "Edge", "100-119"),
("3b5074b1b5d032e5620f69f9f700ff0e", "curl", "7.x"),
("456523fc94726331a4d5a2e1d40b2cd7", "Python requests", "2.x"),
("e7d705a3286e19ea42f587b344ee6865", "Wget", "1.x"),
("6734f37431670b3ab4292b8f60f29984", "Java", "11+"),
("e35c1eda3a26177f1a3f0aebdc2bd319", "Go", "1.19+"),
(
"chrome_120_win",
"cd08e31494f9531f560d64c695473da9",
"Chrome 120 on Windows 10/11",
_json.dumps([0x1301, 0x1302, 0x1303, 0xc02b, 0xc02f, 0xc02c, 0xc030,
0xcca9, 0xcca8, 0xc013, 0xc014, 0x009c, 0x009d, 0x002f, 0x0035]),
_json.dumps([0, 23, 65281, 10, 11, 35, 16, 5, 13, 18, 51, 45, 43, 27, 17513, 21]),
_json.dumps([0x001d, 0x0017, 0x0018]),
_json.dumps([0]),
),
(
"firefox_121_win",
"579ccef312d18482fc42e2b822ca2430",
"Firefox 121 on Windows 10/11",
_json.dumps([0x1301, 0x1303, 0x1302, 0xc02b, 0xc02f, 0xcca9, 0xcca8,
0xc02c, 0xc030, 0xc013, 0xc014, 0x009c, 0x009d, 0x002f, 0x0035]),
_json.dumps([0, 23, 65281, 10, 11, 35, 16, 5, 34, 51, 43, 13, 45, 28, 21]),
_json.dumps([0x001d, 0x0017, 0x0018, 0x0019]),
_json.dumps([0]),
),
(
"edge_120_win",
"b32309a26951912be7dba376398abc3b",
"Edge 120 on Windows 10/11",
_json.dumps([0x1301, 0x1302, 0x1303, 0xc02b, 0xc02f, 0xc02c, 0xc030,
0xcca9, 0xcca8, 0xc013, 0xc014, 0x009c, 0x009d, 0x002f, 0x0035]),
_json.dumps([0, 23, 65281, 10, 11, 35, 16, 5, 13, 18, 51, 45, 43, 27, 17513, 21]),
_json.dumps([0x001d, 0x0017, 0x0018]),
_json.dumps([0]),
),
(
"chrome_120_linux",
"a17a3bfd385b62b1e15606dbd08c9f89",
"Chrome 120 on Linux",
_json.dumps([0x1301, 0x1302, 0x1303, 0xc02b, 0xc02f, 0xc02c, 0xc030,
0xcca9, 0xcca8, 0xc013, 0xc014, 0x009c, 0x009d, 0x002f, 0x0035]),
_json.dumps([0, 23, 65281, 10, 11, 35, 16, 5, 13, 18, 51, 45, 43, 27, 17513, 21]),
_json.dumps([0x001d, 0x0017, 0x0018]),
_json.dumps([0]),
),
]
conn.executemany(
"INSERT OR IGNORE INTO fingerprints (ja3_hash, client, version) VALUES (?, ?, ?)",
ja3s
"INSERT OR IGNORE INTO ja3_profiles "
"(name, ja3_hash, description, cipher_suites, extensions, elliptic_curves, ec_point_formats) "
"VALUES (?, ?, ?, ?, ?, ?, ?)",
ja3s,
)
conn.commit()
count = conn.execute("SELECT COUNT(*) FROM fingerprints").fetchone()[0]
count = conn.execute("SELECT COUNT(*) FROM ja3_profiles").fetchone()[0]
conn.close()
print(f" ja3_fingerprints.db: {count} JA3 fingerprints")
print(f" ja3_fingerprints.db: {count} JA3 profiles")
def main():