From 0f754cfb7fd248f46f687cdf8025841bf6120ac2 Mon Sep 17 00:00:00 2001 From: Cobra Date: Tue, 7 Apr 2026 13:05:31 -0400 Subject: [PATCH] Fix DB table/column mismatch and stale PID noise on startup build_data.py created 'profiles' but mac_manager queries 'mac_profiles'. Columns category/model renamed to device_type/device_name. device_type values aligned with _CATEGORY_MAP in mac_manager (streaming, smart_tv, phone, tablet, smart_speaker, iot, printer, gaming). ja3_fingerprints.db table renamed from 'fingerprints' to 'ja3_profiles' and expanded with all columns ja3_spoofer queries (name, description, cipher_suites, extensions, elliptic_curves, ec_point_formats as JSON). StateManager.reset_module_status() clears stale 'running' entries on startup so watchdog doesn't flag old-session PIDs as dead modules. Engine.start_all() calls reset before launch and does a 3s post-startup liveness check, logging any modules that crashed immediately after fork. --- core/engine.py | 35 +++++++++- core/state.py | 22 +++++++ scripts/build_data.py | 145 ++++++++++++++++++++++++++++-------------- 3 files changed, 152 insertions(+), 50 deletions(-) diff --git a/core/engine.py b/core/engine.py index 826963b..f787316 100644 --- a/core/engine.py +++ b/core/engine.py @@ -363,8 +363,12 @@ class Engine: def start_all(self) -> dict[str, bool]: """Start all registered modules in dependency order.""" + # Clear stale PIDs from any previous run so the watchdog does not + # report modules from the old session as dead. + self.state.reset_module_status() + order = _topo_sort(self._modules) - + # Instantiate and start CaptureBus for passive modules iface = self.config.get("network", {}).get("primary_interface") if iface: @@ -376,17 +380,42 @@ class Engine: except Exception as e: logger.error("Failed to start CaptureBus: %s", e) self.capture_bus = None - + # Inject capture_bus into module configs for passive modules # Some modules use "capture_bus", others "_capture_bus" — inject both for name, entry in self._modules.items(): if getattr(entry.module_class, "requires_capture_bus", False): entry.config["capture_bus"] = self.capture_bus entry.config["_capture_bus"] = self.capture_bus - + results = {} for name in order: results[name] = self.start(name) + + # Post-startup liveness check: wait for module processes to settle, + # then verify each one is still alive and log a clear summary. + time.sleep(3) + alive, dead = [], [] + with self._lock: + for name, entry in self._modules.items(): + if results.get(name): + if entry.process and entry.process.is_alive(): + alive.append(name) + else: + dead.append(name) + # Update state so watchdog doesn't double-report + self.state.set_module_status(name, "stopped") + + if dead: + logger.error( + "Startup check: %d modules died immediately after launch: %s", + len(dead), ", ".join(dead), + ) + logger.info( + "Startup check complete — running: %d, failed: %d", + len(alive), len(dead), + ) + return results def stop_all(self, timeout: float = 5.0) -> None: diff --git a/core/state.py b/core/state.py index df886f8..3df972b 100644 --- a/core/state.py +++ b/core/state.py @@ -232,6 +232,28 @@ class StateManager: for r in rows } + def reset_module_status(self) -> None: + """Mark all running modules as stopped. + + Called at engine startup to clear stale PIDs from a previous run. + Without this, the watchdog picks up old PIDs and reports dead modules + that were never started in the current session. + """ + conn = self._get_write_conn() + try: + conn.execute("BEGIN IMMEDIATE") + conn.execute( + "UPDATE module_status SET status='stopped', pid=NULL, updated=? " + "WHERE status='running'", + (time.time(),), + ) + conn.commit() + except Exception: + try: + conn.rollback() + except Exception: + pass + # ------------------------------------------------------------------ # Write queue # ------------------------------------------------------------------ diff --git a/scripts/build_data.py b/scripts/build_data.py index 6678346..faea8ea 100755 --- a/scripts/build_data.py +++ b/scripts/build_data.py @@ -21,19 +21,22 @@ from pathlib import Path def create_innocuous_macs_db(db_path: str) -> None: """Create innocuous MAC profiles database with ~50 real device profiles. - Each profile includes OUI, vendor, model, DHCP hostname pattern, + Each profile includes OUI, vendor, device_name, DHCP hostname pattern, DHCP vendor class, TTL, TCP window size, and OS family -- everything needed to impersonate the device at the network fingerprint level. + + Table name and column names match what mac_manager.py queries: + mac_profiles(device_type, device_name, vendor, oui, ...) """ conn = sqlite3.connect(db_path) conn.execute("PRAGMA journal_mode=WAL") conn.executescript(""" - CREATE TABLE IF NOT EXISTS profiles ( + CREATE TABLE IF NOT EXISTS mac_profiles ( id INTEGER PRIMARY KEY AUTOINCREMENT, - category TEXT NOT NULL, + device_type TEXT NOT NULL, oui TEXT NOT NULL, vendor TEXT NOT NULL, - model TEXT NOT NULL, + device_name TEXT NOT NULL, dhcp_hostname TEXT, dhcp_vendor_class TEXT, ttl INTEGER DEFAULT 64, @@ -41,8 +44,8 @@ def create_innocuous_macs_db(db_path: str) -> None: os_family TEXT DEFAULT 'Linux' ); - CREATE INDEX IF NOT EXISTS idx_profiles_category ON profiles(category); - CREATE INDEX IF NOT EXISTS idx_profiles_oui ON profiles(oui); + CREATE INDEX IF NOT EXISTS idx_mac_profiles_type ON mac_profiles(device_type); + CREATE INDEX IF NOT EXISTS idx_mac_profiles_oui ON mac_profiles(oui); """) profiles = [ @@ -83,17 +86,17 @@ def create_innocuous_macs_db(db_path: str) -> None: "Sony-TV", "Sony", 64, 65535, "Android"), # ── Smart speakers / assistants ──────────────────────────────── - ("speaker", "48:A6:B8", "Sonos", "Sonos One", + ("smart_speaker", "48:A6:B8", "Sonos", "Sonos One", "Sonos-", "Sonos", 64, 65535, "Linux"), - ("speaker", "5C:AA:FD", "Sonos", "Sonos Beam", + ("smart_speaker", "5C:AA:FD", "Sonos", "Sonos Beam", "Sonos-", "Sonos", 64, 65535, "Linux"), - ("speaker", "30:FD:38", "Google", "Google Home Mini", + ("smart_speaker", "30:FD:38", "Google", "Google Home Mini", "Google-Home-", "Google", 64, 65535, "Linux"), - ("speaker", "1C:F2:9A", "Google", "Google Nest Hub", + ("smart_speaker", "1C:F2:9A", "Google", "Google Nest Hub", "Google-Nest-", "Google", 64, 65535, "Linux"), - ("speaker", "68:54:FD", "Amazon", "Echo Dot", + ("smart_speaker", "68:54:FD", "Amazon", "Echo Dot", "amazon-", "AmazonEcho", 64, 26883, "Android"), - ("speaker", "74:C2:46", "Amazon", "Echo Show", + ("smart_speaker", "74:C2:46", "Amazon", "Echo Show", "amazon-", "AmazonEcho", 64, 26883, "Android"), # ── Smart home / IoT ────────────────────────────────────────── @@ -148,30 +151,40 @@ def create_innocuous_macs_db(db_path: str) -> None: ("gaming", "7C:BB:8A", "Nintendo", "Nintendo Switch", "Nintendo-", "Nintendo", 64, 65535, "Nintendo"), + # ── Phones / tablets ────────────────────────────────────────── + ("phone", "A4:83:E7", "Apple", "iPhone 15 Pro", + "iphone", "dhcpcd-6.x.x", 64, 65535, "iOS"), + ("phone", "40:CB:C0", "Apple", "iPad Air", + "ipad", "dhcpcd-6.x.x", 64, 65535, "iPadOS"), + ("tablet", "3C:28:6D", "Samsung", "Galaxy Tab S9", + "Samsung-Tab", "dhcpcd-6.x.x", 64, 65535, "Android"), + ("phone", "DC:1A:C5", "Samsung", "Galaxy S24", + "Galaxy-S", "dhcpcd-6.x.x", 64, 65535, "Android"), + # ── Network gear (hide as infrastructure) ───────────────────── - ("network", "B4:FB:E4", "Ubiquiti", "UniFi AP", + ("iot", "B4:FB:E4", "Ubiquiti", "UniFi AP", "UAP-", "Ubiquiti", 64, 65535, "Linux"), - ("network", "78:8A:20", "Ubiquiti", "USG Gateway", + ("iot", "78:8A:20", "Ubiquiti", "USG Gateway", "USG-", "Ubiquiti", 64, 65535, "Linux"), - ("network", "14:CC:20", "TP-Link", "TP-Link Archer AX50", + ("iot", "14:CC:20", "TP-Link", "TP-Link Archer AX50", "TL-", "TP-Link", 64, 65535, "Linux"), - ("network", "AC:84:C6", "TP-Link", "TP-Link Deco M5", + ("iot", "AC:84:C6", "TP-Link", "TP-Link Deco M5", "Deco-", "TP-Link", 64, 65535, "Linux"), - ("network", "20:A6:CD", "Netgear", "Netgear Orbi", + ("iot", "20:A6:CD", "Netgear", "Netgear Orbi", "Orbi-", "Netgear", 64, 65535, "Linux"), - ("network", "C4:04:15", "Netgear", "Netgear Nighthawk", + ("iot", "C4:04:15", "Netgear", "Netgear Nighthawk", "NETGEAR-", "Netgear", 64, 65535, "Linux"), ] conn.executemany(""" - INSERT INTO profiles (category, oui, vendor, model, - dhcp_hostname, dhcp_vendor_class, - ttl, tcp_window, os_family) + INSERT INTO mac_profiles (device_type, oui, vendor, device_name, + dhcp_hostname, dhcp_vendor_class, + ttl, tcp_window, os_family) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?) """, profiles) conn.commit() - count = conn.execute("SELECT COUNT(*) FROM profiles").fetchone()[0] + count = conn.execute("SELECT COUNT(*) FROM mac_profiles").fetchone()[0] conn.close() print(f" innocuous_macs.db: {count} device profiles") @@ -372,47 +385,85 @@ def create_ja3_fingerprints_db(db_path: str) -> None: JA3 hashes uniquely identify TLS client implementations. Used by ja3_spoofer to match outbound traffic to common browsers. + + Table name and column names match what ja3_spoofer.py queries: + ja3_profiles(name, ja3_hash, description, cipher_suites, + extensions, elliptic_curves, ec_point_formats) + All list columns are stored as JSON arrays. """ conn = sqlite3.connect(db_path) conn.execute("PRAGMA journal_mode=WAL") conn.executescript(""" - CREATE TABLE IF NOT EXISTS fingerprints ( - ja3_hash TEXT PRIMARY KEY, - client TEXT NOT NULL, - version TEXT DEFAULT '' + CREATE TABLE IF NOT EXISTS ja3_profiles ( + name TEXT PRIMARY KEY, + ja3_hash TEXT NOT NULL, + description TEXT DEFAULT '', + cipher_suites TEXT NOT NULL DEFAULT '[]', + extensions TEXT NOT NULL DEFAULT '[]', + elliptic_curves TEXT NOT NULL DEFAULT '[]', + ec_point_formats TEXT NOT NULL DEFAULT '[0]' ); - CREATE INDEX IF NOT EXISTS idx_ja3_client ON fingerprints(client); + CREATE INDEX IF NOT EXISTS idx_ja3_hash ON ja3_profiles(ja3_hash); """) - # Common JA3 hashes -- these rotate with browser versions but - # provide a baseline for spoofing. Updated at runtime via - # ja3er.com or similar feeds. + # Mirror of BUILTIN_PROFILES in ja3_spoofer.py — ensures the DB has + # the same data available for external queries and future updates. + import json as _json ja3s = [ - ("cd08e31494f9531f560d64c695473da9", "Chrome", "120+"), - ("b32309a26951912be7dba376398abc3b", "Chrome", "100-119"), - ("eb1d94daa7e0344597e756a1fb6e7054", "Firefox", "120+"), - ("e4bb02b26cf670ba1d2e4942440a1cfc", "Firefox", "100-119"), - ("773906b0efdefa24a7f2b8eb6985bf37", "Safari", "17+"), - ("2ce0b4f7f6e119e26091e74e5c635cca", "Safari", "16"), - ("56c70e24355b14b540300c0c15971b8a", "Edge", "120+"), - ("555af378b96073da365e42db3052cf7a", "Edge", "100-119"), - ("3b5074b1b5d032e5620f69f9f700ff0e", "curl", "7.x"), - ("456523fc94726331a4d5a2e1d40b2cd7", "Python requests", "2.x"), - ("e7d705a3286e19ea42f587b344ee6865", "Wget", "1.x"), - ("6734f37431670b3ab4292b8f60f29984", "Java", "11+"), - ("e35c1eda3a26177f1a3f0aebdc2bd319", "Go", "1.19+"), + ( + "chrome_120_win", + "cd08e31494f9531f560d64c695473da9", + "Chrome 120 on Windows 10/11", + _json.dumps([0x1301, 0x1302, 0x1303, 0xc02b, 0xc02f, 0xc02c, 0xc030, + 0xcca9, 0xcca8, 0xc013, 0xc014, 0x009c, 0x009d, 0x002f, 0x0035]), + _json.dumps([0, 23, 65281, 10, 11, 35, 16, 5, 13, 18, 51, 45, 43, 27, 17513, 21]), + _json.dumps([0x001d, 0x0017, 0x0018]), + _json.dumps([0]), + ), + ( + "firefox_121_win", + "579ccef312d18482fc42e2b822ca2430", + "Firefox 121 on Windows 10/11", + _json.dumps([0x1301, 0x1303, 0x1302, 0xc02b, 0xc02f, 0xcca9, 0xcca8, + 0xc02c, 0xc030, 0xc013, 0xc014, 0x009c, 0x009d, 0x002f, 0x0035]), + _json.dumps([0, 23, 65281, 10, 11, 35, 16, 5, 34, 51, 43, 13, 45, 28, 21]), + _json.dumps([0x001d, 0x0017, 0x0018, 0x0019]), + _json.dumps([0]), + ), + ( + "edge_120_win", + "b32309a26951912be7dba376398abc3b", + "Edge 120 on Windows 10/11", + _json.dumps([0x1301, 0x1302, 0x1303, 0xc02b, 0xc02f, 0xc02c, 0xc030, + 0xcca9, 0xcca8, 0xc013, 0xc014, 0x009c, 0x009d, 0x002f, 0x0035]), + _json.dumps([0, 23, 65281, 10, 11, 35, 16, 5, 13, 18, 51, 45, 43, 27, 17513, 21]), + _json.dumps([0x001d, 0x0017, 0x0018]), + _json.dumps([0]), + ), + ( + "chrome_120_linux", + "a17a3bfd385b62b1e15606dbd08c9f89", + "Chrome 120 on Linux", + _json.dumps([0x1301, 0x1302, 0x1303, 0xc02b, 0xc02f, 0xc02c, 0xc030, + 0xcca9, 0xcca8, 0xc013, 0xc014, 0x009c, 0x009d, 0x002f, 0x0035]), + _json.dumps([0, 23, 65281, 10, 11, 35, 16, 5, 13, 18, 51, 45, 43, 27, 17513, 21]), + _json.dumps([0x001d, 0x0017, 0x0018]), + _json.dumps([0]), + ), ] conn.executemany( - "INSERT OR IGNORE INTO fingerprints (ja3_hash, client, version) VALUES (?, ?, ?)", - ja3s + "INSERT OR IGNORE INTO ja3_profiles " + "(name, ja3_hash, description, cipher_suites, extensions, elliptic_curves, ec_point_formats) " + "VALUES (?, ?, ?, ?, ?, ?, ?)", + ja3s, ) conn.commit() - count = conn.execute("SELECT COUNT(*) FROM fingerprints").fetchone()[0] + count = conn.execute("SELECT COUNT(*) FROM ja3_profiles").fetchone()[0] conn.close() - print(f" ja3_fingerprints.db: {count} JA3 fingerprints") + print(f" ja3_fingerprints.db: {count} JA3 profiles") def main():