#!/usr/bin/env python3
"""
net_alerter.py — Net alerter persistent daemon.
Monitors device presence via DHCP sniffing + Netlink neighbor events.
Zero active probing. No traffic generated.

Three concurrent threads:
1. DHCP sniffer — AF_PACKET raw socket, captures DHCP traffic passively
2. RTM_NEWNEIGH watcher — Netlink socket, kernel pushes new ARP neighbor events
3. RTM_DELNEIGH watcher — Netlink socket, kernel pushes deleted ARP neighbor events
"""

import hashlib
import json
import logging
import os
import re
import socket
import struct
import subprocess
import sys
import threading
import time
import urllib.parse
import urllib.request
import sqlite3
from pathlib import Path

# --- Config (from .env or env vars) ---
LOG_FILE = "/opt/net_alerter/net_alerter.log"
OUI_DB_PATH = "/opt/net_alerter/oui.db"
ENV_PATH = Path(os.getenv("NET_ALERTER_ENV", "/opt/net_alerter/.env"))
BLE_NAMES_FILE = Path("/opt/net_alerter/ble_names.json")
BLE_CORRELATION_WINDOW = 60  # seconds — how far back to look for nearby BLE devices
ARP_SCAN_INTERVAL = int(os.getenv("NET_ALERTER_ARP_SCAN_INTERVAL", "15"))

known_devices = {}  # {mac: {ip, hostname, vendor, first_seen, last_seen, infrastructure (opt)}}
known_lock = threading.Lock()
infrastructure_ips = set()  # IPs that should never trigger alerts (gateway, broadcast, self)
infrastructure_macs = set()  # MACs that should never be treated as personal devices
departure_timers = {}  # {mac: threading.Timer} for 15-min departure debounce
departure_lock = threading.Lock()  # Protects departure_timers and last_departed_time
last_departed_time = {}  # {mac: timestamp} when device was actually alerted as departed

# Passive observation tracking for iOS departure detection
last_seen = {}  # {mac: float} — updated by passive ARP/mDNS capture
last_seen_lock = threading.Lock()

# Interface flap detection — suppresses departure storms
_flap_window = []          # [(timestamp, mac)]
_flap_lock = threading.Lock()
_interface_recovering = False
_recovery_timer = None


# Network equipment OUI prefixes (first 3 octets, uppercase no colons)
# Auto-exclusion for known infrastructure devices
NETWORK_EQUIPMENT_OUIS = {
    # Ubiquiti
    "0418D6", "24A43C", "44D9E7", "68729C", "788A20", "802AA8",
    "9C05D6", "DC9FDB", "E063DA", "F09FC2", "6C63F8", "B4FBE4",
    "00156D", "002722", "04186F",
    # Cisco
    "000C29", "001A2F", "001BB1", "001E13", "001F26", "002155",
    "0060B0", "00902B", "008024", "0090AB", "00E014", "001011",
    # Aruba
    "000B86", "00FC8B", "040E3C", "1C28AF", "40E3D6", "6C29D1",
    "84D47E", "884774", "94B40F", "AC3744",
    # TP-Link
    "000AEB", "001777", "1C61B4", "5C63BF", "6046D9", "A84E3F",
    "B4B024", "C46AB7", "EC086B", "F81A67",
    # Netgear
    "001B2F", "00146C", "002196", "00A0CC", "081102", "1C1E29",
    "206A8A", "30469A", "4F2AFF", "6CB0CE", "8C3BAD",
    # MikroTik
    "4C5E0C", "B8069F", "C4AD34", "CC2DE0", "D4CA6D", "E48D8C",
    # Ruckus
    "000C67", "00249B", "04BD88", "14778B", "2C5D93", "34A84E",
}

# Churn tracking — auto-suppresses devices that cycle 3+ times in 30min
_churn_tracker = {}       # {mac: [timestamps of departure events]}
_churn_lock = threading.Lock()
CHURN_THRESHOLD = 3       # cycles in window before suppression
CHURN_WINDOW_SEC = 1800   # 30 minutes
WATCHDOG_TIMEOUT_SEC = int(os.getenv("NET_ALERTER_WATCHDOG_TIMEOUT", "180"))
DEPARTURE_DEBOUNCE_SEC = int(os.getenv("NET_ALERTER_DEPARTURE_DEBOUNCE", "180"))

# Mobile device OUI prefixes for personal device detection
MOBILE_DEVICE_OUIS = {
    # Apple
    "28A02E", "A4C3F0", "00A0C6", "38C086", "5C4950", "7061D0", "88A29E",
    "A81D6A", "A8BBCF", "AC3744", "B0EE7B", "B82DFE", "BC9674", "C8DFC0",
    "D4996F", "D8BB85", "E0D55E", "F4CE46", "F8FF9B", "FCFC48",
    # Google Pixel
    "28876D", "34E5FB", "4C80F3", "70EBF2", "AC16E5", "D0A05F", "E4C547",
    # Samsung
    "00166B", "001843", "001D67", "001FAF", "0060B0", "084ECE", "0825F3",
    "088038", "0A49EF", "0C1432", "0EBB1B", "1851BF", "205C85", "20C18B",
    "240A64", "2C5A7B", "32A639", "360DFE", "3A590D", "3CC06C", "441735",
    "44B16C", "48D76E", "58D55C", "5C2EE4", "68C903", "6AC4C6", "6EAA0A",
    "7045AC", "7499EA", "74D4DD", "781F02", "789098", "7C59B9", "7E0A6D",
    # OnePlus
    "00166B", "001BF3", "001C47", "002000", "0026F2", "3C28CB", "38DB23",
    "42BAFC", "5C55F9", "681E7D", "A8178E", "AA6B1F", "BC1E1A", "EA96D7",
    # Xiaomi
    "0022AA", "002255", "003677", "3418C2", "48EE0C", "54EF25", "7418DB",
    "7C8B5C", "7E4A5D", "868A5A", "8A6FB8", "8C8E14", "8EBE59", "A0871B",
    "B8328D", "B832D6", "BA5294", "BC7C47", "C8816D", "E4AF17", "E899C3",
    # Huawei
    "00E04C", "001097", "001093", "001A2B", "001D3C", "001E37", "001F3E",
    "001ECE", "002074", "00207B", "0020E2", "0020F1", "0020F5", "002268",
    "002269", "00226A",
    # LG
    "001E4B", "001E8D", "00218B", "002215", "0022FB", "0023B2", "0023F8",
    "002655", "00266C", "00AB8B", "30F9ED", "D8D0DF",
    # HTC
    "001072", "001058", "0010C6", "001153", "001167", "0011BD", "00121F",
    "00121C", "001212", "001236", "00124B", "00131D", "0013D4", "0013E8",
    # Motorola
    "001555", "001620", "001645", "001693", "0016B8", "001730", "0017F6",
    "001822", "001888", "0018F8", "001913", "001985", "001A45", "001A92",
    "001AFC",
    # Garmin (GPS watches/wearables with WiFi — do not use private MACs)
    "086698", "F8B5AB", "E87941", "1430C6", "801F02", "68F3B1", "283B96", "24DEC6",
    # Fitbit (wearables with WiFi sync — do not use private MACs)
    "E890AC", "88A70E", "E407AA", "A41566", "008B17",
}

# Personal device tracking
personal_devices = set()  # {mac: ...} of detected personal device MACs
personal_names = set()  # {name: ...} device names for auto-discovery via mDNS
device_labels: dict = {}  # {normalized_mac: friendly_name} from NET_ALERTER_DEVICE_LABELS
personal_lock = threading.Lock()
location_occupancy = "UNKNOWN"  # Current location state: VACANT, OCCUPIED, UNKNOWN
occupancy_lock = threading.Lock()

# Multi-source device identity store (zero-config enrollment)
# Key: stable identity (BLE Handoff sequence anchor or DHCP Option 55 hash)
# Value: {identity_id, wifi_macs, ble_macs, last_seen, signal_count, enrolled, dhcp_fingerprint, handoff_seq, first_seen}
device_store = {}  # {identity_id: device_record}
device_store_lock = threading.RLock()

# BLE advertisement parsing constants
BLE_COMPANY_ID_APPLE = 0x004C  # Little-endian in AD structure
BLE_HANDOFF_MSG_TYPE = 0x0C
BLE_HANDOFF_SEQ_OFFSET = 4  # Bytes 4-5 of handoff payload contain seq number


# --- Multi-source device identity functions ---

def _dhcp_fingerprint_hash(option55_bytes: bytes) -> str:
    """Compute stable fingerprint of DHCP Option 55 (Parameter Request List).

    Returns hex digest of first 8 bytes of SHA256 hash.
    """
    if not option55_bytes:
        return ""
    return hashlib.sha256(option55_bytes).hexdigest()[:16]


def _create_identity_record(identity_id: str) -> dict:
    """Create a new device identity record."""
    return {
        'identity_id': identity_id,
        'wifi_macs': set(),
        'ble_macs': set(),
        'last_seen': {},  # {mac: timestamp}
        'signal_types': set(),  # Set of distinct signal categories: {'ble'} or {'wifi'} or both
        'enrolled': False,
        'dhcp_fingerprint': "",  # DHCP Option 55 fingerprint (hex string)
        'handoff_seq': None,  # Last BLE Handoff sequence number seen
        'first_seen': time.time(),
    }


def _fire_active_probes(identity_id: str, mac: str, signal_type: str, dhcp_fp: str) -> None:
    """Disabled — passive-only operation. No active probes sent."""
    return


def _fire_active_probes_DISABLED(identity_id: str, mac: str, signal_type: str, dhcp_fp: str) -> None:
    def _probe_worker():
        try:
            with device_store_lock:
                if identity_id not in device_store:
                    return
                dev = device_store[identity_id]

                # Collect all known MACs for this identity
                all_macs = dev['wifi_macs'] | dev['ble_macs']

                # Try to find IP from last_seen tracking
                ip_for_probe = None
                for m in all_macs:
                    if m in dev['last_seen']:
                        # Check if we can infer IP from known_devices (if MAC was previously seen)
                        with known_lock:
                            if m in known_devices:
                                ip_for_probe = known_devices[m].get('ip')
                                break

            if ip_for_probe:
                logging.info(f"New device identity {identity_id}: firing active probes for IP {ip_for_probe}")
                try:
                    # Attempt ARP ping using subprocess arping (lightweight, no extra deps)
                    subprocess.run(['arping', '-c', '1', '-w', '1', ip_for_probe],
                                   stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, timeout=2)
                    logging.debug(f"ARP probe sent to {ip_for_probe} (identity {identity_id})")
                except (FileNotFoundError, subprocess.TimeoutExpired, Exception):
                    # arping not available or timed out — try ping as fallback
                    try:
                        subprocess.run(['ping', '-c', '1', '-W', '1', ip_for_probe],
                                       stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, timeout=2)
                        logging.debug(f"ICMP ping sent to {ip_for_probe} (identity {identity_id})")
                    except (subprocess.TimeoutExpired, Exception):
                        pass
            else:
                logging.debug(f"New device identity {identity_id}: no IP available for active probes yet")
        except Exception as e:
            logging.debug(f"Error in active probe thread for {identity_id}: {e}")

    # Fire probe in background thread to avoid blocking
    probe_thread = threading.Thread(target=_probe_worker, daemon=True)
    probe_thread.start()


def _fire_enrollment_callback(identity_id: str) -> None:
    """Called when a device meets enrollment criteria (1+ signal types).

    Logs enrollment, sends no alert (silent enrollment).
    Caller must hold device_store_lock.
    """
    if identity_id not in device_store:
        return
    dev = device_store[identity_id]
    if not dev['enrolled']:
        dev['enrolled'] = True
        all_macs = dev['wifi_macs'] | dev['ble_macs']
        logging.info(f"Device enrolled (multi-source identity {identity_id}): {len(all_macs)} MAC(s), signal_types={dev['signal_types']}")


def _correlate_or_create_identity(mac: str, signal_type: str, dhcp_fp: str = "", handoff_seq: int = None) -> str:
    """
    Correlate a MAC with an existing identity or create a new one.

    signal_type: "ble", "dhcp", "arp", or "mdns"

    Lookup strategy:
    0. Check if MAC already exists in any identity's MAC sets
    1. If handoff_seq provided, look for matching BLE handoff seq anchor
    2. If dhcp_fp provided, look for matching DHCP fingerprint
    3. If neither, create new provisional identity

    Returns the identity_id (stable key).
    """
    with device_store_lock:
        # Determine signal category: 'ble', 'mdns', or 'wifi' (for dhcp/arp)
        if signal_type == 'ble':
            signal_category = 'ble'
        elif signal_type == 'mdns':
            signal_category = 'mdns'
        else:  # dhcp, arp
            signal_category = 'wifi'

        # Strategy 0: Check if MAC already exists in any identity
        for iid, dev in device_store.items():
            if mac in dev['wifi_macs'] or mac in dev['ble_macs']:
                # Found existing identity for this MAC — update instead of creating new
                dev['last_seen'][mac] = time.time()
                dev['signal_types'].add(signal_category)
                logging.debug(f"MAC {mac} already linked to identity {iid}, updated {signal_category} signal")
                # Check if meets enrollment criteria: (2+ signal types) OR (mdns present)
                if not dev['enrolled'] and len(dev['signal_types']) >= 1:
                    _fire_enrollment_callback(iid)
                return iid

        # Strategy 1: Match by BLE Handoff sequence (most stable)
        if handoff_seq is not None:
            for iid, dev in device_store.items():
                if dev['handoff_seq'] == handoff_seq:
                    # Found matching handoff anchor — add this MAC
                    dev['ble_macs'].add(mac)
                    dev['last_seen'][mac] = time.time()
                    dev['signal_types'].add('ble')
                    logging.debug(f"BLE MAC rotation detected: {mac} linked to identity {iid} (seq {handoff_seq})")
                    return iid

        # Strategy 2: Match by DHCP fingerprint
        if dhcp_fp:
            for iid, dev in device_store.items():
                if dev['dhcp_fingerprint'] == dhcp_fp and dev['dhcp_fingerprint']:
                    # Found matching DHCP fingerprint — add this WiFi MAC
                    if mac not in dev['wifi_macs']:
                        old_macs = dev['wifi_macs'].copy()
                        dev['wifi_macs'].add(mac)
                        dev['last_seen'][mac] = time.time()
                        if old_macs:
                            logging.debug(f"MAC rotation (DHCP fingerprint match): {old_macs} → {mac}, identity {iid}")
                    # Add wifi signal type if not already present
                    dev['signal_types'].add('wifi')
                    # Check if meets enrollment criteria: (2+ signal types) OR (mdns present)
                    if not dev['enrolled'] and len(dev['signal_types']) >= 1:
                        _fire_enrollment_callback(iid)
                    return iid

        # Strategy 3: Create new provisional identity
        # Use DHCP fingerprint or handoff seq as primary key if available, else random
        if dhcp_fp:
            identity_id = f"dhcp:{dhcp_fp}"
        elif handoff_seq is not None:
            identity_id = f"ble_seq:{handoff_seq}"
        else:
            # Provisional identity based on first MAC seen
            identity_id = f"prov:{mac}"

        # Check if this identity already exists (e.g., multiple mDNS/ARP signals for same MAC)
        if identity_id in device_store:
            dev = device_store[identity_id]
            # Update existing record instead of overwriting
            if signal_type == "ble":
                dev['ble_macs'].add(mac)
                if handoff_seq is not None:
                    dev['handoff_seq'] = handoff_seq
                dev['signal_types'].add('ble')
            else:
                dev['wifi_macs'].add(mac)
                if dhcp_fp:
                    dev['dhcp_fingerprint'] = dhcp_fp
                dev['signal_types'].add(signal_category)
            dev['last_seen'][mac] = time.time()
            # Check if meets enrollment criteria: (2+ signal types) OR (mdns present)
            if not dev['enrolled'] and len(dev['signal_types']) >= 1:
                _fire_enrollment_callback(identity_id)
            logging.debug(f"Updated device identity {identity_id} with {signal_type} signal (MAC: {mac})")
        else:
            # Create new identity
            dev = _create_identity_record(identity_id)
            is_new_identity = True

            # Add initial signal
            if signal_type == "ble":
                dev['ble_macs'].add(mac)
                if handoff_seq is not None:
                    dev['handoff_seq'] = handoff_seq
                dev['signal_types'].add('ble')
            else:
                dev['wifi_macs'].add(mac)
                if dhcp_fp:
                    dev['dhcp_fingerprint'] = dhcp_fp
                dev['signal_types'].add(signal_category)

            dev['last_seen'][mac] = time.time()
            device_store[identity_id] = dev
            logging.debug(f"Created new device identity {identity_id} from {signal_type} signal")

            # Fire active probes for new identity (non-blocking)
            _fire_active_probes(identity_id, mac, signal_type, dhcp_fp)

        return identity_id


def _ingest_signal(mac: str, signal_type: str, dhcp_fp: str = "", handoff_seq: int = None) -> str:
    """
    Ingest a signal from a device source (BLE, DHCP, ARP, mDNS).

    Returns the identity_id it was correlated to.
    Handles enrollment when signal_count reaches 2+ from distinct types or when mdns signal is present.
    """
    identity_id = _correlate_or_create_identity(mac, signal_type, dhcp_fp, handoff_seq)

    with device_store_lock:
        if identity_id not in device_store:
            return identity_id

        dev = device_store[identity_id]
        dev['last_seen'][mac] = time.time()

        # Enrollment check: if we have (2+ distinct signal types) OR (mdns present) and not yet enrolled
        if not dev['enrolled'] and len(dev['signal_types']) >= 1:
            _fire_enrollment_callback(identity_id)

        # Eviction: if device_store exceeds 500 entries, evict 100 oldest by first_seen
        if len(device_store) > 500:
            sorted_ids = sorted(device_store.keys(), key=lambda iid: device_store[iid]['first_seen'])
            for iid in sorted_ids[:100]:
                del device_store[iid]
            logging.debug(f"Evicted 100 oldest device identities; store size now {len(device_store)}")

    return identity_id


def _check_flap(mac: str) -> bool:
    """Return True if this departure is part of an interface flap (suppress it)."""
    global _interface_recovering, _recovery_timer
    now = time.time()
    with _flap_lock:
        _flap_window[:] = [(t, m) for t, m in _flap_window if now - t < 3.0]
        _flap_window.append((now, mac))
        if len(_flap_window) >= 3 and not _interface_recovering:
            _interface_recovering = True
            if _recovery_timer:
                _recovery_timer.cancel()
            _recovery_timer = threading.Timer(60.0, _clear_recovery)
            _recovery_timer.daemon = True
            _recovery_timer.start()
            logging.info("Interface flap detected — suppressing departures for 60s")
        return _interface_recovering


def _clear_recovery():
    global _interface_recovering
    _interface_recovering = False
    logging.info("Interface flap recovery window ended")


def _check_churn(mac: str, ip: str) -> bool:
    """Return True and suppress if this device churns too frequently."""
    # Explicitly labeled personal devices are never churn-suppressed into infrastructure
    with personal_lock:
        if mac in device_labels or mac in personal_devices:
            return False
    now = time.time()
    with _churn_lock:
        events = _churn_tracker.get(mac, [])
        events = [t for t in events if now - t < CHURN_WINDOW_SEC]
        events.append(now)
        _churn_tracker[mac] = events
        if len(events) >= CHURN_THRESHOLD:
            infrastructure_ips.add(ip)
            logging.info(f"Churn suppression: {ip} (MAC:{mac}) cycled {len(events)}x in {CHURN_WINDOW_SEC//60}min — auto-added to infrastructure")
            return True
    return False
# Top 200 common device OUI prefixes (MAC first 3 octets)
OUI_DICT = {
    "88A29E": "Apple",
    "001A7D": "Apple",
    "185F3F": "Apple",
    "A4C3F0": "Apple",
    "0899D8": "Apple",
    "004096": "Apple",
    "00219B": "Apple",
    "0021E9": "Apple",
    "005973": "Apple",
    "006377": "Apple",
    "0064B9": "Apple",
    "0084F3": "Apple",
    "00A04D": "Apple",
    "00D04B": "Apple",
    "28879F": "Google",
    "2887BA": "Google",
    "5427EB": "Google",
    "542758": "Google",
    "341513": "Amazon",
    "0C47C2": "Amazon",
    "B827EB": "Raspberry Pi",
    "2C56DC": "Raspberry Pi",
    "E45F01": "Raspberry Pi",
    "000D82": "Intel",
    "001025": "Intel",
    "001ABA": "Intel",
    "001F3B": "Intel",
    "001F3C": "Intel",
    "001BA9": "Intel",
    "001BFB": "Intel",
    "001D7A": "Intel",
    "001DB8": "Intel",
    "001E67": "Intel",
    "001F29": "Intel",
    "F44DA2": "Intel",
    "B03C1C": "Intel",
    "00166B": "Qualcomm",
    "001BF3": "Qualcomm",
    "001C47": "Qualcomm",
    "002000": "Qualcomm",
    "0026F2": "Qualcomm",
    "0026F3": "Qualcomm",
    "0026F4": "Qualcomm",
    "002618": "Qualcomm",
    "FFFFFF": "Espressif",
    "687DA8": "Espressif",
    "A01D48": "Espressif",
    "30AEA4": "Espressif",
    "B4E63B": "Espressif",
    "80E386": "TP-Link",
    "8CBF26": "TP-Link",
    "90843F": "TP-Link",
    "A838B4": "TP-Link",
    "C81F66": "TP-Link",
    "DCFEEE": "TP-Link",
    "F81A67": "TP-Link",
    "689B5A": "Netgear",
    "6C46CB": "Netgear",
    "78A21B": "Netgear",
    "94B3D5": "Netgear",
    "B0EE7B": "Netgear",
    "B4FBE4": "Netgear",
    "C8D7B4": "Netgear",
    "D8BB85": "Netgear",
    "F8AB05": "Netgear",
    "001D2F": "Cisco",
    "001930": "Cisco",
    "001A2F": "Cisco",
    "001C0E": "Cisco",
    "001CED": "Cisco",
    "001EBD": "Cisco",
    "001F6C": "Cisco",
    "0021A0": "Cisco",
    "002678": "Cisco",
    "0026CB": "Cisco",
    "00266E": "Cisco",
    "00269F": "Cisco",
    "0026A6": "Cisco",
    "002700": "Cisco",
    "0026FA": "Cisco",
    "00267B": "Cisco",
    "0C1234": "Cisco",
    "001BD7": "Dell",
    "001BDB": "Dell",
    "001A6B": "Dell",
    "003067": "Dell",
    "00188B": "Dell",
    "001B21": "Dell",
    "001E67": "Dell",
    "0026F7": "Dell",
    "00242A": "Dell",
    "001C23": "HP",
    "001E0B": "HP",
    "001EBB": "HP",
    "001F2F": "HP",
    "001BFC": "HP",
    "002219": "HP",
    "0022A6": "HP",
    "002264": "HP",
    "0024A9": "HP",
    "0026B9": "HP",
    "0026DF": "HP",
    "001A6C": "Lenovo",
    "001DFB": "Lenovo",
    "001F3A": "Lenovo",
    "001A73": "Lenovo",
    "00238B": "Lenovo",
    "002264": "Lenovo",
    "00B0D0": "Lenovo",
    "9C2EBF": "Lenovo",
    "F0DE8D": "Lenovo",
    "00E04C": "Huawei",
    "001097": "Huawei",
    "001093": "Huawei",
    "001A2B": "Huawei",
    "001D3C": "Huawei",
    "001E37": "Huawei",
    "001F3E": "Huawei",
    "001ECE": "Huawei",
    "002074": "Huawei",
    "00207B": "Huawei",
    "0020E2": "Huawei",
    "0020F1": "Huawei",
    "0020F5": "Huawei",
    "002268": "Huawei",
    "002269": "Huawei",
    "00226A": "Huawei",
    "0022AA": "Xiaomi",
    "002255": "Xiaomi",
    "003677": "Xiaomi",
    "3418C2": "Xiaomi",
    "50EB71": "Xiaomi",
    "5CF4D4": "Xiaomi",
    "78111D": "Xiaomi",
    "78E1D3": "Xiaomi",
    "A4ABDA": "Xiaomi",
    "E4F47A": "Xiaomi",
    "F45EAB": "Xiaomi",
    "A4D18D": "OnePlus",
    "50F5DA": "OnePlus",
    "8C03BE": "OnePlus",
    "002261": "LG",
    "002265": "LG",
    "0022D2": "LG",
    "0022D3": "LG",
    "000426": "LG",
    "001854": "LG",
    "001E8F": "LG",
    "001F5B": "LG",
    "002038": "LG",
    "0020A6": "LG",
    "0020F8": "LG",
    "002220": "LG",
    "00222D": "LG",
    "0027E4": "LG",
    "00242C": "LG",
    "002531": "LG",
    "001054": "Sony",
    "001458": "Sony",
    "0014EA": "Sony",
    "001525": "Sony",
    "00188A": "Sony",
    "001B44": "Sony",
    "00215E": "Sony",
    "002178": "Sony",
    "0021C5": "Sony",
    "002333": "Sony",
    "002481": "Sony",
    "0026FF": "Sony",
    "002702": "Nintendo",
    "0008FE": "Nintendo",
    "0013F7": "Nintendo",
    "001BDB": "Nintendo",
    "001EEE": "Nintendo",
    "001FEE": "Nintendo",
    "0020F8": "Nintendo",
    "002095": "Nintendo",
    "0020FA": "Nintendo",
    "00227B": "Nintendo",
}


def setup_logging():
    """Setup logging to file + stdout."""
    formatter = logging.Formatter(
        fmt='%(asctime)s [%(levelname)s] %(message)s',
        datefmt='%Y-%m-%dT%H:%M:%S'
    )

    # File handler
    Path(LOG_FILE).parent.mkdir(parents=True, exist_ok=True)
    file_handler = logging.FileHandler(LOG_FILE)
    file_handler.setFormatter(formatter)

    # Stdout handler
    stdout_handler = logging.StreamHandler(sys.stdout)
    stdout_handler.setFormatter(formatter)

    logger = logging.getLogger()
    logger.setLevel(logging.DEBUG)
    logger.addHandler(file_handler)
    logger.addHandler(stdout_handler)

    return logger


# MAC address validation regex: exactly 12 hex digits (after normalization)
_MAC_RE = re.compile(r'^[0-9A-F]{12}$')

def _normalize_mac(mac: str) -> str:
    """Normalize MAC address: remove colons/hyphens, uppercase. Returns empty string if invalid."""
    normalized = mac.replace(":", "").replace("-", "").upper()
    if not _MAC_RE.match(normalized):
        return ""
    return normalized


def load_personal_macs_from_env() -> None:
    """Parse NET_ALERTER_PERSONAL_MACS and NET_ALERTER_PERSONAL_NAMES env vars.
    
    Adds MACs from NET_ALERTER_PERSONAL_MACS to personal_devices set.
    Adds device names from NET_ALERTER_PERSONAL_NAMES to personal_names set for mDNS discovery.
    """
    macs_str = os.getenv("NET_ALERTER_PERSONAL_MACS", "")
    names_str = os.getenv("NET_ALERTER_PERSONAL_NAMES", "")
    
    with personal_lock:
        # Load explicit MACs
        if macs_str.strip():
            macs = [m.strip() for m in macs_str.split(",") if m.strip()]
            for mac in macs:
                normalized = _normalize_mac(mac)
                if not normalized:  # Skip invalid MACs
                    logging.warning(f"Skipping invalid MAC from NET_ALERTER_PERSONAL_MACS: {mac}")
                    continue
                personal_devices.add(normalized)
        
        # Load device names for mDNS auto-discovery
        if names_str.strip():
            names = [n.strip() for n in names_str.split(",") if n.strip()]
            for name in names:
                personal_names.add(name)
        
        # Load device labels (MAC=Name pairs for readable alerts)
        labels_str = os.getenv("NET_ALERTER_DEVICE_LABELS", "")
        if labels_str.strip():
            for pair in labels_str.split(","):
                if "=" not in pair:
                    continue
                mac_part, _, name_part = pair.partition("=")
                normalized = _normalize_mac(mac_part.strip())
                if normalized:
                    device_labels[normalized] = name_part.strip()

        # Log summary
        if personal_devices or personal_names:
            logging.info(f"Loaded {len(personal_devices)} personal device MAC(s) and {len(personal_names)} personal name(s)")


def load_infrastructure_macs_from_env() -> None:
    """Parse NET_ALERTER_INFRA_MACS env var into infrastructure_macs set."""
    global infrastructure_macs
    macs_str = _read_env_value("NET_ALERTER_INFRA_MACS") or os.getenv("NET_ALERTER_INFRA_MACS", "")
    for mac in (m.strip() for m in macs_str.split(",") if m.strip()):
        normalized = _normalize_mac(mac)
        if normalized:
            infrastructure_macs.add(normalized)
            logging.info(f"Seeded infrastructure MAC {mac} — excluded from personal device detection")
        else:
            logging.warning(f"Skipping invalid MAC from NET_ALERTER_INFRA_MACS: {mac}")


def _read_env_value(key: str) -> str:
    """Read a single environment variable from .env file or system env."""
    if ENV_PATH.exists():
        try:
            for line in ENV_PATH.read_text().splitlines():
                line = line.strip()
                if not line or line.startswith("#"):
                    continue
                if "=" not in line:
                    continue
                k, v = line.split("=", 1)
                if k == key:
                    return v
        except Exception as e:
            logging.error(f"Failed to read {key} from .env: {e}")
    return os.getenv(key, "")


def load_env():
    """Parse .env file manually (key=value), no dependency required.

    Note: Matrix credentials (MATRIX_HOMESERVER, MATRIX_ACCESS_TOKEN, MATRIX_ROOM_ID)
    are NOT cached in globals. They are read at call time in send_alert() to minimize
    credential lifetime in process memory.
    """
    if not ENV_PATH.exists():
        logging.warning(f".env not found at {ENV_PATH}, using env vars only")
        load_personal_macs_from_env()
        return

    try:
        for line in ENV_PATH.read_text().splitlines():
            line = line.strip()
            if not line or line.startswith("#"):
                continue
            if "=" not in line:
                continue
            # Only load non-credential config here
            # Credentials are read at call time to minimize exposure
        logging.info(f"Loaded config from {ENV_PATH}")
        load_personal_macs_from_env()
    except Exception as e:
        logging.error(f"Failed to load .env: {e}")


def is_personal_device(mac: str) -> bool:
    """
    Check if MAC belongs to a phone or wearable.

    Three signals, checked in order:
    1. Explicit config — device_labels or personal_devices (operator-configured)
    2. OUI table — MOBILE_DEVICE_OUIS covers phones + Garmin/Fitbit (no private MACs)
    3. LAA bit — locally-administered bit set = iOS 14+/Android 10+ private MAC = phone/watch
    """
    mac_normalized = _normalize_mac(mac)
    if not mac_normalized:
        return False

    oui = mac_normalized[:6]

    with personal_lock:
        if mac_normalized in personal_devices or mac_normalized in device_labels:
            return True

    if oui in MOBILE_DEVICE_OUIS:
        return True

    # Infrastructure MACs are never personal, regardless of LAA bit
    if mac_normalized in infrastructure_macs:
        return False

    # Locally-administered bit: modern phones randomize MACs; TVs/infra don't
    try:
        return bool(int(mac_normalized[:2], 16) & 0x02)
    except ValueError:
        return False


def lookup_oui(mac: str) -> str:
    """
    Look up OUI vendor from MAC address (first 3 octets).
    Tries SQLite database first, falls back to inline dictionary.
    Returns vendor name or 'unknown' if not found.
    """
    if not mac or mac == "00:00:00:00:00:00":
        return "unknown"

    # Try database first
    oui_prefix = mac[:8].lower()
    try:
        conn = sqlite3.connect(OUI_DB_PATH, timeout=1)
        conn.row_factory = sqlite3.Row
        cursor = conn.cursor()
        result = cursor.execute("SELECT vendor FROM oui WHERE prefix = ?", (oui_prefix,)).fetchone()
        conn.close()
        if result:
            return result['vendor']
    except Exception:
        # Log at DEBUG only, no output noise
        logging.debug(f"OUI database lookup failed for {oui_prefix}", exc_info=False)

    # Fall back to inline dictionary
    oui_prefix_upper = oui_prefix.replace(":", "").upper()
    return OUI_DICT.get(oui_prefix_upper, "unknown")


def seed_from_arp_cache():
    """Read /proc/net/arp, populate known_devices without alerting."""
    global known_devices

    try:
        lines = Path("/proc/net/arp").read_text().splitlines()
        for line in lines:
            if line.startswith("IP"):
                continue
            fields = line.split()
            if len(fields) < 4:
                continue

            ip = fields[0]
            mac = fields[3].lower()
            flags = fields[2]

            if mac == "00:00:00:00:00:00" or flags == "0x0":
                continue

            # Personal devices (phones, wearables — any LAA MAC, known mobile OUI,
            # or explicitly labeled MAC) must NOT be silently seeded. The operator
            # needs ARRIVED for every device they don't already know about.
            if is_personal_device(mac):
                with last_seen_lock:
                    if mac not in last_seen:
                        last_seen[mac] = time.time()
                continue

            vendor = lookup_oui(mac)
            with known_lock:
                if mac not in known_devices:
                    dev = {
                        'ip': ip,
                        'hostname': ip,
                        'vendor': vendor,
                        'first_seen': time.time(),
                        'last_seen': time.time()
                    }
                    # Mark infrastructure IPs so they never alert
                    if ip in infrastructure_ips:
                        dev['infrastructure'] = True
                    known_devices[mac] = dev
                    # Seed last_seen so the watchdog has a baseline.
                    # Devices that are genuinely active will refresh this via ARP
                    # requests or mDNS; AP proxy ghosts (only ARP replies) won't.
                    with last_seen_lock:
                        if mac not in last_seen:
                            last_seen[mac] = time.time()

        logging.info(f"Seeded {len(known_devices)} devices from ARP cache")
    except Exception as e:
        logging.error(f"Failed to seed from ARP cache: {e}")


def send_alert(msg: str) -> None:
    """Send alert to Matrix room.

    Credentials are read at call time from .env/env vars to minimize
    their lifetime in process memory.
    """
    token = _read_env_value("MATRIX_ACCESS_TOKEN")
    if not token:
        logging.warning("No MATRIX_ACCESS_TOKEN set, skipping alert")
        return

    homeserver = _read_env_value("MATRIX_HOMESERVER")
    if not homeserver:
        logging.warning("No MATRIX_HOMESERVER set, skipping alert")
        return
    room_id = _read_env_value("MATRIX_ROOM_ID")
    if not room_id:
        logging.warning("No MATRIX_ROOM_ID set, skipping alert")
        return

    url = (
        f"{homeserver}/_matrix/client/v3/rooms/"
        f"{urllib.parse.quote(room_id, safe='')}/send/m.room.message"
    )
    payload = json.dumps({"msgtype": "m.text", "body": msg}).encode()
    req = urllib.request.Request(
        url,
        data=payload,
        headers={
            "Authorization": f"Bearer {token}",
            "Content-Type": "application/json",
            "User-Agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36",
        },
        method="POST",
    )
    try:
        with urllib.request.urlopen(req, timeout=10) as r:
            pass
        logging.debug(f"Alert sent to Matrix")
    except urllib.error.HTTPError as e:
        logging.error(f"Matrix send failed with HTTP {e.code}")
    except Exception as e:
        logging.error(f"Matrix send exception: {e}")


def fmt_duration(secs: float) -> str:
    """Format duration in seconds to human-readable format."""
    if secs < 60:
        return f"{int(secs)}s"
    elif secs < 3600:
        m = int(secs // 60)
        s = int(secs % 60)
        return f"{m}m {s}s" if s else f"{m}m"
    else:
        h = int(secs // 3600)
        m = int((secs % 3600) // 60)
        return f"{h}h {m}m" if m else f"{h}h"


def format_arrival(hostname: str, ip: str, vendor: str, mac: str) -> str:
    """Format arrival alert message. Label always wins when set; hostname is fallback."""
    label = device_labels.get(_normalize_mac(mac))
    display = label or (hostname if hostname and hostname != ip else None)
    if display:
        return f"[NET] ARRIVED: {display} ({ip})"
    return f"[NET] ARRIVED: {ip} [{vendor}] MAC:{mac}"


def format_departure(hostname: str, ip: str, vendor: str, mac: str, duration: str) -> str:
    """Format departure alert message. Label always wins when set; hostname is fallback."""
    label = device_labels.get(_normalize_mac(mac))
    display = label or (hostname if hostname and hostname != ip else None)
    if display:
        return f"[NET] DEPARTED: {display} ({ip}) — present {duration}"
    return f"[NET] DEPARTED: {ip} [{vendor}] MAC:{mac} — present {duration}"


def _update_occupancy_state_unlocked() -> None:
    """
    Internal: Update location occupancy state.
    Assumes known_lock AND occupancy_lock are already held by caller.
    """
    global location_occupancy

    # Delegate to is_personal_device: checks explicit config, OUI table (phones + Garmin/Fitbit),
    # and LAA bit (iOS 14+/Android 10+ private MACs). No pre-config needed for modern phones.
    _is_personal = is_personal_device

    active_enrolled_count = 0
    for mac, dev in known_devices.items():
        if _is_personal(mac) and not dev.get('departing', False) and dev['ip'] not in infrastructure_ips:
            active_enrolled_count += 1

    # Determine new occupancy state
    new_occupancy = "OCCUPIED" if active_enrolled_count > 0 else "VACANT"

    # Fire alert on all transitions including startup UNKNOWN → OCCUPIED
    if new_occupancy != location_occupancy:
        if new_occupancy == "OCCUPIED":
            present = []
            for mac, dev in known_devices.items():
                if _is_personal(mac) and not dev.get('departing', False) and dev['ip'] not in infrastructure_ips:
                    label = device_labels.get(_normalize_mac(mac)) or mac[-8:]
                    present.append(label)
            msg = f"[NET] Location: OCCUPIED ({', '.join(present)})"
        else:
            msg = f"[NET] Location: VACANT"
        logging.info(msg)
        send_alert(msg)

        location_occupancy = new_occupancy
        logging.info(f"Location occupancy updated to {location_occupancy} ({active_enrolled_count} personal devices active)")


def update_occupancy_state() -> None:
    """
    Update location occupancy state based on presence of personal devices.
    Fires Matrix alerts on all VACANT/OCCUPIED transitions including startup.
    Thread-safe: acquires locks in order: known_lock, then occupancy_lock.
    """
    with known_lock:
        with occupancy_lock:
            _update_occupancy_state_unlocked()


def _log_ble_correlation(wifi_mac: str) -> None:
    """Log nearby Apple BLE devices seen within BLE_CORRELATION_WINDOW seconds of a WiFi arrival."""
    try:
        if not BLE_NAMES_FILE.exists():
            return
        data = json.loads(BLE_NAMES_FILE.read_text())
        now = time.time()
        nearby = []
        for ble_mac, entry in data.items():
            age = now - entry.get("last_seen", 0)
            if age <= BLE_CORRELATION_WINDOW:
                name = entry.get("name", "")
                nearby.append(f"{ble_mac}" + (f" ({name})" if name else ""))
        if nearby:
            logging.info(f"[BLE-CORR] WiFi arrival {wifi_mac} — nearby Apple BLE: {', '.join(nearby)}")
    except Exception as e:
        logging.debug(f"BLE correlation read failed: {e}")


def on_arrival(mac: str, ip: str, hostname: str = "") -> None:
    """Handle device arrival."""
    # Infrastructure IPs never trigger alerts
    if ip in infrastructure_ips:
        logging.debug(f"Ignoring infrastructure IP {ip} (MAC:{mac})")
        return
    # OUI-based auto-exclusion for network equipment
    oui = mac.replace(":", "").upper()[:6]
    if oui in NETWORK_EQUIPMENT_OUIS:
        infrastructure_ips.add(ip)
        logging.info(f"OUI auto-exclusion: {ip} (MAC:{mac} OUI:{oui}) — network equipment detected")
        return

    hostname = hostname or ip
    now = time.time()

    # Lock order: known_lock, then departure_lock, then occupancy_lock
    # Extract timers to cancel OUTSIDE all locks to avoid deadlock
    timers_to_cancel = []
    
    with departure_lock:
        if mac in departure_timers:
            timers_to_cancel.append(departure_timers.pop(mac))
    
    # Cancel all timers BEFORE acquiring any other locks
    for timer in timers_to_cancel:
        timer.cancel()
    
    with known_lock:
        with departure_lock:
            # Check if device marked departing
            if mac in known_devices and known_devices[mac].get('departing'):
                depart_initiated = known_devices[mac].get('depart_initiated', now)
                absence_secs = now - depart_initiated
                if absence_secs < 120:
                    # True rapid flap (transient Netlink/DHCP event) — suppress
                    known_devices[mac]['departing'] = False
                    with occupancy_lock:
                        _update_occupancy_state_unlocked()
                    last_departed_time.pop(mac, None)
                    logging.debug(f"Device {mac} rapid-flap suppressed ({int(absence_secs)}s)")
                    return
                else:
                    # Watchdog-confirmed absence — real re-arrival, let it alert
                    logging.debug(f"Device {mac} returning after {int(absence_secs)}s absence — treating as arrival")
                    known_devices.pop(mac)
                    last_departed_time.pop(mac, None)
            
            # Log if we cancelled a timer
            if timers_to_cancel:
                logging.debug(f"Cancelled departure timer for {mac} — device re-arrived before debounce expired")

        if mac in last_departed_time:
            time_since_departure = now - last_departed_time[mac]
            if time_since_departure < 300:  # 5 min
                logging.debug(f"Suppressing re-arrival alert for {mac} (departed {int(time_since_departure)}s ago)")
                # Re-add to known_devices to track presence
                if mac not in known_devices:
                    vendor = lookup_oui(mac)
                    known_devices[mac] = {
                        'ip': ip,
                        'hostname': hostname,
                        'vendor': vendor,
                        'first_seen': now,
                        'last_seen': now
                    }
                else:
                    known_devices[mac]['last_seen'] = now
                with occupancy_lock:
                    _update_occupancy_state_unlocked()
                return

        # Still inside known_lock, but not inside departure_lock now
        if mac in known_devices:
            # Device is already known
            dev = known_devices[mac]
            dev['last_seen'] = now
            # Upgrade from ARP-probe placeholder to real IP
            if dev.get('ip') in ('0.0.0.0', 'unknown') and ip not in ('0.0.0.0', 'unknown'):
                dev['ip'] = ip
                logging.debug(f"Updated {mac} IP from placeholder to {ip}")

            # DHCP renewal dedup: if last_seen < 30 min, skip alert
            if now - dev['first_seen'] < 1800:  # 30 min
                logging.debug(f"Skipping DHCP renewal alert for {mac} (seen {int(now - dev['first_seen'])}s ago)")
                with occupancy_lock:
                    _update_occupancy_state_unlocked()
                return

            # Not a re-arrival, update last_seen and don't alert (already known)
            with occupancy_lock:
                _update_occupancy_state_unlocked()
            return

        # New device
        vendor = lookup_oui(mac)
        known_devices[mac] = {
            'ip': ip,
            'hostname': hostname,
            'vendor': vendor,
            'first_seen': now,
            'last_seen': now
        }

    # Only alert for phones/wearables. Everything else is enrolled silently.
    if not is_personal_device(mac):
        logging.debug(f"Silent enroll {mac} — not a phone/wearable")
        update_occupancy_state()
        return

    # Delay alert 2 seconds so mDNS can update the hostname before we send.
    # The timer callback reads from known_devices at fire time to get the real name.
    def _send_arrival_alert():
        with known_lock:
            dev = known_devices.get(mac)
            if dev is None:
                return  # Device departed before timer fired
            resolved_hostname = dev.get('hostname', hostname)
            resolved_ip = dev.get('ip', ip)
            resolved_vendor = dev.get('vendor', vendor)
        msg = format_arrival(resolved_hostname, resolved_ip, resolved_vendor or "unknown", mac)
        logging.info(msg)
        send_alert(msg)
        _log_ble_correlation(mac)
        update_occupancy_state()

    t = threading.Timer(2.0, _send_arrival_alert)
    t.daemon = True
    t.start()


def on_departure(mac: str) -> None:
    """
    Handle device departure with debounce.
    Only sends alert if device still gone after DEPARTURE_DEBOUNCE_SEC.
    """
    with known_lock:
        if mac not in known_devices:
            return

        # Check if infrastructure IP (never alert)
        dev = known_devices[mac]
        if dev['ip'] in infrastructure_ips:
            logging.debug(f"Ignoring departure for infrastructure IP {dev['ip']} (MAC:{mac})")
            return

        # Only alert departure for phones/wearables
        if not is_personal_device(mac):
            known_devices.pop(mac, None)
            logging.debug(f"Silent depart {mac} — not a phone/wearable")
            return

        # Check if this is part of an interface flap (suppress simultaneous mass-departure)
        if _check_flap(mac):
            logging.debug(f"Flap suppression: ignoring departure for {mac}")
            return
        # Churn-based auto-suppression: if device cycles 3+ times in 30min, suppress
        if _check_churn(mac, dev['ip']):
            return

        dev_copy = dev.copy()
        # Mark departing instead of popping — allows re-arrival to detect and cancel timer
        known_devices[mac]['departing'] = True
        known_devices[mac]['depart_initiated'] = time.time()

    # Start departure debounce timer
    # Extract timer to cancel outside of lock to avoid deadlock
    timer_to_cancel = None
    with departure_lock:
        # Cancel any existing timer for this MAC
        if mac in departure_timers:
            timer_to_cancel = departure_timers.pop(mac)
    
    # Cancel outside lock
    if timer_to_cancel:
        timer_to_cancel.cancel()
    
    with departure_lock:
        def send_departure_alert():
            """Callback to send alert after debounce expires."""
            # Lock order: known_lock, then departure_lock, then occupancy_lock
            with known_lock:
                if not known_devices.get(mac, {}).get('departing'):
                    return  # device re-arrived, departure was cancelled
                # Device still departed — send alert and remove from tracking
                known_devices.pop(mac, None)

                with departure_lock:
                    last_departed_time[mac] = time.time()
                    if mac in departure_timers:
                        del departure_timers[mac]

                # Update occupancy state after device removal (still inside known_lock)
                with occupancy_lock:
                    _update_occupancy_state_unlocked()

            duration = fmt_duration(time.time() - dev_copy['first_seen'])
            msg = format_departure(dev_copy['hostname'], dev_copy['ip'], dev_copy['vendor'] or "unknown", mac, duration)
            logging.info(msg)
            send_alert(msg)

        timer = threading.Timer(float(DEPARTURE_DEBOUNCE_SEC), send_departure_alert)
        timer.daemon = True
        timer.name = f'departure-debounce-{mac}'
        departure_timers[mac] = timer
        timer.start()

    logging.debug(f"Departure debounce started for {mac}, will alert in {DEPARTURE_DEBOUNCE_SEC}s if still gone")


def get_primary_interface() -> str:
    """Get primary network interface (not lo)."""
    try:
        with open('/proc/net/route') as f:
            for line in f.readlines()[1:]:
                parts = line.split()
                if len(parts) > 1 and parts[0] != 'lo' and parts[1] == '00000000':
                    return parts[0]

        with open('/proc/net/route') as f:
            for line in f.readlines()[1:]:
                parts = line.split()
                if parts[0] != 'lo':
                    return parts[0]
    except Exception:
        pass

    return 'eth0'


def get_local_ip(iface: str) -> str:
    """Get the local IP address for the given interface."""
    try:
        import subprocess
        result = subprocess.run(['ip', 'addr', 'show', iface], capture_output=True, text=True, timeout=5)
        if result.returncode == 0:
            for line in result.stdout.split('\n'):
                if 'inet ' in line:
                    parts = line.strip().split()
                    if len(parts) >= 2:
                        ip_with_mask = parts[1]
                        ip = ip_with_mask.split('/')[0]
                        return ip
    except Exception as e:
        logging.warning(f"Failed to get local IP for {iface}: {e}")
    return ""


def seed_infrastructure_ips(iface: str) -> None:
    """
    Seed infrastructure IPs that should never trigger alerts.
    
    Seeding steps:
    1. Self IP (this device's interface IP)
    2. Gateway IP (from 'ip route show default')
    3. Broadcast IP (x.x.x.255)
    4. Network equipment by OUI (Ubiquiti, Cisco, Aruba, TP-Link, Netgear, MikroTik, Ruckus)
    5. Operator-provided IPs via NET_ALERTER_INFRA_IPS env var (comma-separated)
    
    NET_ALERTER_INFRA_IPS is a post-deploy configuration for operators to add
    infrastructure IPs known after deployment (access points, switches, etc).
    """
    global infrastructure_ips
    infrastructure_ips.clear()

    # Get self IP
    self_ip = get_local_ip(iface)
    if self_ip:
        infrastructure_ips.add(self_ip)
        logging.info(f"Seeded self IP {self_ip} as infrastructure")

    # Read gateway from 'ip route show default'
    try:
        import subprocess
        result = subprocess.run(['ip', 'route', 'show', 'default'], capture_output=True, text=True, timeout=5)
        if result.returncode == 0:
            # Parse: default via 192.168.1.1 dev eth0 proto dhcp metric 100
            for line in result.stdout.strip().split('\n'):
                if 'via' in line:
                    parts = line.split()
                    for i, part in enumerate(parts):
                        if part == 'via' and i + 1 < len(parts):
                            gateway_ip = parts[i + 1]
                            infrastructure_ips.add(gateway_ip)
                            logging.info(f"Seeded gateway {gateway_ip} as infrastructure")
                            break
    except Exception as e:
        logging.warning(f"Failed to read gateway: {e}")

    # Add broadcast (x.x.x.255 for most networks)
    if self_ip:
        try:
            parts = self_ip.rsplit('.', 1)
            if len(parts) == 2:
                broadcast_ip = parts[0] + ".255"
                infrastructure_ips.add(broadcast_ip)
                logging.info(f"Seeded broadcast {broadcast_ip} as infrastructure")
        except Exception:
            pass

    # Auto-exclude known network equipment by OUI
    with known_lock:
        for mac, dev in list(known_devices.items()):
            oui = mac.replace(":", "").upper()[:6]
            if oui in NETWORK_EQUIPMENT_OUIS:
                infrastructure_ips.add(dev['ip'])
                logging.info(f"OUI auto-exclusion: {dev['ip']} (MAC:{mac} OUI:{oui}) — network equipment detected")
    logging.info(f"Infrastructure IPs: {infrastructure_ips}")

    # Allow operator to add additional IPs via env var (comma-separated)
    extra = os.environ.get("NET_ALERTER_INFRA_IPS", "")
    for ip in (x.strip() for x in extra.split(",") if x.strip()):
        infrastructure_ips.add(ip)
        logging.info(f"Seeded extra infrastructure IP {ip} from env")


def _get_iface_mac(iface: str) -> bytes:
    """Return interface MAC as 6 bytes, or zero bytes on failure."""
    try:
        raw = open(f"/sys/class/net/{iface}/address").read().strip()
        return bytes(int(x, 16) for x in raw.split(':'))
    except Exception:
        return b'\x00' * 6


def arp_scanner(iface: str) -> None:
    """Active ARP scan thread.

    Sends ARP who-has to every IP in the local /24 subnet every ARP_SCAN_INTERVAL
    seconds. Every device that replies triggers on_arrival, bypassing AP ARP proxy.
    This is the reliable presence detection path for devices with existing DHCP leases
    that never broadcast ARP on their own.
    """
    logging.info(f"ARP scanner started on {iface} (interval={ARP_SCAN_INTERVAL}s)")
    src_mac = _get_iface_mac(iface)

    while True:
        try:
            src_ip = get_local_ip(iface)
            if not src_ip:
                time.sleep(ARP_SCAN_INTERVAL)
                continue

            prefix = '.'.join(src_ip.split('.')[:3])
            src_ip_b = socket.inet_aton(src_ip)
            broadcast = b'\xff\xff\xff\xff\xff\xff'

            sock = socket.socket(socket.AF_PACKET, socket.SOCK_RAW, socket.htons(0x0806))
            sock.bind((iface, 0))
            sock.settimeout(0.1)

            # Send ARP who-has for every IP in the /24
            for i in range(1, 255):
                target_ip = f"{prefix}.{i}"
                if target_ip == src_ip:
                    continue
                eth = broadcast + src_mac + b'\x08\x06'
                arp = struct.pack('!HHBBH', 1, 0x0800, 6, 4, 1)
                arp += src_mac + src_ip_b + b'\x00' * 6 + socket.inet_aton(target_ip)
                try:
                    sock.send(eth + arp)
                except Exception:
                    pass

            # Collect replies for 2 seconds
            deadline = time.time() + 2.0
            while time.time() < deadline:
                try:
                    data = sock.recv(60)
                    if len(data) < 42 or data[12:14] != b'\x08\x06':
                        continue
                    if struct.unpack('!H', data[20:22])[0] != 2:  # opcode must be reply
                        continue
                    sender_mac = ':'.join(f'{b:02x}' for b in data[22:28])
                    sender_ip = socket.inet_ntoa(data[28:32])
                    if sender_mac in ('00:00:00:00:00:00', 'ff:ff:ff:ff:ff:ff'):
                        continue
                    _update_last_seen(sender_mac)
                    on_arrival(sender_mac, sender_ip)
                except socket.timeout:
                    continue
                except Exception:
                    break

            sock.close()

        except Exception as e:
            logging.error(f"ARP scanner error: {e}")

        time.sleep(ARP_SCAN_INTERVAL)


def _find_monitor_iface(primary_iface: str) -> str:
    """Return a second WiFi interface name for monitor mode, or '' if none found."""
    try:
        result = subprocess.run(['iw', 'dev'], capture_output=True, text=True, timeout=5)
        for line in result.stdout.splitlines():
            line = line.strip()
            if line.startswith('Interface '):
                iface = line.split()[1]
                if iface != primary_iface:
                    return iface
    except Exception:
        pass
    return ''


def _enable_monitor_mode(iface: str) -> bool:
    """Put iface into 802.11 monitor mode. Returns True on success."""
    try:
        subprocess.run(['ip', 'link', 'set', iface, 'down'],
                       check=True, capture_output=True, timeout=5)
        subprocess.run(['iw', 'dev', iface, 'set', 'type', 'monitor'],
                       check=True, capture_output=True, timeout=5)
        subprocess.run(['ip', 'link', 'set', iface, 'up'],
                       check=True, capture_output=True, timeout=5)
        return True
    except Exception as e:
        logging.warning(f"Monitor mode setup failed on {iface}: {e}")
        return False


def _parse_80211_frame(frame: bytes) -> None:
    """Parse a radiotap+802.11 frame and call on_arrival for transmitting personal devices.

    addr2 is always the transmitter. We care about:
    - Management frames from stations (assoc req subtype=0, auth subtype=11)
    - Data frames with ToDS=1 (station → AP)

    Probe requests (subtype=4) are excluded — modern iOS uses random MACs for those.
    """
    try:
        if len(frame) < 4 or frame[0] != 0:  # radiotap version must be 0
            return
        rt_len = struct.unpack_from('<H', frame, 2)[0]
        if rt_len > len(frame) - 24:
            return

        dot11 = frame[rt_len:]
        if len(dot11) < 24:
            return

        fc = struct.unpack_from('<H', dot11, 0)[0]
        frame_type = (fc >> 2) & 0x3
        frame_subtype = (fc >> 4) & 0xF

        # addr2 = transmitter (bytes 10-15)
        src_mac = ':'.join(f'{b:02x}' for b in dot11[10:16])
        if src_mac in ('00:00:00:00:00:00', 'ff:ff:ff:ff:ff:ff'):
            return

        is_station_tx = False
        if frame_type == 0 and frame_subtype in (0, 11):  # assoc req, auth
            is_station_tx = True
        elif frame_type == 2:  # data
            to_ds = (fc >> 8) & 0x1
            from_ds = (fc >> 9) & 0x1
            if to_ds and not from_ds:  # STA → AP
                is_station_tx = True

        if not is_station_tx:
            return

        _update_last_seen(src_mac)

        if not is_personal_device(src_mac):
            return

        with known_lock:
            already_known = src_mac in known_devices
        if not already_known:
            on_arrival(src_mac, 'unknown')

    except Exception:
        pass


def monitor_sniffer(iface: str) -> None:
    """802.11 monitor mode sniffer thread.

    Reads raw radiotap+802.11 frames and calls on_arrival for personal devices
    seen transmitting. Bypasses AP entirely — works even when AP proxy hides
    all ARP/DHCP traffic.
    """
    if not _enable_monitor_mode(iface):
        logging.warning(f"Monitor mode unavailable on {iface}, monitor sniffer disabled")
        return

    logging.info(f"Monitor mode sniffer started on {iface}")

    try:
        sock = socket.socket(socket.AF_PACKET, socket.SOCK_RAW, socket.htons(0x0003))
        sock.bind((iface, 0))
        sock.settimeout(1.0)
    except Exception as e:
        logging.error(f"Monitor mode socket failed on {iface}: {e}")
        return

    while True:
        try:
            frame = sock.recv(2048)
            _parse_80211_frame(frame)
        except socket.timeout:
            continue
        except Exception as e:
            logging.debug(f"Monitor sniffer recv error: {e}")
            time.sleep(0.1)


def ble_sniffer() -> None:
    """
    BLE sniffer thread.
    Parses Apple Continuity Protocol Handoff advertisements to track device identity
    across BLE MAC rotations using monotonically-incrementing sequence numbers.

    Attempts HCI raw socket first, falls back to hcidump subprocess.
    Thread exits gracefully on hardware errors.
    """
    try:
        # Try raw HCI socket approach
        try:
            s = socket.socket(socket.AF_BLUETOOTH, socket.SOCK_RAW, socket.BTPROTO_HCI)
            s.bind((0,))  # Bind to hci0 (first controller)

            # Set HCI filter to receive LE Meta events only (simplification: receive all for now)
            # This would require HCI_FILTER socket option, but basic approach is simpler
            logging.info("BLE sniffer started (HCI raw socket)")

            while True:
                try:
                    data = s.recv(4096)
                    _parse_hci_event(data)
                except Exception as e:
                    logging.debug(f"HCI socket error: {e}")
                    time.sleep(1)
        except (FileNotFoundError, PermissionError, OSError) as e:
            logging.debug(f"HCI socket unavailable ({type(e).__name__}), trying hcidump fallback")

            # Fallback: run hcidump subprocess and parse output
            try:
                proc = subprocess.Popen(
                    ['hcidump', '-R', '-i', 'hci0'],
                    stdout=subprocess.PIPE,
                    stderr=subprocess.PIPE,
                    text=False,
                    bufsize=0
                )
                logging.info("BLE sniffer started (hcidump subprocess)")

                while True:
                    line = proc.stdout.readline()
                    if not line:
                        break
                    try:
                        # hcidump outputs hex dumps; parse them
                        _parse_hcidump_line(line)
                    except Exception:
                        pass
            except FileNotFoundError:
                logging.warning("hcidump not found; BLE sniffing unavailable (install bluez)")
                return

    except Exception as e:
        logging.error(f"BLE sniffer fatal error: {e}")


def _parse_hci_event(data: bytes) -> None:
    """Parse HCI event packet for LE Meta events with Apple Continuity data."""
    try:
        # HCI packet: type (1) + data
        # This is a simplified implementation; full HCI parsing is complex
        # For now, we'll use the hcidump fallback which is more reliable
        pass
    except Exception:
        pass


def _parse_hcidump_line(line: bytes) -> None:
    """Parse a line from hcidump output to extract Apple Continuity Handoff data.

    hcidump -R outputs hex dumps like:
    > 04 3E 2B 02 01 00 19 00 01 02 03 04 05 06 07 08 ...

    Extracts BLE MAC and Handoff sequence number, then calls _ingest_signal().
    Returns None silently if parsing fails (incomplete packet).
    """
    try:
        # Parse hcidump hex output: skip leading ">" or "<" and split on whitespace
        hex_str = line.decode('utf-8', errors='replace').strip()
        if not hex_str or hex_str[0] not in '><':
            return

        # Extract hex bytes
        hex_parts = hex_str[1:].split()
        data = bytes([int(h, 16) for h in hex_parts if h])

        if len(data) < 7:
            return  # Too short for HCI LE advertising report

        # Check for LE Meta Event (HCI packet type 04, HCI event 3E)
        if data[0] != 0x04 or data[1] != 0x3E:
            return

        # Skip HCI event header: type(1) event(1) len(1) subevt(1) = 4 bytes
        if len(data) < 12:  # Need at least event header + minimal adv report
            return

        # HCI_LE_Meta_Event (0x3E), subevent is at offset 3
        subevent = data[3]
        if subevent != 0x02:  # LE Advertising Report subevent
            return

        # LE Advertising Report structure: subevent(1) num_reports(1) event_type(1) addr_type(1) addr(6) len(1) data(variable)
        if len(data) < 13:  # Minimum for header + MAC
            return

        num_reports = data[4]
        if num_reports < 1:
            return

        event_type = data[5]
        addr_type = data[6]

        # Extract BLE MAC (6 bytes, little-endian on wire)
        ble_mac_raw = data[7:13]
        ble_mac = ':'.join(f'{b:02x}' for b in reversed(ble_mac_raw))

        # Advertisement data length
        if len(data) < 14:
            return
        ad_data_len = data[13]
        if len(data) < 14 + ad_data_len:
            return  # Incomplete packet

        # Parse advertisement data for Apple Company ID (0x004C little-endian = 4C 00)
        ad_data = data[14:14 + ad_data_len]
        offset = 0

        while offset < len(ad_data) - 1:
            ad_len = ad_data[offset]
            if ad_len == 0 or offset + 1 + ad_len > len(ad_data):
                break

            ad_type = ad_data[offset + 1]
            ad_payload = ad_data[offset + 2:offset + 1 + ad_len]

            # Look for Manufacturer Specific Data (type 0xFF) with Apple Company ID (0x004C)
            if ad_type == 0xFF and len(ad_payload) >= 2:
                company_id = struct.unpack('<H', ad_payload[:2])[0]
                if company_id == BLE_COMPANY_ID_APPLE and len(ad_payload) >= BLE_HANDOFF_SEQ_OFFSET + 2:
                    # Check for Handoff message type (0x0C)
                    if ad_payload[2] == BLE_HANDOFF_MSG_TYPE:
                        # Extract sequence number from bytes 4-5 (big-endian)
                        seq = struct.unpack('!H', ad_payload[BLE_HANDOFF_SEQ_OFFSET:BLE_HANDOFF_SEQ_OFFSET + 2])[0]
                        _ingest_signal(ble_mac, 'ble', handoff_seq=seq)
                        return

            offset += 1 + ad_len

    except Exception:
        return  # Silently fail on malformed packets


def dhcp_sniffer(iface: str) -> None:
    """
    DHCP sniffer thread.
    AF_PACKET raw socket, captures DHCP traffic passively.
    """
    try:
        s = socket.socket(socket.AF_PACKET, socket.SOCK_RAW, socket.htons(0x0003))
        s.bind((iface, 0))
        logging.info(f"DHCP sniffer started on {iface}")

        while True:
            try:
                data, _ = s.recvfrom(65535)
                parse_frame(data)
            except Exception as e:
                logging.error(f"DHCP sniffer error: {e}")
                time.sleep(1)
    except Exception as e:
        logging.error(f"Failed to start DHCP sniffer: {e}")



def _update_last_seen(mac: str) -> None:
    """Update the last_seen timestamp for a MAC address (passive observation)."""
    with last_seen_lock:
        last_seen[mac] = time.time()

def _extract_mdns_hostname(payload: bytes) -> str:
    """Scan mDNS DNS payload for an A record with a .local name. Returns display name (sans .local) or ''."""
    try:
        if len(payload) < 12:
            return ""
        qdcount = struct.unpack('!H', payload[4:6])[0]
        ancount = struct.unpack('!H', payload[6:8])[0]
        nscount = struct.unpack('!H', payload[8:10])[0]
        arcount = struct.unpack('!H', payload[10:12])[0]
        offset = 12
        for _ in range(min(qdcount, 50)):
            offset = _skip_dns_name(payload, offset)
            if offset is None or offset + 4 > len(payload):
                return ""
            offset += 4
        for _ in range(min(ancount + nscount + arcount, 200)):
            if offset is None or offset + 10 > len(payload):
                break
            name_offset = offset
            offset = _skip_dns_name(payload, offset)
            if offset is None or offset + 10 > len(payload):
                break
            record_type = struct.unpack('!H', payload[offset:offset+2])[0]
            rdlen = struct.unpack('!H', payload[offset+8:offset+10])[0]
            offset += 10
            if record_type == 1:  # A record — name is the device hostname
                name = _extract_dns_name(payload, name_offset)
                if name and name.endswith('.local'):
                    return name[:-6]
            if offset + rdlen > len(payload):
                break
            offset += rdlen
    except Exception:
        pass
    return ""


def _parse_mdns_for_names(payload: bytes, mac: str) -> None:
    """Parse mDNS DNS message to extract PTR/SRV/A record names and check against personal_names.

    If a record name matches a name in personal_names, add the source MAC to personal_devices.
    """
    try:
        if len(payload) < 12:  # DNS header minimum size
            return
        
        # DNS header: ID(2) Flags(2) QDCOUNT(2) ANCOUNT(2) NSCOUNT(2) ARCOUNT(2)
        qdcount = struct.unpack('!H', payload[4:6])[0]
        ancount = struct.unpack('!H', payload[6:8])[0]
        
        # Skip questions section (we only care about answers/records)
        offset = 12
        for _ in range(qdcount):
            offset = _skip_dns_name(payload, offset)
            if offset is None or offset + 4 > len(payload):
                return
            offset += 4  # QTYPE + QCLASS
        
        # Parse answers/authority/additional records
        nscount = struct.unpack('!H', payload[8:10])[0]
        arcount = struct.unpack('!H', payload[10:12])[0]
        # Cap total records to prevent unbounded loop on crafted packets
        total_records = min(ancount + nscount + arcount, 1000)
        for _ in range(total_records):
            if offset is None or offset + 10 > len(payload):
                break
            
            # Parse record name
            name_offset = offset
            offset = _skip_dns_name(payload, offset)
            if offset is None:
                break
            
            # Parse TYPE(2) CLASS(2) TTL(4) RDLEN(2)
            if offset + 10 > len(payload):
                break
            
            record_type, record_class, ttl, rdlen = struct.unpack('!HHIH', payload[offset:offset+10])
            offset += 10
            
            # Extract the name string from the record
            name = _extract_dns_name(payload, name_offset)
            if name:
                # A record (type 1): record name IS the Bonjour hostname (e.g. Johns-iPhone.local)
                # Update known_devices hostname if currently unset or just an IP address
                if record_type == 1 and name.endswith('.local'):
                    display_name = name[:-6]  # strip .local
                    mac_normalized = _normalize_mac(mac)
                    if mac_normalized and display_name:
                        with known_lock:
                            dev = known_devices.get(mac_normalized)
                            if dev:
                                current = dev.get('hostname', '')
                                # Only overwrite if hostname is an IP or unset
                                try:
                                    socket.inet_aton(current)
                                    is_ip = True
                                except Exception:
                                    is_ip = not current or current == mac_normalized
                                if is_ip or not current:
                                    dev['hostname'] = display_name
                                    logging.debug(f"mDNS hostname update: {mac_normalized} → {display_name}")

                if personal_names:
                    with personal_lock:
                        for pname in personal_names:
                            if pname.lower() in name.lower():
                                mac_normalized = _normalize_mac(mac)
                                if mac_normalized:
                                    personal_devices.add(mac_normalized)
                                    logging.debug(f"mDNS auto-discovery: added {mac_normalized} for name match '{pname}' in '{name}'")
                                break

            # Skip RDATA
            if offset + rdlen > len(payload):
                break
            offset += rdlen
    
    except Exception as e:
        logging.debug(f"mDNS parse error: {e}")


def _skip_dns_name(payload: bytes, offset: int) -> int:
    """Skip a DNS name in the payload, handling compression pointers. Returns new offset or None on error."""
    try:
        while offset < len(payload):
            length = payload[offset]
            if length == 0:
                return offset + 1
            if (length & 0xC0) == 0xC0:  # Pointer
                return offset + 2
            offset += 1 + length
        return None
    except Exception:
        return None


def _extract_dns_name(payload: bytes, offset: int) -> str:
    """Extract a DNS name from payload at offset. Handles compression pointers."""
    try:
        parts = []
        visited = set()
        
        while offset < len(payload):
            if offset in visited:  # Prevent infinite loops
                break
            visited.add(offset)
            
            length = payload[offset]
            offset += 1
            
            if length == 0:
                break
            elif (length & 0xC0) == 0xC0:  # Pointer
                if offset >= len(payload):
                    break
                pointer_offset = struct.unpack('!H', bytes([(length & 0x3F), payload[offset]]))[0]
                # Bounds check before recursing: pointer must not point backwards or beyond payload
                if pointer_offset >= offset or pointer_offset >= len(payload):
                    break
                offset = pointer_offset
                continue
            
            # Regular label
            if offset + length > len(payload):
                break
            label = payload[offset:offset + length].decode('utf-8', errors='replace')
            parts.append(label)
            offset += length
        
        return '.'.join(parts) if parts else ""
    except Exception:
        return ""

def parse_frame(data: bytes) -> None:
    """Parse Ethernet frame: ARP, mDNS, and DHCP packets for passive observation and event triggering."""
    try:
        if len(data) < 14:
            return

        # Ethernet frame — extract source MAC (bytes 6:12)
        src_mac = ':'.join(f'{b:02x}' for b in data[6:12])
        
        # Ethernet type field
        eth_type = struct.unpack('!H', data[12:14])[0]
        
        # Branch A: ARP frame (ethertype 0x0806)
        if eth_type == 0x0806:
            opcode = struct.unpack('!H', data[20:22])[0] if len(data) >= 22 else 0
            # Opcode 1 (request) is a genuine device signal.
            # Opcode 2 (reply) may be an AP ARP proxy forgery — EXCEPT for explicitly
            # labeled personal devices where we trust the frame regardless, because
            # iPhones with privacy MACs often only emit opcode=2 after initial association.
            with personal_lock:
                is_labeled = _normalize_mac(src_mac) in device_labels
            if opcode == 1 or (opcode == 2 and is_labeled):
                _update_last_seen(src_mac)
                # Personal devices first seen via ARP get no DHCP/Netlink event.
                # Fire on_arrival so they get ARRIVED alert and occupancy counts.
                if is_personal_device(src_mac) and len(data) >= 32:
                    with known_lock:
                        already_known = src_mac in known_devices
                    if not already_known:
                        spa = socket.inet_ntoa(data[28:32])
                        on_arrival(src_mac, spa)
            # Ingest ARP signal for multi-source device identity
            _ingest_signal(src_mac, 'arp')
            return
        
        # For IP frames, extract IP header info
        if eth_type != 0x0800:  # not IPv4
            return

        # IP header
        if len(data) < 23:
            return
        proto = data[23]
        if proto != 17:  # not UDP
            return

        # UDP header
        if len(data) < 38:
            return
        src_port = struct.unpack('!H', data[34:36])[0]
        dst_port = struct.unpack('!H', data[36:38])[0]
        
        # Branch B: mDNS frame (IPv4 UDP dst port 5353)
        if dst_port == 5353:
            _update_last_seen(src_mac)
            # mDNS is the primary arrival indicator for Apple devices — they send it
            # on WiFi join before or instead of DHCP. Fire on_arrival so the device
            # gets an ARRIVED alert and is counted in occupancy.
            if len(data) >= 30:
                src_ip = socket.inet_ntoa(data[26:30])
                with known_lock:
                    already_known = src_mac in known_devices
                if not already_known:
                    # Try to extract Bonjour hostname from this frame before alerting
                    mdns_hostname = _extract_mdns_hostname(data[42:]) if len(data) >= 42 else ""
                    on_arrival(src_mac, src_ip, mdns_hostname)
            # Ingest mDNS signal for multi-source device identity
            _ingest_signal(src_mac, 'mdns')
            # Parse mDNS DNS message for device names (updates hostname on subsequent frames)
            if len(data) >= 42:
                _parse_mdns_for_names(data[42:], src_mac)
            return
        
        # Branch C: DHCP frame (UDP ports 67/68)
        if not (src_port in (67, 68) and dst_port in (67, 68)):
            return

        # DHCP payload (starts at byte 42)
        if len(data) < 42:
            return
        dhcp = data[42:]
        if len(dhcp) < 236:
            return

        # MAC address (chaddr at offset 28, 6 bytes)
        mac_bytes = dhcp[28:34]
        mac = ':'.join(f'{b:02x}' for b in mac_bytes)
        if mac in ('00:00:00:00:00:00', 'ff:ff:ff:ff:ff:ff'):
            return

        # Update last_seen for DHCP packets
        _update_last_seen(mac)

        # Parse DHCP options (start at byte 240)
        if len(dhcp) < 240:
            return

        msg_type = None
        hostname = ""
        requested_ip = ""
        ciaddr = socket.inet_ntoa(dhcp[12:16])
        dhcp_option55 = None  # Parameter Request List for device fingerprinting

        i = 240
        while i < len(dhcp):
            opt = dhcp[i]
            if opt == 255:
                break
            if opt == 0:
                i += 1
                continue
            if i + 1 >= len(dhcp):
                break

            length = dhcp[i + 1]
            if i + 2 + length > len(dhcp):
                break

            val = dhcp[i + 2:i + 2 + length]

            if opt == 53 and length == 1:  # DHCP message type
                msg_type = val[0]
            elif opt == 12:  # Hostname
                hostname = val.decode('utf-8', errors='replace').strip('\x00')
            elif opt == 50 and length == 4:  # Requested IP
                requested_ip = socket.inet_ntoa(val)
            elif opt == 55:  # Parameter Request List (Option 55)
                # Bounds check already done above; safe to use val
                dhcp_option55 = val

            i += 2 + length

        # Ingest DHCP signal for multi-source device identity
        if msg_type in (1, 3) and dhcp_option55:
            dhcp_fp = _dhcp_fingerprint_hash(dhcp_option55)
            _ingest_signal(mac, 'dhcp', dhcp_fp=dhcp_fp)

        # Process based on message type
        if msg_type in (1, 3):  # Discover or Request (arrival)
            ip = requested_ip or ciaddr
            if ip and ip != '0.0.0.0':
                on_arrival(mac, ip, hostname)
        elif msg_type == 7:  # Release (departure)
            on_departure(mac)

    except Exception as e:
        logging.debug(f"Frame parse error: {e}")


# Netlink constants
NETLINK_ROUTE = 0
RTMGRP_NEIGH = 0x4
RTM_NEWNEIGH = 28
RTM_DELNEIGH = 29
NUD_REACHABLE = 0x02
NUD_STALE = 0x04
NUD_DELAY = 0x08
NUD_PROBE = 0x10
NDA_DST = 1
NDA_LLADDR = 2


def netlink_watcher() -> None:
    """
    Netlink watcher thread.
    Kernel pushes RTM_NEWNEIGH and RTM_DELNEIGH events.
    """
    try:
        s = socket.socket(socket.AF_NETLINK, socket.SOCK_RAW, NETLINK_ROUTE)
        s.bind((os.getpid(), RTMGRP_NEIGH))
        logging.info("Netlink neighbor watcher started")

        while True:
            try:
                data = s.recv(65535)
                parse_netlink(data)
            except Exception as e:
                logging.error(f"Netlink watcher error: {e}")
                time.sleep(1)
    except Exception as e:
        logging.error(f"Failed to start Netlink watcher: {e}")


def parse_netlink(data: bytes) -> None:
    """Parse Netlink neighbor messages."""
    try:
        offset = 0
        while offset < len(data):
            if offset + 16 > len(data):
                break

            nlmsg_len, nlmsg_type, _, _, _ = struct.unpack_from('=IHHII', data, offset)
            if nlmsg_len < 16:
                break

            payload = data[offset + 16:offset + nlmsg_len]

            if nlmsg_type in (RTM_NEWNEIGH, RTM_DELNEIGH):
                parse_ndmsg(payload, nlmsg_type)

            offset += (nlmsg_len + 3) & ~3

    except Exception as e:
        logging.debug(f"Netlink parse error: {e}")


def parse_ndmsg(data: bytes, msg_type: int) -> None:
    """Parse ndmsg (neighbor discovery message)."""
    try:
        if len(data) < 12:
            return

        state = struct.unpack_from('=H', data, 8)[0]

        mac = None
        ip = None
        offset = 12

        while offset + 4 <= len(data):
            rta_len, rta_type = struct.unpack_from('=HH', data, offset)
            if rta_len < 4:
                break

            val = data[offset + 4:offset + rta_len]

            if rta_type == NDA_LLADDR and len(val) == 6:
                mac = ':'.join(f'{b:02x}' for b in val)
            elif rta_type == NDA_DST:
                if len(val) == 4:
                    ip = socket.inet_ntoa(val)

            offset += (rta_len + 3) & ~3

        if not mac or mac in ('00:00:00:00:00:00', 'ff:ff:ff:ff:ff:ff'):
            return

        if msg_type == RTM_NEWNEIGH and (state & NUD_REACHABLE):
            on_arrival(mac, ip or 'unknown')
        elif msg_type == RTM_DELNEIGH:
            on_departure(mac)

    except Exception as e:
        logging.debug(f"ndmsg parse error: {e}")


def personal_watchdog() -> None:
    """Watchdog thread: fires on_departure for personal devices silent > WATCHDOG_TIMEOUT_SEC.
    
    Monitors last_seen timestamps for personal devices and triggers departure alerts
    if a device goes silent for longer than the configured timeout.
    """
    while True:
        try:
            time.sleep(60)  # Check every minute
            now = time.time()
            
            # Track all personal devices currently in known_devices (not just enrolled set)
            with known_lock:
                tracked = {mac for mac in known_devices if is_personal_device(mac)}

            for mac in tracked:
                # Get last_seen timestamp
                with last_seen_lock:
                    ts = last_seen.get(mac)
                
                if ts is None:
                    continue
                
                # Check if device is silent
                if now - ts > WATCHDOG_TIMEOUT_SEC:
                    # Only fire if device is currently known and not already departing
                    should_depart = False
                    with known_lock:
                        if mac in known_devices and not known_devices[mac].get('departing'):
                            should_depart = True

                    if should_depart:
                        logging.info(f"Watchdog: personal device {mac} silent >{WATCHDOG_TIMEOUT_SEC}s, triggering departure")
                        on_departure(mac)
        
        except Exception as e:
            logging.error(f"Watchdog thread error: {e}")

def main() -> None:
    """Main entry point."""
    logger = setup_logging()
    load_env()

    iface = get_primary_interface()
    logging.info(f"Net alerter starting on interface {iface}")

    seed_infrastructure_ips(iface)
    load_infrastructure_macs_from_env()
    seed_from_arp_cache()

    # Start monitor threads
    threads = [
        threading.Thread(target=dhcp_sniffer, args=(iface,), daemon=True, name='dhcp-sniffer'),
        threading.Thread(target=netlink_watcher, daemon=True, name='netlink-watcher'),
        threading.Thread(target=personal_watchdog, daemon=True, name='personal-watchdog'),
        threading.Thread(target=ble_sniffer, daemon=True, name='ble-sniffer'),
        threading.Thread(target=arp_scanner, args=(iface,), daemon=True, name='arp-scanner'),
    ]

    monitor_iface = _find_monitor_iface(iface)
    if monitor_iface:
        logging.info(f"Monitor mode interface found: {monitor_iface}")
        threads.append(threading.Thread(target=monitor_sniffer, args=(monitor_iface,), daemon=True, name='monitor-sniffer'))

    for t in threads:
        t.start()

    logging.info("Net alerter running — DHCP sniffer + Netlink neighbor watcher + personal watchdog + BLE sniffer + ARP scanner active")

    # Keep main thread alive
    try:
        while True:
            time.sleep(60)
            with known_lock:
                num_known = len(known_devices)
            with device_store_lock:
                num_identities = len(device_store)
                enrolled_count = sum(1 for d in device_store.values() if d['enrolled'])
            logging.debug(f"Tracking {num_known} legacy devices; {num_identities} device identities ({enrolled_count} enrolled)")
    except KeyboardInterrupt:
        logging.info("Shutting down")
        sys.exit(0)


if __name__ == '__main__':
    main()
