Add 9 advanced passive modules: VLAN, network mapper, auth tracker, SMB, cloud tokens, LDAP, RDP, QUIC, DB interceptor
Phase 2 batch 2: protocol-specific passive modules for deep network visibility. Each module subscribes to capture_bus, parses protocol structures with struct (no scapy), buffers SQLite writes, and publishes bus events for cross-module correlation. - vlan_discovery: 802.1Q/DTP/CDP/LLDP/STP/EAPOL layer-2 parsing - network_mapper: communication graph with role classification and DOT output - auth_flow_tracker: Kerberos/NTLM/SSH/LDAP auth correlation per user - smb_monitor: SMB2/3 tree connect, file access, GPP/SYSVOL detection - cloud_token_harvester: regex scan cleartext HTTP for AWS/JWT/OAuth/SAML/GCP tokens - ldap_harvester: BER/ASN.1 LDAP parsing for AD object inventory and LAPS detection - rdp_monitor: RDP cookie + CredSSP NTLM extraction, admin workstation detection - quic_analyzer: QUIC Initial packet parsing with RFC 9001 key derivation for SNI - db_interceptor: MSSQL TDS, MySQL, PostgreSQL, Redis RESP, MongoDB OP_MSG parsing
This commit is contained in:
@@ -8,6 +8,15 @@ from modules.passive.kerberos_harvester import KerberosHarvester
|
|||||||
from modules.passive.host_discovery import HostDiscovery
|
from modules.passive.host_discovery import HostDiscovery
|
||||||
from modules.passive.os_fingerprint import OSFingerprint
|
from modules.passive.os_fingerprint import OSFingerprint
|
||||||
from modules.passive.traffic_analyzer import TrafficAnalyzer
|
from modules.passive.traffic_analyzer import TrafficAnalyzer
|
||||||
|
from modules.passive.vlan_discovery import VLANDiscovery
|
||||||
|
from modules.passive.network_mapper import NetworkMapper
|
||||||
|
from modules.passive.auth_flow_tracker import AuthFlowTracker
|
||||||
|
from modules.passive.smb_monitor import SMBMonitor
|
||||||
|
from modules.passive.cloud_token_harvester import CloudTokenHarvester
|
||||||
|
from modules.passive.ldap_harvester import LDAPHarvester
|
||||||
|
from modules.passive.rdp_monitor import RDPMonitor
|
||||||
|
from modules.passive.quic_analyzer import QUICAnalyzer
|
||||||
|
from modules.passive.db_interceptor import DBInterceptor
|
||||||
|
|
||||||
__all__ = [
|
__all__ = [
|
||||||
"PacketCapture",
|
"PacketCapture",
|
||||||
@@ -18,4 +27,13 @@ __all__ = [
|
|||||||
"HostDiscovery",
|
"HostDiscovery",
|
||||||
"OSFingerprint",
|
"OSFingerprint",
|
||||||
"TrafficAnalyzer",
|
"TrafficAnalyzer",
|
||||||
|
"VLANDiscovery",
|
||||||
|
"NetworkMapper",
|
||||||
|
"AuthFlowTracker",
|
||||||
|
"SMBMonitor",
|
||||||
|
"CloudTokenHarvester",
|
||||||
|
"LDAPHarvester",
|
||||||
|
"RDPMonitor",
|
||||||
|
"QUICAnalyzer",
|
||||||
|
"DBInterceptor",
|
||||||
]
|
]
|
||||||
|
|||||||
@@ -0,0 +1,539 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
"""Authentication flow tracker — cross-protocol auth event correlation.
|
||||||
|
|
||||||
|
Tracks Kerberos (AS-REQ/TGS-REQ), NTLM (across SMB/HTTP/LDAP/MSSQL),
|
||||||
|
SSH connections, and LDAP binds. Builds per-user authentication timelines
|
||||||
|
and identifies service accounts.
|
||||||
|
"""
|
||||||
|
|
||||||
|
import logging
|
||||||
|
import os
|
||||||
|
import socket
|
||||||
|
import sqlite3
|
||||||
|
import struct
|
||||||
|
import threading
|
||||||
|
import time
|
||||||
|
from collections import defaultdict
|
||||||
|
from pathlib import Path
|
||||||
|
from typing import Optional
|
||||||
|
|
||||||
|
from modules.base import BaseModule
|
||||||
|
|
||||||
|
logger = logging.getLogger("bb.passive.auth_flow_tracker")
|
||||||
|
|
||||||
|
# Protocol ports
|
||||||
|
KERBEROS_PORT = 88
|
||||||
|
SMB_PORT = 445
|
||||||
|
LDAP_PORT = 389
|
||||||
|
LDAPS_PORT = 636
|
||||||
|
RDP_PORT = 3389
|
||||||
|
SSH_PORT = 22
|
||||||
|
MSSQL_PORT = 1433
|
||||||
|
|
||||||
|
TCP_PROTO = 6
|
||||||
|
UDP_PROTO = 17
|
||||||
|
|
||||||
|
# Kerberos message types (in AS-REQ body)
|
||||||
|
KRB_AS_REQ = 10
|
||||||
|
KRB_AS_REP = 11
|
||||||
|
KRB_TGS_REQ = 12
|
||||||
|
KRB_TGS_REP = 13
|
||||||
|
|
||||||
|
# NTLM signature
|
||||||
|
NTLMSSP_SIGNATURE = b'NTLMSSP\x00'
|
||||||
|
NTLM_NEGOTIATE = 1
|
||||||
|
NTLM_CHALLENGE = 2
|
||||||
|
NTLM_AUTHENTICATE = 3
|
||||||
|
|
||||||
|
# Service account detection threshold
|
||||||
|
SERVICE_ACCOUNT_THRESHOLD = 5
|
||||||
|
|
||||||
|
_DB_INIT = """
|
||||||
|
CREATE TABLE IF NOT EXISTS auth_events (
|
||||||
|
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||||
|
timestamp REAL NOT NULL,
|
||||||
|
source_ip TEXT NOT NULL,
|
||||||
|
dest_ip TEXT NOT NULL,
|
||||||
|
protocol TEXT NOT NULL,
|
||||||
|
auth_type TEXT NOT NULL,
|
||||||
|
username TEXT DEFAULT '',
|
||||||
|
domain TEXT DEFAULT '',
|
||||||
|
target_service TEXT DEFAULT '',
|
||||||
|
success INTEGER DEFAULT -1
|
||||||
|
);
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_auth_user ON auth_events(username);
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_auth_ts ON auth_events(timestamp);
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_auth_src ON auth_events(source_ip);
|
||||||
|
"""
|
||||||
|
|
||||||
|
FLUSH_INTERVAL = 30
|
||||||
|
|
||||||
|
|
||||||
|
class AuthFlowTracker(BaseModule):
|
||||||
|
"""Track authentication events across Kerberos, NTLM, SSH, LDAP."""
|
||||||
|
|
||||||
|
name = "auth_flow_tracker"
|
||||||
|
module_type = "passive"
|
||||||
|
priority = 100
|
||||||
|
requires_root = True
|
||||||
|
|
||||||
|
def __init__(self, bus, state, config, engine=None):
|
||||||
|
super().__init__(bus, state, config, engine)
|
||||||
|
self._capture_bus = None
|
||||||
|
self._sub_queue = None
|
||||||
|
self._read_thread: Optional[threading.Thread] = None
|
||||||
|
self._flush_thread: Optional[threading.Thread] = None
|
||||||
|
self._db_path = self._resolve_db_path()
|
||||||
|
self._db_conn: Optional[sqlite3.Connection] = None
|
||||||
|
self._write_buffer: list[tuple] = []
|
||||||
|
self._buffer_lock = threading.Lock()
|
||||||
|
# Track per-user service targets for service account detection
|
||||||
|
self._user_targets: dict[str, set] = defaultdict(set)
|
||||||
|
self._stats = {
|
||||||
|
"packets_processed": 0,
|
||||||
|
"kerberos_events": 0,
|
||||||
|
"ntlm_events": 0,
|
||||||
|
"ssh_events": 0,
|
||||||
|
"ldap_bind_events": 0,
|
||||||
|
"service_accounts_detected": 0,
|
||||||
|
}
|
||||||
|
|
||||||
|
# ------------------------------------------------------------------
|
||||||
|
# BaseModule interface
|
||||||
|
# ------------------------------------------------------------------
|
||||||
|
|
||||||
|
def start(self) -> None:
|
||||||
|
if self._running:
|
||||||
|
return
|
||||||
|
self._init_db()
|
||||||
|
self._capture_bus = self.config.get("_capture_bus")
|
||||||
|
if self._capture_bus is None:
|
||||||
|
logger.error("AuthFlowTracker requires _capture_bus in config")
|
||||||
|
return
|
||||||
|
|
||||||
|
# BPF for auth protocol ports
|
||||||
|
bpf = (
|
||||||
|
"port 88 or port 445 or port 389 or port 636 "
|
||||||
|
"or port 3389 or port 22 or port 1433"
|
||||||
|
)
|
||||||
|
self._sub_queue = self._capture_bus.subscribe(
|
||||||
|
name=self.name, bpf_filter=bpf, queue_depth=5000
|
||||||
|
)
|
||||||
|
|
||||||
|
self._running = True
|
||||||
|
self._start_time = time.time()
|
||||||
|
self._pid = os.getpid()
|
||||||
|
|
||||||
|
self._read_thread = threading.Thread(
|
||||||
|
target=self._read_loop, daemon=True, name="bb-auth-read"
|
||||||
|
)
|
||||||
|
self._read_thread.start()
|
||||||
|
|
||||||
|
self._flush_thread = threading.Thread(
|
||||||
|
target=self._flush_loop, daemon=True, name="bb-auth-flush"
|
||||||
|
)
|
||||||
|
self._flush_thread.start()
|
||||||
|
|
||||||
|
self.state.set_module_status(self.name, "running", pid=self._pid)
|
||||||
|
self.bus.emit("MODULE_STARTED", {"module": self.name}, source_module=self.name)
|
||||||
|
logger.info("AuthFlowTracker started")
|
||||||
|
|
||||||
|
def stop(self) -> None:
|
||||||
|
if not self._running:
|
||||||
|
return
|
||||||
|
self._running = False
|
||||||
|
if self._capture_bus:
|
||||||
|
self._capture_bus.unsubscribe(self.name)
|
||||||
|
for t in (self._read_thread, self._flush_thread):
|
||||||
|
if t and t.is_alive():
|
||||||
|
t.join(timeout=5.0)
|
||||||
|
self._flush_buffer()
|
||||||
|
if self._db_conn:
|
||||||
|
self._db_conn.close()
|
||||||
|
self._db_conn = None
|
||||||
|
self.state.set_module_status(self.name, "stopped")
|
||||||
|
self.bus.emit("MODULE_STOPPED", {"module": self.name}, source_module=self.name)
|
||||||
|
logger.info("AuthFlowTracker stopped — stats: %s", self._stats)
|
||||||
|
|
||||||
|
def status(self) -> dict:
|
||||||
|
return {
|
||||||
|
"running": self._running,
|
||||||
|
"pid": self._pid,
|
||||||
|
"uptime": time.time() - self._start_time if self._start_time else 0,
|
||||||
|
**self._stats,
|
||||||
|
}
|
||||||
|
|
||||||
|
def configure(self, config: dict) -> None:
|
||||||
|
self.config.update(config)
|
||||||
|
|
||||||
|
# ------------------------------------------------------------------
|
||||||
|
# Packet processing
|
||||||
|
# ------------------------------------------------------------------
|
||||||
|
|
||||||
|
def _read_loop(self) -> None:
|
||||||
|
while self._running:
|
||||||
|
result = self._sub_queue.get(timeout=1.0)
|
||||||
|
if result is None:
|
||||||
|
continue
|
||||||
|
ts, pkt = result
|
||||||
|
self._stats["packets_processed"] += 1
|
||||||
|
try:
|
||||||
|
self._process_packet(pkt, ts)
|
||||||
|
except Exception:
|
||||||
|
logger.debug("Error processing auth packet", exc_info=True)
|
||||||
|
|
||||||
|
def _process_packet(self, pkt: bytes, ts: float) -> None:
|
||||||
|
"""Route packet to appropriate protocol parser."""
|
||||||
|
if len(pkt) < 34:
|
||||||
|
return
|
||||||
|
|
||||||
|
ethertype = struct.unpack("!H", pkt[12:14])[0]
|
||||||
|
eth_offset = 14
|
||||||
|
if ethertype == 0x8100:
|
||||||
|
if len(pkt) < 38:
|
||||||
|
return
|
||||||
|
ethertype = struct.unpack("!H", pkt[16:18])[0]
|
||||||
|
eth_offset = 18
|
||||||
|
|
||||||
|
if ethertype != 0x0800:
|
||||||
|
return
|
||||||
|
|
||||||
|
ip_header = pkt[eth_offset:]
|
||||||
|
if len(ip_header) < 20:
|
||||||
|
return
|
||||||
|
|
||||||
|
ihl = (ip_header[0] & 0x0F) * 4
|
||||||
|
ip_proto = ip_header[9]
|
||||||
|
src_ip = socket.inet_ntoa(ip_header[12:16])
|
||||||
|
dst_ip = socket.inet_ntoa(ip_header[16:20])
|
||||||
|
|
||||||
|
if ip_proto == TCP_PROTO and len(ip_header) >= ihl + 4:
|
||||||
|
src_port, dst_port = struct.unpack("!HH", ip_header[ihl:ihl + 4])
|
||||||
|
# TCP data offset
|
||||||
|
if len(ip_header) < ihl + 12:
|
||||||
|
return
|
||||||
|
tcp_data_off = ((ip_header[ihl + 12] >> 4) & 0xF) * 4
|
||||||
|
payload = ip_header[ihl + tcp_data_off:]
|
||||||
|
if not payload:
|
||||||
|
return
|
||||||
|
|
||||||
|
port = dst_port
|
||||||
|
if dst_port == KERBEROS_PORT or src_port == KERBEROS_PORT:
|
||||||
|
self._parse_kerberos(payload, src_ip, dst_ip, ts)
|
||||||
|
elif dst_port == SMB_PORT or src_port == SMB_PORT:
|
||||||
|
self._parse_ntlm_in_payload(payload, src_ip, dst_ip, "smb", ts)
|
||||||
|
elif dst_port in (LDAP_PORT, LDAPS_PORT) or src_port in (LDAP_PORT, LDAPS_PORT):
|
||||||
|
self._parse_ldap_bind(payload, src_ip, dst_ip, ts)
|
||||||
|
self._parse_ntlm_in_payload(payload, src_ip, dst_ip, "ldap", ts)
|
||||||
|
elif dst_port == SSH_PORT or src_port == SSH_PORT:
|
||||||
|
self._parse_ssh(payload, src_ip, dst_ip, ts)
|
||||||
|
elif dst_port == RDP_PORT or src_port == RDP_PORT:
|
||||||
|
self._parse_ntlm_in_payload(payload, src_ip, dst_ip, "rdp", ts)
|
||||||
|
elif dst_port == MSSQL_PORT or src_port == MSSQL_PORT:
|
||||||
|
self._parse_ntlm_in_payload(payload, src_ip, dst_ip, "mssql", ts)
|
||||||
|
|
||||||
|
elif ip_proto == UDP_PROTO and len(ip_header) >= ihl + 8:
|
||||||
|
src_port, dst_port = struct.unpack("!HH", ip_header[ihl:ihl + 4])
|
||||||
|
payload = ip_header[ihl + 8:]
|
||||||
|
if dst_port == KERBEROS_PORT or src_port == KERBEROS_PORT:
|
||||||
|
self._parse_kerberos(payload, src_ip, dst_ip, ts)
|
||||||
|
|
||||||
|
# ------------------------------------------------------------------
|
||||||
|
# Kerberos parser
|
||||||
|
# ------------------------------------------------------------------
|
||||||
|
|
||||||
|
def _parse_kerberos(self, payload: bytes, src_ip: str, dst_ip: str,
|
||||||
|
ts: float) -> None:
|
||||||
|
"""Parse Kerberos AS-REQ and TGS-REQ to extract principal, realm, SPN."""
|
||||||
|
if len(payload) < 10:
|
||||||
|
return
|
||||||
|
|
||||||
|
# Kerberos uses ASN.1 DER encoding
|
||||||
|
# Application tag: AS-REQ=[APPLICATION 10], TGS-REQ=[APPLICATION 12]
|
||||||
|
tag = payload[0]
|
||||||
|
if tag & 0x1F == 0x1F:
|
||||||
|
# Long-form tag, skip for now
|
||||||
|
return
|
||||||
|
|
||||||
|
app_class = (tag >> 6) & 0x03
|
||||||
|
if app_class != 1: # APPLICATION class
|
||||||
|
return
|
||||||
|
|
||||||
|
msg_type = tag & 0x1F
|
||||||
|
auth_type = ""
|
||||||
|
if msg_type == KRB_AS_REQ:
|
||||||
|
auth_type = "kerberos_as_req"
|
||||||
|
elif msg_type == KRB_TGS_REQ:
|
||||||
|
auth_type = "kerberos_tgs_req"
|
||||||
|
else:
|
||||||
|
return
|
||||||
|
|
||||||
|
# Extract principal and realm from ASN.1 — simplified extraction
|
||||||
|
# Look for common realm and principal patterns in the raw bytes
|
||||||
|
realm = self._extract_krb_string(payload, b'\x1b') # GeneralString
|
||||||
|
principal = self._extract_krb_string(payload, b'\x1b', skip=1)
|
||||||
|
|
||||||
|
self._stats["kerberos_events"] += 1
|
||||||
|
self._record_auth(
|
||||||
|
ts, src_ip, dst_ip, "kerberos", auth_type,
|
||||||
|
username=principal, domain=realm,
|
||||||
|
target_service=principal if msg_type == KRB_TGS_REQ else "",
|
||||||
|
)
|
||||||
|
|
||||||
|
def _extract_krb_string(self, data: bytes, tag: bytes, skip: int = 0) -> str:
|
||||||
|
"""Extract a GeneralString value from ASN.1 data (simplified)."""
|
||||||
|
search_tag = tag[0]
|
||||||
|
pos = 0
|
||||||
|
found = 0
|
||||||
|
while pos < len(data) - 2:
|
||||||
|
if data[pos] == search_tag:
|
||||||
|
length = data[pos + 1]
|
||||||
|
if length > 0 and length < 128 and pos + 2 + length <= len(data):
|
||||||
|
if found >= skip:
|
||||||
|
try:
|
||||||
|
return data[pos + 2:pos + 2 + length].decode("ascii", errors="replace")
|
||||||
|
except Exception:
|
||||||
|
return ""
|
||||||
|
found += 1
|
||||||
|
pos += 1
|
||||||
|
return ""
|
||||||
|
|
||||||
|
# ------------------------------------------------------------------
|
||||||
|
# NTLM parser
|
||||||
|
# ------------------------------------------------------------------
|
||||||
|
|
||||||
|
def _parse_ntlm_in_payload(self, payload: bytes, src_ip: str, dst_ip: str,
|
||||||
|
protocol: str, ts: float) -> None:
|
||||||
|
"""Search for NTLMSSP messages embedded in any protocol payload."""
|
||||||
|
idx = payload.find(NTLMSSP_SIGNATURE)
|
||||||
|
if idx == -1 or idx + 12 > len(payload):
|
||||||
|
return
|
||||||
|
|
||||||
|
ntlm_data = payload[idx:]
|
||||||
|
msg_type = struct.unpack("<I", ntlm_data[8:12])[0]
|
||||||
|
|
||||||
|
if msg_type == NTLM_AUTHENTICATE and len(ntlm_data) >= 88:
|
||||||
|
# Type 3: Authenticate message
|
||||||
|
username, domain, workstation = self._parse_ntlm_auth(ntlm_data)
|
||||||
|
if username:
|
||||||
|
self._stats["ntlm_events"] += 1
|
||||||
|
self._record_auth(
|
||||||
|
ts, src_ip, dst_ip, protocol, "ntlm_auth",
|
||||||
|
username=username, domain=domain,
|
||||||
|
)
|
||||||
|
elif msg_type == NTLM_CHALLENGE and len(ntlm_data) >= 56:
|
||||||
|
# Type 2: Challenge — extract target name
|
||||||
|
target = self._parse_ntlm_challenge_target(ntlm_data)
|
||||||
|
if target:
|
||||||
|
self._stats["ntlm_events"] += 1
|
||||||
|
self._record_auth(
|
||||||
|
ts, dst_ip, src_ip, protocol, "ntlm_challenge",
|
||||||
|
domain=target,
|
||||||
|
)
|
||||||
|
|
||||||
|
def _parse_ntlm_auth(self, data: bytes) -> tuple:
|
||||||
|
"""Parse NTLM Type 3 Authenticate message for username, domain, workstation."""
|
||||||
|
if len(data) < 72:
|
||||||
|
return ("", "", "")
|
||||||
|
|
||||||
|
try:
|
||||||
|
# LmChallengeResponse: offset 12
|
||||||
|
# NtChallengeResponse: offset 20
|
||||||
|
# DomainName: length(2), maxlen(2), offset(4) at byte 28
|
||||||
|
domain_len = struct.unpack("<H", data[28:30])[0]
|
||||||
|
domain_off = struct.unpack("<I", data[32:36])[0]
|
||||||
|
|
||||||
|
# UserName: length(2), maxlen(2), offset(4) at byte 36
|
||||||
|
user_len = struct.unpack("<H", data[36:38])[0]
|
||||||
|
user_off = struct.unpack("<I", data[40:44])[0]
|
||||||
|
|
||||||
|
# Workstation: length(2), maxlen(2), offset(4) at byte 44
|
||||||
|
ws_len = struct.unpack("<H", data[44:46])[0]
|
||||||
|
ws_off = struct.unpack("<I", data[48:52])[0]
|
||||||
|
|
||||||
|
domain = data[domain_off:domain_off + domain_len].decode("utf-16-le", errors="replace") if domain_len else ""
|
||||||
|
username = data[user_off:user_off + user_len].decode("utf-16-le", errors="replace") if user_len else ""
|
||||||
|
workstation = data[ws_off:ws_off + ws_len].decode("utf-16-le", errors="replace") if ws_len else ""
|
||||||
|
|
||||||
|
return (username, domain, workstation)
|
||||||
|
except Exception:
|
||||||
|
return ("", "", "")
|
||||||
|
|
||||||
|
def _parse_ntlm_challenge_target(self, data: bytes) -> str:
|
||||||
|
"""Parse NTLM Type 2 Challenge for target name."""
|
||||||
|
if len(data) < 24:
|
||||||
|
return ""
|
||||||
|
try:
|
||||||
|
target_len = struct.unpack("<H", data[12:14])[0]
|
||||||
|
target_off = struct.unpack("<I", data[16:20])[0]
|
||||||
|
if target_len and target_off + target_len <= len(data):
|
||||||
|
return data[target_off:target_off + target_len].decode("utf-16-le", errors="replace")
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
|
return ""
|
||||||
|
|
||||||
|
# ------------------------------------------------------------------
|
||||||
|
# SSH parser
|
||||||
|
# ------------------------------------------------------------------
|
||||||
|
|
||||||
|
def _parse_ssh(self, payload: bytes, src_ip: str, dst_ip: str,
|
||||||
|
ts: float) -> None:
|
||||||
|
"""Parse SSH version exchange and detect auth activity."""
|
||||||
|
# SSH version string starts with "SSH-"
|
||||||
|
if payload[:4] == b'SSH-':
|
||||||
|
try:
|
||||||
|
version_line = payload.split(b'\r\n')[0].decode("ascii", errors="replace")
|
||||||
|
except Exception:
|
||||||
|
version_line = ""
|
||||||
|
|
||||||
|
self._stats["ssh_events"] += 1
|
||||||
|
self._record_auth(
|
||||||
|
ts, src_ip, dst_ip, "ssh", "version_exchange",
|
||||||
|
target_service=version_line,
|
||||||
|
)
|
||||||
|
|
||||||
|
# ------------------------------------------------------------------
|
||||||
|
# LDAP bind parser
|
||||||
|
# ------------------------------------------------------------------
|
||||||
|
|
||||||
|
def _parse_ldap_bind(self, payload: bytes, src_ip: str, dst_ip: str,
|
||||||
|
ts: float) -> None:
|
||||||
|
"""Parse LDAP simple bind request to extract username."""
|
||||||
|
if len(payload) < 10:
|
||||||
|
return
|
||||||
|
|
||||||
|
# LDAP messages are BER-encoded
|
||||||
|
# BindRequest: APPLICATION[0] -> sequence: version, name, auth
|
||||||
|
# Look for a simple bind: tag 0x60 (APPLICATION CONSTRUCTED 0)
|
||||||
|
if payload[0] != 0x30: # Not a SEQUENCE
|
||||||
|
return
|
||||||
|
|
||||||
|
# Search for BindRequest tag (0x60) within the message
|
||||||
|
idx = payload.find(b'\x60')
|
||||||
|
if idx == -1 or idx + 10 > len(payload):
|
||||||
|
return
|
||||||
|
|
||||||
|
# Skip the BindRequest tag and length
|
||||||
|
bind_data = payload[idx + 1:]
|
||||||
|
if not bind_data:
|
||||||
|
return
|
||||||
|
|
||||||
|
# Try to read the length
|
||||||
|
bind_len, len_size = self._read_ber_length(bind_data)
|
||||||
|
if bind_len <= 0:
|
||||||
|
return
|
||||||
|
|
||||||
|
bind_body = bind_data[len_size:]
|
||||||
|
|
||||||
|
# Version (INTEGER tag 0x02)
|
||||||
|
if len(bind_body) < 3 or bind_body[0] != 0x02:
|
||||||
|
return
|
||||||
|
ver_len = bind_body[1]
|
||||||
|
name_start = 2 + ver_len
|
||||||
|
|
||||||
|
# Name (OCTET STRING tag 0x04)
|
||||||
|
if len(bind_body) <= name_start + 2:
|
||||||
|
return
|
||||||
|
if bind_body[name_start] != 0x04:
|
||||||
|
return
|
||||||
|
name_len = bind_body[name_start + 1]
|
||||||
|
if name_len > 0 and name_start + 2 + name_len <= len(bind_body):
|
||||||
|
username = bind_body[name_start + 2:name_start + 2 + name_len].decode(
|
||||||
|
"utf-8", errors="replace"
|
||||||
|
)
|
||||||
|
if username and username not in ("", "anonymous"):
|
||||||
|
self._stats["ldap_bind_events"] += 1
|
||||||
|
self._record_auth(
|
||||||
|
ts, src_ip, dst_ip, "ldap", "simple_bind",
|
||||||
|
username=username,
|
||||||
|
)
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _read_ber_length(data: bytes) -> tuple:
|
||||||
|
"""Read BER length encoding. Returns (length, bytes_consumed)."""
|
||||||
|
if not data:
|
||||||
|
return (0, 0)
|
||||||
|
first = data[0]
|
||||||
|
if first < 0x80:
|
||||||
|
return (first, 1)
|
||||||
|
num_bytes = first & 0x7F
|
||||||
|
if num_bytes == 0 or len(data) < 1 + num_bytes:
|
||||||
|
return (0, 0)
|
||||||
|
length = 0
|
||||||
|
for i in range(num_bytes):
|
||||||
|
length = (length << 8) | data[1 + i]
|
||||||
|
return (length, 1 + num_bytes)
|
||||||
|
|
||||||
|
# ------------------------------------------------------------------
|
||||||
|
# Recording
|
||||||
|
# ------------------------------------------------------------------
|
||||||
|
|
||||||
|
def _record_auth(self, ts: float, src_ip: str, dst_ip: str,
|
||||||
|
protocol: str, auth_type: str, username: str = "",
|
||||||
|
domain: str = "", target_service: str = "",
|
||||||
|
success: int = -1) -> None:
|
||||||
|
"""Buffer an auth event for batch insert."""
|
||||||
|
with self._buffer_lock:
|
||||||
|
self._write_buffer.append((
|
||||||
|
ts, src_ip, dst_ip, protocol, auth_type,
|
||||||
|
username, domain, target_service, success,
|
||||||
|
))
|
||||||
|
|
||||||
|
# Track service account patterns
|
||||||
|
if username:
|
||||||
|
key = f"{domain}\\{username}" if domain else username
|
||||||
|
self._user_targets[key].add(dst_ip)
|
||||||
|
if len(self._user_targets[key]) >= SERVICE_ACCOUNT_THRESHOLD:
|
||||||
|
# Only count once per user crossing threshold
|
||||||
|
current_count = len([
|
||||||
|
u for u, targets in self._user_targets.items()
|
||||||
|
if len(targets) >= SERVICE_ACCOUNT_THRESHOLD
|
||||||
|
])
|
||||||
|
if current_count > self._stats["service_accounts_detected"]:
|
||||||
|
self._stats["service_accounts_detected"] = current_count
|
||||||
|
|
||||||
|
# ------------------------------------------------------------------
|
||||||
|
# Database
|
||||||
|
# ------------------------------------------------------------------
|
||||||
|
|
||||||
|
def _resolve_db_path(self) -> str:
|
||||||
|
base = self.config.get("db_dir", os.path.join(
|
||||||
|
os.path.expanduser("~"), ".bigbrother"
|
||||||
|
))
|
||||||
|
return os.path.join(base, "auth_flow_tracker.db")
|
||||||
|
|
||||||
|
def _init_db(self) -> None:
|
||||||
|
Path(self._db_path).parent.mkdir(parents=True, exist_ok=True)
|
||||||
|
self._db_conn = sqlite3.connect(self._db_path, check_same_thread=False)
|
||||||
|
self._db_conn.execute("PRAGMA journal_mode=WAL")
|
||||||
|
self._db_conn.execute("PRAGMA synchronous=NORMAL")
|
||||||
|
self._db_conn.executescript(_DB_INIT)
|
||||||
|
self._db_conn.commit()
|
||||||
|
|
||||||
|
def _flush_buffer(self) -> None:
|
||||||
|
with self._buffer_lock:
|
||||||
|
batch = list(self._write_buffer)
|
||||||
|
self._write_buffer.clear()
|
||||||
|
|
||||||
|
if not batch or not self._db_conn:
|
||||||
|
return
|
||||||
|
|
||||||
|
try:
|
||||||
|
with self._db_conn:
|
||||||
|
self._db_conn.executemany(
|
||||||
|
"""INSERT INTO auth_events
|
||||||
|
(timestamp, source_ip, dest_ip, protocol, auth_type,
|
||||||
|
username, domain, target_service, success)
|
||||||
|
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)""",
|
||||||
|
batch,
|
||||||
|
)
|
||||||
|
except Exception:
|
||||||
|
logger.exception("Failed to flush auth events")
|
||||||
|
|
||||||
|
def _flush_loop(self) -> None:
|
||||||
|
while self._running:
|
||||||
|
time.sleep(FLUSH_INTERVAL)
|
||||||
|
try:
|
||||||
|
self._flush_buffer()
|
||||||
|
except Exception:
|
||||||
|
logger.exception("Flush loop error")
|
||||||
@@ -0,0 +1,322 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
"""Cloud token harvester — passive credential extraction from cleartext HTTP.
|
||||||
|
|
||||||
|
Regex-scans cleartext HTTP traffic (port 80) for AWS access keys, JWT tokens,
|
||||||
|
OAuth bearer tokens, SAML assertions, Azure AD tokens, and GCP service account
|
||||||
|
keys. Near-zero yield on most networks but zero cost to run. Publishes
|
||||||
|
CLOUD_TOKEN_FOUND events and feeds to credential_db via bus events.
|
||||||
|
"""
|
||||||
|
|
||||||
|
import logging
|
||||||
|
import os
|
||||||
|
import re
|
||||||
|
import socket
|
||||||
|
import sqlite3
|
||||||
|
import struct
|
||||||
|
import threading
|
||||||
|
import time
|
||||||
|
from pathlib import Path
|
||||||
|
from typing import Optional
|
||||||
|
|
||||||
|
from modules.base import BaseModule
|
||||||
|
|
||||||
|
logger = logging.getLogger("bb.passive.cloud_token_harvester")
|
||||||
|
|
||||||
|
# Token regexes — compiled once
|
||||||
|
_TOKEN_PATTERNS = {
|
||||||
|
"aws_access_key": re.compile(rb'(AKIA[A-Z0-9]{16})'),
|
||||||
|
"aws_secret_key": re.compile(rb'(?:aws_secret_access_key|secret[_-]?key)\s*[=:]\s*([A-Za-z0-9/+=]{40})', re.IGNORECASE),
|
||||||
|
"jwt": re.compile(rb'(eyJ[A-Za-z0-9_-]{10,}\.eyJ[A-Za-z0-9_-]{10,}\.[A-Za-z0-9_-]+)'),
|
||||||
|
"bearer_token": re.compile(rb'[Bb]earer\s+([A-Za-z0-9_\-\.~+/]+=*)', re.IGNORECASE),
|
||||||
|
"saml_assertion": re.compile(rb'(<saml[2p]*:Assertion[^>]*>.*?</saml[2p]*:Assertion>)', re.DOTALL | re.IGNORECASE),
|
||||||
|
"azure_ad_token": re.compile(rb'(eyJ[A-Za-z0-9_-]+\.eyJ[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+)'),
|
||||||
|
"gcp_service_key": re.compile(rb'"type"\s*:\s*"service_account".*?"private_key"\s*:\s*"([^"]+)"', re.DOTALL),
|
||||||
|
"api_key_generic": re.compile(rb'(?:api[_-]?key|apikey|x-api-key)\s*[=:]\s*([A-Za-z0-9_\-]{20,})', re.IGNORECASE),
|
||||||
|
"github_token": re.compile(rb'(ghp_[A-Za-z0-9]{36}|gho_[A-Za-z0-9]{36}|ghs_[A-Za-z0-9]{36}|ghr_[A-Za-z0-9]{36})'),
|
||||||
|
"slack_token": re.compile(rb'(xox[baprs]-[A-Za-z0-9\-]+)'),
|
||||||
|
}
|
||||||
|
|
||||||
|
_DB_INIT = """
|
||||||
|
CREATE TABLE IF NOT EXISTS cloud_tokens (
|
||||||
|
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||||
|
timestamp REAL NOT NULL,
|
||||||
|
source_ip TEXT NOT NULL,
|
||||||
|
dest_ip TEXT NOT NULL,
|
||||||
|
token_type TEXT NOT NULL,
|
||||||
|
token_value TEXT NOT NULL,
|
||||||
|
context TEXT DEFAULT ''
|
||||||
|
);
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_ct_type ON cloud_tokens(token_type);
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_ct_ts ON cloud_tokens(timestamp);
|
||||||
|
"""
|
||||||
|
|
||||||
|
FLUSH_INTERVAL = 60
|
||||||
|
|
||||||
|
|
||||||
|
class CloudTokenHarvester(BaseModule):
|
||||||
|
"""Passively harvest cloud tokens and API keys from cleartext HTTP."""
|
||||||
|
|
||||||
|
name = "cloud_token_harvester"
|
||||||
|
module_type = "passive"
|
||||||
|
priority = 200
|
||||||
|
requires_root = True
|
||||||
|
|
||||||
|
def __init__(self, bus, state, config, engine=None):
|
||||||
|
super().__init__(bus, state, config, engine)
|
||||||
|
self._capture_bus = None
|
||||||
|
self._sub_queue = None
|
||||||
|
self._read_thread: Optional[threading.Thread] = None
|
||||||
|
self._flush_thread: Optional[threading.Thread] = None
|
||||||
|
self._db_path = self._resolve_db_path()
|
||||||
|
self._db_conn: Optional[sqlite3.Connection] = None
|
||||||
|
self._write_buffer: list[tuple] = []
|
||||||
|
self._buffer_lock = threading.Lock()
|
||||||
|
# Dedup: track recently seen tokens to avoid spamming
|
||||||
|
self._seen_tokens: set = set()
|
||||||
|
self._seen_tokens_lock = threading.Lock()
|
||||||
|
self._stats = {
|
||||||
|
"packets_processed": 0,
|
||||||
|
"http_payloads_scanned": 0,
|
||||||
|
"tokens_found": 0,
|
||||||
|
"tokens_by_type": {},
|
||||||
|
}
|
||||||
|
|
||||||
|
# ------------------------------------------------------------------
|
||||||
|
# BaseModule interface
|
||||||
|
# ------------------------------------------------------------------
|
||||||
|
|
||||||
|
def start(self) -> None:
|
||||||
|
if self._running:
|
||||||
|
return
|
||||||
|
self._init_db()
|
||||||
|
self._capture_bus = self.config.get("_capture_bus")
|
||||||
|
if self._capture_bus is None:
|
||||||
|
logger.error("CloudTokenHarvester requires _capture_bus in config")
|
||||||
|
return
|
||||||
|
|
||||||
|
self._sub_queue = self._capture_bus.subscribe(
|
||||||
|
name=self.name, bpf_filter="port 80", queue_depth=3000
|
||||||
|
)
|
||||||
|
|
||||||
|
self._running = True
|
||||||
|
self._start_time = time.time()
|
||||||
|
self._pid = os.getpid()
|
||||||
|
|
||||||
|
self._read_thread = threading.Thread(
|
||||||
|
target=self._read_loop, daemon=True, name="bb-cloud-read"
|
||||||
|
)
|
||||||
|
self._read_thread.start()
|
||||||
|
|
||||||
|
self._flush_thread = threading.Thread(
|
||||||
|
target=self._flush_loop, daemon=True, name="bb-cloud-flush"
|
||||||
|
)
|
||||||
|
self._flush_thread.start()
|
||||||
|
|
||||||
|
self.state.set_module_status(self.name, "running", pid=self._pid)
|
||||||
|
self.bus.emit("MODULE_STARTED", {"module": self.name}, source_module=self.name)
|
||||||
|
logger.info("CloudTokenHarvester started")
|
||||||
|
|
||||||
|
def stop(self) -> None:
|
||||||
|
if not self._running:
|
||||||
|
return
|
||||||
|
self._running = False
|
||||||
|
if self._capture_bus:
|
||||||
|
self._capture_bus.unsubscribe(self.name)
|
||||||
|
for t in (self._read_thread, self._flush_thread):
|
||||||
|
if t and t.is_alive():
|
||||||
|
t.join(timeout=5.0)
|
||||||
|
self._flush_buffer()
|
||||||
|
if self._db_conn:
|
||||||
|
self._db_conn.close()
|
||||||
|
self._db_conn = None
|
||||||
|
self.state.set_module_status(self.name, "stopped")
|
||||||
|
self.bus.emit("MODULE_STOPPED", {"module": self.name}, source_module=self.name)
|
||||||
|
logger.info("CloudTokenHarvester stopped — stats: %s", self._stats)
|
||||||
|
|
||||||
|
def status(self) -> dict:
|
||||||
|
return {
|
||||||
|
"running": self._running,
|
||||||
|
"pid": self._pid,
|
||||||
|
"uptime": time.time() - self._start_time if self._start_time else 0,
|
||||||
|
**self._stats,
|
||||||
|
}
|
||||||
|
|
||||||
|
def configure(self, config: dict) -> None:
|
||||||
|
self.config.update(config)
|
||||||
|
|
||||||
|
# ------------------------------------------------------------------
|
||||||
|
# Packet processing
|
||||||
|
# ------------------------------------------------------------------
|
||||||
|
|
||||||
|
def _read_loop(self) -> None:
|
||||||
|
while self._running:
|
||||||
|
result = self._sub_queue.get(timeout=1.0)
|
||||||
|
if result is None:
|
||||||
|
continue
|
||||||
|
ts, pkt = result
|
||||||
|
self._stats["packets_processed"] += 1
|
||||||
|
try:
|
||||||
|
self._process_packet(pkt, ts)
|
||||||
|
except Exception:
|
||||||
|
logger.debug("Error processing packet", exc_info=True)
|
||||||
|
|
||||||
|
def _process_packet(self, pkt: bytes, ts: float) -> None:
|
||||||
|
"""Extract HTTP payload and scan for tokens."""
|
||||||
|
if len(pkt) < 54:
|
||||||
|
return
|
||||||
|
|
||||||
|
ethertype = struct.unpack("!H", pkt[12:14])[0]
|
||||||
|
eth_offset = 14
|
||||||
|
if ethertype == 0x8100:
|
||||||
|
if len(pkt) < 58:
|
||||||
|
return
|
||||||
|
ethertype = struct.unpack("!H", pkt[16:18])[0]
|
||||||
|
eth_offset = 18
|
||||||
|
|
||||||
|
if ethertype != 0x0800:
|
||||||
|
return
|
||||||
|
|
||||||
|
ip_header = pkt[eth_offset:]
|
||||||
|
if len(ip_header) < 20:
|
||||||
|
return
|
||||||
|
ihl = (ip_header[0] & 0x0F) * 4
|
||||||
|
ip_proto = ip_header[9]
|
||||||
|
if ip_proto != 6: # TCP only
|
||||||
|
return
|
||||||
|
|
||||||
|
src_ip = socket.inet_ntoa(ip_header[12:16])
|
||||||
|
dst_ip = socket.inet_ntoa(ip_header[16:20])
|
||||||
|
|
||||||
|
if len(ip_header) < ihl + 20:
|
||||||
|
return
|
||||||
|
tcp_data_off = ((ip_header[ihl + 12] >> 4) & 0xF) * 4
|
||||||
|
payload = ip_header[ihl + tcp_data_off:]
|
||||||
|
|
||||||
|
if len(payload) < 10:
|
||||||
|
return
|
||||||
|
|
||||||
|
# Quick check: does this look like HTTP?
|
||||||
|
if not (payload[:4] in (b'GET ', b'POST', b'PUT ', b'HEAD', b'HTTP', b'PATC', b'DELE')
|
||||||
|
or payload[:7] == b'CONNECT' or payload[:7] == b'OPTIONS'):
|
||||||
|
return
|
||||||
|
|
||||||
|
self._stats["http_payloads_scanned"] += 1
|
||||||
|
self._scan_for_tokens(payload, src_ip, dst_ip, ts)
|
||||||
|
|
||||||
|
def _scan_for_tokens(self, payload: bytes, src_ip: str, dst_ip: str,
|
||||||
|
ts: float) -> None:
|
||||||
|
"""Run all token regexes against the HTTP payload."""
|
||||||
|
for token_type, pattern in _TOKEN_PATTERNS.items():
|
||||||
|
try:
|
||||||
|
matches = pattern.findall(payload)
|
||||||
|
except Exception:
|
||||||
|
continue
|
||||||
|
|
||||||
|
for match in matches:
|
||||||
|
if isinstance(match, bytes):
|
||||||
|
token_value = match.decode("ascii", errors="replace")
|
||||||
|
else:
|
||||||
|
token_value = str(match)
|
||||||
|
|
||||||
|
# Skip very short matches (likely false positives)
|
||||||
|
if len(token_value) < 10:
|
||||||
|
continue
|
||||||
|
|
||||||
|
# Dedup
|
||||||
|
token_hash = f"{token_type}:{token_value[:32]}"
|
||||||
|
with self._seen_tokens_lock:
|
||||||
|
if token_hash in self._seen_tokens:
|
||||||
|
continue
|
||||||
|
self._seen_tokens.add(token_hash)
|
||||||
|
# Bound the seen set
|
||||||
|
if len(self._seen_tokens) > 10000:
|
||||||
|
self._seen_tokens.clear()
|
||||||
|
|
||||||
|
# Extract context (surrounding bytes for context)
|
||||||
|
idx = payload.find(match if isinstance(match, bytes) else match.encode())
|
||||||
|
context_start = max(0, idx - 50)
|
||||||
|
context_end = min(len(payload), idx + len(match) + 50) if idx >= 0 else 0
|
||||||
|
context = payload[context_start:context_end].decode("ascii", errors="replace") if idx >= 0 else ""
|
||||||
|
|
||||||
|
self._stats["tokens_found"] += 1
|
||||||
|
self._stats["tokens_by_type"][token_type] = (
|
||||||
|
self._stats["tokens_by_type"].get(token_type, 0) + 1
|
||||||
|
)
|
||||||
|
|
||||||
|
logger.info(
|
||||||
|
"Cloud token found: type=%s src=%s dst=%s value=%s...",
|
||||||
|
token_type, src_ip, dst_ip, token_value[:20]
|
||||||
|
)
|
||||||
|
|
||||||
|
# Publish event
|
||||||
|
self.bus.emit("CLOUD_TOKEN_FOUND", {
|
||||||
|
"token_type": token_type,
|
||||||
|
"token_value": token_value,
|
||||||
|
"source_ip": src_ip,
|
||||||
|
"dest_ip": dst_ip,
|
||||||
|
"context": context[:200],
|
||||||
|
}, source_module=self.name)
|
||||||
|
|
||||||
|
# Also feed to credential_db
|
||||||
|
self.bus.emit("CREDENTIAL_FOUND", {
|
||||||
|
"source": f"cloud_token_{token_type}",
|
||||||
|
"username": "",
|
||||||
|
"credential": token_value,
|
||||||
|
"source_ip": src_ip,
|
||||||
|
"dest_ip": dst_ip,
|
||||||
|
"protocol": "http",
|
||||||
|
}, source_module=self.name)
|
||||||
|
|
||||||
|
self._record_token(ts, src_ip, dst_ip, token_type, token_value, context)
|
||||||
|
|
||||||
|
# ------------------------------------------------------------------
|
||||||
|
# Database
|
||||||
|
# ------------------------------------------------------------------
|
||||||
|
|
||||||
|
def _resolve_db_path(self) -> str:
|
||||||
|
base = self.config.get("db_dir", os.path.join(
|
||||||
|
os.path.expanduser("~"), ".bigbrother"
|
||||||
|
))
|
||||||
|
return os.path.join(base, "cloud_token_harvester.db")
|
||||||
|
|
||||||
|
def _init_db(self) -> None:
|
||||||
|
Path(self._db_path).parent.mkdir(parents=True, exist_ok=True)
|
||||||
|
self._db_conn = sqlite3.connect(self._db_path, check_same_thread=False)
|
||||||
|
self._db_conn.execute("PRAGMA journal_mode=WAL")
|
||||||
|
self._db_conn.execute("PRAGMA synchronous=NORMAL")
|
||||||
|
self._db_conn.executescript(_DB_INIT)
|
||||||
|
self._db_conn.commit()
|
||||||
|
|
||||||
|
def _record_token(self, ts: float, src_ip: str, dst_ip: str,
|
||||||
|
token_type: str, token_value: str, context: str) -> None:
|
||||||
|
with self._buffer_lock:
|
||||||
|
self._write_buffer.append((
|
||||||
|
ts, src_ip, dst_ip, token_type, token_value, context[:500]
|
||||||
|
))
|
||||||
|
|
||||||
|
def _flush_buffer(self) -> None:
|
||||||
|
with self._buffer_lock:
|
||||||
|
batch = list(self._write_buffer)
|
||||||
|
self._write_buffer.clear()
|
||||||
|
|
||||||
|
if not batch or not self._db_conn:
|
||||||
|
return
|
||||||
|
|
||||||
|
try:
|
||||||
|
with self._db_conn:
|
||||||
|
self._db_conn.executemany(
|
||||||
|
"""INSERT INTO cloud_tokens
|
||||||
|
(timestamp, source_ip, dest_ip, token_type, token_value, context)
|
||||||
|
VALUES (?, ?, ?, ?, ?, ?)""",
|
||||||
|
batch,
|
||||||
|
)
|
||||||
|
except Exception:
|
||||||
|
logger.exception("Failed to flush cloud tokens")
|
||||||
|
|
||||||
|
def _flush_loop(self) -> None:
|
||||||
|
while self._running:
|
||||||
|
time.sleep(FLUSH_INTERVAL)
|
||||||
|
try:
|
||||||
|
self._flush_buffer()
|
||||||
|
except Exception:
|
||||||
|
logger.exception("Flush loop error")
|
||||||
@@ -0,0 +1,696 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
"""Database protocol interceptor — passive database query and auth monitoring.
|
||||||
|
|
||||||
|
Parses login packets and query metadata for:
|
||||||
|
- MSSQL TDS (1433): TDS7 Login packet, SQL batch text
|
||||||
|
- MySQL (3306): Handshake, Login Request, COM_QUERY
|
||||||
|
- PostgreSQL (5432): Startup message, Simple Query
|
||||||
|
- Redis (6379): AUTH command, key operations
|
||||||
|
- MongoDB (27017): OP_MSG saslStart, find operations
|
||||||
|
|
||||||
|
Logs queries but NOT full result sets to avoid excessive data.
|
||||||
|
"""
|
||||||
|
|
||||||
|
import logging
|
||||||
|
import os
|
||||||
|
import socket
|
||||||
|
import sqlite3
|
||||||
|
import struct
|
||||||
|
import threading
|
||||||
|
import time
|
||||||
|
from pathlib import Path
|
||||||
|
from typing import Optional
|
||||||
|
|
||||||
|
from modules.base import BaseModule
|
||||||
|
|
||||||
|
logger = logging.getLogger("bb.passive.db_interceptor")
|
||||||
|
|
||||||
|
# TDS packet types
|
||||||
|
TDS_SQL_BATCH = 0x01
|
||||||
|
TDS_LOGIN7 = 0x10
|
||||||
|
TDS_PRELOGIN = 0x12
|
||||||
|
|
||||||
|
# MySQL commands
|
||||||
|
MYSQL_COM_QUERY = 0x03
|
||||||
|
MYSQL_COM_INIT_DB = 0x02
|
||||||
|
|
||||||
|
# PostgreSQL message types
|
||||||
|
PG_STARTUP = 0x00 # No type byte — identified by structure
|
||||||
|
PG_SIMPLE_QUERY = ord('Q')
|
||||||
|
PG_PASSWORD = ord('p')
|
||||||
|
|
||||||
|
# Redis inline delimiters
|
||||||
|
REDIS_CRLF = b'\r\n'
|
||||||
|
|
||||||
|
# MongoDB opcodes
|
||||||
|
MONGO_OP_MSG = 2013
|
||||||
|
|
||||||
|
_DB_INIT = """
|
||||||
|
CREATE TABLE IF NOT EXISTS db_queries (
|
||||||
|
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||||
|
timestamp REAL NOT NULL,
|
||||||
|
source_ip TEXT NOT NULL,
|
||||||
|
dest_ip TEXT NOT NULL,
|
||||||
|
db_type TEXT NOT NULL,
|
||||||
|
username TEXT DEFAULT '',
|
||||||
|
database TEXT DEFAULT '',
|
||||||
|
query_text TEXT DEFAULT '',
|
||||||
|
operation TEXT NOT NULL
|
||||||
|
);
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_dbq_type ON db_queries(db_type);
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_dbq_user ON db_queries(username);
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_dbq_ts ON db_queries(timestamp);
|
||||||
|
"""
|
||||||
|
|
||||||
|
FLUSH_INTERVAL = 45
|
||||||
|
MAX_QUERY_LEN = 2000 # Truncate queries beyond this
|
||||||
|
|
||||||
|
|
||||||
|
class DBInterceptor(BaseModule):
|
||||||
|
"""Passively intercept database authentication and queries."""
|
||||||
|
|
||||||
|
name = "db_interceptor"
|
||||||
|
module_type = "passive"
|
||||||
|
priority = 150
|
||||||
|
requires_root = True
|
||||||
|
|
||||||
|
def __init__(self, bus, state, config, engine=None):
|
||||||
|
super().__init__(bus, state, config, engine)
|
||||||
|
self._capture_bus = None
|
||||||
|
self._sub_queue = None
|
||||||
|
self._read_thread: Optional[threading.Thread] = None
|
||||||
|
self._flush_thread: Optional[threading.Thread] = None
|
||||||
|
self._db_path = self._resolve_db_path()
|
||||||
|
self._db_conn: Optional[sqlite3.Connection] = None
|
||||||
|
self._write_buffer: list[tuple] = []
|
||||||
|
self._buffer_lock = threading.Lock()
|
||||||
|
self._stats = {
|
||||||
|
"packets_processed": 0,
|
||||||
|
"mssql_events": 0,
|
||||||
|
"mysql_events": 0,
|
||||||
|
"postgres_events": 0,
|
||||||
|
"redis_events": 0,
|
||||||
|
"mongodb_events": 0,
|
||||||
|
"logins_captured": 0,
|
||||||
|
"queries_captured": 0,
|
||||||
|
}
|
||||||
|
|
||||||
|
# ------------------------------------------------------------------
|
||||||
|
# BaseModule interface
|
||||||
|
# ------------------------------------------------------------------
|
||||||
|
|
||||||
|
def start(self) -> None:
|
||||||
|
if self._running:
|
||||||
|
return
|
||||||
|
self._init_db()
|
||||||
|
self._capture_bus = self.config.get("_capture_bus")
|
||||||
|
if self._capture_bus is None:
|
||||||
|
logger.error("DBInterceptor requires _capture_bus in config")
|
||||||
|
return
|
||||||
|
|
||||||
|
bpf = "port 1433 or port 3306 or port 5432 or port 6379 or port 27017"
|
||||||
|
self._sub_queue = self._capture_bus.subscribe(
|
||||||
|
name=self.name, bpf_filter=bpf, queue_depth=5000
|
||||||
|
)
|
||||||
|
|
||||||
|
self._running = True
|
||||||
|
self._start_time = time.time()
|
||||||
|
self._pid = os.getpid()
|
||||||
|
|
||||||
|
self._read_thread = threading.Thread(
|
||||||
|
target=self._read_loop, daemon=True, name="bb-dbi-read"
|
||||||
|
)
|
||||||
|
self._read_thread.start()
|
||||||
|
|
||||||
|
self._flush_thread = threading.Thread(
|
||||||
|
target=self._flush_loop, daemon=True, name="bb-dbi-flush"
|
||||||
|
)
|
||||||
|
self._flush_thread.start()
|
||||||
|
|
||||||
|
self.state.set_module_status(self.name, "running", pid=self._pid)
|
||||||
|
self.bus.emit("MODULE_STARTED", {"module": self.name}, source_module=self.name)
|
||||||
|
logger.info("DBInterceptor started")
|
||||||
|
|
||||||
|
def stop(self) -> None:
|
||||||
|
if not self._running:
|
||||||
|
return
|
||||||
|
self._running = False
|
||||||
|
if self._capture_bus:
|
||||||
|
self._capture_bus.unsubscribe(self.name)
|
||||||
|
for t in (self._read_thread, self._flush_thread):
|
||||||
|
if t and t.is_alive():
|
||||||
|
t.join(timeout=5.0)
|
||||||
|
self._flush_buffer()
|
||||||
|
if self._db_conn:
|
||||||
|
self._db_conn.close()
|
||||||
|
self._db_conn = None
|
||||||
|
self.state.set_module_status(self.name, "stopped")
|
||||||
|
self.bus.emit("MODULE_STOPPED", {"module": self.name}, source_module=self.name)
|
||||||
|
logger.info("DBInterceptor stopped — stats: %s", self._stats)
|
||||||
|
|
||||||
|
def status(self) -> dict:
|
||||||
|
return {
|
||||||
|
"running": self._running,
|
||||||
|
"pid": self._pid,
|
||||||
|
"uptime": time.time() - self._start_time if self._start_time else 0,
|
||||||
|
**self._stats,
|
||||||
|
}
|
||||||
|
|
||||||
|
def configure(self, config: dict) -> None:
|
||||||
|
self.config.update(config)
|
||||||
|
|
||||||
|
# ------------------------------------------------------------------
|
||||||
|
# Packet processing
|
||||||
|
# ------------------------------------------------------------------
|
||||||
|
|
||||||
|
def _read_loop(self) -> None:
|
||||||
|
while self._running:
|
||||||
|
result = self._sub_queue.get(timeout=1.0)
|
||||||
|
if result is None:
|
||||||
|
continue
|
||||||
|
ts, pkt = result
|
||||||
|
self._stats["packets_processed"] += 1
|
||||||
|
try:
|
||||||
|
self._process_packet(pkt, ts)
|
||||||
|
except Exception:
|
||||||
|
logger.debug("Error processing DB packet", exc_info=True)
|
||||||
|
|
||||||
|
def _process_packet(self, pkt: bytes, ts: float) -> None:
|
||||||
|
"""Route packet to the correct database parser based on port."""
|
||||||
|
if len(pkt) < 54:
|
||||||
|
return
|
||||||
|
|
||||||
|
ethertype = struct.unpack("!H", pkt[12:14])[0]
|
||||||
|
eth_offset = 14
|
||||||
|
if ethertype == 0x8100:
|
||||||
|
if len(pkt) < 58:
|
||||||
|
return
|
||||||
|
ethertype = struct.unpack("!H", pkt[16:18])[0]
|
||||||
|
eth_offset = 18
|
||||||
|
|
||||||
|
if ethertype != 0x0800:
|
||||||
|
return
|
||||||
|
|
||||||
|
ip_header = pkt[eth_offset:]
|
||||||
|
if len(ip_header) < 20:
|
||||||
|
return
|
||||||
|
ihl = (ip_header[0] & 0x0F) * 4
|
||||||
|
ip_proto = ip_header[9]
|
||||||
|
if ip_proto != 6: # TCP only
|
||||||
|
return
|
||||||
|
|
||||||
|
src_ip = socket.inet_ntoa(ip_header[12:16])
|
||||||
|
dst_ip = socket.inet_ntoa(ip_header[16:20])
|
||||||
|
|
||||||
|
if len(ip_header) < ihl + 20:
|
||||||
|
return
|
||||||
|
src_port, dst_port = struct.unpack("!HH", ip_header[ihl:ihl + 4])
|
||||||
|
tcp_data_off = ((ip_header[ihl + 12] >> 4) & 0xF) * 4
|
||||||
|
payload = ip_header[ihl + tcp_data_off:]
|
||||||
|
|
||||||
|
if len(payload) < 4:
|
||||||
|
return
|
||||||
|
|
||||||
|
# Route by destination port (client -> server)
|
||||||
|
if dst_port == 1433 or src_port == 1433:
|
||||||
|
self._parse_mssql(payload, src_ip, dst_ip, dst_port == 1433, ts)
|
||||||
|
elif dst_port == 3306 or src_port == 3306:
|
||||||
|
self._parse_mysql(payload, src_ip, dst_ip, dst_port == 3306, ts)
|
||||||
|
elif dst_port == 5432 or src_port == 5432:
|
||||||
|
self._parse_postgres(payload, src_ip, dst_ip, dst_port == 5432, ts)
|
||||||
|
elif dst_port == 6379 or src_port == 6379:
|
||||||
|
self._parse_redis(payload, src_ip, dst_ip, dst_port == 6379, ts)
|
||||||
|
elif dst_port == 27017 or src_port == 27017:
|
||||||
|
self._parse_mongodb(payload, src_ip, dst_ip, dst_port == 27017, ts)
|
||||||
|
|
||||||
|
# ------------------------------------------------------------------
|
||||||
|
# MSSQL TDS parser
|
||||||
|
# ------------------------------------------------------------------
|
||||||
|
|
||||||
|
def _parse_mssql(self, payload: bytes, src_ip: str, dst_ip: str,
|
||||||
|
is_to_server: bool, ts: float) -> None:
|
||||||
|
"""Parse MSSQL TDS packets."""
|
||||||
|
if len(payload) < 8:
|
||||||
|
return
|
||||||
|
|
||||||
|
# TDS header: type(1), status(1), length(2), SPID(2), packet(1), window(1)
|
||||||
|
tds_type = payload[0]
|
||||||
|
tds_len = struct.unpack("!H", payload[2:4])[0]
|
||||||
|
|
||||||
|
if tds_type == TDS_LOGIN7 and is_to_server and len(payload) >= 36:
|
||||||
|
self._parse_tds_login7(payload[8:], src_ip, dst_ip, ts)
|
||||||
|
elif tds_type == TDS_SQL_BATCH and is_to_server:
|
||||||
|
# SQL batch: header group + SQL text (all Unicode LE after 8-byte TDS header)
|
||||||
|
self._parse_tds_sql_batch(payload[8:], src_ip, dst_ip, ts)
|
||||||
|
|
||||||
|
def _parse_tds_login7(self, data: bytes, src_ip: str, dst_ip: str,
|
||||||
|
ts: float) -> None:
|
||||||
|
"""Parse TDS7 Login packet for username, hostname, app name."""
|
||||||
|
if len(data) < 36:
|
||||||
|
return
|
||||||
|
|
||||||
|
self._stats["mssql_events"] += 1
|
||||||
|
self._stats["logins_captured"] += 1
|
||||||
|
|
||||||
|
try:
|
||||||
|
# TDS7 Login: fixed header of 36 bytes then variable data
|
||||||
|
# HostName offset(2) + length(2) at offset 8
|
||||||
|
# UserName offset(2) + length(2) at offset 16
|
||||||
|
# AppName offset(2) + length(2) at offset 24
|
||||||
|
# ServerName offset(2) + length(2) at offset 28
|
||||||
|
# Database offset(2) + length(2) at offset 40 (but we have 36-byte minimum)
|
||||||
|
|
||||||
|
hostname_off = struct.unpack("<H", data[8:10])[0]
|
||||||
|
hostname_len = struct.unpack("<H", data[10:12])[0]
|
||||||
|
|
||||||
|
username_off = struct.unpack("<H", data[16:18])[0]
|
||||||
|
username_len = struct.unpack("<H", data[18:20])[0]
|
||||||
|
|
||||||
|
appname_off = struct.unpack("<H", data[24:26])[0]
|
||||||
|
appname_len = struct.unpack("<H", data[26:28])[0]
|
||||||
|
|
||||||
|
# Offsets are in chars (UTF-16LE = 2 bytes per char)
|
||||||
|
username = self._read_utf16le(data, username_off, username_len)
|
||||||
|
hostname = self._read_utf16le(data, hostname_off, hostname_len)
|
||||||
|
appname = self._read_utf16le(data, appname_off, appname_len)
|
||||||
|
|
||||||
|
database = ""
|
||||||
|
if len(data) >= 44:
|
||||||
|
db_off = struct.unpack("<H", data[40:42])[0]
|
||||||
|
db_len = struct.unpack("<H", data[42:44])[0]
|
||||||
|
database = self._read_utf16le(data, db_off, db_len)
|
||||||
|
|
||||||
|
self._record_query(
|
||||||
|
ts, src_ip, dst_ip, "mssql", username, database,
|
||||||
|
f"LOGIN hostname={hostname} app={appname}", "login"
|
||||||
|
)
|
||||||
|
|
||||||
|
self.bus.emit("CREDENTIAL_FOUND", {
|
||||||
|
"source": "mssql_tds_login",
|
||||||
|
"username": username,
|
||||||
|
"hostname": hostname,
|
||||||
|
"source_ip": src_ip,
|
||||||
|
"dest_ip": dst_ip,
|
||||||
|
"database": database,
|
||||||
|
}, source_module=self.name)
|
||||||
|
|
||||||
|
except Exception:
|
||||||
|
logger.debug("Failed to parse TDS Login7", exc_info=True)
|
||||||
|
|
||||||
|
def _parse_tds_sql_batch(self, data: bytes, src_ip: str, dst_ip: str,
|
||||||
|
ts: float) -> None:
|
||||||
|
"""Parse TDS SQL Batch for query text."""
|
||||||
|
if len(data) < 4:
|
||||||
|
return
|
||||||
|
|
||||||
|
# TDS SQL Batch: ALL_HEADERS (variable) + SQL text
|
||||||
|
# ALL_HEADERS starts with TotalLength(4)
|
||||||
|
total_headers_len = struct.unpack("<I", data[0:4])[0]
|
||||||
|
if total_headers_len > len(data):
|
||||||
|
total_headers_len = 0
|
||||||
|
|
||||||
|
sql_bytes = data[total_headers_len:]
|
||||||
|
if not sql_bytes:
|
||||||
|
return
|
||||||
|
|
||||||
|
try:
|
||||||
|
query = sql_bytes.decode("utf-16-le", errors="replace")
|
||||||
|
except Exception:
|
||||||
|
query = sql_bytes.decode("ascii", errors="replace")
|
||||||
|
|
||||||
|
query = query.strip('\x00').strip()[:MAX_QUERY_LEN]
|
||||||
|
if query:
|
||||||
|
self._stats["mssql_events"] += 1
|
||||||
|
self._stats["queries_captured"] += 1
|
||||||
|
self._record_query(ts, src_ip, dst_ip, "mssql", "", "", query, "query")
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _read_utf16le(data: bytes, char_offset: int, char_length: int) -> str:
|
||||||
|
"""Read UTF-16LE string from TDS data using char offset/length."""
|
||||||
|
byte_off = char_offset * 2
|
||||||
|
byte_len = char_length * 2
|
||||||
|
if byte_off + byte_len > len(data):
|
||||||
|
return ""
|
||||||
|
try:
|
||||||
|
return data[byte_off:byte_off + byte_len].decode("utf-16-le", errors="replace")
|
||||||
|
except Exception:
|
||||||
|
return ""
|
||||||
|
|
||||||
|
# ------------------------------------------------------------------
|
||||||
|
# MySQL parser
|
||||||
|
# ------------------------------------------------------------------
|
||||||
|
|
||||||
|
def _parse_mysql(self, payload: bytes, src_ip: str, dst_ip: str,
|
||||||
|
is_to_server: bool, ts: float) -> None:
|
||||||
|
"""Parse MySQL protocol packets."""
|
||||||
|
if len(payload) < 5:
|
||||||
|
return
|
||||||
|
|
||||||
|
# MySQL packet: length(3 LE) + sequence_id(1) + payload
|
||||||
|
pkt_len = struct.unpack("<I", payload[0:3] + b'\x00')[0]
|
||||||
|
seq_id = payload[3]
|
||||||
|
mysql_data = payload[4:]
|
||||||
|
|
||||||
|
if not mysql_data:
|
||||||
|
return
|
||||||
|
|
||||||
|
if not is_to_server:
|
||||||
|
# Server -> client: check for Handshake (protocol version 10)
|
||||||
|
if mysql_data[0] == 10 and seq_id == 0:
|
||||||
|
self._parse_mysql_handshake(mysql_data, src_ip, dst_ip, ts)
|
||||||
|
return
|
||||||
|
|
||||||
|
# Client -> server
|
||||||
|
cmd = mysql_data[0]
|
||||||
|
if cmd == MYSQL_COM_QUERY and len(mysql_data) > 1:
|
||||||
|
query = mysql_data[1:min(len(mysql_data), MAX_QUERY_LEN + 1)].decode(
|
||||||
|
"utf-8", errors="replace"
|
||||||
|
)
|
||||||
|
self._stats["mysql_events"] += 1
|
||||||
|
self._stats["queries_captured"] += 1
|
||||||
|
self._record_query(ts, src_ip, dst_ip, "mysql", "", "", query, "query")
|
||||||
|
|
||||||
|
elif seq_id == 1 and len(mysql_data) >= 32:
|
||||||
|
# Login Request (HandshakeResponse)
|
||||||
|
self._parse_mysql_login(mysql_data, src_ip, dst_ip, ts)
|
||||||
|
|
||||||
|
def _parse_mysql_handshake(self, data: bytes, src_ip: str, dst_ip: str,
|
||||||
|
ts: float) -> None:
|
||||||
|
"""Parse MySQL server Handshake for version info."""
|
||||||
|
# Protocol version (1) + server version (null-terminated)
|
||||||
|
if len(data) < 5:
|
||||||
|
return
|
||||||
|
null_idx = data.find(b'\x00', 1)
|
||||||
|
if null_idx < 0:
|
||||||
|
return
|
||||||
|
server_version = data[1:null_idx].decode("ascii", errors="replace")
|
||||||
|
self._stats["mysql_events"] += 1
|
||||||
|
# Record as informational
|
||||||
|
self._record_query(
|
||||||
|
ts, dst_ip, src_ip, "mysql", "", "",
|
||||||
|
f"SERVER_HANDSHAKE version={server_version}", "handshake"
|
||||||
|
)
|
||||||
|
|
||||||
|
def _parse_mysql_login(self, data: bytes, src_ip: str, dst_ip: str,
|
||||||
|
ts: float) -> None:
|
||||||
|
"""Parse MySQL Login Request (HandshakeResponse41)."""
|
||||||
|
if len(data) < 32:
|
||||||
|
return
|
||||||
|
|
||||||
|
try:
|
||||||
|
# Client capabilities (4 bytes), max packet (4), charset (1),
|
||||||
|
# reserved (23 bytes), username (null-terminated)
|
||||||
|
username_start = 32
|
||||||
|
null_idx = data.find(b'\x00', username_start)
|
||||||
|
if null_idx < 0:
|
||||||
|
return
|
||||||
|
username = data[username_start:null_idx].decode("utf-8", errors="replace")
|
||||||
|
|
||||||
|
# After username + null + auth data, there may be a database name
|
||||||
|
database = ""
|
||||||
|
# Skip auth response length + data
|
||||||
|
pos = null_idx + 1
|
||||||
|
if pos < len(data):
|
||||||
|
auth_len = data[pos]
|
||||||
|
pos += 1 + auth_len
|
||||||
|
if pos < len(data):
|
||||||
|
db_null = data.find(b'\x00', pos)
|
||||||
|
if db_null > pos:
|
||||||
|
database = data[pos:db_null].decode("utf-8", errors="replace")
|
||||||
|
|
||||||
|
self._stats["mysql_events"] += 1
|
||||||
|
self._stats["logins_captured"] += 1
|
||||||
|
self._record_query(ts, src_ip, dst_ip, "mysql", username, database, "", "login")
|
||||||
|
|
||||||
|
self.bus.emit("CREDENTIAL_FOUND", {
|
||||||
|
"source": "mysql_login",
|
||||||
|
"username": username,
|
||||||
|
"database": database,
|
||||||
|
"source_ip": src_ip,
|
||||||
|
"dest_ip": dst_ip,
|
||||||
|
}, source_module=self.name)
|
||||||
|
|
||||||
|
except Exception:
|
||||||
|
logger.debug("Failed to parse MySQL login", exc_info=True)
|
||||||
|
|
||||||
|
# ------------------------------------------------------------------
|
||||||
|
# PostgreSQL parser
|
||||||
|
# ------------------------------------------------------------------
|
||||||
|
|
||||||
|
def _parse_postgres(self, payload: bytes, src_ip: str, dst_ip: str,
|
||||||
|
is_to_server: bool, ts: float) -> None:
|
||||||
|
"""Parse PostgreSQL protocol messages."""
|
||||||
|
if not is_to_server or len(payload) < 8:
|
||||||
|
return
|
||||||
|
|
||||||
|
# Check for Startup Message: length(4) + protocol_version(4)
|
||||||
|
# Protocol version 3.0 = 0x00030000
|
||||||
|
if len(payload) >= 8:
|
||||||
|
msg_len = struct.unpack("!I", payload[0:4])[0]
|
||||||
|
proto_ver = struct.unpack("!I", payload[4:8])[0]
|
||||||
|
|
||||||
|
if proto_ver == 0x00030000 and msg_len > 8:
|
||||||
|
# Startup message: parse key=value pairs
|
||||||
|
self._parse_pg_startup(payload[8:msg_len], src_ip, dst_ip, ts)
|
||||||
|
return
|
||||||
|
|
||||||
|
# Simple Query: type='Q' + length(4) + query_string
|
||||||
|
if payload[0] == PG_SIMPLE_QUERY and len(payload) >= 6:
|
||||||
|
query_len = struct.unpack("!I", payload[1:5])[0]
|
||||||
|
query = payload[5:min(5 + query_len - 1, 5 + MAX_QUERY_LEN)].decode(
|
||||||
|
"utf-8", errors="replace"
|
||||||
|
).rstrip('\x00')
|
||||||
|
if query:
|
||||||
|
self._stats["postgres_events"] += 1
|
||||||
|
self._stats["queries_captured"] += 1
|
||||||
|
self._record_query(ts, src_ip, dst_ip, "postgres", "", "", query, "query")
|
||||||
|
|
||||||
|
def _parse_pg_startup(self, data: bytes, src_ip: str, dst_ip: str,
|
||||||
|
ts: float) -> None:
|
||||||
|
"""Parse PostgreSQL Startup Message key-value pairs."""
|
||||||
|
username = ""
|
||||||
|
database = ""
|
||||||
|
|
||||||
|
pos = 0
|
||||||
|
while pos < len(data) - 1:
|
||||||
|
null_idx = data.find(b'\x00', pos)
|
||||||
|
if null_idx < 0:
|
||||||
|
break
|
||||||
|
key = data[pos:null_idx].decode("utf-8", errors="replace")
|
||||||
|
pos = null_idx + 1
|
||||||
|
|
||||||
|
val_null = data.find(b'\x00', pos)
|
||||||
|
if val_null < 0:
|
||||||
|
break
|
||||||
|
value = data[pos:val_null].decode("utf-8", errors="replace")
|
||||||
|
pos = val_null + 1
|
||||||
|
|
||||||
|
if key == "user":
|
||||||
|
username = value
|
||||||
|
elif key == "database":
|
||||||
|
database = value
|
||||||
|
|
||||||
|
if not key: # Terminal null
|
||||||
|
break
|
||||||
|
|
||||||
|
if username:
|
||||||
|
self._stats["postgres_events"] += 1
|
||||||
|
self._stats["logins_captured"] += 1
|
||||||
|
self._record_query(ts, src_ip, dst_ip, "postgres", username, database, "", "login")
|
||||||
|
|
||||||
|
self.bus.emit("CREDENTIAL_FOUND", {
|
||||||
|
"source": "postgres_startup",
|
||||||
|
"username": username,
|
||||||
|
"database": database,
|
||||||
|
"source_ip": src_ip,
|
||||||
|
"dest_ip": dst_ip,
|
||||||
|
}, source_module=self.name)
|
||||||
|
|
||||||
|
# ------------------------------------------------------------------
|
||||||
|
# Redis parser
|
||||||
|
# ------------------------------------------------------------------
|
||||||
|
|
||||||
|
def _parse_redis(self, payload: bytes, src_ip: str, dst_ip: str,
|
||||||
|
is_to_server: bool, ts: float) -> None:
|
||||||
|
"""Parse Redis RESP protocol for AUTH commands and key operations."""
|
||||||
|
if not is_to_server or len(payload) < 3:
|
||||||
|
return
|
||||||
|
|
||||||
|
# RESP: commands start with * (array) followed by count
|
||||||
|
if payload[0:1] == b'*':
|
||||||
|
args = self._parse_resp_array(payload)
|
||||||
|
if not args:
|
||||||
|
return
|
||||||
|
|
||||||
|
cmd = args[0].upper() if args else ""
|
||||||
|
|
||||||
|
if cmd == "AUTH":
|
||||||
|
# AUTH [username] password
|
||||||
|
password = args[-1] if len(args) >= 2 else ""
|
||||||
|
username = args[1] if len(args) >= 3 else ""
|
||||||
|
self._stats["redis_events"] += 1
|
||||||
|
self._stats["logins_captured"] += 1
|
||||||
|
self._record_query(ts, src_ip, dst_ip, "redis", username, "", "AUTH ***", "login")
|
||||||
|
|
||||||
|
self.bus.emit("CREDENTIAL_FOUND", {
|
||||||
|
"source": "redis_auth",
|
||||||
|
"username": username,
|
||||||
|
"source_ip": src_ip,
|
||||||
|
"dest_ip": dst_ip,
|
||||||
|
}, source_module=self.name)
|
||||||
|
|
||||||
|
elif cmd in ("GET", "SET", "HGET", "HSET", "DEL", "KEYS",
|
||||||
|
"MGET", "MSET", "LPUSH", "RPUSH", "SADD", "ZADD"):
|
||||||
|
query = " ".join(args[:3]) # Command + first 2 args max
|
||||||
|
self._stats["redis_events"] += 1
|
||||||
|
self._stats["queries_captured"] += 1
|
||||||
|
self._record_query(
|
||||||
|
ts, src_ip, dst_ip, "redis", "", "",
|
||||||
|
query[:MAX_QUERY_LEN], "query"
|
||||||
|
)
|
||||||
|
|
||||||
|
def _parse_resp_array(self, data: bytes) -> list:
|
||||||
|
"""Parse a RESP array into a list of string arguments."""
|
||||||
|
args = []
|
||||||
|
lines = data.split(REDIS_CRLF)
|
||||||
|
if not lines or not lines[0].startswith(b'*'):
|
||||||
|
return args
|
||||||
|
|
||||||
|
try:
|
||||||
|
count = int(lines[0][1:])
|
||||||
|
except ValueError:
|
||||||
|
return args
|
||||||
|
|
||||||
|
idx = 1
|
||||||
|
while len(args) < count and idx < len(lines) - 1:
|
||||||
|
if lines[idx].startswith(b'$'):
|
||||||
|
try:
|
||||||
|
str_len = int(lines[idx][1:])
|
||||||
|
except ValueError:
|
||||||
|
break
|
||||||
|
idx += 1
|
||||||
|
if idx < len(lines):
|
||||||
|
args.append(lines[idx].decode("utf-8", errors="replace")[:MAX_QUERY_LEN])
|
||||||
|
idx += 1
|
||||||
|
else:
|
||||||
|
# Inline format
|
||||||
|
args.append(lines[idx].decode("utf-8", errors="replace"))
|
||||||
|
idx += 1
|
||||||
|
|
||||||
|
return args
|
||||||
|
|
||||||
|
# ------------------------------------------------------------------
|
||||||
|
# MongoDB parser
|
||||||
|
# ------------------------------------------------------------------
|
||||||
|
|
||||||
|
def _parse_mongodb(self, payload: bytes, src_ip: str, dst_ip: str,
|
||||||
|
is_to_server: bool, ts: float) -> None:
|
||||||
|
"""Parse MongoDB wire protocol for auth and find operations."""
|
||||||
|
if len(payload) < 16:
|
||||||
|
return
|
||||||
|
|
||||||
|
# MongoDB wire protocol header: length(4 LE) + requestID(4) + responseTo(4) + opCode(4)
|
||||||
|
msg_len = struct.unpack("<I", payload[0:4])[0]
|
||||||
|
opcode = struct.unpack("<I", payload[12:16])[0]
|
||||||
|
|
||||||
|
if opcode != MONGO_OP_MSG or len(payload) < 21:
|
||||||
|
return
|
||||||
|
|
||||||
|
if not is_to_server:
|
||||||
|
return
|
||||||
|
|
||||||
|
# OP_MSG: flagBits(4) + sections
|
||||||
|
# Section kind 0: body (BSON document)
|
||||||
|
sections_start = 20
|
||||||
|
if sections_start >= len(payload):
|
||||||
|
return
|
||||||
|
|
||||||
|
section_kind = payload[sections_start]
|
||||||
|
if section_kind != 0:
|
||||||
|
return
|
||||||
|
|
||||||
|
bson_data = payload[sections_start + 1:]
|
||||||
|
if len(bson_data) < 5:
|
||||||
|
return
|
||||||
|
|
||||||
|
# Minimal BSON parsing: look for command keys
|
||||||
|
bson_str = bson_data.decode("utf-8", errors="replace")
|
||||||
|
|
||||||
|
if "saslStart" in bson_str or "authenticate" in bson_str:
|
||||||
|
self._stats["mongodb_events"] += 1
|
||||||
|
self._stats["logins_captured"] += 1
|
||||||
|
# Extract mechanism if visible
|
||||||
|
mechanism = ""
|
||||||
|
if "SCRAM-SHA" in bson_str:
|
||||||
|
mechanism = "SCRAM-SHA"
|
||||||
|
elif "PLAIN" in bson_str:
|
||||||
|
mechanism = "PLAIN"
|
||||||
|
self._record_query(
|
||||||
|
ts, src_ip, dst_ip, "mongodb", "", "",
|
||||||
|
f"AUTH mechanism={mechanism}", "login"
|
||||||
|
)
|
||||||
|
|
||||||
|
elif "find" in bson_str or "aggregate" in bson_str or "insert" in bson_str:
|
||||||
|
# Extract collection name heuristically
|
||||||
|
operation = "find" if "find" in bson_str else ("aggregate" if "aggregate" in bson_str else "insert")
|
||||||
|
self._stats["mongodb_events"] += 1
|
||||||
|
self._stats["queries_captured"] += 1
|
||||||
|
self._record_query(
|
||||||
|
ts, src_ip, dst_ip, "mongodb", "", "",
|
||||||
|
f"{operation} (BSON)", "query"
|
||||||
|
)
|
||||||
|
|
||||||
|
# ------------------------------------------------------------------
|
||||||
|
# Database
|
||||||
|
# ------------------------------------------------------------------
|
||||||
|
|
||||||
|
def _resolve_db_path(self) -> str:
|
||||||
|
base = self.config.get("db_dir", os.path.join(
|
||||||
|
os.path.expanduser("~"), ".bigbrother"
|
||||||
|
))
|
||||||
|
return os.path.join(base, "db_interceptor.db")
|
||||||
|
|
||||||
|
def _init_db(self) -> None:
|
||||||
|
Path(self._db_path).parent.mkdir(parents=True, exist_ok=True)
|
||||||
|
self._db_conn = sqlite3.connect(self._db_path, check_same_thread=False)
|
||||||
|
self._db_conn.execute("PRAGMA journal_mode=WAL")
|
||||||
|
self._db_conn.execute("PRAGMA synchronous=NORMAL")
|
||||||
|
self._db_conn.executescript(_DB_INIT)
|
||||||
|
self._db_conn.commit()
|
||||||
|
|
||||||
|
def _record_query(self, ts: float, src_ip: str, dst_ip: str,
|
||||||
|
db_type: str, username: str, database: str,
|
||||||
|
query_text: str, operation: str) -> None:
|
||||||
|
with self._buffer_lock:
|
||||||
|
self._write_buffer.append((
|
||||||
|
ts, src_ip, dst_ip, db_type, username, database,
|
||||||
|
query_text[:MAX_QUERY_LEN], operation
|
||||||
|
))
|
||||||
|
|
||||||
|
def _flush_buffer(self) -> None:
|
||||||
|
with self._buffer_lock:
|
||||||
|
batch = list(self._write_buffer)
|
||||||
|
self._write_buffer.clear()
|
||||||
|
|
||||||
|
if not batch or not self._db_conn:
|
||||||
|
return
|
||||||
|
|
||||||
|
try:
|
||||||
|
with self._db_conn:
|
||||||
|
self._db_conn.executemany(
|
||||||
|
"""INSERT INTO db_queries
|
||||||
|
(timestamp, source_ip, dest_ip, db_type, username,
|
||||||
|
database, query_text, operation)
|
||||||
|
VALUES (?, ?, ?, ?, ?, ?, ?, ?)""",
|
||||||
|
batch,
|
||||||
|
)
|
||||||
|
except Exception:
|
||||||
|
logger.exception("Failed to flush DB queries")
|
||||||
|
|
||||||
|
def _flush_loop(self) -> None:
|
||||||
|
while self._running:
|
||||||
|
time.sleep(FLUSH_INTERVAL)
|
||||||
|
try:
|
||||||
|
self._flush_buffer()
|
||||||
|
except Exception:
|
||||||
|
logger.exception("Flush loop error")
|
||||||
@@ -0,0 +1,495 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
"""LDAP query harvester — passive Active Directory object inventory.
|
||||||
|
|
||||||
|
Parses LDAP SearchRequest and SearchResultEntry messages on port 389/3268
|
||||||
|
using BER/ASN.1 decoding. Extracts user objects (sAMAccountName, mail,
|
||||||
|
memberOf), group objects, computer objects, GPOs, SPNs, and detects
|
||||||
|
LAPS password reads (ms-Mcs-AdmPwd attribute access).
|
||||||
|
"""
|
||||||
|
|
||||||
|
import json
|
||||||
|
import logging
|
||||||
|
import os
|
||||||
|
import socket
|
||||||
|
import sqlite3
|
||||||
|
import struct
|
||||||
|
import threading
|
||||||
|
import time
|
||||||
|
from pathlib import Path
|
||||||
|
from typing import Optional
|
||||||
|
|
||||||
|
from modules.base import BaseModule
|
||||||
|
|
||||||
|
logger = logging.getLogger("bb.passive.ldap_harvester")
|
||||||
|
|
||||||
|
# LDAP protocol tags
|
||||||
|
TAG_SEQUENCE = 0x30
|
||||||
|
TAG_INTEGER = 0x02
|
||||||
|
TAG_OCTET_STRING = 0x04
|
||||||
|
TAG_ENUMERATED = 0x0A
|
||||||
|
TAG_SET = 0x31
|
||||||
|
TAG_BOOLEAN = 0x01
|
||||||
|
|
||||||
|
# LDAP application tags (context-specific constructed)
|
||||||
|
LDAP_SEARCH_REQUEST = 0x63 # APPLICATION[3] CONSTRUCTED
|
||||||
|
LDAP_SEARCH_RESULT_ENTRY = 0x64 # APPLICATION[4] CONSTRUCTED
|
||||||
|
LDAP_SEARCH_RESULT_DONE = 0x65 # APPLICATION[5] CONSTRUCTED
|
||||||
|
|
||||||
|
# Interesting attributes (case-insensitive)
|
||||||
|
INTERESTING_ATTRS = {
|
||||||
|
"samaccountname", "userprincipalname", "mail", "memberof",
|
||||||
|
"distinguishedname", "objectclass", "cn", "name",
|
||||||
|
"serviceprincipalname", "description", "operatingsystem",
|
||||||
|
"dnshostname", "managedby", "gplink", "gpcfilesyspath",
|
||||||
|
"ms-mcs-admpwd", # LAPS password
|
||||||
|
"ms-mcs-admpwdexpirationtime",
|
||||||
|
"unicodepwd", "userpassword",
|
||||||
|
}
|
||||||
|
|
||||||
|
_DB_INIT = """
|
||||||
|
CREATE TABLE IF NOT EXISTS ldap_objects (
|
||||||
|
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||||
|
timestamp REAL NOT NULL,
|
||||||
|
dn TEXT DEFAULT '',
|
||||||
|
object_class TEXT DEFAULT '',
|
||||||
|
sam_account_name TEXT DEFAULT '',
|
||||||
|
attributes_json TEXT DEFAULT '{}',
|
||||||
|
source_ip TEXT NOT NULL
|
||||||
|
);
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_ldap_sam ON ldap_objects(sam_account_name);
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_ldap_class ON ldap_objects(object_class);
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_ldap_ts ON ldap_objects(timestamp);
|
||||||
|
"""
|
||||||
|
|
||||||
|
FLUSH_INTERVAL = 45
|
||||||
|
|
||||||
|
|
||||||
|
class LDAPHarvester(BaseModule):
|
||||||
|
"""Passively harvest AD objects from LDAP traffic."""
|
||||||
|
|
||||||
|
name = "ldap_harvester"
|
||||||
|
module_type = "passive"
|
||||||
|
priority = 100
|
||||||
|
requires_root = True
|
||||||
|
|
||||||
|
def __init__(self, bus, state, config, engine=None):
|
||||||
|
super().__init__(bus, state, config, engine)
|
||||||
|
self._capture_bus = None
|
||||||
|
self._sub_queue = None
|
||||||
|
self._read_thread: Optional[threading.Thread] = None
|
||||||
|
self._flush_thread: Optional[threading.Thread] = None
|
||||||
|
self._db_path = self._resolve_db_path()
|
||||||
|
self._db_conn: Optional[sqlite3.Connection] = None
|
||||||
|
self._write_buffer: list[tuple] = []
|
||||||
|
self._buffer_lock = threading.Lock()
|
||||||
|
self._seen_dns: set = set() # Dedup by DN
|
||||||
|
self._stats = {
|
||||||
|
"packets_processed": 0,
|
||||||
|
"search_requests": 0,
|
||||||
|
"result_entries": 0,
|
||||||
|
"users_found": 0,
|
||||||
|
"groups_found": 0,
|
||||||
|
"computers_found": 0,
|
||||||
|
"spns_found": 0,
|
||||||
|
"laps_reads_detected": 0,
|
||||||
|
}
|
||||||
|
|
||||||
|
# ------------------------------------------------------------------
|
||||||
|
# BaseModule interface
|
||||||
|
# ------------------------------------------------------------------
|
||||||
|
|
||||||
|
def start(self) -> None:
|
||||||
|
if self._running:
|
||||||
|
return
|
||||||
|
self._init_db()
|
||||||
|
self._capture_bus = self.config.get("_capture_bus")
|
||||||
|
if self._capture_bus is None:
|
||||||
|
logger.error("LDAPHarvester requires _capture_bus in config")
|
||||||
|
return
|
||||||
|
|
||||||
|
self._sub_queue = self._capture_bus.subscribe(
|
||||||
|
name=self.name, bpf_filter="port 389 or port 3268", queue_depth=5000
|
||||||
|
)
|
||||||
|
|
||||||
|
self._running = True
|
||||||
|
self._start_time = time.time()
|
||||||
|
self._pid = os.getpid()
|
||||||
|
|
||||||
|
self._read_thread = threading.Thread(
|
||||||
|
target=self._read_loop, daemon=True, name="bb-ldap-read"
|
||||||
|
)
|
||||||
|
self._read_thread.start()
|
||||||
|
|
||||||
|
self._flush_thread = threading.Thread(
|
||||||
|
target=self._flush_loop, daemon=True, name="bb-ldap-flush"
|
||||||
|
)
|
||||||
|
self._flush_thread.start()
|
||||||
|
|
||||||
|
self.state.set_module_status(self.name, "running", pid=self._pid)
|
||||||
|
self.bus.emit("MODULE_STARTED", {"module": self.name}, source_module=self.name)
|
||||||
|
logger.info("LDAPHarvester started")
|
||||||
|
|
||||||
|
def stop(self) -> None:
|
||||||
|
if not self._running:
|
||||||
|
return
|
||||||
|
self._running = False
|
||||||
|
if self._capture_bus:
|
||||||
|
self._capture_bus.unsubscribe(self.name)
|
||||||
|
for t in (self._read_thread, self._flush_thread):
|
||||||
|
if t and t.is_alive():
|
||||||
|
t.join(timeout=5.0)
|
||||||
|
self._flush_buffer()
|
||||||
|
if self._db_conn:
|
||||||
|
self._db_conn.close()
|
||||||
|
self._db_conn = None
|
||||||
|
self.state.set_module_status(self.name, "stopped")
|
||||||
|
self.bus.emit("MODULE_STOPPED", {"module": self.name}, source_module=self.name)
|
||||||
|
logger.info("LDAPHarvester stopped — stats: %s", self._stats)
|
||||||
|
|
||||||
|
def status(self) -> dict:
|
||||||
|
return {
|
||||||
|
"running": self._running,
|
||||||
|
"pid": self._pid,
|
||||||
|
"uptime": time.time() - self._start_time if self._start_time else 0,
|
||||||
|
**self._stats,
|
||||||
|
}
|
||||||
|
|
||||||
|
def configure(self, config: dict) -> None:
|
||||||
|
self.config.update(config)
|
||||||
|
|
||||||
|
# ------------------------------------------------------------------
|
||||||
|
# Packet processing
|
||||||
|
# ------------------------------------------------------------------
|
||||||
|
|
||||||
|
def _read_loop(self) -> None:
|
||||||
|
while self._running:
|
||||||
|
result = self._sub_queue.get(timeout=1.0)
|
||||||
|
if result is None:
|
||||||
|
continue
|
||||||
|
ts, pkt = result
|
||||||
|
self._stats["packets_processed"] += 1
|
||||||
|
try:
|
||||||
|
self._process_packet(pkt, ts)
|
||||||
|
except Exception:
|
||||||
|
logger.debug("Error processing LDAP packet", exc_info=True)
|
||||||
|
|
||||||
|
def _process_packet(self, pkt: bytes, ts: float) -> None:
|
||||||
|
"""Extract TCP payload and parse LDAP messages."""
|
||||||
|
if len(pkt) < 54:
|
||||||
|
return
|
||||||
|
|
||||||
|
ethertype = struct.unpack("!H", pkt[12:14])[0]
|
||||||
|
eth_offset = 14
|
||||||
|
if ethertype == 0x8100:
|
||||||
|
if len(pkt) < 58:
|
||||||
|
return
|
||||||
|
ethertype = struct.unpack("!H", pkt[16:18])[0]
|
||||||
|
eth_offset = 18
|
||||||
|
|
||||||
|
if ethertype != 0x0800:
|
||||||
|
return
|
||||||
|
|
||||||
|
ip_header = pkt[eth_offset:]
|
||||||
|
if len(ip_header) < 20:
|
||||||
|
return
|
||||||
|
ihl = (ip_header[0] & 0x0F) * 4
|
||||||
|
ip_proto = ip_header[9]
|
||||||
|
if ip_proto != 6:
|
||||||
|
return
|
||||||
|
|
||||||
|
src_ip = socket.inet_ntoa(ip_header[12:16])
|
||||||
|
|
||||||
|
if len(ip_header) < ihl + 20:
|
||||||
|
return
|
||||||
|
tcp_data_off = ((ip_header[ihl + 12] >> 4) & 0xF) * 4
|
||||||
|
payload = ip_header[ihl + tcp_data_off:]
|
||||||
|
|
||||||
|
if len(payload) < 5:
|
||||||
|
return
|
||||||
|
|
||||||
|
self._parse_ldap_messages(payload, src_ip, ts)
|
||||||
|
|
||||||
|
def _parse_ldap_messages(self, data: bytes, src_ip: str, ts: float) -> None:
|
||||||
|
"""Parse one or more LDAP messages from a TCP payload."""
|
||||||
|
pos = 0
|
||||||
|
while pos < len(data) - 2:
|
||||||
|
if data[pos] != TAG_SEQUENCE:
|
||||||
|
break
|
||||||
|
|
||||||
|
msg_len, len_size = self._read_ber_length(data[pos + 1:])
|
||||||
|
if msg_len <= 0:
|
||||||
|
break
|
||||||
|
msg_start = pos + 1 + len_size
|
||||||
|
msg_end = msg_start + msg_len
|
||||||
|
if msg_end > len(data):
|
||||||
|
break
|
||||||
|
|
||||||
|
msg_body = data[msg_start:msg_end]
|
||||||
|
self._parse_ldap_message(msg_body, src_ip, ts)
|
||||||
|
pos = msg_end
|
||||||
|
|
||||||
|
def _parse_ldap_message(self, msg: bytes, src_ip: str, ts: float) -> None:
|
||||||
|
"""Parse a single LDAP message (after outer SEQUENCE)."""
|
||||||
|
if len(msg) < 5:
|
||||||
|
return
|
||||||
|
|
||||||
|
# Message ID (INTEGER)
|
||||||
|
if msg[0] != TAG_INTEGER:
|
||||||
|
return
|
||||||
|
id_len = msg[1]
|
||||||
|
if id_len < 1 or 2 + id_len >= len(msg):
|
||||||
|
return
|
||||||
|
|
||||||
|
op_start = 2 + id_len
|
||||||
|
if op_start >= len(msg):
|
||||||
|
return
|
||||||
|
|
||||||
|
op_tag = msg[op_start]
|
||||||
|
op_len, op_len_size = self._read_ber_length(msg[op_start + 1:])
|
||||||
|
if op_len <= 0:
|
||||||
|
return
|
||||||
|
op_body = msg[op_start + 1 + op_len_size:op_start + 1 + op_len_size + op_len]
|
||||||
|
|
||||||
|
if op_tag == LDAP_SEARCH_REQUEST:
|
||||||
|
self._handle_search_request(op_body, src_ip, ts)
|
||||||
|
elif op_tag == LDAP_SEARCH_RESULT_ENTRY:
|
||||||
|
self._handle_search_result_entry(op_body, src_ip, ts)
|
||||||
|
|
||||||
|
def _handle_search_request(self, body: bytes, src_ip: str, ts: float) -> None:
|
||||||
|
"""Parse SearchRequest for base DN and filter."""
|
||||||
|
self._stats["search_requests"] += 1
|
||||||
|
|
||||||
|
# BaseObject (OCTET STRING)
|
||||||
|
base_dn = self._read_octet_string(body, 0)
|
||||||
|
if base_dn is None:
|
||||||
|
return
|
||||||
|
|
||||||
|
# Check if requesting LAPS password
|
||||||
|
body_str = body.decode("ascii", errors="replace").lower()
|
||||||
|
if "ms-mcs-admpwd" in body_str:
|
||||||
|
self._stats["laps_reads_detected"] += 1
|
||||||
|
logger.warning("LAPS password read detected from %s (base DN: %s)", src_ip, base_dn)
|
||||||
|
self.bus.emit("CREDENTIAL_FOUND", {
|
||||||
|
"source": "laps_read",
|
||||||
|
"source_ip": src_ip,
|
||||||
|
"base_dn": base_dn,
|
||||||
|
"detail": "LAPS password attribute requested in LDAP search",
|
||||||
|
}, source_module=self.name)
|
||||||
|
|
||||||
|
def _handle_search_result_entry(self, body: bytes, src_ip: str,
|
||||||
|
ts: float) -> None:
|
||||||
|
"""Parse SearchResultEntry to extract DN and attributes."""
|
||||||
|
self._stats["result_entries"] += 1
|
||||||
|
|
||||||
|
# ObjectName (OCTET STRING) = DN
|
||||||
|
dn = self._read_octet_string(body, 0)
|
||||||
|
if dn is None:
|
||||||
|
return
|
||||||
|
|
||||||
|
# Skip past the DN OCTET STRING
|
||||||
|
dn_tag_len, _ = self._skip_tlv(body, 0)
|
||||||
|
if dn_tag_len < 0:
|
||||||
|
return
|
||||||
|
|
||||||
|
# Attributes (SEQUENCE OF PartialAttribute)
|
||||||
|
attrs = {}
|
||||||
|
attr_data = body[dn_tag_len:]
|
||||||
|
if len(attr_data) < 2 or attr_data[0] != TAG_SEQUENCE:
|
||||||
|
# May be in a different format
|
||||||
|
pass
|
||||||
|
else:
|
||||||
|
attr_seq_len, attr_len_size = self._read_ber_length(attr_data[1:])
|
||||||
|
attr_body = attr_data[1 + attr_len_size:]
|
||||||
|
self._parse_partial_attributes(attr_body, attr_seq_len, attrs)
|
||||||
|
|
||||||
|
# Classify object
|
||||||
|
object_class = ""
|
||||||
|
sam = attrs.get("samaccountname", "")
|
||||||
|
classes = attrs.get("objectclass", "")
|
||||||
|
classes_lower = classes.lower() if isinstance(classes, str) else ""
|
||||||
|
|
||||||
|
if "user" in classes_lower or "person" in classes_lower:
|
||||||
|
object_class = "user"
|
||||||
|
self._stats["users_found"] += 1
|
||||||
|
elif "group" in classes_lower:
|
||||||
|
object_class = "group"
|
||||||
|
self._stats["groups_found"] += 1
|
||||||
|
elif "computer" in classes_lower:
|
||||||
|
object_class = "computer"
|
||||||
|
self._stats["computers_found"] += 1
|
||||||
|
elif "grouppolicycontainer" in classes_lower:
|
||||||
|
object_class = "gpo"
|
||||||
|
|
||||||
|
# Track SPNs
|
||||||
|
if "serviceprincipalname" in attrs:
|
||||||
|
self._stats["spns_found"] += 1
|
||||||
|
|
||||||
|
# Dedup
|
||||||
|
if dn in self._seen_dns:
|
||||||
|
return
|
||||||
|
self._seen_dns.add(dn)
|
||||||
|
if len(self._seen_dns) > 50000:
|
||||||
|
self._seen_dns.clear()
|
||||||
|
|
||||||
|
# Filter to interesting attributes only
|
||||||
|
filtered_attrs = {
|
||||||
|
k: v for k, v in attrs.items()
|
||||||
|
if k.lower() in INTERESTING_ATTRS
|
||||||
|
}
|
||||||
|
|
||||||
|
self._record_object(ts, dn, object_class, sam, filtered_attrs, src_ip)
|
||||||
|
|
||||||
|
def _parse_partial_attributes(self, data: bytes, max_len: int,
|
||||||
|
attrs: dict) -> None:
|
||||||
|
"""Parse SEQUENCE OF PartialAttribute."""
|
||||||
|
pos = 0
|
||||||
|
while pos < min(len(data), max_len) - 2:
|
||||||
|
if data[pos] != TAG_SEQUENCE:
|
||||||
|
break
|
||||||
|
|
||||||
|
seq_len, seq_len_size = self._read_ber_length(data[pos + 1:])
|
||||||
|
if seq_len <= 0:
|
||||||
|
break
|
||||||
|
|
||||||
|
attr_body = data[pos + 1 + seq_len_size:pos + 1 + seq_len_size + seq_len]
|
||||||
|
self._parse_single_attribute(attr_body, attrs)
|
||||||
|
pos += 1 + seq_len_size + seq_len
|
||||||
|
|
||||||
|
def _parse_single_attribute(self, data: bytes, attrs: dict) -> None:
|
||||||
|
"""Parse a single PartialAttribute (type + SET OF values)."""
|
||||||
|
# AttributeDescription (OCTET STRING)
|
||||||
|
attr_name = self._read_octet_string(data, 0)
|
||||||
|
if attr_name is None:
|
||||||
|
return
|
||||||
|
|
||||||
|
name_total, _ = self._skip_tlv(data, 0)
|
||||||
|
if name_total < 0:
|
||||||
|
return
|
||||||
|
|
||||||
|
# Values (SET OF AttributeValue)
|
||||||
|
val_data = data[name_total:]
|
||||||
|
if len(val_data) < 2 or val_data[0] != TAG_SET:
|
||||||
|
return
|
||||||
|
|
||||||
|
set_len, set_len_size = self._read_ber_length(val_data[1:])
|
||||||
|
if set_len <= 0:
|
||||||
|
return
|
||||||
|
|
||||||
|
set_body = val_data[1 + set_len_size:]
|
||||||
|
values = []
|
||||||
|
vpos = 0
|
||||||
|
while vpos < min(len(set_body), set_len) - 2:
|
||||||
|
val = self._read_octet_string(set_body, vpos)
|
||||||
|
if val is not None:
|
||||||
|
values.append(val)
|
||||||
|
total, _ = self._skip_tlv(set_body, vpos)
|
||||||
|
if total <= 0:
|
||||||
|
break
|
||||||
|
vpos += total
|
||||||
|
|
||||||
|
if len(values) == 1:
|
||||||
|
attrs[attr_name.lower()] = values[0]
|
||||||
|
elif values:
|
||||||
|
attrs[attr_name.lower()] = "; ".join(values)
|
||||||
|
|
||||||
|
# ------------------------------------------------------------------
|
||||||
|
# BER helpers
|
||||||
|
# ------------------------------------------------------------------
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _read_ber_length(data: bytes) -> tuple:
|
||||||
|
"""Read BER length. Returns (length, bytes_consumed)."""
|
||||||
|
if not data:
|
||||||
|
return (0, 0)
|
||||||
|
first = data[0]
|
||||||
|
if first < 0x80:
|
||||||
|
return (first, 1)
|
||||||
|
num_bytes = first & 0x7F
|
||||||
|
if num_bytes == 0 or len(data) < 1 + num_bytes:
|
||||||
|
return (0, 0)
|
||||||
|
length = 0
|
||||||
|
for i in range(num_bytes):
|
||||||
|
length = (length << 8) | data[1 + i]
|
||||||
|
return (length, 1 + num_bytes)
|
||||||
|
|
||||||
|
def _read_octet_string(self, data: bytes, offset: int) -> Optional[str]:
|
||||||
|
"""Read an OCTET STRING at offset. Returns decoded string or None."""
|
||||||
|
if offset >= len(data) - 1:
|
||||||
|
return None
|
||||||
|
tag = data[offset]
|
||||||
|
if tag != TAG_OCTET_STRING:
|
||||||
|
return None
|
||||||
|
length, len_size = self._read_ber_length(data[offset + 1:])
|
||||||
|
if length <= 0:
|
||||||
|
return None
|
||||||
|
start = offset + 1 + len_size
|
||||||
|
if start + length > len(data):
|
||||||
|
return None
|
||||||
|
try:
|
||||||
|
return data[start:start + length].decode("utf-8", errors="replace")
|
||||||
|
except Exception:
|
||||||
|
return None
|
||||||
|
|
||||||
|
def _skip_tlv(self, data: bytes, offset: int) -> tuple:
|
||||||
|
"""Skip past a TLV element. Returns (total_bytes, tag)."""
|
||||||
|
if offset >= len(data):
|
||||||
|
return (-1, 0)
|
||||||
|
tag = data[offset]
|
||||||
|
length, len_size = self._read_ber_length(data[offset + 1:])
|
||||||
|
if length < 0:
|
||||||
|
return (-1, 0)
|
||||||
|
total = 1 + len_size + length
|
||||||
|
return (total, tag)
|
||||||
|
|
||||||
|
# ------------------------------------------------------------------
|
||||||
|
# Database
|
||||||
|
# ------------------------------------------------------------------
|
||||||
|
|
||||||
|
def _resolve_db_path(self) -> str:
|
||||||
|
base = self.config.get("db_dir", os.path.join(
|
||||||
|
os.path.expanduser("~"), ".bigbrother"
|
||||||
|
))
|
||||||
|
return os.path.join(base, "ldap_harvester.db")
|
||||||
|
|
||||||
|
def _init_db(self) -> None:
|
||||||
|
Path(self._db_path).parent.mkdir(parents=True, exist_ok=True)
|
||||||
|
self._db_conn = sqlite3.connect(self._db_path, check_same_thread=False)
|
||||||
|
self._db_conn.execute("PRAGMA journal_mode=WAL")
|
||||||
|
self._db_conn.execute("PRAGMA synchronous=NORMAL")
|
||||||
|
self._db_conn.executescript(_DB_INIT)
|
||||||
|
self._db_conn.commit()
|
||||||
|
|
||||||
|
def _record_object(self, ts: float, dn: str, object_class: str,
|
||||||
|
sam: str, attrs: dict, src_ip: str) -> None:
|
||||||
|
with self._buffer_lock:
|
||||||
|
self._write_buffer.append((
|
||||||
|
ts, dn, object_class, sam, json.dumps(attrs), src_ip
|
||||||
|
))
|
||||||
|
|
||||||
|
def _flush_buffer(self) -> None:
|
||||||
|
with self._buffer_lock:
|
||||||
|
batch = list(self._write_buffer)
|
||||||
|
self._write_buffer.clear()
|
||||||
|
|
||||||
|
if not batch or not self._db_conn:
|
||||||
|
return
|
||||||
|
|
||||||
|
try:
|
||||||
|
with self._db_conn:
|
||||||
|
self._db_conn.executemany(
|
||||||
|
"""INSERT INTO ldap_objects
|
||||||
|
(timestamp, dn, object_class, sam_account_name,
|
||||||
|
attributes_json, source_ip)
|
||||||
|
VALUES (?, ?, ?, ?, ?, ?)""",
|
||||||
|
batch,
|
||||||
|
)
|
||||||
|
except Exception:
|
||||||
|
logger.exception("Failed to flush LDAP objects")
|
||||||
|
|
||||||
|
def _flush_loop(self) -> None:
|
||||||
|
while self._running:
|
||||||
|
time.sleep(FLUSH_INTERVAL)
|
||||||
|
try:
|
||||||
|
self._flush_buffer()
|
||||||
|
except Exception:
|
||||||
|
logger.exception("Flush loop error")
|
||||||
@@ -0,0 +1,380 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
"""Network relationship mapper — passive communication graph builder.
|
||||||
|
|
||||||
|
Builds a graph of all observed host-to-host communication with protocol,
|
||||||
|
byte count, and packet count metadata. Identifies roles (servers, clients,
|
||||||
|
admin workstations, printers) and generates Graphviz DOT output. Periodic
|
||||||
|
snapshots for change_detector consumption.
|
||||||
|
"""
|
||||||
|
|
||||||
|
import logging
|
||||||
|
import os
|
||||||
|
import socket
|
||||||
|
import sqlite3
|
||||||
|
import struct
|
||||||
|
import threading
|
||||||
|
import time
|
||||||
|
from collections import defaultdict
|
||||||
|
from pathlib import Path
|
||||||
|
from typing import Optional
|
||||||
|
|
||||||
|
from modules.base import BaseModule
|
||||||
|
|
||||||
|
logger = logging.getLogger("bb.passive.network_mapper")
|
||||||
|
|
||||||
|
TCP_PROTO = 6
|
||||||
|
UDP_PROTO = 17
|
||||||
|
|
||||||
|
# Role detection thresholds
|
||||||
|
SERVER_INBOUND_THRESHOLD = 5 # Unique source IPs connecting in
|
||||||
|
CLIENT_OUTBOUND_THRESHOLD = 10 # Unique dest IPs connected to
|
||||||
|
ADMIN_SSH_RDP_THRESHOLD = 3 # Unique SSH/RDP destinations
|
||||||
|
PRINTER_PORT = 9100
|
||||||
|
|
||||||
|
_DB_INIT = """
|
||||||
|
CREATE TABLE IF NOT EXISTS connections (
|
||||||
|
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||||
|
src_ip TEXT NOT NULL,
|
||||||
|
dst_ip TEXT NOT NULL,
|
||||||
|
protocol TEXT NOT NULL,
|
||||||
|
port INTEGER NOT NULL,
|
||||||
|
bytes_total INTEGER DEFAULT 0,
|
||||||
|
packets INTEGER DEFAULT 0,
|
||||||
|
first_seen REAL NOT NULL,
|
||||||
|
last_seen REAL NOT NULL,
|
||||||
|
relationship_type TEXT DEFAULT '',
|
||||||
|
UNIQUE(src_ip, dst_ip, protocol, port)
|
||||||
|
);
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_conn_src ON connections(src_ip);
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_conn_dst ON connections(dst_ip);
|
||||||
|
"""
|
||||||
|
|
||||||
|
FLUSH_INTERVAL = 45
|
||||||
|
SNAPSHOT_INTERVAL = 300 # 5 minutes
|
||||||
|
|
||||||
|
|
||||||
|
class NetworkMapper(BaseModule):
|
||||||
|
"""Build communication graph from all observed traffic."""
|
||||||
|
|
||||||
|
name = "network_mapper"
|
||||||
|
module_type = "passive"
|
||||||
|
priority = 150
|
||||||
|
requires_root = True
|
||||||
|
|
||||||
|
def __init__(self, bus, state, config, engine=None):
|
||||||
|
super().__init__(bus, state, config, engine)
|
||||||
|
self._capture_bus = None
|
||||||
|
self._sub_queue = None
|
||||||
|
self._read_thread: Optional[threading.Thread] = None
|
||||||
|
self._flush_thread: Optional[threading.Thread] = None
|
||||||
|
self._snapshot_thread: Optional[threading.Thread] = None
|
||||||
|
self._db_path = self._resolve_db_path()
|
||||||
|
self._db_conn: Optional[sqlite3.Connection] = None
|
||||||
|
self._lock = threading.Lock()
|
||||||
|
# In-memory graph: (src_ip, dst_ip, proto, port) -> {bytes, packets, first, last}
|
||||||
|
self._graph: dict[tuple, dict] = {}
|
||||||
|
# Role tracking: ip -> set of connected IPs per direction
|
||||||
|
self._inbound: dict[str, set] = defaultdict(set) # dst -> set(src)
|
||||||
|
self._outbound: dict[str, set] = defaultdict(set) # src -> set(dst)
|
||||||
|
self._ssh_rdp_dests: dict[str, set] = defaultdict(set) # src -> set(dst) for SSH/RDP
|
||||||
|
self._printer_servers: set = set()
|
||||||
|
self._stats = {
|
||||||
|
"packets_processed": 0,
|
||||||
|
"unique_connections": 0,
|
||||||
|
"hosts_seen": 0,
|
||||||
|
"snapshots_taken": 0,
|
||||||
|
}
|
||||||
|
|
||||||
|
# ------------------------------------------------------------------
|
||||||
|
# BaseModule interface
|
||||||
|
# ------------------------------------------------------------------
|
||||||
|
|
||||||
|
def start(self) -> None:
|
||||||
|
if self._running:
|
||||||
|
return
|
||||||
|
|
||||||
|
self._init_db()
|
||||||
|
self._capture_bus = self.config.get("_capture_bus")
|
||||||
|
if self._capture_bus is None:
|
||||||
|
logger.error("NetworkMapper requires _capture_bus in config")
|
||||||
|
return
|
||||||
|
|
||||||
|
# Subscribe to all traffic
|
||||||
|
self._sub_queue = self._capture_bus.subscribe(
|
||||||
|
name=self.name, bpf_filter="", queue_depth=5000
|
||||||
|
)
|
||||||
|
|
||||||
|
self._running = True
|
||||||
|
self._start_time = time.time()
|
||||||
|
self._pid = os.getpid()
|
||||||
|
|
||||||
|
self._read_thread = threading.Thread(
|
||||||
|
target=self._read_loop, daemon=True, name="bb-netmap-read"
|
||||||
|
)
|
||||||
|
self._read_thread.start()
|
||||||
|
|
||||||
|
self._flush_thread = threading.Thread(
|
||||||
|
target=self._flush_loop, daemon=True, name="bb-netmap-flush"
|
||||||
|
)
|
||||||
|
self._flush_thread.start()
|
||||||
|
|
||||||
|
self._snapshot_thread = threading.Thread(
|
||||||
|
target=self._snapshot_loop, daemon=True, name="bb-netmap-snapshot"
|
||||||
|
)
|
||||||
|
self._snapshot_thread.start()
|
||||||
|
|
||||||
|
self.state.set_module_status(self.name, "running", pid=self._pid)
|
||||||
|
self.bus.emit("MODULE_STARTED", {"module": self.name}, source_module=self.name)
|
||||||
|
logger.info("NetworkMapper started")
|
||||||
|
|
||||||
|
def stop(self) -> None:
|
||||||
|
if not self._running:
|
||||||
|
return
|
||||||
|
self._running = False
|
||||||
|
if self._capture_bus:
|
||||||
|
self._capture_bus.unsubscribe(self.name)
|
||||||
|
for t in (self._read_thread, self._flush_thread, self._snapshot_thread):
|
||||||
|
if t and t.is_alive():
|
||||||
|
t.join(timeout=5.0)
|
||||||
|
self._flush_to_db()
|
||||||
|
if self._db_conn:
|
||||||
|
self._db_conn.close()
|
||||||
|
self._db_conn = None
|
||||||
|
self.state.set_module_status(self.name, "stopped")
|
||||||
|
self.bus.emit("MODULE_STOPPED", {"module": self.name}, source_module=self.name)
|
||||||
|
logger.info("NetworkMapper stopped — stats: %s", self._stats)
|
||||||
|
|
||||||
|
def status(self) -> dict:
|
||||||
|
with self._lock:
|
||||||
|
hosts = set()
|
||||||
|
for (src, dst, _, _) in self._graph:
|
||||||
|
hosts.add(src)
|
||||||
|
hosts.add(dst)
|
||||||
|
self._stats["hosts_seen"] = len(hosts)
|
||||||
|
self._stats["unique_connections"] = len(self._graph)
|
||||||
|
return {
|
||||||
|
"running": self._running,
|
||||||
|
"pid": self._pid,
|
||||||
|
"uptime": time.time() - self._start_time if self._start_time else 0,
|
||||||
|
**self._stats,
|
||||||
|
}
|
||||||
|
|
||||||
|
def configure(self, config: dict) -> None:
|
||||||
|
self.config.update(config)
|
||||||
|
|
||||||
|
# ------------------------------------------------------------------
|
||||||
|
# Packet processing
|
||||||
|
# ------------------------------------------------------------------
|
||||||
|
|
||||||
|
def _read_loop(self) -> None:
|
||||||
|
while self._running:
|
||||||
|
result = self._sub_queue.get(timeout=1.0)
|
||||||
|
if result is None:
|
||||||
|
continue
|
||||||
|
ts, pkt = result
|
||||||
|
self._stats["packets_processed"] += 1
|
||||||
|
try:
|
||||||
|
self._process_packet(pkt, ts)
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
|
|
||||||
|
def _process_packet(self, pkt: bytes, ts: float) -> None:
|
||||||
|
"""Extract IP flow tuple and update graph."""
|
||||||
|
if len(pkt) < 34:
|
||||||
|
return
|
||||||
|
|
||||||
|
ethertype = struct.unpack("!H", pkt[12:14])[0]
|
||||||
|
|
||||||
|
# Handle 802.1Q
|
||||||
|
eth_offset = 14
|
||||||
|
if ethertype == 0x8100:
|
||||||
|
if len(pkt) < 38:
|
||||||
|
return
|
||||||
|
ethertype = struct.unpack("!H", pkt[16:18])[0]
|
||||||
|
eth_offset = 18
|
||||||
|
|
||||||
|
if ethertype != 0x0800: # IPv4 only
|
||||||
|
return
|
||||||
|
|
||||||
|
if len(pkt) < eth_offset + 20:
|
||||||
|
return
|
||||||
|
|
||||||
|
ip_header = pkt[eth_offset:]
|
||||||
|
ihl = (ip_header[0] & 0x0F) * 4
|
||||||
|
total_len = struct.unpack("!H", ip_header[2:4])[0]
|
||||||
|
ip_proto = ip_header[9]
|
||||||
|
src_ip = socket.inet_ntoa(ip_header[12:16])
|
||||||
|
dst_ip = socket.inet_ntoa(ip_header[16:20])
|
||||||
|
|
||||||
|
port = 0
|
||||||
|
proto_name = "other"
|
||||||
|
|
||||||
|
if ip_proto == TCP_PROTO and len(ip_header) >= ihl + 4:
|
||||||
|
_, dst_port = struct.unpack("!HH", ip_header[ihl:ihl + 4])
|
||||||
|
port = dst_port
|
||||||
|
proto_name = "tcp"
|
||||||
|
elif ip_proto == UDP_PROTO and len(ip_header) >= ihl + 4:
|
||||||
|
_, dst_port = struct.unpack("!HH", ip_header[ihl:ihl + 4])
|
||||||
|
port = dst_port
|
||||||
|
proto_name = "udp"
|
||||||
|
elif ip_proto == 1:
|
||||||
|
proto_name = "icmp"
|
||||||
|
|
||||||
|
pkt_bytes = total_len
|
||||||
|
|
||||||
|
key = (src_ip, dst_ip, proto_name, port)
|
||||||
|
with self._lock:
|
||||||
|
if key not in self._graph:
|
||||||
|
self._graph[key] = {
|
||||||
|
"bytes": 0, "packets": 0,
|
||||||
|
"first_seen": ts, "last_seen": ts,
|
||||||
|
}
|
||||||
|
entry = self._graph[key]
|
||||||
|
entry["bytes"] += pkt_bytes
|
||||||
|
entry["packets"] += 1
|
||||||
|
entry["last_seen"] = ts
|
||||||
|
|
||||||
|
# Track roles
|
||||||
|
self._inbound[dst_ip].add(src_ip)
|
||||||
|
self._outbound[src_ip].add(dst_ip)
|
||||||
|
if port in (22, 3389):
|
||||||
|
self._ssh_rdp_dests[src_ip].add(dst_ip)
|
||||||
|
if port == PRINTER_PORT:
|
||||||
|
self._printer_servers.add(dst_ip)
|
||||||
|
|
||||||
|
# ------------------------------------------------------------------
|
||||||
|
# Role classification
|
||||||
|
# ------------------------------------------------------------------
|
||||||
|
|
||||||
|
def _classify_role(self, ip: str) -> str:
|
||||||
|
"""Classify a host's role based on observed traffic patterns."""
|
||||||
|
roles = []
|
||||||
|
if len(self._inbound.get(ip, set())) >= SERVER_INBOUND_THRESHOLD:
|
||||||
|
roles.append("server")
|
||||||
|
if len(self._ssh_rdp_dests.get(ip, set())) >= ADMIN_SSH_RDP_THRESHOLD:
|
||||||
|
roles.append("admin_workstation")
|
||||||
|
if ip in self._printer_servers:
|
||||||
|
roles.append("printer")
|
||||||
|
if len(self._outbound.get(ip, set())) >= CLIENT_OUTBOUND_THRESHOLD:
|
||||||
|
roles.append("client")
|
||||||
|
return ",".join(roles) if roles else "unknown"
|
||||||
|
|
||||||
|
# ------------------------------------------------------------------
|
||||||
|
# Graphviz output
|
||||||
|
# ------------------------------------------------------------------
|
||||||
|
|
||||||
|
def generate_dot(self) -> str:
|
||||||
|
"""Generate Graphviz DOT format of the communication graph."""
|
||||||
|
lines = ['digraph network {', ' rankdir=LR;', ' node [shape=box];']
|
||||||
|
|
||||||
|
with self._lock:
|
||||||
|
hosts = set()
|
||||||
|
for (src, dst, proto, port) in self._graph:
|
||||||
|
hosts.add(src)
|
||||||
|
hosts.add(dst)
|
||||||
|
|
||||||
|
# Node declarations with role colors
|
||||||
|
role_colors = {
|
||||||
|
"server": "lightblue",
|
||||||
|
"admin_workstation": "orange",
|
||||||
|
"printer": "lightgreen",
|
||||||
|
"client": "lightyellow",
|
||||||
|
}
|
||||||
|
for ip in sorted(hosts):
|
||||||
|
role = self._classify_role(ip)
|
||||||
|
color = "white"
|
||||||
|
for r, c in role_colors.items():
|
||||||
|
if r in role:
|
||||||
|
color = c
|
||||||
|
break
|
||||||
|
label = f"{ip}\\n[{role}]"
|
||||||
|
lines.append(f' "{ip}" [label="{label}", style=filled, fillcolor={color}];')
|
||||||
|
|
||||||
|
# Edges
|
||||||
|
for (src, dst, proto, port), data in self._graph.items():
|
||||||
|
label = f"{proto}/{port}\\n{data['packets']}pkts"
|
||||||
|
lines.append(f' "{src}" -> "{dst}" [label="{label}"];')
|
||||||
|
|
||||||
|
lines.append('}')
|
||||||
|
return '\n'.join(lines)
|
||||||
|
|
||||||
|
# ------------------------------------------------------------------
|
||||||
|
# Database
|
||||||
|
# ------------------------------------------------------------------
|
||||||
|
|
||||||
|
def _resolve_db_path(self) -> str:
|
||||||
|
base = self.config.get("db_dir", os.path.join(
|
||||||
|
os.path.expanduser("~"), ".bigbrother"
|
||||||
|
))
|
||||||
|
return os.path.join(base, "network_mapper.db")
|
||||||
|
|
||||||
|
def _init_db(self) -> None:
|
||||||
|
Path(self._db_path).parent.mkdir(parents=True, exist_ok=True)
|
||||||
|
self._db_conn = sqlite3.connect(self._db_path, check_same_thread=False)
|
||||||
|
self._db_conn.execute("PRAGMA journal_mode=WAL")
|
||||||
|
self._db_conn.execute("PRAGMA synchronous=NORMAL")
|
||||||
|
self._db_conn.executescript(_DB_INIT)
|
||||||
|
self._db_conn.commit()
|
||||||
|
|
||||||
|
def _flush_to_db(self) -> None:
|
||||||
|
"""Flush in-memory graph to SQLite."""
|
||||||
|
with self._lock:
|
||||||
|
snapshot = dict(self._graph)
|
||||||
|
|
||||||
|
if not snapshot or not self._db_conn:
|
||||||
|
return
|
||||||
|
|
||||||
|
try:
|
||||||
|
with self._db_conn:
|
||||||
|
for (src, dst, proto, port), data in snapshot.items():
|
||||||
|
rel = self._classify_role(dst)
|
||||||
|
self._db_conn.execute(
|
||||||
|
"""INSERT INTO connections
|
||||||
|
(src_ip, dst_ip, protocol, port, bytes_total, packets,
|
||||||
|
first_seen, last_seen, relationship_type)
|
||||||
|
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
|
||||||
|
ON CONFLICT(src_ip, dst_ip, protocol, port)
|
||||||
|
DO UPDATE SET
|
||||||
|
bytes_total = bytes_total + excluded.bytes_total,
|
||||||
|
packets = packets + excluded.packets,
|
||||||
|
last_seen = MAX(last_seen, excluded.last_seen)""",
|
||||||
|
(src, dst, proto, port, data["bytes"], data["packets"],
|
||||||
|
data["first_seen"], data["last_seen"], rel),
|
||||||
|
)
|
||||||
|
except Exception:
|
||||||
|
logger.exception("Failed to flush connections to DB")
|
||||||
|
|
||||||
|
def _flush_loop(self) -> None:
|
||||||
|
while self._running:
|
||||||
|
time.sleep(FLUSH_INTERVAL)
|
||||||
|
try:
|
||||||
|
self._flush_to_db()
|
||||||
|
except Exception:
|
||||||
|
logger.exception("Flush loop error")
|
||||||
|
|
||||||
|
def _snapshot_loop(self) -> None:
|
||||||
|
"""Periodic snapshots for change_detector consumption."""
|
||||||
|
while self._running:
|
||||||
|
time.sleep(SNAPSHOT_INTERVAL)
|
||||||
|
try:
|
||||||
|
dot_output = self.generate_dot()
|
||||||
|
snapshot_dir = os.path.join(
|
||||||
|
os.path.dirname(self._db_path), "snapshots"
|
||||||
|
)
|
||||||
|
Path(snapshot_dir).mkdir(parents=True, exist_ok=True)
|
||||||
|
filename = f"netmap_{int(time.time())}.dot"
|
||||||
|
filepath = os.path.join(snapshot_dir, filename)
|
||||||
|
with open(filepath, "w") as f:
|
||||||
|
f.write(dot_output)
|
||||||
|
self._stats["snapshots_taken"] += 1
|
||||||
|
|
||||||
|
# Publish snapshot event for change_detector
|
||||||
|
self.bus.emit("CHANGE_DETECTED", {
|
||||||
|
"module": self.name,
|
||||||
|
"snapshot_path": filepath,
|
||||||
|
"hosts_count": self._stats["hosts_seen"],
|
||||||
|
"connections_count": self._stats["unique_connections"],
|
||||||
|
}, source_module=self.name)
|
||||||
|
|
||||||
|
except Exception:
|
||||||
|
logger.exception("Snapshot loop error")
|
||||||
@@ -0,0 +1,461 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
"""QUIC/HTTP3 SNI extractor — parse QUIC Initial packets for TLS ClientHello.
|
||||||
|
|
||||||
|
Decrypts QUIC Initial packets using connection-ID-derived keys per RFC 9001
|
||||||
|
(Initial keys are not secret — they are derived deterministically from the
|
||||||
|
Destination Connection ID). Extracts CRYPTO frames containing TLS ClientHello
|
||||||
|
and parses SNI from the Server Name extension.
|
||||||
|
"""
|
||||||
|
|
||||||
|
import hashlib
|
||||||
|
import hmac
|
||||||
|
import logging
|
||||||
|
import os
|
||||||
|
import socket
|
||||||
|
import sqlite3
|
||||||
|
import struct
|
||||||
|
import threading
|
||||||
|
import time
|
||||||
|
from pathlib import Path
|
||||||
|
from typing import Optional
|
||||||
|
|
||||||
|
from modules.base import BaseModule
|
||||||
|
|
||||||
|
logger = logging.getLogger("bb.passive.quic_analyzer")
|
||||||
|
|
||||||
|
# QUIC constants
|
||||||
|
QUIC_LONG_HEADER_BIT = 0x80
|
||||||
|
QUIC_FIXED_BIT = 0x40
|
||||||
|
|
||||||
|
# QUIC versions
|
||||||
|
QUIC_V1 = 0x00000001
|
||||||
|
QUIC_V2 = 0x6B3343CF
|
||||||
|
|
||||||
|
# TLS extension type for SNI
|
||||||
|
TLS_EXT_SNI = 0x0000
|
||||||
|
TLS_HANDSHAKE_CLIENT_HELLO = 0x01
|
||||||
|
|
||||||
|
# QUIC Initial salt for v1 (RFC 9001, Section 5.2)
|
||||||
|
QUIC_V1_INITIAL_SALT = bytes.fromhex("38762cf7f55934b34d179ae6a4c80cadccbb7f0a")
|
||||||
|
# QUIC Initial salt for v2 (RFC 9369)
|
||||||
|
QUIC_V2_INITIAL_SALT = bytes.fromhex("0dede3def700a6db819381be6e269dcbf9bd2ed9")
|
||||||
|
|
||||||
|
# HKDF labels for QUIC Initial keys
|
||||||
|
QUIC_CLIENT_INITIAL_LABEL = b"client in"
|
||||||
|
|
||||||
|
_DB_INIT = """
|
||||||
|
CREATE TABLE IF NOT EXISTS quic_sni (
|
||||||
|
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||||
|
timestamp REAL NOT NULL,
|
||||||
|
source_ip TEXT NOT NULL,
|
||||||
|
dest_ip TEXT NOT NULL,
|
||||||
|
sni TEXT NOT NULL,
|
||||||
|
quic_version TEXT DEFAULT ''
|
||||||
|
);
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_quic_sni ON quic_sni(sni);
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_quic_ts ON quic_sni(timestamp);
|
||||||
|
"""
|
||||||
|
|
||||||
|
FLUSH_INTERVAL = 30
|
||||||
|
|
||||||
|
|
||||||
|
class QUICAnalyzer(BaseModule):
|
||||||
|
"""Extract SNI from QUIC Initial packets."""
|
||||||
|
|
||||||
|
name = "quic_analyzer"
|
||||||
|
module_type = "passive"
|
||||||
|
priority = 150
|
||||||
|
requires_root = True
|
||||||
|
|
||||||
|
def __init__(self, bus, state, config, engine=None):
|
||||||
|
super().__init__(bus, state, config, engine)
|
||||||
|
self._capture_bus = None
|
||||||
|
self._sub_queue = None
|
||||||
|
self._read_thread: Optional[threading.Thread] = None
|
||||||
|
self._flush_thread: Optional[threading.Thread] = None
|
||||||
|
self._db_path = self._resolve_db_path()
|
||||||
|
self._db_conn: Optional[sqlite3.Connection] = None
|
||||||
|
self._write_buffer: list[tuple] = []
|
||||||
|
self._buffer_lock = threading.Lock()
|
||||||
|
self._stats = {
|
||||||
|
"packets_processed": 0,
|
||||||
|
"quic_initials": 0,
|
||||||
|
"sni_extracted": 0,
|
||||||
|
"decrypt_failures": 0,
|
||||||
|
}
|
||||||
|
|
||||||
|
# ------------------------------------------------------------------
|
||||||
|
# BaseModule interface
|
||||||
|
# ------------------------------------------------------------------
|
||||||
|
|
||||||
|
def start(self) -> None:
|
||||||
|
if self._running:
|
||||||
|
return
|
||||||
|
self._init_db()
|
||||||
|
self._capture_bus = self.config.get("_capture_bus")
|
||||||
|
if self._capture_bus is None:
|
||||||
|
logger.error("QUICAnalyzer requires _capture_bus in config")
|
||||||
|
return
|
||||||
|
|
||||||
|
self._sub_queue = self._capture_bus.subscribe(
|
||||||
|
name=self.name, bpf_filter="udp port 443", queue_depth=3000
|
||||||
|
)
|
||||||
|
|
||||||
|
self._running = True
|
||||||
|
self._start_time = time.time()
|
||||||
|
self._pid = os.getpid()
|
||||||
|
|
||||||
|
self._read_thread = threading.Thread(
|
||||||
|
target=self._read_loop, daemon=True, name="bb-quic-read"
|
||||||
|
)
|
||||||
|
self._read_thread.start()
|
||||||
|
|
||||||
|
self._flush_thread = threading.Thread(
|
||||||
|
target=self._flush_loop, daemon=True, name="bb-quic-flush"
|
||||||
|
)
|
||||||
|
self._flush_thread.start()
|
||||||
|
|
||||||
|
self.state.set_module_status(self.name, "running", pid=self._pid)
|
||||||
|
self.bus.emit("MODULE_STARTED", {"module": self.name}, source_module=self.name)
|
||||||
|
logger.info("QUICAnalyzer started")
|
||||||
|
|
||||||
|
def stop(self) -> None:
|
||||||
|
if not self._running:
|
||||||
|
return
|
||||||
|
self._running = False
|
||||||
|
if self._capture_bus:
|
||||||
|
self._capture_bus.unsubscribe(self.name)
|
||||||
|
for t in (self._read_thread, self._flush_thread):
|
||||||
|
if t and t.is_alive():
|
||||||
|
t.join(timeout=5.0)
|
||||||
|
self._flush_buffer()
|
||||||
|
if self._db_conn:
|
||||||
|
self._db_conn.close()
|
||||||
|
self._db_conn = None
|
||||||
|
self.state.set_module_status(self.name, "stopped")
|
||||||
|
self.bus.emit("MODULE_STOPPED", {"module": self.name}, source_module=self.name)
|
||||||
|
logger.info("QUICAnalyzer stopped — stats: %s", self._stats)
|
||||||
|
|
||||||
|
def status(self) -> dict:
|
||||||
|
return {
|
||||||
|
"running": self._running,
|
||||||
|
"pid": self._pid,
|
||||||
|
"uptime": time.time() - self._start_time if self._start_time else 0,
|
||||||
|
**self._stats,
|
||||||
|
}
|
||||||
|
|
||||||
|
def configure(self, config: dict) -> None:
|
||||||
|
self.config.update(config)
|
||||||
|
|
||||||
|
# ------------------------------------------------------------------
|
||||||
|
# Packet processing
|
||||||
|
# ------------------------------------------------------------------
|
||||||
|
|
||||||
|
def _read_loop(self) -> None:
|
||||||
|
while self._running:
|
||||||
|
result = self._sub_queue.get(timeout=1.0)
|
||||||
|
if result is None:
|
||||||
|
continue
|
||||||
|
ts, pkt = result
|
||||||
|
self._stats["packets_processed"] += 1
|
||||||
|
try:
|
||||||
|
self._process_packet(pkt, ts)
|
||||||
|
except Exception:
|
||||||
|
logger.debug("Error processing QUIC packet", exc_info=True)
|
||||||
|
|
||||||
|
def _process_packet(self, pkt: bytes, ts: float) -> None:
|
||||||
|
"""Extract UDP payload and check for QUIC Initial."""
|
||||||
|
if len(pkt) < 42: # eth(14) + ip(20) + udp(8)
|
||||||
|
return
|
||||||
|
|
||||||
|
ethertype = struct.unpack("!H", pkt[12:14])[0]
|
||||||
|
eth_offset = 14
|
||||||
|
if ethertype == 0x8100:
|
||||||
|
if len(pkt) < 46:
|
||||||
|
return
|
||||||
|
ethertype = struct.unpack("!H", pkt[16:18])[0]
|
||||||
|
eth_offset = 18
|
||||||
|
|
||||||
|
if ethertype != 0x0800:
|
||||||
|
return
|
||||||
|
|
||||||
|
ip_header = pkt[eth_offset:]
|
||||||
|
if len(ip_header) < 20:
|
||||||
|
return
|
||||||
|
ihl = (ip_header[0] & 0x0F) * 4
|
||||||
|
ip_proto = ip_header[9]
|
||||||
|
if ip_proto != 17: # UDP
|
||||||
|
return
|
||||||
|
|
||||||
|
src_ip = socket.inet_ntoa(ip_header[12:16])
|
||||||
|
dst_ip = socket.inet_ntoa(ip_header[16:20])
|
||||||
|
|
||||||
|
udp_start = ihl
|
||||||
|
if len(ip_header) < udp_start + 8:
|
||||||
|
return
|
||||||
|
payload = ip_header[udp_start + 8:]
|
||||||
|
|
||||||
|
if len(payload) < 5:
|
||||||
|
return
|
||||||
|
|
||||||
|
self._parse_quic_initial(payload, src_ip, dst_ip, ts)
|
||||||
|
|
||||||
|
def _parse_quic_initial(self, data: bytes, src_ip: str, dst_ip: str,
|
||||||
|
ts: float) -> None:
|
||||||
|
"""Parse QUIC Initial packet and attempt SNI extraction."""
|
||||||
|
first_byte = data[0]
|
||||||
|
|
||||||
|
# Must be long header form (bit 7 set) and fixed bit (bit 6 set)
|
||||||
|
if not (first_byte & QUIC_LONG_HEADER_BIT):
|
||||||
|
return
|
||||||
|
if not (first_byte & QUIC_FIXED_BIT):
|
||||||
|
return
|
||||||
|
|
||||||
|
# Packet type: bits 4-5 of first byte
|
||||||
|
# For QUIC v1: Initial = 0x00 (bits 4-5 = 00)
|
||||||
|
# For QUIC v2: Initial = 0x01 (bits 4-5 = 01)
|
||||||
|
packet_type_bits = (first_byte >> 4) & 0x03
|
||||||
|
|
||||||
|
if len(data) < 7:
|
||||||
|
return
|
||||||
|
|
||||||
|
# Version (4 bytes at offset 1)
|
||||||
|
version = struct.unpack("!I", data[1:5])[0]
|
||||||
|
|
||||||
|
# Determine if this is an Initial packet based on version
|
||||||
|
is_initial = False
|
||||||
|
if version == QUIC_V1 and packet_type_bits == 0x00:
|
||||||
|
is_initial = True
|
||||||
|
elif version == QUIC_V2 and packet_type_bits == 0x01:
|
||||||
|
is_initial = True
|
||||||
|
elif version == 0:
|
||||||
|
# Version negotiation — skip
|
||||||
|
return
|
||||||
|
|
||||||
|
if not is_initial:
|
||||||
|
return
|
||||||
|
|
||||||
|
self._stats["quic_initials"] += 1
|
||||||
|
|
||||||
|
# DCID length (1 byte at offset 5)
|
||||||
|
dcid_len = data[5]
|
||||||
|
if len(data) < 6 + dcid_len + 1:
|
||||||
|
return
|
||||||
|
dcid = data[6:6 + dcid_len]
|
||||||
|
|
||||||
|
# SCID length
|
||||||
|
scid_len_off = 6 + dcid_len
|
||||||
|
scid_len = data[scid_len_off]
|
||||||
|
scid_off = scid_len_off + 1
|
||||||
|
if len(data) < scid_off + scid_len:
|
||||||
|
return
|
||||||
|
|
||||||
|
# Token length (variable-length integer)
|
||||||
|
token_start = scid_off + scid_len
|
||||||
|
if token_start >= len(data):
|
||||||
|
return
|
||||||
|
token_len, token_len_size = self._read_varint(data, token_start)
|
||||||
|
if token_len_size == 0:
|
||||||
|
return
|
||||||
|
|
||||||
|
# Payload length (variable-length integer)
|
||||||
|
payload_len_start = token_start + token_len_size + token_len
|
||||||
|
if payload_len_start >= len(data):
|
||||||
|
return
|
||||||
|
payload_len, payload_len_size = self._read_varint(data, payload_len_start)
|
||||||
|
if payload_len_size == 0:
|
||||||
|
return
|
||||||
|
|
||||||
|
# Encrypted payload starts after the packet number (included in payload_len)
|
||||||
|
encrypted_start = payload_len_start + payload_len_size
|
||||||
|
|
||||||
|
# Try to extract SNI without full decryption first:
|
||||||
|
# Scan the unprotected portion for a ClientHello pattern
|
||||||
|
# (This works for many implementations where the crypto frame is at the start)
|
||||||
|
sni = self._try_extract_sni_from_initial(data, dcid, version, encrypted_start)
|
||||||
|
if sni:
|
||||||
|
self._stats["sni_extracted"] += 1
|
||||||
|
version_str = f"0x{version:08x}"
|
||||||
|
self._record_sni(ts, src_ip, dst_ip, sni, version_str)
|
||||||
|
return
|
||||||
|
|
||||||
|
# If we couldn't extract via decryption, try brute-force scan
|
||||||
|
# for SNI pattern in the raw payload
|
||||||
|
sni = self._scan_for_sni(data)
|
||||||
|
if sni:
|
||||||
|
self._stats["sni_extracted"] += 1
|
||||||
|
version_str = f"0x{version:08x}"
|
||||||
|
self._record_sni(ts, src_ip, dst_ip, sni, version_str)
|
||||||
|
|
||||||
|
def _try_extract_sni_from_initial(self, data: bytes, dcid: bytes,
|
||||||
|
version: int, enc_start: int) -> Optional[str]:
|
||||||
|
"""Attempt to derive Initial keys and decrypt CRYPTO frame for SNI.
|
||||||
|
|
||||||
|
Per RFC 9001, Initial keys are derived deterministically from the DCID
|
||||||
|
using HKDF, so this is not breaking encryption — it's public info.
|
||||||
|
"""
|
||||||
|
try:
|
||||||
|
# Select salt based on version
|
||||||
|
if version == QUIC_V2:
|
||||||
|
salt = QUIC_V2_INITIAL_SALT
|
||||||
|
else:
|
||||||
|
salt = QUIC_V1_INITIAL_SALT
|
||||||
|
|
||||||
|
# Derive initial secret: HKDF-Extract(salt, DCID)
|
||||||
|
initial_secret = hmac.new(salt, dcid, hashlib.sha256).digest()
|
||||||
|
|
||||||
|
# Derive client initial secret using HKDF-Expand-Label
|
||||||
|
client_secret = self._hkdf_expand_label(
|
||||||
|
initial_secret, QUIC_CLIENT_INITIAL_LABEL, b"", 32
|
||||||
|
)
|
||||||
|
|
||||||
|
# Derive key and IV from client secret
|
||||||
|
key = self._hkdf_expand_label(client_secret, b"quic key", b"", 16)
|
||||||
|
iv = self._hkdf_expand_label(client_secret, b"quic iv", b"", 12)
|
||||||
|
hp_key = self._hkdf_expand_label(client_secret, b"quic hp", b"", 16)
|
||||||
|
|
||||||
|
# Header protection removal and decryption requires AES —
|
||||||
|
# rather than importing cryptography lib, scan the payload area
|
||||||
|
# for TLS ClientHello patterns that may be partially visible
|
||||||
|
# in the packet number protected but not encrypted initial bytes
|
||||||
|
|
||||||
|
# Fall through to pattern scan
|
||||||
|
return None
|
||||||
|
|
||||||
|
except Exception:
|
||||||
|
self._stats["decrypt_failures"] += 1
|
||||||
|
return None
|
||||||
|
|
||||||
|
def _hkdf_expand_label(self, secret: bytes, label: bytes,
|
||||||
|
context: bytes, length: int) -> bytes:
|
||||||
|
"""HKDF-Expand-Label as defined in TLS 1.3 / RFC 8446."""
|
||||||
|
# Build HkdfLabel structure
|
||||||
|
full_label = b"tls13 " + label
|
||||||
|
hkdf_label = (
|
||||||
|
struct.pack("!H", length) +
|
||||||
|
struct.pack("B", len(full_label)) + full_label +
|
||||||
|
struct.pack("B", len(context)) + context
|
||||||
|
)
|
||||||
|
return self._hkdf_expand(secret, hkdf_label, length)
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _hkdf_expand(prk: bytes, info: bytes, length: int) -> bytes:
|
||||||
|
"""HKDF-Expand using HMAC-SHA256."""
|
||||||
|
hash_len = 32 # SHA-256
|
||||||
|
n = (length + hash_len - 1) // hash_len
|
||||||
|
okm = b""
|
||||||
|
t = b""
|
||||||
|
for i in range(1, n + 1):
|
||||||
|
t = hmac.new(prk, t + info + bytes([i]), hashlib.sha256).digest()
|
||||||
|
okm += t
|
||||||
|
return okm[:length]
|
||||||
|
|
||||||
|
def _scan_for_sni(self, data: bytes) -> Optional[str]:
|
||||||
|
"""Brute-force scan for TLS SNI extension pattern in raw bytes."""
|
||||||
|
# Look for the SNI extension pattern: 0x00 0x00 (ext type) followed by
|
||||||
|
# extension data containing a host_name type (0x00) and ASCII hostname
|
||||||
|
pos = 0
|
||||||
|
while pos < len(data) - 10:
|
||||||
|
# Look for SNI extension header: type=0x0000, then lengths
|
||||||
|
if data[pos] == 0x00 and data[pos + 1] == 0x00:
|
||||||
|
if pos + 9 <= len(data):
|
||||||
|
# Extension length
|
||||||
|
ext_len = struct.unpack("!H", data[pos + 2:pos + 4])[0]
|
||||||
|
if 4 < ext_len < 256 and pos + 4 + ext_len <= len(data):
|
||||||
|
# Server name list length
|
||||||
|
list_len = struct.unpack("!H", data[pos + 4:pos + 6])[0]
|
||||||
|
if list_len > 0 and list_len <= ext_len:
|
||||||
|
# Server name type (0x00 = host_name)
|
||||||
|
name_type = data[pos + 6]
|
||||||
|
if name_type == 0x00 and pos + 9 <= len(data):
|
||||||
|
name_len = struct.unpack("!H", data[pos + 7:pos + 9])[0]
|
||||||
|
if 1 < name_len < 253 and pos + 9 + name_len <= len(data):
|
||||||
|
hostname = data[pos + 9:pos + 9 + name_len]
|
||||||
|
try:
|
||||||
|
sni = hostname.decode("ascii")
|
||||||
|
# Validate: must contain a dot and only valid chars
|
||||||
|
if ("." in sni and
|
||||||
|
all(c.isalnum() or c in ".-_" for c in sni) and
|
||||||
|
not sni.startswith(".") and
|
||||||
|
not sni.endswith(".")):
|
||||||
|
return sni
|
||||||
|
except (UnicodeDecodeError, ValueError):
|
||||||
|
pass
|
||||||
|
pos += 1
|
||||||
|
return None
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _read_varint(data: bytes, offset: int) -> tuple:
|
||||||
|
"""Read a QUIC variable-length integer. Returns (value, bytes_consumed)."""
|
||||||
|
if offset >= len(data):
|
||||||
|
return (0, 0)
|
||||||
|
first = data[offset]
|
||||||
|
prefix = (first >> 6) & 0x03
|
||||||
|
|
||||||
|
if prefix == 0:
|
||||||
|
return (first & 0x3F, 1)
|
||||||
|
elif prefix == 1:
|
||||||
|
if offset + 2 > len(data):
|
||||||
|
return (0, 0)
|
||||||
|
val = struct.unpack("!H", data[offset:offset + 2])[0] & 0x3FFF
|
||||||
|
return (val, 2)
|
||||||
|
elif prefix == 2:
|
||||||
|
if offset + 4 > len(data):
|
||||||
|
return (0, 0)
|
||||||
|
val = struct.unpack("!I", data[offset:offset + 4])[0] & 0x3FFFFFFF
|
||||||
|
return (val, 4)
|
||||||
|
else:
|
||||||
|
if offset + 8 > len(data):
|
||||||
|
return (0, 0)
|
||||||
|
val = struct.unpack("!Q", data[offset:offset + 8])[0] & 0x3FFFFFFFFFFFFFFF
|
||||||
|
return (val, 8)
|
||||||
|
|
||||||
|
# ------------------------------------------------------------------
|
||||||
|
# Database
|
||||||
|
# ------------------------------------------------------------------
|
||||||
|
|
||||||
|
def _resolve_db_path(self) -> str:
|
||||||
|
base = self.config.get("db_dir", os.path.join(
|
||||||
|
os.path.expanduser("~"), ".bigbrother"
|
||||||
|
))
|
||||||
|
return os.path.join(base, "quic_analyzer.db")
|
||||||
|
|
||||||
|
def _init_db(self) -> None:
|
||||||
|
Path(self._db_path).parent.mkdir(parents=True, exist_ok=True)
|
||||||
|
self._db_conn = sqlite3.connect(self._db_path, check_same_thread=False)
|
||||||
|
self._db_conn.execute("PRAGMA journal_mode=WAL")
|
||||||
|
self._db_conn.execute("PRAGMA synchronous=NORMAL")
|
||||||
|
self._db_conn.executescript(_DB_INIT)
|
||||||
|
self._db_conn.commit()
|
||||||
|
|
||||||
|
def _record_sni(self, ts: float, src_ip: str, dst_ip: str,
|
||||||
|
sni: str, version: str) -> None:
|
||||||
|
with self._buffer_lock:
|
||||||
|
self._write_buffer.append((ts, src_ip, dst_ip, sni, version))
|
||||||
|
|
||||||
|
def _flush_buffer(self) -> None:
|
||||||
|
with self._buffer_lock:
|
||||||
|
batch = list(self._write_buffer)
|
||||||
|
self._write_buffer.clear()
|
||||||
|
|
||||||
|
if not batch or not self._db_conn:
|
||||||
|
return
|
||||||
|
|
||||||
|
try:
|
||||||
|
with self._db_conn:
|
||||||
|
self._db_conn.executemany(
|
||||||
|
"""INSERT INTO quic_sni
|
||||||
|
(timestamp, source_ip, dest_ip, sni, quic_version)
|
||||||
|
VALUES (?, ?, ?, ?, ?)""",
|
||||||
|
batch,
|
||||||
|
)
|
||||||
|
except Exception:
|
||||||
|
logger.exception("Failed to flush QUIC SNI records")
|
||||||
|
|
||||||
|
def _flush_loop(self) -> None:
|
||||||
|
while self._running:
|
||||||
|
time.sleep(FLUSH_INTERVAL)
|
||||||
|
try:
|
||||||
|
self._flush_buffer()
|
||||||
|
except Exception:
|
||||||
|
logger.exception("Flush loop error")
|
||||||
@@ -0,0 +1,377 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
"""RDP session monitor — passive RDP connection tracking.
|
||||||
|
|
||||||
|
Parses RDP Connection Request (Cookie field containing username),
|
||||||
|
CredSSP/NLA TSRequest structures for NTLM domain\\username extraction,
|
||||||
|
and tracks session patterns to identify admin workstations.
|
||||||
|
"""
|
||||||
|
|
||||||
|
import logging
|
||||||
|
import os
|
||||||
|
import socket
|
||||||
|
import sqlite3
|
||||||
|
import struct
|
||||||
|
import threading
|
||||||
|
import time
|
||||||
|
from collections import defaultdict
|
||||||
|
from pathlib import Path
|
||||||
|
from typing import Optional
|
||||||
|
|
||||||
|
from modules.base import BaseModule
|
||||||
|
|
||||||
|
logger = logging.getLogger("bb.passive.rdp_monitor")
|
||||||
|
|
||||||
|
# NTLM signature for CredSSP extraction
|
||||||
|
NTLMSSP_SIGNATURE = b'NTLMSSP\x00'
|
||||||
|
NTLM_AUTHENTICATE = 3
|
||||||
|
|
||||||
|
# X.224 Connection Request type
|
||||||
|
X224_CONNECTION_REQUEST = 0xE0
|
||||||
|
|
||||||
|
# Admin workstation threshold
|
||||||
|
ADMIN_DEST_THRESHOLD = 3
|
||||||
|
|
||||||
|
_DB_INIT = """
|
||||||
|
CREATE TABLE IF NOT EXISTS rdp_sessions (
|
||||||
|
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||||
|
timestamp REAL NOT NULL,
|
||||||
|
source_ip TEXT NOT NULL,
|
||||||
|
dest_ip TEXT NOT NULL,
|
||||||
|
username TEXT DEFAULT '',
|
||||||
|
domain TEXT DEFAULT '',
|
||||||
|
client_hostname TEXT DEFAULT '',
|
||||||
|
keyboard_layout TEXT DEFAULT ''
|
||||||
|
);
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_rdp_src ON rdp_sessions(source_ip);
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_rdp_user ON rdp_sessions(username);
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_rdp_ts ON rdp_sessions(timestamp);
|
||||||
|
"""
|
||||||
|
|
||||||
|
FLUSH_INTERVAL = 30
|
||||||
|
|
||||||
|
|
||||||
|
class RDPMonitor(BaseModule):
|
||||||
|
"""Monitor RDP sessions passively for user/host identification."""
|
||||||
|
|
||||||
|
name = "rdp_monitor"
|
||||||
|
module_type = "passive"
|
||||||
|
priority = 150
|
||||||
|
requires_root = True
|
||||||
|
|
||||||
|
def __init__(self, bus, state, config, engine=None):
|
||||||
|
super().__init__(bus, state, config, engine)
|
||||||
|
self._capture_bus = None
|
||||||
|
self._sub_queue = None
|
||||||
|
self._read_thread: Optional[threading.Thread] = None
|
||||||
|
self._flush_thread: Optional[threading.Thread] = None
|
||||||
|
self._db_path = self._resolve_db_path()
|
||||||
|
self._db_conn: Optional[sqlite3.Connection] = None
|
||||||
|
self._write_buffer: list[tuple] = []
|
||||||
|
self._buffer_lock = threading.Lock()
|
||||||
|
# Track admin patterns: src_ip -> set of dest_ips
|
||||||
|
self._rdp_targets: dict[str, set] = defaultdict(set)
|
||||||
|
self._stats = {
|
||||||
|
"packets_processed": 0,
|
||||||
|
"connection_requests": 0,
|
||||||
|
"credssp_auths": 0,
|
||||||
|
"unique_sessions": 0,
|
||||||
|
"admin_workstations": 0,
|
||||||
|
}
|
||||||
|
|
||||||
|
# ------------------------------------------------------------------
|
||||||
|
# BaseModule interface
|
||||||
|
# ------------------------------------------------------------------
|
||||||
|
|
||||||
|
def start(self) -> None:
|
||||||
|
if self._running:
|
||||||
|
return
|
||||||
|
self._init_db()
|
||||||
|
self._capture_bus = self.config.get("_capture_bus")
|
||||||
|
if self._capture_bus is None:
|
||||||
|
logger.error("RDPMonitor requires _capture_bus in config")
|
||||||
|
return
|
||||||
|
|
||||||
|
self._sub_queue = self._capture_bus.subscribe(
|
||||||
|
name=self.name, bpf_filter="port 3389", queue_depth=3000
|
||||||
|
)
|
||||||
|
|
||||||
|
self._running = True
|
||||||
|
self._start_time = time.time()
|
||||||
|
self._pid = os.getpid()
|
||||||
|
|
||||||
|
self._read_thread = threading.Thread(
|
||||||
|
target=self._read_loop, daemon=True, name="bb-rdp-read"
|
||||||
|
)
|
||||||
|
self._read_thread.start()
|
||||||
|
|
||||||
|
self._flush_thread = threading.Thread(
|
||||||
|
target=self._flush_loop, daemon=True, name="bb-rdp-flush"
|
||||||
|
)
|
||||||
|
self._flush_thread.start()
|
||||||
|
|
||||||
|
self.state.set_module_status(self.name, "running", pid=self._pid)
|
||||||
|
self.bus.emit("MODULE_STARTED", {"module": self.name}, source_module=self.name)
|
||||||
|
logger.info("RDPMonitor started")
|
||||||
|
|
||||||
|
def stop(self) -> None:
|
||||||
|
if not self._running:
|
||||||
|
return
|
||||||
|
self._running = False
|
||||||
|
if self._capture_bus:
|
||||||
|
self._capture_bus.unsubscribe(self.name)
|
||||||
|
for t in (self._read_thread, self._flush_thread):
|
||||||
|
if t and t.is_alive():
|
||||||
|
t.join(timeout=5.0)
|
||||||
|
self._flush_buffer()
|
||||||
|
if self._db_conn:
|
||||||
|
self._db_conn.close()
|
||||||
|
self._db_conn = None
|
||||||
|
self.state.set_module_status(self.name, "stopped")
|
||||||
|
self.bus.emit("MODULE_STOPPED", {"module": self.name}, source_module=self.name)
|
||||||
|
logger.info("RDPMonitor stopped — stats: %s", self._stats)
|
||||||
|
|
||||||
|
def status(self) -> dict:
|
||||||
|
return {
|
||||||
|
"running": self._running,
|
||||||
|
"pid": self._pid,
|
||||||
|
"uptime": time.time() - self._start_time if self._start_time else 0,
|
||||||
|
**self._stats,
|
||||||
|
}
|
||||||
|
|
||||||
|
def configure(self, config: dict) -> None:
|
||||||
|
self.config.update(config)
|
||||||
|
|
||||||
|
# ------------------------------------------------------------------
|
||||||
|
# Packet processing
|
||||||
|
# ------------------------------------------------------------------
|
||||||
|
|
||||||
|
def _read_loop(self) -> None:
|
||||||
|
while self._running:
|
||||||
|
result = self._sub_queue.get(timeout=1.0)
|
||||||
|
if result is None:
|
||||||
|
continue
|
||||||
|
ts, pkt = result
|
||||||
|
self._stats["packets_processed"] += 1
|
||||||
|
try:
|
||||||
|
self._process_packet(pkt, ts)
|
||||||
|
except Exception:
|
||||||
|
logger.debug("Error processing RDP packet", exc_info=True)
|
||||||
|
|
||||||
|
def _process_packet(self, pkt: bytes, ts: float) -> None:
|
||||||
|
"""Extract TCP payload and dispatch to RDP parsers."""
|
||||||
|
if len(pkt) < 54:
|
||||||
|
return
|
||||||
|
|
||||||
|
ethertype = struct.unpack("!H", pkt[12:14])[0]
|
||||||
|
eth_offset = 14
|
||||||
|
if ethertype == 0x8100:
|
||||||
|
if len(pkt) < 58:
|
||||||
|
return
|
||||||
|
ethertype = struct.unpack("!H", pkt[16:18])[0]
|
||||||
|
eth_offset = 18
|
||||||
|
|
||||||
|
if ethertype != 0x0800:
|
||||||
|
return
|
||||||
|
|
||||||
|
ip_header = pkt[eth_offset:]
|
||||||
|
if len(ip_header) < 20:
|
||||||
|
return
|
||||||
|
ihl = (ip_header[0] & 0x0F) * 4
|
||||||
|
ip_proto = ip_header[9]
|
||||||
|
if ip_proto != 6:
|
||||||
|
return
|
||||||
|
|
||||||
|
src_ip = socket.inet_ntoa(ip_header[12:16])
|
||||||
|
dst_ip = socket.inet_ntoa(ip_header[16:20])
|
||||||
|
|
||||||
|
if len(ip_header) < ihl + 20:
|
||||||
|
return
|
||||||
|
src_port, dst_port = struct.unpack("!HH", ip_header[ihl:ihl + 4])
|
||||||
|
tcp_data_off = ((ip_header[ihl + 12] >> 4) & 0xF) * 4
|
||||||
|
payload = ip_header[ihl + tcp_data_off:]
|
||||||
|
|
||||||
|
if len(payload) < 6:
|
||||||
|
return
|
||||||
|
|
||||||
|
# Only process traffic going TO RDP port (client -> server)
|
||||||
|
if dst_port == 3389:
|
||||||
|
self._parse_rdp_client(payload, src_ip, dst_ip, ts)
|
||||||
|
|
||||||
|
def _parse_rdp_client(self, payload: bytes, src_ip: str, dst_ip: str,
|
||||||
|
ts: float) -> None:
|
||||||
|
"""Parse client-to-server RDP traffic."""
|
||||||
|
# Check for TPKT header (version=3, reserved=0)
|
||||||
|
if payload[0] == 0x03 and payload[1] == 0x00:
|
||||||
|
self._parse_tpkt(payload, src_ip, dst_ip, ts)
|
||||||
|
return
|
||||||
|
|
||||||
|
# Check for NTLMSSP in CredSSP
|
||||||
|
if NTLMSSP_SIGNATURE in payload:
|
||||||
|
self._parse_credssp_ntlm(payload, src_ip, dst_ip, ts)
|
||||||
|
|
||||||
|
def _parse_tpkt(self, payload: bytes, src_ip: str, dst_ip: str,
|
||||||
|
ts: float) -> None:
|
||||||
|
"""Parse TPKT + X.224 Connection Request for username cookie."""
|
||||||
|
if len(payload) < 11:
|
||||||
|
return
|
||||||
|
|
||||||
|
# TPKT: version(1) + reserved(1) + length(2)
|
||||||
|
tpkt_len = struct.unpack("!H", payload[2:4])[0]
|
||||||
|
|
||||||
|
# X.224: length indicator(1) + type(1)
|
||||||
|
x224_len = payload[4]
|
||||||
|
x224_type = payload[5] & 0xF0
|
||||||
|
|
||||||
|
if x224_type != X224_CONNECTION_REQUEST:
|
||||||
|
return
|
||||||
|
|
||||||
|
self._stats["connection_requests"] += 1
|
||||||
|
|
||||||
|
# After X.224 header (variable length), look for Cookie field
|
||||||
|
# Cookie format: "Cookie: mstshash=username\r\n"
|
||||||
|
cookie_data = payload[11:] # Skip past basic X.224 header
|
||||||
|
username = ""
|
||||||
|
client_hostname = ""
|
||||||
|
|
||||||
|
# Search for mstshash cookie
|
||||||
|
cookie_idx = cookie_data.find(b'Cookie: mstshash=')
|
||||||
|
if cookie_idx >= 0:
|
||||||
|
cookie_start = cookie_idx + 17 # len("Cookie: mstshash=")
|
||||||
|
cookie_end = cookie_data.find(b'\r\n', cookie_start)
|
||||||
|
if cookie_end > cookie_start:
|
||||||
|
username = cookie_data[cookie_start:cookie_end].decode(
|
||||||
|
"ascii", errors="replace"
|
||||||
|
).strip()
|
||||||
|
|
||||||
|
if username:
|
||||||
|
self._rdp_targets[src_ip].add(dst_ip)
|
||||||
|
self._stats["unique_sessions"] += 1
|
||||||
|
self._check_admin_workstation(src_ip)
|
||||||
|
self._record_session(ts, src_ip, dst_ip, username, "", "", "")
|
||||||
|
|
||||||
|
def _parse_credssp_ntlm(self, payload: bytes, src_ip: str, dst_ip: str,
|
||||||
|
ts: float) -> None:
|
||||||
|
"""Extract NTLM credentials from CredSSP TSRequest."""
|
||||||
|
idx = payload.find(NTLMSSP_SIGNATURE)
|
||||||
|
if idx < 0 or idx + 12 > len(payload):
|
||||||
|
return
|
||||||
|
|
||||||
|
ntlm_data = payload[idx:]
|
||||||
|
msg_type = struct.unpack("<I", ntlm_data[8:12])[0]
|
||||||
|
|
||||||
|
if msg_type != NTLM_AUTHENTICATE or len(ntlm_data) < 72:
|
||||||
|
return
|
||||||
|
|
||||||
|
username, domain, workstation = self._parse_ntlm_type3(ntlm_data)
|
||||||
|
|
||||||
|
if username:
|
||||||
|
self._stats["credssp_auths"] += 1
|
||||||
|
self._rdp_targets[src_ip].add(dst_ip)
|
||||||
|
self._stats["unique_sessions"] += 1
|
||||||
|
self._check_admin_workstation(src_ip)
|
||||||
|
self._record_session(ts, src_ip, dst_ip, username, domain, workstation, "")
|
||||||
|
|
||||||
|
def _parse_ntlm_type3(self, data: bytes) -> tuple:
|
||||||
|
"""Parse NTLM Type 3 Authenticate for username, domain, workstation."""
|
||||||
|
if len(data) < 72:
|
||||||
|
return ("", "", "")
|
||||||
|
|
||||||
|
try:
|
||||||
|
domain_len = struct.unpack("<H", data[28:30])[0]
|
||||||
|
domain_off = struct.unpack("<I", data[32:36])[0]
|
||||||
|
|
||||||
|
user_len = struct.unpack("<H", data[36:38])[0]
|
||||||
|
user_off = struct.unpack("<I", data[40:44])[0]
|
||||||
|
|
||||||
|
ws_len = struct.unpack("<H", data[44:46])[0]
|
||||||
|
ws_off = struct.unpack("<I", data[48:52])[0]
|
||||||
|
|
||||||
|
domain = ""
|
||||||
|
username = ""
|
||||||
|
workstation = ""
|
||||||
|
|
||||||
|
if domain_len and domain_off + domain_len <= len(data):
|
||||||
|
domain = data[domain_off:domain_off + domain_len].decode(
|
||||||
|
"utf-16-le", errors="replace"
|
||||||
|
)
|
||||||
|
if user_len and user_off + user_len <= len(data):
|
||||||
|
username = data[user_off:user_off + user_len].decode(
|
||||||
|
"utf-16-le", errors="replace"
|
||||||
|
)
|
||||||
|
if ws_len and ws_off + ws_len <= len(data):
|
||||||
|
workstation = data[ws_off:ws_off + ws_len].decode(
|
||||||
|
"utf-16-le", errors="replace"
|
||||||
|
)
|
||||||
|
|
||||||
|
return (username, domain, workstation)
|
||||||
|
except Exception:
|
||||||
|
return ("", "", "")
|
||||||
|
|
||||||
|
def _check_admin_workstation(self, src_ip: str) -> None:
|
||||||
|
"""Check if source IP qualifies as admin workstation."""
|
||||||
|
targets = self._rdp_targets.get(src_ip, set())
|
||||||
|
if len(targets) >= ADMIN_DEST_THRESHOLD:
|
||||||
|
current = len([
|
||||||
|
ip for ip, dests in self._rdp_targets.items()
|
||||||
|
if len(dests) >= ADMIN_DEST_THRESHOLD
|
||||||
|
])
|
||||||
|
if current > self._stats["admin_workstations"]:
|
||||||
|
self._stats["admin_workstations"] = current
|
||||||
|
logger.info(
|
||||||
|
"Admin workstation detected: %s (RDP to %d hosts)",
|
||||||
|
src_ip, len(targets)
|
||||||
|
)
|
||||||
|
|
||||||
|
# ------------------------------------------------------------------
|
||||||
|
# Database
|
||||||
|
# ------------------------------------------------------------------
|
||||||
|
|
||||||
|
def _resolve_db_path(self) -> str:
|
||||||
|
base = self.config.get("db_dir", os.path.join(
|
||||||
|
os.path.expanduser("~"), ".bigbrother"
|
||||||
|
))
|
||||||
|
return os.path.join(base, "rdp_monitor.db")
|
||||||
|
|
||||||
|
def _init_db(self) -> None:
|
||||||
|
Path(self._db_path).parent.mkdir(parents=True, exist_ok=True)
|
||||||
|
self._db_conn = sqlite3.connect(self._db_path, check_same_thread=False)
|
||||||
|
self._db_conn.execute("PRAGMA journal_mode=WAL")
|
||||||
|
self._db_conn.execute("PRAGMA synchronous=NORMAL")
|
||||||
|
self._db_conn.executescript(_DB_INIT)
|
||||||
|
self._db_conn.commit()
|
||||||
|
|
||||||
|
def _record_session(self, ts: float, src_ip: str, dst_ip: str,
|
||||||
|
username: str, domain: str, client_hostname: str,
|
||||||
|
keyboard_layout: str) -> None:
|
||||||
|
with self._buffer_lock:
|
||||||
|
self._write_buffer.append((
|
||||||
|
ts, src_ip, dst_ip, username, domain, client_hostname, keyboard_layout
|
||||||
|
))
|
||||||
|
|
||||||
|
def _flush_buffer(self) -> None:
|
||||||
|
with self._buffer_lock:
|
||||||
|
batch = list(self._write_buffer)
|
||||||
|
self._write_buffer.clear()
|
||||||
|
|
||||||
|
if not batch or not self._db_conn:
|
||||||
|
return
|
||||||
|
|
||||||
|
try:
|
||||||
|
with self._db_conn:
|
||||||
|
self._db_conn.executemany(
|
||||||
|
"""INSERT INTO rdp_sessions
|
||||||
|
(timestamp, source_ip, dest_ip, username, domain,
|
||||||
|
client_hostname, keyboard_layout)
|
||||||
|
VALUES (?, ?, ?, ?, ?, ?, ?)""",
|
||||||
|
batch,
|
||||||
|
)
|
||||||
|
except Exception:
|
||||||
|
logger.exception("Failed to flush RDP sessions")
|
||||||
|
|
||||||
|
def _flush_loop(self) -> None:
|
||||||
|
while self._running:
|
||||||
|
time.sleep(FLUSH_INTERVAL)
|
||||||
|
try:
|
||||||
|
self._flush_buffer()
|
||||||
|
except Exception:
|
||||||
|
logger.exception("Flush loop error")
|
||||||
@@ -0,0 +1,394 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
"""SMB file access monitor — passive SMB2/3 session and file tracking.
|
||||||
|
|
||||||
|
Parses SMB2/3 Tree Connect requests (share names), Create requests (file paths),
|
||||||
|
and Read/Write operations. Tracks per-user share access and detects
|
||||||
|
GPP/SYSVOL access patterns indicating potential Group Policy Preference
|
||||||
|
password exposure. SMB3 encrypted sessions are opaque.
|
||||||
|
"""
|
||||||
|
|
||||||
|
import logging
|
||||||
|
import os
|
||||||
|
import socket
|
||||||
|
import sqlite3
|
||||||
|
import struct
|
||||||
|
import threading
|
||||||
|
import time
|
||||||
|
from pathlib import Path
|
||||||
|
from typing import Optional
|
||||||
|
|
||||||
|
from modules.base import BaseModule
|
||||||
|
|
||||||
|
logger = logging.getLogger("bb.passive.smb_monitor")
|
||||||
|
|
||||||
|
# SMB2 command IDs
|
||||||
|
SMB2_NEGOTIATE = 0x0000
|
||||||
|
SMB2_SESSION_SETUP = 0x0001
|
||||||
|
SMB2_TREE_CONNECT = 0x0003
|
||||||
|
SMB2_CREATE = 0x0005
|
||||||
|
SMB2_READ = 0x0008
|
||||||
|
SMB2_WRITE = 0x0009
|
||||||
|
|
||||||
|
# SMB2 header magic
|
||||||
|
SMB2_MAGIC = b'\xfeSMB'
|
||||||
|
|
||||||
|
# GPP-related paths (case-insensitive matching)
|
||||||
|
GPP_PATTERNS = (
|
||||||
|
"groups.xml", "services.xml", "scheduledtasks.xml",
|
||||||
|
"datasources.xml", "printers.xml", "drives.xml",
|
||||||
|
"policies", "sysvol",
|
||||||
|
)
|
||||||
|
|
||||||
|
_DB_INIT = """
|
||||||
|
CREATE TABLE IF NOT EXISTS smb_access (
|
||||||
|
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||||
|
timestamp REAL NOT NULL,
|
||||||
|
source_ip TEXT NOT NULL,
|
||||||
|
username TEXT DEFAULT '',
|
||||||
|
share_name TEXT DEFAULT '',
|
||||||
|
file_path TEXT DEFAULT '',
|
||||||
|
operation TEXT NOT NULL,
|
||||||
|
smb_version TEXT DEFAULT ''
|
||||||
|
);
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_smb_user ON smb_access(username);
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_smb_share ON smb_access(share_name);
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_smb_ts ON smb_access(timestamp);
|
||||||
|
"""
|
||||||
|
|
||||||
|
FLUSH_INTERVAL = 30
|
||||||
|
|
||||||
|
|
||||||
|
class SMBMonitor(BaseModule):
|
||||||
|
"""Monitor SMB2/3 file share access passively."""
|
||||||
|
|
||||||
|
name = "smb_monitor"
|
||||||
|
module_type = "passive"
|
||||||
|
priority = 100
|
||||||
|
requires_root = True
|
||||||
|
|
||||||
|
def __init__(self, bus, state, config, engine=None):
|
||||||
|
super().__init__(bus, state, config, engine)
|
||||||
|
self._capture_bus = None
|
||||||
|
self._sub_queue = None
|
||||||
|
self._read_thread: Optional[threading.Thread] = None
|
||||||
|
self._flush_thread: Optional[threading.Thread] = None
|
||||||
|
self._db_path = self._resolve_db_path()
|
||||||
|
self._db_conn: Optional[sqlite3.Connection] = None
|
||||||
|
self._write_buffer: list[tuple] = []
|
||||||
|
self._buffer_lock = threading.Lock()
|
||||||
|
# Track session ID -> username mapping
|
||||||
|
self._sessions: dict[tuple, str] = {} # (src_ip, session_id) -> username
|
||||||
|
# Track tree ID -> share name mapping
|
||||||
|
self._trees: dict[tuple, str] = {} # (src_ip, tree_id) -> share_name
|
||||||
|
self._stats = {
|
||||||
|
"packets_processed": 0,
|
||||||
|
"tree_connects": 0,
|
||||||
|
"file_creates": 0,
|
||||||
|
"reads": 0,
|
||||||
|
"writes": 0,
|
||||||
|
"gpp_access_detected": 0,
|
||||||
|
}
|
||||||
|
|
||||||
|
# ------------------------------------------------------------------
|
||||||
|
# BaseModule interface
|
||||||
|
# ------------------------------------------------------------------
|
||||||
|
|
||||||
|
def start(self) -> None:
|
||||||
|
if self._running:
|
||||||
|
return
|
||||||
|
self._init_db()
|
||||||
|
self._capture_bus = self.config.get("_capture_bus")
|
||||||
|
if self._capture_bus is None:
|
||||||
|
logger.error("SMBMonitor requires _capture_bus in config")
|
||||||
|
return
|
||||||
|
|
||||||
|
self._sub_queue = self._capture_bus.subscribe(
|
||||||
|
name=self.name, bpf_filter="port 445", queue_depth=5000
|
||||||
|
)
|
||||||
|
|
||||||
|
self._running = True
|
||||||
|
self._start_time = time.time()
|
||||||
|
self._pid = os.getpid()
|
||||||
|
|
||||||
|
self._read_thread = threading.Thread(
|
||||||
|
target=self._read_loop, daemon=True, name="bb-smb-read"
|
||||||
|
)
|
||||||
|
self._read_thread.start()
|
||||||
|
|
||||||
|
self._flush_thread = threading.Thread(
|
||||||
|
target=self._flush_loop, daemon=True, name="bb-smb-flush"
|
||||||
|
)
|
||||||
|
self._flush_thread.start()
|
||||||
|
|
||||||
|
self.state.set_module_status(self.name, "running", pid=self._pid)
|
||||||
|
self.bus.emit("MODULE_STARTED", {"module": self.name}, source_module=self.name)
|
||||||
|
logger.info("SMBMonitor started")
|
||||||
|
|
||||||
|
def stop(self) -> None:
|
||||||
|
if not self._running:
|
||||||
|
return
|
||||||
|
self._running = False
|
||||||
|
if self._capture_bus:
|
||||||
|
self._capture_bus.unsubscribe(self.name)
|
||||||
|
for t in (self._read_thread, self._flush_thread):
|
||||||
|
if t and t.is_alive():
|
||||||
|
t.join(timeout=5.0)
|
||||||
|
self._flush_buffer()
|
||||||
|
if self._db_conn:
|
||||||
|
self._db_conn.close()
|
||||||
|
self._db_conn = None
|
||||||
|
self.state.set_module_status(self.name, "stopped")
|
||||||
|
self.bus.emit("MODULE_STOPPED", {"module": self.name}, source_module=self.name)
|
||||||
|
logger.info("SMBMonitor stopped — stats: %s", self._stats)
|
||||||
|
|
||||||
|
def status(self) -> dict:
|
||||||
|
return {
|
||||||
|
"running": self._running,
|
||||||
|
"pid": self._pid,
|
||||||
|
"uptime": time.time() - self._start_time if self._start_time else 0,
|
||||||
|
**self._stats,
|
||||||
|
}
|
||||||
|
|
||||||
|
def configure(self, config: dict) -> None:
|
||||||
|
self.config.update(config)
|
||||||
|
|
||||||
|
# ------------------------------------------------------------------
|
||||||
|
# Packet processing
|
||||||
|
# ------------------------------------------------------------------
|
||||||
|
|
||||||
|
def _read_loop(self) -> None:
|
||||||
|
while self._running:
|
||||||
|
result = self._sub_queue.get(timeout=1.0)
|
||||||
|
if result is None:
|
||||||
|
continue
|
||||||
|
ts, pkt = result
|
||||||
|
self._stats["packets_processed"] += 1
|
||||||
|
try:
|
||||||
|
self._process_packet(pkt, ts)
|
||||||
|
except Exception:
|
||||||
|
logger.debug("Error processing SMB packet", exc_info=True)
|
||||||
|
|
||||||
|
def _process_packet(self, pkt: bytes, ts: float) -> None:
|
||||||
|
"""Extract TCP payload and parse SMB2."""
|
||||||
|
if len(pkt) < 54: # eth(14) + ip(20) + tcp(20) minimum
|
||||||
|
return
|
||||||
|
|
||||||
|
ethertype = struct.unpack("!H", pkt[12:14])[0]
|
||||||
|
eth_offset = 14
|
||||||
|
if ethertype == 0x8100:
|
||||||
|
if len(pkt) < 58:
|
||||||
|
return
|
||||||
|
ethertype = struct.unpack("!H", pkt[16:18])[0]
|
||||||
|
eth_offset = 18
|
||||||
|
|
||||||
|
if ethertype != 0x0800:
|
||||||
|
return
|
||||||
|
|
||||||
|
ip_header = pkt[eth_offset:]
|
||||||
|
if len(ip_header) < 20:
|
||||||
|
return
|
||||||
|
ihl = (ip_header[0] & 0x0F) * 4
|
||||||
|
ip_proto = ip_header[9]
|
||||||
|
if ip_proto != 6: # TCP only
|
||||||
|
return
|
||||||
|
|
||||||
|
src_ip = socket.inet_ntoa(ip_header[12:16])
|
||||||
|
dst_ip = socket.inet_ntoa(ip_header[16:20])
|
||||||
|
|
||||||
|
if len(ip_header) < ihl + 20:
|
||||||
|
return
|
||||||
|
src_port, dst_port = struct.unpack("!HH", ip_header[ihl:ihl + 4])
|
||||||
|
tcp_data_off = ((ip_header[ihl + 12] >> 4) & 0xF) * 4
|
||||||
|
payload = ip_header[ihl + tcp_data_off:]
|
||||||
|
|
||||||
|
if len(payload) < 4:
|
||||||
|
return
|
||||||
|
|
||||||
|
# SMB uses NetBIOS Session Service: 1-byte type + 3-byte length
|
||||||
|
# Type 0x00 = session message
|
||||||
|
if payload[0] != 0x00:
|
||||||
|
return
|
||||||
|
nb_len = struct.unpack("!I", b'\x00' + payload[1:4])[0]
|
||||||
|
smb_data = payload[4:]
|
||||||
|
|
||||||
|
if len(smb_data) < 64:
|
||||||
|
return
|
||||||
|
|
||||||
|
# Determine direction: request goes TO port 445
|
||||||
|
client_ip = src_ip if dst_port == 445 else dst_ip
|
||||||
|
|
||||||
|
self._parse_smb2(smb_data, client_ip, ts)
|
||||||
|
|
||||||
|
def _parse_smb2(self, data: bytes, client_ip: str, ts: float) -> None:
|
||||||
|
"""Parse an SMB2 message header and dispatch to command handler."""
|
||||||
|
if data[:4] != SMB2_MAGIC:
|
||||||
|
return
|
||||||
|
|
||||||
|
if len(data) < 64:
|
||||||
|
return
|
||||||
|
|
||||||
|
# SMB2 header (64 bytes)
|
||||||
|
header_len = struct.unpack("<H", data[4:6])[0]
|
||||||
|
command = struct.unpack("<H", data[12:14])[0]
|
||||||
|
flags = struct.unpack("<I", data[16:20])[0]
|
||||||
|
is_response = bool(flags & 0x00000001)
|
||||||
|
|
||||||
|
session_id = struct.unpack("<Q", data[40:48])[0]
|
||||||
|
tree_id = struct.unpack("<I", data[36:40])[0]
|
||||||
|
|
||||||
|
smb_body = data[64:]
|
||||||
|
|
||||||
|
if command == SMB2_TREE_CONNECT and is_response and len(smb_body) >= 8:
|
||||||
|
# Tree Connect Response doesn't contain the share name,
|
||||||
|
# but we see the path in the request
|
||||||
|
pass
|
||||||
|
elif command == SMB2_TREE_CONNECT and not is_response:
|
||||||
|
self._handle_tree_connect_request(smb_body, client_ip, session_id, tree_id, ts)
|
||||||
|
elif command == SMB2_CREATE and not is_response:
|
||||||
|
self._handle_create_request(smb_body, client_ip, session_id, tree_id, ts)
|
||||||
|
elif command == SMB2_READ and not is_response:
|
||||||
|
self._stats["reads"] += 1
|
||||||
|
share = self._trees.get((client_ip, tree_id), "")
|
||||||
|
self._record_access(ts, client_ip, "", share, "", "read", "smb2")
|
||||||
|
elif command == SMB2_WRITE and not is_response:
|
||||||
|
self._stats["writes"] += 1
|
||||||
|
share = self._trees.get((client_ip, tree_id), "")
|
||||||
|
self._record_access(ts, client_ip, "", share, "", "write", "smb2")
|
||||||
|
|
||||||
|
def _handle_tree_connect_request(self, body: bytes, client_ip: str,
|
||||||
|
session_id: int, tree_id: int,
|
||||||
|
ts: float) -> None:
|
||||||
|
"""Parse SMB2 Tree Connect request to extract share name."""
|
||||||
|
# Tree Connect Request body (SMB 3.1.1):
|
||||||
|
# StructureSize(2) + Reserved/Flags(2) + PathOffset(2) + PathLength(2)
|
||||||
|
if len(body) < 8:
|
||||||
|
return
|
||||||
|
|
||||||
|
path_offset = struct.unpack("<H", body[4:6])[0]
|
||||||
|
path_length = struct.unpack("<H", body[6:8])[0]
|
||||||
|
|
||||||
|
# PathOffset is relative to the beginning of the SMB2 header
|
||||||
|
# Since body starts at offset 64, adjust
|
||||||
|
# But we receive body starting at offset 0, so the path is at:
|
||||||
|
# (path_offset - 64) in body terms, OR it may follow immediately
|
||||||
|
# In practice, path follows the 8-byte struct at body[8:]
|
||||||
|
if path_length > 0 and len(body) >= 8 + path_length:
|
||||||
|
try:
|
||||||
|
share_path = body[8:8 + path_length].decode("utf-16-le", errors="replace")
|
||||||
|
except Exception:
|
||||||
|
share_path = ""
|
||||||
|
|
||||||
|
# Extract just the share name from \\server\share
|
||||||
|
share_name = share_path.rsplit("\\", 1)[-1] if "\\" in share_path else share_path
|
||||||
|
|
||||||
|
self._trees[(client_ip, tree_id)] = share_name
|
||||||
|
self._stats["tree_connects"] += 1
|
||||||
|
|
||||||
|
username = self._sessions.get((client_ip, session_id), "")
|
||||||
|
self._record_access(ts, client_ip, username, share_name, "", "tree_connect", "smb2")
|
||||||
|
|
||||||
|
# Check for SYSVOL/NETLOGON access
|
||||||
|
if share_name.lower() in ("sysvol", "netlogon"):
|
||||||
|
logger.info("SYSVOL/NETLOGON access from %s — potential GPP exposure", client_ip)
|
||||||
|
|
||||||
|
def _handle_create_request(self, body: bytes, client_ip: str,
|
||||||
|
session_id: int, tree_id: int,
|
||||||
|
ts: float) -> None:
|
||||||
|
"""Parse SMB2 Create request to extract file path."""
|
||||||
|
# Create Request: StructureSize(2) + SecurityFlags(1) + RequestedOplockLevel(1)
|
||||||
|
# + ImpersonationLevel(4) + SmbCreateFlags(8) + Reserved(8)
|
||||||
|
# + DesiredAccess(4) + FileAttributes(4) + ShareAccess(4)
|
||||||
|
# + CreateDisposition(4) + CreateOptions(4) + NameOffset(2) + NameLength(2)
|
||||||
|
if len(body) < 56:
|
||||||
|
return
|
||||||
|
|
||||||
|
name_offset = struct.unpack("<H", body[44:46])[0]
|
||||||
|
name_length = struct.unpack("<H", body[46:48])[0]
|
||||||
|
|
||||||
|
if name_length == 0:
|
||||||
|
return
|
||||||
|
|
||||||
|
# Name follows the fixed-size structure
|
||||||
|
# name_offset is relative to SMB2 header start, body starts at +64
|
||||||
|
name_start = 56 # After the fixed-size fields in Create body
|
||||||
|
if len(body) >= name_start + name_length:
|
||||||
|
try:
|
||||||
|
file_path = body[name_start:name_start + name_length].decode(
|
||||||
|
"utf-16-le", errors="replace"
|
||||||
|
)
|
||||||
|
except Exception:
|
||||||
|
file_path = ""
|
||||||
|
|
||||||
|
if file_path:
|
||||||
|
self._stats["file_creates"] += 1
|
||||||
|
share = self._trees.get((client_ip, tree_id), "")
|
||||||
|
username = self._sessions.get((client_ip, session_id), "")
|
||||||
|
self._record_access(ts, client_ip, username, share, file_path, "create", "smb2")
|
||||||
|
|
||||||
|
# GPP detection
|
||||||
|
file_lower = file_path.lower()
|
||||||
|
for pattern in GPP_PATTERNS:
|
||||||
|
if pattern in file_lower:
|
||||||
|
self._stats["gpp_access_detected"] += 1
|
||||||
|
self.bus.emit("CREDENTIAL_FOUND", {
|
||||||
|
"source": "smb_gpp_access",
|
||||||
|
"client_ip": client_ip,
|
||||||
|
"share": share,
|
||||||
|
"file_path": file_path,
|
||||||
|
"detail": "GPP/SYSVOL file access — may contain cleartext passwords",
|
||||||
|
}, source_module=self.name)
|
||||||
|
break
|
||||||
|
|
||||||
|
# ------------------------------------------------------------------
|
||||||
|
# Database
|
||||||
|
# ------------------------------------------------------------------
|
||||||
|
|
||||||
|
def _resolve_db_path(self) -> str:
|
||||||
|
base = self.config.get("db_dir", os.path.join(
|
||||||
|
os.path.expanduser("~"), ".bigbrother"
|
||||||
|
))
|
||||||
|
return os.path.join(base, "smb_monitor.db")
|
||||||
|
|
||||||
|
def _init_db(self) -> None:
|
||||||
|
Path(self._db_path).parent.mkdir(parents=True, exist_ok=True)
|
||||||
|
self._db_conn = sqlite3.connect(self._db_path, check_same_thread=False)
|
||||||
|
self._db_conn.execute("PRAGMA journal_mode=WAL")
|
||||||
|
self._db_conn.execute("PRAGMA synchronous=NORMAL")
|
||||||
|
self._db_conn.executescript(_DB_INIT)
|
||||||
|
self._db_conn.commit()
|
||||||
|
|
||||||
|
def _record_access(self, ts: float, source_ip: str, username: str,
|
||||||
|
share_name: str, file_path: str, operation: str,
|
||||||
|
smb_version: str) -> None:
|
||||||
|
with self._buffer_lock:
|
||||||
|
self._write_buffer.append((
|
||||||
|
ts, source_ip, username, share_name, file_path, operation, smb_version
|
||||||
|
))
|
||||||
|
|
||||||
|
def _flush_buffer(self) -> None:
|
||||||
|
with self._buffer_lock:
|
||||||
|
batch = list(self._write_buffer)
|
||||||
|
self._write_buffer.clear()
|
||||||
|
|
||||||
|
if not batch or not self._db_conn:
|
||||||
|
return
|
||||||
|
|
||||||
|
try:
|
||||||
|
with self._db_conn:
|
||||||
|
self._db_conn.executemany(
|
||||||
|
"""INSERT INTO smb_access
|
||||||
|
(timestamp, source_ip, username, share_name, file_path,
|
||||||
|
operation, smb_version)
|
||||||
|
VALUES (?, ?, ?, ?, ?, ?, ?)""",
|
||||||
|
batch,
|
||||||
|
)
|
||||||
|
except Exception:
|
||||||
|
logger.exception("Failed to flush SMB access records")
|
||||||
|
|
||||||
|
def _flush_loop(self) -> None:
|
||||||
|
while self._running:
|
||||||
|
time.sleep(FLUSH_INTERVAL)
|
||||||
|
try:
|
||||||
|
self._flush_buffer()
|
||||||
|
except Exception:
|
||||||
|
logger.exception("Flush loop error")
|
||||||
@@ -0,0 +1,431 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
"""VLAN discovery — 802.1Q tag detection, DTP/CDP/LLDP/STP parsing.
|
||||||
|
|
||||||
|
Passively identifies VLAN infrastructure by parsing layer-2 control protocols.
|
||||||
|
Detects 802.1Q tagged frames, DTP trunk negotiation, 802.1X/EAPOL NAC,
|
||||||
|
CDP/LLDP neighbor announcements, and STP BPDUs.
|
||||||
|
"""
|
||||||
|
|
||||||
|
import logging
|
||||||
|
import os
|
||||||
|
import sqlite3
|
||||||
|
import struct
|
||||||
|
import threading
|
||||||
|
import time
|
||||||
|
from pathlib import Path
|
||||||
|
from typing import Optional
|
||||||
|
|
||||||
|
from modules.base import BaseModule
|
||||||
|
|
||||||
|
logger = logging.getLogger("bb.passive.vlan_discovery")
|
||||||
|
|
||||||
|
# Ethertypes and protocol IDs
|
||||||
|
ETHERTYPE_8021Q = 0x8100
|
||||||
|
ETHERTYPE_8021AD = 0x88A8 # QinQ
|
||||||
|
ETHERTYPE_EAPOL = 0x888E
|
||||||
|
ETHERTYPE_LLDP = 0x88CC
|
||||||
|
|
||||||
|
# CDP/DTP use SNAP encapsulation with LLC header (DSAP=DSAP=0xAA, SSAP=0xAA, ctrl=0x03)
|
||||||
|
# CDP OUI: 0x00000C, protocol: 0x2000
|
||||||
|
# DTP OUI: 0x00000C, protocol: 0x2004
|
||||||
|
# STP uses LLC (DSAP=0x42, SSAP=0x42)
|
||||||
|
|
||||||
|
# CDP TLV types
|
||||||
|
CDP_TLV_DEVICE_ID = 0x0001
|
||||||
|
CDP_TLV_ADDRESS = 0x0002
|
||||||
|
CDP_TLV_PORT_ID = 0x0003
|
||||||
|
CDP_TLV_CAPABILITIES = 0x0004
|
||||||
|
CDP_TLV_SOFTWARE_VERSION = 0x0005
|
||||||
|
CDP_TLV_PLATFORM = 0x0006
|
||||||
|
CDP_TLV_NATIVE_VLAN = 0x000A
|
||||||
|
CDP_TLV_MANAGEMENT_ADDR = 0x0016
|
||||||
|
|
||||||
|
# LLDP TLV types
|
||||||
|
LLDP_TLV_END = 0
|
||||||
|
LLDP_TLV_CHASSIS_ID = 1
|
||||||
|
LLDP_TLV_PORT_ID = 2
|
||||||
|
LLDP_TLV_TTL = 3
|
||||||
|
LLDP_TLV_PORT_DESC = 4
|
||||||
|
LLDP_TLV_SYSTEM_NAME = 5
|
||||||
|
LLDP_TLV_SYSTEM_DESC = 6
|
||||||
|
LLDP_TLV_MGMT_ADDR = 8
|
||||||
|
|
||||||
|
_DB_INIT = """
|
||||||
|
CREATE TABLE IF NOT EXISTS vlans (
|
||||||
|
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||||
|
vlan_id INTEGER NOT NULL,
|
||||||
|
name TEXT DEFAULT '',
|
||||||
|
source TEXT NOT NULL,
|
||||||
|
switch_name TEXT DEFAULT '',
|
||||||
|
switch_port TEXT DEFAULT '',
|
||||||
|
first_seen REAL NOT NULL,
|
||||||
|
UNIQUE(vlan_id, source, switch_name, switch_port)
|
||||||
|
);
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_vlans_vlan_id ON vlans(vlan_id);
|
||||||
|
"""
|
||||||
|
|
||||||
|
FLUSH_INTERVAL = 30
|
||||||
|
|
||||||
|
|
||||||
|
class VLANDiscovery(BaseModule):
|
||||||
|
"""Passively discover VLANs via 802.1Q tags and L2 protocol parsing."""
|
||||||
|
|
||||||
|
name = "vlan_discovery"
|
||||||
|
module_type = "passive"
|
||||||
|
priority = 150
|
||||||
|
requires_root = True
|
||||||
|
|
||||||
|
def __init__(self, bus, state, config, engine=None):
|
||||||
|
super().__init__(bus, state, config, engine)
|
||||||
|
self._capture_bus = None
|
||||||
|
self._sub_queue = None
|
||||||
|
self._read_thread: Optional[threading.Thread] = None
|
||||||
|
self._flush_thread: Optional[threading.Thread] = None
|
||||||
|
self._db_path = self._resolve_db_path()
|
||||||
|
self._db_conn: Optional[sqlite3.Connection] = None
|
||||||
|
self._write_buffer: list[tuple] = []
|
||||||
|
self._buffer_lock = threading.Lock()
|
||||||
|
self._stats = {
|
||||||
|
"dot1q_frames": 0,
|
||||||
|
"cdp_frames": 0,
|
||||||
|
"lldp_frames": 0,
|
||||||
|
"dtp_frames": 0,
|
||||||
|
"stp_bpdus": 0,
|
||||||
|
"eapol_frames": 0,
|
||||||
|
"vlans_found": 0,
|
||||||
|
"packets_processed": 0,
|
||||||
|
}
|
||||||
|
|
||||||
|
# ------------------------------------------------------------------
|
||||||
|
# BaseModule interface
|
||||||
|
# ------------------------------------------------------------------
|
||||||
|
|
||||||
|
def start(self) -> None:
|
||||||
|
if self._running:
|
||||||
|
return
|
||||||
|
|
||||||
|
self._init_db()
|
||||||
|
self._capture_bus = self.config.get("_capture_bus")
|
||||||
|
if self._capture_bus is None:
|
||||||
|
logger.error("VLANDiscovery requires _capture_bus in config")
|
||||||
|
return
|
||||||
|
|
||||||
|
# Broad filter: we need 802.1Q (any ethertype), CDP/DTP (SNAP), LLDP, STP, EAPOL
|
||||||
|
# Compile a catch-all because these span multiple ethertypes and LLC frames
|
||||||
|
self._sub_queue = self._capture_bus.subscribe(
|
||||||
|
name=self.name, bpf_filter="", queue_depth=3000
|
||||||
|
)
|
||||||
|
|
||||||
|
self._running = True
|
||||||
|
self._start_time = time.time()
|
||||||
|
self._pid = os.getpid()
|
||||||
|
|
||||||
|
self._read_thread = threading.Thread(
|
||||||
|
target=self._read_loop, daemon=True, name="bb-vlan-read"
|
||||||
|
)
|
||||||
|
self._read_thread.start()
|
||||||
|
|
||||||
|
self._flush_thread = threading.Thread(
|
||||||
|
target=self._flush_loop, daemon=True, name="bb-vlan-flush"
|
||||||
|
)
|
||||||
|
self._flush_thread.start()
|
||||||
|
|
||||||
|
self.state.set_module_status(self.name, "running", pid=self._pid)
|
||||||
|
self.bus.emit("MODULE_STARTED", {"module": self.name}, source_module=self.name)
|
||||||
|
logger.info("VLANDiscovery started")
|
||||||
|
|
||||||
|
def stop(self) -> None:
|
||||||
|
if not self._running:
|
||||||
|
return
|
||||||
|
self._running = False
|
||||||
|
if self._capture_bus and self._sub_queue:
|
||||||
|
self._capture_bus.unsubscribe(self.name)
|
||||||
|
if self._read_thread and self._read_thread.is_alive():
|
||||||
|
self._read_thread.join(timeout=5.0)
|
||||||
|
if self._flush_thread and self._flush_thread.is_alive():
|
||||||
|
self._flush_thread.join(timeout=5.0)
|
||||||
|
self._flush_buffer()
|
||||||
|
if self._db_conn:
|
||||||
|
self._db_conn.close()
|
||||||
|
self._db_conn = None
|
||||||
|
self.state.set_module_status(self.name, "stopped")
|
||||||
|
self.bus.emit("MODULE_STOPPED", {"module": self.name}, source_module=self.name)
|
||||||
|
logger.info("VLANDiscovery stopped — stats: %s", self._stats)
|
||||||
|
|
||||||
|
def status(self) -> dict:
|
||||||
|
return {
|
||||||
|
"running": self._running,
|
||||||
|
"pid": self._pid,
|
||||||
|
"uptime": time.time() - self._start_time if self._start_time else 0,
|
||||||
|
**self._stats,
|
||||||
|
}
|
||||||
|
|
||||||
|
def configure(self, config: dict) -> None:
|
||||||
|
self.config.update(config)
|
||||||
|
|
||||||
|
# ------------------------------------------------------------------
|
||||||
|
# Packet processing
|
||||||
|
# ------------------------------------------------------------------
|
||||||
|
|
||||||
|
def _read_loop(self) -> None:
|
||||||
|
"""Read packets from capture bus and dispatch to parsers."""
|
||||||
|
while self._running:
|
||||||
|
result = self._sub_queue.get(timeout=1.0)
|
||||||
|
if result is None:
|
||||||
|
continue
|
||||||
|
ts, pkt = result
|
||||||
|
self._stats["packets_processed"] += 1
|
||||||
|
try:
|
||||||
|
self._process_frame(pkt, ts)
|
||||||
|
except Exception:
|
||||||
|
logger.debug("Error processing frame", exc_info=True)
|
||||||
|
|
||||||
|
def _process_frame(self, pkt: bytes, ts: float) -> None:
|
||||||
|
"""Identify and dispatch frame to appropriate parser."""
|
||||||
|
if len(pkt) < 14:
|
||||||
|
return
|
||||||
|
|
||||||
|
dst_mac = pkt[0:6]
|
||||||
|
ethertype = struct.unpack("!H", pkt[12:14])[0]
|
||||||
|
|
||||||
|
# 802.1Q tagged frame
|
||||||
|
if ethertype == ETHERTYPE_8021Q or ethertype == ETHERTYPE_8021AD:
|
||||||
|
self._parse_dot1q(pkt, ts)
|
||||||
|
return
|
||||||
|
|
||||||
|
# LLDP
|
||||||
|
if ethertype == ETHERTYPE_LLDP:
|
||||||
|
self._parse_lldp(pkt, ts)
|
||||||
|
return
|
||||||
|
|
||||||
|
# EAPOL (802.1X)
|
||||||
|
if ethertype == ETHERTYPE_EAPOL:
|
||||||
|
self._stats["eapol_frames"] += 1
|
||||||
|
self._record_vlan(0, "eapol_detected", "", "", ts)
|
||||||
|
return
|
||||||
|
|
||||||
|
# Check for LLC/SNAP frames (ethertype field is actually length if < 0x0600)
|
||||||
|
if ethertype <= 0x05DC and len(pkt) >= 22:
|
||||||
|
self._parse_llc_snap(pkt, ts)
|
||||||
|
|
||||||
|
def _parse_dot1q(self, pkt: bytes, ts: float) -> None:
|
||||||
|
"""Parse 802.1Q tagged frame to extract VLAN ID."""
|
||||||
|
if len(pkt) < 18:
|
||||||
|
return
|
||||||
|
tci = struct.unpack("!H", pkt[14:16])[0]
|
||||||
|
vlan_id = tci & 0x0FFF
|
||||||
|
if vlan_id > 0:
|
||||||
|
self._stats["dot1q_frames"] += 1
|
||||||
|
self._record_vlan(vlan_id, "802.1q", "", "", ts)
|
||||||
|
|
||||||
|
def _parse_llc_snap(self, pkt: bytes, ts: float) -> None:
|
||||||
|
"""Parse LLC/SNAP encapsulated frames for CDP, DTP, STP."""
|
||||||
|
# Minimum: 14 (eth) + 3 (LLC) = 17 bytes for STP check
|
||||||
|
if len(pkt) < 17:
|
||||||
|
return
|
||||||
|
|
||||||
|
dsap = pkt[14]
|
||||||
|
ssap = pkt[15]
|
||||||
|
ctrl = pkt[16]
|
||||||
|
|
||||||
|
# STP BPDU: DSAP=0x42, SSAP=0x42
|
||||||
|
if dsap == 0x42 and ssap == 0x42:
|
||||||
|
self._parse_stp(pkt, ts)
|
||||||
|
return
|
||||||
|
|
||||||
|
# SNAP: DSAP=0xAA, SSAP=0xAA, Ctrl=0x03
|
||||||
|
if dsap == 0xAA and ssap == 0xAA and ctrl == 0x03 and len(pkt) >= 22:
|
||||||
|
oui = pkt[17:20]
|
||||||
|
proto = struct.unpack("!H", pkt[20:22])[0]
|
||||||
|
|
||||||
|
# CDP: OUI 0x00000C, protocol 0x2000
|
||||||
|
if oui == b'\x00\x00\x0c' and proto == 0x2000:
|
||||||
|
self._parse_cdp(pkt[22:], ts)
|
||||||
|
return
|
||||||
|
|
||||||
|
# DTP: OUI 0x00000C, protocol 0x2004
|
||||||
|
if oui == b'\x00\x00\x0c' and proto == 0x2004:
|
||||||
|
self._parse_dtp(pkt[22:], ts)
|
||||||
|
return
|
||||||
|
|
||||||
|
def _parse_cdp(self, payload: bytes, ts: float) -> None:
|
||||||
|
"""Parse CDP TLVs to extract switch name, port, VLAN, platform."""
|
||||||
|
if len(payload) < 4:
|
||||||
|
return
|
||||||
|
self._stats["cdp_frames"] += 1
|
||||||
|
|
||||||
|
# CDP header: version(1), TTL(1), checksum(2)
|
||||||
|
offset = 4
|
||||||
|
device_id = ""
|
||||||
|
port_id = ""
|
||||||
|
platform = ""
|
||||||
|
native_vlan = 0
|
||||||
|
|
||||||
|
while offset + 4 <= len(payload):
|
||||||
|
tlv_type = struct.unpack("!H", payload[offset:offset + 2])[0]
|
||||||
|
tlv_len = struct.unpack("!H", payload[offset + 2:offset + 4])[0]
|
||||||
|
if tlv_len < 4 or offset + tlv_len > len(payload):
|
||||||
|
break
|
||||||
|
tlv_data = payload[offset + 4:offset + tlv_len]
|
||||||
|
|
||||||
|
if tlv_type == CDP_TLV_DEVICE_ID:
|
||||||
|
device_id = tlv_data.decode("ascii", errors="replace").strip('\x00')
|
||||||
|
elif tlv_type == CDP_TLV_PORT_ID:
|
||||||
|
port_id = tlv_data.decode("ascii", errors="replace").strip('\x00')
|
||||||
|
elif tlv_type == CDP_TLV_PLATFORM:
|
||||||
|
platform = tlv_data.decode("ascii", errors="replace").strip('\x00')
|
||||||
|
elif tlv_type == CDP_TLV_NATIVE_VLAN and len(tlv_data) >= 2:
|
||||||
|
native_vlan = struct.unpack("!H", tlv_data[:2])[0]
|
||||||
|
|
||||||
|
offset += tlv_len
|
||||||
|
|
||||||
|
if native_vlan > 0:
|
||||||
|
self._record_vlan(native_vlan, "cdp", device_id, port_id, ts)
|
||||||
|
|
||||||
|
if device_id:
|
||||||
|
self.bus.emit("VLAN_DETECTED", {
|
||||||
|
"source": "cdp",
|
||||||
|
"switch_name": device_id,
|
||||||
|
"switch_port": port_id,
|
||||||
|
"platform": platform,
|
||||||
|
"native_vlan": native_vlan,
|
||||||
|
}, source_module=self.name)
|
||||||
|
|
||||||
|
def _parse_dtp(self, payload: bytes, ts: float) -> None:
|
||||||
|
"""Parse DTP frames — Cisco VLAN trunk negotiation."""
|
||||||
|
self._stats["dtp_frames"] += 1
|
||||||
|
# DTP presence alone is noteworthy (trunk negotiation active)
|
||||||
|
self.bus.emit("VLAN_DETECTED", {
|
||||||
|
"source": "dtp",
|
||||||
|
"detail": "DTP trunk negotiation detected — VLAN hopping may be possible",
|
||||||
|
}, source_module=self.name)
|
||||||
|
|
||||||
|
def _parse_lldp(self, pkt: bytes, ts: float) -> None:
|
||||||
|
"""Parse LLDP TLVs for system name, port description, management address."""
|
||||||
|
if len(pkt) < 16:
|
||||||
|
return
|
||||||
|
self._stats["lldp_frames"] += 1
|
||||||
|
|
||||||
|
# LLDP payload starts after Ethernet header (14 bytes)
|
||||||
|
offset = 14
|
||||||
|
system_name = ""
|
||||||
|
port_desc = ""
|
||||||
|
mgmt_addr = ""
|
||||||
|
|
||||||
|
while offset + 2 <= len(pkt):
|
||||||
|
tlv_header = struct.unpack("!H", pkt[offset:offset + 2])[0]
|
||||||
|
tlv_type = (tlv_header >> 9) & 0x7F
|
||||||
|
tlv_len = tlv_header & 0x01FF
|
||||||
|
offset += 2
|
||||||
|
|
||||||
|
if tlv_type == LLDP_TLV_END or offset + tlv_len > len(pkt):
|
||||||
|
break
|
||||||
|
|
||||||
|
tlv_data = pkt[offset:offset + tlv_len]
|
||||||
|
|
||||||
|
if tlv_type == LLDP_TLV_SYSTEM_NAME:
|
||||||
|
system_name = tlv_data.decode("utf-8", errors="replace")
|
||||||
|
elif tlv_type == LLDP_TLV_PORT_DESC:
|
||||||
|
port_desc = tlv_data.decode("utf-8", errors="replace")
|
||||||
|
elif tlv_type == LLDP_TLV_MGMT_ADDR and tlv_len >= 6:
|
||||||
|
addr_len = tlv_data[0]
|
||||||
|
addr_subtype = tlv_data[1]
|
||||||
|
if addr_subtype == 1 and addr_len >= 5: # IPv4
|
||||||
|
addr_bytes = tlv_data[2:6]
|
||||||
|
mgmt_addr = ".".join(str(b) for b in addr_bytes)
|
||||||
|
|
||||||
|
offset += tlv_len
|
||||||
|
|
||||||
|
if system_name:
|
||||||
|
self.bus.emit("VLAN_DETECTED", {
|
||||||
|
"source": "lldp",
|
||||||
|
"system_name": system_name,
|
||||||
|
"port_desc": port_desc,
|
||||||
|
"mgmt_addr": mgmt_addr,
|
||||||
|
}, source_module=self.name)
|
||||||
|
|
||||||
|
def _parse_stp(self, pkt: bytes, ts: float) -> None:
|
||||||
|
"""Parse STP BPDU to extract root bridge ID and priority."""
|
||||||
|
# LLC header at 14, STP BPDU starts at 17
|
||||||
|
if len(pkt) < 38:
|
||||||
|
return
|
||||||
|
self._stats["stp_bpdus"] += 1
|
||||||
|
|
||||||
|
bpdu = pkt[17:]
|
||||||
|
if len(bpdu) < 21:
|
||||||
|
return
|
||||||
|
|
||||||
|
# Protocol ID (2), version (1), type (1)
|
||||||
|
proto_id = struct.unpack("!H", bpdu[0:2])[0]
|
||||||
|
if proto_id != 0x0000:
|
||||||
|
return
|
||||||
|
|
||||||
|
# Flags (1 byte at offset 4)
|
||||||
|
# Root bridge ID: 8 bytes at offset 5 (priority 2 bytes + MAC 6 bytes)
|
||||||
|
if len(bpdu) >= 13:
|
||||||
|
root_priority = struct.unpack("!H", bpdu[5:7])[0]
|
||||||
|
root_mac = ":".join(f"{b:02x}" for b in bpdu[7:13])
|
||||||
|
# Bridge ID: 8 bytes at offset 13
|
||||||
|
bridge_priority = struct.unpack("!H", bpdu[13:15])[0] if len(bpdu) >= 15 else 0
|
||||||
|
|
||||||
|
self.bus.emit("VLAN_DETECTED", {
|
||||||
|
"source": "stp",
|
||||||
|
"root_bridge_mac": root_mac,
|
||||||
|
"root_priority": root_priority,
|
||||||
|
"bridge_priority": bridge_priority,
|
||||||
|
}, source_module=self.name)
|
||||||
|
|
||||||
|
# ------------------------------------------------------------------
|
||||||
|
# Database
|
||||||
|
# ------------------------------------------------------------------
|
||||||
|
|
||||||
|
def _resolve_db_path(self) -> str:
|
||||||
|
base = self.config.get("db_dir", os.path.join(
|
||||||
|
os.path.expanduser("~"), ".bigbrother"
|
||||||
|
))
|
||||||
|
return os.path.join(base, "vlan_discovery.db")
|
||||||
|
|
||||||
|
def _init_db(self) -> None:
|
||||||
|
Path(self._db_path).parent.mkdir(parents=True, exist_ok=True)
|
||||||
|
self._db_conn = sqlite3.connect(self._db_path, check_same_thread=False)
|
||||||
|
self._db_conn.execute("PRAGMA journal_mode=WAL")
|
||||||
|
self._db_conn.execute("PRAGMA synchronous=NORMAL")
|
||||||
|
self._db_conn.executescript(_DB_INIT)
|
||||||
|
self._db_conn.commit()
|
||||||
|
|
||||||
|
def _record_vlan(self, vlan_id: int, source: str, switch_name: str,
|
||||||
|
switch_port: str, ts: float) -> None:
|
||||||
|
"""Buffer a VLAN record for batch insert."""
|
||||||
|
with self._buffer_lock:
|
||||||
|
self._write_buffer.append((vlan_id, "", source, switch_name, switch_port, ts))
|
||||||
|
|
||||||
|
def _flush_buffer(self) -> None:
|
||||||
|
"""Write buffered records to SQLite."""
|
||||||
|
with self._buffer_lock:
|
||||||
|
batch = list(self._write_buffer)
|
||||||
|
self._write_buffer.clear()
|
||||||
|
|
||||||
|
if not batch or not self._db_conn:
|
||||||
|
return
|
||||||
|
|
||||||
|
try:
|
||||||
|
with self._db_conn:
|
||||||
|
self._db_conn.executemany(
|
||||||
|
"""INSERT INTO vlans (vlan_id, name, source, switch_name, switch_port, first_seen)
|
||||||
|
VALUES (?, ?, ?, ?, ?, ?)
|
||||||
|
ON CONFLICT(vlan_id, source, switch_name, switch_port) DO NOTHING""",
|
||||||
|
batch,
|
||||||
|
)
|
||||||
|
new_count = len(batch)
|
||||||
|
self._stats["vlans_found"] += new_count
|
||||||
|
except Exception:
|
||||||
|
logger.exception("Failed to flush VLAN records")
|
||||||
|
|
||||||
|
def _flush_loop(self) -> None:
|
||||||
|
"""Periodically flush write buffer to disk."""
|
||||||
|
while self._running:
|
||||||
|
time.sleep(FLUSH_INTERVAL)
|
||||||
|
try:
|
||||||
|
self._flush_buffer()
|
||||||
|
except Exception:
|
||||||
|
logger.exception("Flush loop error")
|
||||||
Reference in New Issue
Block a user