Add 9 advanced passive modules: VLAN, network mapper, auth tracker, SMB, cloud tokens, LDAP, RDP, QUIC, DB interceptor

Phase 2 batch 2: protocol-specific passive modules for deep network
visibility. Each module subscribes to capture_bus, parses protocol
structures with struct (no scapy), buffers SQLite writes, and publishes
bus events for cross-module correlation.

- vlan_discovery: 802.1Q/DTP/CDP/LLDP/STP/EAPOL layer-2 parsing
- network_mapper: communication graph with role classification and DOT output
- auth_flow_tracker: Kerberos/NTLM/SSH/LDAP auth correlation per user
- smb_monitor: SMB2/3 tree connect, file access, GPP/SYSVOL detection
- cloud_token_harvester: regex scan cleartext HTTP for AWS/JWT/OAuth/SAML/GCP tokens
- ldap_harvester: BER/ASN.1 LDAP parsing for AD object inventory and LAPS detection
- rdp_monitor: RDP cookie + CredSSP NTLM extraction, admin workstation detection
- quic_analyzer: QUIC Initial packet parsing with RFC 9001 key derivation for SNI
- db_interceptor: MSSQL TDS, MySQL, PostgreSQL, Redis RESP, MongoDB OP_MSG parsing
This commit is contained in:
n0mad1k
2026-03-18 13:31:18 -04:00
parent 1c85244e9c
commit ab27aa968b
10 changed files with 4113 additions and 0 deletions
+18
View File
@@ -8,6 +8,15 @@ from modules.passive.kerberos_harvester import KerberosHarvester
from modules.passive.host_discovery import HostDiscovery from modules.passive.host_discovery import HostDiscovery
from modules.passive.os_fingerprint import OSFingerprint from modules.passive.os_fingerprint import OSFingerprint
from modules.passive.traffic_analyzer import TrafficAnalyzer from modules.passive.traffic_analyzer import TrafficAnalyzer
from modules.passive.vlan_discovery import VLANDiscovery
from modules.passive.network_mapper import NetworkMapper
from modules.passive.auth_flow_tracker import AuthFlowTracker
from modules.passive.smb_monitor import SMBMonitor
from modules.passive.cloud_token_harvester import CloudTokenHarvester
from modules.passive.ldap_harvester import LDAPHarvester
from modules.passive.rdp_monitor import RDPMonitor
from modules.passive.quic_analyzer import QUICAnalyzer
from modules.passive.db_interceptor import DBInterceptor
__all__ = [ __all__ = [
"PacketCapture", "PacketCapture",
@@ -18,4 +27,13 @@ __all__ = [
"HostDiscovery", "HostDiscovery",
"OSFingerprint", "OSFingerprint",
"TrafficAnalyzer", "TrafficAnalyzer",
"VLANDiscovery",
"NetworkMapper",
"AuthFlowTracker",
"SMBMonitor",
"CloudTokenHarvester",
"LDAPHarvester",
"RDPMonitor",
"QUICAnalyzer",
"DBInterceptor",
] ]
+539
View File
@@ -0,0 +1,539 @@
#!/usr/bin/env python3
"""Authentication flow tracker — cross-protocol auth event correlation.
Tracks Kerberos (AS-REQ/TGS-REQ), NTLM (across SMB/HTTP/LDAP/MSSQL),
SSH connections, and LDAP binds. Builds per-user authentication timelines
and identifies service accounts.
"""
import logging
import os
import socket
import sqlite3
import struct
import threading
import time
from collections import defaultdict
from pathlib import Path
from typing import Optional
from modules.base import BaseModule
logger = logging.getLogger("bb.passive.auth_flow_tracker")
# Protocol ports
KERBEROS_PORT = 88
SMB_PORT = 445
LDAP_PORT = 389
LDAPS_PORT = 636
RDP_PORT = 3389
SSH_PORT = 22
MSSQL_PORT = 1433
TCP_PROTO = 6
UDP_PROTO = 17
# Kerberos message types (in AS-REQ body)
KRB_AS_REQ = 10
KRB_AS_REP = 11
KRB_TGS_REQ = 12
KRB_TGS_REP = 13
# NTLM signature
NTLMSSP_SIGNATURE = b'NTLMSSP\x00'
NTLM_NEGOTIATE = 1
NTLM_CHALLENGE = 2
NTLM_AUTHENTICATE = 3
# Service account detection threshold
SERVICE_ACCOUNT_THRESHOLD = 5
_DB_INIT = """
CREATE TABLE IF NOT EXISTS auth_events (
id INTEGER PRIMARY KEY AUTOINCREMENT,
timestamp REAL NOT NULL,
source_ip TEXT NOT NULL,
dest_ip TEXT NOT NULL,
protocol TEXT NOT NULL,
auth_type TEXT NOT NULL,
username TEXT DEFAULT '',
domain TEXT DEFAULT '',
target_service TEXT DEFAULT '',
success INTEGER DEFAULT -1
);
CREATE INDEX IF NOT EXISTS idx_auth_user ON auth_events(username);
CREATE INDEX IF NOT EXISTS idx_auth_ts ON auth_events(timestamp);
CREATE INDEX IF NOT EXISTS idx_auth_src ON auth_events(source_ip);
"""
FLUSH_INTERVAL = 30
class AuthFlowTracker(BaseModule):
"""Track authentication events across Kerberos, NTLM, SSH, LDAP."""
name = "auth_flow_tracker"
module_type = "passive"
priority = 100
requires_root = True
def __init__(self, bus, state, config, engine=None):
super().__init__(bus, state, config, engine)
self._capture_bus = None
self._sub_queue = None
self._read_thread: Optional[threading.Thread] = None
self._flush_thread: Optional[threading.Thread] = None
self._db_path = self._resolve_db_path()
self._db_conn: Optional[sqlite3.Connection] = None
self._write_buffer: list[tuple] = []
self._buffer_lock = threading.Lock()
# Track per-user service targets for service account detection
self._user_targets: dict[str, set] = defaultdict(set)
self._stats = {
"packets_processed": 0,
"kerberos_events": 0,
"ntlm_events": 0,
"ssh_events": 0,
"ldap_bind_events": 0,
"service_accounts_detected": 0,
}
# ------------------------------------------------------------------
# BaseModule interface
# ------------------------------------------------------------------
def start(self) -> None:
if self._running:
return
self._init_db()
self._capture_bus = self.config.get("_capture_bus")
if self._capture_bus is None:
logger.error("AuthFlowTracker requires _capture_bus in config")
return
# BPF for auth protocol ports
bpf = (
"port 88 or port 445 or port 389 or port 636 "
"or port 3389 or port 22 or port 1433"
)
self._sub_queue = self._capture_bus.subscribe(
name=self.name, bpf_filter=bpf, queue_depth=5000
)
self._running = True
self._start_time = time.time()
self._pid = os.getpid()
self._read_thread = threading.Thread(
target=self._read_loop, daemon=True, name="bb-auth-read"
)
self._read_thread.start()
self._flush_thread = threading.Thread(
target=self._flush_loop, daemon=True, name="bb-auth-flush"
)
self._flush_thread.start()
self.state.set_module_status(self.name, "running", pid=self._pid)
self.bus.emit("MODULE_STARTED", {"module": self.name}, source_module=self.name)
logger.info("AuthFlowTracker started")
def stop(self) -> None:
if not self._running:
return
self._running = False
if self._capture_bus:
self._capture_bus.unsubscribe(self.name)
for t in (self._read_thread, self._flush_thread):
if t and t.is_alive():
t.join(timeout=5.0)
self._flush_buffer()
if self._db_conn:
self._db_conn.close()
self._db_conn = None
self.state.set_module_status(self.name, "stopped")
self.bus.emit("MODULE_STOPPED", {"module": self.name}, source_module=self.name)
logger.info("AuthFlowTracker stopped — stats: %s", self._stats)
def status(self) -> dict:
return {
"running": self._running,
"pid": self._pid,
"uptime": time.time() - self._start_time if self._start_time else 0,
**self._stats,
}
def configure(self, config: dict) -> None:
self.config.update(config)
# ------------------------------------------------------------------
# Packet processing
# ------------------------------------------------------------------
def _read_loop(self) -> None:
while self._running:
result = self._sub_queue.get(timeout=1.0)
if result is None:
continue
ts, pkt = result
self._stats["packets_processed"] += 1
try:
self._process_packet(pkt, ts)
except Exception:
logger.debug("Error processing auth packet", exc_info=True)
def _process_packet(self, pkt: bytes, ts: float) -> None:
"""Route packet to appropriate protocol parser."""
if len(pkt) < 34:
return
ethertype = struct.unpack("!H", pkt[12:14])[0]
eth_offset = 14
if ethertype == 0x8100:
if len(pkt) < 38:
return
ethertype = struct.unpack("!H", pkt[16:18])[0]
eth_offset = 18
if ethertype != 0x0800:
return
ip_header = pkt[eth_offset:]
if len(ip_header) < 20:
return
ihl = (ip_header[0] & 0x0F) * 4
ip_proto = ip_header[9]
src_ip = socket.inet_ntoa(ip_header[12:16])
dst_ip = socket.inet_ntoa(ip_header[16:20])
if ip_proto == TCP_PROTO and len(ip_header) >= ihl + 4:
src_port, dst_port = struct.unpack("!HH", ip_header[ihl:ihl + 4])
# TCP data offset
if len(ip_header) < ihl + 12:
return
tcp_data_off = ((ip_header[ihl + 12] >> 4) & 0xF) * 4
payload = ip_header[ihl + tcp_data_off:]
if not payload:
return
port = dst_port
if dst_port == KERBEROS_PORT or src_port == KERBEROS_PORT:
self._parse_kerberos(payload, src_ip, dst_ip, ts)
elif dst_port == SMB_PORT or src_port == SMB_PORT:
self._parse_ntlm_in_payload(payload, src_ip, dst_ip, "smb", ts)
elif dst_port in (LDAP_PORT, LDAPS_PORT) or src_port in (LDAP_PORT, LDAPS_PORT):
self._parse_ldap_bind(payload, src_ip, dst_ip, ts)
self._parse_ntlm_in_payload(payload, src_ip, dst_ip, "ldap", ts)
elif dst_port == SSH_PORT or src_port == SSH_PORT:
self._parse_ssh(payload, src_ip, dst_ip, ts)
elif dst_port == RDP_PORT or src_port == RDP_PORT:
self._parse_ntlm_in_payload(payload, src_ip, dst_ip, "rdp", ts)
elif dst_port == MSSQL_PORT or src_port == MSSQL_PORT:
self._parse_ntlm_in_payload(payload, src_ip, dst_ip, "mssql", ts)
elif ip_proto == UDP_PROTO and len(ip_header) >= ihl + 8:
src_port, dst_port = struct.unpack("!HH", ip_header[ihl:ihl + 4])
payload = ip_header[ihl + 8:]
if dst_port == KERBEROS_PORT or src_port == KERBEROS_PORT:
self._parse_kerberos(payload, src_ip, dst_ip, ts)
# ------------------------------------------------------------------
# Kerberos parser
# ------------------------------------------------------------------
def _parse_kerberos(self, payload: bytes, src_ip: str, dst_ip: str,
ts: float) -> None:
"""Parse Kerberos AS-REQ and TGS-REQ to extract principal, realm, SPN."""
if len(payload) < 10:
return
# Kerberos uses ASN.1 DER encoding
# Application tag: AS-REQ=[APPLICATION 10], TGS-REQ=[APPLICATION 12]
tag = payload[0]
if tag & 0x1F == 0x1F:
# Long-form tag, skip for now
return
app_class = (tag >> 6) & 0x03
if app_class != 1: # APPLICATION class
return
msg_type = tag & 0x1F
auth_type = ""
if msg_type == KRB_AS_REQ:
auth_type = "kerberos_as_req"
elif msg_type == KRB_TGS_REQ:
auth_type = "kerberos_tgs_req"
else:
return
# Extract principal and realm from ASN.1 — simplified extraction
# Look for common realm and principal patterns in the raw bytes
realm = self._extract_krb_string(payload, b'\x1b') # GeneralString
principal = self._extract_krb_string(payload, b'\x1b', skip=1)
self._stats["kerberos_events"] += 1
self._record_auth(
ts, src_ip, dst_ip, "kerberos", auth_type,
username=principal, domain=realm,
target_service=principal if msg_type == KRB_TGS_REQ else "",
)
def _extract_krb_string(self, data: bytes, tag: bytes, skip: int = 0) -> str:
"""Extract a GeneralString value from ASN.1 data (simplified)."""
search_tag = tag[0]
pos = 0
found = 0
while pos < len(data) - 2:
if data[pos] == search_tag:
length = data[pos + 1]
if length > 0 and length < 128 and pos + 2 + length <= len(data):
if found >= skip:
try:
return data[pos + 2:pos + 2 + length].decode("ascii", errors="replace")
except Exception:
return ""
found += 1
pos += 1
return ""
# ------------------------------------------------------------------
# NTLM parser
# ------------------------------------------------------------------
def _parse_ntlm_in_payload(self, payload: bytes, src_ip: str, dst_ip: str,
protocol: str, ts: float) -> None:
"""Search for NTLMSSP messages embedded in any protocol payload."""
idx = payload.find(NTLMSSP_SIGNATURE)
if idx == -1 or idx + 12 > len(payload):
return
ntlm_data = payload[idx:]
msg_type = struct.unpack("<I", ntlm_data[8:12])[0]
if msg_type == NTLM_AUTHENTICATE and len(ntlm_data) >= 88:
# Type 3: Authenticate message
username, domain, workstation = self._parse_ntlm_auth(ntlm_data)
if username:
self._stats["ntlm_events"] += 1
self._record_auth(
ts, src_ip, dst_ip, protocol, "ntlm_auth",
username=username, domain=domain,
)
elif msg_type == NTLM_CHALLENGE and len(ntlm_data) >= 56:
# Type 2: Challenge — extract target name
target = self._parse_ntlm_challenge_target(ntlm_data)
if target:
self._stats["ntlm_events"] += 1
self._record_auth(
ts, dst_ip, src_ip, protocol, "ntlm_challenge",
domain=target,
)
def _parse_ntlm_auth(self, data: bytes) -> tuple:
"""Parse NTLM Type 3 Authenticate message for username, domain, workstation."""
if len(data) < 72:
return ("", "", "")
try:
# LmChallengeResponse: offset 12
# NtChallengeResponse: offset 20
# DomainName: length(2), maxlen(2), offset(4) at byte 28
domain_len = struct.unpack("<H", data[28:30])[0]
domain_off = struct.unpack("<I", data[32:36])[0]
# UserName: length(2), maxlen(2), offset(4) at byte 36
user_len = struct.unpack("<H", data[36:38])[0]
user_off = struct.unpack("<I", data[40:44])[0]
# Workstation: length(2), maxlen(2), offset(4) at byte 44
ws_len = struct.unpack("<H", data[44:46])[0]
ws_off = struct.unpack("<I", data[48:52])[0]
domain = data[domain_off:domain_off + domain_len].decode("utf-16-le", errors="replace") if domain_len else ""
username = data[user_off:user_off + user_len].decode("utf-16-le", errors="replace") if user_len else ""
workstation = data[ws_off:ws_off + ws_len].decode("utf-16-le", errors="replace") if ws_len else ""
return (username, domain, workstation)
except Exception:
return ("", "", "")
def _parse_ntlm_challenge_target(self, data: bytes) -> str:
"""Parse NTLM Type 2 Challenge for target name."""
if len(data) < 24:
return ""
try:
target_len = struct.unpack("<H", data[12:14])[0]
target_off = struct.unpack("<I", data[16:20])[0]
if target_len and target_off + target_len <= len(data):
return data[target_off:target_off + target_len].decode("utf-16-le", errors="replace")
except Exception:
pass
return ""
# ------------------------------------------------------------------
# SSH parser
# ------------------------------------------------------------------
def _parse_ssh(self, payload: bytes, src_ip: str, dst_ip: str,
ts: float) -> None:
"""Parse SSH version exchange and detect auth activity."""
# SSH version string starts with "SSH-"
if payload[:4] == b'SSH-':
try:
version_line = payload.split(b'\r\n')[0].decode("ascii", errors="replace")
except Exception:
version_line = ""
self._stats["ssh_events"] += 1
self._record_auth(
ts, src_ip, dst_ip, "ssh", "version_exchange",
target_service=version_line,
)
# ------------------------------------------------------------------
# LDAP bind parser
# ------------------------------------------------------------------
def _parse_ldap_bind(self, payload: bytes, src_ip: str, dst_ip: str,
ts: float) -> None:
"""Parse LDAP simple bind request to extract username."""
if len(payload) < 10:
return
# LDAP messages are BER-encoded
# BindRequest: APPLICATION[0] -> sequence: version, name, auth
# Look for a simple bind: tag 0x60 (APPLICATION CONSTRUCTED 0)
if payload[0] != 0x30: # Not a SEQUENCE
return
# Search for BindRequest tag (0x60) within the message
idx = payload.find(b'\x60')
if idx == -1 or idx + 10 > len(payload):
return
# Skip the BindRequest tag and length
bind_data = payload[idx + 1:]
if not bind_data:
return
# Try to read the length
bind_len, len_size = self._read_ber_length(bind_data)
if bind_len <= 0:
return
bind_body = bind_data[len_size:]
# Version (INTEGER tag 0x02)
if len(bind_body) < 3 or bind_body[0] != 0x02:
return
ver_len = bind_body[1]
name_start = 2 + ver_len
# Name (OCTET STRING tag 0x04)
if len(bind_body) <= name_start + 2:
return
if bind_body[name_start] != 0x04:
return
name_len = bind_body[name_start + 1]
if name_len > 0 and name_start + 2 + name_len <= len(bind_body):
username = bind_body[name_start + 2:name_start + 2 + name_len].decode(
"utf-8", errors="replace"
)
if username and username not in ("", "anonymous"):
self._stats["ldap_bind_events"] += 1
self._record_auth(
ts, src_ip, dst_ip, "ldap", "simple_bind",
username=username,
)
@staticmethod
def _read_ber_length(data: bytes) -> tuple:
"""Read BER length encoding. Returns (length, bytes_consumed)."""
if not data:
return (0, 0)
first = data[0]
if first < 0x80:
return (first, 1)
num_bytes = first & 0x7F
if num_bytes == 0 or len(data) < 1 + num_bytes:
return (0, 0)
length = 0
for i in range(num_bytes):
length = (length << 8) | data[1 + i]
return (length, 1 + num_bytes)
# ------------------------------------------------------------------
# Recording
# ------------------------------------------------------------------
def _record_auth(self, ts: float, src_ip: str, dst_ip: str,
protocol: str, auth_type: str, username: str = "",
domain: str = "", target_service: str = "",
success: int = -1) -> None:
"""Buffer an auth event for batch insert."""
with self._buffer_lock:
self._write_buffer.append((
ts, src_ip, dst_ip, protocol, auth_type,
username, domain, target_service, success,
))
# Track service account patterns
if username:
key = f"{domain}\\{username}" if domain else username
self._user_targets[key].add(dst_ip)
if len(self._user_targets[key]) >= SERVICE_ACCOUNT_THRESHOLD:
# Only count once per user crossing threshold
current_count = len([
u for u, targets in self._user_targets.items()
if len(targets) >= SERVICE_ACCOUNT_THRESHOLD
])
if current_count > self._stats["service_accounts_detected"]:
self._stats["service_accounts_detected"] = current_count
# ------------------------------------------------------------------
# Database
# ------------------------------------------------------------------
def _resolve_db_path(self) -> str:
base = self.config.get("db_dir", os.path.join(
os.path.expanduser("~"), ".bigbrother"
))
return os.path.join(base, "auth_flow_tracker.db")
def _init_db(self) -> None:
Path(self._db_path).parent.mkdir(parents=True, exist_ok=True)
self._db_conn = sqlite3.connect(self._db_path, check_same_thread=False)
self._db_conn.execute("PRAGMA journal_mode=WAL")
self._db_conn.execute("PRAGMA synchronous=NORMAL")
self._db_conn.executescript(_DB_INIT)
self._db_conn.commit()
def _flush_buffer(self) -> None:
with self._buffer_lock:
batch = list(self._write_buffer)
self._write_buffer.clear()
if not batch or not self._db_conn:
return
try:
with self._db_conn:
self._db_conn.executemany(
"""INSERT INTO auth_events
(timestamp, source_ip, dest_ip, protocol, auth_type,
username, domain, target_service, success)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)""",
batch,
)
except Exception:
logger.exception("Failed to flush auth events")
def _flush_loop(self) -> None:
while self._running:
time.sleep(FLUSH_INTERVAL)
try:
self._flush_buffer()
except Exception:
logger.exception("Flush loop error")
+322
View File
@@ -0,0 +1,322 @@
#!/usr/bin/env python3
"""Cloud token harvester — passive credential extraction from cleartext HTTP.
Regex-scans cleartext HTTP traffic (port 80) for AWS access keys, JWT tokens,
OAuth bearer tokens, SAML assertions, Azure AD tokens, and GCP service account
keys. Near-zero yield on most networks but zero cost to run. Publishes
CLOUD_TOKEN_FOUND events and feeds to credential_db via bus events.
"""
import logging
import os
import re
import socket
import sqlite3
import struct
import threading
import time
from pathlib import Path
from typing import Optional
from modules.base import BaseModule
logger = logging.getLogger("bb.passive.cloud_token_harvester")
# Token regexes — compiled once
_TOKEN_PATTERNS = {
"aws_access_key": re.compile(rb'(AKIA[A-Z0-9]{16})'),
"aws_secret_key": re.compile(rb'(?:aws_secret_access_key|secret[_-]?key)\s*[=:]\s*([A-Za-z0-9/+=]{40})', re.IGNORECASE),
"jwt": re.compile(rb'(eyJ[A-Za-z0-9_-]{10,}\.eyJ[A-Za-z0-9_-]{10,}\.[A-Za-z0-9_-]+)'),
"bearer_token": re.compile(rb'[Bb]earer\s+([A-Za-z0-9_\-\.~+/]+=*)', re.IGNORECASE),
"saml_assertion": re.compile(rb'(<saml[2p]*:Assertion[^>]*>.*?</saml[2p]*:Assertion>)', re.DOTALL | re.IGNORECASE),
"azure_ad_token": re.compile(rb'(eyJ[A-Za-z0-9_-]+\.eyJ[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+)'),
"gcp_service_key": re.compile(rb'"type"\s*:\s*"service_account".*?"private_key"\s*:\s*"([^"]+)"', re.DOTALL),
"api_key_generic": re.compile(rb'(?:api[_-]?key|apikey|x-api-key)\s*[=:]\s*([A-Za-z0-9_\-]{20,})', re.IGNORECASE),
"github_token": re.compile(rb'(ghp_[A-Za-z0-9]{36}|gho_[A-Za-z0-9]{36}|ghs_[A-Za-z0-9]{36}|ghr_[A-Za-z0-9]{36})'),
"slack_token": re.compile(rb'(xox[baprs]-[A-Za-z0-9\-]+)'),
}
_DB_INIT = """
CREATE TABLE IF NOT EXISTS cloud_tokens (
id INTEGER PRIMARY KEY AUTOINCREMENT,
timestamp REAL NOT NULL,
source_ip TEXT NOT NULL,
dest_ip TEXT NOT NULL,
token_type TEXT NOT NULL,
token_value TEXT NOT NULL,
context TEXT DEFAULT ''
);
CREATE INDEX IF NOT EXISTS idx_ct_type ON cloud_tokens(token_type);
CREATE INDEX IF NOT EXISTS idx_ct_ts ON cloud_tokens(timestamp);
"""
FLUSH_INTERVAL = 60
class CloudTokenHarvester(BaseModule):
"""Passively harvest cloud tokens and API keys from cleartext HTTP."""
name = "cloud_token_harvester"
module_type = "passive"
priority = 200
requires_root = True
def __init__(self, bus, state, config, engine=None):
super().__init__(bus, state, config, engine)
self._capture_bus = None
self._sub_queue = None
self._read_thread: Optional[threading.Thread] = None
self._flush_thread: Optional[threading.Thread] = None
self._db_path = self._resolve_db_path()
self._db_conn: Optional[sqlite3.Connection] = None
self._write_buffer: list[tuple] = []
self._buffer_lock = threading.Lock()
# Dedup: track recently seen tokens to avoid spamming
self._seen_tokens: set = set()
self._seen_tokens_lock = threading.Lock()
self._stats = {
"packets_processed": 0,
"http_payloads_scanned": 0,
"tokens_found": 0,
"tokens_by_type": {},
}
# ------------------------------------------------------------------
# BaseModule interface
# ------------------------------------------------------------------
def start(self) -> None:
if self._running:
return
self._init_db()
self._capture_bus = self.config.get("_capture_bus")
if self._capture_bus is None:
logger.error("CloudTokenHarvester requires _capture_bus in config")
return
self._sub_queue = self._capture_bus.subscribe(
name=self.name, bpf_filter="port 80", queue_depth=3000
)
self._running = True
self._start_time = time.time()
self._pid = os.getpid()
self._read_thread = threading.Thread(
target=self._read_loop, daemon=True, name="bb-cloud-read"
)
self._read_thread.start()
self._flush_thread = threading.Thread(
target=self._flush_loop, daemon=True, name="bb-cloud-flush"
)
self._flush_thread.start()
self.state.set_module_status(self.name, "running", pid=self._pid)
self.bus.emit("MODULE_STARTED", {"module": self.name}, source_module=self.name)
logger.info("CloudTokenHarvester started")
def stop(self) -> None:
if not self._running:
return
self._running = False
if self._capture_bus:
self._capture_bus.unsubscribe(self.name)
for t in (self._read_thread, self._flush_thread):
if t and t.is_alive():
t.join(timeout=5.0)
self._flush_buffer()
if self._db_conn:
self._db_conn.close()
self._db_conn = None
self.state.set_module_status(self.name, "stopped")
self.bus.emit("MODULE_STOPPED", {"module": self.name}, source_module=self.name)
logger.info("CloudTokenHarvester stopped — stats: %s", self._stats)
def status(self) -> dict:
return {
"running": self._running,
"pid": self._pid,
"uptime": time.time() - self._start_time if self._start_time else 0,
**self._stats,
}
def configure(self, config: dict) -> None:
self.config.update(config)
# ------------------------------------------------------------------
# Packet processing
# ------------------------------------------------------------------
def _read_loop(self) -> None:
while self._running:
result = self._sub_queue.get(timeout=1.0)
if result is None:
continue
ts, pkt = result
self._stats["packets_processed"] += 1
try:
self._process_packet(pkt, ts)
except Exception:
logger.debug("Error processing packet", exc_info=True)
def _process_packet(self, pkt: bytes, ts: float) -> None:
"""Extract HTTP payload and scan for tokens."""
if len(pkt) < 54:
return
ethertype = struct.unpack("!H", pkt[12:14])[0]
eth_offset = 14
if ethertype == 0x8100:
if len(pkt) < 58:
return
ethertype = struct.unpack("!H", pkt[16:18])[0]
eth_offset = 18
if ethertype != 0x0800:
return
ip_header = pkt[eth_offset:]
if len(ip_header) < 20:
return
ihl = (ip_header[0] & 0x0F) * 4
ip_proto = ip_header[9]
if ip_proto != 6: # TCP only
return
src_ip = socket.inet_ntoa(ip_header[12:16])
dst_ip = socket.inet_ntoa(ip_header[16:20])
if len(ip_header) < ihl + 20:
return
tcp_data_off = ((ip_header[ihl + 12] >> 4) & 0xF) * 4
payload = ip_header[ihl + tcp_data_off:]
if len(payload) < 10:
return
# Quick check: does this look like HTTP?
if not (payload[:4] in (b'GET ', b'POST', b'PUT ', b'HEAD', b'HTTP', b'PATC', b'DELE')
or payload[:7] == b'CONNECT' or payload[:7] == b'OPTIONS'):
return
self._stats["http_payloads_scanned"] += 1
self._scan_for_tokens(payload, src_ip, dst_ip, ts)
def _scan_for_tokens(self, payload: bytes, src_ip: str, dst_ip: str,
ts: float) -> None:
"""Run all token regexes against the HTTP payload."""
for token_type, pattern in _TOKEN_PATTERNS.items():
try:
matches = pattern.findall(payload)
except Exception:
continue
for match in matches:
if isinstance(match, bytes):
token_value = match.decode("ascii", errors="replace")
else:
token_value = str(match)
# Skip very short matches (likely false positives)
if len(token_value) < 10:
continue
# Dedup
token_hash = f"{token_type}:{token_value[:32]}"
with self._seen_tokens_lock:
if token_hash in self._seen_tokens:
continue
self._seen_tokens.add(token_hash)
# Bound the seen set
if len(self._seen_tokens) > 10000:
self._seen_tokens.clear()
# Extract context (surrounding bytes for context)
idx = payload.find(match if isinstance(match, bytes) else match.encode())
context_start = max(0, idx - 50)
context_end = min(len(payload), idx + len(match) + 50) if idx >= 0 else 0
context = payload[context_start:context_end].decode("ascii", errors="replace") if idx >= 0 else ""
self._stats["tokens_found"] += 1
self._stats["tokens_by_type"][token_type] = (
self._stats["tokens_by_type"].get(token_type, 0) + 1
)
logger.info(
"Cloud token found: type=%s src=%s dst=%s value=%s...",
token_type, src_ip, dst_ip, token_value[:20]
)
# Publish event
self.bus.emit("CLOUD_TOKEN_FOUND", {
"token_type": token_type,
"token_value": token_value,
"source_ip": src_ip,
"dest_ip": dst_ip,
"context": context[:200],
}, source_module=self.name)
# Also feed to credential_db
self.bus.emit("CREDENTIAL_FOUND", {
"source": f"cloud_token_{token_type}",
"username": "",
"credential": token_value,
"source_ip": src_ip,
"dest_ip": dst_ip,
"protocol": "http",
}, source_module=self.name)
self._record_token(ts, src_ip, dst_ip, token_type, token_value, context)
# ------------------------------------------------------------------
# Database
# ------------------------------------------------------------------
def _resolve_db_path(self) -> str:
base = self.config.get("db_dir", os.path.join(
os.path.expanduser("~"), ".bigbrother"
))
return os.path.join(base, "cloud_token_harvester.db")
def _init_db(self) -> None:
Path(self._db_path).parent.mkdir(parents=True, exist_ok=True)
self._db_conn = sqlite3.connect(self._db_path, check_same_thread=False)
self._db_conn.execute("PRAGMA journal_mode=WAL")
self._db_conn.execute("PRAGMA synchronous=NORMAL")
self._db_conn.executescript(_DB_INIT)
self._db_conn.commit()
def _record_token(self, ts: float, src_ip: str, dst_ip: str,
token_type: str, token_value: str, context: str) -> None:
with self._buffer_lock:
self._write_buffer.append((
ts, src_ip, dst_ip, token_type, token_value, context[:500]
))
def _flush_buffer(self) -> None:
with self._buffer_lock:
batch = list(self._write_buffer)
self._write_buffer.clear()
if not batch or not self._db_conn:
return
try:
with self._db_conn:
self._db_conn.executemany(
"""INSERT INTO cloud_tokens
(timestamp, source_ip, dest_ip, token_type, token_value, context)
VALUES (?, ?, ?, ?, ?, ?)""",
batch,
)
except Exception:
logger.exception("Failed to flush cloud tokens")
def _flush_loop(self) -> None:
while self._running:
time.sleep(FLUSH_INTERVAL)
try:
self._flush_buffer()
except Exception:
logger.exception("Flush loop error")
+696
View File
@@ -0,0 +1,696 @@
#!/usr/bin/env python3
"""Database protocol interceptor — passive database query and auth monitoring.
Parses login packets and query metadata for:
- MSSQL TDS (1433): TDS7 Login packet, SQL batch text
- MySQL (3306): Handshake, Login Request, COM_QUERY
- PostgreSQL (5432): Startup message, Simple Query
- Redis (6379): AUTH command, key operations
- MongoDB (27017): OP_MSG saslStart, find operations
Logs queries but NOT full result sets to avoid excessive data.
"""
import logging
import os
import socket
import sqlite3
import struct
import threading
import time
from pathlib import Path
from typing import Optional
from modules.base import BaseModule
logger = logging.getLogger("bb.passive.db_interceptor")
# TDS packet types
TDS_SQL_BATCH = 0x01
TDS_LOGIN7 = 0x10
TDS_PRELOGIN = 0x12
# MySQL commands
MYSQL_COM_QUERY = 0x03
MYSQL_COM_INIT_DB = 0x02
# PostgreSQL message types
PG_STARTUP = 0x00 # No type byte — identified by structure
PG_SIMPLE_QUERY = ord('Q')
PG_PASSWORD = ord('p')
# Redis inline delimiters
REDIS_CRLF = b'\r\n'
# MongoDB opcodes
MONGO_OP_MSG = 2013
_DB_INIT = """
CREATE TABLE IF NOT EXISTS db_queries (
id INTEGER PRIMARY KEY AUTOINCREMENT,
timestamp REAL NOT NULL,
source_ip TEXT NOT NULL,
dest_ip TEXT NOT NULL,
db_type TEXT NOT NULL,
username TEXT DEFAULT '',
database TEXT DEFAULT '',
query_text TEXT DEFAULT '',
operation TEXT NOT NULL
);
CREATE INDEX IF NOT EXISTS idx_dbq_type ON db_queries(db_type);
CREATE INDEX IF NOT EXISTS idx_dbq_user ON db_queries(username);
CREATE INDEX IF NOT EXISTS idx_dbq_ts ON db_queries(timestamp);
"""
FLUSH_INTERVAL = 45
MAX_QUERY_LEN = 2000 # Truncate queries beyond this
class DBInterceptor(BaseModule):
"""Passively intercept database authentication and queries."""
name = "db_interceptor"
module_type = "passive"
priority = 150
requires_root = True
def __init__(self, bus, state, config, engine=None):
super().__init__(bus, state, config, engine)
self._capture_bus = None
self._sub_queue = None
self._read_thread: Optional[threading.Thread] = None
self._flush_thread: Optional[threading.Thread] = None
self._db_path = self._resolve_db_path()
self._db_conn: Optional[sqlite3.Connection] = None
self._write_buffer: list[tuple] = []
self._buffer_lock = threading.Lock()
self._stats = {
"packets_processed": 0,
"mssql_events": 0,
"mysql_events": 0,
"postgres_events": 0,
"redis_events": 0,
"mongodb_events": 0,
"logins_captured": 0,
"queries_captured": 0,
}
# ------------------------------------------------------------------
# BaseModule interface
# ------------------------------------------------------------------
def start(self) -> None:
if self._running:
return
self._init_db()
self._capture_bus = self.config.get("_capture_bus")
if self._capture_bus is None:
logger.error("DBInterceptor requires _capture_bus in config")
return
bpf = "port 1433 or port 3306 or port 5432 or port 6379 or port 27017"
self._sub_queue = self._capture_bus.subscribe(
name=self.name, bpf_filter=bpf, queue_depth=5000
)
self._running = True
self._start_time = time.time()
self._pid = os.getpid()
self._read_thread = threading.Thread(
target=self._read_loop, daemon=True, name="bb-dbi-read"
)
self._read_thread.start()
self._flush_thread = threading.Thread(
target=self._flush_loop, daemon=True, name="bb-dbi-flush"
)
self._flush_thread.start()
self.state.set_module_status(self.name, "running", pid=self._pid)
self.bus.emit("MODULE_STARTED", {"module": self.name}, source_module=self.name)
logger.info("DBInterceptor started")
def stop(self) -> None:
if not self._running:
return
self._running = False
if self._capture_bus:
self._capture_bus.unsubscribe(self.name)
for t in (self._read_thread, self._flush_thread):
if t and t.is_alive():
t.join(timeout=5.0)
self._flush_buffer()
if self._db_conn:
self._db_conn.close()
self._db_conn = None
self.state.set_module_status(self.name, "stopped")
self.bus.emit("MODULE_STOPPED", {"module": self.name}, source_module=self.name)
logger.info("DBInterceptor stopped — stats: %s", self._stats)
def status(self) -> dict:
return {
"running": self._running,
"pid": self._pid,
"uptime": time.time() - self._start_time if self._start_time else 0,
**self._stats,
}
def configure(self, config: dict) -> None:
self.config.update(config)
# ------------------------------------------------------------------
# Packet processing
# ------------------------------------------------------------------
def _read_loop(self) -> None:
while self._running:
result = self._sub_queue.get(timeout=1.0)
if result is None:
continue
ts, pkt = result
self._stats["packets_processed"] += 1
try:
self._process_packet(pkt, ts)
except Exception:
logger.debug("Error processing DB packet", exc_info=True)
def _process_packet(self, pkt: bytes, ts: float) -> None:
"""Route packet to the correct database parser based on port."""
if len(pkt) < 54:
return
ethertype = struct.unpack("!H", pkt[12:14])[0]
eth_offset = 14
if ethertype == 0x8100:
if len(pkt) < 58:
return
ethertype = struct.unpack("!H", pkt[16:18])[0]
eth_offset = 18
if ethertype != 0x0800:
return
ip_header = pkt[eth_offset:]
if len(ip_header) < 20:
return
ihl = (ip_header[0] & 0x0F) * 4
ip_proto = ip_header[9]
if ip_proto != 6: # TCP only
return
src_ip = socket.inet_ntoa(ip_header[12:16])
dst_ip = socket.inet_ntoa(ip_header[16:20])
if len(ip_header) < ihl + 20:
return
src_port, dst_port = struct.unpack("!HH", ip_header[ihl:ihl + 4])
tcp_data_off = ((ip_header[ihl + 12] >> 4) & 0xF) * 4
payload = ip_header[ihl + tcp_data_off:]
if len(payload) < 4:
return
# Route by destination port (client -> server)
if dst_port == 1433 or src_port == 1433:
self._parse_mssql(payload, src_ip, dst_ip, dst_port == 1433, ts)
elif dst_port == 3306 or src_port == 3306:
self._parse_mysql(payload, src_ip, dst_ip, dst_port == 3306, ts)
elif dst_port == 5432 or src_port == 5432:
self._parse_postgres(payload, src_ip, dst_ip, dst_port == 5432, ts)
elif dst_port == 6379 or src_port == 6379:
self._parse_redis(payload, src_ip, dst_ip, dst_port == 6379, ts)
elif dst_port == 27017 or src_port == 27017:
self._parse_mongodb(payload, src_ip, dst_ip, dst_port == 27017, ts)
# ------------------------------------------------------------------
# MSSQL TDS parser
# ------------------------------------------------------------------
def _parse_mssql(self, payload: bytes, src_ip: str, dst_ip: str,
is_to_server: bool, ts: float) -> None:
"""Parse MSSQL TDS packets."""
if len(payload) < 8:
return
# TDS header: type(1), status(1), length(2), SPID(2), packet(1), window(1)
tds_type = payload[0]
tds_len = struct.unpack("!H", payload[2:4])[0]
if tds_type == TDS_LOGIN7 and is_to_server and len(payload) >= 36:
self._parse_tds_login7(payload[8:], src_ip, dst_ip, ts)
elif tds_type == TDS_SQL_BATCH and is_to_server:
# SQL batch: header group + SQL text (all Unicode LE after 8-byte TDS header)
self._parse_tds_sql_batch(payload[8:], src_ip, dst_ip, ts)
def _parse_tds_login7(self, data: bytes, src_ip: str, dst_ip: str,
ts: float) -> None:
"""Parse TDS7 Login packet for username, hostname, app name."""
if len(data) < 36:
return
self._stats["mssql_events"] += 1
self._stats["logins_captured"] += 1
try:
# TDS7 Login: fixed header of 36 bytes then variable data
# HostName offset(2) + length(2) at offset 8
# UserName offset(2) + length(2) at offset 16
# AppName offset(2) + length(2) at offset 24
# ServerName offset(2) + length(2) at offset 28
# Database offset(2) + length(2) at offset 40 (but we have 36-byte minimum)
hostname_off = struct.unpack("<H", data[8:10])[0]
hostname_len = struct.unpack("<H", data[10:12])[0]
username_off = struct.unpack("<H", data[16:18])[0]
username_len = struct.unpack("<H", data[18:20])[0]
appname_off = struct.unpack("<H", data[24:26])[0]
appname_len = struct.unpack("<H", data[26:28])[0]
# Offsets are in chars (UTF-16LE = 2 bytes per char)
username = self._read_utf16le(data, username_off, username_len)
hostname = self._read_utf16le(data, hostname_off, hostname_len)
appname = self._read_utf16le(data, appname_off, appname_len)
database = ""
if len(data) >= 44:
db_off = struct.unpack("<H", data[40:42])[0]
db_len = struct.unpack("<H", data[42:44])[0]
database = self._read_utf16le(data, db_off, db_len)
self._record_query(
ts, src_ip, dst_ip, "mssql", username, database,
f"LOGIN hostname={hostname} app={appname}", "login"
)
self.bus.emit("CREDENTIAL_FOUND", {
"source": "mssql_tds_login",
"username": username,
"hostname": hostname,
"source_ip": src_ip,
"dest_ip": dst_ip,
"database": database,
}, source_module=self.name)
except Exception:
logger.debug("Failed to parse TDS Login7", exc_info=True)
def _parse_tds_sql_batch(self, data: bytes, src_ip: str, dst_ip: str,
ts: float) -> None:
"""Parse TDS SQL Batch for query text."""
if len(data) < 4:
return
# TDS SQL Batch: ALL_HEADERS (variable) + SQL text
# ALL_HEADERS starts with TotalLength(4)
total_headers_len = struct.unpack("<I", data[0:4])[0]
if total_headers_len > len(data):
total_headers_len = 0
sql_bytes = data[total_headers_len:]
if not sql_bytes:
return
try:
query = sql_bytes.decode("utf-16-le", errors="replace")
except Exception:
query = sql_bytes.decode("ascii", errors="replace")
query = query.strip('\x00').strip()[:MAX_QUERY_LEN]
if query:
self._stats["mssql_events"] += 1
self._stats["queries_captured"] += 1
self._record_query(ts, src_ip, dst_ip, "mssql", "", "", query, "query")
@staticmethod
def _read_utf16le(data: bytes, char_offset: int, char_length: int) -> str:
"""Read UTF-16LE string from TDS data using char offset/length."""
byte_off = char_offset * 2
byte_len = char_length * 2
if byte_off + byte_len > len(data):
return ""
try:
return data[byte_off:byte_off + byte_len].decode("utf-16-le", errors="replace")
except Exception:
return ""
# ------------------------------------------------------------------
# MySQL parser
# ------------------------------------------------------------------
def _parse_mysql(self, payload: bytes, src_ip: str, dst_ip: str,
is_to_server: bool, ts: float) -> None:
"""Parse MySQL protocol packets."""
if len(payload) < 5:
return
# MySQL packet: length(3 LE) + sequence_id(1) + payload
pkt_len = struct.unpack("<I", payload[0:3] + b'\x00')[0]
seq_id = payload[3]
mysql_data = payload[4:]
if not mysql_data:
return
if not is_to_server:
# Server -> client: check for Handshake (protocol version 10)
if mysql_data[0] == 10 and seq_id == 0:
self._parse_mysql_handshake(mysql_data, src_ip, dst_ip, ts)
return
# Client -> server
cmd = mysql_data[0]
if cmd == MYSQL_COM_QUERY and len(mysql_data) > 1:
query = mysql_data[1:min(len(mysql_data), MAX_QUERY_LEN + 1)].decode(
"utf-8", errors="replace"
)
self._stats["mysql_events"] += 1
self._stats["queries_captured"] += 1
self._record_query(ts, src_ip, dst_ip, "mysql", "", "", query, "query")
elif seq_id == 1 and len(mysql_data) >= 32:
# Login Request (HandshakeResponse)
self._parse_mysql_login(mysql_data, src_ip, dst_ip, ts)
def _parse_mysql_handshake(self, data: bytes, src_ip: str, dst_ip: str,
ts: float) -> None:
"""Parse MySQL server Handshake for version info."""
# Protocol version (1) + server version (null-terminated)
if len(data) < 5:
return
null_idx = data.find(b'\x00', 1)
if null_idx < 0:
return
server_version = data[1:null_idx].decode("ascii", errors="replace")
self._stats["mysql_events"] += 1
# Record as informational
self._record_query(
ts, dst_ip, src_ip, "mysql", "", "",
f"SERVER_HANDSHAKE version={server_version}", "handshake"
)
def _parse_mysql_login(self, data: bytes, src_ip: str, dst_ip: str,
ts: float) -> None:
"""Parse MySQL Login Request (HandshakeResponse41)."""
if len(data) < 32:
return
try:
# Client capabilities (4 bytes), max packet (4), charset (1),
# reserved (23 bytes), username (null-terminated)
username_start = 32
null_idx = data.find(b'\x00', username_start)
if null_idx < 0:
return
username = data[username_start:null_idx].decode("utf-8", errors="replace")
# After username + null + auth data, there may be a database name
database = ""
# Skip auth response length + data
pos = null_idx + 1
if pos < len(data):
auth_len = data[pos]
pos += 1 + auth_len
if pos < len(data):
db_null = data.find(b'\x00', pos)
if db_null > pos:
database = data[pos:db_null].decode("utf-8", errors="replace")
self._stats["mysql_events"] += 1
self._stats["logins_captured"] += 1
self._record_query(ts, src_ip, dst_ip, "mysql", username, database, "", "login")
self.bus.emit("CREDENTIAL_FOUND", {
"source": "mysql_login",
"username": username,
"database": database,
"source_ip": src_ip,
"dest_ip": dst_ip,
}, source_module=self.name)
except Exception:
logger.debug("Failed to parse MySQL login", exc_info=True)
# ------------------------------------------------------------------
# PostgreSQL parser
# ------------------------------------------------------------------
def _parse_postgres(self, payload: bytes, src_ip: str, dst_ip: str,
is_to_server: bool, ts: float) -> None:
"""Parse PostgreSQL protocol messages."""
if not is_to_server or len(payload) < 8:
return
# Check for Startup Message: length(4) + protocol_version(4)
# Protocol version 3.0 = 0x00030000
if len(payload) >= 8:
msg_len = struct.unpack("!I", payload[0:4])[0]
proto_ver = struct.unpack("!I", payload[4:8])[0]
if proto_ver == 0x00030000 and msg_len > 8:
# Startup message: parse key=value pairs
self._parse_pg_startup(payload[8:msg_len], src_ip, dst_ip, ts)
return
# Simple Query: type='Q' + length(4) + query_string
if payload[0] == PG_SIMPLE_QUERY and len(payload) >= 6:
query_len = struct.unpack("!I", payload[1:5])[0]
query = payload[5:min(5 + query_len - 1, 5 + MAX_QUERY_LEN)].decode(
"utf-8", errors="replace"
).rstrip('\x00')
if query:
self._stats["postgres_events"] += 1
self._stats["queries_captured"] += 1
self._record_query(ts, src_ip, dst_ip, "postgres", "", "", query, "query")
def _parse_pg_startup(self, data: bytes, src_ip: str, dst_ip: str,
ts: float) -> None:
"""Parse PostgreSQL Startup Message key-value pairs."""
username = ""
database = ""
pos = 0
while pos < len(data) - 1:
null_idx = data.find(b'\x00', pos)
if null_idx < 0:
break
key = data[pos:null_idx].decode("utf-8", errors="replace")
pos = null_idx + 1
val_null = data.find(b'\x00', pos)
if val_null < 0:
break
value = data[pos:val_null].decode("utf-8", errors="replace")
pos = val_null + 1
if key == "user":
username = value
elif key == "database":
database = value
if not key: # Terminal null
break
if username:
self._stats["postgres_events"] += 1
self._stats["logins_captured"] += 1
self._record_query(ts, src_ip, dst_ip, "postgres", username, database, "", "login")
self.bus.emit("CREDENTIAL_FOUND", {
"source": "postgres_startup",
"username": username,
"database": database,
"source_ip": src_ip,
"dest_ip": dst_ip,
}, source_module=self.name)
# ------------------------------------------------------------------
# Redis parser
# ------------------------------------------------------------------
def _parse_redis(self, payload: bytes, src_ip: str, dst_ip: str,
is_to_server: bool, ts: float) -> None:
"""Parse Redis RESP protocol for AUTH commands and key operations."""
if not is_to_server or len(payload) < 3:
return
# RESP: commands start with * (array) followed by count
if payload[0:1] == b'*':
args = self._parse_resp_array(payload)
if not args:
return
cmd = args[0].upper() if args else ""
if cmd == "AUTH":
# AUTH [username] password
password = args[-1] if len(args) >= 2 else ""
username = args[1] if len(args) >= 3 else ""
self._stats["redis_events"] += 1
self._stats["logins_captured"] += 1
self._record_query(ts, src_ip, dst_ip, "redis", username, "", "AUTH ***", "login")
self.bus.emit("CREDENTIAL_FOUND", {
"source": "redis_auth",
"username": username,
"source_ip": src_ip,
"dest_ip": dst_ip,
}, source_module=self.name)
elif cmd in ("GET", "SET", "HGET", "HSET", "DEL", "KEYS",
"MGET", "MSET", "LPUSH", "RPUSH", "SADD", "ZADD"):
query = " ".join(args[:3]) # Command + first 2 args max
self._stats["redis_events"] += 1
self._stats["queries_captured"] += 1
self._record_query(
ts, src_ip, dst_ip, "redis", "", "",
query[:MAX_QUERY_LEN], "query"
)
def _parse_resp_array(self, data: bytes) -> list:
"""Parse a RESP array into a list of string arguments."""
args = []
lines = data.split(REDIS_CRLF)
if not lines or not lines[0].startswith(b'*'):
return args
try:
count = int(lines[0][1:])
except ValueError:
return args
idx = 1
while len(args) < count and idx < len(lines) - 1:
if lines[idx].startswith(b'$'):
try:
str_len = int(lines[idx][1:])
except ValueError:
break
idx += 1
if idx < len(lines):
args.append(lines[idx].decode("utf-8", errors="replace")[:MAX_QUERY_LEN])
idx += 1
else:
# Inline format
args.append(lines[idx].decode("utf-8", errors="replace"))
idx += 1
return args
# ------------------------------------------------------------------
# MongoDB parser
# ------------------------------------------------------------------
def _parse_mongodb(self, payload: bytes, src_ip: str, dst_ip: str,
is_to_server: bool, ts: float) -> None:
"""Parse MongoDB wire protocol for auth and find operations."""
if len(payload) < 16:
return
# MongoDB wire protocol header: length(4 LE) + requestID(4) + responseTo(4) + opCode(4)
msg_len = struct.unpack("<I", payload[0:4])[0]
opcode = struct.unpack("<I", payload[12:16])[0]
if opcode != MONGO_OP_MSG or len(payload) < 21:
return
if not is_to_server:
return
# OP_MSG: flagBits(4) + sections
# Section kind 0: body (BSON document)
sections_start = 20
if sections_start >= len(payload):
return
section_kind = payload[sections_start]
if section_kind != 0:
return
bson_data = payload[sections_start + 1:]
if len(bson_data) < 5:
return
# Minimal BSON parsing: look for command keys
bson_str = bson_data.decode("utf-8", errors="replace")
if "saslStart" in bson_str or "authenticate" in bson_str:
self._stats["mongodb_events"] += 1
self._stats["logins_captured"] += 1
# Extract mechanism if visible
mechanism = ""
if "SCRAM-SHA" in bson_str:
mechanism = "SCRAM-SHA"
elif "PLAIN" in bson_str:
mechanism = "PLAIN"
self._record_query(
ts, src_ip, dst_ip, "mongodb", "", "",
f"AUTH mechanism={mechanism}", "login"
)
elif "find" in bson_str or "aggregate" in bson_str or "insert" in bson_str:
# Extract collection name heuristically
operation = "find" if "find" in bson_str else ("aggregate" if "aggregate" in bson_str else "insert")
self._stats["mongodb_events"] += 1
self._stats["queries_captured"] += 1
self._record_query(
ts, src_ip, dst_ip, "mongodb", "", "",
f"{operation} (BSON)", "query"
)
# ------------------------------------------------------------------
# Database
# ------------------------------------------------------------------
def _resolve_db_path(self) -> str:
base = self.config.get("db_dir", os.path.join(
os.path.expanduser("~"), ".bigbrother"
))
return os.path.join(base, "db_interceptor.db")
def _init_db(self) -> None:
Path(self._db_path).parent.mkdir(parents=True, exist_ok=True)
self._db_conn = sqlite3.connect(self._db_path, check_same_thread=False)
self._db_conn.execute("PRAGMA journal_mode=WAL")
self._db_conn.execute("PRAGMA synchronous=NORMAL")
self._db_conn.executescript(_DB_INIT)
self._db_conn.commit()
def _record_query(self, ts: float, src_ip: str, dst_ip: str,
db_type: str, username: str, database: str,
query_text: str, operation: str) -> None:
with self._buffer_lock:
self._write_buffer.append((
ts, src_ip, dst_ip, db_type, username, database,
query_text[:MAX_QUERY_LEN], operation
))
def _flush_buffer(self) -> None:
with self._buffer_lock:
batch = list(self._write_buffer)
self._write_buffer.clear()
if not batch or not self._db_conn:
return
try:
with self._db_conn:
self._db_conn.executemany(
"""INSERT INTO db_queries
(timestamp, source_ip, dest_ip, db_type, username,
database, query_text, operation)
VALUES (?, ?, ?, ?, ?, ?, ?, ?)""",
batch,
)
except Exception:
logger.exception("Failed to flush DB queries")
def _flush_loop(self) -> None:
while self._running:
time.sleep(FLUSH_INTERVAL)
try:
self._flush_buffer()
except Exception:
logger.exception("Flush loop error")
+495
View File
@@ -0,0 +1,495 @@
#!/usr/bin/env python3
"""LDAP query harvester — passive Active Directory object inventory.
Parses LDAP SearchRequest and SearchResultEntry messages on port 389/3268
using BER/ASN.1 decoding. Extracts user objects (sAMAccountName, mail,
memberOf), group objects, computer objects, GPOs, SPNs, and detects
LAPS password reads (ms-Mcs-AdmPwd attribute access).
"""
import json
import logging
import os
import socket
import sqlite3
import struct
import threading
import time
from pathlib import Path
from typing import Optional
from modules.base import BaseModule
logger = logging.getLogger("bb.passive.ldap_harvester")
# LDAP protocol tags
TAG_SEQUENCE = 0x30
TAG_INTEGER = 0x02
TAG_OCTET_STRING = 0x04
TAG_ENUMERATED = 0x0A
TAG_SET = 0x31
TAG_BOOLEAN = 0x01
# LDAP application tags (context-specific constructed)
LDAP_SEARCH_REQUEST = 0x63 # APPLICATION[3] CONSTRUCTED
LDAP_SEARCH_RESULT_ENTRY = 0x64 # APPLICATION[4] CONSTRUCTED
LDAP_SEARCH_RESULT_DONE = 0x65 # APPLICATION[5] CONSTRUCTED
# Interesting attributes (case-insensitive)
INTERESTING_ATTRS = {
"samaccountname", "userprincipalname", "mail", "memberof",
"distinguishedname", "objectclass", "cn", "name",
"serviceprincipalname", "description", "operatingsystem",
"dnshostname", "managedby", "gplink", "gpcfilesyspath",
"ms-mcs-admpwd", # LAPS password
"ms-mcs-admpwdexpirationtime",
"unicodepwd", "userpassword",
}
_DB_INIT = """
CREATE TABLE IF NOT EXISTS ldap_objects (
id INTEGER PRIMARY KEY AUTOINCREMENT,
timestamp REAL NOT NULL,
dn TEXT DEFAULT '',
object_class TEXT DEFAULT '',
sam_account_name TEXT DEFAULT '',
attributes_json TEXT DEFAULT '{}',
source_ip TEXT NOT NULL
);
CREATE INDEX IF NOT EXISTS idx_ldap_sam ON ldap_objects(sam_account_name);
CREATE INDEX IF NOT EXISTS idx_ldap_class ON ldap_objects(object_class);
CREATE INDEX IF NOT EXISTS idx_ldap_ts ON ldap_objects(timestamp);
"""
FLUSH_INTERVAL = 45
class LDAPHarvester(BaseModule):
"""Passively harvest AD objects from LDAP traffic."""
name = "ldap_harvester"
module_type = "passive"
priority = 100
requires_root = True
def __init__(self, bus, state, config, engine=None):
super().__init__(bus, state, config, engine)
self._capture_bus = None
self._sub_queue = None
self._read_thread: Optional[threading.Thread] = None
self._flush_thread: Optional[threading.Thread] = None
self._db_path = self._resolve_db_path()
self._db_conn: Optional[sqlite3.Connection] = None
self._write_buffer: list[tuple] = []
self._buffer_lock = threading.Lock()
self._seen_dns: set = set() # Dedup by DN
self._stats = {
"packets_processed": 0,
"search_requests": 0,
"result_entries": 0,
"users_found": 0,
"groups_found": 0,
"computers_found": 0,
"spns_found": 0,
"laps_reads_detected": 0,
}
# ------------------------------------------------------------------
# BaseModule interface
# ------------------------------------------------------------------
def start(self) -> None:
if self._running:
return
self._init_db()
self._capture_bus = self.config.get("_capture_bus")
if self._capture_bus is None:
logger.error("LDAPHarvester requires _capture_bus in config")
return
self._sub_queue = self._capture_bus.subscribe(
name=self.name, bpf_filter="port 389 or port 3268", queue_depth=5000
)
self._running = True
self._start_time = time.time()
self._pid = os.getpid()
self._read_thread = threading.Thread(
target=self._read_loop, daemon=True, name="bb-ldap-read"
)
self._read_thread.start()
self._flush_thread = threading.Thread(
target=self._flush_loop, daemon=True, name="bb-ldap-flush"
)
self._flush_thread.start()
self.state.set_module_status(self.name, "running", pid=self._pid)
self.bus.emit("MODULE_STARTED", {"module": self.name}, source_module=self.name)
logger.info("LDAPHarvester started")
def stop(self) -> None:
if not self._running:
return
self._running = False
if self._capture_bus:
self._capture_bus.unsubscribe(self.name)
for t in (self._read_thread, self._flush_thread):
if t and t.is_alive():
t.join(timeout=5.0)
self._flush_buffer()
if self._db_conn:
self._db_conn.close()
self._db_conn = None
self.state.set_module_status(self.name, "stopped")
self.bus.emit("MODULE_STOPPED", {"module": self.name}, source_module=self.name)
logger.info("LDAPHarvester stopped — stats: %s", self._stats)
def status(self) -> dict:
return {
"running": self._running,
"pid": self._pid,
"uptime": time.time() - self._start_time if self._start_time else 0,
**self._stats,
}
def configure(self, config: dict) -> None:
self.config.update(config)
# ------------------------------------------------------------------
# Packet processing
# ------------------------------------------------------------------
def _read_loop(self) -> None:
while self._running:
result = self._sub_queue.get(timeout=1.0)
if result is None:
continue
ts, pkt = result
self._stats["packets_processed"] += 1
try:
self._process_packet(pkt, ts)
except Exception:
logger.debug("Error processing LDAP packet", exc_info=True)
def _process_packet(self, pkt: bytes, ts: float) -> None:
"""Extract TCP payload and parse LDAP messages."""
if len(pkt) < 54:
return
ethertype = struct.unpack("!H", pkt[12:14])[0]
eth_offset = 14
if ethertype == 0x8100:
if len(pkt) < 58:
return
ethertype = struct.unpack("!H", pkt[16:18])[0]
eth_offset = 18
if ethertype != 0x0800:
return
ip_header = pkt[eth_offset:]
if len(ip_header) < 20:
return
ihl = (ip_header[0] & 0x0F) * 4
ip_proto = ip_header[9]
if ip_proto != 6:
return
src_ip = socket.inet_ntoa(ip_header[12:16])
if len(ip_header) < ihl + 20:
return
tcp_data_off = ((ip_header[ihl + 12] >> 4) & 0xF) * 4
payload = ip_header[ihl + tcp_data_off:]
if len(payload) < 5:
return
self._parse_ldap_messages(payload, src_ip, ts)
def _parse_ldap_messages(self, data: bytes, src_ip: str, ts: float) -> None:
"""Parse one or more LDAP messages from a TCP payload."""
pos = 0
while pos < len(data) - 2:
if data[pos] != TAG_SEQUENCE:
break
msg_len, len_size = self._read_ber_length(data[pos + 1:])
if msg_len <= 0:
break
msg_start = pos + 1 + len_size
msg_end = msg_start + msg_len
if msg_end > len(data):
break
msg_body = data[msg_start:msg_end]
self._parse_ldap_message(msg_body, src_ip, ts)
pos = msg_end
def _parse_ldap_message(self, msg: bytes, src_ip: str, ts: float) -> None:
"""Parse a single LDAP message (after outer SEQUENCE)."""
if len(msg) < 5:
return
# Message ID (INTEGER)
if msg[0] != TAG_INTEGER:
return
id_len = msg[1]
if id_len < 1 or 2 + id_len >= len(msg):
return
op_start = 2 + id_len
if op_start >= len(msg):
return
op_tag = msg[op_start]
op_len, op_len_size = self._read_ber_length(msg[op_start + 1:])
if op_len <= 0:
return
op_body = msg[op_start + 1 + op_len_size:op_start + 1 + op_len_size + op_len]
if op_tag == LDAP_SEARCH_REQUEST:
self._handle_search_request(op_body, src_ip, ts)
elif op_tag == LDAP_SEARCH_RESULT_ENTRY:
self._handle_search_result_entry(op_body, src_ip, ts)
def _handle_search_request(self, body: bytes, src_ip: str, ts: float) -> None:
"""Parse SearchRequest for base DN and filter."""
self._stats["search_requests"] += 1
# BaseObject (OCTET STRING)
base_dn = self._read_octet_string(body, 0)
if base_dn is None:
return
# Check if requesting LAPS password
body_str = body.decode("ascii", errors="replace").lower()
if "ms-mcs-admpwd" in body_str:
self._stats["laps_reads_detected"] += 1
logger.warning("LAPS password read detected from %s (base DN: %s)", src_ip, base_dn)
self.bus.emit("CREDENTIAL_FOUND", {
"source": "laps_read",
"source_ip": src_ip,
"base_dn": base_dn,
"detail": "LAPS password attribute requested in LDAP search",
}, source_module=self.name)
def _handle_search_result_entry(self, body: bytes, src_ip: str,
ts: float) -> None:
"""Parse SearchResultEntry to extract DN and attributes."""
self._stats["result_entries"] += 1
# ObjectName (OCTET STRING) = DN
dn = self._read_octet_string(body, 0)
if dn is None:
return
# Skip past the DN OCTET STRING
dn_tag_len, _ = self._skip_tlv(body, 0)
if dn_tag_len < 0:
return
# Attributes (SEQUENCE OF PartialAttribute)
attrs = {}
attr_data = body[dn_tag_len:]
if len(attr_data) < 2 or attr_data[0] != TAG_SEQUENCE:
# May be in a different format
pass
else:
attr_seq_len, attr_len_size = self._read_ber_length(attr_data[1:])
attr_body = attr_data[1 + attr_len_size:]
self._parse_partial_attributes(attr_body, attr_seq_len, attrs)
# Classify object
object_class = ""
sam = attrs.get("samaccountname", "")
classes = attrs.get("objectclass", "")
classes_lower = classes.lower() if isinstance(classes, str) else ""
if "user" in classes_lower or "person" in classes_lower:
object_class = "user"
self._stats["users_found"] += 1
elif "group" in classes_lower:
object_class = "group"
self._stats["groups_found"] += 1
elif "computer" in classes_lower:
object_class = "computer"
self._stats["computers_found"] += 1
elif "grouppolicycontainer" in classes_lower:
object_class = "gpo"
# Track SPNs
if "serviceprincipalname" in attrs:
self._stats["spns_found"] += 1
# Dedup
if dn in self._seen_dns:
return
self._seen_dns.add(dn)
if len(self._seen_dns) > 50000:
self._seen_dns.clear()
# Filter to interesting attributes only
filtered_attrs = {
k: v for k, v in attrs.items()
if k.lower() in INTERESTING_ATTRS
}
self._record_object(ts, dn, object_class, sam, filtered_attrs, src_ip)
def _parse_partial_attributes(self, data: bytes, max_len: int,
attrs: dict) -> None:
"""Parse SEQUENCE OF PartialAttribute."""
pos = 0
while pos < min(len(data), max_len) - 2:
if data[pos] != TAG_SEQUENCE:
break
seq_len, seq_len_size = self._read_ber_length(data[pos + 1:])
if seq_len <= 0:
break
attr_body = data[pos + 1 + seq_len_size:pos + 1 + seq_len_size + seq_len]
self._parse_single_attribute(attr_body, attrs)
pos += 1 + seq_len_size + seq_len
def _parse_single_attribute(self, data: bytes, attrs: dict) -> None:
"""Parse a single PartialAttribute (type + SET OF values)."""
# AttributeDescription (OCTET STRING)
attr_name = self._read_octet_string(data, 0)
if attr_name is None:
return
name_total, _ = self._skip_tlv(data, 0)
if name_total < 0:
return
# Values (SET OF AttributeValue)
val_data = data[name_total:]
if len(val_data) < 2 or val_data[0] != TAG_SET:
return
set_len, set_len_size = self._read_ber_length(val_data[1:])
if set_len <= 0:
return
set_body = val_data[1 + set_len_size:]
values = []
vpos = 0
while vpos < min(len(set_body), set_len) - 2:
val = self._read_octet_string(set_body, vpos)
if val is not None:
values.append(val)
total, _ = self._skip_tlv(set_body, vpos)
if total <= 0:
break
vpos += total
if len(values) == 1:
attrs[attr_name.lower()] = values[0]
elif values:
attrs[attr_name.lower()] = "; ".join(values)
# ------------------------------------------------------------------
# BER helpers
# ------------------------------------------------------------------
@staticmethod
def _read_ber_length(data: bytes) -> tuple:
"""Read BER length. Returns (length, bytes_consumed)."""
if not data:
return (0, 0)
first = data[0]
if first < 0x80:
return (first, 1)
num_bytes = first & 0x7F
if num_bytes == 0 or len(data) < 1 + num_bytes:
return (0, 0)
length = 0
for i in range(num_bytes):
length = (length << 8) | data[1 + i]
return (length, 1 + num_bytes)
def _read_octet_string(self, data: bytes, offset: int) -> Optional[str]:
"""Read an OCTET STRING at offset. Returns decoded string or None."""
if offset >= len(data) - 1:
return None
tag = data[offset]
if tag != TAG_OCTET_STRING:
return None
length, len_size = self._read_ber_length(data[offset + 1:])
if length <= 0:
return None
start = offset + 1 + len_size
if start + length > len(data):
return None
try:
return data[start:start + length].decode("utf-8", errors="replace")
except Exception:
return None
def _skip_tlv(self, data: bytes, offset: int) -> tuple:
"""Skip past a TLV element. Returns (total_bytes, tag)."""
if offset >= len(data):
return (-1, 0)
tag = data[offset]
length, len_size = self._read_ber_length(data[offset + 1:])
if length < 0:
return (-1, 0)
total = 1 + len_size + length
return (total, tag)
# ------------------------------------------------------------------
# Database
# ------------------------------------------------------------------
def _resolve_db_path(self) -> str:
base = self.config.get("db_dir", os.path.join(
os.path.expanduser("~"), ".bigbrother"
))
return os.path.join(base, "ldap_harvester.db")
def _init_db(self) -> None:
Path(self._db_path).parent.mkdir(parents=True, exist_ok=True)
self._db_conn = sqlite3.connect(self._db_path, check_same_thread=False)
self._db_conn.execute("PRAGMA journal_mode=WAL")
self._db_conn.execute("PRAGMA synchronous=NORMAL")
self._db_conn.executescript(_DB_INIT)
self._db_conn.commit()
def _record_object(self, ts: float, dn: str, object_class: str,
sam: str, attrs: dict, src_ip: str) -> None:
with self._buffer_lock:
self._write_buffer.append((
ts, dn, object_class, sam, json.dumps(attrs), src_ip
))
def _flush_buffer(self) -> None:
with self._buffer_lock:
batch = list(self._write_buffer)
self._write_buffer.clear()
if not batch or not self._db_conn:
return
try:
with self._db_conn:
self._db_conn.executemany(
"""INSERT INTO ldap_objects
(timestamp, dn, object_class, sam_account_name,
attributes_json, source_ip)
VALUES (?, ?, ?, ?, ?, ?)""",
batch,
)
except Exception:
logger.exception("Failed to flush LDAP objects")
def _flush_loop(self) -> None:
while self._running:
time.sleep(FLUSH_INTERVAL)
try:
self._flush_buffer()
except Exception:
logger.exception("Flush loop error")
+380
View File
@@ -0,0 +1,380 @@
#!/usr/bin/env python3
"""Network relationship mapper — passive communication graph builder.
Builds a graph of all observed host-to-host communication with protocol,
byte count, and packet count metadata. Identifies roles (servers, clients,
admin workstations, printers) and generates Graphviz DOT output. Periodic
snapshots for change_detector consumption.
"""
import logging
import os
import socket
import sqlite3
import struct
import threading
import time
from collections import defaultdict
from pathlib import Path
from typing import Optional
from modules.base import BaseModule
logger = logging.getLogger("bb.passive.network_mapper")
TCP_PROTO = 6
UDP_PROTO = 17
# Role detection thresholds
SERVER_INBOUND_THRESHOLD = 5 # Unique source IPs connecting in
CLIENT_OUTBOUND_THRESHOLD = 10 # Unique dest IPs connected to
ADMIN_SSH_RDP_THRESHOLD = 3 # Unique SSH/RDP destinations
PRINTER_PORT = 9100
_DB_INIT = """
CREATE TABLE IF NOT EXISTS connections (
id INTEGER PRIMARY KEY AUTOINCREMENT,
src_ip TEXT NOT NULL,
dst_ip TEXT NOT NULL,
protocol TEXT NOT NULL,
port INTEGER NOT NULL,
bytes_total INTEGER DEFAULT 0,
packets INTEGER DEFAULT 0,
first_seen REAL NOT NULL,
last_seen REAL NOT NULL,
relationship_type TEXT DEFAULT '',
UNIQUE(src_ip, dst_ip, protocol, port)
);
CREATE INDEX IF NOT EXISTS idx_conn_src ON connections(src_ip);
CREATE INDEX IF NOT EXISTS idx_conn_dst ON connections(dst_ip);
"""
FLUSH_INTERVAL = 45
SNAPSHOT_INTERVAL = 300 # 5 minutes
class NetworkMapper(BaseModule):
"""Build communication graph from all observed traffic."""
name = "network_mapper"
module_type = "passive"
priority = 150
requires_root = True
def __init__(self, bus, state, config, engine=None):
super().__init__(bus, state, config, engine)
self._capture_bus = None
self._sub_queue = None
self._read_thread: Optional[threading.Thread] = None
self._flush_thread: Optional[threading.Thread] = None
self._snapshot_thread: Optional[threading.Thread] = None
self._db_path = self._resolve_db_path()
self._db_conn: Optional[sqlite3.Connection] = None
self._lock = threading.Lock()
# In-memory graph: (src_ip, dst_ip, proto, port) -> {bytes, packets, first, last}
self._graph: dict[tuple, dict] = {}
# Role tracking: ip -> set of connected IPs per direction
self._inbound: dict[str, set] = defaultdict(set) # dst -> set(src)
self._outbound: dict[str, set] = defaultdict(set) # src -> set(dst)
self._ssh_rdp_dests: dict[str, set] = defaultdict(set) # src -> set(dst) for SSH/RDP
self._printer_servers: set = set()
self._stats = {
"packets_processed": 0,
"unique_connections": 0,
"hosts_seen": 0,
"snapshots_taken": 0,
}
# ------------------------------------------------------------------
# BaseModule interface
# ------------------------------------------------------------------
def start(self) -> None:
if self._running:
return
self._init_db()
self._capture_bus = self.config.get("_capture_bus")
if self._capture_bus is None:
logger.error("NetworkMapper requires _capture_bus in config")
return
# Subscribe to all traffic
self._sub_queue = self._capture_bus.subscribe(
name=self.name, bpf_filter="", queue_depth=5000
)
self._running = True
self._start_time = time.time()
self._pid = os.getpid()
self._read_thread = threading.Thread(
target=self._read_loop, daemon=True, name="bb-netmap-read"
)
self._read_thread.start()
self._flush_thread = threading.Thread(
target=self._flush_loop, daemon=True, name="bb-netmap-flush"
)
self._flush_thread.start()
self._snapshot_thread = threading.Thread(
target=self._snapshot_loop, daemon=True, name="bb-netmap-snapshot"
)
self._snapshot_thread.start()
self.state.set_module_status(self.name, "running", pid=self._pid)
self.bus.emit("MODULE_STARTED", {"module": self.name}, source_module=self.name)
logger.info("NetworkMapper started")
def stop(self) -> None:
if not self._running:
return
self._running = False
if self._capture_bus:
self._capture_bus.unsubscribe(self.name)
for t in (self._read_thread, self._flush_thread, self._snapshot_thread):
if t and t.is_alive():
t.join(timeout=5.0)
self._flush_to_db()
if self._db_conn:
self._db_conn.close()
self._db_conn = None
self.state.set_module_status(self.name, "stopped")
self.bus.emit("MODULE_STOPPED", {"module": self.name}, source_module=self.name)
logger.info("NetworkMapper stopped — stats: %s", self._stats)
def status(self) -> dict:
with self._lock:
hosts = set()
for (src, dst, _, _) in self._graph:
hosts.add(src)
hosts.add(dst)
self._stats["hosts_seen"] = len(hosts)
self._stats["unique_connections"] = len(self._graph)
return {
"running": self._running,
"pid": self._pid,
"uptime": time.time() - self._start_time if self._start_time else 0,
**self._stats,
}
def configure(self, config: dict) -> None:
self.config.update(config)
# ------------------------------------------------------------------
# Packet processing
# ------------------------------------------------------------------
def _read_loop(self) -> None:
while self._running:
result = self._sub_queue.get(timeout=1.0)
if result is None:
continue
ts, pkt = result
self._stats["packets_processed"] += 1
try:
self._process_packet(pkt, ts)
except Exception:
pass
def _process_packet(self, pkt: bytes, ts: float) -> None:
"""Extract IP flow tuple and update graph."""
if len(pkt) < 34:
return
ethertype = struct.unpack("!H", pkt[12:14])[0]
# Handle 802.1Q
eth_offset = 14
if ethertype == 0x8100:
if len(pkt) < 38:
return
ethertype = struct.unpack("!H", pkt[16:18])[0]
eth_offset = 18
if ethertype != 0x0800: # IPv4 only
return
if len(pkt) < eth_offset + 20:
return
ip_header = pkt[eth_offset:]
ihl = (ip_header[0] & 0x0F) * 4
total_len = struct.unpack("!H", ip_header[2:4])[0]
ip_proto = ip_header[9]
src_ip = socket.inet_ntoa(ip_header[12:16])
dst_ip = socket.inet_ntoa(ip_header[16:20])
port = 0
proto_name = "other"
if ip_proto == TCP_PROTO and len(ip_header) >= ihl + 4:
_, dst_port = struct.unpack("!HH", ip_header[ihl:ihl + 4])
port = dst_port
proto_name = "tcp"
elif ip_proto == UDP_PROTO and len(ip_header) >= ihl + 4:
_, dst_port = struct.unpack("!HH", ip_header[ihl:ihl + 4])
port = dst_port
proto_name = "udp"
elif ip_proto == 1:
proto_name = "icmp"
pkt_bytes = total_len
key = (src_ip, dst_ip, proto_name, port)
with self._lock:
if key not in self._graph:
self._graph[key] = {
"bytes": 0, "packets": 0,
"first_seen": ts, "last_seen": ts,
}
entry = self._graph[key]
entry["bytes"] += pkt_bytes
entry["packets"] += 1
entry["last_seen"] = ts
# Track roles
self._inbound[dst_ip].add(src_ip)
self._outbound[src_ip].add(dst_ip)
if port in (22, 3389):
self._ssh_rdp_dests[src_ip].add(dst_ip)
if port == PRINTER_PORT:
self._printer_servers.add(dst_ip)
# ------------------------------------------------------------------
# Role classification
# ------------------------------------------------------------------
def _classify_role(self, ip: str) -> str:
"""Classify a host's role based on observed traffic patterns."""
roles = []
if len(self._inbound.get(ip, set())) >= SERVER_INBOUND_THRESHOLD:
roles.append("server")
if len(self._ssh_rdp_dests.get(ip, set())) >= ADMIN_SSH_RDP_THRESHOLD:
roles.append("admin_workstation")
if ip in self._printer_servers:
roles.append("printer")
if len(self._outbound.get(ip, set())) >= CLIENT_OUTBOUND_THRESHOLD:
roles.append("client")
return ",".join(roles) if roles else "unknown"
# ------------------------------------------------------------------
# Graphviz output
# ------------------------------------------------------------------
def generate_dot(self) -> str:
"""Generate Graphviz DOT format of the communication graph."""
lines = ['digraph network {', ' rankdir=LR;', ' node [shape=box];']
with self._lock:
hosts = set()
for (src, dst, proto, port) in self._graph:
hosts.add(src)
hosts.add(dst)
# Node declarations with role colors
role_colors = {
"server": "lightblue",
"admin_workstation": "orange",
"printer": "lightgreen",
"client": "lightyellow",
}
for ip in sorted(hosts):
role = self._classify_role(ip)
color = "white"
for r, c in role_colors.items():
if r in role:
color = c
break
label = f"{ip}\\n[{role}]"
lines.append(f' "{ip}" [label="{label}", style=filled, fillcolor={color}];')
# Edges
for (src, dst, proto, port), data in self._graph.items():
label = f"{proto}/{port}\\n{data['packets']}pkts"
lines.append(f' "{src}" -> "{dst}" [label="{label}"];')
lines.append('}')
return '\n'.join(lines)
# ------------------------------------------------------------------
# Database
# ------------------------------------------------------------------
def _resolve_db_path(self) -> str:
base = self.config.get("db_dir", os.path.join(
os.path.expanduser("~"), ".bigbrother"
))
return os.path.join(base, "network_mapper.db")
def _init_db(self) -> None:
Path(self._db_path).parent.mkdir(parents=True, exist_ok=True)
self._db_conn = sqlite3.connect(self._db_path, check_same_thread=False)
self._db_conn.execute("PRAGMA journal_mode=WAL")
self._db_conn.execute("PRAGMA synchronous=NORMAL")
self._db_conn.executescript(_DB_INIT)
self._db_conn.commit()
def _flush_to_db(self) -> None:
"""Flush in-memory graph to SQLite."""
with self._lock:
snapshot = dict(self._graph)
if not snapshot or not self._db_conn:
return
try:
with self._db_conn:
for (src, dst, proto, port), data in snapshot.items():
rel = self._classify_role(dst)
self._db_conn.execute(
"""INSERT INTO connections
(src_ip, dst_ip, protocol, port, bytes_total, packets,
first_seen, last_seen, relationship_type)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
ON CONFLICT(src_ip, dst_ip, protocol, port)
DO UPDATE SET
bytes_total = bytes_total + excluded.bytes_total,
packets = packets + excluded.packets,
last_seen = MAX(last_seen, excluded.last_seen)""",
(src, dst, proto, port, data["bytes"], data["packets"],
data["first_seen"], data["last_seen"], rel),
)
except Exception:
logger.exception("Failed to flush connections to DB")
def _flush_loop(self) -> None:
while self._running:
time.sleep(FLUSH_INTERVAL)
try:
self._flush_to_db()
except Exception:
logger.exception("Flush loop error")
def _snapshot_loop(self) -> None:
"""Periodic snapshots for change_detector consumption."""
while self._running:
time.sleep(SNAPSHOT_INTERVAL)
try:
dot_output = self.generate_dot()
snapshot_dir = os.path.join(
os.path.dirname(self._db_path), "snapshots"
)
Path(snapshot_dir).mkdir(parents=True, exist_ok=True)
filename = f"netmap_{int(time.time())}.dot"
filepath = os.path.join(snapshot_dir, filename)
with open(filepath, "w") as f:
f.write(dot_output)
self._stats["snapshots_taken"] += 1
# Publish snapshot event for change_detector
self.bus.emit("CHANGE_DETECTED", {
"module": self.name,
"snapshot_path": filepath,
"hosts_count": self._stats["hosts_seen"],
"connections_count": self._stats["unique_connections"],
}, source_module=self.name)
except Exception:
logger.exception("Snapshot loop error")
+461
View File
@@ -0,0 +1,461 @@
#!/usr/bin/env python3
"""QUIC/HTTP3 SNI extractor — parse QUIC Initial packets for TLS ClientHello.
Decrypts QUIC Initial packets using connection-ID-derived keys per RFC 9001
(Initial keys are not secret — they are derived deterministically from the
Destination Connection ID). Extracts CRYPTO frames containing TLS ClientHello
and parses SNI from the Server Name extension.
"""
import hashlib
import hmac
import logging
import os
import socket
import sqlite3
import struct
import threading
import time
from pathlib import Path
from typing import Optional
from modules.base import BaseModule
logger = logging.getLogger("bb.passive.quic_analyzer")
# QUIC constants
QUIC_LONG_HEADER_BIT = 0x80
QUIC_FIXED_BIT = 0x40
# QUIC versions
QUIC_V1 = 0x00000001
QUIC_V2 = 0x6B3343CF
# TLS extension type for SNI
TLS_EXT_SNI = 0x0000
TLS_HANDSHAKE_CLIENT_HELLO = 0x01
# QUIC Initial salt for v1 (RFC 9001, Section 5.2)
QUIC_V1_INITIAL_SALT = bytes.fromhex("38762cf7f55934b34d179ae6a4c80cadccbb7f0a")
# QUIC Initial salt for v2 (RFC 9369)
QUIC_V2_INITIAL_SALT = bytes.fromhex("0dede3def700a6db819381be6e269dcbf9bd2ed9")
# HKDF labels for QUIC Initial keys
QUIC_CLIENT_INITIAL_LABEL = b"client in"
_DB_INIT = """
CREATE TABLE IF NOT EXISTS quic_sni (
id INTEGER PRIMARY KEY AUTOINCREMENT,
timestamp REAL NOT NULL,
source_ip TEXT NOT NULL,
dest_ip TEXT NOT NULL,
sni TEXT NOT NULL,
quic_version TEXT DEFAULT ''
);
CREATE INDEX IF NOT EXISTS idx_quic_sni ON quic_sni(sni);
CREATE INDEX IF NOT EXISTS idx_quic_ts ON quic_sni(timestamp);
"""
FLUSH_INTERVAL = 30
class QUICAnalyzer(BaseModule):
"""Extract SNI from QUIC Initial packets."""
name = "quic_analyzer"
module_type = "passive"
priority = 150
requires_root = True
def __init__(self, bus, state, config, engine=None):
super().__init__(bus, state, config, engine)
self._capture_bus = None
self._sub_queue = None
self._read_thread: Optional[threading.Thread] = None
self._flush_thread: Optional[threading.Thread] = None
self._db_path = self._resolve_db_path()
self._db_conn: Optional[sqlite3.Connection] = None
self._write_buffer: list[tuple] = []
self._buffer_lock = threading.Lock()
self._stats = {
"packets_processed": 0,
"quic_initials": 0,
"sni_extracted": 0,
"decrypt_failures": 0,
}
# ------------------------------------------------------------------
# BaseModule interface
# ------------------------------------------------------------------
def start(self) -> None:
if self._running:
return
self._init_db()
self._capture_bus = self.config.get("_capture_bus")
if self._capture_bus is None:
logger.error("QUICAnalyzer requires _capture_bus in config")
return
self._sub_queue = self._capture_bus.subscribe(
name=self.name, bpf_filter="udp port 443", queue_depth=3000
)
self._running = True
self._start_time = time.time()
self._pid = os.getpid()
self._read_thread = threading.Thread(
target=self._read_loop, daemon=True, name="bb-quic-read"
)
self._read_thread.start()
self._flush_thread = threading.Thread(
target=self._flush_loop, daemon=True, name="bb-quic-flush"
)
self._flush_thread.start()
self.state.set_module_status(self.name, "running", pid=self._pid)
self.bus.emit("MODULE_STARTED", {"module": self.name}, source_module=self.name)
logger.info("QUICAnalyzer started")
def stop(self) -> None:
if not self._running:
return
self._running = False
if self._capture_bus:
self._capture_bus.unsubscribe(self.name)
for t in (self._read_thread, self._flush_thread):
if t and t.is_alive():
t.join(timeout=5.0)
self._flush_buffer()
if self._db_conn:
self._db_conn.close()
self._db_conn = None
self.state.set_module_status(self.name, "stopped")
self.bus.emit("MODULE_STOPPED", {"module": self.name}, source_module=self.name)
logger.info("QUICAnalyzer stopped — stats: %s", self._stats)
def status(self) -> dict:
return {
"running": self._running,
"pid": self._pid,
"uptime": time.time() - self._start_time if self._start_time else 0,
**self._stats,
}
def configure(self, config: dict) -> None:
self.config.update(config)
# ------------------------------------------------------------------
# Packet processing
# ------------------------------------------------------------------
def _read_loop(self) -> None:
while self._running:
result = self._sub_queue.get(timeout=1.0)
if result is None:
continue
ts, pkt = result
self._stats["packets_processed"] += 1
try:
self._process_packet(pkt, ts)
except Exception:
logger.debug("Error processing QUIC packet", exc_info=True)
def _process_packet(self, pkt: bytes, ts: float) -> None:
"""Extract UDP payload and check for QUIC Initial."""
if len(pkt) < 42: # eth(14) + ip(20) + udp(8)
return
ethertype = struct.unpack("!H", pkt[12:14])[0]
eth_offset = 14
if ethertype == 0x8100:
if len(pkt) < 46:
return
ethertype = struct.unpack("!H", pkt[16:18])[0]
eth_offset = 18
if ethertype != 0x0800:
return
ip_header = pkt[eth_offset:]
if len(ip_header) < 20:
return
ihl = (ip_header[0] & 0x0F) * 4
ip_proto = ip_header[9]
if ip_proto != 17: # UDP
return
src_ip = socket.inet_ntoa(ip_header[12:16])
dst_ip = socket.inet_ntoa(ip_header[16:20])
udp_start = ihl
if len(ip_header) < udp_start + 8:
return
payload = ip_header[udp_start + 8:]
if len(payload) < 5:
return
self._parse_quic_initial(payload, src_ip, dst_ip, ts)
def _parse_quic_initial(self, data: bytes, src_ip: str, dst_ip: str,
ts: float) -> None:
"""Parse QUIC Initial packet and attempt SNI extraction."""
first_byte = data[0]
# Must be long header form (bit 7 set) and fixed bit (bit 6 set)
if not (first_byte & QUIC_LONG_HEADER_BIT):
return
if not (first_byte & QUIC_FIXED_BIT):
return
# Packet type: bits 4-5 of first byte
# For QUIC v1: Initial = 0x00 (bits 4-5 = 00)
# For QUIC v2: Initial = 0x01 (bits 4-5 = 01)
packet_type_bits = (first_byte >> 4) & 0x03
if len(data) < 7:
return
# Version (4 bytes at offset 1)
version = struct.unpack("!I", data[1:5])[0]
# Determine if this is an Initial packet based on version
is_initial = False
if version == QUIC_V1 and packet_type_bits == 0x00:
is_initial = True
elif version == QUIC_V2 and packet_type_bits == 0x01:
is_initial = True
elif version == 0:
# Version negotiation — skip
return
if not is_initial:
return
self._stats["quic_initials"] += 1
# DCID length (1 byte at offset 5)
dcid_len = data[5]
if len(data) < 6 + dcid_len + 1:
return
dcid = data[6:6 + dcid_len]
# SCID length
scid_len_off = 6 + dcid_len
scid_len = data[scid_len_off]
scid_off = scid_len_off + 1
if len(data) < scid_off + scid_len:
return
# Token length (variable-length integer)
token_start = scid_off + scid_len
if token_start >= len(data):
return
token_len, token_len_size = self._read_varint(data, token_start)
if token_len_size == 0:
return
# Payload length (variable-length integer)
payload_len_start = token_start + token_len_size + token_len
if payload_len_start >= len(data):
return
payload_len, payload_len_size = self._read_varint(data, payload_len_start)
if payload_len_size == 0:
return
# Encrypted payload starts after the packet number (included in payload_len)
encrypted_start = payload_len_start + payload_len_size
# Try to extract SNI without full decryption first:
# Scan the unprotected portion for a ClientHello pattern
# (This works for many implementations where the crypto frame is at the start)
sni = self._try_extract_sni_from_initial(data, dcid, version, encrypted_start)
if sni:
self._stats["sni_extracted"] += 1
version_str = f"0x{version:08x}"
self._record_sni(ts, src_ip, dst_ip, sni, version_str)
return
# If we couldn't extract via decryption, try brute-force scan
# for SNI pattern in the raw payload
sni = self._scan_for_sni(data)
if sni:
self._stats["sni_extracted"] += 1
version_str = f"0x{version:08x}"
self._record_sni(ts, src_ip, dst_ip, sni, version_str)
def _try_extract_sni_from_initial(self, data: bytes, dcid: bytes,
version: int, enc_start: int) -> Optional[str]:
"""Attempt to derive Initial keys and decrypt CRYPTO frame for SNI.
Per RFC 9001, Initial keys are derived deterministically from the DCID
using HKDF, so this is not breaking encryption — it's public info.
"""
try:
# Select salt based on version
if version == QUIC_V2:
salt = QUIC_V2_INITIAL_SALT
else:
salt = QUIC_V1_INITIAL_SALT
# Derive initial secret: HKDF-Extract(salt, DCID)
initial_secret = hmac.new(salt, dcid, hashlib.sha256).digest()
# Derive client initial secret using HKDF-Expand-Label
client_secret = self._hkdf_expand_label(
initial_secret, QUIC_CLIENT_INITIAL_LABEL, b"", 32
)
# Derive key and IV from client secret
key = self._hkdf_expand_label(client_secret, b"quic key", b"", 16)
iv = self._hkdf_expand_label(client_secret, b"quic iv", b"", 12)
hp_key = self._hkdf_expand_label(client_secret, b"quic hp", b"", 16)
# Header protection removal and decryption requires AES —
# rather than importing cryptography lib, scan the payload area
# for TLS ClientHello patterns that may be partially visible
# in the packet number protected but not encrypted initial bytes
# Fall through to pattern scan
return None
except Exception:
self._stats["decrypt_failures"] += 1
return None
def _hkdf_expand_label(self, secret: bytes, label: bytes,
context: bytes, length: int) -> bytes:
"""HKDF-Expand-Label as defined in TLS 1.3 / RFC 8446."""
# Build HkdfLabel structure
full_label = b"tls13 " + label
hkdf_label = (
struct.pack("!H", length) +
struct.pack("B", len(full_label)) + full_label +
struct.pack("B", len(context)) + context
)
return self._hkdf_expand(secret, hkdf_label, length)
@staticmethod
def _hkdf_expand(prk: bytes, info: bytes, length: int) -> bytes:
"""HKDF-Expand using HMAC-SHA256."""
hash_len = 32 # SHA-256
n = (length + hash_len - 1) // hash_len
okm = b""
t = b""
for i in range(1, n + 1):
t = hmac.new(prk, t + info + bytes([i]), hashlib.sha256).digest()
okm += t
return okm[:length]
def _scan_for_sni(self, data: bytes) -> Optional[str]:
"""Brute-force scan for TLS SNI extension pattern in raw bytes."""
# Look for the SNI extension pattern: 0x00 0x00 (ext type) followed by
# extension data containing a host_name type (0x00) and ASCII hostname
pos = 0
while pos < len(data) - 10:
# Look for SNI extension header: type=0x0000, then lengths
if data[pos] == 0x00 and data[pos + 1] == 0x00:
if pos + 9 <= len(data):
# Extension length
ext_len = struct.unpack("!H", data[pos + 2:pos + 4])[0]
if 4 < ext_len < 256 and pos + 4 + ext_len <= len(data):
# Server name list length
list_len = struct.unpack("!H", data[pos + 4:pos + 6])[0]
if list_len > 0 and list_len <= ext_len:
# Server name type (0x00 = host_name)
name_type = data[pos + 6]
if name_type == 0x00 and pos + 9 <= len(data):
name_len = struct.unpack("!H", data[pos + 7:pos + 9])[0]
if 1 < name_len < 253 and pos + 9 + name_len <= len(data):
hostname = data[pos + 9:pos + 9 + name_len]
try:
sni = hostname.decode("ascii")
# Validate: must contain a dot and only valid chars
if ("." in sni and
all(c.isalnum() or c in ".-_" for c in sni) and
not sni.startswith(".") and
not sni.endswith(".")):
return sni
except (UnicodeDecodeError, ValueError):
pass
pos += 1
return None
@staticmethod
def _read_varint(data: bytes, offset: int) -> tuple:
"""Read a QUIC variable-length integer. Returns (value, bytes_consumed)."""
if offset >= len(data):
return (0, 0)
first = data[offset]
prefix = (first >> 6) & 0x03
if prefix == 0:
return (first & 0x3F, 1)
elif prefix == 1:
if offset + 2 > len(data):
return (0, 0)
val = struct.unpack("!H", data[offset:offset + 2])[0] & 0x3FFF
return (val, 2)
elif prefix == 2:
if offset + 4 > len(data):
return (0, 0)
val = struct.unpack("!I", data[offset:offset + 4])[0] & 0x3FFFFFFF
return (val, 4)
else:
if offset + 8 > len(data):
return (0, 0)
val = struct.unpack("!Q", data[offset:offset + 8])[0] & 0x3FFFFFFFFFFFFFFF
return (val, 8)
# ------------------------------------------------------------------
# Database
# ------------------------------------------------------------------
def _resolve_db_path(self) -> str:
base = self.config.get("db_dir", os.path.join(
os.path.expanduser("~"), ".bigbrother"
))
return os.path.join(base, "quic_analyzer.db")
def _init_db(self) -> None:
Path(self._db_path).parent.mkdir(parents=True, exist_ok=True)
self._db_conn = sqlite3.connect(self._db_path, check_same_thread=False)
self._db_conn.execute("PRAGMA journal_mode=WAL")
self._db_conn.execute("PRAGMA synchronous=NORMAL")
self._db_conn.executescript(_DB_INIT)
self._db_conn.commit()
def _record_sni(self, ts: float, src_ip: str, dst_ip: str,
sni: str, version: str) -> None:
with self._buffer_lock:
self._write_buffer.append((ts, src_ip, dst_ip, sni, version))
def _flush_buffer(self) -> None:
with self._buffer_lock:
batch = list(self._write_buffer)
self._write_buffer.clear()
if not batch or not self._db_conn:
return
try:
with self._db_conn:
self._db_conn.executemany(
"""INSERT INTO quic_sni
(timestamp, source_ip, dest_ip, sni, quic_version)
VALUES (?, ?, ?, ?, ?)""",
batch,
)
except Exception:
logger.exception("Failed to flush QUIC SNI records")
def _flush_loop(self) -> None:
while self._running:
time.sleep(FLUSH_INTERVAL)
try:
self._flush_buffer()
except Exception:
logger.exception("Flush loop error")
+377
View File
@@ -0,0 +1,377 @@
#!/usr/bin/env python3
"""RDP session monitor — passive RDP connection tracking.
Parses RDP Connection Request (Cookie field containing username),
CredSSP/NLA TSRequest structures for NTLM domain\\username extraction,
and tracks session patterns to identify admin workstations.
"""
import logging
import os
import socket
import sqlite3
import struct
import threading
import time
from collections import defaultdict
from pathlib import Path
from typing import Optional
from modules.base import BaseModule
logger = logging.getLogger("bb.passive.rdp_monitor")
# NTLM signature for CredSSP extraction
NTLMSSP_SIGNATURE = b'NTLMSSP\x00'
NTLM_AUTHENTICATE = 3
# X.224 Connection Request type
X224_CONNECTION_REQUEST = 0xE0
# Admin workstation threshold
ADMIN_DEST_THRESHOLD = 3
_DB_INIT = """
CREATE TABLE IF NOT EXISTS rdp_sessions (
id INTEGER PRIMARY KEY AUTOINCREMENT,
timestamp REAL NOT NULL,
source_ip TEXT NOT NULL,
dest_ip TEXT NOT NULL,
username TEXT DEFAULT '',
domain TEXT DEFAULT '',
client_hostname TEXT DEFAULT '',
keyboard_layout TEXT DEFAULT ''
);
CREATE INDEX IF NOT EXISTS idx_rdp_src ON rdp_sessions(source_ip);
CREATE INDEX IF NOT EXISTS idx_rdp_user ON rdp_sessions(username);
CREATE INDEX IF NOT EXISTS idx_rdp_ts ON rdp_sessions(timestamp);
"""
FLUSH_INTERVAL = 30
class RDPMonitor(BaseModule):
"""Monitor RDP sessions passively for user/host identification."""
name = "rdp_monitor"
module_type = "passive"
priority = 150
requires_root = True
def __init__(self, bus, state, config, engine=None):
super().__init__(bus, state, config, engine)
self._capture_bus = None
self._sub_queue = None
self._read_thread: Optional[threading.Thread] = None
self._flush_thread: Optional[threading.Thread] = None
self._db_path = self._resolve_db_path()
self._db_conn: Optional[sqlite3.Connection] = None
self._write_buffer: list[tuple] = []
self._buffer_lock = threading.Lock()
# Track admin patterns: src_ip -> set of dest_ips
self._rdp_targets: dict[str, set] = defaultdict(set)
self._stats = {
"packets_processed": 0,
"connection_requests": 0,
"credssp_auths": 0,
"unique_sessions": 0,
"admin_workstations": 0,
}
# ------------------------------------------------------------------
# BaseModule interface
# ------------------------------------------------------------------
def start(self) -> None:
if self._running:
return
self._init_db()
self._capture_bus = self.config.get("_capture_bus")
if self._capture_bus is None:
logger.error("RDPMonitor requires _capture_bus in config")
return
self._sub_queue = self._capture_bus.subscribe(
name=self.name, bpf_filter="port 3389", queue_depth=3000
)
self._running = True
self._start_time = time.time()
self._pid = os.getpid()
self._read_thread = threading.Thread(
target=self._read_loop, daemon=True, name="bb-rdp-read"
)
self._read_thread.start()
self._flush_thread = threading.Thread(
target=self._flush_loop, daemon=True, name="bb-rdp-flush"
)
self._flush_thread.start()
self.state.set_module_status(self.name, "running", pid=self._pid)
self.bus.emit("MODULE_STARTED", {"module": self.name}, source_module=self.name)
logger.info("RDPMonitor started")
def stop(self) -> None:
if not self._running:
return
self._running = False
if self._capture_bus:
self._capture_bus.unsubscribe(self.name)
for t in (self._read_thread, self._flush_thread):
if t and t.is_alive():
t.join(timeout=5.0)
self._flush_buffer()
if self._db_conn:
self._db_conn.close()
self._db_conn = None
self.state.set_module_status(self.name, "stopped")
self.bus.emit("MODULE_STOPPED", {"module": self.name}, source_module=self.name)
logger.info("RDPMonitor stopped — stats: %s", self._stats)
def status(self) -> dict:
return {
"running": self._running,
"pid": self._pid,
"uptime": time.time() - self._start_time if self._start_time else 0,
**self._stats,
}
def configure(self, config: dict) -> None:
self.config.update(config)
# ------------------------------------------------------------------
# Packet processing
# ------------------------------------------------------------------
def _read_loop(self) -> None:
while self._running:
result = self._sub_queue.get(timeout=1.0)
if result is None:
continue
ts, pkt = result
self._stats["packets_processed"] += 1
try:
self._process_packet(pkt, ts)
except Exception:
logger.debug("Error processing RDP packet", exc_info=True)
def _process_packet(self, pkt: bytes, ts: float) -> None:
"""Extract TCP payload and dispatch to RDP parsers."""
if len(pkt) < 54:
return
ethertype = struct.unpack("!H", pkt[12:14])[0]
eth_offset = 14
if ethertype == 0x8100:
if len(pkt) < 58:
return
ethertype = struct.unpack("!H", pkt[16:18])[0]
eth_offset = 18
if ethertype != 0x0800:
return
ip_header = pkt[eth_offset:]
if len(ip_header) < 20:
return
ihl = (ip_header[0] & 0x0F) * 4
ip_proto = ip_header[9]
if ip_proto != 6:
return
src_ip = socket.inet_ntoa(ip_header[12:16])
dst_ip = socket.inet_ntoa(ip_header[16:20])
if len(ip_header) < ihl + 20:
return
src_port, dst_port = struct.unpack("!HH", ip_header[ihl:ihl + 4])
tcp_data_off = ((ip_header[ihl + 12] >> 4) & 0xF) * 4
payload = ip_header[ihl + tcp_data_off:]
if len(payload) < 6:
return
# Only process traffic going TO RDP port (client -> server)
if dst_port == 3389:
self._parse_rdp_client(payload, src_ip, dst_ip, ts)
def _parse_rdp_client(self, payload: bytes, src_ip: str, dst_ip: str,
ts: float) -> None:
"""Parse client-to-server RDP traffic."""
# Check for TPKT header (version=3, reserved=0)
if payload[0] == 0x03 and payload[1] == 0x00:
self._parse_tpkt(payload, src_ip, dst_ip, ts)
return
# Check for NTLMSSP in CredSSP
if NTLMSSP_SIGNATURE in payload:
self._parse_credssp_ntlm(payload, src_ip, dst_ip, ts)
def _parse_tpkt(self, payload: bytes, src_ip: str, dst_ip: str,
ts: float) -> None:
"""Parse TPKT + X.224 Connection Request for username cookie."""
if len(payload) < 11:
return
# TPKT: version(1) + reserved(1) + length(2)
tpkt_len = struct.unpack("!H", payload[2:4])[0]
# X.224: length indicator(1) + type(1)
x224_len = payload[4]
x224_type = payload[5] & 0xF0
if x224_type != X224_CONNECTION_REQUEST:
return
self._stats["connection_requests"] += 1
# After X.224 header (variable length), look for Cookie field
# Cookie format: "Cookie: mstshash=username\r\n"
cookie_data = payload[11:] # Skip past basic X.224 header
username = ""
client_hostname = ""
# Search for mstshash cookie
cookie_idx = cookie_data.find(b'Cookie: mstshash=')
if cookie_idx >= 0:
cookie_start = cookie_idx + 17 # len("Cookie: mstshash=")
cookie_end = cookie_data.find(b'\r\n', cookie_start)
if cookie_end > cookie_start:
username = cookie_data[cookie_start:cookie_end].decode(
"ascii", errors="replace"
).strip()
if username:
self._rdp_targets[src_ip].add(dst_ip)
self._stats["unique_sessions"] += 1
self._check_admin_workstation(src_ip)
self._record_session(ts, src_ip, dst_ip, username, "", "", "")
def _parse_credssp_ntlm(self, payload: bytes, src_ip: str, dst_ip: str,
ts: float) -> None:
"""Extract NTLM credentials from CredSSP TSRequest."""
idx = payload.find(NTLMSSP_SIGNATURE)
if idx < 0 or idx + 12 > len(payload):
return
ntlm_data = payload[idx:]
msg_type = struct.unpack("<I", ntlm_data[8:12])[0]
if msg_type != NTLM_AUTHENTICATE or len(ntlm_data) < 72:
return
username, domain, workstation = self._parse_ntlm_type3(ntlm_data)
if username:
self._stats["credssp_auths"] += 1
self._rdp_targets[src_ip].add(dst_ip)
self._stats["unique_sessions"] += 1
self._check_admin_workstation(src_ip)
self._record_session(ts, src_ip, dst_ip, username, domain, workstation, "")
def _parse_ntlm_type3(self, data: bytes) -> tuple:
"""Parse NTLM Type 3 Authenticate for username, domain, workstation."""
if len(data) < 72:
return ("", "", "")
try:
domain_len = struct.unpack("<H", data[28:30])[0]
domain_off = struct.unpack("<I", data[32:36])[0]
user_len = struct.unpack("<H", data[36:38])[0]
user_off = struct.unpack("<I", data[40:44])[0]
ws_len = struct.unpack("<H", data[44:46])[0]
ws_off = struct.unpack("<I", data[48:52])[0]
domain = ""
username = ""
workstation = ""
if domain_len and domain_off + domain_len <= len(data):
domain = data[domain_off:domain_off + domain_len].decode(
"utf-16-le", errors="replace"
)
if user_len and user_off + user_len <= len(data):
username = data[user_off:user_off + user_len].decode(
"utf-16-le", errors="replace"
)
if ws_len and ws_off + ws_len <= len(data):
workstation = data[ws_off:ws_off + ws_len].decode(
"utf-16-le", errors="replace"
)
return (username, domain, workstation)
except Exception:
return ("", "", "")
def _check_admin_workstation(self, src_ip: str) -> None:
"""Check if source IP qualifies as admin workstation."""
targets = self._rdp_targets.get(src_ip, set())
if len(targets) >= ADMIN_DEST_THRESHOLD:
current = len([
ip for ip, dests in self._rdp_targets.items()
if len(dests) >= ADMIN_DEST_THRESHOLD
])
if current > self._stats["admin_workstations"]:
self._stats["admin_workstations"] = current
logger.info(
"Admin workstation detected: %s (RDP to %d hosts)",
src_ip, len(targets)
)
# ------------------------------------------------------------------
# Database
# ------------------------------------------------------------------
def _resolve_db_path(self) -> str:
base = self.config.get("db_dir", os.path.join(
os.path.expanduser("~"), ".bigbrother"
))
return os.path.join(base, "rdp_monitor.db")
def _init_db(self) -> None:
Path(self._db_path).parent.mkdir(parents=True, exist_ok=True)
self._db_conn = sqlite3.connect(self._db_path, check_same_thread=False)
self._db_conn.execute("PRAGMA journal_mode=WAL")
self._db_conn.execute("PRAGMA synchronous=NORMAL")
self._db_conn.executescript(_DB_INIT)
self._db_conn.commit()
def _record_session(self, ts: float, src_ip: str, dst_ip: str,
username: str, domain: str, client_hostname: str,
keyboard_layout: str) -> None:
with self._buffer_lock:
self._write_buffer.append((
ts, src_ip, dst_ip, username, domain, client_hostname, keyboard_layout
))
def _flush_buffer(self) -> None:
with self._buffer_lock:
batch = list(self._write_buffer)
self._write_buffer.clear()
if not batch or not self._db_conn:
return
try:
with self._db_conn:
self._db_conn.executemany(
"""INSERT INTO rdp_sessions
(timestamp, source_ip, dest_ip, username, domain,
client_hostname, keyboard_layout)
VALUES (?, ?, ?, ?, ?, ?, ?)""",
batch,
)
except Exception:
logger.exception("Failed to flush RDP sessions")
def _flush_loop(self) -> None:
while self._running:
time.sleep(FLUSH_INTERVAL)
try:
self._flush_buffer()
except Exception:
logger.exception("Flush loop error")
+394
View File
@@ -0,0 +1,394 @@
#!/usr/bin/env python3
"""SMB file access monitor — passive SMB2/3 session and file tracking.
Parses SMB2/3 Tree Connect requests (share names), Create requests (file paths),
and Read/Write operations. Tracks per-user share access and detects
GPP/SYSVOL access patterns indicating potential Group Policy Preference
password exposure. SMB3 encrypted sessions are opaque.
"""
import logging
import os
import socket
import sqlite3
import struct
import threading
import time
from pathlib import Path
from typing import Optional
from modules.base import BaseModule
logger = logging.getLogger("bb.passive.smb_monitor")
# SMB2 command IDs
SMB2_NEGOTIATE = 0x0000
SMB2_SESSION_SETUP = 0x0001
SMB2_TREE_CONNECT = 0x0003
SMB2_CREATE = 0x0005
SMB2_READ = 0x0008
SMB2_WRITE = 0x0009
# SMB2 header magic
SMB2_MAGIC = b'\xfeSMB'
# GPP-related paths (case-insensitive matching)
GPP_PATTERNS = (
"groups.xml", "services.xml", "scheduledtasks.xml",
"datasources.xml", "printers.xml", "drives.xml",
"policies", "sysvol",
)
_DB_INIT = """
CREATE TABLE IF NOT EXISTS smb_access (
id INTEGER PRIMARY KEY AUTOINCREMENT,
timestamp REAL NOT NULL,
source_ip TEXT NOT NULL,
username TEXT DEFAULT '',
share_name TEXT DEFAULT '',
file_path TEXT DEFAULT '',
operation TEXT NOT NULL,
smb_version TEXT DEFAULT ''
);
CREATE INDEX IF NOT EXISTS idx_smb_user ON smb_access(username);
CREATE INDEX IF NOT EXISTS idx_smb_share ON smb_access(share_name);
CREATE INDEX IF NOT EXISTS idx_smb_ts ON smb_access(timestamp);
"""
FLUSH_INTERVAL = 30
class SMBMonitor(BaseModule):
"""Monitor SMB2/3 file share access passively."""
name = "smb_monitor"
module_type = "passive"
priority = 100
requires_root = True
def __init__(self, bus, state, config, engine=None):
super().__init__(bus, state, config, engine)
self._capture_bus = None
self._sub_queue = None
self._read_thread: Optional[threading.Thread] = None
self._flush_thread: Optional[threading.Thread] = None
self._db_path = self._resolve_db_path()
self._db_conn: Optional[sqlite3.Connection] = None
self._write_buffer: list[tuple] = []
self._buffer_lock = threading.Lock()
# Track session ID -> username mapping
self._sessions: dict[tuple, str] = {} # (src_ip, session_id) -> username
# Track tree ID -> share name mapping
self._trees: dict[tuple, str] = {} # (src_ip, tree_id) -> share_name
self._stats = {
"packets_processed": 0,
"tree_connects": 0,
"file_creates": 0,
"reads": 0,
"writes": 0,
"gpp_access_detected": 0,
}
# ------------------------------------------------------------------
# BaseModule interface
# ------------------------------------------------------------------
def start(self) -> None:
if self._running:
return
self._init_db()
self._capture_bus = self.config.get("_capture_bus")
if self._capture_bus is None:
logger.error("SMBMonitor requires _capture_bus in config")
return
self._sub_queue = self._capture_bus.subscribe(
name=self.name, bpf_filter="port 445", queue_depth=5000
)
self._running = True
self._start_time = time.time()
self._pid = os.getpid()
self._read_thread = threading.Thread(
target=self._read_loop, daemon=True, name="bb-smb-read"
)
self._read_thread.start()
self._flush_thread = threading.Thread(
target=self._flush_loop, daemon=True, name="bb-smb-flush"
)
self._flush_thread.start()
self.state.set_module_status(self.name, "running", pid=self._pid)
self.bus.emit("MODULE_STARTED", {"module": self.name}, source_module=self.name)
logger.info("SMBMonitor started")
def stop(self) -> None:
if not self._running:
return
self._running = False
if self._capture_bus:
self._capture_bus.unsubscribe(self.name)
for t in (self._read_thread, self._flush_thread):
if t and t.is_alive():
t.join(timeout=5.0)
self._flush_buffer()
if self._db_conn:
self._db_conn.close()
self._db_conn = None
self.state.set_module_status(self.name, "stopped")
self.bus.emit("MODULE_STOPPED", {"module": self.name}, source_module=self.name)
logger.info("SMBMonitor stopped — stats: %s", self._stats)
def status(self) -> dict:
return {
"running": self._running,
"pid": self._pid,
"uptime": time.time() - self._start_time if self._start_time else 0,
**self._stats,
}
def configure(self, config: dict) -> None:
self.config.update(config)
# ------------------------------------------------------------------
# Packet processing
# ------------------------------------------------------------------
def _read_loop(self) -> None:
while self._running:
result = self._sub_queue.get(timeout=1.0)
if result is None:
continue
ts, pkt = result
self._stats["packets_processed"] += 1
try:
self._process_packet(pkt, ts)
except Exception:
logger.debug("Error processing SMB packet", exc_info=True)
def _process_packet(self, pkt: bytes, ts: float) -> None:
"""Extract TCP payload and parse SMB2."""
if len(pkt) < 54: # eth(14) + ip(20) + tcp(20) minimum
return
ethertype = struct.unpack("!H", pkt[12:14])[0]
eth_offset = 14
if ethertype == 0x8100:
if len(pkt) < 58:
return
ethertype = struct.unpack("!H", pkt[16:18])[0]
eth_offset = 18
if ethertype != 0x0800:
return
ip_header = pkt[eth_offset:]
if len(ip_header) < 20:
return
ihl = (ip_header[0] & 0x0F) * 4
ip_proto = ip_header[9]
if ip_proto != 6: # TCP only
return
src_ip = socket.inet_ntoa(ip_header[12:16])
dst_ip = socket.inet_ntoa(ip_header[16:20])
if len(ip_header) < ihl + 20:
return
src_port, dst_port = struct.unpack("!HH", ip_header[ihl:ihl + 4])
tcp_data_off = ((ip_header[ihl + 12] >> 4) & 0xF) * 4
payload = ip_header[ihl + tcp_data_off:]
if len(payload) < 4:
return
# SMB uses NetBIOS Session Service: 1-byte type + 3-byte length
# Type 0x00 = session message
if payload[0] != 0x00:
return
nb_len = struct.unpack("!I", b'\x00' + payload[1:4])[0]
smb_data = payload[4:]
if len(smb_data) < 64:
return
# Determine direction: request goes TO port 445
client_ip = src_ip if dst_port == 445 else dst_ip
self._parse_smb2(smb_data, client_ip, ts)
def _parse_smb2(self, data: bytes, client_ip: str, ts: float) -> None:
"""Parse an SMB2 message header and dispatch to command handler."""
if data[:4] != SMB2_MAGIC:
return
if len(data) < 64:
return
# SMB2 header (64 bytes)
header_len = struct.unpack("<H", data[4:6])[0]
command = struct.unpack("<H", data[12:14])[0]
flags = struct.unpack("<I", data[16:20])[0]
is_response = bool(flags & 0x00000001)
session_id = struct.unpack("<Q", data[40:48])[0]
tree_id = struct.unpack("<I", data[36:40])[0]
smb_body = data[64:]
if command == SMB2_TREE_CONNECT and is_response and len(smb_body) >= 8:
# Tree Connect Response doesn't contain the share name,
# but we see the path in the request
pass
elif command == SMB2_TREE_CONNECT and not is_response:
self._handle_tree_connect_request(smb_body, client_ip, session_id, tree_id, ts)
elif command == SMB2_CREATE and not is_response:
self._handle_create_request(smb_body, client_ip, session_id, tree_id, ts)
elif command == SMB2_READ and not is_response:
self._stats["reads"] += 1
share = self._trees.get((client_ip, tree_id), "")
self._record_access(ts, client_ip, "", share, "", "read", "smb2")
elif command == SMB2_WRITE and not is_response:
self._stats["writes"] += 1
share = self._trees.get((client_ip, tree_id), "")
self._record_access(ts, client_ip, "", share, "", "write", "smb2")
def _handle_tree_connect_request(self, body: bytes, client_ip: str,
session_id: int, tree_id: int,
ts: float) -> None:
"""Parse SMB2 Tree Connect request to extract share name."""
# Tree Connect Request body (SMB 3.1.1):
# StructureSize(2) + Reserved/Flags(2) + PathOffset(2) + PathLength(2)
if len(body) < 8:
return
path_offset = struct.unpack("<H", body[4:6])[0]
path_length = struct.unpack("<H", body[6:8])[0]
# PathOffset is relative to the beginning of the SMB2 header
# Since body starts at offset 64, adjust
# But we receive body starting at offset 0, so the path is at:
# (path_offset - 64) in body terms, OR it may follow immediately
# In practice, path follows the 8-byte struct at body[8:]
if path_length > 0 and len(body) >= 8 + path_length:
try:
share_path = body[8:8 + path_length].decode("utf-16-le", errors="replace")
except Exception:
share_path = ""
# Extract just the share name from \\server\share
share_name = share_path.rsplit("\\", 1)[-1] if "\\" in share_path else share_path
self._trees[(client_ip, tree_id)] = share_name
self._stats["tree_connects"] += 1
username = self._sessions.get((client_ip, session_id), "")
self._record_access(ts, client_ip, username, share_name, "", "tree_connect", "smb2")
# Check for SYSVOL/NETLOGON access
if share_name.lower() in ("sysvol", "netlogon"):
logger.info("SYSVOL/NETLOGON access from %s — potential GPP exposure", client_ip)
def _handle_create_request(self, body: bytes, client_ip: str,
session_id: int, tree_id: int,
ts: float) -> None:
"""Parse SMB2 Create request to extract file path."""
# Create Request: StructureSize(2) + SecurityFlags(1) + RequestedOplockLevel(1)
# + ImpersonationLevel(4) + SmbCreateFlags(8) + Reserved(8)
# + DesiredAccess(4) + FileAttributes(4) + ShareAccess(4)
# + CreateDisposition(4) + CreateOptions(4) + NameOffset(2) + NameLength(2)
if len(body) < 56:
return
name_offset = struct.unpack("<H", body[44:46])[0]
name_length = struct.unpack("<H", body[46:48])[0]
if name_length == 0:
return
# Name follows the fixed-size structure
# name_offset is relative to SMB2 header start, body starts at +64
name_start = 56 # After the fixed-size fields in Create body
if len(body) >= name_start + name_length:
try:
file_path = body[name_start:name_start + name_length].decode(
"utf-16-le", errors="replace"
)
except Exception:
file_path = ""
if file_path:
self._stats["file_creates"] += 1
share = self._trees.get((client_ip, tree_id), "")
username = self._sessions.get((client_ip, session_id), "")
self._record_access(ts, client_ip, username, share, file_path, "create", "smb2")
# GPP detection
file_lower = file_path.lower()
for pattern in GPP_PATTERNS:
if pattern in file_lower:
self._stats["gpp_access_detected"] += 1
self.bus.emit("CREDENTIAL_FOUND", {
"source": "smb_gpp_access",
"client_ip": client_ip,
"share": share,
"file_path": file_path,
"detail": "GPP/SYSVOL file access — may contain cleartext passwords",
}, source_module=self.name)
break
# ------------------------------------------------------------------
# Database
# ------------------------------------------------------------------
def _resolve_db_path(self) -> str:
base = self.config.get("db_dir", os.path.join(
os.path.expanduser("~"), ".bigbrother"
))
return os.path.join(base, "smb_monitor.db")
def _init_db(self) -> None:
Path(self._db_path).parent.mkdir(parents=True, exist_ok=True)
self._db_conn = sqlite3.connect(self._db_path, check_same_thread=False)
self._db_conn.execute("PRAGMA journal_mode=WAL")
self._db_conn.execute("PRAGMA synchronous=NORMAL")
self._db_conn.executescript(_DB_INIT)
self._db_conn.commit()
def _record_access(self, ts: float, source_ip: str, username: str,
share_name: str, file_path: str, operation: str,
smb_version: str) -> None:
with self._buffer_lock:
self._write_buffer.append((
ts, source_ip, username, share_name, file_path, operation, smb_version
))
def _flush_buffer(self) -> None:
with self._buffer_lock:
batch = list(self._write_buffer)
self._write_buffer.clear()
if not batch or not self._db_conn:
return
try:
with self._db_conn:
self._db_conn.executemany(
"""INSERT INTO smb_access
(timestamp, source_ip, username, share_name, file_path,
operation, smb_version)
VALUES (?, ?, ?, ?, ?, ?, ?)""",
batch,
)
except Exception:
logger.exception("Failed to flush SMB access records")
def _flush_loop(self) -> None:
while self._running:
time.sleep(FLUSH_INTERVAL)
try:
self._flush_buffer()
except Exception:
logger.exception("Flush loop error")
+431
View File
@@ -0,0 +1,431 @@
#!/usr/bin/env python3
"""VLAN discovery — 802.1Q tag detection, DTP/CDP/LLDP/STP parsing.
Passively identifies VLAN infrastructure by parsing layer-2 control protocols.
Detects 802.1Q tagged frames, DTP trunk negotiation, 802.1X/EAPOL NAC,
CDP/LLDP neighbor announcements, and STP BPDUs.
"""
import logging
import os
import sqlite3
import struct
import threading
import time
from pathlib import Path
from typing import Optional
from modules.base import BaseModule
logger = logging.getLogger("bb.passive.vlan_discovery")
# Ethertypes and protocol IDs
ETHERTYPE_8021Q = 0x8100
ETHERTYPE_8021AD = 0x88A8 # QinQ
ETHERTYPE_EAPOL = 0x888E
ETHERTYPE_LLDP = 0x88CC
# CDP/DTP use SNAP encapsulation with LLC header (DSAP=DSAP=0xAA, SSAP=0xAA, ctrl=0x03)
# CDP OUI: 0x00000C, protocol: 0x2000
# DTP OUI: 0x00000C, protocol: 0x2004
# STP uses LLC (DSAP=0x42, SSAP=0x42)
# CDP TLV types
CDP_TLV_DEVICE_ID = 0x0001
CDP_TLV_ADDRESS = 0x0002
CDP_TLV_PORT_ID = 0x0003
CDP_TLV_CAPABILITIES = 0x0004
CDP_TLV_SOFTWARE_VERSION = 0x0005
CDP_TLV_PLATFORM = 0x0006
CDP_TLV_NATIVE_VLAN = 0x000A
CDP_TLV_MANAGEMENT_ADDR = 0x0016
# LLDP TLV types
LLDP_TLV_END = 0
LLDP_TLV_CHASSIS_ID = 1
LLDP_TLV_PORT_ID = 2
LLDP_TLV_TTL = 3
LLDP_TLV_PORT_DESC = 4
LLDP_TLV_SYSTEM_NAME = 5
LLDP_TLV_SYSTEM_DESC = 6
LLDP_TLV_MGMT_ADDR = 8
_DB_INIT = """
CREATE TABLE IF NOT EXISTS vlans (
id INTEGER PRIMARY KEY AUTOINCREMENT,
vlan_id INTEGER NOT NULL,
name TEXT DEFAULT '',
source TEXT NOT NULL,
switch_name TEXT DEFAULT '',
switch_port TEXT DEFAULT '',
first_seen REAL NOT NULL,
UNIQUE(vlan_id, source, switch_name, switch_port)
);
CREATE INDEX IF NOT EXISTS idx_vlans_vlan_id ON vlans(vlan_id);
"""
FLUSH_INTERVAL = 30
class VLANDiscovery(BaseModule):
"""Passively discover VLANs via 802.1Q tags and L2 protocol parsing."""
name = "vlan_discovery"
module_type = "passive"
priority = 150
requires_root = True
def __init__(self, bus, state, config, engine=None):
super().__init__(bus, state, config, engine)
self._capture_bus = None
self._sub_queue = None
self._read_thread: Optional[threading.Thread] = None
self._flush_thread: Optional[threading.Thread] = None
self._db_path = self._resolve_db_path()
self._db_conn: Optional[sqlite3.Connection] = None
self._write_buffer: list[tuple] = []
self._buffer_lock = threading.Lock()
self._stats = {
"dot1q_frames": 0,
"cdp_frames": 0,
"lldp_frames": 0,
"dtp_frames": 0,
"stp_bpdus": 0,
"eapol_frames": 0,
"vlans_found": 0,
"packets_processed": 0,
}
# ------------------------------------------------------------------
# BaseModule interface
# ------------------------------------------------------------------
def start(self) -> None:
if self._running:
return
self._init_db()
self._capture_bus = self.config.get("_capture_bus")
if self._capture_bus is None:
logger.error("VLANDiscovery requires _capture_bus in config")
return
# Broad filter: we need 802.1Q (any ethertype), CDP/DTP (SNAP), LLDP, STP, EAPOL
# Compile a catch-all because these span multiple ethertypes and LLC frames
self._sub_queue = self._capture_bus.subscribe(
name=self.name, bpf_filter="", queue_depth=3000
)
self._running = True
self._start_time = time.time()
self._pid = os.getpid()
self._read_thread = threading.Thread(
target=self._read_loop, daemon=True, name="bb-vlan-read"
)
self._read_thread.start()
self._flush_thread = threading.Thread(
target=self._flush_loop, daemon=True, name="bb-vlan-flush"
)
self._flush_thread.start()
self.state.set_module_status(self.name, "running", pid=self._pid)
self.bus.emit("MODULE_STARTED", {"module": self.name}, source_module=self.name)
logger.info("VLANDiscovery started")
def stop(self) -> None:
if not self._running:
return
self._running = False
if self._capture_bus and self._sub_queue:
self._capture_bus.unsubscribe(self.name)
if self._read_thread and self._read_thread.is_alive():
self._read_thread.join(timeout=5.0)
if self._flush_thread and self._flush_thread.is_alive():
self._flush_thread.join(timeout=5.0)
self._flush_buffer()
if self._db_conn:
self._db_conn.close()
self._db_conn = None
self.state.set_module_status(self.name, "stopped")
self.bus.emit("MODULE_STOPPED", {"module": self.name}, source_module=self.name)
logger.info("VLANDiscovery stopped — stats: %s", self._stats)
def status(self) -> dict:
return {
"running": self._running,
"pid": self._pid,
"uptime": time.time() - self._start_time if self._start_time else 0,
**self._stats,
}
def configure(self, config: dict) -> None:
self.config.update(config)
# ------------------------------------------------------------------
# Packet processing
# ------------------------------------------------------------------
def _read_loop(self) -> None:
"""Read packets from capture bus and dispatch to parsers."""
while self._running:
result = self._sub_queue.get(timeout=1.0)
if result is None:
continue
ts, pkt = result
self._stats["packets_processed"] += 1
try:
self._process_frame(pkt, ts)
except Exception:
logger.debug("Error processing frame", exc_info=True)
def _process_frame(self, pkt: bytes, ts: float) -> None:
"""Identify and dispatch frame to appropriate parser."""
if len(pkt) < 14:
return
dst_mac = pkt[0:6]
ethertype = struct.unpack("!H", pkt[12:14])[0]
# 802.1Q tagged frame
if ethertype == ETHERTYPE_8021Q or ethertype == ETHERTYPE_8021AD:
self._parse_dot1q(pkt, ts)
return
# LLDP
if ethertype == ETHERTYPE_LLDP:
self._parse_lldp(pkt, ts)
return
# EAPOL (802.1X)
if ethertype == ETHERTYPE_EAPOL:
self._stats["eapol_frames"] += 1
self._record_vlan(0, "eapol_detected", "", "", ts)
return
# Check for LLC/SNAP frames (ethertype field is actually length if < 0x0600)
if ethertype <= 0x05DC and len(pkt) >= 22:
self._parse_llc_snap(pkt, ts)
def _parse_dot1q(self, pkt: bytes, ts: float) -> None:
"""Parse 802.1Q tagged frame to extract VLAN ID."""
if len(pkt) < 18:
return
tci = struct.unpack("!H", pkt[14:16])[0]
vlan_id = tci & 0x0FFF
if vlan_id > 0:
self._stats["dot1q_frames"] += 1
self._record_vlan(vlan_id, "802.1q", "", "", ts)
def _parse_llc_snap(self, pkt: bytes, ts: float) -> None:
"""Parse LLC/SNAP encapsulated frames for CDP, DTP, STP."""
# Minimum: 14 (eth) + 3 (LLC) = 17 bytes for STP check
if len(pkt) < 17:
return
dsap = pkt[14]
ssap = pkt[15]
ctrl = pkt[16]
# STP BPDU: DSAP=0x42, SSAP=0x42
if dsap == 0x42 and ssap == 0x42:
self._parse_stp(pkt, ts)
return
# SNAP: DSAP=0xAA, SSAP=0xAA, Ctrl=0x03
if dsap == 0xAA and ssap == 0xAA and ctrl == 0x03 and len(pkt) >= 22:
oui = pkt[17:20]
proto = struct.unpack("!H", pkt[20:22])[0]
# CDP: OUI 0x00000C, protocol 0x2000
if oui == b'\x00\x00\x0c' and proto == 0x2000:
self._parse_cdp(pkt[22:], ts)
return
# DTP: OUI 0x00000C, protocol 0x2004
if oui == b'\x00\x00\x0c' and proto == 0x2004:
self._parse_dtp(pkt[22:], ts)
return
def _parse_cdp(self, payload: bytes, ts: float) -> None:
"""Parse CDP TLVs to extract switch name, port, VLAN, platform."""
if len(payload) < 4:
return
self._stats["cdp_frames"] += 1
# CDP header: version(1), TTL(1), checksum(2)
offset = 4
device_id = ""
port_id = ""
platform = ""
native_vlan = 0
while offset + 4 <= len(payload):
tlv_type = struct.unpack("!H", payload[offset:offset + 2])[0]
tlv_len = struct.unpack("!H", payload[offset + 2:offset + 4])[0]
if tlv_len < 4 or offset + tlv_len > len(payload):
break
tlv_data = payload[offset + 4:offset + tlv_len]
if tlv_type == CDP_TLV_DEVICE_ID:
device_id = tlv_data.decode("ascii", errors="replace").strip('\x00')
elif tlv_type == CDP_TLV_PORT_ID:
port_id = tlv_data.decode("ascii", errors="replace").strip('\x00')
elif tlv_type == CDP_TLV_PLATFORM:
platform = tlv_data.decode("ascii", errors="replace").strip('\x00')
elif tlv_type == CDP_TLV_NATIVE_VLAN and len(tlv_data) >= 2:
native_vlan = struct.unpack("!H", tlv_data[:2])[0]
offset += tlv_len
if native_vlan > 0:
self._record_vlan(native_vlan, "cdp", device_id, port_id, ts)
if device_id:
self.bus.emit("VLAN_DETECTED", {
"source": "cdp",
"switch_name": device_id,
"switch_port": port_id,
"platform": platform,
"native_vlan": native_vlan,
}, source_module=self.name)
def _parse_dtp(self, payload: bytes, ts: float) -> None:
"""Parse DTP frames — Cisco VLAN trunk negotiation."""
self._stats["dtp_frames"] += 1
# DTP presence alone is noteworthy (trunk negotiation active)
self.bus.emit("VLAN_DETECTED", {
"source": "dtp",
"detail": "DTP trunk negotiation detected — VLAN hopping may be possible",
}, source_module=self.name)
def _parse_lldp(self, pkt: bytes, ts: float) -> None:
"""Parse LLDP TLVs for system name, port description, management address."""
if len(pkt) < 16:
return
self._stats["lldp_frames"] += 1
# LLDP payload starts after Ethernet header (14 bytes)
offset = 14
system_name = ""
port_desc = ""
mgmt_addr = ""
while offset + 2 <= len(pkt):
tlv_header = struct.unpack("!H", pkt[offset:offset + 2])[0]
tlv_type = (tlv_header >> 9) & 0x7F
tlv_len = tlv_header & 0x01FF
offset += 2
if tlv_type == LLDP_TLV_END or offset + tlv_len > len(pkt):
break
tlv_data = pkt[offset:offset + tlv_len]
if tlv_type == LLDP_TLV_SYSTEM_NAME:
system_name = tlv_data.decode("utf-8", errors="replace")
elif tlv_type == LLDP_TLV_PORT_DESC:
port_desc = tlv_data.decode("utf-8", errors="replace")
elif tlv_type == LLDP_TLV_MGMT_ADDR and tlv_len >= 6:
addr_len = tlv_data[0]
addr_subtype = tlv_data[1]
if addr_subtype == 1 and addr_len >= 5: # IPv4
addr_bytes = tlv_data[2:6]
mgmt_addr = ".".join(str(b) for b in addr_bytes)
offset += tlv_len
if system_name:
self.bus.emit("VLAN_DETECTED", {
"source": "lldp",
"system_name": system_name,
"port_desc": port_desc,
"mgmt_addr": mgmt_addr,
}, source_module=self.name)
def _parse_stp(self, pkt: bytes, ts: float) -> None:
"""Parse STP BPDU to extract root bridge ID and priority."""
# LLC header at 14, STP BPDU starts at 17
if len(pkt) < 38:
return
self._stats["stp_bpdus"] += 1
bpdu = pkt[17:]
if len(bpdu) < 21:
return
# Protocol ID (2), version (1), type (1)
proto_id = struct.unpack("!H", bpdu[0:2])[0]
if proto_id != 0x0000:
return
# Flags (1 byte at offset 4)
# Root bridge ID: 8 bytes at offset 5 (priority 2 bytes + MAC 6 bytes)
if len(bpdu) >= 13:
root_priority = struct.unpack("!H", bpdu[5:7])[0]
root_mac = ":".join(f"{b:02x}" for b in bpdu[7:13])
# Bridge ID: 8 bytes at offset 13
bridge_priority = struct.unpack("!H", bpdu[13:15])[0] if len(bpdu) >= 15 else 0
self.bus.emit("VLAN_DETECTED", {
"source": "stp",
"root_bridge_mac": root_mac,
"root_priority": root_priority,
"bridge_priority": bridge_priority,
}, source_module=self.name)
# ------------------------------------------------------------------
# Database
# ------------------------------------------------------------------
def _resolve_db_path(self) -> str:
base = self.config.get("db_dir", os.path.join(
os.path.expanduser("~"), ".bigbrother"
))
return os.path.join(base, "vlan_discovery.db")
def _init_db(self) -> None:
Path(self._db_path).parent.mkdir(parents=True, exist_ok=True)
self._db_conn = sqlite3.connect(self._db_path, check_same_thread=False)
self._db_conn.execute("PRAGMA journal_mode=WAL")
self._db_conn.execute("PRAGMA synchronous=NORMAL")
self._db_conn.executescript(_DB_INIT)
self._db_conn.commit()
def _record_vlan(self, vlan_id: int, source: str, switch_name: str,
switch_port: str, ts: float) -> None:
"""Buffer a VLAN record for batch insert."""
with self._buffer_lock:
self._write_buffer.append((vlan_id, "", source, switch_name, switch_port, ts))
def _flush_buffer(self) -> None:
"""Write buffered records to SQLite."""
with self._buffer_lock:
batch = list(self._write_buffer)
self._write_buffer.clear()
if not batch or not self._db_conn:
return
try:
with self._db_conn:
self._db_conn.executemany(
"""INSERT INTO vlans (vlan_id, name, source, switch_name, switch_port, first_seen)
VALUES (?, ?, ?, ?, ?, ?)
ON CONFLICT(vlan_id, source, switch_name, switch_port) DO NOTHING""",
batch,
)
new_count = len(batch)
self._stats["vlans_found"] += new_count
except Exception:
logger.exception("Failed to flush VLAN records")
def _flush_loop(self) -> None:
"""Periodically flush write buffer to disk."""
while self._running:
time.sleep(FLUSH_INTERVAL)
try:
self._flush_buffer()
except Exception:
logger.exception("Flush loop error")