Initial build: geo-scout geo-targeted network scanner
masscan port discovery → nmap fingerprinting → probe engine → pattern/version matching Dynamic YAML inputs: countries.yaml (country codes + CIDR resolution) and targets.yaml (fingerprint probes) RIR delegated stats for authoritative country CIDR data (no external API) Supports tcp_banner, http, https, rtsp, udp probe types Version extraction and comparison operators (<=, >=, ==, etc.) SQLite results per engagement, CSV/JSON export format-spec/ docs for generating input files externally
This commit is contained in:
n0mad1k
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
+104
@@ -0,0 +1,104 @@
|
||||
import ipaddress
|
||||
import json
|
||||
import os
|
||||
import time
|
||||
import urllib.request
|
||||
from pathlib import Path
|
||||
|
||||
CACHE_DIR = Path(__file__).parent.parent / "data" / "cidr-cache"
|
||||
CACHE_TTL = 86400 * 7 # 7 days
|
||||
|
||||
RIR_URLS = [
|
||||
"https://ftp.ripe.net/pub/stats/ripencc/delegated-ripencc-latest",
|
||||
"https://ftp.arin.net/pub/stats/arin/delegated-arin-extended-latest",
|
||||
"https://ftp.apnic.net/pub/stats/apnic/delegated-apnic-latest",
|
||||
"https://ftp.lacnic.net/pub/stats/lacnic/delegated-lacnic-latest",
|
||||
"https://ftp.afrinic.net/pub/stats/afrinic/delegated-afrinic-latest",
|
||||
]
|
||||
|
||||
RIR_NAMES = ["ripe", "arin", "apnic", "lacnic", "afrinic"]
|
||||
|
||||
|
||||
def _cache_path(rir: str) -> Path:
|
||||
return CACHE_DIR / f"{rir}.txt"
|
||||
|
||||
|
||||
def _index_path() -> Path:
|
||||
return CACHE_DIR / "index.json"
|
||||
|
||||
|
||||
def _is_stale(path: Path) -> bool:
|
||||
if not path.exists():
|
||||
return True
|
||||
return time.time() - path.stat().st_mtime > CACHE_TTL
|
||||
|
||||
|
||||
def fetch_rir_data(force: bool = False) -> None:
|
||||
CACHE_DIR.mkdir(parents=True, exist_ok=True)
|
||||
for name, url in zip(RIR_NAMES, RIR_URLS):
|
||||
path = _cache_path(name)
|
||||
if not force and not _is_stale(path):
|
||||
continue
|
||||
print(f" Fetching {name}...", end=" ", flush=True)
|
||||
try:
|
||||
req = urllib.request.Request(url, headers={"User-Agent": "geo-scout/1.0"})
|
||||
with urllib.request.urlopen(req, timeout=30) as resp:
|
||||
path.write_bytes(resp.read())
|
||||
print("done")
|
||||
except Exception as e:
|
||||
print(f"failed ({e})")
|
||||
|
||||
|
||||
def _build_index() -> dict[str, list[str]]:
|
||||
index: dict[str, list[str]] = {}
|
||||
for name in RIR_NAMES:
|
||||
path = _cache_path(name)
|
||||
if not path.exists():
|
||||
continue
|
||||
for line in path.read_text(errors="replace").splitlines():
|
||||
if line.startswith("#") or not line.strip():
|
||||
continue
|
||||
parts = line.split("|")
|
||||
if len(parts) < 7:
|
||||
continue
|
||||
_, cc, record_type, start, value, _, status = parts[:7]
|
||||
if record_type != "ipv4" or status not in ("allocated", "assigned"):
|
||||
continue
|
||||
if not cc or cc == "*":
|
||||
continue
|
||||
cc = cc.upper()
|
||||
try:
|
||||
count = int(value)
|
||||
prefix_len = 32 - (count - 1).bit_length()
|
||||
cidr = f"{start}/{prefix_len}"
|
||||
ipaddress.ip_network(cidr, strict=False)
|
||||
except (ValueError, TypeError):
|
||||
continue
|
||||
index.setdefault(cc, []).append(cidr)
|
||||
return index
|
||||
|
||||
|
||||
def load_index(force_rebuild: bool = False) -> dict[str, list[str]]:
|
||||
idx_path = _index_path()
|
||||
if not force_rebuild and idx_path.exists() and not _is_stale(idx_path):
|
||||
return json.loads(idx_path.read_text())
|
||||
index = _build_index()
|
||||
idx_path.write_text(json.dumps(index))
|
||||
return index
|
||||
|
||||
|
||||
def get_cidrs(country_code: str, index: dict[str, list[str]]) -> list[str]:
|
||||
return index.get(country_code.upper(), [])
|
||||
|
||||
|
||||
def merge_cidrs(cidrs: list[str], exclude: list[str]) -> list[str]:
|
||||
if not exclude:
|
||||
return cidrs
|
||||
exclude_nets = [ipaddress.ip_network(c, strict=False) for c in exclude]
|
||||
result = []
|
||||
for cidr in cidrs:
|
||||
net = ipaddress.ip_network(cidr, strict=False)
|
||||
excluded = any(net.overlaps(ex) for ex in exclude_nets)
|
||||
if not excluded:
|
||||
result.append(cidr)
|
||||
return result
|
||||
@@ -0,0 +1,117 @@
|
||||
import json
|
||||
import sqlite3
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
|
||||
RESULTS_DIR = Path(__file__).parent.parent / "results"
|
||||
|
||||
|
||||
def get_db(engagement: str) -> sqlite3.Connection:
|
||||
RESULTS_DIR.mkdir(parents=True, exist_ok=True)
|
||||
db_path = RESULTS_DIR / f"{engagement}.db"
|
||||
conn = sqlite3.connect(str(db_path))
|
||||
conn.row_factory = sqlite3.Row
|
||||
_init(conn)
|
||||
return conn
|
||||
|
||||
|
||||
def _init(conn: sqlite3.Connection) -> None:
|
||||
conn.executescript("""
|
||||
CREATE TABLE IF NOT EXISTS runs (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
started_at TEXT NOT NULL,
|
||||
finished_at TEXT,
|
||||
countries TEXT,
|
||||
targets TEXT,
|
||||
total_hosts INTEGER DEFAULT 0,
|
||||
total_matches INTEGER DEFAULT 0
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS hits (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
run_id INTEGER NOT NULL,
|
||||
ip TEXT NOT NULL,
|
||||
port INTEGER NOT NULL,
|
||||
country_code TEXT,
|
||||
target_name TEXT NOT NULL,
|
||||
tags TEXT,
|
||||
probe_type TEXT,
|
||||
confidence TEXT,
|
||||
version TEXT,
|
||||
nmap_service TEXT,
|
||||
nmap_banner TEXT,
|
||||
found_at TEXT NOT NULL,
|
||||
FOREIGN KEY (run_id) REFERENCES runs(id)
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_hits_ip ON hits(ip);
|
||||
CREATE INDEX IF NOT EXISTS idx_hits_target ON hits(target_name);
|
||||
CREATE INDEX IF NOT EXISTS idx_hits_country ON hits(country_code);
|
||||
CREATE INDEX IF NOT EXISTS idx_hits_run ON hits(run_id);
|
||||
""")
|
||||
conn.commit()
|
||||
|
||||
|
||||
def start_run(conn: sqlite3.Connection, countries: list, targets: list) -> int:
|
||||
cur = conn.execute(
|
||||
"INSERT INTO runs (started_at, countries, targets) VALUES (?, ?, ?)",
|
||||
(datetime.utcnow().isoformat(), json.dumps(countries), json.dumps(targets)),
|
||||
)
|
||||
conn.commit()
|
||||
return cur.lastrowid
|
||||
|
||||
|
||||
def finish_run(conn: sqlite3.Connection, run_id: int, total_hosts: int, total_matches: int) -> None:
|
||||
conn.execute(
|
||||
"UPDATE runs SET finished_at=?, total_hosts=?, total_matches=? WHERE id=?",
|
||||
(datetime.utcnow().isoformat(), total_hosts, total_matches, run_id),
|
||||
)
|
||||
conn.commit()
|
||||
|
||||
|
||||
def insert_hit(conn: sqlite3.Connection, run_id: int, hit: dict) -> None:
|
||||
conn.execute(
|
||||
"""INSERT INTO hits
|
||||
(run_id, ip, port, country_code, target_name, tags, probe_type,
|
||||
confidence, version, nmap_service, nmap_banner, found_at)
|
||||
VALUES (?,?,?,?,?,?,?,?,?,?,?,?)""",
|
||||
(
|
||||
run_id,
|
||||
hit["ip"],
|
||||
hit["port"],
|
||||
hit.get("country_code", ""),
|
||||
hit["target_name"],
|
||||
json.dumps(hit.get("tags", [])),
|
||||
hit.get("probe_type", ""),
|
||||
hit.get("confidence", ""),
|
||||
hit.get("version", ""),
|
||||
hit.get("nmap_service", ""),
|
||||
hit.get("nmap_banner", ""),
|
||||
datetime.utcnow().isoformat(),
|
||||
),
|
||||
)
|
||||
conn.commit()
|
||||
|
||||
|
||||
def export_json(conn: sqlite3.Connection, run_id: int | None = None) -> list[dict]:
|
||||
if run_id:
|
||||
rows = conn.execute("SELECT * FROM hits WHERE run_id=?", (run_id,)).fetchall()
|
||||
else:
|
||||
rows = conn.execute("SELECT * FROM hits").fetchall()
|
||||
return [dict(r) for r in rows]
|
||||
|
||||
|
||||
def export_csv_lines(conn: sqlite3.Connection, run_id: int | None = None) -> list[str]:
|
||||
header = "ip,port,country_code,target_name,confidence,version,nmap_service,nmap_banner,found_at"
|
||||
if run_id:
|
||||
rows = conn.execute("SELECT * FROM hits WHERE run_id=?", (run_id,)).fetchall()
|
||||
else:
|
||||
rows = conn.execute("SELECT * FROM hits ORDER BY found_at").fetchall()
|
||||
lines = [header]
|
||||
for r in rows:
|
||||
lines.append(",".join([
|
||||
str(r["ip"]), str(r["port"]), r["country_code"] or "",
|
||||
r["target_name"], r["confidence"] or "", r["version"] or "",
|
||||
r["nmap_service"] or "", r["nmap_banner"] or "", r["found_at"],
|
||||
]))
|
||||
return lines
|
||||
@@ -0,0 +1,90 @@
|
||||
import re
|
||||
from typing import Optional
|
||||
from .prober import ProbeResult
|
||||
|
||||
|
||||
def _version_compare(extracted: str, operator: str, threshold: str) -> bool:
|
||||
def to_tuple(v: str):
|
||||
try:
|
||||
return tuple(int(x) for x in v.split("."))
|
||||
except ValueError:
|
||||
return (v,)
|
||||
|
||||
ev = to_tuple(extracted)
|
||||
tv = to_tuple(threshold)
|
||||
if operator == "<=":
|
||||
return ev <= tv
|
||||
elif operator == ">=":
|
||||
return ev >= tv
|
||||
elif operator == "<":
|
||||
return ev < tv
|
||||
elif operator == ">":
|
||||
return ev > tv
|
||||
elif operator == "==":
|
||||
return ev == tv
|
||||
elif operator == "!=":
|
||||
return ev != tv
|
||||
return False
|
||||
|
||||
|
||||
def _get_search_text(result: ProbeResult, match_in) -> str:
|
||||
if isinstance(match_in, list):
|
||||
parts = []
|
||||
if "body" in match_in:
|
||||
parts.append(result.body or result.banner)
|
||||
if "headers" in match_in:
|
||||
parts.append(" ".join(f"{k}: {v}" for k, v in result.headers.items()))
|
||||
return "\n".join(parts)
|
||||
if match_in == "headers":
|
||||
return " ".join(f"{k}: {v}" for k, v in result.headers.items())
|
||||
return result.body or result.banner
|
||||
|
||||
|
||||
def match_probe(result: ProbeResult, probe: dict) -> tuple[bool, Optional[str]]:
|
||||
if not result.success:
|
||||
return False, None
|
||||
|
||||
match_in = probe.get("match_in", "body")
|
||||
text = _get_search_text(result, match_in)
|
||||
|
||||
patterns = probe.get("patterns", [])
|
||||
all_patterns = probe.get("all_patterns", [])
|
||||
|
||||
if patterns:
|
||||
if not any(re.search(p, text) for p in patterns):
|
||||
return False, None
|
||||
|
||||
if all_patterns:
|
||||
if not all(re.search(p, text) for p in all_patterns):
|
||||
return False, None
|
||||
|
||||
extracted_version = None
|
||||
version_extract = probe.get("version_extract")
|
||||
if version_extract:
|
||||
m = re.search(version_extract, text)
|
||||
if m:
|
||||
extracted_version = m.group(1)
|
||||
|
||||
version_compare = probe.get("version_compare")
|
||||
if version_compare and extracted_version:
|
||||
op = version_compare.get("operator", "==")
|
||||
val = str(version_compare.get("value", ""))
|
||||
if not _version_compare(extracted_version, op, val):
|
||||
return False, None
|
||||
elif version_compare and not extracted_version:
|
||||
return False, None
|
||||
|
||||
return True, extracted_version
|
||||
|
||||
|
||||
def match_target(host: str, port: int, probe_results: list[tuple[dict, ProbeResult]]) -> list[dict]:
|
||||
matches = []
|
||||
for probe, result in probe_results:
|
||||
matched, version = match_probe(result, probe)
|
||||
if matched:
|
||||
matches.append({
|
||||
"probe_type": result.probe_type,
|
||||
"confidence": probe.get("confidence", "medium"),
|
||||
"version": version,
|
||||
})
|
||||
return matches
|
||||
+125
@@ -0,0 +1,125 @@
|
||||
import socket
|
||||
import ssl
|
||||
import urllib.request
|
||||
import urllib.error
|
||||
from dataclasses import dataclass, field
|
||||
|
||||
CONNECT_TIMEOUT = 5
|
||||
READ_TIMEOUT = 8
|
||||
|
||||
|
||||
@dataclass
|
||||
class ProbeResult:
|
||||
probe_type: str
|
||||
success: bool
|
||||
banner: str = ""
|
||||
body: str = ""
|
||||
headers: dict = field(default_factory=dict)
|
||||
error: str = ""
|
||||
|
||||
|
||||
def _tcp_banner(host: str, port: int) -> ProbeResult:
|
||||
try:
|
||||
sock = socket.create_connection((host, port), timeout=CONNECT_TIMEOUT)
|
||||
sock.settimeout(READ_TIMEOUT)
|
||||
try:
|
||||
data = sock.recv(4096)
|
||||
return ProbeResult("tcp_banner", True, banner=data.decode("utf-8", errors="replace"))
|
||||
except socket.timeout:
|
||||
return ProbeResult("tcp_banner", True, banner="")
|
||||
finally:
|
||||
sock.close()
|
||||
except Exception as e:
|
||||
return ProbeResult("tcp_banner", False, error=str(e))
|
||||
|
||||
|
||||
def _http_probe(host: str, port: int, probe: dict, use_ssl: bool) -> ProbeResult:
|
||||
scheme = "https" if use_ssl else "http"
|
||||
path = probe.get("path", "/")
|
||||
method = probe.get("method", "GET").upper()
|
||||
extra_headers = probe.get("headers", {})
|
||||
body_data = probe.get("body", "")
|
||||
|
||||
url = f"{scheme}://{host}:{port}{path}"
|
||||
data = body_data.encode() if body_data else None
|
||||
|
||||
ctx = ssl.create_default_context()
|
||||
ctx.check_hostname = False
|
||||
ctx.verify_mode = ssl.CERT_NONE
|
||||
|
||||
req = urllib.request.Request(url, data=data, method=method)
|
||||
req.add_header("User-Agent", "Mozilla/5.0")
|
||||
req.add_header("Connection", "close")
|
||||
for k, v in extra_headers.items():
|
||||
req.add_header(k, v)
|
||||
|
||||
try:
|
||||
handler = urllib.request.HTTPSHandler(context=ctx) if use_ssl else urllib.request.HTTPHandler()
|
||||
opener = urllib.request.build_opener(handler)
|
||||
opener.addheaders = []
|
||||
with opener.open(req, timeout=READ_TIMEOUT) as resp:
|
||||
raw = resp.read(65536)
|
||||
body = raw.decode("utf-8", errors="replace")
|
||||
headers = dict(resp.headers)
|
||||
return ProbeResult(
|
||||
"https" if use_ssl else "http",
|
||||
True,
|
||||
body=body,
|
||||
headers=headers,
|
||||
)
|
||||
except urllib.error.HTTPError as e:
|
||||
try:
|
||||
body = e.read(16384).decode("utf-8", errors="replace")
|
||||
except Exception:
|
||||
body = ""
|
||||
return ProbeResult("https" if use_ssl else "http", True, body=body, headers=dict(e.headers))
|
||||
except Exception as e:
|
||||
return ProbeResult("https" if use_ssl else "http", False, error=str(e))
|
||||
|
||||
|
||||
def _rtsp_probe(host: str, port: int) -> ProbeResult:
|
||||
try:
|
||||
sock = socket.create_connection((host, port), timeout=CONNECT_TIMEOUT)
|
||||
sock.settimeout(READ_TIMEOUT)
|
||||
request = f"OPTIONS * RTSP/1.0\r\nCSeq: 1\r\nUser-Agent: geo-scout\r\n\r\n"
|
||||
sock.sendall(request.encode())
|
||||
try:
|
||||
data = sock.recv(4096)
|
||||
return ProbeResult("rtsp", True, banner=data.decode("utf-8", errors="replace"))
|
||||
except socket.timeout:
|
||||
return ProbeResult("rtsp", False, error="timeout")
|
||||
finally:
|
||||
sock.close()
|
||||
except Exception as e:
|
||||
return ProbeResult("rtsp", False, error=str(e))
|
||||
|
||||
|
||||
def _udp_probe(host: str, port: int) -> ProbeResult:
|
||||
try:
|
||||
sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
|
||||
sock.settimeout(CONNECT_TIMEOUT)
|
||||
sock.sendto(b"", (host, port))
|
||||
try:
|
||||
data, _ = sock.recvfrom(4096)
|
||||
return ProbeResult("udp", True, banner=data.decode("utf-8", errors="replace"))
|
||||
except socket.timeout:
|
||||
return ProbeResult("udp", False, error="timeout")
|
||||
finally:
|
||||
sock.close()
|
||||
except Exception as e:
|
||||
return ProbeResult("udp", False, error=str(e))
|
||||
|
||||
|
||||
def run_probe(host: str, port: int, probe: dict) -> ProbeResult:
|
||||
ptype = probe.get("type", "tcp_banner")
|
||||
if ptype == "tcp_banner":
|
||||
return _tcp_banner(host, port)
|
||||
elif ptype == "http":
|
||||
return _http_probe(host, port, probe, use_ssl=False)
|
||||
elif ptype == "https":
|
||||
return _http_probe(host, port, probe, use_ssl=True)
|
||||
elif ptype == "rtsp":
|
||||
return _rtsp_probe(host, port)
|
||||
elif ptype == "udp":
|
||||
return _udp_probe(host, port)
|
||||
return ProbeResult(ptype, False, error=f"unknown probe type: {ptype}")
|
||||
+118
@@ -0,0 +1,118 @@
|
||||
import json
|
||||
import shutil
|
||||
import subprocess
|
||||
import tempfile
|
||||
import xml.etree.ElementTree as ET
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
def check_deps() -> list[str]:
|
||||
missing = []
|
||||
for tool in ("masscan", "nmap"):
|
||||
if not shutil.which(tool):
|
||||
missing.append(tool)
|
||||
return missing
|
||||
|
||||
|
||||
def masscan_sweep(cidrs: list[str], ports: list[int], rate: int, output_dir: Path) -> list[dict]:
|
||||
if not cidrs or not ports:
|
||||
return []
|
||||
|
||||
port_str = ",".join(str(p) for p in sorted(set(ports)))
|
||||
cidr_file = output_dir / "targets.txt"
|
||||
cidr_file.write_text("\n".join(cidrs))
|
||||
out_file = output_dir / "masscan.json"
|
||||
|
||||
cmd = [
|
||||
"masscan",
|
||||
f"--rate={rate}",
|
||||
f"--ports={port_str}",
|
||||
"-iL", str(cidr_file),
|
||||
"-oJ", str(out_file),
|
||||
"--output-format", "json",
|
||||
"--wait", "3",
|
||||
]
|
||||
|
||||
try:
|
||||
subprocess.run(cmd, capture_output=True, timeout=3600)
|
||||
except subprocess.TimeoutExpired:
|
||||
pass
|
||||
except FileNotFoundError:
|
||||
raise RuntimeError("masscan not found — install it first")
|
||||
|
||||
if not out_file.exists():
|
||||
return []
|
||||
|
||||
raw = out_file.read_text(errors="replace").strip()
|
||||
if not raw or raw == "[]":
|
||||
return []
|
||||
|
||||
raw = raw.rstrip(",\n")
|
||||
if not raw.endswith("]"):
|
||||
raw += "]"
|
||||
if not raw.startswith("["):
|
||||
raw = "[" + raw
|
||||
|
||||
try:
|
||||
data = json.loads(raw)
|
||||
except json.JSONDecodeError:
|
||||
return []
|
||||
|
||||
results = []
|
||||
for entry in data:
|
||||
ip = entry.get("ip")
|
||||
for port_entry in entry.get("ports", []):
|
||||
port = port_entry.get("port")
|
||||
if ip and port:
|
||||
results.append({"ip": ip, "port": port})
|
||||
return results
|
||||
|
||||
|
||||
def nmap_fingerprint(host: str, ports: list[int], output_dir: Path) -> dict:
|
||||
if not ports:
|
||||
return {}
|
||||
|
||||
port_str = ",".join(str(p) for p in sorted(set(ports)))
|
||||
out_file = output_dir / f"nmap_{host.replace('.', '_')}.xml"
|
||||
|
||||
cmd = [
|
||||
"nmap",
|
||||
"-sV", "--version-intensity", "5",
|
||||
"-p", port_str,
|
||||
"-T4",
|
||||
"--open",
|
||||
"-oX", str(out_file),
|
||||
host,
|
||||
]
|
||||
|
||||
try:
|
||||
subprocess.run(cmd, capture_output=True, timeout=60)
|
||||
except (subprocess.TimeoutExpired, FileNotFoundError):
|
||||
return {}
|
||||
|
||||
if not out_file.exists():
|
||||
return {}
|
||||
|
||||
try:
|
||||
tree = ET.parse(out_file)
|
||||
except ET.ParseError:
|
||||
return {}
|
||||
|
||||
result = {}
|
||||
for port_el in tree.findall(".//port"):
|
||||
portid = int(port_el.get("portid", 0))
|
||||
state = port_el.find("state")
|
||||
if state is None or state.get("state") != "open":
|
||||
continue
|
||||
service = port_el.find("service")
|
||||
info = {}
|
||||
if service is not None:
|
||||
info["service"] = service.get("name", "")
|
||||
info["product"] = service.get("product", "")
|
||||
info["version"] = service.get("version", "")
|
||||
info["extrainfo"] = service.get("extrainfo", "")
|
||||
info["banner"] = " ".join(filter(None, [
|
||||
info["product"], info["version"], info["extrainfo"]
|
||||
]))
|
||||
result[portid] = info
|
||||
return result
|
||||
Reference in New Issue
Block a user