Phase-accurate webrunner cost/time estimator with masscan-Tor OPSEC warnings
Estimator rewrite: - Phase-decomposed estimate_scan_hours: masscan, nmap-only, nmap-fingerprint, probe, nuclei phases each timed separately - Accepts 6 tuning params (nmap_timing, nmap_workers, nuclei_rate, nuclei_concurrency, hit_rate, use_tor) — was previously ignoring all of them - nmap-only mode fixed: was underestimated ~100x (treated nmap like packet scanning); now uses NMAP_TIME_BY_TIMING table per timing template - masscan+nuclei mode now properly modeled (was falling through to masscan-only formula, missing the nuclei phase entirely) - Tor multiplier scoped per-phase: applied to nmap/nuclei/probe only. Masscan uses raw sockets and Tor cannot proxy raw sockets — was previously applying Tor multiplier to masscan rate (incorrect) - 5min/node provisioning overhead added (matters vs Linode 1h billing min) - build_estimate_table accepts tuning dict, threads through to engine Operator-facing changes (deploy_webrunner.py): - Tuning section moved BEFORE estimate display so numbers reflect actual operator choices, not defaults - Estimate header now shows active tuning values when set - Caveats block printed below estimate explaining empirical limits (hit rate range, nuclei template variance, nmap timing tradeoffs, provisioning, billing minimums) - CRITICAL OPSEC banner displayed when Tor enabled with masscan modes: loud red warning that masscan SYN packets bypass Tor and reveal cloud node IP. Includes confirmation gate — operator must affirm they understand before proceeding. Suggests nmap-only mode for full Tor coverage if attribution risk is unacceptable Why: operator was getting estimates off by 100x for nmap-only and 0% accurate for masscan+nuclei. Tuning controls were invisible to the estimator. Tor + masscan was silently broken-by-design (raw sockets bypass proxychains) without operator awareness.
This commit is contained in:
n0mad1k
+106
-20
@@ -48,20 +48,101 @@ BILLING_MINIMUM = {
|
||||
}
|
||||
|
||||
|
||||
NMAP_TIME_PER_HOST_SEC = 10 # empirical: nmap -sV -T4 per host, seconds
|
||||
MASSCAN_HIT_RATE = 0.01 # fraction of scanned IPs that have open ports
|
||||
_NMAP_WORKERS = 10 # must match WEBRUNNER_NMAP_WORKERS default
|
||||
# ── Empirical timing constants ────────────────────────────────────────────────
|
||||
# Per-host nmap time by timing template (seconds, -sV --version-intensity 5)
|
||||
NMAP_TIME_BY_TIMING = {
|
||||
1: 60.0, # T1 sneaky
|
||||
2: 30.0, # T2 polite
|
||||
3: 15.0, # T3 normal
|
||||
4: 10.0, # T4 aggressive (baseline)
|
||||
}
|
||||
|
||||
# Average per-target time for typical nuclei CVE template (1–3 HTTP requests).
|
||||
# Heavy templates with many requests/matchers will run 2–5× longer.
|
||||
NUCLEI_TIME_PER_TARGET_SEC = 5.0
|
||||
|
||||
# TCP banner grab time (geo-scout probe phase)
|
||||
PROBE_TIME_PER_HOST_SEC = 3.0
|
||||
|
||||
# Default fraction of scanned IPs with open ports on common ports.
|
||||
# Real-world range: 0.5%–5% depending on ports & geography.
|
||||
MASSCAN_HIT_RATE = 0.01
|
||||
|
||||
# Defaults — must match WEBRUNNER tuning defaults
|
||||
_NMAP_WORKERS = 10
|
||||
_NUCLEI_RATE = 150
|
||||
_NUCLEI_CONCURRENCY = 25
|
||||
|
||||
# Tor latency multiplier — applies ONLY to TCP probe phases (nmap/nuclei/probe).
|
||||
# Masscan uses raw sockets and bypasses proxychains entirely — Tor cannot
|
||||
# protect masscan SYN packets. The cloud node's IP is exposed to every
|
||||
# masscan target regardless of this setting.
|
||||
TOR_PHASE_MULTIPLIER = 5.0
|
||||
|
||||
# Per-node provisioning overhead (apt install, optional nuclei download).
|
||||
# Nodes provision in parallel so this is roughly constant.
|
||||
PROVISION_OVERHEAD_HOURS = 5 / 60
|
||||
|
||||
|
||||
def estimate_scan_hours(ip_count: int, n_ports: int, rate: int, scan_mode: str = 'masscan-only') -> float:
|
||||
def estimate_scan_hours(
|
||||
ip_count: int,
|
||||
n_ports: int,
|
||||
rate: int,
|
||||
scan_mode: str = 'masscan-only',
|
||||
*,
|
||||
nmap_timing: int = 4,
|
||||
nmap_workers: int = _NMAP_WORKERS,
|
||||
nuclei_rate: int = _NUCLEI_RATE,
|
||||
nuclei_concurrency: int = _NUCLEI_CONCURRENCY,
|
||||
hit_rate: float = MASSCAN_HIT_RATE,
|
||||
use_tor: bool = False,
|
||||
) -> float:
|
||||
"""Phase-decomposed scan time estimate. Returns hours per node."""
|
||||
if rate <= 0:
|
||||
return 0.0
|
||||
masscan_hours = (ip_count * n_ports / rate) / 3600
|
||||
return PROVISION_OVERHEAD_HOURS
|
||||
|
||||
nmap_workers = max(nmap_workers, 1)
|
||||
seconds = 0.0
|
||||
|
||||
# masscan phase: raw sockets, no Tor penalty (Tor cannot proxy raw sockets)
|
||||
if scan_mode in ('masscan-only', 'masscan+nmap', 'geo-scout', 'masscan+nuclei'):
|
||||
seconds += ip_count * n_ports / rate
|
||||
|
||||
# nmap-only phase: every IP gets full -sV fingerprint
|
||||
if scan_mode == 'nmap-only':
|
||||
per_host = NMAP_TIME_BY_TIMING.get(nmap_timing, 10.0)
|
||||
if use_tor:
|
||||
per_host *= TOR_PHASE_MULTIPLIER
|
||||
seconds += (ip_count * per_host) / nmap_workers
|
||||
|
||||
# nmap fingerprint phase: only on masscan hits
|
||||
if scan_mode in ('masscan+nmap', 'geo-scout'):
|
||||
nmap_hosts = ip_count * MASSCAN_HIT_RATE
|
||||
nmap_hours = (nmap_hosts * NMAP_TIME_PER_HOST_SEC) / (_NMAP_WORKERS * 3600)
|
||||
return masscan_hours + nmap_hours
|
||||
return masscan_hours
|
||||
nmap_hosts = ip_count * hit_rate
|
||||
per_host = NMAP_TIME_BY_TIMING.get(nmap_timing, 10.0)
|
||||
if use_tor:
|
||||
per_host *= TOR_PHASE_MULTIPLIER
|
||||
seconds += (nmap_hosts * per_host) / nmap_workers
|
||||
|
||||
# probe banner phase: geo-scout only, on masscan hits
|
||||
if scan_mode == 'geo-scout':
|
||||
probe_hosts = ip_count * hit_rate
|
||||
per_probe = PROBE_TIME_PER_HOST_SEC
|
||||
if use_tor:
|
||||
per_probe *= TOR_PHASE_MULTIPLIER
|
||||
seconds += (probe_hosts * per_probe) / nmap_workers
|
||||
|
||||
# nuclei phase: only on masscan-discovered ip:port pairs
|
||||
if scan_mode == 'masscan+nuclei':
|
||||
nuclei_targets = ip_count * hit_rate
|
||||
per_target = NUCLEI_TIME_PER_TARGET_SEC
|
||||
if use_tor:
|
||||
per_target *= TOR_PHASE_MULTIPLIER
|
||||
rate_throughput = float(nuclei_rate)
|
||||
parallel_throughput = nuclei_concurrency / per_target
|
||||
effective_rps = max(min(rate_throughput, parallel_throughput), 1.0)
|
||||
seconds += nuclei_targets / effective_rps
|
||||
|
||||
return seconds / 3600 + PROVISION_OVERHEAD_HOURS
|
||||
|
||||
|
||||
def node_cost(provider: str, instance_type: str, scan_hours: float) -> float:
|
||||
@@ -84,30 +165,35 @@ def fmt_ip_count(n: int) -> str:
|
||||
return str(n)
|
||||
|
||||
|
||||
# Tor adds ~5x latency overhead for TCP probes; masscan bypasses proxychains (raw sockets)
|
||||
# so only nmap/probe phases are affected — modes that include masscan see partial impact
|
||||
TOR_RATE_MULTIPLIER = 0.2
|
||||
|
||||
|
||||
def build_estimate_table(
|
||||
total_ips: int,
|
||||
n_ports: int,
|
||||
providers: list[str],
|
||||
scan_mode: str,
|
||||
use_tor: bool = False,
|
||||
tuning: dict | None = None,
|
||||
) -> list[dict]:
|
||||
mode_rate = SCAN_MODES.get(scan_mode, {'rate': 3000})['rate']
|
||||
if use_tor and scan_mode != 'masscan-only':
|
||||
mode_rate = int(mode_rate * TOR_RATE_MULTIPLIER)
|
||||
tuning = tuning or {}
|
||||
masscan_rate = int(tuning.get('masscan_rate') or SCAN_MODES.get(scan_mode, {'rate': 3000})['rate'])
|
||||
|
||||
kwargs = dict(
|
||||
nmap_timing=int(tuning.get('nmap_timing', 4)),
|
||||
nmap_workers=int(tuning.get('nmap_workers', _NMAP_WORKERS)),
|
||||
nuclei_rate=int(tuning.get('nuclei_rate', _NUCLEI_RATE)),
|
||||
nuclei_concurrency=int(tuning.get('nuclei_concurrency', _NUCLEI_CONCURRENCY)),
|
||||
hit_rate=float(tuning.get('hit_rate', MASSCAN_HIT_RATE)),
|
||||
use_tor=use_tor,
|
||||
)
|
||||
|
||||
rows = []
|
||||
for preset_key, preset in PRESETS.items():
|
||||
n_chunks = max(1, math.ceil(total_ips / preset['chunk_size']))
|
||||
typical_ips = min(preset['chunk_size'], total_ips)
|
||||
hours_per_node = estimate_scan_hours(typical_ips, n_ports, mode_rate, scan_mode)
|
||||
hours_per_node = estimate_scan_hours(typical_ips, n_ports, masscan_rate, scan_mode, **kwargs)
|
||||
total_cost = 0.0
|
||||
for i in range(n_chunks):
|
||||
chunk_ips = min(preset['chunk_size'], total_ips - i * preset['chunk_size'])
|
||||
chunk_hours = estimate_scan_hours(chunk_ips, n_ports, mode_rate, scan_mode)
|
||||
chunk_hours = estimate_scan_hours(chunk_ips, n_ports, masscan_rate, scan_mode, **kwargs)
|
||||
provider = providers[i % len(providers)]
|
||||
instance = DEFAULT_INSTANCE.get(provider, 'g6-standard-2')
|
||||
total_cost += node_cost(provider, instance, chunk_hours)
|
||||
|
||||
Reference in New Issue
Block a user