Add masscan+nuclei mode, operator tuning controls, and vars file support to webrunner

- New scan mode: masscan+nuclei — masscan finds open ip:port pairs,
  nuclei runs operator-supplied CVE template against discovered hosts
- Per-country vulnerable host count in merge output via CIDR→country lookup
- Tuning args on every scan: --nmap-timing, --nmap-timeout, --nmap-workers,
  --nuclei-rate, --nuclei-concurrency, --nuclei-timeout, --masscan-rate
- Vars file support: operator provides scan_vars.yaml to pre-fill all
  tuning settings without interactive prompts
- Templates copied to nodes at provision time — no live fetches (OPSEC)
- run_scan.yml passes all tuning args with Ansible defaults as fallback
- configure_node.yml installs nuclei binary + uploads template on demand
- Updated deployment summary shows mode-specific tuning params
- countries-format.md and targets-format.md updated for new capabilities
- scan_vars.yaml.example documents all configurable settings with comments
This commit is contained in:
n0mad1k
2026-05-02 14:49:24 -04:00
parent 9d8839e1ad
commit 13ce8a9b8a
9 changed files with 406 additions and 13 deletions
+104
View File
@@ -112,6 +112,24 @@ def _select_scan_mode() -> str:
return 'geo-scout'
def _load_vars_file(path: str) -> dict:
if not path:
return {}
p = Path(path)
if not p.exists():
print(f"{COLORS['YELLOW']} Vars file not found: {path} — continuing interactively{COLORS['RESET']}")
return {}
import yaml
try:
with open(p) as f:
data = yaml.safe_load(f) or {}
print(f"{COLORS['GREEN']} Loaded vars: {p.name}{COLORS['RESET']}")
return data
except Exception as e:
print(f"{COLORS['YELLOW']} Could not parse vars file ({e}) — continuing interactively{COLORS['RESET']}")
return {}
def _get_targets_info(scan_mode: str) -> tuple[str, list[int]]:
if scan_mode == 'geo-scout':
default = str(WEBRUNNER_INPUTS / 'targets.yaml')
@@ -141,6 +159,55 @@ def _get_targets_info(scan_mode: str) -> tuple[str, list[int]]:
return "", sorted(set(ports_list)) or [80, 443, 22]
def _get_nuclei_info(vars_overrides: dict) -> tuple[str, str]:
"""Returns (local_template_path, remote_template_path)."""
print(f"\n{COLORS['BLUE']}Nuclei Template:{COLORS['RESET']}")
if 'nuclei_template' in vars_overrides:
local_path = vars_overrides['nuclei_template']
print(f" Template: {local_path} (from vars file)")
else:
local_path = input(" Path to nuclei template (.yaml): ").strip()
if not local_path or not Path(local_path).exists():
print(f"{COLORS['RED']} Template not found: {local_path}{COLORS['RESET']}")
return "", ""
return local_path, "/root/webrunner/nuclei_template.yaml"
def _get_tuning_params(scan_mode: str, default_rate: int, vars_overrides: dict) -> dict:
"""Gather advanced tuning params. Returns dict of tuning config keys."""
tuning: dict = {}
show_header = True
def _prompt(label: str, key: str, default, cast=int) -> None:
nonlocal show_header
if show_header:
print(f"\n{COLORS['BLUE']}Advanced Tuning (Enter = use default):{COLORS['RESET']}")
show_header = False
if key in vars_overrides:
tuning[key] = cast(vars_overrides[key])
print(f" {label}: {tuning[key]} (vars file)")
else:
raw = input(f" {label} [{default}]: ").strip()
tuning[key] = cast(raw) if raw else default
_prompt("masscan rate (pkt/s)", "masscan_rate", default_rate)
if scan_mode in ('masscan+nmap', 'geo-scout', 'masscan+nuclei'):
pass # masscan_rate already handled
if scan_mode in ('masscan+nmap', 'geo-scout'):
_prompt("nmap timing T1-T4", "nmap_timing", 4)
_prompt("nmap per-host timeout (s)", "nmap_timeout", 60)
_prompt("nmap parallel workers", "nmap_workers", 10)
if scan_mode == 'masscan+nuclei':
_prompt("nuclei rate limit (req/s)", "nuclei_rate", 150)
_prompt("nuclei concurrency", "nuclei_concurrency", 25)
_prompt("nuclei timeout (s)", "nuclei_timeout", 10)
return tuning
def _show_estimate_table(total_ips: int, n_ports: int, providers: list[str], scan_mode: str, use_tor: bool = False):
rows = build_estimate_table(total_ips, n_ports, providers, scan_mode, use_tor=use_tor)
@@ -188,6 +255,10 @@ def gather_webrunner_parameters() -> dict | None:
config['deployment_id'] = generate_deployment_id()
print(f"Deployment ID: {COLORS['CYAN']}{config['deployment_id']}{COLORS['RESET']}")
# Optional vars file — pre-fills tuning defaults
vars_raw = input("Vars file (optional, skip to configure interactively): ").strip()
vars_overrides = _load_vars_file(vars_raw)
# Engagement name — auto if blank
raw_eng = input(f"Engagement name [{config['deployment_id']}]: ").strip()
config['engagement'] = raw_eng or config['deployment_id']
@@ -232,6 +303,16 @@ def gather_webrunner_parameters() -> dict | None:
scan_mode = _select_scan_mode()
config['scan_mode'] = scan_mode
# Nuclei template (masscan+nuclei only)
config['nuclei_template_local'] = ''
config['nuclei_template_remote'] = ''
if scan_mode == 'masscan+nuclei':
local_tmpl, remote_tmpl = _get_nuclei_info(vars_overrides)
if not local_tmpl:
return None
config['nuclei_template_local'] = local_tmpl
config['nuclei_template_remote'] = remote_tmpl
# Targets / ports
targets_file, ports = _get_targets_info(scan_mode)
config['targets_file'] = targets_file
@@ -299,6 +380,8 @@ def gather_webrunner_parameters() -> dict | None:
config['preset'] = preset_key
config['chunk_size'] = chunk_size
config['cidr_country_map_file'] = '' # filled in execute_webrunner_deployment
# Distribute CIDRs into node chunks
all_cidrs: list[str] = []
for cc in country_codes:
@@ -355,6 +438,11 @@ def gather_webrunner_parameters() -> dict | None:
if config['operator_ip']:
print(f"{COLORS['GREEN']}Operator IP: {config['operator_ip']}{COLORS['RESET']}")
# Advanced tuning
default_rate = config['masscan_rate']
tuning = _get_tuning_params(scan_mode, default_rate, vars_overrides)
config.update(tuning)
# OPSEC / teardown options
print(f"\n{COLORS['BLUE']}Deployment Options:{COLORS['RESET']}")
@@ -404,6 +492,12 @@ def execute_webrunner_deployment(config: dict):
print(f" Nodes: {n_nodes} ({config['preset']}, {fmt_ip_count(config['chunk_size'])}/node)")
print(f" Scan mode: {config['scan_mode']}")
print(f" Ports: {', '.join(str(p) for p in config['ports'][:8])}{'...' if len(config['ports']) > 8 else ''}")
print(f" masscan rate: {config.get('masscan_rate', '')} pkt/s")
if config['scan_mode'] in ('masscan+nmap', 'geo-scout'):
print(f" nmap: T{config.get('nmap_timing', 4)} timeout={config.get('nmap_timeout', 60)}s workers={config.get('nmap_workers', 10)}")
if config['scan_mode'] == 'masscan+nuclei':
tmpl_name = os.path.basename(config.get('nuclei_template_local', ''))
print(f" nuclei: {tmpl_name} rate={config.get('nuclei_rate', 150)} concurrency={config.get('nuclei_concurrency', 25)} timeout={config.get('nuclei_timeout', 10)}s")
print(f" Total IPs: {fmt_ip_count(config['total_ips'])}")
print(f" Tor routing: {'Yes' if config['use_tor'] else 'No'}")
print(f" Enhanced OPSEC: {'Yes' if config['enhanced_opsec'] else 'No'}")
@@ -421,6 +515,16 @@ def execute_webrunner_deployment(config: dict):
# regardless of playbook-relative CWD
logs_dir = os.path.abspath(os.path.join(_base, 'logs'))
os.makedirs(logs_dir, exist_ok=True)
# Save CIDR→country map for merge_results per-country attribution
cc_map_file = os.path.join(logs_dir, f"cidr_country_map_{config['deployment_id']}.json")
if config.get('country_codes'):
from utils.chunk_utils import get_country_cidrs
cc_cidrs_saved = get_country_cidrs(config['country_codes'], {})
with open(cc_map_file, 'w') as f:
json.dump(cc_cidrs_saved, f)
config['cidr_country_map_file'] = cc_map_file
chunks_file = os.path.join(logs_dir, f"node_chunks_{config['deployment_id']}.json")
with open(chunks_file, 'w') as f:
json.dump(node_chunks, f)
+15 -3
View File
@@ -1,9 +1,21 @@
# countries.yaml — Format Specification
## Purpose
Defines which countries to scan and scan parameters per country.
This file is read by WEBRUNNER at runtime. The scanner resolves each
country code to its CIDR ranges using RIR delegated stats files.
Defines which countries to scan and optional per-country exclusions.
Used by all WEBRUNNER scan modes. The scanner resolves each country code
to its CIDR ranges using RIR delegated stats files (ARIN, RIPE, APNIC,
LACNIC, AFRINIC — cached 24h locally).
## Scope options
- **Targeted countries**: list specific ISO codes in this file
- **Single country**: just one entry
- **Global sweep**: include every country you want — WEBRUNNER distributes
CIDRs across nodes automatically regardless of count
## Relationship to scan_vars.yaml
`scan_profile.rate` in this file sets the masscan default, but it is
overridden by `masscan_rate` in `scan_vars.yaml` or the interactive tuning
prompt. Prefer `scan_vars.yaml` for operator-level tuning.
## Schema
@@ -0,0 +1,31 @@
# scan_vars.yaml — WEBRUNNER pre-configuration
# Copy this file, fill in values, and provide the path at the "Vars file" prompt.
# All fields are optional. Omit any field to be prompted interactively.
# ── masscan ───────────────────────────────────────────────────────────────────
masscan_rate: 5000 # packets/sec (default: mode-dependent, 300010000)
# Lower for clients with strict IPS/rate-limiting
# ── nmap (masscan+nmap and geo-scout modes) ───────────────────────────────────
nmap_timing: 3 # T1=sneaky T2=polite T3=normal T4=aggressive (default: 4)
nmap_timeout: 120 # per-host timeout in seconds (default: 60)
# Increase for slow/filtered networks
nmap_workers: 10 # parallel nmap threads per node (default: 10)
# ── nuclei (masscan+nuclei mode only) ─────────────────────────────────────────
nuclei_template: "/path/to/your/cve-template.yaml"
# Local path — copied to nodes at provision time
# Never fetched from the internet during scans
nuclei_rate: 150 # requests/sec rate limit (default: 150)
# Lower for clients with WAF/rate-limiting
nuclei_concurrency: 25 # concurrent goroutines (default: 25)
nuclei_timeout: 10 # per-request timeout in seconds (default: 10)
# ── example: conservative client profile ─────────────────────────────────────
# masscan_rate: 1000
# nmap_timing: 2
# nmap_timeout: 180
# nmap_workers: 5
# nuclei_rate: 50
# nuclei_concurrency: 10
# nuclei_timeout: 20
+39 -3
View File
@@ -1,9 +1,45 @@
# targets.yaml — Format Specification
## Purpose
Defines what to look for during a scan. Each target is a fingerprint with
one or more probes. The scanner runs masscan to find open ports, then fires
each probe against matching hosts, and applies pattern/version matching.
Defines what to look for during **geo-scout** mode scans. Each target is a
fingerprint with one or more probes. The scanner runs masscan to find open
ports, then fires each probe against matching hosts, and applies
pattern/version matching.
For **masscan+nuclei** mode, use a nuclei template instead (see below).
targets.yaml is ignored in nuclei mode.
## Nuclei template mode (masscan+nuclei)
Provide a standard nuclei YAML template file. WEBRUNNER:
1. Runs masscan to find open `ip:port` pairs
2. Feeds those pairs as targets to nuclei with your template
3. Outputs per-host match results + per-country vulnerable host counts
**OPSEC note:** Templates are copied to nodes at provision time from your
local path. No live template fetches happen during scans.
**Template path:** Specify in `scan_vars.yaml` (`nuclei_template: /path/to/template.yaml`)
or enter the path at the interactive prompt.
### Minimal nuclei template structure
```yaml
id: cve-2024-example
info:
name: Example CVE
severity: critical
tags: [cve, rce]
http:
- method: GET
path:
- "{{BaseURL}}/vulnerable/endpoint"
matchers:
- type: word
words:
- "vulnerable_string"
```
---
## Schema
@@ -15,6 +15,7 @@
- nmap
- python3
- python3-pip
- curl
state: present
retries: 3
delay: 10
@@ -76,3 +77,22 @@
mode: '0644'
when: scan_mode == 'geo-scout' and targets_file != ""
ignore_errors: true
- name: Install nuclei vulnerability scanner
shell: |
if ! command -v nuclei &>/dev/null; then
curl -sL "https://github.com/projectdiscovery/nuclei/releases/download/v3.3.9/nuclei_3.3.9_linux_amd64.tar.gz" | tar -xz -C /usr/local/bin nuclei
chmod +x /usr/local/bin/nuclei
fi
args:
executable: /bin/bash
when: scan_mode == 'masscan+nuclei'
retries: 2
delay: 5
- name: Upload nuclei template
copy:
src: "{{ nuclei_template_local }}"
dest: /root/webrunner/nuclei_template.yaml
mode: '0644'
when: scan_mode == 'masscan+nuclei' and nuclei_template_local | default('') != ''
+51
View File
@@ -5,16 +5,42 @@ Run by Ansible on the controller after all nodes complete.
"""
import argparse
import ipaddress
import json
import sys
from pathlib import Path
def _build_cidr_map(cc_cidrs: dict) -> list[tuple]:
result = []
for cc, cidrs in cc_cidrs.items():
for cidr in cidrs:
try:
net = ipaddress.ip_network(cidr, strict=False)
result.append((net, cc.upper()))
except ValueError:
pass
result.sort(key=lambda x: x[0].prefixlen, reverse=True)
return result
def _lookup_country(ip: str, cidr_map: list[tuple]) -> str:
try:
addr = ipaddress.ip_address(ip)
except ValueError:
return ""
for net, cc in cidr_map:
if addr in net:
return cc
return ""
def main():
parser = argparse.ArgumentParser()
parser.add_argument("--results-dir", required=True)
parser.add_argument("--output", required=True)
parser.add_argument("--deployment-id", required=True)
parser.add_argument("--cidr-map", default="", help="JSON file mapping country codes to CIDR lists")
args = parser.parse_args()
results_dir = Path(args.results_dir)
@@ -24,6 +50,14 @@ def main():
print(f"No result files found in {results_dir}", file=sys.stderr)
sys.exit(1)
cidr_map: list[tuple] = []
if args.cidr_map:
try:
cc_cidrs = json.loads(Path(args.cidr_map).read_text())
cidr_map = _build_cidr_map(cc_cidrs)
except (OSError, json.JSONDecodeError):
pass
merged: dict = {
"deployment_id": args.deployment_id,
"nodes": [],
@@ -51,16 +85,33 @@ def main():
key = f"{entry.get('ip')}:{entry.get('port')}"
if key not in seen:
seen.add(key)
if cidr_map:
entry["country"] = _lookup_country(entry.get("ip", ""), cidr_map)
merged["results"].append(entry)
merged["total_results"] = len(merged["results"])
if cidr_map:
country_counts: dict[str, int] = {}
for entry in merged["results"]:
cc = entry.get("country", "")
country_counts[cc] = country_counts.get(cc, 0) + 1
merged["country_summary"] = dict(
sorted(country_counts.items(), key=lambda x: x[1], reverse=True)
)
Path(args.output).write_text(json.dumps(merged, indent=2))
print(f"Merged {len(node_files)} node(s) — {merged['total_results']} unique results")
for n in merged["nodes"]:
print(f" {n['node_name']}: {n['result_count']} results ({n['scan_mode']})")
if cidr_map and merged.get("country_summary"):
print("\nVulnerable hosts by country:")
for cc, count in list(merged["country_summary"].items())[:20]:
label = cc if cc else "unknown"
print(f" {label:<6} {count}")
if __name__ == "__main__":
main()
+113 -3
View File
@@ -7,6 +7,7 @@ Reads cidrs.txt, runs scan pipeline, writes results.json.
import argparse
import json
import os
import re
import subprocess
import sys
import time
@@ -16,6 +17,8 @@ from pathlib import Path
WORKDIR = Path("/root/webrunner")
NMAP_WORKERS = int(os.environ.get("WEBRUNNER_NMAP_WORKERS", "10"))
NMAP_TIMING = int(os.environ.get("WEBRUNNER_NMAP_TIMING", "4"))
NMAP_TIMEOUT = int(os.environ.get("WEBRUNNER_NMAP_TIMEOUT", "60"))
def log(msg: str):
@@ -83,11 +86,11 @@ def run_nmap(ip: str, ports: list[int]) -> dict:
cmd = [
"nmap", "-sV", "--version-intensity", "5",
"-p", port_str, "-T4", "--open",
"-p", port_str, f"-T{NMAP_TIMING}", "--open",
"-oX", str(out_file), ip,
]
try:
subprocess.run(cmd, capture_output=True, timeout=60, check=False)
subprocess.run(cmd, capture_output=True, timeout=NMAP_TIMEOUT, check=False)
except (subprocess.TimeoutExpired, FileNotFoundError):
return {}
@@ -267,15 +270,116 @@ def scan_geo_scout(ports: str, rate: int, node_name: str) -> list[dict]:
return results
def run_nuclei(targets: list[str], template: str, rate: int, concurrency: int, timeout: int) -> list[dict]:
targets_file = WORKDIR / "nuclei_targets.txt"
targets_file.write_text("\n".join(targets) + "\n")
out_file = WORKDIR / "nuclei_results.jsonl"
out_file.unlink(missing_ok=True)
cmd = [
"nuclei",
"-t", template,
"-l", str(targets_file),
"-rl", str(rate),
"-c", str(concurrency),
"-timeout", str(timeout),
"-j",
"-o", str(out_file),
"-silent",
"-no-color",
"-disable-update-check",
]
log(f"nuclei starting: template={template} targets={len(targets)} rate={rate} concurrency={concurrency}")
try:
subprocess.run(cmd, timeout=43200, check=False)
except subprocess.TimeoutExpired:
log("nuclei timed out after 12h")
if not out_file.exists():
return []
results = []
for line in out_file.read_text(errors="replace").splitlines():
line = line.strip()
if not line:
continue
try:
entry = json.loads(line)
except json.JSONDecodeError:
continue
host = entry.get("host", "")
ip = entry.get("ip", "")
if not ip and host:
raw = host.split("//")[-1].split("/")[0]
ip_part = raw.rsplit(":", 1)[0] if ":" in raw else raw
m = re.match(r"^(\d+\.\d+\.\d+\.\d+)$", ip_part)
ip = m.group(1) if m else ip_part
port = 0
matched_at = entry.get("matched-at", host)
raw_host = matched_at.split("//")[-1].split("/")[0]
if ":" in raw_host:
try:
port = int(raw_host.rsplit(":", 1)[1])
except (ValueError, IndexError):
pass
results.append({
"ip": ip,
"port": port,
"template_id": entry.get("template-id", ""),
"vuln_name": entry.get("info", {}).get("name", ""),
"severity": entry.get("info", {}).get("severity", ""),
"matched_at": entry.get("matched-at", ""),
"vuln": True,
})
log(f"nuclei found {len(results)} vulnerable hosts/services")
return results
def scan_masscan_nuclei(ports: str, rate: int, template: str, nuclei_rate: int, nuclei_concurrency: int, nuclei_timeout: int, node_name: str) -> list[dict]:
if not template:
log("ERROR: --template required for masscan+nuclei mode")
return []
hits = run_masscan(ports, rate)
if not hits:
return []
targets = [f"{h['ip']}:{h['port']}" for h in hits]
log(f"nuclei scanning {len(targets)} ip:port targets from masscan...")
return run_nuclei(targets, template, nuclei_rate, nuclei_concurrency, nuclei_timeout)
def main():
parser = argparse.ArgumentParser()
parser.add_argument("--mode", required=True,
choices=["masscan-only", "nmap-only", "masscan+nmap", "geo-scout"])
choices=["masscan-only", "nmap-only", "masscan+nmap", "geo-scout", "masscan+nuclei"])
parser.add_argument("--ports", required=True)
parser.add_argument("--rate", type=int, default=3000)
parser.add_argument("--node-name", default="node")
parser.add_argument("--template", default="")
parser.add_argument("--nmap-timing", type=int, default=None, choices=[1, 2, 3, 4])
parser.add_argument("--nmap-timeout", type=int, default=None)
parser.add_argument("--nmap-workers", type=int, default=None)
parser.add_argument("--nuclei-rate", type=int, default=150)
parser.add_argument("--nuclei-concurrency", type=int, default=25)
parser.add_argument("--nuclei-timeout", type=int, default=10)
args = parser.parse_args()
global NMAP_WORKERS, NMAP_TIMING, NMAP_TIMEOUT
if args.nmap_workers is not None:
NMAP_WORKERS = args.nmap_workers
if args.nmap_timing is not None:
NMAP_TIMING = args.nmap_timing
if args.nmap_timeout is not None:
NMAP_TIMEOUT = args.nmap_timeout
log(f"WEBRUNNER node scanner starting — mode={args.mode} node={args.node_name}")
if args.mode == "masscan-only":
@@ -286,6 +390,12 @@ def main():
results = scan_masscan_nmap(args.ports, args.rate, args.node_name)
elif args.mode == "geo-scout":
results = scan_geo_scout(args.ports, args.rate, args.node_name)
elif args.mode == "masscan+nuclei":
results = scan_masscan_nuclei(
args.ports, args.rate, args.template,
args.nuclei_rate, args.nuclei_concurrency, args.nuclei_timeout,
args.node_name,
)
else:
log(f"Unknown mode: {args.mode}")
sys.exit(1)
+28
View File
@@ -22,6 +22,20 @@
- "{{ masscan_rate }}"
- --node-name
- "{{ node_name }}"
- --nmap-timing
- "{{ nmap_timing | default(4) }}"
- --nmap-timeout
- "{{ nmap_timeout | default(60) }}"
- --nmap-workers
- "{{ nmap_workers | default(10) }}"
- --nuclei-rate
- "{{ nuclei_rate | default(150) }}"
- --nuclei-concurrency
- "{{ nuclei_concurrency | default(25) }}"
- --nuclei-timeout
- "{{ nuclei_timeout | default(10) }}"
- --template
- "{{ nuclei_template_remote | default('') }}"
args:
chdir: /root/webrunner
register: scan_output
@@ -46,6 +60,20 @@
- "{{ masscan_rate }}"
- --node-name
- "{{ node_name }}"
- --nmap-timing
- "{{ nmap_timing | default(4) }}"
- --nmap-timeout
- "{{ nmap_timeout | default(60) }}"
- --nmap-workers
- "{{ nmap_workers | default(10) }}"
- --nuclei-rate
- "{{ nuclei_rate | default(150) }}"
- --nuclei-concurrency
- "{{ nuclei_concurrency | default(25) }}"
- --nuclei-timeout
- "{{ nuclei_timeout | default(10) }}"
- --template
- "{{ nuclei_template_remote | default('') }}"
args:
chdir: /root/webrunner
register: scan_output
+5 -4
View File
@@ -22,10 +22,11 @@ INSTANCE_RATES = {
}
SCAN_MODES = {
'geo-scout': {'rate': 3000, 'desc': 'masscan + nmap + probe fingerprinting'},
'masscan-only': {'rate': 10000, 'desc': 'masscan port discovery only'},
'nmap-only': {'rate': 500, 'desc': 'nmap full fingerprint only'},
'masscan+nmap': {'rate': 5000, 'desc': 'masscan + nmap (no probes)'},
'geo-scout': {'rate': 3000, 'desc': 'masscan + nmap + probe fingerprinting'},
'masscan-only': {'rate': 10000, 'desc': 'masscan port discovery only'},
'nmap-only': {'rate': 500, 'desc': 'nmap full fingerprint only'},
'masscan+nmap': {'rate': 5000, 'desc': 'masscan + nmap (no probes)'},
'masscan+nuclei': {'rate': 5000, 'desc': 'masscan discovery + nuclei CVE template'},
}
PRESETS = {