Add WEBRUNNER distributed geo-targeted recon module
Multi-provider (Linode/AWS/FlokiNET) scan orchestration: - Flatten all country CIDRs, chunk by IP count (sprint/balanced/economy presets) - Assign chunks round-robin across providers - Cost+time estimate table before deploy - Ansible provisions N nodes in parallel, runs masscan/nmap/probe per node - Results fetched back and merged into unified JSON report - Scanner IPs logged per engagement for client IR reporting - Operator IP whitelisting, enhanced OPSEC mode, YAML targets.yaml support
This commit is contained in:
n0mad1k
@@ -47,6 +47,7 @@ def main_menu():
|
||||
print(f"11) Deploy Privacy Server {COLORS['GREEN']}(Phantom — VPN, DNS, Matrix, etc.){COLORS['RESET']}")
|
||||
print(f"12) Tools & Utilities")
|
||||
print(f"13) Cleanup & Teardown")
|
||||
print(f"14) WEBRUNNER {COLORS['CYAN']}(Distributed Geo-Targeted Recon){COLORS['RESET']}")
|
||||
print(f"\n99) Exit")
|
||||
|
||||
choice = input(f"\nSelect an option: ")
|
||||
@@ -74,6 +75,8 @@ def main_menu():
|
||||
tools_menu()
|
||||
elif choice == "13":
|
||||
cleanup_menu()
|
||||
elif choice == "14":
|
||||
deploy_webrunner()
|
||||
elif choice == "99":
|
||||
print(f"\n{COLORS['GREEN']}Exiting C2ingRed. Goodbye!{COLORS['RESET']}")
|
||||
sys.exit(0)
|
||||
@@ -212,6 +215,22 @@ def deploy_attack_box():
|
||||
if attack_box_module:
|
||||
attack_box_module.attack_box_menu()
|
||||
|
||||
def deploy_webrunner():
|
||||
"""Launch the WEBRUNNER distributed geo-targeted recon module"""
|
||||
archive_logs_before_deployment()
|
||||
|
||||
wr_module_path = os.path.join(os.path.dirname(__file__), 'modules', 'webrunner', 'deploy_webrunner.py')
|
||||
|
||||
if not os.path.exists(wr_module_path):
|
||||
print(f"\n{COLORS['RED']}WEBRUNNER module not found at: {wr_module_path}{COLORS['RESET']}")
|
||||
wait_for_input()
|
||||
return
|
||||
|
||||
wr_module = import_module_from_path('deploy_webrunner', wr_module_path)
|
||||
if wr_module:
|
||||
wr_module.webrunner_menu()
|
||||
|
||||
|
||||
def tools_menu():
|
||||
"""Display the tools submenu"""
|
||||
while True:
|
||||
|
||||
@@ -0,0 +1,381 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
WEBRUNNER — Distributed geo-targeted recon scanning module for c2itall
|
||||
Provisions cloud nodes across multiple providers, distributes CIDR space,
|
||||
runs masscan/nmap/geo-scout in parallel, collects and merges results.
|
||||
"""
|
||||
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
_base = os.path.join(os.path.dirname(__file__), '..', '..')
|
||||
if _base not in sys.path:
|
||||
sys.path.insert(0, _base)
|
||||
|
||||
from utils.common import (
|
||||
COLORS, clear_screen, print_banner, wait_for_input,
|
||||
get_public_ip, load_vars_file,
|
||||
)
|
||||
from utils.ssh_utils import generate_ssh_key
|
||||
from utils.name_generator import generate_deployment_id
|
||||
from utils.naming_utils import get_deployment_name_with_options
|
||||
from utils.deployment_engine import execute_playbook, set_provider_environment
|
||||
from utils.chunk_utils import get_country_cidrs, chunk_cidrs, ip_count
|
||||
from utils.provider_rates import (
|
||||
PRESETS, SCAN_MODES, DEFAULT_INSTANCE,
|
||||
build_estimate_table, fmt_hours, fmt_ip_count,
|
||||
)
|
||||
|
||||
GEO_SCOUT_INPUTS = Path(__file__).parents[3] / 'geo-scout' / 'inputs'
|
||||
|
||||
SUPPORTED_PROVIDERS = ['linode', 'aws', 'flokinet']
|
||||
PROVIDER_LABELS = {'linode': 'Linode', 'aws': 'AWS', 'flokinet': 'FlokiNET'}
|
||||
|
||||
|
||||
# ── menu ──────────────────────────────────────────────────────────────────────
|
||||
|
||||
def webrunner_menu():
|
||||
while True:
|
||||
clear_screen()
|
||||
print_banner()
|
||||
print(f"{COLORS['CYAN']}WEBRUNNER — Distributed Geo-Targeted Recon{COLORS['RESET']}")
|
||||
print(f"{COLORS['CYAN']}==========================================={COLORS['RESET']}")
|
||||
print(f"1) New Scan Deployment")
|
||||
print(f"\n99) Return to Main Menu")
|
||||
|
||||
choice = input(f"\nSelect: ").strip()
|
||||
if choice == "1":
|
||||
config = gather_webrunner_parameters()
|
||||
if config:
|
||||
execute_webrunner_deployment(config)
|
||||
wait_for_input()
|
||||
elif choice == "99":
|
||||
break
|
||||
else:
|
||||
print(f"{COLORS['RED']}Invalid option.{COLORS['RESET']}")
|
||||
wait_for_input()
|
||||
|
||||
|
||||
# ── parameter gathering ───────────────────────────────────────────────────────
|
||||
|
||||
def _select_countries() -> tuple[list[str], dict[str, list], str]:
|
||||
print(f"\n{COLORS['BLUE']}Country Selection:{COLORS['RESET']}")
|
||||
print(f"1) Use geo-scout countries.yaml")
|
||||
print(f"2) Enter country codes manually")
|
||||
|
||||
choice = input("Select [1]: ").strip() or "1"
|
||||
exclude_map: dict[str, list] = {}
|
||||
|
||||
if choice == "1":
|
||||
default_path = str(GEO_SCOUT_INPUTS / 'countries.yaml')
|
||||
raw = input(f"Path [{default_path}]: ").strip()
|
||||
path = Path(raw) if raw else GEO_SCOUT_INPUTS / 'countries.yaml'
|
||||
|
||||
if not path.exists():
|
||||
print(f"{COLORS['RED']}File not found: {path}{COLORS['RESET']}")
|
||||
return [], {}, ""
|
||||
|
||||
import yaml
|
||||
with open(path) as f:
|
||||
data = yaml.safe_load(f)
|
||||
|
||||
countries = data.get('countries', [])
|
||||
codes = [c['code'].upper() for c in countries]
|
||||
for c in countries:
|
||||
if c.get('exclude_cidrs'):
|
||||
exclude_map[c['code'].upper()] = c['exclude_cidrs']
|
||||
return codes, exclude_map, str(path)
|
||||
|
||||
raw = input("Country codes (comma-separated, e.g. RU,IR,CN): ").strip()
|
||||
codes = [c.strip().upper() for c in raw.split(',') if c.strip()]
|
||||
return codes, exclude_map, "manual"
|
||||
|
||||
|
||||
def _select_scan_mode() -> str:
|
||||
print(f"\n{COLORS['BLUE']}Scan Mode:{COLORS['RESET']}")
|
||||
modes = list(SCAN_MODES.items())
|
||||
for i, (key, val) in enumerate(modes, 1):
|
||||
print(f"{i}) {key:<20} {val['desc']}")
|
||||
|
||||
choice = input("Select [1]: ").strip() or "1"
|
||||
try:
|
||||
idx = int(choice) - 1
|
||||
if 0 <= idx < len(modes):
|
||||
return modes[idx][0]
|
||||
except ValueError:
|
||||
pass
|
||||
return 'geo-scout'
|
||||
|
||||
|
||||
def _select_providers() -> list[str]:
|
||||
print(f"\n{COLORS['BLUE']}Provider Selection (multi-select):{COLORS['RESET']}")
|
||||
for i, key in enumerate(SUPPORTED_PROVIDERS, 1):
|
||||
print(f"{i}) {PROVIDER_LABELS[key]}")
|
||||
|
||||
raw = input("Select providers (e.g. 1 or 1,2) [1]: ").strip() or "1"
|
||||
selected = []
|
||||
for part in raw.split(','):
|
||||
try:
|
||||
idx = int(part.strip()) - 1
|
||||
if 0 <= idx < len(SUPPORTED_PROVIDERS):
|
||||
key = SUPPORTED_PROVIDERS[idx]
|
||||
if key not in selected:
|
||||
selected.append(key)
|
||||
except ValueError:
|
||||
pass
|
||||
return selected or ['linode']
|
||||
|
||||
|
||||
def _get_targets_info(scan_mode: str) -> tuple[str, list[int]]:
|
||||
if scan_mode == 'geo-scout':
|
||||
default = str(GEO_SCOUT_INPUTS / 'targets.yaml')
|
||||
raw = input(f"Path to targets.yaml [{default}]: ").strip()
|
||||
path = raw or default
|
||||
|
||||
import yaml
|
||||
try:
|
||||
with open(path) as f:
|
||||
data = yaml.safe_load(f)
|
||||
ports: set[int] = set()
|
||||
for t in data.get('targets', []):
|
||||
ports.update(t.get('ports', []))
|
||||
return path, sorted(ports)
|
||||
except Exception as e:
|
||||
print(f"{COLORS['YELLOW']}Could not load targets ({e}), using defaults.{COLORS['RESET']}")
|
||||
return path, [22, 80, 443, 8080, 8443]
|
||||
|
||||
default_ports = "22,80,443,8080,8443"
|
||||
raw = input(f"Ports to scan [{default_ports}]: ").strip() or default_ports
|
||||
ports_list: list[int] = []
|
||||
for p in raw.split(','):
|
||||
try:
|
||||
ports_list.append(int(p.strip()))
|
||||
except ValueError:
|
||||
pass
|
||||
return "", sorted(set(ports_list)) or [80, 443, 22]
|
||||
|
||||
|
||||
def _show_estimate_table(total_ips: int, n_ports: int, providers: list[str], scan_mode: str):
|
||||
rows = build_estimate_table(total_ips, n_ports, providers, scan_mode)
|
||||
|
||||
W = COLORS['WHITE']
|
||||
C = COLORS['CYAN']
|
||||
Y = COLORS['YELLOW']
|
||||
G = COLORS['GREEN']
|
||||
R = COLORS['RESET']
|
||||
|
||||
print(f"\n{C}{'─' * 70}{R}")
|
||||
print(f"{C} WEBRUNNER — Cost & Time Estimate{R}")
|
||||
print(f"{W} Total: {fmt_ip_count(total_ips)} IPs {n_ports} ports Mode: {scan_mode}{R}")
|
||||
print(f"{W} Providers: {', '.join(PROVIDER_LABELS[p] for p in providers)}{R}")
|
||||
print(f"{C}{'─' * 70}{R}")
|
||||
print(f" {'Preset':<14} {'Nodes':>6} {'IPs/Node':>10} {'Time/Node':>12} {'Total $':>10}")
|
||||
print(f" {'─' * 56}")
|
||||
|
||||
for row in rows:
|
||||
star = " *" if row['preset'] == 'balanced' else " "
|
||||
color = G if row['preset'] == 'balanced' else W
|
||||
ips_per_node = total_ips // row['n_nodes'] if row['n_nodes'] else total_ips
|
||||
print(
|
||||
f"{color}{star}{row['label']:<12} {row['n_nodes']:>6} "
|
||||
f"{fmt_ip_count(ips_per_node):>10} "
|
||||
f"{fmt_hours(row['hours_per_node']):>12} "
|
||||
f"${row['total_cost_usd']:>9.2f}{R}"
|
||||
)
|
||||
|
||||
print(f"{C}{'─' * 70}{R}")
|
||||
print(f"{Y} * = recommended (Pareto-optimal: speed vs. billing minimum){R}\n")
|
||||
|
||||
|
||||
def gather_webrunner_parameters() -> dict | None:
|
||||
clear_screen()
|
||||
print_banner()
|
||||
print(f"{COLORS['CYAN']}WEBRUNNER — New Scan Deployment{COLORS['RESET']}")
|
||||
print(f"{COLORS['CYAN']}================================{COLORS['RESET']}\n")
|
||||
|
||||
engagement = input("Engagement name: ").strip()
|
||||
if not engagement:
|
||||
print(f"{COLORS['RED']}Engagement name required.{COLORS['RESET']}")
|
||||
wait_for_input()
|
||||
return None
|
||||
|
||||
country_codes, exclude_map, _ = _select_countries()
|
||||
if not country_codes:
|
||||
print(f"{COLORS['RED']}No countries selected.{COLORS['RESET']}")
|
||||
wait_for_input()
|
||||
return None
|
||||
|
||||
print(f"{COLORS['GREEN']}Countries: {', '.join(country_codes)}{COLORS['RESET']}")
|
||||
|
||||
scan_mode = _select_scan_mode()
|
||||
providers = _select_providers()
|
||||
print(f"{COLORS['GREEN']}Providers: {', '.join(PROVIDER_LABELS[p] for p in providers)}{COLORS['RESET']}")
|
||||
|
||||
targets_file, ports = _get_targets_info(scan_mode)
|
||||
|
||||
print(f"\n{COLORS['CYAN']}[*] Resolving CIDR data for {len(country_codes)} countries...{COLORS['RESET']}")
|
||||
cc_cidrs = get_country_cidrs(country_codes, exclude_map)
|
||||
total_ips = sum(
|
||||
sum(ip_count(c) for c in cidrs)
|
||||
for cidrs in cc_cidrs.values()
|
||||
)
|
||||
|
||||
if total_ips == 0:
|
||||
print(f"{COLORS['RED']}No IPs resolved. Ensure geo-scout RIR data is available.{COLORS['RESET']}")
|
||||
wait_for_input()
|
||||
return None
|
||||
|
||||
print(f"{COLORS['GREEN']}Total: {fmt_ip_count(total_ips)} IPs across {len(country_codes)} countries{COLORS['RESET']}")
|
||||
|
||||
_show_estimate_table(total_ips, len(ports), providers, scan_mode)
|
||||
|
||||
print(f"{COLORS['BLUE']}Select deployment preset:{COLORS['RESET']}")
|
||||
preset_list = list(PRESETS.items())
|
||||
for i, (key, val) in enumerate(preset_list, 1):
|
||||
star = " (recommended)" if key == 'balanced' else ""
|
||||
print(f"{i}) {val['label']}{star} — {val['desc']}")
|
||||
print(f"4) Custom chunk size")
|
||||
|
||||
preset_choice = input("Select [2]: ").strip() or "2"
|
||||
|
||||
if preset_choice == "4":
|
||||
raw = input("IPs per node: ").strip()
|
||||
try:
|
||||
chunk_size = int(raw.replace(',', '').replace('_', ''))
|
||||
preset_key = 'custom'
|
||||
except ValueError:
|
||||
chunk_size = PRESETS['balanced']['chunk_size']
|
||||
preset_key = 'balanced'
|
||||
else:
|
||||
try:
|
||||
idx = int(preset_choice) - 1
|
||||
if 0 <= idx < len(preset_list):
|
||||
preset_key, preset_val = preset_list[idx]
|
||||
chunk_size = preset_val['chunk_size']
|
||||
else:
|
||||
preset_key = 'balanced'
|
||||
chunk_size = PRESETS['balanced']['chunk_size']
|
||||
except ValueError:
|
||||
preset_key = 'balanced'
|
||||
chunk_size = PRESETS['balanced']['chunk_size']
|
||||
|
||||
opsec_raw = input(f"\nEnhanced OPSEC mode? (randomize names, auto-teardown) [y/N]: ").strip().lower()
|
||||
enhanced_opsec = opsec_raw in ['y', 'yes']
|
||||
|
||||
print(f"\n{COLORS['CYAN']}[*] Generating deployment assets...{COLORS['RESET']}")
|
||||
deployment_id = generate_deployment_id()
|
||||
webrunner_name = get_deployment_name_with_options('webrunner', deployment_id, prefix='wr-')
|
||||
|
||||
raw_key_path = generate_ssh_key(webrunner_name)
|
||||
if not raw_key_path:
|
||||
print(f"{COLORS['RED']}Failed to generate SSH key.{COLORS['RESET']}")
|
||||
wait_for_input()
|
||||
return None
|
||||
|
||||
config: dict = {
|
||||
'engagement': engagement,
|
||||
'deployment_id': deployment_id,
|
||||
'deployment_type': 'webrunner',
|
||||
'webrunner_deployment': True,
|
||||
'webrunner_name': webrunner_name,
|
||||
'ssh_key_path': f"{raw_key_path}.pub",
|
||||
'ssh_key_name': os.path.basename(raw_key_path),
|
||||
'providers': providers,
|
||||
'scan_mode': scan_mode,
|
||||
'preset': preset_key,
|
||||
'chunk_size': chunk_size,
|
||||
'country_codes': country_codes,
|
||||
'ports': ports,
|
||||
'ports_str': ','.join(str(p) for p in ports),
|
||||
'targets_file': targets_file,
|
||||
'masscan_rate': SCAN_MODES.get(scan_mode, {}).get('rate', 3000),
|
||||
'enhanced_opsec': enhanced_opsec,
|
||||
'linode_instance_type': DEFAULT_INSTANCE.get('linode', 'g6-standard-2'),
|
||||
'linode_region': 'us-east',
|
||||
'aws_instance_type': DEFAULT_INSTANCE.get('aws', 't3.small'),
|
||||
'aws_region': 'us-east-1',
|
||||
}
|
||||
|
||||
operator_ip = get_public_ip()
|
||||
if operator_ip:
|
||||
config['operator_ip'] = operator_ip
|
||||
print(f"{COLORS['GREEN']}Operator IP: {operator_ip}{COLORS['RESET']}")
|
||||
|
||||
for provider in providers:
|
||||
vars_data = load_vars_file(provider)
|
||||
if vars_data:
|
||||
config.update({k: v for k, v in vars_data.items() if k not in config})
|
||||
|
||||
print(f"{COLORS['CYAN']}[*] Distributing {fmt_ip_count(total_ips)} IPs across nodes...{COLORS['RESET']}")
|
||||
all_cidrs: list[str] = []
|
||||
for cc in country_codes:
|
||||
all_cidrs.extend(cc_cidrs.get(cc.upper(), []))
|
||||
|
||||
chunks = chunk_cidrs(all_cidrs, chunk_size)
|
||||
node_chunks = [
|
||||
{
|
||||
'idx': i,
|
||||
'node_name': f"{webrunner_name}-{i + 1:02d}",
|
||||
'provider': providers[i % len(providers)],
|
||||
'cidrs': chunk['cidrs'],
|
||||
'ip_count': chunk['ip_count'],
|
||||
}
|
||||
for i, chunk in enumerate(chunks)
|
||||
]
|
||||
|
||||
print(f"{COLORS['GREEN']}Nodes: {len(node_chunks)} ({preset_key}, {fmt_ip_count(chunk_size)}/node){COLORS['RESET']}")
|
||||
|
||||
print(f"\n{COLORS['YELLOW']}{'─' * 50}{COLORS['RESET']}")
|
||||
print(f" Deployment: {webrunner_name}")
|
||||
print(f" Nodes: {len(node_chunks)}")
|
||||
print(f" Providers: {', '.join(PROVIDER_LABELS[p] for p in providers)}")
|
||||
print(f" Scan mode: {scan_mode}")
|
||||
print(f" Ports: {', '.join(str(p) for p in ports[:8])}{'...' if len(ports) > 8 else ''}")
|
||||
print(f" Total IPs: {fmt_ip_count(total_ips)}")
|
||||
print(f"{COLORS['YELLOW']}{'─' * 50}{COLORS['RESET']}")
|
||||
|
||||
confirm = input(f"\nDeploy {len(node_chunks)} nodes? [y/N]: ").strip().lower()
|
||||
if confirm not in ['y', 'yes']:
|
||||
print(f"{COLORS['YELLOW']}Deployment cancelled.{COLORS['RESET']}")
|
||||
return None
|
||||
|
||||
config['node_chunks'] = node_chunks
|
||||
config['total_ips'] = total_ips
|
||||
return config
|
||||
|
||||
|
||||
# ── execution ─────────────────────────────────────────────────────────────────
|
||||
|
||||
def execute_webrunner_deployment(config: dict):
|
||||
deployment_id = config['deployment_id']
|
||||
node_chunks = config.pop('node_chunks')
|
||||
|
||||
os.makedirs('logs', exist_ok=True)
|
||||
chunks_file = f"logs/node_chunks_{deployment_id}.json"
|
||||
with open(chunks_file, 'w') as f:
|
||||
json.dump(node_chunks, f)
|
||||
config['node_chunks_file'] = chunks_file
|
||||
config['scanner_ip_log'] = f"logs/scanner_ips_{config['webrunner_name']}.txt"
|
||||
config['results_dir'] = f"logs/webrunner_{deployment_id}"
|
||||
|
||||
for provider in config['providers']:
|
||||
set_provider_environment({**config, 'provider': provider})
|
||||
|
||||
playbook = 'providers/webrunner.yml'
|
||||
n_nodes = len(node_chunks)
|
||||
|
||||
print(f"\n{COLORS['CYAN']}[*] Launching WEBRUNNER — {n_nodes} nodes across "
|
||||
f"{', '.join(PROVIDER_LABELS[p] for p in config['providers'])}{COLORS['RESET']}")
|
||||
print(f"{COLORS['YELLOW']} Provisioning may take several minutes per node.{COLORS['RESET']}\n")
|
||||
|
||||
success = execute_playbook(playbook, config)
|
||||
|
||||
if success:
|
||||
print(f"\n{COLORS['GREEN']}[+] WEBRUNNER complete.{COLORS['RESET']}")
|
||||
print(f" Results: logs/webrunner_{deployment_id}/")
|
||||
print(f" Scanner IPs: logs/scanner_ips_{config['webrunner_name']}.txt")
|
||||
print(f" Log: logs/deployment_{deployment_id}.log")
|
||||
else:
|
||||
print(f"\n{COLORS['RED']}[-] WEBRUNNER failed. Check: logs/deployment_{deployment_id}.log{COLORS['RESET']}")
|
||||
@@ -0,0 +1,19 @@
|
||||
---
|
||||
# Fetch scan results from node back to controller
|
||||
|
||||
- name: Check results exist on {{ node_name }}
|
||||
stat:
|
||||
path: /root/webrunner/results.json
|
||||
register: results_stat
|
||||
|
||||
- name: Fetch results from {{ node_name }}
|
||||
fetch:
|
||||
src: /root/webrunner/results.json
|
||||
dest: "{{ results_dir }}/{{ node_name }}_results.json"
|
||||
flat: true
|
||||
when: results_stat.stat.exists
|
||||
|
||||
- name: Warn if no results from {{ node_name }}
|
||||
debug:
|
||||
msg: "No results.json found on {{ node_name }} — scan may have failed"
|
||||
when: not results_stat.stat.exists
|
||||
@@ -0,0 +1,40 @@
|
||||
---
|
||||
# Configure a WEBRUNNER scan node — install deps, create workspace
|
||||
|
||||
- name: Update apt cache
|
||||
apt:
|
||||
update_cache: true
|
||||
cache_valid_time: 3600
|
||||
retries: 3
|
||||
delay: 10
|
||||
|
||||
- name: Install scan tools
|
||||
apt:
|
||||
name:
|
||||
- masscan
|
||||
- nmap
|
||||
- python3
|
||||
- python3-pip
|
||||
state: present
|
||||
retries: 3
|
||||
delay: 10
|
||||
|
||||
- name: Create webrunner workspace
|
||||
file:
|
||||
path: /root/webrunner
|
||||
state: directory
|
||||
mode: '0700'
|
||||
|
||||
- name: Copy node scanner script
|
||||
copy:
|
||||
src: "{{ playbook_dir }}/../modules/webrunner/tasks/node_scanner.py"
|
||||
dest: /root/webrunner/node_scanner.py
|
||||
mode: '0755'
|
||||
|
||||
- name: Copy targets.yaml for geo-scout mode
|
||||
copy:
|
||||
src: "{{ targets_file }}"
|
||||
dest: /root/webrunner/targets.yaml
|
||||
mode: '0644'
|
||||
when: scan_mode == 'geo-scout' and targets_file != ""
|
||||
ignore_errors: true
|
||||
@@ -0,0 +1,66 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
WEBRUNNER merge — combine per-node results.json files into a unified report.
|
||||
Run by Ansible on the controller after all nodes complete.
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--results-dir", required=True)
|
||||
parser.add_argument("--output", required=True)
|
||||
parser.add_argument("--deployment-id", required=True)
|
||||
args = parser.parse_args()
|
||||
|
||||
results_dir = Path(args.results_dir)
|
||||
node_files = sorted(results_dir.glob("*_results.json"))
|
||||
|
||||
if not node_files:
|
||||
print(f"No result files found in {results_dir}", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
|
||||
merged: dict = {
|
||||
"deployment_id": args.deployment_id,
|
||||
"nodes": [],
|
||||
"total_results": 0,
|
||||
"results": [],
|
||||
}
|
||||
|
||||
seen: set[str] = set() # deduplicate by ip:port
|
||||
|
||||
for f in node_files:
|
||||
try:
|
||||
data = json.loads(f.read_text())
|
||||
except (json.JSONDecodeError, OSError) as e:
|
||||
print(f"Skipping {f.name}: {e}", file=sys.stderr)
|
||||
continue
|
||||
|
||||
node_summary = {
|
||||
"node_name": data.get("node_name", f.stem),
|
||||
"scan_mode": data.get("scan_mode", "unknown"),
|
||||
"result_count": data.get("result_count", 0),
|
||||
}
|
||||
merged["nodes"].append(node_summary)
|
||||
|
||||
for entry in data.get("results", []):
|
||||
key = f"{entry.get('ip')}:{entry.get('port')}"
|
||||
if key not in seen:
|
||||
seen.add(key)
|
||||
merged["results"].append(entry)
|
||||
|
||||
merged["total_results"] = len(merged["results"])
|
||||
|
||||
Path(args.output).write_text(json.dumps(merged, indent=2))
|
||||
|
||||
print(f"Merged {len(node_files)} node(s) — {merged['total_results']} unique results")
|
||||
for n in merged["nodes"]:
|
||||
print(f" {n['node_name']}: {n['result_count']} results ({n['scan_mode']})")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,292 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
WEBRUNNER node scanner — runs on each cloud node.
|
||||
Reads cidrs.txt, runs scan pipeline, writes results.json.
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
import xml.etree.ElementTree as ET
|
||||
from pathlib import Path
|
||||
|
||||
WORKDIR = Path("/root/webrunner")
|
||||
|
||||
|
||||
def log(msg: str):
|
||||
ts = time.strftime("%H:%M:%S")
|
||||
print(f"[{ts}] {msg}", flush=True)
|
||||
|
||||
|
||||
def run_masscan(ports: str, rate: int) -> list[dict]:
|
||||
cidr_file = WORKDIR / "cidrs.txt"
|
||||
out_file = WORKDIR / "masscan.json"
|
||||
out_file.unlink(missing_ok=True)
|
||||
|
||||
cmd = [
|
||||
"masscan",
|
||||
f"--rate={rate}",
|
||||
f"--ports={ports}",
|
||||
"-iL", str(cidr_file),
|
||||
"-oJ", str(out_file),
|
||||
"--wait", "3",
|
||||
]
|
||||
log(f"masscan starting: rate={rate} ports={ports}")
|
||||
try:
|
||||
subprocess.run(cmd, timeout=36000, check=False)
|
||||
except subprocess.TimeoutExpired:
|
||||
log("masscan timed out after 10h")
|
||||
|
||||
if not out_file.exists():
|
||||
return []
|
||||
|
||||
raw = out_file.read_text(errors="replace").strip()
|
||||
if not raw or raw == "[]":
|
||||
return []
|
||||
raw = raw.rstrip(",\n")
|
||||
if not raw.endswith("]"):
|
||||
raw += "]"
|
||||
if not raw.startswith("["):
|
||||
raw = "[" + raw
|
||||
try:
|
||||
data = json.loads(raw)
|
||||
except json.JSONDecodeError:
|
||||
log("masscan JSON parse error")
|
||||
return []
|
||||
|
||||
hits = []
|
||||
for entry in data:
|
||||
ip = entry.get("ip")
|
||||
for pe in entry.get("ports", []):
|
||||
port = pe.get("port")
|
||||
if ip and port:
|
||||
hits.append({"ip": ip, "port": port})
|
||||
log(f"masscan found {len(hits)} open port/host pairs")
|
||||
return hits
|
||||
|
||||
|
||||
def group_hits_by_ip(hits: list[dict]) -> dict[str, list[int]]:
|
||||
result: dict[str, list[int]] = {}
|
||||
for h in hits:
|
||||
result.setdefault(h["ip"], []).append(h["port"])
|
||||
return result
|
||||
|
||||
|
||||
def run_nmap(ip: str, ports: list[int]) -> dict:
|
||||
port_str = ",".join(str(p) for p in sorted(set(ports)))
|
||||
out_file = WORKDIR / f"nmap_{ip.replace('.', '_')}.xml"
|
||||
|
||||
cmd = [
|
||||
"nmap", "-sV", "--version-intensity", "5",
|
||||
"-p", port_str, "-T4", "--open",
|
||||
"-oX", str(out_file), ip,
|
||||
]
|
||||
try:
|
||||
subprocess.run(cmd, capture_output=True, timeout=60, check=False)
|
||||
except (subprocess.TimeoutExpired, FileNotFoundError):
|
||||
return {}
|
||||
|
||||
if not out_file.exists():
|
||||
return {}
|
||||
try:
|
||||
tree = ET.parse(out_file)
|
||||
except ET.ParseError:
|
||||
return {}
|
||||
|
||||
result = {}
|
||||
for port_el in tree.findall(".//port"):
|
||||
portid = int(port_el.get("portid", 0))
|
||||
state = port_el.find("state")
|
||||
if state is None or state.get("state") != "open":
|
||||
continue
|
||||
svc = port_el.find("service")
|
||||
info: dict = {}
|
||||
if svc is not None:
|
||||
info["service"] = svc.get("name", "")
|
||||
info["product"] = svc.get("product", "")
|
||||
info["version"] = svc.get("version", "")
|
||||
info["banner"] = " ".join(filter(None, [
|
||||
svc.get("product", ""), svc.get("version", ""), svc.get("extrainfo", "")
|
||||
]))
|
||||
result[portid] = info
|
||||
return result
|
||||
|
||||
|
||||
def probe_service(ip: str, port: int) -> str:
|
||||
import socket
|
||||
try:
|
||||
with socket.create_connection((ip, port), timeout=3) as s:
|
||||
s.settimeout(3)
|
||||
try:
|
||||
s.send(b"HEAD / HTTP/1.0\r\nHost: %b\r\n\r\n" % ip.encode())
|
||||
banner = s.recv(512).decode(errors="replace").strip()
|
||||
return banner[:200]
|
||||
except Exception:
|
||||
try:
|
||||
banner = s.recv(512).decode(errors="replace").strip()
|
||||
return banner[:200]
|
||||
except Exception:
|
||||
return ""
|
||||
except Exception:
|
||||
return ""
|
||||
|
||||
|
||||
def scan_masscan_only(ports: str, rate: int, node_name: str) -> list[dict]:
|
||||
hits = run_masscan(ports, rate)
|
||||
results = [{"ip": h["ip"], "port": h["port"]} for h in hits]
|
||||
return results
|
||||
|
||||
|
||||
def scan_nmap_only(ports: str, node_name: str) -> list[dict]:
|
||||
cidr_file = WORKDIR / "cidrs.txt"
|
||||
cidrs = [l.strip() for l in cidr_file.read_text().splitlines() if l.strip()]
|
||||
port_str = ports
|
||||
out_file = WORKDIR / "nmap_sweep.xml"
|
||||
|
||||
targets_arg = " ".join(cidrs[:50]) # nmap handles CIDRs natively
|
||||
cmd = [
|
||||
"nmap", "-sV", "--version-intensity", "3",
|
||||
"-p", port_str, "-T4", "--open",
|
||||
"-oX", str(out_file),
|
||||
] + cidrs
|
||||
|
||||
log(f"nmap starting across {len(cidrs)} CIDRs")
|
||||
try:
|
||||
subprocess.run(cmd, timeout=36000, check=False)
|
||||
except subprocess.TimeoutExpired:
|
||||
log("nmap timed out")
|
||||
|
||||
if not out_file.exists():
|
||||
return []
|
||||
|
||||
results = []
|
||||
try:
|
||||
tree = ET.parse(out_file)
|
||||
except ET.ParseError:
|
||||
return []
|
||||
|
||||
for host_el in tree.findall(".//host"):
|
||||
addr_el = host_el.find("address[@addrtype='ipv4']")
|
||||
if addr_el is None:
|
||||
continue
|
||||
ip = addr_el.get("addr", "")
|
||||
for port_el in host_el.findall(".//port"):
|
||||
state = port_el.find("state")
|
||||
if state is None or state.get("state") != "open":
|
||||
continue
|
||||
portid = int(port_el.get("portid", 0))
|
||||
svc = port_el.find("service")
|
||||
entry: dict = {"ip": ip, "port": portid}
|
||||
if svc is not None:
|
||||
entry["service"] = svc.get("name", "")
|
||||
entry["banner"] = " ".join(filter(None, [
|
||||
svc.get("product", ""), svc.get("version", ""), svc.get("extrainfo", "")
|
||||
]))
|
||||
results.append(entry)
|
||||
|
||||
log(f"nmap found {len(results)} open ports")
|
||||
return results
|
||||
|
||||
|
||||
def scan_masscan_nmap(ports: str, rate: int, node_name: str) -> list[dict]:
|
||||
hits = run_masscan(ports, rate)
|
||||
if not hits:
|
||||
return []
|
||||
|
||||
by_ip = group_hits_by_ip(hits)
|
||||
log(f"nmap fingerprinting {len(by_ip)} hosts...")
|
||||
results = []
|
||||
for idx, (ip, ip_ports) in enumerate(by_ip.items()):
|
||||
nmap_info = run_nmap(ip, ip_ports)
|
||||
for port in ip_ports:
|
||||
entry: dict = {"ip": ip, "port": port}
|
||||
if port in nmap_info:
|
||||
entry.update(nmap_info[port])
|
||||
results.append(entry)
|
||||
if (idx + 1) % 50 == 0:
|
||||
log(f" nmap: {idx + 1}/{len(by_ip)} hosts done")
|
||||
return results
|
||||
|
||||
|
||||
def scan_geo_scout(ports: str, rate: int, node_name: str) -> list[dict]:
|
||||
hits = run_masscan(ports, rate)
|
||||
if not hits:
|
||||
return []
|
||||
|
||||
by_ip = group_hits_by_ip(hits)
|
||||
log(f"nmap + probe fingerprinting {len(by_ip)} hosts...")
|
||||
|
||||
targets_file = WORKDIR / "targets.yaml"
|
||||
target_ports: dict[int, dict] = {}
|
||||
if targets_file.exists():
|
||||
try:
|
||||
import yaml
|
||||
with open(targets_file) as f:
|
||||
tdata = yaml.safe_load(f)
|
||||
for t in tdata.get("targets", []):
|
||||
for p in t.get("ports", []):
|
||||
target_ports[p] = t
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
results = []
|
||||
for idx, (ip, ip_ports) in enumerate(by_ip.items()):
|
||||
nmap_info = run_nmap(ip, ip_ports)
|
||||
for port in ip_ports:
|
||||
entry: dict = {"ip": ip, "port": port}
|
||||
if port in nmap_info:
|
||||
entry.update(nmap_info[port])
|
||||
# Banner grab / probe
|
||||
banner = probe_service(ip, port)
|
||||
if banner:
|
||||
entry["probe_banner"] = banner
|
||||
# Match against target profile
|
||||
if port in target_ports:
|
||||
t = target_ports[port]
|
||||
entry["target_name"] = t.get("name", "")
|
||||
entry["target_desc"] = t.get("description", "")
|
||||
results.append(entry)
|
||||
if (idx + 1) % 50 == 0:
|
||||
log(f" geo-scout: {idx + 1}/{len(by_ip)} hosts done")
|
||||
return results
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--mode", required=True,
|
||||
choices=["masscan-only", "nmap-only", "masscan+nmap", "geo-scout"])
|
||||
parser.add_argument("--ports", required=True)
|
||||
parser.add_argument("--rate", type=int, default=3000)
|
||||
parser.add_argument("--node-name", default="node")
|
||||
args = parser.parse_args()
|
||||
|
||||
log(f"WEBRUNNER node scanner starting — mode={args.mode} node={args.node_name}")
|
||||
|
||||
if args.mode == "masscan-only":
|
||||
results = scan_masscan_only(args.ports, args.rate, args.node_name)
|
||||
elif args.mode == "nmap-only":
|
||||
results = scan_nmap_only(args.ports, args.node_name)
|
||||
elif args.mode == "masscan+nmap":
|
||||
results = scan_masscan_nmap(args.ports, args.rate, args.node_name)
|
||||
elif args.mode == "geo-scout":
|
||||
results = scan_geo_scout(args.ports, args.rate, args.node_name)
|
||||
else:
|
||||
log(f"Unknown mode: {args.mode}")
|
||||
sys.exit(1)
|
||||
|
||||
out = {
|
||||
"node_name": args.node_name,
|
||||
"scan_mode": args.mode,
|
||||
"result_count": len(results),
|
||||
"results": results,
|
||||
}
|
||||
|
||||
out_file = WORKDIR / "results.json"
|
||||
out_file.write_text(json.dumps(out, indent=2))
|
||||
log(f"Done — {len(results)} results written to {out_file}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,29 @@
|
||||
---
|
||||
# Run the assigned scan on this WEBRUNNER node
|
||||
# Per-node vars: node_name, node_cidrs, node_ip_count, node_idx
|
||||
# Global vars (from extra-vars): scan_mode, ports_str, masscan_rate, webrunner_name, deployment_id
|
||||
|
||||
- name: Write CIDR list for {{ node_name }}
|
||||
copy:
|
||||
content: "{{ node_cidrs | join('\n') }}\n"
|
||||
dest: /root/webrunner/cidrs.txt
|
||||
mode: '0644'
|
||||
|
||||
- name: Run scan on {{ node_name }} ({{ node_ip_count }} IPs, mode={{ scan_mode }})
|
||||
command: >
|
||||
python3 /root/webrunner/node_scanner.py
|
||||
--mode {{ scan_mode }}
|
||||
--ports {{ ports_str }}
|
||||
--rate {{ masscan_rate }}
|
||||
--node-name {{ node_name }}
|
||||
args:
|
||||
chdir: /root/webrunner
|
||||
register: scan_output
|
||||
async: 43200
|
||||
poll: 60
|
||||
ignore_errors: true
|
||||
|
||||
- name: Show scan output for {{ node_name }}
|
||||
debug:
|
||||
var: scan_output.stdout_lines
|
||||
when: scan_output.stdout_lines is defined
|
||||
@@ -0,0 +1,98 @@
|
||||
---
|
||||
# AWS provision tasks for one WEBRUNNER node
|
||||
# Called in a loop — loop_var: node_chunk
|
||||
# Requires: aws_access_key, aws_secret_key, aws_region, aws_instance_type,
|
||||
# ssh_key_name, webrunner_name, deployment_id, operator_ip,
|
||||
# scanner_ip_log, results_dir
|
||||
|
||||
- name: Read public key for {{ node_chunk.node_name }}
|
||||
slurp:
|
||||
src: "~/.ssh/{{ ssh_key_name }}.pub"
|
||||
register: wr_pubkey_aws
|
||||
|
||||
- name: Import SSH key to AWS ({{ node_chunk.node_name }})
|
||||
amazon.aws.ec2_key:
|
||||
name: "{{ webrunner_name }}"
|
||||
key_material: "{{ wr_pubkey_aws.content | b64decode | trim }}"
|
||||
region: "{{ aws_region | default('us-east-1') }}"
|
||||
aws_access_key: "{{ aws_access_key }}"
|
||||
aws_secret_key: "{{ aws_secret_key }}"
|
||||
state: present
|
||||
ignore_errors: true
|
||||
|
||||
- name: Create security group for {{ node_chunk.node_name }}
|
||||
amazon.aws.ec2_security_group:
|
||||
name: "wr-{{ deployment_id }}-sg"
|
||||
description: "WEBRUNNER {{ deployment_id }} scanner nodes"
|
||||
region: "{{ aws_region | default('us-east-1') }}"
|
||||
aws_access_key: "{{ aws_access_key }}"
|
||||
aws_secret_key: "{{ aws_secret_key }}"
|
||||
rules:
|
||||
- proto: tcp
|
||||
ports: [22]
|
||||
cidr_ip: "{{ operator_ip | default('0.0.0.0/0') }}/32"
|
||||
rules_egress:
|
||||
- proto: all
|
||||
cidr_ip: "0.0.0.0/0"
|
||||
tags:
|
||||
Name: "wr-{{ deployment_id }}-sg"
|
||||
DeploymentID: "{{ deployment_id }}"
|
||||
state: present
|
||||
register: wr_sg
|
||||
ignore_errors: true
|
||||
|
||||
- name: Launch EC2 instance {{ node_chunk.node_name }}
|
||||
amazon.aws.ec2_instance:
|
||||
name: "{{ node_chunk.node_name }}"
|
||||
key_name: "{{ webrunner_name }}"
|
||||
instance_type: "{{ aws_instance_type | default('t3.small') }}"
|
||||
image_id: "{{ aws_ami | default('ami-0c55b159cbfafe1f0') }}"
|
||||
region: "{{ aws_region | default('us-east-1') }}"
|
||||
aws_access_key: "{{ aws_access_key }}"
|
||||
aws_secret_key: "{{ aws_secret_key }}"
|
||||
security_group: "wr-{{ deployment_id }}-sg"
|
||||
network:
|
||||
assign_public_ip: true
|
||||
tags:
|
||||
Name: "{{ node_chunk.node_name }}"
|
||||
DeploymentID: "{{ deployment_id }}"
|
||||
webrunner: "{{ webrunner_name }}"
|
||||
wait: true
|
||||
state: running
|
||||
register: wr_ec2
|
||||
|
||||
- name: Extract EC2 public IP
|
||||
set_fact:
|
||||
wr_node_ip: "{{ wr_ec2.instances[0].public_ip_address }}"
|
||||
|
||||
- name: Log scanner IP
|
||||
lineinfile:
|
||||
path: "{{ scanner_ip_log }}"
|
||||
line: "{{ node_chunk.node_name }}: {{ wr_node_ip }}"
|
||||
create: true
|
||||
|
||||
- name: Wait for SSH on {{ node_chunk.node_name }} ({{ wr_node_ip }})
|
||||
wait_for:
|
||||
host: "{{ wr_node_ip }}"
|
||||
port: 22
|
||||
delay: 30
|
||||
timeout: 300
|
||||
|
||||
- name: Add {{ node_chunk.node_name }} to inventory
|
||||
add_host:
|
||||
name: "{{ wr_node_ip }}"
|
||||
groups: webrunner_nodes
|
||||
ansible_host: "{{ wr_node_ip }}"
|
||||
ansible_user: admin
|
||||
ansible_ssh_private_key_file: "~/.ssh/{{ ssh_key_name }}"
|
||||
ansible_ssh_common_args: "-o StrictHostKeyChecking=no"
|
||||
node_name: "{{ node_chunk.node_name }}"
|
||||
node_cidrs: "{{ node_chunk.cidrs }}"
|
||||
node_ip_count: "{{ node_chunk.ip_count }}"
|
||||
node_idx: "{{ node_chunk.idx }}"
|
||||
ec2_instance_id: "{{ wr_ec2.instances[0].instance_id }}"
|
||||
provider: aws
|
||||
|
||||
- name: Show {{ node_chunk.node_name }} ready
|
||||
debug:
|
||||
msg: "AWS node ready: {{ node_chunk.node_name }} @ {{ wr_node_ip }} ({{ node_chunk.ip_count | int | string }} IPs)"
|
||||
@@ -0,0 +1,12 @@
|
||||
---
|
||||
# FlokiNET provision tasks for WEBRUNNER nodes
|
||||
# FlokiNET does not expose a public API — manual provisioning required.
|
||||
|
||||
- name: FlokiNET not supported for automated WEBRUNNER deployment
|
||||
fail:
|
||||
msg: >
|
||||
FlokiNET does not provide a public provisioning API.
|
||||
To use FlokiNET nodes with WEBRUNNER, provision VPS instances manually,
|
||||
add their IPs to the [webrunner_nodes] inventory group, and re-run
|
||||
the scan plays directly. Node chunk {{ node_chunk.node_name }} assigned
|
||||
{{ node_chunk.ip_count }} IPs.
|
||||
@@ -0,0 +1,90 @@
|
||||
---
|
||||
# Linode provision tasks for one WEBRUNNER node
|
||||
# Called in a loop — loop_var: node_chunk
|
||||
# Requires: linode_token, ssh_key_name, webrunner_name, deployment_id,
|
||||
# linode_instance_type, linode_region, scanner_ip_log, results_dir
|
||||
|
||||
- name: Generate root password for {{ node_chunk.node_name }}
|
||||
set_fact:
|
||||
wr_root_pass: "{{ lookup('password', '/dev/null chars=ascii_letters,digits length=32') }}"
|
||||
|
||||
- name: Read public key
|
||||
slurp:
|
||||
src: "~/.ssh/{{ ssh_key_name }}.pub"
|
||||
register: wr_pubkey
|
||||
|
||||
- name: Register SSH key in Linode ({{ node_chunk.node_name }})
|
||||
uri:
|
||||
url: "https://api.linode.com/v4/profile/sshkeys"
|
||||
method: POST
|
||||
headers:
|
||||
Authorization: "Bearer {{ linode_token }}"
|
||||
Content-Type: "application/json"
|
||||
body_format: json
|
||||
body:
|
||||
label: "{{ webrunner_name }}"
|
||||
ssh_key: "{{ wr_pubkey.content | b64decode | trim }}"
|
||||
status_code: [200, 201]
|
||||
ignore_errors: true
|
||||
|
||||
- name: Create Linode instance {{ node_chunk.node_name }}
|
||||
uri:
|
||||
url: "https://api.linode.com/v4/linode/instances"
|
||||
method: POST
|
||||
headers:
|
||||
Authorization: "Bearer {{ linode_token }}"
|
||||
Content-Type: "application/json"
|
||||
body_format: json
|
||||
body:
|
||||
label: "{{ node_chunk.node_name }}"
|
||||
type: "{{ linode_instance_type | default('g6-standard-2') }}"
|
||||
region: "{{ linode_region | default('us-east') }}"
|
||||
image: "linode/debian12"
|
||||
root_pass: "{{ wr_root_pass }}"
|
||||
authorized_keys:
|
||||
- "{{ wr_pubkey.content | b64decode | trim }}"
|
||||
booted: true
|
||||
backups_enabled: false
|
||||
private_ip: false
|
||||
tags:
|
||||
- "webrunner"
|
||||
- "{{ webrunner_name }}"
|
||||
- "{{ deployment_id }}"
|
||||
status_code: [200, 201]
|
||||
register: wr_linode
|
||||
|
||||
- name: Extract node IP
|
||||
set_fact:
|
||||
wr_node_ip: "{{ wr_linode.json.ipv4[0] }}"
|
||||
|
||||
- name: Log scanner IP
|
||||
lineinfile:
|
||||
path: "{{ scanner_ip_log }}"
|
||||
line: "{{ node_chunk.node_name }}: {{ wr_node_ip }}"
|
||||
create: true
|
||||
|
||||
- name: Wait for SSH on {{ node_chunk.node_name }} ({{ wr_node_ip }})
|
||||
wait_for:
|
||||
host: "{{ wr_node_ip }}"
|
||||
port: 22
|
||||
delay: 30
|
||||
timeout: 300
|
||||
|
||||
- name: Add {{ node_chunk.node_name }} to inventory
|
||||
add_host:
|
||||
name: "{{ wr_node_ip }}"
|
||||
groups: webrunner_nodes
|
||||
ansible_host: "{{ wr_node_ip }}"
|
||||
ansible_user: root
|
||||
ansible_ssh_private_key_file: "~/.ssh/{{ ssh_key_name }}"
|
||||
ansible_ssh_common_args: "-o StrictHostKeyChecking=no"
|
||||
node_name: "{{ node_chunk.node_name }}"
|
||||
node_cidrs: "{{ node_chunk.cidrs }}"
|
||||
node_ip_count: "{{ node_chunk.ip_count }}"
|
||||
node_idx: "{{ node_chunk.idx }}"
|
||||
linode_instance_id: "{{ wr_linode.json.id }}"
|
||||
provider: linode
|
||||
|
||||
- name: Show {{ node_chunk.node_name }} ready
|
||||
debug:
|
||||
msg: "Linode node ready: {{ node_chunk.node_name }} @ {{ wr_node_ip }} ({{ node_chunk.ip_count | int | string }} IPs)"
|
||||
@@ -0,0 +1,92 @@
|
||||
---
|
||||
# WEBRUNNER — Distributed geo-targeted recon
|
||||
# Multi-provider: Linode, AWS, FlokiNET
|
||||
# Controller: ~/tools/c2itall/ (CWD when ansible-playbook runs)
|
||||
|
||||
- name: WEBRUNNER — Provision scan nodes
|
||||
hosts: localhost
|
||||
connection: local
|
||||
gather_facts: false
|
||||
vars:
|
||||
ansible_python_interpreter: "{{ ansible_playbook_python }}"
|
||||
ssh_key_name: "{{ ssh_key_path | basename | regex_replace('\\.pub$', '') }}"
|
||||
all_node_chunks: "{{ lookup('file', node_chunks_file) | from_json }}"
|
||||
|
||||
tasks:
|
||||
- name: Ensure results directory
|
||||
file:
|
||||
path: "{{ results_dir }}"
|
||||
state: directory
|
||||
mode: '0755'
|
||||
|
||||
- name: Initialize scanner IP log
|
||||
copy:
|
||||
content: "# WEBRUNNER scanner IPs — {{ webrunner_name }}\n"
|
||||
dest: "{{ scanner_ip_log }}"
|
||||
mode: '0644'
|
||||
force: false
|
||||
|
||||
- name: Provision Linode nodes
|
||||
include_tasks: "Linode/webrunner_provision_tasks.yml"
|
||||
loop: "{{ all_node_chunks | selectattr('provider', 'equalto', 'linode') | list }}"
|
||||
loop_var: node_chunk
|
||||
|
||||
- name: Provision AWS nodes
|
||||
include_tasks: "AWS/webrunner_provision_tasks.yml"
|
||||
loop: "{{ all_node_chunks | selectattr('provider', 'equalto', 'aws') | list }}"
|
||||
loop_var: node_chunk
|
||||
|
||||
- name: Provision FlokiNET nodes
|
||||
include_tasks: "FlokiNET/webrunner_provision_tasks.yml"
|
||||
loop: "{{ all_node_chunks | selectattr('provider', 'equalto', 'flokinet') | list }}"
|
||||
loop_var: node_chunk
|
||||
|
||||
|
||||
- name: WEBRUNNER — Configure and scan
|
||||
hosts: webrunner_nodes
|
||||
gather_facts: false
|
||||
become: true
|
||||
|
||||
tasks:
|
||||
- name: Wait for SSH
|
||||
wait_for_connection:
|
||||
delay: 20
|
||||
timeout: 300
|
||||
|
||||
- name: Configure node
|
||||
include_tasks: "{{ playbook_dir }}/../modules/webrunner/tasks/configure_node.yml"
|
||||
|
||||
- name: Run scan
|
||||
include_tasks: "{{ playbook_dir }}/../modules/webrunner/tasks/run_scan.yml"
|
||||
|
||||
- name: Collect results
|
||||
include_tasks: "{{ playbook_dir }}/../modules/webrunner/tasks/collect_results.yml"
|
||||
|
||||
|
||||
- name: WEBRUNNER — Merge and report
|
||||
hosts: localhost
|
||||
connection: local
|
||||
gather_facts: false
|
||||
|
||||
tasks:
|
||||
- name: Merge per-node results
|
||||
command: >
|
||||
python3 {{ playbook_dir }}/../modules/webrunner/tasks/merge_results.py
|
||||
--results-dir {{ results_dir }}
|
||||
--output {{ results_dir }}/merged_results.json
|
||||
--deployment-id {{ deployment_id }}
|
||||
register: merge_out
|
||||
ignore_errors: true
|
||||
|
||||
- name: Show merge output
|
||||
debug:
|
||||
var: merge_out.stdout_lines
|
||||
when: merge_out.stdout_lines is defined and merge_out.stdout_lines | length > 0
|
||||
|
||||
- name: Summary
|
||||
debug:
|
||||
msg:
|
||||
- "WEBRUNNER complete — {{ webrunner_name }}"
|
||||
- "Results: {{ results_dir }}/merged_results.json"
|
||||
- "Scanner IPs: {{ scanner_ip_log }}"
|
||||
- "Log: logs/deployment_{{ deployment_id }}.log"
|
||||
@@ -0,0 +1,71 @@
|
||||
#!/usr/bin/env python3
|
||||
import ipaddress
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
def _try_load_geo_scout() -> bool:
|
||||
candidates = [
|
||||
Path(__file__).parents[2] / 'geo-scout',
|
||||
Path.home() / 'tools' / 'geo-scout',
|
||||
]
|
||||
for p in candidates:
|
||||
if (p / 'lib' / 'cidr.py').exists():
|
||||
if str(p) not in sys.path:
|
||||
sys.path.insert(0, str(p))
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def ip_count(cidr: str) -> int:
|
||||
try:
|
||||
return ipaddress.ip_network(cidr, strict=False).num_addresses
|
||||
except ValueError:
|
||||
return 0
|
||||
|
||||
|
||||
def get_country_cidrs(country_codes: list[str], exclude_map: dict[str, list] | None = None) -> dict[str, list[str]]:
|
||||
if exclude_map is None:
|
||||
exclude_map = {}
|
||||
|
||||
_try_load_geo_scout()
|
||||
try:
|
||||
from lib.cidr import fetch_rir_data, load_index, get_cidrs, merge_cidrs
|
||||
except ImportError:
|
||||
return {cc.upper(): [] for cc in country_codes}
|
||||
|
||||
fetch_rir_data()
|
||||
index = load_index()
|
||||
result = {}
|
||||
for cc in country_codes:
|
||||
cc = cc.upper()
|
||||
cidrs = get_cidrs(cc, index)
|
||||
cidrs = merge_cidrs(cidrs, exclude_map.get(cc, []))
|
||||
result[cc] = cidrs
|
||||
return result
|
||||
|
||||
|
||||
def chunk_cidrs(all_cidrs: list[str], chunk_size: int) -> list[dict]:
|
||||
chunks = []
|
||||
current_cidrs: list[str] = []
|
||||
current_count = 0
|
||||
|
||||
for cidr in all_cidrs:
|
||||
n = ip_count(cidr)
|
||||
if n == 0:
|
||||
continue
|
||||
if current_count > 0 and current_count + n > chunk_size * 1.5:
|
||||
chunks.append({'cidrs': current_cidrs, 'ip_count': current_count})
|
||||
current_cidrs = []
|
||||
current_count = 0
|
||||
current_cidrs.append(cidr)
|
||||
current_count += n
|
||||
if current_count >= chunk_size:
|
||||
chunks.append({'cidrs': current_cidrs, 'ip_count': current_count})
|
||||
current_cidrs = []
|
||||
current_count = 0
|
||||
|
||||
if current_cidrs:
|
||||
chunks.append({'cidrs': current_cidrs, 'ip_count': current_count})
|
||||
|
||||
return chunks
|
||||
@@ -215,9 +215,10 @@ def show_naming_relationship(name, deployment_id, deployment_type):
|
||||
'c2': 's-', # s for server
|
||||
'tracker': 't-',
|
||||
'attack_box': 'a-',
|
||||
'payload': 'p-'
|
||||
'payload': 'p-',
|
||||
'webrunner': 'wr-',
|
||||
}
|
||||
|
||||
|
||||
expected_prefix = prefix_map.get(deployment_type, '')
|
||||
|
||||
if expected_prefix and name.startswith(expected_prefix):
|
||||
@@ -239,6 +240,7 @@ def get_deployment_type_prefix(deployment_type):
|
||||
'tracker': 't-',
|
||||
'attack_box': 'a-',
|
||||
'payload': 'p-',
|
||||
'phishing': 'p-'
|
||||
'phishing': 'p-',
|
||||
'webrunner': 'wr-',
|
||||
}
|
||||
return prefix_map.get(deployment_type, '')
|
||||
|
||||
@@ -0,0 +1,100 @@
|
||||
#!/usr/bin/env python3
|
||||
import math
|
||||
|
||||
INSTANCE_RATES = {
|
||||
'linode': {
|
||||
'g6-nanode-1': 0.0075,
|
||||
'g6-standard-2': 0.018,
|
||||
'g6-standard-4': 0.036,
|
||||
'g6-standard-8': 0.072,
|
||||
},
|
||||
'aws': {
|
||||
't3.micro': 0.0104,
|
||||
't3.small': 0.0208,
|
||||
't3.medium': 0.0416,
|
||||
't3.large': 0.0832,
|
||||
},
|
||||
'flokinet': {
|
||||
'vps-1': 0.0083,
|
||||
'vps-2': 0.0139,
|
||||
'vps-4': 0.0278,
|
||||
},
|
||||
}
|
||||
|
||||
SCAN_MODES = {
|
||||
'geo-scout': {'rate': 3000, 'desc': 'masscan + nmap + probe fingerprinting'},
|
||||
'masscan-only': {'rate': 10000, 'desc': 'masscan port discovery only'},
|
||||
'nmap-only': {'rate': 500, 'desc': 'nmap full fingerprint only'},
|
||||
'masscan+nmap': {'rate': 5000, 'desc': 'masscan + nmap (no probes)'},
|
||||
}
|
||||
|
||||
PRESETS = {
|
||||
'sprint': {'chunk_size': 500_000, 'label': 'Sprint', 'desc': '~500K IPs/node'},
|
||||
'balanced': {'chunk_size': 2_000_000, 'label': 'Balanced', 'desc': '~2M IPs/node (recommended)'},
|
||||
'economy': {'chunk_size': 5_000_000, 'label': 'Economy', 'desc': '~5M IPs/node'},
|
||||
}
|
||||
|
||||
DEFAULT_INSTANCE = {
|
||||
'linode': 'g6-standard-2',
|
||||
'aws': 't3.small',
|
||||
'flokinet': 'vps-2',
|
||||
}
|
||||
|
||||
BILLING_MINIMUM = {
|
||||
'linode': 1.0,
|
||||
'aws': 0.017, # billed per second, ~1 min minimum in practice
|
||||
'flokinet': 1.0,
|
||||
}
|
||||
|
||||
|
||||
def estimate_scan_hours(ip_count: int, n_ports: int, rate: int) -> float:
|
||||
if rate <= 0:
|
||||
return 0.0
|
||||
return (ip_count * n_ports / rate) / 3600
|
||||
|
||||
|
||||
def node_cost(provider: str, instance_type: str, scan_hours: float) -> float:
|
||||
rate = INSTANCE_RATES.get(provider, {}).get(instance_type, 0.018)
|
||||
billed_hours = max(scan_hours, BILLING_MINIMUM.get(provider, 1.0))
|
||||
return rate * billed_hours
|
||||
|
||||
|
||||
def fmt_hours(h: float) -> str:
|
||||
total_mins = int(h * 60)
|
||||
hrs, mins = divmod(total_mins, 60)
|
||||
return f"{hrs}h {mins:02d}m" if hrs else f"{mins}m"
|
||||
|
||||
|
||||
def fmt_ip_count(n: int) -> str:
|
||||
if n >= 1_000_000:
|
||||
return f"{n / 1_000_000:.1f}M"
|
||||
if n >= 1_000:
|
||||
return f"{n / 1_000:.0f}K"
|
||||
return str(n)
|
||||
|
||||
|
||||
def build_estimate_table(
|
||||
total_ips: int,
|
||||
n_ports: int,
|
||||
providers: list[str],
|
||||
scan_mode: str,
|
||||
) -> list[dict]:
|
||||
mode_rate = SCAN_MODES.get(scan_mode, {'rate': 3000})['rate']
|
||||
rows = []
|
||||
for preset_key, preset in PRESETS.items():
|
||||
n_chunks = max(1, math.ceil(total_ips / preset['chunk_size']))
|
||||
hours = estimate_scan_hours(preset['chunk_size'], n_ports, mode_rate)
|
||||
total_cost = 0.0
|
||||
for i in range(n_chunks):
|
||||
provider = providers[i % len(providers)]
|
||||
instance = DEFAULT_INSTANCE.get(provider, 'g6-standard-2')
|
||||
total_cost += node_cost(provider, instance, hours)
|
||||
rows.append({
|
||||
'preset': preset_key,
|
||||
'label': preset['label'],
|
||||
'desc': preset['desc'],
|
||||
'n_nodes': n_chunks,
|
||||
'hours_per_node': hours,
|
||||
'total_cost_usd': total_cost,
|
||||
})
|
||||
return rows
|
||||
Reference in New Issue
Block a user