Add per-node random regions and AWS parallel provisioning to WEBRUNNER
Each node chunk now carries its own region field, assigned round-robin across the full provider region list unless the user pinned a specific region. AWS uses the same async/poll:0 pattern as Linode but with a serial per-unique-region setup phase (key import + SG) before firing all creates in parallel. Teardown uses hostvars ec2_region per node so multi-region AWS teardowns hit the right endpoint. Infisical replaces interactive credential prompts for both providers. Linode vars.yaml instance type corrected to g6-nanode-1.
This commit is contained in:
n0mad1k
@@ -305,19 +305,51 @@ def gather_webrunner_parameters() -> dict | None:
|
||||
all_cidrs.extend(cc_cidrs.get(cc.upper(), []))
|
||||
|
||||
chunks = chunk_cidrs(all_cidrs, chunk_size)
|
||||
node_chunks = [
|
||||
{
|
||||
|
||||
# Build per-provider region pools for round-robin assignment
|
||||
provider_regions: dict[str, list[str]] = {}
|
||||
for p in providers:
|
||||
if p == 'linode':
|
||||
provider_regions[p] = config.get('linode_regions', [config.get('linode_region', 'us-east')])
|
||||
elif p == 'aws':
|
||||
provider_regions[p] = config.get('aws_regions', [config.get('aws_region', 'us-east-1')])
|
||||
elif p == 'flokinet':
|
||||
provider_regions[p] = [config.get('flokinet_region', 'default')]
|
||||
else:
|
||||
provider_regions[p] = ['default']
|
||||
|
||||
provider_counters: dict[str, int] = {p: 0 for p in providers}
|
||||
node_chunks = []
|
||||
for i, chunk in enumerate(chunks):
|
||||
provider = providers[i % len(providers)]
|
||||
regions = provider_regions[provider]
|
||||
region = regions[provider_counters[provider] % len(regions)]
|
||||
provider_counters[provider] += 1
|
||||
node_chunks.append({
|
||||
'idx': i,
|
||||
'node_name': f"{config['webrunner_name']}-{i + 1:02d}",
|
||||
'provider': providers[i % len(providers)],
|
||||
'provider': provider,
|
||||
'region': region,
|
||||
'cidrs': chunk['cidrs'],
|
||||
'ip_count': chunk['ip_count'],
|
||||
}
|
||||
for i, chunk in enumerate(chunks)
|
||||
]
|
||||
})
|
||||
|
||||
config['node_chunks'] = node_chunks
|
||||
print(f"{COLORS['GREEN']}Nodes: {len(node_chunks)} ({preset_key}, {fmt_ip_count(chunk_size)}/node){COLORS['RESET']}")
|
||||
|
||||
# Warn if any provider's node count exceeds safe per-region quota
|
||||
_PROVIDER_CAPS = {'linode': 20, 'aws': 32, 'flokinet': 10}
|
||||
provider_node_counts: dict[str, int] = {}
|
||||
for nc in node_chunks:
|
||||
provider_node_counts[nc['provider']] = provider_node_counts.get(nc['provider'], 0) + 1
|
||||
for p, count in provider_node_counts.items():
|
||||
n_regions = len(provider_regions.get(p, ['default']))
|
||||
cap = _PROVIDER_CAPS.get(p, 20)
|
||||
per_region = (count + n_regions - 1) // n_regions
|
||||
if per_region > cap:
|
||||
print(f"{COLORS['YELLOW']} Warning: {count} {PROVIDER_LABELS[p]} nodes across {n_regions} region(s) "
|
||||
f"= ~{per_region}/region; default quota is ~{cap}/region.{COLORS['RESET']}")
|
||||
|
||||
# Operator IP
|
||||
config['operator_ip'] = get_public_ip()
|
||||
if config['operator_ip']:
|
||||
|
||||
+118
-9
@@ -12,6 +12,7 @@
|
||||
ssh_key_name: "{{ ssh_key_path | basename | regex_replace('\\.pub$', '') }}"
|
||||
all_node_chunks: "{{ lookup('file', node_chunks_file) | from_json }}"
|
||||
linode_chunks: "{{ all_node_chunks | selectattr('provider', 'equalto', 'linode') | list }}"
|
||||
aws_chunks: "{{ all_node_chunks | selectattr('provider', 'equalto', 'aws') | list }}"
|
||||
|
||||
tasks:
|
||||
- name: Ensure results directory
|
||||
@@ -64,7 +65,7 @@
|
||||
body:
|
||||
label: "{{ item.node_name }}"
|
||||
type: "{{ linode_instance_type | default('g6-nanode-1') }}"
|
||||
region: "{{ linode_region | default('us-east') }}"
|
||||
region: "{{ item.region | default('us-east') }}"
|
||||
image: "linode/debian12"
|
||||
root_pass: "{{ wr_root_pass }}"
|
||||
authorized_keys:
|
||||
@@ -120,20 +121,126 @@
|
||||
|
||||
- name: Show provisioned nodes
|
||||
debug:
|
||||
msg: "Node ready: {{ linode_chunks[idx].node_name }} @ {{ item.json.ipv4[0] }}"
|
||||
msg: "Node ready: {{ linode_chunks[idx].node_name }} @ {{ item.json.ipv4[0] }} ({{ linode_chunks[idx].region }})"
|
||||
loop: "{{ wr_create_results.results }}"
|
||||
loop_control:
|
||||
index_var: idx
|
||||
|
||||
when: linode_chunks | length > 0
|
||||
|
||||
# ── AWS / FlokiNET remain serial until those providers are implemented ─────
|
||||
# ── AWS — per-unique-region setup then parallel creates ───────────────────
|
||||
|
||||
- name: Provision AWS nodes
|
||||
include_tasks: "AWS/webrunner_provision_tasks.yml"
|
||||
loop: "{{ all_node_chunks | selectattr('provider', 'equalto', 'aws') | list }}"
|
||||
loop_control:
|
||||
loop_var: node_chunk
|
||||
- block:
|
||||
- name: Read SSH public key for AWS
|
||||
slurp:
|
||||
src: "~/.ssh/{{ ssh_key_name }}.pub"
|
||||
register: aws_pubkey
|
||||
|
||||
- name: Get unique AWS regions
|
||||
set_fact:
|
||||
aws_unique_regions: "{{ aws_chunks | map(attribute='region') | unique | list }}"
|
||||
|
||||
- name: Import SSH key pair per region
|
||||
amazon.aws.ec2_key:
|
||||
access_key: "{{ lookup('env', 'AWS_ACCESS_KEY_ID') }}"
|
||||
secret_key: "{{ lookup('env', 'AWS_SECRET_ACCESS_KEY') }}"
|
||||
name: "{{ webrunner_name }}"
|
||||
key_material: "{{ aws_pubkey.content | b64decode | trim }}"
|
||||
region: "{{ item }}"
|
||||
state: present
|
||||
loop: "{{ aws_unique_regions }}"
|
||||
ignore_errors: true
|
||||
|
||||
- name: Create security group per region
|
||||
amazon.aws.ec2_security_group:
|
||||
access_key: "{{ lookup('env', 'AWS_ACCESS_KEY_ID') }}"
|
||||
secret_key: "{{ lookup('env', 'AWS_SECRET_ACCESS_KEY') }}"
|
||||
name: "webrunner-{{ webrunner_name }}"
|
||||
description: "WEBRUNNER scan node SG"
|
||||
region: "{{ item }}"
|
||||
rules:
|
||||
- proto: tcp
|
||||
ports: [22]
|
||||
cidr_ip: "{{ (operator_ip ~ '/32') if operator_ip else '0.0.0.0/0' }}"
|
||||
rules_egress:
|
||||
- proto: all
|
||||
cidr_ip: "0.0.0.0/0"
|
||||
state: present
|
||||
loop: "{{ aws_unique_regions }}"
|
||||
ignore_errors: true
|
||||
|
||||
- name: Fire all AWS instance creates
|
||||
amazon.aws.ec2_instance:
|
||||
access_key: "{{ lookup('env', 'AWS_ACCESS_KEY_ID') }}"
|
||||
secret_key: "{{ lookup('env', 'AWS_SECRET_ACCESS_KEY') }}"
|
||||
name: "{{ item.node_name }}"
|
||||
instance_type: "{{ aws_instance_type | default('t3.micro') }}"
|
||||
region: "{{ item.region }}"
|
||||
image_id: "{{ ami_map[item.region] }}"
|
||||
key_name: "{{ webrunner_name }}"
|
||||
security_groups:
|
||||
- "webrunner-{{ webrunner_name }}"
|
||||
network:
|
||||
assign_public_ip: true
|
||||
tags:
|
||||
Name: "{{ item.node_name }}"
|
||||
webrunner: "{{ webrunner_name }}"
|
||||
deployment_id: "{{ deployment_id }}"
|
||||
state: running
|
||||
wait: true
|
||||
wait_timeout: 300
|
||||
loop: "{{ aws_chunks }}"
|
||||
async: 600
|
||||
poll: 0
|
||||
register: aws_create_jobs
|
||||
|
||||
- name: Wait for all AWS creates to confirm
|
||||
async_status:
|
||||
jid: "{{ item.ansible_job_id }}"
|
||||
loop: "{{ aws_create_jobs.results }}"
|
||||
register: aws_create_results
|
||||
until: aws_create_results.finished
|
||||
retries: 60
|
||||
delay: 10
|
||||
|
||||
- name: Add all AWS nodes to inventory
|
||||
add_host:
|
||||
name: "{{ item.instances[0].public_ip_address }}"
|
||||
groups: webrunner_nodes
|
||||
ansible_host: "{{ item.instances[0].public_ip_address }}"
|
||||
ansible_user: admin
|
||||
ansible_ssh_private_key_file: "~/.ssh/{{ ssh_key_name }}"
|
||||
ansible_ssh_common_args: "-o StrictHostKeyChecking=no"
|
||||
node_name: "{{ aws_chunks[idx].node_name }}"
|
||||
node_cidrs: "{{ aws_chunks[idx].cidrs }}"
|
||||
node_ip_count: "{{ aws_chunks[idx].ip_count }}"
|
||||
node_idx: "{{ aws_chunks[idx].idx }}"
|
||||
ec2_instance_id: "{{ item.instances[0].instance_id }}"
|
||||
ec2_region: "{{ aws_chunks[idx].region }}"
|
||||
provider: aws
|
||||
loop: "{{ aws_create_results.results }}"
|
||||
loop_control:
|
||||
index_var: idx
|
||||
|
||||
- name: Log AWS scanner IPs
|
||||
lineinfile:
|
||||
path: "{{ scanner_ip_log }}"
|
||||
line: "{{ aws_chunks[idx].node_name }}: {{ item.instances[0].public_ip_address }}"
|
||||
create: true
|
||||
loop: "{{ aws_create_results.results }}"
|
||||
loop_control:
|
||||
index_var: idx
|
||||
|
||||
- name: Show provisioned AWS nodes
|
||||
debug:
|
||||
msg: "Node ready: {{ aws_chunks[idx].node_name }} @ {{ item.instances[0].public_ip_address }} ({{ aws_chunks[idx].region }})"
|
||||
loop: "{{ aws_create_results.results }}"
|
||||
loop_control:
|
||||
index_var: idx
|
||||
|
||||
when: aws_chunks | length > 0
|
||||
|
||||
# ── FlokiNET ──────────────────────────────────────────────────────────────
|
||||
|
||||
- name: Provision FlokiNET nodes
|
||||
include_tasks: "FlokiNET/webrunner_provision_tasks.yml"
|
||||
@@ -215,9 +322,11 @@
|
||||
|
||||
- name: Teardown AWS nodes
|
||||
amazon.aws.ec2_instance:
|
||||
access_key: "{{ lookup('env', 'AWS_ACCESS_KEY_ID') }}"
|
||||
secret_key: "{{ lookup('env', 'AWS_SECRET_ACCESS_KEY') }}"
|
||||
instance_ids:
|
||||
- "{{ hostvars[item]['ec2_instance_id'] }}"
|
||||
region: "{{ aws_region | default('us-east-1') }}"
|
||||
region: "{{ hostvars[item]['ec2_region'] }}"
|
||||
state: terminated
|
||||
loop: "{{ groups['webrunner_nodes'] | default([]) }}"
|
||||
when:
|
||||
|
||||
+63
-50
@@ -3,73 +3,86 @@
|
||||
AWS provider utilities for C2ingRed deployment system
|
||||
"""
|
||||
|
||||
import random
|
||||
import os
|
||||
import subprocess
|
||||
import logging
|
||||
from .common import COLORS, load_vars_file, confirm_action
|
||||
from .common import COLORS, load_vars_file
|
||||
|
||||
def get_aws_credentials(provider_vars=None):
|
||||
"""Get AWS credentials from user or vars file"""
|
||||
if not provider_vars:
|
||||
provider_vars = load_vars_file('aws')
|
||||
|
||||
default_aws_key = provider_vars.get('aws_access_key', '')
|
||||
default_aws_secret = provider_vars.get('aws_secret_key', '')
|
||||
|
||||
print(f"\n{COLORS['BLUE']}AWS Configuration{COLORS['RESET']}")
|
||||
aws_key = input(f"AWS Access Key [{'*****' if default_aws_key else 'leave blank to use AWS CLI profile'}]: ") or default_aws_key
|
||||
aws_secret = input(f"AWS Secret Key [{'*****' if default_aws_secret else 'leave blank to use AWS CLI profile'}]: ") or default_aws_secret
|
||||
|
||||
return {
|
||||
'aws_access_key': aws_key,
|
||||
'aws_secret_key': aws_secret
|
||||
}
|
||||
"""Get AWS credentials — Infisical first, vars.yaml fallback, then prompt."""
|
||||
key = ''
|
||||
secret = ''
|
||||
|
||||
def select_aws_region(provider_vars=None, component=None):
|
||||
"""Let the user select an AWS region"""
|
||||
try:
|
||||
key = subprocess.check_output(
|
||||
[os.path.expanduser('~/.local/bin/creds'), 'get', 'AWS_ACCESS_KEY_ID', 'homelab'],
|
||||
text=True, stderr=subprocess.DEVNULL,
|
||||
).strip()
|
||||
secret = subprocess.check_output(
|
||||
[os.path.expanduser('~/.local/bin/creds'), 'get', 'AWS_SECRET_ACCESS_KEY', 'homelab'],
|
||||
text=True, stderr=subprocess.DEVNULL,
|
||||
).strip()
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
if not key:
|
||||
if not provider_vars:
|
||||
provider_vars = load_vars_file('aws')
|
||||
key = provider_vars.get('aws_access_key', '')
|
||||
secret = provider_vars.get('aws_secret_key', '')
|
||||
if key and 'YOUR_AWS' in key:
|
||||
key = ''
|
||||
secret = ''
|
||||
|
||||
print(f"\n{COLORS['BLUE']}AWS Configuration{COLORS['RESET']}")
|
||||
if key:
|
||||
print(f" AWS credentials loaded")
|
||||
else:
|
||||
key = input("AWS Access Key: ").strip()
|
||||
secret = input("AWS Secret Key: ").strip()
|
||||
|
||||
return {'aws_access_key': key, 'aws_secret_key': secret}
|
||||
|
||||
def select_aws_regions(provider_vars=None) -> list[str]:
|
||||
"""Return region list — single entry if user specified one, all regions if blank (random per-node)."""
|
||||
if not provider_vars:
|
||||
provider_vars = load_vars_file('aws')
|
||||
|
||||
regions = provider_vars.get('aws_region_choices', [])
|
||||
component_str = f" for {component}" if component else ""
|
||||
|
||||
if not regions:
|
||||
print(f"{COLORS['YELLOW']}No regions found for AWS, using us-east-1{COLORS['RESET']}")
|
||||
return "us-east-1"
|
||||
|
||||
print(f"\nAvailable AWS regions{component_str}:")
|
||||
|
||||
regions = provider_vars.get('aws_region_choices', ['us-east-1'])
|
||||
|
||||
print(f"\nAvailable AWS regions:")
|
||||
for i, region in enumerate(regions, 1):
|
||||
print(f" {i}. {region}")
|
||||
|
||||
region_input = input(f"\nSelect region{component_str} (number or leave blank for random): ")
|
||||
|
||||
print(f" {i:2}. {region}")
|
||||
|
||||
region_input = input("\nSelect region (number or leave blank for random per-node): ").strip()
|
||||
|
||||
if not region_input:
|
||||
return random.choice(regions)
|
||||
|
||||
return regions
|
||||
|
||||
try:
|
||||
region_choice = int(region_input)
|
||||
if 1 <= region_choice <= len(regions):
|
||||
return regions[region_choice - 1]
|
||||
else:
|
||||
print(f"{COLORS['RED']}Invalid choice, using random region{COLORS['RESET']}")
|
||||
return random.choice(regions)
|
||||
idx = int(region_input)
|
||||
if 1 <= idx <= len(regions):
|
||||
return [regions[idx - 1]]
|
||||
except ValueError:
|
||||
print(f"{COLORS['RED']}Invalid input, using random region{COLORS['RESET']}")
|
||||
return random.choice(regions)
|
||||
pass
|
||||
|
||||
print(f"{COLORS['RED']}Invalid input, using random per-node regions{COLORS['RESET']}")
|
||||
return regions
|
||||
|
||||
def gather_aws_config():
|
||||
"""Gather all AWS-specific configuration"""
|
||||
provider_vars = load_vars_file('aws')
|
||||
config = {}
|
||||
|
||||
# Get credentials
|
||||
|
||||
aws_creds = get_aws_credentials(provider_vars)
|
||||
config.update(aws_creds)
|
||||
|
||||
# Get region
|
||||
config['aws_region'] = select_aws_region(provider_vars)
|
||||
|
||||
# Additional AWS-specific settings
|
||||
|
||||
aws_regions = select_aws_regions(provider_vars)
|
||||
config['aws_regions'] = aws_regions
|
||||
config['aws_region'] = aws_regions[0]
|
||||
|
||||
config['ami_map'] = provider_vars.get('ami_map', {})
|
||||
config['aws_instance_type'] = provider_vars.get('aws_instance_type', 't3.micro')
|
||||
config['aws_volume_size'] = provider_vars.get('aws_volume_size', 20)
|
||||
|
||||
|
||||
return config
|
||||
|
||||
+27
-39
@@ -4,7 +4,6 @@ Linode provider utilities for C2ingRed deployment system
|
||||
"""
|
||||
|
||||
import os
|
||||
import random
|
||||
import logging
|
||||
import subprocess
|
||||
from .common import COLORS, load_vars_file
|
||||
@@ -13,7 +12,6 @@ def get_linode_credentials(provider_vars=None):
|
||||
"""Get Linode API token — Infisical first, vars.yaml fallback, then prompt."""
|
||||
default_token = ''
|
||||
|
||||
# Try Infisical first
|
||||
try:
|
||||
default_token = subprocess.check_output(
|
||||
[os.path.expanduser('~/.local/bin/creds'), 'get', 'LINODE_TOKEN', 'homelab'],
|
||||
@@ -22,7 +20,6 @@ def get_linode_credentials(provider_vars=None):
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
# Fall back to vars.yaml
|
||||
if not default_token:
|
||||
if not provider_vars:
|
||||
provider_vars = load_vars_file('linode')
|
||||
@@ -37,56 +34,47 @@ def get_linode_credentials(provider_vars=None):
|
||||
|
||||
return {'linode_token': token}
|
||||
|
||||
def select_linode_region(provider_vars=None, component=None):
|
||||
"""Let the user select a Linode region"""
|
||||
def select_linode_regions(provider_vars=None) -> list[str]:
|
||||
"""Return region list — single entry if user specified one, all regions if blank (random per-node)."""
|
||||
if not provider_vars:
|
||||
provider_vars = load_vars_file('linode')
|
||||
|
||||
regions = provider_vars.get('region_choices', [])
|
||||
component_str = f" for {component}" if component else ""
|
||||
|
||||
if not regions:
|
||||
print(f"{COLORS['YELLOW']}No regions found for Linode, using us-east{COLORS['RESET']}")
|
||||
return "us-east"
|
||||
|
||||
print(f"\nAvailable Linode regions{component_str}:")
|
||||
|
||||
regions = provider_vars.get('region_choices', ['us-east'])
|
||||
|
||||
print(f"\nAvailable Linode regions:")
|
||||
for i, region in enumerate(regions, 1):
|
||||
print(f" {i}. {region}")
|
||||
|
||||
region_input = input(f"\nSelect region{component_str} (number or leave blank for random): ")
|
||||
|
||||
print(f" {i:2}. {region}")
|
||||
|
||||
region_input = input("\nSelect region (number or leave blank for random per-node): ").strip()
|
||||
|
||||
if not region_input:
|
||||
# Use random from reliable regions only (first 8 are most reliable)
|
||||
reliable_regions = regions[:8] if len(regions) >= 8 else regions
|
||||
return random.choice(reliable_regions)
|
||||
|
||||
return regions
|
||||
|
||||
try:
|
||||
region_choice = int(region_input)
|
||||
if 1 <= region_choice <= len(regions):
|
||||
return regions[region_choice - 1]
|
||||
else:
|
||||
print(f"{COLORS['RED']}Invalid choice, using random region{COLORS['RESET']}")
|
||||
return random.choice(regions)
|
||||
idx = int(region_input)
|
||||
if 1 <= idx <= len(regions):
|
||||
return [regions[idx - 1]]
|
||||
except ValueError:
|
||||
print(f"{COLORS['RED']}Invalid input, using random region{COLORS['RESET']}")
|
||||
return random.choice(regions)
|
||||
pass
|
||||
|
||||
print(f"{COLORS['RED']}Invalid input, using random per-node regions{COLORS['RESET']}")
|
||||
return regions
|
||||
|
||||
def gather_linode_config():
|
||||
"""Gather all Linode-specific configuration"""
|
||||
provider_vars = load_vars_file('linode')
|
||||
config = {}
|
||||
|
||||
# Get credentials
|
||||
|
||||
linode_creds = get_linode_credentials(provider_vars)
|
||||
if not linode_creds:
|
||||
return None
|
||||
config.update(linode_creds)
|
||||
|
||||
# Get region
|
||||
config['linode_region'] = select_linode_region(provider_vars)
|
||||
|
||||
# Additional Linode-specific settings
|
||||
|
||||
linode_regions = select_linode_regions(provider_vars)
|
||||
config['linode_regions'] = linode_regions
|
||||
config['linode_region'] = linode_regions[0]
|
||||
|
||||
config['linode_instance_type'] = provider_vars.get('linode_instance_type', 'g6-nanode-1')
|
||||
config['linode_image'] = provider_vars.get('linode_image', 'linode/kali')
|
||||
|
||||
config['linode_image'] = provider_vars.get('linode_image', 'linode/debian12')
|
||||
|
||||
return config
|
||||
|
||||
Reference in New Issue
Block a user