From 78da72e9f61b7a755568ef982551e5b72619c887 Mon Sep 17 00:00:00 2001 From: n0mad1k Date: Thu, 30 Apr 2026 23:48:36 -0400 Subject: [PATCH] Add per-node random regions and AWS parallel provisioning to WEBRUNNER Each node chunk now carries its own region field, assigned round-robin across the full provider region list unless the user pinned a specific region. AWS uses the same async/poll:0 pattern as Linode but with a serial per-unique-region setup phase (key import + SG) before firing all creates in parallel. Teardown uses hostvars ec2_region per node so multi-region AWS teardowns hit the right endpoint. Infisical replaces interactive credential prompts for both providers. Linode vars.yaml instance type corrected to g6-nanode-1. --- modules/webrunner/deploy_webrunner.py | 44 +++++++-- providers/webrunner.yml | 127 ++++++++++++++++++++++++-- utils/aws_utils.py | 113 +++++++++++++---------- utils/linode_utils.py | 66 ++++++------- 4 files changed, 246 insertions(+), 104 deletions(-) diff --git a/modules/webrunner/deploy_webrunner.py b/modules/webrunner/deploy_webrunner.py index 3cfe030..22712f3 100644 --- a/modules/webrunner/deploy_webrunner.py +++ b/modules/webrunner/deploy_webrunner.py @@ -305,19 +305,51 @@ def gather_webrunner_parameters() -> dict | None: all_cidrs.extend(cc_cidrs.get(cc.upper(), [])) chunks = chunk_cidrs(all_cidrs, chunk_size) - node_chunks = [ - { + + # Build per-provider region pools for round-robin assignment + provider_regions: dict[str, list[str]] = {} + for p in providers: + if p == 'linode': + provider_regions[p] = config.get('linode_regions', [config.get('linode_region', 'us-east')]) + elif p == 'aws': + provider_regions[p] = config.get('aws_regions', [config.get('aws_region', 'us-east-1')]) + elif p == 'flokinet': + provider_regions[p] = [config.get('flokinet_region', 'default')] + else: + provider_regions[p] = ['default'] + + provider_counters: dict[str, int] = {p: 0 for p in providers} + node_chunks = [] + for i, chunk in enumerate(chunks): + provider = providers[i % len(providers)] + regions = provider_regions[provider] + region = regions[provider_counters[provider] % len(regions)] + provider_counters[provider] += 1 + node_chunks.append({ 'idx': i, 'node_name': f"{config['webrunner_name']}-{i + 1:02d}", - 'provider': providers[i % len(providers)], + 'provider': provider, + 'region': region, 'cidrs': chunk['cidrs'], 'ip_count': chunk['ip_count'], - } - for i, chunk in enumerate(chunks) - ] + }) + config['node_chunks'] = node_chunks print(f"{COLORS['GREEN']}Nodes: {len(node_chunks)} ({preset_key}, {fmt_ip_count(chunk_size)}/node){COLORS['RESET']}") + # Warn if any provider's node count exceeds safe per-region quota + _PROVIDER_CAPS = {'linode': 20, 'aws': 32, 'flokinet': 10} + provider_node_counts: dict[str, int] = {} + for nc in node_chunks: + provider_node_counts[nc['provider']] = provider_node_counts.get(nc['provider'], 0) + 1 + for p, count in provider_node_counts.items(): + n_regions = len(provider_regions.get(p, ['default'])) + cap = _PROVIDER_CAPS.get(p, 20) + per_region = (count + n_regions - 1) // n_regions + if per_region > cap: + print(f"{COLORS['YELLOW']} Warning: {count} {PROVIDER_LABELS[p]} nodes across {n_regions} region(s) " + f"= ~{per_region}/region; default quota is ~{cap}/region.{COLORS['RESET']}") + # Operator IP config['operator_ip'] = get_public_ip() if config['operator_ip']: diff --git a/providers/webrunner.yml b/providers/webrunner.yml index ea61bbf..a478171 100644 --- a/providers/webrunner.yml +++ b/providers/webrunner.yml @@ -12,6 +12,7 @@ ssh_key_name: "{{ ssh_key_path | basename | regex_replace('\\.pub$', '') }}" all_node_chunks: "{{ lookup('file', node_chunks_file) | from_json }}" linode_chunks: "{{ all_node_chunks | selectattr('provider', 'equalto', 'linode') | list }}" + aws_chunks: "{{ all_node_chunks | selectattr('provider', 'equalto', 'aws') | list }}" tasks: - name: Ensure results directory @@ -64,7 +65,7 @@ body: label: "{{ item.node_name }}" type: "{{ linode_instance_type | default('g6-nanode-1') }}" - region: "{{ linode_region | default('us-east') }}" + region: "{{ item.region | default('us-east') }}" image: "linode/debian12" root_pass: "{{ wr_root_pass }}" authorized_keys: @@ -120,20 +121,126 @@ - name: Show provisioned nodes debug: - msg: "Node ready: {{ linode_chunks[idx].node_name }} @ {{ item.json.ipv4[0] }}" + msg: "Node ready: {{ linode_chunks[idx].node_name }} @ {{ item.json.ipv4[0] }} ({{ linode_chunks[idx].region }})" loop: "{{ wr_create_results.results }}" loop_control: index_var: idx when: linode_chunks | length > 0 - # ── AWS / FlokiNET remain serial until those providers are implemented ───── + # ── AWS — per-unique-region setup then parallel creates ─────────────────── - - name: Provision AWS nodes - include_tasks: "AWS/webrunner_provision_tasks.yml" - loop: "{{ all_node_chunks | selectattr('provider', 'equalto', 'aws') | list }}" - loop_control: - loop_var: node_chunk + - block: + - name: Read SSH public key for AWS + slurp: + src: "~/.ssh/{{ ssh_key_name }}.pub" + register: aws_pubkey + + - name: Get unique AWS regions + set_fact: + aws_unique_regions: "{{ aws_chunks | map(attribute='region') | unique | list }}" + + - name: Import SSH key pair per region + amazon.aws.ec2_key: + access_key: "{{ lookup('env', 'AWS_ACCESS_KEY_ID') }}" + secret_key: "{{ lookup('env', 'AWS_SECRET_ACCESS_KEY') }}" + name: "{{ webrunner_name }}" + key_material: "{{ aws_pubkey.content | b64decode | trim }}" + region: "{{ item }}" + state: present + loop: "{{ aws_unique_regions }}" + ignore_errors: true + + - name: Create security group per region + amazon.aws.ec2_security_group: + access_key: "{{ lookup('env', 'AWS_ACCESS_KEY_ID') }}" + secret_key: "{{ lookup('env', 'AWS_SECRET_ACCESS_KEY') }}" + name: "webrunner-{{ webrunner_name }}" + description: "WEBRUNNER scan node SG" + region: "{{ item }}" + rules: + - proto: tcp + ports: [22] + cidr_ip: "{{ (operator_ip ~ '/32') if operator_ip else '0.0.0.0/0' }}" + rules_egress: + - proto: all + cidr_ip: "0.0.0.0/0" + state: present + loop: "{{ aws_unique_regions }}" + ignore_errors: true + + - name: Fire all AWS instance creates + amazon.aws.ec2_instance: + access_key: "{{ lookup('env', 'AWS_ACCESS_KEY_ID') }}" + secret_key: "{{ lookup('env', 'AWS_SECRET_ACCESS_KEY') }}" + name: "{{ item.node_name }}" + instance_type: "{{ aws_instance_type | default('t3.micro') }}" + region: "{{ item.region }}" + image_id: "{{ ami_map[item.region] }}" + key_name: "{{ webrunner_name }}" + security_groups: + - "webrunner-{{ webrunner_name }}" + network: + assign_public_ip: true + tags: + Name: "{{ item.node_name }}" + webrunner: "{{ webrunner_name }}" + deployment_id: "{{ deployment_id }}" + state: running + wait: true + wait_timeout: 300 + loop: "{{ aws_chunks }}" + async: 600 + poll: 0 + register: aws_create_jobs + + - name: Wait for all AWS creates to confirm + async_status: + jid: "{{ item.ansible_job_id }}" + loop: "{{ aws_create_jobs.results }}" + register: aws_create_results + until: aws_create_results.finished + retries: 60 + delay: 10 + + - name: Add all AWS nodes to inventory + add_host: + name: "{{ item.instances[0].public_ip_address }}" + groups: webrunner_nodes + ansible_host: "{{ item.instances[0].public_ip_address }}" + ansible_user: admin + ansible_ssh_private_key_file: "~/.ssh/{{ ssh_key_name }}" + ansible_ssh_common_args: "-o StrictHostKeyChecking=no" + node_name: "{{ aws_chunks[idx].node_name }}" + node_cidrs: "{{ aws_chunks[idx].cidrs }}" + node_ip_count: "{{ aws_chunks[idx].ip_count }}" + node_idx: "{{ aws_chunks[idx].idx }}" + ec2_instance_id: "{{ item.instances[0].instance_id }}" + ec2_region: "{{ aws_chunks[idx].region }}" + provider: aws + loop: "{{ aws_create_results.results }}" + loop_control: + index_var: idx + + - name: Log AWS scanner IPs + lineinfile: + path: "{{ scanner_ip_log }}" + line: "{{ aws_chunks[idx].node_name }}: {{ item.instances[0].public_ip_address }}" + create: true + loop: "{{ aws_create_results.results }}" + loop_control: + index_var: idx + + - name: Show provisioned AWS nodes + debug: + msg: "Node ready: {{ aws_chunks[idx].node_name }} @ {{ item.instances[0].public_ip_address }} ({{ aws_chunks[idx].region }})" + loop: "{{ aws_create_results.results }}" + loop_control: + index_var: idx + + when: aws_chunks | length > 0 + + # ── FlokiNET ────────────────────────────────────────────────────────────── - name: Provision FlokiNET nodes include_tasks: "FlokiNET/webrunner_provision_tasks.yml" @@ -215,9 +322,11 @@ - name: Teardown AWS nodes amazon.aws.ec2_instance: + access_key: "{{ lookup('env', 'AWS_ACCESS_KEY_ID') }}" + secret_key: "{{ lookup('env', 'AWS_SECRET_ACCESS_KEY') }}" instance_ids: - "{{ hostvars[item]['ec2_instance_id'] }}" - region: "{{ aws_region | default('us-east-1') }}" + region: "{{ hostvars[item]['ec2_region'] }}" state: terminated loop: "{{ groups['webrunner_nodes'] | default([]) }}" when: diff --git a/utils/aws_utils.py b/utils/aws_utils.py index 8744fc9..c7203f2 100644 --- a/utils/aws_utils.py +++ b/utils/aws_utils.py @@ -3,73 +3,86 @@ AWS provider utilities for C2ingRed deployment system """ -import random +import os +import subprocess import logging -from .common import COLORS, load_vars_file, confirm_action +from .common import COLORS, load_vars_file def get_aws_credentials(provider_vars=None): - """Get AWS credentials from user or vars file""" - if not provider_vars: - provider_vars = load_vars_file('aws') - - default_aws_key = provider_vars.get('aws_access_key', '') - default_aws_secret = provider_vars.get('aws_secret_key', '') - - print(f"\n{COLORS['BLUE']}AWS Configuration{COLORS['RESET']}") - aws_key = input(f"AWS Access Key [{'*****' if default_aws_key else 'leave blank to use AWS CLI profile'}]: ") or default_aws_key - aws_secret = input(f"AWS Secret Key [{'*****' if default_aws_secret else 'leave blank to use AWS CLI profile'}]: ") or default_aws_secret - - return { - 'aws_access_key': aws_key, - 'aws_secret_key': aws_secret - } + """Get AWS credentials — Infisical first, vars.yaml fallback, then prompt.""" + key = '' + secret = '' -def select_aws_region(provider_vars=None, component=None): - """Let the user select an AWS region""" + try: + key = subprocess.check_output( + [os.path.expanduser('~/.local/bin/creds'), 'get', 'AWS_ACCESS_KEY_ID', 'homelab'], + text=True, stderr=subprocess.DEVNULL, + ).strip() + secret = subprocess.check_output( + [os.path.expanduser('~/.local/bin/creds'), 'get', 'AWS_SECRET_ACCESS_KEY', 'homelab'], + text=True, stderr=subprocess.DEVNULL, + ).strip() + except Exception: + pass + + if not key: + if not provider_vars: + provider_vars = load_vars_file('aws') + key = provider_vars.get('aws_access_key', '') + secret = provider_vars.get('aws_secret_key', '') + if key and 'YOUR_AWS' in key: + key = '' + secret = '' + + print(f"\n{COLORS['BLUE']}AWS Configuration{COLORS['RESET']}") + if key: + print(f" AWS credentials loaded") + else: + key = input("AWS Access Key: ").strip() + secret = input("AWS Secret Key: ").strip() + + return {'aws_access_key': key, 'aws_secret_key': secret} + +def select_aws_regions(provider_vars=None) -> list[str]: + """Return region list — single entry if user specified one, all regions if blank (random per-node).""" if not provider_vars: provider_vars = load_vars_file('aws') - - regions = provider_vars.get('aws_region_choices', []) - component_str = f" for {component}" if component else "" - - if not regions: - print(f"{COLORS['YELLOW']}No regions found for AWS, using us-east-1{COLORS['RESET']}") - return "us-east-1" - - print(f"\nAvailable AWS regions{component_str}:") + + regions = provider_vars.get('aws_region_choices', ['us-east-1']) + + print(f"\nAvailable AWS regions:") for i, region in enumerate(regions, 1): - print(f" {i}. {region}") - - region_input = input(f"\nSelect region{component_str} (number or leave blank for random): ") - + print(f" {i:2}. {region}") + + region_input = input("\nSelect region (number or leave blank for random per-node): ").strip() + if not region_input: - return random.choice(regions) - + return regions + try: - region_choice = int(region_input) - if 1 <= region_choice <= len(regions): - return regions[region_choice - 1] - else: - print(f"{COLORS['RED']}Invalid choice, using random region{COLORS['RESET']}") - return random.choice(regions) + idx = int(region_input) + if 1 <= idx <= len(regions): + return [regions[idx - 1]] except ValueError: - print(f"{COLORS['RED']}Invalid input, using random region{COLORS['RESET']}") - return random.choice(regions) + pass + + print(f"{COLORS['RED']}Invalid input, using random per-node regions{COLORS['RESET']}") + return regions def gather_aws_config(): """Gather all AWS-specific configuration""" provider_vars = load_vars_file('aws') config = {} - - # Get credentials + aws_creds = get_aws_credentials(provider_vars) config.update(aws_creds) - - # Get region - config['aws_region'] = select_aws_region(provider_vars) - - # Additional AWS-specific settings + + aws_regions = select_aws_regions(provider_vars) + config['aws_regions'] = aws_regions + config['aws_region'] = aws_regions[0] + + config['ami_map'] = provider_vars.get('ami_map', {}) config['aws_instance_type'] = provider_vars.get('aws_instance_type', 't3.micro') config['aws_volume_size'] = provider_vars.get('aws_volume_size', 20) - + return config diff --git a/utils/linode_utils.py b/utils/linode_utils.py index 24a7cb2..72a3141 100644 --- a/utils/linode_utils.py +++ b/utils/linode_utils.py @@ -4,7 +4,6 @@ Linode provider utilities for C2ingRed deployment system """ import os -import random import logging import subprocess from .common import COLORS, load_vars_file @@ -13,7 +12,6 @@ def get_linode_credentials(provider_vars=None): """Get Linode API token — Infisical first, vars.yaml fallback, then prompt.""" default_token = '' - # Try Infisical first try: default_token = subprocess.check_output( [os.path.expanduser('~/.local/bin/creds'), 'get', 'LINODE_TOKEN', 'homelab'], @@ -22,7 +20,6 @@ def get_linode_credentials(provider_vars=None): except Exception: pass - # Fall back to vars.yaml if not default_token: if not provider_vars: provider_vars = load_vars_file('linode') @@ -37,56 +34,47 @@ def get_linode_credentials(provider_vars=None): return {'linode_token': token} -def select_linode_region(provider_vars=None, component=None): - """Let the user select a Linode region""" +def select_linode_regions(provider_vars=None) -> list[str]: + """Return region list — single entry if user specified one, all regions if blank (random per-node).""" if not provider_vars: provider_vars = load_vars_file('linode') - - regions = provider_vars.get('region_choices', []) - component_str = f" for {component}" if component else "" - - if not regions: - print(f"{COLORS['YELLOW']}No regions found for Linode, using us-east{COLORS['RESET']}") - return "us-east" - - print(f"\nAvailable Linode regions{component_str}:") + + regions = provider_vars.get('region_choices', ['us-east']) + + print(f"\nAvailable Linode regions:") for i, region in enumerate(regions, 1): - print(f" {i}. {region}") - - region_input = input(f"\nSelect region{component_str} (number or leave blank for random): ") - + print(f" {i:2}. {region}") + + region_input = input("\nSelect region (number or leave blank for random per-node): ").strip() + if not region_input: - # Use random from reliable regions only (first 8 are most reliable) - reliable_regions = regions[:8] if len(regions) >= 8 else regions - return random.choice(reliable_regions) - + return regions + try: - region_choice = int(region_input) - if 1 <= region_choice <= len(regions): - return regions[region_choice - 1] - else: - print(f"{COLORS['RED']}Invalid choice, using random region{COLORS['RESET']}") - return random.choice(regions) + idx = int(region_input) + if 1 <= idx <= len(regions): + return [regions[idx - 1]] except ValueError: - print(f"{COLORS['RED']}Invalid input, using random region{COLORS['RESET']}") - return random.choice(regions) + pass + + print(f"{COLORS['RED']}Invalid input, using random per-node regions{COLORS['RESET']}") + return regions def gather_linode_config(): """Gather all Linode-specific configuration""" provider_vars = load_vars_file('linode') config = {} - - # Get credentials + linode_creds = get_linode_credentials(provider_vars) if not linode_creds: return None config.update(linode_creds) - - # Get region - config['linode_region'] = select_linode_region(provider_vars) - - # Additional Linode-specific settings + + linode_regions = select_linode_regions(provider_vars) + config['linode_regions'] = linode_regions + config['linode_region'] = linode_regions[0] + config['linode_instance_type'] = provider_vars.get('linode_instance_type', 'g6-nanode-1') - config['linode_image'] = provider_vars.get('linode_image', 'linode/kali') - + config['linode_image'] = provider_vars.get('linode_image', 'linode/debian12') + return config