Add per-node random regions and AWS parallel provisioning to WEBRUNNER

Each node chunk now carries its own region field, assigned round-robin
across the full provider region list unless the user pinned a specific
region. AWS uses the same async/poll:0 pattern as Linode but with a
serial per-unique-region setup phase (key import + SG) before firing all
creates in parallel. Teardown uses hostvars ec2_region per node so
multi-region AWS teardowns hit the right endpoint. Infisical replaces
interactive credential prompts for both providers. Linode vars.yaml
instance type corrected to g6-nanode-1.

modules/webrunner/deploy_webrunner.py

@@ -305,19 +305,51 @@ def gather_webrunner_parameters() -> dict | None:
         all_cidrs.extend(cc_cidrs.get(cc.upper(), []))
 
     chunks = chunk_cidrs(all_cidrs, chunk_size)
-    node_chunks = [
+
-        {
+    # Build per-provider region pools for round-robin assignment
+    provider_regions: dict[str, list[str]] = {}
+    for p in providers:
+        if p == 'linode':
+            provider_regions[p] = config.get('linode_regions', [config.get('linode_region', 'us-east')])
+        elif p == 'aws':
+            provider_regions[p] = config.get('aws_regions', [config.get('aws_region', 'us-east-1')])
+        elif p == 'flokinet':
+            provider_regions[p] = [config.get('flokinet_region', 'default')]
+        else:
+            provider_regions[p] = ['default']
+
+    provider_counters: dict[str, int] = {p: 0 for p in providers}
+    node_chunks = []
+    for i, chunk in enumerate(chunks):
+        provider = providers[i % len(providers)]
+        regions = provider_regions[provider]
+        region = regions[provider_counters[provider] % len(regions)]
+        provider_counters[provider] += 1
+        node_chunks.append({
             'idx': i,
             'node_name': f"{config['webrunner_name']}-{i + 1:02d}",
-            'provider': providers[i % len(providers)],
+            'provider': provider,
+            'region': region,
             'cidrs': chunk['cidrs'],
             'ip_count': chunk['ip_count'],
-        }
+        })
-        for i, chunk in enumerate(chunks)
+
-    ]
     config['node_chunks'] = node_chunks
     print(f"{COLORS['GREEN']}Nodes: {len(node_chunks)} ({preset_key}, {fmt_ip_count(chunk_size)}/node){COLORS['RESET']}")
 
+    # Warn if any provider's node count exceeds safe per-region quota
+    _PROVIDER_CAPS = {'linode': 20, 'aws': 32, 'flokinet': 10}
+    provider_node_counts: dict[str, int] = {}
+    for nc in node_chunks:
+        provider_node_counts[nc['provider']] = provider_node_counts.get(nc['provider'], 0) + 1
+    for p, count in provider_node_counts.items():
+        n_regions = len(provider_regions.get(p, ['default']))
+        cap = _PROVIDER_CAPS.get(p, 20)
+        per_region = (count + n_regions - 1) // n_regions
+        if per_region > cap:
+            print(f"{COLORS['YELLOW']}  Warning: {count} {PROVIDER_LABELS[p]} nodes across {n_regions} region(s) "
+                  f"= ~{per_region}/region; default quota is ~{cap}/region.{COLORS['RESET']}")
+
     # Operator IP
     config['operator_ip'] = get_public_ip()
     if config['operator_ip']:

providers/webrunner.yml

@@ -12,6 +12,7 @@
     ssh_key_name: "{{ ssh_key_path | basename | regex_replace('\\.pub$', '') }}"
     all_node_chunks: "{{ lookup('file', node_chunks_file) | from_json }}"
     linode_chunks: "{{ all_node_chunks | selectattr('provider', 'equalto', 'linode') | list }}"
+    aws_chunks: "{{ all_node_chunks | selectattr('provider', 'equalto', 'aws') | list }}"
 
   tasks:
     - name: Ensure results directory
@@ -64,7 +65,7 @@
             body:
               label: "{{ item.node_name }}"
               type: "{{ linode_instance_type | default('g6-nanode-1') }}"
-              region: "{{ linode_region | default('us-east') }}"
+              region: "{{ item.region | default('us-east') }}"
               image: "linode/debian12"
               root_pass: "{{ wr_root_pass }}"
               authorized_keys:
@@ -120,20 +121,126 @@
 
         - name: Show provisioned nodes
           debug:
-            msg: "Node ready: {{ linode_chunks[idx].node_name }} @ {{ item.json.ipv4[0] }}"
+            msg: "Node ready: {{ linode_chunks[idx].node_name }} @ {{ item.json.ipv4[0] }} ({{ linode_chunks[idx].region }})"
           loop: "{{ wr_create_results.results }}"
           loop_control:
             index_var: idx
 
       when: linode_chunks | length > 0
 
-    # ── AWS / FlokiNET remain serial until those providers are implemented ─────
+    # ── AWS — per-unique-region setup then parallel creates ───────────────────
 
-    - name: Provision AWS nodes
+    - block:
-      include_tasks: "AWS/webrunner_provision_tasks.yml"
+        - name: Read SSH public key for AWS
-      loop: "{{ all_node_chunks | selectattr('provider', 'equalto', 'aws') | list }}"
+          slurp:
-      loop_control:
+            src: "~/.ssh/{{ ssh_key_name }}.pub"
-        loop_var: node_chunk
+          register: aws_pubkey
+
+        - name: Get unique AWS regions
+          set_fact:
+            aws_unique_regions: "{{ aws_chunks | map(attribute='region') | unique | list }}"
+
+        - name: Import SSH key pair per region
+          amazon.aws.ec2_key:
+            access_key: "{{ lookup('env', 'AWS_ACCESS_KEY_ID') }}"
+            secret_key: "{{ lookup('env', 'AWS_SECRET_ACCESS_KEY') }}"
+            name: "{{ webrunner_name }}"
+            key_material: "{{ aws_pubkey.content | b64decode | trim }}"
+            region: "{{ item }}"
+            state: present
+          loop: "{{ aws_unique_regions }}"
+          ignore_errors: true
+
+        - name: Create security group per region
+          amazon.aws.ec2_security_group:
+            access_key: "{{ lookup('env', 'AWS_ACCESS_KEY_ID') }}"
+            secret_key: "{{ lookup('env', 'AWS_SECRET_ACCESS_KEY') }}"
+            name: "webrunner-{{ webrunner_name }}"
+            description: "WEBRUNNER scan node SG"
+            region: "{{ item }}"
+            rules:
+              - proto: tcp
+                ports: [22]
+                cidr_ip: "{{ (operator_ip ~ '/32') if operator_ip else '0.0.0.0/0' }}"
+            rules_egress:
+              - proto: all
+                cidr_ip: "0.0.0.0/0"
+            state: present
+          loop: "{{ aws_unique_regions }}"
+          ignore_errors: true
+
+        - name: Fire all AWS instance creates
+          amazon.aws.ec2_instance:
+            access_key: "{{ lookup('env', 'AWS_ACCESS_KEY_ID') }}"
+            secret_key: "{{ lookup('env', 'AWS_SECRET_ACCESS_KEY') }}"
+            name: "{{ item.node_name }}"
+            instance_type: "{{ aws_instance_type | default('t3.micro') }}"
+            region: "{{ item.region }}"
+            image_id: "{{ ami_map[item.region] }}"
+            key_name: "{{ webrunner_name }}"
+            security_groups:
+              - "webrunner-{{ webrunner_name }}"
+            network:
+              assign_public_ip: true
+            tags:
+              Name: "{{ item.node_name }}"
+              webrunner: "{{ webrunner_name }}"
+              deployment_id: "{{ deployment_id }}"
+            state: running
+            wait: true
+            wait_timeout: 300
+          loop: "{{ aws_chunks }}"
+          async: 600
+          poll: 0
+          register: aws_create_jobs
+
+        - name: Wait for all AWS creates to confirm
+          async_status:
+            jid: "{{ item.ansible_job_id }}"
+          loop: "{{ aws_create_jobs.results }}"
+          register: aws_create_results
+          until: aws_create_results.finished
+          retries: 60
+          delay: 10
+
+        - name: Add all AWS nodes to inventory
+          add_host:
+            name: "{{ item.instances[0].public_ip_address }}"
+            groups: webrunner_nodes
+            ansible_host: "{{ item.instances[0].public_ip_address }}"
+            ansible_user: admin
+            ansible_ssh_private_key_file: "~/.ssh/{{ ssh_key_name }}"
+            ansible_ssh_common_args: "-o StrictHostKeyChecking=no"
+            node_name: "{{ aws_chunks[idx].node_name }}"
+            node_cidrs: "{{ aws_chunks[idx].cidrs }}"
+            node_ip_count: "{{ aws_chunks[idx].ip_count }}"
+            node_idx: "{{ aws_chunks[idx].idx }}"
+            ec2_instance_id: "{{ item.instances[0].instance_id }}"
+            ec2_region: "{{ aws_chunks[idx].region }}"
+            provider: aws
+          loop: "{{ aws_create_results.results }}"
+          loop_control:
+            index_var: idx
+
+        - name: Log AWS scanner IPs
+          lineinfile:
+            path: "{{ scanner_ip_log }}"
+            line: "{{ aws_chunks[idx].node_name }}: {{ item.instances[0].public_ip_address }}"
+            create: true
+          loop: "{{ aws_create_results.results }}"
+          loop_control:
+            index_var: idx
+
+        - name: Show provisioned AWS nodes
+          debug:
+            msg: "Node ready: {{ aws_chunks[idx].node_name }} @ {{ item.instances[0].public_ip_address }} ({{ aws_chunks[idx].region }})"
+          loop: "{{ aws_create_results.results }}"
+          loop_control:
+            index_var: idx
+
+      when: aws_chunks | length > 0
+
+    # ── FlokiNET ──────────────────────────────────────────────────────────────
 
     - name: Provision FlokiNET nodes
       include_tasks: "FlokiNET/webrunner_provision_tasks.yml"
@@ -215,9 +322,11 @@
 
     - name: Teardown AWS nodes
       amazon.aws.ec2_instance:
+        access_key: "{{ lookup('env', 'AWS_ACCESS_KEY_ID') }}"
+        secret_key: "{{ lookup('env', 'AWS_SECRET_ACCESS_KEY') }}"
         instance_ids:
           - "{{ hostvars[item]['ec2_instance_id'] }}"
-        region: "{{ aws_region | default('us-east-1') }}"
+        region: "{{ hostvars[item]['ec2_region'] }}"
         state: terminated
       loop: "{{ groups['webrunner_nodes'] | default([]) }}"
       when:

utils/aws_utils.py

@@ -3,73 +3,86 @@
 AWS provider utilities for C2ingRed deployment system
 """
 
-import random
+import os
+import subprocess
 import logging
-from .common import COLORS, load_vars_file, confirm_action
+from .common import COLORS, load_vars_file
 
 def get_aws_credentials(provider_vars=None):
-    """Get AWS credentials from user or vars file"""
+    """Get AWS credentials — Infisical first, vars.yaml fallback, then prompt."""
-    if not provider_vars:
+    key = ''
-        provider_vars = load_vars_file('aws')
+    secret = ''
-    
-    default_aws_key = provider_vars.get('aws_access_key', '')
-    default_aws_secret = provider_vars.get('aws_secret_key', '')
-    
-    print(f"\n{COLORS['BLUE']}AWS Configuration{COLORS['RESET']}")
-    aws_key = input(f"AWS Access Key [{'*****' if default_aws_key else 'leave blank to use AWS CLI profile'}]: ") or default_aws_key
-    aws_secret = input(f"AWS Secret Key [{'*****' if default_aws_secret else 'leave blank to use AWS CLI profile'}]: ") or default_aws_secret
-    
-    return {
-        'aws_access_key': aws_key,
-        'aws_secret_key': aws_secret
-    }
 
-def select_aws_region(provider_vars=None, component=None):
+    try:
-    """Let the user select an AWS region"""
+        key = subprocess.check_output(
+            [os.path.expanduser('~/.local/bin/creds'), 'get', 'AWS_ACCESS_KEY_ID', 'homelab'],
+            text=True, stderr=subprocess.DEVNULL,
+        ).strip()
+        secret = subprocess.check_output(
+            [os.path.expanduser('~/.local/bin/creds'), 'get', 'AWS_SECRET_ACCESS_KEY', 'homelab'],
+            text=True, stderr=subprocess.DEVNULL,
+        ).strip()
+    except Exception:
+        pass
+
+    if not key:
+        if not provider_vars:
+            provider_vars = load_vars_file('aws')
+        key = provider_vars.get('aws_access_key', '')
+        secret = provider_vars.get('aws_secret_key', '')
+        if key and 'YOUR_AWS' in key:
+            key = ''
+            secret = ''
+
+    print(f"\n{COLORS['BLUE']}AWS Configuration{COLORS['RESET']}")
+    if key:
+        print(f"  AWS credentials loaded")
+    else:
+        key = input("AWS Access Key: ").strip()
+        secret = input("AWS Secret Key: ").strip()
+
+    return {'aws_access_key': key, 'aws_secret_key': secret}
+
+def select_aws_regions(provider_vars=None) -> list[str]:
+    """Return region list — single entry if user specified one, all regions if blank (random per-node)."""
     if not provider_vars:
         provider_vars = load_vars_file('aws')
-    
+
-    regions = provider_vars.get('aws_region_choices', [])
+    regions = provider_vars.get('aws_region_choices', ['us-east-1'])
-    component_str = f" for {component}" if component else ""
+
-    
+    print(f"\nAvailable AWS regions:")
-    if not regions:
-        print(f"{COLORS['YELLOW']}No regions found for AWS, using us-east-1{COLORS['RESET']}")
-        return "us-east-1"
-    
-    print(f"\nAvailable AWS regions{component_str}:")
     for i, region in enumerate(regions, 1):
-        print(f"  {i}. {region}")
+        print(f"  {i:2}. {region}")
-    
+
-    region_input = input(f"\nSelect region{component_str} (number or leave blank for random): ")
+    region_input = input("\nSelect region (number or leave blank for random per-node): ").strip()
-    
+
     if not region_input:
-        return random.choice(regions)
+        return regions
-    
+
     try:
-        region_choice = int(region_input)
+        idx = int(region_input)
-        if 1 <= region_choice <= len(regions):
+        if 1 <= idx <= len(regions):
-            return regions[region_choice - 1]
+            return [regions[idx - 1]]
-        else:
-            print(f"{COLORS['RED']}Invalid choice, using random region{COLORS['RESET']}")
-            return random.choice(regions)
     except ValueError:
-        print(f"{COLORS['RED']}Invalid input, using random region{COLORS['RESET']}")
+        pass
-        return random.choice(regions)
+
+    print(f"{COLORS['RED']}Invalid input, using random per-node regions{COLORS['RESET']}")
+    return regions
 
 def gather_aws_config():
     """Gather all AWS-specific configuration"""
     provider_vars = load_vars_file('aws')
     config = {}
-    
+
-    # Get credentials
     aws_creds = get_aws_credentials(provider_vars)
     config.update(aws_creds)
-    
+
-    # Get region
+    aws_regions = select_aws_regions(provider_vars)
-    config['aws_region'] = select_aws_region(provider_vars)
+    config['aws_regions'] = aws_regions
-    
+    config['aws_region'] = aws_regions[0]
-    # Additional AWS-specific settings
+
+    config['ami_map'] = provider_vars.get('ami_map', {})
     config['aws_instance_type'] = provider_vars.get('aws_instance_type', 't3.micro')
     config['aws_volume_size'] = provider_vars.get('aws_volume_size', 20)
-    
+
     return config

utils/linode_utils.py

@@ -4,7 +4,6 @@ Linode provider utilities for C2ingRed deployment system
 """
 
 import os
-import random
 import logging
 import subprocess
 from .common import COLORS, load_vars_file
@@ -13,7 +12,6 @@ def get_linode_credentials(provider_vars=None):
     """Get Linode API token — Infisical first, vars.yaml fallback, then prompt."""
     default_token = ''
 
-    # Try Infisical first
     try:
         default_token = subprocess.check_output(
             [os.path.expanduser('~/.local/bin/creds'), 'get', 'LINODE_TOKEN', 'homelab'],
@@ -22,7 +20,6 @@ def get_linode_credentials(provider_vars=None):
     except Exception:
         pass
 
-    # Fall back to vars.yaml
     if not default_token:
         if not provider_vars:
             provider_vars = load_vars_file('linode')
@@ -37,56 +34,47 @@ def get_linode_credentials(provider_vars=None):
 
     return {'linode_token': token}
 
-def select_linode_region(provider_vars=None, component=None):
+def select_linode_regions(provider_vars=None) -> list[str]:
-    """Let the user select a Linode region"""
+    """Return region list — single entry if user specified one, all regions if blank (random per-node)."""
     if not provider_vars:
         provider_vars = load_vars_file('linode')
-    
+
-    regions = provider_vars.get('region_choices', [])
+    regions = provider_vars.get('region_choices', ['us-east'])
-    component_str = f" for {component}" if component else ""
+
-    
+    print(f"\nAvailable Linode regions:")
-    if not regions:
-        print(f"{COLORS['YELLOW']}No regions found for Linode, using us-east{COLORS['RESET']}")
-        return "us-east"
-    
-    print(f"\nAvailable Linode regions{component_str}:")
     for i, region in enumerate(regions, 1):
-        print(f"  {i}. {region}")
+        print(f"  {i:2}. {region}")
-    
+
-    region_input = input(f"\nSelect region{component_str} (number or leave blank for random): ")
+    region_input = input("\nSelect region (number or leave blank for random per-node): ").strip()
-    
+
     if not region_input:
-        # Use random from reliable regions only (first 8 are most reliable)
+        return regions
-        reliable_regions = regions[:8] if len(regions) >= 8 else regions
+
-        return random.choice(reliable_regions)
-    
     try:
-        region_choice = int(region_input)
+        idx = int(region_input)
-        if 1 <= region_choice <= len(regions):
+        if 1 <= idx <= len(regions):
-            return regions[region_choice - 1]
+            return [regions[idx - 1]]
-        else:
-            print(f"{COLORS['RED']}Invalid choice, using random region{COLORS['RESET']}")
-            return random.choice(regions)
     except ValueError:
-        print(f"{COLORS['RED']}Invalid input, using random region{COLORS['RESET']}")
+        pass
-        return random.choice(regions)
+
+    print(f"{COLORS['RED']}Invalid input, using random per-node regions{COLORS['RESET']}")
+    return regions
 
 def gather_linode_config():
     """Gather all Linode-specific configuration"""
     provider_vars = load_vars_file('linode')
     config = {}
-    
+
-    # Get credentials
     linode_creds = get_linode_credentials(provider_vars)
     if not linode_creds:
         return None
     config.update(linode_creds)
-    
+
-    # Get region
+    linode_regions = select_linode_regions(provider_vars)
-    config['linode_region'] = select_linode_region(provider_vars)
+    config['linode_regions'] = linode_regions
-    
+    config['linode_region'] = linode_regions[0]
-    # Additional Linode-specific settings
+
     config['linode_instance_type'] = provider_vars.get('linode_instance_type', 'g6-nanode-1')
-    config['linode_image'] = provider_vars.get('linode_image', 'linode/kali')
+    config['linode_image'] = provider_vars.get('linode_image', 'linode/debian12')
-    
+
     return config