diff --git a/providers/Linode/webrunner_provision_tasks.yml.retired b/providers/Linode/webrunner_provision_tasks.yml.retired new file mode 100644 index 0000000..c28ebf0 --- /dev/null +++ b/providers/Linode/webrunner_provision_tasks.yml.retired @@ -0,0 +1,90 @@ +--- +# Linode provision tasks for one WEBRUNNER node +# Called in a loop — loop_var: node_chunk +# Requires: linode_token, ssh_key_name, webrunner_name, deployment_id, +# linode_instance_type, linode_region, scanner_ip_log, results_dir + +- name: Generate root password for {{ node_chunk.node_name }} + set_fact: + wr_root_pass: "{{ lookup('password', '/dev/null chars=ascii_letters,digits length=32') }}" + +- name: Read public key + slurp: + src: "~/.ssh/{{ ssh_key_name }}.pub" + register: wr_pubkey + +- name: Register SSH key in Linode ({{ node_chunk.node_name }}) + uri: + url: "https://api.linode.com/v4/profile/sshkeys" + method: POST + headers: + Authorization: "Bearer {{ linode_token }}" + Content-Type: "application/json" + body_format: json + body: + label: "{{ webrunner_name }}" + ssh_key: "{{ wr_pubkey.content | b64decode | trim }}" + status_code: [200, 201] + ignore_errors: true + +- name: Create Linode instance {{ node_chunk.node_name }} + uri: + url: "https://api.linode.com/v4/linode/instances" + method: POST + headers: + Authorization: "Bearer {{ linode_token }}" + Content-Type: "application/json" + body_format: json + body: + label: "{{ node_chunk.node_name }}" + type: "{{ linode_instance_type | default('g6-standard-2') }}" + region: "{{ linode_region | default('us-east') }}" + image: "linode/debian12" + root_pass: "{{ wr_root_pass }}" + authorized_keys: + - "{{ wr_pubkey.content | b64decode | trim }}" + booted: true + backups_enabled: false + private_ip: false + tags: + - "webrunner" + - "{{ webrunner_name }}" + - "{{ deployment_id }}" + status_code: [200, 201] + register: wr_linode + +- name: Extract node IP + set_fact: + wr_node_ip: "{{ wr_linode.json.ipv4[0] }}" + +- name: Log scanner IP + lineinfile: + path: "{{ scanner_ip_log }}" + line: "{{ node_chunk.node_name }}: {{ wr_node_ip }}" + create: true + +- name: Wait for SSH on {{ node_chunk.node_name }} ({{ wr_node_ip }}) + wait_for: + host: "{{ wr_node_ip }}" + port: 22 + delay: 30 + timeout: 300 + +- name: Add {{ node_chunk.node_name }} to inventory + add_host: + name: "{{ wr_node_ip }}" + groups: webrunner_nodes + ansible_host: "{{ wr_node_ip }}" + ansible_user: root + ansible_ssh_private_key_file: "~/.ssh/{{ ssh_key_name }}" + ansible_ssh_common_args: "-o StrictHostKeyChecking=no" + node_name: "{{ node_chunk.node_name }}" + node_cidrs: "{{ node_chunk.cidrs }}" + node_ip_count: "{{ node_chunk.ip_count }}" + node_idx: "{{ node_chunk.idx }}" + linode_instance_id: "{{ wr_linode.json.id }}" + provider: linode + +- name: Show {{ node_chunk.node_name }} ready + debug: + msg: "Linode node ready: {{ node_chunk.node_name }} @ {{ wr_node_ip }} ({{ node_chunk.ip_count | int | string }} IPs)" diff --git a/providers/webrunner.yml b/providers/webrunner.yml index 5223f07..d844bce 100644 --- a/providers/webrunner.yml +++ b/providers/webrunner.yml @@ -11,6 +11,7 @@ ansible_python_interpreter: "{{ ansible_playbook_python }}" ssh_key_name: "{{ ssh_key_path | basename | regex_replace('\\.pub$', '') }}" all_node_chunks: "{{ lookup('file', node_chunks_file) | from_json }}" + linode_chunks: "{{ all_node_chunks | selectattr('provider', 'equalto', 'linode') | list }}" tasks: - name: Ensure results directory @@ -26,11 +27,107 @@ mode: '0644' force: false - - name: Provision Linode nodes - include_tasks: "Linode/webrunner_provision_tasks.yml" - loop: "{{ all_node_chunks | selectattr('provider', 'equalto', 'linode') | list }}" - loop_control: - loop_var: node_chunk + # ── Linode — all nodes created in parallel ──────────────────────────────── + + - block: + - name: Generate shared root password + set_fact: + wr_root_pass: "{{ lookup('password', '/dev/null chars=ascii_letters,digits length=32') }}" + + - name: Read SSH public key + slurp: + src: "~/.ssh/{{ ssh_key_name }}.pub" + register: wr_pubkey + + - name: Register SSH key with Linode + uri: + url: "https://api.linode.com/v4/profile/sshkeys" + method: POST + headers: + Authorization: "Bearer {{ lookup('env', 'LINODE_TOKEN') }}" + Content-Type: "application/json" + body_format: json + body: + label: "{{ webrunner_name }}" + ssh_key: "{{ wr_pubkey.content | b64decode | trim }}" + status_code: [200, 201, 422] + ignore_errors: true + + - name: Fire all Linode instance creates + uri: + url: "https://api.linode.com/v4/linode/instances" + method: POST + headers: + Authorization: "Bearer {{ lookup('env', 'LINODE_TOKEN') }}" + Content-Type: "application/json" + body_format: json + body: + label: "{{ item.node_name }}" + type: "{{ linode_instance_type | default('g6-standard-2') }}" + region: "{{ linode_region | default('us-east') }}" + image: "linode/debian12" + root_pass: "{{ wr_root_pass }}" + authorized_keys: + - "{{ wr_pubkey.content | b64decode | trim }}" + booted: true + backups_enabled: false + private_ip: false + tags: + - "webrunner" + - "{{ webrunner_name }}" + - "{{ deployment_id }}" + status_code: [200, 201] + loop: "{{ linode_chunks }}" + async: 300 + poll: 0 + register: wr_create_jobs + + - name: Wait for all creates to confirm + async_status: + jid: "{{ item.ansible_job_id }}" + loop: "{{ wr_create_jobs.results }}" + register: wr_create_results + until: wr_create_results.finished + retries: 30 + delay: 10 + + - name: Add all Linode nodes to inventory + add_host: + name: "{{ item.json.ipv4[0] }}" + groups: webrunner_nodes + ansible_host: "{{ item.json.ipv4[0] }}" + ansible_user: root + ansible_ssh_private_key_file: "~/.ssh/{{ ssh_key_name }}" + ansible_ssh_common_args: "-o StrictHostKeyChecking=no" + node_name: "{{ linode_chunks[idx].node_name }}" + node_cidrs: "{{ linode_chunks[idx].cidrs }}" + node_ip_count: "{{ linode_chunks[idx].ip_count }}" + node_idx: "{{ linode_chunks[idx].idx }}" + linode_instance_id: "{{ item.json.id }}" + provider: linode + loop: "{{ wr_create_results.results }}" + loop_control: + index_var: idx + + - name: Log scanner IPs + lineinfile: + path: "{{ scanner_ip_log }}" + line: "{{ linode_chunks[idx].node_name }}: {{ item.json.ipv4[0] }}" + create: true + loop: "{{ wr_create_results.results }}" + loop_control: + index_var: idx + + - name: Show provisioned nodes + debug: + msg: "Node ready: {{ linode_chunks[idx].node_name }} @ {{ item.json.ipv4[0] }}" + loop: "{{ wr_create_results.results }}" + loop_control: + index_var: idx + + when: linode_chunks | length > 0 + + # ── AWS / FlokiNET remain serial until those providers are implemented ───── - name: Provision AWS nodes include_tasks: "AWS/webrunner_provision_tasks.yml" @@ -49,6 +146,7 @@ hosts: webrunner_nodes gather_facts: false become: true + strategy: free tasks: - name: Wait for SSH diff --git a/utils/deployment_engine.py b/utils/deployment_engine.py index f88aa8b..f0a7c7f 100644 --- a/utils/deployment_engine.py +++ b/utils/deployment_engine.py @@ -164,6 +164,10 @@ def execute_playbook(playbook, config): '--extra-vars', create_extra_vars(config) ] + # Webrunner runs many nodes in parallel — increase fork count + if 'webrunner' in playbook: + cmd += ['-f', '50'] + # Write sensitive vars to a temp file (0o600) so they reach Ansible as # variables without appearing in process listings or the main extra-vars JSON. # Read from environment (set by set_provider_environment) so this works even