Merge pull request #24 from cisco-open/fr/cluster_updated

allow for deployment of vmanages with cluster subnet configured on AWS

.gitignore

@@ -13,3 +13,4 @@ deployment_results
 playbooks/deployment_results
 playbooks/results
 dev_dir_tmp
+.idea

galaxy.yml

@@ -1,6 +1,6 @@
 namespace: cisco
 name: sdwan_deployment
-version: 0.2.2
+version: 0.2.3
 readme: README.md
 authors:
   - Arkadiusz Cichon <[email protected]>

roles/aws_controllers/defaults/main.yml

@@ -40,6 +40,7 @@ default_vbond_ip: 192.168.1.199  # default ips from official Cisco guides
 aws_vmanage_ami_id: null
 aws_vmanage_instance_type: "c5.9xlarge"
 
+vmanage_default_persona: "COMPUTE_AND_DATA"
 site_id_vmanage: 100
 
 vmanage_instances: []

roles/aws_controllers/tasks/aws_vmanage_ec2_instance.yml

@@ -14,7 +14,8 @@
 - name: Filter required subnets for instance creation. Set aws_mgmt_subnet and aws_transport_subnet facts
   ansible.builtin.set_fact:
     aws_mgmt_subnet: "{{ aws_subnets_config | selectattr('tags.VPN', 'equalto', '512') | list | first }}"
-    aws_transport_subnet: "{{ aws_subnets_config | selectattr('tags.VPN', 'equalto', '0') | list | first }}"
+    aws_transport_subnet: "{{ aws_subnets_config | selectattr('tags.Name', 'search', 'transport') | list | first }}"
+    aws_cluster_subnet: "{{ aws_subnets_config | selectattr('tags.Name', 'search', 'cluster') | list | first | default(omit) }}"
 
 # Note that we don't have mechanism yet to detect if there are already required interfaces present on AWS for
 # that particular machine. We should include here some check to verify that they exist and skip if present,
@@ -31,7 +32,11 @@
       Machine: "{{ hostname }}"
       VPN: "{{ subnet_item.tags.VPN }}"
   register: network_interfaces_vmanage
-  loop: "{{ [aws_mgmt_subnet, aws_transport_subnet] }}"
+  loop: >
+        {{
+            [aws_mgmt_subnet, aws_transport_subnet]
+            + ([aws_cluster_subnet] if aws_cluster_subnet is defined else [])
+        }}
   loop_control:
     loop_var: subnet_item
     label: "nic-{{ subnet_item.tags.Name }}"
@@ -40,11 +45,19 @@
   ansible.builtin.set_fact:
     aws_network_interfaces: "{{ network_interfaces_vmanage.results | map(attribute='interface') | list }}"
 
-- name: Filter aws_network_interfaces for instance creation. Set aws_mgmt_nic and aws_transport_nic facts
+- name: Filter aws_network_interfaces for instance creation. Set NIC facts
   ansible.builtin.set_fact:
     aws_mgmt_nic: "{{ aws_network_interfaces | selectattr('tags.VPN', 'equalto', '512') | list | first }}"
-    aws_transport_nic: "{{ aws_network_interfaces | selectattr('tags.VPN', 'equalto', '0') | list | first }}"
+    aws_transport_nic: "{{ aws_network_interfaces | selectattr('tags.Name', 'search', 'transport') | list | first }}"
+    aws_cluster_nic: "{{ aws_network_interfaces | selectattr('tags.Name', 'search', 'cluster') | list | first | default(omit) }}"
 
+- name: Set vmanage cluster related facts
+  ansible.builtin.set_fact:
+    vmanage_cluster_private_ip: "{{ aws_cluster_nic.private_ip_address }}"
+    vmanage_cluster_prefix: "{{ aws_cluster_subnet.cidr_block.split('/')[1] }}"
+  when:
+    - aws_cluster_nic is defined
+    - aws_cluster_subnet is defined
 
 # EIPs
 - name: Associate EIP with network interfaces
@@ -68,11 +81,11 @@
   delay: 3
   until: eip_vmanage is succeeded
 
-
 # cloud-init
-- name: Set path for bootstrap configuration
+- name: Set path for bootstrap configuration and vmanage persona
   ansible.builtin.set_fact:
     generated_userdata_vmanage: "{{ userdata_vmanage_path }}-{{ hostname }}"
+    vmanage_persona: "{{ persona | default(vmanage_default_persona) }}"
   changed_when: true
 
 - name: Template userdata file for vManage
@@ -81,6 +94,20 @@
     dest: "{{ generated_userdata_vmanage }}"
     mode: "0644"
 
+- name: Set vManage interfaces list
+  ansible.builtin.set_fact:
+    vmanage_interfaces:
+      - id: "{{ aws_mgmt_nic.id }}"
+        device_index: 0
+        description: "{{ aws_mgmt_nic.tags.Name }}"
+      - id: "{{ aws_transport_nic.id }}"
+        device_index: 1
+        description: "{{ aws_transport_nic.tags.Name }}"
+
+- name: Add vManage cluster interface to vManage interfaces list
+  ansible.builtin.set_fact:
+    vmanage_interfaces: "{{ vmanage_interfaces + [{'id': aws_cluster_nic.id, 'device_index': 2, 'description': aws_cluster_nic.tags.Name}] }}"
+  when: aws_cluster_nic is defined
 
 # vManage
 - name: Launch vManage
@@ -95,13 +122,7 @@
     key_name: "{{ aws_key_name | default('') | bool | ternary(aws_key_name, omit) }}"
     network:
       assign_public_ip: false
-      interfaces:
-        - id: "{{ aws_mgmt_nic.id }}"
-          device_index: 0
-          description: "{{ aws_mgmt_nic.tags.Name }}"
-        - id: "{{ aws_transport_nic.id }}"
-          device_index: 1
-          description: "{{ aws_transport_nic.tags.Name }}"
+      interfaces: "{{ vmanage_interfaces }}"
     name: "{{ hostname }}"
     tags:
       Name: "{{ hostname }}"
@@ -127,6 +148,8 @@
       admin_password: "{{ admin_password }}"
       mgmt_public_ip: "{{ eip_vmanage.results[0].public_ip }}"
       transport_public_ip: "{{ eip_vmanage.results[1].public_ip }}"
+      cluster_private_ip: "{{ aws_cluster_nic.private_ip_address | default(omit) }}"
+      persona: "{{ vmanage_persona }}"
   changed_when: true
   notify: Show deployment_facts
   register: _vmanage_facts
@@ -151,7 +174,22 @@
     marker: "\n--------------    ec2 vManage   --------------\n"
     content: "{{ ec2_vmanage | to_nice_yaml }}"
 
-- name: Allow traffic outside VPC for vManage IP addresses
+- name: Set security group rules
+  ansible.builtin.set_fact:
+    sg_rules:
+      - proto: all  # proto all means: `ports: all`
+        cidr_ip: "{{ eip_vmanage.results[0].public_ip }}/32"
+        rule_desc: "{{ hostname }} - mgmt (VPN 512)"
+      - proto: all
+        cidr_ip: "{{ eip_vmanage.results[1].public_ip }}/32"
+        rule_desc: "{{ hostname }} - transport (VPN 0)"
+
+- name: Update security group rules to handle cluster traffic
+  ansible.builtin.set_fact:
+    vmanage_interfaces: "{{ sg_rules + [{'proto': 'all', 'cidr_ip': vmanage_cluster_private_ip + '/32', 'rule_desc': hostname + ' - cluster (VPN 0)'}] }}"
+  when: cluster_private_ip is defined
+
+- name: Configure security group
   amazon.aws.ec2_security_group:
     name: "{{ aws_security_group_config.group_name }}"
     description: "Security Group for SD-WAN instances"
@@ -160,13 +198,7 @@
     purge_rules: false
     purge_tags: false
     purge_rules_egress: false
-    rules:
-      - proto: all  # proto all means: `ports: all`
-        cidr_ip: "{{ eip_vmanage.results[0].public_ip }}/32"
-        rule_desc: "{{ hostname }} - mgmt (VPN 512)"
-      - proto: all
-        cidr_ip: "{{ eip_vmanage.results[1].public_ip }}/32"
-        rule_desc: "{{ hostname }} - transport (VPN 0)"
+    rules: "{{ sg_rules }}"
     rules_egress: []
   register: allow_traffic
   retries: 3

roles/aws_controllers/tasks/main.yml

@@ -83,6 +83,7 @@
     hostname: "{{ instance_item.hostname }}"
     system_ip: "{{ instance_item.system_ip }}"
     site_id: "{{ instance_item.site_id }}"
+    persona: "{{ instance_item.persona }}"
   loop: "{{ vmanage_instances }}"
   loop_control:
     loop_var: instance_item

roles/aws_controllers/templates/userdata_vmanage.j2

@@ -33,7 +33,7 @@ write_files:
 - path: /opt/web-app/etc/persona
   owner: vmanage:vmanage-admin
   permissions: '0644'
-  content: '{"persona":"COMPUTE_AND_DATA"}'
+  content: '{"persona":"{{ vmanage_persona }}"}'
 - path: /etc/default/personality
   content: "vmanage\n"
 - path: /etc/default/inited
@@ -97,4 +97,76 @@ write_files:
         </vpn-instance>
       </vpn>
     </config>
+
+{% if vmanage_cluster_private_ip is defined %}
+- path: /home/admin/customized.cfg
+  content: |
+    vpn 0
+        interface eth2
+            ip address {{ vmanage_cluster_private_ip }}/{{ vmanage_cluster_prefix }}
+            no shutdown
+        !
+    !
+
+bootcmd:
+  - |
+    cat << 'EOF' > /etc/customized.sh
+    #!/bin/bash
+    MAX_WAIT=1800
+    WAIT_TIMER=30
+    MAX_RETRY=20
+    RETRY_TIMER=5
+
+    success_file_name=/home/admin/success.log
+    failure_file_name=/home/admin/failure.log
+
+    start_time=`date +%s`
+    while true
+    do
+        end_time=`date +%s`
+
+        CONFD_IPC_ACCESS_FILE="/etc/confd/confd_ipc_secret"
+        cloud_init_rerun="/home/admin/customized.cfg"
+        export CONFD_IPC_ACCESS_FILE=/etc/confd/confd_ipc_secret
+
+        if [ -f "$CONFD_IPC_ACCESS_FILE" ] && [ -f "$cloud_init_rerun" ];then
+            sleep $WAIT_TIMER
+
+            break_flap=0
+            for retry_timer in `seq $MAX_RETRY`
+            do
+              confd_load -H -dd -lm -e -F c $cloud_init_rerun >> "$success_file_name" 2>&1
+              if [ $? -eq 0 ];then
+                echo success >> "$success_file_name"
+                mv $cloud_init_rerun ${cloud_init_rerun}.bak
+                break_flap=1
+                break
+              fi
+              sleep $RETRY_TIMER
+              echo "Retry: $retry_timer" >> "$success_file_name"
+            done
+
+            if [ $break_flap -eq 0 ];then
+              echo "Load configure error" >> "$success_file_name"
+            else
+              echo "Load configure success" >> "$success_file_name"
+            fi
+
+            break
+        fi
+
+        if [ $((end_time - start_time)) -gt $MAX_WAIT ];then
+            echo "start_time: $start_time" >> "$failure_file_name"
+            echo "end_time: $end_time" >> "$failure_file_name"
+            echo "end_time - start_time: $((end_time - start_time))" >> "$failure_file_name"
+            echo "MAX_WAIT: $MAX_WAIT" >> "$failure_file_name"
+            break
+        fi;
+
+        sleep 0.1
+
+    done
+    EOF
+  - (nohup bash /etc/customized.sh >> /home/admin/customized_config.log 2>&1 )&
+{% endif %}
 --===============8815267485200512281==

roles/aws_network_infrastructure/defaults/main.yml

@@ -27,7 +27,8 @@ aws_vpc_cidr: 10.0.0.0/16  # default ips from official Cisco guides
 aws_igw_name: "{{ aws_resources_prefix }}-igw"
 
 # SUBNETs
-aws_subnets:
+aws_subnets: |
+  {% filter from_yaml %}
   - name: "{{ aws_resources_prefix }}-mgmt-subnet-512"
     subnet_cidr: "10.0.1.0/24"  # default ips from official Cisco guides
     VPN: 512
@@ -36,10 +37,13 @@ aws_subnets:
     subnet_cidr: "10.0.2.0/24"  # default ips from official Cisco guides
     VPN: 0
     type: transport
-  # - name: "{{ aws_resources_prefix }}-cluster-subnet-0"
-  #   subnet_cidr: "10.0.3.0/24"
-  #   VPN: 0
-  #   type: cluster
+  {% if vmanage_instances is defined and vmanage_instances | length > 2 %}
+  - name: "{{ aws_resources_prefix }}-cluster-subnet-0"
+    subnet_cidr: "10.0.3.0/24"
+    VPN: 0
+    type: cluster
+  {% endif %}
+  {% endfilter %}
 
 # ROUTE TABLEs
 aws_route_table_name: "{{ aws_resources_prefix }}-rtab"

roles/aws_network_infrastructure/tasks/aws_create_network_infrastructure.yml

@@ -63,7 +63,7 @@
     state: present
     vpc_id: "{{ _created_vpc.vpc.id }}"
     cidr: "{{ subnet_config.subnet_cidr }}"
-    map_public: true
+    map_public: "{{ subnet_config.type != 'cluster' }}"
     region: "{{ aws_region }}"
     az: "{{ aws_availibility_zone }}"
     tags:
@@ -115,6 +115,7 @@
     aws_route_tables:
       - "{{ _created_route_table.results[0] }}"
       - "{{ _created_route_table.results[1] }}"
+      - "{{ _created_route_table.results[2] | default(omit) }}"
 
 - name: Copy aws_route_tables resources information to resources file
   ansible.builtin.blockinfile:

roles/aws_teardown/tasks/ec2_instance.yml

@@ -13,6 +13,7 @@
 
 - name: Terminate all instances in a specific VPC
   amazon.aws.ec2_instance:
+    region: "{{ aws_region }}"
     state: terminated
     instance_ids: "{{ instance_item.instance_id }}"
   loop: "{{ ec2_info.instances }}"

roles/azure_controllers/tasks/azure_vbond_vm.yml

@@ -20,8 +20,8 @@
       VPN: "{{ subnet_item.VPN }}"
       Subnet: "{{ subnet_item.name }}"
   loop:
-    - "{{ transport_subnet }}"
     - "{{ mgmt_subnet }}"
+    - "{{ transport_subnet }}"
   loop_control:
     loop_var: subnet_item
   register: public_ip_addresses

roles/azure_controllers/tasks/azure_vmanage_vm.yml

@@ -144,10 +144,16 @@
     vmanage_transport_private_ip: "{{ az_transport_nic.ip_configuration.private_ip_address }}"
     vmanage_mgmt_public_ip: "{{ az_mgmt_public_ip.ip_address }}"
     vmanage_transport_public_ip: "{{ az_transport_public_ip.ip_address }}"
-    vmanage_cluster_private_ip: "{{ az_cluster_nic.ip_configuration.private_ip_address | default(omit) }}"
-    vmanage_cluster_prefix: "{{ cluster_subnet.cidr | ansible.utils.ipaddr('prefix') | default(omit) }}"
     vmanage_persona: "{{ persona | default(vmanage_default_persona) }}"
 
+- name: "Set vmanage cluster related facts"
+  ansible.builtin.set_fact:
+    vmanage_cluster_private_ip: "{{ az_cluster_nic.ip_configuration.private_ip_address }}"
+    vmanage_cluster_prefix: "{{ cluster_subnet.cidr | ansible.utils.ipaddr('prefix') }}"
+  when:
+    - az_cluster_nic is defined
+    - cluster_subnet is defined
+
 - name: "Set vpn0_default_gateway fact from VPN 0 subnet value"
   ansible.builtin.set_fact:
     vpn0_default_gateway: "{{ subnet.cidr | ansible.utils.ipaddr('1') | ansible.utils.ipaddr('address') }}"

roles/azure_controllers/tasks/azure_vsmart_vm.yml

@@ -20,8 +20,8 @@
       VPN: "{{ subnet_item.VPN }}"
       Subnet: "{{ subnet_item.name }}"
   loop:
-    - "{{ transport_subnet }}"
     - "{{ mgmt_subnet }}"
+    - "{{ transport_subnet }}"
   loop_control:
     loop_var: subnet_item
   register: public_ip_addresses