feat: add local forwarder configuration (#304)

Co-authored-by: Marco Vito Moscaritolo <[email protected]>

.github/workflows/pr-checker.yml

@@ -20,6 +20,7 @@ jobs:
       fail-fast: false
       matrix:
         scenario:
+          - agent-local-forwarder
           - agent-smoke-kmodule
           - agent-smoke-legacy-ebpf
           - agent-smoke-universal-ebpf
@@ -43,7 +44,7 @@ jobs:
         run: |
           python -m pip install --upgrade pip
           pip install -r molecule/requirements.txt
-          ansible-galaxy collection install amazon.aws community.crypto
+          ansible-galaxy collection install amazon.aws ansible.posix community.crypto community.docker
 
       - name: Expand templates for CI
         run: |

destroy.yml

@@ -0,0 +1,53 @@
+---
+- name: Destroy
+  hosts: localhost
+  connection: local
+  gather_facts: false
+  no_log: "{{ molecule_no_log }}"
+  tags:
+    - always
+  tasks:
+    - name: Set async_dir for HOME env
+      ansible.builtin.set_fact:
+        ansible_async_dir: "{{ lookup('env', 'HOME') }}/.ansible_async/"
+      when: (lookup('env', 'HOME'))
+
+    - name: Destroy molecule instance(s)
+      community.docker.docker_container:
+        name: "{{ item.name }}"
+        docker_host: "{{ item.docker_host | default(lookup('env', 'DOCKER_HOST') or 'unix://var/run/docker.sock') }}"
+        cacert_path: "{{ item.cacert_path | default((lookup('env', 'DOCKER_CERT_PATH') + '/ca.pem') if lookup('env', 'DOCKER_CERT_PATH') else omit) }}"
+        cert_path: "{{ item.cert_path | default((lookup('env', 'DOCKER_CERT_PATH') + '/cert.pem') if lookup('env', 'DOCKER_CERT_PATH') else omit) }}"
+        key_path: "{{ item.key_path | default((lookup('env', 'DOCKER_CERT_PATH') + '/key.pem') if lookup('env', 'DOCKER_CERT_PATH') else omit) }}"
+        tls_verify: "{{ item.tls_verify | default(lookup('env', 'DOCKER_TLS_VERIFY')) or false }}"
+        state: absent
+        force_kill: "{{ item.force_kill | default(true) }}"
+        keep_volumes: "{{ item.keep_volumes | default(true) }}"
+        container_default_behavior: >-
+          {{ item.container_default_behavior
+             | default('compatibility' if ansible_version.full is version_compare('2.10', '>=') else omit) }}
+      register: server
+      loop: "{{ molecule_yml.platforms }}"
+      loop_control:
+        label: "{{ item.name }}"
+      no_log: false
+      async: 7200
+      poll: 0
+
+    - name: Wait for instance(s) deletion to complete
+      ansible.builtin.async_status:
+        jid: "{{ item.ansible_job_id }}"
+      register: docker_jobs
+      until: docker_jobs.finished
+      retries: 300
+      loop: "{{ server.results }}"
+      loop_control:
+        label: "{{ item.item.name }}"
+
+    - name: Delete docker networks(s)
+      ansible.builtin.include_tasks: tasks/delete_network.yml
+      loop: "{{ molecule_yml.platforms | molecule_get_docker_networks() }}"
+      loop_control:
+        label: "{{ item.name }}"
+      no_log: false
+

molecule/agent-local-forwarder/Dockerfile.j2

@@ -0,0 +1,22 @@
+# Molecule managed
+
+{% if item.registry is defined %}
+FROM {{ item.registry.url }}/{{ item.image }}
+{% else %}
+FROM {{ item.image }}
+{% endif %}
+
+{% if item.env is defined %}
+{% for var, value in item.env.items() %}
+{% if value %}
+ENV {{ var }} {{ value }}
+{% endif %}
+{% endfor %}
+{% endif %}
+
+RUN if [ $(command -v apt-get) ]; then export DEBIAN_FRONTEND=noninteractive && apt-get update && apt-get install -y python3 sudo bash ca-certificates iproute2 python3-apt aptitude rsync && apt-get clean && rm -rf /var/lib/apt/lists/*; \
+    elif [ $(command -v dnf) ]; then dnf makecache && dnf --assumeyes install /usr/bin/python3 /usr/bin/python3-config /usr/bin/dnf-3 sudo bash iproute rsync && dnf clean all; \
+    elif [ $(command -v yum) ]; then yum makecache fast && yum install -y /usr/bin/python /usr/bin/python2-config sudo yum-plugin-ovl bash iproute rsync && sed -i 's/plugins=0/plugins=1/g' /etc/yum.conf && yum clean all; \
+    elif [ $(command -v zypper) ]; then zypper refresh && zypper install -y python3 sudo bash iproute2 rsync && zypper clean -a; \
+    elif [ $(command -v apk) ]; then apk update && apk add --no-cache python3 sudo bash ca-certificates rsync; \
+    elif [ $(command -v xbps-install) ]; then xbps-install -Syu && xbps-install -y python3 sudo bash ca-certificates iproute2 rsync && xbps-remove -O; fi

molecule/agent-local-forwarder/converge.yml.template

@@ -0,0 +1,28 @@
+---
+- name: Converge
+  hosts: all
+  strategy: free
+  roles:
+    - role: sysdig.agent.agent_install
+  vars:
+    configuration:
+      connection:
+        access_key: ${AGENT_ACCESS_KEY}
+        custom_collector:
+          url: ${COLLECTOR_URL}
+          port: ${COLLECTOR_PORT}
+      agent:
+        driver:
+          install_build_dependencies: false
+          type: universal_ebpf
+    features:
+      security:
+        local_forwarder:
+          enabled: true
+          integrations:
+            - channels:
+                - SECURE_EVENTS_POLICIES
+                - ACTIVITY_AUDIT
+              configuration:
+                output: stdout
+                type: LOCAL

molecule/agent-local-forwarder/create.yml

@@ -0,0 +1,202 @@
+---
+- name: Create
+  hosts: localhost
+  connection: local
+  gather_facts: false
+  no_log: "{{ molecule_no_log }}"
+  vars:
+    molecule_labels:
+      owner: molecule
+  tags:
+    - always
+  tasks:
+    - name: Set async_dir for HOME env
+      ansible.builtin.set_fact:
+        ansible_async_dir: "{{ lookup('env', 'HOME') }}/.ansible_async/"
+      when: (lookup('env', 'HOME'))
+
+    - name: Log into a Docker registry
+      community.docker.docker_login:
+        username: "{{ item.registry.credentials.username }}"
+        password: "{{ item.registry.credentials.password }}"
+        email: "{{ item.registry.credentials.email | default(omit) }}"
+        registry: "{{ item.registry.url }}"
+        docker_host: "{{ item.docker_host | default(lookup('env', 'DOCKER_HOST') or 'unix://var/run/docker.sock') }}"
+        cacert_path: "{{ item.cacert_path | default((lookup('env', 'DOCKER_CERT_PATH') + '/ca.pem') if lookup('env', 'DOCKER_CERT_PATH') else omit) }}"
+        cert_path: "{{ item.cert_path | default((lookup('env', 'DOCKER_CERT_PATH') + '/cert.pem') if lookup('env', 'DOCKER_CERT_PATH') else omit) }}"
+        key_path: "{{ item.key_path | default((lookup('env', 'DOCKER_CERT_PATH') + '/key.pem') if lookup('env', 'DOCKER_CERT_PATH') else omit) }}"
+        tls_verify: "{{ item.tls_verify | default(lookup('env', 'DOCKER_TLS_VERIFY')) or false }}"
+      with_items: "{{ molecule_yml.platforms }}"
+      when:
+        - item.registry is defined
+        - item.registry.url is defined and item.registry.url
+        - item.registry.credentials is defined
+        - item.registry.credentials.username is defined
+      no_log: true
+
+    - name: Check presence of custom Dockerfiles
+      ansible.builtin.stat:
+        path: "{{ molecule_scenario_directory + '/' + (item.dockerfile | default('Dockerfile.j2')) }}"
+      loop: "{{ molecule_yml.platforms }}"
+      register: dockerfile_stats
+
+    - name: Create Dockerfiles from image names
+      ansible.builtin.template:
+        # when using embedded playbooks the dockerfile is alongside them
+        src: "{%- if dockerfile_stats.results[i].stat.exists -%}\
+          {{ molecule_scenario_directory + '/' + (item.dockerfile | default('Dockerfile.j2')) }}\
+          {%- else -%}\
+          {{ playbook_dir + '/Dockerfile.j2' }}\
+          {%- endif -%}"
+        dest: "{{ molecule_ephemeral_directory }}/Dockerfile_{{ item.image | regex_replace('[^a-zA-Z0-9_]', '_') }}"
+        mode: "0600"
+      loop: "{{ molecule_yml.platforms }}"
+      loop_control:
+        index_var: i
+      when: not item.pre_build_image | default(false)
+      register: platforms
+
+    - name: Synchronization the context
+      ansible.posix.synchronize:
+        src: "{%- if dockerfile_stats.results[i].stat.exists -%}\
+          {{ molecule_scenario_directory + '/' }}\
+          {%- else -%}\
+          {{ playbook_dir + '/' }}\
+          {%- endif -%}"
+        dest: "{{ molecule_ephemeral_directory }}"
+        rsync_opts:
+          - "--exclude=molecule.yml"
+      loop: "{{ molecule_yml.platforms }}"
+      loop_control:
+        index_var: i
+      when: not item.pre_build_image | default(false)
+      delegate_to: localhost
+
+    - name: Discover local Docker images
+      community.docker.docker_image_info:
+        name: "molecule_local/{{ item.item.name }}"
+        docker_host: "{{ item.item.docker_host | default(lookup('env', 'DOCKER_HOST') or 'unix://var/run/docker.sock') }}"
+        cacert_path: "{{ item.cacert_path | default((lookup('env', 'DOCKER_CERT_PATH') + '/ca.pem') if lookup('env', 'DOCKER_CERT_PATH') else omit) }}"
+        cert_path: "{{ item.cert_path | default((lookup('env', 'DOCKER_CERT_PATH') + '/cert.pem') if lookup('env', 'DOCKER_CERT_PATH') else omit) }}"
+        key_path: "{{ item.key_path | default((lookup('env', 'DOCKER_CERT_PATH') + '/key.pem') if lookup('env', 'DOCKER_CERT_PATH') else omit) }}"
+        tls_verify: "{{ item.tls_verify | default(lookup('env', 'DOCKER_TLS_VERIFY')) or false }}"
+      with_items: "{{ platforms.results }}"
+      when:
+        - not item.pre_build_image | default(false)
+      register: docker_images
+
+    - name: Create docker network(s)
+      ansible.builtin.include_tasks: tasks/create_network.yml
+      with_items: "{{ molecule_yml.platforms | molecule_get_docker_networks(molecule_labels) }}"
+      loop_control:
+        label: "{{ item.name }}"
+      no_log: false
+
+    - name: Build an Ansible compatible image (new) # noqa: no-handler
+      when:
+        - platforms.changed or docker_images.results | map(attribute='images') | select('equalto', []) | list | count >= 0
+        - not item.item.pre_build_image | default(false)
+      community.docker.docker_image:
+        build:
+          path: "{{ molecule_ephemeral_directory }}"
+          dockerfile: "{{ item.invocation.module_args.dest }}"
+          pull: "{{ item.item.pull | default(true) }}"
+          network: "{{ item.item.network_mode | default(omit) }}"
+          args: "{{ item.item.buildargs | default(omit) }}"
+          platform: "{{ item.item.platform | default(omit) }}"
+        name: "molecule_local/{{ item.item.image }}"
+        docker_host: "{{ item.item.docker_host | default(lookup('env', 'DOCKER_HOST') or 'unix://var/run/docker.sock') }}"
+        cacert_path: "{{ item.cacert_path | default((lookup('env', 'DOCKER_CERT_PATH') + '/ca.pem') if lookup('env', 'DOCKER_CERT_PATH') else omit) }}"
+        cert_path: "{{ item.cert_path | default((lookup('env', 'DOCKER_CERT_PATH') + '/cert.pem') if lookup('env', 'DOCKER_CERT_PATH') else omit) }}"
+        key_path: "{{ item.key_path | default((lookup('env', 'DOCKER_CERT_PATH') + '/key.pem') if lookup('env', 'DOCKER_CERT_PATH') else omit) }}"
+        tls_verify: "{{ item.tls_verify | default(lookup('env', 'DOCKER_TLS_VERIFY')) or false }}"
+        force_source: "{{ item.item.force | default(true) }}"
+        source: build
+      with_items: "{{ platforms.results }}"
+      loop_control:
+        label: "molecule_local/{{ item.item.image }}"
+      no_log: false
+      register: result
+      until: result is not failed
+      retries: 3
+      delay: 30
+
+    - name: Determine the CMD directives
+      ansible.builtin.set_fact:
+        command_directives_dict: >-
+          {{ command_directives_dict | default({}) |
+             combine({item.name: item.command | default('bash -c "while true; do sleep 10000; done"')})
+          }}
+      with_items: "{{ molecule_yml.platforms }}"
+      when: item.override_command | default(true)
+
+    - name: Create molecule instance(s)
+      community.docker.docker_container:
+        name: "{{ item.name }}"
+        docker_host: "{{ item.docker_host | default(lookup('env', 'DOCKER_HOST') or 'unix://var/run/docker.sock') }}"
+        cacert_path: "{{ item.cacert_path | default((lookup('env', 'DOCKER_CERT_PATH') + '/ca.pem') if lookup('env', 'DOCKER_CERT_PATH') else omit) }}"
+        cert_path: "{{ item.cert_path | default((lookup('env', 'DOCKER_CERT_PATH') + '/cert.pem') if lookup('env', 'DOCKER_CERT_PATH') else omit) }}"
+        key_path: "{{ item.key_path | default((lookup('env', 'DOCKER_CERT_PATH') + '/key.pem') if lookup('env', 'DOCKER_CERT_PATH') else omit) }}"
+        tls_verify: "{{ item.tls_verify | default(lookup('env', 'DOCKER_TLS_VERIFY')) or false }}"
+        hostname: "{{ item.hostname | default(item.name) }}"
+        image: "{{ item.pre_build_image | default(false) | ternary('', 'molecule_local/') }}{{ item.image }}"
+        pull: "{{ item.pull | default(omit) }}"
+        memory: "{{ item.memory | default(omit) }}"
+        memory_swap: "{{ item.memory_swap | default(omit) }}"
+        state: started
+        recreate: false
+        log_driver: json-file
+        command: "{{ (command_directives_dict | default({}))[item.name] | default(omit) }}"
+        command_handling: "{{ item.command_handling | default('compatibility') }}"
+        user: "{{ item.user | default(omit) }}"
+        pid_mode: "{{ item.pid_mode | default(omit) }}"
+        runtime: "{{ item.runtime | default(omit) }}"
+        privileged: "{{ item.privileged | default(omit) }}"
+        security_opts: "{{ item.security_opts | default(omit) }}"
+        devices: "{{ item.devices | default(omit) }}"
+        links: "{{ item.links | default(omit) }}"
+        volumes: "{{ item.volumes | default(omit) }}"
+        mounts: "{{ item.mounts | default(omit) }}"
+        tmpfs: "{{ item.tmpfs | default(omit) }}"
+        capabilities: "{{ item.capabilities | default(omit) }}"
+        sysctls: "{{ item.sysctls | default(omit) }}"
+        exposed_ports: "{{ item.exposed_ports | default(omit) }}"
+        published_ports: "{{ item.published_ports | default(omit) }}"
+        ulimits: "{{ item.ulimits | default(omit) }}"
+        networks: "{{ item.networks | default(omit) }}"
+        network_mode: "{{ item.network_mode | default(omit) }}"
+        networks_cli_compatible: "{{ item.networks_cli_compatible | default(true) }}"
+        purge_networks: "{{ item.purge_networks | default(omit) }}"
+        dns_servers: "{{ item.dns_servers | default(omit) }}"
+        etc_hosts: "{{ item.etc_hosts | default(omit) }}"
+        env: "{{ item.env | default(omit) }}"
+        restart_policy: "{{ item.restart_policy | default(omit) }}"
+        restart_retries: "{{ item.restart_retries | default(omit) }}"
+        tty: "{{ item.tty | default(omit) }}"
+        labels: "{{ molecule_labels | combine(item.labels | default({})) }}"
+        container_default_behavior: >-
+          {{ item.container_default_behavior
+             | default('compatibility' if ansible_version.full is version_compare('2.10', '>=') else omit) }}
+        stop_signal: "{{ item.stop_signal | default(omit) }}"
+        kill_signal: "{{ item.kill_signal | default(omit) }}"
+        cgroupns_mode: "{{ item.cgroupns_mode | default(omit) }}"
+        shm_size: "{{ item.shm_size | default(omit) }}"
+        platform: "{{ item.platform | default(omit) }}"
+        comparisons:
+          platform: ignore
+      register: server
+      with_items: "{{ molecule_yml.platforms }}"
+      loop_control:
+        label: "{{ item.name }}"
+      no_log: false
+      async: 7200
+      poll: 0
+
+    - name: Wait for instance(s) creation to complete
+      ansible.builtin.async_status:
+        jid: "{{ item.ansible_job_id }}"
+      register: docker_jobs
+      until: docker_jobs.finished
+      retries: 300
+      with_items: "{{ server.results }}"
+

molecule/agent-local-forwarder/filter_plugins/get_docker_networks.py

@@ -0,0 +1,37 @@
+"""Embedded ansible filter used by Molecule Docker driver create playbook."""
+
+
+def get_docker_networks(data, labels={}):
+    """Get list of docker networks."""
+    network_list = []
+    network_names = []
+    for platform in data:
+        if "docker_networks" in platform:
+            for docker_network in platform["docker_networks"]:
+                if "labels" not in docker_network:
+                    docker_network["labels"] = {}
+                for key in labels:
+                    docker_network["labels"][key] = labels[key]
+
+                if "name" in docker_network:
+                    network_list.append(docker_network)
+                    network_names.append(docker_network["name"])
+
+        # If a network name is defined for a platform but is not defined in
+        # docker_networks, add it to the network list.
+        if "networks" in platform:
+            for network in platform["networks"]:
+                if "name" in network:
+                    name = network["name"]
+                    if name not in network_names:
+                        network_list.append({"name": name, "labels": labels})
+    return network_list
+
+
+class FilterModule:
+    """Core Molecule filter plugins."""
+
+    def filters(self):
+        return {
+            "molecule_get_docker_networks": get_docker_networks,
+        }

molecule/agent-local-forwarder/molecule.yml

@@ -0,0 +1,32 @@
+---
+dependency:
+  name: galaxy
+driver:
+  name: docker
+platforms:
+  - name: centos-stream8
+    image: quay.io/centos/centos:stream8
+    pre_build_image: true
+    privileged: true
+provisioner:
+  name: ansible
+scenario:
+  create_sequence:
+    - create
+    - prepare
+  converge_sequence:
+    - create
+    - prepare
+    - converge
+  destroy_sequence:
+    - destroy
+  test_sequence:
+    - create
+    - prepare
+    - converge
+    - verify
+    - destroy
+verifier:
+  name: testinfra
+  additional_files_or_dirs:
+    - tests