[7.16](backport #2064) fix: update stack deployment to follow Users&Roles best practices

.ci/.e2e-tests-for-elastic-agent.yaml

@@ -36,7 +36,7 @@ SUITES:
         platforms: ["debian_arm64", "debian_amd64"]
       - name: "APM Integration"
         tags: "apm_server"
-        platforms: ["fleet_elastic_pkg"]
+        platforms: ["debian_amd64"]
       - name: "Linux Integration"
         tags: "linux_integration"
         platforms: ["debian_arm64", "debian_amd64"]

.ci/.e2e-tests.yaml

@@ -47,7 +47,7 @@ SUITES:
         platforms: ["debian_arm64", "debian_amd64"]
       - name: "APM Integration"
         tags: "apm_server"
-        platforms: ["fleet_elastic_pkg"]
+        platforms: ["debian_amd64"]
       - name: "Linux Integration"
         tags: "linux_integration"
         platforms: ["debian_arm64", "debian_amd64"]

.ci/Jenkinsfile

@@ -215,7 +215,7 @@ pipeline {
                     ansible(
                       stackWorkspace,
                       env.RUN_ID.split('-')[0],
-                      "-i \"${stackRunner.ip},\" -t setup-stack --extra-vars=\"${LABELS_STRING} nodeLabel=stack nodeUser=${stackMachine.username} nodeImage=${stackMachine.image} nodeInstanceType=${stackMachine.instance_type}\""
+                      "-i \"${stackRunner.ip},\" -t setup-stack --extra-vars=\"${LABELS_STRING} stackRunner=${stackRunner.ip} nodeLabel=stack nodeUser=${stackMachine.username} nodeImage=${stackMachine.image} nodeInstanceType=${stackMachine.instance_type}\""
                     )
                   }
 

.ci/ansible/files/sshd_config

@@ -0,0 +1,2 @@
+ClientAliveInterval 60
+ClientAliveCountMax 10

.ci/ansible/playbook.yml

@@ -65,6 +65,11 @@
     tags:
       - setup-stack
 
+  - name: Configure test script
+    include_tasks: tasks/setup_test_script.yml
+    tags:
+      - setup-stack
+
   - name: Add SSH keys to stack
     include_tasks: tasks/install_ssh_keys.yml
     tags:
@@ -77,19 +82,20 @@
 
   - name: Configure stack files
     ansible.builtin.replace:
-      path: "/home/{{ ansible_user }}/e2e-testing/cli/config/compose/profiles/fleet/default/kibana.config.yml"
+      path: '{{ item.path }}'
       regexp: '{{ item.old }}'
       replace: '{{ item.new }}'
     loop:
-      - { old: 'http://elasticsearch', new: 'http://{{ inventory_hostname }}' }
-      - { old: 'http://fleet-server', new: 'http://{{ inventory_hostname }}' }
-      - { old: 'http://package-registry:8080', new: 'https://epr-staging.elastic.co' }
+      - { old: 'http://elasticsearch', new: 'http://{{ inventory_hostname }}', path: "/home/{{ ansible_user }}/e2e-testing/cli/config/compose/profiles/fleet/default/kibana.config.yml" }
+      - { old: 'http://fleet-server', new: 'http://{{ inventory_hostname }}', path: "/home/{{ ansible_user }}/e2e-testing/cli/config/compose/profiles/fleet/default/kibana.config.yml" }
+      - { old: 'http://kibana', new: 'http://{{ inventory_hostname }}', path: "/home/{{ ansible_user }}/e2e-testing/cli/config/compose/services/elastic-agent/fleet-server/docker-compose.yml" }
+      - { old: 'http://package-registry:8080', new: 'https://epr-staging.elastic.co', path: "/home/{{ ansible_user }}/e2e-testing/cli/config/compose/profiles/fleet/default/kibana.config.yml" }
     tags:
       - setup-stack
 
-  - name: Start stack
+  - name: Start stack without any tag to spin up the stack using Fleet's bootstrapping code
     become: true
-    shell: docker-compose -f /home/{{ansible_user}}/e2e-testing/cli/config/compose/profiles/fleet/docker-compose.yml up -d
+    shell: "/home/{{ ansible_user }}/e2e-testing/.ci/scripts/functional-test.sh 'non-existing-tag'"
     tags:
       - setup-stack
 
@@ -107,19 +113,22 @@
     - role: geerlingguy.docker
       docker_daemon_options:
         default-ulimit: ["nofile=1024000:1024000"]
-      when: "'kubernetes-autodiscover' in suite or 'helm' in suite or 'fleet_amd64' in nodeLabel or 'fleet_elastic_pkg' in nodeLabel"
+      when:
+        - '"arm64" not in nodeLabel'
+        - 'ansible_os_family not in ["Suse"]'
     - role: andrewrothstein.kubectl
-      when: "'kubernetes-autodiscover' in suite or 'helm' in suite"
+      when: suite in ["kubernetes-autodiscover", "helm"]
     - role: andrewrothstein.kind
-      when: "'kubernetes-autodiscover' in suite or 'helm' in suite"
+      when: suite in ["kubernetes-autodiscover", "helm"]
     - role: geerlingguy.helm
-      when: "'kubernetes-autodiscover' in suite or 'helm' in suite"
+      when: suite in ["kubernetes-autodiscover", "helm"]
     - role: gantsign.golang
       vars:
         golang_version: 1.16.3
   vars:
     ansible_python_interpreter: "auto"
     ansible_user: "{{nodeUser}}"
+    pip_package: "python3-pip"
   tasks:
   - name: Install deps
     include_tasks: tasks/install_deps.yml

.ci/ansible/requirements.yml

@@ -1,4 +1,6 @@
+- src: geerlingguy.pip
 - src: geerlingguy.docker
+- src: geerlingguy.docker_arm
 - src: geerlingguy.helm
 - src: andrewrothstein.kubectl
 - src: andrewrothstein.kind

.ci/ansible/tasks/install_deps.yml

@@ -22,6 +22,7 @@
     name:
       - autoconf
       - bison
+      - docker
       - flex
       - gcc
       - gcc-c++
@@ -55,3 +56,18 @@
 - name: Install ssh-import-id python package to copy public SSH keys from Github accounts
   pip:
     name: ssh-import-id
+
+- name: Set sshd configuration for client alive settings
+  ansible.builtin.copy:
+    src: sshd_config
+    dest: /etc/ssh/sshd_config
+    owner: "{{ ansible_user }}"
+    group: "{{ ansible_user }}"
+    mode: '0600'
+  when: ansible_distribution in ["CentOS", "Debian", "Fedora", "RedHat", "Ubuntu"]
+
+- name: Install Docker for ARM (Debian, Ubuntu)
+  ansible.builtin.shell: curl -fsSL test.docker.com -o get-docker.sh && sh get-docker.sh
+  when:
+    - ansible_distribution in ["Debian", "Ubuntu"]
+    - '"arm64" in nodeLabel'

.ci/ansible/tasks/runners.yml

@@ -52,7 +52,7 @@
     - start-node
 
 - name: Wait for SSH to come up
-  wait_for: host={{ nodeItem.public_ip }} port=22 delay=10 timeout=60
+  wait_for: host={{ nodeItem.public_ip }} port=22 delay=10
   loop: "{{ ec2.instances }}"
   loop_control:
     loop_var: nodeItem

.ci/ansible/tasks/setup_test_script.yml

@@ -1,4 +1,18 @@
 ---
+- name: Extend environment for Stack Bootstrapping
+  lineinfile:
+    state: present
+    line: "{{ item }}"
+    insertafter: EOF
+    dest: "/home/{{ ansible_user }}/e2e-testing/.env"
+    create: yes
+  with_items:
+    - "export SUITE=fleet"
+    - "export PROVIDER=docker"
+    - "export DEVELOPER_MODE=true"
+    - "export SKIP_PULL=1"
+  when: "'stack' == nodeLabel"
+
 - name: Extend environment for Fleet testing
   lineinfile:
     state: present
@@ -15,8 +29,10 @@
     - "export KIBANA_URL=http://{{stackRunner}}:5601"
     - "export FLEET_URL=http://{{stackRunner}}:8220"
     - "export SKIP_PULL=1"
-
-  when: "'fleet' == suite"
+  when:
+    - suite is defined
+    - stackRunner is defined
+    - suite == "fleet"
 
 - name: Extend environment for Fleet with elastic-package testing
   lineinfile:
@@ -41,7 +57,9 @@
   with_items:
     - "export SUITE={{ lookup('env', 'SUITE') or 'kubernetes-autodiscover' }}"
     - "export PROVIDER={{ lookup('env', 'PROVIDER') or 'docker' }}"
-  when: "'kubernetes-autodiscover' == suite"
+  when:
+    - suite is defined
+    - suite == "kubernetes-autodiscover"
 
 - name: Extend environment for Helm testing
   lineinfile:
@@ -53,7 +71,9 @@
   with_items:
     - "export SUITE={{ lookup('env', 'SUITE') or 'helm' }}"
     - "export PROVIDER={{ lookup('env', 'PROVIDER') or 'docker' }}"
-  when: "'helm' == suite"
+  when:
+    - suite is defined
+    - suite == "helm"
 
 - name: Extend environment
   lineinfile:
@@ -66,7 +86,7 @@
     - "export STACK_VERSION={{ lookup('file', '{{workspace}}/.stack-version') or '8.0.0-SNAPSHOT' }}"
     - "export BEAT_VERSION={{ lookup('file', '{{workspace}}/.stack-version') or '8.0.0-SNAPSHOT' }}"
     - "export ELASTIC_APM_GLOBAL_LABELS={{ lookup('env', 'ELASTIC_APM_GLOBAL_LABELS') }}"
-    - "export PATH=$PATH:/opt/go/{{golang_version}}/bin"
+    - "export PATH=$PATH:/opt/go/{{golang_version}}/bin:/usr/local/bin"
 
 - name: Create Fleet test script file
   become: no

cli/config/compose/profiles/fleet/default/kibana.config.yml

@@ -5,7 +5,7 @@ server.host: "0.0.0.0"
 telemetry.enabled: false
 
 elasticsearch.hosts: [ "http://elasticsearch:9200" ]
-elasticsearch.username: elastic
+elasticsearch.username: admin
 elasticsearch.password: changeme
 xpack.monitoring.ui.container.elasticsearch.enabled: true
 

cli/config/compose/profiles/fleet/docker-compose.yml

@@ -14,12 +14,18 @@ services:
       - xpack.license.self_generated.type=trial
       - xpack.security.enabled=true
       - xpack.security.authc.api_key.enabled=true
-      - ELASTIC_USERNAME=elastic
+      - xpack.security.authc.token.enabled=true
+      - xpack.security.authc.token.timeout=60m
+      - ELASTIC_USERNAME=admin
       - ELASTIC_PASSWORD=changeme
     image: "docker.elastic.co/elasticsearch/elasticsearch:${stackVersion:-7.16.4-047be5da-SNAPSHOT}"
     platform: ${stackPlatform:-linux/amd64}
     ports:
       - "9200:9200"
+    volumes:
+      - ./elasticsearch-roles.yml:/usr/share/elasticsearch/config/roles.yml
+      - ./elasticsearch-users:/usr/share/elasticsearch/config/users
+      - ./elasticsearch-users_roles:/usr/share/elasticsearch/config/users_roles
   kibana:
     depends_on:
       elasticsearch:
@@ -34,23 +40,3 @@ services:
       - "5601:5601"
     volumes:
       - ./${kibanaProfile:-default}/kibana.config.yml:/usr/share/kibana/config/kibana.yml
-  fleet-server:
-    image: "docker.elastic.co/beats/elastic-agent:${stackVersion:-7.16.4-047be5da-SNAPSHOT}"
-    depends_on:
-      elasticsearch:
-        condition: service_healthy
-      kibana:
-        condition: service_healthy
-    ports:
-      - "8220:8220"
-    healthcheck:
-      test: "curl -f http://127.0.0.1:8220/api/status | grep HEALTHY 2>&1 >/dev/null"
-      retries: 12
-      interval: 5s
-    environment:
-      - "FLEET_SERVER_ENABLE=1"
-      - "FLEET_SERVER_INSECURE_HTTP=1"
-      - "KIBANA_FLEET_SETUP=1"
-      - "KIBANA_FLEET_HOST=http://kibana:5601"
-      - "FLEET_SERVER_HOST=0.0.0.0"
-      - "FLEET_SERVER_PORT=8220"

cli/config/compose/profiles/fleet/elasticsearch-roles.yml

@@ -0,0 +1,34 @@
+---
+apm_server:
+  cluster: ['manage_ilm', 'manage_security', 'manage_api_key']
+  indices:
+    - names: ['apm-*', 'logs-apm*', 'metrics-apm*', 'traces-apm*']
+      privileges: ['write', 'create_index', 'manage', 'manage_ilm']
+  applications:
+    - application: 'apm'
+      privileges: ['sourcemap:write', 'event:write', 'config_agent:read']
+      resources: '*'
+beats:
+  cluster: ['manage_index_templates', 'monitor', 'manage_ingest_pipelines', 'manage_ilm', 'manage_security', 'manage_api_key']
+  indices:
+    - names: ['filebeat-*', 'shrink-filebeat-*']
+      privileges: ['all']
+filebeat:
+  cluster: ['manage_index_templates', 'monitor', 'manage_ingest_pipelines', 'manage_ilm']
+  indices:
+    - names: ['filebeat-*', 'shrink-filebeat-*']
+      privileges: ['all']
+heartbeat:
+  cluster: ['manage_index_templates', 'monitor', 'manage_ingest_pipelines', 'manage_ilm']
+  indices:
+    - names: ['heartbeat-*', 'shrink-heartbeat-*']
+      privileges: ['all']
+metricbeat:
+  cluster: ['manage_index_templates', 'monitor', 'manage_ingest_pipelines', 'manage_ilm']
+  indices:
+    - names: ['metricbeat-*', 'shrink-metricbeat-*']
+      privileges: ['all']
+opbeans:
+  indices:
+    - names: ['opbeans-*']
+      privileges: ['write', 'read']

cli/config/compose/profiles/fleet/elasticsearch-users

@@ -0,0 +1,9 @@
+admin:$2a$10$xiY0ZzOKmDDN1p3if4t4muUBwh2.bFHADoMRAWQgSClm4ZJ4132Y.
+apm_server_user:$2a$10$iTy29qZaCSVn4FXlIjertuO8YfYVLCbvoUAJ3idaXfLRclg9GXdGG
+apm_user_ro:$2a$10$hQfy2o2u33SapUClsx8NCuRMpQyHP9b2l4t3QqrBA.5xXN2S.nT4u
+beats_user:$2a$10$LRpKi4/Q3Qo4oIbiu26rH.FNIL4aOH4aj2Kwi58FkMo1z9FgJONn2
+filebeat_user:$2a$10$sFxIEX8tKyOYgsbJLbUhTup76ssvSD3L4T0H6Raaxg4ewuNr.lUFC
+heartbeat_user:$2a$10$nKUGDr/V5ClfliglJhfy8.oEkjrDtklGQfhd9r9NoFqQeoNxr7uUK
+kibana_system_user:$2a$10$nN6sRtQl2KX9Gn8kV/.NpOLSk6Jwn8TehEDnZ7aaAgzyl/dy5PYzW
+metricbeat_user:$2a$10$5PyTd121U2ZXnFk9NyqxPuLxdptKbB8nK5egt6M5/4xrKUkk.GReG
+opbeans_user:$2a$10$iTy29qZaCSVn4FXlIjertuO8YfYVLCbvoUAJ3idaXfLRclg9GXdGG

cli/config/compose/profiles/fleet/elasticsearch-users_roles

@@ -0,0 +1,13 @@
+apm_server:apm_server_user
+apm_system:apm_server_user
+apm_user:apm_server_user,apm_user_ro
+beats:beats_user
+beats_system:beats_user,filebeat_user,heartbeat_user,metricbeat_user
+filebeat:filebeat_user
+heartbeat:heartbeat_user
+ingest_admin:apm_server_user
+kibana_system:admin,kibana_system_user
+kibana_user:apm_server_user,apm_user_ro,beats_user,filebeat_user,heartbeat_user,metricbeat_user,opbeans_user
+metricbeat:metricbeat_user
+opbeans:opbeans_user
+superuser:admin

cli/config/compose/profiles/fleet/preconfigured-policies/kibana.config.yml

@@ -5,7 +5,7 @@ server.host: "0.0.0.0"
 telemetry.enabled: false
 
 elasticsearch.hosts: [ "http://elasticsearch:9200" ]
-elasticsearch.username: elastic
+elasticsearch.username: admin
 elasticsearch.password: changeme
 xpack.monitoring.ui.container.elasticsearch.enabled: true
 

cli/config/compose/services/elastic-agent/docker-compose.yml

@@ -8,6 +8,8 @@ services:
       kibana:
         condition: service_healthy
     environment:
+      - "ELASTICSEARCH_USERNAME=admin"
+      - "ELASTICSEARCH_PASSWORD=changeme"
       - "FLEET_SERVER_ENABLE=${fleetServerMode:-0}"
       - "FLEET_SERVER_INSECURE_HTTP=${fleetServerMode:-0}"
       - "FLEET_ENROLL=${fleetEnroll:-1}"
@@ -16,4 +18,4 @@ services:
       - "FLEET_URL=${fleetUrl:-}"
     platform: ${stackPlatform:-linux/amd64}
     ports:
-      - "127.0.0.1:${fleetServerPort:-8220}:8220"
+      - "${fleetServerPort:-8220}:8220"

cli/config/compose/services/elastic-agent/fleet-server/docker-compose.yml

@@ -0,0 +1,27 @@
+version: '2.4'
+services:
+  fleet-server:
+    image: "docker.elastic.co/${elasticAgentDockerNamespace:-beats}/elastic-agent${elasticAgentDockerImageSuffix}:${elasticAgentTag:-8.1.0-aa69d697-SNAPSHOT}"
+    depends_on:
+      elasticsearch:
+        condition: service_healthy
+      kibana:
+        condition: service_healthy
+    environment:
+      - "ELASTICSEARCH_USERNAME=admin"
+      - "ELASTICSEARCH_PASSWORD=changeme"
+      - "FLEET_SERVER_ENABLE=${fleetServerMode:-0}"
+      - "FLEET_SERVER_HOST=0.0.0.0"
+      - "FLEET_SERVER_INSECURE_HTTP=${fleetServerMode:-0}"
+      - "FLEET_SERVER_PORT=${fleetServerPort:-8220}"
+      - "FLEET_SERVER_SERVICE_TOKEN=${fleetServerServiceToken:-}"
+      - "FLEET_SERVER_POLICY_ID=${fleetServerPolicyId:-}"
+      - "FLEET_ENROLL=${fleetEnroll:-1}"
+      - "FLEET_ENROLLMENT_TOKEN=${fleetEnrollmentToken:-}"
+      - "FLEET_INSECURE=${fleetInsecure:-0}"
+      - "FLEET_URL=${fleetUrl:-}"
+      - "KIBANA_FLEET_HOST=http://kibana:5601"
+      - "KIBANA_FLEET_SETUP=${fleetServerMode:-0}"
+    platform: ${stackPlatform:-linux/amd64}
+    ports:
+      - "${fleetServerPort:-8220}:8220"

e2e/_suites/fleet/features/apm_integration.feature

@@ -15,7 +15,6 @@ Examples: default
   | default |
 
 @ubi8
-@skip:arm64
 Examples: Ubi8
 | image   |
 | ubi8    |