fix: update stack deployment to follow Users&Roles best practices (#2064

)

* bump stack version 8.1.0-aa69d697

* fix: use new kibana roles and users

We are going to use 'admin' everywhere

* feat: add a method for checking ES cluster health

* fix: create fleet-server after getting a service token from elasticsearch

* fix: start stack using Fleet's test suite code

Instead of calling the compose, we are calling the bootstrapping code for
the Fleet test suite but without any valid tag. Because we are setting
DEVELOPER_MODE=true for the stack node, it will keep the stack even
though the scenarios and tags finished. We also pass a non-existing gherkin
tag, to avoid running any scenario but the bootstrap code, only.

* fix: typo

* fix: selective execution of the .env for fleet suite

* chore: try 'not in'

* chore: use AND conditionals as a list

* fix: check for stckRunner to be defined

* fix: pass stackRunner var to the stack creation

* fix: check for suite is defined first

* fix: check for suite var

* chore: use multiline for when condirtionals

* fix docker install

Signed-off-by: Adam Stokes <[email protected]>

* fix suite definition for autodiscover

Signed-off-by: Adam Stokes <[email protected]>

* add kubectl to path

Signed-off-by: Adam Stokes <[email protected]>

* chore: bump elastic-agent versions to 8.1.hashed snapshot

* fix: use docker provider for APM integration

It will run on Debian AMD/ARM and SLES15

* chore: add client alive SSH settings

* Revert "chore: add client alive SSH settings"

This reverts commit 306551c.

* chore: define SSHD server settings for runners

* chore: skip ubi8 scenarios

We need to adapt them to the dnew deployment model. See #2088

* fix: transform response from bytes to string

* fix: properly read Input Streams and Vars

* fix: expose port for 0.0.0.0

* fix: streams could go empty

* fix: expose port for 0.0.0.0

* fix: support checking for process count in containers

* chore: unskip apm-server on ubi8

* chore: always install docker on runners

* chore: bump elastic-package to v0.36.0

* chore: use elastic-package for apm-server scenarios

* chore: use elastic-package for apm-server scenarios

* Revert "chore: use elastic-package for apm-server scenarios"

This reverts commit b5896a8.

* Revert "chore: use elastic-package for apm-server scenarios"

This reverts commit c6c29ac.

* chore: run dockerised tests only on debian

* chore: do not print out tar extract command

* fix: keep a Docker deployer for docker-based tests

* fix: install docker for ARM

* fix: remove invalid role vars for ARM

* fix: install python-pip on ARM first

* fix: install docker on ARM properly

See https://www.docker.com/blog/getting-started-with-docker-for-arm-on-linux/

* fix: install docker on Suse

* fix: typo

Co-authored-by: Noémi Ványi <[email protected]>

* Fix error checking for revoked enroll token

Signed-off-by: Adam Stokes <[email protected]>

* fix logging

Signed-off-by: Adam Stokes <[email protected]>

* chore: use empty streams if error

* fix: pass fleet-server policy to fleet-server on bootstrap

* fix: retrieve default fleet-server policy instead of creating a new one

* chore: restart services with restart command

Instead of calling stop & start right after it, we are leveraging services
ability to be restarted. For linux, systemctl will use "restart", for MacOS
it will use "stop and start", for Windows, it's not supported yet

* chore: increase expire timeout of the service token to the max (1h)

Co-authored-by: apmmachine <[email protected]>
Co-authored-by: Adam Stokes <[email protected]>
Co-authored-by: Noémi Ványi <[email protected]>

.ci/.e2e-tests-for-elastic-agent.yaml

@@ -40,7 +40,7 @@ SUITES:
         platforms: ["debian_arm64", "debian_amd64", "sles15"]
       - name: "APM Integration"
         tags: "apm_server"
-        platforms: ["fleet_elastic_pkg"]
+        platforms: ["debian_amd64"]
       - name: "Linux Integration"
         tags: "linux_integration"
         platforms: ["debian_arm64", "debian_amd64", "sles15"]

.ci/.e2e-tests.yaml

@@ -51,7 +51,7 @@ SUITES:
         platforms: ["debian_arm64", "debian_amd64", "sles15"]
       - name: "APM Integration"
         tags: "apm_server"
-        platforms: ["fleet_elastic_pkg"]
+        platforms: ["debian_amd64"]
       - name: "Linux Integration"
         tags: "linux_integration"
         platforms: ["debian_arm64", "debian_amd64", "sles15"]

.ci/Jenkinsfile

@@ -215,7 +215,7 @@ pipeline {
                     ansible(
                       stackWorkspace,
                       env.RUN_ID.split('-')[0],
-                      "-i \"${stackRunner.ip},\" -t setup-stack --extra-vars=\"${LABELS_STRING} nodeLabel=stack nodeUser=${stackMachine.username} nodeImage=${stackMachine.image} nodeInstanceType=${stackMachine.instance_type}\""
+                      "-i \"${stackRunner.ip},\" -t setup-stack --extra-vars=\"${LABELS_STRING} stackRunner=${stackRunner.ip} nodeLabel=stack nodeUser=${stackMachine.username} nodeImage=${stackMachine.image} nodeInstanceType=${stackMachine.instance_type}\""
                     )
                   }
 

.ci/ansible/files/sshd_config

@@ -0,0 +1,2 @@
+ClientAliveInterval 60
+ClientAliveCountMax 10

.ci/ansible/playbook.yml

@@ -65,26 +65,32 @@
     tags:
       - setup-stack
 
+  - name: Configure test script
+    include_tasks: tasks/setup_test_script.yml
+    tags:
+      - setup-stack
+
   - name: Add SSH keys to stack
     include_tasks: tasks/install_ssh_keys.yml
     tags:
       - setup-stack
 
   - name: Configure stack files
     ansible.builtin.replace:
-      path: "/home/{{ ansible_user }}/e2e-testing/cli/config/compose/profiles/fleet/default/kibana.config.yml"
+      path: '{{ item.path }}'
       regexp: '{{ item.old }}'
       replace: '{{ item.new }}'
     loop:
-      - { old: 'http://elasticsearch', new: 'http://{{ inventory_hostname }}' }
-      - { old: 'http://fleet-server', new: 'http://{{ inventory_hostname }}' }
-      - { old: 'http://package-registry:8080', new: 'https://epr-staging.elastic.co' }
+      - { old: 'http://elasticsearch', new: 'http://{{ inventory_hostname }}', path: "/home/{{ ansible_user }}/e2e-testing/cli/config/compose/profiles/fleet/default/kibana.config.yml" }
+      - { old: 'http://fleet-server', new: 'http://{{ inventory_hostname }}', path: "/home/{{ ansible_user }}/e2e-testing/cli/config/compose/profiles/fleet/default/kibana.config.yml" }
+      - { old: 'http://kibana', new: 'http://{{ inventory_hostname }}', path: "/home/{{ ansible_user }}/e2e-testing/cli/config/compose/services/elastic-agent/fleet-server/docker-compose.yml" }
+      - { old: 'http://package-registry:8080', new: 'https://epr-staging.elastic.co', path: "/home/{{ ansible_user }}/e2e-testing/cli/config/compose/profiles/fleet/default/kibana.config.yml" }
     tags:
       - setup-stack
 
-  - name: Start stack
+  - name: Start stack without any tag to spin up the stack using Fleet's bootstrapping code
     become: true
-    shell: docker-compose -f /home/{{ansible_user}}/e2e-testing/cli/config/compose/profiles/fleet/docker-compose.yml up -d
+    shell: "/home/{{ ansible_user }}/e2e-testing/.ci/scripts/functional-test.sh 'non-existing-tag'"
     tags:
       - setup-stack
 
@@ -102,19 +108,22 @@
     - role: geerlingguy.docker
       docker_daemon_options:
         default-ulimit: ["nofile=1024000:1024000"]
-      when: "'kubernetes-autodiscover' in suite or 'helm' in suite or 'fleet_amd64' in nodeLabel or 'fleet_elastic_pkg' in nodeLabel"
+      when:
+        - '"arm64" not in nodeLabel'
+        - 'ansible_os_family not in ["Suse"]'
     - role: andrewrothstein.kubectl
-      when: "'kubernetes-autodiscover' in suite or 'helm' in suite"
+      when: suite in ["kubernetes-autodiscover", "helm"]
     - role: andrewrothstein.kind
-      when: "'kubernetes-autodiscover' in suite or 'helm' in suite"
+      when: suite in ["kubernetes-autodiscover", "helm"]
     - role: geerlingguy.helm
-      when: "'kubernetes-autodiscover' in suite or 'helm' in suite"
+      when: suite in ["kubernetes-autodiscover", "helm"]
     - role: gantsign.golang
       vars:
         golang_version: 1.17
   vars:
     ansible_python_interpreter: "auto"
     ansible_user: "{{nodeUser}}"
+    pip_package: "python3-pip"
   tasks:
   - name: Install deps
     include_tasks: tasks/install_deps.yml

.ci/ansible/requirements.yml

@@ -1,4 +1,6 @@
+- src: geerlingguy.pip
 - src: geerlingguy.docker
+- src: geerlingguy.docker_arm
 - src: geerlingguy.helm
 - src: andrewrothstein.kubectl
 - src: andrewrothstein.kind

.ci/ansible/tasks/install_deps.yml

@@ -18,6 +18,7 @@
     name:
       - autoconf
       - bison
+      - docker
       - flex
       - gcc
       - gcc-c++
@@ -51,3 +52,18 @@
 - name: Install ssh-import-id python package to copy public SSH keys from Github accounts
   pip:
     name: ssh-import-id
+
+- name: Set sshd configuration for client alive settings
+  ansible.builtin.copy:
+    src: sshd_config
+    dest: /etc/ssh/sshd_config
+    owner: "{{ ansible_user }}"
+    group: "{{ ansible_user }}"
+    mode: '0600'
+  when: ansible_distribution in ["CentOS", "Debian", "Fedora", "RedHat", "Ubuntu"]
+
+- name: Install Docker for ARM (Debian, Ubuntu)
+  ansible.builtin.shell: curl -fsSL test.docker.com -o get-docker.sh && sh get-docker.sh
+  when:
+    - ansible_distribution in ["Debian", "Ubuntu"]
+    - '"arm64" in nodeLabel'

.ci/ansible/tasks/runners.yml

@@ -52,7 +52,7 @@
     - start-node
 
 - name: Wait for SSH to come up
-  wait_for: host={{ nodeItem.public_ip }} port=22 delay=10 timeout=60
+  wait_for: host={{ nodeItem.public_ip }} port=22 delay=10
   loop: "{{ ec2.instances }}"
   loop_control:
     loop_var: nodeItem

.ci/ansible/tasks/setup_test_script.yml

@@ -1,4 +1,18 @@
 ---
+- name: Extend environment for Stack Bootstrapping
+  lineinfile:
+    state: present
+    line: "{{ item }}"
+    insertafter: EOF
+    dest: "/home/{{ ansible_user }}/e2e-testing/.env"
+    create: yes
+  with_items:
+    - "export SUITE=fleet"
+    - "export PROVIDER=docker"
+    - "export DEVELOPER_MODE=true"
+    - "export SKIP_PULL=1"
+  when: "'stack' == nodeLabel"
+
 - name: Extend environment for Fleet testing
   lineinfile:
     state: present
@@ -15,8 +29,10 @@
     - "export KIBANA_URL=http://{{stackRunner}}:5601"
     - "export FLEET_URL=http://{{stackRunner}}:8220"
     - "export SKIP_PULL=1"
-
-  when: "'fleet' == suite"
+  when:
+    - suite is defined
+    - stackRunner is defined
+    - suite == "fleet"
 
 - name: Extend environment for Fleet with elastic-package testing
   lineinfile:
@@ -41,7 +57,9 @@
   with_items:
     - "export SUITE={{ lookup('env', 'SUITE') or 'kubernetes-autodiscover' }}"
     - "export PROVIDER={{ lookup('env', 'PROVIDER') or 'docker' }}"
-  when: "'kubernetes-autodiscover' == suite"
+  when:
+    - suite is defined
+    - suite == "kubernetes-autodiscover"
 
 - name: Extend environment for Helm testing
   lineinfile:
@@ -53,7 +71,9 @@
   with_items:
     - "export SUITE={{ lookup('env', 'SUITE') or 'helm' }}"
     - "export PROVIDER={{ lookup('env', 'PROVIDER') or 'docker' }}"
-  when: "'helm' == suite"
+  when:
+    - suite is defined
+    - suite == "helm"
 
 - name: Extend environment
   lineinfile:
@@ -66,7 +86,7 @@
     - "export STACK_VERSION={{ lookup('file', '{{workspace}}/.stack-version') or '8.0.0-SNAPSHOT' }}"
     - "export BEAT_VERSION={{ lookup('file', '{{workspace}}/.stack-version') or '8.0.0-SNAPSHOT' }}"
     - "export ELASTIC_APM_GLOBAL_LABELS={{ lookup('env', 'ELASTIC_APM_GLOBAL_LABELS') }}"
-    - "export PATH=$PATH:/opt/go/{{golang_version}}/bin"
+    - "export PATH=$PATH:/opt/go/{{golang_version}}/bin:/usr/local/bin"
 
 - name: Create Fleet test script file
   become: no

cli/config/compose/profiles/fleet/default/kibana.config.yml

@@ -5,7 +5,7 @@ server.host: "0.0.0.0"
 telemetry.enabled: false
 
 elasticsearch.hosts: [ "http://elasticsearch:9200" ]
-elasticsearch.username: elastic
+elasticsearch.username: admin
 elasticsearch.password: changeme
 xpack.monitoring.ui.container.elasticsearch.enabled: true
 

cli/config/compose/profiles/fleet/docker-compose.yml

@@ -14,12 +14,18 @@ services:
       - xpack.license.self_generated.type=trial
       - xpack.security.enabled=true
       - xpack.security.authc.api_key.enabled=true
-      - ELASTIC_USERNAME=elastic
+      - xpack.security.authc.token.enabled=true
+      - xpack.security.authc.token.timeout=60m
+      - ELASTIC_USERNAME=admin
       - ELASTIC_PASSWORD=changeme
     image: "docker.elastic.co/elasticsearch/elasticsearch:${stackVersion:-8.1.0-aa69d697-SNAPSHOT}"
     platform: ${stackPlatform:-linux/amd64}
     ports:
       - "9200:9200"
+    volumes:
+      - ./elasticsearch-roles.yml:/usr/share/elasticsearch/config/roles.yml
+      - ./elasticsearch-users:/usr/share/elasticsearch/config/users
+      - ./elasticsearch-users_roles:/usr/share/elasticsearch/config/users_roles
   kibana:
     depends_on:
       elasticsearch:
@@ -34,23 +40,3 @@ services:
       - "5601:5601"
     volumes:
       - ./${kibanaProfile:-default}/kibana.config.yml:/usr/share/kibana/config/kibana.yml
-  fleet-server:
-    image: "docker.elastic.co/beats/elastic-agent-complete:${stackVersion:-8.1.0-aa69d697-SNAPSHOT}"
-    depends_on:
-      elasticsearch:
-        condition: service_healthy
-      kibana:
-        condition: service_healthy
-    ports:
-      - "8220:8220"
-    healthcheck:
-      test: "curl -f http://127.0.0.1:8220/api/status | grep HEALTHY 2>&1 >/dev/null"
-      retries: 12
-      interval: 5s
-    environment:
-      - "FLEET_SERVER_ENABLE=1"
-      - "FLEET_SERVER_INSECURE_HTTP=1"
-      - "KIBANA_FLEET_SETUP=1"
-      - "KIBANA_FLEET_HOST=http://kibana:5601"
-      - "FLEET_SERVER_HOST=0.0.0.0"
-      - "FLEET_SERVER_PORT=8220"

cli/config/compose/profiles/fleet/elasticsearch-roles.yml

@@ -0,0 +1,34 @@
+---
+apm_server:
+  cluster: ['manage_ilm', 'manage_security', 'manage_api_key']
+  indices:
+    - names: ['apm-*', 'logs-apm*', 'metrics-apm*', 'traces-apm*']
+      privileges: ['write', 'create_index', 'manage', 'manage_ilm']
+  applications:
+    - application: 'apm'
+      privileges: ['sourcemap:write', 'event:write', 'config_agent:read']
+      resources: '*'
+beats:
+  cluster: ['manage_index_templates', 'monitor', 'manage_ingest_pipelines', 'manage_ilm', 'manage_security', 'manage_api_key']
+  indices:
+    - names: ['filebeat-*', 'shrink-filebeat-*']
+      privileges: ['all']
+filebeat:
+  cluster: ['manage_index_templates', 'monitor', 'manage_ingest_pipelines', 'manage_ilm']
+  indices:
+    - names: ['filebeat-*', 'shrink-filebeat-*']
+      privileges: ['all']
+heartbeat:
+  cluster: ['manage_index_templates', 'monitor', 'manage_ingest_pipelines', 'manage_ilm']
+  indices:
+    - names: ['heartbeat-*', 'shrink-heartbeat-*']
+      privileges: ['all']
+metricbeat:
+  cluster: ['manage_index_templates', 'monitor', 'manage_ingest_pipelines', 'manage_ilm']
+  indices:
+    - names: ['metricbeat-*', 'shrink-metricbeat-*']
+      privileges: ['all']
+opbeans:
+  indices:
+    - names: ['opbeans-*']
+      privileges: ['write', 'read']

cli/config/compose/profiles/fleet/elasticsearch-users

@@ -0,0 +1,9 @@
+admin:$2a$10$xiY0ZzOKmDDN1p3if4t4muUBwh2.bFHADoMRAWQgSClm4ZJ4132Y.
+apm_server_user:$2a$10$iTy29qZaCSVn4FXlIjertuO8YfYVLCbvoUAJ3idaXfLRclg9GXdGG
+apm_user_ro:$2a$10$hQfy2o2u33SapUClsx8NCuRMpQyHP9b2l4t3QqrBA.5xXN2S.nT4u
+beats_user:$2a$10$LRpKi4/Q3Qo4oIbiu26rH.FNIL4aOH4aj2Kwi58FkMo1z9FgJONn2
+filebeat_user:$2a$10$sFxIEX8tKyOYgsbJLbUhTup76ssvSD3L4T0H6Raaxg4ewuNr.lUFC
+heartbeat_user:$2a$10$nKUGDr/V5ClfliglJhfy8.oEkjrDtklGQfhd9r9NoFqQeoNxr7uUK
+kibana_system_user:$2a$10$nN6sRtQl2KX9Gn8kV/.NpOLSk6Jwn8TehEDnZ7aaAgzyl/dy5PYzW
+metricbeat_user:$2a$10$5PyTd121U2ZXnFk9NyqxPuLxdptKbB8nK5egt6M5/4xrKUkk.GReG
+opbeans_user:$2a$10$iTy29qZaCSVn4FXlIjertuO8YfYVLCbvoUAJ3idaXfLRclg9GXdGG

cli/config/compose/profiles/fleet/elasticsearch-users_roles

@@ -0,0 +1,13 @@
+apm_server:apm_server_user
+apm_system:apm_server_user
+apm_user:apm_server_user,apm_user_ro
+beats:beats_user
+beats_system:beats_user,filebeat_user,heartbeat_user,metricbeat_user
+filebeat:filebeat_user
+heartbeat:heartbeat_user
+ingest_admin:apm_server_user
+kibana_system:admin,kibana_system_user
+kibana_user:apm_server_user,apm_user_ro,beats_user,filebeat_user,heartbeat_user,metricbeat_user,opbeans_user
+metricbeat:metricbeat_user
+opbeans:opbeans_user
+superuser:admin

cli/config/compose/profiles/fleet/preconfigured-policies/kibana.config.yml

@@ -5,7 +5,7 @@ server.host: "0.0.0.0"
 telemetry.enabled: false
 
 elasticsearch.hosts: [ "http://elasticsearch:9200" ]
-elasticsearch.username: elastic
+elasticsearch.username: admin
 elasticsearch.password: changeme
 xpack.monitoring.ui.container.elasticsearch.enabled: true
 

cli/config/compose/services/elastic-agent/docker-compose.yml

@@ -1,13 +1,15 @@
 version: '2.4'
 services:
   elastic-agent:
-    image: "docker.elastic.co/${elasticAgentDockerNamespace:-beats}/elastic-agent${elasticAgentDockerImageSuffix}:${elasticAgentTag:-8.0.0-SNAPSHOT}"
+    image: "docker.elastic.co/${elasticAgentDockerNamespace:-beats}/elastic-agent${elasticAgentDockerImageSuffix}:${elasticAgentTag:-8.1.0-aa69d697-SNAPSHOT}"
     depends_on:
       elasticsearch:
         condition: service_healthy
       kibana:
         condition: service_healthy
     environment:
+      - "ELASTICSEARCH_USERNAME=admin"
+      - "ELASTICSEARCH_PASSWORD=changeme"
       - "FLEET_SERVER_ENABLE=${fleetServerMode:-0}"
       - "FLEET_SERVER_INSECURE_HTTP=${fleetServerMode:-0}"
       - "FLEET_ENROLL=${fleetEnroll:-1}"
@@ -16,4 +18,4 @@ services:
       - "FLEET_URL=${fleetUrl:-}"
     platform: ${stackPlatform:-linux/amd64}
     ports:
-      - "127.0.0.1:${fleetServerPort:-8220}:8220"
+      - "${fleetServerPort:-8220}:8220"

cli/config/compose/services/elastic-agent/fleet-server/docker-compose.yml

@@ -0,0 +1,27 @@
+version: '2.4'
+services:
+  fleet-server:
+    image: "docker.elastic.co/${elasticAgentDockerNamespace:-beats}/elastic-agent${elasticAgentDockerImageSuffix}:${elasticAgentTag:-8.1.0-aa69d697-SNAPSHOT}"
+    depends_on:
+      elasticsearch:
+        condition: service_healthy
+      kibana:
+        condition: service_healthy
+    environment:
+      - "ELASTICSEARCH_USERNAME=admin"
+      - "ELASTICSEARCH_PASSWORD=changeme"
+      - "FLEET_SERVER_ENABLE=${fleetServerMode:-0}"
+      - "FLEET_SERVER_HOST=0.0.0.0"
+      - "FLEET_SERVER_INSECURE_HTTP=${fleetServerMode:-0}"
+      - "FLEET_SERVER_PORT=${fleetServerPort:-8220}"
+      - "FLEET_SERVER_SERVICE_TOKEN=${fleetServerServiceToken:-}"
+      - "FLEET_SERVER_POLICY_ID=${fleetServerPolicyId:-}"
+      - "FLEET_ENROLL=${fleetEnroll:-1}"
+      - "FLEET_ENROLLMENT_TOKEN=${fleetEnrollmentToken:-}"
+      - "FLEET_INSECURE=${fleetInsecure:-0}"
+      - "FLEET_URL=${fleetUrl:-}"
+      - "KIBANA_FLEET_HOST=http://kibana:5601"
+      - "KIBANA_FLEET_SETUP=${fleetServerMode:-0}"
+    platform: ${stackPlatform:-linux/amd64}
+    ports:
+      - "${fleetServerPort:-8220}:8220"

e2e/_suites/fleet/features/apm_integration.feature

@@ -15,7 +15,6 @@ Examples: default
   | default |
 
 @ubi8
-@skip:arm64
 Examples: Ubi8
 | image   |
 | ubi8    |