[7.16](backport #2064) fix: update stack deployment to follow Users&R…

…oles best practices (#2103) * fix: update stack deployment to follow Users&Roles best practices (#2064) * bump stack version 8.1.0-aa69d697 * fix: use new kibana roles and users We are going to use 'admin' everywhere * feat: add a method for checking ES cluster health * fix: create fleet-server after getting a service token from elasticsearch * fix: start stack using Fleet's test suite code Instead of calling the compose, we are calling the bootstrapping code for the Fleet test suite but without any valid tag. Because we are setting DEVELOPER_MODE=true for the stack node, it will keep the stack even though the scenarios and tags finished. We also pass a non-existing gherkin tag, to avoid running any scenario but the bootstrap code, only. * fix: typo * fix: selective execution of the .env for fleet suite * chore: try 'not in' * chore: use AND conditionals as a list * fix: check for stckRunner to be defined * fix: pass stackRunner var to the stack creation * fix: check for suite is defined first * fix: check for suite var * chore: use multiline for when condirtionals * fix docker install Signed-off-by: Adam Stokes <[email protected]> * fix suite definition for autodiscover Signed-off-by: Adam Stokes <[email protected]> * add kubectl to path Signed-off-by: Adam Stokes <[email protected]> * chore: bump elastic-agent versions to 8.1.hashed snapshot * fix: use docker provider for APM integration It will run on Debian AMD/ARM and SLES15 * chore: add client alive SSH settings * Revert "chore: add client alive SSH settings" This reverts commit 306551c. * chore: define SSHD server settings for runners * chore: skip ubi8 scenarios We need to adapt them to the dnew deployment model. See #2088 * fix: transform response from bytes to string * fix: properly read Input Streams and Vars * fix: expose port for 0.0.0.0 * fix: streams could go empty * fix: expose port for 0.0.0.0 * fix: support checking for process count in containers * chore: unskip apm-server on ubi8 * chore: always install docker on runners * chore: bump elastic-package to v0.36.0 * chore: use elastic-package for apm-server scenarios * chore: use elastic-package for apm-server scenarios * Revert "chore: use elastic-package for apm-server scenarios" This reverts commit b5896a8. * Revert "chore: use elastic-package for apm-server scenarios" This reverts commit c6c29ac. * chore: run dockerised tests only on debian * chore: do not print out tar extract command * fix: keep a Docker deployer for docker-based tests * fix: install docker for ARM * fix: remove invalid role vars for ARM * fix: install python-pip on ARM first * fix: install docker on ARM properly See https://www.docker.com/blog/getting-started-with-docker-for-arm-on-linux/ * fix: install docker on Suse * fix: typo Co-authored-by: Noémi Ványi <[email protected]> * Fix error checking for revoked enroll token Signed-off-by: Adam Stokes <[email protected]> * fix logging Signed-off-by: Adam Stokes <[email protected]> * chore: use empty streams if error * fix: pass fleet-server policy to fleet-server on bootstrap * fix: retrieve default fleet-server policy instead of creating a new one * chore: restart services with restart command Instead of calling stop & start right after it, we are leveraging services ability to be restarted. For linux, systemctl will use "restart", for MacOS it will use "stop and start", for Windows, it's not supported yet * chore: increase expire timeout of the service token to the max (1h) Co-authored-by: apmmachine <[email protected]> Co-authored-by: Adam Stokes <[email protected]> Co-authored-by: Noémi Ványi <[email protected]> (cherry picked from commit a31f807) # Conflicts: # cli/config/compose/profiles/fleet/docker-compose.yml # cli/config/compose/services/elastic-agent/docker-compose.yml # go.mod # go.sum # internal/deploy/base_test.go * fix: resolve conflicts * chore: run go mod tidy * fix: resolve more conflicts Co-authored-by: Manuel de la Peña <[email protected]>
elastic · Feb 8, 2022 · 5bc38e2 · 5bc38e2
1 parent a3e3f45
commit 5bc38e2
Show file tree

Hide file tree

Showing 40 changed files with 1,228 additions and 615 deletions.
diff --git a/.ci/.e2e-tests-for-elastic-agent.yaml b/.ci/.e2e-tests-for-elastic-agent.yaml
@@ -36,7 +36,7 @@ SUITES:
         platforms: ["debian_arm64", "debian_amd64"]
       - name: "APM Integration"
         tags: "apm_server"
-        platforms: ["fleet_elastic_pkg"]
+        platforms: ["debian_amd64"]
       - name: "Linux Integration"
         tags: "linux_integration"
         platforms: ["debian_arm64", "debian_amd64"]

diff --git a/.ci/.e2e-tests.yaml b/.ci/.e2e-tests.yaml
@@ -47,7 +47,7 @@ SUITES:
         platforms: ["debian_arm64", "debian_amd64"]
       - name: "APM Integration"
         tags: "apm_server"
-        platforms: ["fleet_elastic_pkg"]
+        platforms: ["debian_amd64"]
       - name: "Linux Integration"
         tags: "linux_integration"
         platforms: ["debian_arm64", "debian_amd64"]

diff --git a/.ci/Jenkinsfile b/.ci/Jenkinsfile
@@ -215,7 +215,7 @@ pipeline {
                     ansible(
                       stackWorkspace,
                       env.RUN_ID.split('-')[0],
-                      "-i \"${stackRunner.ip},\" -t setup-stack --extra-vars=\"${LABELS_STRING} nodeLabel=stack nodeUser=${stackMachine.username} nodeImage=${stackMachine.image} nodeInstanceType=${stackMachine.instance_type}\""
+                      "-i \"${stackRunner.ip},\" -t setup-stack --extra-vars=\"${LABELS_STRING} stackRunner=${stackRunner.ip} nodeLabel=stack nodeUser=${stackMachine.username} nodeImage=${stackMachine.image} nodeInstanceType=${stackMachine.instance_type}\""
                     )
                   }
 

diff --git a/.ci/ansible/files/sshd_config b/.ci/ansible/files/sshd_config
@@ -0,0 +1,2 @@
+ClientAliveInterval 60
+ClientAliveCountMax 10
diff --git a/.ci/ansible/playbook.yml b/.ci/ansible/playbook.yml
@@ -65,6 +65,11 @@
     tags:
       - setup-stack
 
+  - name: Configure test script
+    include_tasks: tasks/setup_test_script.yml
+    tags:
+      - setup-stack
+
   - name: Add SSH keys to stack
     include_tasks: tasks/install_ssh_keys.yml
     tags:
@@ -77,19 +82,20 @@
 
   - name: Configure stack files
     ansible.builtin.replace:
-      path: "/home/{{ ansible_user }}/e2e-testing/cli/config/compose/profiles/fleet/default/kibana.config.yml"
+      path: '{{ item.path }}'
       regexp: '{{ item.old }}'
       replace: '{{ item.new }}'
     loop:
-      - { old: 'http://elasticsearch', new: 'http://{{ inventory_hostname }}' }
-      - { old: 'http://fleet-server', new: 'http://{{ inventory_hostname }}' }
-      - { old: 'http://package-registry:8080', new: 'https://epr-staging.elastic.co' }
+      - { old: 'http://elasticsearch', new: 'http://{{ inventory_hostname }}', path: "/home/{{ ansible_user }}/e2e-testing/cli/config/compose/profiles/fleet/default/kibana.config.yml" }
+      - { old: 'http://fleet-server', new: 'http://{{ inventory_hostname }}', path: "/home/{{ ansible_user }}/e2e-testing/cli/config/compose/profiles/fleet/default/kibana.config.yml" }
+      - { old: 'http://kibana', new: 'http://{{ inventory_hostname }}', path: "/home/{{ ansible_user }}/e2e-testing/cli/config/compose/services/elastic-agent/fleet-server/docker-compose.yml" }
+      - { old: 'http://package-registry:8080', new: 'https://epr-staging.elastic.co', path: "/home/{{ ansible_user }}/e2e-testing/cli/config/compose/profiles/fleet/default/kibana.config.yml" }
     tags:
       - setup-stack
 
-  - name: Start stack
+  - name: Start stack without any tag to spin up the stack using Fleet's bootstrapping code
     become: true
-    shell: docker-compose -f /home/{{ansible_user}}/e2e-testing/cli/config/compose/profiles/fleet/docker-compose.yml up -d
+    shell: "/home/{{ ansible_user }}/e2e-testing/.ci/scripts/functional-test.sh 'non-existing-tag'"
     tags:
       - setup-stack
 
@@ -107,19 +113,22 @@
     - role: geerlingguy.docker
       docker_daemon_options:
         default-ulimit: ["nofile=1024000:1024000"]
-      when: "'kubernetes-autodiscover' in suite or 'helm' in suite or 'fleet_amd64' in nodeLabel or 'fleet_elastic_pkg' in nodeLabel"
+      when:
+        - '"arm64" not in nodeLabel'
+        - 'ansible_os_family not in ["Suse"]'
     - role: andrewrothstein.kubectl
-      when: "'kubernetes-autodiscover' in suite or 'helm' in suite"
+      when: suite in ["kubernetes-autodiscover", "helm"]
     - role: andrewrothstein.kind
-      when: "'kubernetes-autodiscover' in suite or 'helm' in suite"
+      when: suite in ["kubernetes-autodiscover", "helm"]
     - role: geerlingguy.helm
-      when: "'kubernetes-autodiscover' in suite or 'helm' in suite"
+      when: suite in ["kubernetes-autodiscover", "helm"]
     - role: gantsign.golang
       vars:
         golang_version: 1.16.3
   vars:
     ansible_python_interpreter: "auto"
     ansible_user: "{{nodeUser}}"
+    pip_package: "python3-pip"
   tasks:
   - name: Install deps
     include_tasks: tasks/install_deps.yml

diff --git a/.ci/ansible/requirements.yml b/.ci/ansible/requirements.yml
@@ -1,4 +1,6 @@
+- src: geerlingguy.pip
 - src: geerlingguy.docker
+- src: geerlingguy.docker_arm
 - src: geerlingguy.helm
 - src: andrewrothstein.kubectl
 - src: andrewrothstein.kind

diff --git a/.ci/ansible/tasks/install_deps.yml b/.ci/ansible/tasks/install_deps.yml
@@ -22,6 +22,7 @@
     name:
       - autoconf
       - bison
+      - docker
       - flex
       - gcc
       - gcc-c++
@@ -55,3 +56,18 @@
 - name: Install ssh-import-id python package to copy public SSH keys from Github accounts
   pip:
     name: ssh-import-id
+
+- name: Set sshd configuration for client alive settings
+  ansible.builtin.copy:
+    src: sshd_config
+    dest: /etc/ssh/sshd_config
+    owner: "{{ ansible_user }}"
+    group: "{{ ansible_user }}"
+    mode: '0600'
+  when: ansible_distribution in ["CentOS", "Debian", "Fedora", "RedHat", "Ubuntu"]
+
+- name: Install Docker for ARM (Debian, Ubuntu)
+  ansible.builtin.shell: curl -fsSL test.docker.com -o get-docker.sh && sh get-docker.sh
+  when:
+    - ansible_distribution in ["Debian", "Ubuntu"]
+    - '"arm64" in nodeLabel'
diff --git a/.ci/ansible/tasks/runners.yml b/.ci/ansible/tasks/runners.yml
@@ -52,7 +52,7 @@
     - start-node
 
 - name: Wait for SSH to come up
-  wait_for: host={{ nodeItem.public_ip }} port=22 delay=10 timeout=60
+  wait_for: host={{ nodeItem.public_ip }} port=22 delay=10
   loop: "{{ ec2.instances }}"
   loop_control:
     loop_var: nodeItem

diff --git a/.ci/ansible/tasks/setup_test_script.yml b/.ci/ansible/tasks/setup_test_script.yml
@@ -1,4 +1,18 @@
 ---
+- name: Extend environment for Stack Bootstrapping
+  lineinfile:
+    state: present
+    line: "{{ item }}"
+    insertafter: EOF
+    dest: "/home/{{ ansible_user }}/e2e-testing/.env"
+    create: yes
+  with_items:
+    - "export SUITE=fleet"
+    - "export PROVIDER=docker"
+    - "export DEVELOPER_MODE=true"
+    - "export SKIP_PULL=1"
+  when: "'stack' == nodeLabel"
+
 - name: Extend environment for Fleet testing
   lineinfile:
     state: present
@@ -15,8 +29,10 @@
     - "export KIBANA_URL=http://{{stackRunner}}:5601"
     - "export FLEET_URL=http://{{stackRunner}}:8220"
     - "export SKIP_PULL=1"
-
-  when: "'fleet' == suite"
+  when:
+    - suite is defined
+    - stackRunner is defined
+    - suite == "fleet"
 
 - name: Extend environment for Fleet with elastic-package testing
   lineinfile:
@@ -41,7 +57,9 @@
   with_items:
     - "export SUITE={{ lookup('env', 'SUITE') or 'kubernetes-autodiscover' }}"
     - "export PROVIDER={{ lookup('env', 'PROVIDER') or 'docker' }}"
-  when: "'kubernetes-autodiscover' == suite"
+  when:
+    - suite is defined
+    - suite == "kubernetes-autodiscover"
 
 - name: Extend environment for Helm testing
   lineinfile:
@@ -53,7 +71,9 @@
   with_items:
     - "export SUITE={{ lookup('env', 'SUITE') or 'helm' }}"
     - "export PROVIDER={{ lookup('env', 'PROVIDER') or 'docker' }}"
-  when: "'helm' == suite"
+  when:
+    - suite is defined
+    - suite == "helm"
 
 - name: Extend environment
   lineinfile:
@@ -66,7 +86,7 @@
     - "export STACK_VERSION={{ lookup('file', '{{workspace}}/.stack-version') or '8.0.0-SNAPSHOT' }}"
     - "export BEAT_VERSION={{ lookup('file', '{{workspace}}/.stack-version') or '8.0.0-SNAPSHOT' }}"
     - "export ELASTIC_APM_GLOBAL_LABELS={{ lookup('env', 'ELASTIC_APM_GLOBAL_LABELS') }}"
-    - "export PATH=$PATH:/opt/go/{{golang_version}}/bin"
+    - "export PATH=$PATH:/opt/go/{{golang_version}}/bin:/usr/local/bin"
 
 - name: Create Fleet test script file
   become: no

diff --git a/cli/config/compose/profiles/fleet/default/kibana.config.yml b/cli/config/compose/profiles/fleet/default/kibana.config.yml
@@ -5,7 +5,7 @@ server.host: "0.0.0.0"
 telemetry.enabled: false
 
 elasticsearch.hosts: [ "http://elasticsearch:9200" ]
-elasticsearch.username: elastic
+elasticsearch.username: admin
 elasticsearch.password: changeme
 xpack.monitoring.ui.container.elasticsearch.enabled: true
 

diff --git a/cli/config/compose/profiles/fleet/docker-compose.yml b/cli/config/compose/profiles/fleet/docker-compose.yml
@@ -14,12 +14,18 @@ services:
       - xpack.license.self_generated.type=trial
       - xpack.security.enabled=true
       - xpack.security.authc.api_key.enabled=true
-      - ELASTIC_USERNAME=elastic
+      - xpack.security.authc.token.enabled=true
+      - xpack.security.authc.token.timeout=60m
+      - ELASTIC_USERNAME=admin
       - ELASTIC_PASSWORD=changeme
     image: "docker.elastic.co/elasticsearch/elasticsearch:${stackVersion:-7.16.4-047be5da-SNAPSHOT}"
     platform: ${stackPlatform:-linux/amd64}
     ports:
       - "9200:9200"
+    volumes:
+      - ./elasticsearch-roles.yml:/usr/share/elasticsearch/config/roles.yml
+      - ./elasticsearch-users:/usr/share/elasticsearch/config/users
+      - ./elasticsearch-users_roles:/usr/share/elasticsearch/config/users_roles
   kibana:
     depends_on:
       elasticsearch:
@@ -34,23 +40,3 @@ services:
       - "5601:5601"
     volumes:
       - ./${kibanaProfile:-default}/kibana.config.yml:/usr/share/kibana/config/kibana.yml
-  fleet-server:
-    image: "docker.elastic.co/beats/elastic-agent:${stackVersion:-7.16.4-047be5da-SNAPSHOT}"
-    depends_on:
-      elasticsearch:
-        condition: service_healthy
-      kibana:
-        condition: service_healthy
-    ports:
-      - "8220:8220"
-    healthcheck:
-      test: "curl -f http://127.0.0.1:8220/api/status | grep HEALTHY 2>&1 >/dev/null"
-      retries: 12
-      interval: 5s
-    environment:
-      - "FLEET_SERVER_ENABLE=1"
-      - "FLEET_SERVER_INSECURE_HTTP=1"
-      - "KIBANA_FLEET_SETUP=1"
-      - "KIBANA_FLEET_HOST=http://kibana:5601"
-      - "FLEET_SERVER_HOST=0.0.0.0"
-      - "FLEET_SERVER_PORT=8220"
diff --git a/cli/config/compose/profiles/fleet/elasticsearch-roles.yml b/cli/config/compose/profiles/fleet/elasticsearch-roles.yml
@@ -0,0 +1,34 @@
+---
+apm_server:
+  cluster: ['manage_ilm', 'manage_security', 'manage_api_key']
+  indices:
+    - names: ['apm-*', 'logs-apm*', 'metrics-apm*', 'traces-apm*']
+      privileges: ['write', 'create_index', 'manage', 'manage_ilm']
+  applications:
+    - application: 'apm'
+      privileges: ['sourcemap:write', 'event:write', 'config_agent:read']
+      resources: '*'
+beats:
+  cluster: ['manage_index_templates', 'monitor', 'manage_ingest_pipelines', 'manage_ilm', 'manage_security', 'manage_api_key']
+  indices:
+    - names: ['filebeat-*', 'shrink-filebeat-*']
+      privileges: ['all']
+filebeat:
+  cluster: ['manage_index_templates', 'monitor', 'manage_ingest_pipelines', 'manage_ilm']
+  indices:
+    - names: ['filebeat-*', 'shrink-filebeat-*']
+      privileges: ['all']
+heartbeat:
+  cluster: ['manage_index_templates', 'monitor', 'manage_ingest_pipelines', 'manage_ilm']
+  indices:
+    - names: ['heartbeat-*', 'shrink-heartbeat-*']
+      privileges: ['all']
+metricbeat:
+  cluster: ['manage_index_templates', 'monitor', 'manage_ingest_pipelines', 'manage_ilm']
+  indices:
+    - names: ['metricbeat-*', 'shrink-metricbeat-*']
+      privileges: ['all']
+opbeans:
+  indices:
+    - names: ['opbeans-*']
+      privileges: ['write', 'read']
diff --git a/cli/config/compose/profiles/fleet/elasticsearch-users b/cli/config/compose/profiles/fleet/elasticsearch-users
@@ -0,0 +1,9 @@
+admin:$2a$10$xiY0ZzOKmDDN1p3if4t4muUBwh2.bFHADoMRAWQgSClm4ZJ4132Y.
+apm_server_user:$2a$10$iTy29qZaCSVn4FXlIjertuO8YfYVLCbvoUAJ3idaXfLRclg9GXdGG
+apm_user_ro:$2a$10$hQfy2o2u33SapUClsx8NCuRMpQyHP9b2l4t3QqrBA.5xXN2S.nT4u
+beats_user:$2a$10$LRpKi4/Q3Qo4oIbiu26rH.FNIL4aOH4aj2Kwi58FkMo1z9FgJONn2
+filebeat_user:$2a$10$sFxIEX8tKyOYgsbJLbUhTup76ssvSD3L4T0H6Raaxg4ewuNr.lUFC
+heartbeat_user:$2a$10$nKUGDr/V5ClfliglJhfy8.oEkjrDtklGQfhd9r9NoFqQeoNxr7uUK
+kibana_system_user:$2a$10$nN6sRtQl2KX9Gn8kV/.NpOLSk6Jwn8TehEDnZ7aaAgzyl/dy5PYzW
+metricbeat_user:$2a$10$5PyTd121U2ZXnFk9NyqxPuLxdptKbB8nK5egt6M5/4xrKUkk.GReG
+opbeans_user:$2a$10$iTy29qZaCSVn4FXlIjertuO8YfYVLCbvoUAJ3idaXfLRclg9GXdGG
diff --git a/cli/config/compose/profiles/fleet/elasticsearch-users_roles b/cli/config/compose/profiles/fleet/elasticsearch-users_roles
@@ -0,0 +1,13 @@
+apm_server:apm_server_user
+apm_system:apm_server_user
+apm_user:apm_server_user,apm_user_ro
+beats:beats_user
+beats_system:beats_user,filebeat_user,heartbeat_user,metricbeat_user
+filebeat:filebeat_user
+heartbeat:heartbeat_user
+ingest_admin:apm_server_user
+kibana_system:admin,kibana_system_user
+kibana_user:apm_server_user,apm_user_ro,beats_user,filebeat_user,heartbeat_user,metricbeat_user,opbeans_user
+metricbeat:metricbeat_user
+opbeans:opbeans_user
+superuser:admin
diff --git a/cli/config/compose/profiles/fleet/preconfigured-policies/kibana.config.yml b/cli/config/compose/profiles/fleet/preconfigured-policies/kibana.config.yml
@@ -5,7 +5,7 @@ server.host: "0.0.0.0"
 telemetry.enabled: false
 
 elasticsearch.hosts: [ "http://elasticsearch:9200" ]
-elasticsearch.username: elastic
+elasticsearch.username: admin
 elasticsearch.password: changeme
 xpack.monitoring.ui.container.elasticsearch.enabled: true
 

diff --git a/cli/config/compose/services/elastic-agent/docker-compose.yml b/cli/config/compose/services/elastic-agent/docker-compose.yml
@@ -8,6 +8,8 @@ services:
       kibana:
         condition: service_healthy
     environment:
+      - "ELASTICSEARCH_USERNAME=admin"
+      - "ELASTICSEARCH_PASSWORD=changeme"
       - "FLEET_SERVER_ENABLE=${fleetServerMode:-0}"
       - "FLEET_SERVER_INSECURE_HTTP=${fleetServerMode:-0}"
       - "FLEET_ENROLL=${fleetEnroll:-1}"
@@ -16,4 +18,4 @@ services:
       - "FLEET_URL=${fleetUrl:-}"
     platform: ${stackPlatform:-linux/amd64}
     ports:
-      - "127.0.0.1:${fleetServerPort:-8220}:8220"
+      - "${fleetServerPort:-8220}:8220"
diff --git a/cli/config/compose/services/elastic-agent/fleet-server/docker-compose.yml b/cli/config/compose/services/elastic-agent/fleet-server/docker-compose.yml
@@ -0,0 +1,27 @@
+version: '2.4'
+services:
+  fleet-server:
+    image: "docker.elastic.co/${elasticAgentDockerNamespace:-beats}/elastic-agent${elasticAgentDockerImageSuffix}:${elasticAgentTag:-8.1.0-aa69d697-SNAPSHOT}"
+    depends_on:
+      elasticsearch:
+        condition: service_healthy
+      kibana:
+        condition: service_healthy
+    environment:
+      - "ELASTICSEARCH_USERNAME=admin"
+      - "ELASTICSEARCH_PASSWORD=changeme"
+      - "FLEET_SERVER_ENABLE=${fleetServerMode:-0}"
+      - "FLEET_SERVER_HOST=0.0.0.0"
+      - "FLEET_SERVER_INSECURE_HTTP=${fleetServerMode:-0}"
+      - "FLEET_SERVER_PORT=${fleetServerPort:-8220}"
+      - "FLEET_SERVER_SERVICE_TOKEN=${fleetServerServiceToken:-}"
+      - "FLEET_SERVER_POLICY_ID=${fleetServerPolicyId:-}"
+      - "FLEET_ENROLL=${fleetEnroll:-1}"
+      - "FLEET_ENROLLMENT_TOKEN=${fleetEnrollmentToken:-}"
+      - "FLEET_INSECURE=${fleetInsecure:-0}"
+      - "FLEET_URL=${fleetUrl:-}"
+      - "KIBANA_FLEET_HOST=http://kibana:5601"
+      - "KIBANA_FLEET_SETUP=${fleetServerMode:-0}"
+    platform: ${stackPlatform:-linux/amd64}
+    ports:
+      - "${fleetServerPort:-8220}:8220"
diff --git a/e2e/_suites/fleet/features/apm_integration.feature b/e2e/_suites/fleet/features/apm_integration.feature
@@ -15,7 +15,6 @@ Examples: default
   | default |
 
 @ubi8
-@skip:arm64
 Examples: Ubi8
 | image   |
 | ubi8    |