[Agent] Docker image for integration tests (#16898)

[Agent] Docker image for integration tests (#16898)

dev-tools/mage/dockerbuilder.go

@@ -120,8 +120,8 @@ func (b *dockerBuilder) prepareBuild() error {
 		"ModulesDirs": b.modulesDirs(),
 	}
 
-	return filepath.Walk(templatesDir, func(path string, info os.FileInfo, _ error) error {
-		if !info.IsDir() {
+	err = filepath.Walk(templatesDir, func(path string, info os.FileInfo, _ error) error {
+		if !info.IsDir() && !isDockerFile(path) {
 			target := strings.TrimSuffix(
 				filepath.Join(b.buildDir, filepath.Base(path)),
 				".tmpl",
@@ -134,6 +134,52 @@ func (b *dockerBuilder) prepareBuild() error {
 		}
 		return nil
 	})
+
+	if err != nil {
+		return err
+	}
+
+	return b.expandDockerfile(templatesDir, data)
+}
+
+func isDockerFile(path string) bool {
+	path = filepath.Base(path)
+	return strings.HasPrefix(path, "Dockerfile") || strings.HasPrefix(path, "docker-entrypoint")
+}
+
+func (b *dockerBuilder) expandDockerfile(templatesDir string, data map[string]interface{}) error {
+	// has specific dockerfile
+	dockerfile := fmt.Sprintf("Dockerfile.%s.tmpl", b.imageName)
+	_, err := os.Stat(filepath.Join(templatesDir, dockerfile))
+	if err != nil {
+		// specific missing fallback to generic
+		dockerfile = "Dockerfile.tmpl"
+	}
+
+	entrypoint := fmt.Sprintf("docker-entrypoint.%s.tmpl", b.imageName)
+	_, err = os.Stat(filepath.Join(templatesDir, entrypoint))
+	if err != nil {
+		// specific missing fallback to generic
+		entrypoint = "docker-entrypoint.tmpl"
+	}
+
+	type fileExpansion struct {
+		source string
+		target string
+	}
+	for _, file := range []fileExpansion{{dockerfile, "Dockerfile.tmpl"}, {entrypoint, "docker-entrypoint.tmpl"}} {
+		target := strings.TrimSuffix(
+			filepath.Join(b.buildDir, file.target),
+			".tmpl",
+		)
+		path := filepath.Join(templatesDir, file.source)
+		err = b.ExpandFile(path, target, data)
+		if err != nil {
+			return errors.Wrapf(err, "expanding template '%s' to '%s'", path, target)
+		}
+	}
+
+	return nil
 }
 
 func (b *dockerBuilder) dockerBuild() (string, error) {

dev-tools/packaging/packages.yml

@@ -160,7 +160,7 @@ shared:
     <<: *agent_binary_spec
     extra_vars:
       from: 'centos:7'
-      user: '{{ .BeatName }}'
+      user: 'root'
       linux_capabilities: ''
     files:
       'agent.yml':

dev-tools/packaging/templates/docker/Dockerfile.agent.tmpl

@@ -0,0 +1,52 @@
+{{- $beatHome := printf "%s/%s" "/usr/share" .BeatName }}
+{{- $beatBinary := printf "%s/%s" $beatHome .BeatName }}
+{{- $repoInfo := repo }}
+
+FROM {{ .from }}
+
+LABEL \
+  org.label-schema.build-date="{{ date }}" \
+  org.label-schema.schema-version="1.0" \
+  org.label-schema.vendor="{{ .BeatVendor }}" \
+  org.label-schema.license="{{ .License }}" \
+  org.label-schema.name="{{ .BeatName }}" \
+  org.label-schema.version="{{ beat_version }}" \
+  org.label-schema.url="{{ .BeatURL }}" \
+  org.label-schema.vcs-url="{{ $repoInfo.RootImportPath }}" \
+  org.label-schema.vcs-ref="{{ commit }}" \
+  license="{{ .License }}" \
+  description="{{ .BeatDescription }}"
+
+ENV ELASTIC_CONTAINER "true"
+ENV PATH={{ $beatHome }}:$PATH
+
+COPY beat {{ $beatHome }}
+COPY docker-entrypoint /usr/local/bin/docker-entrypoint
+RUN chmod 755 /usr/local/bin/docker-entrypoint
+
+RUN groupadd --gid 1000 {{ .BeatName }}
+
+RUN mkdir {{ $beatHome }}/data {{ $beatHome }}/logs && \
+    chown -R root:{{ .BeatName }} {{ $beatHome }} && \
+    find {{ $beatHome }} -type d -exec chmod 0750 {} \; && \
+    find {{ $beatHome }} -type f -exec chmod 0640 {} \; && \
+    chmod 0750 {{ $beatBinary }} && \
+{{- if .linux_capabilities }}
+    setcap {{ .linux_capabilities }} {{ $beatBinary }} && \
+{{- end }}
+{{- range $i, $modulesd := .ModulesDirs }}
+    chmod 0770 {{ $beatHome}}/{{ $modulesd }} && \
+{{- end }}
+    chmod 0770 {{ $beatHome }}/data {{ $beatHome }}/logs
+
+{{- if ne .user "root" }}
+RUN useradd -M --uid 1000 --gid 1000 --home {{ $beatHome }} {{ .user }}
+{{- end }}
+USER {{ .user }}
+
+{{- range $i, $port := .ExposePorts }}
+EXPOSE {{ $port }}
+{{- end }}
+
+WORKDIR {{ $beatHome }}
+ENTRYPOINT ["/usr/local/bin/docker-entrypoint"]

dev-tools/packaging/templates/docker/docker-entrypoint.agent.tmpl

@@ -0,0 +1,55 @@
+#!/bin/bash
+
+set -eo pipefail
+
+# Environment variables used
+# FLEET_ADMIN_PASSWORD - used for new fleet user [elastic]
+# FLEET_ADMIN_USERNAME - used for new fleet user [changeme]
+# FLEET_CONFIG_ID - config related to new token [defaul]
+# FLEET_ENROLLMENT_TOKEN - existing enrollment token to be used for enroll
+# FLEET_ENROLL - if set to 1 enroll will be performed
+# FLEET_SETUP - if  set to 1 fleet setup will be performed
+# FLEET_TOKEN_NAME - token name for a token to be created
+# KIBANA_HOST - actual kibana host [http://localhost:5601]
+# KIBANA_PASSWORD - password for accessing kibana API [changeme]
+# KIBANA_USERNAME - username for accessing kibana API [elastic]
+
+function setup(){
+  curl -X POST  ${KIBANA_HOST:-http://localhost:5601}/api/ingest_manager/setup -H 'kbn-xsrf: true' -u ${KIBANA_USERNAME:-elastic}:${KIBANA_PASSWORD:-changeme}
+  curl -X POST  ${KIBANA_HOST:-http://localhost:5601}/api/ingest_manager/fleet/setup \
+    -H 'Content-Type: application/json' \
+    -H 'kbn-xsrf: true' \
+    -d '{"admin_username":"'"${FLEET_ADMIN_USERNAME:-elastic}"'","admin_password":"'"${FLEET_ADMIN_PASSWORD:-changeme}"'"}' \
+    -u ${KIBANA_USERNAME:-elastic}:${KIBANA_PASSWORD:-changeme}
+}
+
+function enroll(){
+    local enrollResp
+    local apiKey
+
+    if [[ -n "${FLEET_ENROLLMENT_TOKEN}" ]] && [[ ${FLEET_ENROLLMENT_TOKEN} == 1 ]]; then
+      apikey = "${FLEET_ENROLLMENT_TOKEN}"
+    else
+      enrollResp=$(curl -X POST  ${KIBANA_HOST:-http://localhost:5601}/api/ingest_manager/fleet/enrollment-api-keys \
+        -H 'Content-Type: application/json' \
+        -H 'kbn-xsrf: true' \
+        -u ${KIBANA_USERNAME:-elastic}:${KIBANA_PASSWORD:-changeme} \
+        -d '{"name":"'"${FLEET_TOKEN_NAME:-demotoken}"'","config_id":"'"${FLEET_CONFIG_ID:-default}"'"}')
+
+      local exitCode=$?
+      if [ $exitCode -ne 0 ]; then
+        exit $exitCode
+      fi
+
+      apikey=$(echo $enrollResp | jq -r '.item.api_key')
+    fi
+
+    ./{{ .BeatName }} enroll ${KIBANA_HOST:-http://localhost:5601} $apikey -f
+}
+yum install -y epel-release
+yum install -y jq
+
+if [[ -n "${FLEET_SETUP}" ]] && [[ ${FLEET_SETUP} == 1 ]]; then setup; fi
+if [[ -n "${FLEET_ENROLL}" ]] && [[ ${FLEET_ENROLL} == 1 ]]; then enroll; fi
+
+exec {{ .BeatName }} run "$@"

x-pack/agent/_meta/agent.docker.yml

@@ -4,9 +4,9 @@
 outputs:
   default:
     type: elasticsearch
-    hosts: [127.0.0.1:9200, 127.0.0.1:9300]
-    username: elastic
-    password: changeme
+    hosts: '${ELASTICSEARCH_HOSTS:http://elasticsearch:9200}'
+    username: '${ELASTICSEARCH_USERNAME:elastic}'
+    password: '${ELASTICSEARCH_PASSWORD:changeme}'
 
 datasources:
   - namespace: default

x-pack/agent/agent.docker.yml

@@ -4,9 +4,9 @@
 outputs:
   default:
     type: elasticsearch
-    hosts: [127.0.0.1:9200, 127.0.0.1:9300]
-    username: elastic
-    password: changeme
+    hosts: '${ELASTICSEARCH_HOSTS:http://elasticsearch:9200}'
+    username: '${ELASTICSEARCH_USERNAME:elastic}'
+    password: '${ELASTICSEARCH_PASSWORD:changeme}'
 
 datasources:
   - namespace: default