From 49fe4c949ba095f528d6ee2294990c5d3b82a54b Mon Sep 17 00:00:00 2001 From: Michal Pristas Date: Mon, 16 Mar 2020 11:44:29 +0100 Subject: [PATCH] [Agent] Docker image for integration tests (#16898) [Agent] Docker image for integration tests (#16898) --- dev-tools/mage/dockerbuilder.go | 50 ++++++++++++++++- dev-tools/packaging/packages.yml | 2 +- .../templates/docker/Dockerfile.agent.tmpl | 52 ++++++++++++++++++ .../docker/docker-entrypoint.agent.tmpl | 55 +++++++++++++++++++ x-pack/agent/_meta/agent.docker.yml | 6 +- x-pack/agent/agent.docker.yml | 6 +- 6 files changed, 162 insertions(+), 9 deletions(-) create mode 100644 dev-tools/packaging/templates/docker/Dockerfile.agent.tmpl create mode 100644 dev-tools/packaging/templates/docker/docker-entrypoint.agent.tmpl diff --git a/dev-tools/mage/dockerbuilder.go b/dev-tools/mage/dockerbuilder.go index 046a476957b..adc30aabfd7 100644 --- a/dev-tools/mage/dockerbuilder.go +++ b/dev-tools/mage/dockerbuilder.go @@ -120,8 +120,8 @@ func (b *dockerBuilder) prepareBuild() error { "ModulesDirs": b.modulesDirs(), } - return filepath.Walk(templatesDir, func(path string, info os.FileInfo, _ error) error { - if !info.IsDir() { + err = filepath.Walk(templatesDir, func(path string, info os.FileInfo, _ error) error { + if !info.IsDir() && !isDockerFile(path) { target := strings.TrimSuffix( filepath.Join(b.buildDir, filepath.Base(path)), ".tmpl", @@ -134,6 +134,52 @@ func (b *dockerBuilder) prepareBuild() error { } return nil }) + + if err != nil { + return err + } + + return b.expandDockerfile(templatesDir, data) +} + +func isDockerFile(path string) bool { + path = filepath.Base(path) + return strings.HasPrefix(path, "Dockerfile") || strings.HasPrefix(path, "docker-entrypoint") +} + +func (b *dockerBuilder) expandDockerfile(templatesDir string, data map[string]interface{}) error { + // has specific dockerfile + dockerfile := fmt.Sprintf("Dockerfile.%s.tmpl", b.imageName) + _, err := os.Stat(filepath.Join(templatesDir, dockerfile)) + if err != nil { + // specific missing fallback to generic + dockerfile = "Dockerfile.tmpl" + } + + entrypoint := fmt.Sprintf("docker-entrypoint.%s.tmpl", b.imageName) + _, err = os.Stat(filepath.Join(templatesDir, entrypoint)) + if err != nil { + // specific missing fallback to generic + entrypoint = "docker-entrypoint.tmpl" + } + + type fileExpansion struct { + source string + target string + } + for _, file := range []fileExpansion{{dockerfile, "Dockerfile.tmpl"}, {entrypoint, "docker-entrypoint.tmpl"}} { + target := strings.TrimSuffix( + filepath.Join(b.buildDir, file.target), + ".tmpl", + ) + path := filepath.Join(templatesDir, file.source) + err = b.ExpandFile(path, target, data) + if err != nil { + return errors.Wrapf(err, "expanding template '%s' to '%s'", path, target) + } + } + + return nil } func (b *dockerBuilder) dockerBuild() (string, error) { diff --git a/dev-tools/packaging/packages.yml b/dev-tools/packaging/packages.yml index 4b155cd39b1..fd4e83acce6 100644 --- a/dev-tools/packaging/packages.yml +++ b/dev-tools/packaging/packages.yml @@ -160,7 +160,7 @@ shared: <<: *agent_binary_spec extra_vars: from: 'centos:7' - user: '{{ .BeatName }}' + user: 'root' linux_capabilities: '' files: 'agent.yml': diff --git a/dev-tools/packaging/templates/docker/Dockerfile.agent.tmpl b/dev-tools/packaging/templates/docker/Dockerfile.agent.tmpl new file mode 100644 index 00000000000..5ac98b09129 --- /dev/null +++ b/dev-tools/packaging/templates/docker/Dockerfile.agent.tmpl @@ -0,0 +1,52 @@ +{{- $beatHome := printf "%s/%s" "/usr/share" .BeatName }} +{{- $beatBinary := printf "%s/%s" $beatHome .BeatName }} +{{- $repoInfo := repo }} + +FROM {{ .from }} + +LABEL \ + org.label-schema.build-date="{{ date }}" \ + org.label-schema.schema-version="1.0" \ + org.label-schema.vendor="{{ .BeatVendor }}" \ + org.label-schema.license="{{ .License }}" \ + org.label-schema.name="{{ .BeatName }}" \ + org.label-schema.version="{{ beat_version }}" \ + org.label-schema.url="{{ .BeatURL }}" \ + org.label-schema.vcs-url="{{ $repoInfo.RootImportPath }}" \ + org.label-schema.vcs-ref="{{ commit }}" \ + license="{{ .License }}" \ + description="{{ .BeatDescription }}" + +ENV ELASTIC_CONTAINER "true" +ENV PATH={{ $beatHome }}:$PATH + +COPY beat {{ $beatHome }} +COPY docker-entrypoint /usr/local/bin/docker-entrypoint +RUN chmod 755 /usr/local/bin/docker-entrypoint + +RUN groupadd --gid 1000 {{ .BeatName }} + +RUN mkdir {{ $beatHome }}/data {{ $beatHome }}/logs && \ + chown -R root:{{ .BeatName }} {{ $beatHome }} && \ + find {{ $beatHome }} -type d -exec chmod 0750 {} \; && \ + find {{ $beatHome }} -type f -exec chmod 0640 {} \; && \ + chmod 0750 {{ $beatBinary }} && \ +{{- if .linux_capabilities }} + setcap {{ .linux_capabilities }} {{ $beatBinary }} && \ +{{- end }} +{{- range $i, $modulesd := .ModulesDirs }} + chmod 0770 {{ $beatHome}}/{{ $modulesd }} && \ +{{- end }} + chmod 0770 {{ $beatHome }}/data {{ $beatHome }}/logs + +{{- if ne .user "root" }} +RUN useradd -M --uid 1000 --gid 1000 --home {{ $beatHome }} {{ .user }} +{{- end }} +USER {{ .user }} + +{{- range $i, $port := .ExposePorts }} +EXPOSE {{ $port }} +{{- end }} + +WORKDIR {{ $beatHome }} +ENTRYPOINT ["/usr/local/bin/docker-entrypoint"] diff --git a/dev-tools/packaging/templates/docker/docker-entrypoint.agent.tmpl b/dev-tools/packaging/templates/docker/docker-entrypoint.agent.tmpl new file mode 100644 index 00000000000..29ed2a60beb --- /dev/null +++ b/dev-tools/packaging/templates/docker/docker-entrypoint.agent.tmpl @@ -0,0 +1,55 @@ +#!/bin/bash + +set -eo pipefail + +# Environment variables used +# FLEET_ADMIN_PASSWORD - used for new fleet user [elastic] +# FLEET_ADMIN_USERNAME - used for new fleet user [changeme] +# FLEET_CONFIG_ID - config related to new token [defaul] +# FLEET_ENROLLMENT_TOKEN - existing enrollment token to be used for enroll +# FLEET_ENROLL - if set to 1 enroll will be performed +# FLEET_SETUP - if set to 1 fleet setup will be performed +# FLEET_TOKEN_NAME - token name for a token to be created +# KIBANA_HOST - actual kibana host [http://localhost:5601] +# KIBANA_PASSWORD - password for accessing kibana API [changeme] +# KIBANA_USERNAME - username for accessing kibana API [elastic] + +function setup(){ + curl -X POST ${KIBANA_HOST:-http://localhost:5601}/api/ingest_manager/setup -H 'kbn-xsrf: true' -u ${KIBANA_USERNAME:-elastic}:${KIBANA_PASSWORD:-changeme} + curl -X POST ${KIBANA_HOST:-http://localhost:5601}/api/ingest_manager/fleet/setup \ + -H 'Content-Type: application/json' \ + -H 'kbn-xsrf: true' \ + -d '{"admin_username":"'"${FLEET_ADMIN_USERNAME:-elastic}"'","admin_password":"'"${FLEET_ADMIN_PASSWORD:-changeme}"'"}' \ + -u ${KIBANA_USERNAME:-elastic}:${KIBANA_PASSWORD:-changeme} +} + +function enroll(){ + local enrollResp + local apiKey + + if [[ -n "${FLEET_ENROLLMENT_TOKEN}" ]] && [[ ${FLEET_ENROLLMENT_TOKEN} == 1 ]]; then + apikey = "${FLEET_ENROLLMENT_TOKEN}" + else + enrollResp=$(curl -X POST ${KIBANA_HOST:-http://localhost:5601}/api/ingest_manager/fleet/enrollment-api-keys \ + -H 'Content-Type: application/json' \ + -H 'kbn-xsrf: true' \ + -u ${KIBANA_USERNAME:-elastic}:${KIBANA_PASSWORD:-changeme} \ + -d '{"name":"'"${FLEET_TOKEN_NAME:-demotoken}"'","config_id":"'"${FLEET_CONFIG_ID:-default}"'"}') + + local exitCode=$? + if [ $exitCode -ne 0 ]; then + exit $exitCode + fi + + apikey=$(echo $enrollResp | jq -r '.item.api_key') + fi + + ./{{ .BeatName }} enroll ${KIBANA_HOST:-http://localhost:5601} $apikey -f +} +yum install -y epel-release +yum install -y jq + +if [[ -n "${FLEET_SETUP}" ]] && [[ ${FLEET_SETUP} == 1 ]]; then setup; fi +if [[ -n "${FLEET_ENROLL}" ]] && [[ ${FLEET_ENROLL} == 1 ]]; then enroll; fi + +exec {{ .BeatName }} run "$@" diff --git a/x-pack/agent/_meta/agent.docker.yml b/x-pack/agent/_meta/agent.docker.yml index 5ac56bef8e4..eb0e4d3771c 100644 --- a/x-pack/agent/_meta/agent.docker.yml +++ b/x-pack/agent/_meta/agent.docker.yml @@ -4,9 +4,9 @@ outputs: default: type: elasticsearch - hosts: [127.0.0.1:9200, 127.0.0.1:9300] - username: elastic - password: changeme + hosts: '${ELASTICSEARCH_HOSTS:http://elasticsearch:9200}' + username: '${ELASTICSEARCH_USERNAME:elastic}' + password: '${ELASTICSEARCH_PASSWORD:changeme}' datasources: - namespace: default diff --git a/x-pack/agent/agent.docker.yml b/x-pack/agent/agent.docker.yml index 5ac56bef8e4..eb0e4d3771c 100644 --- a/x-pack/agent/agent.docker.yml +++ b/x-pack/agent/agent.docker.yml @@ -4,9 +4,9 @@ outputs: default: type: elasticsearch - hosts: [127.0.0.1:9200, 127.0.0.1:9300] - username: elastic - password: changeme + hosts: '${ELASTICSEARCH_HOSTS:http://elasticsearch:9200}' + username: '${ELASTICSEARCH_USERNAME:elastic}' + password: '${ELASTICSEARCH_PASSWORD:changeme}' datasources: - namespace: default