From c1daa3c0799c131a168a86515398cba357722092 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=28=E2=95=AF=C2=B0=E2=96=A1=C2=B0=EF=BC=89=E2=95=AF?= =?UTF-8?q?=EF=B8=B5=20u=E1=B4=89=C7=9DssnH=20=C9=90=C9=9F=C9=90=CA=87soW?= Date: Tue, 22 Nov 2022 22:25:30 +0400 Subject: [PATCH] introducing apm-server to elastdocker MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: (╯°□°）╯︵ uᴉǝssnH ɐɟɐʇsoW --- .env | 4 ++ Makefile | 2 +- README.md | 25 ++++++++ apm-server/Dockerfile | 5 ++ apm-server/config/apm-server.yml | 101 +++++++++++++++++++++++++++++++ docker-compose.yml | 28 +++++++++ kibana/config/kibana.yml | 18 +++++- setup/instances.yml | 9 ++- 8 files changed, 189 insertions(+), 3 deletions(-) create mode 100644 apm-server/Dockerfile create mode 100644 apm-server/config/apm-server.yml diff --git a/.env b/.env index 4f80c86..694b0c1 100644 --- a/.env +++ b/.env @@ -17,6 +17,9 @@ KIBANA_PORT=5601 LOGSTASH_HOST=logstash LOGSTASH_PORT=8080 +APMSERVER_HOST=apm-server +APMSERVER_PORT=8200 + #----------- Credientals ------------------------# # Username & Password for Admin Elasticsearch cluster. # This is used to set the password at setup, and used by others to connect to Elasticsearch at runtime. @@ -25,6 +28,7 @@ ELASTIC_USERNAME=elastic ELASTIC_PASSWORD=changeme AWS_ACCESS_KEY_ID=nottherealid AWS_SECRET_ACCESS_KEY=notherealsecret +ELASTIC_APM_SECRET_TOKEN=secrettokengoeshere #----------- Cluster ----------------------------# ELASTIC_CLUSTER_NAME=elastdocker-cluster diff --git a/Makefile b/Makefile index 9e5388b..cba1ac3 100644 --- a/Makefile +++ b/Makefile @@ -5,7 +5,7 @@ COMPOSE_MONITORING := -f docker-compose.yml -f docker-compose.monitor.yml COMPOSE_LOGGING := -f docker-compose.yml -f docker-compose.logs.yml COMPOSE_TOOLS := -f docker-compose.yml -f docker-compose.tools.yml COMPOSE_NODES := -f docker-compose.yml -f docker-compose.nodes.yml -ELK_SERVICES := elasticsearch logstash kibana +ELK_SERVICES := elasticsearch logstash kibana apm-server ELK_LOG_COLLECTION := filebeat ELK_MONITORING := elasticsearch-exporter logstash-exporter filebeat-cluster-logs ELK_TOOLS := rubban diff --git a/README.md b/README.md index 6204416..a291209 100644 --- a/README.md +++ b/README.md @@ -241,7 +241,32 @@ If you started Prometheus Exporters using `make monitoring` command. Prometheus ![Metrics](https://user-images.githubusercontent.com/16992394/78685076-89a58900-78f1-11ea-959b-ce374fe51500.jpg) +# Working with Elastic APM +After completing the setup step, you will notice a container named apm-server which gives you deeper visibility into your applications and can help you to identify and resolve root cause issues with correlated traces, logs, and metrics. + +## Authenticating with Elastic APM + +In order to authenticate with Elastic APM, you will need the following: + +- The value of `ELASTIC_APM_SECRET_TOKEN` defined in `.env` file as we have [secret token](https://www.elastic.co/guide/en/apm/guide/master/secret-token.html) enabled by default +- The ability to reach port `8200` +- Install elastic apm client in your application e.g. for NodeJS based applications you need to install [elastic-apm-node](https://www.elastic.co/guide/en/apm/agent/nodejs/master/typescript.html) +- Import the package in your application and call the start function, In case of NodeJS based application you can do the following: + +``` +const apm = require('elastic-apm-node').start({ + serviceName: 'foobar', + secretToken: process.env.ELASTIC_APM_SECRET_TOKEN, + + // https is enabled by default as per elastdocker configuration + serverUrl: 'https://localhost:8200', +}) +``` +> Make sure that the agent is started before you require any other modules in your Node.js application - i.e. before express, http, etc. as mentioned in [Elastic APM Agent - NodeJS initialization](https://www.elastic.co/guide/en/apm/agent/nodejs/master/express.html#express-initialization) + +For more details or other languages you can check the following: +- [APM Agents in different languages](https://www.elastic.co/guide/en/apm/agent/index.html) # License [MIT License](https://raw.githubusercontent.com/sherifabdlnaby/elastdocker/master/LICENSE) Copyright (c) 2020 Sherif Abdel-Naby diff --git a/apm-server/Dockerfile b/apm-server/Dockerfile new file mode 100644 index 0000000..92efc37 --- /dev/null +++ b/apm-server/Dockerfile @@ -0,0 +1,5 @@ +ARG ELK_VERSION + +# https://github.com/elastic/apm-server +FROM docker.elastic.co/apm/apm-server:${ELK_VERSION} +ARG ELK_VERSION diff --git a/apm-server/config/apm-server.yml b/apm-server/config/apm-server.yml new file mode 100644 index 0000000..68a6559 --- /dev/null +++ b/apm-server/config/apm-server.yml @@ -0,0 +1,101 @@ +######################### APM Server Configuration ######################### + +################################ APM Server ################################ + +apm-server: + # Defines the host and port the server is listening on. Use "unix:/path/to.sock" to listen on a unix domain socket. + host: "0.0.0.0:8200" + + + #---------------------------- APM Server - Secure Communication with Agents ---------------------------- + + # Enable authentication using Secret token + auth: + secret_token: '${ELASTIC_APM_SECRET_TOKEN}' + + # Enable secure communication between APM agents and the server. By default ssl is disabled. + ssl: + enabled: true + + # Path to file containing the certificate for server authentication. + # Needs to be configured when ssl is enabled. + certificate: "/certs/apm-server.crt" + + # Path to file containing server certificate key. + # Needs to be configured when ssl is enabled. + key: "/certs/apm-server.key" + +#================================ Outputs ================================= + +# Configure the output to use when sending the data collected by apm-server. + +#-------------------------- Elasticsearch output -------------------------- +output.elasticsearch: + # Array of hosts to connect to. + # Scheme and port can be left out and will be set to the default (`http` and `9200`). + # In case you specify and additional path, the scheme is required: `http://elasticsearch:9200/path`. + # IPv6 addresses should always be defined as: `https://[2001:db8::1]:9200`. + hosts: '${ELASTICSEARCH_HOST_PORT}' + + # Boolean flag to enable or disable the output module. + enabled: true + + # Protocol - either `http` (default) or `https`. + protocol: "https" + + # Authentication credentials + username: '${ELASTIC_USERNAME}' + password: '${ELASTIC_PASSWORD}' + + # Enable custom SSL settings. Set to false to ignore custom SSL settings for secure communication. + ssl.enabled: true + + # List of root certificates for HTTPS server verifications. + ssl.certificate_authorities: ["/certs/ca.crt"] + + # Certificate for SSL client authentication. + ssl.certificate: "/certs/apm-server.crt" + + # Client Certificate Key + ssl.key: "/certs/apm-server.key" + +#============================= X-pack Monitoring ============================= + +# APM server can export internal metrics to a central Elasticsearch monitoring +# cluster. This requires x-pack monitoring to be enabled in Elasticsearch. The +# reporting is disabled by default. + +# Set to true to enable the monitoring reporter. +monitoring.enabled: true + +# Most settings from the Elasticsearch output are accepted here as well. +# Note that these settings should be configured to point to your Elasticsearch *monitoring* cluster. +# Any setting that is not set is automatically inherited from the Elasticsearch +# output configuration. This means that if you have the Elasticsearch output configured, +# you can simply uncomment the following line. +monitoring.elasticsearch: + + # Protocol - either `http` (default) or `https`. + protocol: "https" + + # Authentication credentials + username: '${ELASTIC_USERNAME}' + password: '${ELASTIC_PASSWORD}' + + # Array of hosts to connect to. + # Scheme and port can be left out and will be set to the default (`http` and `9200`). + # In case you specify and additional path, the scheme is required: `http://elasticsearch:9200/path`. + # IPv6 addresses should always be defined as: `https://[2001:db8::1]:9200`. + hosts: '${ELASTICSEARCH_HOST_PORT}' + + # Enable custom SSL settings. Set to false to ignore custom SSL settings for secure communication. + ssl.enabled: true + + # List of root certificates for HTTPS server verifications. + ssl.certificate_authorities: ["/certs/ca.crt"] + + # Certificate for SSL client authentication. + ssl.certificate: "/certs/apm-server.crt" + + # Client Certificate Key + ssl.key: "/certs/apm-server.key" diff --git a/docker-compose.yml b/docker-compose.yml index 3ad2834..4844380 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -25,6 +25,10 @@ secrets: file: ./secrets/certs/kibana/kibana.crt kibana.key: file: ./secrets/certs/kibana/kibana.key + apm-server.certificate: + file: ./secrets/certs/apm-server/apm-server.crt + apm-server.key: + file: ./secrets/certs/apm-server/apm-server.key services: elasticsearch: @@ -120,3 +124,27 @@ services: target: /certs/kibana.key ports: - "5601:5601" + + apm-server: + image: elastdocker/apm-server:${ELK_VERSION} + build: + context: apm-server/ + args: + ELK_VERSION: $ELK_VERSION + restart: unless-stopped + ports: + - "8200:8200" + volumes: + - ./apm-server/config/apm-server.yml:/usr/share/apm-server/apm-server.yml:ro + environment: + ELASTIC_USERNAME: ${ELASTIC_USERNAME} + ELASTIC_PASSWORD: ${ELASTIC_PASSWORD} + ELASTICSEARCH_HOST_PORT: https://${ELASTICSEARCH_HOST}:${ELASTICSEARCH_PORT} + ELASTIC_APM_SECRET_TOKEN: ${ELASTIC_APM_SECRET_TOKEN} + secrets: + - source: elastic.ca + target: /certs/ca.crt + - source: apm-server.certificate + target: /certs/apm-server.crt + - source: apm-server.key + target: /certs/apm-server.key diff --git a/kibana/config/kibana.yml b/kibana/config/kibana.yml index 79e6372..b2adbf9 100644 --- a/kibana/config/kibana.yml +++ b/kibana/config/kibana.yml @@ -21,6 +21,22 @@ xpack.reporting.encryptionKey: RSCueeHKzrqzOVTJhkjt17EMnzM96LlN elasticsearch.serviceAccountToken: "${KIBANA_SERVICE_ACCOUNT_TOKEN}" elasticsearch.ssl.certificateAuthorities: [ "/certs/ca.crt" ] +## Add policy for apm-server integration +xpack.fleet.packages: + - name: apm + version: latest +xpack.fleet.agentPolicies: + - name: Agent policy 1 + id: agent-policy-1 + namespace: default + monitoring_enabled: + - logs + - metrics + package_policies: + - name: apm-1 + id: default-apm + package: + name: apm ## Misc elasticsearch.requestTimeout: 90000 @@ -29,4 +45,4 @@ elasticsearch.requestTimeout: 90000 ## ElastAlert Plugin #elastalert-kibana-plugin.serverHost: elastalert -#elastalert-kibana-plugin.serverPort: 3030 \ No newline at end of file +#elastalert-kibana-plugin.serverPort: 3030 diff --git a/setup/instances.yml b/setup/instances.yml index 2025ecb..853ea40 100644 --- a/setup/instances.yml +++ b/setup/instances.yml @@ -11,4 +11,11 @@ instances: - kibana - localhost ip: - - 127.0.0.1 \ No newline at end of file + - 127.0.0.1 + + - name: apm-server + dns: + - apm-server + - localhost + ip: + - 127.0.0.1