From 03b0d54b1c9cb8d69242813b594860d310f0f59d Mon Sep 17 00:00:00 2001 From: algarfer Date: Thu, 18 Apr 2024 17:32:48 +0200 Subject: [PATCH 1/3] Small fixes elastic --- gatewayservice/filebeat.yml | 2 +- gatewayservice/gateway-service.js | 1 - jordi/filebeat.yml | 2 +- logging/setup/roles/custom_user.json | 45 ---------------------------- logging/setup/start.sh | 3 -- proxy/filebeat.yml | 2 +- userhistory/filebeat.yml | 2 +- users/authservice/filebeat.yml | 2 +- users/userservice/filebeat.yml | 2 +- 9 files changed, 6 insertions(+), 55 deletions(-) delete mode 100644 logging/setup/roles/custom_user.json diff --git a/gatewayservice/filebeat.yml b/gatewayservice/filebeat.yml index edc4384d..7c7991a4 100644 --- a/gatewayservice/filebeat.yml +++ b/gatewayservice/filebeat.yml @@ -175,7 +175,7 @@ output.logstash: mem: events: 4096 flush.min_events: 2048 - flush.timeout: 60s + flush.timeout: 15s # ================================= Processors ================================= processors: diff --git a/gatewayservice/gateway-service.js b/gatewayservice/gateway-service.js index 8dfb9b57..e2084d3f 100644 --- a/gatewayservice/gateway-service.js +++ b/gatewayservice/gateway-service.js @@ -20,7 +20,6 @@ const logger = winston.createLogger({ const app = express(); const port = 8000; -app.use(cors()); app.use(express.json()); app.use(require("./middleware/ReqLoggerMiddleware")(logger.info.bind(logger))) diff --git a/jordi/filebeat.yml b/jordi/filebeat.yml index 750fbd06..651f7aec 100644 --- a/jordi/filebeat.yml +++ b/jordi/filebeat.yml @@ -175,7 +175,7 @@ output.logstash: mem: events: 4096 flush.min_events: 2048 - flush.timeout: 60s + flush.timeout: 15s # ================================= Processors ================================= processors: diff --git a/logging/setup/roles/custom_user.json b/logging/setup/roles/custom_user.json deleted file mode 100644 index d7cf962b..00000000 --- a/logging/setup/roles/custom_user.json +++ /dev/null @@ -1,45 +0,0 @@ -{ - "cluster": [ - "all" - ], - "indices" : [ - { - "names" : [ - ".kibana*", - ".reporting*", - ".apm-agent-configuration", - ".apm-custom-link", - ".elastic-connectors", - ".elastic-connectors-v1", - "apm-*", - "tasks", - "spaces", - "ml", - "rollup", - "ilm", - "watcher", - "myindex-*" - ], - "privileges" : [ - "all", - "read" - ] - } - ], - "applications": [ - { - "application": "kibana-.kibana", - "privileges": [ - "all" - ], - "resources": [ - "*" - ] - } - ], - "run_as": [], - "metadata": {}, - "transient_metadata": { - "enabled": true - } -} \ No newline at end of file diff --git a/logging/setup/start.sh b/logging/setup/start.sh index bafb6cd5..e05112a6 100644 --- a/logging/setup/start.sh +++ b/logging/setup/start.sh @@ -13,7 +13,6 @@ declare -A users_passwords users_passwords=( [logstash_internal]="${LOGSTASH_INTERNAL_PASSWORD:-}" [kibana_system]="${KIBANA_SYSTEM_PASSWORD:-}" - ["${CUSTOM_USER:-user}"]="${CUSTOM_USER_PASSWORD:-}" # [metricbeat_internal]="${METRICBEAT_INTERNAL_PASSWORD:-}" # [filebeat_internal]="${FILEBEAT_INTERNAL_PASSWORD:-}" # [heartbeat_internal]="${HEARTBEAT_INTERNAL_PASSWORD:-}" @@ -24,7 +23,6 @@ users_passwords=( declare -A users_roles users_roles=( [logstash_internal]='logstash_writer' - ["${CUSTOM_USER:-user}"]="custom_user" # [metricbeat_internal]='metricbeat_writer' # [filebeat_internal]='filebeat_writer' # [heartbeat_internal]='heartbeat_writer' @@ -37,7 +35,6 @@ users_roles=( declare -A roles_files roles_files=( [logstash_writer]='logstash_writer.json' - [custom_user]="custom_user.json" # [metricbeat_writer]='metricbeat_writer.json' # [filebeat_writer]='filebeat_writer.json' # [heartbeat_writer]='heartbeat_writer.json' diff --git a/proxy/filebeat.yml b/proxy/filebeat.yml index 16f1041e..13763e9e 100644 --- a/proxy/filebeat.yml +++ b/proxy/filebeat.yml @@ -175,7 +175,7 @@ output.logstash: mem: events: 4096 flush.min_events: 2048 - flush.timeout: 60s + flush.timeout: 15s # ================================= Processors ================================= processors: diff --git a/userhistory/filebeat.yml b/userhistory/filebeat.yml index a8ec248b..83b32d43 100644 --- a/userhistory/filebeat.yml +++ b/userhistory/filebeat.yml @@ -175,7 +175,7 @@ output.logstash: mem: events: 4096 flush.min_events: 2048 - flush.timeout: 60s + flush.timeout: 15s # ================================= Processors ================================= processors: diff --git a/users/authservice/filebeat.yml b/users/authservice/filebeat.yml index 5fa320b4..b6406805 100644 --- a/users/authservice/filebeat.yml +++ b/users/authservice/filebeat.yml @@ -175,7 +175,7 @@ output.logstash: mem: events: 4096 flush.min_events: 2048 - flush.timeout: 60s + flush.timeout: 15s # ================================= Processors ================================= processors: diff --git a/users/userservice/filebeat.yml b/users/userservice/filebeat.yml index 8f0cf2f5..a93963d5 100644 --- a/users/userservice/filebeat.yml +++ b/users/userservice/filebeat.yml @@ -175,7 +175,7 @@ output.logstash: mem: events: 4096 flush.min_events: 2048 - flush.timeout: 60s + flush.timeout: 15s # ================================= Processors ================================= processors: From 93679e0dd6013a3e4e915aa4aafa300dc828b55f Mon Sep 17 00:00:00 2001 From: algarfer Date: Thu, 18 Apr 2024 17:52:48 +0200 Subject: [PATCH 2/3] Added grafana to production --- .github/workflows/deploy.yml | 20 +- docker-compose.yml | 16 +- gatewayservice/monitoring/grafana/grafana.ini | 1654 +++++++++++++++++ .../monitoring/prometheus/prometheus.yml | 9 +- jordi/jordi-service.js | 5 + jordi/package-lock.json | 145 +- jordi/package.json | 1 + proxy/default.conf | 6 + userhistory/history-service.js | 5 + userhistory/package-lock.json | 145 +- userhistory/package.json | 1 + users/authservice/auth-service.js | 5 + users/authservice/package-lock.json | 141 ++ users/authservice/package.json | 1 + users/userservice/package-lock.json | 141 ++ users/userservice/package.json | 1 + users/userservice/user-service.js | 5 + 17 files changed, 2277 insertions(+), 24 deletions(-) create mode 100644 gatewayservice/monitoring/grafana/grafana.ini diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index 404320c2..aec6038c 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -387,12 +387,14 @@ jobs: key: ${{ secrets.DEPLOY_KEY }} command: | - mkdir -p wiq_es05b - cd wiq_es05b - rm -rf ./* - wget https://raw.githubusercontent.com/arquisoft/wiq_es05b/master/docker-compose.yml -O docker-compose.yml - wget https://raw.githubusercontent.com/arquisoft/wiq_es05b/master/.env -O .env - docker logout ghcr.io - docker logout ghcr - docker image prune -f - docker compose --profile prod up -d --pull always + mkdir -p wiq_es05b + cd wiq_es05b + rm -rf ./* + wget https://raw.githubusercontent.com/arquisoft/wiq_es05b/master/docker-compose.yml -O docker-compose.yml + wget https://raw.githubusercontent.com/arquisoft/wiq_es05b/master/.env -O .env + echo "GF_SECURITY_ADMIN_USER=${{ secrets.GF_SECURITY_ADMIN_USER }}" >> .env + echo "GF_SECURITY_ADMIN_PASSWORD=${{ secrets.GF_SECURITY_ADMIN_PASSWORD }}" >> .env + docker logout ghcr.io + docker logout ghcr + docker image prune -f + docker compose --profile prod up -d --pull always diff --git a/docker-compose.yml b/docker-compose.yml index 1ed54ccb..b0e48178 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -156,8 +156,6 @@ services: ELASTIC_PASSWORD: ${ELASTIC_PASSWORD:-changeme} LOGSTASH_INTERNAL_PASSWORD: ${LOGSTASH_INTERNAL_PASSWORD:-changeme} KIBANA_SYSTEM_PASSWORD: ${KIBANA_SYSTEM_PASSWORD:-changeme} - CUSTOM_USER: ${CUSTOM_USER:-user} - CUSTOM_USER_PASSWORD: ${CUSTOM_USER_PASSWORD:-changeme} networks: - mynetwork @@ -201,7 +199,7 @@ services: prometheus: image: prom/prometheus container_name: prometheus-${teamname:-wiq_es05b} - profiles: ["dev"] + profiles: ["dev", "prod"] networks: - mynetwork volumes: @@ -215,17 +213,21 @@ services: grafana: image: grafana/grafana container_name: grafana-${teamname:-wiq_es05b} - profiles: ["dev"] + profiles: ["dev", "prod"] networks: - mynetwork volumes: - grafana_data:/var/lib/grafana - ./gatewayservice/monitoring/grafana/provisioning:/etc/grafana/provisioning + - ./gatewayservice/monitoring/grafana/grafana.ini:/etc/grafana/grafana.ini environment: - GF_SERVER_HTTP_PORT=9091 - - GF_AUTH_DISABLE_LOGIN_FORM=true - - GF_AUTH_ANONYMOUS_ENABLED=true - - GF_AUTH_ANONYMOUS_ORG_ROLE=Admin +# - GF_AUTH_DISABLE_LOGIN_FORM=true +# - GF_AUTH_ANONYMOUS_ENABLED=true +# - GF_AUTH_ANONYMOUS_ORG_ROLE=Admin + - GF_SECURITY_ADMIN_USER=${GF_SECURITY_ADMIN_USER:-admin} + - GF_SECURITY_ADMIN_PASSWORD=${GF_SECURITY_ADMIN_PASSWORD:-changeme} + - GF_SERVER_SERVE_FROM_SUB_PATH=${GF_SERVER_SERVE_FROM_SUB_PATH:-false} ports: - "9091:9091" depends_on: diff --git a/gatewayservice/monitoring/grafana/grafana.ini b/gatewayservice/monitoring/grafana/grafana.ini new file mode 100644 index 00000000..884ec51a --- /dev/null +++ b/gatewayservice/monitoring/grafana/grafana.ini @@ -0,0 +1,1654 @@ +##################### Grafana Configuration Example ##################### +# +# Everything has defaults so you only need to uncomment things you want to +# change + +# possible values : production, development +;app_mode = production + +# instance name, defaults to HOSTNAME environment variable value or hostname if HOSTNAME var is empty +;instance_name = ${HOSTNAME} + +# force migration will run migrations that might cause dataloss +# Deprecated, use clean_upgrade option in [unified_alerting.upgrade] instead. +;force_migration = false + +#################################### Paths #################################### +[paths] +# Path to where grafana can store temp files, sessions, and the sqlite3 db (if that is used) +;data = /var/lib/grafana + +# Temporary files in `data` directory older than given duration will be removed +;temp_data_lifetime = 24h + +# Directory where grafana can store logs +;logs = /var/log/grafana + +# Directory where grafana will automatically scan and look for plugins +;plugins = /var/lib/grafana/plugins + +# folder that contains provisioning config files that grafana will apply on startup and while running. +;provisioning = conf/provisioning + +#################################### Server #################################### +[server] +# Protocol (http, https, h2, socket) +;protocol = http + +# This is the minimum TLS version allowed. By default, this value is empty. Accepted values are: TLS1.2, TLS1.3. If nothing is set TLS1.2 would be taken +;min_tls_version = "" + +# The ip address to bind to, empty will bind to all interfaces +;http_addr = + +# The http port to use +;http_port = 3000 + +# The public facing domain name used to access grafana from a browser +;domain = localhost + +# Redirect to correct domain if host header does not match domain +# Prevents DNS rebinding attacks +;enforce_domain = false + +# The full public facing url you use in browser, used for redirects and emails +# If you use reverse proxy and sub path specify full url (with sub path) +root_url = %(protocol)s://%(domain)s:%(http_port)s/grafana + +# Serve Grafana from subpath specified in `root_url` setting. By default it is set to `false` for compatibility reasons. +serve_from_sub_path = true +# Log web requests +;router_logging = false + +# the path relative working path +;static_root_path = public + +# enable gzip +;enable_gzip = false + +# https certs & key file +;cert_file = +;cert_key = + +# Unix socket gid +# Changing the gid of a file without privileges requires that the target group is in the group of the process and that the process is the file owner +# It is recommended to set the gid as http server user gid +# Not set when the value is -1 +;socket_gid = + +# Unix socket mode +;socket_mode = + +# Unix socket path +;socket = + +# CDN Url +;cdn_url = + +# Sets the maximum time using a duration format (5s/5m/5ms) before timing out read of an incoming request and closing idle connections. +# `0` means there is no timeout for reading the request. +;read_timeout = 0 + +# This setting enables you to specify additional headers that the server adds to HTTP(S) responses. +[server.custom_response_headers] +#exampleHeader1 = exampleValue1 +#exampleHeader2 = exampleValue2 + +#################################### GRPC Server ######################### +;[grpc_server] +;network = "tcp" +;address = "127.0.0.1:10000" +;use_tls = false +;cert_file = +;key_file = + +#################################### Database #################################### +[database] +# You can configure the database connection by specifying type, host, name, user and password +# as separate properties or as on string using the url properties. + +# Either "mysql", "postgres" or "sqlite3", it's your choice +;type = sqlite3 +;host = 127.0.0.1:3306 +;name = grafana +;user = root +# If the password contains # or ; you have to wrap it with triple quotes. Ex """#password;""" +;password = + +# Use either URL or the previous fields to configure the database +# Example: mysql://user:secret@host:port/database +;url = + +# For "postgres", use either "disable", "require" or "verify-full" +# For "mysql", use either "true", "false", or "skip-verify". +;ssl_mode = disable + +# For "postregs", use either "1" to enable or "0" to disable SNI +;ssl_sni = + +# Database drivers may support different transaction isolation levels. +# Currently, only "mysql" driver supports isolation levels. +# If the value is empty - driver's default isolation level is applied. +# For "mysql" use "READ-UNCOMMITTED", "READ-COMMITTED", "REPEATABLE-READ" or "SERIALIZABLE". +;isolation_level = + +;ca_cert_path = +;client_key_path = +;client_cert_path = +;server_cert_name = + +# For "sqlite3" only, path relative to data_path setting +;path = grafana.db + +# Max idle conn setting default is 2 +;max_idle_conn = 2 + +# Max conn setting default is 0 (mean not set) +;max_open_conn = + +# Connection Max Lifetime default is 14400 (means 14400 seconds or 4 hours) +;conn_max_lifetime = 14400 + +# Set to true to log the sql calls and execution times. +;log_queries = + +# For "sqlite3" only. cache mode setting used for connecting to the database. (private, shared) +;cache_mode = private + +# For "sqlite3" only. Enable/disable Write-Ahead Logging, https://sqlite.org/wal.html. Default is false. +;wal = false + +# For "mysql" only if migrationLocking feature toggle is set. How many seconds to wait before failing to lock the database for the migrations, default is 0. +;locking_attempt_timeout_sec = 0 + +# For "sqlite" only. How many times to retry query in case of database is locked failures. Default is 0 (disabled). +;query_retries = 0 + +# For "sqlite" only. How many times to retry transaction in case of database is locked failures. Default is 5. +;transaction_retries = 5 + +# Set to true to add metrics and tracing for database queries. +;instrument_queries = false + +################################### Data sources ######################### +[datasources] +# Upper limit of data sources that Grafana will return. This limit is a temporary configuration and it will be deprecated when pagination will be introduced on the list data sources API. +;datasource_limit = 5000 + +#################################### Cache server ############################# +[remote_cache] +# Either "redis", "memcached" or "database" default is "database" +;type = database + +# cache connectionstring options +# database: will use Grafana primary database. +# redis: config like redis server e.g. `addr=127.0.0.1:6379,pool_size=100,db=0,ssl=false`. Only addr is required. ssl may be 'true', 'false', or 'insecure'. +# memcache: 127.0.0.1:11211 +;connstr = + +# prefix prepended to all the keys in the remote cache +; prefix = + +# This enables encryption of values stored in the remote cache +;encryption = + +#################################### Data proxy ########################### +[dataproxy] + +# This enables data proxy logging, default is false +;logging = false + +# How long the data proxy waits to read the headers of the response before timing out, default is 30 seconds. +# This setting also applies to core backend HTTP data sources where query requests use an HTTP client with timeout set. +;timeout = 30 + +# How long the data proxy waits to establish a TCP connection before timing out, default is 10 seconds. +;dialTimeout = 10 + +# How many seconds the data proxy waits before sending a keepalive probe request. +;keep_alive_seconds = 30 + +# How many seconds the data proxy waits for a successful TLS Handshake before timing out. +;tls_handshake_timeout_seconds = 10 + +# How many seconds the data proxy will wait for a server's first response headers after +# fully writing the request headers if the request has an "Expect: 100-continue" +# header. A value of 0 will result in the body being sent immediately, without +# waiting for the server to approve. +;expect_continue_timeout_seconds = 1 + +# Optionally limits the total number of connections per host, including connections in the dialing, +# active, and idle states. On limit violation, dials will block. +# A value of zero (0) means no limit. +;max_conns_per_host = 0 + +# The maximum number of idle connections that Grafana will keep alive. +;max_idle_connections = 100 + +# How many seconds the data proxy keeps an idle connection open before timing out. +;idle_conn_timeout_seconds = 90 + +# If enabled and user is not anonymous, data proxy will add X-Grafana-User header with username into the request, default is false. +;send_user_header = false + +# Limit the amount of bytes that will be read/accepted from responses of outgoing HTTP requests. +;response_limit = 0 + +# Limits the number of rows that Grafana will process from SQL data sources. +;row_limit = 1000000 + +# Sets a custom value for the `User-Agent` header for outgoing data proxy requests. If empty, the default value is `Grafana/` (for example `Grafana/9.0.0`). +;user_agent = + +#################################### Analytics #################################### +[analytics] +# Server reporting, sends usage counters to stats.grafana.org every 24 hours. +# No ip addresses are being tracked, only simple counters to track +# running instances, dashboard and error counts. It is very helpful to us. +# Change this option to false to disable reporting. +;reporting_enabled = true + +# The name of the distributor of the Grafana instance. Ex hosted-grafana, grafana-labs +;reporting_distributor = grafana-labs + +# Set to false to disable all checks to https://grafana.com +# for new versions of grafana. The check is used +# in some UI views to notify that a grafana update exists. +# This option does not cause any auto updates, nor send any information +# only a GET request to https://grafana.com/api/grafana/versions/stable to get the latest version. +;check_for_updates = true + +# Set to false to disable all checks to https://grafana.com +# for new versions of plugins. The check is used +# in some UI views to notify that a plugin update exists. +# This option does not cause any auto updates, nor send any information +# only a GET request to https://grafana.com to get the latest versions. +;check_for_plugin_updates = true + +# Google Analytics universal tracking code, only enabled if you specify an id here +;google_analytics_ua_id = + +# Google Analytics 4 tracking code, only enabled if you specify an id here +;google_analytics_4_id = + +# When Google Analytics 4 Enhanced event measurement is enabled, we will try to avoid sending duplicate events and let Google Analytics 4 detect navigation changes, etc. +;google_analytics_4_send_manual_page_views = false + +# Google Tag Manager ID, only enabled if you specify an id here +;google_tag_manager_id = + +# Rudderstack write key, enabled only if rudderstack_data_plane_url is also set +;rudderstack_write_key = + +# Rudderstack data plane url, enabled only if rudderstack_write_key is also set +;rudderstack_data_plane_url = + +# Rudderstack SDK url, optional, only valid if rudderstack_write_key and rudderstack_data_plane_url is also set +;rudderstack_sdk_url = + +# Rudderstack Config url, optional, used by Rudderstack SDK to fetch source config +;rudderstack_config_url = + +# Rudderstack Integrations URL, optional. Only valid if you pass the SDK version 1.1 or higher +;rudderstack_integrations_url = + +# Intercom secret, optional, used to hash user_id before passing to Intercom via Rudderstack +;intercom_secret = + +# Controls if the UI contains any links to user feedback forms +;feedback_links_enabled = true + +#################################### Security #################################### +[security] +# disable creation of admin user on first start of grafana +;disable_initial_admin_creation = false + +# default admin user, created on startup +;admin_user = admin + +# default admin password, can be changed before first start of grafana, or in profile settings +;admin_password = admin + +# default admin email, created on startup +;admin_email = admin@localhost + +# used for signing +;secret_key = SW2YcwTIb9zpOOhoPsMm + +# current key provider used for envelope encryption, default to static value specified by secret_key +;encryption_provider = secretKey.v1 + +# list of configured key providers, space separated (Enterprise only): e.g., awskms.v1 azurekv.v1 +;available_encryption_providers = + +# disable gravatar profile images +;disable_gravatar = false + +# data source proxy whitelist (ip_or_domain:port separated by spaces) +;data_source_proxy_whitelist = + +# disable protection against brute force login attempts +;disable_brute_force_login_protection = false + +# set to true if you host Grafana behind HTTPS. default is false. +;cookie_secure = false + +# set cookie SameSite attribute. defaults to `lax`. can be set to "lax", "strict", "none" and "disabled" +;cookie_samesite = lax + +# set to true if you want to allow browsers to render Grafana in a ,