From 08c3b5fc904f4c250dfe194dbfbfb5c0ecb1764a Mon Sep 17 00:00:00 2001 From: Ryan McCue Date: Tue, 20 Aug 2024 15:44:06 +1000 Subject: [PATCH 1/2] Remove Elasticsearch configuration This is instead moving into the Enhanced Search package. --- .../class-docker-compose-generator.php | 179 ------------------ 1 file changed, 179 deletions(-) diff --git a/inc/composer/class-docker-compose-generator.php b/inc/composer/class-docker-compose-generator.php index c8843ca..81b12cc 100644 --- a/inc/composer/class-docker-compose-generator.php +++ b/inc/composer/class-docker-compose-generator.php @@ -165,7 +165,6 @@ protected function get_php_reusable() : array { "proxy:{$this->hostname}", "proxy:pinpoint-{$this->hostname}", "proxy:cognito-{$this->hostname}", - "proxy:elasticsearch-{$this->hostname}", "proxy:s3-{$this->hostname}", "proxy:s3-{$this->project_name}.localhost", ], @@ -189,8 +188,6 @@ protected function get_php_reusable() : array { 'PAGER' => 'more', 'HM_ENV_ARCHITECTURE' => 'local-server', 'HM_DEPLOYMENT_REVISION' => 'dev', - 'ELASTICSEARCH_HOST' => 'elasticsearch', - 'ELASTICSEARCH_PORT' => 9200, 'AWS_XRAY_DAEMON_HOST' => 'xray', 'S3_UPLOADS_ENDPOINT' => Command::set_url_scheme( "https://s3-{$this->hostname}/{$this->bucket_name}/" ), 'S3_UPLOADS_BUCKET' => "{$this->bucket_name}", @@ -215,12 +212,6 @@ protected function get_php_reusable() : array { ], ]; - if ( $this->get_config()['elasticsearch'] ) { - $services['depends_on']['elasticsearch'] = [ - 'condition' => 'service_healthy', - ]; - } - // Forward CI env var - set by Travis, Circle CI, GH Actions and more... if ( getenv( 'CI' ) ) { $services['environment']['CI'] = getenv( 'CI' ); @@ -476,130 +467,6 @@ protected function get_service_db() : array { ]; } - /** - * Get the Elasticsearch service. - * - * @return array - */ - protected function get_service_elasticsearch() : array { - $mem_limit = getenv( 'ES_MEM_LIMIT' ) ?: '1g'; - - $version_map = [ - '7.10' => 'humanmade/altis-local-server-elasticsearch:4.1.0', - '7' => 'humanmade/altis-local-server-elasticsearch:4.1.0', - '6.8' => 'humanmade/altis-local-server-elasticsearch:3.1.0', - '6' => 'humanmade/altis-local-server-elasticsearch:3.1.0', - '6.3' => 'humanmade/altis-local-server-elasticsearch:3.0.0', - ]; - - $this->check_elasticsearch_version( array_keys( $version_map ) ); - - $image = $version_map[ $this->get_elasticsearch_version() ]; - - return [ - 'elasticsearch' => [ - 'image' => $image, - 'restart' => 'unless-stopped', - 'container_name' => "{$this->project_name}-es", - 'ulimits' => [ - 'memlock' => [ - 'soft' => -1, - 'hard' => -1, - ], - ], - 'mem_limit' => $mem_limit, - 'volumes' => [ - 'es-data:/usr/share/elasticsearch/data', - "{$this->root_dir}/content/uploads/es-packages:/usr/share/elasticsearch/config/packages", - ], - 'ports' => [ - '9200', - ], - 'networks' => [ - 'proxy', - 'default', - ], - 'healthcheck' => [ - 'test' => [ - 'CMD-SHELL', - 'curl --silent --fail localhost:9200/_cluster/health || exit 1', - ], - 'interval' => '5s', - 'timeout' => '5s', - 'retries' => 25, - ], - 'labels' => [ - 'traefik.port=9200', - 'traefik.protocol=http', - 'traefik.docker.network=proxy', - "traefik.frontend.rule=HostRegexp:elasticsearch-{$this->hostname}", - "traefik.domain=elasticsearch-{$this->hostname}", - ], - 'environment' => [ - 'http.max_content_length=10mb', - // Force ES into single-node mode (otherwise defaults to zen discovery as - // network.host is set in the default config). - 'discovery.type=single-node', - // Use max container memory limit as the max JVM heap allocation value. - "ES_JAVA_OPTS=-Xms512m -Xmx{$mem_limit}", - ], - ], - ]; - } - - /** - * Get the Kibana service. - * - * @return array - */ - protected function get_service_kibana() : array { - - $version_map = [ - '7.10' => 'humanmade/altis-local-server-kibana:1.1.1', - '7' => 'humanmade/altis-local-server-kibana:1.1.1', - '6.8' => 'blacktop/kibana:6.8', - '6' => 'blacktop/kibana:6.8', - '6.3' => 'blacktop/kibana:6.3', - ]; - - $this->check_elasticsearch_version( array_keys( $version_map ) ); - - $image = $version_map[ $this->get_elasticsearch_version() ]; - - $yml_file = 'kibana.yml'; - if ( version_compare( $this->get_elasticsearch_version(), '7', '>=' ) ) { - $yml_file = 'kibana-7.yml'; - } - - return [ - 'kibana' => [ - 'image' => $image, - 'container_name' => "{$this->project_name}-kibana", - 'networks' => [ - 'proxy', - 'default', - ], - 'ports' => [ - '5601', - ], - 'labels' => [ - 'traefik.port=5601', - 'traefik.protocol=http', - 'traefik.docker.network=proxy', - "traefik.frontend.rule=Host:{$this->hostname};PathPrefix:/kibana", - ], - 'depends_on' => [ - 'elasticsearch' => [ - 'condition' => 'service_healthy', - ], - ], - 'volumes' => [ - "{$this->config_dir}/{$yml_file}:/usr/share/kibana/config/kibana.yml", - ], - ], - ]; - } - /** * Get the S3 service. * @@ -836,10 +703,6 @@ public function get_array() : array { $services = array_merge( $services, $this->get_service_cavalcade() ); } - if ( $this->get_config()['elasticsearch'] ) { - $services = array_merge( $services, $this->get_service_elasticsearch() ); - } - $services = array_merge( $services, $this->get_service_mailhog() @@ -857,10 +720,6 @@ public function get_array() : array { $services = array_merge( $services, $this->get_service_analytics() ); } - if ( $this->get_config()['kibana'] && $this->get_config()['elasticsearch'] ) { - $services = array_merge( $services, $this->get_service_kibana() ); - } - if ( strpos( $this->args['xdebug'] ?? false, 'profile' ) !== false ) { $services = array_merge( $services, $this->get_service_webgrind() ); } @@ -882,7 +741,6 @@ public function get_array() : array { ], 'volumes' => [ 'db-data' => null, - 'es-data' => null, 'tmp' => null, 's3' => null, 'socket' => null, @@ -973,7 +831,6 @@ protected function get_config() : array { 'tachyon' => $modules['media']['tachyon'] ?? true, 'analytics' => $analytics_enabled, 'cavalcade' => $modules['cloud']['cavalcade'] ?? true, - 'elasticsearch' => ( $analytics_enabled || $search_enabled ) ? '7' : false, 'kibana' => ( $analytics_enabled || $search_enabled ), 'afterburner' => false, 'xray' => $modules['cloud']['xray'] ?? true, @@ -986,42 +843,6 @@ protected function get_config() : array { return array_merge( $defaults, $modules['local-server'] ?? [] ); } - /** - * Get the configured Elasticsearch version. - * - * @return int - */ - protected function get_elasticsearch_version() : string { - if ( ! empty( $this->get_config()['elasticsearch'] ) ) { - return (string) $this->get_config()['elasticsearch']; - } - - return '7'; - } - - /** - * Check the configured Elasticsearch version in config. - * - * @param array $versions List of available version numbers. - * @return void - */ - protected function check_elasticsearch_version( array $versions ) { - $versions = array_map( 'strval', $versions ); - rsort( $versions ); - if ( in_array( $this->get_elasticsearch_version(), $versions, true ) ) { - return; - } - - echo sprintf( - "The configured elasticsearch version \"%s\" is not supported.\nTry one of the following:\n - %s\n", - // phpcs:ignore HM.Security.EscapeOutput.OutputNotEscaped - $this->get_elasticsearch_version(), - // phpcs:ignore HM.Security.EscapeOutput.OutputNotEscaped - implode( "\n - ", $versions ) - ); - exit( 1 ); - } - /** * Get the main application volume adjusted for sharing config options. * From ff559c4797497e99f9e8ba49c41b66a309ed6845 Mon Sep 17 00:00:00 2001 From: Ryan McCue Date: Wed, 21 Aug 2024 13:41:06 +1000 Subject: [PATCH 2/2] Update Elasticsearch docs for new config --- docs/elasticsearch.md | 62 +++++++++++++++++++++++++++++++++++++------ 1 file changed, 54 insertions(+), 8 deletions(-) diff --git a/docs/elasticsearch.md b/docs/elasticsearch.md index b91a153..ce5e387 100644 --- a/docs/elasticsearch.md +++ b/docs/elasticsearch.md @@ -1,20 +1,44 @@ -# ElasticSearch +# Elasticsearch + +Elasticsearch is available on some Altis plans, and is fully integrated into Altis with the [Enhanced Search module](docs://search/), enabling enhanced search and relevancy as well as powering the analytics data query layer. + +(Note: Altis uses [OpenDistro for Elasticsearch](https://opendistro.github.io/for-elasticsearch-docs/) and [OpenSearch](https://opensearch.org/), which are compatible with Elasticsearch.) + + +## Enabling + +Elasticsearch support in Local Server is enabled when the Enhanced Search module is installed. It can be disabled through configuration if desired: + +```json +{ + "extra": { + "altis": { + "modules": { + "search": { + "local": { + "enabled": false + } + } + } + } + } +} +``` -ElasticSearch is an integral component of Altis, enabling enhanced search and relevancy as well as powering the analytics data query -layer. ## Available Versions -Elasticsearch defaults to version 7.10 however you can change the version in your config if your cloud environments have not yet -been updated and you need to match them: +Elasticsearch defaults to version 7.10, however you can change the version in your config if using a different version. This should match your cloud environments; consult the Settings > Environment page for your environment for details about the configuration being used. ```json { "extra": { "altis": { "modules": { - "local-server": { - "elasticsearch": "6.8" + "search": { + "local": { + "version": "6.8" + } } } } @@ -32,6 +56,7 @@ You can also use the major version on its own to get the latest minor version, f **Note**: If your device has an ARM chip you must use Elasticsearch 7 or higher. + ## Kibana Local Server provides [Kibana](https://www.elastic.co/products/kibana) out of the box, a powerful tool for viewing indexes, creating @@ -39,7 +64,24 @@ and debugging queries and more. Kibana is available at [`/kibana/`](internal://site/kibana/). -The version will always match the current Elasticsearch version. +The version will always match the current Elasticsearch version. Kibana is enabled in Local Server by default when Enhanced Search is installed, but can be disabled via configuration: + +```json +{ + "extra": { + "altis": { + "modules": { + "search": { + "local": { + "kibana": false + } + } + } + } + } +} +``` + ### Adding Index Patterns @@ -61,6 +103,7 @@ current version. You can add additional index patterns from the Management section at any time. + ### Developing & Debugging Queries Use the "Dev Tools" tab to enter and run queries. This provides useful features including linting and autocompletion based on your @@ -68,6 +111,7 @@ data. ![Kibana "Dev Tools" panel](./assets/kibana-dev-tools.png) + ### Viewing & Understanding Data The easiest way to view your data is in the Discover tab. You will need to create some index patterns first before you can explore @@ -78,6 +122,7 @@ types. ![Kibana Discover panel](./assets/kibana-discover.png) + ## Accessing ElasticSearch Directly The ElasticSearch host name is not directly exposed however you can find the dynamic port and IP to connect to by @@ -95,6 +140,7 @@ Copy the mapped IP and port (`0.0.0.0:32871` in the example above) and use it to curl -XGET http://0.0.0.0:32871 ``` + ## ElasticSearch Memory Limit ElasticSearch requires more memory on certain operating systems such as Ubuntu or when using Continuous Integration services. If