From 1585ec2c1c19962d2c5879bc6fdc018bc0c45fb1 Mon Sep 17 00:00:00 2001 From: Julien Mailleret Date: Mon, 27 May 2019 18:16:50 +0200 Subject: [PATCH] remove file script feature File scripts have been removed since elasticsearch 6.0 (https://www.elastic.co/guide/en/elasticsearch/reference/6.0/breaking_60_scripting_changes.html#_file_scripts_removed) --- README.md | 51 ++++++++----------- defaults/main.yml | 1 - files/scripts/calculate-score.groovy | 1 - tasks/elasticsearch-scripts.yml | 26 ---------- tasks/main.yml | 6 --- .../helpers/serverspec/multi_spec.rb | 11 ---- .../helpers/serverspec/shared_spec.rb | 10 ---- test/integration/multi.yml | 2 - 8 files changed, 22 insertions(+), 86 deletions(-) delete mode 100644 files/scripts/calculate-score.groovy delete mode 100644 tasks/elasticsearch-scripts.yml diff --git a/README.md b/README.md index 703ed5b2..949140d5 100644 --- a/README.md +++ b/README.md @@ -26,10 +26,10 @@ Create your Ansible playbook with your own tasks, and include the role elasticse ansible-galaxy install elastic.elasticsearch,7.0.1 ``` -Then create your playbook yaml adding the role elasticsearch. By default, the user is only required to specify a unique es_instance_name per role application. This should be unique per node. +Then create your playbook yaml adding the role elasticsearch. By default, the user is only required to specify a unique es_instance_name per role application. This should be unique per node. The application of the elasticsearch role results in the installation of a node on a host. -The simplest configuration therefore consists of: +The simplest configuration therefore consists of: ```yaml - name: Simple Example @@ -93,7 +93,7 @@ The `PATTERN` is a kitchen pattern which can match multiple suites. To run all t $ make converge PATTERN=centos-7 ``` -The default version is 7.x. If you want to test 6.x you can override it with the `VERSION` variable, for example: +The default version is 7.x. If you want to test 6.x you can override it with the `VERSION` variable, for example: ```sh $ make converge VERSION=6.x PATTERN=oss-centos-7 ``` @@ -105,7 +105,7 @@ $ make destroy-all ### Basic Elasticsearch Configuration -All Elasticsearch configuration parameters are supported. This is achieved using a configuration map parameter 'es_config' which is serialized into the elasticsearch.yml file. +All Elasticsearch configuration parameters are supported. This is achieved using a configuration map parameter 'es_config' which is serialized into the elasticsearch.yml file. The use of a map ensures the Ansible playbook does not need to be updated to reflect new/deprecated/plugin configuration parameters. In addition to the es_config map, several other parameters are supported for additional functions e.g. script installation. These can be found in the role's defaults/main.yml file. @@ -131,7 +131,6 @@ The following illustrates applying configuration parameters to an Elasticsearch node.data: false node.master: true bootstrap.memory_lock: true - es_scripts: false es_templates: false es_version_lock: false es_heap_size: 1g @@ -143,9 +142,9 @@ Whilst the role installs Elasticsearch with the default configuration parameters * ```es_config['http.port']``` - the http port for the node * ```es_config['transport.port']``` - the transport port for the node * ```es_config['discovery.seed_hosts']``` - the unicast discovery list, in the comma separated format ```":,:"``` (typically the clusters dedicated masters) -* ```es_config['network.host']``` - sets both network.bind_host and network.publish_host to the same host value. The network.bind_host setting allows to control the host different network components will bind on. +* ```es_config['network.host']``` - sets both network.bind_host and network.publish_host to the same host value. The network.bind_host setting allows to control the host different network components will bind on. -The network.publish_host setting allows to control the host the node will publish itself within the cluster so other nodes will be able to connect to it. +The network.publish_host setting allows to control the host the node will publish itself within the cluster so other nodes will be able to connect to it. See https://www.elastic.co/guide/en/elasticsearch/reference/current/modules-network.html for further details on default binding behaviour and available options. The role makes no attempt to enforce the setting of these are requires users to specify them appropriately. IT is recommended master nodes are listed and thus deployed first where possible. @@ -171,7 +170,6 @@ A more complex example: node.data: false node.master: true bootstrap.memory_lock: true - es_scripts: false es_templates: false es_version_lock: false es_heap_size: 1g @@ -186,12 +184,12 @@ A more complex example: #### Important Note -**The role uses es_api_host and es_api_port to communicate with the node for actions only achievable via http e.g. to install templates and to check the NODE IS ACTIVE. These default to "localhost" and 9200 respectively. +**The role uses es_api_host and es_api_port to communicate with the node for actions only achievable via http e.g. to install templates and to check the NODE IS ACTIVE. These default to "localhost" and 9200 respectively. If the node is deployed to bind on either a different host or port, these must be changed.** ### Multi Node Server Installations -The application of the elasticsearch role results in the installation of a node on a host. Specifying the role multiple times for a host therefore results in the installation of multiple nodes for the host. +The application of the elasticsearch role results in the installation of a node on a host. Specifying the role multiple times for a host therefore results in the installation of multiple nodes for the host. An example of a two server deployment is shown below. The first server holds the master and is thus declared first. Whilst not mandatory, this is recommended in any multi node cluster configuration. The second server hosts two data nodes. @@ -212,7 +210,6 @@ An example of a two server deployment is shown below. The first server holds th node.data: false node.master: true bootstrap.memory_lock: false - es_scripts: false es_templates: false es_version_lock: false ansible_user: ansible @@ -224,7 +221,7 @@ An example of a two server deployment is shown below. The first server holds th - role: elastic.elasticsearch vars: es_instance_name: "node1" - es_data_dirs: + es_data_dirs: - "/opt/elasticsearch" es_config: cluster.name: "test-cluster" @@ -234,14 +231,13 @@ An example of a two server deployment is shown below. The first server holds th node.data: true node.master: false bootstrap.memory_lock: false - es_scripts: false es_templates: false es_version_lock: false ansible_user: ansible es_api_port: 9200 es_plugins: - plugin: ingest-attachment - + - hosts: data_nodes roles: - role: elastic.elasticsearch @@ -256,7 +252,6 @@ An example of a two server deployment is shown below. The first server holds th node.master: false bootstrap.memory_lock: false cluster.name: "test-cluster" - es_scripts: false es_templates: false es_version_lock: false es_api_port: 9201 @@ -317,8 +312,8 @@ es_users: - power_user - user ``` - - + + * ```es_roles``` - Elasticsearch roles can be declared here as yml. Two sub keys 'native' and 'file' determine how the role is created i.e. either through a file or http(native) call. Beneath each key list the roles with appropriate permissions, using the file based format described [here] (https://www.elastic.co/guide/en/x-pack/current/file-realm.html) e.g. ```yaml @@ -360,13 +355,13 @@ es_roles: - write - delete - create_index -``` - +``` + * ```es_xpack_license``` - X-Pack license. The license is a json blob. Set the variable directly (possibly protected by Ansible vault) or from a file in the Ansible project on the control machine via a lookup: ```yaml es_xpack_license: "{{ lookup('file', playbook_dir + '/files/' + es_cluster_name + '/license.json') }}" -``` +``` X-Pack configuration parameters can be added to the elasticsearch.yml file using the normal `es_config` parameter. @@ -397,7 +392,7 @@ In addition to es_config, the following parameters allow the customization of th * ```es_plugins``` an array of plugin definitions e.g.: ```yaml es_plugins: - - plugin: ingest-attachment + - plugin: ingest-attachment ``` * ```es_path_repo``` Sets the whitelist for allowing local back-up repositories * ```es_action_auto_create_index ``` Sets the value for auto index creation, use the syntax below for specifying indexes (else true/false): @@ -413,7 +408,7 @@ In addition to es_config, the following parameters allow the customization of th * ```es_custom_package_url``` the URL to the rpm or deb package for Ansible to install. When using this you will also need to set `es_use_repository: false` and make sure that the `es_version` matches the version being installed from your custom URL. E.g. `es_custom_package_url: https://downloads.example.com/elasticsearch.rpm` Earlier examples illustrate the installation of plugins using `es_plugins`. For officially supported plugins no version or source delimiter is required. The plugin script will determine the appropriate plugin version based on the target Elasticsearch version. For community based plugins include the full url. This approach should NOT be used for the X-Pack plugin. See X-Pack below for details here. - + If installing Monitoring or Alerting, ensure the license plugin is also specified. Security configuration currently has limited support, but more support is planned for later versions. To configure X-pack to send mail, the following configuration can be added to the role. When require_auth is true, you will also need to provide the user and password. If not these can be removed: @@ -434,7 +429,7 @@ To configure X-pack to send mail, the following configuration can be added to th * ```es_user_id``` - default is undefined. * ```es_group_id``` - default is undefined. -Both ```es_user_id``` and ```es_group_id``` must be set for the user and group ids to be set. +Both ```es_user_id``` and ```es_group_id``` must be set for the user and group ids to be set. By default, each node on a host will be installed to use unique pid, plugin, work, data and log directories. These directories are created, using the instance and host name, beneath default locations ] controlled by the following parameters: @@ -443,11 +438,9 @@ controlled by the following parameters: * ```es_data_dirs``` - defaults to "/var/lib/elasticsearch". This can be a list or comma separated string e.g. ["/opt/elasticsearch/data-1","/opt/elasticsearch/data-2"] or "/opt/elasticsearch/data-1,/opt/elasticsearch/data-2" * ```es_log_dir``` - defaults to "/var/log/elasticsearch". * ```es_restart_on_change``` - defaults to true. If false, changes will not result in Elasticsearch being restarted. -* ```es_plugins_reinstall``` - defaults to false. If true, all currently installed plugins will be removed from a node. Listed plugins will then be re-installed. +* ```es_plugins_reinstall``` - defaults to false. If true, all currently installed plugins will be removed from a node. Listed plugins will then be re-installed. -This role ships with sample scripts and templates located in the [files/scripts/](files/scripts) and [files/templates/](files/templates) directories, respectively. These variables are used with the Ansible [with_fileglob](http://docs.ansible.com/ansible/playbooks_loops.html#id4) loop. When setting the globs, be sure to use an absolute path. -* ```es_scripts_fileglob``` - defaults to `/files/scripts/`. -* ```es_templates_fileglob``` - defaults to `/files/templates/`. +This role ships with sample templates located in the [files/templates/](files/templates) directory. `es_templates_fileglob` variable (defaults to `/files/templates/`) is used with the Ansible [with_fileglob](http://docs.ansible.com/ansible/playbooks_loops.html#id4) loop. When setting the globs, be sure to use an absolute path. ### Proxy @@ -460,7 +453,7 @@ To define proxy only for a particular plugin during its installation: ```yaml es_plugins: - - plugin: ingest-attachment + - plugin: ingest-attachment proxy_host: proxy.example.com proxy_port: 8080 ``` @@ -472,7 +465,7 @@ To define proxy only for a particular plugin during its installation: * The role assumes the user/group exists on the server. The elasticsearch packages create the default elasticsearch user. If this needs to be changed, ensure the user exists. * The playbook relies on the inventory_name of each host to ensure its directories are unique * Changing an instance_name for a role application will result in the installation of a new component. The previous component will remain. -* KitchenCI has been used for testing. This is used to confirm images reach the correct state after a play is first applied. We currently test the latest version of 7.x and 6.x on all supported platforms. +* KitchenCI has been used for testing. This is used to confirm images reach the correct state after a play is first applied. We currently test the latest version of 7.x and 6.x on all supported platforms. * The role aims to be idempotent. Running the role multiple times, with no changes, should result in no state change on the server. If the configuration is changed, these will be applied and Elasticsearch restarted where required. * Systemd is used for Ubuntu versions >= 15, Debian >=8, Centos >=7. All other versions use init for service scripts. * In order to run x-pack tests a license file with security enabled is required. A trial license is appropriate. Set the environment variable `ES_XPACK_LICENSE_FILE` to the full path of the license file prior to running tests. diff --git a/defaults/main.yml b/defaults/main.yml index ec69115c..30687a85 100644 --- a/defaults/main.yml +++ b/defaults/main.yml @@ -15,7 +15,6 @@ es_java_install: true update_java: false es_restart_on_change: true es_plugins_reinstall: false -es_scripts: false es_templates: false es_user: elasticsearch es_group: elasticsearch diff --git a/files/scripts/calculate-score.groovy b/files/scripts/calculate-score.groovy deleted file mode 100644 index 442c25ca..00000000 --- a/files/scripts/calculate-score.groovy +++ /dev/null @@ -1 +0,0 @@ -log(_score * 2) + my_modifier \ No newline at end of file diff --git a/tasks/elasticsearch-scripts.yml b/tasks/elasticsearch-scripts.yml deleted file mode 100644 index e38c3b4c..00000000 --- a/tasks/elasticsearch-scripts.yml +++ /dev/null @@ -1,26 +0,0 @@ ---- - -- name: set fact es_script_dir - set_fact: es_script_dir={{ es_conf_dir }}/{{es_instance_name}} - tags: - - always - -- name: set fact es_script_dir when path.scripts - set_fact: es_script_dir={{es_config['path.scripts']}} - when: es_config['path.scripts'] is defined - tags: - - always - -- name: Create script dir - become: yes - file: state=directory path={{ es_script_dir }} owner={{ es_user }} group={{ es_group }} recurse=yes - -- name: Copy default scripts to elasticsearch - become: yes - copy: src=scripts dest={{ es_script_dir }} owner={{ es_user }} group={{ es_group }} - when: es_scripts_fileglob is not defined - -- name: Copy scripts to elasticsearch - become: yes - copy: src={{ item }} dest={{ es_script_dir }} owner={{ es_user }} group={{ es_group }} - with_fileglob: "{{ es_scripts_fileglob | default('') }}" diff --git a/tasks/main.yml b/tasks/main.yml index 465540c9..c0551519 100644 --- a/tasks/main.yml +++ b/tasks/main.yml @@ -39,12 +39,6 @@ tags: - config -- name: include elasticsearch-scripts.yml - include: elasticsearch-scripts.yml - when: es_scripts - tags: - - scripts - - name: include elasticsearch-plugins.yml include: elasticsearch-plugins.yml when: es_plugins is defined or es_plugins_reinstall diff --git a/test/integration/helpers/serverspec/multi_spec.rb b/test/integration/helpers/serverspec/multi_spec.rb index 278ed42e..f961f56f 100644 --- a/test/integration/helpers/serverspec/multi_spec.rb +++ b/test/integration/helpers/serverspec/multi_spec.rb @@ -64,17 +64,6 @@ end end - #Confirm scripts are on both nodes - describe file('/etc/elasticsearch/master/scripts') do - it { should be_directory } - it { should be_owned_by 'elasticsearch' } - end - - describe file('/etc/elasticsearch/master/scripts/calculate-score.groovy') do - it { should be_file } - it { should be_owned_by 'elasticsearch' } - end - #Confirm that the data directory has only been set for the first node describe file('/opt/elasticsearch/master/localhost-master') do it { should be_directory } diff --git a/test/integration/helpers/serverspec/shared_spec.rb b/test/integration/helpers/serverspec/shared_spec.rb index 0402dca5..596cd427 100644 --- a/test/integration/helpers/serverspec/shared_spec.rb +++ b/test/integration/helpers/serverspec/shared_spec.rb @@ -90,16 +90,6 @@ end end end - if vars['es_scripts'] - describe file("/etc/elasticsearch/#{vars['es_instance_name']}/scripts") do - it { should be_directory } - it { should be_owned_by 'elasticsearch' } - end - describe file("/etc/elasticsearch/#{vars['es_instance_name']}/scripts/calculate-score.groovy") do - it { should be_file } - it { should be_owned_by 'elasticsearch' } - end - end describe file('/etc/init.d/elasticsearch') do it { should_not exist } end diff --git a/test/integration/multi.yml b/test/integration/multi.yml index f018bd4f..8f1b8e07 100644 --- a/test/integration/multi.yml +++ b/test/integration/multi.yml @@ -25,7 +25,6 @@ bootstrap.memory_lock: true es_config: "{{ es_config_7x if es_major_version == '7.x' else es_config_6x }}" es_enable_xpack: false - es_scripts: true es_templates: true es_heap_size: "1g" es_api_port: 9200 @@ -40,7 +39,6 @@ - elasticsearch vars: es_enable_xpack: false - es_scripts: true es_templates: true es_heap_size: "1g" es_api_port: 9201