From 33c17a1c81e7f3a3bda8ed54743ee25a24a3d0c2 Mon Sep 17 00:00:00 2001 From: cicharka Date: Wed, 9 Feb 2022 13:09:07 +0100 Subject: [PATCH 1/7] Fix rsync and schema validation for backup/recovery Bug #2942 rsync command fails trying to copy artifacts * With new version of ansible we can use private_key option for synchronize module, therefore there's no need to use rsh Bug #2930 Backup/recovery commands fail when default configuration for backup attached to cluster-config.yml * extend run_for_individual_documents method so it can choose relevant schema for validated document --- .../tasks/common/download_via_rsync.yml | 16 ++++++++------- .../tasks/common/upload_via_rsync.yml | 19 +++++++++--------- cli/src/schema/SchemaValidator.py | 6 +++++- docs/home/howto/BACKUP.md | 20 +++++++++++-------- 4 files changed, 36 insertions(+), 25 deletions(-) diff --git a/ansible/playbooks/roles/backup/tasks/common/download_via_rsync.yml b/ansible/playbooks/roles/backup/tasks/common/download_via_rsync.yml index 768e055813..bfe75fffba 100644 --- a/ansible/playbooks/roles/backup/tasks/common/download_via_rsync.yml +++ b/ansible/playbooks/roles/backup/tasks/common/download_via_rsync.yml @@ -75,14 +75,16 @@ dest: "{{ backup_destination_dir }}" src: "{{ item }}" checksum: true - rsync_opts: - - --rsh={{ rsh }} - vars: - # this fixes / replaces incorrect path to the private key file that synchronize provides - # (setting private_key parameter has no effect whatsoever, looks like a bug tbh) - rsh: >- - /usr/bin/ssh -S none -i {{ private_key_file.path }} -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null + private_key: "{{ private_key_file.path }}" loop: "{{ artifacts }}" + # Legacy code used in eariler version of Ansible + # rsync_opts: + # - --rsh={{ rsh }} + # vars: + # # this fixes / replaces incorrect path to the private key file that synchronize provides + # # (setting private_key parameter has no effect whatsoever, looks like a bug tbh) + # rsh: >- + # /usr/bin/ssh -S none -i {{ private_key_file.path }} -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null - name: Remove copied artifacts from source file: diff --git a/ansible/playbooks/roles/recovery/tasks/common/upload_via_rsync.yml b/ansible/playbooks/roles/recovery/tasks/common/upload_via_rsync.yml index 427e8d2d1e..1b46eae77e 100644 --- a/ansible/playbooks/roles/recovery/tasks/common/upload_via_rsync.yml +++ b/ansible/playbooks/roles/recovery/tasks/common/upload_via_rsync.yml @@ -70,12 +70,13 @@ dest: "{{ recovery_dir }}/" src: "{{ item }}" checksum: true - rsync_opts: - - --rsh={{ rsh }} - vars: - # this fixes / replaces incorrect path to the private key file that synchronize provides - # (setting private_key parameter has no effect whatsoever, looks like a bug tbh) - rsh: >- - /usr/bin/ssh -S none -i {{ private_key_file.path }} -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null - loop: >- - {{ artifacts }} + private_key: "{{ private_key_file.path }}" + loop: "{{ artifacts }}" + # Legacy code used in earlier version of Ansible + # rsync_opts: + # - --rsh={{ rsh }} + # vars: + # # this fixes / replaces incorrect path to the private key file that synchronize provides + # # (setting private_key parameter has no effect whatsoever, looks like a bug tbh) + # rsh: >- + # /usr/bin/ssh -S none -i {{ private_key_file.path }} -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null diff --git a/cli/src/schema/SchemaValidator.py b/cli/src/schema/SchemaValidator.py index c2a93ef808..a8f6ea6a18 100644 --- a/cli/src/schema/SchemaValidator.py +++ b/cli/src/schema/SchemaValidator.py @@ -47,7 +47,11 @@ def validate_document(self, doc, schema): def run_for_individual_documents(self): for doc in self.validation_docs: # Load document schema - schema = load_schema_obj(types.VALIDATION, self.provider, doc.kind) + if 'backup' in doc.kind or 'recovery' in doc.kind: + schema = load_schema_obj(types.VALIDATION, self.provider, doc.kind) + else: + schema = self.get_base_schema(doc.kind) + schema['properties']['specification'] = load_schema_obj(types.VALIDATION, self.provider, doc.kind) # Include "definitions" schema['definitions'] = self.definitions diff --git a/docs/home/howto/BACKUP.md b/docs/home/howto/BACKUP.md index bc601be64f..2731a9b103 100644 --- a/docs/home/howto/BACKUP.md +++ b/docs/home/howto/BACKUP.md @@ -17,17 +17,18 @@ filesystem. See [How to store backup](#2-how-to-store-backup) chapter. ## 1. How to perform backup -#### Backup configuration file and command +### Backup configuration file and command Copy default configuration for backup from ``defaults/configuration/backup.yml`` into newly created backup.yml config -file, and enable backup for chosen components by setting up ``enabled`` parameter to ``true``. +file, supply correct provider and enable backup for chosen components by setting up ``enabled`` parameter to ``true``. This config may also be attached to cluster-config.yml -``` +```yaml kind: configuration/backup title: Backup Config name: default +provider: azure specification: components: load_balancer: @@ -48,7 +49,7 @@ specification: Run ``epicli backup`` command: -``` +```shell epicli backup -f backup.yml -b build_folder ``` @@ -80,15 +81,16 @@ machine's disk drive. This is not recommended. ### Recovery configuration file and command Copy existing default configuration from ``defaults/configuration/recovery.yml`` into newly created recovery.yml config -file, and set ``enabled`` parameter for component to recovery. It's possible to choose snapshot name by passing date and -time part of snapshot name. If snapshot name is not provided, the latest one will be restored. +file, supply correct provider and set ``enabled`` parameter for component to recovery. It's possible to choose snapshot +name by passing date and time part of snapshot name. If snapshot name is not provided, the latest one will be restored. This config may also be attached to cluster-config.yml -``` +```yaml kind: configuration/recovery title: Recovery Config name: default +provider: azure specification: components: load_balancer: @@ -110,7 +112,9 @@ specification: Run ``epicli recovery`` command: -``epicli recovery -f recovery.yml -b build_folder`` +```shell +epicli recovery -f recovery.yml -b build_folder +``` If recovery config is attached to cluster-config.yml, use this file instead of ``recovery.yml``. From 6502598380a00fc7ed82d851a28cec170705df5c Mon Sep 17 00:00:00 2001 From: cicharka Date: Wed, 9 Feb 2022 13:52:11 +0100 Subject: [PATCH 2/7] * Changelog updated --- docs/changelogs/CHANGELOG-2.0.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/docs/changelogs/CHANGELOG-2.0.md b/docs/changelogs/CHANGELOG-2.0.md index cb86d15c4b..319d803baa 100644 --- a/docs/changelogs/CHANGELOG-2.0.md +++ b/docs/changelogs/CHANGELOG-2.0.md @@ -23,8 +23,13 @@ - [#2945](https://github.com/epiphany-platform/epiphany/issues/2945) - epicli apply sleeps 10 seconds after creating inventory - [#2968](https://github.com/epiphany-platform/epiphany/issues/2968) - `epicli init` should generate `specification.cloud.subscription_name` for minimal cluster config - [#2940](https://github.com/epiphany-platform/epiphany/issues/2940) - firewalld.service unit could not be found on host however ansible_facts sees it as defined +<<<<<<< HEAD - [#2979](https://github.com/epiphany-platform/epiphany/issues/2979) - Restore the possibility of choosing the availability zone in AWS - [#2984](https://github.com/epiphany-platform/epiphany/issues/2984) - Validation blocks overwriting of destination_address_prefix in NSG rules, which is 0.0.0.0/0 by default +======= +- [#2942](https://github.com/epiphany-platform/epiphany/issues/2942) - rsync command fails trying to copy artifacts +- [#2930](https://github.com/epiphany-platform/epiphany/issues/2930) - Backup/recovery commands fail when default configuration for backup attached to cluster-config.yml +>>>>>>> 71b9c4f1 (* Changelog updated) ### Updated From 959244960b7b42bdf618bc1d3d9e2a82b6b2e9b4 Mon Sep 17 00:00:00 2001 From: cicharka Date: Mon, 14 Feb 2022 09:34:10 +0100 Subject: [PATCH 3/7] * remove legacy code --- .../roles/backup/tasks/common/download_via_rsync.yml | 8 -------- .../roles/recovery/tasks/common/upload_via_rsync.yml | 8 -------- 2 files changed, 16 deletions(-) diff --git a/ansible/playbooks/roles/backup/tasks/common/download_via_rsync.yml b/ansible/playbooks/roles/backup/tasks/common/download_via_rsync.yml index bfe75fffba..ee0aec3576 100644 --- a/ansible/playbooks/roles/backup/tasks/common/download_via_rsync.yml +++ b/ansible/playbooks/roles/backup/tasks/common/download_via_rsync.yml @@ -77,14 +77,6 @@ checksum: true private_key: "{{ private_key_file.path }}" loop: "{{ artifacts }}" - # Legacy code used in eariler version of Ansible - # rsync_opts: - # - --rsh={{ rsh }} - # vars: - # # this fixes / replaces incorrect path to the private key file that synchronize provides - # # (setting private_key parameter has no effect whatsoever, looks like a bug tbh) - # rsh: >- - # /usr/bin/ssh -S none -i {{ private_key_file.path }} -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null - name: Remove copied artifacts from source file: diff --git a/ansible/playbooks/roles/recovery/tasks/common/upload_via_rsync.yml b/ansible/playbooks/roles/recovery/tasks/common/upload_via_rsync.yml index 1b46eae77e..2ec9a38ec6 100644 --- a/ansible/playbooks/roles/recovery/tasks/common/upload_via_rsync.yml +++ b/ansible/playbooks/roles/recovery/tasks/common/upload_via_rsync.yml @@ -72,11 +72,3 @@ checksum: true private_key: "{{ private_key_file.path }}" loop: "{{ artifacts }}" - # Legacy code used in earlier version of Ansible - # rsync_opts: - # - --rsh={{ rsh }} - # vars: - # # this fixes / replaces incorrect path to the private key file that synchronize provides - # # (setting private_key parameter has no effect whatsoever, looks like a bug tbh) - # rsh: >- - # /usr/bin/ssh -S none -i {{ private_key_file.path }} -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null From c68dd59b0ec3001c6860398d00a240064c651ec6 Mon Sep 17 00:00:00 2001 From: cicharka Date: Wed, 23 Feb 2022 10:17:43 +0100 Subject: [PATCH 4/7] Additional check to verify if backup/recovery present in input docs --- cli/src/commands/BackupRecoveryBase.py | 9 ++++++--- cli/src/schema/SchemaValidator.py | 6 +----- 2 files changed, 7 insertions(+), 8 deletions(-) diff --git a/cli/src/commands/BackupRecoveryBase.py b/cli/src/commands/BackupRecoveryBase.py index d7f7e3f0e8..e5fff5b9c4 100644 --- a/cli/src/commands/BackupRecoveryBase.py +++ b/cli/src/commands/BackupRecoveryBase.py @@ -10,7 +10,7 @@ from cli.src.helpers.data_loader import load_schema_obj, load_yamls_file from cli.src.helpers.data_loader import types as data_types from cli.src.helpers.doc_list_helpers import (ExpectedSingleResultException, - select_single) + select_single, select_all) from cli.src.helpers.yaml_helpers import dump from cli.src.schema.DefaultMerger import DefaultMerger from cli.src.schema.SchemaValidator import SchemaValidator @@ -48,8 +48,11 @@ def _process_input_docs(self): self.manifest_docs = load_manifest(self.build_directory) self.cluster_model = select_single(self.manifest_docs, lambda x: x.kind == 'epiphany-cluster') - # Load backup / recovery configuration documents - self.input_docs = load_yamls_file(self.file) + # Load only backup / recovery configuration documents + loaded_docs = load_yamls_file(self.file) + self.input_docs = select_all(loaded_docs, lambda x: x.kind == ('configuration/backup','configuration/recovery')) + if self.input_docs < 1: + raise Exception('No documents for backup or recovery in input file.') # Validate input documents with SchemaValidator(self.cluster_model.provider, self.input_docs) as schema_validator: diff --git a/cli/src/schema/SchemaValidator.py b/cli/src/schema/SchemaValidator.py index a8f6ea6a18..c2a93ef808 100644 --- a/cli/src/schema/SchemaValidator.py +++ b/cli/src/schema/SchemaValidator.py @@ -47,11 +47,7 @@ def validate_document(self, doc, schema): def run_for_individual_documents(self): for doc in self.validation_docs: # Load document schema - if 'backup' in doc.kind or 'recovery' in doc.kind: - schema = load_schema_obj(types.VALIDATION, self.provider, doc.kind) - else: - schema = self.get_base_schema(doc.kind) - schema['properties']['specification'] = load_schema_obj(types.VALIDATION, self.provider, doc.kind) + schema = load_schema_obj(types.VALIDATION, self.provider, doc.kind) # Include "definitions" schema['definitions'] = self.definitions From 72a1f228dbf972e2f790464126274016c318a934 Mon Sep 17 00:00:00 2001 From: cicharka Date: Wed, 23 Feb 2022 10:19:23 +0100 Subject: [PATCH 5/7] docs: remove info about backup/recovery in data.yml --- docs/home/howto/BACKUP.md | 8 -------- 1 file changed, 8 deletions(-) diff --git a/docs/home/howto/BACKUP.md b/docs/home/howto/BACKUP.md index 2731a9b103..45ee9378dc 100644 --- a/docs/home/howto/BACKUP.md +++ b/docs/home/howto/BACKUP.md @@ -22,8 +22,6 @@ filesystem. See [How to store backup](#2-how-to-store-backup) chapter. Copy default configuration for backup from ``defaults/configuration/backup.yml`` into newly created backup.yml config file, supply correct provider and enable backup for chosen components by setting up ``enabled`` parameter to ``true``. -This config may also be attached to cluster-config.yml - ```yaml kind: configuration/backup title: Backup Config @@ -53,8 +51,6 @@ Run ``epicli backup`` command: epicli backup -f backup.yml -b build_folder ``` -If backup config is attached to cluster-config.yml, use this file instead of ``backup.yml``. - ## 2. How to store backup Backup location is defined in ``backup`` role as ``backup_destination_host`` and ``backup_destination_dir``. Default @@ -84,8 +80,6 @@ Copy existing default configuration from ``defaults/configuration/recovery.yml`` file, supply correct provider and set ``enabled`` parameter for component to recovery. It's possible to choose snapshot name by passing date and time part of snapshot name. If snapshot name is not provided, the latest one will be restored. -This config may also be attached to cluster-config.yml - ```yaml kind: configuration/recovery title: Recovery Config @@ -116,8 +110,6 @@ Run ``epicli recovery`` command: epicli recovery -f recovery.yml -b build_folder ``` -If recovery config is attached to cluster-config.yml, use this file instead of ``recovery.yml``. - ## 4. How backup and recovery work ### Load Balancer From 1e1007ffb863a52de550d150c46348787cc5a425 Mon Sep 17 00:00:00 2001 From: cicharka Date: Wed, 23 Feb 2022 10:22:13 +0100 Subject: [PATCH 6/7] Changelog update --- docs/changelogs/CHANGELOG-2.0.md | 3 --- 1 file changed, 3 deletions(-) diff --git a/docs/changelogs/CHANGELOG-2.0.md b/docs/changelogs/CHANGELOG-2.0.md index 319d803baa..a09e254e8d 100644 --- a/docs/changelogs/CHANGELOG-2.0.md +++ b/docs/changelogs/CHANGELOG-2.0.md @@ -23,13 +23,10 @@ - [#2945](https://github.com/epiphany-platform/epiphany/issues/2945) - epicli apply sleeps 10 seconds after creating inventory - [#2968](https://github.com/epiphany-platform/epiphany/issues/2968) - `epicli init` should generate `specification.cloud.subscription_name` for minimal cluster config - [#2940](https://github.com/epiphany-platform/epiphany/issues/2940) - firewalld.service unit could not be found on host however ansible_facts sees it as defined -<<<<<<< HEAD - [#2979](https://github.com/epiphany-platform/epiphany/issues/2979) - Restore the possibility of choosing the availability zone in AWS - [#2984](https://github.com/epiphany-platform/epiphany/issues/2984) - Validation blocks overwriting of destination_address_prefix in NSG rules, which is 0.0.0.0/0 by default -======= - [#2942](https://github.com/epiphany-platform/epiphany/issues/2942) - rsync command fails trying to copy artifacts - [#2930](https://github.com/epiphany-platform/epiphany/issues/2930) - Backup/recovery commands fail when default configuration for backup attached to cluster-config.yml ->>>>>>> 71b9c4f1 (* Changelog updated) ### Updated From cecbc2812ea487103a62598fa213f6c023df8c59 Mon Sep 17 00:00:00 2001 From: cicharka Date: Wed, 23 Feb 2022 11:23:15 +0100 Subject: [PATCH 7/7] Fix for input_docs len comparison --- cli/src/commands/BackupRecoveryBase.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cli/src/commands/BackupRecoveryBase.py b/cli/src/commands/BackupRecoveryBase.py index e5fff5b9c4..34484b0f49 100644 --- a/cli/src/commands/BackupRecoveryBase.py +++ b/cli/src/commands/BackupRecoveryBase.py @@ -50,8 +50,8 @@ def _process_input_docs(self): # Load only backup / recovery configuration documents loaded_docs = load_yamls_file(self.file) - self.input_docs = select_all(loaded_docs, lambda x: x.kind == ('configuration/backup','configuration/recovery')) - if self.input_docs < 1: + self.input_docs = select_all(loaded_docs, lambda x: x.kind in ['configuration/backup', 'configuration/recovery']) + if len(self.input_docs) < 1: raise Exception('No documents for backup or recovery in input file.') # Validate input documents