From f1786b1d5dd52a7c60bad87e405d432ce5a2a195 Mon Sep 17 00:00:00 2001 From: adi88d Date: Sun, 25 Jun 2023 11:31:30 +0300 Subject: [PATCH 1/9] skip core pack installation --- Tests/Marketplace/search_and_uninstall_pack.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/Tests/Marketplace/search_and_uninstall_pack.py b/Tests/Marketplace/search_and_uninstall_pack.py index 34090def17d8..2af3aed57420 100644 --- a/Tests/Marketplace/search_and_uninstall_pack.py +++ b/Tests/Marketplace/search_and_uninstall_pack.py @@ -11,6 +11,7 @@ from Tests.Marketplace.search_and_install_packs import install_packs from time import sleep +UNREMOVABLE_PACKS = ['Base', 'CoreAlertFields', 'Core'] def get_all_installed_packs(client: demisto_client): """ @@ -35,8 +36,9 @@ def get_all_installed_packs(client: demisto_client): installed_packs_ids_str = ', '.join(installed_packs_ids) logging.debug( f'The following packs are currently installed from a previous build run:\n{installed_packs_ids_str}') - if 'Base' in installed_packs_ids: - installed_packs_ids.remove('Base') + for pack in UNREMOVABLE_PACKS: + if pack in installed_packs_ids: + installed_packs_ids.remove('Base') return installed_packs_ids else: result_object = ast.literal_eval(response_data) @@ -153,7 +155,7 @@ def wait_for_uninstallation_to_complete(client: demisto_client): installed_packs_amount_history, failed_uninstall_attempt_count = len(installed_packs), 0 # new calculation for num of retries retries = math.ceil(len(installed_packs) / 2) - while len(installed_packs) > 1: + while len(installed_packs) > len(UNREMOVABLE_PACKS): if retry > retries: raise Exception('Waiting time for packs to be uninstalled has passed, there are still installed ' 'packs. Aborting.') From b3e984392790e4afec06df77c23e59c03cdc38c8 Mon Sep 17 00:00:00 2001 From: adi88d Date: Sun, 25 Jun 2023 12:03:45 +0300 Subject: [PATCH 2/9] update reset_base_pack_version method --- .../Marketplace/search_and_uninstall_pack.py | 78 ++++++++++--------- 1 file changed, 43 insertions(+), 35 deletions(-) diff --git a/Tests/Marketplace/search_and_uninstall_pack.py b/Tests/Marketplace/search_and_uninstall_pack.py index 2af3aed57420..a6f3f75ce3d7 100644 --- a/Tests/Marketplace/search_and_uninstall_pack.py +++ b/Tests/Marketplace/search_and_uninstall_pack.py @@ -38,7 +38,7 @@ def get_all_installed_packs(client: demisto_client): f'The following packs are currently installed from a previous build run:\n{installed_packs_ids_str}') for pack in UNREMOVABLE_PACKS: if pack in installed_packs_ids: - installed_packs_ids.remove('Base') + installed_packs_ids.remove(pack) return installed_packs_ids else: result_object = ast.literal_eval(response_data) @@ -94,9 +94,9 @@ def uninstall_all_packs(client: demisto_client, hostname): return True -def reset_base_pack_version(client: demisto_client): +def reset_core_pack_version(client: demisto_client): """ - Resets base pack version to prod version. + Resets core pack version to prod version. Args: client (demisto_client): The client to connect to. @@ -104,38 +104,46 @@ def reset_base_pack_version(client: demisto_client): """ host = client.api_client.configuration.host.replace('https://api-', 'https://') # disable-secrets-detection - try: - # make the search request - response_data, status_code, _ = demisto_client.generic_request_func(client, - path='/contentpacks/marketplace/Base', - method='GET', - accept='application/json', - _request_timeout=None) - if 200 <= status_code < 300: - result_object = ast.literal_eval(response_data) - - if result_object and result_object.get('currentVersion'): - logging.debug('Found Base pack in bucket!') - - pack_data = { - 'id': result_object.get('id'), - 'version': result_object.get('currentVersion') - } - # install latest version of Base pack - logging.info(f'updating base pack to version {result_object.get("currentVersion")}') - return install_packs(client, host, [pack_data], False) - + is_succeed = True + + for pack_id in UNREMOVABLE_PACKS: + + try: + # make the search request + response_data, status_code, _ = demisto_client.generic_request_func(client, + path=f'/contentpacks/marketplace/{pack_id}', + method='GET', + accept='application/json', + _request_timeout=None) + if 200 <= status_code < 300: + result_object = ast.literal_eval(response_data) + + if result_object and result_object.get('currentVersion'): + logging.debug(f'Found {pack_id} pack in bucket!') + + pack_data = { + 'id': result_object.get('id'), + 'version': result_object.get('currentVersion') + } + # install latest version of the pack + logging.info(f'updating {pack_id} pack to version {result_object.get("currentVersion")}') + installation_status = install_packs(client, host, [pack_data], False) + if not installation_status: + is_succeed = False + + else: + raise Exception(f'Did not find {pack_id} pack') else: - raise Exception('Did not find Base pack') - else: - result_object = ast.literal_eval(response_data) - msg = result_object.get('message', '') - err_msg = f'Search request for base pack, failed with status code ' \ - f'{status_code}\n{msg}' - raise Exception(err_msg) - except Exception: - logging.exception('Search request Base pack has failed.') - return False + result_object = ast.literal_eval(response_data) + msg = result_object.get('message', '') + err_msg = f'Search request for {pack_id} pack, failed with status code ' \ + f'{status_code}\n{msg}' + raise Exception(err_msg) + except Exception: + logging.exception(f'Search request {pack_id} pack has failed.') + is_succeed = False + + return is_succeed def wait_for_uninstallation_to_complete(client: demisto_client): @@ -231,7 +239,7 @@ def main(): # We are syncing marketplace since we are copying production bucket to build bucket and if packs were configured # in earlier builds they will appear in the bucket as it is cached. sync_marketplace(client=client) - success = reset_base_pack_version(client) and uninstall_all_packs(client, + success = reset_core_pack_version(client) and uninstall_all_packs(client, host) and wait_for_uninstallation_to_complete( client) sync_marketplace(client=client) From ac48ccbc59fe926e45aa1b2d20e8c2a74521ecbe Mon Sep 17 00:00:00 2001 From: adi88d Date: Sun, 25 Jun 2023 12:22:19 +0300 Subject: [PATCH 3/9] update unitests --- Tests/Marketplace/Tests/search_and_unistall_pack_test.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Tests/Marketplace/Tests/search_and_unistall_pack_test.py b/Tests/Marketplace/Tests/search_and_unistall_pack_test.py index 8f6b34cda02c..bd31c0b05679 100644 --- a/Tests/Marketplace/Tests/search_and_unistall_pack_test.py +++ b/Tests/Marketplace/Tests/search_and_unistall_pack_test.py @@ -97,12 +97,12 @@ def test_uninstall_all_packs(mocker): assert success is True -def test_reset_base_pack_version(mocker): +def test_reset_core_pack_version(mocker): """ Given - Base pack with different version than production. When - - Updating Base pack to prod version. + - Updating core pack to prod version. Then - Ensure the pack version gets reset. """ @@ -110,14 +110,14 @@ def test_reset_base_pack_version(mocker): client.api_client.configuration.host = 'https://api-someurl.com/' # disable-secrets-detection mocker.patch.object(demisto_client, 'generic_request_func', side_effect=mocked_generic_request_func) mocker.patch('Tests.Marketplace.search_and_uninstall_pack.install_packs', return_value=True) - success = script.reset_base_pack_version(client) + success = script.reset_core_pack_version(client) assert success is True @pytest.mark.parametrize('def_call, return_val', [ (script.get_all_installed_packs, None), - (script.reset_base_pack_version, False) + (script.reset_core_pack_version, False) ]) def test_exception_reset_base_pack(def_call, return_val, mocker): """ From 191abfb2b9ad0ca0b2f9212f0b376d42e6ebe538 Mon Sep 17 00:00:00 2001 From: adi88d Date: Sun, 25 Jun 2023 12:58:31 +0300 Subject: [PATCH 4/9] update mocked_generic_request_func --- Tests/Marketplace/Tests/search_and_unistall_pack_test.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/Tests/Marketplace/Tests/search_and_unistall_pack_test.py b/Tests/Marketplace/Tests/search_and_unistall_pack_test.py index bd31c0b05679..61fecea2e5c1 100644 --- a/Tests/Marketplace/Tests/search_and_unistall_pack_test.py +++ b/Tests/Marketplace/Tests/search_and_unistall_pack_test.py @@ -35,6 +35,10 @@ def mocked_generic_request_func(self, path: str, method, body=None, accept=None, _request_timeout=None): if path == '/contentpacks/marketplace/Base': return MOCK_BASE_SEARCH_RESULTS, 200, None + if path == '/contentpacks/marketplace/Core': + return MOCK_BASE_SEARCH_RESULTS, 200, None + if path == '/contentpacks/marketplace/CoreAlertFields': + return MOCK_BASE_SEARCH_RESULTS, 200, None elif path == '/contentpacks/metadata/installed': return MOCK_PACKS_INSTALLATED_RESULT, 200, None elif path == '/contentpacks/installed/delete': From d019c41c136f41f64525b416002e3c4942c15095 Mon Sep 17 00:00:00 2001 From: adi88d Date: Sun, 25 Jun 2023 13:56:51 +0300 Subject: [PATCH 5/9] fix flake8 --- Tests/Marketplace/search_and_uninstall_pack.py | 1 + 1 file changed, 1 insertion(+) diff --git a/Tests/Marketplace/search_and_uninstall_pack.py b/Tests/Marketplace/search_and_uninstall_pack.py index a6f3f75ce3d7..3c24ddf5b9b4 100644 --- a/Tests/Marketplace/search_and_uninstall_pack.py +++ b/Tests/Marketplace/search_and_uninstall_pack.py @@ -13,6 +13,7 @@ UNREMOVABLE_PACKS = ['Base', 'CoreAlertFields', 'Core'] + def get_all_installed_packs(client: demisto_client): """ From 4d2d394969ad30cfb1413f595d4866e4f7b3295d Mon Sep 17 00:00:00 2001 From: adi88d Date: Sun, 25 Jun 2023 17:18:27 +0300 Subject: [PATCH 6/9] use search_and_install_packs_and_their_dependencies --- .../Tests/search_and_unistall_pack_test.py | 25 +---------- .../Marketplace/search_and_uninstall_pack.py | 45 ++----------------- 2 files changed, 5 insertions(+), 65 deletions(-) diff --git a/Tests/Marketplace/Tests/search_and_unistall_pack_test.py b/Tests/Marketplace/Tests/search_and_unistall_pack_test.py index 61fecea2e5c1..0f41f56bd7a3 100644 --- a/Tests/Marketplace/Tests/search_and_unistall_pack_test.py +++ b/Tests/Marketplace/Tests/search_and_unistall_pack_test.py @@ -35,10 +35,6 @@ def mocked_generic_request_func(self, path: str, method, body=None, accept=None, _request_timeout=None): if path == '/contentpacks/marketplace/Base': return MOCK_BASE_SEARCH_RESULTS, 200, None - if path == '/contentpacks/marketplace/Core': - return MOCK_BASE_SEARCH_RESULTS, 200, None - if path == '/contentpacks/marketplace/CoreAlertFields': - return MOCK_BASE_SEARCH_RESULTS, 200, None elif path == '/contentpacks/metadata/installed': return MOCK_PACKS_INSTALLATED_RESULT, 200, None elif path == '/contentpacks/installed/delete': @@ -101,27 +97,8 @@ def test_uninstall_all_packs(mocker): assert success is True -def test_reset_core_pack_version(mocker): - """ - Given - - Base pack with different version than production. - When - - Updating core pack to prod version. - Then - - Ensure the pack version gets reset. - """ - client = MockClient() - client.api_client.configuration.host = 'https://api-someurl.com/' # disable-secrets-detection - mocker.patch.object(demisto_client, 'generic_request_func', side_effect=mocked_generic_request_func) - mocker.patch('Tests.Marketplace.search_and_uninstall_pack.install_packs', return_value=True) - success = script.reset_core_pack_version(client) - - assert success is True - - @pytest.mark.parametrize('def_call, return_val', [ - (script.get_all_installed_packs, None), - (script.reset_core_pack_version, False) + (script.get_all_installed_packs, None) ]) def test_exception_reset_base_pack(def_call, return_val, mocker): """ diff --git a/Tests/Marketplace/search_and_uninstall_pack.py b/Tests/Marketplace/search_and_uninstall_pack.py index 3c24ddf5b9b4..7d999322306e 100644 --- a/Tests/Marketplace/search_and_uninstall_pack.py +++ b/Tests/Marketplace/search_and_uninstall_pack.py @@ -8,7 +8,7 @@ from Tests.configure_and_test_integration_instances import CloudBuild from Tests.scripts.utils import logging_wrapper as logging from Tests.scripts.utils.log_util import install_logging -from Tests.Marketplace.search_and_install_packs import install_packs +from Tests.Marketplace.configure_and_install_packs import search_and_install_packs_and_their_dependencies from time import sleep UNREMOVABLE_PACKS = ['Base', 'CoreAlertFields', 'Core'] @@ -105,46 +105,9 @@ def reset_core_pack_version(client: demisto_client): """ host = client.api_client.configuration.host.replace('https://api-', 'https://') # disable-secrets-detection - is_succeed = True - - for pack_id in UNREMOVABLE_PACKS: - - try: - # make the search request - response_data, status_code, _ = demisto_client.generic_request_func(client, - path=f'/contentpacks/marketplace/{pack_id}', - method='GET', - accept='application/json', - _request_timeout=None) - if 200 <= status_code < 300: - result_object = ast.literal_eval(response_data) - - if result_object and result_object.get('currentVersion'): - logging.debug(f'Found {pack_id} pack in bucket!') - - pack_data = { - 'id': result_object.get('id'), - 'version': result_object.get('currentVersion') - } - # install latest version of the pack - logging.info(f'updating {pack_id} pack to version {result_object.get("currentVersion")}') - installation_status = install_packs(client, host, [pack_data], False) - if not installation_status: - is_succeed = False - - else: - raise Exception(f'Did not find {pack_id} pack') - else: - result_object = ast.literal_eval(response_data) - msg = result_object.get('message', '') - err_msg = f'Search request for {pack_id} pack, failed with status code ' \ - f'{status_code}\n{msg}' - raise Exception(err_msg) - except Exception: - logging.exception(f'Search request {pack_id} pack has failed.') - is_succeed = False - - return is_succeed + _, success = search_and_install_packs_and_their_dependencies(UNREMOVABLE_PACKS, client, host, True) + + return success def wait_for_uninstallation_to_complete(client: demisto_client): From 4aa1b1d6bc4c1a88a8dc91b258d6e62b0f8a8f3f Mon Sep 17 00:00:00 2001 From: adi88d Date: Sun, 25 Jun 2023 17:24:08 +0300 Subject: [PATCH 7/9] use search_and_install_packs_and_their_dependencies --- Tests/Marketplace/search_and_uninstall_pack.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/Tests/Marketplace/search_and_uninstall_pack.py b/Tests/Marketplace/search_and_uninstall_pack.py index 7d999322306e..8683b7d89195 100644 --- a/Tests/Marketplace/search_and_uninstall_pack.py +++ b/Tests/Marketplace/search_and_uninstall_pack.py @@ -105,8 +105,10 @@ def reset_core_pack_version(client: demisto_client): """ host = client.api_client.configuration.host.replace('https://api-', 'https://') # disable-secrets-detection - _, success = search_and_install_packs_and_their_dependencies(UNREMOVABLE_PACKS, client, host, True) - + _, success = search_and_install_packs_and_their_dependencies(pack_ids=UNREMOVABLE_PACKS, + client=client, + hostname=host, + install_packs_one_by_one=True) return success From 0309c596129a16b762679571ef5ce745e88eaaa1 Mon Sep 17 00:00:00 2001 From: adi88d Date: Mon, 26 Jun 2023 09:31:55 +0300 Subject: [PATCH 8/9] increase Sanity Test - Playbook with Unmockable Whois timeout --- Tests/conf.json | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Tests/conf.json b/Tests/conf.json index 8fa6457a6b47..f6cf6ba2697c 100644 --- a/Tests/conf.json +++ b/Tests/conf.json @@ -715,7 +715,8 @@ { "integrations": "Whois", "playbookID": "Sanity Test - Playbook with Unmockable Whois Integration", - "fromversion": "6.5.0" + "fromversion": "6.5.0", + "timeout": "1000" }, { "integrations": "HelloWorld", From 2b6746aa21a4e2769e0bb147d7a7ebebfdfa9d3a Mon Sep 17 00:00:00 2001 From: adi88d Date: Mon, 26 Jun 2023 11:02:30 +0300 Subject: [PATCH 9/9] revert conf.json --- Tests/conf.json | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/Tests/conf.json b/Tests/conf.json index f6cf6ba2697c..8fa6457a6b47 100644 --- a/Tests/conf.json +++ b/Tests/conf.json @@ -715,8 +715,7 @@ { "integrations": "Whois", "playbookID": "Sanity Test - Playbook with Unmockable Whois Integration", - "fromversion": "6.5.0", - "timeout": "1000" + "fromversion": "6.5.0" }, { "integrations": "HelloWorld",