From a026b76342c895203ee1666018ff52dea5561f61 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Lipovsk=C3=BD?= Date: Mon, 8 Apr 2024 13:20:19 +0200 Subject: [PATCH] Adding support of multiple operators in one fragment [CLOUDDST-22242] --- iib/workers/tasks/fbc_utils.py | 14 ++- iib/workers/tasks/opm_operations.py | 90 ++++++++++--------- .../test_workers/test_tasks/test_fbc_utils.py | 5 +- .../test_tasks/test_opm_operations.py | 29 +++--- 4 files changed, 69 insertions(+), 69 deletions(-) diff --git a/iib/workers/tasks/fbc_utils.py b/iib/workers/tasks/fbc_utils.py index c85cc389..2767502e 100644 --- a/iib/workers/tasks/fbc_utils.py +++ b/iib/workers/tasks/fbc_utils.py @@ -4,7 +4,7 @@ import logging import shutil -from typing import Tuple +from typing import Tuple, List from iib.exceptions import IIBError from iib.workers.config import get_worker_config from iib.common.tracing import instrument_tracing @@ -91,13 +91,13 @@ def merge_catalogs_dirs(src_config: str, dest_config: str): shutil.copytree(src_config, dest_config, dirs_exist_ok=True) -def extract_fbc_fragment(temp_dir: str, fbc_fragment: str) -> Tuple[str, str]: +def extract_fbc_fragment(temp_dir: str, fbc_fragment: str) -> Tuple[str, List[str]]: """ - Extract operator package from the fbc_fragment image. + Extract operator packages from the fbc_fragment image. :param str temp_dir: base temp directory for IIB request. :param str fbc_fragment: pull specification of fbc_fragment in the IIB request. - :return: fbc_fragment path, fbc_operator_package. + :return: fbc_fragment path, fbc_operator_packages. :rtype: tuple """ from iib.workers.tasks.build import _copy_files_from_image @@ -111,10 +111,8 @@ def extract_fbc_fragment(temp_dir: str, fbc_fragment: str) -> Tuple[str, str]: log.info("fbc_fragment extracted at %s", fbc_fragment_path) operator_packages = os.listdir(fbc_fragment_path) - log.info("fbc_fragment contains package %s", operator_packages) + log.info("fbc_fragment contains packages %s", operator_packages) if not operator_packages: raise IIBError("No operator packages in fbc_fragment %s", fbc_fragment) - if len(operator_packages) > 1: - raise IIBError("More than 1 package is present in fbc_fragment %s", fbc_fragment) - return fbc_fragment_path, operator_packages[0] + return fbc_fragment_path, operator_packages diff --git a/iib/workers/tasks/opm_operations.py b/iib/workers/tasks/opm_operations.py index b0eb234a..a400de02 100644 --- a/iib/workers/tasks/opm_operations.py +++ b/iib/workers/tasks/opm_operations.py @@ -819,57 +819,56 @@ def opm_registry_add_fbc_fragment( """ set_request_state(request_id, 'in_progress', 'Extracting operator package from fbc_fragment') # fragment path will look like /tmp/iib-**/fbc-fragment - fragment_path, fragment_operator = extract_fbc_fragment( + fragment_path, fragment_operators = extract_fbc_fragment( temp_dir=temp_dir, fbc_fragment=fbc_fragment ) - is_operator_in_db, index_db_path = verify_operator_exists( + # the dir where all the configs from from_index are stored + # this will look like /tmp/iib-**/configs + from_index_configs_dir = get_catalog_dir(from_index=from_index, base_dir=temp_dir) + log.info("The content of from_index configs located at %s", from_index_configs_dir) + + operators_in_db, index_db_path = verify_operators_exists( from_index=from_index, base_dir=temp_dir, - operator_package=fragment_operator, + operator_packages=fragment_operators, overwrite_from_index_token=overwrite_from_index_token, ) - # the dir where all the configs from from_index is stored - # this will look like /tmp/iib-**/configs - from_index_configs_dir = get_catalog_dir(from_index=from_index, base_dir=temp_dir) - - log.info("The content of from_index configs located at %s", from_index_configs_dir) + if operators_in_db: + log.info('Removing %s from %s index.db ', operators_in_db, from_index) + _opm_registry_rm(index_db_path=index_db_path, operators=operators_in_db, base_dir=temp_dir) - if is_operator_in_db: - log.info('Removing %s from %s index.db ', fragment_operator, from_index) - _opm_registry_rm( - index_db_path=index_db_path, operators=[fragment_operator], base_dir=temp_dir - ) - # migated_catalog_dir path will look like /tmp/iib-**/catalog - migated_catalog_dir, _ = opm_migrate( + # migrated_catalog_dir path will look like /tmp/iib-**/catalog + migrated_catalog_dir, _ = opm_migrate( index_db=index_db_path, base_dir=temp_dir, generate_cache=False, ) - log.info("Migated catalog after removing from db at %s", migated_catalog_dir) + log.info("Migrated catalog after removing from db at %s", migrated_catalog_dir) # copy the content of migrated_catalog to from_index's config - log.info("Copying content of %s to %s", migated_catalog_dir, from_index_configs_dir) - for operator_package in os.listdir(migated_catalog_dir): + log.info("Copying content of %s to %s", migrated_catalog_dir, from_index_configs_dir) + for operator_package in os.listdir(migrated_catalog_dir): shutil.copytree( - os.path.join(migated_catalog_dir, operator_package), + os.path.join(migrated_catalog_dir, operator_package), os.path.join(from_index_configs_dir, operator_package), dirs_exist_ok=True, ) - # copy fragment_operator to from_index configs - set_request_state(request_id, 'in_progress', 'Adding fbc_fragment to from_index') - fragment_opr_src_path = os.path.join(fragment_path, fragment_operator) - fragment_opr_dest_path = os.path.join(from_index_configs_dir, fragment_operator) - if os.path.exists(fragment_opr_dest_path): - shutil.rmtree(fragment_opr_dest_path) - log.info( - "Copying content of %s to %s", - fragment_opr_src_path, - fragment_opr_dest_path, - ) - shutil.copytree(fragment_opr_src_path, fragment_opr_dest_path) + for fragment_operator in fragment_operators: + # copy fragment_operator to from_index configs + set_request_state(request_id, 'in_progress', 'Adding fbc_fragment to from_index') + fragment_opr_src_path = os.path.join(fragment_path, fragment_operator) + fragment_opr_dest_path = os.path.join(from_index_configs_dir, fragment_operator) + if os.path.exists(fragment_opr_dest_path): + shutil.rmtree(fragment_opr_dest_path) + log.info( + "Copying content of %s to %s", + fragment_opr_src_path, + fragment_opr_dest_path, + ) + shutil.copytree(fragment_opr_src_path, fragment_opr_dest_path) local_cache_path = os.path.join(temp_dir, 'cache') generate_cache_locally( @@ -886,32 +885,32 @@ def opm_registry_add_fbc_fragment( ) -def verify_operator_exists( +def verify_operators_exists( from_index: str, base_dir: str, - operator_package: str, + operator_packages: List[str], overwrite_from_index_token: Optional[str], ): """ - Check if operator exists in index image. + Check if operators exists in index image. :param str from_index: index in which operator existence is checked :param str base_dir: base temp directory for IIB request - :param str operator_package: operator_package to check + :param list(str) operator_packages: operator_package to check :param str overwrite_from_index_token: token used to access the image - :return: is_package_in_index, index_db_path - :rtype: (str, str) + :return: packages_in_index, index_db_path + :rtype: (list, str) """ from iib.workers.tasks.build import terminate_process, get_bundle_json from iib.workers.tasks.iib_static_types import BundleImage from iib.workers.tasks.utils import run_cmd from iib.workers.tasks.utils import set_registry_token - is_package_in_index = False + packages_in_index: List[str] = [] - log.info("Verifying if operator package %s exists in index %s", operator_package, from_index) + log.info("Verifying if operator packages %s exists in index %s", operator_packages, from_index) - # check if operater package exists in hidden index.db + # check if operator packages exists in hidden index.db with set_registry_token(overwrite_from_index_token, from_index, append=True): index_db_path = get_hidden_index_database(from_index=from_index, base_dir=base_dir) @@ -924,10 +923,13 @@ def verify_operator_exists( present_bundles: List[BundleImage] = get_bundle_json(bundles) for bundle in present_bundles: - if bundle['packageName'] == operator_package: - is_package_in_index = True - log.info("operator package %s found in index_db %s", operator_package, index_db_path) - return is_package_in_index, index_db_path + if bundle['packageName'] in operator_packages: + packages_in_index.append(bundle['packageName']) + + if packages_in_index: + log.info("operator packages found in index_db %s: %s", index_db_path, packages_in_index) + + return packages_in_index, index_db_path @retry( diff --git a/tests/test_workers/test_tasks/test_fbc_utils.py b/tests/test_workers/test_tasks/test_fbc_utils.py index 9337b1c6..dbc056a3 100644 --- a/tests/test_workers/test_tasks/test_fbc_utils.py +++ b/tests/test_workers/test_tasks/test_fbc_utils.py @@ -144,15 +144,14 @@ def test_merge_catalogs_dirs_raise(mock_isdir, mock_cpt, tmpdir): @pytest.mark.parametrize('ldr_output', [['testoperator'], ['test1', 'test2'], []]) @mock.patch('os.listdir') @mock.patch('iib.workers.tasks.build._copy_files_from_image') -def test_extract_fbc_fragment_one_operator(mock_cffi, mock_osldr, ldr_output, tmpdir): +def test_extract_fbc_fragment(mock_cffi, mock_osldr, ldr_output, tmpdir): test_fbc_fragment = "example.com/test/fbc_fragment:latest" mock_osldr.return_value = ldr_output fbc_fragment_path = os.path.join(tmpdir, get_worker_config()['temp_fbc_fragment_path']) - if len(ldr_output) != 1: + if not ldr_output: with pytest.raises(IIBError): extract_fbc_fragment(tmpdir, test_fbc_fragment) - else: extract_fbc_fragment(tmpdir, test_fbc_fragment) mock_cffi.assert_has_calls([mock.call(test_fbc_fragment, '/configs', fbc_fragment_path)]) diff --git a/tests/test_workers/test_tasks/test_opm_operations.py b/tests/test_workers/test_tasks/test_opm_operations.py index bca4b249..a2f0e660 100644 --- a/tests/test_workers/test_tasks/test_opm_operations.py +++ b/tests/test_workers/test_tasks/test_opm_operations.py @@ -648,7 +648,8 @@ def test_verify_cache_insertion_edit_dockerfile_failed(): @pytest.mark.parametrize( - 'is_operator_exists, index_db_path', [(True, "index_path"), (False, "index_path")] + 'operators_exists, index_db_path', + [(['test-operator'], "index_path"), ([], "index_path")], ) @mock.patch('iib.workers.tasks.opm_operations.opm_generate_dockerfile') @mock.patch('iib.workers.tasks.opm_operations.generate_cache_locally') @@ -658,7 +659,7 @@ def test_verify_cache_insertion_edit_dockerfile_failed(): @mock.patch('iib.workers.tasks.opm_operations.opm_migrate') @mock.patch('iib.workers.tasks.opm_operations._opm_registry_rm') @mock.patch('iib.workers.tasks.opm_operations.get_catalog_dir') -@mock.patch('iib.workers.tasks.opm_operations.verify_operator_exists') +@mock.patch('iib.workers.tasks.opm_operations.verify_operators_exists') @mock.patch('iib.workers.tasks.opm_operations.extract_fbc_fragment') @mock.patch('iib.workers.tasks.opm_operations.set_request_state') def test_opm_registry_add_fbc_fragment( @@ -673,16 +674,16 @@ def test_opm_registry_add_fbc_fragment( mock_rmt, mock_gcc, mock_ogd, - is_operator_exists, + operators_exists, index_db_path, tmpdir, ): from_index = "example.com/test/index" binary_image = "example.com/ose/binary" fbc_fragment = "example.com/test/fragment" - fbc_fragment_operator = "test-operator" - mock_eff.return_value = (os.path.join(tmpdir, "fbc_fragment"), fbc_fragment_operator) - mock_voe.return_value = is_operator_exists, index_db_path + fbc_fragment_operators = ["test-operator"] + mock_eff.return_value = (os.path.join(tmpdir, "fbc_fragment"), fbc_fragment_operators) + mock_voe.return_value = operators_exists, index_db_path mock_gcr.return_value = os.path.join(tmpdir, "configs") mock_om.return_value = os.path.join(tmpdir, "catalog"), None mock_ldr.return_value = [ @@ -695,13 +696,13 @@ def test_opm_registry_add_fbc_fragment( mock_voe.assert_called_with( from_index=from_index, base_dir=tmpdir, - operator_package=fbc_fragment_operator, + operator_packages=fbc_fragment_operators, overwrite_from_index_token=None, ) mock_gcr.assert_called_with(from_index=from_index, base_dir=tmpdir) - if is_operator_exists: + if operators_exists: mock_orr.assert_called_with( - index_db_path=index_db_path, operators=[fbc_fragment_operator], base_dir=tmpdir + index_db_path=index_db_path, operators=fbc_fragment_operators, base_dir=tmpdir ) mock_om.assert_called_with(index_db=index_db_path, base_dir=tmpdir, generate_cache=False) mock_cpt.assert_has_calls( @@ -721,8 +722,8 @@ def test_opm_registry_add_fbc_fragment( mock_cpt.assert_has_calls( [ mock.call( - os.path.join(tmpdir, "fbc_fragment", fbc_fragment_operator), - os.path.join(tmpdir, "configs", fbc_fragment_operator), + os.path.join(tmpdir, "fbc_fragment", fbc_fragment_operators[0]), + os.path.join(tmpdir, "configs", fbc_fragment_operators[0]), ) ] ) @@ -736,12 +737,12 @@ def test_opm_registry_add_fbc_fragment( ( '{"packageName": "test-operator", "version": "v1.0", "bundlePath":"bundle1"\n}' '\n{\n"packageName": "package2", "version": "v2.0", "bundlePath":"bundle2"}', - True, + ["test-operator"], ), ( '{"packageName": "package1", "version": "v1.0", "bundlePath":"bundle1"\n}' '\n{\n"packageName": "package2", "version": "v2.0", "bundlePath":"bundle2"}', - False, + [], ), ], ) @@ -757,7 +758,7 @@ def test_verify_operator_exists( mock_ors.return_value = 500, mock.MagicMock() mock_rc.return_value = bundles_in_db index_db_path = os.path.join(tmpdir, get_worker_config()['temp_index_db_path']) - package_exists, index_db_path = opm_operations.verify_operator_exists( + package_exists, index_db_path = opm_operations.verify_operators_exists( from_index, tmpdir, 'test-operator', None ) mock_ors.assert_has_calls([mock.call(db_path=index_db_path)])