diff --git a/golem/apps/__init__.py b/golem/apps/__init__.py index 5097c6464c..879e354a34 100644 --- a/golem/apps/__init__.py +++ b/golem/apps/__init__.py @@ -1,11 +1,12 @@ import hashlib import logging from pathlib import Path -from typing import Dict, Any, Iterator, Type +from typing import Dict, Any, Iterator, Type, Tuple from dataclasses import dataclass, field from dataclasses_json import dataclass_json, config from marshmallow import fields as mm_fields +from pathvalidate import sanitize_filename from golem.marketplace import ( RequestorMarketStrategy, @@ -81,11 +82,17 @@ def load_app_from_json_file(json_file: Path) -> AppDefinition: raise ValueError(msg) -def load_apps_from_dir(app_dir: Path) -> Iterator[AppDefinition]: +def load_apps_from_dir(app_dir: Path) -> Iterator[Tuple[Path, AppDefinition]]: """ Read every file in the given directory and attempt to parse it. Ignore files which don't contain valid app definitions. """ for json_file in app_dir.iterdir(): try: - yield load_app_from_json_file(json_file) + yield (json_file, load_app_from_json_file(json_file)) except ValueError: continue + + +def app_json_file_name(app_def: AppDefinition) -> str: + filename = f"{app_def.name}_{app_def.version}_{app_def.id}.json" + filename = sanitize_filename(filename, replacement_text="_") + return filename diff --git a/golem/apps/default.py b/golem/apps/default.py index 9c5f90e25a..5a8874b81f 100644 --- a/golem/apps/default.py +++ b/golem/apps/default.py @@ -2,9 +2,10 @@ from typing import List from golem_task_api.envs import DOCKER_CPU_ENV_ID -from pathvalidate import sanitize_filename -from golem.apps import AppId, AppDefinition, save_app_to_json_file +from golem.apps import ( + AppId, AppDefinition, save_app_to_json_file, app_json_file_name, +) from golem.marketplace import RequestorBrassMarketStrategy BlenderAppDefinition = AppDefinition( @@ -31,16 +32,13 @@ def save_built_in_app_definitions(path: Path) -> List[AppId]: - app_ids = [] - + new_app_ids = [] for app_id, app in APPS.items(): - - filename = f"{app.name}_{app.version}_{app_id}.json" - filename = sanitize_filename(filename, replacement_text="_") + filename = app_json_file_name(app) json_file = path / filename if not json_file.exists(): save_app_to_json_file(app, json_file) - app_ids.append(app_id) + new_app_ids.append(app_id) - return app_ids + return new_app_ids diff --git a/golem/apps/manager.py b/golem/apps/manager.py index 3b5bb1129e..8d0cea6840 100644 --- a/golem/apps/manager.py +++ b/golem/apps/manager.py @@ -1,7 +1,9 @@ import logging from typing import Dict, List, Tuple +from pathlib import Path -from golem.apps import AppId, AppDefinition +from golem.apps import AppId, AppDefinition, load_apps_from_dir +from golem.apps.default import save_built_in_app_definitions from golem.model import AppConfiguration logger = logging.getLogger(__name__) @@ -10,9 +12,23 @@ class AppManager: """ Manager class for applications using Task API. """ - def __init__(self) -> None: + def __init__(self, app_dir: Path, save_apps=True) -> None: self._apps: Dict[AppId, AppDefinition] = {} self._state = AppStates() + self._app_file_names: Dict[AppId, Path] = dict() + + # Save build in apps, then load apps from path + new_apps: List[AppId] = [] + if save_apps: + new_apps = save_built_in_app_definitions(app_dir) + for app_def_path, app_def in load_apps_from_dir(app_dir): + self.register_app(app_def) + self._app_file_names[app_def.id] = app_def_path + for app_id in new_apps: + self.set_enabled(app_id, True) + + def registered(self, app_id) -> bool: + return app_id in self._apps def register_app(self, app: AppDefinition) -> None: """ Register an application in the manager. """ @@ -58,26 +74,33 @@ def app(self, app_id: AppId) -> AppDefinition: """ Get an app with given ID (assuming it is registered). """ return self._apps[app_id] + def delete(self, app_id: AppId) -> bool: + # Delete self._state from the database first + del self._state[app_id] + del self._apps[app_id] + self._app_file_names[app_id].unlink() + return True + class AppStates: - def __contains__(self, item): - if not isinstance(item, str): - self._raise_no_str_type(item) + def __contains__(self, key): + if not isinstance(key, str): + self._raise_no_str_type(key) return AppConfiguration.select(AppConfiguration.app_id) \ - .where(AppConfiguration.app_id == item) \ + .where(AppConfiguration.app_id == key) \ .exists() - def __getitem__(self, item): - if not isinstance(item, str): - self._raise_no_str_type(item) + def __getitem__(self, key): + if not isinstance(key, str): + self._raise_no_str_type(key) try: return AppConfiguration \ - .get(AppConfiguration.app_id == item) \ + .get(AppConfiguration.app_id == key) \ .enabled except AppConfiguration.DoesNotExist: - raise KeyError(item) + raise KeyError(key) def __setitem__(self, key, val): if not isinstance(key, str): @@ -87,6 +110,14 @@ def __setitem__(self, key, val): AppConfiguration.insert(app_id=key, enabled=val).upsert().execute() + def __delitem__(self, key): + try: + AppConfiguration.delete() \ + .where(AppConfiguration.app_id == key).execute() + except AppConfiguration.DoesNotExist: + logger.warning('Can not delete app, not found. id=%e', key) + raise KeyError(key) + @staticmethod - def _raise_no_str_type(item): - raise TypeError(f"Key is of type {type(item)}; str expected") + def _raise_no_str_type(key): + raise TypeError(f"Key is of type {type(key)}; str expected") diff --git a/golem/apps/rpc.py b/golem/apps/rpc.py new file mode 100644 index 0000000000..4427a3a2dc --- /dev/null +++ b/golem/apps/rpc.py @@ -0,0 +1,53 @@ +import logging +import typing + +from golem.rpc import utils as rpc_utils + +if typing.TYPE_CHECKING: + # pylint:disable=unused-import, ungrouped-imports + from golem.apps.manager import AppManager + +logger = logging.getLogger(__name__) + + +class ClientAppProvider: + def __init__(self, app_manager: 'AppManager'): + self.app_manager = app_manager + + @rpc_utils.expose('apps.list') + def apps_list(self): + logger.debug('apps.list called from rpc') + result = [] + for app_id, app_def in self.app_manager.apps(): + logger.debug('app_id=%r, app_def=%r', app_id, app_def) + app_result = { + 'id': app_id, + 'name': app_def.name, + 'version': app_def.version, + 'enabled': self.app_manager.enabled(app_id), + } + # TODO: Add full argument for more values + result.append(app_result) + logger.info('Listing apps. count=%r', len(result)) + return result + + @rpc_utils.expose('apps.update') + def apps_update(self, app_id, enabled): + logger.debug( + 'apps.update called from rpc. app_id=%r, enabled=%r', + app_id, + enabled, + ) + if not self.app_manager.registered(app_id): + raise Exception(f"App not found, please check the app_id={app_id}") + self.app_manager.set_enabled(app_id, bool(enabled)) + logger.info('Updated app. app_id=%r, enabled=%r', app_id, enabled) + return "App state updated." + + @rpc_utils.expose('apps.delete') + def apps_delete(self, app_id): + logger.debug('apps.delete called from rpc. app_id=%r', app_id) + if not self.app_manager.delete(app_id): + raise Exception(f"Failed to delete app. app_id={app_id}") + logger.info('Deleted app. app_id=%r', app_id) + return "App deleted with success." diff --git a/golem/client.py b/golem/client.py index ef76e70187..e58ef94023 100644 --- a/golem/client.py +++ b/golem/client.py @@ -265,7 +265,11 @@ def get_wamp_rpc_mapping(self): from golem.network.concent import soft_switch as concent_soft_switch from golem.rpc.api import ethereum_ as api_ethereum from golem.task import rpc as task_rpc + from golem.apps import rpc as apps_rpc task_rpc_provider = task_rpc.ClientProvider(self) + app_rpc_provider = apps_rpc.ClientAppProvider( + self.task_server.app_manager + ) providers = ( self, concent_soft_switch, @@ -276,6 +280,7 @@ def get_wamp_rpc_mapping(self): self.environments_manager, self.transaction_system, task_rpc_provider, + app_rpc_provider, api_ethereum.ETSProvider(self.transaction_system), ) mapping = {} diff --git a/golem/task/taskserver.py b/golem/task/taskserver.py index a6c4a17468..5754b7a2db 100644 --- a/golem/task/taskserver.py +++ b/golem/task/taskserver.py @@ -32,12 +32,10 @@ from twisted.internet.defer import inlineCallbacks, Deferred, \ TimeoutError as DeferredTimeoutError -import golem.apps from apps.appsmanager import AppsManager from apps.core.task.coretask import CoreTask from golem import constants as gconst from golem.apps import manager as app_manager -from golem.apps.default import save_built_in_app_definitions from golem.clientconfigdescriptor import ClientConfigDescriptor from golem.core.common import ( short_node_id, @@ -154,14 +152,7 @@ def __init__( dev_mode=task_api_dev_mode, ) - app_dir = self.get_app_dir() - built_in_apps = save_built_in_app_definitions(app_dir) - - self.app_manager = app_mgr = app_manager.AppManager() - for app_def in golem.apps.load_apps_from_dir(app_dir): - app_mgr.register_app(app_def) - for app_id in built_in_apps: - app_mgr.set_enabled(app_id, True) + self.app_manager = app_manager.AppManager(self.get_app_dir()) self.node = node self.task_archiver = task_archiver @@ -182,7 +173,7 @@ def __init__( ) self.requested_task_manager = RequestedTaskManager( - app_manager=app_mgr, + app_manager=self.app_manager, env_manager=new_env_manager, public_key=self.keys_auth.public_key, root_path=Path(TaskServer.__get_task_manager_root(client.datadir)), diff --git a/tests/golem/apps/test_app_manager.py b/tests/golem/apps/test_app_manager.py index 07db03fc2b..79dd61af26 100644 --- a/tests/golem/apps/test_app_manager.py +++ b/tests/golem/apps/test_app_manager.py @@ -1,3 +1,5 @@ +from mock import Mock + from golem.apps.manager import AppManager from golem.apps import ( AppDefinition, @@ -22,7 +24,9 @@ class AppManagerTestBase(DatabaseFixture): def setUp(self): super().setUp() - self.app_manager = AppManager() + app_path = self.new_path / 'apps' + app_path.mkdir(exist_ok=True) + self.app_manager = AppManager(app_path, False) class TestRegisterApp(AppManagerTestBase): @@ -38,6 +42,15 @@ def test_re_register(self): with self.assertRaises(ValueError): self.app_manager.register_app(APP_DEF) + def test_delete_app(self): + self.app_manager.register_app(APP_DEF) + self.app_manager._app_file_names[APP_ID] = mocked_file = Mock() + mocked_file.unlink = Mock() + self.assertEqual(self.app_manager.apps(), [(APP_ID, APP_DEF)]) + self.app_manager.delete(APP_ID) + self.assertEqual(self.app_manager.apps(), []) + mocked_file.unlink.assert_called_once_with() + class TestSetEnabled(AppManagerTestBase): @@ -109,4 +122,4 @@ def test_register(self): app_file.write_text(APP_DEF.to_json(), encoding='utf-8') bogus_file.write_text('(╯°□°)╯︵ ┻━┻', encoding='utf-8') loaded_apps = list(load_apps_from_dir(self.new_path)) - self.assertEqual(loaded_apps, [APP_DEF]) + self.assertEqual(loaded_apps, [(app_file, APP_DEF)]) diff --git a/tests/golem/apps/test_app_rpc.py b/tests/golem/apps/test_app_rpc.py new file mode 100644 index 0000000000..b9bf3e462f --- /dev/null +++ b/tests/golem/apps/test_app_rpc.py @@ -0,0 +1,81 @@ +import pytest +from mock import Mock + +from golem.apps.rpc import ClientAppProvider +from golem.apps.manager import AppManager +from golem.apps.default import BlenderAppDefinition +from golem.testutils import pytest_database_fixture # noqa pylint: disable=unused-import + + +class TestClientAppProvider: + @pytest.fixture(autouse=True) + def setup_method(self): + self._app_manger = Mock(spec=AppManager) + self._handler = ClientAppProvider(self._app_manger) + + def test_list(self): + # given + mocked_apps = [(BlenderAppDefinition.id, BlenderAppDefinition)] + self._app_manger.apps = Mock( + return_value=mocked_apps + ) + + # when + result = self._handler.apps_list() + + # then + assert len(result) == len(mocked_apps), \ + 'count of result does not match input count' + assert result[0]['id'] == mocked_apps[0][0], \ + 'the first returned app id does not match input' + assert self._app_manger.apps.called_once_with() + + def test_update(self): + # given + app_id = 'a' + enabled = True + + # when + result = self._handler.apps_update(app_id, enabled) + + # then + self._app_manger.registered.called_once_with(app_id) + self._app_manger.set_enabled.called_once_with(app_id, enabled) + assert result == 'App state updated.' + + def test_update_not_registered(self): + # given + app_id = 'a' + enabled = True + self._app_manger.registered.return_value = False + + # when + with pytest.raises(Exception): + self._handler.apps_update(app_id, enabled) + + # then + self._app_manger.registered.called_once_with(app_id) + self._app_manger.set_enabled.assert_not_called() + + def test_delete(self): + # given + app_id = 'a' + + # when + result = self._handler.apps_delete(app_id) + + # then + self._app_manger.delete.called_once_with(app_id) + assert result == 'App deleted with success.' + + def test_delete_failed(self): + # given + app_id = 'a' + self._app_manger.delete.return_value = False + + # when + with pytest.raises(Exception): + self._handler.apps_delete(app_id) + + # then + self._app_manger.delete.called_once_with(app_id) diff --git a/tests/golem/core/test_fileshelper.py b/tests/golem/core/test_fileshelper.py index 4064585bf4..b8a1b0b737 100644 --- a/tests/golem/core/test_fileshelper.py +++ b/tests/golem/core/test_fileshelper.py @@ -210,6 +210,8 @@ def tearDown(self): if os.path.isdir(self.testdir): shutil.rmtree(self.testdir) + super().tearDown() + class TestDu(TestDirFixture): @@ -290,6 +292,7 @@ def tearDown(self): os.rmdir(self.test_dir2) os.rmdir(self.test_dir1) + super().tearDown() def test_find_file_with_ext(self): """ Test find_file_with_ext method """ diff --git a/tests/golem/network/concent/test_received_handler.py b/tests/golem/network/concent/test_received_handler.py index 039b927c5a..4b16fb809a 100644 --- a/tests/golem/network/concent/test_received_handler.py +++ b/tests/golem/network/concent/test_received_handler.py @@ -187,6 +187,7 @@ def tearDown(self): # Remove registered handlers del self.task_server gc.collect() + super().tearDown() class IsOursTest(TaskServerMessageHandlerTestBase): diff --git a/tests/golem/verifier/test_blenderverifier.py b/tests/golem/verifier/test_blenderverifier.py index 44e65b1a4e..1b8bcfd33d 100644 --- a/tests/golem/verifier/test_blenderverifier.py +++ b/tests/golem/verifier/test_blenderverifier.py @@ -29,6 +29,7 @@ logger = logging.getLogger(__name__) + @pytest.mark.slow @pytest.mark.skipif( not is_linux(), @@ -71,6 +72,7 @@ def tearDown(self): # Try again after 3 seconds sleep(3) self.remove_files() + super().tearDown() def remove_files(self): above_tmp_dir = os.path.dirname(self.tempdir)