task-api - Added rpc calls for managing apps

golem/apps/__init__.py

@@ -1,11 +1,12 @@
 import hashlib
 import logging
 from pathlib import Path
-from typing import Dict, Any, Iterator, Type
+from typing import Dict, Any, Iterator, Type, Tuple
 
 from dataclasses import dataclass, field
 from dataclasses_json import dataclass_json, config
 from marshmallow import fields as mm_fields
+from pathvalidate import sanitize_filename
 
 from golem.marketplace import (
     RequestorMarketStrategy,
@@ -81,11 +82,17 @@ def load_app_from_json_file(json_file: Path) -> AppDefinition:
         raise ValueError(msg)
 
 
-def load_apps_from_dir(app_dir: Path) -> Iterator[AppDefinition]:
+def load_apps_from_dir(app_dir: Path) -> Iterator[Tuple[Path, AppDefinition]]:
     """ Read every file in the given directory and attempt to parse it. Ignore
         files which don't contain valid app definitions. """
     for json_file in app_dir.iterdir():
         try:
-            yield load_app_from_json_file(json_file)
+            yield (json_file, load_app_from_json_file(json_file))
         except ValueError:
             continue
+
+
+def app_json_file_name(app_def: AppDefinition) -> str:
+    filename = f"{app_def.name}_{app_def.version}_{app_def.id}.json"
+    filename = sanitize_filename(filename, replacement_text="_")
+    return filename

golem/apps/default.py

@@ -2,9 +2,10 @@
 from typing import List
 
 from golem_task_api.envs import DOCKER_CPU_ENV_ID
-from pathvalidate import sanitize_filename
 
-from golem.apps import AppId, AppDefinition, save_app_to_json_file
+from golem.apps import (
+    AppId, AppDefinition, save_app_to_json_file, app_json_file_name,
+)
 from golem.marketplace import RequestorBrassMarketStrategy
 
 BlenderAppDefinition = AppDefinition(
@@ -31,16 +32,13 @@
 
 
 def save_built_in_app_definitions(path: Path) -> List[AppId]:
-    app_ids = []
-
+    new_app_ids = []
     for app_id, app in APPS.items():
-
-        filename = f"{app.name}_{app.version}_{app_id}.json"
-        filename = sanitize_filename(filename, replacement_text="_")
+        filename = app_json_file_name(app)
         json_file = path / filename
 
         if not json_file.exists():
             save_app_to_json_file(app, json_file)
-            app_ids.append(app_id)
+            new_app_ids.append(app_id)
 
-    return app_ids
+    return new_app_ids

golem/apps/manager.py

@@ -1,7 +1,9 @@
 import logging
 from typing import Dict, List, Tuple
+from pathlib import Path
 
-from golem.apps import AppId, AppDefinition
+from golem.apps import AppId, AppDefinition, load_apps_from_dir
+from golem.apps.default import save_built_in_app_definitions
 from golem.model import AppConfiguration
 
 logger = logging.getLogger(__name__)
@@ -10,9 +12,23 @@
 class AppManager:
     """ Manager class for applications using Task API. """
 
-    def __init__(self) -> None:
+    def __init__(self, app_dir: Path, save_apps=True) -> None:
         self._apps: Dict[AppId, AppDefinition] = {}
         self._state = AppStates()
+        self._app_file_names: Dict[AppId, Path] = dict()
+
+        # Save build in apps, then load apps from path
+        new_apps: List[AppId] = []
+        if save_apps:
+            new_apps = save_built_in_app_definitions(app_dir)
+        for app_def_path, app_def in load_apps_from_dir(app_dir):
+            self.register_app(app_def)
+            self._app_file_names[app_def.id] = app_def_path
+        for app_id in new_apps:
+            self.set_enabled(app_id, True)
+
+    def registered(self, app_id) -> bool:
+        return app_id in self._apps
 
     def register_app(self, app: AppDefinition) -> None:
         """ Register an application in the manager. """
@@ -58,26 +74,33 @@ def app(self, app_id: AppId) -> AppDefinition:
         """ Get an app with given ID (assuming it is registered). """
         return self._apps[app_id]
 
+    def delete(self, app_id: AppId) -> bool:
+        # Delete self._state from the database first
+        del self._state[app_id]
+        del self._apps[app_id]
+        self._app_file_names[app_id].unlink()
+        return True
+
 
 class AppStates:
 
-    def __contains__(self, item):
-        if not isinstance(item, str):
-            self._raise_no_str_type(item)
+    def __contains__(self, key):
+        if not isinstance(key, str):
+            self._raise_no_str_type(key)
 
         return AppConfiguration.select(AppConfiguration.app_id) \
-            .where(AppConfiguration.app_id == item) \
+            .where(AppConfiguration.app_id == key) \
             .exists()
 
-    def __getitem__(self, item):
-        if not isinstance(item, str):
-            self._raise_no_str_type(item)
+    def __getitem__(self, key):
+        if not isinstance(key, str):
+            self._raise_no_str_type(key)
         try:
             return AppConfiguration \
-                .get(AppConfiguration.app_id == item) \
+                .get(AppConfiguration.app_id == key) \
                 .enabled
         except AppConfiguration.DoesNotExist:
-            raise KeyError(item)
+            raise KeyError(key)
 
     def __setitem__(self, key, val):
         if not isinstance(key, str):
@@ -87,6 +110,14 @@ def __setitem__(self, key, val):
 
         AppConfiguration.insert(app_id=key, enabled=val).upsert().execute()
 
+    def __delitem__(self, key):
+        try:
+            AppConfiguration.delete() \
+                .where(AppConfiguration.app_id == key).execute()
+        except AppConfiguration.DoesNotExist:
+            logger.warning('Can not delete app, not found. id=%e', key)
+            raise KeyError(key)
+
     @staticmethod
-    def _raise_no_str_type(item):
-        raise TypeError(f"Key is of type {type(item)}; str expected")
+    def _raise_no_str_type(key):
+        raise TypeError(f"Key is of type {type(key)}; str expected")

golem/apps/rpc.py

@@ -0,0 +1,53 @@
+import logging
+import typing
+
+from golem.rpc import utils as rpc_utils
+
+if typing.TYPE_CHECKING:
+    # pylint:disable=unused-import, ungrouped-imports
+    from golem.apps.manager import AppManager
+
+logger = logging.getLogger(__name__)
+
+
+class ClientAppProvider:
+    def __init__(self, app_manager: 'AppManager'):
+        self.app_manager = app_manager
+
+    @rpc_utils.expose('apps.list')
+    def apps_list(self):
+        logger.debug('apps.list called from rpc')
+        result = []
+        for app_id, app_def in self.app_manager.apps():
+            logger.debug('app_id=%r, app_def=%r', app_id, app_def)
+            app_result = {
+                'id': app_id,
+                'name': app_def.name,
+                'version': app_def.version,
+                'enabled': self.app_manager.enabled(app_id),
+            }
+            # TODO: Add full argument for more values
+            result.append(app_result)
+        logger.info('Listing apps. count=%r', len(result))
+        return result
+
+    @rpc_utils.expose('apps.update')
+    def apps_update(self, app_id, enabled):
+        logger.debug(
+            'apps.update called from rpc. app_id=%r, enabled=%r',
+            app_id,
+            enabled,
+        )
+        if not self.app_manager.registered(app_id):
+            raise Exception(f"App not found, please check the app_id={app_id}")
+        self.app_manager.set_enabled(app_id, bool(enabled))
+        logger.info('Updated app. app_id=%r, enabled=%r', app_id, enabled)
+        return "App state updated."
+
+    @rpc_utils.expose('apps.delete')
+    def apps_delete(self, app_id):
+        logger.debug('apps.delete called from rpc. app_id=%r', app_id)
+        if not self.app_manager.delete(app_id):
+            raise Exception(f"Failed to delete app. app_id={app_id}")
+        logger.info('Deleted app. app_id=%r', app_id)
+        return "App deleted with success."

golem/client.py

@@ -265,7 +265,11 @@ def get_wamp_rpc_mapping(self):
         from golem.network.concent import soft_switch as concent_soft_switch
         from golem.rpc.api import ethereum_ as api_ethereum
         from golem.task import rpc as task_rpc
+        from golem.apps import rpc as apps_rpc
         task_rpc_provider = task_rpc.ClientProvider(self)
+        app_rpc_provider = apps_rpc.ClientAppProvider(
+            self.task_server.app_manager
+        )
         providers = (
             self,
             concent_soft_switch,
@@ -276,6 +280,7 @@ def get_wamp_rpc_mapping(self):
             self.environments_manager,
             self.transaction_system,
             task_rpc_provider,
+            app_rpc_provider,
             api_ethereum.ETSProvider(self.transaction_system),
         )
         mapping = {}

golem/task/taskserver.py

@@ -32,12 +32,10 @@
 from twisted.internet.defer import inlineCallbacks, Deferred, \
     TimeoutError as DeferredTimeoutError
 
-import golem.apps
 from apps.appsmanager import AppsManager
 from apps.core.task.coretask import CoreTask
 from golem import constants as gconst
 from golem.apps import manager as app_manager
-from golem.apps.default import save_built_in_app_definitions
 from golem.clientconfigdescriptor import ClientConfigDescriptor
 from golem.core.common import (
     short_node_id,
@@ -154,14 +152,7 @@ def __init__(
             dev_mode=task_api_dev_mode,
         )
 
-        app_dir = self.get_app_dir()
-        built_in_apps = save_built_in_app_definitions(app_dir)
-
-        self.app_manager = app_mgr = app_manager.AppManager()
-        for app_def in golem.apps.load_apps_from_dir(app_dir):
-            app_mgr.register_app(app_def)
-        for app_id in built_in_apps:
-            app_mgr.set_enabled(app_id, True)
+        self.app_manager = app_manager.AppManager(self.get_app_dir())
 
         self.node = node
         self.task_archiver = task_archiver
@@ -182,7 +173,7 @@ def __init__(
         )
 
         self.requested_task_manager = RequestedTaskManager(
-            app_manager=app_mgr,
+            app_manager=self.app_manager,
             env_manager=new_env_manager,
             public_key=self.keys_auth.public_key,
             root_path=Path(TaskServer.__get_task_manager_root(client.datadir)),

tests/golem/apps/test_app_manager.py

@@ -1,3 +1,5 @@
+from mock import Mock
+
 from golem.apps.manager import AppManager
 from golem.apps import (
     AppDefinition,
@@ -22,7 +24,9 @@ class AppManagerTestBase(DatabaseFixture):
 
     def setUp(self):
         super().setUp()
-        self.app_manager = AppManager()
+        app_path = self.new_path / 'apps'
+        app_path.mkdir(exist_ok=True)
+        self.app_manager = AppManager(app_path, False)
 
 
 class TestRegisterApp(AppManagerTestBase):
@@ -38,6 +42,15 @@ def test_re_register(self):
         with self.assertRaises(ValueError):
             self.app_manager.register_app(APP_DEF)
 
+    def test_delete_app(self):
+        self.app_manager.register_app(APP_DEF)
+        self.app_manager._app_file_names[APP_ID] = mocked_file = Mock()
+        mocked_file.unlink = Mock()
+        self.assertEqual(self.app_manager.apps(), [(APP_ID, APP_DEF)])
+        self.app_manager.delete(APP_ID)
+        self.assertEqual(self.app_manager.apps(), [])
+        mocked_file.unlink.assert_called_once_with()
+
 
 class TestSetEnabled(AppManagerTestBase):
 
@@ -109,4 +122,4 @@ def test_register(self):
         app_file.write_text(APP_DEF.to_json(), encoding='utf-8')
         bogus_file.write_text('(╯°□°）╯︵ ┻━┻', encoding='utf-8')
         loaded_apps = list(load_apps_from_dir(self.new_path))
-        self.assertEqual(loaded_apps, [APP_DEF])
+        self.assertEqual(loaded_apps, [(app_file, APP_DEF)])

tests/golem/apps/test_app_rpc.py

@@ -0,0 +1,81 @@
+import pytest
+from mock import Mock
+
+from golem.apps.rpc import ClientAppProvider
+from golem.apps.manager import AppManager
+from golem.apps.default import BlenderAppDefinition
+from golem.testutils import pytest_database_fixture  # noqa pylint: disable=unused-import
+
+
+class TestClientAppProvider:
+    @pytest.fixture(autouse=True)
+    def setup_method(self):
+        self._app_manger = Mock(spec=AppManager)
+        self._handler = ClientAppProvider(self._app_manger)
+
+    def test_list(self):
+        # given
+        mocked_apps = [(BlenderAppDefinition.id, BlenderAppDefinition)]
+        self._app_manger.apps = Mock(
+            return_value=mocked_apps
+        )
+
+        # when
+        result = self._handler.apps_list()
+
+        # then
+        assert len(result) == len(mocked_apps), \
+            'count of result does not match input count'
+        assert result[0]['id'] == mocked_apps[0][0], \
+            'the first returned app id does not match input'
+        assert self._app_manger.apps.called_once_with()
+
+    def test_update(self):
+        # given
+        app_id = 'a'
+        enabled = True
+
+        # when
+        result = self._handler.apps_update(app_id, enabled)
+
+        # then
+        self._app_manger.registered.called_once_with(app_id)
+        self._app_manger.set_enabled.called_once_with(app_id, enabled)
+        assert result == 'App state updated.'
+
+    def test_update_not_registered(self):
+        # given
+        app_id = 'a'
+        enabled = True
+        self._app_manger.registered.return_value = False
+
+        # when
+        with pytest.raises(Exception):
+            self._handler.apps_update(app_id, enabled)
+
+        # then
+        self._app_manger.registered.called_once_with(app_id)
+        self._app_manger.set_enabled.assert_not_called()
+
+    def test_delete(self):
+        # given
+        app_id = 'a'
+
+        # when
+        result = self._handler.apps_delete(app_id)
+
+        # then
+        self._app_manger.delete.called_once_with(app_id)
+        assert result == 'App deleted with success.'
+
+    def test_delete_failed(self):
+        # given
+        app_id = 'a'
+        self._app_manger.delete.return_value = False
+
+        # when
+        with pytest.raises(Exception):
+            self._handler.apps_delete(app_id)
+
+        # then
+        self._app_manger.delete.called_once_with(app_id)