feat: add configuration with config file

powerapi-ng · Aug 30, 2021 · 26771f6 · 26771f6
1 parent 2216980
commit 26771f6
Show file tree

Hide file tree

Showing 12 changed files with 244 additions and 90 deletions.
diff --git a/powerapi/cli/__init__.py b/powerapi/cli/__init__.py
@@ -26,5 +26,4 @@
 # CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
 # OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-
-
+from .config_validator import ConfigValidator
diff --git a/powerapi/cli/config_validator.py b/powerapi/cli/config_validator.py
@@ -0,0 +1,62 @@
+# Copyright (c) 2018, INRIA
+# Copyright (c) 2018, University of Lille
+# All rights reserved.
+
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are met:
+
+# * Redistributions of source code must retain the above copyright notice, this
+#   list of conditions and the following disclaimer.
+
+# * Redistributions in binary form must reproduce the above copyright notice,
+#   this list of conditions and the following disclaimer in the documentation
+#   and/or other materials provided with the distribution.
+
+# * Neither the name of the copyright holder nor the names of its
+#   contributors may be used to endorse or promote products derived from
+#   this software without specific prior written permission.
+
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
+# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+from typing import Dict
+import logging
+
+class ConfigValidator:
+    @staticmethod
+    def validate(config: Dict):
+
+        if 'verbose' not in config:
+            config['verbose'] = logging.NOTSET
+        if 'stream' not in config:
+            config['stream'] = False
+        if 'output' not in config:
+            logging.error("no output configuration found")
+            return False
+
+        for output_type in config['output']:
+            output_config = config['output'][output_type]
+            if 'model' not in  output_config:
+                output_config['model'] = 'HWPCReport'
+            if 'name' not in  output_config:
+                output_config['name'] = 'default_pusher'
+
+        if 'input' not in config:
+            logging.error("no input configuration found")
+            return False
+
+        for input_type in config['input']:
+            input_config = config['input'][input_type]
+            if 'model' not in input_config:
+                input_config['model'] = 'PowerReport'
+            if 'name' not in input_config:
+                input_config['name'] = 'default_puller'
+        return True
+
diff --git a/powerapi/cli/parser.py b/powerapi/cli/parser.py
@@ -474,6 +474,7 @@ def _action(arg, val, args, acc):
             args, subparse_result = subparser.subparse(args)
 
             acc[arg][subparser.name] = subparse_result
+            # acc[arg] = subparse_result
             return args, acc
 
         if component_type not in self.subparsers_group:
@@ -511,14 +512,14 @@ def _action(arg, val, args, acc):
                 raise NoNameSpecifiedForComponentException(component_type)
 
             component_name = subparse_result['name']
+            del subparse_result['name']
 
-            if subparser.name not in acc[arg]:
-                acc[arg][subparser.name] = {}
-
-            if component_name in acc[arg][subparser.name]:
+            if component_name in acc[arg]:
                 raise ComponentAlreadyExistException(component_name)
 
-            acc[arg][subparser.name][component_name] = subparse_result
+            acc[arg][component_name] = subparse_result
+            acc[arg][component_name]['type'] = subparser.name
+
             return args, acc
 
         if 'name' not in subparser.actions:

diff --git a/powerapi/cli/tools.py b/powerapi/cli/tools.py
@@ -128,8 +128,8 @@ def __init__(self):
                                      help_str='specify a database output : --db_output database_name ARG1 ARG2 ...')
 
         subparser_prom_output = ComponentSubParser('prom')
-        subparser_prom_output.add_argument('a', 'addr', help='specify server address')
         subparser_prom_output.add_argument('t', 'tags', help='specify report tags')
+        subparser_prom_output.add_argument('u', 'uri', help='specify server uri')
         subparser_prom_output.add_argument('p', 'port', help='specify server port', type=int)
         subparser_prom_output.add_argument('M', 'metric_name', help='speify metric name')
         subparser_prom_output.add_argument('d', 'metric_description', help='specify metric description', default='energy consumption')
@@ -142,8 +142,8 @@ def __init__(self):
                                      help_str='specify a database output : --db_output database_name ARG1 ARG2 ...')
 
         subparser_direct_prom_output = ComponentSubParser('direct_prom')
-        subparser_direct_prom_output.add_argument('a', 'addr', help='specify server address')
         subparser_direct_prom_output.add_argument('t', 'tags', help='specify report tags')
+        subparser_direct_prom_output.add_argument('a', 'uri', help='specify server uri')
         subparser_direct_prom_output.add_argument('p', 'port', help='specify server port', type=int)
         subparser_direct_prom_output.add_argument('M', 'metric_name', help='speify metric name')
         subparser_direct_prom_output.add_argument('d', 'metric_description', help='specify metric description', default='energy consumption')
@@ -229,19 +229,19 @@ def generate(self, config: Dict) -> Dict[str, Tuple[Type[Actor], StartMessage]]:
 
         actors = {}
 
-        for component_type, components_list in config[self.component_group_name].items():
-            for component_name, component_config in components_list.items():
-                try:
-                    actors[component_name] = self._gen_actor(component_type, component_config, config)
-                except KeyError as exn:
-                    msg = 'CLI error : argument ' + exn.args[0]
-                    msg += ' needed with --output ' + component_type
-                    print(msg, file=sys.stderr)
-                    sys.exit()
+        for component_name, component_config in config[self.component_group_name].items():
+            component_type = component_config['type']
+            try:
+                actors[component_name] = self._gen_actor(component_type, component_config, config, component_name)
+            except KeyError as exn:
+                msg = 'CLI error : argument ' + exn.args[0]
+                msg += ' needed with --output ' + component_type
+                print(msg, file=sys.stderr)
+                sys.exit()
 
         return actors
 
-    def _gen_actor(self, component_name: str, component_config: Dict, main_config: Dict) -> Tuple[Type[Actor], StartMessage]:
+    def _gen_actor(self, component_type: str, component_config: Dict, main_config: Dict, component_name: str) -> Tuple[Type[Actor], StartMessage]:
         raise NotImplementedError()
 
 
@@ -305,9 +305,9 @@ def __init__(self, component_group_name):
                                            files=[] if 'files' not in db_config else db_config['files']),
             'influxdb': lambda db_config: InfluxDB(db_config['model'], db_config['uri'], db_config['port'], db_config['db'], gen_tag_list(db_config)),
             'opentsdb': lambda db_config: OpenTSDB(db_config['model'], db_config['uri'], db_config['port'], db_config['metric_name']),
-            'prom': lambda db_config: PrometheusDB(db_config['model'], db_config['port'], db_config['addr'], db_config['metric_name'],
+            'prom': lambda db_config: PrometheusDB(db_config['model'], db_config['port'], db_config['uri'], db_config['metric_name'],
                                                    db_config['metric_description'], db_config['aggregation_period'], gen_tag_list(db_config)),
-            'direct_prom': lambda db_config: DirectPrometheusDB(db_config['model'], db_config['port'], db_config['addr'], db_config['metric_name'],
+            'direct_prom': lambda db_config: DirectPrometheusDB(db_config['model'], db_config['port'], db_config['uri'], db_config['metric_name'],
                                                                 db_config['metric_description'], gen_tag_list(db_config)),
             'virtiofs': lambda db_config: VirtioFSDB(db_config['model'], db_config['vm_name_regexp'], db_config['root_directory_name'], db_config['vm_directory_name_prefix'], db_config['vm_directory_name_suffix']),
         }

diff --git a/tests/acceptation/crash/test_crash_formula_medium_error.py b/tests/acceptation/crash/test_crash_formula_medium_error.py
@@ -144,9 +144,18 @@ def mongodb_content():
 
 
 def test_run_mongo(mongo_database, shutdown_system):
-    config = {'verbose': True, 'stream': False,
-              'output': {'mongodb': {'test_pusher': {'model': 'PowerReport', 'name': 'test_pusher', 'uri': MONGO_URI, 'db': MONGO_DATABASE_NAME, 'collection': MONGO_OUTPUT_COLLECTION_NAME}}},
-              'input': {'mongodb': {'test_puller': {'model': 'HWPCReport', 'name': 'test_puller', 'uri': MONGO_URI, 'db': MONGO_DATABASE_NAME, 'collection': MONGO_INPUT_COLLECTION_NAME}}}
+    config = {'verbose': True,
+              'stream': False,
+              'output': {'test_pusher': {'type': 'mongodb',
+                                         'model': 'PowerReport',
+                                         'uri': MONGO_URI,
+                                         'db': MONGO_DATABASE_NAME,
+                                         'collection': MONGO_OUTPUT_COLLECTION_NAME}},
+              'input': {'test_puller': {'type': 'mongodb',
+                                        'model': 'HWPCReport',
+                                        'uri': MONGO_URI,
+                                        'db': MONGO_DATABASE_NAME,
+                                        'collection': MONGO_INPUT_COLLECTION_NAME}}
               }
 
     supervisor = Supervisor()

diff --git a/tests/acceptation/crash/test_crash_formula_minor_error.py b/tests/acceptation/crash/test_crash_formula_minor_error.py
@@ -135,11 +135,20 @@ def mongodb_content():
 
 
 def test_run_mongo(mongo_database, shutdown_system):
-    config = {'verbose': True, 'stream': False,
-              'output': {'mongodb': {'test_pusher': {'model': 'PowerReport', 'name': 'test_pusher', 'uri': MONGO_URI, 'db': MONGO_DATABASE_NAME, 'collection': MONGO_OUTPUT_COLLECTION_NAME}}},
-              'input': {'mongodb': {'test_puller': {'model': 'HWPCReport', 'name': 'test_puller', 'uri': MONGO_URI, 'db': MONGO_DATABASE_NAME, 'collection': MONGO_INPUT_COLLECTION_NAME}}}
+    config = {'verbose': True,
+              'stream': False,
+              'output': {'test_pusher': {'type': 'mongodb',
+                                         'model': 'PowerReport',
+                                         'uri': MONGO_URI,
+                                         'db': MONGO_DATABASE_NAME,
+                                         'collection': MONGO_OUTPUT_COLLECTION_NAME}},
+              'input': {'test_puller': {'type': 'mongodb',
+                                        'model': 'HWPCReport',
+                                        'name': 'test_puller',
+                                        'uri': MONGO_URI,
+                                        'db': MONGO_DATABASE_NAME,
+                                        'collection': MONGO_INPUT_COLLECTION_NAME}}
               }
-
     supervisor = Supervisor()
     launch_simple_architecture(config, supervisor)
     supervisor.monitor()

diff --git a/tests/acceptation/test_simple_architecture.py b/tests/acceptation/test_simple_architecture.py
@@ -123,9 +123,19 @@ def mongodb_content():
     return extract_rapl_reports_with_2_sockets(10)
 
 def test_run_mongo(mongo_database, shutdown_system):
-    config = {'verbose': True, 'stream': False,
-              'output': {'mongodb': {'test_pusher': {'tags': 'socket', 'model': 'PowerReport', 'name': 'test_pusher', 'uri': MONGO_URI, 'db': MONGO_DATABASE_NAME, 'collection': MONGO_OUTPUT_COLLECTION_NAME}}},
-              'input': {'mongodb': {'test_puller': {'model': 'HWPCReport', 'name': 'test_puller', 'uri': MONGO_URI, 'db': MONGO_DATABASE_NAME, 'collection': MONGO_INPUT_COLLECTION_NAME}}}
+    config = {'verbose': True,
+              'stream': False,
+              'output': {'test_pusher': {'type': 'mongodb',
+                                         'tags': 'socket',
+                                         'model': 'PowerReport',
+                                         'uri': MONGO_URI,
+                                         'db': MONGO_DATABASE_NAME,
+                                         'collection': MONGO_OUTPUT_COLLECTION_NAME}},
+              'input': {'test_puller': {'type': 'mongodb',
+                                        'model': 'HWPCReport',
+                                        'uri': MONGO_URI,
+                                        'db': MONGO_DATABASE_NAME,
+                                        'collection': MONGO_INPUT_COLLECTION_NAME}}
               }
 
     supervisor = Supervisor()
@@ -158,9 +168,19 @@ def check_influx_db(influx_client):
 
 
 def test_run_mongo_to_influx(mongo_database, influx_database, shutdown_system):
-    config = {'verbose': True, 'stream': False,
-              'output': {'influxdb': {'test_pusher': {'tags': 'socket', 'model': 'PowerReport', 'name': 'test_pusher', 'uri': INFLUX_URI, 'port': INFLUX_PORT, 'db': INFLUX_DBNAME}}},
-              'input': {'mongodb': {'test_puller': {'model': 'HWPCReport', 'name': 'test_puller', 'uri': MONGO_URI, 'db': MONGO_DATABASE_NAME, 'collection': MONGO_INPUT_COLLECTION_NAME}}}
+    config = {'verbose': True,
+              'stream': False,
+              'output': {'test_pusher': {'type': 'influxdb',
+                                         'tags': 'socket',
+                                         'model': 'PowerReport',
+                                         'uri': INFLUX_URI,
+                                         'port': INFLUX_PORT,
+                                         'db': INFLUX_DBNAME}},
+              'input': {{'test_puller': {'type': 'mongodb',
+                                         'model': 'HWPCReport',
+                                         'uri': MONGO_URI,
+                                         'db': MONGO_DATABASE_NAME,
+                                         'collection': MONGO_INPUT_COLLECTION_NAME}}}
               }
 
     supervisor = Supervisor()
@@ -220,9 +240,17 @@ def check_output_file():
 
 
 def test_run_csv_to_csv(files, shutdown_system):
-    config = {'verbose': True, 'stream': False,
-              'output': {'csv': {'test_pusher': {'tags': 'socket', 'model': 'PowerReport', 'name': 'pusher', 'directory': OUTPUT_PATH}}},
-              'input': {'csv': {'test_puller': {'files': FILES, 'model': 'HWPCReport', 'name': 'puller'}}},
+    config = {'verbose': True,
+              'stream': False,
+              'output': {'test_pusher': {'type': 'csv',
+                                         'tags': 'socket',
+                                         'model': 'PowerReport',
+                                         'name': 'pusher',
+                                         'directory': OUTPUT_PATH}},
+              'input': {'test_puller': {'type': 'csv',
+                                        'files': FILES,
+                                        'model': 'HWPCReport',
+                                        'name': 'puller'}},
               }
     supervisor = Supervisor()
     launch_simple_architecture(config, supervisor)
@@ -249,10 +277,16 @@ def check_db_socket():
 
 
 def test_run_socket_to_mongo(mongo_database, unused_tcp_port, shutdown_system):
-    config = {'verbose': True, 'stream': False,
-              'output': {'mongodb': {'test_pusher': {'model': 'PowerReport', 'name': 'test_pusher', 'uri': MONGO_URI, 'db': MONGO_DATABASE_NAME,
-                                                     'collection': MONGO_OUTPUT_COLLECTION_NAME}}},
-              'input': {'socket': {'test_puller': {'port': unused_tcp_port, 'model': 'HWPCReport', 'name': 'puller'}}},
+    config = {'verbose': True,
+              'stream': False,
+              'output': {'test_pusher': {'type': 'mongodb',
+                                         'model': 'PowerReport',
+                                         'uri': MONGO_URI,
+                                         'db': MONGO_DATABASE_NAME,
+                                         'collection': MONGO_OUTPUT_COLLECTION_NAME}},
+              'input': {'test_puller': {'type': 'socket',
+                                        'port': unused_tcp_port,
+                                        'model': 'HWPCReport'}},
               }
     supervisor = Supervisor()
     launch_simple_architecture(config, supervisor)
@@ -262,10 +296,16 @@ def test_run_socket_to_mongo(mongo_database, unused_tcp_port, shutdown_system):
     check_db_socket()
 
 def test_run_socket_with_delay_between_message_to_mongo(mongo_database, unused_tcp_port, shutdown_system):
-    config = {'verbose': True, 'stream': False,
-              'output': {'mongodb': {'test_pusher': {'model': 'PowerReport', 'name': 'test_pusher', 'uri': MONGO_URI, 'db': MONGO_DATABASE_NAME,
-                                                     'collection': MONGO_OUTPUT_COLLECTION_NAME}}},
-              'input': {'socket': {'test_puller': {'port': unused_tcp_port, 'model': 'HWPCReport', 'name': 'test_puller'}}},
+    config = {'verbose': True,
+              'stream': False,
+              'output': {'test_pusher': {'type': 'mongodb',
+                                         'model': 'PowerReport',
+                                         'uri': MONGO_URI,
+                                         'db': MONGO_DATABASE_NAME,
+                                         'collection': MONGO_OUTPUT_COLLECTION_NAME}},
+              'input': {'test_puller': {'type': 'socket',
+                                        'port': unused_tcp_port,
+                                        'model': 'HWPCReport'}},
               }
     supervisor = Supervisor()
     launch_simple_architecture(config, supervisor)

diff --git a/tests/acceptation/test_simple_architecture_with_libvirt_mapper.py b/tests/acceptation/test_simple_architecture_with_libvirt_mapper.py
@@ -90,8 +90,6 @@ def check_db():
             {'timestamp': ts, 'sensor': report['sensor'],
              'target': UUID_1}) == 2
 
-
-
 def filter_rule(msg):
     return True
 
@@ -125,17 +123,17 @@ def launch_simple_architecture(config, supervisor):
 def test_run(mocked_libvirt, mongo_database, shutdown_system):
     config = {'verbose': True,
               'stream': False,
-              'input': {'mongodb': {'test_puller': {'uri': MONGO_URI,
-                                               'db': MONGO_DATABASE_NAME,
-                                               'collection': MONGO_INPUT_COLLECTION_NAME,
-                                               'model': 'HWPCReport',
-                                               'name': 'test_puller',
-                                               }}},
-              'output': {'mongodb': {'test_pusher': {'uri': MONGO_URI,
-                                                'db': MONGO_DATABASE_NAME,
-                                                'collection': MONGO_OUTPUT_COLLECTION_NAME,
-                                                'model': 'PowerReport',
-                                                'name': 'test_pusher'}}},
+              'input': {'puller': {'type': 'mongodb',
+                                   'uri': MONGO_URI,
+                                   'db': MONGO_DATABASE_NAME,
+                                   'collection': MONGO_INPUT_COLLECTION_NAME,
+                                   'model': 'HWPCReport',
+                                   }},
+              'output': {'pusher': {'type': 'mongodb',
+                                    'uri': MONGO_URI,
+                                    'db': MONGO_DATABASE_NAME,
+                                    'collection': MONGO_OUTPUT_COLLECTION_NAME,
+                                    'model': 'PowerReport'}},
               'report_modifier': {'libvirt_mapper': {'uri': '',
                                                      'domain_regexp': REGEXP}}
               }