Platform rework

.github/workflows/demo.yml

@@ -65,7 +65,7 @@ jobs:
     - name: Initialize MLonMCU environment
       run: |
         source .venv/bin/activate
-        mlonmcu init home/ --non-interactive --template ${{ matrix.template }} --clone-models
+        mlonmcu init home/ --non-interactive --template ${{ matrix.template }}
     - name: Setup MLonMCU dependencies
       run: |
         source .venv/bin/activate

.gitmodules

@@ -1,6 +1,3 @@
 [submodule "models"]
 	path = models
 	url = https://github.com/tum-ei-eda/mlonmcu-models.git
-[submodule "sw"]
-	path = sw
-	url = https://github.com/tum-ei-eda/mlonmcu-sw.git

MANIFEST.in

@@ -5,6 +5,7 @@ include LICENSE
 include README.md
 
 recursive-include tests *
+recursive-include resources *
 recursive-exclude * __pycache__
 recursive-exclude * *.py[co]
 

TODO.md

@@ -15,8 +15,6 @@ This is a list of pending tasks which did not make it into an own issue.
 
 - [ ] Implement: `mlonmcu models [filter]` on cli
 
-- [ ] Add postprocess and visualization features
-
 - [ ] Autotuning for ETISS targets
 
 - [ ] IPYNB: Add Google Colab support?
@@ -27,8 +25,6 @@ This is a list of pending tasks which did not make it into an own issue.
 
 - [ ] Finish main README
 
-- [ ] Add table with supported frontends/features/frameworks/backends/targets
-
 - [ ] Explain sessions and runs
 
 - [ ] Explain features

ipynb/Demo.ipynb

@@ -144,7 +144,7 @@
    "source": [
     "# WARNING: execute this only once! Alternatively remove the environment directory beforehand to start from scratch.\n",
     "!test -d /tmp/mlonmcu_env && echo \"Skipped initialization on environment\" \\\n",
-    "  || mlonmcu init /tmp/mlonmcu_env --non-interactive --clone-models"
+    "  || mlonmcu init /tmp/mlonmcu_env --non-interactive"
    ]
   },
   {

mlonmcu/__init__.py

@@ -23,5 +23,3 @@
 
 from mlonmcu.version import __version__
 
-# import mlonmcu.cli
-# import mlonmcu.config

mlonmcu/cli/common.py

@@ -20,7 +20,8 @@
 import multiprocessing
 import logging
 from mlonmcu.target import SUPPORTED_TARGETS
-from mlonmcu.platform import SUPPORTED_PLATFORMS
+
+from mlonmcu.platform import get_platforms
 from mlonmcu.session.postprocess import SUPPORTED_POSTPROCESSES
 from mlonmcu.feature.features import get_available_feature_names
 from mlonmcu.logging import get_logger, set_log_level
@@ -46,18 +47,23 @@ def add_flow_options(parser):
         "--target",
         type=str,
         metavar="TARGET",
-        choices=SUPPORTED_TARGETS.keys(),
+        # choices=SUPPORTED_TARGETS.keys(),
         action="append",
         # default=None,
         # nargs=1,
-        help="The target device/architecture (choices: %(choices)s)",
+        help="The target device/architecture (choices: See --list-targets)",
+    )
+    flow_parser.add_argument(  # TODO: move to compile.py?
+        "--list-targets",
+        action="store_true",
+        help="List the supported targets in the environment",
     )
     flow_parser.add_argument(
         # "-p",
         "--platform",
         type=str,
         metavar="PLATFORM",
-        choices=SUPPORTED_PLATFORMS.keys(),
+        choices=get_platforms().keys(),
         default=None,
         nargs=1,
         help="Explicitly choose the platforms to use (choices: %(choices)s)",

mlonmcu/cli/compile.py

@@ -35,6 +35,7 @@
 from mlonmcu.flow.backend import Backend
 from mlonmcu.flow.framework import Framework
 from mlonmcu.session.run import RunStage
+from mlonmcu.platform.lookup import get_platforms_targets
 
 
 def add_compile_options(parser):
@@ -74,6 +75,12 @@ def _handle(args, context):
         targets = context.environment.get_default_targets()
         assert len(targets) > 0, "TODO"
 
+    if args.platform:  # TODO: move this somewhere else
+        platform_names = args.platform
+    else:
+        platform_names = context.environment.lookup_platform_configs(names_only=True)
+    platform_targets = get_platforms_targets(context)
+
     debug = args.debug
     assert len(context.sessions) > 0  # TODO: automatically request session if no active one is available
     session = context.sessions[-1]
@@ -82,12 +89,13 @@ def _handle(args, context):
         for target_name in targets:
             for n in num:
                 new_run = run.copy()
-                if args.platform:  # TODO: move this somewhere else
-                    platform_name = args.platform[0]
-                else:
-                    platform_name = "mlif"
+                platform_name = None
+                for platform in platform_names:
+                    if target_name in platform_targets[platform]:
+                        platform_name = platform
+                assert platform_name is not None, f"Unable to find a suitable platform for the target '{target_name}'"
+                new_run.add_platform_by_name(platform_name, context=context)
                 new_run.add_target_by_name(target_name, context=context)
-                new_run.add_platform_by_name(platform_name, context=context)  # TODO: do this implicitly
                 new_run.debug = debug
                 new_run.num = n
                 new_runs.append(new_run)

mlonmcu/cli/flow.py

@@ -22,14 +22,16 @@
 
 import multiprocessing
 
+import mlonmcu.context
+import mlonmcu.platform.lookup
 import mlonmcu.cli.load as load
 import mlonmcu.cli.tune as tune
 import mlonmcu.cli.build as build
 import mlonmcu.cli.compile as compile_  # compile is a builtin name
 import mlonmcu.cli.debug as debug
 import mlonmcu.cli.test as test
 import mlonmcu.cli.run as run
-from mlonmcu.cli.common import add_flow_options, add_common_options
+from mlonmcu.cli.common import add_flow_options, add_common_options, add_context_options
 
 # from .trace import get_trace_parser
 
@@ -44,6 +46,7 @@ def get_parser(subparsers, parent=None):
     )
     parser.set_defaults(func=handle)
     add_common_options(parser)
+    add_context_options(parser)
     add_flow_options(parser)
     subparsers = parser.add_subparsers(dest="subcommand2")  # this line changed
     load_parser = load.get_parser(subparsers)
@@ -55,10 +58,18 @@ def get_parser(subparsers, parent=None):
     test_parser = test.get_parser(subparsers)
 
 
+def handle_list_targets(args):
+    with mlonmcu.context.MlonMcuContext(path=args.home, lock=True) as context:
+        mlonmcu.platform.lookup.print_summary(context=context)
+
+
 def handle(args):
     """Callback function which will be called to process the flow subcommand"""
-    if hasattr(args, "flow_func"):
-        args.flow_func(args)
+    if args.list_targets:
+        handle_list_targets(args)
     else:
-        print("Invalid command. Check 'mlonmcu flow --help' for the available subcommands!")
-        sys.exit(1)
+        if hasattr(args, "flow_func"):
+            args.flow_func(args)
+        else:
+            print("Invalid command. Check 'mlonmcu flow --help' for the available subcommands!")
+            sys.exit(1)

mlonmcu/cli/init.py

@@ -77,13 +77,6 @@ def add_init_options(parser):
         action="store_true",
         help="Clone models directory into environment",
     )
-    init_parser.add_argument(
-        "--skip-sw",
-        dest="skip_sw",
-        default=None,
-        action="store_true",
-        help="Clone sw directory into environment",
-    )
     init_parser.add_argument(
         "--register",
         default=None,
@@ -116,7 +109,6 @@ def handle(args):
         create_venv=args.venv,
         interactive=not args.non_interactive,
         clone_models=args.clone_models,
-        skip_sw=args.skip_sw,
         register=args.register,
         template=args.template[0] if isinstance(args.template, list) else args.template,
         allow_exists=args.allow_exists,

mlonmcu/cli/tune.py

@@ -59,6 +59,8 @@ def handle(args, ctx=None):
     if ctx:
         handle_build(args, context)
     else:
+        args.features.append("autotune")  # TODO: enable autotuning automatically?
+        # args.features.append("autotuned")  # ?
         with mlonmcu.context.MlonMcuContext(path=args.home, lock=True) as context:
             handle_build(args, context)
             kickoff_runs(args, RunStage.TUNE, context)

mlonmcu/environment/init.py

@@ -53,10 +53,6 @@ def clone_models_repo(
     git.Repo.clone_from(url, dest)
 
 
-def clone_sw_repo(dest, url="https://github.com/tum-ei-eda/mlonmcu-sw.git"):  # TODO: how to get submodule url/ref?
-    git.Repo.clone_from(url, dest)
-
-
 def create_venv_directory(base, hidden=True):
     if not isinstance(base, Path):
         base = Path(base)
@@ -72,7 +68,6 @@ def initialize_environment(
     interactive=True,
     create_venv=None,
     clone_models=None,
-    skip_sw=None,
     allow_exists=None,
     register=None,
     template=None,
@@ -156,26 +151,14 @@ def initialize_environment(
             clone_models is None
             and ask_user(
                 "Clone mlonmcu-models repository into environment?",
-                default=False,
+                default=True,
                 interactive=interactive,
             )
         ):
             clone_models_repo(models_subdir)
         else:
             subdirs.append("models")
 
-    sw_subdir = Path(target_dir) / "sw"
-    if not sw_subdir.is_dir():
-        if (skip_sw == False) or (
-            skip_sw is None
-            and ask_user(
-                "Clone mlonmcu-sw repository into environment?",
-                default=True,
-                interactive=interactive,
-            )
-        ):
-            clone_sw_repo(sw_subdir)
-
     print("Initializing directories in environment:", " ".join(subdirs))
     create_environment_directories(target_dir, subdirs)
 

mlonmcu/feature/feature.py

@@ -161,13 +161,11 @@ def get_platform_config(self, platform):
     def add_platform_config(self, platform, config):
         config.update(self.get_platform_config(platform))
 
-    def get_cmake_args(self):
-        return []
-
-    def add_cmake_args(self, args):
-        args += self.get_cmake_args()
+    def get_platform_defs(self, platform):
+        return {}
 
-    # TODO: alternative mlif.cmake_args appenden?
+    def add_platform_defs(self, platform, defs):
+        defs.update(self.get_platform_defs(platform))
 
 
 class SetupFeature(FeatureBase):  # TODO: alternative: CacheFeature

mlonmcu/feature/features.py

@@ -90,12 +90,12 @@ def get_backend_config(self, backend):
         # TODO: TFLM also compatible?
         return {f"{backend}.debug_arena": self.enabled}
 
-    # def get_platform_config(self):
-    #    return {"mlif.debug_arena": True}
-
-    def get_cmake_args(self):
-        val = "ON" if self.enabled else "OFF"
-        return [f"-DDEBUG_ARENA={val}"]
+    def get_platform_defs(self, platform):
+        if platform == "espidf":
+            val = "y" if self.enabled else "n"  # TODO: bool to string at later step?
+        else:
+            val = "ON" if self.enabled else "OFF"
+        return {"DEBUG_ARENA": val}
 
 
 @register_feature("validate")
@@ -105,6 +105,7 @@ class Validate(FrontendFeature, PlatformFeature):
     DEFAULTS = {
         **FeatureBase.DEFAULTS,
         "allow_missing": True,
+        "fail_on_error": None,
     }
 
     def __init__(self, config=None):
@@ -114,17 +115,21 @@ def __init__(self, config=None):
     def allow_missing(self):
         return bool(self.config["allow_missing"])
 
+    @property
+    def fail_on_error(self):
+        return self.config["fail_on_error"]
+
     def get_frontend_config(self, frontend):
         if not self.allow_missing:
             raise NotImplementedError
         return {f"{frontend}.use_inout_data": True}
 
     def get_platform_config(self, platform):
         assert platform == "mlif", f"Unsupported feature '{self.name}' for platform '{platform}'"
-        return {f"{platform}.ignore_data": False}
-
-    # def get_cmake_args(self):
-    #     pass
+        return filter_none({
+            f"{platform}.ignore_data": False,
+            f"{platform}.fail_on_error": self.fail_on_error,
+        })
 
 
 @register_feature("muriscvnn")
@@ -391,7 +396,7 @@ def get_backend_config(self, backend):
 
 
 @register_feature("packed")
-class Packed(FrameworkFeature, FrontendFeature, BackendFeature, SetupFeature, PlatformFeature):
+class Packed(FrameworkFeature, FrontendFeature, BackendFeature, SetupFeature):
     """Sub-8-bit and sparsity feature for TFLite Micro kernels."""
 
     def __init__(self, config=None):
@@ -410,10 +415,6 @@ def get_backend_config(self, backend):
     def get_required_cache_flags(self):
         return {"tflmc.exe": ["packed"]}
 
-    def get_cmake_args(self):
-        val = "ON" if self.enabled else "OFF"
-        return [f"-DDEBUG_ARENA={val}"]
-
 
 @register_feature("packing")
 class Packing(FrontendFeature):

mlonmcu/flow/backend.py

@@ -136,19 +136,15 @@ def export_code(self, path):
                     logger.info(f"Exporting artifact: {artifact.name}")
                     outfile.write(artifact.content)
 
-    def get_cmake_args(self):
-        assert self.name is not None
-        return [f"-DBACKEND={self.name}"]
-
-    def add_cmake_args(self, args):
-        args += self.get_cmake_args()
-
-    def get_espidf_defs(self):
-        assert self.name is not None
-        return {"MLONMCU_BACKEND": self.name}
+    def get_platform_defs(self, platform):
+        if platform == "espidf":
+            backend_upper = self.name.upper()
+            return {f"MLONMCU_BACKEND_{backend_upper}": True}
+        else:
+            return {"MLONMCU_BACKEND": self.name}
 
-    def add_espidf_defs(self, defs):
-        defs.update(self.get_espidf_defs())
+    def add_platform_defs(self, platform, defs):
+        defs.update(self.get_platform_defs(platform))
 
 
 def get_parser(backend_name, features, required, defaults):