From 9a81c345f9e83c72bad48974ce43e94a2ee7d96a Mon Sep 17 00:00:00 2001 From: Philipp van Kempen Date: Tue, 10 Jan 2023 18:10:50 +0100 Subject: [PATCH 001/105] [WIP] implement microtvm_gvsoc target --- .../microtvm/microtvm_gvsoc_target.py | 97 +++++++++++++++++++ mlonmcu/platform/microtvm/microtvm_target.py | 2 + mlonmcu/setup/gen_requirements.py | 2 + mlonmcu/setup/setup.py | 6 ++ mlonmcu/setup/tasks/__init__.py | 1 + mlonmcu/setup/tasks/ekut.py | 58 +++++++++++ 6 files changed, 166 insertions(+) create mode 100644 mlonmcu/platform/microtvm/microtvm_gvsoc_target.py create mode 100644 mlonmcu/setup/tasks/ekut.py diff --git a/mlonmcu/platform/microtvm/microtvm_gvsoc_target.py b/mlonmcu/platform/microtvm/microtvm_gvsoc_target.py new file mode 100644 index 000000000..510393a90 --- /dev/null +++ b/mlonmcu/platform/microtvm/microtvm_gvsoc_target.py @@ -0,0 +1,97 @@ +# +# Copyright (c) 2022 TUM Department of Electrical and Computer Engineering. +# +# This file is part of MLonMCU. +# See https://github.com/tum-ei-eda/mlonmcu.git for further info. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +from pathlib import Path + +from mlonmcu.target.target import Target + +from mlonmcu.logging import get_logger + +from .microtvm_template_target import TemplateMicroTvmPlatformTarget + +logger = get_logger() + + +class GVSocMicroTvmPlatformTarget(TemplateMicroTvmPlatformTarget): + # FEATURES = TemplateMicroTvmPlatformTarget.FEATURES + ["xpulp"] + FEATURES = TemplateMicroTvmPlatformTarget.FEATURES + + DEFAULTS = { + **TemplateMicroTvmPlatformTarget.DEFAULTS, + # "verbose": True, + "compiler": "gcc", + "project_type": "host_driven", + # "xpulp_version": None, # None means that xpulp extension is not used, + # "model": "pulp", + } + REQUIRED = Target.REQUIRED + [ + "gvsoc.exe", + "pulp_freertos.support_dir", + "pulp_freertos.config_dir", + "pulp_freertos.install_dir", + "microtvm_gvsoc.template", + "hannah_tvm.src_dir", + ] + + def __init__(self, name=None, features=None, config=None): + super().__init__(name=name, features=features, config=config) + self.template_path = self.microtvm_gvsoc_template + # TODO: integrate into TVM build config + self.option_names = [ + # "verbose", + "project_type", + "compiler", + ] + + @property + def microtvm_gvsoc_template(self): + return Path(self.config["microtvm_gvsoc.template"]) + + @property + def hannah_tvm_src_dir(self): + return Path(self.config["hannah_tvm.src_dir"]) + + @property + def compiler(self): + return self.config["compiler"] + + def get_project_options(self): + ret = super().get_project_options() + # TODO + ret.update( + { + # "gvsoc_exe": str(self.gvsoc_exe), + } + ) + return ret + + def update_environment(self, env): + super().update_environment(env) + p = env.get("PYTHONPATH", "") + env["PYTHONPATH"] = f"{self.hannah_tvm_src_dir}:{p}" + + # TODO + # if "PATH" in env: + # env["PATH"] = str(self.riscv_gcc_install_dir / "bin") + ":" + env["PATH"] + # else: + # env["PATH"] = str(self.riscv_gcc_install_dir / "bin") + + def get_backend_config(self, backend): + ret = {} + # TODO + return ret diff --git a/mlonmcu/platform/microtvm/microtvm_target.py b/mlonmcu/platform/microtvm/microtvm_target.py index 7df0c878f..c37308e10 100644 --- a/mlonmcu/platform/microtvm/microtvm_target.py +++ b/mlonmcu/platform/microtvm/microtvm_target.py @@ -30,6 +30,7 @@ from .microtvm_host_target import HostMicroTvmPlatformTarget from .microtvm_etissvp_target import EtissvpMicroTvmPlatformTarget from .microtvm_spike_target import SpikeMicroTvmPlatformTarget +from .microtvm_gvsoc_target import GVSocMicroTvmPlatformTarget logger = get_logger() @@ -59,6 +60,7 @@ def get_microtvm_platform_targets(): register_microtvm_platform_target("microtvm_etissvp", EtissvpMicroTvmPlatformTarget) register_microtvm_platform_target("microtvm_espidf", EspidfMicroTvmPlatformTarget) register_microtvm_platform_target("microtvm_spike", SpikeMicroTvmPlatformTarget) +register_microtvm_platform_target("microtvm_gvsoc", GVSocMicroTvmPlatformTarget) def create_microtvm_platform_target(name, platform, base=Target): diff --git a/mlonmcu/setup/gen_requirements.py b/mlonmcu/setup/gen_requirements.py index 235fad38d..7f74408c8 100644 --- a/mlonmcu/setup/gen_requirements.py +++ b/mlonmcu/setup/gen_requirements.py @@ -125,6 +125,7 @@ ["pyserial", "pyusb"], ), ), + ("microtvm_gvsoc", ("Requirements for microtvm_gvsoc target", ["hydra-core"])), # Provide support for moiopt. ("moiopt", ("Requirements for moiopt", ["ortools"])), # Provide support for onnx. @@ -221,6 +222,7 @@ ("gdbgui", "==0.13.2.0"), ("graphviz", None), ("humanize", None), + ("hydra-core", None), ("idf-component-manager", "~=1.0"), ("itsdangerous", "<2.1"), ("jinja2", "<3.1"), diff --git a/mlonmcu/setup/setup.py b/mlonmcu/setup/setup.py index 29220b419..b90fd20eb 100644 --- a/mlonmcu/setup/setup.py +++ b/mlonmcu/setup/setup.py @@ -232,6 +232,12 @@ def feature_enabled_and_supported(obj, feature): f.write(f"{d}{os.linesep}") logger.info("add dependencies for etiss") break + for config in config_pools: + if "microtvm_gvsoc" in config.name and config.enabled: + for d in requirements["microtvm_gvsoc"][1]: + f.write(f"{d}{os.linesep}") + logger.info("add dependencies for microtvm_gvsoc") + break for config in config_pools: if "visualize" in config.name and config.enabled: for d in requirements["visualize"][1]: diff --git a/mlonmcu/setup/tasks/__init__.py b/mlonmcu/setup/tasks/__init__.py index 215db2fc1..68d1fea90 100644 --- a/mlonmcu/setup/tasks/__init__.py +++ b/mlonmcu/setup/tasks/__init__.py @@ -38,3 +38,4 @@ from .utvmcg import * # noqa: F401, F403 from .zephyr import * # noqa: F401, F403 from .pulp import * # noqa: F401, F403 +from .ekut import * # noqa: F401, F403 diff --git a/mlonmcu/setup/tasks/ekut.py b/mlonmcu/setup/tasks/ekut.py new file mode 100644 index 000000000..e3db23a3a --- /dev/null +++ b/mlonmcu/setup/tasks/ekut.py @@ -0,0 +1,58 @@ +# +# Copyright (c) 2022 TUM Department of Electrical and Computer Engineering. +# +# This file is part of MLonMCU. +# See https://github.com/tum-ei-eda/mlonmcu.git for further info. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +"""Definition of tasks used to dynamically install MLonMCU dependencies""" + +import multiprocessing + +from mlonmcu.setup.task import TaskType +from mlonmcu.context.context import MlonMcuContext +from mlonmcu.setup import utils +from mlonmcu.logging import get_logger + +from .common import get_task_factory + +logger = get_logger() +Tasks = get_task_factory() + +############## +# hannah-tvm # +############## + + +def _validate_hannah_tvm(context: MlonMcuContext, params=None): + return context.environment.has_target("microtvm_gvsoc") + + +@Tasks.provides(["hannah_tvm.src_dir", "microtvm_gvsoc.template"]) +@Tasks.validate(_validate_hannah_tvm) +@Tasks.register(category=TaskType.TARGET) +def clone_hannah_tvm( + context: MlonMcuContext, params=None, rebuild=False, verbose=False, threads=multiprocessing.cpu_count() +): + """Clone the hannah-tvm repository.""" + hannahTvmName = utils.makeDirName("hannah_tvm") + hannahTvmSrcDir = context.environment.paths["deps"].path / "src" / hannahTvmName + if ( + rebuild + or not utils.is_populated(hannahTvmSrcDir) + ): + pulpRtosRepo = context.environment.repos["hannah_tvm"] + utils.clone(pulpRtosRepo.url, hannahTvmSrcDir, branch=pulpRtosRepo.ref, refresh=rebuild, recursive=True) + context.cache["hannah_tvm.src_dir"] = hannahTvmSrcDir + context.cache["microtvm_gvsoc.template"] = hannahTvmSrcDir / "template" / "gvsoc" From 4f9ef92f25560105b6ab35ac8d09d426405a0ed1 Mon Sep 17 00:00:00 2001 From: Philipp van Kempen Date: Fri, 12 Jan 2024 14:23:48 +0100 Subject: [PATCH 002/105] feature.py: RunFeatures can not enable subprocesses --- mlonmcu/feature/feature.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/mlonmcu/feature/feature.py b/mlonmcu/feature/feature.py index 6c747d064..4f45ad0a6 100644 --- a/mlonmcu/feature/feature.py +++ b/mlonmcu/feature/feature.py @@ -229,3 +229,9 @@ def get_run_config(self): def add_run_config(self, config): config.update(self.get_run_config()) + + def get_postprocesses(self): + return [] + + def add_postprocesses(self, postprocesses): + postprocesses.extend(self.get_postprocesses()) From f7bbf57c9146cd6a1f26e0d035ac80e2c0baa39e Mon Sep 17 00:00:00 2001 From: Philipp van Kempen Date: Fri, 12 Jan 2024 14:26:09 +0100 Subject: [PATCH 003/105] WIP: add new features and postprocesses for validate_new --- mlonmcu/feature/features.py | 169 +++++++++++++++++++++++ mlonmcu/models/frontend.py | 265 +++++++++++++++++++++++++++++++++++- 2 files changed, 432 insertions(+), 2 deletions(-) diff --git a/mlonmcu/feature/features.py b/mlonmcu/feature/features.py index 1b2e098a8..2d7ed3471 100644 --- a/mlonmcu/feature/features.py +++ b/mlonmcu/feature/features.py @@ -2217,3 +2217,172 @@ def add_target_config(self, target, config): assert self.name not in extra_plugin_config extra_plugin_config[self.name]["baseaddr"] = self.base_addr config.update({f"{target}.extra_plugin_config": extra_plugin_config}) + + +@register_feature("gen_data") +class GenData(FrontendFeature): # TODO: use custom stage instead of LOAD + """TODO""" + + DEFAULTS = { + **FeatureBase.DEFAULTS, + "fill_mode": "file", # Allowed: random, ones, zeros, file, dataset + "file": "auto", # Only relevant if fill_mode=file + "number": 10, # generate up to number samples (may be less if file has less inputs) + "fmt": "npy", # Allowed: npy, npz + } + + def __init__(self, features=None, config=None): + super().__init__("gen_data", features=features, config=config) + + @property + def fill_mode(self): + value = self.config["fill_mode"] + assert value in ["random", "ones", "zeros", "file", "dataset"] + return value + + @property + def file(self): + value = self.config["file"] + return value + + @property + def number(self): + return int(self.config["number"]) + + @property + def fmt(self): + value = self.config["fmt"] + assert value in ["npy", "npz"] + return value + + def get_frontend_config(self, frontend): + assert frontend in ["tflite"] + return { + f"{frontend}.gen_data": self.enabled, + f"{frontend}.gen_data_fill_mode": self.fill_mode, + f"{frontend}.gen_data_file": self.file, + f"{frontend}.gen_data_number": self.number, + f"{frontend}.gen_data_fmt": self.fmt, + } + + +@register_feature("gen_ref_data", depends=["gen_data"]) +class GenRefData(FrontendFeature): # TODO: use custom stage instead of LOAD + """TODO""" + + DEFAULTS = { + **FeatureBase.DEFAULTS, + "mode": "file", # Allowed: file, model + "file": "auto", # Only relevant if mode=file + "fmt": "npy", # Allowed: npy, npz + } + + def __init__(self, features=None, config=None): + super().__init__("gen_ref_data", features=features, config=config) + + @property + def mode(self): + value = self.config["mode"] + assert value in ["file", "model"] + return value + + @property + def file(self): + value = self.config["file"] + return value + + @property + def fmt(self): + value = self.config["fmt"] + assert value in ["npy", "npz"] + return value + + def get_frontend_config(self, frontend): + assert frontend in ["tflite"] + return { + f"{frontend}.gen_ref_data": self.enabled, + f"{frontend}.gen_ref_data_mode": self.mode, + f"{frontend}.gen_ref_data_file": self.file, + f"{frontend}.gen_ref_data_fmt": self.fmt, + } + + +@register_feature("set_inputs") +class SetInputs(PlatformFeature): # TODO: use custom stage instead of LOAD + """TODO""" + + DEFAULTS = { + **FeatureBase.DEFAULTS, + "interface": "auto", # Allowed: auto, rom, filesystem, stdout, uart + } + + def __init__(self, features=None, config=None): + super().__init__("set_inputs", features=features, config=config) + + @property + def mode(self): + value = self.config["interface"] + assert value in ["auto", "rom", "filesystem", "stdin", "uart"] + return value + + def get_platform_config(self, platform): + assert platform in ["mlif", "tvm", "microtvm"] + # if tvm/microtvm: allow using --fill-mode provided by tvmc run + return { + f"{platform}.set_inputs": self.enabled, + f"{platform}.set_inputs_interface": self.interface, + } + + +@register_feature("get_outputs") +class GetOutputs(PlatformFeature): # TODO: use custom stage instead of LOAD + """TODO""" + + DEFAULTS = { + **FeatureBase.DEFAULTS, + "interface": "auto", # Allowed: auto, filesystem, stdout, uart + "fmt": "npy", # Allowed: npz, npz + } + + def __init__(self, features=None, config=None): + super().__init__("gen_outputs", features=features, config=config) + + @property + def mode(self): + value = self.config["interface"] + assert value in ["auto", "filesystem", "stdout", "uart"] + return value + + @property + def fmt(self): + value = self.config["fmt"] + assert value in ["npy", "npz"] + return value + + def get_platform_config(self, platform): + assert platform in ["mlif", "tvm", "microtvm"] + return { + f"{platform}.get_outputs": self.enabled, + f"{platform}.get_outputs_interface": self.interface, + f"{platform}.get_outputs_fmt": self.fmt, + } + + +@register_feature("validate_new", depends=["gen_data", "gen_ref_data", "set_inputs", "get_outputs"]) +class ValidateNew(RunFeature): + """TODO""" + + DEFAULTS = { + **FeatureBase.DEFAULTS, + } + + def __init__(self, features=None, config=None): + super().__init__("validate_new", features=features, config=config) + + def get_postprocesses(self): + # config = {} + # from mlonmcu.session.postprocess import ValidateOutputsPostprocess + # validate_outputs_postprocess = ValidateOutputsPostprocess(features=[], config=config) + # return [validate_outputs_postprocess] + return ["validate_outputs"] +>>>>>>> a3ccb1a1... WIP: add new features and postprocesses for validate_new diff --git a/mlonmcu/models/frontend.py b/mlonmcu/models/frontend.py index b27753579..1cf9b1a7b 100644 --- a/mlonmcu/models/frontend.py +++ b/mlonmcu/models/frontend.py @@ -24,6 +24,8 @@ from abc import ABC, abstractmethod from typing import Tuple, List +import numpy as np + from mlonmcu.feature.features import get_matching_features from mlonmcu.models.model import ( ModelFormats, @@ -55,7 +57,17 @@ class Frontend(ABC): DEFAULTS = { "use_inout_data": False, - # TODO: print_outputs for frontends + # the following should be configured using gen_data feature + "gen_data": False, + "gen_data_fill_mode": None, + "gen_data_file": None, + "gen_data_number": None, + "gen_data_fmt": None, + # the following should be configured using gen_ref_data feature + "gen_ref_data": False, + "gen_ref_data_mode": None, + "gen_ref_data_file": None, + "gen_ref_data_fmt": None, } REQUIRED = set() @@ -84,6 +96,52 @@ def use_inout_data(self): value = self.config["use_inout_data"] return str2bool(value) if not isinstance(value, (bool, int)) else value + @property + def gen_data(self): + value = self.config["gen_data"] + return str2bool(value) if not isinstance(value, (bool, int)) else value + + @property + def gen_data_fill_mode(self): + value = self.config["gen_data_fill_mode"] + assert value in ["random", "ones", "zeros", "file", "dataset"] + return value + + @property + def gen_data_file(self): + return self.config["gen_data_file"] + + @property + def gen_data_number(self): + return int(self.config["gen_data_number"]) + + @property + def gen_data_fmt(self): + value = self.config["gen_data_fmt"] + assert value in ["npy", "npz"] + return value + + @property + def gen_ref_data(self): + value = self.config["gen_data"] + return str2bool(value) if not isinstance(value, (bool, int)) else value + + @property + def gen_ref_data_mode(self): + value = self.config["gen_ref_data_mode"] + assert value in ["file", "model"] + return value + + @property + def gen_ref_data_file(self): + return self.config["gen_ref_data_file"] + + @property + def gen_ref_data_fmt(self): + value = self.config["gen_ref_data_fmt"] + assert value in ["npy", "npz"] + return value + def supports_formats(self, ins=None, outs=None): """Returs true if the frontend can handle at least one combination of input and output formats.""" assert ins is not None or outs is not None, "Please provide a list of input formats, outputs formats or both" @@ -145,7 +203,7 @@ def process_metadata(self, model, cfg=None): input_shapes[name] = shape if name and ty: input_types[name] = ty - if self.use_inout_data: + if self.use_inout_data or (self.gen_data and self.gen_data_fill_mode == "file" and self.gen_data_file == "auto"): if "example_input" in inp and "path" in inp["example_input"]: in_data_dir = Path(inp["example_input"]["path"]) # TODO: this will only work with relative paths to model dir! (Fallback to parent directories?) @@ -212,6 +270,209 @@ def process_metadata(self, model, cfg=None): cfg.update({f"{model.name}.input_types": input_types}) if len(output_types) > 0: cfg.update({f"{model.name}.output_types": output_types}) + # flattened version + if len(input_shapes) > 0: + assert len(input_types) in [len(input_shapes), 0] + input_names = list(input_shapes.keys()) + elif len(input_shapes) > 0: + input_names = list(input_types.keys()) + else: + input_names = [] + if len(output_shapes) > 0: + assert len(output_types) in [len(output_shapes), 0] + output_names = list(output_shapes.keys()) + elif len(output_shapes) > 0: + output_names = list(output_types.keys()) + else: + output_names = [] + model_info_dict = { + "input_names": input_names, + "output_names": output_names, + "input_shapes": list(input_shapes.values()), + "output_shapes": list(output_shapes.values()), + "input_types": list(input_types.values()), + "output_types": list(output_types.values()), + } + print("model_info_dict", model_info_dict) + artifacts = [] + if self.gen_data: + inputs_data = [] + if self.gen_data_fill_mode == "random": + raise NotImplementedError + elif self.gen_data_fill_mode == "zeros": + raise NotImplementedError + elif self.gen_data_fill_mode == "ones": + raise NotImplementedError + elif self.gen_data_fill_mode == "file": + if self.gen_data_file == "auto": + len(in_paths) > 0 + print("in_paths", in_paths) + if len(in_paths) == 1: + if in_paths[0].is_dir(): + files = list(in_paths[0].iterdir()) + else: + files = in_paths + temp = {} + for file in files: + if not isinstance(file, Path): + file = Path(file) + assert file.is_file() + print("file", file) + basename, ext = file.stem, file.suffix + if ext == ".bin": + if "_" in basename: + i, ii = basename.split("_", 1) + i = int(i) + ii = int(ii) + else: + i = int(basename) + ii = 0 + with open(file, "rb") as f: + data = f.read() + if i not in temp: + temp[i] = {} + temp[i][ii] = data + elif ext in [".npy", ".npz"]: + raise NotImplementedError + else: + raise RuntimeError(f"Unsupported ext: {ext}") + # print("temp", temp) + for i in range(min(self.gen_data_number, len(temp))): + print("i", i) + assert i in temp + data = {} + for ii, input_name in enumerate(input_names): + print("ii", ii) + assert ii in temp[i] + # assert input_name in input_types + dtype = input_types.get(input_name, "int8") # TODO: require dtype! + arr = np.frombuffer(temp[i][ii], dtype=dtype) + assert ii < len(input_shapes) + shape = input_shapes[input_name] + arr = np.reshape(arr, shape) + data[input_name] = arr + inputs_data.append(data) + else: + assert self.gen_data_file is not None, "Missing value for gen_data_file" + file = Path(self.gen_data_file) + assert file.is_file(), f"File not found: {file}" + # for i, input_name in enumerate(input_names): + + elif self.gen_data_fill_mode == "dataset": + raise NotImplementedError + else: + raise RuntimeError(f"unsupported fill_mode: {self.gen_data_fill_mode}") + print("inputs_data", inputs_data) + fmt = self.gen_data_fmt + if fmt == "npy": + with tempfile.TemporaryDirectory() as tmpdirname: + tempfilename = Path(tmpdirname) / "inputs.npy" + np.save(tempfilename, inputs_data) + with open(tempfilename, "rb") as f: + raw = f.read() + elif fmt == "npz": + raise NotImplementedError + else: + raise RuntimeError(f"Unsupported fmt: {fmt}") + assert raw + inputs_data_artifact = Artifact(f"inputs.{fmt}", raw=raw, fmt=ArtifactFormat.BIN, flags=("inputs", fmt)) + artifacts.append(inputs_data_artifact) + if self.gen_ref_data: + outputs_data = [] + if self.gen_ref_data_mode == "model": + raise NotImplementedError + elif self.gen_ref_data_mode == "file": + if self.gen_ref_data_file == "auto": + len(out_paths) > 0 + if len(out_paths) == 1: + if out_paths[0].is_dir(): + files = list(out_paths[0].iterdir()) + else: + files = out_paths + temp = {} + for file in files: + if not isinstance(file, Path): + file = Path(file) + assert file.is_file() + print("file", file) + basename, ext = file.stem, file.suffix + if ext == ".bin": + if "_" in basename: + i, ii = basename.split("_", 1) + i = int(i) + ii = int(ii) + else: + i = int(basename) + ii = 0 + with open(file, "rb") as f: + data = f.read() + if i not in temp: + temp[i] = {} + temp[i][ii] = data + elif ext in [".npy", ".npz"]: + raise NotImplementedError + else: + raise RuntimeError(f"Unsupported ext: {ext}") + # print("temp", temp) + # TODO: handle case where there are more output samples than input samples? + for i in range(len(temp)): + print("i", i) + assert i in temp + data = {} + for ii, output_name in enumerate(output_names): + print("ii", ii) + assert ii in temp[i] + # assert output_name in output_types + dtype = output_types.get(output_name, "float32") # TODO: require dtype! + arr = np.frombuffer(temp[i][ii], dtype=dtype) + assert ii < len(output_shapes) + shape = output_shapes[output_name] + arr = np.reshape(arr, shape) + data[output_name] = arr + outputs_data.append(data) + else: + assert self.gen_data_file is not None, "Missing value for gen_data_file" + file = Path(self.gen_data_file) + assert file.is_file(), f"File not found: {file}" + else: + raise RuntimeError(f"unsupported fill_mode: {self.gen_ref_data_mode}") + print("outputs_data", outputs_data) + fmt = self.gen_data_fmt + if fmt == "npy": + with tempfile.TemporaryDirectory() as tmpdirname: + tempfilename = Path(tmpdirname) / "outputs_ref.npy" + np.save(tempfilename, outputs_data) + with open(tempfilename, "rb") as f: + raw = f.read() + elif fmt == "npz": + raise NotImplementedError + else: + raise RuntimeError(f"Unsupported fmt: {fmt}") + assert raw + outputs_data_artifact = Artifact(f"outputs.{fmt}", raw=raw, fmt=ArtifactFormat.BIN, flags=("outputs", fmt)) + artifacts.append(outputs_data_artifact) + # nested version + # model_info_dict = { + # "inputs": [ + # { + # "name": "input_1", + # "shape": [1, 1014], + # "type": "int8", + # } + # ], + # "outputs": [ + # { + # "name": "output", + # "shape": [1, 10], + # "type": "int8", + # } + # ], + # } + import yaml + content = yaml.dump(model_info_dict) + model_info_artifact = Artifact("model_info.yml", content=content, fmt=ArtifactFormat.TEXT, flags=("model_info",)) + artifacts.append(model_info_artifact) + return artifacts def generate(self, model) -> Tuple[dict, dict]: artifacts = [] From 8036b3ff3ee2741e51e2ade62f802acc3cdc338f Mon Sep 17 00:00:00 2001 From: Philipp van Kempen Date: Fri, 12 Jan 2024 14:26:39 +0100 Subject: [PATCH 004/105] frontends.py: fix typo in process_metadata --- mlonmcu/models/frontend.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mlonmcu/models/frontend.py b/mlonmcu/models/frontend.py index 1cf9b1a7b..cbb062ead 100644 --- a/mlonmcu/models/frontend.py +++ b/mlonmcu/models/frontend.py @@ -227,7 +227,7 @@ def process_metadata(self, model, cfg=None): out_data_dir = Path(outp["test_output_path"]) out_path = model_dir / out_data_dir assert ( - in_path.is_dir() + out_path.is_dir() ), f"Output data directory defined in model metadata does not exist: {out_path}" out_paths.append(out_path) else: From aea3130549ca9bb39c59253e5be9f544a163a586 Mon Sep 17 00:00:00 2001 From: Philipp van Kempen Date: Fri, 12 Jan 2024 14:28:16 +0100 Subject: [PATCH 005/105] run.py: allow process_metadata to return artifacts --- mlonmcu/session/run.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/mlonmcu/session/run.py b/mlonmcu/session/run.py index 7abae9859..18fb89163 100644 --- a/mlonmcu/session/run.py +++ b/mlonmcu/session/run.py @@ -982,7 +982,15 @@ def load(self): # The following is very very dirty but required to update arena sizes via model metadata... cfg_new = {} if isinstance(self.model, Model): - self.frontend.process_metadata(self.model, cfg=cfg_new) + artifacts_ = self.frontend.process_metadata(self.model, cfg=cfg_new) + if artifacts_ is not None: + if isinstance(artifacts, dict): + assert "default" in artifacts.keys() + artifacts["default"].extend(artifacts_) + # ignore subs for now + else: + assert isinstance(artifacts, list) + artifacts.extend(artifacts_) if len(cfg_new) > 0: for key, value in cfg_new.items(): component, name = key.split(".")[:2] From b0fc7ed475556e5bc050dc785fd9ac7664f5a438 Mon Sep 17 00:00:00 2001 From: Philipp van Kempen Date: Fri, 12 Jan 2024 14:28:31 +0100 Subject: [PATCH 006/105] WIP: add new features and postprocesses for validate_new --- mlonmcu/models/frontend.py | 4 ++-- mlonmcu/session/run.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/mlonmcu/models/frontend.py b/mlonmcu/models/frontend.py index cbb062ead..f2e7e3a84 100644 --- a/mlonmcu/models/frontend.py +++ b/mlonmcu/models/frontend.py @@ -222,7 +222,7 @@ def process_metadata(self, model, cfg=None): output_shapes[name] = shape if name and ty: output_types[name] = ty - if self.use_inout_data: + if self.use_inout_data or (self.gen_ref_data and self.gen_ref_data_mode == "file" and self.gen_ref_data_file == "auto"): if "test_output_path" in outp: out_data_dir = Path(outp["test_output_path"]) out_path = model_dir / out_data_dir @@ -572,7 +572,7 @@ def produce_artifacts(self, model): # TODO: frontend parsed metadata instead of lookup.py? # TODO: how to find inout_data? class TfLiteFrontend(SimpleFrontend): - FEATURES = Frontend.FEATURES | {"visualize", "split_layers", "tflite_analyze"} + FEATURES = Frontend.FEATURES | {"visualize", "split_layers", "tflite_analyze", "gen_data", "gen_ref_data"} DEFAULTS = { **Frontend.DEFAULTS, diff --git a/mlonmcu/session/run.py b/mlonmcu/session/run.py index 18fb89163..ebb84d100 100644 --- a/mlonmcu/session/run.py +++ b/mlonmcu/session/run.py @@ -74,7 +74,7 @@ def add_any(new, base=None, append=True): class Run: """A run is single model/backend/framework/target combination with a given set of features and configs.""" - FEATURES = {"autotune", "target_optimized"} + FEATURES = {"autotune", "target_optimized", "validate_new"} DEFAULTS = { "export_optional": False, From a87b42a63788f912b28a8bdc6a1d626c7a106608 Mon Sep 17 00:00:00 2001 From: Philipp van Kempen Date: Mon, 15 Jan 2024 09:46:07 +0100 Subject: [PATCH 007/105] fix print_top dtype --- mlonmcu/flow/tvm/backend/tvmc_utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mlonmcu/flow/tvm/backend/tvmc_utils.py b/mlonmcu/flow/tvm/backend/tvmc_utils.py index f923342fb..522a1061a 100644 --- a/mlonmcu/flow/tvm/backend/tvmc_utils.py +++ b/mlonmcu/flow/tvm/backend/tvmc_utils.py @@ -176,7 +176,7 @@ def get_data_tvmc_args(mode=None, ins_file=None, outs_file=None, print_top=10): ret.extend(["--outputs", outs_file]) if print_top is not None and print_top > 0: - ret.extend(["--print-top", print_top]) + ret.extend(["--print-top", str(print_top)]) return ret From d9a91f77eb0142112e5c30b79ea031fd2ccd6aaf Mon Sep 17 00:00:00 2001 From: Philipp van Kempen Date: Mon, 15 Jan 2024 09:46:45 +0100 Subject: [PATCH 008/105] add quant/dequant details to model_info --- mlonmcu/models/frontend.py | 42 ++++++++++++++++++++++++++++++++------ 1 file changed, 36 insertions(+), 6 deletions(-) diff --git a/mlonmcu/models/frontend.py b/mlonmcu/models/frontend.py index f2e7e3a84..fc1a78eed 100644 --- a/mlonmcu/models/frontend.py +++ b/mlonmcu/models/frontend.py @@ -191,6 +191,8 @@ def process_metadata(self, model, cfg=None): output_shapes = {} input_types = {} output_types = {} + input_quant_details = {} + output_quant_details = {} if metadata is not None and "network_parameters" in metadata: network = metadata["network_parameters"] assert "input_nodes" in network @@ -198,11 +200,20 @@ def process_metadata(self, model, cfg=None): for inp in ins: name = inp.get("name", None) shape = inp.get("shape", None) - ty = inp.get("type", None) + ty = inp.get("dtype", None) + if ty is None: + ty = inp.get("type", None) # legacy + quantize = inp.get("quantize", None) if name and shape: input_shapes[name] = shape if name and ty: input_types[name] = ty + if name and quantize: + quant_scale = quantize.get("scale", None) + quant_zero_shift = quantize.get("zero_shift", None) + quant_dtype = quantize.get("dtype", None) + quant_details = [quant_scale, quant_zero_shift, quant_dtype] + input_quant_details[name] = quant_details if self.use_inout_data or (self.gen_data and self.gen_data_fill_mode == "file" and self.gen_data_file == "auto"): if "example_input" in inp and "path" in inp["example_input"]: in_data_dir = Path(inp["example_input"]["path"]) @@ -217,11 +228,20 @@ def process_metadata(self, model, cfg=None): for outp in outs: name = outp.get("name", None) shape = outp.get("shape", None) - ty = outp.get("type", None) + ty = outp.get("dtype", None) + if ty is None: + ty = outp.get("type", None) # legacy + dequantize = outp.get("dequantize", None) if name and shape: output_shapes[name] = shape if name and ty: output_types[name] = ty + if name and dequantize: + quant_scale = dequantize.get("scale", None) + quant_zero_shift = dequantize.get("zero_shift", None) + quant_dtype = dequantize.get("dtype", None) + quant_details = [quant_scale, quant_zero_shift, quant_dtype] + output_quant_details[name] = quant_details if self.use_inout_data or (self.gen_ref_data and self.gen_ref_data_mode == "file" and self.gen_ref_data_file == "auto"): if "test_output_path" in outp: out_data_dir = Path(outp["test_output_path"]) @@ -292,6 +312,8 @@ def process_metadata(self, model, cfg=None): "output_shapes": list(output_shapes.values()), "input_types": list(input_types.values()), "output_types": list(output_types.values()), + "input_quant_details": list(input_quant_details.values()), + "output_quant_details": list(output_quant_details.values()), } print("model_info_dict", model_info_dict) artifacts = [] @@ -344,8 +366,12 @@ def process_metadata(self, model, cfg=None): for ii, input_name in enumerate(input_names): print("ii", ii) assert ii in temp[i] - # assert input_name in input_types - dtype = input_types.get(input_name, "int8") # TODO: require dtype! + assert input_name in input_types, f"Unknown dtype for input: {input_name}" + dtype = input_types[input_name] + quant = input_quant_details.get(name, None) + if quant: + _, _, ty = quant + dtype = ty arr = np.frombuffer(temp[i][ii], dtype=dtype) assert ii < len(input_shapes) shape = input_shapes[input_name] @@ -422,8 +448,12 @@ def process_metadata(self, model, cfg=None): for ii, output_name in enumerate(output_names): print("ii", ii) assert ii in temp[i] - # assert output_name in output_types - dtype = output_types.get(output_name, "float32") # TODO: require dtype! + assert output_name in output_types, f"Unknown dtype for output: {output_name}" + dtype = output_types[output_name] + dequant = output_quant_details.get(name, None) + if dequant: + _, _, ty = dequant + dtype = ty arr = np.frombuffer(temp[i][ii], dtype=dtype) assert ii < len(output_shapes) shape = output_shapes[output_name] From 4d4c98cf89162830ffbb6480c20c20f5f0a5a0d7 Mon Sep 17 00:00:00 2001 From: Philipp van Kempen Date: Mon, 15 Jan 2024 09:47:28 +0100 Subject: [PATCH 009/105] WIP: add new features and postprocesses for validate_new --- mlonmcu/models/frontend.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mlonmcu/models/frontend.py b/mlonmcu/models/frontend.py index fc1a78eed..3b7e4278a 100644 --- a/mlonmcu/models/frontend.py +++ b/mlonmcu/models/frontend.py @@ -479,7 +479,7 @@ def process_metadata(self, model, cfg=None): else: raise RuntimeError(f"Unsupported fmt: {fmt}") assert raw - outputs_data_artifact = Artifact(f"outputs.{fmt}", raw=raw, fmt=ArtifactFormat.BIN, flags=("outputs", fmt)) + outputs_data_artifact = Artifact(f"outputs_ref.{fmt}", raw=raw, fmt=ArtifactFormat.BIN, flags=("outputs_ref", fmt)) artifacts.append(outputs_data_artifact) # nested version # model_info_dict = { From 3ffc3c6277a7b5fdcaecfb4b9f5bf70998f91b1f Mon Sep 17 00:00:00 2001 From: Philipp van Kempen Date: Mon, 15 Jan 2024 09:47:45 +0100 Subject: [PATCH 010/105] add assertion to SimpleFrontend --- mlonmcu/models/frontend.py | 1 + 1 file changed, 1 insertion(+) diff --git a/mlonmcu/models/frontend.py b/mlonmcu/models/frontend.py index 3b7e4278a..ec92ed717 100644 --- a/mlonmcu/models/frontend.py +++ b/mlonmcu/models/frontend.py @@ -590,6 +590,7 @@ def produce_artifacts(self, model): assert len(self.input_formats) == len(self.output_formats) == len(model.paths) == 1 artifacts = [] name = model.name + assert "/" not in name path = model.paths[0] ext = self.input_formats[0].extension with open(path, "rb") as handle: # TODO: is an onnx model raw data or text? From fb4a2179078046a4a9ac82ff797a169356133f4a Mon Sep 17 00:00:00 2001 From: Philipp van Kempen Date: Mon, 15 Jan 2024 09:49:50 +0100 Subject: [PATCH 011/105] WIP: add new features and postprocesses for validate_new (tvm platform) --- mlonmcu/platform/tvm/tvm_target.py | 11 +++--- mlonmcu/platform/tvm/tvm_target_platform.py | 40 ++++++++++++++++----- 2 files changed, 38 insertions(+), 13 deletions(-) diff --git a/mlonmcu/platform/tvm/tvm_target.py b/mlonmcu/platform/tvm/tvm_target.py index b3c6d7bc5..10f88508a 100644 --- a/mlonmcu/platform/tvm/tvm_target.py +++ b/mlonmcu/platform/tvm/tvm_target.py @@ -61,8 +61,8 @@ def exec(self, program, *args, cwd=os.getcwd(), **kwargs): if self.timeout_sec > 0: raise NotImplementedError - ret = self.platform.run(program, self) - return ret + ret, artifacts = self.platform.run(program, self) + return ret, artifacts def parse_stdout(self, out): mean_ms = None @@ -91,10 +91,11 @@ def parse_stdout(self, out): return mean_ms, median_ms, max_ms, min_ms, std_ms def get_metrics(self, elf, directory, handle_exit=None): + artifacts = [] if self.print_outputs: - out = self.exec(elf, cwd=directory, live=True, handle_exit=handle_exit) + out, artifacts = self.exec(elf, cwd=directory, live=True, handle_exit=handle_exit) else: - out = self.exec( + out, artifacts = self.exec( elf, cwd=directory, live=False, @@ -153,7 +154,7 @@ def extract_cols(line): metrics_.add("Runtime [s]", float(item["Duration (us)"]) / 1e6) metrics[item["Name"]] = metrics_ - return metrics, out, [] + return metrics, out, artifacts def get_arch(self): return "unkwown" diff --git a/mlonmcu/platform/tvm/tvm_target_platform.py b/mlonmcu/platform/tvm/tvm_target_platform.py index c521f57e0..317d7e23f 100644 --- a/mlonmcu/platform/tvm/tvm_target_platform.py +++ b/mlonmcu/platform/tvm/tvm_target_platform.py @@ -17,12 +17,15 @@ # limitations under the License. # """TVM Target Platform""" +import tempfile +from pathlib import Path from mlonmcu.config import str2bool from .tvm_rpc_platform import TvmRpcPlatform from ..platform import TargetPlatform from mlonmcu.target import get_targets from mlonmcu.target.target import Target from .tvm_target import create_tvm_platform_target +from mlonmcu.artifact import Artifact, ArtifactFormat from mlonmcu.flow.tvm.backend.tvmc_utils import ( get_bench_tvmc_args, get_data_tvmc_args, @@ -121,10 +124,10 @@ def create_target(self, name): base = Target return create_tvm_platform_target(name, self, base=base) - def get_tvmc_run_args(self): + def get_tvmc_run_args(self, ins_file=None, outs_file=None): return [ *get_data_tvmc_args( - mode=self.fill_mode, ins_file=self.ins_file, outs_file=self.outs_file, print_top=self.print_top + mode=self.fill_mode, ins_file=ins_file, outs_file=outs_file, print_top=self.print_top ), *get_bench_tvmc_args( print_time=True, profile=self.profile, end_to_end=False, repeat=self.repeat, number=self.number @@ -132,18 +135,39 @@ def get_tvmc_run_args(self): *get_rpc_tvmc_args(self.use_rpc, self.rpc_key, self.rpc_hostname, self.rpc_port), ] - def invoke_tvmc_run(self, *args, target=None): + def invoke_tvmc_run(self, *args, target=None, **kwargs): assert target is not None, "Target required for tvmc run" combined_args = [] combined_args.extend(["--device", target.device]) - return self.invoke_tvmc("run", *args) + return self.invoke_tvmc("run", *args, **kwargs) def run(self, elf, target, timeout=120): # TODO: implement timeout # Here, elf is actually a directory # TODO: replace workaround with possibility to pass TAR directly tar_path = str(elf) - args = [tar_path] + self.get_tvmc_run_args() - output = self.invoke_tvmc_run(*args, target=target) - - return output + in_path = self.ins_file + out_path = self.outs_file + set_inputs = False + get_outputs = True + artifacts = [] + with tempfile.TemporaryDirectory() as tmp_dir: + if set_inputs and in_path is None: + in_path = Path(tmp_dir) / "ins.npz" + # TODO: populate + if get_outputs and out_path is None: + out_path = Path(tmp_dir) / "outs.npz" + args = [tar_path] + self.get_tvmc_run_args(ins_file=in_path, outs_file=out_path) + output = self.invoke_tvmc_run(*args, target=target, cwd=tmp_dir) + if get_outputs and self.outs_file is None: + import numpy as np + with np.load(out_path) as out_data: + outs_data = [dict(out_data)] + outs_path = Path(tmp_dir) / "outputs.npy" + np.save(outs_path, outs_data) + with open(outs_path, "rb") as f: + outs_raw = f.read() + outputs_artifact = Artifact("outputs.npy", raw=outs_raw, fmt=ArtifactFormat.BIN, flags=("outputs", "npy")) + artifacts.append(outputs_artifact) + + return output, artifacts From 59c89dc369b741ee546ed6bafa1e6df8dba63c3d Mon Sep 17 00:00:00 2001 From: Philipp van Kempen Date: Mon, 15 Jan 2024 09:50:04 +0100 Subject: [PATCH 012/105] fix print_top dtype 2 --- mlonmcu/platform/tvm/tvm_target_platform.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/mlonmcu/platform/tvm/tvm_target_platform.py b/mlonmcu/platform/tvm/tvm_target_platform.py index 317d7e23f..c609f548e 100644 --- a/mlonmcu/platform/tvm/tvm_target_platform.py +++ b/mlonmcu/platform/tvm/tvm_target_platform.py @@ -75,7 +75,8 @@ def outs_file(self): @property def print_top(self): - return self.config["print_top"] + value = self.config["print_top"] + return int(value) if isinstance(value, str) else None @property def profile(self): From 2e10d319e13a9e505c150cb4d0801f40032ec6f9 Mon Sep 17 00:00:00 2001 From: Philipp van Kempen Date: Mon, 15 Jan 2024 09:50:43 +0100 Subject: [PATCH 013/105] WIP: add new features and postprocesses for validate_new (validate_outputs postprocess) --- mlonmcu/session/postprocess/postprocesses.py | 103 +++++++++++++++++++ 1 file changed, 103 insertions(+) diff --git a/mlonmcu/session/postprocess/postprocesses.py b/mlonmcu/session/postprocess/postprocesses.py index 99a9ccc3f..aba4d7265 100644 --- a/mlonmcu/session/postprocess/postprocesses.py +++ b/mlonmcu/session/postprocess/postprocesses.py @@ -1455,3 +1455,106 @@ def post_run(self, report, artifacts): report.post_df = post_df assert self.to_file or self.to_df, "Either to_file or to_df have to be true" return ret_artifacts + + +class ValidateOutputsPostprocess(RunPostprocess): + """Postprocess for comparing model outputs with golden reference.""" + + DEFAULTS = { + **RunPostprocess.DEFAULTS, + "atol": 0.0, + "rtol": 0.0, + "report": False + } + + def __init__(self, features=None, config=None): + super().__init__("validate_outputs", features=features, config=config) + + @property + def atol(self): + """Get atol property.""" + value = self.config["atol"] + return float(value) + + @property + def rtol(self): + """Get rtol property.""" + value = self.config["rtol"] + return float(value) + + @property + def report(self): + """Get report property.""" + value = self.config["report"] + return str2bool(value) if not isinstance(value, (bool, int)) else value + + def post_run(self, report, artifacts): + """Called at the end of a run.""" + model_info_artifact = lookup_artifacts(artifacts, name="model_info.yml", first_only=True) + assert len(model_info_artifact) == 1, "Could not find artifact: model_info.yml" + model_info_artifact = model_info_artifact[0] + import yaml + model_info_data = yaml.safe_load(model_info_artifact.content) + print("model_info_data", model_info_data) + if len(model_info_data["output_names"]) > 1: + raise NotImplementedError("Multi-outputs not yet supported.") + outputs_ref_artifact = lookup_artifacts(artifacts, name="outputs_ref.npy", first_only=True) + assert len(outputs_ref_artifact) == 1, "Could not find artifact: outputs_ref.npy" + outputs_ref_artifact = outputs_ref_artifact[0] + import numpy as np + outputs_ref = np.load(outputs_ref_artifact.path, allow_pickle=True) + import copy + # outputs = copy.deepcopy(outputs_ref) + # outputs[1][list(outputs[1].keys())[0]][0] = 42 + outputs_artifact = lookup_artifacts(artifacts, name="outputs.npy", first_only=True) + assert len(outputs_artifact) == 1, "Could not find artifact: outputs.npy" + outputs_artifact = outputs_artifact[0] + outputs = np.load(outputs_artifact.path, allow_pickle=True) + compared = 0 + matching = 0 + missing = 0 + for i, output_ref in enumerate(outputs_ref): + if i >= len(outputs): + logger.warning("Missing output sample") + missing += 1 + break + output = outputs[i] + ii = 0 + for out_name, out_ref_data in output_ref.items(): + if out_name in output: + out_data = output[out_name] + elif ii < len(output): + if isinstance(output, dict): + # fallback for custom name-based npy dict + out_data = list(output.values())[ii] + else: # fallback for index-based npy array + assert isinstance(output, (list, np.array)), "expected dict, list of np.array type" + out_data = output[ii] + else: + RuntimeError(f"Output not found: {out_name}") + # optional dequantize + quant = model_info_data.get("output_quant_details", None) + if quant: + assert ii < len(quant) + quant = quant[ii] + if quant: + quant_scale, quant_zero_point, quant_dtype = quant + if quant_dtype: + if out_data.dtype.name != quant_dtype: + # need to dequantize here + assert out_data.dtype.name in ["int8"], "Dequantization only supported for int8 input" + assert quant_dtype in ["float32"], "Dequantization only supported for float32 output" + out_data = (out_data.astype("float32") - quant_zero_point) * quant_scale + if np.allclose(out_data, out_ref_data, rtol=0, atol=0): + matching += 1 + compared += 1 + ii += 1 + if self.report: + raise NotImplementedError + if self.atol: + raise NotImplementedError + if self.rtol: + raise NotImplementedError + res = f"{matching}/{compared} ({int(matching/compared*100)}%)" + report.post_df["Validation Result"] = res + return [] From bb5d710655be4df92bd037cb977ac11667b9bb45 Mon Sep 17 00:00:00 2001 From: Philipp van Kempen Date: Tue, 16 Jan 2024 14:03:29 +0100 Subject: [PATCH 014/105] WIP: add new features and postprocesses for validate_new --- mlonmcu/feature/features.py | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/mlonmcu/feature/features.py b/mlonmcu/feature/features.py index 2d7ed3471..7c2151e66 100644 --- a/mlonmcu/feature/features.py +++ b/mlonmcu/feature/features.py @@ -2313,20 +2313,22 @@ class SetInputs(PlatformFeature): # TODO: use custom stage instead of LOAD DEFAULTS = { **FeatureBase.DEFAULTS, - "interface": "auto", # Allowed: auto, rom, filesystem, stdout, uart + "interface": "auto", # Allowed: auto, rom, filesystem, stdin, stdin_raw, uart } def __init__(self, features=None, config=None): super().__init__("set_inputs", features=features, config=config) @property - def mode(self): + def interface(self): value = self.config["interface"] - assert value in ["auto", "rom", "filesystem", "stdin", "uart"] + assert value in ["auto", "rom", "filesystem", "stdin", "stdin_raw", "uart"] return value def get_platform_config(self, platform): assert platform in ["mlif", "tvm", "microtvm"] + # if platform in ["tvm", "mircotvm"]: + # assert self.interface in ["auto", "filesystem"] # if tvm/microtvm: allow using --fill-mode provided by tvmc run return { f"{platform}.set_inputs": self.enabled, @@ -2340,17 +2342,17 @@ class GetOutputs(PlatformFeature): # TODO: use custom stage instead of LOAD DEFAULTS = { **FeatureBase.DEFAULTS, - "interface": "auto", # Allowed: auto, filesystem, stdout, uart + "interface": "auto", # Allowed: auto, filesystem, stdout, stdout_raw, uart "fmt": "npy", # Allowed: npz, npz } def __init__(self, features=None, config=None): - super().__init__("gen_outputs", features=features, config=config) + super().__init__("get_outputs", features=features, config=config) @property - def mode(self): + def interface(self): value = self.config["interface"] - assert value in ["auto", "filesystem", "stdout", "uart"] + assert value in ["auto", "filesystem", "stdout", "stdout_raw", "uart"] return value @property @@ -2361,6 +2363,8 @@ def fmt(self): def get_platform_config(self, platform): assert platform in ["mlif", "tvm", "microtvm"] + # if platform in ["tvm", "mircotvm"]: + # assert self.interface in ["auto", "filesystem", "stdout"] return { f"{platform}.get_outputs": self.enabled, f"{platform}.get_outputs_interface": self.interface, From c8b0fab9547c6b01367f605cf286caae47c69dad Mon Sep 17 00:00:00 2001 From: Philipp van Kempen Date: Tue, 16 Jan 2024 14:03:54 +0100 Subject: [PATCH 015/105] tvmc_utils: disable print_top by default --- mlonmcu/flow/tvm/backend/tvmc_utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mlonmcu/flow/tvm/backend/tvmc_utils.py b/mlonmcu/flow/tvm/backend/tvmc_utils.py index 522a1061a..8bcb6894b 100644 --- a/mlonmcu/flow/tvm/backend/tvmc_utils.py +++ b/mlonmcu/flow/tvm/backend/tvmc_utils.py @@ -164,7 +164,7 @@ def get_tvmrt_tvmc_args(runtime="crt", system_lib=True, link_params=True): return ret -def get_data_tvmc_args(mode=None, ins_file=None, outs_file=None, print_top=10): +def get_data_tvmc_args(mode=None, ins_file=None, outs_file=None, print_top=None): ret = [] if ins_file is not None: ret.extend(["--inputs", ins_file]) From d497193727d5f609078b338e824deec076050348 Mon Sep 17 00:00:00 2001 From: Philipp van Kempen Date: Tue, 16 Jan 2024 14:05:24 +0100 Subject: [PATCH 016/105] WIP: add new features and postprocesses for validate_new (mlif) --- mlonmcu/platform/mlif/mlif.py | 53 +++++++++ mlonmcu/platform/mlif/mlif_target.py | 162 +++++++++++++++++++++++++++ 2 files changed, 215 insertions(+) diff --git a/mlonmcu/platform/mlif/mlif.py b/mlonmcu/platform/mlif/mlif.py index 4358a0531..cb79a5bef 100644 --- a/mlonmcu/platform/mlif/mlif.py +++ b/mlonmcu/platform/mlif/mlif.py @@ -57,6 +57,8 @@ class MlifPlatform(CompilePlatform, TargetPlatform): "auto_vectorize", "benchmark", "xpulp", + "set_inputs", + "get_outputs", } # TODO: allow Feature-Features with automatic resolution of initialization order ) @@ -82,6 +84,11 @@ class MlifPlatform(CompilePlatform, TargetPlatform): "fuse_ld": None, "strip_strings": False, "goal": "generic_mlonmcu", # Use 'generic_mlif' for older version of MLIF + "set_inputs": False, + "set_inputs_interface": None, + "get_outputs": False, + "get_outputs_interface": None, + "get_outputs_fmt": None, } REQUIRED = {"mlif.src_dir"} @@ -100,6 +107,52 @@ def __init__(self, features=None, config=None): def goal(self): return self.config["goal"] + @property + def set_inputs(self): + value = self.config["set_inputs"] + return str2bool(value) if not isinstance(value, (bool, int)) else value + + @property + def set_inputs_interface(self): + value = self.config["set_inputs_interface"] + return value + + @property + def get_outputs(self): + value = self.config["get_outputs"] + return str2bool(value) if not isinstance(value, (bool, int)) else value + + @property + def get_outputs_interface(self): + value = self.config["get_outputs_interface"] + return value + + @property + def get_outputs_fmt(self): + value = self.config["get_outputs_fmt"] # TODO: use + return value + + @property + def inputs_artifact(self): + # THIS IS A HACK (get inputs fom artifacts!) + lookup_path = self.build_dir.parent / "inputs.npy" + if lookup_path.is_file(): + return lookup_path + else: + logger.warning("Artifact 'inputs.npz' not found!") + return None + + @property + def model_info_file(self): + # THIS IS A HACK (get inputs fom artifacts!) + lookup_path = self.build_dir.parent / "model_info.yml" + if lookup_path.is_file(): + return lookup_path + else: + logger.warning("Artifact 'model_info.yml' not found!") + return None + + def gen_data_artifact(self): in_paths = self.input_data_path if not isinstance(in_paths, list): diff --git a/mlonmcu/platform/mlif/mlif_target.py b/mlonmcu/platform/mlif/mlif_target.py index f8249aa13..854e6a2ac 100644 --- a/mlonmcu/platform/mlif/mlif_target.py +++ b/mlonmcu/platform/mlif/mlif_target.py @@ -16,9 +16,12 @@ # See the License for the specific language governing permissions and # limitations under the License. # +import os +from pathlib import Path from enum import IntEnum from mlonmcu.target import get_targets, Target +from mlonmcu.artifact import Artifact, ArtifactFormat from mlonmcu.logging import get_logger logger = get_logger() @@ -49,6 +52,165 @@ def __init__(self, features=None, config=None): self.platform = platform self.validation_result = None + def exec(self, program, *args, cwd=os.getcwd(), **kwargs): + ins_file = None + num_inputs = 0 + batch_size = 5 # idea is to process i.e. 5 samples per batch, repeating until no samples remain + in_interface = None + out_interface = None + if self.platform.set_inputs: + # first figure out how many inputs are provided + assert self.platform.inputs_artifact is not None + import numpy as np + data = np.load(self.platform.inputs_artifact, allow_pickle=True) + # print("data", data, type(data)) + num_inputs = len(data) + in_interface = self.platform.set_inputs_interface + if in_interface == "auto": + if self.supports_filesystem: + in_interface = "filesystem" + # TODO: eventually update batch_size + elif self.supports_stdin: + in_interface = "stdin_raw" + # TODO: also allow stdin? + # TODO: eventually update batch_size + else: # Fallback + in_interface = "rom" + batch_size = 1e6 # all inputs are in already compiled into program + else: + assert in_interface in ["filesystem", "stdin", "stdin_raw", "rom"] + if in_interface == "filesystem": + pass + elif in_interface == "stdin": + pass + elif in_interface == "stdin_raw": + pass + elif in_interface == "rom": + pass # nothing to do + outs_file = None + encoding = "utf-8" + if self.platform.get_outputs: + out_interface = self.platform.get_outputs_interface + if out_interface == "auto": + if self.supports_filesystem: + out_interface = "filesystem" + elif self.supports_stdout: + out_interface = "stdout_raw" + # TODO: support stdout? + else: + assert out_interface in ["filesystem", "stdout", "stdout_raw"] + if out_interface == "filesystem": + pass + elif out_interface == "stdout": + pass + elif out_interface == "stdout_raw": + encoding = None + ret = "" + artifacts = [] + num_batches = max(round(num_inputs / batch_size), 1) + processed_inputs = 0 + remaining_inputs = num_inputs + outs_data = [] + stdin_data = None + for idx in range(num_batches): + # print("idx", idx) + # current_batch_size = max(min(batch_size, remaining_inputs), 1) + if processed_inputs < num_inputs: + if in_interface == "filesystem": + batch_data = data[idx * batch_size:((idx + 1) * batch_size)] + # print("batch_data", batch_data, type(batch_data)) + ins_file = Path(cwd) / "ins.npy" + np.save(ins_file, batch_data) + elif in_interface == "stdin": + raise NotImplementedError + elif in_interface == "stdin_raw": + batch_data = data[idx * batch_size:((idx + 1) * batch_size)] + # print("batch_data", batch_data, type(batch_data)) + stdin_data = b"" + for cur_data in batch_data: + # print("cur_data", cur_data) + for key, value in cur_data.items(): + # print("key", key) + # print("value", value, type(value)) + # print("value.tostring", value.tostring()) + stdin_data += value.tostring() + # TODO: check that stdin_data has expected size + # This is just a placeholder example! + # stdin_data = "input[0] = {0, 1, 2, ...};\nDONE\n""".encode() + # stdin_data *= 200 + # raise NotImplementedError + # TODO: generate input stream here! + + ret_, artifacts_ = super().exec(program, *args, cwd=cwd, **kwargs, stdin_data=stdin_data, encoding=encoding) + if self.platform.get_outputs: + if out_interface == "filesystem": + import numpy as np + outs_file = Path(cwd) / "outs.npy" + with np.load(outs_file) as out_data: + outs_data.extend(dict(out_data)) + elif out_interface == "stdout": + # TODO: get output_data from stdout + raise NotImplementedError + elif out_interface == "stdout_raw": + # DUMMY BELOW + model_info_file = self.platform.model_info_file # TODO: replace workaround (add model info to platform?) + assert model_info_file is not None + import yaml + with open(model_info_file, "r") as f: + model_info_data = yaml.safe_load(f) + # print("model_info_data", model_info_data) + # dtype = "int8" + # shape = [1, 10] + # input("!") + # print("ret_", ret_, type(ret_)) + # out_idx = 0 + x = ret_ # Does this copy? + while True: + out_data_temp = {} + # substr = ret_[ret_.find("-?-".encode())+3:ret_.find("-!-".encode())] + # print("substr", substr, len(substr)) + found_start = x.find("-?-".encode()) + # print("found_start", found_start) + if found_start < 0: + break + x = x[found_start+3:] + # print("x[:20]", x[:20]) + found_end = x.find("-!-".encode()) + # print("found_end", found_end) + assert found_end >= 0 + x_ = x[:found_end] + x = x[found_end+3:] + # print("x[:20]", x[:20]) + # print("x_", x_) + # out_idx += 1 + dtype = model_info_data["output_types"][0] + arr = np.frombuffer(x_, dtype=dtype) + # print("arr", arr) + shape = model_info_data["output_shapes"][0] + arr = arr.reshape(shape) + # print("arr2", arr) + assert len(model_info_data["output_names"]) == 1, "Multi-output models not yet supported" + out_name = model_info_data["output_names"][0] + out_data_temp[out_name] = arr + outs_data.append(out_data_temp) + # {"output_0": arr}]) + ret_ = ret_.decode("utf-8", errors="replace") + # raise NotImplementedError + else: + assert False + ret += ret_ + artifacts += artifacts_ + # print("outs_data", outs_data) + # input("$") + if len(outs_data) > 0: + outs_path = Path(cwd) / "outputs.npy" + np.save(outs_path, outs_data) + with open(outs_path, "rb") as f: + outs_raw = f.read() + outputs_artifact = Artifact("outputs.npy", raw=outs_raw, fmt=ArtifactFormat.BIN, flags=("outputs", "npy")) + artifacts.append(outputs_artifact) + return ret, artifacts + def get_metrics(self, elf, directory, handle_exit=None): # This is wrapper around the original exec function to catch special return codes thrown by the inout data # feature (TODO: catch edge cases: no input data available (skipped) and no return code (real hardware)) From 844020c8874db2a14d32d4c27a7c7091129a4798 Mon Sep 17 00:00:00 2001 From: Philipp van Kempen Date: Tue, 16 Jan 2024 14:06:15 +0100 Subject: [PATCH 017/105] WIP: add new features and postprocesses for validate_new (tvm) --- mlonmcu/platform/tvm/tvm_target.py | 102 +++++++++++++++++++- mlonmcu/platform/tvm/tvm_target_platform.py | 83 +++++++++++----- 2 files changed, 159 insertions(+), 26 deletions(-) diff --git a/mlonmcu/platform/tvm/tvm_target.py b/mlonmcu/platform/tvm/tvm_target.py index 10f88508a..bdc37977e 100644 --- a/mlonmcu/platform/tvm/tvm_target.py +++ b/mlonmcu/platform/tvm/tvm_target.py @@ -18,9 +18,11 @@ # import re import os +from pathlib import Path from mlonmcu.target.target import Target from mlonmcu.target.metrics import Metrics +from mlonmcu.artifact import Artifact, ArtifactFormat from mlonmcu.logging import get_logger @@ -61,7 +63,97 @@ def exec(self, program, *args, cwd=os.getcwd(), **kwargs): if self.timeout_sec > 0: raise NotImplementedError - ret, artifacts = self.platform.run(program, self) + ins_file = None + num_inputs = 0 + batch_size = 1 + if self.platform.set_inputs: + interface = self.platform.set_inputs_interface + if interface == "auto": + if self.supports_filesystem: + interface = "filesystem" + else: + assert interface in ["filesystem"] + if interface == "filesystem": + ins_file = self.platform.ins_file + if ins_file is None: + assert self.platform.inputs_artifact is not None + import numpy as np + data = np.load(self.platform.inputs_artifact, allow_pickle=True) + print("data", data, type(data)) + num_inputs = len(data) + ins_file = Path(cwd) / "ins.npz" + outs_file = None + print_top = self.platform.print_top + print("before IF") + if self.platform.get_outputs: + print("IF") + interface = self.platform.get_outputs_interface + if interface == "auto": + if self.supports_filesystem: + interface = "filesystem" + elif self.supports_stdout: + interface = "stdout" + else: + assert interface in ["filesystem", "stdout"] + if interface == "filesystem": + outs_file = self.platform.outs_file + if outs_file is None: + outs_file = Path(cwd) / "outs.npz" + elif interface == "stdout": + print_top = 1e6 + + + ret = "" + artifacts = [] + num_batches = max(round(num_inputs / batch_size), 1) + processed_inputs = 0 + remaining_inputs = num_inputs + outs_data = [] + for idx in range(num_batches): + print("idx", idx) + current_batch_size = max(min(batch_size, remaining_inputs), 1) + assert current_batch_size == 1 + if processed_inputs < num_inputs: + print("!!!") + in_data = data[idx] + print("in_data", in_data, type(in_data)) + np.savez(ins_file, **in_data) + print("saved!") + processed_inputs += 1 + remaining_inputs -= 1 + else: + ins_file = None + ret_, artifacts_ = self.platform.run(program, self, cwd=cwd, ins_file=ins_file, outs_file=outs_file, print_top=print_top) + ret += ret_ + print("self.platform.get_outputs", self.platform.get_outputs) + if self.platform.get_outputs: + interface = self.platform.get_outputs_interface + # print("interface", interface) + if interface == "auto": + if self.supports_filesystem: + interface = "filesystem" + elif self.supports_stdout: + interface = "stdout" + else: + assert interface in ["filesystem", "stdout"] + if interface == "filesystem": + import numpy as np + with np.load(outs_file) as out_data: + outs_data.append(dict(out_data)) + elif interface == "stdout": + raise NotImplementedError + else: + assert False + print("outs_data", outs_data) + # input("$") + if len(outs_data) > 0: + outs_path = Path(cwd) / "outputs.npy" + np.save(outs_path, outs_data) + with open(outs_path, "rb") as f: + outs_raw = f.read() + outputs_artifact = Artifact("outputs.npy", raw=outs_raw, fmt=ArtifactFormat.BIN, flags=("outputs", "npy")) + artifacts.append(outputs_artifact) + return ret, artifacts def parse_stdout(self, out): @@ -163,4 +255,12 @@ def update_environment(self, env): # TODO: implement in base class? pass + @property + def supports_filesystem(self): + return True + + @property + def supports_stdout(self): + return True + return TvmPlatformTarget diff --git a/mlonmcu/platform/tvm/tvm_target_platform.py b/mlonmcu/platform/tvm/tvm_target_platform.py index c609f548e..cd2623938 100644 --- a/mlonmcu/platform/tvm/tvm_target_platform.py +++ b/mlonmcu/platform/tvm/tvm_target_platform.py @@ -17,7 +17,7 @@ # limitations under the License. # """TVM Target Platform""" -import tempfile +import os from pathlib import Path from mlonmcu.config import str2bool from .tvm_rpc_platform import TvmRpcPlatform @@ -31,6 +31,9 @@ get_data_tvmc_args, get_rpc_tvmc_args, ) +from mlonmcu.logging import get_logger + +logger = get_logger() class TvmTargetPlatform(TargetPlatform, TvmRpcPlatform): @@ -42,6 +45,8 @@ class TvmTargetPlatform(TargetPlatform, TvmRpcPlatform): | { "benchmark", "tvm_profile", + "set_inputs", + "get_outputs", } ) @@ -57,6 +62,11 @@ class TvmTargetPlatform(TargetPlatform, TvmRpcPlatform): "number": 1, "aggregate": "none", # Allowed: avg, max, min, none, all "total_time": False, + "set_inputs": False, + "set_inputs_interface": None, + "get_outputs": False, + "get_outputs_interface": None, + "get_outputs_fmt": None, } REQUIRED = TargetPlatform.REQUIRED | TvmRpcPlatform.REQUIRED @@ -102,6 +112,41 @@ def total_time(self): value = self.config["total_time"] return str2bool(value) if not isinstance(value, (bool, int)) else value + @property + def set_inputs(self): + value = self.config["set_inputs"] + return str2bool(value) if not isinstance(value, (bool, int)) else value + + @property + def set_inputs_interface(self): + value = self.config["set_inputs_interface"] + return value + + @property + def get_outputs(self): + value = self.config["get_outputs"] + return str2bool(value) if not isinstance(value, (bool, int)) else value + + @property + def get_outputs_interface(self): + value = self.config["get_outputs_interface"] + return value + + @property + def get_outputs_fmt(self): + value = self.config["get_outputs_fmt"] # TODO: use + return value + + @property + def inputs_artifact(self): + # THIS IS A HACK (get inputs fom artifacts!) + lookup_path = self.project_dir.parent / "inputs.npy" + if lookup_path.is_file(): + return lookup_path + else: + logger.warning("Artifact 'inputs.npz' not found!") + return None + def flash(self, elf, target, timeout=120): raise NotImplementedError @@ -142,33 +187,21 @@ def invoke_tvmc_run(self, *args, target=None, **kwargs): combined_args.extend(["--device", target.device]) return self.invoke_tvmc("run", *args, **kwargs) - def run(self, elf, target, timeout=120): + def run(self, elf, target, timeout=120, cwd=os.getcwd(), ins_file=None, outs_file=None, print_top=None): + artifacts = [] # TODO: implement timeout # Here, elf is actually a directory # TODO: replace workaround with possibility to pass TAR directly tar_path = str(elf) - in_path = self.ins_file - out_path = self.outs_file - set_inputs = False - get_outputs = True - artifacts = [] - with tempfile.TemporaryDirectory() as tmp_dir: - if set_inputs and in_path is None: - in_path = Path(tmp_dir) / "ins.npz" - # TODO: populate - if get_outputs and out_path is None: - out_path = Path(tmp_dir) / "outs.npz" - args = [tar_path] + self.get_tvmc_run_args(ins_file=in_path, outs_file=out_path) - output = self.invoke_tvmc_run(*args, target=target, cwd=tmp_dir) - if get_outputs and self.outs_file is None: - import numpy as np - with np.load(out_path) as out_data: - outs_data = [dict(out_data)] - outs_path = Path(tmp_dir) / "outputs.npy" - np.save(outs_path, outs_data) - with open(outs_path, "rb") as f: - outs_raw = f.read() - outputs_artifact = Artifact("outputs.npy", raw=outs_raw, fmt=ArtifactFormat.BIN, flags=("outputs", "npy")) - artifacts.append(outputs_artifact) + # in_path = self.ins_file + # out_path = self.outs_file + # set_inputs = False + # if set_inputs and in_path is None: + # in_path = Path(cwd) / "ins.npz" + # # TODO: populate + # if self.get_outputs and self.get_outputs_interface == "filesystem" and out_path is None: + # out_path = Path(cwd) / "outs.npz" + args = [tar_path] + self.get_tvmc_run_args(ins_file=ins_file, outs_file=outs_file, print_top=print_top) + output = self.invoke_tvmc_run(*args, target=target, cwd=cwd) return output, artifacts From 9a23c7fa92d5b72e3c6797b4ea946e2c69b07047 Mon Sep 17 00:00:00 2001 From: Philipp van Kempen Date: Tue, 16 Jan 2024 14:06:31 +0100 Subject: [PATCH 018/105] tvmc_utils: disable print_top by default 2 --- mlonmcu/platform/tvm/tvm_target_platform.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mlonmcu/platform/tvm/tvm_target_platform.py b/mlonmcu/platform/tvm/tvm_target_platform.py index cd2623938..2defae063 100644 --- a/mlonmcu/platform/tvm/tvm_target_platform.py +++ b/mlonmcu/platform/tvm/tvm_target_platform.py @@ -170,10 +170,10 @@ def create_target(self, name): base = Target return create_tvm_platform_target(name, self, base=base) - def get_tvmc_run_args(self, ins_file=None, outs_file=None): + def get_tvmc_run_args(self, ins_file=None, outs_file=None, print_top=None): return [ *get_data_tvmc_args( - mode=self.fill_mode, ins_file=ins_file, outs_file=outs_file, print_top=self.print_top + mode=self.fill_mode, ins_file=ins_file, outs_file=outs_file, print_top=print_top ), *get_bench_tvmc_args( print_time=True, profile=self.profile, end_to_end=False, repeat=self.repeat, number=self.number From 119e13cf09d6fa4816d1e9b781c4432d78abb45b Mon Sep 17 00:00:00 2001 From: Philipp van Kempen Date: Tue, 16 Jan 2024 14:06:59 +0100 Subject: [PATCH 019/105] WIP: add new features and postprocesses for validate_new (postproc atol rtol) --- mlonmcu/session/postprocess/postprocesses.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mlonmcu/session/postprocess/postprocesses.py b/mlonmcu/session/postprocess/postprocesses.py index aba4d7265..86a617512 100644 --- a/mlonmcu/session/postprocess/postprocesses.py +++ b/mlonmcu/session/postprocess/postprocesses.py @@ -1545,7 +1545,7 @@ def post_run(self, report, artifacts): assert out_data.dtype.name in ["int8"], "Dequantization only supported for int8 input" assert quant_dtype in ["float32"], "Dequantization only supported for float32 output" out_data = (out_data.astype("float32") - quant_zero_point) * quant_scale - if np.allclose(out_data, out_ref_data, rtol=0, atol=0): + if np.allclose(out_data, out_ref_data, rtol=0.1, atol=0.1): matching += 1 compared += 1 ii += 1 From fec58b470a81aa2093a2d0c280b09d4a3b5f2cd9 Mon Sep 17 00:00:00 2001 From: Philipp van Kempen Date: Tue, 16 Jan 2024 14:12:20 +0100 Subject: [PATCH 020/105] target/common.yml: Allow overriding stdout encoding for execute utility --- mlonmcu/setup/utils.py | 21 +++++++++++++++++---- 1 file changed, 17 insertions(+), 4 deletions(-) diff --git a/mlonmcu/setup/utils.py b/mlonmcu/setup/utils.py index 200f8d468..a8bc309c8 100644 --- a/mlonmcu/setup/utils.py +++ b/mlonmcu/setup/utils.py @@ -195,6 +195,7 @@ def execute( print_func: Callable = print, handle_exit: Optional[Callable] = None, err_func: Callable = logger.error, + encoding: Optional[str] = "utf-8", prefix: str = "", **kwargs, ) -> str: @@ -214,6 +215,8 @@ def execute( Handler for exit code. err_func : Callable Function which should be used to print errors. + encoding: str, optional + Used encoding for the stdout. kwargs: dict Arbitrary keyword arguments passed through to the subprocess. @@ -241,14 +244,19 @@ def execute( ) as process: try: for line in process.stdout: - new_line = prefix + line.decode(errors="replace") + if encoding: + line = line.decode(encoding, errors="replace") + new_line = prefix + line out_str = out_str + new_line print_func(new_line.replace("\n", "")) exit_code = None while exit_code is None: exit_code = process.poll() if handle_exit is not None: - exit_code = handle_exit(exit_code, out=out_str) + out_str_ = out_str + if encoding is None: + out_str_ = out_str_.decode("utf-8", errors="ignore") + exit_code = handle_exit(exit_code, out=out_str_) assert exit_code == 0, "The process returned an non-zero exit code {}! (CMD: `{}`)".format( exit_code, " ".join(list(map(str, args))) ) @@ -259,12 +267,17 @@ def execute( else: try: p = subprocess.Popen([i for i in args], **kwargs, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) - out_str = p.communicate()[0].decode(errors="replace") + out_str = p.communicate()[0] + if encoding: + out_str = out_str.decode(encoding, errors="replace") out_str = prefix + out_str exit_code = p.poll() # print_func(out_str) if handle_exit is not None: - exit_code = handle_exit(exit_code, out=out_str) + out_str_ = out_str + if encoding is None: + out_str_ = out_str_.decode("utf-8", errors="ignore") + exit_code = handle_exit(exit_code, out=out_str_) if exit_code != 0: err_func(out_str) assert exit_code == 0, "The process returned an non-zero exit code {}! (CMD: `{}`)".format( From a2426f5b71acfa769f9c9a47016b03bc212acf7e Mon Sep 17 00:00:00 2001 From: Philipp van Kempen Date: Tue, 16 Jan 2024 14:13:09 +0100 Subject: [PATCH 021/105] target/common.py: Add stdin_data argument (needs live=False) --- mlonmcu/setup/utils.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/mlonmcu/setup/utils.py b/mlonmcu/setup/utils.py index a8bc309c8..1bec7b2bd 100644 --- a/mlonmcu/setup/utils.py +++ b/mlonmcu/setup/utils.py @@ -196,6 +196,7 @@ def execute( handle_exit: Optional[Callable] = None, err_func: Callable = logger.error, encoding: Optional[str] = "utf-8", + stdin_data: Optional[bytes] = None, prefix: str = "", **kwargs, ) -> str: @@ -217,6 +218,8 @@ def execute( Function which should be used to print errors. encoding: str, optional Used encoding for the stdout. + stdin_data: bytes, optional + Send this to the stdin of the process. kwargs: dict Arbitrary keyword arguments passed through to the subprocess. @@ -243,6 +246,10 @@ def execute( stderr=subprocess.STDOUT, ) as process: try: + if stdin_data: + raise RuntimeError("stdin_data only supported if live=False") + # not working... + # process.stdin.write(stdin_data) for line in process.stdout: if encoding: line = line.decode(encoding, errors="replace") @@ -267,7 +274,10 @@ def execute( else: try: p = subprocess.Popen([i for i in args], **kwargs, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) - out_str = p.communicate()[0] + if stdin_data: + out_str = p.communicate(input=stdin_data)[0] + else: + out_str = p.communicate()[0] if encoding: out_str = out_str.decode(encoding, errors="replace") out_str = prefix + out_str From 3a0679b45b552d881f7b2a68233e0f773f995957 Mon Sep 17 00:00:00 2001 From: Philipp van Kempen Date: Tue, 16 Jan 2024 14:33:24 +0100 Subject: [PATCH 022/105] target.exec may return artifacts now --- mlonmcu/target/riscv/spike.py | 8 ++++---- mlonmcu/target/target.py | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/mlonmcu/target/riscv/spike.py b/mlonmcu/target/riscv/spike.py index 0d327d834..ca5295cab 100644 --- a/mlonmcu/target/riscv/spike.py +++ b/mlonmcu/target/riscv/spike.py @@ -152,7 +152,7 @@ def exec(self, program, *args, cwd=os.getcwd(), **kwargs): cwd=cwd, **kwargs, ) - return ret + return ret, [] def parse_stdout(self, out, metrics, exit_code=0): add_bench_metrics(out, metrics, exit_code != 0, target_name=self.name) @@ -176,9 +176,9 @@ def _handle_exit(code, out=None): start_time = time.time() if self.print_outputs: - out = self.exec(elf, *args, cwd=directory, live=True, handle_exit=_handle_exit) + out, artifacts = self.exec(elf, *args, cwd=directory, live=True, handle_exit=_handle_exit) else: - out = self.exec( + out, artifacts = self.exec( elf, *args, cwd=directory, live=False, print_func=lambda *args, **kwargs: None, handle_exit=_handle_exit ) # TODO: do something with out? @@ -196,7 +196,7 @@ def _handle_exit(code, out=None): if diff > 0: metrics.add("MIPS", (sim_insns / diff) / 1e6, True) - return metrics, out, [] + return metrics, out, artifacts def get_platform_defs(self, platform): ret = {} diff --git a/mlonmcu/target/target.py b/mlonmcu/target/target.py index 04a01614f..a4cab9ef3 100644 --- a/mlonmcu/target/target.py +++ b/mlonmcu/target/target.py @@ -167,7 +167,7 @@ def generate(self, elf) -> Tuple[dict, dict]: metrics = [] total = 1 + (self.repeat if self.repeat else 0) # We only save the stdout and artifacts of the last execution - # Callect metrics from all runs to aggregate them in a callback with high priority + # Collect metrics from all runs to aggregate them in a callback with high priority artifacts_ = [] # if self.dir is None: # self.dir = Path( From 3ee0b4baac3912c13444f392cf4eb6d980cc0308 Mon Sep 17 00:00:00 2001 From: Philipp van Kempen Date: Tue, 16 Jan 2024 14:33:51 +0100 Subject: [PATCH 023/105] target.py: add utils for choosing io interfaces --- mlonmcu/target/target.py | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/mlonmcu/target/target.py b/mlonmcu/target/target.py index a4cab9ef3..bbba3f5e0 100644 --- a/mlonmcu/target/target.py +++ b/mlonmcu/target/target.py @@ -279,3 +279,23 @@ def get_hardware_details(self): "max-vthread-extent": 0, "warp-size": 0, } + + @property + def supports_filesystem(self): + return False + + @property + def supports_stdout(self): + return True + + @property + def supports_stdin(self): + return False + + @property + def supports_argv(self): + return False + + @property + def supports_uart(self): + return False From 10c1106dbcb87026cb889592ace3798f6aecd4dc Mon Sep 17 00:00:00 2001 From: Philipp van Kempen Date: Tue, 16 Jan 2024 14:58:58 +0100 Subject: [PATCH 024/105] Add ValidationNew.md --- ValidationNew.md | 126 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 126 insertions(+) create mode 100644 ValidationNew.md diff --git a/ValidationNew.md b/ValidationNew.md new file mode 100644 index 000000000..d2c0550ec --- /dev/null +++ b/ValidationNew.md @@ -0,0 +1,126 @@ +# MLonMCU - Validation (new) + +## Updates + +- `mlonmcu-sw` (`$MLONMCU_HOME/models/`): + - `resnet/definition.yml` + - Add input/output dtype + - Add dequantize details (taken from `mlif_override.cpp`) + - `resnet/support/mlif_override.cpp` + - Comment out old validation code + - Add example code to dump raw inputs/outputs via stdin/stdout +- `mlonmcu-sw` (`$MLONMCU_HOME/deps/src/mlif`): + - Nothing changed (yet) +- `mlonmcu`: + - `mlonmcu/target/common.py` + - Allow overriding used encoding for stdout (Required for dumping raw data) + - Add optional `stdin_data` argument for passing input to process (Only works with `live=False` aka. `-c {target}.print_outputs=0`) + - `mlonmcu/target/target.py` / `mlonmcu/target/riscv/spike.py` + - `target.exec()` may return artifacts now + - Add checks: `supports_filesystem`, `supports_stdout`, `supports_stdin`, `supports_argv`, `supports_uart` + - `mlonmcu/feature/feature.py`: + - Add `gen_data` feature + - Add `gen_ref_data` feature + - Add `set_inputs` feature + - Add `get_outputs` feature + - Add wrapper `validate_new` feature (Limitation: can not enable postprocess yet) + - `mlonmcu/models/frontend.py` + - Parse input/output types from `definition.yml` + - Parse quantization/dequantization details from `definition.yml` + - Create `model_info.yml` Artifact to pass input/output dtypes/shapes/quant to later stages + - Implement `gen_data` functionality (including workaround to convert raw `inputs/0.bin` to numpy) + - Implement dummy `gen_ref_data` functionality + - `mlonmcu/platform/mlif/mlif.py` / `mlonmcu/platform/mlif/mlif_target.py` + - Implement `set_inputs` functionality (stdin_raw only) + - Implement `get_outputs` functionality (stdout_raw only) + - Implement batching (run simulation serveral {num_inputs/batch_size} times, with different inputs) + - `mlonmcu/platform/tvm/tvm_target_platform.py` / `mlonmcu/platform/tvm/tvm_target.py` + - Implement `set_inputs` functionality (filesystem only) + - Implement `get_outputs` functionality (filesystem only) + - Implement batching (run simulation serveral {num_inputs/batch_size} times, with different inputs) + - `mlonmcu/session/postprocess/postprocesses.py` + - Add `ValidateOutputs` postprocess (WIP) + + + + +## Examples + +``` +# platform: tvm target: tvm_cpu` +# implemented: +python3 -m mlonmcu.cli.main flow run resnet -v \ + --target tvm_cpu --backend tvmllvm \ + --feature validate_new --post validate_outputs \ + -c tvm.print_outputs=1 -c tvm_cpu.print_outputs=1 + -c set_inputs.interface=filesystem -c get_outputs.interface=filesystem +python3 -m mlonmcu.cli.main flow run resnet -v \ + --target tvm_cpu --backend tvmllvm \ + --feature validate_new --post validate_outputs \ + -c tvm.print_outputs=1 -c tvm_cpu.print_outputs=1 + -c set_inputs.interface=auto -c get_outputs.interface=auto + +# not implemented: +python3 -m mlonmcu.cli.main flow run resnet -v \ + --target tvm_cpu --backend tvmllvm \ + --feature validate_new --post validate_outputs \ + -c tvm.print_outputs=1 -c tvm_cpu.print_outputs=1 + -c set_inputs.interface=filesystem -c get_outputs.interface=stdout + +# platform: mlif target: spike +# implemented: +python3 -m mlonmcu.cli.main flow run resnet -v \ + --target spike --backend tvmaotplus \ + -f validate \ + --feature validate_new --post validate_outputs \ + -c mlif.print_outputs=1 -c spike.print_outputs=0 \ + -c set_inputs.interface=stdin_raw -c get_outputs.interface=stdout_raw + +# not implemented: +# python3 -m mlonmcu.cli.main flow run resnet -v \ + --target spike --backend tvmaotplus \ + -f validate \ + --feature validate_new --post validate_outputs \ + -c mlif.print_outputs=1 -c spike.print_outputs=1 \ + -c set_inputs.interface=stdin -c get_outputs.interface=stdout +# python3 -m mlonmcu.cli.main flow run resnet -v \ + --target spike --backend tvmaotplus \ + -f validate \ + --feature validate_new --post validate_outputs \ + -c mlif.print_outputs=1 -c spike.print_outputs=1 \ + -c set_inputs.interface=filesystem -c get_outputs.interface=filesystem +# python3 -m mlonmcu.cli.main flow run resnet -v \ + --target spike --backend tvmaotplus \ + -f validate \ + --feature validate_new --post validate_outputs \ + -c mlif.print_outputs=1 -c spike.print_outputs=0 \ + -c set_inputs.interface=auto -c get_outputs.interface=auto +# combinations (e.g. filesystem+stdout) should also be tested! + +# platform: mlif target: host_x86` +# TODO: should support same configs as spike target +``` + +## TODOs +- [ ] Fix broken targets (due to refactoring of `self.exec`) -> PHILIPP +- [ ] Add missing target checks (see above) -> PHILIPP +- [ ] Update `definition.yml` for other models in `mlonmcu-sw` (At least `aww`, `vww`, `toycar`) -> LIU +- [ ] Refactor model support (see `mlomcu_sw/lib/ml_interface`) to be aware of output/input tensor index (maybe even name?) und sample index -> PHILIPP +- [ ] Write generator for custom `mlif_override.cpp` (based on `model_info.yml` + `in_interface` + `out_interface` (+ `inputs.npy`)) -> LIU +- [ ] Eliminate hacks used to get `model_info.yml` and `inputs.yml` in RUN stage -> PHILIPP +- [ ] Implement missing interfaces for tvm (out: `stdout`) -> LIU +- [ ] Implement missing interfaces for mlif platform (in: `filesystem`, `stdin`; out: `filesystem`, `stdout`) -> LIU +- [ ] Implement missing interfaces for mlif platform (in: `rom`) -> PHILIPP) +- [ ] Add support for multi-output/multi-input -> PHILIPP/LIU +- [ ] Update `gen_data` & `gen_ref_data` feature (see NotImplementedErrors, respect fmt,...) -> LIU +- [ ] Move `gen_data` & `gen_ref_data` from LOAD stage to custom stage (remove dependency on tflite frontend) -> PHILIPP +- [ ] Test with targets: `tvm_cpu`, `host_x86`, `spike` (See example commands above) -> LIU +- [ ] Extend `validate_outputs` postprocess (Add `report`, implement `atol`/`rtol`, `fail_on_error`, `top-k`,...) -> LIU +- [ ] Add more validation data (at least 10 samples per model, either manually or using `gen_ref_outputs`) -> LIU +- [ ] Generate validation reports for a few models (`aww`, `vww`, `resnet`, `toycar`) and at least backends (`tvmaotplus`, `tflmi`) -> LIU +- [ ] Cleanup codebases (lint, remove prints, reuse code, create helper functions,...) -> LIU/PHILIPP +- [ ] Document usage of new validation feature -> LIU +- [ ] Add tests -> LIU/PHILIPP +- [ ] Streamline `model_info.yml` with BUILD stage `ModelInfo` -> PHILIPP +- [ ] Improve artifacts handling -> PHILIPP +- [ ] Support automatic quantization of inputs (See `vww` and `toycar`) -> PHILIPP/LIU From 0f6c94822bfe1c856a2a9cf285f12ca5ec28fff9 Mon Sep 17 00:00:00 2001 From: Philipp van Kempen Date: Tue, 16 Jan 2024 15:30:53 +0100 Subject: [PATCH 025/105] Update ValidationNew.md --- ValidationNew.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ValidationNew.md b/ValidationNew.md index d2c0550ec..70baead92 100644 --- a/ValidationNew.md +++ b/ValidationNew.md @@ -52,12 +52,12 @@ python3 -m mlonmcu.cli.main flow run resnet -v \ --target tvm_cpu --backend tvmllvm \ --feature validate_new --post validate_outputs \ - -c tvm.print_outputs=1 -c tvm_cpu.print_outputs=1 + -c tvm.print_outputs=1 -c tvm_cpu.print_outputs=1 \ -c set_inputs.interface=filesystem -c get_outputs.interface=filesystem python3 -m mlonmcu.cli.main flow run resnet -v \ --target tvm_cpu --backend tvmllvm \ --feature validate_new --post validate_outputs \ - -c tvm.print_outputs=1 -c tvm_cpu.print_outputs=1 + -c tvm.print_outputs=1 -c tvm_cpu.print_outputs=1 \ -c set_inputs.interface=auto -c get_outputs.interface=auto # not implemented: From 06e08a7cd668aacebc344eb8aa62566b3f4ce00c Mon Sep 17 00:00:00 2001 From: Philipp van Kempen Date: Tue, 16 Jan 2024 20:06:31 +0100 Subject: [PATCH 026/105] Update ValidationNew.md --- ValidationNew.md | 54 ++++++++++++++++++++++++------------------------ 1 file changed, 27 insertions(+), 27 deletions(-) diff --git a/ValidationNew.md b/ValidationNew.md index 70baead92..b0d75fb62 100644 --- a/ValidationNew.md +++ b/ValidationNew.md @@ -2,7 +2,7 @@ ## Updates -- `mlonmcu-sw` (`$MLONMCU_HOME/models/`): +- `mlonmcu-models` (`$MLONMCU_HOME/models/`, Branch: `refactor-validate`): - `resnet/definition.yml` - Add input/output dtype - Add dequantize details (taken from `mlif_override.cpp`) @@ -11,7 +11,7 @@ - Add example code to dump raw inputs/outputs via stdin/stdout - `mlonmcu-sw` (`$MLONMCU_HOME/deps/src/mlif`): - Nothing changed (yet) -- `mlonmcu`: +- `mlonmcu` (Branch: `refactor-validate`): - `mlonmcu/target/common.py` - Allow overriding used encoding for stdout (Required for dumping raw data) - Add optional `stdin_data` argument for passing input to process (Only works with `live=False` aka. `-c {target}.print_outputs=0`) @@ -61,40 +61,40 @@ python3 -m mlonmcu.cli.main flow run resnet -v \ -c set_inputs.interface=auto -c get_outputs.interface=auto # not implemented: -python3 -m mlonmcu.cli.main flow run resnet -v \ - --target tvm_cpu --backend tvmllvm \ - --feature validate_new --post validate_outputs \ - -c tvm.print_outputs=1 -c tvm_cpu.print_outputs=1 - -c set_inputs.interface=filesystem -c get_outputs.interface=stdout +# python3 -m mlonmcu.cli.main flow run resnet -v \ +# --target tvm_cpu --backend tvmllvm \ +# --feature validate_new --post validate_outputs \ +# -c tvm.print_outputs=1 -c tvm_cpu.print_outputs=1 +# -c set_inputs.interface=filesystem -c get_outputs.interface=stdout # platform: mlif target: spike # implemented: python3 -m mlonmcu.cli.main flow run resnet -v \ - --target spike --backend tvmaotplus \ - -f validate \ - --feature validate_new --post validate_outputs \ - -c mlif.print_outputs=1 -c spike.print_outputs=0 \ - -c set_inputs.interface=stdin_raw -c get_outputs.interface=stdout_raw +--target spike --backend tvmaotplus \ +-f validate \ +--feature validate_new --post validate_outputs \ +-c mlif.print_outputs=1 -c spike.print_outputs=0 \ +-c set_inputs.interface=stdin_raw -c get_outputs.interface=stdout_raw # not implemented: # python3 -m mlonmcu.cli.main flow run resnet -v \ - --target spike --backend tvmaotplus \ - -f validate \ - --feature validate_new --post validate_outputs \ - -c mlif.print_outputs=1 -c spike.print_outputs=1 \ - -c set_inputs.interface=stdin -c get_outputs.interface=stdout +# --target spike --backend tvmaotplus \ +# -f validate \ +# --feature validate_new --post validate_outputs \ +# -c mlif.print_outputs=1 -c spike.print_outputs=1 \ +# -c set_inputs.interface=stdin -c get_outputs.interface=stdout # python3 -m mlonmcu.cli.main flow run resnet -v \ - --target spike --backend tvmaotplus \ - -f validate \ - --feature validate_new --post validate_outputs \ - -c mlif.print_outputs=1 -c spike.print_outputs=1 \ - -c set_inputs.interface=filesystem -c get_outputs.interface=filesystem +# --target spike --backend tvmaotplus \ +# -f validate \ +# --feature validate_new --post validate_outputs \ +# -c mlif.print_outputs=1 -c spike.print_outputs=1 \ +# -c set_inputs.interface=filesystem -c get_outputs.interface=filesystem # python3 -m mlonmcu.cli.main flow run resnet -v \ - --target spike --backend tvmaotplus \ - -f validate \ - --feature validate_new --post validate_outputs \ - -c mlif.print_outputs=1 -c spike.print_outputs=0 \ - -c set_inputs.interface=auto -c get_outputs.interface=auto +# --target spike --backend tvmaotplus \ +# -f validate \ +# --feature validate_new --post validate_outputs \ +# -c mlif.print_outputs=1 -c spike.print_outputs=0 \ +# -c set_inputs.interface=auto -c get_outputs.interface=auto # combinations (e.g. filesystem+stdout) should also be tested! # platform: mlif target: host_x86` From 6ec80903d1ee32f8c3f5985fc3fadf139f461a10 Mon Sep 17 00:00:00 2001 From: Philipp van Kempen Date: Wed, 17 Jan 2024 09:46:35 +0100 Subject: [PATCH 027/105] tflite frontend: add inference method --- mlonmcu/models/frontend.py | 39 +++++++++++++++++++++++++++++++++++++- 1 file changed, 38 insertions(+), 1 deletion(-) diff --git a/mlonmcu/models/frontend.py b/mlonmcu/models/frontend.py index ec92ed717..eb3794356 100644 --- a/mlonmcu/models/frontend.py +++ b/mlonmcu/models/frontend.py @@ -22,7 +22,7 @@ import multiprocessing from pathlib import Path from abc import ABC, abstractmethod -from typing import Tuple, List +from typing import Tuple, List, Dict import numpy as np @@ -142,6 +142,9 @@ def gen_ref_data_fmt(self): assert value in ["npy", "npz"] return value + def inference(self, model: Model, input_data: Dict[str, np.array]): + raise NotImplementedError + def supports_formats(self, ins=None, outs=None): """Returs true if the frontend can handle at least one combination of input and output formats.""" assert ins is not None or outs is not None, "Please provide a list of input formats, outputs formats or both" @@ -654,6 +657,40 @@ def analyze_enable(self): def analyze_script(self): return self.config["analyze_script"] + def inference(self, model: Model, input_data: Dict[str, np.array], quant=False, dequant=False): + import tensorflow as tf + model_path = str(model.paths[0]) + interpreter = tf.lite.Interpreter(model_path=model_path) + input_details = interpreter.get_input_details() + output_details = interpreter.get_output_details() + interpreter.allocate_tensors() + assert len(input_details) == 1, "Multi-inputs not yet supported" + input_type = input_details[0]["dtype"] + input_name = input_details[0]["name"] + input_shape = input_details[0]["shape"] + assert input_name in input_data, f"Input {input_name} fot found in data" + np_features = input_data[input_name] + if quant and input_type == np.int8: + input_scale, input_zero_point = input_details[0]['quantization'] + np_features = (np_features / input_scale) + input_zero_point + np_features = np.around(np_features) + np_features = np_features.astype(input_type) + np_features = np_features.reshape(input_shape) + interpreter.set_tensor(input_details[0]['index'], np_features) + interpreter.invoke() + output = interpreter.get_tensor(output_details[0]['index']) + + # If the output type is int8 (quantized model), rescale data + assert len(output_details) == 1, "Multi-outputs not yet supported" + output_type = output_details[0]["dtype"] + output_name = output_details[0]["name"] + if dequant and output_type == np.int8: + output_scale, output_zero_point = output_details[0]["quantization"] + output = output_scale * (output.astype(np.float32) - output_zero_point) + + # Print the results of inference + return {output_name: output} + def produce_artifacts(self, model): assert len(self.input_formats) == len(model.paths) == 1 artifacts = [] From c132768bc7efca3303b1aa6a225947d087381ae8 Mon Sep 17 00:00:00 2001 From: Philipp van Kempen Date: Wed, 17 Jan 2024 09:47:10 +0100 Subject: [PATCH 028/105] continue working on gen_ref_data feature --- mlonmcu/models/frontend.py | 44 +++++++++++++++++++++++++++++++++----- 1 file changed, 39 insertions(+), 5 deletions(-) diff --git a/mlonmcu/models/frontend.py b/mlonmcu/models/frontend.py index eb3794356..db2759fa5 100644 --- a/mlonmcu/models/frontend.py +++ b/mlonmcu/models/frontend.py @@ -322,10 +322,35 @@ def process_metadata(self, model, cfg=None): artifacts = [] if self.gen_data: inputs_data = [] - if self.gen_data_fill_mode == "random": - raise NotImplementedError - elif self.gen_data_fill_mode == "zeros": - raise NotImplementedError + if self.gen_data_fill_mode in ["zeros", "ones", "random"]: + for i in range(self.gen_data_number): + print("i", i) + data = {} + for ii, input_name in enumerate(input_names): + print("ii", ii) + assert input_name in input_types, f"Unknown dtype for input: {input_name}" + dtype = input_types[input_name] + quant = input_quant_details.get(name, None) + if quant: + _, _, ty = quant + dtype = ty + assert input_name in input_shapes, f"Unknown shape for input: {input_name}" + shape = input_shapes[input_name] + if self.gen_data_fill_mode == "zeros": + arr = np.zeros(shape, dtype=dtype) + elif self.gen_data_fill_mode == "ones": + arr = np.ones(shape, dtype=dtype) + elif self.gen_data_fill_mode == "ones": + if "float" in dtype: + arr = np.rand(*shape).astype(dtype) + elif "int" in dtype: + arr = np.random.randint(np.iinfo(dtype).min, np.iinfo(dtype).max, size=shape, dtype=dtype) + else: + assert False + else: + assert False + data[input_name] = arr + inputs_data.append(data) elif self.gen_data_fill_mode == "ones": raise NotImplementedError elif self.gen_data_fill_mode == "file": @@ -409,7 +434,16 @@ def process_metadata(self, model, cfg=None): if self.gen_ref_data: outputs_data = [] if self.gen_ref_data_mode == "model": - raise NotImplementedError + assert len(inputs_data) > 0 + for i, input_data in enumerate(inputs_data): + print("i", i) + print("model", model, type(model)) + # input("321?") + output_data = self.inference(model, input_data, quant=False, dequant=False) + print("output_data", output_data) + outputs_data.append(output_data) + # input("321!") + elif self.gen_ref_data_mode == "file": if self.gen_ref_data_file == "auto": len(out_paths) > 0 From 603b1a5943ffc44e36d18ed1cc4f41dd15fa25ef Mon Sep 17 00:00:00 2001 From: Philipp van Kempen Date: Wed, 17 Jan 2024 09:47:29 +0100 Subject: [PATCH 029/105] tflite frontend: prints (WIP) --- mlonmcu/models/frontend.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/mlonmcu/models/frontend.py b/mlonmcu/models/frontend.py index db2759fa5..48c88167e 100644 --- a/mlonmcu/models/frontend.py +++ b/mlonmcu/models/frontend.py @@ -698,6 +698,13 @@ def inference(self, model: Model, input_data: Dict[str, np.array], quant=False, input_details = interpreter.get_input_details() output_details = interpreter.get_output_details() interpreter.allocate_tensors() + print() + print("Input details:") + print(input_details) + print() + print("Output details:") + print(output_details) + print() assert len(input_details) == 1, "Multi-inputs not yet supported" input_type = input_details[0]["dtype"] input_name = input_details[0]["name"] @@ -706,10 +713,14 @@ def inference(self, model: Model, input_data: Dict[str, np.array], quant=False, np_features = input_data[input_name] if quant and input_type == np.int8: input_scale, input_zero_point = input_details[0]['quantization'] + print("Input scale:", input_scale) + print("Input zero point:", input_zero_point) + print() np_features = (np_features / input_scale) + input_zero_point np_features = np.around(np_features) np_features = np_features.astype(input_type) np_features = np_features.reshape(input_shape) + print("np_features", np_features) interpreter.set_tensor(input_details[0]['index'], np_features) interpreter.invoke() output = interpreter.get_tensor(output_details[0]['index']) @@ -720,9 +731,14 @@ def inference(self, model: Model, input_data: Dict[str, np.array], quant=False, output_name = output_details[0]["name"] if dequant and output_type == np.int8: output_scale, output_zero_point = output_details[0]["quantization"] + print("Raw output scores:", output) + print("Output scale:", output_scale) + print("Output zero point:", output_zero_point) + print() output = output_scale * (output.astype(np.float32) - output_zero_point) # Print the results of inference + print("Inference output:", output, type(output)) return {output_name: output} def produce_artifacts(self, model): From 87610a3f421cea40b81af28f43ad5cf6e908e2e3 Mon Sep 17 00:00:00 2001 From: Philipp van Kempen Date: Wed, 17 Jan 2024 10:23:35 +0100 Subject: [PATCH 030/105] continue working on gen_ref_data feature 2 --- mlonmcu/models/frontend.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mlonmcu/models/frontend.py b/mlonmcu/models/frontend.py index 48c88167e..ff6c7efa6 100644 --- a/mlonmcu/models/frontend.py +++ b/mlonmcu/models/frontend.py @@ -340,7 +340,7 @@ def process_metadata(self, model, cfg=None): arr = np.zeros(shape, dtype=dtype) elif self.gen_data_fill_mode == "ones": arr = np.ones(shape, dtype=dtype) - elif self.gen_data_fill_mode == "ones": + elif self.gen_data_fill_mode == "random": if "float" in dtype: arr = np.rand(*shape).astype(dtype) elif "int" in dtype: @@ -439,7 +439,7 @@ def process_metadata(self, model, cfg=None): print("i", i) print("model", model, type(model)) # input("321?") - output_data = self.inference(model, input_data, quant=False, dequant=False) + output_data = self.inference(model, input_data, quant=False, dequant=True) print("output_data", output_data) outputs_data.append(output_data) # input("321!") From c6b20aa1bacfb86adf973b11669a9d37fca86071 Mon Sep 17 00:00:00 2001 From: Philipp van Kempen Date: Wed, 17 Jan 2024 10:24:04 +0100 Subject: [PATCH 031/105] validate_outputs: set rtol=0 for testing --- mlonmcu/session/postprocess/postprocesses.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/mlonmcu/session/postprocess/postprocesses.py b/mlonmcu/session/postprocess/postprocesses.py index 86a617512..a3b3e93d7 100644 --- a/mlonmcu/session/postprocess/postprocesses.py +++ b/mlonmcu/session/postprocess/postprocesses.py @@ -1545,7 +1545,12 @@ def post_run(self, report, artifacts): assert out_data.dtype.name in ["int8"], "Dequantization only supported for int8 input" assert quant_dtype in ["float32"], "Dequantization only supported for float32 output" out_data = (out_data.astype("float32") - quant_zero_point) * quant_scale - if np.allclose(out_data, out_ref_data, rtol=0.1, atol=0.1): + print("out_data", out_data) + print("out_ref_data", out_ref_data) + assert out_data.dtype == out_ref_data.dtype, "dtype missmatch" + assert out_data.shape == out_ref_data.shape, "shape missmatch" + # if np.allclose(out_data, out_ref_data, rtol=0, atol=0): + if np.allclose(out_data, out_ref_data, rtol=0, atol=0.1): matching += 1 compared += 1 ii += 1 From 67048cac5b4865329441acec9075ca8d6840ef37 Mon Sep 17 00:00:00 2001 From: Philipp van Kempen Date: Wed, 17 Jan 2024 22:41:28 +0100 Subject: [PATCH 032/105] tflite frontend: extract model info from tflite file if definition.yml missing --- mlonmcu/models/frontend.py | 69 ++++++++++++++++++++++++++++++-------- 1 file changed, 55 insertions(+), 14 deletions(-) diff --git a/mlonmcu/models/frontend.py b/mlonmcu/models/frontend.py index ff6c7efa6..2c35cc0a0 100644 --- a/mlonmcu/models/frontend.py +++ b/mlonmcu/models/frontend.py @@ -145,6 +145,9 @@ def gen_ref_data_fmt(self): def inference(self, model: Model, input_data: Dict[str, np.array]): raise NotImplementedError + def extract_model_info(self, model: Model): + raise NotImplementedError + def supports_formats(self, ins=None, outs=None): """Returs true if the frontend can handle at least one combination of input and output formats.""" assert ins is not None or outs is not None, "Please provide a list of input formats, outputs formats or both" @@ -269,6 +272,17 @@ def process_metadata(self, model, cfg=None): flattened = {f"{backend}.{key}": value for key, value in backend_options[backend].items()} cfg.update(flattened) + if len(input_shapes) > 0: + assert len(input_types) in [len(input_shapes), 0] + input_names = list(input_shapes.keys()) + elif len(input_shapes) > 0: + input_names = list(input_types.keys()) + else: + input_names = [] + + if metadata is None: + input_names, input_shapes, input_types, input_quant_details, output_names, output_shapes, output_types, output_quant_details = self.extract_model_info(model) + # Detect model support code (Allow overwrite in metadata YAML) support_path = model_dir / "support" if support_path.is_dir(): @@ -294,13 +308,6 @@ def process_metadata(self, model, cfg=None): if len(output_types) > 0: cfg.update({f"{model.name}.output_types": output_types}) # flattened version - if len(input_shapes) > 0: - assert len(input_types) in [len(input_shapes), 0] - input_names = list(input_shapes.keys()) - elif len(input_shapes) > 0: - input_names = list(input_types.keys()) - else: - input_names = [] if len(output_shapes) > 0: assert len(output_types) in [len(output_shapes), 0] output_names = list(output_shapes.keys()) @@ -330,10 +337,10 @@ def process_metadata(self, model, cfg=None): print("ii", ii) assert input_name in input_types, f"Unknown dtype for input: {input_name}" dtype = input_types[input_name] - quant = input_quant_details.get(name, None) + quant = input_quant_details.get(input_name, None) if quant: _, _, ty = quant - dtype = ty + # dtype = ty assert input_name in input_shapes, f"Unknown shape for input: {input_name}" shape = input_shapes[input_name] if self.gen_data_fill_mode == "zeros": @@ -342,7 +349,7 @@ def process_metadata(self, model, cfg=None): arr = np.ones(shape, dtype=dtype) elif self.gen_data_fill_mode == "random": if "float" in dtype: - arr = np.rand(*shape).astype(dtype) + arr = np.random.rand(*shape).astype(dtype) elif "int" in dtype: arr = np.random.randint(np.iinfo(dtype).min, np.iinfo(dtype).max, size=shape, dtype=dtype) else: @@ -350,9 +357,8 @@ def process_metadata(self, model, cfg=None): else: assert False data[input_name] = arr + assert len(data) > 0 inputs_data.append(data) - elif self.gen_data_fill_mode == "ones": - raise NotImplementedError elif self.gen_data_fill_mode == "file": if self.gen_data_file == "auto": len(in_paths) > 0 @@ -387,6 +393,7 @@ def process_metadata(self, model, cfg=None): else: raise RuntimeError(f"Unsupported ext: {ext}") # print("temp", temp) + assert len(temp) > 0 for i in range(min(self.gen_data_number, len(temp))): print("i", i) assert i in temp @@ -396,7 +403,7 @@ def process_metadata(self, model, cfg=None): assert ii in temp[i] assert input_name in input_types, f"Unknown dtype for input: {input_name}" dtype = input_types[input_name] - quant = input_quant_details.get(name, None) + quant = input_quant_details.get(input_name, None) if quant: _, _, ty = quant dtype = ty @@ -487,7 +494,7 @@ def process_metadata(self, model, cfg=None): assert ii in temp[i] assert output_name in output_types, f"Unknown dtype for output: {output_name}" dtype = output_types[output_name] - dequant = output_quant_details.get(name, None) + dequant = output_quant_details.get(output_name, None) if dequant: _, _, ty = dequant dtype = ty @@ -691,6 +698,40 @@ def analyze_enable(self): def analyze_script(self): return self.config["analyze_script"] + def extract_model_info(self, model: Model): + import tensorflow as tf + model_path = str(model.paths[0]) + interpreter = tf.lite.Interpreter(model_path=model_path) + input_details = interpreter.get_input_details() + output_details = interpreter.get_output_details() + input_names = [] + input_shapes = {} + input_types = {} + input_quant_details = {} + output_names = [] + output_shapes = {} + output_types = {} + output_quant_details = {} + for inp in input_details: + name = str(inp["name"]) + input_names.append(name) + input_shapes[name] = inp["shape"].tolist() + input_types[name] = np.dtype(inp["dtype"]).name + if "quantization" in inp: + scale, zero_point = inp["quantization"] + quant = [scale, zero_point, "float32"] + input_quant_details[name] = quant + for outp in output_details: + name = str(outp["name"]) + output_names.append(name) + output_shapes[name] = outp["shape"].tolist() + output_types[name] = np.dtype(outp["dtype"]).name + if "quantization" in outp: + scale, zero_point = outp["quantization"] + quant = [scale, zero_point, "float32"] + output_quant_details[name] = quant + return input_names, input_shapes, input_types, input_quant_details, output_names, output_shapes, output_types, output_quant_details + def inference(self, model: Model, input_data: Dict[str, np.array], quant=False, dequant=False): import tensorflow as tf model_path = str(model.paths[0]) From 9553d0f7b3d6c9e626fd85830955d90362aede32 Mon Sep 17 00:00:00 2001 From: Philipp van Kempen Date: Wed, 17 Jan 2024 22:45:29 +0100 Subject: [PATCH 033/105] update md --- ValidationNew.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ValidationNew.md b/ValidationNew.md index b0d75fb62..603d08a48 100644 --- a/ValidationNew.md +++ b/ValidationNew.md @@ -112,7 +112,7 @@ python3 -m mlonmcu.cli.main flow run resnet -v \ - [ ] Implement missing interfaces for mlif platform (in: `filesystem`, `stdin`; out: `filesystem`, `stdout`) -> LIU - [ ] Implement missing interfaces for mlif platform (in: `rom`) -> PHILIPP) - [ ] Add support for multi-output/multi-input -> PHILIPP/LIU -- [ ] Update `gen_data` & `gen_ref_data` feature (see NotImplementedErrors, respect fmt,...) -> LIU +- [x] Update `gen_data` & `gen_ref_data` feature (see NotImplementedErrors, respect fmt,...) - [ ] Move `gen_data` & `gen_ref_data` from LOAD stage to custom stage (remove dependency on tflite frontend) -> PHILIPP - [ ] Test with targets: `tvm_cpu`, `host_x86`, `spike` (See example commands above) -> LIU - [ ] Extend `validate_outputs` postprocess (Add `report`, implement `atol`/`rtol`, `fail_on_error`, `top-k`,...) -> LIU From 6f2668884761984b9fb376aa5d64cd27db30ae9f Mon Sep 17 00:00:00 2001 From: Philipp van Kempen Date: Wed, 24 Jan 2024 14:05:32 +0100 Subject: [PATCH 034/105] mlif: support generated model_support files for validation feature --- mlonmcu/models/utils.py | 17 +++ mlonmcu/platform/mlif/mlif.py | 219 +++++++++++++++++++++++++++++++++- 2 files changed, 232 insertions(+), 4 deletions(-) diff --git a/mlonmcu/models/utils.py b/mlonmcu/models/utils.py index 20f7ad308..a914ee8c7 100644 --- a/mlonmcu/models/utils.py +++ b/mlonmcu/models/utils.py @@ -79,6 +79,23 @@ def fill_data_source(in_bufs, out_bufs): return out +def fill_data_source_inputs_only(in_bufs): + # out = '#include "ml_interface.h"\n' + out = "#include \n" + out += "const int num_data_buffers_in = " + str(sum([len(buf) for buf in in_bufs])) + ";\n" + for i, buf in enumerate(in_bufs): + for j in range(len(buf)): + out += "const unsigned char data_buffer_in_" + str(i) + "_" + str(j) + "[] = {" + buf[j] + "};\n" + var_in = "const unsigned char *const data_buffers_in[] = {" + var_insz = "const size_t data_size_in[] = {" + for i, buf in enumerate(in_bufs): + for j in range(len(buf)): + var_in += "data_buffer_in_" + str(i) + "_" + str(j) + ", " + var_insz += "sizeof(data_buffer_in_" + str(i) + "_" + str(j) + "), " + out += var_in + "};\n" + var_insz + "};\n" + return out + + def lookup_data_buffers(input_paths, output_paths): assert len(input_paths) > 0 legacy = False diff --git a/mlonmcu/platform/mlif/mlif.py b/mlonmcu/platform/mlif/mlif.py index cb79a5bef..777a8ad36 100644 --- a/mlonmcu/platform/mlif/mlif.py +++ b/mlonmcu/platform/mlif/mlif.py @@ -30,7 +30,7 @@ from mlonmcu.logging import get_logger from mlonmcu.target import get_targets from mlonmcu.target.target import Target -from mlonmcu.models.utils import get_data_source +from mlonmcu.models.utils import get_data_source, fill_data_source_inputs_only from ..platform import CompilePlatform, TargetPlatform from .mlif_target import get_mlif_platform_targets, create_mlif_platform_target @@ -89,6 +89,10 @@ class MlifPlatform(CompilePlatform, TargetPlatform): "get_outputs": False, "get_outputs_interface": None, "get_outputs_fmt": None, + "batch_size": None, + "model_support_file": None, + "model_support_dir": None, + "model_support_lib": None, } REQUIRED = {"mlif.src_dir"} @@ -132,6 +136,13 @@ def get_outputs_fmt(self): value = self.config["get_outputs_fmt"] # TODO: use return value + @property + def batch_size(self): + value = self.config["batch_size"] # TODO: use + if isinstance(value, str): + value = int(value) + return value + @property def inputs_artifact(self): # THIS IS A HACK (get inputs fom artifacts!) @@ -152,7 +163,6 @@ def model_info_file(self): logger.warning("Artifact 'model_info.yml' not found!") return None - def gen_data_artifact(self): in_paths = self.input_data_path if not isinstance(in_paths, list): @@ -265,9 +275,20 @@ def validate_outputs(self): def toolchain(self): return str(self.config["toolchain"]) + @property + def model_support_file(self): + value = self.config["model_support_file"] # TODO: use + return value + @property def model_support_dir(self): - return self.config["model_support_dir"] + value = self.config["model_support_dir"] # TODO: use + return value + + @property + def model_support_lib(self): + value = self.config["model_support_lib"] # TODO: use + return value @property def prebuild_lib_dir(self): @@ -339,6 +360,8 @@ def get_definitions(self): definitions["TOOLCHAIN"] = self.toolchain definitions["QUIET"] = self.mem_only definitions["SKIP_CHECK"] = self.skip_check + if self.batch_size is not None: + definitions["BATCH_SIZE"] = self.batch_size if self.num_threads is not None: definitions["SUBPROJECT_THREADS"] = self.num_threads if self.toolchain == "llvm" and self.llvm_dir is None: @@ -357,8 +380,12 @@ def get_definitions(self): definitions["ENABLE_GC"] = self.garbage_collect if self.slim_cpp is not None: definitions["SLIM_CPP"] = self.slim_cpp + if self.model_support_file is not None: + definitions["MODEL_SUPPORT_FILE"] = self.model_support_file if self.model_support_dir is not None: definitions["MODEL_SUPPORT_DIR"] = self.model_support_dir + if self.model_support_lib is not None: + definitions["MODEL_SUPPORT_LIB"] = self.model_support_lib if self.fuse_ld is not None: definitions["FUSE_LD"] = self.fuse_ld if self.strip_strings is not None: @@ -386,7 +413,192 @@ def prepare_environment(self): env["PATH"] = path_new return env + def select_set_inputs_interface(self, target, batch_size): + in_interface = self.set_inputs_interface + if in_interface == "auto": + if target.supports_filesystem: + in_interface = "filesystem" + elif target.supports_stdin: + in_interface = "stdin_raw" + # TODO: also allow stdin? + else: # Fallback + in_interface = "rom" + assert in_interface in ["filesystem", "stdin", "stdin_raw", "rom"] + if batch_size is None: + if in_interface == "rom": + batch_size = 1e6 # all inputs are in already compiled into program + else: + batch_size = 10 + return in_interface, batch_size + + def select_get_outputs_interface(self, target, batch_size): + out_interface = self.get_outputs_interface + if out_interface == "auto": + if target.supports_filesystem: + out_interface = "filesystem" + elif target.supports_stdin: + out_interface = "stdout_raw" + # TODO: also allow stdout? + else: # Fallback + out_interface = "ram" + assert out_interface in ["filesystem", "stdout", "stdout_raw", "ram"] + if batch_size is None: + batch_size = 10 + return out_interface, batch_size + + def generate_model_support_code(self, in_interface, out_interface, batch_size): + code = "" + code += """ +#include "quantize.h" +#include "printing.h" +#include "exit.h" +// #include "ml_interface.h" +#include +#include +#include +#include +#include + +extern "C" { +int mlif_process_inputs(size_t, bool*); +int mlif_process_outputs(size_t); +void *mlif_input_ptr(int); +void *mlif_output_ptr(int); +int mlif_input_sz(int); +int mlif_output_sz(int); +int mlif_num_inputs(); +int mlif_num_outputs(); +} +""" + if in_interface == "rom": + assert self.inputs_artifact is not None + import numpy as np + data = np.load(self.inputs_artifact, allow_pickle=True) + in_bufs = [] + for i, ins_data in enumerate(data): + temp = [] + for j, in_data in enumerate(ins_data.values()): + byte_data = in_data.tobytes() + temp2 = ", ".join(["0x{:02x}".format(x) for x in byte_data] + [""]) + temp.append(temp2) + in_bufs.append(temp) + + code += fill_data_source_inputs_only(in_bufs) + code += """ +int mlif_process_inputs(size_t batch_idx, bool *new_) +{ + *new_ = true; + int num_inputs = mlif_num_inputs(); + for (int i = 0; i < num_inputs; i++) + { + int idx = num_inputs * batch_idx + i; + int size = mlif_input_sz(i); + char* model_input_ptr = (char*)mlif_input_ptr(i); + if (idx >= num_data_buffers_in) + { + *new_ = false; + break; + } + if (size != data_size_in[idx]) + { + return EXIT_MLIF_INVALID_SIZE; + } + memcpy(model_input_ptr, data_buffers_in[idx], size); + } + return 0; +} +""" + elif in_interface == "stdin_raw": + code += """ +int mlif_process_inputs(size_t batch_idx, bool *new_) +{ + char ch; + *new_ = true; + for (int i = 0; i < mlif_num_inputs(); i++) + { + int cnt = 0; + int size = mlif_input_sz(i); + char* model_input_ptr = (char*)mlif_input_ptr(i); + while(read(STDIN_FILENO, &ch, 1) > 0) { + // printf("c=%c / %d\\n", ch, ch); + model_input_ptr[cnt] = ch; + cnt++; + if (cnt == size) { + break; + } + } + // printf("cnt=%d in_size=%lu\\n", cnt, in_size); + if (cnt == 0) { + *new_ = false; + return 0; + } + else if (cnt < size) + { + return EXIT_MLIF_INVALID_SIZE; + } + } + return 0; +} +""" + elif in_interface == "stdin": + raise NotImplementedError + elif in_interface == "filesystem": + raise NotImplementedError + else: + assert False + if out_interface == "ram": + raise NotImplementedError + elif out_interface == "stdout_raw": + # TODO: maybe hardcode num_outputs and size here because we know it + # and get rid of loop? + code += """ +int mlif_process_outputs(size_t batch_idx) +{ + for (int i = 0; i < mlif_num_outputs(); i++) + { + int8_t *model_output_ptr = (int8_t*)mlif_output_ptr(i); + int size = mlif_output_sz(i); + // TODO: move markers out of loop + write(1, "-?-", 3); + write(1, model_output_ptr, size); + write(1, "-!-\\n" ,4); + } + return 0; +} +""" + elif out_interface == "stdout": + raise NotImplementedError + elif out_interface == "filesystem": + raise NotImplementedError + else: + assert False + return code + + def generate_model_support(self, target): + artifacts = [] + in_interface = None + batch_size = self.batch_size + if self.set_inputs: + in_interface, batch_size = self.select_set_inputs_interface(target, batch_size) + if self.get_outputs: + out_interface, batch_size = self.select_get_outputs_interface(target, batch_size) + if in_interface or out_interface: + code = self.generate_model_support_code(in_interface, out_interface, batch_size) + code_artifact = Artifact( + "model_support.cpp", content=code, fmt=ArtifactFormat.TEXT, flags=("model_support"), + ) + self.definitions["BATCH_SIZE"] = batch_size + artifacts.append(code_artifact) + return artifacts + def configure(self, target, src, _model): + artifacts = self.generate_model_support(target) + if len(artifacts) > 0: + assert len(artifacts) == 1 + model_support_artifact = artifacts[0] + model_support_file = self.build_dir / model_support_artifact.name + model_support_artifact.export(model_support_file) + self.definitions["MODEL_SUPPORT_FILE"] = model_support_file del target if not isinstance(src, Path): src = Path(src) @@ -397,7 +609,6 @@ def configure(self, target, src, _model): cmakeArgs.append("-DSRC_DIR=" + str(src)) else: raise RuntimeError("Unable to find sources!") - artifacts = [] if self.ignore_data: cmakeArgs.append("-DDATA_SRC=") else: From 969543f075bc51638dc5e02886f5da25efdc6776 Mon Sep 17 00:00:00 2001 From: Philipp van Kempen Date: Wed, 24 Jan 2024 14:06:02 +0100 Subject: [PATCH 035/105] implement mlif template versioning --- mlonmcu/platform/mlif/mlif.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/mlonmcu/platform/mlif/mlif.py b/mlonmcu/platform/mlif/mlif.py index 777a8ad36..9b38671ab 100644 --- a/mlonmcu/platform/mlif/mlif.py +++ b/mlonmcu/platform/mlif/mlif.py @@ -66,6 +66,7 @@ class MlifPlatform(CompilePlatform, TargetPlatform): **CompilePlatform.DEFAULTS, **TargetPlatform.DEFAULTS, "template": "ml_interface", + "template_version": None, "ignore_data": True, "skip_check": False, "fail_on_error": False, # Prefer to add acolum with validation results instead of raising a RuntimeError @@ -252,6 +253,10 @@ def srecord_dir(self): def template(self): return self.config["template"] + @property + def template_version(self): + return self.config["template_version"] + @property def ignore_data(self): value = self.config["ignore_data"] @@ -357,6 +362,8 @@ def close(self): def get_definitions(self): definitions = self.definitions definitions["TEMPLATE"] = self.template + if self.template_version: + definitions["TEMPLATE_VERSION"] = self.template_version definitions["TOOLCHAIN"] = self.toolchain definitions["QUIET"] = self.mem_only definitions["SKIP_CHECK"] = self.skip_check From 4e323cb11240b2c9c0d3a9f8c470f47fe403710a Mon Sep 17 00:00:00 2001 From: Philipp van Kempen Date: Wed, 24 Jan 2024 14:06:50 +0100 Subject: [PATCH 036/105] mlif_target: fix batch size calculation --- mlonmcu/platform/mlif/mlif_target.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/mlonmcu/platform/mlif/mlif_target.py b/mlonmcu/platform/mlif/mlif_target.py index 854e6a2ac..5d8265e3b 100644 --- a/mlonmcu/platform/mlif/mlif_target.py +++ b/mlonmcu/platform/mlif/mlif_target.py @@ -17,6 +17,7 @@ # limitations under the License. # import os +from math import ceil from pathlib import Path from enum import IntEnum @@ -55,9 +56,9 @@ def __init__(self, features=None, config=None): def exec(self, program, *args, cwd=os.getcwd(), **kwargs): ins_file = None num_inputs = 0 - batch_size = 5 # idea is to process i.e. 5 samples per batch, repeating until no samples remain in_interface = None out_interface = None + batch_size = 1 if self.platform.set_inputs: # first figure out how many inputs are provided assert self.platform.inputs_artifact is not None @@ -107,7 +108,7 @@ def exec(self, program, *args, cwd=os.getcwd(), **kwargs): encoding = None ret = "" artifacts = [] - num_batches = max(round(num_inputs / batch_size), 1) + num_batches = max(ceil(num_inputs / batch_size), 1) processed_inputs = 0 remaining_inputs = num_inputs outs_data = [] From 3c917336190b6c1733f82953be0f6bd69a7e8cf4 Mon Sep 17 00:00:00 2001 From: Philipp van Kempen Date: Wed, 24 Jan 2024 14:07:15 +0100 Subject: [PATCH 037/105] mlif_target: cleanup interface selection code --- mlonmcu/platform/mlif/mlif_target.py | 41 +++------------------------- 1 file changed, 4 insertions(+), 37 deletions(-) diff --git a/mlonmcu/platform/mlif/mlif_target.py b/mlonmcu/platform/mlif/mlif_target.py index 5d8265e3b..a666ae0ff 100644 --- a/mlonmcu/platform/mlif/mlif_target.py +++ b/mlonmcu/platform/mlif/mlif_target.py @@ -66,51 +66,18 @@ def exec(self, program, *args, cwd=os.getcwd(), **kwargs): data = np.load(self.platform.inputs_artifact, allow_pickle=True) # print("data", data, type(data)) num_inputs = len(data) - in_interface = self.platform.set_inputs_interface - if in_interface == "auto": - if self.supports_filesystem: - in_interface = "filesystem" - # TODO: eventually update batch_size - elif self.supports_stdin: - in_interface = "stdin_raw" - # TODO: also allow stdin? - # TODO: eventually update batch_size - else: # Fallback - in_interface = "rom" - batch_size = 1e6 # all inputs are in already compiled into program - else: - assert in_interface in ["filesystem", "stdin", "stdin_raw", "rom"] - if in_interface == "filesystem": - pass - elif in_interface == "stdin": - pass - elif in_interface == "stdin_raw": - pass - elif in_interface == "rom": - pass # nothing to do + in_interface, batch_size = self.platform.select_set_inputs_interface(self, self.platform.batch_size) outs_file = None encoding = "utf-8" if self.platform.get_outputs: - out_interface = self.platform.get_outputs_interface - if out_interface == "auto": - if self.supports_filesystem: - out_interface = "filesystem" - elif self.supports_stdout: - out_interface = "stdout_raw" - # TODO: support stdout? - else: - assert out_interface in ["filesystem", "stdout", "stdout_raw"] - if out_interface == "filesystem": - pass - elif out_interface == "stdout": - pass - elif out_interface == "stdout_raw": + out_interface, batch_size = self.platform.select_get_outputs_interface(self, batch_size) + if out_interface == "stdout_raw": encoding = None ret = "" artifacts = [] num_batches = max(ceil(num_inputs / batch_size), 1) processed_inputs = 0 - remaining_inputs = num_inputs + # remaining_inputs = num_inputs outs_data = [] stdin_data = None for idx in range(num_batches): From 8971790afdbc566e6637434b48b66636c8a9f4fb Mon Sep 17 00:00:00 2001 From: Philipp van Kempen Date: Wed, 24 Jan 2024 14:07:40 +0100 Subject: [PATCH 038/105] handle mlif_iunvalid_size error correctly --- mlonmcu/platform/mlif/mlif_target.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/mlonmcu/platform/mlif/mlif_target.py b/mlonmcu/platform/mlif/mlif_target.py index a666ae0ff..85c2baf13 100644 --- a/mlonmcu/platform/mlif/mlif_target.py +++ b/mlonmcu/platform/mlif/mlif_target.py @@ -193,9 +193,10 @@ def _handle_exit(code, out=None): if code in MlifExitCode.values(): reason = MlifExitCode(code).name logger.error("A platform error occured during the simulation. Reason: %s", reason) - self.validation_result = False - if not self.platform.fail_on_error: - code = 0 + if code == MlifExitCode.OUTPUT_MISSMATCH: + self.validation_result = False + if not self.platform.fail_on_error: + code = 0 return code else: From 1e90d0efc9c278885a056c0b6374e6feb7c25806 Mon Sep 17 00:00:00 2001 From: Philipp van Kempen Date: Wed, 24 Jan 2024 16:30:35 +0100 Subject: [PATCH 039/105] pass user/env config to ModelHints --- mlonmcu/models/frontend.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/mlonmcu/models/frontend.py b/mlonmcu/models/frontend.py index 2c35cc0a0..8e8c77e3c 100644 --- a/mlonmcu/models/frontend.py +++ b/mlonmcu/models/frontend.py @@ -281,7 +281,10 @@ def process_metadata(self, model, cfg=None): input_names = [] if metadata is None: - input_names, input_shapes, input_types, input_quant_details, output_names, output_shapes, output_types, output_quant_details = self.extract_model_info(model) + try: + input_names, input_shapes, input_types, input_quant_details, output_names, output_shapes, output_types, output_quant_details = self.extract_model_info(model) + except NotImplementedError: + logger.warning("Model info could not be extracted.") # Detect model support code (Allow overwrite in metadata YAML) support_path = model_dir / "support" From b25cbd9b4ea686853e5c390f84ecfccaf2fa92e4 Mon Sep 17 00:00:00 2001 From: Philipp van Kempen Date: Wed, 24 Jan 2024 16:31:51 +0100 Subject: [PATCH 040/105] mlif: support generated model_support files for validation feature --- mlonmcu/models/frontend.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/mlonmcu/models/frontend.py b/mlonmcu/models/frontend.py index 8e8c77e3c..2d8121f73 100644 --- a/mlonmcu/models/frontend.py +++ b/mlonmcu/models/frontend.py @@ -291,7 +291,8 @@ def process_metadata(self, model, cfg=None): if support_path.is_dir(): assert cfg is not None # TODO: onlu overwrite if unset? - cfg.update({"mlif.model_support_dir": support_path}) + if cfg.get("mlif.model_support_dir", None) is not None: + cfg.update({"mlif.model_support_dir": support_path}) # cfg.update({"espidf.model_support_dir": support_path}) # cfg.update({"zephyr.model_support_dir": support_path}) if len(in_paths) > 0: From 6a6da26bdd03286a6deee1a1d040965760bd8602 Mon Sep 17 00:00:00 2001 From: Philipp van Kempen Date: Wed, 24 Jan 2024 18:28:35 +0100 Subject: [PATCH 041/105] move ModelSupport code into separate class and file --- mlonmcu/platform/mlif/interfaces.py | 252 +++++++++++++++++++++++++++ mlonmcu/platform/mlif/mlif.py | 189 ++------------------ mlonmcu/platform/mlif/mlif_target.py | 46 +++-- 3 files changed, 301 insertions(+), 186 deletions(-) create mode 100644 mlonmcu/platform/mlif/interfaces.py diff --git a/mlonmcu/platform/mlif/interfaces.py b/mlonmcu/platform/mlif/interfaces.py new file mode 100644 index 000000000..0b06d5aa6 --- /dev/null +++ b/mlonmcu/platform/mlif/interfaces.py @@ -0,0 +1,252 @@ +# +# Copyright (c) 2022 TUM Department of Electrical and Computer Engineering. +# +# This file is part of MLonMCU. +# See https://github.com/tum-ei-eda/mlonmcu.git for further info. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +"""MLIF Interfaces""" +from mlonmcu.models.utils import fill_data_source_inputs_only + +MAX_BATCH_SIZE = int(1e6) +DEFAULT_BATCH_SIZE = 10 + + +def get_header(): + return """ +#include "quantize.h" +#include "printing.h" +#include "exit.h" +// #include "ml_interface.h" +#include +#include +#include +#include +#include + +extern "C" { +int mlif_process_inputs(size_t, bool*); +int mlif_process_outputs(size_t); +void *mlif_input_ptr(int); +void *mlif_output_ptr(int); +int mlif_input_sz(int); +int mlif_output_sz(int); +int mlif_num_inputs(); +int mlif_num_outputs(); +} +""" + + +def get_top_rom(inputs_data): + in_bufs = [] + for i, ins_data in enumerate(inputs_data): + temp = [] + for j, in_data in enumerate(ins_data.values()): + byte_data = in_data.tobytes() + temp2 = ", ".join(["0x{:02x}".format(x) for x in byte_data] + [""]) + temp.append(temp2) + in_bufs.append(temp) + + return fill_data_source_inputs_only(in_bufs) + + +def get_process_inputs_head(): + return """ +int mlif_process_inputs(size_t batch_idx, bool *new_) +{ +""" + + +def get_process_inputs_tail(): + return """ +} +""" + + +def get_process_outputs_head(): + return """ +int mlif_process_outputs(size_t batch_idx) +{ +""" + + +def get_process_outputs_tail(): + return """ +} +""" + + +def get_process_inputs_rom(): + return """ + *new_ = true; + int num_inputs = mlif_num_inputs(); + for (int i = 0; i < num_inputs; i++) + { + int idx = num_inputs * batch_idx + i; + int size = mlif_input_sz(i); + char* model_input_ptr = (char*)mlif_input_ptr(i); + if (idx >= num_data_buffers_in) + { + *new_ = false; + break; + } + if (size != data_size_in[idx]) + { + return EXIT_MLIF_INVALID_SIZE; + } + memcpy(model_input_ptr, data_buffers_in[idx], size); + } + return 0; +""" + + +def get_process_inputs_stdin_raw(): + return """ + char ch; + *new_ = true; + for (int i = 0; i < mlif_num_inputs(); i++) + { + int cnt = 0; + int size = mlif_input_sz(i); + char* model_input_ptr = (char*)mlif_input_ptr(i); + while(read(STDIN_FILENO, &ch, 1) > 0) { + // printf("c=%c / %d\\n", ch, ch); + model_input_ptr[cnt] = ch; + cnt++; + if (cnt == size) { + break; + } + } + // printf("cnt=%d in_size=%lu\\n", cnt, in_size); + if (cnt == 0) { + *new_ = false; + return 0; + } + else if (cnt < size) + { + return EXIT_MLIF_INVALID_SIZE; + } + } + return 0; +""" + + +def get_process_outputs_stdout_raw(): + # TODO: maybe hardcode num_outputs and size here because we know it + # and get rid of loop? + return """ + for (int i = 0; i < mlif_num_outputs(); i++) + { + int8_t *model_output_ptr = (int8_t*)mlif_output_ptr(i); + int size = mlif_output_sz(i); + // TODO: move markers out of loop + write(1, "-?-", 3); + write(1, model_output_ptr, size); + write(1, "-!-\\n" ,4); + } + return 0; +""" + + +class ModelSupport: + + def __init__(self, in_interface, out_interface, model_info, target=None, batch_size=None, inputs_data=None): + self.model_info = model_info + self.target = target + self.inputs_data = inputs_data + self.in_interface = in_interface + self.out_interface = out_interface + self.in_interface, self.batch_size = self.select_set_inputs_interface(in_interface, batch_size) + self.out_interface, self.batch_size = self.select_get_outputs_interface(out_interface, self.batch_size) + + def select_set_inputs_interface(self, in_interface, batch_size): + if in_interface == "auto": + assert self.target is not None + if self.target.supports_filesystem: + in_interface = "filesystem" + elif self.target.supports_stdin: + in_interface = "stdin_raw" + # TODO: also allow stdin? + else: # Fallback + in_interface = "rom" + assert in_interface in ["filesystem", "stdin", "stdin_raw", "rom"] + if batch_size is None: + if in_interface == "rom": + batch_size = MAX_BATCH_SIZE # all inputs are in already compiled into program + else: + batch_size = DEFAULT_BATCH_SIZE + return in_interface, batch_size + + def select_get_outputs_interface(self, out_interface, batch_size): + if out_interface == "auto": + assert self.target is not None + if self.target.supports_filesystem: + out_interface = "filesystem" + elif self.target.supports_stdin: + out_interface = "stdout_raw" + # TODO: also allow stdout? + else: # Fallback + out_interface = "ram" + assert out_interface in ["filesystem", "stdout", "stdout_raw", "ram"] + if batch_size is None: + batch_size = DEFAULT_BATCH_SIZE + return out_interface, batch_size + + def generate_header(self): + # TODO: make this configurable + # TODO: do not require C++? + return get_header() + + def generate_top(self): + if self.in_interface == "rom": + return get_top_rom(self.inputs_data) + return "" + + def generate_bottom(self): + return "" + + def generate_process_inputs_body(self): + if self.in_interface == "rom": + return get_process_inputs_rom() + elif self.in_interface == "stdin_raw": + return get_process_inputs_stdin_raw() + raise NotImplementedError # TODO: implement: filesystem (bin+npy), stdout + + def generate_process_outputs_body(self): + if self.out_interface == "stdout_raw": + return get_process_outputs_stdout_raw() + raise NotImplementedError # TODO: implement: filesystem (bin+npy), ram + + def generate_process_inputs(self): + code = "" + code += get_process_inputs_head() + code += self.generate_process_inputs_body() + code += get_process_inputs_tail() + return code + + def generate_process_outputs(self): + code = "" + code += get_process_outputs_head() + code += self.generate_process_outputs_body() + code += get_process_outputs_tail() + return code + + def generate(self): + code = "" + code += self.generate_header() + code += self.generate_top() + code += self.generate_process_inputs() + code += self.generate_process_outputs() + code += self.generate_bottom() + return code diff --git a/mlonmcu/platform/mlif/mlif.py b/mlonmcu/platform/mlif/mlif.py index 9b38671ab..0ff808314 100644 --- a/mlonmcu/platform/mlif/mlif.py +++ b/mlonmcu/platform/mlif/mlif.py @@ -30,9 +30,10 @@ from mlonmcu.logging import get_logger from mlonmcu.target import get_targets from mlonmcu.target.target import Target -from mlonmcu.models.utils import get_data_source, fill_data_source_inputs_only +from mlonmcu.models.utils import get_data_source from ..platform import CompilePlatform, TargetPlatform +from .interfaces import ModelSupport from .mlif_target import get_mlif_platform_targets, create_mlif_platform_target logger = get_logger() @@ -420,181 +421,29 @@ def prepare_environment(self): env["PATH"] = path_new return env - def select_set_inputs_interface(self, target, batch_size): - in_interface = self.set_inputs_interface - if in_interface == "auto": - if target.supports_filesystem: - in_interface = "filesystem" - elif target.supports_stdin: - in_interface = "stdin_raw" - # TODO: also allow stdin? - else: # Fallback - in_interface = "rom" - assert in_interface in ["filesystem", "stdin", "stdin_raw", "rom"] - if batch_size is None: - if in_interface == "rom": - batch_size = 1e6 # all inputs are in already compiled into program - else: - batch_size = 10 - return in_interface, batch_size - - def select_get_outputs_interface(self, target, batch_size): - out_interface = self.get_outputs_interface - if out_interface == "auto": - if target.supports_filesystem: - out_interface = "filesystem" - elif target.supports_stdin: - out_interface = "stdout_raw" - # TODO: also allow stdout? - else: # Fallback - out_interface = "ram" - assert out_interface in ["filesystem", "stdout", "stdout_raw", "ram"] - if batch_size is None: - batch_size = 10 - return out_interface, batch_size - - def generate_model_support_code(self, in_interface, out_interface, batch_size): - code = "" - code += """ -#include "quantize.h" -#include "printing.h" -#include "exit.h" -// #include "ml_interface.h" -#include -#include -#include -#include -#include - -extern "C" { -int mlif_process_inputs(size_t, bool*); -int mlif_process_outputs(size_t); -void *mlif_input_ptr(int); -void *mlif_output_ptr(int); -int mlif_input_sz(int); -int mlif_output_sz(int); -int mlif_num_inputs(); -int mlif_num_outputs(); -} -""" - if in_interface == "rom": - assert self.inputs_artifact is not None - import numpy as np - data = np.load(self.inputs_artifact, allow_pickle=True) - in_bufs = [] - for i, ins_data in enumerate(data): - temp = [] - for j, in_data in enumerate(ins_data.values()): - byte_data = in_data.tobytes() - temp2 = ", ".join(["0x{:02x}".format(x) for x in byte_data] + [""]) - temp.append(temp2) - in_bufs.append(temp) - - code += fill_data_source_inputs_only(in_bufs) - code += """ -int mlif_process_inputs(size_t batch_idx, bool *new_) -{ - *new_ = true; - int num_inputs = mlif_num_inputs(); - for (int i = 0; i < num_inputs; i++) - { - int idx = num_inputs * batch_idx + i; - int size = mlif_input_sz(i); - char* model_input_ptr = (char*)mlif_input_ptr(i); - if (idx >= num_data_buffers_in) - { - *new_ = false; - break; - } - if (size != data_size_in[idx]) - { - return EXIT_MLIF_INVALID_SIZE; - } - memcpy(model_input_ptr, data_buffers_in[idx], size); - } - return 0; -} -""" - elif in_interface == "stdin_raw": - code += """ -int mlif_process_inputs(size_t batch_idx, bool *new_) -{ - char ch; - *new_ = true; - for (int i = 0; i < mlif_num_inputs(); i++) - { - int cnt = 0; - int size = mlif_input_sz(i); - char* model_input_ptr = (char*)mlif_input_ptr(i); - while(read(STDIN_FILENO, &ch, 1) > 0) { - // printf("c=%c / %d\\n", ch, ch); - model_input_ptr[cnt] = ch; - cnt++; - if (cnt == size) { - break; - } - } - // printf("cnt=%d in_size=%lu\\n", cnt, in_size); - if (cnt == 0) { - *new_ = false; - return 0; - } - else if (cnt < size) - { - return EXIT_MLIF_INVALID_SIZE; - } - } - return 0; -} -""" - elif in_interface == "stdin": - raise NotImplementedError - elif in_interface == "filesystem": - raise NotImplementedError - else: - assert False - if out_interface == "ram": - raise NotImplementedError - elif out_interface == "stdout_raw": - # TODO: maybe hardcode num_outputs and size here because we know it - # and get rid of loop? - code += """ -int mlif_process_outputs(size_t batch_idx) -{ - for (int i = 0; i < mlif_num_outputs(); i++) - { - int8_t *model_output_ptr = (int8_t*)mlif_output_ptr(i); - int size = mlif_output_sz(i); - // TODO: move markers out of loop - write(1, "-?-", 3); - write(1, model_output_ptr, size); - write(1, "-!-\\n" ,4); - } - return 0; -} -""" - elif out_interface == "stdout": - raise NotImplementedError - elif out_interface == "filesystem": - raise NotImplementedError - else: - assert False - return code - def generate_model_support(self, target): artifacts = [] - in_interface = None batch_size = self.batch_size - if self.set_inputs: - in_interface, batch_size = self.select_set_inputs_interface(target, batch_size) - if self.get_outputs: - out_interface, batch_size = self.select_get_outputs_interface(target, batch_size) - if in_interface or out_interface: - code = self.generate_model_support_code(in_interface, out_interface, batch_size) + inputs_data = None + if self.inputs_artifact is not None: + inputs_data = np.load(self.inputs_artifact, allow_pickle=True) + if self.model_info_file is not None: + with open(self.model_info_file, "r") as f: + model_info = yaml.safe_load(f) + if self.set_inputs or self.get_outputs: + model_support = ModelSupport( + in_interface=self.set_inputs_interface, + out_interface=self.get_outputs_interface, + model_info=model_info, + target=target, + batch_size=batch_size, + inputs_data=inputs_data, + ) + code = model_support.generate() code_artifact = Artifact( "model_support.cpp", content=code, fmt=ArtifactFormat.TEXT, flags=("model_support"), ) - self.definitions["BATCH_SIZE"] = batch_size + self.definitions["BATCH_SIZE"] = model_support.batch_size artifacts.append(code_artifact) return artifacts diff --git a/mlonmcu/platform/mlif/mlif_target.py b/mlonmcu/platform/mlif/mlif_target.py index 85c2baf13..863feb955 100644 --- a/mlonmcu/platform/mlif/mlif_target.py +++ b/mlonmcu/platform/mlif/mlif_target.py @@ -25,6 +25,8 @@ from mlonmcu.artifact import Artifact, ArtifactFormat from mlonmcu.logging import get_logger +from .interfaces import ModelSupport + logger = get_logger() @@ -59,20 +61,36 @@ def exec(self, program, *args, cwd=os.getcwd(), **kwargs): in_interface = None out_interface = None batch_size = 1 - if self.platform.set_inputs: - # first figure out how many inputs are provided - assert self.platform.inputs_artifact is not None - import numpy as np - data = np.load(self.platform.inputs_artifact, allow_pickle=True) - # print("data", data, type(data)) - num_inputs = len(data) - in_interface, batch_size = self.platform.select_set_inputs_interface(self, self.platform.batch_size) - outs_file = None encoding = "utf-8" - if self.platform.get_outputs: - out_interface, batch_size = self.platform.select_get_outputs_interface(self, batch_size) + model_info_file = self.platform.model_info_file # TODO: replace workaround (add model info to platform?) + if self.platform.set_inputs or self.platform.get_outputs: + # first figure out how many inputs are provided + if model_info_file is not None: + import yaml + with open(model_info_file, "r") as f: + model_info_data = yaml.safe_load(f) + else: + model_info_data = None + if self.platform.inputs_artifact is not None: + import numpy as np + data = np.load(self.platform.inputs_artifact, allow_pickle=True) + num_inputs = len(data) + else: + data = None + model_support = ModelSupport( + in_interface=self.platform.set_inputs_interface, + out_interface=self.platform.get_outputs_interface, + model_info=model_info_data, + target=self, + batch_size=self.platform.batch_size, + inputs_data=data, + ) + in_interface = model_support.in_interface + out_interface = model_support.out_interface + batch_size = model_support.batch_size if out_interface == "stdout_raw": encoding = None + outs_file = None ret = "" artifacts = [] num_batches = max(ceil(num_inputs / batch_size), 1) @@ -121,11 +139,7 @@ def exec(self, program, *args, cwd=os.getcwd(), **kwargs): raise NotImplementedError elif out_interface == "stdout_raw": # DUMMY BELOW - model_info_file = self.platform.model_info_file # TODO: replace workaround (add model info to platform?) - assert model_info_file is not None - import yaml - with open(model_info_file, "r") as f: - model_info_data = yaml.safe_load(f) + assert model_info_data is not None # print("model_info_data", model_info_data) # dtype = "int8" # shape = [1, 10] From 69665a4ace566cccea936b3883b2ba3035f50d51 Mon Sep 17 00:00:00 2001 From: Philipp van Kempen Date: Thu, 25 Jan 2024 12:25:02 +0100 Subject: [PATCH 042/105] mlif.py: add missing imports --- mlonmcu/platform/mlif/mlif.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/mlonmcu/platform/mlif/mlif.py b/mlonmcu/platform/mlif/mlif.py index 0ff808314..1765979f3 100644 --- a/mlonmcu/platform/mlif/mlif.py +++ b/mlonmcu/platform/mlif/mlif.py @@ -20,9 +20,11 @@ import os import tempfile from typing import Tuple - from pathlib import Path +import yaml +import numpy as np + from mlonmcu.config import str2bool from mlonmcu.setup import utils # TODO: Move one level up? from mlonmcu.timeout import exec_timeout From f642c6851afd69cf6b3a496d75b177874e89f017 Mon Sep 17 00:00:00 2001 From: Philipp van Kempen Date: Thu, 25 Jan 2024 12:40:57 +0100 Subject: [PATCH 043/105] update validation readme --- ValidationNew.md | 21 +++++++++++++++++---- 1 file changed, 17 insertions(+), 4 deletions(-) diff --git a/ValidationNew.md b/ValidationNew.md index 603d08a48..aee22fb31 100644 --- a/ValidationNew.md +++ b/ValidationNew.md @@ -9,8 +9,6 @@ - `resnet/support/mlif_override.cpp` - Comment out old validation code - Add example code to dump raw inputs/outputs via stdin/stdout -- `mlonmcu-sw` (`$MLONMCU_HOME/deps/src/mlif`): - - Nothing changed (yet) - `mlonmcu` (Branch: `refactor-validate`): - `mlonmcu/target/common.py` - Allow overriding used encoding for stdout (Required for dumping raw data) @@ -41,7 +39,21 @@ - `mlonmcu/session/postprocess/postprocesses.py` - Add `ValidateOutputs` postprocess (WIP) - +## Updates (25.01.2024) +- `mlonmcu-models` (`$MLONMCU_HOME/models/`, Branch: `refactor-validate`): + - Update some more definition.yml files +- `mlonmcu-sw` (`$MLONMCU_HOME/deps/src/mlif`, Branch: `refactor-validate` **NEW**): + - Added versioning to ml_interface template (Use `-c mlif.template_version=v2` to use the updated mlif) + - Link `mlifio` utilities to ml_interface + - Drop old model support and data handling + - Refactor API to be aware of batch index + - Replace `process_input` and `process_output` functions by `process_inputs` and `process_outputs` processing all inputs at once + - Expose `TVMWrap_GetInputPtr`,... to model support via `mlif_input_ptr`,... + - Model support will determine input size, pointers via above mentioned API instead of arguments. +- `mlonmcu` (Branch: `refactor-validate`): + - TODO + - Allow overriding model support files + - Add generation of model support files ## Examples @@ -99,13 +111,14 @@ python3 -m mlonmcu.cli.main flow run resnet -v \ # platform: mlif target: host_x86` # TODO: should support same configs as spike target +# TODO: gen_data/gen_ref_data ``` ## TODOs - [ ] Fix broken targets (due to refactoring of `self.exec`) -> PHILIPP - [ ] Add missing target checks (see above) -> PHILIPP - [ ] Update `definition.yml` for other models in `mlonmcu-sw` (At least `aww`, `vww`, `toycar`) -> LIU -- [ ] Refactor model support (see `mlomcu_sw/lib/ml_interface`) to be aware of output/input tensor index (maybe even name?) und sample index -> PHILIPP +- [x] Refactor model support (see `mlomcu_sw/lib/ml_interface`) to be aware of output/input tensor index (maybe even name?) und sample index -> PHILIPP - [ ] Write generator for custom `mlif_override.cpp` (based on `model_info.yml` + `in_interface` + `out_interface` (+ `inputs.npy`)) -> LIU - [ ] Eliminate hacks used to get `model_info.yml` and `inputs.yml` in RUN stage -> PHILIPP - [ ] Implement missing interfaces for tvm (out: `stdout`) -> LIU From 77347a9eff8870ceabca85858486fc7426803519 Mon Sep 17 00:00:00 2001 From: Philipp van Kempen Date: Thu, 25 Jan 2024 12:43:31 +0100 Subject: [PATCH 044/105] fix typo in mlonmcu/models/frontend.py --- mlonmcu/models/frontend.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mlonmcu/models/frontend.py b/mlonmcu/models/frontend.py index 2d8121f73..a88d043e5 100644 --- a/mlonmcu/models/frontend.py +++ b/mlonmcu/models/frontend.py @@ -275,7 +275,7 @@ def process_metadata(self, model, cfg=None): if len(input_shapes) > 0: assert len(input_types) in [len(input_shapes), 0] input_names = list(input_shapes.keys()) - elif len(input_shapes) > 0: + elif len(input_types) > 0: input_names = list(input_types.keys()) else: input_names = [] From 7e0f9ae25b84488cc056750b95e8665b0590697e Mon Sep 17 00:00:00 2001 From: Philipp van Kempen Date: Thu, 25 Jan 2024 13:04:26 +0100 Subject: [PATCH 045/105] update validation readme --- ValidationNew.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ValidationNew.md b/ValidationNew.md index aee22fb31..6960f2a84 100644 --- a/ValidationNew.md +++ b/ValidationNew.md @@ -51,7 +51,8 @@ - Expose `TVMWrap_GetInputPtr`,... to model support via `mlif_input_ptr`,... - Model support will determine input size, pointers via above mentioned API instead of arguments. - `mlonmcu` (Branch: `refactor-validate`): - - TODO + - Add `mlif.template_version` to select ml_interface version (v1 vs v2), will be automatic in the future + - Add `mlif.batch_size` config to override default batch size - Allow overriding model support files - Add generation of model support files From 577d0a5a5631d61c1fecbcbc3fa978836543566c Mon Sep 17 00:00:00 2001 From: Philipp van Kempen Date: Thu, 25 Jan 2024 13:09:08 +0100 Subject: [PATCH 046/105] update validation readme --- ValidationNew.md | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/ValidationNew.md b/ValidationNew.md index 6960f2a84..052bda114 100644 --- a/ValidationNew.md +++ b/ValidationNew.md @@ -124,9 +124,10 @@ python3 -m mlonmcu.cli.main flow run resnet -v \ - [ ] Eliminate hacks used to get `model_info.yml` and `inputs.yml` in RUN stage -> PHILIPP - [ ] Implement missing interfaces for tvm (out: `stdout`) -> LIU - [ ] Implement missing interfaces for mlif platform (in: `filesystem`, `stdin`; out: `filesystem`, `stdout`) -> LIU -- [ ] Implement missing interfaces for mlif platform (in: `rom`) -> PHILIPP) +- [x] Implement missing interfaces for mlif platform (in: `rom`) -> PHILIPP) - [ ] Add support for multi-output/multi-input -> PHILIPP/LIU -- [x] Update `gen_data` & `gen_ref_data` feature (see NotImplementedErrors, respect fmt,...) +- [x] Update `gen_data` & `gen_ref_data` feature (see NotImplementedErrors, respect fmt,...) -> PHILIPP +- [x] Implement model-based gen_ref_data mode (currenly only tflite inference) -> PHILIPP - [ ] Move `gen_data` & `gen_ref_data` from LOAD stage to custom stage (remove dependency on tflite frontend) -> PHILIPP - [ ] Test with targets: `tvm_cpu`, `host_x86`, `spike` (See example commands above) -> LIU - [ ] Extend `validate_outputs` postprocess (Add `report`, implement `atol`/`rtol`, `fail_on_error`, `top-k`,...) -> LIU @@ -137,4 +138,4 @@ python3 -m mlonmcu.cli.main flow run resnet -v \ - [ ] Add tests -> LIU/PHILIPP - [ ] Streamline `model_info.yml` with BUILD stage `ModelInfo` -> PHILIPP - [ ] Improve artifacts handling -> PHILIPP -- [ ] Support automatic quantization of inputs (See `vww` and `toycar`) -> PHILIPP/LIU +- [x] Support automatic quantization of inputs (See `vww` and `toycar`) -> PHILIPP/LIU From 676d75ebec5f3b5a9eeda3a158dce647b121a4c0 Mon Sep 17 00:00:00 2001 From: Philipp van Kempen Date: Sun, 28 Jan 2024 01:37:38 +0100 Subject: [PATCH 047/105] update validation readme --- ValidationNew.md | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/ValidationNew.md b/ValidationNew.md index 052bda114..6cfc85453 100644 --- a/ValidationNew.md +++ b/ValidationNew.md @@ -42,6 +42,8 @@ ## Updates (25.01.2024) - `mlonmcu-models` (`$MLONMCU_HOME/models/`, Branch: `refactor-validate`): - Update some more definition.yml files + - Resnet: add support_new dir for ml_interface v2 compatibility + - Resnet: add support_new_mlifio as demo/test of new interfaces - `mlonmcu-sw` (`$MLONMCU_HOME/deps/src/mlif`, Branch: `refactor-validate` **NEW**): - Added versioning to ml_interface template (Use `-c mlif.template_version=v2` to use the updated mlif) - Link `mlifio` utilities to ml_interface @@ -59,6 +61,29 @@ ## Examples +### Generation of Data (NEW) + +``` +python3 -m mlonmcu.cli.main flow load resnet -f gen_data -c gen_data.number=5 -c gen_data.fill_mode=zeros -c run.export_optional=1 +python3 -m mlonmcu.cli.main flow load resnet -f gen_data -c gen_data.number=5 -c gen_data.fill_mode=ones -c run.export_optional=1 +python3 -m mlonmcu.cli.main flow load resnet -f gen_data -f gen_ref_data -c gen_data.number=5 -c gen_data.fill_mode=random -c gen_ref_data.mode=model -c run.export_optional=1 +``` + +### Generated Model Support (NEW) + +``` +# mlifio demo (hardcoded, not functional) +python3 -m mlonmcu.cli.main flow run resnet --target host_x86 -f debug --backend tvmaotplus -c mlif.print_outputs=1 -c host_x86.print_outputs=1 -v -f validate -c mlif.template_version=v2 --parallel -c mlif.model_support_dir=$MLONMCU_HOME/models/resnet/support_new_mlifio -c mlif.batch_size=5 + +# in: rom, out: stdout_raw +python3 -m mlonmcu.cli.main flow run resnet --target host_x86 -f debug --backend tvmaotplus -c mlif.print_outputs=1 -c host_x86.print_outputs=1 -v -f validate -c mlif.template_version=v2 --parallel --feature validate_new --post validate_outputs -c set_inputs.interface=rom get_outputs.interface=stdout_raw -c run.export_optional=1 -c gen_data.number=5 -c gen_data.fill_mode=random -c gen_ref_data.mode=model + +# in: stdin_raw, out: stdout_raw +python3 -m mlonmcu.cli.main flow run resnet --target host_x86 -f debug --backend tvmaotplus -c mlif.print_outputs=1 -c host_x86.print_outputs=1 -v -f validate -c mlif.template_version=v2 --parallel --feature validate_new --post validate_outputs -c set_inputs.interface=stdin_raw get_outputs.interface=stdout_raw -c run.export_optional=1 -c gen_data.number=5 -c gen_data.fill_mode=random -c gen_ref_data.mode=model +``` + +### Full Flow + ``` # platform: tvm target: tvm_cpu` # implemented: From 09aeafd12129d395b956fdf18140a821e6fda390 Mon Sep 17 00:00:00 2001 From: Philipp van Kempen Date: Sun, 28 Jan 2024 01:38:01 +0100 Subject: [PATCH 048/105] fix typo in gen_ref_data property --- mlonmcu/models/frontend.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mlonmcu/models/frontend.py b/mlonmcu/models/frontend.py index a88d043e5..64726d794 100644 --- a/mlonmcu/models/frontend.py +++ b/mlonmcu/models/frontend.py @@ -123,7 +123,7 @@ def gen_data_fmt(self): @property def gen_ref_data(self): - value = self.config["gen_data"] + value = self.config["gen_ref_data"] return str2bool(value) if not isinstance(value, (bool, int)) else value @property From 725ac5dba1a6b17dd4a6dddb3b9bd1795019a076 Mon Sep 17 00:00:00 2001 From: Philipp van Kempen Date: Wed, 28 Feb 2024 17:28:04 +0100 Subject: [PATCH 049/105] fixes --- mlonmcu/models/frontend.py | 91 ++++++++++++-- mlonmcu/platform/mlif/mlif.py | 3 +- mlonmcu/session/postprocess/postprocesses.py | 123 +++++++++++++++++-- 3 files changed, 196 insertions(+), 21 deletions(-) diff --git a/mlonmcu/models/frontend.py b/mlonmcu/models/frontend.py index 64726d794..f04424cb0 100644 --- a/mlonmcu/models/frontend.py +++ b/mlonmcu/models/frontend.py @@ -337,27 +337,78 @@ def process_metadata(self, model, cfg=None): for i in range(self.gen_data_number): print("i", i) data = {} + NEW = True for ii, input_name in enumerate(input_names): print("ii", ii) assert input_name in input_types, f"Unknown dtype for input: {input_name}" dtype = input_types[input_name] quant = input_quant_details.get(input_name, None) + gen_dtype = dtype if quant: _, _, ty = quant - # dtype = ty + assert "float" in ty, "Input already quantized?" + if NEW: + gen_dtype = ty assert input_name in input_shapes, f"Unknown shape for input: {input_name}" shape = input_shapes[input_name] if self.gen_data_fill_mode == "zeros": - arr = np.zeros(shape, dtype=dtype) + arr = np.zeros(shape, dtype=gen_dtype) elif self.gen_data_fill_mode == "ones": - arr = np.ones(shape, dtype=dtype) + arr = np.ones(shape, dtype=gen_dtype) elif self.gen_data_fill_mode == "random": - if "float" in dtype: - arr = np.random.rand(*shape).astype(dtype) - elif "int" in dtype: - arr = np.random.randint(np.iinfo(dtype).min, np.iinfo(dtype).max, size=shape, dtype=dtype) + DIST = "uniform" + if DIST == "uniform": + UPPER = None # TODO: config + LOWER = None # TODO: config + if "float" in gen_dtype: + if UPPER is None: + # UPPER = 1.0 + UPPER = 0.5 + if LOWER is None: + # LOWER = -1.0 + LOWER = -0.5 + elif "int" in gen_dtype: + dtype_info = np.iinfo(gen_dtype), + if UPPER is None: + UPPER = dtype_info.max + else: + assert UPPER <= dtype_info.max, f"Out of dtype bound" + if LOWER is None: + LOWER = dtype_info.min + else: + assert LOWER >= dtype_info.min, f"Out of dtype bound" + else: + raise RuntimeError(f"Unsupported dtype: {gen_dtype}") + RANGE = UPPER - LOWER + print("dtype", dtype) + print("gen_dtype", gen_dtype) + print("UPPER,LOWER,RANGE", UPPER, LOWER, RANGE) + assert RANGE > 0 + arr = np.random.uniform(LOWER, UPPER, shape) + print("arr0", arr) + arr = arr.astype(gen_dtype) + print("arr1", arr) + # input("?=") + # if "float" in dtype: + # arr = np.random.rand(*shape).astype(dtype) + # elif "int" in dtype: + # arr = np.random.randint(np.iinfo(dtype).min, np.iinfo(dtype).max, size=shape, dtype=dtype) + # else: + # assert False + # Quantize if required + if gen_dtype != dtype: + assert "int" in dtype + assert quant + scale, shift, ty = quant + arr = (arr / scale) + shift + print("arr2", arr) + arr = np.around(arr) + print("arr3", arr) + arr = arr.astype(dtype) + print("arr4", arr) + # input("!=") else: - assert False + raise RuntimeError(f"Unsupported distribution: {DIST}") else: assert False data[input_name] = arr @@ -373,10 +424,11 @@ def process_metadata(self, model, cfg=None): else: files = in_paths temp = {} + NEW = True for file in files: if not isinstance(file, Path): file = Path(file) - assert file.is_file() + assert file.is_file(), f"Not found: {file}" print("file", file) basename, ext = file.stem, file.suffix if ext == ".bin": @@ -408,13 +460,28 @@ def process_metadata(self, model, cfg=None): assert input_name in input_types, f"Unknown dtype for input: {input_name}" dtype = input_types[input_name] quant = input_quant_details.get(input_name, None) + gen_dtype = dtype if quant: _, _, ty = quant - dtype = ty - arr = np.frombuffer(temp[i][ii], dtype=dtype) - assert ii < len(input_shapes) + assert "float" in ty, "Input already quantized?" + if NEW: + gen_dtype = ty + arr = np.frombuffer(temp[i][ii], dtype=gen_dtype) + assert input_name in input_shapes, f"Unknown shape for input: {input_name}" shape = input_shapes[input_name] arr = np.reshape(arr, shape) + # Quantize if required + if gen_dtype != dtype: + assert "int" in dtype + assert quant + scale, shift, ty = quant + arr = (arr / scale) + shift + print("arr2", arr) + arr = np.around(arr) + print("arr3", arr) + arr = arr.astype(dtype) + print("arr4", arr) + # input("!=") data[input_name] = arr inputs_data.append(data) else: diff --git a/mlonmcu/platform/mlif/mlif.py b/mlonmcu/platform/mlif/mlif.py index 1765979f3..9aa40c315 100644 --- a/mlonmcu/platform/mlif/mlif.py +++ b/mlonmcu/platform/mlif/mlif.py @@ -470,7 +470,8 @@ def configure(self, target, src, _model): if self.ignore_data: cmakeArgs.append("-DDATA_SRC=") else: - data_artifact = self.gen_data_artifact() + # data_artifact = self.gen_data_artifact() + data_artifact = None if data_artifact: data_file = self.build_dir / data_artifact.name data_artifact.export(data_file) diff --git a/mlonmcu/session/postprocess/postprocesses.py b/mlonmcu/session/postprocess/postprocesses.py index a3b3e93d7..e3c75371b 100644 --- a/mlonmcu/session/postprocess/postprocesses.py +++ b/mlonmcu/session/postprocess/postprocesses.py @@ -1503,7 +1503,7 @@ def post_run(self, report, artifacts): outputs_ref_artifact = outputs_ref_artifact[0] import numpy as np outputs_ref = np.load(outputs_ref_artifact.path, allow_pickle=True) - import copy + # import copy # outputs = copy.deepcopy(outputs_ref) # outputs[1][list(outputs[1].keys())[0]][0] = 42 outputs_artifact = lookup_artifacts(artifacts, name="outputs.npy", first_only=True) @@ -1511,8 +1511,18 @@ def post_run(self, report, artifacts): outputs_artifact = outputs_artifact[0] outputs = np.load(outputs_artifact.path, allow_pickle=True) compared = 0 - matching = 0 + # matching = 0 missing = 0 + metrics = { + "allclose(atol=0.0,rtol=0.0)": 0, + "allclose(atol=0.05,rtol=0.05)": 0, + "allclose(atol=0.1,rtol=0.1)": 0, + "topk(n=1)": 0, + "topk(n=2)": 0, + "topk(n=inf)": 0, + "toy": 0, + "mse": 0, + } for i, output_ref in enumerate(outputs_ref): if i >= len(outputs): logger.warning("Missing output sample") @@ -1533,6 +1543,8 @@ def post_run(self, report, artifacts): else: RuntimeError(f"Output not found: {out_name}") # optional dequantize + # print("out_data_before_quant", out_data) + # print("sum(out_data_before_quant", np.sum(out_data)) quant = model_info_data.get("output_quant_details", None) if quant: assert ii < len(quant) @@ -1545,13 +1557,106 @@ def post_run(self, report, artifacts): assert out_data.dtype.name in ["int8"], "Dequantization only supported for int8 input" assert quant_dtype in ["float32"], "Dequantization only supported for float32 output" out_data = (out_data.astype("float32") - quant_zero_point) * quant_scale - print("out_data", out_data) - print("out_ref_data", out_ref_data) + # print("out_data", out_data) + # print("sum(out_data)", np.sum(out_data)) + # print("out_ref_data", out_ref_data) + # print("sum(out_ref_data)", np.sum(out_ref_data)) + # input("TIAW") assert out_data.dtype == out_ref_data.dtype, "dtype missmatch" assert out_data.shape == out_ref_data.shape, "shape missmatch" # if np.allclose(out_data, out_ref_data, rtol=0, atol=0): - if np.allclose(out_data, out_ref_data, rtol=0, atol=0.1): - matching += 1 + # if np.allclose(out_data, out_ref_data, rtol=0.1, atol=0.1): + # if np.allclose(out_data, out_ref_data, rtol=0.0, atol=0.0): + # if np.allclose(out_data, out_ref_data, rtol=0.01, atol=0.01): + + def mse_helper(data, ref_data, thr): + mse = ((data - ref_data)**2).mean() + print("mse", mse) + return mse < thr + + def toy_helper(data, ref_data, atol, rtol): + data_flat = data.flatten().tolist() + ref_data_flat = ref_data.flatten().tolist() + res = 0 + ref_res = 0 + length = len(data_flat) + for jjj in range(length): + res += data_flat[jjj] ** 2 + ref_res += ref_data_flat[jjj] ** 2 + res /= length + ref_res /= length + print("res", res) + print("ref_res", ref_res) + return np.allclose([res], [ref_res], atol=atol, rtol=rtol) + + def topk_helper(data, ref_data, n): + # TODO: only for classification models! + # TODO: support multi_outputs? + data_sorted_idx = list(reversed(np.argsort(data).tolist()[0])) + ref_data_sorted_idx = list(reversed(np.argsort(ref_data).tolist()[0])) + k = 0 + num_checks = min(n, len(data_sorted_idx)) + assert len(data_sorted_idx) == len(ref_data_sorted_idx) + # print("data_sorted_idx", data_sorted_idx, type(data_sorted_idx)) + # print("ref_data_sorted_idx", ref_data_sorted_idx, type(ref_data_sorted_idx)) + # print("num_checks", num_checks) + for j in range(num_checks): + # print("j", j) + # print(f"data_sorted_idx[{j}]", data_sorted_idx[j], type(data_sorted_idx[j])) + idx = data_sorted_idx[j] + # print("idx", idx) + ref_idx = ref_data_sorted_idx[j] + # print("ref_idx", ref_idx) + if idx == ref_idx: + # print("IF") + k += 1 + else: + # print("ELSE") + if data.tolist()[0][idx] == ref_data.tolist()[0][ref_idx]: + # print("SAME") + k += 1 + else: + # print("BREAK") + break + # print("k", k) + if k < num_checks: + return False + elif k == num_checks: + return True + else: + assert False + for metric_name in metrics: + if "allclose" in metric_name: + if metric_name == "allclose(atol=0.0,rtol=0.0)": + atol = 0.0 + rtol = 0.0 + elif metric_name == "allclose(atol=0.05,rtol=0.05)": + atol = 0.05 + rtol = 0.05 + elif metric_name == "allclose(atol=0.1,rtol=0.1)": + atol = 0.1 + rtol = 0.1 + else: + raise NotImplementedError + if np.allclose(out_data, out_ref_data, rtol=rtol, atol=atol): + metrics[metric_name] += 1 + elif "topk" in metric_name: + if metric_name == "topk(n=1)": + n = 1 + elif metric_name == "topk(n=2)": + n = 2 + elif metric_name == "topk(n=inf)": + n = 1000000 + else: + raise NotImplementedError + if topk_helper(out_data, out_ref_data, n): + metrics[metric_name] += 1 + elif metric_name == "toy": + if toy_helper(out_data, out_ref_data, 0.01, 0.01): + metrics[metric_name] += 1 + elif metric_name == "mse": + if mse_helper(out_data, out_ref_data, 0.1): + metrics[metric_name] += 1 compared += 1 ii += 1 if self.report: @@ -1560,6 +1665,8 @@ def post_run(self, report, artifacts): raise NotImplementedError if self.rtol: raise NotImplementedError - res = f"{matching}/{compared} ({int(matching/compared*100)}%)" - report.post_df["Validation Result"] = res + for metric_name, metric_data in metrics.items(): + matching = metric_data + res = f"{matching}/{compared} ({int(matching/compared*100)}%)" + report.post_df[f"{metric_name}"] = res return [] From 0e839e14b560dd3e12a227baabccfe624db70c8b Mon Sep 17 00:00:00 2001 From: Philipp van Kempen Date: Wed, 20 Mar 2024 16:00:13 +0100 Subject: [PATCH 050/105] validate_outputs: add experimental pm1 metric --- mlonmcu/session/postprocess/postprocesses.py | 72 +++++++++++++++++--- 1 file changed, 61 insertions(+), 11 deletions(-) diff --git a/mlonmcu/session/postprocess/postprocesses.py b/mlonmcu/session/postprocess/postprocesses.py index e3c75371b..f3962404b 100644 --- a/mlonmcu/session/postprocess/postprocesses.py +++ b/mlonmcu/session/postprocess/postprocesses.py @@ -1514,14 +1514,17 @@ def post_run(self, report, artifacts): # matching = 0 missing = 0 metrics = { - "allclose(atol=0.0,rtol=0.0)": 0, - "allclose(atol=0.05,rtol=0.05)": 0, - "allclose(atol=0.1,rtol=0.1)": 0, - "topk(n=1)": 0, - "topk(n=2)": 0, - "topk(n=inf)": 0, - "toy": 0, - "mse": 0, + "allclose(atol=0.0,rtol=0.0)": None, + "allclose(atol=0.05,rtol=0.05)": None, + "allclose(atol=0.1,rtol=0.1)": None, + "topk(n=1)": None, + "topk(n=2)": None, + "topk(n=inf)": None, + "toy": None, + "mse(thr=0.1)": None, + "mse(thr=0.05)": None, + "mse(thr=0.01)": None, + "+-1": None, } for i, output_ref in enumerate(outputs_ref): if i >= len(outputs): @@ -1545,6 +1548,19 @@ def post_run(self, report, artifacts): # optional dequantize # print("out_data_before_quant", out_data) # print("sum(out_data_before_quant", np.sum(out_data)) + + def pm1_helper(data, ref_data): + data_ = data.flatten().tolist() + ref_data_ = ref_data.flatten().tolist() + + length = len(data_) + for jjj in range(length): + diff = abs(data_[jjj] - ref_data_[jjj]) + print("diff", diff) + if diff > 1: + print("r FALSE") + return False + return True quant = model_info_data.get("output_quant_details", None) if quant: assert ii < len(quant) @@ -1556,6 +1572,13 @@ def post_run(self, report, artifacts): # need to dequantize here assert out_data.dtype.name in ["int8"], "Dequantization only supported for int8 input" assert quant_dtype in ["float32"], "Dequantization only supported for float32 output" + for metric_name in metrics: + if metric_name == "+-1": + if metrics[metric_name] is None: + metrics[metric_name] = 0 + out_ref_data_quant = np.around((out_ref_data / quant_scale) + quant_zero_point).astype("int8") + if pm1_helper(out_data, out_ref_data_quant): + metrics[metric_name] += 1 out_data = (out_data.astype("float32") - quant_zero_point) * quant_scale # print("out_data", out_data) # print("sum(out_data)", np.sum(out_data)) @@ -1627,6 +1650,8 @@ def topk_helper(data, ref_data, n): assert False for metric_name in metrics: if "allclose" in metric_name: + if metrics[metric_name] is None: + metrics[metric_name] = 0 if metric_name == "allclose(atol=0.0,rtol=0.0)": atol = 0.0 rtol = 0.0 @@ -1641,6 +1666,11 @@ def topk_helper(data, ref_data, n): if np.allclose(out_data, out_ref_data, rtol=rtol, atol=atol): metrics[metric_name] += 1 elif "topk" in metric_name: + data_len = len(out_data.flatten().tolist()) + if data_len > 25: # Probably no classification + continue + if metrics[metric_name] is None: + metrics[metric_name] = 0 if metric_name == "topk(n=1)": n = 1 elif metric_name == "topk(n=2)": @@ -1652,11 +1682,28 @@ def topk_helper(data, ref_data, n): if topk_helper(out_data, out_ref_data, n): metrics[metric_name] += 1 elif metric_name == "toy": + data_len = len(out_data.flatten().tolist()) + if data_len != 640: + continue + if metrics[metric_name] is None: + metrics[metric_name] = 0 if toy_helper(out_data, out_ref_data, 0.01, 0.01): metrics[metric_name] += 1 elif metric_name == "mse": - if mse_helper(out_data, out_ref_data, 0.1): + if metrics[metric_name] is None: + metrics[metric_name] = 0 + if metric_name == "mse(thr=0.1)": + thr = 0.1 + elif metric_name == "mse(thr=0.05)": + thr = 0.05 + elif metric_name == "mse(thr=0.01)": + thr = 0.01 + else: + raise NotImplementedError + if mse_helper(out_data, out_ref_data, thr): metrics[metric_name] += 1 + elif metric_name == "+-1": + continue compared += 1 ii += 1 if self.report: @@ -1666,7 +1713,10 @@ def topk_helper(data, ref_data, n): if self.rtol: raise NotImplementedError for metric_name, metric_data in metrics.items(): - matching = metric_data - res = f"{matching}/{compared} ({int(matching/compared*100)}%)" + if metric_data is None: + res = "N/A" + else: + matching = metric_data + res = f"{matching}/{compared} ({int(matching/compared*100)}%)" report.post_df[f"{metric_name}"] = res return [] From 36684004f910f41baa77299b0c30ecbbda7e9c80 Mon Sep 17 00:00:00 2001 From: Philipp van Kempen Date: Sun, 31 Mar 2024 13:32:03 +0200 Subject: [PATCH 051/105] frontends: move generation of inout data to different methods --- mlonmcu/models/frontend.py | 622 ++++++++++++++++++++----------------- 1 file changed, 329 insertions(+), 293 deletions(-) diff --git a/mlonmcu/models/frontend.py b/mlonmcu/models/frontend.py index f04424cb0..5a1ec244f 100644 --- a/mlonmcu/models/frontend.py +++ b/mlonmcu/models/frontend.py @@ -188,6 +188,265 @@ def process_features(self, features): def produce_artifacts(self, model): pass + def generate_input_data(self, input_names, input_types, input_shapes, input_quant_details, in_paths): + # TODO: drop self and move method out of frontends.py, support non-tflite models + assert self.gen_data + inputs_data = [] + if self.gen_data_fill_mode in ["zeros", "ones", "random"]: + for i in range(self.gen_data_number): + data = {} + NEW = True + for ii, input_name in enumerate(input_names): + assert input_name in input_types, f"Unknown dtype for input: {input_name}" + dtype = input_types[input_name] + quant = input_quant_details.get(input_name, None) + gen_dtype = dtype + if quant: + _, _, ty = quant + assert "float" in ty, "Input already quantized?" + if NEW: + gen_dtype = ty + assert input_name in input_shapes, f"Unknown shape for input: {input_name}" + shape = input_shapes[input_name] + if self.gen_data_fill_mode == "zeros": + arr = np.zeros(shape, dtype=gen_dtype) + elif self.gen_data_fill_mode == "ones": + arr = np.ones(shape, dtype=gen_dtype) + elif self.gen_data_fill_mode == "random": + DIST = "uniform" + if DIST == "uniform": + UPPER = None # TODO: config + LOWER = None # TODO: config + if "float" in gen_dtype: + if UPPER is None: + # UPPER = 1.0 + UPPER = 0.5 + if LOWER is None: + # LOWER = -1.0 + LOWER = -0.5 + elif "int" in gen_dtype: + dtype_info = (np.iinfo(gen_dtype),) + if UPPER is None: + UPPER = dtype_info.max + else: + assert UPPER <= dtype_info.max, f"Out of dtype bound" + if LOWER is None: + LOWER = dtype_info.min + else: + assert LOWER >= dtype_info.min, f"Out of dtype bound" + else: + raise RuntimeError(f"Unsupported dtype: {gen_dtype}") + RANGE = UPPER - LOWER + assert RANGE > 0 + arr = np.random.uniform(LOWER, UPPER, shape) + arr = arr.astype(gen_dtype) + # input("?=") + # if "float" in dtype: + # arr = np.random.rand(*shape).astype(dtype) + # elif "int" in dtype: + # arr = np.random.randint(np.iinfo(dtype).min, np.iinfo(dtype).max, size=shape, dtype=dtype) + # else: + # assert False + # Quantize if required + if gen_dtype != dtype: + assert "int" in dtype + assert quant + scale, shift, ty = quant + arr = (arr / scale) + shift + arr = np.around(arr) + arr = arr.astype(dtype) + # input("!=") + else: + raise RuntimeError(f"Unsupported distribution: {DIST}") + else: + assert False + data[input_name] = arr + assert len(data) > 0 + inputs_data.append(data) + elif self.gen_data_fill_mode == "file": + if self.gen_data_file == "auto": + len(in_paths) > 0 + if len(in_paths) == 1: + if in_paths[0].is_dir(): + files = list(in_paths[0].iterdir()) + else: + files = in_paths + temp = {} + NEW = True + for file in files: + if not isinstance(file, Path): + file = Path(file) + assert file.is_file(), f"Not found: {file}" + basename, ext = file.stem, file.suffix + if ext == ".bin": + if "_" in basename: + i, ii = basename.split("_", 1) + i = int(i) + ii = int(ii) + else: + i = int(basename) + ii = 0 + with open(file, "rb") as f: + data = f.read() + if i not in temp: + temp[i] = {} + temp[i][ii] = data + elif ext in [".npy", ".npz"]: + raise NotImplementedError + else: + raise RuntimeError(f"Unsupported ext: {ext}") + assert len(temp) > 0 + for i in range(min(self.gen_data_number, len(temp))): + assert i in temp + data = {} + for ii, input_name in enumerate(input_names): + assert ii in temp[i] + assert input_name in input_types, f"Unknown dtype for input: {input_name}" + dtype = input_types[input_name] + quant = input_quant_details.get(input_name, None) + gen_dtype = dtype + if quant: + _, _, ty = quant + assert "float" in ty, "Input already quantized?" + if NEW: + gen_dtype = ty + arr = np.frombuffer(temp[i][ii], dtype=gen_dtype) + assert input_name in input_shapes, f"Unknown shape for input: {input_name}" + shape = input_shapes[input_name] + arr = np.reshape(arr, shape) + # Quantize if required + if gen_dtype != dtype: + assert "int" in dtype + assert quant + scale, shift, ty = quant + arr = (arr / scale) + shift + arr = np.around(arr) + arr = arr.astype(dtype) + # input("!=") + data[input_name] = arr + inputs_data.append(data) + else: + assert self.gen_data_file is not None, "Missing value for gen_data_file" + file = Path(self.gen_data_file) + assert file.is_file(), f"File not found: {file}" + # for i, input_name in enumerate(input_names): + + elif self.gen_data_fill_mode == "dataset": + raise NotImplementedError + else: + raise RuntimeError(f"unsupported fill_mode: {self.gen_data_fill_mode}") + return inputs_data + + def generate_output_ref_data( + self, inputs_data, model, out_paths, output_names, output_types, output_shapes, output_quant_details + ): + assert self.gen_ref_data + outputs_data = [] + if self.gen_ref_data_mode == "model": + assert len(inputs_data) > 0 + for i, input_data in enumerate(inputs_data): + # input("321?") + output_data = self.inference(model, input_data, quant=False, dequant=True) + outputs_data.append(output_data) + # input("321!") + + elif self.gen_ref_data_mode == "file": + if self.gen_ref_data_file == "auto": + len(out_paths) > 0 + if len(out_paths) == 1: + if out_paths[0].is_dir(): + files = list(out_paths[0].iterdir()) + else: + files = out_paths + temp = {} + for file in files: + if not isinstance(file, Path): + file = Path(file) + assert file.is_file() + basename, ext = file.stem, file.suffix + if ext == ".bin": + if "_" in basename: + i, ii = basename.split("_", 1) + i = int(i) + ii = int(ii) + else: + i = int(basename) + ii = 0 + with open(file, "rb") as f: + data = f.read() + if i not in temp: + temp[i] = {} + temp[i][ii] = data + elif ext in [".npy", ".npz"]: + raise NotImplementedError + else: + raise RuntimeError(f"Unsupported ext: {ext}") + # TODO: handle case where there are more output samples than input samples? + for i in range(len(temp)): + assert i in temp + data = {} + for ii, output_name in enumerate(output_names): + assert ii in temp[i] + assert output_name in output_types, f"Unknown dtype for output: {output_name}" + dtype = output_types[output_name] + dequant = output_quant_details.get(output_name, None) + if dequant: + _, _, ty = dequant + dtype = ty + arr = np.frombuffer(temp[i][ii], dtype=dtype) + assert output_name in output_shapes, f"Unknown shape for output: {output_name}" + shape = output_shapes[output_name] + arr = np.reshape(arr, shape) + data[output_name] = arr + outputs_data.append(data) + else: + assert self.gen_data_file is not None, "Missing value for gen_data_file" + file = Path(self.gen_data_file) + assert file.is_file(), f"File not found: {file}" + else: + raise RuntimeError(f"unsupported fill_mode: {self.gen_ref_data_mode}") + return outputs_data + + def generate_model_info( + self, + input_names, + output_names, + input_shapes, + output_shapes, + input_types, + output_types, + input_quant_details, + output_quant_details, + ): + model_info_dict = { + "input_names": input_names, + "output_names": output_names, + "input_shapes": list(input_shapes.values()), + "output_shapes": list(output_shapes.values()), + "input_types": list(input_types.values()), + "output_types": list(output_types.values()), + "input_quant_details": list(input_quant_details.values()), + "output_quant_details": list(output_quant_details.values()), + } + # nested version + # model_info_dict = { + # "inputs": [ + # { + # "name": "input_1", + # "shape": [1, 1014], + # "type": "int8", + # } + # ], + # "outputs": [ + # { + # "name": "output", + # "shape": [1, 10], + # "type": "int8", + # } + # ], + # } + return model_info_dict # TODO: turn into class + def process_metadata(self, model, cfg=None): model_dir = Path(model.paths[0]).parent.resolve() metadata = model.metadata @@ -220,7 +479,9 @@ def process_metadata(self, model, cfg=None): quant_dtype = quantize.get("dtype", None) quant_details = [quant_scale, quant_zero_shift, quant_dtype] input_quant_details[name] = quant_details - if self.use_inout_data or (self.gen_data and self.gen_data_fill_mode == "file" and self.gen_data_file == "auto"): + if self.use_inout_data or ( + self.gen_data and self.gen_data_fill_mode == "file" and self.gen_data_file == "auto" + ): if "example_input" in inp and "path" in inp["example_input"]: in_data_dir = Path(inp["example_input"]["path"]) # TODO: this will only work with relative paths to model dir! (Fallback to parent directories?) @@ -248,7 +509,9 @@ def process_metadata(self, model, cfg=None): quant_dtype = dequantize.get("dtype", None) quant_details = [quant_scale, quant_zero_shift, quant_dtype] output_quant_details[name] = quant_details - if self.use_inout_data or (self.gen_ref_data and self.gen_ref_data_mode == "file" and self.gen_ref_data_file == "auto"): + if self.use_inout_data or ( + self.gen_ref_data and self.gen_ref_data_mode == "file" and self.gen_ref_data_file == "auto" + ): if "test_output_path" in outp: out_data_dir = Path(outp["test_output_path"]) out_path = model_dir / out_data_dir @@ -319,182 +582,31 @@ def process_metadata(self, model, cfg=None): output_names = list(output_types.keys()) else: output_names = [] - model_info_dict = { - "input_names": input_names, - "output_names": output_names, - "input_shapes": list(input_shapes.values()), - "output_shapes": list(output_shapes.values()), - "input_types": list(input_types.values()), - "output_types": list(output_types.values()), - "input_quant_details": list(input_quant_details.values()), - "output_quant_details": list(output_quant_details.values()), - } - print("model_info_dict", model_info_dict) artifacts = [] + inputs_data = None + gen_model_info = True # TODO: move to self (configurable) + if gen_model_info: + model_info_dict = self.generate_model_info( + input_names, + output_names, + input_shapes, + output_shapes, + input_types, + output_types, + input_quant_details, + output_quant_details, + ) + import yaml + + content = yaml.dump(model_info_dict) + model_info_artifact = Artifact( + "model_info.yml", content=content, fmt=ArtifactFormat.TEXT, flags=("model_info",) + ) + artifacts.append(model_info_artifact) if self.gen_data: - inputs_data = [] - if self.gen_data_fill_mode in ["zeros", "ones", "random"]: - for i in range(self.gen_data_number): - print("i", i) - data = {} - NEW = True - for ii, input_name in enumerate(input_names): - print("ii", ii) - assert input_name in input_types, f"Unknown dtype for input: {input_name}" - dtype = input_types[input_name] - quant = input_quant_details.get(input_name, None) - gen_dtype = dtype - if quant: - _, _, ty = quant - assert "float" in ty, "Input already quantized?" - if NEW: - gen_dtype = ty - assert input_name in input_shapes, f"Unknown shape for input: {input_name}" - shape = input_shapes[input_name] - if self.gen_data_fill_mode == "zeros": - arr = np.zeros(shape, dtype=gen_dtype) - elif self.gen_data_fill_mode == "ones": - arr = np.ones(shape, dtype=gen_dtype) - elif self.gen_data_fill_mode == "random": - DIST = "uniform" - if DIST == "uniform": - UPPER = None # TODO: config - LOWER = None # TODO: config - if "float" in gen_dtype: - if UPPER is None: - # UPPER = 1.0 - UPPER = 0.5 - if LOWER is None: - # LOWER = -1.0 - LOWER = -0.5 - elif "int" in gen_dtype: - dtype_info = np.iinfo(gen_dtype), - if UPPER is None: - UPPER = dtype_info.max - else: - assert UPPER <= dtype_info.max, f"Out of dtype bound" - if LOWER is None: - LOWER = dtype_info.min - else: - assert LOWER >= dtype_info.min, f"Out of dtype bound" - else: - raise RuntimeError(f"Unsupported dtype: {gen_dtype}") - RANGE = UPPER - LOWER - print("dtype", dtype) - print("gen_dtype", gen_dtype) - print("UPPER,LOWER,RANGE", UPPER, LOWER, RANGE) - assert RANGE > 0 - arr = np.random.uniform(LOWER, UPPER, shape) - print("arr0", arr) - arr = arr.astype(gen_dtype) - print("arr1", arr) - # input("?=") - # if "float" in dtype: - # arr = np.random.rand(*shape).astype(dtype) - # elif "int" in dtype: - # arr = np.random.randint(np.iinfo(dtype).min, np.iinfo(dtype).max, size=shape, dtype=dtype) - # else: - # assert False - # Quantize if required - if gen_dtype != dtype: - assert "int" in dtype - assert quant - scale, shift, ty = quant - arr = (arr / scale) + shift - print("arr2", arr) - arr = np.around(arr) - print("arr3", arr) - arr = arr.astype(dtype) - print("arr4", arr) - # input("!=") - else: - raise RuntimeError(f"Unsupported distribution: {DIST}") - else: - assert False - data[input_name] = arr - assert len(data) > 0 - inputs_data.append(data) - elif self.gen_data_fill_mode == "file": - if self.gen_data_file == "auto": - len(in_paths) > 0 - print("in_paths", in_paths) - if len(in_paths) == 1: - if in_paths[0].is_dir(): - files = list(in_paths[0].iterdir()) - else: - files = in_paths - temp = {} - NEW = True - for file in files: - if not isinstance(file, Path): - file = Path(file) - assert file.is_file(), f"Not found: {file}" - print("file", file) - basename, ext = file.stem, file.suffix - if ext == ".bin": - if "_" in basename: - i, ii = basename.split("_", 1) - i = int(i) - ii = int(ii) - else: - i = int(basename) - ii = 0 - with open(file, "rb") as f: - data = f.read() - if i not in temp: - temp[i] = {} - temp[i][ii] = data - elif ext in [".npy", ".npz"]: - raise NotImplementedError - else: - raise RuntimeError(f"Unsupported ext: {ext}") - # print("temp", temp) - assert len(temp) > 0 - for i in range(min(self.gen_data_number, len(temp))): - print("i", i) - assert i in temp - data = {} - for ii, input_name in enumerate(input_names): - print("ii", ii) - assert ii in temp[i] - assert input_name in input_types, f"Unknown dtype for input: {input_name}" - dtype = input_types[input_name] - quant = input_quant_details.get(input_name, None) - gen_dtype = dtype - if quant: - _, _, ty = quant - assert "float" in ty, "Input already quantized?" - if NEW: - gen_dtype = ty - arr = np.frombuffer(temp[i][ii], dtype=gen_dtype) - assert input_name in input_shapes, f"Unknown shape for input: {input_name}" - shape = input_shapes[input_name] - arr = np.reshape(arr, shape) - # Quantize if required - if gen_dtype != dtype: - assert "int" in dtype - assert quant - scale, shift, ty = quant - arr = (arr / scale) + shift - print("arr2", arr) - arr = np.around(arr) - print("arr3", arr) - arr = arr.astype(dtype) - print("arr4", arr) - # input("!=") - data[input_name] = arr - inputs_data.append(data) - else: - assert self.gen_data_file is not None, "Missing value for gen_data_file" - file = Path(self.gen_data_file) - assert file.is_file(), f"File not found: {file}" - # for i, input_name in enumerate(input_names): - - elif self.gen_data_fill_mode == "dataset": - raise NotImplementedError - else: - raise RuntimeError(f"unsupported fill_mode: {self.gen_data_fill_mode}") - print("inputs_data", inputs_data) + inputs_data = self.generate_input_data( + input_names, input_types, input_shapes, input_quant_details, in_paths + ) fmt = self.gen_data_fmt if fmt == "npy": with tempfile.TemporaryDirectory() as tmpdirname: @@ -510,78 +622,9 @@ def process_metadata(self, model, cfg=None): inputs_data_artifact = Artifact(f"inputs.{fmt}", raw=raw, fmt=ArtifactFormat.BIN, flags=("inputs", fmt)) artifacts.append(inputs_data_artifact) if self.gen_ref_data: - outputs_data = [] - if self.gen_ref_data_mode == "model": - assert len(inputs_data) > 0 - for i, input_data in enumerate(inputs_data): - print("i", i) - print("model", model, type(model)) - # input("321?") - output_data = self.inference(model, input_data, quant=False, dequant=True) - print("output_data", output_data) - outputs_data.append(output_data) - # input("321!") - - elif self.gen_ref_data_mode == "file": - if self.gen_ref_data_file == "auto": - len(out_paths) > 0 - if len(out_paths) == 1: - if out_paths[0].is_dir(): - files = list(out_paths[0].iterdir()) - else: - files = out_paths - temp = {} - for file in files: - if not isinstance(file, Path): - file = Path(file) - assert file.is_file() - print("file", file) - basename, ext = file.stem, file.suffix - if ext == ".bin": - if "_" in basename: - i, ii = basename.split("_", 1) - i = int(i) - ii = int(ii) - else: - i = int(basename) - ii = 0 - with open(file, "rb") as f: - data = f.read() - if i not in temp: - temp[i] = {} - temp[i][ii] = data - elif ext in [".npy", ".npz"]: - raise NotImplementedError - else: - raise RuntimeError(f"Unsupported ext: {ext}") - # print("temp", temp) - # TODO: handle case where there are more output samples than input samples? - for i in range(len(temp)): - print("i", i) - assert i in temp - data = {} - for ii, output_name in enumerate(output_names): - print("ii", ii) - assert ii in temp[i] - assert output_name in output_types, f"Unknown dtype for output: {output_name}" - dtype = output_types[output_name] - dequant = output_quant_details.get(output_name, None) - if dequant: - _, _, ty = dequant - dtype = ty - arr = np.frombuffer(temp[i][ii], dtype=dtype) - assert ii < len(output_shapes) - shape = output_shapes[output_name] - arr = np.reshape(arr, shape) - data[output_name] = arr - outputs_data.append(data) - else: - assert self.gen_data_file is not None, "Missing value for gen_data_file" - file = Path(self.gen_data_file) - assert file.is_file(), f"File not found: {file}" - else: - raise RuntimeError(f"unsupported fill_mode: {self.gen_ref_data_mode}") - print("outputs_data", outputs_data) + outputs_data = self.generate_output_ref_data( + inputs_data, model, out_paths, output_names, output_types, output_shapes, output_quant_details + ) fmt = self.gen_data_fmt if fmt == "npy": with tempfile.TemporaryDirectory() as tmpdirname: @@ -594,30 +637,10 @@ def process_metadata(self, model, cfg=None): else: raise RuntimeError(f"Unsupported fmt: {fmt}") assert raw - outputs_data_artifact = Artifact(f"outputs_ref.{fmt}", raw=raw, fmt=ArtifactFormat.BIN, flags=("outputs_ref", fmt)) + outputs_data_artifact = Artifact( + f"outputs_ref.{fmt}", raw=raw, fmt=ArtifactFormat.BIN, flags=("outputs_ref", fmt) + ) artifacts.append(outputs_data_artifact) - # nested version - # model_info_dict = { - # "inputs": [ - # { - # "name": "input_1", - # "shape": [1, 1014], - # "type": "int8", - # } - # ], - # "outputs": [ - # { - # "name": "output", - # "shape": [1, 10], - # "type": "int8", - # } - # ], - # } - import yaml - content = yaml.dump(model_info_dict) - model_info_artifact = Artifact("model_info.yml", content=content, fmt=ArtifactFormat.TEXT, flags=("model_info",)) - artifacts.append(model_info_artifact) - return artifacts def generate(self, model) -> Tuple[dict, dict]: artifacts = [] @@ -801,22 +824,33 @@ def extract_model_info(self, model: Model): scale, zero_point = outp["quantization"] quant = [scale, zero_point, "float32"] output_quant_details[name] = quant - return input_names, input_shapes, input_types, input_quant_details, output_names, output_shapes, output_types, output_quant_details + return ( + input_names, + input_shapes, + input_types, + input_quant_details, + output_names, + output_shapes, + output_types, + output_quant_details, + ) - def inference(self, model: Model, input_data: Dict[str, np.array], quant=False, dequant=False): + def inference(self, model: Model, input_data: Dict[str, np.array], quant=False, dequant=False, verbose=False): import tensorflow as tf + model_path = str(model.paths[0]) interpreter = tf.lite.Interpreter(model_path=model_path) input_details = interpreter.get_input_details() output_details = interpreter.get_output_details() interpreter.allocate_tensors() - print() - print("Input details:") - print(input_details) - print() - print("Output details:") - print(output_details) - print() + if verbose: + print() + print("Input details:") + print(input_details) + print() + print("Output details:") + print(output_details) + print() assert len(input_details) == 1, "Multi-inputs not yet supported" input_type = input_details[0]["dtype"] input_name = input_details[0]["name"] @@ -824,18 +858,18 @@ def inference(self, model: Model, input_data: Dict[str, np.array], quant=False, assert input_name in input_data, f"Input {input_name} fot found in data" np_features = input_data[input_name] if quant and input_type == np.int8: - input_scale, input_zero_point = input_details[0]['quantization'] - print("Input scale:", input_scale) - print("Input zero point:", input_zero_point) - print() + input_scale, input_zero_point = input_details[0]["quantization"] + if verbose: + print("Input scale:", input_scale) + print("Input zero point:", input_zero_point) + print() np_features = (np_features / input_scale) + input_zero_point np_features = np.around(np_features) np_features = np_features.astype(input_type) np_features = np_features.reshape(input_shape) - print("np_features", np_features) - interpreter.set_tensor(input_details[0]['index'], np_features) + interpreter.set_tensor(input_details[0]["index"], np_features) interpreter.invoke() - output = interpreter.get_tensor(output_details[0]['index']) + output = interpreter.get_tensor(output_details[0]["index"]) # If the output type is int8 (quantized model), rescale data assert len(output_details) == 1, "Multi-outputs not yet supported" @@ -843,14 +877,16 @@ def inference(self, model: Model, input_data: Dict[str, np.array], quant=False, output_name = output_details[0]["name"] if dequant and output_type == np.int8: output_scale, output_zero_point = output_details[0]["quantization"] - print("Raw output scores:", output) - print("Output scale:", output_scale) - print("Output zero point:", output_zero_point) - print() + if verbose: + print("Raw output scores:", output) + print("Output scale:", output_scale) + print("Output zero point:", output_zero_point) + print() output = output_scale * (output.astype(np.float32) - output_zero_point) - # Print the results of inference - print("Inference output:", output, type(output)) + if verbose: + # Print the results of inference + print("Inference output:", output, type(output)) return {output_name: output} def produce_artifacts(self, model): From f1c75e420a9397e1fab929eca376b16211ba5051 Mon Sep 17 00:00:00 2001 From: Philipp van Kempen Date: Sun, 31 Mar 2024 13:32:44 +0200 Subject: [PATCH 052/105] frontends: allow overriding model-specific paths --- mlonmcu/models/frontend.py | 11 ++++++++++- mlonmcu/models/model.py | 6 +++--- 2 files changed, 13 insertions(+), 4 deletions(-) diff --git a/mlonmcu/models/frontend.py b/mlonmcu/models/frontend.py index 5a1ec244f..1791cb33b 100644 --- a/mlonmcu/models/frontend.py +++ b/mlonmcu/models/frontend.py @@ -526,6 +526,12 @@ def process_metadata(self, model, cfg=None): fallback_out_path = model_dir / "output" if fallback_out_path.is_dir(): out_paths.append(fallback_out_path) + if model.inputs_path: + logger.info("Overriding default model input data with user path") + in_paths = [model.inputs_path] + if model.outputs_path: + logger.info("Overriding default model output data with user path") + out_paths = [model.outputs_path] if metadata is not None and "backends" in metadata: assert cfg is not None @@ -550,7 +556,10 @@ def process_metadata(self, model, cfg=None): logger.warning("Model info could not be extracted.") # Detect model support code (Allow overwrite in metadata YAML) - support_path = model_dir / "support" + if model.support_path: + support_path = model.support_path + else: + support_path = model_dir / "support" if support_path.is_dir(): assert cfg is not None # TODO: onlu overwrite if unset? diff --git a/mlonmcu/models/model.py b/mlonmcu/models/model.py index aa340ffb3..3dd249849 100644 --- a/mlonmcu/models/model.py +++ b/mlonmcu/models/model.py @@ -160,9 +160,9 @@ class Model(Workload): "output_shapes": None, "input_types": None, "output_types": None, - "support_path": "support", - "inputs_path": "input", - "outputs_path": "output", + "support_path": None, + "inputs_path": None, + "outputs_path": None, } def __init__(self, name, paths, config=None, alt=None, formats=ModelFormats.TFLITE): From 4345db5ac84769b04416020dd5214ce641cfdeb4 Mon Sep 17 00:00:00 2001 From: Philipp van Kempen Date: Sun, 31 Mar 2024 13:34:43 +0200 Subject: [PATCH 053/105] lint code --- mlonmcu/models/frontend.py | 12 +++++++++++- mlonmcu/platform/mlif/mlif.py | 5 ++++- mlonmcu/platform/mlif/mlif_target.py | 19 +++++++++++++------ mlonmcu/platform/tvm/tvm_target.py | 11 ++++++++--- mlonmcu/platform/tvm/tvm_target_platform.py | 4 +--- mlonmcu/session/postprocess/postprocesses.py | 17 +++++++++-------- 6 files changed, 46 insertions(+), 22 deletions(-) diff --git a/mlonmcu/models/frontend.py b/mlonmcu/models/frontend.py index 1791cb33b..d013fb5d5 100644 --- a/mlonmcu/models/frontend.py +++ b/mlonmcu/models/frontend.py @@ -551,7 +551,16 @@ def process_metadata(self, model, cfg=None): if metadata is None: try: - input_names, input_shapes, input_types, input_quant_details, output_names, output_shapes, output_types, output_quant_details = self.extract_model_info(model) + ( + input_names, + input_shapes, + input_types, + input_quant_details, + output_names, + output_shapes, + output_types, + output_quant_details, + ) = self.extract_model_info(model) except NotImplementedError: logger.warning("Model info could not be extracted.") @@ -803,6 +812,7 @@ def analyze_script(self): def extract_model_info(self, model: Model): import tensorflow as tf + model_path = str(model.paths[0]) interpreter = tf.lite.Interpreter(model_path=model_path) input_details = interpreter.get_input_details() diff --git a/mlonmcu/platform/mlif/mlif.py b/mlonmcu/platform/mlif/mlif.py index 9aa40c315..1c26848de 100644 --- a/mlonmcu/platform/mlif/mlif.py +++ b/mlonmcu/platform/mlif/mlif.py @@ -443,7 +443,10 @@ def generate_model_support(self, target): ) code = model_support.generate() code_artifact = Artifact( - "model_support.cpp", content=code, fmt=ArtifactFormat.TEXT, flags=("model_support"), + "model_support.cpp", + content=code, + fmt=ArtifactFormat.TEXT, + flags=("model_support"), ) self.definitions["BATCH_SIZE"] = model_support.batch_size artifacts.append(code_artifact) diff --git a/mlonmcu/platform/mlif/mlif_target.py b/mlonmcu/platform/mlif/mlif_target.py index 863feb955..7ee546a09 100644 --- a/mlonmcu/platform/mlif/mlif_target.py +++ b/mlonmcu/platform/mlif/mlif_target.py @@ -67,12 +67,14 @@ def exec(self, program, *args, cwd=os.getcwd(), **kwargs): # first figure out how many inputs are provided if model_info_file is not None: import yaml + with open(model_info_file, "r") as f: model_info_data = yaml.safe_load(f) else: model_info_data = None if self.platform.inputs_artifact is not None: import numpy as np + data = np.load(self.platform.inputs_artifact, allow_pickle=True) num_inputs = len(data) else: @@ -103,14 +105,14 @@ def exec(self, program, *args, cwd=os.getcwd(), **kwargs): # current_batch_size = max(min(batch_size, remaining_inputs), 1) if processed_inputs < num_inputs: if in_interface == "filesystem": - batch_data = data[idx * batch_size:((idx + 1) * batch_size)] + batch_data = data[idx * batch_size : ((idx + 1) * batch_size)] # print("batch_data", batch_data, type(batch_data)) ins_file = Path(cwd) / "ins.npy" np.save(ins_file, batch_data) elif in_interface == "stdin": raise NotImplementedError elif in_interface == "stdin_raw": - batch_data = data[idx * batch_size:((idx + 1) * batch_size)] + batch_data = data[idx * batch_size : ((idx + 1) * batch_size)] # print("batch_data", batch_data, type(batch_data)) stdin_data = b"" for cur_data in batch_data: @@ -127,10 +129,13 @@ def exec(self, program, *args, cwd=os.getcwd(), **kwargs): # raise NotImplementedError # TODO: generate input stream here! - ret_, artifacts_ = super().exec(program, *args, cwd=cwd, **kwargs, stdin_data=stdin_data, encoding=encoding) + ret_, artifacts_ = super().exec( + program, *args, cwd=cwd, **kwargs, stdin_data=stdin_data, encoding=encoding + ) if self.platform.get_outputs: if out_interface == "filesystem": import numpy as np + outs_file = Path(cwd) / "outs.npy" with np.load(outs_file) as out_data: outs_data.extend(dict(out_data)) @@ -155,13 +160,13 @@ def exec(self, program, *args, cwd=os.getcwd(), **kwargs): # print("found_start", found_start) if found_start < 0: break - x = x[found_start+3:] + x = x[found_start + 3 :] # print("x[:20]", x[:20]) found_end = x.find("-!-".encode()) # print("found_end", found_end) assert found_end >= 0 x_ = x[:found_end] - x = x[found_end+3:] + x = x[found_end + 3 :] # print("x[:20]", x[:20]) # print("x_", x_) # out_idx += 1 @@ -189,7 +194,9 @@ def exec(self, program, *args, cwd=os.getcwd(), **kwargs): np.save(outs_path, outs_data) with open(outs_path, "rb") as f: outs_raw = f.read() - outputs_artifact = Artifact("outputs.npy", raw=outs_raw, fmt=ArtifactFormat.BIN, flags=("outputs", "npy")) + outputs_artifact = Artifact( + "outputs.npy", raw=outs_raw, fmt=ArtifactFormat.BIN, flags=("outputs", "npy") + ) artifacts.append(outputs_artifact) return ret, artifacts diff --git a/mlonmcu/platform/tvm/tvm_target.py b/mlonmcu/platform/tvm/tvm_target.py index bdc37977e..115be35b8 100644 --- a/mlonmcu/platform/tvm/tvm_target.py +++ b/mlonmcu/platform/tvm/tvm_target.py @@ -78,6 +78,7 @@ def exec(self, program, *args, cwd=os.getcwd(), **kwargs): if ins_file is None: assert self.platform.inputs_artifact is not None import numpy as np + data = np.load(self.platform.inputs_artifact, allow_pickle=True) print("data", data, type(data)) num_inputs = len(data) @@ -102,7 +103,6 @@ def exec(self, program, *args, cwd=os.getcwd(), **kwargs): elif interface == "stdout": print_top = 1e6 - ret = "" artifacts = [] num_batches = max(round(num_inputs / batch_size), 1) @@ -123,7 +123,9 @@ def exec(self, program, *args, cwd=os.getcwd(), **kwargs): remaining_inputs -= 1 else: ins_file = None - ret_, artifacts_ = self.platform.run(program, self, cwd=cwd, ins_file=ins_file, outs_file=outs_file, print_top=print_top) + ret_, artifacts_ = self.platform.run( + program, self, cwd=cwd, ins_file=ins_file, outs_file=outs_file, print_top=print_top + ) ret += ret_ print("self.platform.get_outputs", self.platform.get_outputs) if self.platform.get_outputs: @@ -138,6 +140,7 @@ def exec(self, program, *args, cwd=os.getcwd(), **kwargs): assert interface in ["filesystem", "stdout"] if interface == "filesystem": import numpy as np + with np.load(outs_file) as out_data: outs_data.append(dict(out_data)) elif interface == "stdout": @@ -151,7 +154,9 @@ def exec(self, program, *args, cwd=os.getcwd(), **kwargs): np.save(outs_path, outs_data) with open(outs_path, "rb") as f: outs_raw = f.read() - outputs_artifact = Artifact("outputs.npy", raw=outs_raw, fmt=ArtifactFormat.BIN, flags=("outputs", "npy")) + outputs_artifact = Artifact( + "outputs.npy", raw=outs_raw, fmt=ArtifactFormat.BIN, flags=("outputs", "npy") + ) artifacts.append(outputs_artifact) return ret, artifacts diff --git a/mlonmcu/platform/tvm/tvm_target_platform.py b/mlonmcu/platform/tvm/tvm_target_platform.py index 2defae063..98cea2eba 100644 --- a/mlonmcu/platform/tvm/tvm_target_platform.py +++ b/mlonmcu/platform/tvm/tvm_target_platform.py @@ -172,9 +172,7 @@ def create_target(self, name): def get_tvmc_run_args(self, ins_file=None, outs_file=None, print_top=None): return [ - *get_data_tvmc_args( - mode=self.fill_mode, ins_file=ins_file, outs_file=outs_file, print_top=print_top - ), + *get_data_tvmc_args(mode=self.fill_mode, ins_file=ins_file, outs_file=outs_file, print_top=print_top), *get_bench_tvmc_args( print_time=True, profile=self.profile, end_to_end=False, repeat=self.repeat, number=self.number ), diff --git a/mlonmcu/session/postprocess/postprocesses.py b/mlonmcu/session/postprocess/postprocesses.py index f3962404b..3783c08a5 100644 --- a/mlonmcu/session/postprocess/postprocesses.py +++ b/mlonmcu/session/postprocess/postprocesses.py @@ -1460,12 +1460,7 @@ def post_run(self, report, artifacts): class ValidateOutputsPostprocess(RunPostprocess): """Postprocess for comparing model outputs with golden reference.""" - DEFAULTS = { - **RunPostprocess.DEFAULTS, - "atol": 0.0, - "rtol": 0.0, - "report": False - } + DEFAULTS = {**RunPostprocess.DEFAULTS, "atol": 0.0, "rtol": 0.0, "report": False} def __init__(self, features=None, config=None): super().__init__("validate_outputs", features=features, config=config) @@ -1494,6 +1489,7 @@ def post_run(self, report, artifacts): assert len(model_info_artifact) == 1, "Could not find artifact: model_info.yml" model_info_artifact = model_info_artifact[0] import yaml + model_info_data = yaml.safe_load(model_info_artifact.content) print("model_info_data", model_info_data) if len(model_info_data["output_names"]) > 1: @@ -1502,6 +1498,7 @@ def post_run(self, report, artifacts): assert len(outputs_ref_artifact) == 1, "Could not find artifact: outputs_ref.npy" outputs_ref_artifact = outputs_ref_artifact[0] import numpy as np + outputs_ref = np.load(outputs_ref_artifact.path, allow_pickle=True) # import copy # outputs = copy.deepcopy(outputs_ref) @@ -1561,6 +1558,7 @@ def pm1_helper(data, ref_data): print("r FALSE") return False return True + quant = model_info_data.get("output_quant_details", None) if quant: assert ii < len(quant) @@ -1576,7 +1574,9 @@ def pm1_helper(data, ref_data): if metric_name == "+-1": if metrics[metric_name] is None: metrics[metric_name] = 0 - out_ref_data_quant = np.around((out_ref_data / quant_scale) + quant_zero_point).astype("int8") + out_ref_data_quant = np.around( + (out_ref_data / quant_scale) + quant_zero_point + ).astype("int8") if pm1_helper(out_data, out_ref_data_quant): metrics[metric_name] += 1 out_data = (out_data.astype("float32") - quant_zero_point) * quant_scale @@ -1593,7 +1593,7 @@ def pm1_helper(data, ref_data): # if np.allclose(out_data, out_ref_data, rtol=0.01, atol=0.01): def mse_helper(data, ref_data, thr): - mse = ((data - ref_data)**2).mean() + mse = ((data - ref_data) ** 2).mean() print("mse", mse) return mse < thr @@ -1648,6 +1648,7 @@ def topk_helper(data, ref_data, n): return True else: assert False + for metric_name in metrics: if "allclose" in metric_name: if metrics[metric_name] is None: From 1a8d9eb0fa7aa8b2828779a5b598d06ab0916b4a Mon Sep 17 00:00:00 2001 From: Philipp van Kempen Date: Tue, 9 Apr 2024 08:22:20 +0200 Subject: [PATCH 054/105] move validate metrics to seperate class & file --- mlonmcu/models/frontend.py | 1 + mlonmcu/session/postprocess/postprocesses.py | 214 ++++--------------- 2 files changed, 38 insertions(+), 177 deletions(-) diff --git a/mlonmcu/models/frontend.py b/mlonmcu/models/frontend.py index d013fb5d5..804917e70 100644 --- a/mlonmcu/models/frontend.py +++ b/mlonmcu/models/frontend.py @@ -659,6 +659,7 @@ def process_metadata(self, model, cfg=None): f"outputs_ref.{fmt}", raw=raw, fmt=ArtifactFormat.BIN, flags=("outputs_ref", fmt) ) artifacts.append(outputs_data_artifact) + return artifacts def generate(self, model) -> Tuple[dict, dict]: artifacts = [] diff --git a/mlonmcu/session/postprocess/postprocesses.py b/mlonmcu/session/postprocess/postprocesses.py index 3783c08a5..c7611dac6 100644 --- a/mlonmcu/session/postprocess/postprocesses.py +++ b/mlonmcu/session/postprocess/postprocesses.py @@ -31,6 +31,7 @@ from mlonmcu.logging import get_logger from .postprocess import SessionPostprocess, RunPostprocess +from .validate_metrics import parse_validate_metrics logger = get_logger() @@ -1460,22 +1461,20 @@ def post_run(self, report, artifacts): class ValidateOutputsPostprocess(RunPostprocess): """Postprocess for comparing model outputs with golden reference.""" - DEFAULTS = {**RunPostprocess.DEFAULTS, "atol": 0.0, "rtol": 0.0, "report": False} + DEFAULTS = { + **RunPostprocess.DEFAULTS, + "report": False, + "validate_metrics": "topk(n=1);topk(n=2)", + } def __init__(self, features=None, config=None): super().__init__("validate_outputs", features=features, config=config) @property - def atol(self): - """Get atol property.""" - value = self.config["atol"] - return float(value) - - @property - def rtol(self): - """Get rtol property.""" - value = self.config["rtol"] - return float(value) + def validate_metrics(self): + """Get validate_metrics property.""" + value = self.config["validate_metrics"] + return value @property def report(self): @@ -1507,22 +1506,24 @@ def post_run(self, report, artifacts): assert len(outputs_artifact) == 1, "Could not find artifact: outputs.npy" outputs_artifact = outputs_artifact[0] outputs = np.load(outputs_artifact.path, allow_pickle=True) - compared = 0 + # compared = 0 # matching = 0 missing = 0 - metrics = { - "allclose(atol=0.0,rtol=0.0)": None, - "allclose(atol=0.05,rtol=0.05)": None, - "allclose(atol=0.1,rtol=0.1)": None, - "topk(n=1)": None, - "topk(n=2)": None, - "topk(n=inf)": None, - "toy": None, - "mse(thr=0.1)": None, - "mse(thr=0.05)": None, - "mse(thr=0.01)": None, - "+-1": None, - } + # metrics = { + # "allclose(atol=0.0,rtol=0.0)": None, + # "allclose(atol=0.05,rtol=0.05)": None, + # "allclose(atol=0.1,rtol=0.1)": None, + # "topk(n=1)": None, + # "topk(n=2)": None, + # "topk(n=inf)": None, + # "toy": None, + # "mse(thr=0.1)": None, + # "mse(thr=0.05)": None, + # "mse(thr=0.01)": None, + # "+-1": None, + # } + validate_metrics_str = self.validate_metrics + validate_metrics = parse_validate_metrics(validate_metrics_str) for i, output_ref in enumerate(outputs_ref): if i >= len(outputs): logger.warning("Missing output sample") @@ -1546,19 +1547,6 @@ def post_run(self, report, artifacts): # print("out_data_before_quant", out_data) # print("sum(out_data_before_quant", np.sum(out_data)) - def pm1_helper(data, ref_data): - data_ = data.flatten().tolist() - ref_data_ = ref_data.flatten().tolist() - - length = len(data_) - for jjj in range(length): - diff = abs(data_[jjj] - ref_data_[jjj]) - print("diff", diff) - if diff > 1: - print("r FALSE") - return False - return True - quant = model_info_data.get("output_quant_details", None) if quant: assert ii < len(quant) @@ -1570,15 +1558,11 @@ def pm1_helper(data, ref_data): # need to dequantize here assert out_data.dtype.name in ["int8"], "Dequantization only supported for int8 input" assert quant_dtype in ["float32"], "Dequantization only supported for float32 output" - for metric_name in metrics: - if metric_name == "+-1": - if metrics[metric_name] is None: - metrics[metric_name] = 0 - out_ref_data_quant = np.around( - (out_ref_data / quant_scale) + quant_zero_point - ).astype("int8") - if pm1_helper(out_data, out_ref_data_quant): - metrics[metric_name] += 1 + out_ref_data_quant = np.around( + (out_ref_data / quant_scale) + quant_zero_point + ).astype("int8") + for vm in validate_metrics: + vm.process(out_data, out_ref_data_quant, quant=True) out_data = (out_data.astype("float32") - quant_zero_point) * quant_scale # print("out_data", out_data) # print("sum(out_data)", np.sum(out_data)) @@ -1587,137 +1571,13 @@ def pm1_helper(data, ref_data): # input("TIAW") assert out_data.dtype == out_ref_data.dtype, "dtype missmatch" assert out_data.shape == out_ref_data.shape, "shape missmatch" - # if np.allclose(out_data, out_ref_data, rtol=0, atol=0): - # if np.allclose(out_data, out_ref_data, rtol=0.1, atol=0.1): - # if np.allclose(out_data, out_ref_data, rtol=0.0, atol=0.0): - # if np.allclose(out_data, out_ref_data, rtol=0.01, atol=0.01): - - def mse_helper(data, ref_data, thr): - mse = ((data - ref_data) ** 2).mean() - print("mse", mse) - return mse < thr - - def toy_helper(data, ref_data, atol, rtol): - data_flat = data.flatten().tolist() - ref_data_flat = ref_data.flatten().tolist() - res = 0 - ref_res = 0 - length = len(data_flat) - for jjj in range(length): - res += data_flat[jjj] ** 2 - ref_res += ref_data_flat[jjj] ** 2 - res /= length - ref_res /= length - print("res", res) - print("ref_res", ref_res) - return np.allclose([res], [ref_res], atol=atol, rtol=rtol) - - def topk_helper(data, ref_data, n): - # TODO: only for classification models! - # TODO: support multi_outputs? - data_sorted_idx = list(reversed(np.argsort(data).tolist()[0])) - ref_data_sorted_idx = list(reversed(np.argsort(ref_data).tolist()[0])) - k = 0 - num_checks = min(n, len(data_sorted_idx)) - assert len(data_sorted_idx) == len(ref_data_sorted_idx) - # print("data_sorted_idx", data_sorted_idx, type(data_sorted_idx)) - # print("ref_data_sorted_idx", ref_data_sorted_idx, type(ref_data_sorted_idx)) - # print("num_checks", num_checks) - for j in range(num_checks): - # print("j", j) - # print(f"data_sorted_idx[{j}]", data_sorted_idx[j], type(data_sorted_idx[j])) - idx = data_sorted_idx[j] - # print("idx", idx) - ref_idx = ref_data_sorted_idx[j] - # print("ref_idx", ref_idx) - if idx == ref_idx: - # print("IF") - k += 1 - else: - # print("ELSE") - if data.tolist()[0][idx] == ref_data.tolist()[0][ref_idx]: - # print("SAME") - k += 1 - else: - # print("BREAK") - break - # print("k", k) - if k < num_checks: - return False - elif k == num_checks: - return True - else: - assert False - - for metric_name in metrics: - if "allclose" in metric_name: - if metrics[metric_name] is None: - metrics[metric_name] = 0 - if metric_name == "allclose(atol=0.0,rtol=0.0)": - atol = 0.0 - rtol = 0.0 - elif metric_name == "allclose(atol=0.05,rtol=0.05)": - atol = 0.05 - rtol = 0.05 - elif metric_name == "allclose(atol=0.1,rtol=0.1)": - atol = 0.1 - rtol = 0.1 - else: - raise NotImplementedError - if np.allclose(out_data, out_ref_data, rtol=rtol, atol=atol): - metrics[metric_name] += 1 - elif "topk" in metric_name: - data_len = len(out_data.flatten().tolist()) - if data_len > 25: # Probably no classification - continue - if metrics[metric_name] is None: - metrics[metric_name] = 0 - if metric_name == "topk(n=1)": - n = 1 - elif metric_name == "topk(n=2)": - n = 2 - elif metric_name == "topk(n=inf)": - n = 1000000 - else: - raise NotImplementedError - if topk_helper(out_data, out_ref_data, n): - metrics[metric_name] += 1 - elif metric_name == "toy": - data_len = len(out_data.flatten().tolist()) - if data_len != 640: - continue - if metrics[metric_name] is None: - metrics[metric_name] = 0 - if toy_helper(out_data, out_ref_data, 0.01, 0.01): - metrics[metric_name] += 1 - elif metric_name == "mse": - if metrics[metric_name] is None: - metrics[metric_name] = 0 - if metric_name == "mse(thr=0.1)": - thr = 0.1 - elif metric_name == "mse(thr=0.05)": - thr = 0.05 - elif metric_name == "mse(thr=0.01)": - thr = 0.01 - else: - raise NotImplementedError - if mse_helper(out_data, out_ref_data, thr): - metrics[metric_name] += 1 - elif metric_name == "+-1": - continue - compared += 1 + + for vm in validate_metrics: + vm.process(out_data, out_ref_data_quant, quant=False) ii += 1 if self.report: raise NotImplementedError - if self.atol: - raise NotImplementedError - if self.rtol: - raise NotImplementedError - for metric_name, metric_data in metrics.items(): - if metric_data is None: - res = "N/A" - else: - matching = metric_data - res = f"{matching}/{compared} ({int(matching/compared*100)}%)" - report.post_df[f"{metric_name}"] = res + for vm in validate_metrics: + res = vm.get_summary() + report.post_df[f"{vm.name}"] = res return [] From bc24886673d6d238b789888dc297a7bb9e84bdbb Mon Sep 17 00:00:00 2001 From: Philipp van Kempen Date: Thu, 11 Apr 2024 11:56:18 +0200 Subject: [PATCH 055/105] platforms: add mlif.needs_model_support --- mlonmcu/platform/mlif/mlif.py | 21 +++++++++++++-------- 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/mlonmcu/platform/mlif/mlif.py b/mlonmcu/platform/mlif/mlif.py index 1c26848de..602357c20 100644 --- a/mlonmcu/platform/mlif/mlif.py +++ b/mlonmcu/platform/mlif/mlif.py @@ -167,6 +167,10 @@ def model_info_file(self): logger.warning("Artifact 'model_info.yml' not found!") return None + @property + def needs_model_support(self): + return self.set_inputs or self.get_outputs + def gen_data_artifact(self): in_paths = self.input_data_path if not isinstance(in_paths, list): @@ -453,14 +457,15 @@ def generate_model_support(self, target): return artifacts def configure(self, target, src, _model): - artifacts = self.generate_model_support(target) - if len(artifacts) > 0: - assert len(artifacts) == 1 - model_support_artifact = artifacts[0] - model_support_file = self.build_dir / model_support_artifact.name - model_support_artifact.export(model_support_file) - self.definitions["MODEL_SUPPORT_FILE"] = model_support_file - del target + if self.needs_model_support: + artifacts = self.generate_model_support(target) + if len(artifacts) > 0: + assert len(artifacts) == 1 + model_support_artifact = artifacts[0] + model_support_file = self.build_dir / model_support_artifact.name + model_support_artifact.export(model_support_file) + self.definitions["MODEL_SUPPORT_FILE"] = model_support_file + del target if not isinstance(src, Path): src = Path(src) cmakeArgs = self.get_cmake_args() From 65233fd42a7a6ade458816ad26a45b52c517da1d Mon Sep 17 00:00:00 2001 From: Philipp van Kempen Date: Thu, 11 Apr 2024 11:56:35 +0200 Subject: [PATCH 056/105] lint --- mlonmcu/session/postprocess/postprocesses.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/mlonmcu/session/postprocess/postprocesses.py b/mlonmcu/session/postprocess/postprocesses.py index c7611dac6..98e105aee 100644 --- a/mlonmcu/session/postprocess/postprocesses.py +++ b/mlonmcu/session/postprocess/postprocesses.py @@ -1558,9 +1558,9 @@ def post_run(self, report, artifacts): # need to dequantize here assert out_data.dtype.name in ["int8"], "Dequantization only supported for int8 input" assert quant_dtype in ["float32"], "Dequantization only supported for float32 output" - out_ref_data_quant = np.around( - (out_ref_data / quant_scale) + quant_zero_point - ).astype("int8") + out_ref_data_quant = np.around((out_ref_data / quant_scale) + quant_zero_point).astype( + "int8" + ) for vm in validate_metrics: vm.process(out_data, out_ref_data_quant, quant=True) out_data = (out_data.astype("float32") - quant_zero_point) * quant_scale From 607c5c6c2c001d78a9b61b235000811d64731646 Mon Sep 17 00:00:00 2001 From: Philipp van Kempen Date: Thu, 11 Apr 2024 14:52:43 +0200 Subject: [PATCH 057/105] fix merge artifact --- mlonmcu/feature/features.py | 1 - 1 file changed, 1 deletion(-) diff --git a/mlonmcu/feature/features.py b/mlonmcu/feature/features.py index 7c2151e66..69cc935b4 100644 --- a/mlonmcu/feature/features.py +++ b/mlonmcu/feature/features.py @@ -2389,4 +2389,3 @@ def get_postprocesses(self): # validate_outputs_postprocess = ValidateOutputsPostprocess(features=[], config=config) # return [validate_outputs_postprocess] return ["validate_outputs"] ->>>>>>> a3ccb1a1... WIP: add new features and postprocesses for validate_new From 5f198eb2da99b6108f256b7e17822e7cc75d25a9 Mon Sep 17 00:00:00 2001 From: Philipp van Kempen Date: Thu, 11 Apr 2024 14:54:08 +0200 Subject: [PATCH 058/105] lint --- mlonmcu/models/frontend.py | 7 ++++--- mlonmcu/platform/tvm/tvm_target_platform.py | 2 -- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/mlonmcu/models/frontend.py b/mlonmcu/models/frontend.py index 804917e70..abab0142c 100644 --- a/mlonmcu/models/frontend.py +++ b/mlonmcu/models/frontend.py @@ -229,11 +229,11 @@ def generate_input_data(self, input_names, input_types, input_shapes, input_quan if UPPER is None: UPPER = dtype_info.max else: - assert UPPER <= dtype_info.max, f"Out of dtype bound" + assert UPPER <= dtype_info.max, "Out of dtype bound" if LOWER is None: LOWER = dtype_info.min else: - assert LOWER >= dtype_info.min, f"Out of dtype bound" + assert LOWER >= dtype_info.min, "Out of dtype bound" else: raise RuntimeError(f"Unsupported dtype: {gen_dtype}") RANGE = UPPER - LOWER @@ -244,7 +244,8 @@ def generate_input_data(self, input_names, input_types, input_shapes, input_quan # if "float" in dtype: # arr = np.random.rand(*shape).astype(dtype) # elif "int" in dtype: - # arr = np.random.randint(np.iinfo(dtype).min, np.iinfo(dtype).max, size=shape, dtype=dtype) + # arr = np.random.randint(np.iinfo(dtype).min, + # np.iinfo(dtype).max, size=shape, dtype=dtype) # else: # assert False # Quantize if required diff --git a/mlonmcu/platform/tvm/tvm_target_platform.py b/mlonmcu/platform/tvm/tvm_target_platform.py index 98cea2eba..608da1def 100644 --- a/mlonmcu/platform/tvm/tvm_target_platform.py +++ b/mlonmcu/platform/tvm/tvm_target_platform.py @@ -18,14 +18,12 @@ # """TVM Target Platform""" import os -from pathlib import Path from mlonmcu.config import str2bool from .tvm_rpc_platform import TvmRpcPlatform from ..platform import TargetPlatform from mlonmcu.target import get_targets from mlonmcu.target.target import Target from .tvm_target import create_tvm_platform_target -from mlonmcu.artifact import Artifact, ArtifactFormat from mlonmcu.flow.tvm.backend.tvmc_utils import ( get_bench_tvmc_args, get_data_tvmc_args, From ab3017a25d69a22e860b19592316b5926ee074ca Mon Sep 17 00:00:00 2001 From: Philipp van Kempen Date: Thu, 11 Apr 2024 15:09:15 +0200 Subject: [PATCH 059/105] add missing file --- .../session/postprocess/validate_metrics.py | 253 ++++++++++++++++++ 1 file changed, 253 insertions(+) create mode 100644 mlonmcu/session/postprocess/validate_metrics.py diff --git a/mlonmcu/session/postprocess/validate_metrics.py b/mlonmcu/session/postprocess/validate_metrics.py new file mode 100644 index 000000000..ff38585ff --- /dev/null +++ b/mlonmcu/session/postprocess/validate_metrics.py @@ -0,0 +1,253 @@ +# +# Copyright (c) 2024 TUM Department of Electrical and Computer Engineering. +# +# This file is part of MLonMCU. +# See https://github.com/tum-ei-eda/mlonmcu.git for further info. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +"""Validation metrics utilities.""" + +import ast +import numpy as np + +from mlonmcu.logging import get_logger + +logger = get_logger() + + +""" + metrics = { + "allclose(atol=0.0,rtol=0.0)": None, + "allclose(atol=0.05,rtol=0.05)": None, + "allclose(atol=0.1,rtol=0.1)": None, + "topk(n=1)": None, + "topk(n=2)": None, + "topk(n=inf)": None, + "toy": None, + "mse(thr=0.1)": None, + "mse(thr=0.05)": None, + "mse(thr=0.01)": None, + "+-1": None, + } +""" + + +class ValidationMetric: + + def __init__(self, name, **cfg): + self.name = name + self.num_total = 0 + self.num_correct = 0 + + def process_(self, out_data, out_data_ref, quant: bool = False): + raise NotImplementedError + + def check(self, out_data, out_data_ref, quant: bool = False): + return out_data.dtype == out_data_ref.dtype + + def process(self, out_data, out_data_ref, quant: bool = False): + if not self.check(out_data, out_data_ref, quant=quant): + return + self.num_total += 1 + if self.process_(out_data, out_data_ref): + self.num_correct += 1 + + def get_summary(self): + if self.num_total == 0: + return "N/A" + return f"{self.num_correct}/{self.num_total} ({int(self.num_correct/self.num_total*100)}%)" + + +class AllCloseMetric(ValidationMetric): + + def __init__(self, name: str, atol: float = 0.0, rtol: float = 0.0): + super().__init__(name) + assert atol >= 0 + self.atol = atol + assert rtol >= 0 + self.rtol = rtol + + def check(self, out_data, out_data_ref, quant: bool = False): + return not quant + + def process_(self, out_data, out_data_ref, quant: bool = False): + return np.allclose(out_data, out_data_ref, rtol=self.rtol, atol=self.atol) + + +class TopKMetric(ValidationMetric): + + def __init__(self, name: str, n: int = 2): + super().__init__(name) + assert n >= 1 + self.n = n + + def check(self, out_data, out_data_ref, quant: bool = False): + data_len = len(out_data.flatten().tolist()) + # Probably no classification + return data_len < 25 and not quant + + def process_(self, out_data, out_data_ref, quant: bool = False): + # TODO: only for classification models! + # TODO: support multi_outputs? + data_sorted_idx = list(reversed(np.argsort(out_data).tolist()[0])) + ref_data_sorted_idx = list(reversed(np.argsort(out_data_ref).tolist()[0])) + k = 0 + num_checks = min(self.n, len(data_sorted_idx)) + assert len(data_sorted_idx) == len(ref_data_sorted_idx) + # print("data_sorted_idx", data_sorted_idx, type(data_sorted_idx)) + # print("ref_data_sorted_idx", ref_data_sorted_idx, type(ref_data_sorted_idx)) + # print("num_checks", num_checks) + for j in range(num_checks): + # print("j", j) + # print(f"data_sorted_idx[{j}]", data_sorted_idx[j], type(data_sorted_idx[j])) + idx = data_sorted_idx[j] + # print("idx", idx) + ref_idx = ref_data_sorted_idx[j] + # print("ref_idx", ref_idx) + if idx == ref_idx: + # print("IF") + k += 1 + else: + # print("ELSE") + if out_data.tolist()[0][idx] == out_data_ref.tolist()[0][ref_idx]: + # print("SAME") + k += 1 + else: + # print("BREAK") + break + # print("k", k) + if k < num_checks: + return False + elif k == num_checks: + return True + else: + assert False + + +class AccuracyMetric(TopKMetric): + + def __init__(self, name: str): + super().__init__(name, n=1) + + +class MSEMetric(ValidationMetric): + + def __init__(self, name: str, thr: int = 0.5): + super().__init__(name) + assert thr >= 0 + self.thr = thr + + def process_(self, out_data, out_data_ref, quant: bool = False): + mse = ((out_data - out_data_ref) ** 2).mean() + print("mse", mse) + return mse < self.thr + + +class ToyScoreMetric(ValidationMetric): + + def __init__(self, name: str, atol: float = 0.1, rtol: float = 0.1): + super().__init__(name) + assert atol >= 0 + self.atol = atol + assert rtol >= 0 + self.rtol = rtol + + def check(self, out_data, out_data_ref, quant: bool = False): + data_len = len(out_data.flatten().tolist()) + return data_len == 640 and not quant + + def process_(self, out_data, out_data_ref, quant: bool = False): + data_flat = out_data.flatten().tolist() + ref_data_flat = out_data_ref.flatten().tolist() + res = 0 + ref_res = 0 + length = len(data_flat) + for jjj in range(length): + res += data_flat[jjj] ** 2 + ref_res += ref_data_flat[jjj] ** 2 + res /= length + ref_res /= length + print("res", res) + print("ref_res", ref_res) + return np.allclose([res], [ref_res], atol=self.atol, rtol=self.rtol) + + +class PlusMinusOneMetric(ValidationMetric): + + def __init__(self, name: str): + super().__init__(name) + + def check(self, out_data, out_data_ref, quant: bool = False): + return "int" in out_data.dtype + + def process_(self, out_data, out_data_ref, quant: bool = False): + data_ = out_data.flatten().tolist() + ref_data_ = out_data_ref.flatten().tolist() + + length = len(data_) + for jjj in range(length): + diff = abs(data_[jjj] - ref_data_[jjj]) + print("diff", diff) + if diff > 1: + print("r FALSE") + return False + return True + + +LOOKUP = { + "allclose": AllCloseMetric, + "topk": TopKMetric, + "acc": AccuracyMetric, + "toy": ToyScoreMetric, + "mse": MSEMetric, + "+-1": PlusMinusOneMetric, + "pm1": PlusMinusOneMetric, +} + + +def parse_validate_metric_args(inp): + ret = {} + for x in inp.split(","): + x = x.strip() + assert "=" in x + key, val = x.split("=", 1) + try: + val = ast.literal_eval(val) + except Exception as e: + raise e + ret[key] = val + return ret + + +def parse_validate_metric(inp): + if "(" in inp: + metric_name, inp_ = inp.split("(") + assert inp_[-1] == ")" + inp_ = inp_[:-1] + metric_args = parse_validate_metric_args(inp_) + else: + metric_name = inp + metric_args = {} + metric_cls = LOOKUP.get(metric_name, None) + assert metric_cls is not None, f"Validate metric not found: {metric_name}" + metric = metric_cls(inp, **metric_args) + return metric + + +def parse_validate_metrics(inp): + ret = [] + for metric_str in inp.split(";"): + metric = parse_validate_metric(metric_str) + ret.append(metric) + return ret From 77f2a400f4ce154472988fc5c7aace8556ad7f9b Mon Sep 17 00:00:00 2001 From: Philipp van Kempen Date: Thu, 23 May 2024 17:16:27 +0200 Subject: [PATCH 060/105] mlif: fixes --- mlonmcu/platform/mlif/mlif.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/mlonmcu/platform/mlif/mlif.py b/mlonmcu/platform/mlif/mlif.py index 602357c20..b1329e1f4 100644 --- a/mlonmcu/platform/mlif/mlif.py +++ b/mlonmcu/platform/mlif/mlif.py @@ -457,8 +457,9 @@ def generate_model_support(self, target): return artifacts def configure(self, target, src, _model): + artifacts = [] if self.needs_model_support: - artifacts = self.generate_model_support(target) + artifacts.extend(self.generate_model_support(target)) if len(artifacts) > 0: assert len(artifacts) == 1 model_support_artifact = artifacts[0] From 95c1dd871870102e0fd34ae6a0d178f4ed776f91 Mon Sep 17 00:00:00 2001 From: Philipp van Kempen Date: Thu, 23 May 2024 17:16:50 +0200 Subject: [PATCH 061/105] postprocesses: add missing registrations --- mlonmcu/session/postprocess/__init__.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/mlonmcu/session/postprocess/__init__.py b/mlonmcu/session/postprocess/__init__.py index 36f2e432b..c3caef59f 100644 --- a/mlonmcu/session/postprocess/__init__.py +++ b/mlonmcu/session/postprocess/__init__.py @@ -32,6 +32,8 @@ CompareRowsPostprocess, AnalyseDumpPostprocess, AnalyseCoreVCountsPostprocess, + ValidateOutputsPostprocess, + ExportOutputsPostprocess, ) SUPPORTED_POSTPROCESSES = { @@ -48,4 +50,6 @@ "compare_rows": CompareRowsPostprocess, "analyse_dump": AnalyseDumpPostprocess, "analyse_corev_counts": AnalyseCoreVCountsPostprocess, + "validate_outputs": ValidateOutputsPostprocess, + "export_outputs": ExportOutputsPostprocess, } From c6b3e94e6762587469a0425d3b5cf3c777ddcbe3 Mon Sep 17 00:00:00 2001 From: Philipp van Kempen Date: Thu, 23 May 2024 17:16:59 +0200 Subject: [PATCH 062/105] postprocesses: fix typo --- mlonmcu/session/postprocess/postprocesses.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mlonmcu/session/postprocess/postprocesses.py b/mlonmcu/session/postprocess/postprocesses.py index 98e105aee..d6aa291ff 100644 --- a/mlonmcu/session/postprocess/postprocesses.py +++ b/mlonmcu/session/postprocess/postprocesses.py @@ -1539,7 +1539,7 @@ def post_run(self, report, artifacts): # fallback for custom name-based npy dict out_data = list(output.values())[ii] else: # fallback for index-based npy array - assert isinstance(output, (list, np.array)), "expected dict, list of np.array type" + assert isinstance(output, (list, np.array)), "expected dict, list or np.array type" out_data = output[ii] else: RuntimeError(f"Output not found: {out_name}") From 2e61212df1ac5a54ba2bf589db0d5546560d143e Mon Sep 17 00:00:00 2001 From: Philipp van Kempen Date: Thu, 23 May 2024 17:18:34 +0200 Subject: [PATCH 063/105] postprocesses: improve validate_outputs --- mlonmcu/session/postprocess/postprocesses.py | 40 +++++++++++++------- 1 file changed, 26 insertions(+), 14 deletions(-) diff --git a/mlonmcu/session/postprocess/postprocesses.py b/mlonmcu/session/postprocess/postprocesses.py index d6aa291ff..b1c0ac525 100644 --- a/mlonmcu/session/postprocess/postprocesses.py +++ b/mlonmcu/session/postprocess/postprocesses.py @@ -1549,21 +1549,33 @@ def post_run(self, report, artifacts): quant = model_info_data.get("output_quant_details", None) if quant: - assert ii < len(quant) - quant = quant[ii] - if quant: + def ref_quant_helper(quant, ref_data): # TODO: move somewhere else + if quant is None: + return data + quant_scale, quant_zero_point, quant_dtype = quant + if quant_dtype is None or ref_data.dtype.name == quant_dtype: + return data + assert out_data.dtype.name in ["float32"], "Quantization only supported for float32 input" + assert quant_dtype in ["int8"], "Quantization only supported for int8 output" + return np.around((out_ref_data / quant_scale) + quant_zero_point).astype( + "int8" + ) + def dequant_helper(quant, data): # TODO: move somewhere else + if quant is None: + return data quant_scale, quant_zero_point, quant_dtype = quant - if quant_dtype: - if out_data.dtype.name != quant_dtype: - # need to dequantize here - assert out_data.dtype.name in ["int8"], "Dequantization only supported for int8 input" - assert quant_dtype in ["float32"], "Dequantization only supported for float32 output" - out_ref_data_quant = np.around((out_ref_data / quant_scale) + quant_zero_point).astype( - "int8" - ) - for vm in validate_metrics: - vm.process(out_data, out_ref_data_quant, quant=True) - out_data = (out_data.astype("float32") - quant_zero_point) * quant_scale + if quant_dtype is None or out_data.dtype.name == quant_dtype: + return data + assert out_data.dtype.name in ["int8"], "Dequantization only supported for int8 input" + assert quant_dtype in ["float32"], "Dequantization only supported for float32 output" + return (out_data.astype("float32") - quant_zero_point) * quant_scale + assert ii < len(quant) + quant_ = quant[ii] + if quant_ is not None: + ref_data_qaunt = ref_quant_helper(quant_, out_ref_data) + for vm in validate_metrics: + vm.process(out_data, out_ref_data_quant, quant=True) + out_data = dequant_helper(quant_, out_data) # print("out_data", out_data) # print("sum(out_data)", np.sum(out_data)) # print("out_ref_data", out_ref_data) From 4ec9d7f0eb75851882369ca80c6eb04f7b1b9dc5 Mon Sep 17 00:00:00 2001 From: Philipp van Kempen Date: Thu, 23 May 2024 17:18:51 +0200 Subject: [PATCH 064/105] postprocesses: implement export_outputs --- mlonmcu/session/postprocess/postprocesses.py | 134 +++++++++++++++++++ 1 file changed, 134 insertions(+) diff --git a/mlonmcu/session/postprocess/postprocesses.py b/mlonmcu/session/postprocess/postprocesses.py index b1c0ac525..aac78b82c 100644 --- a/mlonmcu/session/postprocess/postprocesses.py +++ b/mlonmcu/session/postprocess/postprocesses.py @@ -1593,3 +1593,137 @@ def dequant_helper(quant, data): # TODO: move somewhere else res = vm.get_summary() report.post_df[f"{vm.name}"] = res return [] + + +class ExportOutputsPostprocess(RunPostprocess): + """Postprocess for writing model outputs to a directory.""" + + DEFAULTS = { + **RunPostprocess.DEFAULTS, + "dest": None, # if none: export as artifact + "use_ref": False, + "skip_dequant": False, + "fmt": "bin", + "archive_fmt": None, + } + + def __init__(self, features=None, config=None): + super().__init__("export_outputs", features=features, config=config) + + @property + def dest(self): + """Get dest property.""" + value = self.config["validate_metrics"] + if value is not None: + if not isinstance(value, Path): + assert isinstance(value, str) + value = Path(value) + return value + + def use_ref(self): + """Get use_ref property.""" + value = self.config["use_ref"] + return str2bool(value) if not isinstance(value, (bool, int)) else value + + def skip_dequant(self): + """Get skip_dequant property.""" + value = self.config["skip_dequant"] + return str2bool(value) if not isinstance(value, (bool, int)) else value + + def fmt(self): + """Get fmt property.""" + return self.config["fmt"] + + def archive_fmt(self): + """Get archive_fmt property.""" + return self.config["archive_fmt"] + + def post_run(self, report, artifacts): + """Called at the end of a run.""" + model_info_artifact = lookup_artifacts(artifacts, name="model_info.yml", first_only=True) + assert len(model_info_artifact) == 1, "Could not find artifact: model_info.yml" + model_info_artifact = model_info_artifact[0] + import yaml + + model_info_data = yaml.safe_load(model_info_artifact.content) + # print("model_info_data", model_info_data) + if len(model_info_data["output_names"]) > 1: + raise NotImplementedError("Multi-outputs not yet supported.") + if self.use_ref: + outputs_ref_artifact = lookup_artifacts(artifacts, name="outputs_ref.npy", first_only=True) + assert len(outputs_ref_artifact) == 1, "Could not find artifact: outputs_ref.npy" + outputs_ref_artifact = outputs_ref_artifact[0] + import numpy as np + outputs_ref = np.load(outputs_ref_artifact.path, allow_pickle=True) + outputs = outputs_ref + else: + outputs_artifact = lookup_artifacts(artifacts, name="outputs.npy", first_only=True) + assert len(outputs_artifact) == 1, "Could not find artifact: outputs.npy" + outputs_artifact = outputs_artifact[0] + outputs = np.load(outputs_artifact.path, allow_pickle=True) + if dest is None: + temp_dir = tempfile.TemporaryDirectory() + dest = Path(temp_dir.name) + else: + temp_dir = None + assert dest.is_dir(), f"Not a directory: {dest}" + dest_ = dest + assert self.fmt in ["bin", "npy"], f"Invalid format: {self.fmt}" + filenames = [] + for i, output in enumerate(outputs): + if isinstance(output, dict): # name based lookup + pass + else: # index based lookup + assert isinstance(output, (list, np.array)), "expected dict, list or np.array" + output_names = model_info_data["output_names"] + assert len(output) == len(output_names) + output = {output_names[idx]: out for idx, out in enumerate(output)} + quant = model_info_data.get("output_quant_details", None) + if quant and not self.skip_dequant: + def dequant_helper(quant, data): + if quant is None: + return data + quant_scale, quant_zero_point, quant_dtype = quant + if quant_dtype is None or out_data.dtype.name == quant_dtype: + return data + assert out_data.dtype.name in ["int8"], "Dequantization only supported for int8 input" + assert quant_dtype in ["float32"], "Dequantization only supported for float32 output" + return (out_data.astype("float32") - quant_zero_point) * quant_scale + output = {out_name: dequant_helper(quant[j], outputs[out_name]) for j, out_name in enumerate(output.keys())} + if self.fmt == "npy": + raise NotImplementedError("npy export") + elif self.fmt == "bin": + assert len(output.keys()) == 1, "Multi-outputs not supported" + output_data = list(output.values())[0] + data = output_data.tobytes(order="C") + file_name = f"{i}.bin" + file_dest = dest_ / file_name + filenames.append(file_dest) + with open(dest, "wb") as f: + f.write(data) + else: + assert False, f"fmt not supported: {self.fmt}" + artifacts = [] + archive_fmt = self.archive_fmt + create_artifact = self.dest is None or archive_fmt is not None + if create_artifact: + if archive_fmt is None: + assert self.dest is None + archive_fmt = "tar.gz" # Default fallback + assert self.archive_fmt in ["tar.xz", "tar.gz", "zip"] + archive_name = f"output_data.{archive_fmt}" + archive_path = f"{dest_}.{archive_fmt}" + if archive_fmt == "tar.gz": + import tarfile + with tarfile.open(archive_path, "w:gz") as tar: + for filename in filesnames: + tar.add(filename, arc=dest_) + else: + raise NotImplementedError(f"archive_fmt={archive_fmt}") + with open(archive_path, "rb") as f: + raw = f.read() + artifact = Artifact(archive_name, raw=raw, fmt=ArtifactFormat.BIN) + artifacts.append(artifact) + if temp_dir: + temp_dir.cleanup() + return artifacts From 8cc27b3114aadcdb434585a6340a1ffb11f696a9 Mon Sep 17 00:00:00 2001 From: Philipp van Kempen Date: Thu, 23 May 2024 18:42:21 +0200 Subject: [PATCH 065/105] postprocess fixes --- mlonmcu/session/postprocess/postprocesses.py | 47 ++++++++++--------- .../session/postprocess/validate_metrics.py | 20 +------- 2 files changed, 27 insertions(+), 40 deletions(-) diff --git a/mlonmcu/session/postprocess/postprocesses.py b/mlonmcu/session/postprocess/postprocesses.py index aac78b82c..d6f76f9fb 100644 --- a/mlonmcu/session/postprocess/postprocesses.py +++ b/mlonmcu/session/postprocess/postprocesses.py @@ -21,9 +21,11 @@ import re import ast import tempfile +import tarfile from pathlib import Path from io import StringIO +import numpy as np import pandas as pd from mlonmcu.artifact import Artifact, ArtifactFormat, lookup_artifacts @@ -1549,30 +1551,30 @@ def post_run(self, report, artifacts): quant = model_info_data.get("output_quant_details", None) if quant: - def ref_quant_helper(quant, ref_data): # TODO: move somewhere else + def ref_quant_helper(quant, data): # TODO: move somewhere else if quant is None: return data quant_scale, quant_zero_point, quant_dtype = quant - if quant_dtype is None or ref_data.dtype.name == quant_dtype: + if quant_dtype is None or data.dtype.name == quant_dtype: return data - assert out_data.dtype.name in ["float32"], "Quantization only supported for float32 input" + assert data.dtype.name in ["float32"], "Quantization only supported for float32 input" assert quant_dtype in ["int8"], "Quantization only supported for int8 output" - return np.around((out_ref_data / quant_scale) + quant_zero_point).astype( + return np.around((data / quant_scale) + quant_zero_point).astype( "int8" ) def dequant_helper(quant, data): # TODO: move somewhere else if quant is None: return data quant_scale, quant_zero_point, quant_dtype = quant - if quant_dtype is None or out_data.dtype.name == quant_dtype: + if quant_dtype is None or data.dtype.name == quant_dtype: return data - assert out_data.dtype.name in ["int8"], "Dequantization only supported for int8 input" + assert data.dtype.name in ["int8"], "Dequantization only supported for int8 input" assert quant_dtype in ["float32"], "Dequantization only supported for float32 output" - return (out_data.astype("float32") - quant_zero_point) * quant_scale + return (data.astype("float32") - quant_zero_point) * quant_scale assert ii < len(quant) quant_ = quant[ii] if quant_ is not None: - ref_data_qaunt = ref_quant_helper(quant_, out_ref_data) + out_ref_data_quant = ref_quant_helper(quant_, out_ref_data) for vm in validate_metrics: vm.process(out_data, out_ref_data_quant, quant=True) out_data = dequant_helper(quant_, out_data) @@ -1585,7 +1587,7 @@ def dequant_helper(quant, data): # TODO: move somewhere else assert out_data.shape == out_ref_data.shape, "shape missmatch" for vm in validate_metrics: - vm.process(out_data, out_ref_data_quant, quant=False) + vm.process(out_data, out_ref_data, quant=False) ii += 1 if self.report: raise NotImplementedError @@ -1613,27 +1615,31 @@ def __init__(self, features=None, config=None): @property def dest(self): """Get dest property.""" - value = self.config["validate_metrics"] + value = self.config["dest"] if value is not None: if not isinstance(value, Path): assert isinstance(value, str) value = Path(value) return value + @property def use_ref(self): """Get use_ref property.""" value = self.config["use_ref"] return str2bool(value) if not isinstance(value, (bool, int)) else value + @property def skip_dequant(self): """Get skip_dequant property.""" value = self.config["skip_dequant"] return str2bool(value) if not isinstance(value, (bool, int)) else value + @property def fmt(self): """Get fmt property.""" return self.config["fmt"] + @property def archive_fmt(self): """Get archive_fmt property.""" return self.config["archive_fmt"] @@ -1653,7 +1659,6 @@ def post_run(self, report, artifacts): outputs_ref_artifact = lookup_artifacts(artifacts, name="outputs_ref.npy", first_only=True) assert len(outputs_ref_artifact) == 1, "Could not find artifact: outputs_ref.npy" outputs_ref_artifact = outputs_ref_artifact[0] - import numpy as np outputs_ref = np.load(outputs_ref_artifact.path, allow_pickle=True) outputs = outputs_ref else: @@ -1661,9 +1666,9 @@ def post_run(self, report, artifacts): assert len(outputs_artifact) == 1, "Could not find artifact: outputs.npy" outputs_artifact = outputs_artifact[0] outputs = np.load(outputs_artifact.path, allow_pickle=True) - if dest is None: + if self.dest is None: temp_dir = tempfile.TemporaryDirectory() - dest = Path(temp_dir.name) + dest_ = Path(temp_dir.name) else: temp_dir = None assert dest.is_dir(), f"Not a directory: {dest}" @@ -1684,12 +1689,12 @@ def dequant_helper(quant, data): if quant is None: return data quant_scale, quant_zero_point, quant_dtype = quant - if quant_dtype is None or out_data.dtype.name == quant_dtype: + if quant_dtype is None or data.dtype.name == quant_dtype: return data - assert out_data.dtype.name in ["int8"], "Dequantization only supported for int8 input" + assert data.dtype.name in ["int8"], "Dequantization only supported for int8 input" assert quant_dtype in ["float32"], "Dequantization only supported for float32 output" - return (out_data.astype("float32") - quant_zero_point) * quant_scale - output = {out_name: dequant_helper(quant[j], outputs[out_name]) for j, out_name in enumerate(output.keys())} + return (data.astype("float32") - quant_zero_point) * quant_scale + output = {out_name: dequant_helper(quant[j], output[out_name]) for j, out_name in enumerate(output.keys())} if self.fmt == "npy": raise NotImplementedError("npy export") elif self.fmt == "bin": @@ -1699,7 +1704,7 @@ def dequant_helper(quant, data): file_name = f"{i}.bin" file_dest = dest_ / file_name filenames.append(file_dest) - with open(dest, "wb") as f: + with open(file_dest, "wb") as f: f.write(data) else: assert False, f"fmt not supported: {self.fmt}" @@ -1710,14 +1715,14 @@ def dequant_helper(quant, data): if archive_fmt is None: assert self.dest is None archive_fmt = "tar.gz" # Default fallback - assert self.archive_fmt in ["tar.xz", "tar.gz", "zip"] + assert archive_fmt in ["tar.xz", "tar.gz", "zip"] archive_name = f"output_data.{archive_fmt}" archive_path = f"{dest_}.{archive_fmt}" if archive_fmt == "tar.gz": import tarfile with tarfile.open(archive_path, "w:gz") as tar: - for filename in filesnames: - tar.add(filename, arc=dest_) + for filename in filenames: + tar.add(filename, arcname=dest_) else: raise NotImplementedError(f"archive_fmt={archive_fmt}") with open(archive_path, "rb") as f: diff --git a/mlonmcu/session/postprocess/validate_metrics.py b/mlonmcu/session/postprocess/validate_metrics.py index ff38585ff..615b6ee2f 100644 --- a/mlonmcu/session/postprocess/validate_metrics.py +++ b/mlonmcu/session/postprocess/validate_metrics.py @@ -26,23 +26,6 @@ logger = get_logger() -""" - metrics = { - "allclose(atol=0.0,rtol=0.0)": None, - "allclose(atol=0.05,rtol=0.05)": None, - "allclose(atol=0.1,rtol=0.1)": None, - "topk(n=1)": None, - "topk(n=2)": None, - "topk(n=inf)": None, - "toy": None, - "mse(thr=0.1)": None, - "mse(thr=0.05)": None, - "mse(thr=0.01)": None, - "+-1": None, - } -""" - - class ValidationMetric: def __init__(self, name, **cfg): @@ -150,7 +133,6 @@ def __init__(self, name: str, thr: int = 0.5): def process_(self, out_data, out_data_ref, quant: bool = False): mse = ((out_data - out_data_ref) ** 2).mean() - print("mse", mse) return mse < self.thr @@ -189,7 +171,7 @@ def __init__(self, name: str): super().__init__(name) def check(self, out_data, out_data_ref, quant: bool = False): - return "int" in out_data.dtype + return "int" in out_data.dtype.str def process_(self, out_data, out_data_ref, quant: bool = False): data_ = out_data.flatten().tolist() From 6966e7e0c925bb1c594fe2304c8408c02d0336cf Mon Sep 17 00:00:00 2001 From: Philipp van Kempen Date: Thu, 23 May 2024 18:42:44 +0200 Subject: [PATCH 066/105] utils.execute: fixes for stdin_data feature --- mlonmcu/setup/utils.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/mlonmcu/setup/utils.py b/mlonmcu/setup/utils.py index 1bec7b2bd..5a5c7af0e 100644 --- a/mlonmcu/setup/utils.py +++ b/mlonmcu/setup/utils.py @@ -253,7 +253,9 @@ def execute( for line in process.stdout: if encoding: line = line.decode(encoding, errors="replace") - new_line = prefix + line + new_line = prefix + line + else: + new_line = line out_str = out_str + new_line print_func(new_line.replace("\n", "")) exit_code = None @@ -273,14 +275,15 @@ def execute( os.kill(pid, signal.SIGINT) else: try: - p = subprocess.Popen([i for i in args], **kwargs, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) + p = subprocess.Popen( + [i for i in args], **kwargs, stdout=subprocess.PIPE, stdin=subprocess.PIPE, stderr=subprocess.STDOUT) if stdin_data: out_str = p.communicate(input=stdin_data)[0] else: out_str = p.communicate()[0] if encoding: out_str = out_str.decode(encoding, errors="replace") - out_str = prefix + out_str + out_str = prefix + out_str exit_code = p.poll() # print_func(out_str) if handle_exit is not None: From e7f1be10df53fb8ca581d3fcd9179fe008d10d3e Mon Sep 17 00:00:00 2001 From: Philipp van Kempen Date: Fri, 24 May 2024 12:21:45 +0200 Subject: [PATCH 067/105] models: convert support_path, inputs_path & outputs_path to pathlib.Path --- mlonmcu/models/model.py | 21 ++++++++++++++++++--- 1 file changed, 18 insertions(+), 3 deletions(-) diff --git a/mlonmcu/models/model.py b/mlonmcu/models/model.py index 3dd249849..3bbd1174c 100644 --- a/mlonmcu/models/model.py +++ b/mlonmcu/models/model.py @@ -221,17 +221,32 @@ def output_types(self): @property def support_path(self): - return self.config["support_path"] + value = self.config["support_path"] + if value is not None: + if not isinstance(value, Path): + assert isinstance(value, str) + value = Path(value) + return value @property def inputs_path(self): # TODO: fall back to metadata - return self.config["inputs_path"] + value = self.config["inputs_path"] + if value is not None: + if not isinstance(value, Path): + assert isinstance(value, str) + value = Path(value) + return value @property def outputs_path(self): # TODO: fall back to metadata - return self.config["outputs_path"] + value = self.config["outputs_path"] + if value is not None: + if not isinstance(value, Path): + assert isinstance(value, str) + value = Path(value) + return value @property def skip_check(self): From 69c974bc83d11396506c0174d81339c1704f538b Mon Sep 17 00:00:00 2001 From: Philipp van Kempen Date: Fri, 24 May 2024 12:21:57 +0200 Subject: [PATCH 068/105] postprocess fixes --- mlonmcu/session/postprocess/postprocesses.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/mlonmcu/session/postprocess/postprocesses.py b/mlonmcu/session/postprocess/postprocesses.py index d6f76f9fb..e98bc8219 100644 --- a/mlonmcu/session/postprocess/postprocesses.py +++ b/mlonmcu/session/postprocess/postprocesses.py @@ -1671,8 +1671,8 @@ def post_run(self, report, artifacts): dest_ = Path(temp_dir.name) else: temp_dir = None - assert dest.is_dir(), f"Not a directory: {dest}" - dest_ = dest + assert self.dest.is_dir(), f"Not a directory: {self.dest}" + dest_ = self.dest assert self.fmt in ["bin", "npy"], f"Invalid format: {self.fmt}" filenames = [] for i, output in enumerate(outputs): @@ -1722,7 +1722,7 @@ def dequant_helper(quant, data): import tarfile with tarfile.open(archive_path, "w:gz") as tar: for filename in filenames: - tar.add(filename, arcname=dest_) + tar.add(filename, arcname=filename.name) else: raise NotImplementedError(f"archive_fmt={archive_fmt}") with open(archive_path, "rb") as f: From d8d62698a9bf04cfda89846e72f81d05e008f896 Mon Sep 17 00:00:00 2001 From: Philipp van Kempen Date: Mon, 27 May 2024 09:00:37 +0200 Subject: [PATCH 069/105] frontends: add assertion msg --- mlonmcu/models/frontend.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/mlonmcu/models/frontend.py b/mlonmcu/models/frontend.py index abab0142c..2b70385fa 100644 --- a/mlonmcu/models/frontend.py +++ b/mlonmcu/models/frontend.py @@ -266,7 +266,7 @@ def generate_input_data(self, input_names, input_types, input_shapes, input_quan inputs_data.append(data) elif self.gen_data_fill_mode == "file": if self.gen_data_file == "auto": - len(in_paths) > 0 + assert len(in_paths) > 0, "in_paths is empty" if len(in_paths) == 1: if in_paths[0].is_dir(): files = list(in_paths[0].iterdir()) @@ -360,6 +360,7 @@ def generate_output_ref_data( else: files = out_paths temp = {} + assert len(inputs_data) <= len(files), f"Missing output data for provided inputs. (Expected: {len(inputs_data)}, Got: {len(files)})" for file in files: if not isinstance(file, Path): file = Path(file) From d040b1b4387747167d32c66f0a976b31c3b8d6d6 Mon Sep 17 00:00:00 2001 From: Philipp van Kempen Date: Mon, 27 May 2024 09:00:47 +0200 Subject: [PATCH 070/105] rm print --- mlonmcu/session/postprocess/postprocesses.py | 1 - 1 file changed, 1 deletion(-) diff --git a/mlonmcu/session/postprocess/postprocesses.py b/mlonmcu/session/postprocess/postprocesses.py index e98bc8219..2cacb452e 100644 --- a/mlonmcu/session/postprocess/postprocesses.py +++ b/mlonmcu/session/postprocess/postprocesses.py @@ -1492,7 +1492,6 @@ def post_run(self, report, artifacts): import yaml model_info_data = yaml.safe_load(model_info_artifact.content) - print("model_info_data", model_info_data) if len(model_info_data["output_names"]) > 1: raise NotImplementedError("Multi-outputs not yet supported.") outputs_ref_artifact = lookup_artifacts(artifacts, name="outputs_ref.npy", first_only=True) From 71e7280478095f493983718c7ec1f6def7e11b92 Mon Sep 17 00:00:00 2001 From: Philipp van Kempen Date: Mon, 27 May 2024 16:57:29 +0200 Subject: [PATCH 071/105] introduce new feature: GenRefLabels --- mlonmcu/feature/features.py | 41 +++++++++++++++++++++++++++++++++++++ 1 file changed, 41 insertions(+) diff --git a/mlonmcu/feature/features.py b/mlonmcu/feature/features.py index 69cc935b4..4f90f3256 100644 --- a/mlonmcu/feature/features.py +++ b/mlonmcu/feature/features.py @@ -2307,6 +2307,47 @@ def get_frontend_config(self, frontend): } +@register_feature("gen_ref_labels", depends=["gen_data"]) +class GenRefLabels(FrontendFeature): # TODO: use custom stage instead of LOAD + """TODO""" + + DEFAULTS = { + **FeatureBase.DEFAULTS, + "mode": "file", # Allowed: file, model + "file": "auto", # Only relevant if mode=file + "fmt": "npy", # Allowed: npy, npz, txt + } + + def __init__(self, features=None, config=None): + super().__init__("gen_ref_labels", features=features, config=config) + + @property + def mode(self): + value = self.config["mode"] + assert value in ["file", "model"] + return value + + @property + def file(self): + value = self.config["file"] + return value + + @property + def fmt(self): + value = self.config["fmt"] + assert value in ["npy", "npz"] + return value + + def get_frontend_config(self, frontend): + assert frontend in ["tflite"] + return { + f"{frontend}.gen_ref_labels": self.enabled, + f"{frontend}.gen_ref_labels_mode": self.mode, + f"{frontend}.gen_ref_labels_file": self.file, + f"{frontend}.gen_ref_labels_fmt": self.fmt, + } + + @register_feature("set_inputs") class SetInputs(PlatformFeature): # TODO: use custom stage instead of LOAD """TODO""" From c64bdc3bfedc4d8da624869ac4e4c1d772e72b8a Mon Sep 17 00:00:00 2001 From: Philipp van Kempen Date: Mon, 27 May 2024 16:59:23 +0200 Subject: [PATCH 072/105] introduce new feature: GenRefLabels 2 --- mlonmcu/models/frontend.py | 119 +++++++++++++++++++++++++++++++++++-- 1 file changed, 115 insertions(+), 4 deletions(-) diff --git a/mlonmcu/models/frontend.py b/mlonmcu/models/frontend.py index 2b70385fa..2288d6262 100644 --- a/mlonmcu/models/frontend.py +++ b/mlonmcu/models/frontend.py @@ -68,6 +68,10 @@ class Frontend(ABC): "gen_ref_data_mode": None, "gen_ref_data_file": None, "gen_ref_data_fmt": None, + "gen_ref_labels": False, + "gen_ref_labels_mode": None, + "gen_ref_labels_file": None, + "gen_ref_labels_fmt": None, } REQUIRED = set() @@ -142,6 +146,27 @@ def gen_ref_data_fmt(self): assert value in ["npy", "npz"] return value + @property + def gen_ref_labels(self): + value = self.config["gen_ref_labels"] + return str2bool(value) if not isinstance(value, (bool, int)) else value + + @property + def gen_ref_labels_mode(self): + value = self.config["gen_ref_labels_mode"] + assert value in ["file", "model"] + return value + + @property + def gen_ref_labels_file(self): + return self.config["gen_ref_labels_file"] + + @property + def gen_ref_labels_fmt(self): + value = self.config["gen_ref_labels_fmt"] + assert value in ["npy", "npz", "txt", "csv"] + return value + def inference(self, model: Model, input_data: Dict[str, np.array]): raise NotImplementedError @@ -402,13 +427,58 @@ def generate_output_ref_data( data[output_name] = arr outputs_data.append(data) else: - assert self.gen_data_file is not None, "Missing value for gen_data_file" + assert self.gen_ref_data_file is not None, "Missing value for gen_ref_data_file" file = Path(self.gen_data_file) assert file.is_file(), f"File not found: {file}" + raise NotImplementedError else: raise RuntimeError(f"unsupported fill_mode: {self.gen_ref_data_mode}") return outputs_data + def generate_ref_labels( + self, inputs_data, model, out_labels_paths, output_names, output_types, output_shapes, output_quant_details + ): + assert self.gen_ref_labels + labels = [] + if self.gen_ref_labels_mode == "model": + assert len(inputs_data) > 0 + for i, input_data in enumerate(inputs_data): + output_data = self.inference(model, input_data, quant=False, dequant=True) + assert len(output_data) == 1, "Does not support multi-output classification" + output_data = output_data[list(output_data)[0]] + top_label = np.argmax(output_data) + labels.append(top_label) + + elif self.gen_ref_labels_mode == "file": + if self.gen_ref_labels_file == "auto": + assert len(out_labels_paths) > 0, "labels_paths is empty" + assert len(out_labels_paths) == 1 + file = Path(out_labels_paths[0]) + # file = f"{out_paths[0]}_labels.csv" + print("file", file) + else: + assert self.gen_ref_labels_file is not None, "Missing value for gen_ref_labels_file" + file = Path(self.gen_ref_labels_file) + assert file.is_file(), f"File not found: {file}" + ext = file.suffix + assert len(ext) > 1 + fmt = ext[1:].lower() + if fmt == "csv": + import pandas as pd + labels_df = pd.read_csv(file, sep=",") + assert "i" in labels_df.columns + assert "label_idx" in labels_df.columns + print("len(inputs_data)", len(inputs_data)) + print("len(labels_df)", len(labels_df)) + assert len(inputs_data) <= len(labels_df) + labels_df.sort_values("i", inplace=True) + labels = list(labels_df["label_idx"].astype(int))[:len(inputs_data)] + else: + raise NotImplementedError(f"Fmt not supported: {fmt}") + else: + raise RuntimeError(f"unsupported fill_mode: {self.gen_ref_labels_mode}") + return labels + def generate_model_info( self, input_names, @@ -454,6 +524,7 @@ def process_metadata(self, model, cfg=None): metadata = model.metadata in_paths = [] out_paths = [] + labels_paths = [] input_shapes = {} output_shapes = {} input_types = {} @@ -521,6 +592,14 @@ def process_metadata(self, model, cfg=None): out_path.is_dir() ), f"Output data directory defined in model metadata does not exist: {out_path}" out_paths.append(out_path) + if self.gen_ref_labels and self.gen_ref_labels_mode == "file" and self.gen_ref_labels_file == "auto": + if "test_labels_file" in outp: + labels_file = Path(outp["test_labels_file"]) + labels_path = model_dir / labels_file + assert ( + labels_path.is_file() + ), f"Labels file defined in model metadata does not exist: {labels_path}" + labels_paths.append(labels_path) else: fallback_in_path = model_dir / "input" if fallback_in_path.is_dir(): @@ -528,12 +607,18 @@ def process_metadata(self, model, cfg=None): fallback_out_path = model_dir / "output" if fallback_out_path.is_dir(): out_paths.append(fallback_out_path) + fallback_labels_path = model_dir / "output_labels.csv" + if fallback_labels_path.is_file(): + labels_paths.append(fallback_labels_path) if model.inputs_path: logger.info("Overriding default model input data with user path") in_paths = [model.inputs_path] if model.outputs_path: logger.info("Overriding default model output data with user path") out_paths = [model.outputs_path] + if model.output_labels_path: # TODO + logger.info("Overriding default model output labels with user path") + labels_paths = [model.output_labels_path] if metadata is not None and "backends" in metadata: assert cfg is not None @@ -586,6 +671,8 @@ def process_metadata(self, model, cfg=None): cfg.update({"mlif.output_data_path": out_paths}) # cfg.update({"espidf.output_data_path": out_paths}) # cfg.update({"zephyr.output_data_path": out_paths}) + if len(labels_paths) > 0: + cfg.update({"mlif.output_labels_path": labels_paths}) if len(input_shapes) > 0: cfg.update({f"{model.name}.input_shapes": input_shapes}) if len(output_shapes) > 0: @@ -657,10 +744,34 @@ def process_metadata(self, model, cfg=None): else: raise RuntimeError(f"Unsupported fmt: {fmt}") assert raw - outputs_data_artifact = Artifact( + outputs_ref_artifact = Artifact( f"outputs_ref.{fmt}", raw=raw, fmt=ArtifactFormat.BIN, flags=("outputs_ref", fmt) ) - artifacts.append(outputs_data_artifact) + artifacts.append(outputs_ref_artifact) + if self.gen_ref_labels: + labels_ref = self.generate_ref_labels( + inputs_data, model, labels_paths, output_names, output_types, output_shapes, output_quant_details + ) + fmt = self.gen_ref_labels_fmt + if fmt == "npy": + with tempfile.TemporaryDirectory() as tmpdirname: + tempfilename = Path(tmpdirname) / "labels.npy" + np.save(tempfilename, labels_ref) + with open(tempfilename, "rb") as f: + raw = f.read() + elif fmt == "npz": + raise NotImplementedError + elif fmt == "txt": + raise NotImplementedError + elif fmt == "csv": + raise NotImplementedError + else: + raise RuntimeError(f"Unsupported fmt: {fmt}") + assert raw + labels_ref_artifact = Artifact( + f"labels_ref.{fmt}", raw=raw, fmt=ArtifactFormat.BIN, flags=("labels_ref", fmt) + ) + artifacts.append(labels_ref_artifact) return artifacts def generate(self, model) -> Tuple[dict, dict]: @@ -762,7 +873,7 @@ def produce_artifacts(self, model): # TODO: frontend parsed metadata instead of lookup.py? # TODO: how to find inout_data? class TfLiteFrontend(SimpleFrontend): - FEATURES = Frontend.FEATURES | {"visualize", "split_layers", "tflite_analyze", "gen_data", "gen_ref_data"} + FEATURES = Frontend.FEATURES | {"visualize", "split_layers", "tflite_analyze", "gen_data", "gen_ref_data", "gen_ref_labels"} DEFAULTS = { **Frontend.DEFAULTS, From 983989efdc94ba7b52f9abeccde9b3de2ae2c47f Mon Sep 17 00:00:00 2001 From: Philipp van Kempen Date: Mon, 27 May 2024 17:01:12 +0200 Subject: [PATCH 073/105] frontend: handle dtype ranges --- mlonmcu/models/frontend.py | 68 +++++++++++++++++++++++++++++++------- 1 file changed, 56 insertions(+), 12 deletions(-) diff --git a/mlonmcu/models/frontend.py b/mlonmcu/models/frontend.py index 2288d6262..33570aa75 100644 --- a/mlonmcu/models/frontend.py +++ b/mlonmcu/models/frontend.py @@ -213,7 +213,7 @@ def process_features(self, features): def produce_artifacts(self, model): pass - def generate_input_data(self, input_names, input_types, input_shapes, input_quant_details, in_paths): + def generate_input_data(self, input_names, input_types, input_shapes, input_ranges, input_quant_details, in_paths): # TODO: drop self and move method out of frontends.py, support non-tflite models assert self.gen_data inputs_data = [] @@ -225,6 +225,7 @@ def generate_input_data(self, input_names, input_types, input_shapes, input_quan assert input_name in input_types, f"Unknown dtype for input: {input_name}" dtype = input_types[input_name] quant = input_quant_details.get(input_name, None) + rng = input_ranges.get(input_name, None) gen_dtype = dtype if quant: _, _, ty = quant @@ -240,8 +241,11 @@ def generate_input_data(self, input_names, input_types, input_shapes, input_quan elif self.gen_data_fill_mode == "random": DIST = "uniform" if DIST == "uniform": - UPPER = None # TODO: config - LOWER = None # TODO: config + UPPER = None + LOWER = None + if rng is not None: + assert len(rng) == 2, "Range should be a tuple (lower, upper)" + LOWER, UPPER = rng if "float" in gen_dtype: if UPPER is None: # UPPER = 1.0 @@ -261,6 +265,7 @@ def generate_input_data(self, input_names, input_types, input_shapes, input_quan assert LOWER >= dtype_info.min, "Out of dtype bound" else: raise RuntimeError(f"Unsupported dtype: {gen_dtype}") + assert LOWER <= UPPER RANGE = UPPER - LOWER assert RANGE > 0 arr = np.random.uniform(LOWER, UPPER, shape) @@ -274,9 +279,10 @@ def generate_input_data(self, input_names, input_types, input_shapes, input_quan # else: # assert False # Quantize if required - if gen_dtype != dtype: + # if gen_dtype != dtype: + if quant: assert "int" in dtype - assert quant + # assert quant scale, shift, ty = quant arr = (arr / scale) + shift arr = np.around(arr) @@ -330,6 +336,7 @@ def generate_input_data(self, input_names, input_types, input_shapes, input_quan assert input_name in input_types, f"Unknown dtype for input: {input_name}" dtype = input_types[input_name] quant = input_quant_details.get(input_name, None) + rng = input_ranges.get(input_name, None) gen_dtype = dtype if quant: _, _, ty = quant @@ -343,12 +350,33 @@ def generate_input_data(self, input_names, input_types, input_shapes, input_quan # Quantize if required if gen_dtype != dtype: assert "int" in dtype - assert quant - scale, shift, ty = quant + # assert quant + scale, shift, ty, qrng = quant + if qrng is not None: + assert len(qrng) == 2, "Range should be a tuple (lower, upper)" + lower, upper = qrng + assert lower <= upper + CLIP_INPUTS = True + if CLIP_INPUTS: + arr = np.clip(arr, lower, upper) + else: + assert np.min(arr) >= lower or np.isclose(np.min(arr), lower), "Range missmatch (lower)" + assert np.max(arr) <= upper or np.isclose(np.max(arr), upper), "Range missmatch (upper)" arr = (arr / scale) + shift arr = np.around(arr) arr = arr.astype(dtype) # input("!=") + if rng is not None: + # TODO: Move shared code! + assert len(rng) == 2, "Range should be a tuple (lower, upper)" + lower, upper = rng + assert lower <= upper + CLIP_INPUTS = True + if CLIP_INPUTS: + arr = np.clip(arr, lower, upper) + else: + assert np.min(arr) >= lower or np.isclose(np.min(arr), lower), "Range missmatch (lower)" + assert np.max(arr) <= upper or np.isclose(np.max(arr), upper), "Range missmatch (upper)" data[input_name] = arr inputs_data.append(data) else: @@ -487,6 +515,8 @@ def generate_model_info( output_shapes, input_types, output_types, + input_ranges, + output_ranges, input_quant_details, output_quant_details, ): @@ -497,6 +527,8 @@ def generate_model_info( "output_shapes": list(output_shapes.values()), "input_types": list(input_types.values()), "output_types": list(output_types.values()), + "input_ranges": list(input_ranges.values()), + "output_ranges": list(output_ranges.values()), "input_quant_details": list(input_quant_details.values()), "output_quant_details": list(output_quant_details.values()), } @@ -529,6 +561,8 @@ def process_metadata(self, model, cfg=None): output_shapes = {} input_types = {} output_types = {} + input_ranges = {} + output_ranges = {} input_quant_details = {} output_quant_details = {} if metadata is not None and "network_parameters" in metadata: @@ -541,16 +575,20 @@ def process_metadata(self, model, cfg=None): ty = inp.get("dtype", None) if ty is None: ty = inp.get("type", None) # legacy + rng = inp.get("range", None) quantize = inp.get("quantize", None) if name and shape: input_shapes[name] = shape if name and ty: input_types[name] = ty + if name and rng: + input_ranges[name] = rng if name and quantize: quant_scale = quantize.get("scale", None) quant_zero_shift = quantize.get("zero_shift", None) quant_dtype = quantize.get("dtype", None) - quant_details = [quant_scale, quant_zero_shift, quant_dtype] + quant_range = quantize.get("range", None) + quant_details = [quant_scale, quant_zero_shift, quant_dtype, quant_range] input_quant_details[name] = quant_details if self.use_inout_data or ( self.gen_data and self.gen_data_fill_mode == "file" and self.gen_data_file == "auto" @@ -571,16 +609,20 @@ def process_metadata(self, model, cfg=None): ty = outp.get("dtype", None) if ty is None: ty = outp.get("type", None) # legacy + rng = outp.get("range", None) dequantize = outp.get("dequantize", None) if name and shape: output_shapes[name] = shape if name and ty: output_types[name] = ty + if name and rng: + output_ranges[name] = rng if name and dequantize: quant_scale = dequantize.get("scale", None) quant_zero_shift = dequantize.get("zero_shift", None) quant_dtype = dequantize.get("dtype", None) - quant_details = [quant_scale, quant_zero_shift, quant_dtype] + quant_range = dequantize.get("range", None) + quant_details = [quant_scale, quant_zero_shift, quant_dtype, quant_range] output_quant_details[name] = quant_details if self.use_inout_data or ( self.gen_ref_data and self.gen_ref_data_mode == "file" and self.gen_ref_data_file == "auto" @@ -700,6 +742,8 @@ def process_metadata(self, model, cfg=None): output_shapes, input_types, output_types, + input_ranges, + output_ranges, input_quant_details, output_quant_details, ) @@ -712,7 +756,7 @@ def process_metadata(self, model, cfg=None): artifacts.append(model_info_artifact) if self.gen_data: inputs_data = self.generate_input_data( - input_names, input_types, input_shapes, input_quant_details, in_paths + input_names, input_types, input_shapes, input_ranges, input_quant_details, in_paths ) fmt = self.gen_data_fmt if fmt == "npy": @@ -729,14 +773,14 @@ def process_metadata(self, model, cfg=None): inputs_data_artifact = Artifact(f"inputs.{fmt}", raw=raw, fmt=ArtifactFormat.BIN, flags=("inputs", fmt)) artifacts.append(inputs_data_artifact) if self.gen_ref_data: - outputs_data = self.generate_output_ref_data( + outputs_ref_data = self.generate_output_ref_data( inputs_data, model, out_paths, output_names, output_types, output_shapes, output_quant_details ) fmt = self.gen_data_fmt if fmt == "npy": with tempfile.TemporaryDirectory() as tmpdirname: tempfilename = Path(tmpdirname) / "outputs_ref.npy" - np.save(tempfilename, outputs_data) + np.save(tempfilename, outputs_ref_data) with open(tempfilename, "rb") as f: raw = f.read() elif fmt == "npz": From 9a15a322acee5fb3d75d7565108149818668818f Mon Sep 17 00:00:00 2001 From: Philipp van Kempen Date: Mon, 27 May 2024 17:01:38 +0200 Subject: [PATCH 074/105] frontend: fix assertions --- mlonmcu/models/frontend.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mlonmcu/models/frontend.py b/mlonmcu/models/frontend.py index 33570aa75..00eb531d3 100644 --- a/mlonmcu/models/frontend.py +++ b/mlonmcu/models/frontend.py @@ -406,7 +406,7 @@ def generate_output_ref_data( elif self.gen_ref_data_mode == "file": if self.gen_ref_data_file == "auto": - len(out_paths) > 0 + assert len(out_paths) > 0, "out_paths is empty" if len(out_paths) == 1: if out_paths[0].is_dir(): files = list(out_paths[0].iterdir()) From 86d7aa9fe7d33884c077dbb9ab85ad3923d7bbd3 Mon Sep 17 00:00:00 2001 From: Philipp van Kempen Date: Mon, 27 May 2024 17:06:01 +0200 Subject: [PATCH 075/105] frontends: find handling of aww input shift --- mlonmcu/models/frontend.py | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/mlonmcu/models/frontend.py b/mlonmcu/models/frontend.py index 00eb531d3..7b1f422dc 100644 --- a/mlonmcu/models/frontend.py +++ b/mlonmcu/models/frontend.py @@ -339,8 +339,8 @@ def generate_input_data(self, input_names, input_types, input_shapes, input_rang rng = input_ranges.get(input_name, None) gen_dtype = dtype if quant: - _, _, ty = quant - assert "float" in ty, "Input already quantized?" + _, _, ty, _ = quant + # assert "float" in ty, "Input already quantized?" if NEW: gen_dtype = ty arr = np.frombuffer(temp[i][ii], dtype=gen_dtype) @@ -348,7 +348,8 @@ def generate_input_data(self, input_names, input_types, input_shapes, input_rang shape = input_shapes[input_name] arr = np.reshape(arr, shape) # Quantize if required - if gen_dtype != dtype: + # if gen_dtype != dtype: + if True: assert "int" in dtype # assert quant scale, shift, ty, qrng = quant @@ -446,7 +447,7 @@ def generate_output_ref_data( dtype = output_types[output_name] dequant = output_quant_details.get(output_name, None) if dequant: - _, _, ty = dequant + _, _, ty, _ = dequant dtype = ty arr = np.frombuffer(temp[i][ii], dtype=dtype) assert output_name in output_shapes, f"Unknown shape for output: {output_name}" @@ -482,8 +483,6 @@ def generate_ref_labels( assert len(out_labels_paths) > 0, "labels_paths is empty" assert len(out_labels_paths) == 1 file = Path(out_labels_paths[0]) - # file = f"{out_paths[0]}_labels.csv" - print("file", file) else: assert self.gen_ref_labels_file is not None, "Missing value for gen_ref_labels_file" file = Path(self.gen_ref_labels_file) @@ -496,8 +495,6 @@ def generate_ref_labels( labels_df = pd.read_csv(file, sep=",") assert "i" in labels_df.columns assert "label_idx" in labels_df.columns - print("len(inputs_data)", len(inputs_data)) - print("len(labels_df)", len(labels_df)) assert len(inputs_data) <= len(labels_df) labels_df.sort_values("i", inplace=True) labels = list(labels_df["label_idx"].astype(int))[:len(inputs_data)] From 7b435d9018a08bb87b5e54b49bd22bc1a0531d7a Mon Sep 17 00:00:00 2001 From: Philipp van Kempen Date: Mon, 27 May 2024 17:06:14 +0200 Subject: [PATCH 076/105] introduce new feature: GenRefLabels 3 --- mlonmcu/models/model.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/mlonmcu/models/model.py b/mlonmcu/models/model.py index 3bbd1174c..eafe3aa61 100644 --- a/mlonmcu/models/model.py +++ b/mlonmcu/models/model.py @@ -163,6 +163,7 @@ class Model(Workload): "support_path": None, "inputs_path": None, "outputs_path": None, + "output_labels_path": None, } def __init__(self, name, paths, config=None, alt=None, formats=ModelFormats.TFLITE): @@ -248,6 +249,16 @@ def outputs_path(self): value = Path(value) return value + @property + def output_labels_path(self): + # TODO: fall back to metadata + value = self.config["output_labels_path"] + if value is not None: + if not isinstance(value, Path): + assert isinstance(value, str) + value = Path(value) + return value + @property def skip_check(self): if len(self.formats) == 0: From 916f38a44c37331b35e13c611ab78acf97a79a83 Mon Sep 17 00:00:00 2001 From: Philipp van Kempen Date: Mon, 27 May 2024 17:07:11 +0200 Subject: [PATCH 077/105] introduce validate_labels postprocess --- mlonmcu/session/postprocess/__init__.py | 2 + mlonmcu/session/postprocess/postprocesses.py | 71 ++++++++++++++++++++ 2 files changed, 73 insertions(+) diff --git a/mlonmcu/session/postprocess/__init__.py b/mlonmcu/session/postprocess/__init__.py index c3caef59f..c02fa9400 100644 --- a/mlonmcu/session/postprocess/__init__.py +++ b/mlonmcu/session/postprocess/__init__.py @@ -33,6 +33,7 @@ AnalyseDumpPostprocess, AnalyseCoreVCountsPostprocess, ValidateOutputsPostprocess, + ValidateLabelsPostprocess, ExportOutputsPostprocess, ) @@ -51,5 +52,6 @@ "analyse_dump": AnalyseDumpPostprocess, "analyse_corev_counts": AnalyseCoreVCountsPostprocess, "validate_outputs": ValidateOutputsPostprocess, + "validate_labels": ValidateLabelsPostprocess, "export_outputs": ExportOutputsPostprocess, } diff --git a/mlonmcu/session/postprocess/postprocesses.py b/mlonmcu/session/postprocess/postprocesses.py index 2cacb452e..c726a5cf1 100644 --- a/mlonmcu/session/postprocess/postprocesses.py +++ b/mlonmcu/session/postprocess/postprocesses.py @@ -1596,6 +1596,77 @@ def dequant_helper(quant, data): # TODO: move somewhere else return [] +class ValidateLabelsPostprocess(RunPostprocess): + """Postprocess for comparing model outputs with golden reference.""" + + DEFAULTS = { + **RunPostprocess.DEFAULTS, + "report": False, + "classify_metrics": "topk_label(n=1);topk_label(n=2)", + } + + def __init__(self, features=None, config=None): + super().__init__("validate_labels", features=features, config=config) + + @property + def classify_metrics(self): + """Get classify_metrics property.""" + value = self.config["classify_metrics"] + return value + + @property + def report(self): + """Get report property.""" + value = self.config["report"] + return str2bool(value) if not isinstance(value, (bool, int)) else value + + def post_run(self, report, artifacts): + """Called at the end of a run.""" + model_info_artifact = lookup_artifacts(artifacts, name="model_info.yml", first_only=True) + assert len(model_info_artifact) == 1, "Could not find artifact: model_info.yml" + model_info_artifact = model_info_artifact[0] + import yaml + + model_info_data = yaml.safe_load(model_info_artifact.content) + if len(model_info_data["output_names"]) > 1: + raise NotImplementedError("Multi-outputs not yet supported.") + labels_ref_artifact = lookup_artifacts(artifacts, name="labels_ref.npy", first_only=True) + assert len(labels_ref_artifact) == 1, "Could not find artifact: labels_ref.npy (Run classify_labels postprocess first!)" + labels_ref_artifact = labels_ref_artifact[0] + import numpy as np + + labels_ref = np.load(labels_ref_artifact.path, allow_pickle=True) + outputs_artifact = lookup_artifacts(artifacts, name="outputs.npy", first_only=True) + assert len(outputs_artifact) == 1, "Could not find artifact: outputs.npy" + outputs_artifact = outputs_artifact[0] + outputs = np.load(outputs_artifact.path, allow_pickle=True) + missing = 0 + classify_metrics_str = self.classify_metrics + classify_metrics = parse_classify_metrics(classify_metrics_str) + for i, output in enumerate(outputs): + if isinstance(output, dict): # name based lookup + pass + else: # index based lookup + assert isinstance(output, (list, np.array)), "expected dict, list or np.array" + output_names = model_info_data["output_names"] + assert len(output) == len(output_names) + output = {output_names[idx]: out for idx, out in enumerate(output)} + assert len(output) == 1, "Only supporting single-output models" + out_data = output[list(output.keys())[0]] + # print("out_data", out_data) + assert i < len(labels_ref), "Missing reference labels" + label_ref = labels_ref[i] + # print("label_ref", label_ref) + for cm in classify_metrics: + cm.process(out_data, label_ref, quant=False) + if self.report: + raise NotImplementedError + for cm in classify_metrics: + res = cm.get_summary() + report.post_df[f"{cm.name}"] = res + return [] + + class ExportOutputsPostprocess(RunPostprocess): """Postprocess for writing model outputs to a directory.""" From 79baeda3f458a55a13247e95f4cef9c03d5bc7fd Mon Sep 17 00:00:00 2001 From: Philipp van Kempen Date: Mon, 27 May 2024 17:07:46 +0200 Subject: [PATCH 078/105] introduce classify metrics --- mlonmcu/session/postprocess/postprocesses.py | 2 +- .../session/postprocess/validate_metrics.py | 131 ++++++++++++++++-- 2 files changed, 118 insertions(+), 15 deletions(-) diff --git a/mlonmcu/session/postprocess/postprocesses.py b/mlonmcu/session/postprocess/postprocesses.py index c726a5cf1..f2292d2ef 100644 --- a/mlonmcu/session/postprocess/postprocesses.py +++ b/mlonmcu/session/postprocess/postprocesses.py @@ -33,7 +33,7 @@ from mlonmcu.logging import get_logger from .postprocess import SessionPostprocess, RunPostprocess -from .validate_metrics import parse_validate_metrics +from .validate_metrics import parse_validate_metrics, parse_classify_metrics logger = get_logger() diff --git a/mlonmcu/session/postprocess/validate_metrics.py b/mlonmcu/session/postprocess/validate_metrics.py index 615b6ee2f..e56656daa 100644 --- a/mlonmcu/session/postprocess/validate_metrics.py +++ b/mlonmcu/session/postprocess/validate_metrics.py @@ -52,6 +52,32 @@ def get_summary(self): return f"{self.num_correct}/{self.num_total} ({int(self.num_correct/self.num_total*100)}%)" +class ClassifyMetric: + + def __init__(self, name, **cfg): + self.name = name + self.num_total = 0 + self.num_correct = 0 + + def process_(self, out_data, label_ref, quant: bool = False): + raise NotImplementedError + + def check(self, out_data, label_ref, quant: bool = False): + return out_data.dtype == out_data_ref.dtype + + def process(self, out_data, label_ref, quant: bool = False): + if not self.check(out_data, label_ref, quant=quant): + return + self.num_total += 1 + if self.process_(out_data, label_ref): + self.num_correct += 1 + + def get_summary(self): + if self.num_total == 0: + return "N/A" + return f"{self.num_correct}/{self.num_total} ({int(self.num_correct/self.num_total*100)}%)" + + class AllCloseMetric(ValidationMetric): def __init__(self, name: str, atol: float = 0.0, rtol: float = 0.0): @@ -88,28 +114,16 @@ def process_(self, out_data, out_data_ref, quant: bool = False): k = 0 num_checks = min(self.n, len(data_sorted_idx)) assert len(data_sorted_idx) == len(ref_data_sorted_idx) - # print("data_sorted_idx", data_sorted_idx, type(data_sorted_idx)) - # print("ref_data_sorted_idx", ref_data_sorted_idx, type(ref_data_sorted_idx)) - # print("num_checks", num_checks) for j in range(num_checks): - # print("j", j) - # print(f"data_sorted_idx[{j}]", data_sorted_idx[j], type(data_sorted_idx[j])) idx = data_sorted_idx[j] - # print("idx", idx) ref_idx = ref_data_sorted_idx[j] - # print("ref_idx", ref_idx) if idx == ref_idx: - # print("IF") k += 1 else: - # print("ELSE") if out_data.tolist()[0][idx] == out_data_ref.tolist()[0][ref_idx]: - # print("SAME") k += 1 else: - # print("BREAK") break - # print("k", k) if k < num_checks: return False elif k == num_checks: @@ -118,12 +132,87 @@ def process_(self, out_data, out_data_ref, quant: bool = False): assert False +class TopKLabelsMetric(ClassifyMetric): + + def __init__(self, name: str, n: int = 2): + super().__init__(name) + assert n >= 1 + self.n = n + + def check(self, out_data, label_ref, quant: bool = False): + data_len = len(out_data.flatten().tolist()) + # Probably no classification + return data_len < 25 + + def process_(self, out_data, label_ref, quant: bool = False): + # print("process_") + # print("out_data", out_data) + # print("label_ref", label_ref) + data_sorted_idx = list(reversed(np.argsort(out_data).tolist()[0])) + # print("data_sorted_idx", data_sorted_idx) + data_sorted_idx_trunc = data_sorted_idx[:self.n] + # print("data_sorted_idx_trunc", data_sorted_idx_trunc) + res = label_ref in data_sorted_idx_trunc + # print("res", res) + # TODO: handle same values? + # input("111") + return res + + +class ConfusionMatrixMetric(ValidationMetric): + + def __init__(self, name: str): + super().__init__(name) + self.temp = {} + self.num_correct_per_class = {} + + def check(self, out_data, label_ref, quant: bool = False): + data_len = len(out_data.flatten().tolist()) + # Probably no classification + return data_len < 25 and not quant + + def process_(self, out_data, label_ref, quant: bool = False): + data_sorted_idx = list(reversed(np.argsort(out_data).tolist()[0])) + label = data_sorted_idx[0] + correct = label_ref == label + # TODO: handle same values? + return correct, label + + def process(self, out_data, label_ref, quant: bool = False): + print("ConfusionMatrixMetric.process") + if not self.check(out_data, label_ref, quant=quant): + return + self.num_total += 1 + correct, label = self.process_(out_data, label_ref) + if correct: + self.num_correct += 1 + if label_ref not in self.num_correct_per_class: + self.num_correct_per_class[label_ref] = 0 + self.num_correct_per_class[label_ref] += 1 + temp_ = self.temp.get(label_ref, {}) + if label not in temp_: + temp_[label] = 0 + temp_[label] += 1 + self.temp[label_ref] = temp_ + + def get_summary(self): + if self.num_total == 0: + return "N/A" + return f"{self.temp}" + + class AccuracyMetric(TopKMetric): def __init__(self, name: str): super().__init__(name, n=1) +class AccuracyLabelsMetric(TopKLabelsMetric): + + def __init__(self, name: str): + super().__init__(name, n=1) + + class MSEMetric(ValidationMetric): def __init__(self, name: str, thr: int = 0.5): @@ -197,6 +286,12 @@ def process_(self, out_data, out_data_ref, quant: bool = False): "pm1": PlusMinusOneMetric, } +LABELS_LOOKUP = { + "topk_label": TopKLabelsMetric, + "acc_label": AccuracyLabelsMetric, + "confusion_matrix": ConfusionMatrixMetric, +} + def parse_validate_metric_args(inp): ret = {} @@ -212,7 +307,7 @@ def parse_validate_metric_args(inp): return ret -def parse_validate_metric(inp): +def parse_validate_metric(inp, lookup=LOOKUP): if "(" in inp: metric_name, inp_ = inp.split("(") assert inp_[-1] == ")" @@ -221,7 +316,7 @@ def parse_validate_metric(inp): else: metric_name = inp metric_args = {} - metric_cls = LOOKUP.get(metric_name, None) + metric_cls = lookup.get(metric_name, None) assert metric_cls is not None, f"Validate metric not found: {metric_name}" metric = metric_cls(inp, **metric_args) return metric @@ -233,3 +328,11 @@ def parse_validate_metrics(inp): metric = parse_validate_metric(metric_str) ret.append(metric) return ret + + +def parse_classify_metrics(inp): + ret = [] + for metric_str in inp.split(";"): + metric = parse_validate_metric(metric_str, lookup=LABELS_LOOKUP) + ret.append(metric) + return ret From 852736e9dda661652015a99a77e9290d20631267 Mon Sep 17 00:00:00 2001 From: Philipp van Kempen Date: Mon, 27 May 2024 17:09:01 +0200 Subject: [PATCH 079/105] validate_outputs: add validate_range option (on by default) 2 --- mlonmcu/session/postprocess/postprocesses.py | 44 ++++++++++++++++++-- 1 file changed, 40 insertions(+), 4 deletions(-) diff --git a/mlonmcu/session/postprocess/postprocesses.py b/mlonmcu/session/postprocess/postprocesses.py index f2292d2ef..1782cb16b 100644 --- a/mlonmcu/session/postprocess/postprocesses.py +++ b/mlonmcu/session/postprocess/postprocesses.py @@ -1467,6 +1467,7 @@ class ValidateOutputsPostprocess(RunPostprocess): **RunPostprocess.DEFAULTS, "report": False, "validate_metrics": "topk(n=1);topk(n=2)", + "validate_range": True, } def __init__(self, features=None, config=None): @@ -1484,6 +1485,12 @@ def report(self): value = self.config["report"] return str2bool(value) if not isinstance(value, (bool, int)) else value + @property + def validate_range(self): + """Get validate_range property.""" + value = self.config["validate_range"] + return str2bool(value) if not isinstance(value, (bool, int)) else value + def post_run(self, report, artifacts): """Called at the end of a run.""" model_info_artifact = lookup_artifacts(artifacts, name="model_info.yml", first_only=True) @@ -1549,27 +1556,56 @@ def post_run(self, report, artifacts): # print("sum(out_data_before_quant", np.sum(out_data)) quant = model_info_data.get("output_quant_details", None) + rng = model_info_data.get("output_ranges", None) if quant: def ref_quant_helper(quant, data): # TODO: move somewhere else if quant is None: return data - quant_scale, quant_zero_point, quant_dtype = quant + quant_scale, quant_zero_point, quant_dtype, quant_range = quant if quant_dtype is None or data.dtype.name == quant_dtype: return data assert data.dtype.name in ["float32"], "Quantization only supported for float32 input" assert quant_dtype in ["int8"], "Quantization only supported for int8 output" + if quant_range and self.validate_range: + assert len(quant_range) == 2, "Range should be a tuple (lower, upper)" + lower, upper = quant_range + # print("quant_range", quant_range) + # print("np.min(data)", np.min(data)) + # print("np.max(data)", np.max(data)) + assert lower <= upper + assert np.min(data) >= lower and np.max(data) <= upper, "Range missmatch" + return np.around((data / quant_scale) + quant_zero_point).astype( "int8" ) def dequant_helper(quant, data): # TODO: move somewhere else if quant is None: return data - quant_scale, quant_zero_point, quant_dtype = quant + quant_scale, quant_zero_point, quant_dtype, quant_range = quant if quant_dtype is None or data.dtype.name == quant_dtype: return data assert data.dtype.name in ["int8"], "Dequantization only supported for int8 input" assert quant_dtype in ["float32"], "Dequantization only supported for float32 output" - return (data.astype("float32") - quant_zero_point) * quant_scale + ret = (data.astype("float32") - quant_zero_point) * quant_scale + if quant_range and self.validate_range: + assert len(quant_range) == 2, "Range should be a tuple (lower, upper)" + # print("quant_range", quant_range) + # print("np.min(ret)", np.min(ret)) + # print("np.max(ret)", np.max(ret)) + lower, upper = quant_range + assert lower <= upper + assert np.min(ret) >= lower and np.max(ret) <= upper, "Range missmatch" + return ret + assert ii < len(rng) + rng_ = rng[ii] + if rng_ and self.validate_range: + assert len(rng_) == 2, "Range should be a tuple (lower, upper)" + lower, upper = rng_ + assert lower <= upper + # print("rng_", rng_) + # print("np.min(out_data)", np.min(out_data)) + # print("np.max(out_data)", np.max(out_data)) + assert np.min(out_data) >= lower and np.max(out_data) <= upper, "Range missmatch" assert ii < len(quant) quant_ = quant[ii] if quant_ is not None: @@ -1758,7 +1794,7 @@ def post_run(self, report, artifacts): def dequant_helper(quant, data): if quant is None: return data - quant_scale, quant_zero_point, quant_dtype = quant + quant_scale, quant_zero_point, quant_dtype, quant_range = quant if quant_dtype is None or data.dtype.name == quant_dtype: return data assert data.dtype.name in ["int8"], "Dequantization only supported for int8 input" From 65499caa52c3c3c86e6b73d53e049a11034761f6 Mon Sep 17 00:00:00 2001 From: Philipp van Kempen Date: Mon, 27 May 2024 17:10:05 +0200 Subject: [PATCH 080/105] run: add error message if not frontend can be initialized --- mlonmcu/session/run.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/mlonmcu/session/run.py b/mlonmcu/session/run.py index ebb84d100..e3e64bdb2 100644 --- a/mlonmcu/session/run.py +++ b/mlonmcu/session/run.py @@ -505,15 +505,22 @@ def add_frontend_by_name(self, frontend_name, context=None): def add_frontends_by_name(self, frontend_names, context=None): """Helper function to initialize and configure frontends by their names.""" frontends = [] + reasons = {} for name in frontend_names: try: assert context is not None and context.environment.has_frontend( name ), f"The frontend '{name}' is not enabled for this environment" frontends.append(self.init_component(SUPPORTED_FRONTENDS[name], context=context)) - except Exception: + except Exception as e: + reasons[name] = str(e) continue assert len(frontends) > 0, "No compatible frontend was found" + if len(frontends) == 0: + if reasons: + logger.error("Initialization of frontends was no successfull. Reasons: %s", reasons) + else: + raise RuntimeError(f"No compatible frontend was found.") self.add_frontends(frontends) def add_backend_by_name(self, backend_name, context=None): From cbff929e1445a74b825e692a567b8d1041773c19 Mon Sep 17 00:00:00 2001 From: Philipp van Kempen Date: Tue, 28 May 2024 13:38:16 +0200 Subject: [PATCH 081/105] update validate_metrics --- mlonmcu/session/postprocess/postprocesses.py | 5 ++-- .../session/postprocess/validate_metrics.py | 29 +++++++++++-------- 2 files changed, 20 insertions(+), 14 deletions(-) diff --git a/mlonmcu/session/postprocess/postprocesses.py b/mlonmcu/session/postprocess/postprocesses.py index 1782cb16b..1135f1398 100644 --- a/mlonmcu/session/postprocess/postprocesses.py +++ b/mlonmcu/session/postprocess/postprocesses.py @@ -1514,6 +1514,7 @@ def post_run(self, report, artifacts): assert len(outputs_artifact) == 1, "Could not find artifact: outputs.npy" outputs_artifact = outputs_artifact[0] outputs = np.load(outputs_artifact.path, allow_pickle=True) + in_data = None # compared = 0 # matching = 0 missing = 0 @@ -1611,7 +1612,7 @@ def dequant_helper(quant, data): # TODO: move somewhere else if quant_ is not None: out_ref_data_quant = ref_quant_helper(quant_, out_ref_data) for vm in validate_metrics: - vm.process(out_data, out_ref_data_quant, quant=True) + vm.process(out_data, out_ref_data_quant, in_data=in_data, quant=True) out_data = dequant_helper(quant_, out_data) # print("out_data", out_data) # print("sum(out_data)", np.sum(out_data)) @@ -1622,7 +1623,7 @@ def dequant_helper(quant, data): # TODO: move somewhere else assert out_data.shape == out_ref_data.shape, "shape missmatch" for vm in validate_metrics: - vm.process(out_data, out_ref_data, quant=False) + vm.process(out_data, out_ref_data, in_data=in_data, quant=False) ii += 1 if self.report: raise NotImplementedError diff --git a/mlonmcu/session/postprocess/validate_metrics.py b/mlonmcu/session/postprocess/validate_metrics.py index e56656daa..361444502 100644 --- a/mlonmcu/session/postprocess/validate_metrics.py +++ b/mlonmcu/session/postprocess/validate_metrics.py @@ -20,6 +20,7 @@ import ast import numpy as np +from typing import Optional from mlonmcu.logging import get_logger @@ -33,13 +34,13 @@ def __init__(self, name, **cfg): self.num_total = 0 self.num_correct = 0 - def process_(self, out_data, out_data_ref, quant: bool = False): + def process_(self, out_data, out_data_ref, in_data: Optional[np.array] = None, quant: bool = False): raise NotImplementedError def check(self, out_data, out_data_ref, quant: bool = False): return out_data.dtype == out_data_ref.dtype - def process(self, out_data, out_data_ref, quant: bool = False): + def process(self, out_data, out_data_ref, in_data: Optional[np.array] = None, quant: bool = False): if not self.check(out_data, out_data_ref, quant=quant): return self.num_total += 1 @@ -90,7 +91,7 @@ def __init__(self, name: str, atol: float = 0.0, rtol: float = 0.0): def check(self, out_data, out_data_ref, quant: bool = False): return not quant - def process_(self, out_data, out_data_ref, quant: bool = False): + def process_(self, out_data, out_data_ref, in_data: Optional[np.array] = None, quant: bool = False): return np.allclose(out_data, out_data_ref, rtol=self.rtol, atol=self.atol) @@ -106,7 +107,7 @@ def check(self, out_data, out_data_ref, quant: bool = False): # Probably no classification return data_len < 25 and not quant - def process_(self, out_data, out_data_ref, quant: bool = False): + def process_(self, out_data, out_data_ref, in_data: Optional[np.array] = None, quant: bool = False): # TODO: only for classification models! # TODO: support multi_outputs? data_sorted_idx = list(reversed(np.argsort(out_data).tolist()[0])) @@ -220,7 +221,7 @@ def __init__(self, name: str, thr: int = 0.5): assert thr >= 0 self.thr = thr - def process_(self, out_data, out_data_ref, quant: bool = False): + def process_(self, out_data, out_data_ref, in_data: Optional[np.array] = None, quant: bool = False): mse = ((out_data - out_data_ref) ** 2).mean() return mse < self.thr @@ -238,15 +239,19 @@ def check(self, out_data, out_data_ref, quant: bool = False): data_len = len(out_data.flatten().tolist()) return data_len == 640 and not quant - def process_(self, out_data, out_data_ref, quant: bool = False): - data_flat = out_data.flatten().tolist() - ref_data_flat = out_data_ref.flatten().tolist() + def process_(self, out_data, out_data_ref, in_data: Optional[np.array] = None, quant: bool = False): + assert in_data is not None + in_data_flat = in_data.flatten().tolist() + out_data_flat = out_data.flatten().tolist() + ref_out_data_flat = out_data_ref.flatten().tolist() res = 0 ref_res = 0 - length = len(data_flat) + length = len(out_data_flat) for jjj in range(length): - res += data_flat[jjj] ** 2 - ref_res += ref_data_flat[jjj] ** 2 + res = (in_data_flat[jjj] - out_data_flat[jjj]) + res += res ** 2 + ref_res = (in_data_flat[jjj] - ref_out_data_flat[jjj]) + ref_res += ref_res ** 2 res /= length ref_res /= length print("res", res) @@ -262,7 +267,7 @@ def __init__(self, name: str): def check(self, out_data, out_data_ref, quant: bool = False): return "int" in out_data.dtype.str - def process_(self, out_data, out_data_ref, quant: bool = False): + def process_(self, out_data, out_data_ref, in_data: Optional[np.array] = None, quant: bool = False): data_ = out_data.flatten().tolist() ref_data_ = out_data_ref.flatten().tolist() From cbc5a158107aa8cb498f6120c379918bc5bfe70f Mon Sep 17 00:00:00 2001 From: Philipp van Kempen Date: Wed, 29 May 2024 10:56:52 +0200 Subject: [PATCH 082/105] lint code --- mlonmcu/models/frontend.py | 24 +++++++++++++++---- mlonmcu/platform/mlif/interfaces.py | 1 - mlonmcu/session/postprocess/postprocesses.py | 18 ++++++++++---- .../session/postprocess/validate_metrics.py | 21 ++++------------ mlonmcu/setup/utils.py | 3 ++- 5 files changed, 39 insertions(+), 28 deletions(-) diff --git a/mlonmcu/models/frontend.py b/mlonmcu/models/frontend.py index 7b1f422dc..2f07a27ef 100644 --- a/mlonmcu/models/frontend.py +++ b/mlonmcu/models/frontend.py @@ -361,8 +361,12 @@ def generate_input_data(self, input_names, input_types, input_shapes, input_rang if CLIP_INPUTS: arr = np.clip(arr, lower, upper) else: - assert np.min(arr) >= lower or np.isclose(np.min(arr), lower), "Range missmatch (lower)" - assert np.max(arr) <= upper or np.isclose(np.max(arr), upper), "Range missmatch (upper)" + assert np.min(arr) >= lower or np.isclose( + np.min(arr), lower + ), "Range missmatch (lower)" + assert np.max(arr) <= upper or np.isclose( + np.max(arr), upper + ), "Range missmatch (upper)" arr = (arr / scale) + shift arr = np.around(arr) arr = arr.astype(dtype) @@ -414,7 +418,9 @@ def generate_output_ref_data( else: files = out_paths temp = {} - assert len(inputs_data) <= len(files), f"Missing output data for provided inputs. (Expected: {len(inputs_data)}, Got: {len(files)})" + assert len(inputs_data) <= len( + files + ), f"Missing output data for provided inputs. (Expected: {len(inputs_data)}, Got: {len(files)})" for file in files: if not isinstance(file, Path): file = Path(file) @@ -492,12 +498,13 @@ def generate_ref_labels( fmt = ext[1:].lower() if fmt == "csv": import pandas as pd + labels_df = pd.read_csv(file, sep=",") assert "i" in labels_df.columns assert "label_idx" in labels_df.columns assert len(inputs_data) <= len(labels_df) labels_df.sort_values("i", inplace=True) - labels = list(labels_df["label_idx"].astype(int))[:len(inputs_data)] + labels = list(labels_df["label_idx"].astype(int))[: len(inputs_data)] else: raise NotImplementedError(f"Fmt not supported: {fmt}") else: @@ -914,7 +921,14 @@ def produce_artifacts(self, model): # TODO: frontend parsed metadata instead of lookup.py? # TODO: how to find inout_data? class TfLiteFrontend(SimpleFrontend): - FEATURES = Frontend.FEATURES | {"visualize", "split_layers", "tflite_analyze", "gen_data", "gen_ref_data", "gen_ref_labels"} + FEATURES = Frontend.FEATURES | { + "visualize", + "split_layers", + "tflite_analyze", + "gen_data", + "gen_ref_data", + "gen_ref_labels", + } DEFAULTS = { **Frontend.DEFAULTS, diff --git a/mlonmcu/platform/mlif/interfaces.py b/mlonmcu/platform/mlif/interfaces.py index 0b06d5aa6..a54cc3ec3 100644 --- a/mlonmcu/platform/mlif/interfaces.py +++ b/mlonmcu/platform/mlif/interfaces.py @@ -160,7 +160,6 @@ def get_process_outputs_stdout_raw(): class ModelSupport: - def __init__(self, in_interface, out_interface, model_info, target=None, batch_size=None, inputs_data=None): self.model_info = model_info self.target = target diff --git a/mlonmcu/session/postprocess/postprocesses.py b/mlonmcu/session/postprocess/postprocesses.py index 1135f1398..28e921cb4 100644 --- a/mlonmcu/session/postprocess/postprocesses.py +++ b/mlonmcu/session/postprocess/postprocesses.py @@ -1559,6 +1559,7 @@ def post_run(self, report, artifacts): quant = model_info_data.get("output_quant_details", None) rng = model_info_data.get("output_ranges", None) if quant: + def ref_quant_helper(quant, data): # TODO: move somewhere else if quant is None: return data @@ -1576,9 +1577,8 @@ def ref_quant_helper(quant, data): # TODO: move somewhere else assert lower <= upper assert np.min(data) >= lower and np.max(data) <= upper, "Range missmatch" - return np.around((data / quant_scale) + quant_zero_point).astype( - "int8" - ) + return np.around((data / quant_scale) + quant_zero_point).astype("int8") + def dequant_helper(quant, data): # TODO: move somewhere else if quant is None: return data @@ -1597,6 +1597,7 @@ def dequant_helper(quant, data): # TODO: move somewhere else assert lower <= upper assert np.min(ret) >= lower and np.max(ret) <= upper, "Range missmatch" return ret + assert ii < len(rng) rng_ = rng[ii] if rng_ and self.validate_range: @@ -1668,7 +1669,9 @@ def post_run(self, report, artifacts): if len(model_info_data["output_names"]) > 1: raise NotImplementedError("Multi-outputs not yet supported.") labels_ref_artifact = lookup_artifacts(artifacts, name="labels_ref.npy", first_only=True) - assert len(labels_ref_artifact) == 1, "Could not find artifact: labels_ref.npy (Run classify_labels postprocess first!)" + assert ( + len(labels_ref_artifact) == 1 + ), "Could not find artifact: labels_ref.npy (Run classify_labels postprocess first!)" labels_ref_artifact = labels_ref_artifact[0] import numpy as np @@ -1792,6 +1795,7 @@ def post_run(self, report, artifacts): output = {output_names[idx]: out for idx, out in enumerate(output)} quant = model_info_data.get("output_quant_details", None) if quant and not self.skip_dequant: + def dequant_helper(quant, data): if quant is None: return data @@ -1801,7 +1805,10 @@ def dequant_helper(quant, data): assert data.dtype.name in ["int8"], "Dequantization only supported for int8 input" assert quant_dtype in ["float32"], "Dequantization only supported for float32 output" return (data.astype("float32") - quant_zero_point) * quant_scale - output = {out_name: dequant_helper(quant[j], output[out_name]) for j, out_name in enumerate(output.keys())} + + output = { + out_name: dequant_helper(quant[j], output[out_name]) for j, out_name in enumerate(output.keys()) + } if self.fmt == "npy": raise NotImplementedError("npy export") elif self.fmt == "bin": @@ -1827,6 +1834,7 @@ def dequant_helper(quant, data): archive_path = f"{dest_}.{archive_fmt}" if archive_fmt == "tar.gz": import tarfile + with tarfile.open(archive_path, "w:gz") as tar: for filename in filenames: tar.add(filename, arcname=filename.name) diff --git a/mlonmcu/session/postprocess/validate_metrics.py b/mlonmcu/session/postprocess/validate_metrics.py index 361444502..219fc38e9 100644 --- a/mlonmcu/session/postprocess/validate_metrics.py +++ b/mlonmcu/session/postprocess/validate_metrics.py @@ -28,7 +28,6 @@ class ValidationMetric: - def __init__(self, name, **cfg): self.name = name self.num_total = 0 @@ -54,7 +53,6 @@ def get_summary(self): class ClassifyMetric: - def __init__(self, name, **cfg): self.name = name self.num_total = 0 @@ -80,7 +78,6 @@ def get_summary(self): class AllCloseMetric(ValidationMetric): - def __init__(self, name: str, atol: float = 0.0, rtol: float = 0.0): super().__init__(name) assert atol >= 0 @@ -96,7 +93,6 @@ def process_(self, out_data, out_data_ref, in_data: Optional[np.array] = None, q class TopKMetric(ValidationMetric): - def __init__(self, name: str, n: int = 2): super().__init__(name) assert n >= 1 @@ -134,7 +130,6 @@ def process_(self, out_data, out_data_ref, in_data: Optional[np.array] = None, q class TopKLabelsMetric(ClassifyMetric): - def __init__(self, name: str, n: int = 2): super().__init__(name) assert n >= 1 @@ -151,7 +146,7 @@ def process_(self, out_data, label_ref, quant: bool = False): # print("label_ref", label_ref) data_sorted_idx = list(reversed(np.argsort(out_data).tolist()[0])) # print("data_sorted_idx", data_sorted_idx) - data_sorted_idx_trunc = data_sorted_idx[:self.n] + data_sorted_idx_trunc = data_sorted_idx[: self.n] # print("data_sorted_idx_trunc", data_sorted_idx_trunc) res = label_ref in data_sorted_idx_trunc # print("res", res) @@ -161,7 +156,6 @@ def process_(self, out_data, label_ref, quant: bool = False): class ConfusionMatrixMetric(ValidationMetric): - def __init__(self, name: str): super().__init__(name) self.temp = {} @@ -203,19 +197,16 @@ def get_summary(self): class AccuracyMetric(TopKMetric): - def __init__(self, name: str): super().__init__(name, n=1) class AccuracyLabelsMetric(TopKLabelsMetric): - def __init__(self, name: str): super().__init__(name, n=1) class MSEMetric(ValidationMetric): - def __init__(self, name: str, thr: int = 0.5): super().__init__(name) assert thr >= 0 @@ -227,7 +218,6 @@ def process_(self, out_data, out_data_ref, in_data: Optional[np.array] = None, q class ToyScoreMetric(ValidationMetric): - def __init__(self, name: str, atol: float = 0.1, rtol: float = 0.1): super().__init__(name) assert atol >= 0 @@ -248,10 +238,10 @@ def process_(self, out_data, out_data_ref, in_data: Optional[np.array] = None, q ref_res = 0 length = len(out_data_flat) for jjj in range(length): - res = (in_data_flat[jjj] - out_data_flat[jjj]) - res += res ** 2 - ref_res = (in_data_flat[jjj] - ref_out_data_flat[jjj]) - ref_res += ref_res ** 2 + res = in_data_flat[jjj] - out_data_flat[jjj] + res += res**2 + ref_res = in_data_flat[jjj] - ref_out_data_flat[jjj] + ref_res += ref_res**2 res /= length ref_res /= length print("res", res) @@ -260,7 +250,6 @@ def process_(self, out_data, out_data_ref, in_data: Optional[np.array] = None, q class PlusMinusOneMetric(ValidationMetric): - def __init__(self, name: str): super().__init__(name) diff --git a/mlonmcu/setup/utils.py b/mlonmcu/setup/utils.py index 5a5c7af0e..5b22a0fa8 100644 --- a/mlonmcu/setup/utils.py +++ b/mlonmcu/setup/utils.py @@ -276,7 +276,8 @@ def execute( else: try: p = subprocess.Popen( - [i for i in args], **kwargs, stdout=subprocess.PIPE, stdin=subprocess.PIPE, stderr=subprocess.STDOUT) + [i for i in args], **kwargs, stdout=subprocess.PIPE, stdin=subprocess.PIPE, stderr=subprocess.STDOUT + ) if stdin_data: out_str = p.communicate(input=stdin_data)[0] else: From 73e945b9d99f2410167189d2c904c4b14de7baef Mon Sep 17 00:00:00 2001 From: Philipp van Kempen Date: Wed, 29 May 2024 10:58:48 +0200 Subject: [PATCH 083/105] lint code 2 --- mlonmcu/session/postprocess/postprocesses.py | 7 +++---- mlonmcu/session/postprocess/validate_metrics.py | 2 +- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/mlonmcu/session/postprocess/postprocesses.py b/mlonmcu/session/postprocess/postprocesses.py index 28e921cb4..e395c37d6 100644 --- a/mlonmcu/session/postprocess/postprocesses.py +++ b/mlonmcu/session/postprocess/postprocesses.py @@ -21,7 +21,6 @@ import re import ast import tempfile -import tarfile from pathlib import Path from io import StringIO @@ -1517,7 +1516,7 @@ def post_run(self, report, artifacts): in_data = None # compared = 0 # matching = 0 - missing = 0 + # missing = 0 # metrics = { # "allclose(atol=0.0,rtol=0.0)": None, # "allclose(atol=0.05,rtol=0.05)": None, @@ -1536,7 +1535,7 @@ def post_run(self, report, artifacts): for i, output_ref in enumerate(outputs_ref): if i >= len(outputs): logger.warning("Missing output sample") - missing += 1 + # missing += 1 break output = outputs[i] ii = 0 @@ -1680,7 +1679,7 @@ def post_run(self, report, artifacts): assert len(outputs_artifact) == 1, "Could not find artifact: outputs.npy" outputs_artifact = outputs_artifact[0] outputs = np.load(outputs_artifact.path, allow_pickle=True) - missing = 0 + # missing = 0 classify_metrics_str = self.classify_metrics classify_metrics = parse_classify_metrics(classify_metrics_str) for i, output in enumerate(outputs): diff --git a/mlonmcu/session/postprocess/validate_metrics.py b/mlonmcu/session/postprocess/validate_metrics.py index 219fc38e9..35b8db3fe 100644 --- a/mlonmcu/session/postprocess/validate_metrics.py +++ b/mlonmcu/session/postprocess/validate_metrics.py @@ -62,7 +62,7 @@ def process_(self, out_data, label_ref, quant: bool = False): raise NotImplementedError def check(self, out_data, label_ref, quant: bool = False): - return out_data.dtype == out_data_ref.dtype + return True def process(self, out_data, label_ref, quant: bool = False): if not self.check(out_data, label_ref, quant=quant): From ed66f54a7b1c6a7a001914a2543dcad9870b98a1 Mon Sep 17 00:00:00 2001 From: Philipp van Kempen Date: Sat, 25 May 2024 08:23:39 +0200 Subject: [PATCH 084/105] t# This is a combination of 2 commits. etiss: enable simple_mem_system.error_on_invalid_access etiss: error_on_invalid_access fix --- mlonmcu/target/riscv/etiss.py | 1 + 1 file changed, 1 insertion(+) diff --git a/mlonmcu/target/riscv/etiss.py b/mlonmcu/target/riscv/etiss.py index ad3fc6b4e..70ccc679b 100644 --- a/mlonmcu/target/riscv/etiss.py +++ b/mlonmcu/target/riscv/etiss.py @@ -335,6 +335,7 @@ def max_block_size(self): def get_ini_bool_config(self): ret = { "arch.enable_semihosting": True, + "simple_mem_system.error_on_invalid_access": not self.allow_error, } ret.update(self.extra_string_config) return ret From 7cde2378d3aedddbbce1bb1326fe90fdd4cb71e5 Mon Sep 17 00:00:00 2001 From: Philipp van Kempen Date: Sat, 25 May 2024 08:25:50 +0200 Subject: [PATCH 085/105] etiss: add comments on simple_mem_system.memseg_mode_* --- mlonmcu/target/riscv/etiss.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/mlonmcu/target/riscv/etiss.py b/mlonmcu/target/riscv/etiss.py index 70ccc679b..26a08cf7f 100644 --- a/mlonmcu/target/riscv/etiss.py +++ b/mlonmcu/target/riscv/etiss.py @@ -343,6 +343,9 @@ def get_ini_bool_config(self): def get_ini_string_config(self): ret = { "arch.cpu": self.cpu_arch, + # Mode will be overwritten by elf... + # "simple_mem_system.memseg_mode_00": "RX", + # "simple_mem_system.memseg_mode_01": "RWX", } if self.jit is not None: ret["jit.type"] = f"{self.jit}JIT" From 5bb56009ca3a635977fef868c7cb7968d9aa0dd1 Mon Sep 17 00:00:00 2001 From: Philipp van Kempen Date: Sat, 25 May 2024 08:51:03 +0200 Subject: [PATCH 086/105] etiss: move rom start --- mlonmcu/target/riscv/etiss.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mlonmcu/target/riscv/etiss.py b/mlonmcu/target/riscv/etiss.py index 26a08cf7f..4e0275958 100644 --- a/mlonmcu/target/riscv/etiss.py +++ b/mlonmcu/target/riscv/etiss.py @@ -62,9 +62,9 @@ class EtissTarget(RISCVTarget): "plugins": [], "verbose": False, "cpu_arch": None, - "rom_start": 0x0, + "rom_start": 0x1000000, "rom_size": 0x800000, # 8 MB - "ram_start": 0x800000, + "ram_start": 0x1000000 + 0x800000, "ram_size": 0x4000000, # 64 MB "cycle_time_ps": 31250, # 32 MHz "enable_vext": False, From 37c54be80f46a9d4770e3c6909f681c87ee52430 Mon Sep 17 00:00:00 2001 From: Philipp van Kempen Date: Wed, 29 May 2024 14:48:12 +0200 Subject: [PATCH 087/105] etiss: add option for running etiss directly, e.g. without run_helper.sh --- mlonmcu/target/riscv/etiss.py | 103 ++++++++++++++++++---------------- 1 file changed, 56 insertions(+), 47 deletions(-) diff --git a/mlonmcu/target/riscv/etiss.py b/mlonmcu/target/riscv/etiss.py index 4e0275958..ac2e8b264 100644 --- a/mlonmcu/target/riscv/etiss.py +++ b/mlonmcu/target/riscv/etiss.py @@ -88,6 +88,7 @@ class EtissTarget(RISCVTarget): "extra_bool_config": {}, "extra_string_config": {}, "extra_plugin_config": {}, + "use_run_helper": True, } REQUIRED = RISCVTarget.REQUIRED | {"etiss.src_dir", "etiss.install_dir", "etissvp.script"} @@ -312,6 +313,11 @@ def allow_error(self): value = self.config["allow_error"] return str2bool(value) if not isinstance(value, (bool, int)) else value + @property + def use_run_helper(self): + value = self.config["use_run_helper"] + return str2bool(value) if not isinstance(value, (bool, int)) else value + @property def vext_spec(self): return float(self.config["vext_spec"]) @@ -420,54 +426,57 @@ def write_ini(self, path): def exec(self, program, *args, cwd=os.getcwd(), **kwargs): """Use target to execute a executable with given arguments""" - etiss_script_args = [] - if len(self.extra_args) > 0: - etiss_script_args.extend(self.extra_args.split(" ")) - - # TODO: this is outdated - # TODO: validate features (attach xor noattach!) - if self.debug_etiss: - etiss_script_args.append("gdb") - if self.gdbserver_enable: - etiss_script_args.append("tgdb") - if not self.gdbserver_attach: - etiss_script_args.append("noattach") - if self.trace_memory: - etiss_script_args.append("trace") - etiss_script_args.append("nodmi") - if self.verbose: - etiss_script_args.append("v") - # Alternative to stdout parsing: etiss_script_args.append("--vp.stats_file_path=stats.json") - - # TODO: working directory? - etiss_ini = os.path.join(cwd, "custom.ini") - self.write_ini(etiss_ini) - etiss_script_args.append("-i" + etiss_ini) - for plugin in self.plugins: - etiss_script_args.extend(["-p", plugin]) - - # if self.timeout_sec > 0: - if False: - ret = exec_timeout( - self.timeout_sec, - execute, - Path(self.etiss_script).resolve(), - program, - *etiss_script_args, - *args, - cwd=cwd, - **kwargs, - ) + if self.use_run_helper: + etiss_script_args = [] + if len(self.extra_args) > 0: + etiss_script_args.extend(self.extra_args.split(" ")) + + # TODO: this is outdated + # TODO: validate features (attach xor noattach!) + if self.debug_etiss: + etiss_script_args.append("gdb") + if self.gdbserver_enable: + etiss_script_args.append("tgdb") + if not self.gdbserver_attach: + etiss_script_args.append("noattach") + if self.trace_memory: + etiss_script_args.append("trace") + etiss_script_args.append("nodmi") + if self.verbose: + etiss_script_args.append("v") + # Alternative to stdout parsing: etiss_script_args.append("--vp.stats_file_path=stats.json") + + # TODO: working directory? + etiss_ini = os.path.join(cwd, "custom.ini") + self.write_ini(etiss_ini) + etiss_script_args.append("-i" + etiss_ini) + for plugin in self.plugins: + etiss_script_args.extend(["-p", plugin]) + + # if self.timeout_sec > 0: + if False: + ret = exec_timeout( + self.timeout_sec, + execute, + Path(self.etiss_script).resolve(), + program, + *etiss_script_args, + *args, + cwd=cwd, + **kwargs, + ) + else: + ret = execute( + Path(self.etiss_script).resolve(), + program, + *etiss_script_args, + *args, + cwd=cwd, + **kwargs, + ) + return ret, [] else: - ret = execute( - Path(self.etiss_script).resolve(), - program, - *etiss_script_args, - *args, - cwd=cwd, - **kwargs, - ) - return ret, [] + raise NotImplementedError def parse_exit(self, out): exit_code = super().parse_exit(out) From 635a85d5b9393d02e8a6766b1dac2e0fb31922f7 Mon Sep 17 00:00:00 2001 From: Philipp van Kempen Date: Wed, 29 May 2024 16:13:27 +0200 Subject: [PATCH 088/105] refactor etiss target and add more cfgs --- mlonmcu/target/riscv/etiss.py | 262 +++++++++++++++++++++++++--------- 1 file changed, 197 insertions(+), 65 deletions(-) diff --git a/mlonmcu/target/riscv/etiss.py b/mlonmcu/target/riscv/etiss.py index ac2e8b264..fe02cf6ab 100644 --- a/mlonmcu/target/riscv/etiss.py +++ b/mlonmcu/target/riscv/etiss.py @@ -89,8 +89,17 @@ class EtissTarget(RISCVTarget): "extra_string_config": {}, "extra_plugin_config": {}, "use_run_helper": True, + "exit_on_loop": False, + "log_pc": False, + "log_level": None, + "enable_semihosting": True, + "output_path_prefix": "", + "jit_gcc_cleanup": True, + "jit_verify": False, + "jit_debug": False, + "load_integrated_libraries": True, } - REQUIRED = RISCVTarget.REQUIRED | {"etiss.src_dir", "etiss.install_dir", "etissvp.script"} + REQUIRED = RISCVTarget.REQUIRED | {"etiss.src_dir", "etiss.install_dir", "etissvp.exe", "etissvp.script"} def __init__(self, name="etiss", features=None, config=None): super().__init__(name, features=features, config=config) @@ -109,6 +118,10 @@ def etiss_dir(self): def etiss_script(self): return self.config["etissvp.script"] + @property + def etiss_exe(self): + return self.config["etissvp.exe"] + @property def gdbserver_enable(self): value = self.config["gdbserver_enable"] @@ -133,11 +146,22 @@ def trace_memory(self): value = self.config["trace_memory"] return str2bool(value) if not isinstance(value, (bool, int)) else value + @property + def enable_dmi(self): + return False + # return not self.trace_memory + @property def plugins(self): value = self.config["plugins"] return str2list(value) if isinstance(value, str) else value + def get_plugin_names(self): + ret = self.plugins + if self.gdbserver_enable: + ret.append("gdbserver") + return list(set(ret)) + @property def verbose(self): value = self.config["verbose"] @@ -318,6 +342,52 @@ def use_run_helper(self): value = self.config["use_run_helper"] return str2bool(value) if not isinstance(value, (bool, int)) else value + @property + def exit_on_loop(self): + value = self.config["exit_on_loop"] + return str2bool(value) if not isinstance(value, (bool, int)) else value + + @property + def log_pc(self): + value = self.config["log_pc"] + return str2bool(value) if not isinstance(value, (bool, int)) else value + + @property + def log_level(self): + value = self.config["log_level"] + if isinstance(value, str): + value = int(value) + return value + + @property + def enable_semihosting(self): + value = self.config["enable_semihosting"] + return str2bool(value) if not isinstance(value, (bool, int)) else value + + @property + def output_path_prefix(self): + return self.config["output_path_prefix"] + + @property + def jit_gcc_cleanup(self): + value = self.config["jit_gcc_cleanup"] + return str2bool(value) if not isinstance(value, (bool, int)) else value + + @property + def jit_verify(self): + value = self.config["jit_verify"] + return str2bool(value) if not isinstance(value, (bool, int)) else value + + @property + def jit_debug(self): + value = self.config["jit_debug"] + return str2bool(value) if not isinstance(value, (bool, int)) else value + + @property + def load_integrated_libraries(self): + value = self.config["load_integrated_libraries"] + return str2bool(value) if not isinstance(value, (bool, int)) else value + @property def vext_spec(self): return float(self.config["vext_spec"]) @@ -338,27 +408,46 @@ def max_block_size(self): value = int(value) return value - def get_ini_bool_config(self): + def get_ini_bool_config(self, override=None): + override = {k: v for k, v in override.items() if isinstance(v, bool)} + ret = { - "arch.enable_semihosting": True, + "arch.enable_semihosting": self.enable_semihosting, "simple_mem_system.error_on_invalid_access": not self.allow_error, + "jit.verify": self.jit_verify, + "jit.debug": self.jit_debug, + "etiss.load_integrated_libraries": self.load_integrated_libraries, } - ret.update(self.extra_string_config) + if not self.use_run_helper: + ret["simple_mem_system.print_dbus_access"] = self.trace_memory + ret["simple_mem_system.print_to_file"] = self.trace_memory + ret["etiss.exit_on_loop"] = self.exit_on_loop + ret["etiss.log_pc"] = self.log_pc + ret["etiss.enable_dmi"] = self.enable_dmi + if self.jit == "GCC": + ret["jit.gcc.cleanup"] = self.jit_gcc_cleanup + + ret.update(self.extra_bool_config) + ret.update(override) return ret - def get_ini_string_config(self): + def get_ini_string_config(self, override=None): + override = {k: v for k, v in override.items() if isinstance(v, (str, Path))} ret = { "arch.cpu": self.cpu_arch, # Mode will be overwritten by elf... # "simple_mem_system.memseg_mode_00": "RX", # "simple_mem_system.memseg_mode_01": "RWX", + "etiss.output_path_prefix": self.output_path_prefix, } if self.jit is not None: ret["jit.type"] = f"{self.jit}JIT" ret.update(self.extra_string_config) + ret.update(override) return ret - def get_ini_int_config(self): + def get_ini_int_config(self, override=None): + override = {k: v for k, v in override.items() if isinstance(v, int)} ret = { "simple_mem_system.memseg_origin_00": self.rom_start, "simple_mem_system.memseg_length_00": self.rom_size, @@ -378,105 +467,146 @@ def get_ini_int_config(self): ret["arch.rv32imacfdpv.vlen"] = self.vlen if self.elen > 0: ret["arch.rv32imacfdpv.elen"] = self.elen + log_level = self.log_level + if log_level is None and not self.use_run_helper: + log_level = 5 if self.verbose else 4 + if log_level is not None: + ret["etiss.loglevel"] = log_level + # TODO + # ETISS::CPU_quantum_ps=100000 + # ETISS::write_pc_trace_from_time_us=0 + # ETISS::write_pc_trace_until_time_us=3000000 + # ETISS::sim_mode=0 + # vp::simulation_time_us=20000000 ret.update(self.extra_int_config) + ret.update(override) return ret def get_ini_plugin_config(self): ret = {} if self.gdbserver_enable: - # This could also be accomplished using `--plugin.gdbserver.port` on the cmdline - ret["gdbserver"] = { - "port": self.gdbserver_port, - } + if not self.use_run_helper: + ret["gdbserver"] = { + "port": self.gdbserver_port, + } ret.update(self.extra_plugin_config) # TODO: merge nested dict instead of overriding return ret - def write_ini(self, path): + def write_ini(self, path, override=None): # TODO: Either create artifact for ini or prefer to use cmdline args. with open(path, "w") as f: - ini_bool = self.get_ini_bool_config() + ini_bool = self.get_ini_bool_config(override=override) if len(ini_bool) > 0: f.write("[BoolConfigurations]\n") for key, value in ini_bool.items(): assert isinstance(value, bool) val = "true" if value else "false" f.write(f"{key}={val}\n") - ini_string = self.get_ini_string_config() + ini_string = self.get_ini_string_config(override=override) if len(ini_string) > 0: f.write("[StringConfigurations]\n") for key, value in ini_string.items(): + if isinstance(value, Path): + value = str(value) assert isinstance(value, str) f.write(f"{key}={value}\n") - ini_int = self.get_ini_int_config() + ini_int = self.get_ini_int_config(override=override) if len(ini_int) > 0: f.write("[IntConfigurations]\n") for key, value in ini_int.items(): assert isinstance(value, int) f.write(f"{key}={value}\n") ini_plugin = self.get_ini_plugin_config() - if len(ini_plugin) > 0: - for name, cfg in ini_plugin.items(): - f.write(f"[Plugin {name}]\n") - for key, value in cfg.items(): - if isinstance(value, bool): - val = "true" if value else "false" - else: - val = value - f.write(f"plugin.{name}.{key}={val}\n") + for plugin_name in self.get_plugin_names(): + f.write(f"[Plugin {plugin_name}]\n") + cfg = ini_plugin.pop(plugin_name, {}) + for key, value in cfg.items(): + if isinstance(value, bool): + val = "true" if value else "false" + else: + val = value + f.write(f"plugin.{plugin_name}.{key}={val}\n") + # Check for remaining configs + for plugin_name, cfg in ini_plugin.items(): + if len(cfg) == 0: + continue + logger.warning("Skipping config %s for disabled plugin %s", cfg, plugin_name) def exec(self, program, *args, cwd=os.getcwd(), **kwargs): """Use target to execute a executable with given arguments""" - if self.use_run_helper: - etiss_script_args = [] - if len(self.extra_args) > 0: - etiss_script_args.extend(self.extra_args.split(" ")) - - # TODO: this is outdated - # TODO: validate features (attach xor noattach!) - if self.debug_etiss: + etiss_script_args = [] + if len(self.extra_args) > 0: + if not self.use_run_helper: + raise NotImplementedError("etiss.extra_args requires etiss.use_run_helper=1") + etiss_script_args.extend(self.extra_args.split(" ")) + + # TODO: this is outdated + # TODO: validate features (attach xor noattach!) + if self.debug_etiss: + if self.use_run_helper: etiss_script_args.append("gdb") - if self.gdbserver_enable: + else: + raise NotImplementedError("etiss.debug_etiss requires etiss.use_run_helper=1") + if self.gdbserver_enable: + if self.use_run_helper: etiss_script_args.append("tgdb") if not self.gdbserver_attach: etiss_script_args.append("noattach") - if self.trace_memory: + etiss_script_args.append("--plugin.gdbserver.port={self.gdbserver_port}") + if self.gdbserver_attach: + raise NotImplementedError("etiss.gdbserver_attach requires etiss.use_run_helper=1") + if self.trace_memory: + if self.use_run_helper: etiss_script_args.append("trace") etiss_script_args.append("nodmi") - if self.verbose: - etiss_script_args.append("v") - # Alternative to stdout parsing: etiss_script_args.append("--vp.stats_file_path=stats.json") - - # TODO: working directory? - etiss_ini = os.path.join(cwd, "custom.ini") - self.write_ini(etiss_ini) - etiss_script_args.append("-i" + etiss_ini) - for plugin in self.plugins: + if self.exit_on_loop: + if self.use_run_helper: + etiss_script_args.append("noloop") + if self.log_pc: + if self.use_run_helper: + etiss_script_args.append("logpc") + if not self.enable_dmi: + if self.use_run_helper: + etiss_script_args.append("nodmi") + if self.verbose: + etiss_script_args.append("v") + # Alternative to stdout parsing: etiss_script_args.append("--vp.stats_file_path=stats.json") + if self.use_run_helper: + for plugin in self.get_plugin_names(): etiss_script_args.extend(["-p", plugin]) - # if self.timeout_sec > 0: - if False: - ret = exec_timeout( - self.timeout_sec, - execute, - Path(self.etiss_script).resolve(), - program, - *etiss_script_args, - *args, - cwd=cwd, - **kwargs, - ) - else: - ret = execute( - Path(self.etiss_script).resolve(), - program, - *etiss_script_args, - *args, - cwd=cwd, - **kwargs, - ) - return ret, [] + # TODO: working directory? + ini_override = {} + if self.use_run_helper: + etiss_script_args.insert(0, program) else: - raise NotImplementedError + ini_override["vp.elf_file"] = program + etiss_ini = os.path.join(cwd, "custom.ini") + self.write_ini(etiss_ini, override=ini_override) + etiss_script_args.append("-i" + etiss_ini) + + # if self.timeout_sec > 0: + script = self.etiss_script if self.use_run_helper else self.etiss_exe + script = Path(script).resolve() + if False: + ret = exec_timeout( + self.timeout_sec, + execute, + script, + *etiss_script_args, + *args, + cwd=cwd, + **kwargs, + ) + else: + ret = execute( + script, + *etiss_script_args, + *args, + cwd=cwd, + **kwargs, + ) + return ret, [] def parse_exit(self, out): exit_code = super().parse_exit(out) @@ -620,6 +750,8 @@ def get_ram_sizes(data): return metrics, out, artifacts def get_target_system(self): + if not self.enable_semihosting: + raise NotImplementedError("etiss.enable_semihosting=0 is not supported anymore") return self.name def get_platform_defs(self, platform): From 4e6e0c52c770d9499add3ca78de02942df9972f2 Mon Sep 17 00:00:00 2001 From: Philipp van Kempen Date: Tue, 27 Aug 2024 10:38:13 +0200 Subject: [PATCH 089/105] ci: update actions to latest versions --- .github/workflows/bench.yml | 4 +- .github/workflows/cicd.yml | 22 +++++----- .github/workflows/container.yml | 40 +++++++++---------- .github/workflows/container_weekly.yml | 36 ++++++++--------- .github/workflows/demo.yml | 4 +- .github/workflows/docker_bench.yml | 2 +- .github/workflows/notebook.yml | 4 +- .github/workflows/refresh_container.yml | 12 +++--- .github/workflows/refresh_container_daily.yml | 24 +++++------ .github/workflows/release.yml | 2 +- .github/workflows/style.yml | 4 +- 11 files changed, 77 insertions(+), 77 deletions(-) diff --git a/.github/workflows/bench.yml b/.github/workflows/bench.yml index 97351d281..393e78432 100644 --- a/.github/workflows/bench.yml +++ b/.github/workflows/bench.yml @@ -40,7 +40,7 @@ jobs: with: submodules: recursive - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} # TODO: caching @@ -76,7 +76,7 @@ jobs: cd scripts/ python bench.py ${{ github.event.inputs.benchmark }} out/ - name: Archive reports - uses: actions/upload-artifact@v2 + uses: actions/upload-artifact@v4 with: name: results path: scripts/out/ diff --git a/.github/workflows/cicd.yml b/.github/workflows/cicd.yml index 2be233dbd..ac9cd411a 100644 --- a/.github/workflows/cicd.yml +++ b/.github/workflows/cicd.yml @@ -46,10 +46,10 @@ jobs: steps: - uses: actions/checkout@v3 - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} - - uses: actions/cache@v3 + - uses: actions/cache@v4 id: cache-venv # name for referring later with: path: | @@ -86,10 +86,10 @@ jobs: steps: - uses: actions/checkout@v3 - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} - - uses: actions/cache@v3 + - uses: actions/cache@v4 id: cache-venv # name for referring later with: path: | @@ -100,7 +100,7 @@ jobs: restore-keys: | build-${{ runner.os }}-${{ matrix.python-version }}-venv-${{ hashFiles('**/requirements*.txt') }} - name: Archive package - uses: actions/upload-artifact@v2 + uses: actions/upload-artifact@v4 with: name: mlonmcu path: dist/ @@ -120,10 +120,10 @@ jobs: steps: - uses: actions/checkout@v3 - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} - - uses: actions/cache@v3 + - uses: actions/cache@v4 id: cache-venv # name for referring later with: path: | @@ -156,7 +156,7 @@ jobs: source .venv/bin/activate make coverage-full - name: Archive code coverage html report - uses: actions/upload-artifact@v2 + uses: actions/upload-artifact@v4 with: name: code-coverage-report path: htmlcov @@ -184,10 +184,10 @@ jobs: steps: - uses: actions/checkout@v3 - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} - - uses: actions/cache@v3 + - uses: actions/cache@v4 id: cache-venv # name for referring later with: path: | @@ -215,7 +215,7 @@ jobs: source .venv/bin/activate make docs - name: Deploy docs - uses: peaceiris/actions-gh-pages@v3 + uses: peaceiris/actions-gh-pages@v4 if: ${{ github.ref == 'refs/heads/main' }} with: github_token: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/container.yml b/.github/workflows/container.yml index cf518cc0a..4f95d860a 100644 --- a/.github/workflows/container.yml +++ b/.github/workflows/container.yml @@ -56,11 +56,11 @@ jobs: with: submodules: recursive - name: Set up QEMU - uses: docker/setup-qemu-action@v2 + uses: docker/setup-qemu-action@v3 - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v2 + uses: docker/setup-buildx-action@v3 - name: Login to GitHub Container Registry - uses: docker/login-action@v2 + uses: docker/login-action@v3 with: registry: ghcr.io username: ${{ github.actor }} @@ -70,11 +70,11 @@ jobs: ./scripts/update_version.sh - name: Lowercase repository url id: lowered - uses: ASzc/change-string-case-action@v5 + uses: ASzc/change-string-case-action@v6 with: string: ${{ github.repository }} - name: Build and push (CMake) - uses: docker/build-push-action@v4 + uses: docker/build-push-action@v6 with: context: . file: docker/Dockerfile @@ -88,7 +88,7 @@ jobs: cache-to: type=inline tags: ghcr.io/${{ steps.lowered.outputs.lowercase }}-cmake:${{ github.event.inputs.version }} - name: Build and push - uses: docker/build-push-action@v4 + uses: docker/build-push-action@v6 with: context: . file: docker/Dockerfile @@ -122,11 +122,11 @@ jobs: with: submodules: recursive - name: Set up QEMU - uses: docker/setup-qemu-action@v2 + uses: docker/setup-qemu-action@v3 - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v2 + uses: docker/setup-buildx-action@v3 - name: Login to GitHub Container Registry - uses: docker/login-action@v2 + uses: docker/login-action@v3 with: registry: ghcr.io username: ${{ github.actor }} @@ -136,11 +136,11 @@ jobs: ./scripts/update_version.sh - name: Lowercase repository url id: lowered - uses: ASzc/change-string-case-action@v5 + uses: ASzc/change-string-case-action@v6 with: string: ${{ github.repository }} # - name: Build and push - # uses: docker/build-push-action@v4 + # uses: docker/build-push-action@v6 # with: # context: . # file: docker/Dockerfile @@ -153,14 +153,14 @@ jobs: # cache-to: type=inline # tags: ghcr.io/${{ steps.lowered.outputs.lowercase }}:${{ github.event.inputs.version }} - name: Login to DockerHub - uses: docker/login-action@v2 + uses: docker/login-action@v3 if: ${{ github.repository == 'tum-ei-eda/mlonmcu' }} with: registry: ${{ secrets.DOCKER_REGISTRY }} username: ${{ secrets.DOCKER_USERNAME }} password: ${{ secrets.DOCKER_PASSWORD }} - name: Build and push to Docker Hub - uses: docker/build-push-action@v4 + uses: docker/build-push-action@v6 if: ${{ github.repository == 'tum-ei-eda/mlonmcu' }} with: context: . @@ -196,22 +196,22 @@ jobs: with: submodules: recursive - name: Set up QEMU - uses: docker/setup-qemu-action@v2 + uses: docker/setup-qemu-action@v3 - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v2 + uses: docker/setup-buildx-action@v3 - name: Login to GitHub Container Registry - uses: docker/login-action@v2 + uses: docker/login-action@v3 with: registry: ghcr.io username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} - name: Lowercase repository url id: lowered - uses: ASzc/change-string-case-action@v5 + uses: ASzc/change-string-case-action@v6 with: string: ${{ github.repository }} # - name: Build and push - # uses: docker/build-push-action@v4 + # uses: docker/build-push-action@v6 # with: # context: . # file: docker/Dockerfile @@ -227,14 +227,14 @@ jobs: # cache-to: type=inline # tags: ghcr.io/${{ steps.lowered.outputs.lowercase }}-bench:${{ github.event.inputs.version }} - name: Login to DockerHub - uses: docker/login-action@v2 + uses: docker/login-action@v3 if: ${{ github.repository == 'tum-ei-eda/mlonmcu' }} with: registry: ${{ secrets.DOCKER_REGISTRY }} username: ${{ secrets.DOCKER_USERNAME }} password: ${{ secrets.DOCKER_PASSWORD }} - name: Build and push to Docker Hub - uses: docker/build-push-action@v4 + uses: docker/build-push-action@v6 if: ${{ github.repository == 'tum-ei-eda/mlonmcu' }} with: context: . diff --git a/.github/workflows/container_weekly.yml b/.github/workflows/container_weekly.yml index 3aaf3157a..22304e257 100644 --- a/.github/workflows/container_weekly.yml +++ b/.github/workflows/container_weekly.yml @@ -47,7 +47,7 @@ jobs: id: timestamp - name: Lowercase repository url id: lowered - uses: ASzc/change-string-case-action@v5 + uses: ASzc/change-string-case-action@v6 with: string: ${{ github.repository }} - name: CFG @@ -113,17 +113,17 @@ jobs: with: ref: ${{ steps.cfg.outputs.branch }} - name: Set up QEMU - uses: docker/setup-qemu-action@v2 + uses: docker/setup-qemu-action@v3 - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v2 + uses: docker/setup-buildx-action@v3 - name: Login to GitHub Container Registry - uses: docker/login-action@v2 + uses: docker/login-action@v3 with: registry: ghcr.io username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} - name: Build and push (CMake) - uses: docker/build-push-action@v4 + uses: docker/build-push-action@v6 with: context: . file: docker/Dockerfile @@ -138,7 +138,7 @@ jobs: cache-to: type=inline tags: ghcr.io/${{ steps.lowered.outputs.lowercase }}-cmake:latest - name: Build and push - uses: docker/build-push-action@v4 + uses: docker/build-push-action@v6 with: context: . file: docker/Dockerfile @@ -166,7 +166,7 @@ jobs: id: timestamp - name: Lowercase repository url id: lowered - uses: ASzc/change-string-case-action@v5 + uses: ASzc/change-string-case-action@v6 with: string: ${{ github.repository }} - name: CFG @@ -232,24 +232,24 @@ jobs: with: ref: ${{ steps.cfg.outputs.branch }} - name: Set up QEMU - uses: docker/setup-qemu-action@v2 + uses: docker/setup-qemu-action@v3 - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v2 + uses: docker/setup-buildx-action@v3 - name: Login to GitHub Container Registry - uses: docker/login-action@v2 + uses: docker/login-action@v3 with: registry: ghcr.io username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} - name: Login to DockerHub - uses: docker/login-action@v2 + uses: docker/login-action@v3 if: ${{ github.repository == 'tum-ei-eda/mlonmcu' }} with: registry: ${{ secrets.DOCKER_REGISTRY }} username: ${{ secrets.DOCKER_USERNAME }} password: ${{ secrets.DOCKER_PASSWORD }} - name: Build and push to Docker Hub - uses: docker/build-push-action@v4 + uses: docker/build-push-action@v6 if: ${{ github.repository == 'tum-ei-eda/mlonmcu' }} with: context: . @@ -278,7 +278,7 @@ jobs: id: timestamp - name: Lowercase repository url id: lowered - uses: ASzc/change-string-case-action@v5 + uses: ASzc/change-string-case-action@v6 with: string: ${{ github.repository }} - name: CFG @@ -344,24 +344,24 @@ jobs: with: ref: ${{ steps.cfg.outputs.branch }} - name: Set up QEMU - uses: docker/setup-qemu-action@v2 + uses: docker/setup-qemu-action@v3 - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v2 + uses: docker/setup-buildx-action@v3 - name: Login to GitHub Container Registry - uses: docker/login-action@v2 + uses: docker/login-action@v3 with: registry: ghcr.io username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} - name: Login to DockerHub - uses: docker/login-action@v2 + uses: docker/login-action@v3 if: ${{ github.repository == 'tum-ei-eda/mlonmcu' }} with: registry: ${{ secrets.DOCKER_REGISTRY }} username: ${{ secrets.DOCKER_USERNAME }} password: ${{ secrets.DOCKER_PASSWORD }} - name: Build and push to Docker Hub - uses: docker/build-push-action@v4 + uses: docker/build-push-action@v6 if: ${{ github.repository == 'tum-ei-eda/mlonmcu' }} with: context: . diff --git a/.github/workflows/demo.yml b/.github/workflows/demo.yml index e1dbd3a57..e04ae280f 100644 --- a/.github/workflows/demo.yml +++ b/.github/workflows/demo.yml @@ -70,7 +70,7 @@ jobs: with: submodules: recursive - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} # TODO: caching @@ -127,7 +127,7 @@ jobs: mlonmcu cleanup -H home/ -f --deps if: ${{ github.event.inputs.artifact == 'true' }} - name: Archive environment (without deps) - uses: actions/upload-artifact@v2 + uses: actions/upload-artifact@v4 with: name: mlonmcu_home path: home/ diff --git a/.github/workflows/docker_bench.yml b/.github/workflows/docker_bench.yml index 5a8f95165..360d7e01f 100644 --- a/.github/workflows/docker_bench.yml +++ b/.github/workflows/docker_bench.yml @@ -50,7 +50,7 @@ jobs: - name: Store cmdline run: echo "${{ github.event.inputs.bench_cmd }}" >> /environment/results/cmd.txt - name: Archive reports - uses: actions/upload-artifact@v2 + uses: actions/upload-artifact@v4 with: name: results path: /environment/results/ diff --git a/.github/workflows/notebook.yml b/.github/workflows/notebook.yml index d764cbc3f..772c728d4 100644 --- a/.github/workflows/notebook.yml +++ b/.github/workflows/notebook.yml @@ -49,7 +49,7 @@ jobs: with: submodules: recursive - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} # TODO: caching @@ -73,7 +73,7 @@ jobs: - name: Get date run: echo "timestamp=`date +%FT%T`" >> $GITHUB_ENV - name: Create Pull Request - uses: peter-evans/create-pull-request@v5 + uses: peter-evans/create-pull-request@v6 if: github.event_name == 'workflow_dispatch' && github.event.inputs.branch != '' with: # token: ${{ secrets.PAT }} diff --git a/.github/workflows/refresh_container.yml b/.github/workflows/refresh_container.yml index ccd828c81..364b5f643 100644 --- a/.github/workflows/refresh_container.yml +++ b/.github/workflows/refresh_container.yml @@ -52,29 +52,29 @@ jobs: with: submodules: recursive - name: Set up QEMU - uses: docker/setup-qemu-action@v2 + uses: docker/setup-qemu-action@v3 - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v2 + uses: docker/setup-buildx-action@v3 - name: Login to GitHub Container Registry - uses: docker/login-action@v2 + uses: docker/login-action@v3 with: registry: ghcr.io username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} - name: Lowercase repository url id: lowered - uses: ASzc/change-string-case-action@v5 + uses: ASzc/change-string-case-action@v6 with: string: ${{ github.repository }} - name: Login to DockerHub - uses: docker/login-action@v2 + uses: docker/login-action@v3 if: ${{ github.repository == 'tum-ei-eda/mlonmcu' }} with: registry: ${{ secrets.DOCKER_REGISTRY }} username: ${{ secrets.DOCKER_USERNAME }} password: ${{ secrets.DOCKER_PASSWORD }} - name: Build and push to Docker Hub - uses: docker/build-push-action@v4 + uses: docker/build-push-action@v6 if: ${{ github.repository == 'tum-ei-eda/mlonmcu' }} with: context: . diff --git a/.github/workflows/refresh_container_daily.yml b/.github/workflows/refresh_container_daily.yml index 1c124eb52..42e5f43d4 100644 --- a/.github/workflows/refresh_container_daily.yml +++ b/.github/workflows/refresh_container_daily.yml @@ -39,7 +39,7 @@ jobs: steps: - name: Lowercase repository url id: lowered - uses: ASzc/change-string-case-action@v5 + uses: ASzc/change-string-case-action@v6 with: string: ${{ github.repository }} - name: Gen string @@ -74,24 +74,24 @@ jobs: with: ref: ${{ github.event.inputs.branch }} - name: Set up QEMU - uses: docker/setup-qemu-action@v2 + uses: docker/setup-qemu-action@v3 - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v2 + uses: docker/setup-buildx-action@v3 - name: Login to GitHub Container Registry - uses: docker/login-action@v2 + uses: docker/login-action@v3 with: registry: ghcr.io username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} - name: Login to DockerHub - uses: docker/login-action@v2 + uses: docker/login-action@v3 if: ${{ github.repository == 'tum-ei-eda/mlonmcu' }} with: registry: ${{ secrets.DOCKER_REGISTRY }} username: ${{ secrets.DOCKER_USERNAME }} password: ${{ secrets.DOCKER_PASSWORD }} - name: Build and push to Docker Hub - uses: docker/build-push-action@v4 + uses: docker/build-push-action@v6 if: ${{ github.repository == 'tum-ei-eda/mlonmcu' }} with: context: . @@ -115,7 +115,7 @@ jobs: steps: - name: Lowercase repository url id: lowered - uses: ASzc/change-string-case-action@v5 + uses: ASzc/change-string-case-action@v6 with: string: ${{ github.repository }} - name: Gen string @@ -150,24 +150,24 @@ jobs: with: ref: ${{ matrix.config.branch }} - name: Set up QEMU - uses: docker/setup-qemu-action@v2 + uses: docker/setup-qemu-action@v3 - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v2 + uses: docker/setup-buildx-action@v3 - name: Login to GitHub Container Registry - uses: docker/login-action@v2 + uses: docker/login-action@v3 with: registry: ghcr.io username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} - name: Login to DockerHub - uses: docker/login-action@v2 + uses: docker/login-action@v3 if: ${{ github.repository == 'tum-ei-eda/mlonmcu' }} with: registry: ${{ secrets.DOCKER_REGISTRY }} username: ${{ secrets.DOCKER_USERNAME }} password: ${{ secrets.DOCKER_PASSWORD }} - name: Build and push to Docker Hub - uses: docker/build-push-action@v4 + uses: docker/build-push-action@v6 if: ${{ github.repository == 'tum-ei-eda/mlonmcu' }} with: context: . diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 05ae1eafe..b08c392b9 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -16,7 +16,7 @@ jobs: uses: actions/checkout@v3 - name: Generate changelog id: changelog - uses: metcalfc/changelog-generator@v0.4.3 + uses: metcalfc/changelog-generator@v4.3.1 with: myToken: ${{ secrets.GITHUB_TOKEN }} - name: Create Release diff --git a/.github/workflows/style.yml b/.github/workflows/style.yml index a8f1cfa10..f852d23f4 100644 --- a/.github/workflows/style.yml +++ b/.github/workflows/style.yml @@ -12,7 +12,7 @@ jobs: uses: actions/checkout@v3 - name: Set up Python - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: "3.10" @@ -20,7 +20,7 @@ jobs: run: pip install black flake8 - name: Run linters - uses: wearerequired/lint-action@v1 + uses: wearerequired/lint-action@v2 with: black: true black_args: "--line-length=120" From 81a128b00af7f20b25b6d921e478d70ed1e8485c Mon Sep 17 00:00:00 2001 From: Philipp van Kempen Date: Tue, 27 Aug 2024 10:40:57 +0200 Subject: [PATCH 090/105] ci: update actions to latest versions --- .github/workflows/bench.yml | 2 +- .github/workflows/cicd.yml | 8 ++++---- .github/workflows/container.yml | 6 +++--- .github/workflows/container_weekly.yml | 6 +++--- .github/workflows/demo.yml | 2 +- .github/workflows/notebook.yml | 2 +- .github/workflows/refresh_container.yml | 2 +- .github/workflows/refresh_container_daily.yml | 4 ++-- .github/workflows/release.yml | 4 ++-- .github/workflows/style.yml | 4 ++-- 10 files changed, 20 insertions(+), 20 deletions(-) diff --git a/.github/workflows/bench.yml b/.github/workflows/bench.yml index 393e78432..7877dcc98 100644 --- a/.github/workflows/bench.yml +++ b/.github/workflows/bench.yml @@ -36,7 +36,7 @@ jobs: matrix: python-version: ["3.10"] steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 with: submodules: recursive - name: Set up Python ${{ matrix.python-version }} diff --git a/.github/workflows/cicd.yml b/.github/workflows/cicd.yml index ac9cd411a..e3a37a528 100644 --- a/.github/workflows/cicd.yml +++ b/.github/workflows/cicd.yml @@ -44,7 +44,7 @@ jobs: matrix: python-version: ["3.7", "3.8", "3.9", "3.10", "3.11"] steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v5 with: @@ -84,7 +84,7 @@ jobs: python-version: ["3.10"] if: github.ref == 'refs/heads/main' || startsWith(github.ref, 'refs/tags/v') steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v5 with: @@ -118,7 +118,7 @@ jobs: matrix: python-version: ["3.10"] steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v5 with: @@ -182,7 +182,7 @@ jobs: matrix: python-version: ["3.10"] steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v5 with: diff --git a/.github/workflows/container.yml b/.github/workflows/container.yml index 4f95d860a..bbf928aad 100644 --- a/.github/workflows/container.yml +++ b/.github/workflows/container.yml @@ -52,7 +52,7 @@ jobs: remove-haskell: 'true' remove-codeql: 'true' - name: Checkout - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: submodules: recursive - name: Set up QEMU @@ -118,7 +118,7 @@ jobs: remove-haskell: 'true' remove-codeql: 'true' - name: Checkout - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: submodules: recursive - name: Set up QEMU @@ -192,7 +192,7 @@ jobs: remove-haskell: 'true' remove-codeql: 'true' - name: Checkout - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: submodules: recursive - name: Set up QEMU diff --git a/.github/workflows/container_weekly.yml b/.github/workflows/container_weekly.yml index 22304e257..131324442 100644 --- a/.github/workflows/container_weekly.yml +++ b/.github/workflows/container_weekly.yml @@ -109,7 +109,7 @@ jobs: remove-haskell: 'true' remove-codeql: 'true' - name: Checkout - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: ref: ${{ steps.cfg.outputs.branch }} - name: Set up QEMU @@ -228,7 +228,7 @@ jobs: remove-haskell: 'true' remove-codeql: 'true' - name: Checkout - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: ref: ${{ steps.cfg.outputs.branch }} - name: Set up QEMU @@ -340,7 +340,7 @@ jobs: remove-haskell: 'true' remove-codeql: 'true' - name: Checkout - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: ref: ${{ steps.cfg.outputs.branch }} - name: Set up QEMU diff --git a/.github/workflows/demo.yml b/.github/workflows/demo.yml index e04ae280f..1e374421f 100644 --- a/.github/workflows/demo.yml +++ b/.github/workflows/demo.yml @@ -66,7 +66,7 @@ jobs: remove-android: 'true' remove-haskell: 'true' remove-codeql: 'true' - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 with: submodules: recursive - name: Set up Python ${{ matrix.python-version }} diff --git a/.github/workflows/notebook.yml b/.github/workflows/notebook.yml index 772c728d4..dbf22692f 100644 --- a/.github/workflows/notebook.yml +++ b/.github/workflows/notebook.yml @@ -45,7 +45,7 @@ jobs: remove-android: 'true' remove-haskell: 'true' remove-codeql: 'true' - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 with: submodules: recursive - name: Set up Python ${{ matrix.python-version }} diff --git a/.github/workflows/refresh_container.yml b/.github/workflows/refresh_container.yml index 364b5f643..f8f957324 100644 --- a/.github/workflows/refresh_container.yml +++ b/.github/workflows/refresh_container.yml @@ -48,7 +48,7 @@ jobs: remove-haskell: 'true' remove-codeql: 'true' - name: Checkout - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: submodules: recursive - name: Set up QEMU diff --git a/.github/workflows/refresh_container_daily.yml b/.github/workflows/refresh_container_daily.yml index 42e5f43d4..a0d944d3f 100644 --- a/.github/workflows/refresh_container_daily.yml +++ b/.github/workflows/refresh_container_daily.yml @@ -70,7 +70,7 @@ jobs: remove-haskell: 'true' remove-codeql: 'true' - name: Checkout - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: ref: ${{ github.event.inputs.branch }} - name: Set up QEMU @@ -146,7 +146,7 @@ jobs: remove-haskell: 'true' remove-codeql: 'true' - name: Checkout - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: ref: ${{ matrix.config.branch }} - name: Set up QEMU diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index b08c392b9..1f6d60459 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -13,7 +13,7 @@ jobs: if: ${{ github.repository == 'tum-ei-eda/mlonmcu' }} steps: - name: Checkout code - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Generate changelog id: changelog uses: metcalfc/changelog-generator@v4.3.1 @@ -40,7 +40,7 @@ jobs: if: ${{ github.repository == 'tum-ei-eda/mlonmcu' }} steps: - name: Checkout code - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Build package run: make dist - name: Publish to PyPI diff --git a/.github/workflows/style.yml b/.github/workflows/style.yml index f852d23f4..c708ed197 100644 --- a/.github/workflows/style.yml +++ b/.github/workflows/style.yml @@ -9,7 +9,7 @@ jobs: steps: - name: Check out Git repository - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Set up Python uses: actions/setup-python@v5 @@ -33,7 +33,7 @@ jobs: steps: - name: Check out Git repository - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Install Python dependencies run: pip install licenseheaders From 97b3553815a19d8f6b07a32418c788f528b173ea Mon Sep 17 00:00:00 2001 From: Philipp van Kempen Date: Mon, 9 Sep 2024 09:13:51 +0200 Subject: [PATCH 091/105] features: comment out unimplemented get_postprocesses --- ValidationNew.md | 166 ------------------------------------ mlonmcu/feature/feature.py | 8 +- mlonmcu/feature/features.py | 12 +-- 3 files changed, 10 insertions(+), 176 deletions(-) delete mode 100644 ValidationNew.md diff --git a/ValidationNew.md b/ValidationNew.md deleted file mode 100644 index 6cfc85453..000000000 --- a/ValidationNew.md +++ /dev/null @@ -1,166 +0,0 @@ -# MLonMCU - Validation (new) - -## Updates - -- `mlonmcu-models` (`$MLONMCU_HOME/models/`, Branch: `refactor-validate`): - - `resnet/definition.yml` - - Add input/output dtype - - Add dequantize details (taken from `mlif_override.cpp`) - - `resnet/support/mlif_override.cpp` - - Comment out old validation code - - Add example code to dump raw inputs/outputs via stdin/stdout -- `mlonmcu` (Branch: `refactor-validate`): - - `mlonmcu/target/common.py` - - Allow overriding used encoding for stdout (Required for dumping raw data) - - Add optional `stdin_data` argument for passing input to process (Only works with `live=False` aka. `-c {target}.print_outputs=0`) - - `mlonmcu/target/target.py` / `mlonmcu/target/riscv/spike.py` - - `target.exec()` may return artifacts now - - Add checks: `supports_filesystem`, `supports_stdout`, `supports_stdin`, `supports_argv`, `supports_uart` - - `mlonmcu/feature/feature.py`: - - Add `gen_data` feature - - Add `gen_ref_data` feature - - Add `set_inputs` feature - - Add `get_outputs` feature - - Add wrapper `validate_new` feature (Limitation: can not enable postprocess yet) - - `mlonmcu/models/frontend.py` - - Parse input/output types from `definition.yml` - - Parse quantization/dequantization details from `definition.yml` - - Create `model_info.yml` Artifact to pass input/output dtypes/shapes/quant to later stages - - Implement `gen_data` functionality (including workaround to convert raw `inputs/0.bin` to numpy) - - Implement dummy `gen_ref_data` functionality - - `mlonmcu/platform/mlif/mlif.py` / `mlonmcu/platform/mlif/mlif_target.py` - - Implement `set_inputs` functionality (stdin_raw only) - - Implement `get_outputs` functionality (stdout_raw only) - - Implement batching (run simulation serveral {num_inputs/batch_size} times, with different inputs) - - `mlonmcu/platform/tvm/tvm_target_platform.py` / `mlonmcu/platform/tvm/tvm_target.py` - - Implement `set_inputs` functionality (filesystem only) - - Implement `get_outputs` functionality (filesystem only) - - Implement batching (run simulation serveral {num_inputs/batch_size} times, with different inputs) - - `mlonmcu/session/postprocess/postprocesses.py` - - Add `ValidateOutputs` postprocess (WIP) - -## Updates (25.01.2024) -- `mlonmcu-models` (`$MLONMCU_HOME/models/`, Branch: `refactor-validate`): - - Update some more definition.yml files - - Resnet: add support_new dir for ml_interface v2 compatibility - - Resnet: add support_new_mlifio as demo/test of new interfaces -- `mlonmcu-sw` (`$MLONMCU_HOME/deps/src/mlif`, Branch: `refactor-validate` **NEW**): - - Added versioning to ml_interface template (Use `-c mlif.template_version=v2` to use the updated mlif) - - Link `mlifio` utilities to ml_interface - - Drop old model support and data handling - - Refactor API to be aware of batch index - - Replace `process_input` and `process_output` functions by `process_inputs` and `process_outputs` processing all inputs at once - - Expose `TVMWrap_GetInputPtr`,... to model support via `mlif_input_ptr`,... - - Model support will determine input size, pointers via above mentioned API instead of arguments. -- `mlonmcu` (Branch: `refactor-validate`): - - Add `mlif.template_version` to select ml_interface version (v1 vs v2), will be automatic in the future - - Add `mlif.batch_size` config to override default batch size - - Allow overriding model support files - - Add generation of model support files - - -## Examples - -### Generation of Data (NEW) - -``` -python3 -m mlonmcu.cli.main flow load resnet -f gen_data -c gen_data.number=5 -c gen_data.fill_mode=zeros -c run.export_optional=1 -python3 -m mlonmcu.cli.main flow load resnet -f gen_data -c gen_data.number=5 -c gen_data.fill_mode=ones -c run.export_optional=1 -python3 -m mlonmcu.cli.main flow load resnet -f gen_data -f gen_ref_data -c gen_data.number=5 -c gen_data.fill_mode=random -c gen_ref_data.mode=model -c run.export_optional=1 -``` - -### Generated Model Support (NEW) - -``` -# mlifio demo (hardcoded, not functional) -python3 -m mlonmcu.cli.main flow run resnet --target host_x86 -f debug --backend tvmaotplus -c mlif.print_outputs=1 -c host_x86.print_outputs=1 -v -f validate -c mlif.template_version=v2 --parallel -c mlif.model_support_dir=$MLONMCU_HOME/models/resnet/support_new_mlifio -c mlif.batch_size=5 - -# in: rom, out: stdout_raw -python3 -m mlonmcu.cli.main flow run resnet --target host_x86 -f debug --backend tvmaotplus -c mlif.print_outputs=1 -c host_x86.print_outputs=1 -v -f validate -c mlif.template_version=v2 --parallel --feature validate_new --post validate_outputs -c set_inputs.interface=rom get_outputs.interface=stdout_raw -c run.export_optional=1 -c gen_data.number=5 -c gen_data.fill_mode=random -c gen_ref_data.mode=model - -# in: stdin_raw, out: stdout_raw -python3 -m mlonmcu.cli.main flow run resnet --target host_x86 -f debug --backend tvmaotplus -c mlif.print_outputs=1 -c host_x86.print_outputs=1 -v -f validate -c mlif.template_version=v2 --parallel --feature validate_new --post validate_outputs -c set_inputs.interface=stdin_raw get_outputs.interface=stdout_raw -c run.export_optional=1 -c gen_data.number=5 -c gen_data.fill_mode=random -c gen_ref_data.mode=model -``` - -### Full Flow - -``` -# platform: tvm target: tvm_cpu` -# implemented: -python3 -m mlonmcu.cli.main flow run resnet -v \ - --target tvm_cpu --backend tvmllvm \ - --feature validate_new --post validate_outputs \ - -c tvm.print_outputs=1 -c tvm_cpu.print_outputs=1 \ - -c set_inputs.interface=filesystem -c get_outputs.interface=filesystem -python3 -m mlonmcu.cli.main flow run resnet -v \ - --target tvm_cpu --backend tvmllvm \ - --feature validate_new --post validate_outputs \ - -c tvm.print_outputs=1 -c tvm_cpu.print_outputs=1 \ - -c set_inputs.interface=auto -c get_outputs.interface=auto - -# not implemented: -# python3 -m mlonmcu.cli.main flow run resnet -v \ -# --target tvm_cpu --backend tvmllvm \ -# --feature validate_new --post validate_outputs \ -# -c tvm.print_outputs=1 -c tvm_cpu.print_outputs=1 -# -c set_inputs.interface=filesystem -c get_outputs.interface=stdout - -# platform: mlif target: spike -# implemented: -python3 -m mlonmcu.cli.main flow run resnet -v \ ---target spike --backend tvmaotplus \ --f validate \ ---feature validate_new --post validate_outputs \ --c mlif.print_outputs=1 -c spike.print_outputs=0 \ --c set_inputs.interface=stdin_raw -c get_outputs.interface=stdout_raw - -# not implemented: -# python3 -m mlonmcu.cli.main flow run resnet -v \ -# --target spike --backend tvmaotplus \ -# -f validate \ -# --feature validate_new --post validate_outputs \ -# -c mlif.print_outputs=1 -c spike.print_outputs=1 \ -# -c set_inputs.interface=stdin -c get_outputs.interface=stdout -# python3 -m mlonmcu.cli.main flow run resnet -v \ -# --target spike --backend tvmaotplus \ -# -f validate \ -# --feature validate_new --post validate_outputs \ -# -c mlif.print_outputs=1 -c spike.print_outputs=1 \ -# -c set_inputs.interface=filesystem -c get_outputs.interface=filesystem -# python3 -m mlonmcu.cli.main flow run resnet -v \ -# --target spike --backend tvmaotplus \ -# -f validate \ -# --feature validate_new --post validate_outputs \ -# -c mlif.print_outputs=1 -c spike.print_outputs=0 \ -# -c set_inputs.interface=auto -c get_outputs.interface=auto -# combinations (e.g. filesystem+stdout) should also be tested! - -# platform: mlif target: host_x86` -# TODO: should support same configs as spike target -# TODO: gen_data/gen_ref_data -``` - -## TODOs -- [ ] Fix broken targets (due to refactoring of `self.exec`) -> PHILIPP -- [ ] Add missing target checks (see above) -> PHILIPP -- [ ] Update `definition.yml` for other models in `mlonmcu-sw` (At least `aww`, `vww`, `toycar`) -> LIU -- [x] Refactor model support (see `mlomcu_sw/lib/ml_interface`) to be aware of output/input tensor index (maybe even name?) und sample index -> PHILIPP -- [ ] Write generator for custom `mlif_override.cpp` (based on `model_info.yml` + `in_interface` + `out_interface` (+ `inputs.npy`)) -> LIU -- [ ] Eliminate hacks used to get `model_info.yml` and `inputs.yml` in RUN stage -> PHILIPP -- [ ] Implement missing interfaces for tvm (out: `stdout`) -> LIU -- [ ] Implement missing interfaces for mlif platform (in: `filesystem`, `stdin`; out: `filesystem`, `stdout`) -> LIU -- [x] Implement missing interfaces for mlif platform (in: `rom`) -> PHILIPP) -- [ ] Add support for multi-output/multi-input -> PHILIPP/LIU -- [x] Update `gen_data` & `gen_ref_data` feature (see NotImplementedErrors, respect fmt,...) -> PHILIPP -- [x] Implement model-based gen_ref_data mode (currenly only tflite inference) -> PHILIPP -- [ ] Move `gen_data` & `gen_ref_data` from LOAD stage to custom stage (remove dependency on tflite frontend) -> PHILIPP -- [ ] Test with targets: `tvm_cpu`, `host_x86`, `spike` (See example commands above) -> LIU -- [ ] Extend `validate_outputs` postprocess (Add `report`, implement `atol`/`rtol`, `fail_on_error`, `top-k`,...) -> LIU -- [ ] Add more validation data (at least 10 samples per model, either manually or using `gen_ref_outputs`) -> LIU -- [ ] Generate validation reports for a few models (`aww`, `vww`, `resnet`, `toycar`) and at least backends (`tvmaotplus`, `tflmi`) -> LIU -- [ ] Cleanup codebases (lint, remove prints, reuse code, create helper functions,...) -> LIU/PHILIPP -- [ ] Document usage of new validation feature -> LIU -- [ ] Add tests -> LIU/PHILIPP -- [ ] Streamline `model_info.yml` with BUILD stage `ModelInfo` -> PHILIPP -- [ ] Improve artifacts handling -> PHILIPP -- [x] Support automatic quantization of inputs (See `vww` and `toycar`) -> PHILIPP/LIU diff --git a/mlonmcu/feature/feature.py b/mlonmcu/feature/feature.py index 4f45ad0a6..e93bc2ef5 100644 --- a/mlonmcu/feature/feature.py +++ b/mlonmcu/feature/feature.py @@ -230,8 +230,8 @@ def get_run_config(self): def add_run_config(self, config): config.update(self.get_run_config()) - def get_postprocesses(self): - return [] + # def get_postprocesses(self): + # return [] - def add_postprocesses(self, postprocesses): - postprocesses.extend(self.get_postprocesses()) + # def add_postprocesses(self, postprocesses): + # postprocesses.extend(self.get_postprocesses()) diff --git a/mlonmcu/feature/features.py b/mlonmcu/feature/features.py index 5222aa22d..47636ff71 100644 --- a/mlonmcu/feature/features.py +++ b/mlonmcu/feature/features.py @@ -2434,9 +2434,9 @@ class ValidateNew(RunFeature): def __init__(self, features=None, config=None): super().__init__("validate_new", features=features, config=config) - def get_postprocesses(self): - # config = {} - # from mlonmcu.session.postprocess import ValidateOutputsPostprocess - # validate_outputs_postprocess = ValidateOutputsPostprocess(features=[], config=config) - # return [validate_outputs_postprocess] - return ["validate_outputs"] + # def get_postprocesses(self): + # # config = {} + # # from mlonmcu.session.postprocess import ValidateOutputsPostprocess + # # validate_outputs_postprocess = ValidateOutputsPostprocess(features=[], config=config) + # # return [validate_outputs_postprocess] + # return ["validate_outputs"] From a6a38c22ab1c4f2815a9fe142909689704d20c99 Mon Sep 17 00:00:00 2001 From: Philipp van Kempen Date: Mon, 9 Sep 2024 09:16:35 +0200 Subject: [PATCH 092/105] features: add docstrings to validation features --- mlonmcu/feature/features.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/mlonmcu/feature/features.py b/mlonmcu/feature/features.py index 47636ff71..ff0d0ac82 100644 --- a/mlonmcu/feature/features.py +++ b/mlonmcu/feature/features.py @@ -2231,7 +2231,7 @@ def add_target_config(self, target, config): @register_feature("gen_data") class GenData(FrontendFeature): # TODO: use custom stage instead of LOAD - """TODO""" + """Generate input data for validation.""" DEFAULTS = { **FeatureBase.DEFAULTS, @@ -2278,7 +2278,7 @@ def get_frontend_config(self, frontend): @register_feature("gen_ref_data", depends=["gen_data"]) class GenRefData(FrontendFeature): # TODO: use custom stage instead of LOAD - """TODO""" + """Generate reference outputs for validation.""" DEFAULTS = { **FeatureBase.DEFAULTS, @@ -2319,7 +2319,7 @@ def get_frontend_config(self, frontend): @register_feature("gen_ref_labels", depends=["gen_data"]) class GenRefLabels(FrontendFeature): # TODO: use custom stage instead of LOAD - """TODO""" + """Generate reference labels for classification.""" DEFAULTS = { **FeatureBase.DEFAULTS, @@ -2360,7 +2360,7 @@ def get_frontend_config(self, frontend): @register_feature("set_inputs") class SetInputs(PlatformFeature): # TODO: use custom stage instead of LOAD - """TODO""" + """Apply test inputs to model.""" DEFAULTS = { **FeatureBase.DEFAULTS, @@ -2389,7 +2389,7 @@ def get_platform_config(self, platform): @register_feature("get_outputs") class GetOutputs(PlatformFeature): # TODO: use custom stage instead of LOAD - """TODO""" + """Extract resulting outputs from model.""" DEFAULTS = { **FeatureBase.DEFAULTS, @@ -2425,7 +2425,7 @@ def get_platform_config(self, platform): @register_feature("validate_new", depends=["gen_data", "gen_ref_data", "set_inputs", "get_outputs"]) class ValidateNew(RunFeature): - """TODO""" + """Wrapper feature for enabling all validatioon related features at once.""" DEFAULTS = { **FeatureBase.DEFAULTS, From 40e90fe67f54a99804d09eea6f8c4c956692d324 Mon Sep 17 00:00:00 2001 From: Philipp van Kempen Date: Mon, 9 Sep 2024 09:17:44 +0200 Subject: [PATCH 093/105] features: cleanup comments --- mlonmcu/feature/features.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/mlonmcu/feature/features.py b/mlonmcu/feature/features.py index ff0d0ac82..26060d708 100644 --- a/mlonmcu/feature/features.py +++ b/mlonmcu/feature/features.py @@ -2378,8 +2378,6 @@ def interface(self): def get_platform_config(self, platform): assert platform in ["mlif", "tvm", "microtvm"] - # if platform in ["tvm", "mircotvm"]: - # assert self.interface in ["auto", "filesystem"] # if tvm/microtvm: allow using --fill-mode provided by tvmc run return { f"{platform}.set_inputs": self.enabled, @@ -2414,8 +2412,6 @@ def fmt(self): def get_platform_config(self, platform): assert platform in ["mlif", "tvm", "microtvm"] - # if platform in ["tvm", "mircotvm"]: - # assert self.interface in ["auto", "filesystem", "stdout"] return { f"{platform}.get_outputs": self.enabled, f"{platform}.get_outputs_interface": self.interface, From 34e0e5d17d119f2c898127ccaf4df875f3897cc7 Mon Sep 17 00:00:00 2001 From: Philipp van Kempen Date: Mon, 9 Sep 2024 09:19:37 +0200 Subject: [PATCH 094/105] tvm_target: rm prints --- mlonmcu/platform/tvm/tvm_target.py | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/mlonmcu/platform/tvm/tvm_target.py b/mlonmcu/platform/tvm/tvm_target.py index d0b9d9847..91e839f13 100644 --- a/mlonmcu/platform/tvm/tvm_target.py +++ b/mlonmcu/platform/tvm/tvm_target.py @@ -80,14 +80,11 @@ def exec(self, program, *args, cwd=os.getcwd(), **kwargs): import numpy as np data = np.load(self.platform.inputs_artifact, allow_pickle=True) - print("data", data, type(data)) num_inputs = len(data) ins_file = Path(cwd) / "ins.npz" outs_file = None print_top = self.platform.print_top - print("before IF") if self.platform.get_outputs: - print("IF") interface = self.platform.get_outputs_interface if interface == "auto": if self.supports_filesystem: @@ -110,15 +107,11 @@ def exec(self, program, *args, cwd=os.getcwd(), **kwargs): remaining_inputs = num_inputs outs_data = [] for idx in range(num_batches): - print("idx", idx) current_batch_size = max(min(batch_size, remaining_inputs), 1) assert current_batch_size == 1 if processed_inputs < num_inputs: - print("!!!") in_data = data[idx] - print("in_data", in_data, type(in_data)) np.savez(ins_file, **in_data) - print("saved!") processed_inputs += 1 remaining_inputs -= 1 else: @@ -127,10 +120,8 @@ def exec(self, program, *args, cwd=os.getcwd(), **kwargs): program, self, cwd=cwd, ins_file=ins_file, outs_file=outs_file, print_top=print_top ) ret += ret_ - print("self.platform.get_outputs", self.platform.get_outputs) if self.platform.get_outputs: interface = self.platform.get_outputs_interface - # print("interface", interface) if interface == "auto": if self.supports_filesystem: interface = "filesystem" @@ -147,8 +138,6 @@ def exec(self, program, *args, cwd=os.getcwd(), **kwargs): raise NotImplementedError else: assert False - print("outs_data", outs_data) - # input("$") if len(outs_data) > 0: outs_path = Path(cwd) / "outputs.npy" np.save(outs_path, outs_data) From d9d6a97a0710e1849d3be550b16d286ca01a5710 Mon Sep 17 00:00:00 2001 From: Philipp van Kempen Date: Mon, 9 Sep 2024 09:22:13 +0200 Subject: [PATCH 095/105] lint --- mlonmcu/session/run.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mlonmcu/session/run.py b/mlonmcu/session/run.py index 037186708..f7e576040 100644 --- a/mlonmcu/session/run.py +++ b/mlonmcu/session/run.py @@ -521,7 +521,7 @@ def add_frontends_by_name(self, frontend_names, context=None): if reasons: logger.error("Initialization of frontends was no successfull. Reasons: %s", reasons) else: - raise RuntimeError(f"No compatible frontend was found.") + raise RuntimeError("No compatible frontend was found.") self.add_frontends(frontends) def add_backend_by_name(self, backend_name, context=None): From aefd53711b8424897fd5f4adb91340f2aab11dc4 Mon Sep 17 00:00:00 2001 From: Philipp van Kempen Date: Mon, 9 Sep 2024 09:28:33 +0200 Subject: [PATCH 096/105] lint --- mlonmcu/setup/tasks/ekut.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/mlonmcu/setup/tasks/ekut.py b/mlonmcu/setup/tasks/ekut.py index e3db23a3a..146422f2d 100644 --- a/mlonmcu/setup/tasks/ekut.py +++ b/mlonmcu/setup/tasks/ekut.py @@ -48,10 +48,7 @@ def clone_hannah_tvm( """Clone the hannah-tvm repository.""" hannahTvmName = utils.makeDirName("hannah_tvm") hannahTvmSrcDir = context.environment.paths["deps"].path / "src" / hannahTvmName - if ( - rebuild - or not utils.is_populated(hannahTvmSrcDir) - ): + if rebuild or not utils.is_populated(hannahTvmSrcDir): pulpRtosRepo = context.environment.repos["hannah_tvm"] utils.clone(pulpRtosRepo.url, hannahTvmSrcDir, branch=pulpRtosRepo.ref, refresh=rebuild, recursive=True) context.cache["hannah_tvm.src_dir"] = hannahTvmSrcDir From 71389578ab8317abb58b349c8b40caf6709b6861 Mon Sep 17 00:00:00 2001 From: Philipp van Kempen Date: Mon, 9 Sep 2024 13:49:06 +0200 Subject: [PATCH 097/105] fix microtvm_gvsoc_target.py typo --- mlonmcu/platform/microtvm/microtvm_gvsoc_target.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mlonmcu/platform/microtvm/microtvm_gvsoc_target.py b/mlonmcu/platform/microtvm/microtvm_gvsoc_target.py index 510393a90..d164a1593 100644 --- a/mlonmcu/platform/microtvm/microtvm_gvsoc_target.py +++ b/mlonmcu/platform/microtvm/microtvm_gvsoc_target.py @@ -39,14 +39,14 @@ class GVSocMicroTvmPlatformTarget(TemplateMicroTvmPlatformTarget): # "xpulp_version": None, # None means that xpulp extension is not used, # "model": "pulp", } - REQUIRED = Target.REQUIRED + [ + REQUIRED = Target.REQUIRED | { "gvsoc.exe", "pulp_freertos.support_dir", "pulp_freertos.config_dir", "pulp_freertos.install_dir", "microtvm_gvsoc.template", "hannah_tvm.src_dir", - ] + } def __init__(self, name=None, features=None, config=None): super().__init__(name=name, features=features, config=config) From 2fd53f04692918bb91ef50ea493301608056b5c7 Mon Sep 17 00:00:00 2001 From: Philipp van Kempen Date: Thu, 12 Sep 2024 12:16:09 +0200 Subject: [PATCH 098/105] dev.yml.j2: update tflite-micro ref --- resources/templates/dev.yml.j2 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/resources/templates/dev.yml.j2 b/resources/templates/dev.yml.j2 index cb54a2668..1df7ba668 100644 --- a/resources/templates/dev.yml.j2 +++ b/resources/templates/dev.yml.j2 @@ -31,7 +31,7 @@ paths: repos: tensorflow: # TODO: rename to tflite-micro? url: "https://github.com/tensorflow/tflite-micro.git" - ref: ca5358f4680dbd94717a4c6bd77186bc1c799e1b + ref: 19aaea85e4679a9a2f265e07ba190ac5ea4d3766 options: single_branch: true # tflite_micro_compiler: From 9c9ef0e398494eaf08dd97e849414e31e326e7ee Mon Sep 17 00:00:00 2001 From: Philipp van Kempen Date: Thu, 12 Sep 2024 12:20:55 +0200 Subject: [PATCH 099/105] corev.yml.j2: update mlif ref --- resources/templates/corev.yml.j2 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/resources/templates/corev.yml.j2 b/resources/templates/corev.yml.j2 index c7cf38dad..9e3bd6a81 100644 --- a/resources/templates/corev.yml.j2 +++ b/resources/templates/corev.yml.j2 @@ -69,7 +69,7 @@ repos: ref: ffeca904368926d60caeb2d97858215626892f35 mlif: url: "https://github.com/tum-ei-eda/mlonmcu-sw.git" - ref: 7ba4ea6992093843720ae3494223cd910a64c828 + ref: 4f89b17aa257afeccbebbb883b775cd9af58b7a0 microtvm_etiss: url: "https://github.com/PhilippvK/microtvm-etiss-template.git" ref: 4460f539f6607b0c8b90321e7cb80e28d1e1fbe2 From e61699f15ed2b4b157112470c9044381307cab82 Mon Sep 17 00:00:00 2001 From: Philipp van Kempen Date: Thu, 26 Sep 2024 20:48:39 +0200 Subject: [PATCH 100/105] mlif: support ccache --- mlonmcu/platform/mlif/mlif.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/mlonmcu/platform/mlif/mlif.py b/mlonmcu/platform/mlif/mlif.py index feb071112..488fc9559 100644 --- a/mlonmcu/platform/mlif/mlif.py +++ b/mlonmcu/platform/mlif/mlif.py @@ -101,6 +101,7 @@ class MlifPlatform(CompilePlatform, TargetPlatform): "fuse_ld": None, "global_isel": False, "extend_attrs": False, + "ccache": False, } REQUIRED = {"mlif.src_dir"} @@ -119,6 +120,11 @@ def __init__(self, features=None, config=None): def goal(self): return self.config["goal"] + @property + def ccache(self): + value = self.config["ccache"] + return str2bool(value) if not isinstance(value, (bool, int)) else value + @property def set_inputs(self): value = self.config["set_inputs"] @@ -431,6 +437,9 @@ def get_definitions(self): definitions["STRIP_STRINGS"] = self.strip_strings if self.unroll_loops is not None: definitions["UNROLL_LOOPS"] = self.unroll_loops + if self.ccache: + definitions["CMAKE_C_COMPILER_LAUNCHER"] = "ccache" # TODO: choose between ccache/sccache + definitions["CMAKE_CXX_COMPILER_LAUNCHER"] = "ccache" # TODO: choose between ccache/sccache return definitions From 3609918d3bdb8c4110eeef1783dfa998bf01451b Mon Sep 17 00:00:00 2001 From: Philipp van Kempen Date: Thu, 26 Sep 2024 20:49:12 +0200 Subject: [PATCH 101/105] mlif: assert that llvm.install_dir exists --- mlonmcu/platform/mlif/mlif.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/mlonmcu/platform/mlif/mlif.py b/mlonmcu/platform/mlif/mlif.py index 488fc9559..d6b68588a 100644 --- a/mlonmcu/platform/mlif/mlif.py +++ b/mlonmcu/platform/mlif/mlif.py @@ -405,10 +405,12 @@ def get_definitions(self): definitions["BATCH_SIZE"] = self.batch_size if self.num_threads is not None: definitions["SUBPROJECT_THREADS"] = self.num_threads - if self.toolchain == "llvm" and self.llvm_dir is None: - raise RuntimeError("Missing config variable: llvm.install_dir") - else: - definitions["LLVM_DIR"] = self.llvm_dir + if self.toolchain == "llvm": + if self.llvm_dir is None: + raise RuntimeError("Missing config variable: llvm.install_dir") + llvm_dir = Path(self.llvm_dir).resolve() + assert llvm_dir.is_dir(), f"llvm.install_dir does not exist: {llvm_dir}" + definitions["LLVM_DIR"] = llvm_dir if self.optimize is not None: definitions["OPTIMIZE"] = self.optimize if self.debug_symbols is not None: From 9efdcb46e0e14f973d05bd6acc70aea26a3918cc Mon Sep 17 00:00:00 2001 From: Philipp van Kempen Date: Fri, 27 Sep 2024 23:42:04 +0200 Subject: [PATCH 102/105] templates: use fixes branch for polybench fork --- resources/templates/ara.yml.j2 | 4 ++-- resources/templates/dev.yml.j2 | 4 ++-- resources/templates/vicuna.yml.j2 | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/resources/templates/ara.yml.j2 b/resources/templates/ara.yml.j2 index 2abf39dcf..11e75695a 100644 --- a/resources/templates/ara.yml.j2 +++ b/resources/templates/ara.yml.j2 @@ -70,8 +70,8 @@ repos: url: "https://github.com/tacle/tacle-bench.git" ref: master polybench: - url: "https://github.com/MatthiasJReisinger/PolyBenchC-4.2.1.git" - ref: master + url: "https://github.com/PhilippvK/PolyBenchC-4.2.1.git" + ref: fixes mibench: url: "https://github.com/embecosm/mibench.git" ref: master diff --git a/resources/templates/dev.yml.j2 b/resources/templates/dev.yml.j2 index 1df7ba668..cec288357 100644 --- a/resources/templates/dev.yml.j2 +++ b/resources/templates/dev.yml.j2 @@ -115,8 +115,8 @@ repos: url: "https://github.com/tacle/tacle-bench.git" ref: master polybench: - url: "https://github.com/MatthiasJReisinger/PolyBenchC-4.2.1.git" - ref: master + url: "https://github.com/PhilippvK/PolyBenchC-4.2.1.git" + ref: fixes mibench: url: "https://github.com/embecosm/mibench.git" ref: master diff --git a/resources/templates/vicuna.yml.j2 b/resources/templates/vicuna.yml.j2 index 3888f2467..15bbd61e5 100644 --- a/resources/templates/vicuna.yml.j2 +++ b/resources/templates/vicuna.yml.j2 @@ -88,8 +88,8 @@ repos: url: "https://github.com/tacle/tacle-bench.git" ref: master polybench: - url: "https://github.com/MatthiasJReisinger/PolyBenchC-4.2.1.git" - ref: master + url: "https://github.com/PhilippvK/PolyBenchC-4.2.1.git" + ref: fixes mibench: url: "https://github.com/embecosm/mibench.git" ref: master From 3a15a042bac1cd544df13714af7b2a9d84f4d0f1 Mon Sep 17 00:00:00 2001 From: Philipp van Kempen Date: Fri, 27 Sep 2024 23:56:05 +0200 Subject: [PATCH 103/105] Polybench: allow passing dataset size --- mlonmcu/models/frontend.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/mlonmcu/models/frontend.py b/mlonmcu/models/frontend.py index 905247fbd..ebec375e0 100644 --- a/mlonmcu/models/frontend.py +++ b/mlonmcu/models/frontend.py @@ -1697,6 +1697,12 @@ def get_platform_config(self, platform): class PolybenchFrontend(SimpleFrontend): + + DEFAULTS = { + **Frontend.DEFAULTS, + "dataset": "large", # mini/small/medium/large/extralarge + } + REQUIRED = {"polybench.src_dir"} def __init__(self, features=None, config=None): @@ -1769,6 +1775,7 @@ def get_platform_defs(self, platform): ret = {} if platform == "mlif": ret["POLYBENCH_DIR"] = Path(self.config["polybench.src_dir"]) + ret["POLYBENCH_DATASET"] = self.config["dataset"].upper() + "_DATASET" return ret def get_platform_config(self, platform): From c7599dfd233982b35152176d3e7a66fc6c529135 Mon Sep 17 00:00:00 2001 From: Philipp van Kempen Date: Fri, 27 Sep 2024 23:57:27 +0200 Subject: [PATCH 104/105] dev.yml.j2: update mlif ref --- resources/templates/dev.yml.j2 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/resources/templates/dev.yml.j2 b/resources/templates/dev.yml.j2 index cec288357..6561a66cc 100644 --- a/resources/templates/dev.yml.j2 +++ b/resources/templates/dev.yml.j2 @@ -81,7 +81,7 @@ repos: ref: fc42c71353d15c564558249bd4f13350119ab6a9 mlif: url: "https://github.com/tum-ei-eda/mlonmcu-sw.git" - ref: 4f89b17aa257afeccbebbb883b775cd9af58b7a0 + ref: f74feb4b27b44f8caa9b89c7df1545579563f5be # espidf: # url: "https://github.com/espressif/esp-idf.git" # ref: release/v4.4 From 7495fadd1b4606b7363a1d594b89510046c7d0b3 Mon Sep 17 00:00:00 2001 From: Philipp van Kempen Date: Mon, 26 Feb 2024 23:37:14 +0100 Subject: [PATCH 105/105] add support for openasip example models --- mlonmcu/models/__init__.py | 2 ++ mlonmcu/models/frontend.py | 48 ++++++++++++++++++++++++++++++++++++++ mlonmcu/models/model.py | 18 ++++++++++++++ 3 files changed, 68 insertions(+) diff --git a/mlonmcu/models/__init__.py b/mlonmcu/models/__init__.py index e2307fbe7..d469e2372 100644 --- a/mlonmcu/models/__init__.py +++ b/mlonmcu/models/__init__.py @@ -33,6 +33,7 @@ MathisFrontend, MibenchFrontend, LayerGenFrontend, + OpenASIPFrontend, ) SUPPORTED_FRONTENDS = { @@ -51,6 +52,7 @@ "mathis": MathisFrontend, "mibench": MibenchFrontend, "layergen": LayerGenFrontend, + "openasip": OpenASIPFrontend, } # TODO: use registry instead __all__ = [ diff --git a/mlonmcu/models/frontend.py b/mlonmcu/models/frontend.py index ebec375e0..23ff846e0 100644 --- a/mlonmcu/models/frontend.py +++ b/mlonmcu/models/frontend.py @@ -38,6 +38,7 @@ DhrystoneProgram, MathisProgram, MibenchProgram, + OpenASIPProgram, ) from mlonmcu.models.lookup import lookup_models from mlonmcu.feature.type import FeatureType @@ -2094,3 +2095,50 @@ def helper(args): artifact = Artifact(f"{name}.{ext}", raw=raw, fmt=ArtifactFormat.RAW, flags=["model"]) artifacts[name] = [artifact] return artifacts, {} + + +class OpenASIPFrontend(SimpleFrontend): + + def __init__(self, features=None, config=None): + super().__init__( + "openasip", + ModelFormats.NONE, + features=features, + config=config, + ) + + @property + def supported_names(self): + return [ + "sha256", + "aes", + "crc", + ] + + # @property + # def skip_backend(self): + # return True + + def lookup_models(self, names, config=None, context=None): + ret = [] + for name in names: + name = name.replace("openasip/", "") + if name in self.supported_names: + hint = OpenASIPProgram( + name, + alt=f"openasip/{name}", + config=config, + ) + ret.append(hint) + return ret + + def generate(self, model) -> Tuple[dict, dict]: + artifacts = [Artifact("dummy_model", raw=bytes(), fmt=ArtifactFormat.RAW, flags=["model", "dummy"])] + + return {"default": artifacts}, {} + + def get_platform_config(self, platform): + ret = {} + if platform == "mlif": + ret["template"] = "openasip" + return ret diff --git a/mlonmcu/models/model.py b/mlonmcu/models/model.py index eafe3aa61..4dd903741 100644 --- a/mlonmcu/models/model.py +++ b/mlonmcu/models/model.py @@ -413,3 +413,21 @@ def get_platform_defs(self, platform): if platform == "mlif": ret["DHRYSTONE_ITERATIONS"] = 10000 return ret + + +class OpenASIPProgram(Program): + DEFAULTS = { + "crc_mode": "both", + } + + @property + def crc_mode(self): + return str(self.config["crc_mode"]) + + def get_platform_defs(self, platform): + ret = {} + if platform == "mlif": + ret["OPENASIP_BENCHMARK"] = self.name + if self.name == "crc": + ret["OPENASIP_CRC_MODE"] = self.crc_mode + return ret