Merge branch 'develop' of github.com:tum-ei-eda/mlonmcu into develop

mlonmcu/target/_target.py

@@ -25,9 +25,11 @@
     AraRtlTarget,
     CV32E40PTarget,
     VicunaTarget,
+    CanMvK230SSHTarget,
 )
 from .arm import Corstone300Target
 from .host_x86 import HostX86Target
+from .host_x86_ssh import HostX86SSHTarget
 
 TARGET_REGISTRY = {}
 
@@ -47,6 +49,7 @@ def get_targets():
 register_target("etiss_pulpino", EtissPulpinoTarget)
 register_target("etiss", EtissTarget)
 register_target("host_x86", HostX86Target)
+register_target("host_x86_ssh", HostX86SSHTarget)
 register_target("corstone300", Corstone300Target)
 register_target("spike", SpikeTarget)
 register_target("ovpsim", OVPSimTarget)
@@ -57,3 +60,4 @@ def get_targets():
 register_target("ara_rtl", AraRtlTarget)
 register_target("cv32e40p", CV32E40PTarget)
 register_target("vicuna", VicunaTarget)
+register_target("canmv_k230_ssh", CanMvK230SSHTarget)

mlonmcu/target/host_x86_ssh.py

@@ -0,0 +1,60 @@
+#
+# Copyright (c) 2022 TUM Department of Electrical and Computer Engineering.
+#
+# This file is part of MLonMCU.
+# See https://github.com/tum-ei-eda/mlonmcu.git for further info.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+"""MLonMCU Host/x86 Target definitions"""
+
+import stat
+from pathlib import Path
+
+from mlonmcu.config import str2bool
+from mlonmcu.setup.utils import execute
+from .common import cli
+from .ssh_target import SSHTarget
+from .host_x86 import HostX86Target
+
+
+class HostX86SSHTarget(SSHTarget, HostX86Target):
+    """TODO"""
+
+    FEATURES = SSHTarget.FEATURES | HostX86Target.FEATURES  # TODO: do not allow gdbserver
+
+    DEFAULTS = {
+        **SSHTarget.DEFAULTS,
+        **HostX86Target.DEFAULTS,
+    }
+
+    def __init__(self, name="host_x86_ssh", features=None, config=None):
+        super().__init__(name, features=features, config=config)
+
+    def exec(self, program, *args, handle_exit=None, **kwargs):
+        if self.gdbserver_enable:
+            raise NotImplementedError("gdbserver via ssh")
+
+        output = self.exec_via_ssh(program, *args, **kwargs)
+        if handle_exit:
+            exit_code = handle_exit(0, out=output)
+            print("exit_code", exit_code)
+            assert exit_code == 0
+        return output, []
+
+    def get_target_system(self):
+        return "host_x86"
+
+
+if __name__ == "__main__":
+    cli(target=HostX86SSHTarget)

mlonmcu/target/riscv/__init__.py

@@ -9,6 +9,7 @@
 from .ara_rtl import AraRtlTarget
 from .cv32e40p import CV32E40PTarget
 from .vicuna import VicunaTarget
+from .canmv_k230_ssh import CanMvK230SSHTarget
 
 __all__ = [
     "EtissPulpinoTarget",

mlonmcu/target/riscv/canmv_k230_ssh.py

@@ -0,0 +1,116 @@
+#
+# Copyright (c) 2022 TUM Department of Electrical and Computer Engineering.
+#
+# This file is part of MLonMCU.
+# See https://github.com/tum-ei-eda/mlonmcu.git for further info.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+"""MLonMCU Spike Target definitions"""
+
+import os
+import time
+# from pathlib import Path
+
+from mlonmcu.logging import get_logger
+from mlonmcu.setup.utils import execute
+from mlonmcu.target.common import cli
+from mlonmcu.target.metrics import Metrics
+from mlonmcu.target.ssh_target import SSHTarget
+from mlonmcu.target.bench import add_bench_metrics
+from .riscv_vext_target import RVVTarget
+
+logger = get_logger()
+
+
+class CanMvK230SSHTarget(SSHTarget, RVVTarget):
+    """TODO."""
+
+    FEATURES = SSHTarget.FEATURES | RVVTarget.FEATURES
+
+    DEFAULTS = {
+        **SSHTarget.DEFAULTS,
+        **RVVTarget.DEFAULTS,
+        "xlen": 64,
+        "vlen": 128,
+    }
+    REQUIRED = SSHTarget.REQUIRED | RVVTarget.REQUIRED
+
+    def __init__(self, name="canmv_k230_ssh", features=None, config=None):
+        super().__init__(name, features=features, config=config)
+
+    def exec(self, program, *args, cwd=os.getcwd(), handle_exit=None, **kwargs):
+        """Use target to execute a executable with given arguments"""
+
+        if self.enable_vext:
+            assert self.vlen == 128, "CanMV K230 only supports VLEN=128"
+
+        if self.timeout_sec > 0:
+            raise NotImplementedError
+
+        output = self.exec_via_ssh(program, *args, **kwargs)
+        if handle_exit:
+            exit_code = handle_exit(0, out=output)
+            print("exit_code", exit_code)
+            assert exit_code == 0
+        return output
+
+    def parse_stdout(self, out, metrics, exit_code=0):
+        add_bench_metrics(out, metrics, exit_code != 0, target_name=self.name)
+
+    def get_metrics(self, elf, directory, *args, handle_exit=None):
+        out = ""
+
+        def _handle_exit(code, out=None):
+            print("_handle_exit (k230)", code)
+            assert out is not None
+            temp = self.parse_exit(out)
+            # TODO: before or after?
+            if temp is None:
+                temp = code
+            if handle_exit is not None:
+                temp = handle_exit(temp, out=out)
+            return temp
+
+        start_time = time.time()
+        if self.print_outputs:
+            out = self.exec(elf, *args, cwd=directory, live=True, handle_exit=_handle_exit)
+        else:
+            out = self.exec(
+                elf, *args, cwd=directory, live=False, print_func=lambda *args, **kwargs: None, handle_exit=_handle_exit
+            )
+        # TODO: do something with out?
+        end_time = time.time()
+        diff = end_time - start_time
+        # size instead of readelf?
+
+        # TODO: get exit code
+        exit_code = 0
+        metrics = Metrics()
+        self.parse_stdout(out, metrics, exit_code=exit_code)
+
+        if metrics.has("Simulated Instructions"):
+            sim_insns = metrics.get("Simulated Instructions")
+            if diff > 0:
+                metrics.add("MIPS", (sim_insns / diff) / 1e6, True)
+
+        return metrics, out, []
+
+    def get_platform_defs(self, platform):
+        ret = {}
+        ret.update(RVVTarget.get_platform_defs(self, platform))
+        return ret
+
+
+if __name__ == "__main__":
+    cli(target=CanMvK230SSHTarget)

mlonmcu/target/ssh_target.py

@@ -0,0 +1,169 @@
+#
+# Copyright (c) 2022 TUM Department of Electrical and Computer Engineering.
+#
+# This file is part of MLonMCU.
+# See https://github.com/tum-ei-eda/mlonmcu.git for further info.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+"""MLonMCU SSH Target definitions"""
+
+import os
+import re
+# import tempfile
+# import time
+import socket
+from pathlib import Path
+
+import paramiko
+from paramiko import BadHostKeyException, AuthenticationException, SSHException
+
+from mlonmcu.config import str2bool
+from .target import Target
+
+
+class SSHTarget(Target):
+    """TODO"""
+
+    DEFAULTS = {
+        **Target.DEFAULTS,
+        "hostname": None,
+        "port": 22,
+        "username": None,
+        "password": None,
+        "ignore_known_hosts": True,
+        "workdir": None,
+    }
+
+    @property
+    def hostname(self):
+        value = self.config["hostname"]
+        assert value is not None, "hostname not defined"
+        return value
+
+    @property
+    def port(self):
+        value = self.config["port"]
+        if isinstance(value, str):
+            value = int(value)
+        assert isinstance(value, int)
+        return value
+
+    @property
+    def username(self):
+        value = self.config["username"]
+        return value
+
+    @property
+    def password(self):
+        value = self.config["password"]
+        return value
+
+    @property
+    def ignore_known_hosts(self):
+        value = self.config["ignore_known_hosts"]
+        return str2bool(value) if not isinstance(value, (bool, int)) else value
+
+    @property
+    def workdir(self):
+        value = self.config["workdir"]
+        if value is not None:
+            if isinstance(value, str):
+                value = Path(value)
+            assert isinstance(value, Path)
+        return value
+
+    def __repr__(self):
+        return f"SSHTarget({self.name})"
+
+    def check_remote(self):
+        ssh = paramiko.SSHClient()
+        try:
+            ssh.connect(self.hostname, port=self.port, username=self.username, password=self.password)
+            # TODO: key_filename=key_file)
+            return True
+        except (BadHostKeyException, AuthenticationException,
+                SSHException, socket.error) as e:
+            print(e)  # TODO: remove
+            return False
+        raise NotImplementedError
+
+    def create_remote_directory(self, path):
+        ssh = paramiko.SSHClient()
+        # ssh.load_host_keys(os.path.expanduser(os.path.join("~", ".ssh", "known_hosts")))
+        if self.ignore_known_hosts:
+            ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+        ssh.connect(self.hostname, port=self.port, username=self.username, password=self.password)
+        command = f"mkdir -p {path}"
+        stdin, stdout, stderr = ssh.exec_command(command)
+        ssh.close()
+
+    def copy_to_remote(self, src, dest):
+        ssh = paramiko.SSHClient()
+        # ssh.load_host_keys(os.path.expanduser(os.path.join("~", ".ssh", "known_hosts")))
+        if self.ignore_known_hosts:
+            ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+        ssh.connect(self.hostname, port=self.port, username=self.username, password=self.password)
+        sftp = ssh.open_sftp()
+        sftp.put(str(src), str(dest))
+        sftp.close()
+        ssh.close()
+
+    def copy_from_remote(self, src, dest):
+        ssh = paramiko.SSHClient()
+        # ssh.load_host_keys(os.path.expanduser(os.path.join("~", ".ssh", "known_hosts")))
+        if self.ignore_known_hosts:
+            ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+        ssh.connect(self.hostname, port=self.port, username=self.username, password=self.password)
+        sftp = ssh.open_sftp()
+        sftp.get(str(src), str(dest))
+        sftp.close()
+        ssh.close()
+
+    def parse_exit(self, out):
+        print("parse_exit (ssh_target)")
+        exit_code = super().parse_exit(out)
+        exit_match = re.search(r"SSH EXIT=(.*)", out)
+        if exit_match:
+            exit_code = int(exit_match.group(1))
+        return exit_code
+
+    def exec_via_ssh(self, program: Path, *args, cwd=os.getcwd(), **kwargs):
+        # TODO: keep connection established!
+        self.check_remote()
+        if self.workdir is None:
+            raise NotImplementedError("temp workdir")
+        else:
+            self.create_remote_directory(self.workdir)
+            workdir = self.workdir
+        remote_program = workdir / program.name
+        self.copy_to_remote(program, remote_program)
+        ssh = paramiko.SSHClient()
+        # ssh.load_host_keys(os.path.expanduser(os.path.join("~", ".ssh", "known_hosts")))
+        if self.ignore_known_hosts:
+            ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+        ssh.connect(self.hostname, port=self.port, username=self.username, password=self.password)
+        args_str = " ".join(args)
+        command = f"cd {workdir} && chmod +x {remote_program} && {remote_program} {args_str}; echo SSH EXIT=$?"
+        stdin, stdout, stderr = ssh.exec_command(command)
+        # print("stdin", stdin)
+        # print("stdout", stdout)
+        # print("stderr", stderr)
+        output = stderr.read().strip() + stdout.read().strip()
+        output = output.decode()
+        if self.print_outputs:
+            print("output", output)  # TODO: cleanup
+        ssh.close()
+        return output
+
+# TODO: logger