diff --git a/.github/workflows/bench.yml b/.github/workflows/bench.yml index 97351d281..7877dcc98 100644 --- a/.github/workflows/bench.yml +++ b/.github/workflows/bench.yml @@ -36,11 +36,11 @@ jobs: matrix: python-version: ["3.10"] steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 with: submodules: recursive - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} # TODO: caching @@ -76,7 +76,7 @@ jobs: cd scripts/ python bench.py ${{ github.event.inputs.benchmark }} out/ - name: Archive reports - uses: actions/upload-artifact@v2 + uses: actions/upload-artifact@v4 with: name: results path: scripts/out/ diff --git a/.github/workflows/cicd.yml b/.github/workflows/cicd.yml index 2be233dbd..e3a37a528 100644 --- a/.github/workflows/cicd.yml +++ b/.github/workflows/cicd.yml @@ -44,12 +44,12 @@ jobs: matrix: python-version: ["3.7", "3.8", "3.9", "3.10", "3.11"] steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} - - uses: actions/cache@v3 + - uses: actions/cache@v4 id: cache-venv # name for referring later with: path: | @@ -84,12 +84,12 @@ jobs: python-version: ["3.10"] if: github.ref == 'refs/heads/main' || startsWith(github.ref, 'refs/tags/v') steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} - - uses: actions/cache@v3 + - uses: actions/cache@v4 id: cache-venv # name for referring later with: path: | @@ -100,7 +100,7 @@ jobs: restore-keys: | build-${{ runner.os }}-${{ matrix.python-version }}-venv-${{ hashFiles('**/requirements*.txt') }} - name: Archive package - uses: actions/upload-artifact@v2 + uses: actions/upload-artifact@v4 with: name: mlonmcu path: dist/ @@ -118,12 +118,12 @@ jobs: matrix: python-version: ["3.10"] steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} - - uses: actions/cache@v3 + - uses: actions/cache@v4 id: cache-venv # name for referring later with: path: | @@ -156,7 +156,7 @@ jobs: source .venv/bin/activate make coverage-full - name: Archive code coverage html report - uses: actions/upload-artifact@v2 + uses: actions/upload-artifact@v4 with: name: code-coverage-report path: htmlcov @@ -182,12 +182,12 @@ jobs: matrix: python-version: ["3.10"] steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} - - uses: actions/cache@v3 + - uses: actions/cache@v4 id: cache-venv # name for referring later with: path: | @@ -215,7 +215,7 @@ jobs: source .venv/bin/activate make docs - name: Deploy docs - uses: peaceiris/actions-gh-pages@v3 + uses: peaceiris/actions-gh-pages@v4 if: ${{ github.ref == 'refs/heads/main' }} with: github_token: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/container.yml b/.github/workflows/container.yml index cf518cc0a..bbf928aad 100644 --- a/.github/workflows/container.yml +++ b/.github/workflows/container.yml @@ -52,15 +52,15 @@ jobs: remove-haskell: 'true' remove-codeql: 'true' - name: Checkout - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: submodules: recursive - name: Set up QEMU - uses: docker/setup-qemu-action@v2 + uses: docker/setup-qemu-action@v3 - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v2 + uses: docker/setup-buildx-action@v3 - name: Login to GitHub Container Registry - uses: docker/login-action@v2 + uses: docker/login-action@v3 with: registry: ghcr.io username: ${{ github.actor }} @@ -70,11 +70,11 @@ jobs: ./scripts/update_version.sh - name: Lowercase repository url id: lowered - uses: ASzc/change-string-case-action@v5 + uses: ASzc/change-string-case-action@v6 with: string: ${{ github.repository }} - name: Build and push (CMake) - uses: docker/build-push-action@v4 + uses: docker/build-push-action@v6 with: context: . file: docker/Dockerfile @@ -88,7 +88,7 @@ jobs: cache-to: type=inline tags: ghcr.io/${{ steps.lowered.outputs.lowercase }}-cmake:${{ github.event.inputs.version }} - name: Build and push - uses: docker/build-push-action@v4 + uses: docker/build-push-action@v6 with: context: . file: docker/Dockerfile @@ -118,15 +118,15 @@ jobs: remove-haskell: 'true' remove-codeql: 'true' - name: Checkout - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: submodules: recursive - name: Set up QEMU - uses: docker/setup-qemu-action@v2 + uses: docker/setup-qemu-action@v3 - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v2 + uses: docker/setup-buildx-action@v3 - name: Login to GitHub Container Registry - uses: docker/login-action@v2 + uses: docker/login-action@v3 with: registry: ghcr.io username: ${{ github.actor }} @@ -136,11 +136,11 @@ jobs: ./scripts/update_version.sh - name: Lowercase repository url id: lowered - uses: ASzc/change-string-case-action@v5 + uses: ASzc/change-string-case-action@v6 with: string: ${{ github.repository }} # - name: Build and push - # uses: docker/build-push-action@v4 + # uses: docker/build-push-action@v6 # with: # context: . # file: docker/Dockerfile @@ -153,14 +153,14 @@ jobs: # cache-to: type=inline # tags: ghcr.io/${{ steps.lowered.outputs.lowercase }}:${{ github.event.inputs.version }} - name: Login to DockerHub - uses: docker/login-action@v2 + uses: docker/login-action@v3 if: ${{ github.repository == 'tum-ei-eda/mlonmcu' }} with: registry: ${{ secrets.DOCKER_REGISTRY }} username: ${{ secrets.DOCKER_USERNAME }} password: ${{ secrets.DOCKER_PASSWORD }} - name: Build and push to Docker Hub - uses: docker/build-push-action@v4 + uses: docker/build-push-action@v6 if: ${{ github.repository == 'tum-ei-eda/mlonmcu' }} with: context: . @@ -192,26 +192,26 @@ jobs: remove-haskell: 'true' remove-codeql: 'true' - name: Checkout - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: submodules: recursive - name: Set up QEMU - uses: docker/setup-qemu-action@v2 + uses: docker/setup-qemu-action@v3 - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v2 + uses: docker/setup-buildx-action@v3 - name: Login to GitHub Container Registry - uses: docker/login-action@v2 + uses: docker/login-action@v3 with: registry: ghcr.io username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} - name: Lowercase repository url id: lowered - uses: ASzc/change-string-case-action@v5 + uses: ASzc/change-string-case-action@v6 with: string: ${{ github.repository }} # - name: Build and push - # uses: docker/build-push-action@v4 + # uses: docker/build-push-action@v6 # with: # context: . # file: docker/Dockerfile @@ -227,14 +227,14 @@ jobs: # cache-to: type=inline # tags: ghcr.io/${{ steps.lowered.outputs.lowercase }}-bench:${{ github.event.inputs.version }} - name: Login to DockerHub - uses: docker/login-action@v2 + uses: docker/login-action@v3 if: ${{ github.repository == 'tum-ei-eda/mlonmcu' }} with: registry: ${{ secrets.DOCKER_REGISTRY }} username: ${{ secrets.DOCKER_USERNAME }} password: ${{ secrets.DOCKER_PASSWORD }} - name: Build and push to Docker Hub - uses: docker/build-push-action@v4 + uses: docker/build-push-action@v6 if: ${{ github.repository == 'tum-ei-eda/mlonmcu' }} with: context: . diff --git a/.github/workflows/container_weekly.yml b/.github/workflows/container_weekly.yml index 3aaf3157a..131324442 100644 --- a/.github/workflows/container_weekly.yml +++ b/.github/workflows/container_weekly.yml @@ -47,7 +47,7 @@ jobs: id: timestamp - name: Lowercase repository url id: lowered - uses: ASzc/change-string-case-action@v5 + uses: ASzc/change-string-case-action@v6 with: string: ${{ github.repository }} - name: CFG @@ -109,21 +109,21 @@ jobs: remove-haskell: 'true' remove-codeql: 'true' - name: Checkout - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: ref: ${{ steps.cfg.outputs.branch }} - name: Set up QEMU - uses: docker/setup-qemu-action@v2 + uses: docker/setup-qemu-action@v3 - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v2 + uses: docker/setup-buildx-action@v3 - name: Login to GitHub Container Registry - uses: docker/login-action@v2 + uses: docker/login-action@v3 with: registry: ghcr.io username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} - name: Build and push (CMake) - uses: docker/build-push-action@v4 + uses: docker/build-push-action@v6 with: context: . file: docker/Dockerfile @@ -138,7 +138,7 @@ jobs: cache-to: type=inline tags: ghcr.io/${{ steps.lowered.outputs.lowercase }}-cmake:latest - name: Build and push - uses: docker/build-push-action@v4 + uses: docker/build-push-action@v6 with: context: . file: docker/Dockerfile @@ -166,7 +166,7 @@ jobs: id: timestamp - name: Lowercase repository url id: lowered - uses: ASzc/change-string-case-action@v5 + uses: ASzc/change-string-case-action@v6 with: string: ${{ github.repository }} - name: CFG @@ -228,28 +228,28 @@ jobs: remove-haskell: 'true' remove-codeql: 'true' - name: Checkout - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: ref: ${{ steps.cfg.outputs.branch }} - name: Set up QEMU - uses: docker/setup-qemu-action@v2 + uses: docker/setup-qemu-action@v3 - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v2 + uses: docker/setup-buildx-action@v3 - name: Login to GitHub Container Registry - uses: docker/login-action@v2 + uses: docker/login-action@v3 with: registry: ghcr.io username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} - name: Login to DockerHub - uses: docker/login-action@v2 + uses: docker/login-action@v3 if: ${{ github.repository == 'tum-ei-eda/mlonmcu' }} with: registry: ${{ secrets.DOCKER_REGISTRY }} username: ${{ secrets.DOCKER_USERNAME }} password: ${{ secrets.DOCKER_PASSWORD }} - name: Build and push to Docker Hub - uses: docker/build-push-action@v4 + uses: docker/build-push-action@v6 if: ${{ github.repository == 'tum-ei-eda/mlonmcu' }} with: context: . @@ -278,7 +278,7 @@ jobs: id: timestamp - name: Lowercase repository url id: lowered - uses: ASzc/change-string-case-action@v5 + uses: ASzc/change-string-case-action@v6 with: string: ${{ github.repository }} - name: CFG @@ -340,28 +340,28 @@ jobs: remove-haskell: 'true' remove-codeql: 'true' - name: Checkout - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: ref: ${{ steps.cfg.outputs.branch }} - name: Set up QEMU - uses: docker/setup-qemu-action@v2 + uses: docker/setup-qemu-action@v3 - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v2 + uses: docker/setup-buildx-action@v3 - name: Login to GitHub Container Registry - uses: docker/login-action@v2 + uses: docker/login-action@v3 with: registry: ghcr.io username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} - name: Login to DockerHub - uses: docker/login-action@v2 + uses: docker/login-action@v3 if: ${{ github.repository == 'tum-ei-eda/mlonmcu' }} with: registry: ${{ secrets.DOCKER_REGISTRY }} username: ${{ secrets.DOCKER_USERNAME }} password: ${{ secrets.DOCKER_PASSWORD }} - name: Build and push to Docker Hub - uses: docker/build-push-action@v4 + uses: docker/build-push-action@v6 if: ${{ github.repository == 'tum-ei-eda/mlonmcu' }} with: context: . diff --git a/.github/workflows/demo.yml b/.github/workflows/demo.yml index e1dbd3a57..1e374421f 100644 --- a/.github/workflows/demo.yml +++ b/.github/workflows/demo.yml @@ -66,11 +66,11 @@ jobs: remove-android: 'true' remove-haskell: 'true' remove-codeql: 'true' - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 with: submodules: recursive - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} # TODO: caching @@ -127,7 +127,7 @@ jobs: mlonmcu cleanup -H home/ -f --deps if: ${{ github.event.inputs.artifact == 'true' }} - name: Archive environment (without deps) - uses: actions/upload-artifact@v2 + uses: actions/upload-artifact@v4 with: name: mlonmcu_home path: home/ diff --git a/.github/workflows/docker_bench.yml b/.github/workflows/docker_bench.yml index 5a8f95165..360d7e01f 100644 --- a/.github/workflows/docker_bench.yml +++ b/.github/workflows/docker_bench.yml @@ -50,7 +50,7 @@ jobs: - name: Store cmdline run: echo "${{ github.event.inputs.bench_cmd }}" >> /environment/results/cmd.txt - name: Archive reports - uses: actions/upload-artifact@v2 + uses: actions/upload-artifact@v4 with: name: results path: /environment/results/ diff --git a/.github/workflows/notebook.yml b/.github/workflows/notebook.yml index d764cbc3f..dbf22692f 100644 --- a/.github/workflows/notebook.yml +++ b/.github/workflows/notebook.yml @@ -45,11 +45,11 @@ jobs: remove-android: 'true' remove-haskell: 'true' remove-codeql: 'true' - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 with: submodules: recursive - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} # TODO: caching @@ -73,7 +73,7 @@ jobs: - name: Get date run: echo "timestamp=`date +%FT%T`" >> $GITHUB_ENV - name: Create Pull Request - uses: peter-evans/create-pull-request@v5 + uses: peter-evans/create-pull-request@v6 if: github.event_name == 'workflow_dispatch' && github.event.inputs.branch != '' with: # token: ${{ secrets.PAT }} diff --git a/.github/workflows/refresh_container.yml b/.github/workflows/refresh_container.yml index ccd828c81..f8f957324 100644 --- a/.github/workflows/refresh_container.yml +++ b/.github/workflows/refresh_container.yml @@ -48,33 +48,33 @@ jobs: remove-haskell: 'true' remove-codeql: 'true' - name: Checkout - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: submodules: recursive - name: Set up QEMU - uses: docker/setup-qemu-action@v2 + uses: docker/setup-qemu-action@v3 - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v2 + uses: docker/setup-buildx-action@v3 - name: Login to GitHub Container Registry - uses: docker/login-action@v2 + uses: docker/login-action@v3 with: registry: ghcr.io username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} - name: Lowercase repository url id: lowered - uses: ASzc/change-string-case-action@v5 + uses: ASzc/change-string-case-action@v6 with: string: ${{ github.repository }} - name: Login to DockerHub - uses: docker/login-action@v2 + uses: docker/login-action@v3 if: ${{ github.repository == 'tum-ei-eda/mlonmcu' }} with: registry: ${{ secrets.DOCKER_REGISTRY }} username: ${{ secrets.DOCKER_USERNAME }} password: ${{ secrets.DOCKER_PASSWORD }} - name: Build and push to Docker Hub - uses: docker/build-push-action@v4 + uses: docker/build-push-action@v6 if: ${{ github.repository == 'tum-ei-eda/mlonmcu' }} with: context: . diff --git a/.github/workflows/refresh_container_daily.yml b/.github/workflows/refresh_container_daily.yml index 1c124eb52..a0d944d3f 100644 --- a/.github/workflows/refresh_container_daily.yml +++ b/.github/workflows/refresh_container_daily.yml @@ -39,7 +39,7 @@ jobs: steps: - name: Lowercase repository url id: lowered - uses: ASzc/change-string-case-action@v5 + uses: ASzc/change-string-case-action@v6 with: string: ${{ github.repository }} - name: Gen string @@ -70,28 +70,28 @@ jobs: remove-haskell: 'true' remove-codeql: 'true' - name: Checkout - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: ref: ${{ github.event.inputs.branch }} - name: Set up QEMU - uses: docker/setup-qemu-action@v2 + uses: docker/setup-qemu-action@v3 - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v2 + uses: docker/setup-buildx-action@v3 - name: Login to GitHub Container Registry - uses: docker/login-action@v2 + uses: docker/login-action@v3 with: registry: ghcr.io username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} - name: Login to DockerHub - uses: docker/login-action@v2 + uses: docker/login-action@v3 if: ${{ github.repository == 'tum-ei-eda/mlonmcu' }} with: registry: ${{ secrets.DOCKER_REGISTRY }} username: ${{ secrets.DOCKER_USERNAME }} password: ${{ secrets.DOCKER_PASSWORD }} - name: Build and push to Docker Hub - uses: docker/build-push-action@v4 + uses: docker/build-push-action@v6 if: ${{ github.repository == 'tum-ei-eda/mlonmcu' }} with: context: . @@ -115,7 +115,7 @@ jobs: steps: - name: Lowercase repository url id: lowered - uses: ASzc/change-string-case-action@v5 + uses: ASzc/change-string-case-action@v6 with: string: ${{ github.repository }} - name: Gen string @@ -146,28 +146,28 @@ jobs: remove-haskell: 'true' remove-codeql: 'true' - name: Checkout - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: ref: ${{ matrix.config.branch }} - name: Set up QEMU - uses: docker/setup-qemu-action@v2 + uses: docker/setup-qemu-action@v3 - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v2 + uses: docker/setup-buildx-action@v3 - name: Login to GitHub Container Registry - uses: docker/login-action@v2 + uses: docker/login-action@v3 with: registry: ghcr.io username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} - name: Login to DockerHub - uses: docker/login-action@v2 + uses: docker/login-action@v3 if: ${{ github.repository == 'tum-ei-eda/mlonmcu' }} with: registry: ${{ secrets.DOCKER_REGISTRY }} username: ${{ secrets.DOCKER_USERNAME }} password: ${{ secrets.DOCKER_PASSWORD }} - name: Build and push to Docker Hub - uses: docker/build-push-action@v4 + uses: docker/build-push-action@v6 if: ${{ github.repository == 'tum-ei-eda/mlonmcu' }} with: context: . diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 05ae1eafe..1f6d60459 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -13,10 +13,10 @@ jobs: if: ${{ github.repository == 'tum-ei-eda/mlonmcu' }} steps: - name: Checkout code - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Generate changelog id: changelog - uses: metcalfc/changelog-generator@v0.4.3 + uses: metcalfc/changelog-generator@v4.3.1 with: myToken: ${{ secrets.GITHUB_TOKEN }} - name: Create Release @@ -40,7 +40,7 @@ jobs: if: ${{ github.repository == 'tum-ei-eda/mlonmcu' }} steps: - name: Checkout code - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Build package run: make dist - name: Publish to PyPI diff --git a/.github/workflows/style.yml b/.github/workflows/style.yml index a8f1cfa10..c708ed197 100644 --- a/.github/workflows/style.yml +++ b/.github/workflows/style.yml @@ -9,10 +9,10 @@ jobs: steps: - name: Check out Git repository - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Set up Python - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: "3.10" @@ -20,7 +20,7 @@ jobs: run: pip install black flake8 - name: Run linters - uses: wearerequired/lint-action@v1 + uses: wearerequired/lint-action@v2 with: black: true black_args: "--line-length=120" @@ -33,7 +33,7 @@ jobs: steps: - name: Check out Git repository - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Install Python dependencies run: pip install licenseheaders diff --git a/mlonmcu/feature/feature.py b/mlonmcu/feature/feature.py index 6c747d064..e93bc2ef5 100644 --- a/mlonmcu/feature/feature.py +++ b/mlonmcu/feature/feature.py @@ -229,3 +229,9 @@ def get_run_config(self): def add_run_config(self, config): config.update(self.get_run_config()) + + # def get_postprocesses(self): + # return [] + + # def add_postprocesses(self, postprocesses): + # postprocesses.extend(self.get_postprocesses()) diff --git a/mlonmcu/feature/features.py b/mlonmcu/feature/features.py index afbfd128d..26060d708 100644 --- a/mlonmcu/feature/features.py +++ b/mlonmcu/feature/features.py @@ -2227,3 +2227,212 @@ def add_target_config(self, target, config): assert self.name not in extra_plugin_config extra_plugin_config[self.name]["baseaddr"] = self.base_addr config.update({f"{target}.extra_plugin_config": extra_plugin_config}) + + +@register_feature("gen_data") +class GenData(FrontendFeature): # TODO: use custom stage instead of LOAD + """Generate input data for validation.""" + + DEFAULTS = { + **FeatureBase.DEFAULTS, + "fill_mode": "file", # Allowed: random, ones, zeros, file, dataset + "file": "auto", # Only relevant if fill_mode=file + "number": 10, # generate up to number samples (may be less if file has less inputs) + "fmt": "npy", # Allowed: npy, npz + } + + def __init__(self, features=None, config=None): + super().__init__("gen_data", features=features, config=config) + + @property + def fill_mode(self): + value = self.config["fill_mode"] + assert value in ["random", "ones", "zeros", "file", "dataset"] + return value + + @property + def file(self): + value = self.config["file"] + return value + + @property + def number(self): + return int(self.config["number"]) + + @property + def fmt(self): + value = self.config["fmt"] + assert value in ["npy", "npz"] + return value + + def get_frontend_config(self, frontend): + assert frontend in ["tflite"] + return { + f"{frontend}.gen_data": self.enabled, + f"{frontend}.gen_data_fill_mode": self.fill_mode, + f"{frontend}.gen_data_file": self.file, + f"{frontend}.gen_data_number": self.number, + f"{frontend}.gen_data_fmt": self.fmt, + } + + +@register_feature("gen_ref_data", depends=["gen_data"]) +class GenRefData(FrontendFeature): # TODO: use custom stage instead of LOAD + """Generate reference outputs for validation.""" + + DEFAULTS = { + **FeatureBase.DEFAULTS, + "mode": "file", # Allowed: file, model + "file": "auto", # Only relevant if mode=file + "fmt": "npy", # Allowed: npy, npz + } + + def __init__(self, features=None, config=None): + super().__init__("gen_ref_data", features=features, config=config) + + @property + def mode(self): + value = self.config["mode"] + assert value in ["file", "model"] + return value + + @property + def file(self): + value = self.config["file"] + return value + + @property + def fmt(self): + value = self.config["fmt"] + assert value in ["npy", "npz"] + return value + + def get_frontend_config(self, frontend): + assert frontend in ["tflite"] + return { + f"{frontend}.gen_ref_data": self.enabled, + f"{frontend}.gen_ref_data_mode": self.mode, + f"{frontend}.gen_ref_data_file": self.file, + f"{frontend}.gen_ref_data_fmt": self.fmt, + } + + +@register_feature("gen_ref_labels", depends=["gen_data"]) +class GenRefLabels(FrontendFeature): # TODO: use custom stage instead of LOAD + """Generate reference labels for classification.""" + + DEFAULTS = { + **FeatureBase.DEFAULTS, + "mode": "file", # Allowed: file, model + "file": "auto", # Only relevant if mode=file + "fmt": "npy", # Allowed: npy, npz, txt + } + + def __init__(self, features=None, config=None): + super().__init__("gen_ref_labels", features=features, config=config) + + @property + def mode(self): + value = self.config["mode"] + assert value in ["file", "model"] + return value + + @property + def file(self): + value = self.config["file"] + return value + + @property + def fmt(self): + value = self.config["fmt"] + assert value in ["npy", "npz"] + return value + + def get_frontend_config(self, frontend): + assert frontend in ["tflite"] + return { + f"{frontend}.gen_ref_labels": self.enabled, + f"{frontend}.gen_ref_labels_mode": self.mode, + f"{frontend}.gen_ref_labels_file": self.file, + f"{frontend}.gen_ref_labels_fmt": self.fmt, + } + + +@register_feature("set_inputs") +class SetInputs(PlatformFeature): # TODO: use custom stage instead of LOAD + """Apply test inputs to model.""" + + DEFAULTS = { + **FeatureBase.DEFAULTS, + "interface": "auto", # Allowed: auto, rom, filesystem, stdin, stdin_raw, uart + } + + def __init__(self, features=None, config=None): + super().__init__("set_inputs", features=features, config=config) + + @property + def interface(self): + value = self.config["interface"] + assert value in ["auto", "rom", "filesystem", "stdin", "stdin_raw", "uart"] + return value + + def get_platform_config(self, platform): + assert platform in ["mlif", "tvm", "microtvm"] + # if tvm/microtvm: allow using --fill-mode provided by tvmc run + return { + f"{platform}.set_inputs": self.enabled, + f"{platform}.set_inputs_interface": self.interface, + } + + +@register_feature("get_outputs") +class GetOutputs(PlatformFeature): # TODO: use custom stage instead of LOAD + """Extract resulting outputs from model.""" + + DEFAULTS = { + **FeatureBase.DEFAULTS, + "interface": "auto", # Allowed: auto, filesystem, stdout, stdout_raw, uart + "fmt": "npy", # Allowed: npz, npz + } + + def __init__(self, features=None, config=None): + super().__init__("get_outputs", features=features, config=config) + + @property + def interface(self): + value = self.config["interface"] + assert value in ["auto", "filesystem", "stdout", "stdout_raw", "uart"] + return value + + @property + def fmt(self): + value = self.config["fmt"] + assert value in ["npy", "npz"] + return value + + def get_platform_config(self, platform): + assert platform in ["mlif", "tvm", "microtvm"] + return { + f"{platform}.get_outputs": self.enabled, + f"{platform}.get_outputs_interface": self.interface, + f"{platform}.get_outputs_fmt": self.fmt, + } + + +@register_feature("validate_new", depends=["gen_data", "gen_ref_data", "set_inputs", "get_outputs"]) +class ValidateNew(RunFeature): + """Wrapper feature for enabling all validatioon related features at once.""" + + DEFAULTS = { + **FeatureBase.DEFAULTS, + } + + def __init__(self, features=None, config=None): + super().__init__("validate_new", features=features, config=config) + + # def get_postprocesses(self): + # # config = {} + # # from mlonmcu.session.postprocess import ValidateOutputsPostprocess + # # validate_outputs_postprocess = ValidateOutputsPostprocess(features=[], config=config) + # # return [validate_outputs_postprocess] + # return ["validate_outputs"] diff --git a/mlonmcu/flow/tvm/backend/tvmc_utils.py b/mlonmcu/flow/tvm/backend/tvmc_utils.py index 0d9cda9ca..bb89a4901 100644 --- a/mlonmcu/flow/tvm/backend/tvmc_utils.py +++ b/mlonmcu/flow/tvm/backend/tvmc_utils.py @@ -164,7 +164,7 @@ def get_tvmrt_tvmc_args(runtime="crt", system_lib=True, link_params=True): return ret -def get_data_tvmc_args(mode=None, ins_file=None, outs_file=None, print_top=10): +def get_data_tvmc_args(mode=None, ins_file=None, outs_file=None, print_top=None): ret = [] if ins_file is not None: ret.extend(["--inputs", ins_file]) @@ -176,7 +176,7 @@ def get_data_tvmc_args(mode=None, ins_file=None, outs_file=None, print_top=10): ret.extend(["--outputs", outs_file]) if print_top is not None and print_top > 0: - ret.extend(["--print-top", print_top]) + ret.extend(["--print-top", str(print_top)]) return ret diff --git a/mlonmcu/models/__init__.py b/mlonmcu/models/__init__.py index e2307fbe7..d469e2372 100644 --- a/mlonmcu/models/__init__.py +++ b/mlonmcu/models/__init__.py @@ -33,6 +33,7 @@ MathisFrontend, MibenchFrontend, LayerGenFrontend, + OpenASIPFrontend, ) SUPPORTED_FRONTENDS = { @@ -51,6 +52,7 @@ "mathis": MathisFrontend, "mibench": MibenchFrontend, "layergen": LayerGenFrontend, + "openasip": OpenASIPFrontend, } # TODO: use registry instead __all__ = [ diff --git a/mlonmcu/models/frontend.py b/mlonmcu/models/frontend.py index fe8b2c469..23ff846e0 100644 --- a/mlonmcu/models/frontend.py +++ b/mlonmcu/models/frontend.py @@ -22,7 +22,9 @@ import multiprocessing from pathlib import Path from abc import ABC, abstractmethod -from typing import Tuple, List +from typing import Tuple, List, Dict + +import numpy as np from mlonmcu.feature.features import get_matching_features from mlonmcu.models.model import ( @@ -36,6 +38,7 @@ DhrystoneProgram, MathisProgram, MibenchProgram, + OpenASIPProgram, ) from mlonmcu.models.lookup import lookup_models from mlonmcu.feature.type import FeatureType @@ -55,7 +58,21 @@ class Frontend(ABC): DEFAULTS = { "use_inout_data": False, - # TODO: print_outputs for frontends + # the following should be configured using gen_data feature + "gen_data": False, + "gen_data_fill_mode": None, + "gen_data_file": None, + "gen_data_number": None, + "gen_data_fmt": None, + # the following should be configured using gen_ref_data feature + "gen_ref_data": False, + "gen_ref_data_mode": None, + "gen_ref_data_file": None, + "gen_ref_data_fmt": None, + "gen_ref_labels": False, + "gen_ref_labels_mode": None, + "gen_ref_labels_file": None, + "gen_ref_labels_fmt": None, } REQUIRED = set() @@ -84,6 +101,79 @@ def use_inout_data(self): value = self.config["use_inout_data"] return str2bool(value) if not isinstance(value, (bool, int)) else value + @property + def gen_data(self): + value = self.config["gen_data"] + return str2bool(value) if not isinstance(value, (bool, int)) else value + + @property + def gen_data_fill_mode(self): + value = self.config["gen_data_fill_mode"] + assert value in ["random", "ones", "zeros", "file", "dataset"] + return value + + @property + def gen_data_file(self): + return self.config["gen_data_file"] + + @property + def gen_data_number(self): + return int(self.config["gen_data_number"]) + + @property + def gen_data_fmt(self): + value = self.config["gen_data_fmt"] + assert value in ["npy", "npz"] + return value + + @property + def gen_ref_data(self): + value = self.config["gen_ref_data"] + return str2bool(value) if not isinstance(value, (bool, int)) else value + + @property + def gen_ref_data_mode(self): + value = self.config["gen_ref_data_mode"] + assert value in ["file", "model"] + return value + + @property + def gen_ref_data_file(self): + return self.config["gen_ref_data_file"] + + @property + def gen_ref_data_fmt(self): + value = self.config["gen_ref_data_fmt"] + assert value in ["npy", "npz"] + return value + + @property + def gen_ref_labels(self): + value = self.config["gen_ref_labels"] + return str2bool(value) if not isinstance(value, (bool, int)) else value + + @property + def gen_ref_labels_mode(self): + value = self.config["gen_ref_labels_mode"] + assert value in ["file", "model"] + return value + + @property + def gen_ref_labels_file(self): + return self.config["gen_ref_labels_file"] + + @property + def gen_ref_labels_fmt(self): + value = self.config["gen_ref_labels_fmt"] + assert value in ["npy", "npz", "txt", "csv"] + return value + + def inference(self, model: Model, input_data: Dict[str, np.array]): + raise NotImplementedError + + def extract_model_info(self, model: Model): + raise NotImplementedError + def supports_formats(self, ins=None, outs=None): """Returs true if the frontend can handle at least one combination of input and output formats.""" assert ins is not None or outs is not None, "Please provide a list of input formats, outputs formats or both" @@ -124,15 +214,362 @@ def process_features(self, features): def produce_artifacts(self, model): pass + def generate_input_data(self, input_names, input_types, input_shapes, input_ranges, input_quant_details, in_paths): + # TODO: drop self and move method out of frontends.py, support non-tflite models + assert self.gen_data + inputs_data = [] + if self.gen_data_fill_mode in ["zeros", "ones", "random"]: + for i in range(self.gen_data_number): + data = {} + NEW = True + for ii, input_name in enumerate(input_names): + assert input_name in input_types, f"Unknown dtype for input: {input_name}" + dtype = input_types[input_name] + quant = input_quant_details.get(input_name, None) + rng = input_ranges.get(input_name, None) + gen_dtype = dtype + if quant: + _, _, ty = quant + assert "float" in ty, "Input already quantized?" + if NEW: + gen_dtype = ty + assert input_name in input_shapes, f"Unknown shape for input: {input_name}" + shape = input_shapes[input_name] + if self.gen_data_fill_mode == "zeros": + arr = np.zeros(shape, dtype=gen_dtype) + elif self.gen_data_fill_mode == "ones": + arr = np.ones(shape, dtype=gen_dtype) + elif self.gen_data_fill_mode == "random": + DIST = "uniform" + if DIST == "uniform": + UPPER = None + LOWER = None + if rng is not None: + assert len(rng) == 2, "Range should be a tuple (lower, upper)" + LOWER, UPPER = rng + if "float" in gen_dtype: + if UPPER is None: + # UPPER = 1.0 + UPPER = 0.5 + if LOWER is None: + # LOWER = -1.0 + LOWER = -0.5 + elif "int" in gen_dtype: + dtype_info = (np.iinfo(gen_dtype),) + if UPPER is None: + UPPER = dtype_info.max + else: + assert UPPER <= dtype_info.max, "Out of dtype bound" + if LOWER is None: + LOWER = dtype_info.min + else: + assert LOWER >= dtype_info.min, "Out of dtype bound" + else: + raise RuntimeError(f"Unsupported dtype: {gen_dtype}") + assert LOWER <= UPPER + RANGE = UPPER - LOWER + assert RANGE > 0 + arr = np.random.uniform(LOWER, UPPER, shape) + arr = arr.astype(gen_dtype) + # input("?=") + # if "float" in dtype: + # arr = np.random.rand(*shape).astype(dtype) + # elif "int" in dtype: + # arr = np.random.randint(np.iinfo(dtype).min, + # np.iinfo(dtype).max, size=shape, dtype=dtype) + # else: + # assert False + # Quantize if required + # if gen_dtype != dtype: + if quant: + assert "int" in dtype + # assert quant + scale, shift, ty = quant + arr = (arr / scale) + shift + arr = np.around(arr) + arr = arr.astype(dtype) + # input("!=") + else: + raise RuntimeError(f"Unsupported distribution: {DIST}") + else: + assert False + data[input_name] = arr + assert len(data) > 0 + inputs_data.append(data) + elif self.gen_data_fill_mode == "file": + if self.gen_data_file == "auto": + assert len(in_paths) > 0, "in_paths is empty" + if len(in_paths) == 1: + if in_paths[0].is_dir(): + files = list(in_paths[0].iterdir()) + else: + files = in_paths + temp = {} + NEW = True + for file in files: + if not isinstance(file, Path): + file = Path(file) + assert file.is_file(), f"Not found: {file}" + basename, ext = file.stem, file.suffix + if ext == ".bin": + if "_" in basename: + i, ii = basename.split("_", 1) + i = int(i) + ii = int(ii) + else: + i = int(basename) + ii = 0 + with open(file, "rb") as f: + data = f.read() + if i not in temp: + temp[i] = {} + temp[i][ii] = data + elif ext in [".npy", ".npz"]: + raise NotImplementedError + else: + raise RuntimeError(f"Unsupported ext: {ext}") + assert len(temp) > 0 + for i in range(min(self.gen_data_number, len(temp))): + assert i in temp + data = {} + for ii, input_name in enumerate(input_names): + assert ii in temp[i] + assert input_name in input_types, f"Unknown dtype for input: {input_name}" + dtype = input_types[input_name] + quant = input_quant_details.get(input_name, None) + rng = input_ranges.get(input_name, None) + gen_dtype = dtype + if quant: + _, _, ty, _ = quant + # assert "float" in ty, "Input already quantized?" + if NEW: + gen_dtype = ty + arr = np.frombuffer(temp[i][ii], dtype=gen_dtype) + assert input_name in input_shapes, f"Unknown shape for input: {input_name}" + shape = input_shapes[input_name] + arr = np.reshape(arr, shape) + # Quantize if required + # if gen_dtype != dtype: + if True: + assert "int" in dtype + # assert quant + scale, shift, ty, qrng = quant + if qrng is not None: + assert len(qrng) == 2, "Range should be a tuple (lower, upper)" + lower, upper = qrng + assert lower <= upper + CLIP_INPUTS = True + if CLIP_INPUTS: + arr = np.clip(arr, lower, upper) + else: + assert np.min(arr) >= lower or np.isclose( + np.min(arr), lower + ), "Range missmatch (lower)" + assert np.max(arr) <= upper or np.isclose( + np.max(arr), upper + ), "Range missmatch (upper)" + arr = (arr / scale) + shift + arr = np.around(arr) + arr = arr.astype(dtype) + # input("!=") + if rng is not None: + # TODO: Move shared code! + assert len(rng) == 2, "Range should be a tuple (lower, upper)" + lower, upper = rng + assert lower <= upper + CLIP_INPUTS = True + if CLIP_INPUTS: + arr = np.clip(arr, lower, upper) + else: + assert np.min(arr) >= lower or np.isclose(np.min(arr), lower), "Range missmatch (lower)" + assert np.max(arr) <= upper or np.isclose(np.max(arr), upper), "Range missmatch (upper)" + data[input_name] = arr + inputs_data.append(data) + else: + assert self.gen_data_file is not None, "Missing value for gen_data_file" + file = Path(self.gen_data_file) + assert file.is_file(), f"File not found: {file}" + # for i, input_name in enumerate(input_names): + + elif self.gen_data_fill_mode == "dataset": + raise NotImplementedError + else: + raise RuntimeError(f"unsupported fill_mode: {self.gen_data_fill_mode}") + return inputs_data + + def generate_output_ref_data( + self, inputs_data, model, out_paths, output_names, output_types, output_shapes, output_quant_details + ): + assert self.gen_ref_data + outputs_data = [] + if self.gen_ref_data_mode == "model": + assert len(inputs_data) > 0 + for i, input_data in enumerate(inputs_data): + # input("321?") + output_data = self.inference(model, input_data, quant=False, dequant=True) + outputs_data.append(output_data) + # input("321!") + + elif self.gen_ref_data_mode == "file": + if self.gen_ref_data_file == "auto": + assert len(out_paths) > 0, "out_paths is empty" + if len(out_paths) == 1: + if out_paths[0].is_dir(): + files = list(out_paths[0].iterdir()) + else: + files = out_paths + temp = {} + assert len(inputs_data) <= len( + files + ), f"Missing output data for provided inputs. (Expected: {len(inputs_data)}, Got: {len(files)})" + for file in files: + if not isinstance(file, Path): + file = Path(file) + assert file.is_file() + basename, ext = file.stem, file.suffix + if ext == ".bin": + if "_" in basename: + i, ii = basename.split("_", 1) + i = int(i) + ii = int(ii) + else: + i = int(basename) + ii = 0 + with open(file, "rb") as f: + data = f.read() + if i not in temp: + temp[i] = {} + temp[i][ii] = data + elif ext in [".npy", ".npz"]: + raise NotImplementedError + else: + raise RuntimeError(f"Unsupported ext: {ext}") + # TODO: handle case where there are more output samples than input samples? + for i in range(len(temp)): + assert i in temp + data = {} + for ii, output_name in enumerate(output_names): + assert ii in temp[i] + assert output_name in output_types, f"Unknown dtype for output: {output_name}" + dtype = output_types[output_name] + dequant = output_quant_details.get(output_name, None) + if dequant: + _, _, ty, _ = dequant + dtype = ty + arr = np.frombuffer(temp[i][ii], dtype=dtype) + assert output_name in output_shapes, f"Unknown shape for output: {output_name}" + shape = output_shapes[output_name] + arr = np.reshape(arr, shape) + data[output_name] = arr + outputs_data.append(data) + else: + assert self.gen_ref_data_file is not None, "Missing value for gen_ref_data_file" + file = Path(self.gen_data_file) + assert file.is_file(), f"File not found: {file}" + raise NotImplementedError + else: + raise RuntimeError(f"unsupported fill_mode: {self.gen_ref_data_mode}") + return outputs_data + + def generate_ref_labels( + self, inputs_data, model, out_labels_paths, output_names, output_types, output_shapes, output_quant_details + ): + assert self.gen_ref_labels + labels = [] + if self.gen_ref_labels_mode == "model": + assert len(inputs_data) > 0 + for i, input_data in enumerate(inputs_data): + output_data = self.inference(model, input_data, quant=False, dequant=True) + assert len(output_data) == 1, "Does not support multi-output classification" + output_data = output_data[list(output_data)[0]] + top_label = np.argmax(output_data) + labels.append(top_label) + + elif self.gen_ref_labels_mode == "file": + if self.gen_ref_labels_file == "auto": + assert len(out_labels_paths) > 0, "labels_paths is empty" + assert len(out_labels_paths) == 1 + file = Path(out_labels_paths[0]) + else: + assert self.gen_ref_labels_file is not None, "Missing value for gen_ref_labels_file" + file = Path(self.gen_ref_labels_file) + assert file.is_file(), f"File not found: {file}" + ext = file.suffix + assert len(ext) > 1 + fmt = ext[1:].lower() + if fmt == "csv": + import pandas as pd + + labels_df = pd.read_csv(file, sep=",") + assert "i" in labels_df.columns + assert "label_idx" in labels_df.columns + assert len(inputs_data) <= len(labels_df) + labels_df.sort_values("i", inplace=True) + labels = list(labels_df["label_idx"].astype(int))[: len(inputs_data)] + else: + raise NotImplementedError(f"Fmt not supported: {fmt}") + else: + raise RuntimeError(f"unsupported fill_mode: {self.gen_ref_labels_mode}") + return labels + + def generate_model_info( + self, + input_names, + output_names, + input_shapes, + output_shapes, + input_types, + output_types, + input_ranges, + output_ranges, + input_quant_details, + output_quant_details, + ): + model_info_dict = { + "input_names": input_names, + "output_names": output_names, + "input_shapes": list(input_shapes.values()), + "output_shapes": list(output_shapes.values()), + "input_types": list(input_types.values()), + "output_types": list(output_types.values()), + "input_ranges": list(input_ranges.values()), + "output_ranges": list(output_ranges.values()), + "input_quant_details": list(input_quant_details.values()), + "output_quant_details": list(output_quant_details.values()), + } + # nested version + # model_info_dict = { + # "inputs": [ + # { + # "name": "input_1", + # "shape": [1, 1014], + # "type": "int8", + # } + # ], + # "outputs": [ + # { + # "name": "output", + # "shape": [1, 10], + # "type": "int8", + # } + # ], + # } + return model_info_dict # TODO: turn into class + def process_metadata(self, model, cfg=None): model_dir = Path(model.paths[0]).parent.resolve() metadata = model.metadata in_paths = [] out_paths = [] + labels_paths = [] input_shapes = {} output_shapes = {} input_types = {} output_types = {} + input_ranges = {} + output_ranges = {} + input_quant_details = {} + output_quant_details = {} if metadata is not None and "network_parameters" in metadata: network = metadata["network_parameters"] assert "input_nodes" in network @@ -140,12 +577,27 @@ def process_metadata(self, model, cfg=None): for inp in ins: name = inp.get("name", None) shape = inp.get("shape", None) - ty = inp.get("type", None) + ty = inp.get("dtype", None) + if ty is None: + ty = inp.get("type", None) # legacy + rng = inp.get("range", None) + quantize = inp.get("quantize", None) if name and shape: input_shapes[name] = shape if name and ty: input_types[name] = ty - if self.use_inout_data: + if name and rng: + input_ranges[name] = rng + if name and quantize: + quant_scale = quantize.get("scale", None) + quant_zero_shift = quantize.get("zero_shift", None) + quant_dtype = quantize.get("dtype", None) + quant_range = quantize.get("range", None) + quant_details = [quant_scale, quant_zero_shift, quant_dtype, quant_range] + input_quant_details[name] = quant_details + if self.use_inout_data or ( + self.gen_data and self.gen_data_fill_mode == "file" and self.gen_data_file == "auto" + ): if "example_input" in inp and "path" in inp["example_input"]: in_data_dir = Path(inp["example_input"]["path"]) # TODO: this will only work with relative paths to model dir! (Fallback to parent directories?) @@ -159,19 +611,42 @@ def process_metadata(self, model, cfg=None): for outp in outs: name = outp.get("name", None) shape = outp.get("shape", None) - ty = outp.get("type", None) + ty = outp.get("dtype", None) + if ty is None: + ty = outp.get("type", None) # legacy + rng = outp.get("range", None) + dequantize = outp.get("dequantize", None) if name and shape: output_shapes[name] = shape if name and ty: output_types[name] = ty - if self.use_inout_data: + if name and rng: + output_ranges[name] = rng + if name and dequantize: + quant_scale = dequantize.get("scale", None) + quant_zero_shift = dequantize.get("zero_shift", None) + quant_dtype = dequantize.get("dtype", None) + quant_range = dequantize.get("range", None) + quant_details = [quant_scale, quant_zero_shift, quant_dtype, quant_range] + output_quant_details[name] = quant_details + if self.use_inout_data or ( + self.gen_ref_data and self.gen_ref_data_mode == "file" and self.gen_ref_data_file == "auto" + ): if "test_output_path" in outp: out_data_dir = Path(outp["test_output_path"]) out_path = model_dir / out_data_dir assert ( - in_path.is_dir() + out_path.is_dir() ), f"Output data directory defined in model metadata does not exist: {out_path}" out_paths.append(out_path) + if self.gen_ref_labels and self.gen_ref_labels_mode == "file" and self.gen_ref_labels_file == "auto": + if "test_labels_file" in outp: + labels_file = Path(outp["test_labels_file"]) + labels_path = model_dir / labels_file + assert ( + labels_path.is_file() + ), f"Labels file defined in model metadata does not exist: {labels_path}" + labels_paths.append(labels_path) else: fallback_in_path = model_dir / "input" if fallback_in_path.is_dir(): @@ -179,6 +654,18 @@ def process_metadata(self, model, cfg=None): fallback_out_path = model_dir / "output" if fallback_out_path.is_dir(): out_paths.append(fallback_out_path) + fallback_labels_path = model_dir / "output_labels.csv" + if fallback_labels_path.is_file(): + labels_paths.append(fallback_labels_path) + if model.inputs_path: + logger.info("Overriding default model input data with user path") + in_paths = [model.inputs_path] + if model.outputs_path: + logger.info("Overriding default model output data with user path") + out_paths = [model.outputs_path] + if model.output_labels_path: # TODO + logger.info("Overriding default model output labels with user path") + labels_paths = [model.output_labels_path] if metadata is not None and "backends" in metadata: assert cfg is not None @@ -188,12 +675,39 @@ def process_metadata(self, model, cfg=None): flattened = {f"{backend}.{key}": value for key, value in backend_options[backend].items()} cfg.update(flattened) + if len(input_shapes) > 0: + assert len(input_types) in [len(input_shapes), 0] + input_names = list(input_shapes.keys()) + elif len(input_types) > 0: + input_names = list(input_types.keys()) + else: + input_names = [] + + if metadata is None: + try: + ( + input_names, + input_shapes, + input_types, + input_quant_details, + output_names, + output_shapes, + output_types, + output_quant_details, + ) = self.extract_model_info(model) + except NotImplementedError: + logger.warning("Model info could not be extracted.") + # Detect model support code (Allow overwrite in metadata YAML) - support_path = model_dir / "support" + if model.support_path: + support_path = model.support_path + else: + support_path = model_dir / "support" if support_path.is_dir(): assert cfg is not None # TODO: onlu overwrite if unset? - cfg.update({"mlif.model_support_dir": support_path}) + if cfg.get("mlif.model_support_dir", None) is not None: + cfg.update({"mlif.model_support_dir": support_path}) # cfg.update({"espidf.model_support_dir": support_path}) # cfg.update({"zephyr.model_support_dir": support_path}) if len(in_paths) > 0: @@ -204,6 +718,8 @@ def process_metadata(self, model, cfg=None): cfg.update({"mlif.output_data_path": out_paths}) # cfg.update({"espidf.output_data_path": out_paths}) # cfg.update({"zephyr.output_data_path": out_paths}) + if len(labels_paths) > 0: + cfg.update({"mlif.output_labels_path": labels_paths}) if len(input_shapes) > 0: cfg.update({f"{model.name}.input_shapes": input_shapes}) if len(output_shapes) > 0: @@ -212,6 +728,100 @@ def process_metadata(self, model, cfg=None): cfg.update({f"{model.name}.input_types": input_types}) if len(output_types) > 0: cfg.update({f"{model.name}.output_types": output_types}) + # flattened version + if len(output_shapes) > 0: + assert len(output_types) in [len(output_shapes), 0] + output_names = list(output_shapes.keys()) + elif len(output_shapes) > 0: + output_names = list(output_types.keys()) + else: + output_names = [] + artifacts = [] + inputs_data = None + gen_model_info = True # TODO: move to self (configurable) + if gen_model_info: + model_info_dict = self.generate_model_info( + input_names, + output_names, + input_shapes, + output_shapes, + input_types, + output_types, + input_ranges, + output_ranges, + input_quant_details, + output_quant_details, + ) + import yaml + + content = yaml.dump(model_info_dict) + model_info_artifact = Artifact( + "model_info.yml", content=content, fmt=ArtifactFormat.TEXT, flags=("model_info",) + ) + artifacts.append(model_info_artifact) + if self.gen_data: + inputs_data = self.generate_input_data( + input_names, input_types, input_shapes, input_ranges, input_quant_details, in_paths + ) + fmt = self.gen_data_fmt + if fmt == "npy": + with tempfile.TemporaryDirectory() as tmpdirname: + tempfilename = Path(tmpdirname) / "inputs.npy" + np.save(tempfilename, inputs_data) + with open(tempfilename, "rb") as f: + raw = f.read() + elif fmt == "npz": + raise NotImplementedError + else: + raise RuntimeError(f"Unsupported fmt: {fmt}") + assert raw + inputs_data_artifact = Artifact(f"inputs.{fmt}", raw=raw, fmt=ArtifactFormat.BIN, flags=("inputs", fmt)) + artifacts.append(inputs_data_artifact) + if self.gen_ref_data: + outputs_ref_data = self.generate_output_ref_data( + inputs_data, model, out_paths, output_names, output_types, output_shapes, output_quant_details + ) + fmt = self.gen_data_fmt + if fmt == "npy": + with tempfile.TemporaryDirectory() as tmpdirname: + tempfilename = Path(tmpdirname) / "outputs_ref.npy" + np.save(tempfilename, outputs_ref_data) + with open(tempfilename, "rb") as f: + raw = f.read() + elif fmt == "npz": + raise NotImplementedError + else: + raise RuntimeError(f"Unsupported fmt: {fmt}") + assert raw + outputs_ref_artifact = Artifact( + f"outputs_ref.{fmt}", raw=raw, fmt=ArtifactFormat.BIN, flags=("outputs_ref", fmt) + ) + artifacts.append(outputs_ref_artifact) + if self.gen_ref_labels: + labels_ref = self.generate_ref_labels( + inputs_data, model, labels_paths, output_names, output_types, output_shapes, output_quant_details + ) + fmt = self.gen_ref_labels_fmt + if fmt == "npy": + with tempfile.TemporaryDirectory() as tmpdirname: + tempfilename = Path(tmpdirname) / "labels.npy" + np.save(tempfilename, labels_ref) + with open(tempfilename, "rb") as f: + raw = f.read() + elif fmt == "npz": + raise NotImplementedError + elif fmt == "txt": + raise NotImplementedError + elif fmt == "csv": + raise NotImplementedError + else: + raise RuntimeError(f"Unsupported fmt: {fmt}") + assert raw + labels_ref_artifact = Artifact( + f"labels_ref.{fmt}", raw=raw, fmt=ArtifactFormat.BIN, flags=("labels_ref", fmt) + ) + artifacts.append(labels_ref_artifact) + return artifacts def generate(self, model) -> Tuple[dict, dict]: artifacts = [] @@ -299,6 +909,7 @@ def produce_artifacts(self, model): assert len(self.input_formats) == len(self.output_formats) == len(model.paths) == 1 artifacts = [] name = model.name + assert "/" not in name path = model.paths[0] ext = self.input_formats[0].extension with open(path, "rb") as handle: # TODO: is an onnx model raw data or text? @@ -311,7 +922,14 @@ def produce_artifacts(self, model): # TODO: frontend parsed metadata instead of lookup.py? # TODO: how to find inout_data? class TfLiteFrontend(SimpleFrontend): - FEATURES = Frontend.FEATURES | {"visualize", "split_layers", "tflite_analyze"} + FEATURES = Frontend.FEATURES | { + "visualize", + "split_layers", + "tflite_analyze", + "gen_data", + "gen_ref_data", + "gen_ref_labels", + } DEFAULTS = { **Frontend.DEFAULTS, @@ -362,6 +980,104 @@ def analyze_enable(self): def analyze_script(self): return self.config["analyze_script"] + def extract_model_info(self, model: Model): + import tensorflow as tf + + model_path = str(model.paths[0]) + interpreter = tf.lite.Interpreter(model_path=model_path) + input_details = interpreter.get_input_details() + output_details = interpreter.get_output_details() + input_names = [] + input_shapes = {} + input_types = {} + input_quant_details = {} + output_names = [] + output_shapes = {} + output_types = {} + output_quant_details = {} + for inp in input_details: + name = str(inp["name"]) + input_names.append(name) + input_shapes[name] = inp["shape"].tolist() + input_types[name] = np.dtype(inp["dtype"]).name + if "quantization" in inp: + scale, zero_point = inp["quantization"] + quant = [scale, zero_point, "float32"] + input_quant_details[name] = quant + for outp in output_details: + name = str(outp["name"]) + output_names.append(name) + output_shapes[name] = outp["shape"].tolist() + output_types[name] = np.dtype(outp["dtype"]).name + if "quantization" in outp: + scale, zero_point = outp["quantization"] + quant = [scale, zero_point, "float32"] + output_quant_details[name] = quant + return ( + input_names, + input_shapes, + input_types, + input_quant_details, + output_names, + output_shapes, + output_types, + output_quant_details, + ) + + def inference(self, model: Model, input_data: Dict[str, np.array], quant=False, dequant=False, verbose=False): + import tensorflow as tf + + model_path = str(model.paths[0]) + interpreter = tf.lite.Interpreter(model_path=model_path) + input_details = interpreter.get_input_details() + output_details = interpreter.get_output_details() + interpreter.allocate_tensors() + if verbose: + print() + print("Input details:") + print(input_details) + print() + print("Output details:") + print(output_details) + print() + assert len(input_details) == 1, "Multi-inputs not yet supported" + input_type = input_details[0]["dtype"] + input_name = input_details[0]["name"] + input_shape = input_details[0]["shape"] + assert input_name in input_data, f"Input {input_name} fot found in data" + np_features = input_data[input_name] + if quant and input_type == np.int8: + input_scale, input_zero_point = input_details[0]["quantization"] + if verbose: + print("Input scale:", input_scale) + print("Input zero point:", input_zero_point) + print() + np_features = (np_features / input_scale) + input_zero_point + np_features = np.around(np_features) + np_features = np_features.astype(input_type) + np_features = np_features.reshape(input_shape) + interpreter.set_tensor(input_details[0]["index"], np_features) + interpreter.invoke() + output = interpreter.get_tensor(output_details[0]["index"]) + + # If the output type is int8 (quantized model), rescale data + assert len(output_details) == 1, "Multi-outputs not yet supported" + output_type = output_details[0]["dtype"] + output_name = output_details[0]["name"] + if dequant and output_type == np.int8: + output_scale, output_zero_point = output_details[0]["quantization"] + if verbose: + print("Raw output scores:", output) + print("Output scale:", output_scale) + print("Output zero point:", output_zero_point) + print() + output = output_scale * (output.astype(np.float32) - output_zero_point) + + if verbose: + # Print the results of inference + print("Inference output:", output, type(output)) + return {output_name: output} + def produce_artifacts(self, model): assert len(self.input_formats) == len(model.paths) == 1 artifacts = [] @@ -982,6 +1698,12 @@ def get_platform_config(self, platform): class PolybenchFrontend(SimpleFrontend): + + DEFAULTS = { + **Frontend.DEFAULTS, + "dataset": "large", # mini/small/medium/large/extralarge + } + REQUIRED = {"polybench.src_dir"} def __init__(self, features=None, config=None): @@ -1054,6 +1776,7 @@ def get_platform_defs(self, platform): ret = {} if platform == "mlif": ret["POLYBENCH_DIR"] = Path(self.config["polybench.src_dir"]) + ret["POLYBENCH_DATASET"] = self.config["dataset"].upper() + "_DATASET" return ret def get_platform_config(self, platform): @@ -1372,3 +2095,50 @@ def helper(args): artifact = Artifact(f"{name}.{ext}", raw=raw, fmt=ArtifactFormat.RAW, flags=["model"]) artifacts[name] = [artifact] return artifacts, {} + + +class OpenASIPFrontend(SimpleFrontend): + + def __init__(self, features=None, config=None): + super().__init__( + "openasip", + ModelFormats.NONE, + features=features, + config=config, + ) + + @property + def supported_names(self): + return [ + "sha256", + "aes", + "crc", + ] + + # @property + # def skip_backend(self): + # return True + + def lookup_models(self, names, config=None, context=None): + ret = [] + for name in names: + name = name.replace("openasip/", "") + if name in self.supported_names: + hint = OpenASIPProgram( + name, + alt=f"openasip/{name}", + config=config, + ) + ret.append(hint) + return ret + + def generate(self, model) -> Tuple[dict, dict]: + artifacts = [Artifact("dummy_model", raw=bytes(), fmt=ArtifactFormat.RAW, flags=["model", "dummy"])] + + return {"default": artifacts}, {} + + def get_platform_config(self, platform): + ret = {} + if platform == "mlif": + ret["template"] = "openasip" + return ret diff --git a/mlonmcu/models/model.py b/mlonmcu/models/model.py index aa340ffb3..4dd903741 100644 --- a/mlonmcu/models/model.py +++ b/mlonmcu/models/model.py @@ -160,9 +160,10 @@ class Model(Workload): "output_shapes": None, "input_types": None, "output_types": None, - "support_path": "support", - "inputs_path": "input", - "outputs_path": "output", + "support_path": None, + "inputs_path": None, + "outputs_path": None, + "output_labels_path": None, } def __init__(self, name, paths, config=None, alt=None, formats=ModelFormats.TFLITE): @@ -221,17 +222,42 @@ def output_types(self): @property def support_path(self): - return self.config["support_path"] + value = self.config["support_path"] + if value is not None: + if not isinstance(value, Path): + assert isinstance(value, str) + value = Path(value) + return value @property def inputs_path(self): # TODO: fall back to metadata - return self.config["inputs_path"] + value = self.config["inputs_path"] + if value is not None: + if not isinstance(value, Path): + assert isinstance(value, str) + value = Path(value) + return value @property def outputs_path(self): # TODO: fall back to metadata - return self.config["outputs_path"] + value = self.config["outputs_path"] + if value is not None: + if not isinstance(value, Path): + assert isinstance(value, str) + value = Path(value) + return value + + @property + def output_labels_path(self): + # TODO: fall back to metadata + value = self.config["output_labels_path"] + if value is not None: + if not isinstance(value, Path): + assert isinstance(value, str) + value = Path(value) + return value @property def skip_check(self): @@ -387,3 +413,21 @@ def get_platform_defs(self, platform): if platform == "mlif": ret["DHRYSTONE_ITERATIONS"] = 10000 return ret + + +class OpenASIPProgram(Program): + DEFAULTS = { + "crc_mode": "both", + } + + @property + def crc_mode(self): + return str(self.config["crc_mode"]) + + def get_platform_defs(self, platform): + ret = {} + if platform == "mlif": + ret["OPENASIP_BENCHMARK"] = self.name + if self.name == "crc": + ret["OPENASIP_CRC_MODE"] = self.crc_mode + return ret diff --git a/mlonmcu/models/utils.py b/mlonmcu/models/utils.py index 20f7ad308..a914ee8c7 100644 --- a/mlonmcu/models/utils.py +++ b/mlonmcu/models/utils.py @@ -79,6 +79,23 @@ def fill_data_source(in_bufs, out_bufs): return out +def fill_data_source_inputs_only(in_bufs): + # out = '#include "ml_interface.h"\n' + out = "#include \n" + out += "const int num_data_buffers_in = " + str(sum([len(buf) for buf in in_bufs])) + ";\n" + for i, buf in enumerate(in_bufs): + for j in range(len(buf)): + out += "const unsigned char data_buffer_in_" + str(i) + "_" + str(j) + "[] = {" + buf[j] + "};\n" + var_in = "const unsigned char *const data_buffers_in[] = {" + var_insz = "const size_t data_size_in[] = {" + for i, buf in enumerate(in_bufs): + for j in range(len(buf)): + var_in += "data_buffer_in_" + str(i) + "_" + str(j) + ", " + var_insz += "sizeof(data_buffer_in_" + str(i) + "_" + str(j) + "), " + out += var_in + "};\n" + var_insz + "};\n" + return out + + def lookup_data_buffers(input_paths, output_paths): assert len(input_paths) > 0 legacy = False diff --git a/mlonmcu/platform/microtvm/microtvm_gvsoc_target.py b/mlonmcu/platform/microtvm/microtvm_gvsoc_target.py new file mode 100644 index 000000000..d164a1593 --- /dev/null +++ b/mlonmcu/platform/microtvm/microtvm_gvsoc_target.py @@ -0,0 +1,97 @@ +# +# Copyright (c) 2022 TUM Department of Electrical and Computer Engineering. +# +# This file is part of MLonMCU. +# See https://github.com/tum-ei-eda/mlonmcu.git for further info. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +from pathlib import Path + +from mlonmcu.target.target import Target + +from mlonmcu.logging import get_logger + +from .microtvm_template_target import TemplateMicroTvmPlatformTarget + +logger = get_logger() + + +class GVSocMicroTvmPlatformTarget(TemplateMicroTvmPlatformTarget): + # FEATURES = TemplateMicroTvmPlatformTarget.FEATURES + ["xpulp"] + FEATURES = TemplateMicroTvmPlatformTarget.FEATURES + + DEFAULTS = { + **TemplateMicroTvmPlatformTarget.DEFAULTS, + # "verbose": True, + "compiler": "gcc", + "project_type": "host_driven", + # "xpulp_version": None, # None means that xpulp extension is not used, + # "model": "pulp", + } + REQUIRED = Target.REQUIRED | { + "gvsoc.exe", + "pulp_freertos.support_dir", + "pulp_freertos.config_dir", + "pulp_freertos.install_dir", + "microtvm_gvsoc.template", + "hannah_tvm.src_dir", + } + + def __init__(self, name=None, features=None, config=None): + super().__init__(name=name, features=features, config=config) + self.template_path = self.microtvm_gvsoc_template + # TODO: integrate into TVM build config + self.option_names = [ + # "verbose", + "project_type", + "compiler", + ] + + @property + def microtvm_gvsoc_template(self): + return Path(self.config["microtvm_gvsoc.template"]) + + @property + def hannah_tvm_src_dir(self): + return Path(self.config["hannah_tvm.src_dir"]) + + @property + def compiler(self): + return self.config["compiler"] + + def get_project_options(self): + ret = super().get_project_options() + # TODO + ret.update( + { + # "gvsoc_exe": str(self.gvsoc_exe), + } + ) + return ret + + def update_environment(self, env): + super().update_environment(env) + p = env.get("PYTHONPATH", "") + env["PYTHONPATH"] = f"{self.hannah_tvm_src_dir}:{p}" + + # TODO + # if "PATH" in env: + # env["PATH"] = str(self.riscv_gcc_install_dir / "bin") + ":" + env["PATH"] + # else: + # env["PATH"] = str(self.riscv_gcc_install_dir / "bin") + + def get_backend_config(self, backend): + ret = {} + # TODO + return ret diff --git a/mlonmcu/platform/microtvm/microtvm_target.py b/mlonmcu/platform/microtvm/microtvm_target.py index 36aad66b8..5e3b5a0b6 100644 --- a/mlonmcu/platform/microtvm/microtvm_target.py +++ b/mlonmcu/platform/microtvm/microtvm_target.py @@ -30,6 +30,7 @@ from .microtvm_host_target import HostMicroTvmPlatformTarget from .microtvm_etiss_target import EtissMicroTvmPlatformTarget from .microtvm_spike_target import SpikeMicroTvmPlatformTarget +from .microtvm_gvsoc_target import GVSocMicroTvmPlatformTarget from .microtvm_corev_ovpsim_target import CoreVOVPSimMicroTvmPlatformTarget from .microtvm_mlonmcu_target import MlonmcuMicroTvmPlatformTarget @@ -61,6 +62,7 @@ def get_microtvm_platform_targets(): register_microtvm_platform_target("microtvm_etiss", EtissMicroTvmPlatformTarget) register_microtvm_platform_target("microtvm_espidf", EspidfMicroTvmPlatformTarget) register_microtvm_platform_target("microtvm_spike", SpikeMicroTvmPlatformTarget) +register_microtvm_platform_target("microtvm_gvsoc", GVSocMicroTvmPlatformTarget) register_microtvm_platform_target("microtvm_corev_ovpsim", CoreVOVPSimMicroTvmPlatformTarget) register_microtvm_platform_target("microtvm_mlonmcu", MlonmcuMicroTvmPlatformTarget) diff --git a/mlonmcu/platform/mlif/interfaces.py b/mlonmcu/platform/mlif/interfaces.py new file mode 100644 index 000000000..a54cc3ec3 --- /dev/null +++ b/mlonmcu/platform/mlif/interfaces.py @@ -0,0 +1,251 @@ +# +# Copyright (c) 2022 TUM Department of Electrical and Computer Engineering. +# +# This file is part of MLonMCU. +# See https://github.com/tum-ei-eda/mlonmcu.git for further info. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +"""MLIF Interfaces""" +from mlonmcu.models.utils import fill_data_source_inputs_only + +MAX_BATCH_SIZE = int(1e6) +DEFAULT_BATCH_SIZE = 10 + + +def get_header(): + return """ +#include "quantize.h" +#include "printing.h" +#include "exit.h" +// #include "ml_interface.h" +#include +#include +#include +#include +#include + +extern "C" { +int mlif_process_inputs(size_t, bool*); +int mlif_process_outputs(size_t); +void *mlif_input_ptr(int); +void *mlif_output_ptr(int); +int mlif_input_sz(int); +int mlif_output_sz(int); +int mlif_num_inputs(); +int mlif_num_outputs(); +} +""" + + +def get_top_rom(inputs_data): + in_bufs = [] + for i, ins_data in enumerate(inputs_data): + temp = [] + for j, in_data in enumerate(ins_data.values()): + byte_data = in_data.tobytes() + temp2 = ", ".join(["0x{:02x}".format(x) for x in byte_data] + [""]) + temp.append(temp2) + in_bufs.append(temp) + + return fill_data_source_inputs_only(in_bufs) + + +def get_process_inputs_head(): + return """ +int mlif_process_inputs(size_t batch_idx, bool *new_) +{ +""" + + +def get_process_inputs_tail(): + return """ +} +""" + + +def get_process_outputs_head(): + return """ +int mlif_process_outputs(size_t batch_idx) +{ +""" + + +def get_process_outputs_tail(): + return """ +} +""" + + +def get_process_inputs_rom(): + return """ + *new_ = true; + int num_inputs = mlif_num_inputs(); + for (int i = 0; i < num_inputs; i++) + { + int idx = num_inputs * batch_idx + i; + int size = mlif_input_sz(i); + char* model_input_ptr = (char*)mlif_input_ptr(i); + if (idx >= num_data_buffers_in) + { + *new_ = false; + break; + } + if (size != data_size_in[idx]) + { + return EXIT_MLIF_INVALID_SIZE; + } + memcpy(model_input_ptr, data_buffers_in[idx], size); + } + return 0; +""" + + +def get_process_inputs_stdin_raw(): + return """ + char ch; + *new_ = true; + for (int i = 0; i < mlif_num_inputs(); i++) + { + int cnt = 0; + int size = mlif_input_sz(i); + char* model_input_ptr = (char*)mlif_input_ptr(i); + while(read(STDIN_FILENO, &ch, 1) > 0) { + // printf("c=%c / %d\\n", ch, ch); + model_input_ptr[cnt] = ch; + cnt++; + if (cnt == size) { + break; + } + } + // printf("cnt=%d in_size=%lu\\n", cnt, in_size); + if (cnt == 0) { + *new_ = false; + return 0; + } + else if (cnt < size) + { + return EXIT_MLIF_INVALID_SIZE; + } + } + return 0; +""" + + +def get_process_outputs_stdout_raw(): + # TODO: maybe hardcode num_outputs and size here because we know it + # and get rid of loop? + return """ + for (int i = 0; i < mlif_num_outputs(); i++) + { + int8_t *model_output_ptr = (int8_t*)mlif_output_ptr(i); + int size = mlif_output_sz(i); + // TODO: move markers out of loop + write(1, "-?-", 3); + write(1, model_output_ptr, size); + write(1, "-!-\\n" ,4); + } + return 0; +""" + + +class ModelSupport: + def __init__(self, in_interface, out_interface, model_info, target=None, batch_size=None, inputs_data=None): + self.model_info = model_info + self.target = target + self.inputs_data = inputs_data + self.in_interface = in_interface + self.out_interface = out_interface + self.in_interface, self.batch_size = self.select_set_inputs_interface(in_interface, batch_size) + self.out_interface, self.batch_size = self.select_get_outputs_interface(out_interface, self.batch_size) + + def select_set_inputs_interface(self, in_interface, batch_size): + if in_interface == "auto": + assert self.target is not None + if self.target.supports_filesystem: + in_interface = "filesystem" + elif self.target.supports_stdin: + in_interface = "stdin_raw" + # TODO: also allow stdin? + else: # Fallback + in_interface = "rom" + assert in_interface in ["filesystem", "stdin", "stdin_raw", "rom"] + if batch_size is None: + if in_interface == "rom": + batch_size = MAX_BATCH_SIZE # all inputs are in already compiled into program + else: + batch_size = DEFAULT_BATCH_SIZE + return in_interface, batch_size + + def select_get_outputs_interface(self, out_interface, batch_size): + if out_interface == "auto": + assert self.target is not None + if self.target.supports_filesystem: + out_interface = "filesystem" + elif self.target.supports_stdin: + out_interface = "stdout_raw" + # TODO: also allow stdout? + else: # Fallback + out_interface = "ram" + assert out_interface in ["filesystem", "stdout", "stdout_raw", "ram"] + if batch_size is None: + batch_size = DEFAULT_BATCH_SIZE + return out_interface, batch_size + + def generate_header(self): + # TODO: make this configurable + # TODO: do not require C++? + return get_header() + + def generate_top(self): + if self.in_interface == "rom": + return get_top_rom(self.inputs_data) + return "" + + def generate_bottom(self): + return "" + + def generate_process_inputs_body(self): + if self.in_interface == "rom": + return get_process_inputs_rom() + elif self.in_interface == "stdin_raw": + return get_process_inputs_stdin_raw() + raise NotImplementedError # TODO: implement: filesystem (bin+npy), stdout + + def generate_process_outputs_body(self): + if self.out_interface == "stdout_raw": + return get_process_outputs_stdout_raw() + raise NotImplementedError # TODO: implement: filesystem (bin+npy), ram + + def generate_process_inputs(self): + code = "" + code += get_process_inputs_head() + code += self.generate_process_inputs_body() + code += get_process_inputs_tail() + return code + + def generate_process_outputs(self): + code = "" + code += get_process_outputs_head() + code += self.generate_process_outputs_body() + code += get_process_outputs_tail() + return code + + def generate(self): + code = "" + code += self.generate_header() + code += self.generate_top() + code += self.generate_process_inputs() + code += self.generate_process_outputs() + code += self.generate_bottom() + return code diff --git a/mlonmcu/platform/mlif/mlif.py b/mlonmcu/platform/mlif/mlif.py index 82a20547c..d6b68588a 100644 --- a/mlonmcu/platform/mlif/mlif.py +++ b/mlonmcu/platform/mlif/mlif.py @@ -20,9 +20,11 @@ import os import tempfile from typing import Tuple - from pathlib import Path +import yaml +import numpy as np + from mlonmcu.config import str2bool from mlonmcu.setup import utils # TODO: Move one level up? from mlonmcu.timeout import exec_timeout @@ -33,6 +35,7 @@ from mlonmcu.models.utils import get_data_source from ..platform import CompilePlatform, TargetPlatform +from .interfaces import ModelSupport from .mlif_target import get_mlif_platform_targets, create_mlif_platform_target logger = get_logger() @@ -57,6 +60,8 @@ class MlifPlatform(CompilePlatform, TargetPlatform): "auto_vectorize", "benchmark", "xpulp", + "set_inputs", + "get_outputs", } # TODO: allow Feature-Features with automatic resolution of initialization order ) @@ -64,6 +69,7 @@ class MlifPlatform(CompilePlatform, TargetPlatform): **CompilePlatform.DEFAULTS, **TargetPlatform.DEFAULTS, "template": "ml_interface", + "template_version": None, "ignore_data": True, "skip_check": False, "fail_on_error": False, # Prefer to add acolum with validation results instead of raising a RuntimeError @@ -82,10 +88,20 @@ class MlifPlatform(CompilePlatform, TargetPlatform): "strip_strings": False, "unroll_loops": None, "goal": "generic_mlonmcu", # Use 'generic_mlif' for older version of MLIF + "set_inputs": False, + "set_inputs_interface": None, + "get_outputs": False, + "get_outputs_interface": None, + "get_outputs_fmt": None, + "batch_size": None, + "model_support_file": None, + "model_support_dir": None, + "model_support_lib": None, # llvm specific (TODO: move to toolchain components) "fuse_ld": None, "global_isel": False, "extend_attrs": False, + "ccache": False, } REQUIRED = {"mlif.src_dir"} @@ -104,6 +120,67 @@ def __init__(self, features=None, config=None): def goal(self): return self.config["goal"] + @property + def ccache(self): + value = self.config["ccache"] + return str2bool(value) if not isinstance(value, (bool, int)) else value + + @property + def set_inputs(self): + value = self.config["set_inputs"] + return str2bool(value) if not isinstance(value, (bool, int)) else value + + @property + def set_inputs_interface(self): + value = self.config["set_inputs_interface"] + return value + + @property + def get_outputs(self): + value = self.config["get_outputs"] + return str2bool(value) if not isinstance(value, (bool, int)) else value + + @property + def get_outputs_interface(self): + value = self.config["get_outputs_interface"] + return value + + @property + def get_outputs_fmt(self): + value = self.config["get_outputs_fmt"] # TODO: use + return value + + @property + def batch_size(self): + value = self.config["batch_size"] # TODO: use + if isinstance(value, str): + value = int(value) + return value + + @property + def inputs_artifact(self): + # THIS IS A HACK (get inputs fom artifacts!) + lookup_path = self.build_dir.parent / "inputs.npy" + if lookup_path.is_file(): + return lookup_path + else: + logger.warning("Artifact 'inputs.npz' not found!") + return None + + @property + def model_info_file(self): + # THIS IS A HACK (get inputs fom artifacts!) + lookup_path = self.build_dir.parent / "model_info.yml" + if lookup_path.is_file(): + return lookup_path + else: + logger.warning("Artifact 'model_info.yml' not found!") + return None + + @property + def needs_model_support(self): + return self.set_inputs or self.get_outputs + def gen_data_artifact(self): in_paths = self.input_data_path if not isinstance(in_paths, list): @@ -193,6 +270,10 @@ def srecord_dir(self): def template(self): return self.config["template"] + @property + def template_version(self): + return self.config["template_version"] + @property def ignore_data(self): value = self.config["ignore_data"] @@ -216,9 +297,20 @@ def validate_outputs(self): def toolchain(self): return str(self.config["toolchain"]) + @property + def model_support_file(self): + value = self.config["model_support_file"] # TODO: use + return value + @property def model_support_dir(self): - return self.config["model_support_dir"] + value = self.config["model_support_dir"] # TODO: use + return value + + @property + def model_support_lib(self): + value = self.config["model_support_lib"] # TODO: use + return value @property def prebuild_lib_dir(self): @@ -304,15 +396,21 @@ def close(self): def get_definitions(self): definitions = self.definitions definitions["TEMPLATE"] = self.template + if self.template_version: + definitions["TEMPLATE_VERSION"] = self.template_version definitions["TOOLCHAIN"] = self.toolchain definitions["QUIET"] = self.mem_only definitions["SKIP_CHECK"] = self.skip_check + if self.batch_size is not None: + definitions["BATCH_SIZE"] = self.batch_size if self.num_threads is not None: definitions["SUBPROJECT_THREADS"] = self.num_threads - if self.toolchain == "llvm" and self.llvm_dir is None: - raise RuntimeError("Missing config variable: llvm.install_dir") - else: - definitions["LLVM_DIR"] = self.llvm_dir + if self.toolchain == "llvm": + if self.llvm_dir is None: + raise RuntimeError("Missing config variable: llvm.install_dir") + llvm_dir = Path(self.llvm_dir).resolve() + assert llvm_dir.is_dir(), f"llvm.install_dir does not exist: {llvm_dir}" + definitions["LLVM_DIR"] = llvm_dir if self.optimize is not None: definitions["OPTIMIZE"] = self.optimize if self.debug_symbols is not None: @@ -325,8 +423,12 @@ def get_definitions(self): definitions["ENABLE_GC"] = self.garbage_collect if self.slim_cpp is not None: definitions["SLIM_CPP"] = self.slim_cpp + if self.model_support_file is not None: + definitions["MODEL_SUPPORT_FILE"] = self.model_support_file if self.model_support_dir is not None: definitions["MODEL_SUPPORT_DIR"] = self.model_support_dir + if self.model_support_lib is not None: + definitions["MODEL_SUPPORT_LIB"] = self.model_support_lib if self.fuse_ld is not None: definitions["FUSE_LD"] = self.fuse_ld if self.global_isel is not None: @@ -337,6 +439,9 @@ def get_definitions(self): definitions["STRIP_STRINGS"] = self.strip_strings if self.unroll_loops is not None: definitions["UNROLL_LOOPS"] = self.unroll_loops + if self.ccache: + definitions["CMAKE_C_COMPILER_LAUNCHER"] = "ccache" # TODO: choose between ccache/sccache + definitions["CMAKE_CXX_COMPILER_LAUNCHER"] = "ccache" # TODO: choose between ccache/sccache return definitions @@ -360,8 +465,46 @@ def prepare_environment(self): env["PATH"] = path_new return env + def generate_model_support(self, target): + artifacts = [] + batch_size = self.batch_size + inputs_data = None + if self.inputs_artifact is not None: + inputs_data = np.load(self.inputs_artifact, allow_pickle=True) + if self.model_info_file is not None: + with open(self.model_info_file, "r") as f: + model_info = yaml.safe_load(f) + if self.set_inputs or self.get_outputs: + model_support = ModelSupport( + in_interface=self.set_inputs_interface, + out_interface=self.get_outputs_interface, + model_info=model_info, + target=target, + batch_size=batch_size, + inputs_data=inputs_data, + ) + code = model_support.generate() + code_artifact = Artifact( + "model_support.cpp", + content=code, + fmt=ArtifactFormat.TEXT, + flags=("model_support"), + ) + self.definitions["BATCH_SIZE"] = model_support.batch_size + artifacts.append(code_artifact) + return artifacts + def configure(self, target, src, _model): - del target + artifacts = [] + if self.needs_model_support: + artifacts.extend(self.generate_model_support(target)) + if len(artifacts) > 0: + assert len(artifacts) == 1 + model_support_artifact = artifacts[0] + model_support_file = self.build_dir / model_support_artifact.name + model_support_artifact.export(model_support_file) + self.definitions["MODEL_SUPPORT_FILE"] = model_support_file + del target if not isinstance(src, Path): src = Path(src) cmakeArgs = self.get_cmake_args() @@ -371,11 +514,11 @@ def configure(self, target, src, _model): cmakeArgs.append("-DSRC_DIR=" + str(src)) else: raise RuntimeError("Unable to find sources!") - artifacts = [] if self.ignore_data: cmakeArgs.append("-DDATA_SRC=") else: - data_artifact = self.gen_data_artifact() + # data_artifact = self.gen_data_artifact() + data_artifact = None if data_artifact: data_file = self.build_dir / data_artifact.name data_artifact.export(data_file) diff --git a/mlonmcu/platform/mlif/mlif_target.py b/mlonmcu/platform/mlif/mlif_target.py index f8249aa13..7ee546a09 100644 --- a/mlonmcu/platform/mlif/mlif_target.py +++ b/mlonmcu/platform/mlif/mlif_target.py @@ -16,11 +16,17 @@ # See the License for the specific language governing permissions and # limitations under the License. # +import os +from math import ceil +from pathlib import Path from enum import IntEnum from mlonmcu.target import get_targets, Target +from mlonmcu.artifact import Artifact, ArtifactFormat from mlonmcu.logging import get_logger +from .interfaces import ModelSupport + logger = get_logger() @@ -49,6 +55,151 @@ def __init__(self, features=None, config=None): self.platform = platform self.validation_result = None + def exec(self, program, *args, cwd=os.getcwd(), **kwargs): + ins_file = None + num_inputs = 0 + in_interface = None + out_interface = None + batch_size = 1 + encoding = "utf-8" + model_info_file = self.platform.model_info_file # TODO: replace workaround (add model info to platform?) + if self.platform.set_inputs or self.platform.get_outputs: + # first figure out how many inputs are provided + if model_info_file is not None: + import yaml + + with open(model_info_file, "r") as f: + model_info_data = yaml.safe_load(f) + else: + model_info_data = None + if self.platform.inputs_artifact is not None: + import numpy as np + + data = np.load(self.platform.inputs_artifact, allow_pickle=True) + num_inputs = len(data) + else: + data = None + model_support = ModelSupport( + in_interface=self.platform.set_inputs_interface, + out_interface=self.platform.get_outputs_interface, + model_info=model_info_data, + target=self, + batch_size=self.platform.batch_size, + inputs_data=data, + ) + in_interface = model_support.in_interface + out_interface = model_support.out_interface + batch_size = model_support.batch_size + if out_interface == "stdout_raw": + encoding = None + outs_file = None + ret = "" + artifacts = [] + num_batches = max(ceil(num_inputs / batch_size), 1) + processed_inputs = 0 + # remaining_inputs = num_inputs + outs_data = [] + stdin_data = None + for idx in range(num_batches): + # print("idx", idx) + # current_batch_size = max(min(batch_size, remaining_inputs), 1) + if processed_inputs < num_inputs: + if in_interface == "filesystem": + batch_data = data[idx * batch_size : ((idx + 1) * batch_size)] + # print("batch_data", batch_data, type(batch_data)) + ins_file = Path(cwd) / "ins.npy" + np.save(ins_file, batch_data) + elif in_interface == "stdin": + raise NotImplementedError + elif in_interface == "stdin_raw": + batch_data = data[idx * batch_size : ((idx + 1) * batch_size)] + # print("batch_data", batch_data, type(batch_data)) + stdin_data = b"" + for cur_data in batch_data: + # print("cur_data", cur_data) + for key, value in cur_data.items(): + # print("key", key) + # print("value", value, type(value)) + # print("value.tostring", value.tostring()) + stdin_data += value.tostring() + # TODO: check that stdin_data has expected size + # This is just a placeholder example! + # stdin_data = "input[0] = {0, 1, 2, ...};\nDONE\n""".encode() + # stdin_data *= 200 + # raise NotImplementedError + # TODO: generate input stream here! + + ret_, artifacts_ = super().exec( + program, *args, cwd=cwd, **kwargs, stdin_data=stdin_data, encoding=encoding + ) + if self.platform.get_outputs: + if out_interface == "filesystem": + import numpy as np + + outs_file = Path(cwd) / "outs.npy" + with np.load(outs_file) as out_data: + outs_data.extend(dict(out_data)) + elif out_interface == "stdout": + # TODO: get output_data from stdout + raise NotImplementedError + elif out_interface == "stdout_raw": + # DUMMY BELOW + assert model_info_data is not None + # print("model_info_data", model_info_data) + # dtype = "int8" + # shape = [1, 10] + # input("!") + # print("ret_", ret_, type(ret_)) + # out_idx = 0 + x = ret_ # Does this copy? + while True: + out_data_temp = {} + # substr = ret_[ret_.find("-?-".encode())+3:ret_.find("-!-".encode())] + # print("substr", substr, len(substr)) + found_start = x.find("-?-".encode()) + # print("found_start", found_start) + if found_start < 0: + break + x = x[found_start + 3 :] + # print("x[:20]", x[:20]) + found_end = x.find("-!-".encode()) + # print("found_end", found_end) + assert found_end >= 0 + x_ = x[:found_end] + x = x[found_end + 3 :] + # print("x[:20]", x[:20]) + # print("x_", x_) + # out_idx += 1 + dtype = model_info_data["output_types"][0] + arr = np.frombuffer(x_, dtype=dtype) + # print("arr", arr) + shape = model_info_data["output_shapes"][0] + arr = arr.reshape(shape) + # print("arr2", arr) + assert len(model_info_data["output_names"]) == 1, "Multi-output models not yet supported" + out_name = model_info_data["output_names"][0] + out_data_temp[out_name] = arr + outs_data.append(out_data_temp) + # {"output_0": arr}]) + ret_ = ret_.decode("utf-8", errors="replace") + # raise NotImplementedError + else: + assert False + ret += ret_ + artifacts += artifacts_ + # print("outs_data", outs_data) + # input("$") + if len(outs_data) > 0: + outs_path = Path(cwd) / "outputs.npy" + np.save(outs_path, outs_data) + with open(outs_path, "rb") as f: + outs_raw = f.read() + outputs_artifact = Artifact( + "outputs.npy", raw=outs_raw, fmt=ArtifactFormat.BIN, flags=("outputs", "npy") + ) + artifacts.append(outputs_artifact) + return ret, artifacts + def get_metrics(self, elf, directory, handle_exit=None): # This is wrapper around the original exec function to catch special return codes thrown by the inout data # feature (TODO: catch edge cases: no input data available (skipped) and no return code (real hardware)) @@ -63,9 +214,10 @@ def _handle_exit(code, out=None): if code in MlifExitCode.values(): reason = MlifExitCode(code).name logger.error("A platform error occured during the simulation. Reason: %s", reason) - self.validation_result = False - if not self.platform.fail_on_error: - code = 0 + if code == MlifExitCode.OUTPUT_MISSMATCH: + self.validation_result = False + if not self.platform.fail_on_error: + code = 0 return code else: diff --git a/mlonmcu/platform/tvm/tvm_target.py b/mlonmcu/platform/tvm/tvm_target.py index b9e387b67..91e839f13 100644 --- a/mlonmcu/platform/tvm/tvm_target.py +++ b/mlonmcu/platform/tvm/tvm_target.py @@ -18,9 +18,11 @@ # import re import os +from pathlib import Path from mlonmcu.target.target import Target from mlonmcu.target.metrics import Metrics +from mlonmcu.artifact import Artifact, ArtifactFormat from mlonmcu.logging import get_logger @@ -61,8 +63,92 @@ def exec(self, program, *args, cwd=os.getcwd(), **kwargs): if self.timeout_sec > 0: raise NotImplementedError - ret = self.platform.run(program, self) - return ret + ins_file = None + num_inputs = 0 + batch_size = 1 + if self.platform.set_inputs: + interface = self.platform.set_inputs_interface + if interface == "auto": + if self.supports_filesystem: + interface = "filesystem" + else: + assert interface in ["filesystem"] + if interface == "filesystem": + ins_file = self.platform.ins_file + if ins_file is None: + assert self.platform.inputs_artifact is not None + import numpy as np + + data = np.load(self.platform.inputs_artifact, allow_pickle=True) + num_inputs = len(data) + ins_file = Path(cwd) / "ins.npz" + outs_file = None + print_top = self.platform.print_top + if self.platform.get_outputs: + interface = self.platform.get_outputs_interface + if interface == "auto": + if self.supports_filesystem: + interface = "filesystem" + elif self.supports_stdout: + interface = "stdout" + else: + assert interface in ["filesystem", "stdout"] + if interface == "filesystem": + outs_file = self.platform.outs_file + if outs_file is None: + outs_file = Path(cwd) / "outs.npz" + elif interface == "stdout": + print_top = 1e6 + + ret = "" + artifacts = [] + num_batches = max(round(num_inputs / batch_size), 1) + processed_inputs = 0 + remaining_inputs = num_inputs + outs_data = [] + for idx in range(num_batches): + current_batch_size = max(min(batch_size, remaining_inputs), 1) + assert current_batch_size == 1 + if processed_inputs < num_inputs: + in_data = data[idx] + np.savez(ins_file, **in_data) + processed_inputs += 1 + remaining_inputs -= 1 + else: + ins_file = None + ret_, artifacts_ = self.platform.run( + program, self, cwd=cwd, ins_file=ins_file, outs_file=outs_file, print_top=print_top + ) + ret += ret_ + if self.platform.get_outputs: + interface = self.platform.get_outputs_interface + if interface == "auto": + if self.supports_filesystem: + interface = "filesystem" + elif self.supports_stdout: + interface = "stdout" + else: + assert interface in ["filesystem", "stdout"] + if interface == "filesystem": + import numpy as np + + with np.load(outs_file) as out_data: + outs_data.append(dict(out_data)) + elif interface == "stdout": + raise NotImplementedError + else: + assert False + if len(outs_data) > 0: + outs_path = Path(cwd) / "outputs.npy" + np.save(outs_path, outs_data) + with open(outs_path, "rb") as f: + outs_raw = f.read() + outputs_artifact = Artifact( + "outputs.npy", raw=outs_raw, fmt=ArtifactFormat.BIN, flags=("outputs", "npy") + ) + artifacts.append(outputs_artifact) + + return ret, artifacts def parse_stdout(self, out): mean_ms = None @@ -91,10 +177,11 @@ def parse_stdout(self, out): return mean_ms, median_ms, max_ms, min_ms, std_ms def get_metrics(self, elf, directory, handle_exit=None): + artifacts = [] if self.print_outputs: - out = self.exec(elf, cwd=directory, live=True, handle_exit=handle_exit) + out, artifacts = self.exec(elf, cwd=directory, live=True, handle_exit=handle_exit) else: - out = self.exec( + out, artifacts = self.exec( elf, cwd=directory, live=False, @@ -153,7 +240,7 @@ def extract_cols(line): metrics_.add("Runtime [s]", float(item["Duration (us)"].replace(",", "")) / 1e6) metrics[item["Name"]] = metrics_ - return metrics, out, [] + return metrics, out, artifacts def get_arch(self): return "unkwown" @@ -162,4 +249,12 @@ def update_environment(self, env): # TODO: implement in base class? pass + @property + def supports_filesystem(self): + return True + + @property + def supports_stdout(self): + return True + return TvmPlatformTarget diff --git a/mlonmcu/platform/tvm/tvm_target_platform.py b/mlonmcu/platform/tvm/tvm_target_platform.py index c521f57e0..608da1def 100644 --- a/mlonmcu/platform/tvm/tvm_target_platform.py +++ b/mlonmcu/platform/tvm/tvm_target_platform.py @@ -17,6 +17,7 @@ # limitations under the License. # """TVM Target Platform""" +import os from mlonmcu.config import str2bool from .tvm_rpc_platform import TvmRpcPlatform from ..platform import TargetPlatform @@ -28,6 +29,9 @@ get_data_tvmc_args, get_rpc_tvmc_args, ) +from mlonmcu.logging import get_logger + +logger = get_logger() class TvmTargetPlatform(TargetPlatform, TvmRpcPlatform): @@ -39,6 +43,8 @@ class TvmTargetPlatform(TargetPlatform, TvmRpcPlatform): | { "benchmark", "tvm_profile", + "set_inputs", + "get_outputs", } ) @@ -54,6 +60,11 @@ class TvmTargetPlatform(TargetPlatform, TvmRpcPlatform): "number": 1, "aggregate": "none", # Allowed: avg, max, min, none, all "total_time": False, + "set_inputs": False, + "set_inputs_interface": None, + "get_outputs": False, + "get_outputs_interface": None, + "get_outputs_fmt": None, } REQUIRED = TargetPlatform.REQUIRED | TvmRpcPlatform.REQUIRED @@ -72,7 +83,8 @@ def outs_file(self): @property def print_top(self): - return self.config["print_top"] + value = self.config["print_top"] + return int(value) if isinstance(value, str) else None @property def profile(self): @@ -98,6 +110,41 @@ def total_time(self): value = self.config["total_time"] return str2bool(value) if not isinstance(value, (bool, int)) else value + @property + def set_inputs(self): + value = self.config["set_inputs"] + return str2bool(value) if not isinstance(value, (bool, int)) else value + + @property + def set_inputs_interface(self): + value = self.config["set_inputs_interface"] + return value + + @property + def get_outputs(self): + value = self.config["get_outputs"] + return str2bool(value) if not isinstance(value, (bool, int)) else value + + @property + def get_outputs_interface(self): + value = self.config["get_outputs_interface"] + return value + + @property + def get_outputs_fmt(self): + value = self.config["get_outputs_fmt"] # TODO: use + return value + + @property + def inputs_artifact(self): + # THIS IS A HACK (get inputs fom artifacts!) + lookup_path = self.project_dir.parent / "inputs.npy" + if lookup_path.is_file(): + return lookup_path + else: + logger.warning("Artifact 'inputs.npz' not found!") + return None + def flash(self, elf, target, timeout=120): raise NotImplementedError @@ -121,29 +168,36 @@ def create_target(self, name): base = Target return create_tvm_platform_target(name, self, base=base) - def get_tvmc_run_args(self): + def get_tvmc_run_args(self, ins_file=None, outs_file=None, print_top=None): return [ - *get_data_tvmc_args( - mode=self.fill_mode, ins_file=self.ins_file, outs_file=self.outs_file, print_top=self.print_top - ), + *get_data_tvmc_args(mode=self.fill_mode, ins_file=ins_file, outs_file=outs_file, print_top=print_top), *get_bench_tvmc_args( print_time=True, profile=self.profile, end_to_end=False, repeat=self.repeat, number=self.number ), *get_rpc_tvmc_args(self.use_rpc, self.rpc_key, self.rpc_hostname, self.rpc_port), ] - def invoke_tvmc_run(self, *args, target=None): + def invoke_tvmc_run(self, *args, target=None, **kwargs): assert target is not None, "Target required for tvmc run" combined_args = [] combined_args.extend(["--device", target.device]) - return self.invoke_tvmc("run", *args) + return self.invoke_tvmc("run", *args, **kwargs) - def run(self, elf, target, timeout=120): + def run(self, elf, target, timeout=120, cwd=os.getcwd(), ins_file=None, outs_file=None, print_top=None): + artifacts = [] # TODO: implement timeout # Here, elf is actually a directory # TODO: replace workaround with possibility to pass TAR directly tar_path = str(elf) - args = [tar_path] + self.get_tvmc_run_args() - output = self.invoke_tvmc_run(*args, target=target) - - return output + # in_path = self.ins_file + # out_path = self.outs_file + # set_inputs = False + # if set_inputs and in_path is None: + # in_path = Path(cwd) / "ins.npz" + # # TODO: populate + # if self.get_outputs and self.get_outputs_interface == "filesystem" and out_path is None: + # out_path = Path(cwd) / "outs.npz" + args = [tar_path] + self.get_tvmc_run_args(ins_file=ins_file, outs_file=outs_file, print_top=print_top) + output = self.invoke_tvmc_run(*args, target=target, cwd=cwd) + + return output, artifacts diff --git a/mlonmcu/session/postprocess/__init__.py b/mlonmcu/session/postprocess/__init__.py index 36f2e432b..c02fa9400 100644 --- a/mlonmcu/session/postprocess/__init__.py +++ b/mlonmcu/session/postprocess/__init__.py @@ -32,6 +32,9 @@ CompareRowsPostprocess, AnalyseDumpPostprocess, AnalyseCoreVCountsPostprocess, + ValidateOutputsPostprocess, + ValidateLabelsPostprocess, + ExportOutputsPostprocess, ) SUPPORTED_POSTPROCESSES = { @@ -48,4 +51,7 @@ "compare_rows": CompareRowsPostprocess, "analyse_dump": AnalyseDumpPostprocess, "analyse_corev_counts": AnalyseCoreVCountsPostprocess, + "validate_outputs": ValidateOutputsPostprocess, + "validate_labels": ValidateLabelsPostprocess, + "export_outputs": ExportOutputsPostprocess, } diff --git a/mlonmcu/session/postprocess/postprocesses.py b/mlonmcu/session/postprocess/postprocesses.py index 72b8ada01..55a8a0682 100644 --- a/mlonmcu/session/postprocess/postprocesses.py +++ b/mlonmcu/session/postprocess/postprocesses.py @@ -24,6 +24,7 @@ from pathlib import Path from io import StringIO +import numpy as np import pandas as pd from mlonmcu.artifact import Artifact, ArtifactFormat, lookup_artifacts @@ -31,6 +32,7 @@ from mlonmcu.logging import get_logger from .postprocess import SessionPostprocess, RunPostprocess +from .validate_metrics import parse_validate_metrics, parse_classify_metrics logger = get_logger() @@ -1455,3 +1457,392 @@ def post_run(self, report, artifacts): report.post_df = post_df assert self.to_file or self.to_df, "Either to_file or to_df have to be true" return ret_artifacts + + +class ValidateOutputsPostprocess(RunPostprocess): + """Postprocess for comparing model outputs with golden reference.""" + + DEFAULTS = { + **RunPostprocess.DEFAULTS, + "report": False, + "validate_metrics": "topk(n=1);topk(n=2)", + "validate_range": True, + } + + def __init__(self, features=None, config=None): + super().__init__("validate_outputs", features=features, config=config) + + @property + def validate_metrics(self): + """Get validate_metrics property.""" + value = self.config["validate_metrics"] + return value + + @property + def report(self): + """Get report property.""" + value = self.config["report"] + return str2bool(value) if not isinstance(value, (bool, int)) else value + + @property + def validate_range(self): + """Get validate_range property.""" + value = self.config["validate_range"] + return str2bool(value) if not isinstance(value, (bool, int)) else value + + def post_run(self, report, artifacts): + """Called at the end of a run.""" + model_info_artifact = lookup_artifacts(artifacts, name="model_info.yml", first_only=True) + assert len(model_info_artifact) == 1, "Could not find artifact: model_info.yml" + model_info_artifact = model_info_artifact[0] + import yaml + + model_info_data = yaml.safe_load(model_info_artifact.content) + if len(model_info_data["output_names"]) > 1: + raise NotImplementedError("Multi-outputs not yet supported.") + outputs_ref_artifact = lookup_artifacts(artifacts, name="outputs_ref.npy", first_only=True) + assert len(outputs_ref_artifact) == 1, "Could not find artifact: outputs_ref.npy" + outputs_ref_artifact = outputs_ref_artifact[0] + import numpy as np + + outputs_ref = np.load(outputs_ref_artifact.path, allow_pickle=True) + # import copy + # outputs = copy.deepcopy(outputs_ref) + # outputs[1][list(outputs[1].keys())[0]][0] = 42 + outputs_artifact = lookup_artifacts(artifacts, name="outputs.npy", first_only=True) + assert len(outputs_artifact) == 1, "Could not find artifact: outputs.npy" + outputs_artifact = outputs_artifact[0] + outputs = np.load(outputs_artifact.path, allow_pickle=True) + in_data = None + # compared = 0 + # matching = 0 + # missing = 0 + # metrics = { + # "allclose(atol=0.0,rtol=0.0)": None, + # "allclose(atol=0.05,rtol=0.05)": None, + # "allclose(atol=0.1,rtol=0.1)": None, + # "topk(n=1)": None, + # "topk(n=2)": None, + # "topk(n=inf)": None, + # "toy": None, + # "mse(thr=0.1)": None, + # "mse(thr=0.05)": None, + # "mse(thr=0.01)": None, + # "+-1": None, + # } + validate_metrics_str = self.validate_metrics + validate_metrics = parse_validate_metrics(validate_metrics_str) + for i, output_ref in enumerate(outputs_ref): + if i >= len(outputs): + logger.warning("Missing output sample") + # missing += 1 + break + output = outputs[i] + ii = 0 + for out_name, out_ref_data in output_ref.items(): + if out_name in output: + out_data = output[out_name] + elif ii < len(output): + if isinstance(output, dict): + # fallback for custom name-based npy dict + out_data = list(output.values())[ii] + else: # fallback for index-based npy array + assert isinstance(output, (list, np.array)), "expected dict, list or np.array type" + out_data = output[ii] + else: + RuntimeError(f"Output not found: {out_name}") + # optional dequantize + # print("out_data_before_quant", out_data) + # print("sum(out_data_before_quant", np.sum(out_data)) + + quant = model_info_data.get("output_quant_details", None) + rng = model_info_data.get("output_ranges", None) + if quant: + + def ref_quant_helper(quant, data): # TODO: move somewhere else + if quant is None: + return data + quant_scale, quant_zero_point, quant_dtype, quant_range = quant + if quant_dtype is None or data.dtype.name == quant_dtype: + return data + assert data.dtype.name in ["float32"], "Quantization only supported for float32 input" + assert quant_dtype in ["int8"], "Quantization only supported for int8 output" + if quant_range and self.validate_range: + assert len(quant_range) == 2, "Range should be a tuple (lower, upper)" + lower, upper = quant_range + # print("quant_range", quant_range) + # print("np.min(data)", np.min(data)) + # print("np.max(data)", np.max(data)) + assert lower <= upper + assert np.min(data) >= lower and np.max(data) <= upper, "Range missmatch" + + return np.around((data / quant_scale) + quant_zero_point).astype("int8") + + def dequant_helper(quant, data): # TODO: move somewhere else + if quant is None: + return data + quant_scale, quant_zero_point, quant_dtype, quant_range = quant + if quant_dtype is None or data.dtype.name == quant_dtype: + return data + assert data.dtype.name in ["int8"], "Dequantization only supported for int8 input" + assert quant_dtype in ["float32"], "Dequantization only supported for float32 output" + ret = (data.astype("float32") - quant_zero_point) * quant_scale + if quant_range and self.validate_range: + assert len(quant_range) == 2, "Range should be a tuple (lower, upper)" + # print("quant_range", quant_range) + # print("np.min(ret)", np.min(ret)) + # print("np.max(ret)", np.max(ret)) + lower, upper = quant_range + assert lower <= upper + assert np.min(ret) >= lower and np.max(ret) <= upper, "Range missmatch" + return ret + + assert ii < len(rng) + rng_ = rng[ii] + if rng_ and self.validate_range: + assert len(rng_) == 2, "Range should be a tuple (lower, upper)" + lower, upper = rng_ + assert lower <= upper + # print("rng_", rng_) + # print("np.min(out_data)", np.min(out_data)) + # print("np.max(out_data)", np.max(out_data)) + assert np.min(out_data) >= lower and np.max(out_data) <= upper, "Range missmatch" + assert ii < len(quant) + quant_ = quant[ii] + if quant_ is not None: + out_ref_data_quant = ref_quant_helper(quant_, out_ref_data) + for vm in validate_metrics: + vm.process(out_data, out_ref_data_quant, in_data=in_data, quant=True) + out_data = dequant_helper(quant_, out_data) + # print("out_data", out_data) + # print("sum(out_data)", np.sum(out_data)) + # print("out_ref_data", out_ref_data) + # print("sum(out_ref_data)", np.sum(out_ref_data)) + # input("TIAW") + assert out_data.dtype == out_ref_data.dtype, "dtype missmatch" + assert out_data.shape == out_ref_data.shape, "shape missmatch" + + for vm in validate_metrics: + vm.process(out_data, out_ref_data, in_data=in_data, quant=False) + ii += 1 + if self.report: + raise NotImplementedError + for vm in validate_metrics: + res = vm.get_summary() + report.post_df[f"{vm.name}"] = res + return [] + + +class ValidateLabelsPostprocess(RunPostprocess): + """Postprocess for comparing model outputs with golden reference.""" + + DEFAULTS = { + **RunPostprocess.DEFAULTS, + "report": False, + "classify_metrics": "topk_label(n=1);topk_label(n=2)", + } + + def __init__(self, features=None, config=None): + super().__init__("validate_labels", features=features, config=config) + + @property + def classify_metrics(self): + """Get classify_metrics property.""" + value = self.config["classify_metrics"] + return value + + @property + def report(self): + """Get report property.""" + value = self.config["report"] + return str2bool(value) if not isinstance(value, (bool, int)) else value + + def post_run(self, report, artifacts): + """Called at the end of a run.""" + model_info_artifact = lookup_artifacts(artifacts, name="model_info.yml", first_only=True) + assert len(model_info_artifact) == 1, "Could not find artifact: model_info.yml" + model_info_artifact = model_info_artifact[0] + import yaml + + model_info_data = yaml.safe_load(model_info_artifact.content) + if len(model_info_data["output_names"]) > 1: + raise NotImplementedError("Multi-outputs not yet supported.") + labels_ref_artifact = lookup_artifacts(artifacts, name="labels_ref.npy", first_only=True) + assert ( + len(labels_ref_artifact) == 1 + ), "Could not find artifact: labels_ref.npy (Run classify_labels postprocess first!)" + labels_ref_artifact = labels_ref_artifact[0] + import numpy as np + + labels_ref = np.load(labels_ref_artifact.path, allow_pickle=True) + outputs_artifact = lookup_artifacts(artifacts, name="outputs.npy", first_only=True) + assert len(outputs_artifact) == 1, "Could not find artifact: outputs.npy" + outputs_artifact = outputs_artifact[0] + outputs = np.load(outputs_artifact.path, allow_pickle=True) + # missing = 0 + classify_metrics_str = self.classify_metrics + classify_metrics = parse_classify_metrics(classify_metrics_str) + for i, output in enumerate(outputs): + if isinstance(output, dict): # name based lookup + pass + else: # index based lookup + assert isinstance(output, (list, np.array)), "expected dict, list or np.array" + output_names = model_info_data["output_names"] + assert len(output) == len(output_names) + output = {output_names[idx]: out for idx, out in enumerate(output)} + assert len(output) == 1, "Only supporting single-output models" + out_data = output[list(output.keys())[0]] + # print("out_data", out_data) + assert i < len(labels_ref), "Missing reference labels" + label_ref = labels_ref[i] + # print("label_ref", label_ref) + for cm in classify_metrics: + cm.process(out_data, label_ref, quant=False) + if self.report: + raise NotImplementedError + for cm in classify_metrics: + res = cm.get_summary() + report.post_df[f"{cm.name}"] = res + return [] + + +class ExportOutputsPostprocess(RunPostprocess): + """Postprocess for writing model outputs to a directory.""" + + DEFAULTS = { + **RunPostprocess.DEFAULTS, + "dest": None, # if none: export as artifact + "use_ref": False, + "skip_dequant": False, + "fmt": "bin", + "archive_fmt": None, + } + + def __init__(self, features=None, config=None): + super().__init__("export_outputs", features=features, config=config) + + @property + def dest(self): + """Get dest property.""" + value = self.config["dest"] + if value is not None: + if not isinstance(value, Path): + assert isinstance(value, str) + value = Path(value) + return value + + @property + def use_ref(self): + """Get use_ref property.""" + value = self.config["use_ref"] + return str2bool(value) if not isinstance(value, (bool, int)) else value + + @property + def skip_dequant(self): + """Get skip_dequant property.""" + value = self.config["skip_dequant"] + return str2bool(value) if not isinstance(value, (bool, int)) else value + + @property + def fmt(self): + """Get fmt property.""" + return self.config["fmt"] + + @property + def archive_fmt(self): + """Get archive_fmt property.""" + return self.config["archive_fmt"] + + def post_run(self, report, artifacts): + """Called at the end of a run.""" + model_info_artifact = lookup_artifacts(artifacts, name="model_info.yml", first_only=True) + assert len(model_info_artifact) == 1, "Could not find artifact: model_info.yml" + model_info_artifact = model_info_artifact[0] + import yaml + + model_info_data = yaml.safe_load(model_info_artifact.content) + # print("model_info_data", model_info_data) + if len(model_info_data["output_names"]) > 1: + raise NotImplementedError("Multi-outputs not yet supported.") + if self.use_ref: + outputs_ref_artifact = lookup_artifacts(artifacts, name="outputs_ref.npy", first_only=True) + assert len(outputs_ref_artifact) == 1, "Could not find artifact: outputs_ref.npy" + outputs_ref_artifact = outputs_ref_artifact[0] + outputs_ref = np.load(outputs_ref_artifact.path, allow_pickle=True) + outputs = outputs_ref + else: + outputs_artifact = lookup_artifacts(artifacts, name="outputs.npy", first_only=True) + assert len(outputs_artifact) == 1, "Could not find artifact: outputs.npy" + outputs_artifact = outputs_artifact[0] + outputs = np.load(outputs_artifact.path, allow_pickle=True) + if self.dest is None: + temp_dir = tempfile.TemporaryDirectory() + dest_ = Path(temp_dir.name) + else: + temp_dir = None + assert self.dest.is_dir(), f"Not a directory: {self.dest}" + dest_ = self.dest + assert self.fmt in ["bin", "npy"], f"Invalid format: {self.fmt}" + filenames = [] + for i, output in enumerate(outputs): + if isinstance(output, dict): # name based lookup + pass + else: # index based lookup + assert isinstance(output, (list, np.array)), "expected dict, list or np.array" + output_names = model_info_data["output_names"] + assert len(output) == len(output_names) + output = {output_names[idx]: out for idx, out in enumerate(output)} + quant = model_info_data.get("output_quant_details", None) + if quant and not self.skip_dequant: + + def dequant_helper(quant, data): + if quant is None: + return data + quant_scale, quant_zero_point, quant_dtype, quant_range = quant + if quant_dtype is None or data.dtype.name == quant_dtype: + return data + assert data.dtype.name in ["int8"], "Dequantization only supported for int8 input" + assert quant_dtype in ["float32"], "Dequantization only supported for float32 output" + return (data.astype("float32") - quant_zero_point) * quant_scale + + output = { + out_name: dequant_helper(quant[j], output[out_name]) for j, out_name in enumerate(output.keys()) + } + if self.fmt == "npy": + raise NotImplementedError("npy export") + elif self.fmt == "bin": + assert len(output.keys()) == 1, "Multi-outputs not supported" + output_data = list(output.values())[0] + data = output_data.tobytes(order="C") + file_name = f"{i}.bin" + file_dest = dest_ / file_name + filenames.append(file_dest) + with open(file_dest, "wb") as f: + f.write(data) + else: + assert False, f"fmt not supported: {self.fmt}" + artifacts = [] + archive_fmt = self.archive_fmt + create_artifact = self.dest is None or archive_fmt is not None + if create_artifact: + if archive_fmt is None: + assert self.dest is None + archive_fmt = "tar.gz" # Default fallback + assert archive_fmt in ["tar.xz", "tar.gz", "zip"] + archive_name = f"output_data.{archive_fmt}" + archive_path = f"{dest_}.{archive_fmt}" + if archive_fmt == "tar.gz": + import tarfile + + with tarfile.open(archive_path, "w:gz") as tar: + for filename in filenames: + tar.add(filename, arcname=filename.name) + else: + raise NotImplementedError(f"archive_fmt={archive_fmt}") + with open(archive_path, "rb") as f: + raw = f.read() + artifact = Artifact(archive_name, raw=raw, fmt=ArtifactFormat.BIN) + artifacts.append(artifact) + if temp_dir: + temp_dir.cleanup() + return artifacts diff --git a/mlonmcu/session/postprocess/validate_metrics.py b/mlonmcu/session/postprocess/validate_metrics.py new file mode 100644 index 000000000..35b8db3fe --- /dev/null +++ b/mlonmcu/session/postprocess/validate_metrics.py @@ -0,0 +1,332 @@ +# +# Copyright (c) 2024 TUM Department of Electrical and Computer Engineering. +# +# This file is part of MLonMCU. +# See https://github.com/tum-ei-eda/mlonmcu.git for further info. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +"""Validation metrics utilities.""" + +import ast +import numpy as np +from typing import Optional + +from mlonmcu.logging import get_logger + +logger = get_logger() + + +class ValidationMetric: + def __init__(self, name, **cfg): + self.name = name + self.num_total = 0 + self.num_correct = 0 + + def process_(self, out_data, out_data_ref, in_data: Optional[np.array] = None, quant: bool = False): + raise NotImplementedError + + def check(self, out_data, out_data_ref, quant: bool = False): + return out_data.dtype == out_data_ref.dtype + + def process(self, out_data, out_data_ref, in_data: Optional[np.array] = None, quant: bool = False): + if not self.check(out_data, out_data_ref, quant=quant): + return + self.num_total += 1 + if self.process_(out_data, out_data_ref): + self.num_correct += 1 + + def get_summary(self): + if self.num_total == 0: + return "N/A" + return f"{self.num_correct}/{self.num_total} ({int(self.num_correct/self.num_total*100)}%)" + + +class ClassifyMetric: + def __init__(self, name, **cfg): + self.name = name + self.num_total = 0 + self.num_correct = 0 + + def process_(self, out_data, label_ref, quant: bool = False): + raise NotImplementedError + + def check(self, out_data, label_ref, quant: bool = False): + return True + + def process(self, out_data, label_ref, quant: bool = False): + if not self.check(out_data, label_ref, quant=quant): + return + self.num_total += 1 + if self.process_(out_data, label_ref): + self.num_correct += 1 + + def get_summary(self): + if self.num_total == 0: + return "N/A" + return f"{self.num_correct}/{self.num_total} ({int(self.num_correct/self.num_total*100)}%)" + + +class AllCloseMetric(ValidationMetric): + def __init__(self, name: str, atol: float = 0.0, rtol: float = 0.0): + super().__init__(name) + assert atol >= 0 + self.atol = atol + assert rtol >= 0 + self.rtol = rtol + + def check(self, out_data, out_data_ref, quant: bool = False): + return not quant + + def process_(self, out_data, out_data_ref, in_data: Optional[np.array] = None, quant: bool = False): + return np.allclose(out_data, out_data_ref, rtol=self.rtol, atol=self.atol) + + +class TopKMetric(ValidationMetric): + def __init__(self, name: str, n: int = 2): + super().__init__(name) + assert n >= 1 + self.n = n + + def check(self, out_data, out_data_ref, quant: bool = False): + data_len = len(out_data.flatten().tolist()) + # Probably no classification + return data_len < 25 and not quant + + def process_(self, out_data, out_data_ref, in_data: Optional[np.array] = None, quant: bool = False): + # TODO: only for classification models! + # TODO: support multi_outputs? + data_sorted_idx = list(reversed(np.argsort(out_data).tolist()[0])) + ref_data_sorted_idx = list(reversed(np.argsort(out_data_ref).tolist()[0])) + k = 0 + num_checks = min(self.n, len(data_sorted_idx)) + assert len(data_sorted_idx) == len(ref_data_sorted_idx) + for j in range(num_checks): + idx = data_sorted_idx[j] + ref_idx = ref_data_sorted_idx[j] + if idx == ref_idx: + k += 1 + else: + if out_data.tolist()[0][idx] == out_data_ref.tolist()[0][ref_idx]: + k += 1 + else: + break + if k < num_checks: + return False + elif k == num_checks: + return True + else: + assert False + + +class TopKLabelsMetric(ClassifyMetric): + def __init__(self, name: str, n: int = 2): + super().__init__(name) + assert n >= 1 + self.n = n + + def check(self, out_data, label_ref, quant: bool = False): + data_len = len(out_data.flatten().tolist()) + # Probably no classification + return data_len < 25 + + def process_(self, out_data, label_ref, quant: bool = False): + # print("process_") + # print("out_data", out_data) + # print("label_ref", label_ref) + data_sorted_idx = list(reversed(np.argsort(out_data).tolist()[0])) + # print("data_sorted_idx", data_sorted_idx) + data_sorted_idx_trunc = data_sorted_idx[: self.n] + # print("data_sorted_idx_trunc", data_sorted_idx_trunc) + res = label_ref in data_sorted_idx_trunc + # print("res", res) + # TODO: handle same values? + # input("111") + return res + + +class ConfusionMatrixMetric(ValidationMetric): + def __init__(self, name: str): + super().__init__(name) + self.temp = {} + self.num_correct_per_class = {} + + def check(self, out_data, label_ref, quant: bool = False): + data_len = len(out_data.flatten().tolist()) + # Probably no classification + return data_len < 25 and not quant + + def process_(self, out_data, label_ref, quant: bool = False): + data_sorted_idx = list(reversed(np.argsort(out_data).tolist()[0])) + label = data_sorted_idx[0] + correct = label_ref == label + # TODO: handle same values? + return correct, label + + def process(self, out_data, label_ref, quant: bool = False): + print("ConfusionMatrixMetric.process") + if not self.check(out_data, label_ref, quant=quant): + return + self.num_total += 1 + correct, label = self.process_(out_data, label_ref) + if correct: + self.num_correct += 1 + if label_ref not in self.num_correct_per_class: + self.num_correct_per_class[label_ref] = 0 + self.num_correct_per_class[label_ref] += 1 + temp_ = self.temp.get(label_ref, {}) + if label not in temp_: + temp_[label] = 0 + temp_[label] += 1 + self.temp[label_ref] = temp_ + + def get_summary(self): + if self.num_total == 0: + return "N/A" + return f"{self.temp}" + + +class AccuracyMetric(TopKMetric): + def __init__(self, name: str): + super().__init__(name, n=1) + + +class AccuracyLabelsMetric(TopKLabelsMetric): + def __init__(self, name: str): + super().__init__(name, n=1) + + +class MSEMetric(ValidationMetric): + def __init__(self, name: str, thr: int = 0.5): + super().__init__(name) + assert thr >= 0 + self.thr = thr + + def process_(self, out_data, out_data_ref, in_data: Optional[np.array] = None, quant: bool = False): + mse = ((out_data - out_data_ref) ** 2).mean() + return mse < self.thr + + +class ToyScoreMetric(ValidationMetric): + def __init__(self, name: str, atol: float = 0.1, rtol: float = 0.1): + super().__init__(name) + assert atol >= 0 + self.atol = atol + assert rtol >= 0 + self.rtol = rtol + + def check(self, out_data, out_data_ref, quant: bool = False): + data_len = len(out_data.flatten().tolist()) + return data_len == 640 and not quant + + def process_(self, out_data, out_data_ref, in_data: Optional[np.array] = None, quant: bool = False): + assert in_data is not None + in_data_flat = in_data.flatten().tolist() + out_data_flat = out_data.flatten().tolist() + ref_out_data_flat = out_data_ref.flatten().tolist() + res = 0 + ref_res = 0 + length = len(out_data_flat) + for jjj in range(length): + res = in_data_flat[jjj] - out_data_flat[jjj] + res += res**2 + ref_res = in_data_flat[jjj] - ref_out_data_flat[jjj] + ref_res += ref_res**2 + res /= length + ref_res /= length + print("res", res) + print("ref_res", ref_res) + return np.allclose([res], [ref_res], atol=self.atol, rtol=self.rtol) + + +class PlusMinusOneMetric(ValidationMetric): + def __init__(self, name: str): + super().__init__(name) + + def check(self, out_data, out_data_ref, quant: bool = False): + return "int" in out_data.dtype.str + + def process_(self, out_data, out_data_ref, in_data: Optional[np.array] = None, quant: bool = False): + data_ = out_data.flatten().tolist() + ref_data_ = out_data_ref.flatten().tolist() + + length = len(data_) + for jjj in range(length): + diff = abs(data_[jjj] - ref_data_[jjj]) + print("diff", diff) + if diff > 1: + print("r FALSE") + return False + return True + + +LOOKUP = { + "allclose": AllCloseMetric, + "topk": TopKMetric, + "acc": AccuracyMetric, + "toy": ToyScoreMetric, + "mse": MSEMetric, + "+-1": PlusMinusOneMetric, + "pm1": PlusMinusOneMetric, +} + +LABELS_LOOKUP = { + "topk_label": TopKLabelsMetric, + "acc_label": AccuracyLabelsMetric, + "confusion_matrix": ConfusionMatrixMetric, +} + + +def parse_validate_metric_args(inp): + ret = {} + for x in inp.split(","): + x = x.strip() + assert "=" in x + key, val = x.split("=", 1) + try: + val = ast.literal_eval(val) + except Exception as e: + raise e + ret[key] = val + return ret + + +def parse_validate_metric(inp, lookup=LOOKUP): + if "(" in inp: + metric_name, inp_ = inp.split("(") + assert inp_[-1] == ")" + inp_ = inp_[:-1] + metric_args = parse_validate_metric_args(inp_) + else: + metric_name = inp + metric_args = {} + metric_cls = lookup.get(metric_name, None) + assert metric_cls is not None, f"Validate metric not found: {metric_name}" + metric = metric_cls(inp, **metric_args) + return metric + + +def parse_validate_metrics(inp): + ret = [] + for metric_str in inp.split(";"): + metric = parse_validate_metric(metric_str) + ret.append(metric) + return ret + + +def parse_classify_metrics(inp): + ret = [] + for metric_str in inp.split(";"): + metric = parse_validate_metric(metric_str, lookup=LABELS_LOOKUP) + ret.append(metric) + return ret diff --git a/mlonmcu/session/run.py b/mlonmcu/session/run.py index 680aef223..f7e576040 100644 --- a/mlonmcu/session/run.py +++ b/mlonmcu/session/run.py @@ -74,7 +74,7 @@ def add_any(new, base=None, append=True): class Run: """A run is single model/backend/framework/target combination with a given set of features and configs.""" - FEATURES = {"autotune", "target_optimized"} + FEATURES = {"autotune", "target_optimized", "validate_new"} DEFAULTS = { "export_optional": False, @@ -506,15 +506,22 @@ def add_frontend_by_name(self, frontend_name, context=None): def add_frontends_by_name(self, frontend_names, context=None): """Helper function to initialize and configure frontends by their names.""" frontends = [] + reasons = {} for name in frontend_names: try: assert context is not None and context.environment.has_frontend( name ), f"The frontend '{name}' is not enabled for this environment" frontends.append(self.init_component(SUPPORTED_FRONTENDS[name], context=context)) - except Exception: + except Exception as e: + reasons[name] = str(e) continue assert len(frontends) > 0, "No compatible frontend was found" + if len(frontends) == 0: + if reasons: + logger.error("Initialization of frontends was no successfull. Reasons: %s", reasons) + else: + raise RuntimeError("No compatible frontend was found.") self.add_frontends(frontends) def add_backend_by_name(self, backend_name, context=None): @@ -993,7 +1000,15 @@ def load(self): # The following is very very dirty but required to update arena sizes via model metadata... cfg_new = {} if isinstance(self.model, Model): - self.frontend.process_metadata(self.model, cfg=cfg_new) + artifacts_ = self.frontend.process_metadata(self.model, cfg=cfg_new) + if artifacts_ is not None: + if isinstance(artifacts, dict): + assert "default" in artifacts.keys() + artifacts["default"].extend(artifacts_) + # ignore subs for now + else: + assert isinstance(artifacts, list) + artifacts.extend(artifacts_) if len(cfg_new) > 0: for key, value in cfg_new.items(): component, name = key.split(".")[:2] diff --git a/mlonmcu/setup/gen_requirements.py b/mlonmcu/setup/gen_requirements.py index a9794d2e0..646d8f013 100644 --- a/mlonmcu/setup/gen_requirements.py +++ b/mlonmcu/setup/gen_requirements.py @@ -126,6 +126,7 @@ ["matplotlib", "pyserial", "pyusb"], ), ), + ("microtvm_gvsoc", ("Requirements for microtvm_gvsoc target", ["hydra-core"])), # Provide support for moiopt. ("moiopt", ("Requirements for moiopt", ["ortools"])), # Provide support for onnx. @@ -223,6 +224,7 @@ ("gdbgui", "==0.13.2.0"), ("graphviz", None), ("humanize", None), + ("hydra-core", None), ("idf-component-manager", "~=1.0"), ("itsdangerous", "<2.1"), ("jinja2", ">=3.1.3"), diff --git a/mlonmcu/setup/setup.py b/mlonmcu/setup/setup.py index f768b70a3..88765e9a0 100644 --- a/mlonmcu/setup/setup.py +++ b/mlonmcu/setup/setup.py @@ -248,6 +248,10 @@ def feature_enabled_and_supported(obj, feature): logger.info("add dependencies for etiss") break for config in config_pools: + if "microtvm_gvsoc" in config.name and config.enabled: + for d in requirements["microtvm_gvsoc"][1]: + f.write(f"{d}{os.linesep}") + logger.info("add dependencies for microtvm_gvsoc") if "gvsoc_pulp" in config.name and config.enabled: for d in requirements["gvsoc_pulp"][1]: f.write(f"{d}{os.linesep}") diff --git a/mlonmcu/setup/tasks/__init__.py b/mlonmcu/setup/tasks/__init__.py index 474cdf30c..39aefe521 100644 --- a/mlonmcu/setup/tasks/__init__.py +++ b/mlonmcu/setup/tasks/__init__.py @@ -38,6 +38,7 @@ from .utvmcg import * # noqa: F401, F403 from .zephyr import * # noqa: F401, F403 from .pulp import * # noqa: F401, F403 +from .ekut import * # noqa: F401, F403 from .ara import * # noqa: F401, F403 from .verilator import * # noqa: F401, F403 from .ovpsim import * # noqa: F401, F403 diff --git a/mlonmcu/setup/tasks/ekut.py b/mlonmcu/setup/tasks/ekut.py new file mode 100644 index 000000000..146422f2d --- /dev/null +++ b/mlonmcu/setup/tasks/ekut.py @@ -0,0 +1,55 @@ +# +# Copyright (c) 2022 TUM Department of Electrical and Computer Engineering. +# +# This file is part of MLonMCU. +# See https://github.com/tum-ei-eda/mlonmcu.git for further info. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +"""Definition of tasks used to dynamically install MLonMCU dependencies""" + +import multiprocessing + +from mlonmcu.setup.task import TaskType +from mlonmcu.context.context import MlonMcuContext +from mlonmcu.setup import utils +from mlonmcu.logging import get_logger + +from .common import get_task_factory + +logger = get_logger() +Tasks = get_task_factory() + +############## +# hannah-tvm # +############## + + +def _validate_hannah_tvm(context: MlonMcuContext, params=None): + return context.environment.has_target("microtvm_gvsoc") + + +@Tasks.provides(["hannah_tvm.src_dir", "microtvm_gvsoc.template"]) +@Tasks.validate(_validate_hannah_tvm) +@Tasks.register(category=TaskType.TARGET) +def clone_hannah_tvm( + context: MlonMcuContext, params=None, rebuild=False, verbose=False, threads=multiprocessing.cpu_count() +): + """Clone the hannah-tvm repository.""" + hannahTvmName = utils.makeDirName("hannah_tvm") + hannahTvmSrcDir = context.environment.paths["deps"].path / "src" / hannahTvmName + if rebuild or not utils.is_populated(hannahTvmSrcDir): + pulpRtosRepo = context.environment.repos["hannah_tvm"] + utils.clone(pulpRtosRepo.url, hannahTvmSrcDir, branch=pulpRtosRepo.ref, refresh=rebuild, recursive=True) + context.cache["hannah_tvm.src_dir"] = hannahTvmSrcDir + context.cache["microtvm_gvsoc.template"] = hannahTvmSrcDir / "template" / "gvsoc" diff --git a/mlonmcu/setup/utils.py b/mlonmcu/setup/utils.py index 0936bace5..24e8b1e46 100644 --- a/mlonmcu/setup/utils.py +++ b/mlonmcu/setup/utils.py @@ -195,6 +195,8 @@ def execute( print_func: Callable = print, handle_exit: Optional[Callable] = None, err_func: Callable = logger.error, + encoding: Optional[str] = "utf-8", + stdin_data: Optional[bytes] = None, prefix: str = "", **kwargs, ) -> str: @@ -214,6 +216,10 @@ def execute( Handler for exit code. err_func : Callable Function which should be used to print errors. + encoding: str, optional + Used encoding for the stdout. + stdin_data: bytes, optional + Send this to the stdin of the process. kwargs: dict Arbitrary keyword arguments passed through to the subprocess. @@ -248,15 +254,26 @@ def args_helper(x): stderr=subprocess.STDOUT, ) as process: try: + if stdin_data: + raise RuntimeError("stdin_data only supported if live=False") + # not working... + # process.stdin.write(stdin_data) for line in process.stdout: - new_line = prefix + line.decode(errors="replace") + if encoding: + line = line.decode(encoding, errors="replace") + new_line = prefix + line + else: + new_line = line out_str = out_str + new_line print_func(new_line.replace("\n", "")) exit_code = None while exit_code is None: exit_code = process.poll() if handle_exit is not None: - exit_code = handle_exit(exit_code, out=out_str) + out_str_ = out_str + if encoding is None: + out_str_ = out_str_.decode("utf-8", errors="ignore") + exit_code = handle_exit(exit_code, out=out_str_) assert exit_code == 0, "The process returned an non-zero exit code {}! (CMD: `{}`)".format( exit_code, " ".join(list(map(args_helper, args))) ) @@ -266,13 +283,23 @@ def args_helper(x): os.kill(pid, signal.SIGINT) else: try: - p = subprocess.Popen([i for i in args], **kwargs, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) - out_str = p.communicate()[0].decode(errors="replace") - out_str = prefix + out_str + p = subprocess.Popen( + [i for i in args], **kwargs, stdout=subprocess.PIPE, stdin=subprocess.PIPE, stderr=subprocess.STDOUT + ) + if stdin_data: + out_str = p.communicate(input=stdin_data)[0] + else: + out_str = p.communicate()[0] + if encoding: + out_str = out_str.decode(encoding, errors="replace") + out_str = prefix + out_str exit_code = p.poll() # print_func(out_str) if handle_exit is not None: - exit_code = handle_exit(exit_code, out=out_str) + out_str_ = out_str + if encoding is None: + out_str_ = out_str_.decode("utf-8", errors="ignore") + exit_code = handle_exit(exit_code, out=out_str_) if exit_code != 0: err_func(out_str) assert exit_code == 0, "The process returned an non-zero exit code {}! (CMD: `{}`)".format( diff --git a/mlonmcu/target/riscv/etiss.py b/mlonmcu/target/riscv/etiss.py index 89cf72995..1b7fc9676 100644 --- a/mlonmcu/target/riscv/etiss.py +++ b/mlonmcu/target/riscv/etiss.py @@ -62,9 +62,9 @@ class EtissTarget(RISCVTarget): "plugins": [], "verbose": False, "cpu_arch": None, - "rom_start": 0x0, + "rom_start": 0x1000000, "rom_size": 0x800000, # 8 MB - "ram_start": 0x800000, + "ram_start": 0x1000000 + 0x800000, "ram_size": 0x4000000, # 64 MB "cycle_time_ps": 31250, # 32 MHz "enable_vext": False, @@ -88,8 +88,18 @@ class EtissTarget(RISCVTarget): "extra_bool_config": {}, "extra_string_config": {}, "extra_plugin_config": {}, + "use_run_helper": True, + "exit_on_loop": False, + "log_pc": False, + "log_level": None, + "enable_semihosting": True, + "output_path_prefix": "", + "jit_gcc_cleanup": True, + "jit_verify": False, + "jit_debug": False, + "load_integrated_libraries": True, } - REQUIRED = RISCVTarget.REQUIRED | {"etiss.src_dir", "etiss.install_dir", "etissvp.script"} + REQUIRED = RISCVTarget.REQUIRED | {"etiss.src_dir", "etiss.install_dir", "etissvp.exe", "etissvp.script"} def __init__(self, name="etiss", features=None, config=None): super().__init__(name, features=features, config=config) @@ -108,6 +118,10 @@ def etiss_dir(self): def etiss_script(self): return self.config["etissvp.script"] + @property + def etiss_exe(self): + return self.config["etissvp.exe"] + @property def gdbserver_enable(self): value = self.config["gdbserver_enable"] @@ -132,11 +146,22 @@ def trace_memory(self): value = self.config["trace_memory"] return str2bool(value) if not isinstance(value, (bool, int)) else value + @property + def enable_dmi(self): + return False + # return not self.trace_memory + @property def plugins(self): value = self.config["plugins"] return str2list(value) if isinstance(value, str) else value + def get_plugin_names(self): + ret = self.plugins + if self.gdbserver_enable: + ret.append("gdbserver") + return list(set(ret)) + @property def verbose(self): value = self.config["verbose"] @@ -312,6 +337,57 @@ def allow_error(self): value = self.config["allow_error"] return str2bool(value) if not isinstance(value, (bool, int)) else value + @property + def use_run_helper(self): + value = self.config["use_run_helper"] + return str2bool(value) if not isinstance(value, (bool, int)) else value + + @property + def exit_on_loop(self): + value = self.config["exit_on_loop"] + return str2bool(value) if not isinstance(value, (bool, int)) else value + + @property + def log_pc(self): + value = self.config["log_pc"] + return str2bool(value) if not isinstance(value, (bool, int)) else value + + @property + def log_level(self): + value = self.config["log_level"] + if isinstance(value, str): + value = int(value) + return value + + @property + def enable_semihosting(self): + value = self.config["enable_semihosting"] + return str2bool(value) if not isinstance(value, (bool, int)) else value + + @property + def output_path_prefix(self): + return self.config["output_path_prefix"] + + @property + def jit_gcc_cleanup(self): + value = self.config["jit_gcc_cleanup"] + return str2bool(value) if not isinstance(value, (bool, int)) else value + + @property + def jit_verify(self): + value = self.config["jit_verify"] + return str2bool(value) if not isinstance(value, (bool, int)) else value + + @property + def jit_debug(self): + value = self.config["jit_debug"] + return str2bool(value) if not isinstance(value, (bool, int)) else value + + @property + def load_integrated_libraries(self): + value = self.config["load_integrated_libraries"] + return str2bool(value) if not isinstance(value, (bool, int)) else value + @property def vext_spec(self): return float(self.config["vext_spec"]) @@ -332,23 +408,46 @@ def max_block_size(self): value = int(value) return value - def get_ini_bool_config(self): + def get_ini_bool_config(self, override=None): + override = {k: v for k, v in override.items() if isinstance(v, bool)} + ret = { - "arch.enable_semihosting": True, + "arch.enable_semihosting": self.enable_semihosting, + "simple_mem_system.error_on_invalid_access": not self.allow_error, + "jit.verify": self.jit_verify, + "jit.debug": self.jit_debug, + "etiss.load_integrated_libraries": self.load_integrated_libraries, } - ret.update(self.extra_string_config) + if not self.use_run_helper: + ret["simple_mem_system.print_dbus_access"] = self.trace_memory + ret["simple_mem_system.print_to_file"] = self.trace_memory + ret["etiss.exit_on_loop"] = self.exit_on_loop + ret["etiss.log_pc"] = self.log_pc + ret["etiss.enable_dmi"] = self.enable_dmi + if self.jit == "GCC": + ret["jit.gcc.cleanup"] = self.jit_gcc_cleanup + + ret.update(self.extra_bool_config) + ret.update(override) return ret - def get_ini_string_config(self): + def get_ini_string_config(self, override=None): + override = {k: v for k, v in override.items() if isinstance(v, (str, Path))} ret = { "arch.cpu": self.cpu_arch, + # Mode will be overwritten by elf... + # "simple_mem_system.memseg_mode_00": "RX", + # "simple_mem_system.memseg_mode_01": "RWX", + "etiss.output_path_prefix": self.output_path_prefix, } if self.jit is not None: ret["jit.type"] = f"{self.jit}JIT" ret.update(self.extra_string_config) + ret.update(override) return ret - def get_ini_int_config(self): + def get_ini_int_config(self, override=None): + override = {k: v for k, v in override.items() if isinstance(v, int)} ret = { "simple_mem_system.memseg_origin_00": self.rom_start, "simple_mem_system.memseg_length_00": self.rom_size, @@ -368,87 +467,132 @@ def get_ini_int_config(self): ret["arch.rv32imacfdpv.vlen"] = self.vlen if self.elen > 0: ret["arch.rv32imacfdpv.elen"] = self.elen + log_level = self.log_level + if log_level is None and not self.use_run_helper: + log_level = 5 if self.verbose else 4 + if log_level is not None: + ret["etiss.loglevel"] = log_level + # TODO + # ETISS::CPU_quantum_ps=100000 + # ETISS::write_pc_trace_from_time_us=0 + # ETISS::write_pc_trace_until_time_us=3000000 + # ETISS::sim_mode=0 + # vp::simulation_time_us=20000000 ret.update(self.extra_int_config) + ret.update(override) return ret def get_ini_plugin_config(self): ret = {} if self.gdbserver_enable: - # This could also be accomplished using `--plugin.gdbserver.port` on the cmdline - ret["gdbserver"] = { - "port": self.gdbserver_port, - } + if not self.use_run_helper: + ret["gdbserver"] = { + "port": self.gdbserver_port, + } ret.update(self.extra_plugin_config) # TODO: merge nested dict instead of overriding return ret - def write_ini(self, path): + def write_ini(self, path, override=None): # TODO: Either create artifact for ini or prefer to use cmdline args. with open(path, "w") as f: - ini_bool = self.get_ini_bool_config() + ini_bool = self.get_ini_bool_config(override=override) if len(ini_bool) > 0: f.write("[BoolConfigurations]\n") for key, value in ini_bool.items(): assert isinstance(value, bool) val = "true" if value else "false" f.write(f"{key}={val}\n") - ini_string = self.get_ini_string_config() + ini_string = self.get_ini_string_config(override=override) if len(ini_string) > 0: f.write("[StringConfigurations]\n") for key, value in ini_string.items(): + if isinstance(value, Path): + value = str(value) assert isinstance(value, str) f.write(f"{key}={value}\n") - ini_int = self.get_ini_int_config() + ini_int = self.get_ini_int_config(override=override) if len(ini_int) > 0: f.write("[IntConfigurations]\n") for key, value in ini_int.items(): assert isinstance(value, int) f.write(f"{key}={value}\n") ini_plugin = self.get_ini_plugin_config() - if len(ini_plugin) > 0: - for name, cfg in ini_plugin.items(): - f.write(f"[Plugin {name}]\n") - for key, value in cfg.items(): - if isinstance(value, bool): - val = "true" if value else "false" - else: - val = value - f.write(f"plugin.{name}.{key}={val}\n") + for plugin_name in self.get_plugin_names(): + f.write(f"[Plugin {plugin_name}]\n") + cfg = ini_plugin.pop(plugin_name, {}) + for key, value in cfg.items(): + if isinstance(value, bool): + val = "true" if value else "false" + else: + val = value + f.write(f"plugin.{plugin_name}.{key}={val}\n") + # Check for remaining configs + for plugin_name, cfg in ini_plugin.items(): + if len(cfg) == 0: + continue + logger.warning("Skipping config %s for disabled plugin %s", cfg, plugin_name) def exec(self, program, *args, cwd=os.getcwd(), **kwargs): """Use target to execute a executable with given arguments""" etiss_script_args = [] if len(self.extra_args) > 0: + if not self.use_run_helper: + raise NotImplementedError("etiss.extra_args requires etiss.use_run_helper=1") etiss_script_args.extend(self.extra_args.split(" ")) # TODO: this is outdated # TODO: validate features (attach xor noattach!) if self.debug_etiss: - etiss_script_args.append("gdb") + if self.use_run_helper: + etiss_script_args.append("gdb") + else: + raise NotImplementedError("etiss.debug_etiss requires etiss.use_run_helper=1") if self.gdbserver_enable: - etiss_script_args.append("tgdb") - if not self.gdbserver_attach: - etiss_script_args.append("noattach") + if self.use_run_helper: + etiss_script_args.append("tgdb") + if not self.gdbserver_attach: + etiss_script_args.append("noattach") + etiss_script_args.append("--plugin.gdbserver.port={self.gdbserver_port}") + if self.gdbserver_attach: + raise NotImplementedError("etiss.gdbserver_attach requires etiss.use_run_helper=1") if self.trace_memory: - etiss_script_args.append("trace") - etiss_script_args.append("nodmi") + if self.use_run_helper: + etiss_script_args.append("trace") + etiss_script_args.append("nodmi") + if self.exit_on_loop: + if self.use_run_helper: + etiss_script_args.append("noloop") + if self.log_pc: + if self.use_run_helper: + etiss_script_args.append("logpc") + if not self.enable_dmi: + if self.use_run_helper: + etiss_script_args.append("nodmi") if self.verbose: etiss_script_args.append("v") # Alternative to stdout parsing: etiss_script_args.append("--vp.stats_file_path=stats.json") + if self.use_run_helper: + for plugin in self.get_plugin_names(): + etiss_script_args.extend(["-p", plugin]) # TODO: working directory? + ini_override = {} + if self.use_run_helper: + etiss_script_args.insert(0, program) + else: + ini_override["vp.elf_file"] = program etiss_ini = os.path.join(cwd, "custom.ini") - self.write_ini(etiss_ini) + self.write_ini(etiss_ini, override=ini_override) etiss_script_args.append("-i" + etiss_ini) - for plugin in self.plugins: - etiss_script_args.extend(["-p", plugin]) # if self.timeout_sec > 0: + script = self.etiss_script if self.use_run_helper else self.etiss_exe + script = Path(script).resolve() if False: ret = exec_timeout( self.timeout_sec, execute, - Path(self.etiss_script).resolve(), - program, + script, *etiss_script_args, *args, cwd=cwd, @@ -456,8 +600,7 @@ def exec(self, program, *args, cwd=os.getcwd(), **kwargs): ) else: ret = execute( - Path(self.etiss_script).resolve(), - program, + script, *etiss_script_args, *args, cwd=cwd, @@ -607,6 +750,8 @@ def get_ram_sizes(data): return metrics, out, artifacts def get_target_system(self): + if not self.enable_semihosting: + raise NotImplementedError("etiss.enable_semihosting=0 is not supported anymore") return self.name def get_platform_defs(self, platform): diff --git a/mlonmcu/target/riscv/spike.py b/mlonmcu/target/riscv/spike.py index 522c68586..27e16a5cd 100644 --- a/mlonmcu/target/riscv/spike.py +++ b/mlonmcu/target/riscv/spike.py @@ -153,7 +153,7 @@ def exec(self, program, *args, cwd=os.getcwd(), **kwargs): cwd=cwd, **kwargs, ) - return ret + return ret, [] def parse_stdout(self, out, metrics, exit_code=0): add_bench_metrics(out, metrics, exit_code != 0, target_name=self.name) @@ -177,9 +177,9 @@ def _handle_exit(code, out=None): start_time = time.time() if self.print_outputs: - out = self.exec(elf, *args, cwd=directory, live=True, handle_exit=_handle_exit) + out, artifacts = self.exec(elf, *args, cwd=directory, live=True, handle_exit=_handle_exit) else: - out = self.exec( + out, artifacts = self.exec( elf, *args, cwd=directory, live=False, print_func=lambda *args, **kwargs: None, handle_exit=_handle_exit ) # TODO: do something with out? @@ -197,7 +197,7 @@ def _handle_exit(code, out=None): if diff > 0: metrics.add("MIPS", (sim_insns / diff) / 1e6, True) - return metrics, out, [] + return metrics, out, artifacts def get_platform_defs(self, platform): ret = {} diff --git a/mlonmcu/target/target.py b/mlonmcu/target/target.py index 6a23bb61a..262babbfc 100644 --- a/mlonmcu/target/target.py +++ b/mlonmcu/target/target.py @@ -172,7 +172,7 @@ def generate(self, elf) -> Tuple[dict, dict]: metrics = [] total = 1 + (self.repeat if self.repeat else 0) # We only save the stdout and artifacts of the last execution - # Callect metrics from all runs to aggregate them in a callback with high priority + # Collect metrics from all runs to aggregate them in a callback with high priority artifacts_ = [] # if self.dir is None: # self.dir = Path( @@ -284,3 +284,23 @@ def get_hardware_details(self): "max-vthread-extent": 0, "warp-size": 0, } + + @property + def supports_filesystem(self): + return False + + @property + def supports_stdout(self): + return True + + @property + def supports_stdin(self): + return False + + @property + def supports_argv(self): + return False + + @property + def supports_uart(self): + return False diff --git a/resources/templates/ara.yml.j2 b/resources/templates/ara.yml.j2 index 2abf39dcf..11e75695a 100644 --- a/resources/templates/ara.yml.j2 +++ b/resources/templates/ara.yml.j2 @@ -70,8 +70,8 @@ repos: url: "https://github.com/tacle/tacle-bench.git" ref: master polybench: - url: "https://github.com/MatthiasJReisinger/PolyBenchC-4.2.1.git" - ref: master + url: "https://github.com/PhilippvK/PolyBenchC-4.2.1.git" + ref: fixes mibench: url: "https://github.com/embecosm/mibench.git" ref: master diff --git a/resources/templates/corev.yml.j2 b/resources/templates/corev.yml.j2 index c7cf38dad..9e3bd6a81 100644 --- a/resources/templates/corev.yml.j2 +++ b/resources/templates/corev.yml.j2 @@ -69,7 +69,7 @@ repos: ref: ffeca904368926d60caeb2d97858215626892f35 mlif: url: "https://github.com/tum-ei-eda/mlonmcu-sw.git" - ref: 7ba4ea6992093843720ae3494223cd910a64c828 + ref: 4f89b17aa257afeccbebbb883b775cd9af58b7a0 microtvm_etiss: url: "https://github.com/PhilippvK/microtvm-etiss-template.git" ref: 4460f539f6607b0c8b90321e7cb80e28d1e1fbe2 diff --git a/resources/templates/dev.yml.j2 b/resources/templates/dev.yml.j2 index cb54a2668..6561a66cc 100644 --- a/resources/templates/dev.yml.j2 +++ b/resources/templates/dev.yml.j2 @@ -31,7 +31,7 @@ paths: repos: tensorflow: # TODO: rename to tflite-micro? url: "https://github.com/tensorflow/tflite-micro.git" - ref: ca5358f4680dbd94717a4c6bd77186bc1c799e1b + ref: 19aaea85e4679a9a2f265e07ba190ac5ea4d3766 options: single_branch: true # tflite_micro_compiler: @@ -81,7 +81,7 @@ repos: ref: fc42c71353d15c564558249bd4f13350119ab6a9 mlif: url: "https://github.com/tum-ei-eda/mlonmcu-sw.git" - ref: 4f89b17aa257afeccbebbb883b775cd9af58b7a0 + ref: f74feb4b27b44f8caa9b89c7df1545579563f5be # espidf: # url: "https://github.com/espressif/esp-idf.git" # ref: release/v4.4 @@ -115,8 +115,8 @@ repos: url: "https://github.com/tacle/tacle-bench.git" ref: master polybench: - url: "https://github.com/MatthiasJReisinger/PolyBenchC-4.2.1.git" - ref: master + url: "https://github.com/PhilippvK/PolyBenchC-4.2.1.git" + ref: fixes mibench: url: "https://github.com/embecosm/mibench.git" ref: master diff --git a/resources/templates/vicuna.yml.j2 b/resources/templates/vicuna.yml.j2 index 3888f2467..15bbd61e5 100644 --- a/resources/templates/vicuna.yml.j2 +++ b/resources/templates/vicuna.yml.j2 @@ -88,8 +88,8 @@ repos: url: "https://github.com/tacle/tacle-bench.git" ref: master polybench: - url: "https://github.com/MatthiasJReisinger/PolyBenchC-4.2.1.git" - ref: master + url: "https://github.com/PhilippvK/PolyBenchC-4.2.1.git" + ref: fixes mibench: url: "https://github.com/embecosm/mibench.git" ref: master