From 6eb812d343b34b446d792412d62a492f9a52b294 Mon Sep 17 00:00:00 2001 From: Mengyi Wang Date: Tue, 9 Apr 2024 19:07:13 +0200 Subject: [PATCH] feat(plugins): add Matter plugin (#4491) This PR adds a new Matter plugin which is useful for building [Matter SDK](https://github.com/project-chip/connectedhomeip) based parts. Co-authored-by: Sheng Yu Co-authored-by: Sergio Schvezov Co-authored-by: Callahan Kovacs --- snapcraft/parts/lifecycle.py | 2 +- snapcraft/parts/plugins/__init__.py | 2 + snapcraft/parts/plugins/matter_sdk_plugin.py | 178 ++++++++++++++++++ snapcraft/parts/plugins/register.py | 2 + .../craft-parts/matter-sdk/snapcraft.yaml | 48 +++++ .../plugins/craft-parts/matter-sdk/task.yaml | 50 +++++ .../parts/plugins/test_matter_sdk_plugin.py | 116 ++++++++++++ tests/unit/parts/test_lifecycle.py | 25 ++- 8 files changed, 420 insertions(+), 3 deletions(-) create mode 100644 snapcraft/parts/plugins/matter_sdk_plugin.py create mode 100644 tests/spread/plugins/craft-parts/matter-sdk/snapcraft.yaml create mode 100644 tests/spread/plugins/craft-parts/matter-sdk/task.yaml create mode 100644 tests/unit/parts/plugins/test_matter_sdk_plugin.py diff --git a/snapcraft/parts/lifecycle.py b/snapcraft/parts/lifecycle.py index 3ccf305d2f..121443069a 100644 --- a/snapcraft/parts/lifecycle.py +++ b/snapcraft/parts/lifecycle.py @@ -50,7 +50,7 @@ import argparse -_EXPERIMENTAL_PLUGINS = ["kernel"] +_EXPERIMENTAL_PLUGINS = ["kernel", "matter-sdk"] def run(command_name: str, parsed_args: "argparse.Namespace") -> None: diff --git a/snapcraft/parts/plugins/__init__.py b/snapcraft/parts/plugins/__init__.py index 4d9336fbc6..6104dfed0c 100644 --- a/snapcraft/parts/plugins/__init__.py +++ b/snapcraft/parts/plugins/__init__.py @@ -21,6 +21,7 @@ from .conda_plugin import CondaPlugin from .flutter_plugin import FlutterPlugin from .kernel_plugin import KernelPlugin +from .matter_sdk_plugin import MatterSdkPlugin from .python_plugin import PythonPlugin from .register import get_plugins, register @@ -28,6 +29,7 @@ "ColconPlugin", "CondaPlugin", "FlutterPlugin", + "MatterSdkPlugin", "KernelPlugin", "PythonPlugin", "get_plugins", diff --git a/snapcraft/parts/plugins/matter_sdk_plugin.py b/snapcraft/parts/plugins/matter_sdk_plugin.py new file mode 100644 index 0000000000..4c50f12996 --- /dev/null +++ b/snapcraft/parts/plugins/matter_sdk_plugin.py @@ -0,0 +1,178 @@ +# -*- Mode:Python; indent-tabs-mode:nil; tab-width:4 -*- +# +# Copyright 2023 Canonical Ltd. +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3 as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +"""The matter SDK plugin.""" +import os +from typing import Any, Dict, List, Set, cast + +from craft_parts import infos, plugins +from overrides import overrides + +# The repository where the matter SDK resides. +MATTER_SDK_REPO = "https://github.com/project-chip/connectedhomeip" + + +class MatterSdkPluginProperties(plugins.PluginProperties, plugins.PluginModel): + """The part properties used by the matter SDK plugin.""" + + matter_sdk_version: str + + @classmethod + @overrides + def unmarshal(cls, data: Dict[str, Any]) -> "MatterSdkPluginProperties": + """Populate class attributes from the part specification. + + :param data: A dictionary containing part properties. + + :return: The populated plugin properties data object. + + :raise pydantic.ValidationError: If validation fails. + """ + plugin_data = plugins.extract_plugin_properties( + data, + plugin_name="matter-sdk", + required=["matter_sdk_version"], + ) + return cls(**plugin_data) + + +class MatterSdkPlugin(plugins.Plugin): + """A plugin for matter SDK project. + + This plugin uses the common plugin keywords. + For more information check the 'plugins' topic. + + Additionally, this plugin uses the following plugin-specific keywords: + - matter-sdk-version + (str, no default) + The matter SDK version to use for the build. + """ + + properties_class = MatterSdkPluginProperties + + def __init__( + self, + *, + properties: plugins.PluginProperties, + part_info: infos.PartInfo, + ) -> None: + super().__init__(properties=properties, part_info=part_info) + + self.matter_sdk_dir = part_info.part_build_dir + self.snap_arch = os.getenv("SNAP_ARCH") + + @overrides + def get_pull_commands(self) -> List[str]: + options = cast(MatterSdkPluginProperties, self._options) + commands = [] + + # Clone Matter SDK repository + commands.extend( + [ + " git init", + f" git remote add origin {MATTER_SDK_REPO}", + f" git fetch --depth 1 origin {options.matter_sdk_version}", + " git checkout FETCH_HEAD", + ] + ) + + # Checkout submodules for Linux platform + commands.extend(["scripts/checkout_submodules.py --shallow --platform linux"]) + + return commands + + @overrides + def get_build_packages(self) -> Set[str]: + return { + "clang", + "cmake", + "generate-ninja", + "git", + "libavahi-client-dev", + "libcairo2-dev", + "libdbus-1-dev", + "libgirepository1.0-dev", + "libglib2.0-dev", + "libreadline-dev", + "libssl-dev", + "ninja-build", + "pkg-config", + "python3-dev", + "python3-pip", + "python3-venv", + "unzip", + "wget", + } + + @overrides + def get_build_environment(self) -> Dict[str, str]: + return {} + + @overrides + def get_build_snaps(self) -> Set[str]: + return set() + + @overrides + def get_build_commands(self) -> List[str]: + commands = [] + + # The project writes its data to /tmp which isn't persisted. + + # Setting TMPDIR env var when running the app isn't sufficient as + # chip_[config,counter,factory,kvs].ini still get written under /tmp. + # The chip-tool currently has no way of overriding the default paths to + # storage and security config files. + + # Snap does not allow bind mounting a persistent directory on /tmp, + # so we need to replace it in the source with another path, e.g. /mnt. + # The consumer snap needs to bind mount a persisted directory within + # the confined snap space on /mnt. + + # Replace storage paths + commands.extend( + [ + r"sed -i 's/\/tmp/\/mnt/g' src/platform/Linux/CHIPLinuxStorage.h", + r"sed -i 's/\/tmp/\/mnt/g' src/platform/Linux/CHIPPlatformConfig.h", + ] + ) + + # Store the initial value of PATH before executing the bootstrap script + commands.extend(["OLD_PATH=$PATH"]) + + # Bootstrapping script for building Matter SDK with minimal "build" requirements + # and setting up the environment. + commands.extend( + ["set +u && source scripts/setup/bootstrap.sh --platform build && set -u"] + ) + + commands.extend(["echo 'Built Matter SDK'"]) + + # Compare the difference between the original PATH and the modified PATH + commands.extend( + [ + 'MATTER_SDK_PATHS="${PATH%$OLD_PATH}"', + ] + ) + + # Prepend the Matter SDK related PATH to the beginning of the PATH environment variable, + # and save it to the staging area as matter-sdk-env.sh file. + commands.extend( + [ + 'echo "export PATH=$MATTER_SDK_PATHS\\$PATH" >> $CRAFT_STAGE/matter-sdk-env.sh', + ] + ) + + return commands diff --git a/snapcraft/parts/plugins/register.py b/snapcraft/parts/plugins/register.py index 6105834649..5d578462a0 100644 --- a/snapcraft/parts/plugins/register.py +++ b/snapcraft/parts/plugins/register.py @@ -23,6 +23,7 @@ from .conda_plugin import CondaPlugin from .flutter_plugin import FlutterPlugin from .kernel_plugin import KernelPlugin +from .matter_sdk_plugin import MatterSdkPlugin from .python_plugin import PythonPlugin @@ -36,6 +37,7 @@ def get_plugins(core22: bool) -> dict[str, PluginType]: "conda": CondaPlugin, "flutter": FlutterPlugin, "python": PythonPlugin, + "matter-sdk": MatterSdkPlugin, } if core22: diff --git a/tests/spread/plugins/craft-parts/matter-sdk/snapcraft.yaml b/tests/spread/plugins/craft-parts/matter-sdk/snapcraft.yaml new file mode 100644 index 0000000000..73766084c8 --- /dev/null +++ b/tests/spread/plugins/craft-parts/matter-sdk/snapcraft.yaml @@ -0,0 +1,48 @@ +name: matter-lighting +summary: Matter plugin test +description: An lighting application to test the matter plugin. +version: "1.0" + +base: core22 + +grade: stable +build-base: core22 +confinement: strict + +layout: + /mnt: + bind: $SNAP_COMMON/mnt + +apps: + matter-lighting: + daemon: simple + command: bin/lighting-app + install-mode: disable + plugs: + - network + - network-bind + - bluez + - avahi-control + +parts: + matter-sdk: + plugin: matter-sdk + matter-sdk-version: "1536ca20c5917578ca40ce509400e97b52751788" # use this commit with ptpython version fix; needs to be updated once matter sdk have a stable release + + lighting: + plugin: nil + after: [matter-sdk] + override-build: | + # Source the Matter SDK environment variables + source $CRAFT_STAGE/matter-sdk-env.sh + + # Build the lighting app for snapcraft spread testing purposes + cd ../../matter-sdk/build/examples/lighting-app/linux + gn gen out/build + ninja -C out/build + + ldd out/build/chip-lighting-app + + mkdir -p $CRAFT_PART_INSTALL/bin + cp out/build/chip-lighting-app $CRAFT_PART_INSTALL/bin/lighting-app + diff --git a/tests/spread/plugins/craft-parts/matter-sdk/task.yaml b/tests/spread/plugins/craft-parts/matter-sdk/task.yaml new file mode 100644 index 0000000000..f293c114cd --- /dev/null +++ b/tests/spread/plugins/craft-parts/matter-sdk/task.yaml @@ -0,0 +1,50 @@ +summary: Craft Parts matter SDK plugin test +manual: true +kill-timeout: 180m + +systems: + - ubuntu-22.04-64 + +prepare: | + #shellcheck source=tests/spread/tools/snapcraft-yaml.sh + . "$TOOLS_DIR/snapcraft-yaml.sh" + set_base "$SNAP/snap/snapcraft.yaml" + +restore: | + cd "$SNAP" + snapcraft clean + rm -f ./*.snap + + #shellcheck source=tests/spread/tools/snapcraft-yaml.sh + . "$TOOLS_DIR/snapcraft-yaml.sh" + restore_yaml "snap/snapcraft.yaml" + +execute: | + cd "$SNAP" + + # Build and install the snap + snapcraft + snap install "${SNAP}*.snap" --dangerous + + start_time=$(date +"%Y-%m-%d %H:%M:%S") + snap start matter-lighting + + # Check if storage path replacement from /tmp to SNAP_COMMON/mnt works + for file in /tmp/chip_*; do + if [ -e "$file" ]; then + echo "Error: $file should not exist." + exit 1 + fi + done + + if [ ! -e "${SNAP_COMMON/mnt/chip_*}" ]; then + echo "Error: ${SNAP_COMMON}/mnt/chip_* does not exist." + exit 1 + fi + + # Check if server initialization is complete for matter-lighting + if ! journalctl --since "$start_time" | grep matter-lighting | grep "CHIP:SVR: Server initialization complete"; then + echo "Error: matter-lighting initialization failed." + exit 1 + fi + \ No newline at end of file diff --git a/tests/unit/parts/plugins/test_matter_sdk_plugin.py b/tests/unit/parts/plugins/test_matter_sdk_plugin.py new file mode 100644 index 0000000000..0431fa1fd0 --- /dev/null +++ b/tests/unit/parts/plugins/test_matter_sdk_plugin.py @@ -0,0 +1,116 @@ +# -*- Mode:Python; indent-tabs-mode:nil; tab-width:4 -*- +# +# Copyright 2023 Canonical Ltd. +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License version 3 as published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this program. If not, see . + + +import pytest +from craft_parts import Part, PartInfo, ProjectInfo + +from snapcraft.parts.plugins import MatterSdkPlugin + +# The repository where the matter SDK resides. +MATTER_SDK_REPO = "https://github.com/project-chip/connectedhomeip" + + +@pytest.fixture(autouse=True) +def part_info(new_dir): + yield PartInfo( + project_info=ProjectInfo( + application_name="test", project_name="test-snap", cache_dir=new_dir + ), + part=Part("my-part", {}), + ) + + +def test_get_pull_commands(part_info): + properties = MatterSdkPlugin.properties_class.unmarshal( + {"matter-sdk-version": "master"} + ) + plugin = MatterSdkPlugin(properties=properties, part_info=part_info) + + sdk_version = properties.matter_sdk_version # type: ignore + + expected_commands = [ + " git init", + f" git remote add origin {MATTER_SDK_REPO}", + f" git fetch --depth 1 origin {sdk_version}", + " git checkout FETCH_HEAD", + "scripts/checkout_submodules.py --shallow --platform linux", + ] + + assert plugin.get_pull_commands() == expected_commands + + +def test_get_build_snaps(part_info): + properties = MatterSdkPlugin.properties_class.unmarshal( + {"matter-sdk-version": "master"} + ) + plugin = MatterSdkPlugin(properties=properties, part_info=part_info) + assert plugin.get_build_snaps() == set() + + +def test_get_build_packages(part_info): + properties = MatterSdkPlugin.properties_class.unmarshal( + {"matter-sdk-version": "master"} + ) + plugin = MatterSdkPlugin(properties=properties, part_info=part_info) + assert plugin.get_build_packages() == { + "clang", + "cmake", + "generate-ninja", + "git", + "libavahi-client-dev", + "libcairo2-dev", + "libdbus-1-dev", + "libgirepository1.0-dev", + "libglib2.0-dev", + "libreadline-dev", + "libssl-dev", + "ninja-build", + "pkg-config", + "python3-dev", + "python3-pip", + "python3-venv", + "unzip", + "wget", + } + + +def test_get_build_environment(part_info): + properties = MatterSdkPlugin.properties_class.unmarshal( + {"matter-sdk-version": "master"} + ) + plugin = MatterSdkPlugin(properties=properties, part_info=part_info) + + assert plugin.get_build_environment() == {} + + +def test_get_build_commands(part_info): + properties = MatterSdkPlugin.properties_class.unmarshal( + {"matter-sdk-version": "master"} + ) + plugin = MatterSdkPlugin(properties=properties, part_info=part_info) + + expected_commands = [ + r"sed -i 's/\/tmp/\/mnt/g' src/platform/Linux/CHIPLinuxStorage.h", + r"sed -i 's/\/tmp/\/mnt/g' src/platform/Linux/CHIPPlatformConfig.h", + "OLD_PATH=$PATH", + "set +u && source scripts/setup/bootstrap.sh --platform build && set -u", + "echo 'Built Matter SDK'", + 'MATTER_SDK_PATHS="${PATH%$OLD_PATH}"', + 'echo "export PATH=$MATTER_SDK_PATHS\\$PATH" >> $CRAFT_STAGE/matter-sdk-env.sh', + ] + + assert plugin.get_build_commands() == expected_commands diff --git a/tests/unit/parts/test_lifecycle.py b/tests/unit/parts/test_lifecycle.py index 54ed44719a..0114f89869 100644 --- a/tests/unit/parts/test_lifecycle.py +++ b/tests/unit/parts/test_lifecycle.py @@ -30,7 +30,7 @@ from snapcraft.elf import ElfFile from snapcraft.models import MANDATORY_ADOPTABLE_FIELDS, Project from snapcraft.parts import lifecycle as parts_lifecycle -from snapcraft.parts.plugins import KernelPlugin +from snapcraft.parts.plugins import KernelPlugin, MatterSdkPlugin from snapcraft.parts.update_metadata import update_project_metadata from snapcraft.utils import get_host_architecture @@ -1155,16 +1155,37 @@ def test_lifecycle_adopt_project_vars(snapcraft_yaml, new_dir): def test_check_experimental_plugins_disabled(snapcraft_yaml, mocker): - mocker.patch("craft_parts.plugins.plugins._PLUGINS", {"kernel": KernelPlugin}) + mocker.patch( + "craft_parts.plugins.plugins._PLUGINS", + {"kernel": KernelPlugin, "matter-sdk": MatterSdkPlugin}, + ) project = Project.unmarshal( snapcraft_yaml(base="core22", parts={"foo": {"plugin": "kernel"}}) ) + with pytest.raises(errors.SnapcraftError) as raised: parts_lifecycle._check_experimental_plugins(project, False) assert str(raised.value) == ( "Plugin 'kernel' in part 'foo' is unstable and may change in the future." ) + project = Project.unmarshal( + snapcraft_yaml( + base="core22", + parts={ + "foo": { + "plugin": "matter-sdk", + "matter-sdk-version": "1536ca20c5917578ca40ce509400e97b52751788", + } + }, + ) + ) + with pytest.raises(errors.SnapcraftError) as raised: + parts_lifecycle._check_experimental_plugins(project, False) + assert str(raised.value) == ( + "Plugin 'matter-sdk' in part 'foo' is unstable and may change in the future." + ) + def test_check_experimental_plugins_enabled(snapcraft_yaml, mocker): mocker.patch("craft_parts.plugins.plugins._PLUGINS", {"kernel": KernelPlugin})