Add rpc builds to python build scripts (#9936)

Add --rpc option which builds applications with the rpc debug
interface enabled.

examples/all-clusters-app/esp32/sdkconfig_m5stack_rpc.defaults

@@ -0,0 +1,56 @@
+#
+#    Copyright (c) 2020 Project CHIP Authors
+#    Copyright (c) 2018 Nest Labs, Inc.
+#    All rights reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+#    Description:
+#      CI uses this to select the ESP32 M5Stack.
+#
+CONFIG_IDF_TARGET="esp32"
+CONFIG_IDF_TARGET_ESP32=y
+CONFIG_DEVICE_TYPE_M5STACK=y
+
+# Default to 921600 baud when flashing and monitoring device
+CONFIG_ESPTOOLPY_BAUD_921600B=y
+CONFIG_ESPTOOLPY_BAUD=921600
+CONFIG_ESPTOOLPY_COMPRESSED=y
+CONFIG_ESPTOOLPY_MONITOR_BAUD_115200B=y
+CONFIG_ESPTOOLPY_MONITOR_BAUD=115200
+
+#enable BT
+CONFIG_BT_ENABLED=y
+CONFIG_BT_NIMBLE_ENABLED=y
+
+#enable lwip ipv6 autoconfig
+CONFIG_LWIP_IPV6_AUTOCONFIG=y
+
+# Use a custom partition table
+CONFIG_PARTITION_TABLE_CUSTOM=y
+CONFIG_PARTITION_TABLE_FILENAME="partitions.csv"
+
+# Vendor and product id
+CONFIG_DEVICE_VENDOR_ID=0x235A
+CONFIG_DEVICE_PRODUCT_ID=0x4541
+
+# Main task needs a bit more stack than the default
+# default is 3584, bump this up to 5k.
+CONFIG_ESP_MAIN_TASK_STACK_SIZE=5120
+
+# PW RPC Debug channel
+CONFIG_EXAMPLE_UART_PORT_NUM=0
+CONFIG_EXAMPLE_UART_BAUD_RATE=115200
+CONFIG_EXAMPLE_UART_RXD=3
+CONFIG_EXAMPLE_UART_TXD=1
+CONFIG_ENABLE_PW_RPC=y

scripts/build/build/__init__.py

@@ -40,6 +40,7 @@ def __init__(self, runner, repository_path: str, output_prefix: str):
     def SetupBuilders(self, platforms: Sequence[Platform],
                       boards: Sequence[Board],
                       applications: Sequence[Application],
+                      enable_rpcs: bool,
                       enable_flashbundle: bool):
         """Configures internal builders for the given platform/board/app combination.
 
@@ -67,7 +68,7 @@ def SetupBuilders(self, platforms: Sequence[Platform],
 
         if not applications:
             applications = set().union(*[
-                TargetRelations.ApplicationsForPlatform(platform)
+                TargetRelations.ApplicationsForPlatform(platform, enable_rpcs)
                 for platform in platforms
             ])
 
@@ -89,7 +90,8 @@ def SetupBuilders(self, platforms: Sequence[Platform],
             for board in sorted(boards):
                 for application in sorted(applications):
                     builder = self.builder_factory.Create(
-                        platform, board, application, enable_flashbundle=enable_flashbundle)
+                        platform, board, application, enable_flashbundle=enable_flashbundle,
+                        enable_rpcs=enable_rpcs)
                     if not builder:
                         logging.debug('Builder not supported for tuple %s/%s/%s', platform,
                                       board, application)

scripts/build/build/factory.py

@@ -29,14 +29,15 @@
 
 class MatchApplication:
 
-    def __init__(self, app, board=None):
+    def __init__(self, app, rpcs_supported, board=None):
         self.app = app
         self.board = board
+        self.rpcs_supported = rpcs_supported
 
-    def Match(self, board: Board, app: Application):
+    def Match(self, board: Board, app: Application, enable_rpcs: bool):
         if app != self.app:
             return False
-        return self.board is None or board == self.board
+        return (self.board is None or board == self.board) and (self.rpcs_supported or not enable_rpcs)
 
 
 class Matcher():
@@ -47,34 +48,36 @@ def __init__(self, builder_class):
         self.app_arguments = {}
         self.board_arguments = {}
 
-    def AcceptApplication(self, __app_key: Application, **kargs):
-        self.app_arguments[MatchApplication(__app_key)] = kargs
+    def AcceptApplication(self, __app_key: Application, rpcs_supported=False, **kargs):
+        self.app_arguments[MatchApplication(__app_key, rpcs_supported)] = kargs
 
-    def AcceptApplicationForBoard(self, __app_key: Application, __board: Board,
+    def AcceptApplicationForBoard(self, __app_key: Application, __board: Board, rpcs_supported=False,
                                   **kargs):
-        self.app_arguments[MatchApplication(__app_key, __board)] = kargs
+        self.app_arguments[MatchApplication(
+            __app_key, rpcs_supported, __board)] = kargs
 
     def AcceptBoard(self, __board_key: Board, **kargs):
         self.board_arguments[__board_key] = kargs
 
     def Create(self, runner, __board_key: Board, __app_key: Application,
-               repo_path: str, **kargs):
+               repo_path: str, enable_rpcs: bool, **kargs):
         """Creates a new builder for the given board/app. """
         if not __board_key in self.board_arguments:
             return None
 
         extra_app_args = None
         for key, value in self.app_arguments.items():
-            if key.Match(__board_key, __app_key):
+            if key.Match(__board_key, __app_key, enable_rpcs):
                 extra_app_args = value
                 break
 
         if extra_app_args is None:
             return None
+        if enable_rpcs:
+            kargs['enable_rpcs'] = True
 
         kargs.update(self.board_arguments[__board_key])
         kargs.update(extra_app_args)
-
         return self.builder_class(repo_path, runner=runner, **kargs)
 
 
@@ -108,8 +111,12 @@ def Create(self, runner, __board_key: Board, __app_key: Application,
 _MATCHERS[Platform.ESP32].AcceptBoard(Board.M5STACK, board=Esp32Board.M5Stack)
 _MATCHERS[Platform.ESP32].AcceptBoard(
     Board.C3DEVKIT, board=Esp32Board.C3DevKit)
-_MATCHERS[Platform.ESP32].AcceptApplication(
-    Application.ALL_CLUSTERS, app=Esp32App.ALL_CLUSTERS)
+_MATCHERS[Platform.ESP32].AcceptApplicationForBoard(
+    Application.ALL_CLUSTERS, Board.M5STACK, app=Esp32App.ALL_CLUSTERS, rpcs_supported=True)
+_MATCHERS[Platform.ESP32].AcceptApplicationForBoard(
+    Application.ALL_CLUSTERS, Board.DEVKITC, app=Esp32App.ALL_CLUSTERS)
+_MATCHERS[Platform.ESP32].AcceptApplicationForBoard(
+    Application.ALL_CLUSTERS, Board.C3DEVKIT, app=Esp32App.ALL_CLUSTERS)
 _MATCHERS[Platform.ESP32].AcceptApplicationForBoard(
     Application.SHELL, Board.DEVKITC, app=Esp32App.SHELL)
 _MATCHERS[Platform.ESP32].AcceptApplicationForBoard(
@@ -125,7 +132,7 @@ def Create(self, runner, __board_key: Board, __app_key: Application,
 _MATCHERS[Platform.EFR32].AcceptBoard(
     Board.BRD4161A, board=Efr32Board.BRD4161A)
 _MATCHERS[Platform.EFR32].AcceptApplication(
-    Application.LIGHT, app=Efr32App.LIGHT)
+    Application.LIGHT, app=Efr32App.LIGHT, rpcs_supported=True)
 _MATCHERS[Platform.EFR32].AcceptApplication(
     Application.LOCK, app=Efr32App.LOCK)
 _MATCHERS[Platform.EFR32].AcceptApplication(
@@ -134,7 +141,8 @@ def Create(self, runner, __board_key: Board, __app_key: Application,
 _MATCHERS[Platform.NRF].AcceptBoard(Board.NRF5340, board=NrfBoard.NRF5340)
 _MATCHERS[Platform.NRF].AcceptBoard(Board.NRF52840, board=NrfBoard.NRF52840)
 _MATCHERS[Platform.NRF].AcceptApplication(Application.LOCK, app=NrfApp.LOCK)
-_MATCHERS[Platform.NRF].AcceptApplication(Application.LIGHT, app=NrfApp.LIGHT)
+_MATCHERS[Platform.NRF].AcceptApplication(
+    Application.LIGHT, app=NrfApp.LIGHT, rpcs_supported=True)
 _MATCHERS[Platform.NRF].AcceptApplication(Application.SHELL, app=NrfApp.SHELL)
 _MATCHERS[Platform.NRF].AcceptApplication(Application.PUMP, app=NrfApp.PUMP)
 _MATCHERS[Platform.NRF].AcceptApplication(
@@ -169,19 +177,21 @@ def __init__(self, runner, repository_path: str, output_prefix: str):
         self.repository_path = repository_path
         self.output_prefix = output_prefix
 
-    def Create(self, platform: Platform, board: Board, app: Application, enable_flashbundle: bool = False):
+    def Create(self, platform: Platform, board: Board, app: Application, enable_rpcs: bool, enable_flashbundle: bool = False):
         """Creates a builder object for the specified arguments. """
 
         builder = _MATCHERS[platform].Create(
             self.runner,
             board,
             app,
             self.repository_path,
+            enable_rpcs,
             output_prefix=self.output_prefix)
 
         if builder:
             builder.SetIdentifier(platform.name.lower(),
-                                  board.name.lower(), app.name.lower())
+                                  board.name.lower(), app.name.lower(),
+                                  enable_rpcs=enable_rpcs)
             builder.enable_flashbundle(enable_flashbundle)
 
         return builder
@@ -206,11 +216,14 @@ def PlatformsForBoard(board: Board) -> Set[Platform]:
         return platforms
 
     @staticmethod
-    def ApplicationsForPlatform(platform: Platform) -> Set[Application]:
+    def ApplicationsForPlatform(platform: Platform, enable_rpcs: bool) -> Set[Application]:
         """What applications are buildable for a specific platform."""
         global _MATCHERS
-        return set(
-            [matcher.app for matcher in _MATCHERS[platform].app_arguments.keys()])
+        apps = set()
+        for matcher in _MATCHERS[platform].app_arguments.keys():
+            if matcher.rpcs_supported or not enable_rpcs:
+                apps.add(matcher.app)
+        return apps
 
     @staticmethod
     def PlatformsForApplication(application: Application) -> Set[Platform]:

scripts/build/build_examples.py

@@ -113,9 +113,15 @@ def ValidateRepoPath(context, parameter, value):
     default=False,
     is_flag=True,
     help='Skip timestaps in log output')
+@click.option(
+    '--rpc',
+    default=False,
+    is_flag=True,
+    help='Build with debug RPCs enabled.'
+)
 @click.pass_context
 def main(context, log_level, platform, board, app, repo, out_prefix, clean,
-         dry_run, dry_run_output, enable_flashbundle, no_log_timestamps):
+         dry_run, dry_run_output, enable_flashbundle, no_log_timestamps, rpc):
     # Ensures somewhat pretty logging of what is going on
     log_fmt = '%(asctime)s %(levelname)-7s %(message)s'
     if no_log_timestamps:
@@ -145,6 +151,7 @@ def main(context, log_level, platform, board, app, repo, out_prefix, clean,
         platforms=[build.Platform.FromArgName(name) for name in platform],
         boards=[build.Board.FromArgName(name) for name in board],
         applications=[build.Application.FromArgName(name) for name in app],
+        enable_rpcs=rpc,
         enable_flashbundle=enable_flashbundle)
 
     if clean:

scripts/build/builders/builder.py

@@ -117,6 +117,8 @@ def CopyArtifacts(self, target_dir: str):
             shutil.copyfile(source_name, target_full_name)
             shutil.copymode(source_name, target_full_name)
 
-    def SetIdentifier(self, platform: str, board: str, app: str):
+    def SetIdentifier(self, platform: str, board: str, app: str, enable_rpcs: bool = False):
         self.identifier = '-'.join([platform, board, app])
+        if enable_rpcs:
+            self.identifier = self.identifier + "-rpc"
         self.output_dir = os.path.join(self.output_prefix, self.identifier)

scripts/build/builders/efr32.py

@@ -69,14 +69,17 @@ def __init__(self,
                  runner,
                  output_prefix: str,
                  app: Efr32App = Efr32App.LIGHT,
-                 board: Efr32Board = Efr32Board.BRD4161A):
+                 board: Efr32Board = Efr32Board.BRD4161A,
+                 enable_rpcs: bool = False):
         super(Efr32Builder, self).__init__(
             root=os.path.join(root, 'examples', app.ExampleName(), 'efr32'),
             runner=runner,
             output_prefix=output_prefix)
-
         self.app = app
         self.board = board
+        self.gn_build_args = ['efr32_board="%s"' % board.GnArgName()]
+        if enable_rpcs:
+            self.gn_build_args.append('import("//with_pw_rpc.gni")')
 
     def GnBuildArgs(self):
         return ['efr32_board="%s"' % self.board.GnArgName()]

scripts/build/builders/esp32.py

@@ -68,17 +68,18 @@ def FlashBundleName(self):
         return self.AppNamePrefix + '.flashbundle.txt'
 
 
-def DefaultsFileName(board: Esp32Board, app: Esp32App):
+def DefaultsFileName(board: Esp32Board, app: Esp32App, enable_rpcs: bool):
     if app != Esp32App.ALL_CLUSTERS:
         # only all-clusters has a specific defaults name
         return None
 
+    rpc = "_rpc" if enable_rpcs else ""
     if board == Esp32Board.DevKitC:
-        return 'sdkconfig.defaults'
+        return 'sdkconfig{}.defaults'.format(rpc)
     elif board == Esp32Board.M5Stack:
-        return 'sdkconfig_m5stack.defaults'
+        return 'sdkconfig_m5stack{}.defaults'.format(rpc)
     elif board == Esp32Board.C3DevKit:
-        return 'sdkconfig_c3devkit.defaults'
+        return 'sdkconfig_c3devkit{}.defaults'.format(rpc)
     else:
         raise Exception('Unknown board type')
 
@@ -90,10 +91,12 @@ def __init__(self,
                  runner,
                  output_prefix: str,
                  board: Esp32Board = Esp32Board.M5Stack,
-                 app: Esp32App = Esp32App.ALL_CLUSTERS):
+                 app: Esp32App = Esp32App.ALL_CLUSTERS,
+                 enable_rpcs: bool = False):
         super(Esp32Builder, self).__init__(root, runner, output_prefix)
         self.board = board
         self.app = app
+        self.enable_rpcs = enable_rpcs
 
     def _IdfEnvExecute(self, cmd, cwd=None, title=None):
         self._Execute(
@@ -105,7 +108,7 @@ def generate(self):
         if os.path.exists(os.path.join(self.output_dir, 'build.ninja')):
             return
 
-        defaults = DefaultsFileName(self.board, self.app)
+        defaults = DefaultsFileName(self.board, self.app, self.enable_rpcs)
 
         cmd = 'idf.py'
 

scripts/build/builders/host.py

@@ -125,6 +125,7 @@ def build_outputs(self):
             self.map_name: os.path.join(self.output_dir, self.map_name)
         }
 
-    def SetIdentifier(self, platform: str, board: str, app: str):
+# todo
+    def SetIdentifier(self, platform: str, board: str, app: str, enable_rpcs: bool = False):
         super(HostBuilder, self).SetIdentifier(
-            self.board.PlatformName(), self.board.BoardName(), app)
+            self.board.PlatformName(), self.board.BoardName(), app, enable_rpcs)

scripts/build/builders/nrf.py

@@ -95,10 +95,12 @@ def __init__(self,
                  runner,
                  output_prefix: str,
                  app: NrfApp = NrfApp.LIGHT,
-                 board: NrfBoard = NrfBoard.NRF52840):
+                 board: NrfBoard = NrfBoard.NRF52840,
+                 enable_rpcs: bool = False):
         super(NrfConnectBuilder, self).__init__(root, runner, output_prefix)
         self.app = app
         self.board = board
+        self.enable_rpcs = enable_rpcs
 
     def generate(self):
         if not os.path.exists(self.output_dir):
@@ -130,12 +132,13 @@ def generate(self):
             cmd = '''
 source "$ZEPHYR_BASE/zephyr-env.sh";
 export GNUARMEMB_TOOLCHAIN_PATH="$PW_PIGWEED_CIPD_INSTALL_DIR";
-west build --cmake-only -d {outdir} -b {board} {sourcedir}
+west build --cmake-only -d {outdir} -b {board} {sourcedir}{rpcs}
         '''.format(
                 outdir=shlex.quote(self.output_dir),
                 board=self.board.GnArgName(),
                 sourcedir=shlex.quote(os.path.join(
-                    self.root, 'examples', self.app.ExampleName(), 'nrfconnect'))
+                    self.root, 'examples', self.app.ExampleName(), 'nrfconnect')),
+                rpcs=" -- -DOVERLAY_CONFIG=rpc.overlay" if self.enable_rpcs else ""
             ).strip()
 
             self._Execute(['bash', '-c', cmd],