Compile android in build_examples.py and move android compilation outside of Github CI

.github/workflows/android.yaml

@@ -15,8 +15,8 @@
 name: Android
 
 on:
-    push:
-    pull_request:
+    workflow_dispatch:
+    # NOTE: not enabled in push/pull_request as this check is available in GCP
 
 concurrency:
     group: ${{ github.ref }}-${{ github.workflow }}-${{ (github.event_name == 'pull_request' && github.event.number) || (github.event_name == 'workflow_dispatch' && github.run_number) || github.sha }}

scripts/build/build/factory.py

@@ -17,11 +17,13 @@
 from typing import Set
 
 from builders.builder import Builder
-from builders.host import HostBuilder, HostApp
-from builders.qpg import QpgBuilder
-from builders.esp32 import Esp32Builder, Esp32Board, Esp32App
+
+from builders.android import AndroidBoard, AndroidBuilder
 from builders.efr32 import Efr32Builder, Efr32App, Efr32Board
+from builders.esp32 import Esp32Builder, Esp32Board, Esp32App
+from builders.host import HostBuilder, HostApp
 from builders.nrf import NrfApp, NrfBoard, NrfConnectBuilder
+from builders.qpg import QpgBuilder
 
 from .targets import Application, Board, Platform
 
@@ -84,13 +86,16 @@ def Create(self, runner, __board_key: Board, __app_key: Application,
     Platform.QPG: Matcher(QpgBuilder),
     Platform.EFR32: Matcher(Efr32Builder),
     Platform.NRF: Matcher(NrfConnectBuilder),
+    Platform.ANDROID: Matcher(AndroidBuilder),
 }
 
 # Matrix of what can be compiled and what build options are required
 # by such compilation
 _MATCHERS[Platform.HOST].AcceptBoard(Board.NATIVE)
-_MATCHERS[Platform.HOST].AcceptApplication(Application.ALL_CLUSTERS, app=HostApp.ALL_CLUSTERS)
-_MATCHERS[Platform.HOST].AcceptApplication(Application.CHIP_TOOL, app=HostApp.CHIP_TOOL)
+_MATCHERS[Platform.HOST].AcceptApplication(
+    Application.ALL_CLUSTERS, app=HostApp.ALL_CLUSTERS)
+_MATCHERS[Platform.HOST].AcceptApplication(
+    Application.CHIP_TOOL, app=HostApp.CHIP_TOOL)
 
 _MATCHERS[Platform.ESP32].AcceptBoard(Board.DEVKITC, board=Esp32Board.DevKitC)
 _MATCHERS[Platform.ESP32].AcceptBoard(Board.M5STACK, board=Esp32Board.M5Stack)
@@ -114,13 +119,18 @@ def Create(self, runner, __board_key: Board, __app_key: Application,
 _MATCHERS[Platform.EFR32].AcceptApplication(
     Application.WINDOW_COVERING, app=Efr32App.WINDOW_COVERING)
 
-
 _MATCHERS[Platform.NRF].AcceptBoard(Board.NRF5340, board=NrfBoard.NRF5340)
 _MATCHERS[Platform.NRF].AcceptBoard(Board.NRF52840, board=NrfBoard.NRF52840)
 _MATCHERS[Platform.NRF].AcceptApplication(Application.LOCK, app=NrfApp.LOCK)
 _MATCHERS[Platform.NRF].AcceptApplication(Application.LIGHT, app=NrfApp.LIGHT)
 _MATCHERS[Platform.NRF].AcceptApplication(Application.SHELL, app=NrfApp.SHELL)
 
+_MATCHERS[Platform.ANDROID].AcceptBoard(Board.ARM, board=AndroidBoard.ARM)
+_MATCHERS[Platform.ANDROID].AcceptBoard(Board.ARM64, board=AndroidBoard.ARM64)
+_MATCHERS[Platform.ANDROID].AcceptBoard(Board.X64, board=AndroidBoard.X64)
+_MATCHERS[Platform.ANDROID].AcceptApplication(Application.CHIP_TOOL)
+
+
 class BuilderFactory:
   """Creates application builders."""
 
@@ -140,7 +150,8 @@ def Create(self, platform: Platform, board: Board, app: Application, enable_flas
         output_prefix=self.output_prefix)
 
     if builder:
-      builder.SetIdentifier(platform.name.lower(), board.name.lower(), app.name.lower())
+      builder.SetIdentifier(platform.name.lower(), board.name.lower(),
+                            app.name.lower())
       builder.enable_flashbundle(enable_flashbundle)
 
     return builder

scripts/build/build/targets.py

@@ -26,6 +26,7 @@ class Platform(IntEnum):
   ESP32 = auto()
   EFR32 = auto()
   NRF = auto()
+  ANDROID = auto()
 
   @property
   def ArgName(self):
@@ -59,6 +60,11 @@ class Board(IntEnum):
   NRF52840 = auto()
   NRF5340 = auto()
 
+  # Android platform
+  ARM = auto()
+  ARM64 = auto()
+  X64 = auto()
+
   @property
   def ArgName(self):
     return self.name.lower()

scripts/build/builders/android.py

@@ -0,0 +1,159 @@
+# Copyright (c) 2021 Project CHIP Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import logging
+import os
+import shlex
+
+from enum import Enum, auto
+
+from .builder import Builder
+
+
+class AndroidBoard(Enum):
+  ARM = auto()
+  ARM64 = auto()
+  X64 = auto()
+
+  def TargetCpuName(self):
+    if self == AndroidBoard.ARM:
+      return 'arm'
+    elif self == AndroidBoard.ARM64:
+      return 'arm64'
+    elif self == AndroidBoard.X64:
+      return 'x64'
+    else:
+      raise Exception('Unknown board type: %r' % self)
+
+
+class AndroidBuilder(Builder):
+
+  def __init__(self, root, runner, output_prefix: str, board: AndroidBoard):
+    super(AndroidBuilder, self).__init__(root, runner, output_prefix)
+    self.board = board
+
+  def validate_build_environment(self):
+    for k in ['ANDROID_NDK_HOME', 'ANDROID_HOME']:
+      if k not in os.environ:
+        raise Exception(
+            'Environment %s missing, cannot build android libraries' % k)
+
+    # SDK manager must be runnable to 'accept licenses'
+    sdk_manager = os.path.join(os.environ['ANDROID_HOME'], 'tools', 'bin',
+                               'sdkmanager')
+    if not (os.path.isfile(sdk_manager) and os.access(sdk_manager, os.X_OK)):
+      raise Exception("'%s' is not executable by the current user" %
+                      sdk_manager)
+
+    # In order to accept a license, the licenses folder is updated with the hash of the
+    # accepted license
+    android_home = os.environ['ANDROID_HOME']
+    licenses = os.path.join(android_home, 'licenses')
+    if not os.path.exists(licenses):
+      # Initial install may not have licenses at all
+      if not os.access(android_home, os.W_OK):
+        raise Exception(
+            "'%s' is NOT writable by the current user (needed to create licenses folder for accept)"
+            % android_home)
+
+    elif not os.access(licenses, os.W_OK):
+      raise Exception(
+          "'%s' is NOT writable by the current user (needed to accept licenses)"
+          % licenses)
+
+  def generate(self):
+    if not os.path.exists(self.output_dir):
+      # NRF does a in-place update  of SDK tools
+      if not self._runner.dry_run:
+        self.validate_build_environment()
+
+      gn_args = {}
+      gn_args['target_os'] = 'android'
+      gn_args['target_cpu'] = self.board.TargetCpuName()
+      gn_args['android_ndk_root'] = os.environ['ANDROID_NDK_HOME']
+      gn_args['android_sdk_root'] = os.environ['ANDROID_HOME']
+
+      args = '--args=%s' % (' '.join([
+          '%s="%s"' % (key, shlex.quote(value))
+          for key, value in gn_args.items()
+      ]))
+
+      self._Execute([
+          'gn', 'gen', '--check', '--fail-on-unused-args', self.output_dir, args
+      ],
+                    title='Generating ' + self.identifier)
+
+      self._Execute([
+          'bash', '-c',
+          'yes | %s/tools/bin/sdkmanager --licenses >/dev/null' %
+          os.environ['ANDROID_HOME']
+      ],
+                    title='Accepting NDK licenses')
+
+  def build(self):
+    self._Execute(['ninja', '-C', self.output_dir],
+                  title='Building JNI ' + self.identifier)
+
+    # NOTE: the following IDE-specific build instructions are NOT used:
+    #  - "rsync -a out/"android_$TARGET_CPU"/lib/*.jar src/android/CHIPTool/app/libs"
+    #    => using the 'ninjaOutputDir' project property instead to take the jar files directly
+    #       from the output
+
+    # JNILibs will be copied as long as they reside in src/main/jniLibs/ABI:
+    #    https://developer.android.com/studio/projects/gradle-external-native-builds#jniLibs
+    self._Execute([
+        'bash', '-c',
+        'rsync -a %s/lib/jni/* %s/src/android/CHIPTool/app/src/main/jniLibs/' %
+        (self.output_dir, self.root)
+    ],
+                  title='Prepare Native libs ' + self.identifier)
+
+    # App compilation
+    self._Execute([
+        '%s/src/android/CHIPTool/gradlew' % self.root, '-p',
+        '%s/src/android/CHIPTool' % self.root,
+        '-PchipSdkJarDir=%s' % os.path.join(self.output_dir, 'lib'),
+        '-PbuildDir=%s' % self.output_dir, 'build'
+    ],
+                  title='Building APP ' + self.identifier)
+
+  def jni_output_libs(self):
+    """Get a dictionary of JNI-required files."""
+    items = {}
+
+    scan_root = os.path.join(self.output_dir, 'lib', 'jni')
+    for root, dirs, files in os.walk(scan_root):
+      dir_name = root[len(scan_root) + 1:]
+      for file_name in files:
+        items[os.path.join(dir_name, file_name)] = os.path.join(root, file_name)
+
+    return items
+
+  def outputs(self):
+    outputs = {
+        'CHIPController.jar':
+            os.path.join(self.output_dir, 'lib', 'CHIPController.jar'),
+        'SetupPayloadParser.jar':
+            os.path.join(self.output_dir, 'lib', 'SetupPayloadParser.jar'),
+        'ChipTool-debug.apk':
+            os.path.join(self.output_dir, 'outputs', 'apk', 'debug',
+                         'app-debug.apk'),
+        'ChipTool-release-unsigned.apk':
+            os.path.join(self.output_dir, 'outputs', 'apk', 'release',
+                         'app-release-unsigned.apk'),
+    }
+
+    outputs.update(self.jni_output_libs())
+
+    return outputs

scripts/build/expected_all_platform_commands.txt

@@ -2,7 +2,7 @@
 gn gen --check --fail-on-unused-args --root={root}/examples/all-clusters-app/linux {out}/{real_platform}-native-all_clusters
 
 # Generating {real_platform}-native-chip_tool
-gn gen --check --fail-on-unused-args --root=/TEST/BUILD/ROOT/examples/chip-tool {out}/{real_platform}-native-chip_tool
+gn gen --check --fail-on-unused-args --root={root}/examples/chip-tool {out}/{real_platform}-native-chip_tool
 
 # Generating qpg-qpg6100-lock
 gn gen --check --fail-on-unused-args --root={root}/examples/lock-app/qpg {out}/qpg-qpg6100-lock
@@ -76,6 +76,24 @@ bash -c 'source "$ZEPHYR_BASE/zephyr-env.sh";
 export GNUARMEMB_TOOLCHAIN_PATH="$PW_PIGWEED_CIPD_INSTALL_DIR";
 west build --cmake-only -d {out}/nrf-nrf5340-shell -b nrf5340dk_nrf5340_cpuapp {root}/examples/shell/nrfconnect'
 
+# Generating android-arm-chip_tool
+gn gen --check --fail-on-unused-args {out}/android-arm-chip_tool '--args=target_os="android" target_cpu="arm" android_ndk_root="TEST_ANDROID_NDK_HOME" android_sdk_root="TEST_ANDROID_HOME"'
+
+# Accepting NDK licenses
+bash -c 'yes | TEST_ANDROID_HOME/tools/bin/sdkmanager --licenses >/dev/null'
+
+# Generating android-arm64-chip_tool
+gn gen --check --fail-on-unused-args {out}/android-arm64-chip_tool '--args=target_os="android" target_cpu="arm64" android_ndk_root="TEST_ANDROID_NDK_HOME" android_sdk_root="TEST_ANDROID_HOME"'
+
+# Accepting NDK licenses
+bash -c 'yes | TEST_ANDROID_HOME/tools/bin/sdkmanager --licenses >/dev/null'
+
+# Generating android-x64-chip_tool
+gn gen --check --fail-on-unused-args {out}/android-x64-chip_tool '--args=target_os="android" target_cpu="x64" android_ndk_root="TEST_ANDROID_NDK_HOME" android_sdk_root="TEST_ANDROID_HOME"'
+
+# Accepting NDK licenses
+bash -c 'yes | TEST_ANDROID_HOME/tools/bin/sdkmanager --licenses >/dev/null'
+
 # Building {real_platform}-native-all_clusters
 ninja -C {out}/{real_platform}-native-all_clusters
 
@@ -130,4 +148,31 @@ ninja -C {out}/nrf-nrf5340-lock
 # Building nrf-nrf5340-shell
 ninja -C {out}/nrf-nrf5340-shell
 
+# Building JNI android-arm-chip_tool
+ninja -C {out}/android-arm-chip_tool
+
+# Prepare Native libs android-arm-chip_tool
+bash -c 'rsync -a {out}/android-arm-chip_tool/lib/jni/* {root}/src/android/CHIPTool/app/src/main/jniLibs/'
+
+# Building APP android-arm-chip_tool
+{root}/src/android/CHIPTool/gradlew -p {root}/src/android/CHIPTool -PchipSdkJarDir={out}/android-arm-chip_tool/lib -PbuildDir={out}/android-arm-chip_tool build
+
+# Building JNI android-arm64-chip_tool
+ninja -C {out}/android-arm64-chip_tool
+
+# Prepare Native libs android-arm64-chip_tool
+bash -c 'rsync -a {out}/android-arm64-chip_tool/lib/jni/* {root}/src/android/CHIPTool/app/src/main/jniLibs/'
+
+# Building APP android-arm64-chip_tool
+{root}/src/android/CHIPTool/gradlew -p {root}/src/android/CHIPTool -PchipSdkJarDir={out}/android-arm64-chip_tool/lib -PbuildDir={out}/android-arm64-chip_tool build
+
+# Building JNI android-x64-chip_tool
+ninja -C {out}/android-x64-chip_tool
+
+# Prepare Native libs android-x64-chip_tool
+bash -c 'rsync -a {out}/android-x64-chip_tool/lib/jni/* {root}/src/android/CHIPTool/app/src/main/jniLibs/'
+
+# Building APP android-x64-chip_tool
+{root}/src/android/CHIPTool/gradlew -p {root}/src/android/CHIPTool -PchipSdkJarDir={out}/android-x64-chip_tool/lib -PbuildDir={out}/android-x64-chip_tool build
+
 

scripts/build/test.py

@@ -29,34 +29,43 @@
 
 SCRIPT_ROOT = os.path.dirname(__file__)
 
+
 def build_expected_output(root: str, out: str) -> List[str]:
-  with open(os.path.join(SCRIPT_ROOT, 'expected_all_platform_commands.txt'), 'rt') as f:
+  with open(
+      os.path.join(SCRIPT_ROOT, 'expected_all_platform_commands.txt'),
+      'rt') as f:
     for l in f.readlines():
-      yield l.replace("{root}", root).replace("{out}", out).replace('{real_platform}', ConcretePlatformName())
-
+      yield l.replace('{root}',
+                      root).replace('{out}',
+                                    out).replace('{real_platform}',
+                                                 ConcretePlatformName())
 
 def build_actual_output(root: str, out: str) -> List[str]:
   # Fake out that we have a project root
-  os.environ['PW_PROJECT_ROOT'] = root
 
   binary = os.path.join(SCRIPT_ROOT, 'build_examples.py')
 
-  retval = subprocess.run([
-    binary,
-    '--platform', 'all',
-    '--log-level', 'FATAL',
-    '--dry-run',
-    '--repo', root,
-    '--out-prefix', out,
-    'build'
-  ], stdout=subprocess.PIPE, check=True, encoding='UTF-8')
-
+  retval = subprocess.run(
+      [
+          binary, '--platform', 'all', '--log-level', 'FATAL', '--dry-run',
+          '--repo', root, '--out-prefix', out, 'build'
+      ],
+      stdout=subprocess.PIPE,
+      check=True,
+      encoding='UTF-8',
+      env={
+          'PW_PROJECT_ROOT': root,
+          'ANDROID_NDK_HOME': 'TEST_ANDROID_NDK_HOME',
+          'ANDROID_HOME': 'TEST_ANDROID_HOME',
+      })
 
   return [l + '\n' for l in retval.stdout.split('\n')]
 
 
 def main():
-  coloredlogs.install(level=logging.INFO, fmt='%(asctime)s %(name)s %(levelname)-7s %(message)s')
+  coloredlogs.install(
+      level=logging.INFO,
+      fmt='%(asctime)s %(name)s %(levelname)-7s %(message)s')
 
   ROOT = '/TEST/BUILD/ROOT'
   OUT = '/OUTPUT/DIR'
@@ -67,12 +76,11 @@ def main():
   diffs = [line for line in difflib.unified_diff(expected, actual)]
 
   if diffs:
-    logging.error("DIFFERENCE between expected and generated output")
+    logging.error('DIFFERENCE between expected and generated output')
     for l in diffs:
-      logging.warning("  " + l.strip())
+      logging.warning('  ' + l.strip())
     sys.exit(1)
 
 
-
-if __name__ == "__main__":
-    main()
+if __name__ == '__main__':
+  main()