[nrfconnect] Reduce size of the Docker image (#7593)

* [nrfconnect] Reduce Docker image size

1. Update nRF Connect SDK in the chip-build-nrf-platform
   Docker image to the latest supported version. Use shallow
   clone to install the new version without the full git
   history.
2. Remove arm-none-eabi-gcc from the image and add make CI
   use the version installed by scripts/bootstrap.sh.

* Update requirements.txt

.github/workflows/examples-nrfconnect.yaml

@@ -55,7 +55,7 @@ jobs:
                    .environment/pigweed-venv/*.log
             - name: Update nRF Connect SDK revision to the currently recommended.
               timeout-minutes: 5
-              run: scripts/run_in_build_env.sh "python3 scripts/setup/nrfconnect/update_ncs.py --update"
+              run: scripts/run_in_build_env.sh "python3 scripts/setup/nrfconnect/update_ncs.py --update --shallow"
             # https://github.com/project-chip/connectedhomeip/issues/3100
             # - name: Initialize CodeQL
             #   uses: github/codeql-action/init@v1

integrations/docker/images/chip-build-esp32/Dockerfile

@@ -9,7 +9,7 @@ RUN set -x \
     && cd /opt/espressif \
     && git clone --progress -b release/v4.2 https://github.com/espressif/esp-idf.git \
     && cd esp-idf \
-    && git reset --hard d9ec7df3931bea46292f015a99d19a28670bd561
+    && git reset --hard d9ec7df3931bea46292f015a99d19a28670bd561 \
     && git submodule update --init --progress \
     && IDF_TOOLS_PATH=/opt/espressif/tools ./install.sh \
     && : # last line

integrations/docker/images/chip-build-nrf-platform/Dockerfile

@@ -2,55 +2,46 @@ ARG VERSION=latest
 FROM connectedhomeip/chip-build:${VERSION}
 
 # Compatible Nordic Connect SDK revision.
-ARG NCS_REVISION=v1.5.0
+ARG NCS_REVISION=910c9ce5ba8ea30155b50b9d12f03ece7123a0c2
 
 # ==================================================
 # nRF Connect SDK dependencies
 # ==================================================
 
 # Tools for building, flashing and accessing device logs
+WORKDIR /opt/NordicSemiconductor/nRF5_tools/install
 RUN set -x \
     && apt-get update \
     && apt-get install --no-install-recommends -fy device-tree-compiler \
-    && (mkdir -p /opt/NordicSemiconductor/nRF5_tools/install && cd /opt/NordicSemiconductor/nRF5_tools/install \
     && curl https://www.nordicsemi.com/-/media/Software-and-other-downloads/Desktop-software/nRF-command-line-tools/sw/Versions-10-x-x/10-12-1/nRFCommandLineTools10121Linuxamd64.tar.gz \
     | tar zxvf - \
     && cd .. \
     && tar xvf install/JLink_Linux_V688a_x86_64.tgz \
     && tar xvf install/nRF-Command-Line-Tools_10_12_1.tar \
-    && rm -rf ./install /var/lib/apt/lists/) \
+    && rm -rf ./install /var/lib/apt/lists/ \
     && : # last line
 
-ENV NRF5_TOOLS_ROOT=/opt/NordicSemiconductor/nRF5_tools
-ENV PATH=${NRF5_TOOLS_ROOT}/JLink_Linux_V688a_x86_64:${PATH}
-ENV PATH=${NRF5_TOOLS_ROOT}/mergehex:${NRF5_TOOLS_ROOT}/nrfjprog:${PATH}
-ENV LD_LIBRARY_PATH=${NRF5_TOOLS_ROOT}/JLink_Linux_V688a_x86_64:${LD_LIBRARY_PATH}
-
-# GNU ARM Embedded toolchain, cross compiler for various platform builds
-RUN set -x \
-    && (mkdir -p /opt/ARM-software && cd /opt/ARM-software \
-    && curl https://armkeil.blob.core.windows.net/developer/Files/downloads/gnu-rm/9-2019q4/gcc-arm-none-eabi-9-2019-q4-major-x86_64-linux.tar.bz2 \
-    | tar jxvf -) \
-    && : # last line
-
-ENV ARM_GCC_INSTALL_ROOT=/opt/ARM-software/gcc-arm-none-eabi-9-2019-q4-major/bin/
-
 # ==================================================
 # nRF Connect SDK
 # ==================================================
+
+WORKDIR /opt/NordicSemiconductor/nrfconnect
 RUN set -x \
-    && (mkdir -p /opt/NordicSemiconductor/nrfconnect && cd /opt/NordicSemiconductor/nrfconnect \
     && python3 -m pip install -U --no-cache-dir pip setuptools wheel cmake west \
-    && west init -m https://github.com/nrfconnect/sdk-nrf --mr $NCS_REVISION \
-    && west update \
+    && west init -m https://github.com/nrfconnect/sdk-nrf --mr "$NCS_REVISION" \
+    && west config update.narrow true \
+    && west config update.fetch smart \
+    && west update -o=--depth=1 \
     && python3 -m pip install --no-cache-dir -r zephyr/scripts/requirements.txt \
     && python3 -m pip install --no-cache-dir -r nrf/scripts/requirements.txt \
-    && python3 -m pip install --no-cache-dir -r bootloader/mcuboot/scripts/requirements.txt) \
-    && echo "source /opt/NordicSemiconductor/nrfconnect/zephyr/zephyr-env.sh" >> ~/.bashrc \
+    && python3 -m pip install --no-cache-dir -r bootloader/mcuboot/scripts/requirements.txt \
     && : # last line
 
+ENV NRF5_TOOLS_ROOT=/opt/NordicSemiconductor/nRF5_tools
+ENV PATH=${NRF5_TOOLS_ROOT}/JLink_Linux_V688a_x86_64:${PATH}
+ENV PATH=${NRF5_TOOLS_ROOT}/mergehex:${NRF5_TOOLS_ROOT}/nrfjprog:${PATH}
+ENV LD_LIBRARY_PATH=${NRF5_TOOLS_ROOT}/JLink_Linux_V688a_x86_64:${LD_LIBRARY_PATH}
 ENV LC_ALL=C.UTF-8
 ENV LANG=C.UTF-8
 ENV ZEPHYR_BASE=/opt/NordicSemiconductor/nrfconnect/zephyr
 ENV ZEPHYR_TOOLCHAIN_VARIANT=gnuarmemb
-ENV GNUARMEMB_TOOLCHAIN_PATH=/opt/ARM-software/gcc-arm-none-eabi-9-2019-q4-major

integrations/docker/images/chip-build-vscode/Dockerfile

@@ -15,7 +15,6 @@ COPY --from=esp32 /opt/espressif/qemu-src /opt/espressif/qemu-src
 
 COPY --from=nrf /opt/NordicSemiconductor/nRF5_tools /opt/NordicSemiconductor/nRF5_tools
 COPY --from=nrf /opt/NordicSemiconductor/nrfconnect /opt/NordicSemiconductor/nrfconnect
-COPY --from=nrf /opt/ARM-software/gcc-arm-none-eabi-9-2019-q4-major /opt/ARM-software/gcc-arm-none-eabi-9-2019-q4-major
 
 COPY --from=android /opt/android/sdk /opt/android/sdk
 COPY --from=android /opt/android/android-ndk-r21b /opt/android/android-ndk-r21b
@@ -31,8 +30,6 @@ ENV QEMU_ESP32=/opt/espressif/qemu/xtensa-softmmu/qemu-system-xtensa
 ENV NRF5_TOOLS_ROOT=/opt/NordicSemiconductor/nRF5_tools
 ENV ZEPHYR_BASE=/opt/NordicSemiconductor/nrfconnect/zephyr
 ENV ZEPHYR_TOOLCHAIN_VARIANT=gnuarmemb
-ENV GNUARMEMB_TOOLCHAIN_PATH=/opt/ARM-software/gcc-arm-none-eabi-9-2019-q4-major
-ENV ARM_GCC_INSTALL_ROOT=/opt/ARM-software/gcc-arm-none-eabi-9-2019-q4-major/bin
 ENV EFR32_BOARD=BRD4161A
 ENV ANDROID_HOME=/opt/android/sdk
 ENV ANDROID_NDK_HOME=/opt/android/android-ndk-r21b

integrations/docker/images/chip-build/version

@@ -1 +1 @@
-0.4.27
+0.4.28

scripts/examples/nrfconnect_example.sh

@@ -16,29 +16,27 @@
 #    limitations under the License.
 #
 
-# Run bootstrap and activate to set up e.g. Pigweed correctly
-source "$(dirname "$0")/../../scripts/activate.sh"
-
 cd "$(dirname "$0")/../../examples"
 
 APP="$1"
 BOARD="$2"
 shift 2
 
-if [[ ! -f "$APP/nrfconnect/CMakeLists.txt" ]]; then
-    echo "Usage: $0 <application>" >&2
+if [[ ! -f "$APP/nrfconnect/CMakeLists.txt" || -z "$BOARD" ]]; then
+    echo "Usage: $0 <application> <board>" >&2
     echo "Applications:" >&2
     ls */nrfconnect/CMakeLists.txt | awk -F/ '{print "  "$1}' >&2
     exit 1
 fi
 
-if [ -z "$BOARD" ]; then
-    echo "No mandatory BOARD argument supplied!"
-    exit 1
-fi
-
 set -x
+
+# Activate Matter environment
+source "../scripts/activate.sh"
+
+# Activate Zephyr environment
 [[ -n $ZEPHYR_BASE ]] && source "$ZEPHYR_BASE/zephyr-env.sh"
+export GNUARMEMB_TOOLCHAIN_PATH="$PW_PIGWEED_CIPD_INSTALL_DIR"
 env
 
 west build -b "$BOARD" -d "$APP/nrfconnect/build/$BOARD" "$APP/nrfconnect" -- "$@"

scripts/requirements.in

@@ -31,7 +31,7 @@ portpicker
 mobly
 
 # zephyr
-west>=0.10.1
+west>=0.11.0
 intelhex
 
 # happy tests

scripts/requirements.txt

@@ -20,34 +20,34 @@ bitstring==3.1.7
     # via -r requirements.in
 brotli==1.0.9
     # via flask-compress
-certifi==2020.12.5
+certifi==2021.5.30
     # via requests
 cffi==1.14.5
     # via cryptography
 chardet==4.0.0
     # via requests
-click==7.1.2
+click==8.0.1
     # via
     #   -r requirements.in
     #   flask
     #   pip-tools
 colorama==0.4.4
     # via west
-coloredlogs==15.0
+coloredlogs==15.0.1
     # via -r requirements.in
-cryptography==3.4.6
+cryptography==3.4.7
     # via -r requirements.in
 cxxfilt==0.2.2
     # via -r requirements.in
 dbus-python==1.2.16 ; sys_platform == "linux"
     # via -r requirements.in
-decorator==4.4.2
+decorator==5.0.9
     # via ipython
-distlib==0.3.1
+distlib==0.3.2
     # via virtualenv
 docopt==0.6.2
     # via pykwalify
-ecdsa==0.16.1
+ecdsa==0.17.0
     # via -r requirements.in
 filelock==3.0.12
     # via virtualenv
@@ -68,28 +68,30 @@ gdbgui==0.13.2.0
     # via -r requirements.in
 gevent==1.5.0
     # via gdbgui
-greenlet==1.0.0
+greenlet==1.1.0
     # via gevent
-humanfriendly==9.1
+humanfriendly==9.2
     # via coloredlogs
 idna==2.10
     # via requests
 intelhex==2.3.0
     # via -r requirements.in
 ipython-genutils==0.2.0
     # via traitlets
-ipython==7.21.0
+ipython==7.24.1
     # via -r requirements.in
-itsdangerous==1.1.0
+itsdangerous==2.0.1
     # via flask
 jedi==0.18.0
     # via ipython
-jinja2==2.11.3
+jinja2==3.0.1
     # via flask
 lockfile==0.12.2
     # via -r requirements.in
-markupsafe==1.1.1
+markupsafe==2.0.1
     # via jinja2
+matplotlib-inline==0.1.2
+    # via ipython
 mobly==1.10.1
     # via -r requirements.in
 numpy==1.20.3
@@ -98,7 +100,7 @@ packaging==20.9
     # via west
 pandas==1.2.4
     # via -r requirements.in
-parso==0.8.1
+parso==0.8.2
     # via jedi
 pep517==0.10.0
     # via pip-tools
@@ -108,15 +110,15 @@ pgi==0.0.11.2 ; sys_platform == "linux"
     # via -r requirements.in
 pickleshare==0.7.5
     # via ipython
-pip-tools==6.0.1
+pip-tools==6.1.0
     # via -r requirements.in
-portpicker==1.3.1
+portpicker==1.4.0
     # via
     #   -r requirements.in
     #   mobly
-prompt-toolkit==3.0.17
+prompt-toolkit==3.0.18
     # via ipython
-protobuf==3.15.6
+protobuf==3.17.3
     # via -r requirements.in
 psutil==5.8.0
     # via
@@ -132,7 +134,7 @@ pygdbmi==0.9.0.2
     # via
     #   -r requirements.in
     #   gdbgui
-pygments==2.8.1
+pygments==2.9.0
     # via
     #   gdbgui
     #   ipython
@@ -150,9 +152,9 @@ python-dateutil==2.8.1
     # via
     #   pandas
     #   pykwalify
-python-engineio==4.0.1
+python-engineio==4.2.0
     # via python-socketio
-python-socketio==5.1.0
+python-socketio==5.3.0
     # via flask-socketio
 pytz==2021.1
     # via pandas
@@ -166,9 +168,9 @@ requests==2.25.1
     # via -r requirements.in
 ruamel.yaml.clib==0.2.2
     # via ruamel.yaml
-ruamel.yaml==0.16.13
+ruamel.yaml==0.17.9
     # via pykwalify
-six==1.15.0
+six==1.16.0
     # via
     #   anytree
     #   ecdsa
@@ -180,18 +182,20 @@ timeout-decorator==0.5.0
 toml==0.10.2
     # via pep517
 traitlets==5.0.5
-    # via ipython
+    # via
+    #   ipython
+    #   matplotlib-inline
 urllib3==1.26.5
     # via requests
-virtualenv==20.4.3
+virtualenv==20.4.7
     # via -r requirements.in
-watchdog==2.0.2
+watchdog==2.1.2
     # via -r requirements.in
 wcwidth==0.2.5
     # via prompt-toolkit
 werkzeug==0.16.1
     # via flask
-west==0.10.1
+west==0.11.0
     # via -r requirements.in
 wheel==0.36.2
     # via -r requirements.in

scripts/setup/nrfconnect/update_ncs.py

@@ -26,7 +26,7 @@ def get_repository_commit_sha(repository_location):
     process = subprocess.run(command, check=True, stdout=subprocess.PIPE)
     return process.stdout.decode('ascii').strip()
 
-def update_ncs(repository_location, revision):
+def update_ncs(repository_location, revision, fetch_shallow):
     # Fetch sdk-nrf to the desired revision.
     command = ['git', '-C', repository_location, 'fetch']
     subprocess.run(command, check=True)
@@ -36,7 +36,8 @@ def update_ncs(repository_location, revision):
     subprocess.run(command, check=True)
 
     # Call west update command to update all projects and submodules used by sdk-nrf.
-    command = ['west', 'update', '-r']
+    command = ['west', 'update']
+    command += ['--fetch', 'smart', '--narrow', '-o=--depth=1'] if fetch_shallow else []
     subprocess.run(command, check=True)
 
 def get_ncs_recommended_revision():
@@ -49,7 +50,7 @@ def get_ncs_recommended_revision():
     except:
         raise RuntimeError("Encountered problem when trying to read .nrfconnect-recommended-revision file.")
 
-def print_messages(messages : list, yellow_text :  bool):
+def print_messages(messages : list, yellow_text : bool):
     # Add colour formatting if yellow text was set
     if yellow_text:
         messages = [f"\33[33m{message}\x1b[0m" for message in messages]
@@ -83,6 +84,7 @@ def main():
         parser = argparse.ArgumentParser(description='Script helping to update nRF Connect SDK to currently recommended revision.')
         parser.add_argument("-c", "--check", help="Check if your current nRF Connect SDK revision is the same as recommended one.", action="store_true")
         parser.add_argument("-u", "--update", help="Update your nRF Connect SDK to currently recommended revision.", action="store_true")
+        parser.add_argument("-s", "--shallow", help="Fetch only specific commits (without the history) when updating nRF Connect SDK.", action="store_true")
         parser.add_argument("-q", "--quiet", help="Don't print any message if the check succeeds.", action="store_true")
         args = parser.parse_args()
 
@@ -104,7 +106,7 @@ def main():
 
         if args.update:
             print("Updating nRF Connect SDK to recommended revision...")
-            update_ncs(ncs_base, recommended_revision)
+            update_ncs(ncs_base, recommended_revision, args.shallow)
 
     except (RuntimeError, subprocess.CalledProcessError) as e:
         print(e)