update docker images and submodule workflow (was drop repos.conf) (pr…

…oject-chip#1508)

* drop repos.conf

* checkout submodules, too

* * update docker images to unbuntu:focal
* rev version to 0.3.0
* fix up build-all.sh to understand the dependencies among the images

* restyled

* use clang

* update docs

* somehow lost dbus, fixed

* don't dist repos.conf

* dang

.devcontainer/Dockerfile

@@ -25,8 +25,6 @@ RUN useradd -s /bin/bash -u $USER_UID -g $USER_GID -G docker -m $USERNAME
 RUN echo $USERNAME ALL=\(root\) NOPASSWD:ALL > /etc/sudoers.d/$USERNAME
 RUN chmod 0440 /etc/sudoers.d/$USERNAME
 
-RUN ln -s /usr/bin/clang-format-9 /usr/local/bin/clang-format
-
 RUN mkdir -p /var/downloads
 RUN cd /var/downloads
 RUN curl -JL https://github.com/microsoft/vscode-cpptools/releases/download/0.27.0/cpptools-linux.vsix | bsdtar -xvf - extension

.devcontainer/devcontainer.json

@@ -8,7 +8,7 @@
         "dockerfile": "Dockerfile",
         "args": {
             //  "BUILD_VERSION": "$(cat integrations/docker/images/chip-build/version)" // trying to get this to work
-            "BUILD_VERSION": "0.2.15"
+            "BUILD_VERSION": "0.3.0"
         }
     },
     "remoteUser": "vscode",

.github/workflows/build.yaml

@@ -13,25 +13,27 @@ jobs:
                 type: [main, clang, linux-embedded, mbedtls]
         env:
             BUILD_TYPE: ${{ matrix.type }}
-            BUILD_VERSION: 0.2.14
-            BUILD_IMAGE: chip-build-openssl
+            BUILD_VERSION: 0.3.0
+            BUILD_IMAGE: chip-build
             BUILD_ORG: connectedhomeip
 
         runs-on: ubuntu-latest
 
         container:
-            image: "connectedhomeip/chip-build-openssl:0.2.18"
+            image: "connectedhomeip/chip-build:0.3.0"
             volumes:
                 - "/tmp/log_output:/tmp/test_logs"
 
         steps:
             - name: Checkout
               uses: actions/checkout@v2
+              with:
+                 submodules: true
             - name: Bootstrap
               run: |
                   case $BUILD_TYPE in
                      "main") export BOOTSTRAP_ARGUMENTS="";;
-                     "clang") export BOOTSTRAP_ARGUMENTS="CC=clang-9 CXX=clang++-9";;
+                     "clang") export BOOTSTRAP_ARGUMENTS="CC=clang CXX=clang++";;
                      "linux-embedded") export BOOTSTRAP_ARGUMENTS="--with-device-layer=linux";;
                      "mbedtls") export BOOTSTRAP_ARGUMENTS="--with-crypto=mbedtls";;
                      *) ;;

.github/workflows/examples.yaml

@@ -10,21 +10,23 @@ jobs:
 
         env:
             BUILD_TYPE: esp32
-            BUILD_VERSION: 0.2.14
+            BUILD_VERSION: 0.3.0
             BUILD_IMAGE: chip-build-esp32
             BUILD_ORG: connectedhomeip
 
         runs-on: ubuntu-latest
 
         container:
-            image: connectedhomeip/chip-build-esp32:0.2.14 # $BUILD_IMAGE/$BUILD_ORG:$BUILD_VERSION
+            image: connectedhomeip/chip-build-esp32:0.3.0 # $BUILD_IMAGE/$BUILD_ORG:$BUILD_VERSION
             volumes:
                 - "/tmp/bloat_reports:/tmp/bloat_reports"
                 - "/tmp/output_binaries:/tmp/output_binaries"
 
         steps:
             - name: Checkout
               uses: actions/checkout@v2
+              with:
+                 submodules: true
             - name: Build example Echo App
               run: scripts/examples/esp_echo_app.sh
             - name: Copy aside build products
@@ -63,21 +65,23 @@ jobs:
 
         env:
             BUILD_TYPE: nrf
-            BUILD_VERSION: 0.2.14
+            BUILD_VERSION: 0.3.0
             BUILD_IMAGE: chip-build-nrf-platform
             BUILD_ORG: connectedhomeip
 
         runs-on: ubuntu-latest
 
         container:
-            image: connectedhomeip/chip-build-nrf-platform:0.2.14 # $BUILD_IMAGE/$BUILD_ORG:$BUILD_VERSION
+            image: connectedhomeip/chip-build-nrf-platform:0.3.0 # $BUILD_IMAGE/$BUILD_ORG:$BUILD_VERSION
             volumes:
                 - "/tmp/bloat_reports:/tmp/bloat_reports"
                 - "/tmp/output_binaries:/tmp/output_binaries"
 
         steps:
             - name: Checkout
               uses: actions/checkout@v2
+              with:
+                 submodules: true
             - name: Build example nRF5 Lock App
               run: scripts/examples/nrf_lock_app.sh
             - name: Preserve artifacts
@@ -90,21 +94,23 @@ jobs:
 
         env:
             BUILD_TYPE: linux
-            BUILD_VERSION: 0.2.14
+            BUILD_VERSION: 0.3.0
             BUILD_IMAGE: chip-build
             BUILD_ORG: connectedhomeip
 
         runs-on: ubuntu-latest
 
         container:
-            image: connectedhomeip/chip-build:0.2.14 # $BUILD_IMAGE/$BUILD_ORG:$BUILD_VERSION
+            image: connectedhomeip/chip-build:0.3.0 # $BUILD_IMAGE/$BUILD_ORG:$BUILD_VERSION
             volumes:
                 - "/tmp/bloat_reports:/tmp/bloat_reports"
                 - "/tmp/output_binaries:/tmp/output_binaries"
 
         steps:
             - name: Checkout
               uses: actions/checkout@v2
+              with:
+                 submodules: true
             - name: Build example Standalone Echo Client
               run: scripts/examples/standalone_echo_client.sh
             - name: Build example Standalone Shell

.github/workflows/qemu.yaml

@@ -10,20 +10,22 @@ jobs:
 
         env:
             BUILD_TYPE: esp32-qemu
-            BUILD_VERSION: 0.2.14
+            BUILD_VERSION: 0.3.0
             BUILD_IMAGE: chip-build-esp32-qemu
             BUILD_ORG: connectedhomeip
 
         runs-on: ubuntu-latest
 
         container:
-            image: connectedhomeip/chip-build-esp32-qemu:0.2.14 # $BUILD_IMAGE/$BUILD_ORG:$BUILD_VERSION
+            image: connectedhomeip/chip-build-esp32-qemu:0.3.0 # $BUILD_IMAGE/$BUILD_ORG:$BUILD_VERSION
             volumes:
                 - "/tmp/log_output:/tmp/test_logs"
 
         steps:
             - name: Checkout
               uses: actions/checkout@v2
+              with:
+                 submodules: true
             - name: Build example Echo App
               run: scripts/examples/esp_echo_app.sh
             - name: Build ESP32 QEMU and Run Tests

Makefile.am

@@ -52,6 +52,7 @@ EXTRA_DIST                               = \
     bootstrap                              \
     bootstrap-configure                    \
     repos.conf                             \
+    .gitmodules                            \
     $(srcdir)/build/autoconf               \
     $(srcdir)/config                       \
     $(srcdir)/config/efr32                 \

docs/BUILDING.md

@@ -30,7 +30,7 @@ want:
 -   GNU make (versions prior to 4.1 may produce spurious warnings/errors)
 -   automake, autoconf, libtool
 -   C and C++ compilers
--   clang-format-9
+-   clang-format (at least version 9)
 -   gcov
 -   pkg-config
 

examples/wifi-echo/server/esp32/README.md

@@ -49,10 +49,10 @@ step. To install these components manually, follow these steps:
 Currently building in VSCode _and_ deploying from native is not supported, so
 make sure the IDF_PATH has been exported(See the manual setup steps above).
 
--   In the root of the example directory, sync the dependencies and source
-    `idf.sh`. Note: This does not have to be repeated for incremental builds.
+-   In the root of the example directory, sync the CHIP tree's submodules and
+    source `idf.sh`. Note: This does not have to be repeated for incremental
+    builds.
 
-          $ make -C third_party/connectedhomeip -f Makefile-bootstrap repos
           $ source idf.sh
 
 -   Next, if you want to use the M5Stack with its display and show a QRCode run

integrations/docker/build.sh

@@ -58,6 +58,12 @@ set -ex
 
 [[ -n $VERSION ]] || die "version cannot be empty"
 
+# go find and build any CHIP images this image is "FROM"
+awk -F/ '/^FROM connectedhomeip/ {print $2}' Dockerfile | while read -r dep; do
+    dep=${dep%:*}
+    (cd "../$dep" && ./build.sh "$@")
+done
+
 BUILD_ARGS=()
 if [[ ${*/--no-cache//} != "${*}" ]]; then
     BUILD_ARGS+=(--no-cache)

integrations/docker/images/build-all.sh

@@ -23,5 +23,5 @@
 
 find . -name Dockerfile | while read -r dockerfile; do
     dir=${dockerfile%/*}
-    (cd "$dir" && ./build.sh) || exit $?
+    (cd "$dir" && ./build.sh "$@") || exit $?
 done

integrations/docker/images/chip-build-esp32/Dockerfile

@@ -3,6 +3,8 @@ FROM connectedhomeip/chip-build:${VERSION}
 
 # Setup the ESP-IDF
 RUN set -x \
+    && apt-get update \
+    && DEBIAN_FRONTEND=noninteractive apt-get install -y python \
     && cd /var \
     && git clone --progress -b release/v4.0 https://github.com/espressif/esp-idf.git \
     && cd esp-idf \

integrations/docker/images/chip-build-nrf-platform/Dockerfile

@@ -16,8 +16,8 @@ RUN set -x \
     && (mkdir /var/nRF5_tools && cd /var/nRF5_tools \
     && curl https://www.nordicsemi.com/-/media/Software-and-other-downloads/Desktop-software/nRF-command-line-tools/sw/Versions-10-x-x/10-7-0/nRFCommandLineTools1070Linuxamd64tar.gz \
     | tar zxvf - \
-    && dpkg -i JLink_Linux_*.deb \
-    && dpkg -i nRF-Command-Line-Tools_*.deb \
+    && dpkg -i --force-depends JLink_Linux_*.deb \
+    && dpkg -i --force-depends nRF-Command-Line-Tools_*.deb \
     && tar zxvf nRF-Command-Line-Tools_*.tar.gz) \
     && rm -rf /var/nRF5_tools/*.tar.gz \
     && rm -rf /var/nRF5_tools/*.deb \

integrations/docker/images/chip-build-vscode/Dockerfile

@@ -1,6 +1,5 @@
 ARG VERSION=latest
 FROM connectedhomeip/chip-build:${VERSION}
-FROM connectedhomeip/chip-build-openssl:${VERSION}
 FROM connectedhomeip/chip-build-nrf-platform:${VERSION}
 FROM connectedhomeip/chip-build-esp32:${VERSION}
 FROM connectedhomeip/chip-build-esp32-qemu:${VERSION}

integrations/docker/images/chip-build/Dockerfile

@@ -1,12 +1,12 @@
-# start with Ubuntu 18.04LTS
-FROM ubuntu:bionic
+# start with Ubuntu 20.04LTS
+FROM ubuntu:focal
 
 VOLUME "/var/source"
 
 # base build and check tools and libraries layer
 RUN set -x \
     && apt-get update \
-    && apt-get install -fy \
+    && DEBIAN_FRONTEND=noninteractive apt-get install -fy \
     git \
     curl \
     jq \
@@ -16,19 +16,19 @@ RUN set -x \
     libtool \
     pkg-config \
     g++ \
-    clang-9 \
-    clang-format-9 \
-    clang-tidy-9 \
+    clang \
+    clang-format \
+    clang-tidy \
+    cmake \
     lcov \
     shellcheck \
     libssl-dev \
     unzip \
     wget \
     libmbedtls-dev \
-    python-pip \
     python3 \
     python3-pip \
-    libusb-1.0 \
+    libusb-1.0-0 \
     libncurses5-dev \
     libncursesw5-dev \
     libnspr4-dev \
@@ -39,25 +39,35 @@ RUN set -x \
     zlib1g-dev \
     libglib2.0-dev \
     libpixman-1-dev \
+    libdbus-1-dev \
     ninja-build \
     && rm -rf /var/lib/apt/lists/ \
     && : # last line
 
+# Python 2 and PIP
 RUN set -x \
-    && pip3 install circleci attrs coloredlogs PyGithub \
+   && apt-get update \
+   && DEBIAN_FRONTEND=noninteractive apt-get install -y software-properties-common \
+   && add-apt-repository universe \
+   && DEBIAN_FRONTEND=noninteractive apt-get install -y python python2 \
+   && curl https://bootstrap.pypa.io/get-pip.py --output get-pip.py \
+   && python2 get-pip.py \
+   && rm -rf /var/lib/apt/lists/ \
+   && : # last line
+
+RUN set -x \
+    && pip3 install circleci attrs coloredlogs PyGithub pygit \
     && : # last line
 
-# Install newer cmake toolchain
+# build and install gn
 RUN set -x \
-    && wget https://cmake.org/files/v3.17/cmake-3.17.2.tar.gz \
-    && tar xzf cmake-3.17.2.tar.gz \
-    && cd cmake-3.17.2 \
-    && ./bootstrap \
-    && make \
-    && make install \
+    && git clone https://gn.googlesource.com/gn \
+    && cd gn \
+    && python3 build/gen.py \
+    && ninja -C out \
+    && cp out/gn /usr/local/bin \
     && cd .. \
-    && rm -rf cmake-3.17.2 \
-    && rm cmake-3.17.2.tar.gz \
+    && rm -rf gn \
     && : # last line
 
 # Install bloat comparison tools

integrations/docker/images/chip-build/version

@@ -1 +1 @@
-0.2.16
+0.3.0

scripts/examples/esp_echo_app.sh

@@ -21,7 +21,6 @@ env
 
 root=examples/wifi-echo/server/esp32/
 
-make -j -f Makefile-bootstrap repos
 # shellcheck source=/dev/null
 source "$root"/idf.sh
 

scripts/helpers/clean_tree.sh

@@ -19,4 +19,4 @@
 set -x
 env
 
-make -f Makefile-bootstrap clean-repos && git clean -Xdf
+git clean -Xdf