Update Dockerfiles to ubuntu:24.04 (#33637)

* Update to ubuntu 24.04 and bump version

* Fix comment

* Dependencies update

* Use the latest gcc by default

* Fix up clang save & cleanup

* More build fixes

* Add more --break-system-packages

* Do not remove libllvm17t64 ... I am unsure about that one

* Remove it after all ... seems some size savings  here

* More cleanup

* Drop alternative install

* Install gcc without a specific version. Undoes #28478

* Need llvm18-dev as well

* Update to lib64 paths

* Also install xz

* xz is in utils

* Fix asr depends when no recommends is installed

* Clean up infineon, make the curl download to NOT be silent

* using -j8 is arbitrary ... use nproc instead

* update what qemu means

* Install patch for the qemu image build

* Adapt Tizen Dockerfiles

* Update ubuntu version

* Update Dockerfile for chip-build-linux-qemu image

* Restyle

* Add ability to skip already built things, otherwise docker keeps re-building stuff

* Docker image inspect is MUCH faster than docker images

* Fix nrf build

* Fix ameba dockerfile build

* Fix cirque image build

* Make imx image work

* Make build ordering predictable

* Fix k32w image build

* Fix telink image build

* Speed up android image build by a lot

* Bump up espressif qemu build, do not use the 3-year old version

* Update versions to match system package versions, so that bootstrap is faster

* Make sure that patch is available in the base image, we seem to use it ocassionally

* One documentation update

* Also allow compilation of mbedos bits in the vscode image

* Include what you use from system

---------

Co-authored-by: Andrei Litvin <[email protected]>
Co-authored-by: Maksymilian Knust <[email protected]>
Co-authored-by: Jakub Latusek <[email protected]>

integrations/docker/build-all.sh

@@ -25,19 +25,33 @@ set -e
 
 SUB_PATH="*"
 ALL_ARGS=""
+SKIP_BUILT=0
 
 for i in "$@"; do
     case $i in
+    -h | --help)
+       echo "Supported arguments:"
+       echo "  -h/--help       This help text"
+       echo "  -p/--path       sub-path in integrations/docker/images to build (defaults to * for everything)"
+       echo "  -skip-built     Do not rebuild images already found as built"
+       exit 0
+       ;;
     -p=* | --path=*)
         SUB_PATH="${i#*=}"
         ;;
+    --skip-built)
+        SKIP_BUILT=1
+	;;
     *)
         ALL_ARGS="$ALL_ARGS ${i#*=}"
         ;;
     esac
 done
 
+VERSION=${DOCKER_BUILD_VERSION:-$(sed 's/ .*//' "$(git rev-parse --show-toplevel)"/integrations/docker/images/base/chip-build/version)}
+
 echo "SUB_PATH: $SUB_PATH"
+echo "VERSION: $VERSION"
 
 function build_image() {
     PARSE_PATH=$1
@@ -46,8 +60,17 @@ function build_image() {
     echo "PARSE_PATH: $PARSE_PATH"
     echo "ARGS_TO_PASS: $ARGS_TO_PASS"
 
-    find "$(git rev-parse --show-toplevel)"/integrations/docker/images/$PARSE_PATH -name Dockerfile ! -path "*chip-cert-bins/*" | while read -r dockerfile; do
-        echo "$(dirname "$dockerfile")"
+    find "$(git rev-parse --show-toplevel)"/integrations/docker/images/$PARSE_PATH -name Dockerfile ! -path "*chip-cert-bins/*" | sort | while read -r dockerfile; do
+        # Images are of the form `ghcr.io/project-chip/{name}` and tagged as "${VERSION}"
+        DOCKER_PATH=$(dirname $dockerfile) # Drop the file name
+        IMAGE_NAME="ghcr.io/project-chip/${DOCKER_PATH##*/}:${VERSION}"     # Drop directory prefix
+
+        if [ $SKIP_BUILT -ne 0 ] && docker image inspect "${IMAGE_NAME}" >/dev/null 2>&1; then
+          echo "Image ${IMAGE_NAME} already exists. Skipping build"
+          continue
+        fi
+
+        echo "BUILDING $(dirname "$dockerfile") (i.e. ${IMAGE_NAME})"
         pushd "$(dirname "$dockerfile")" >/dev/null
         ./build.sh "$ARGS_TO_PASS"
         popd >/dev/null

integrations/docker/images/base/chip-build-minimal/Dockerfile

@@ -1,27 +1,19 @@
 # This minimal build image is intentionally not based on chip-build
-FROM ubuntu:focal
+FROM ubuntu:24.04
 LABEL org.opencontainers.image.source https://github.com/project-chip/connectedhomeip
 
-# ARG NINJA_VERSION=v1.11.1
-ARG GN_HASH=5a004f9427a050c6c393c07ddb85cba8ff3849fa
-
 RUN set -x \
     && apt-get update \
     && DEBIAN_FRONTEND=noninteractive apt-get upgrade -y \
     && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \
-    build-essential ca-certificates git pkg-config python3-venv ninja-build \
+    build-essential \
+    ca-certificates \
+    generate-ninja \
+    git pkg-config \
+    ninja-build \
+    python3-venv \
     && git config --global advice.detachedHead false
 
-# RUN set -x && cd /var/tmp \
-#     && git clone --branch "$NINJA_VERSION" https://github.com/ninja-build/ninja.git \
-#     && ( cd ninja && ./configure.py --bootstrap && install -m 0755 ninja /usr/local/bin/ ) \
-#     && rm -rf ninja
-
-RUN set -x && cd /var/tmp \
-    && git clone https://gn.googlesource.com/gn \
-    && ( cd gn && git checkout "$GN_HASH" && CXX=g++ build/gen.py && ninja -C out && install -m 0755 out/gn /usr/local/bin/ ) \
-    && rm -rf gn
-
 # CHIP build dependencies
 RUN set -x \
     && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \

integrations/docker/images/base/chip-build/Dockerfile

@@ -1,5 +1,4 @@
-# start with Ubuntu 20.04LTS
-FROM ubuntu:focal
+FROM ubuntu:24.04
 LABEL org.opencontainers.image.source https://github.com/project-chip/connectedhomeip
 
 VOLUME "/var/source"
@@ -30,7 +29,7 @@ RUN case ${TARGETPLATFORM} in \
 # base build and check tools and libraries layer
 RUN set -x \
     && apt-get update \
-    && DEBIAN_FRONTEND=noninteractive apt-get install -fy --fix-missing \
+    && DEBIAN_FRONTEND=noninteractive apt-get install -fy --fix-missing --no-install-recommends \
     autoconf \
     automake \
     bison \
@@ -39,23 +38,27 @@ RUN set -x \
     clang \
     clang-format \
     clang-tidy \
+    cmake \
     curl \
     flex \
-    g++-10 \
+    g++ \
+    generate-ninja \
     git \
     git-lfs \
     gperf \
     iproute2 \
+    iwyu \
     jq \
     lcov \
     libavahi-client-dev \
     libavahi-common-dev \
-    libcairo-dev \
     libcairo2-dev \
+    libcairo-dev \
     libdbus-1-dev \
     libdbus-glib-1-dev \
     libdmalloc-dev \
     libgif-dev \
+    libgirepository-1.0-1 \
     libglib2.0-dev \
     libical-dev \
     libjpeg-dev \
@@ -68,8 +71,8 @@ RUN set -x \
     libpango1.0-dev \
     libpixman-1-dev \
     libreadline-dev \
-    libsdl-pango-dev \
     libsdl2-dev \
+    libsdl-pango-dev \
     libssl-dev \
     libtool \
     libudev-dev \
@@ -80,11 +83,19 @@ RUN set -x \
     meson \
     net-tools \
     ninja-build \
+    nodejs \
+    patch \
     pkg-config \
+    python3 \
+    python3-click \
+    python3-coloredlogs \
+    python3-dev \
+    python3-future \
+    python3-pandas \
+    python3-pip \
+    python3-tabulate \
+    python3-venv \
     python-is-python3 \
-    python3.9 \
-    python3.9-dev \
-    python3.9-venv \
     rsync \
     shellcheck \
     strace \
@@ -97,73 +108,19 @@ RUN set -x \
     && git lfs install \
     && : # last line
 
-# Set gcc 10 as a default compiler to work with TSAN
-RUN set -x \
-    && update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-10 10 \
-    && update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-10 10 \
-    && update-alternatives --install /usr/bin/cc cc /usr/bin/gcc 30 \
-    && update-alternatives --set cc /usr/bin/gcc \
-    && update-alternatives --install /usr/bin/c++ c++ /usr/bin/g++ 30 \
-    && update-alternatives --set c++ /usr/bin/g++ \
-    && : # last line
-
-# Cmake v3.23.1
-ENV CMAKE_PLATFORM_VERSION=
-RUN case ${TARGETPLATFORM} in \
-    "linux/amd64") CMAKE_PLATFORM_VERSION="x86_64";; \
-    "linux/arm64") CMAKE_PLATFORM_VERSION="aarch64";; \
-    *) \
-    test -n "$TARGETPLATFORM" \
-    echo "Unsupported platform ${TARGETPLATFORM}" \
-    && return 1 ;\
-    ;; \
-    esac \
-    && set -x \
-    && (cd /tmp \
-    && wget --progress=dot:giga https://github.com/Kitware/CMake/releases/download/v3.23.1/cmake-3.23.1-Linux-$CMAKE_PLATFORM_VERSION.sh \
-    && sh cmake-3.23.1-Linux-$CMAKE_PLATFORM_VERSION.sh --exclude-subdir --prefix=/usr/local \
-    && rm -rf cmake-3.23.1-Linux-$CMAKE_PLATFORM_VERSION.sh) \
-    && exec bash \
-    && : # last line
-
-# Python 3.9 and PIP
-RUN set -x \
-    && DEBIAN_FRONTEND=noninteractive  apt-get update \
-    && DEBIAN_FRONTEND=noninteractive  apt-get install -y libgirepository1.0-dev \
-    && DEBIAN_FRONTEND=noninteractive apt-get install -y software-properties-common \
-    && add-apt-repository universe \
-    && curl https://bootstrap.pypa.io/get-pip.py -o get-pip.py \
-    && python3.9 get-pip.py \
-    && update-alternatives --install /usr/bin/python3 python3 /usr/bin/python3.9 1 \
-    && rm -rf /var/lib/apt/lists/ \
-    && : # last line
-
+# NOTE: this install is NOT ideal as we are installing over system
+#       we should migrate to venvs for packages we do not directly have
+#       provided by the system
 RUN set -x \
-    && pip3 install --no-cache-dir \
+    && pip3 install --break-system-packages --no-cache-dir --break-system-packages \
     attrs \
-    click \
-    coloredlogs \
     cxxfilt \
-    future \
     ghapi \
     mobly \
-    pandas \
     portpicker \
     pygit \
     PyGithub \
     ruff \
-    tabulate \
-    && : # last line
-
-# build and install gn
-RUN set -x \
-    && git clone https://gn.googlesource.com/gn \
-    && cd gn \
-    && python3 build/gen.py \
-    && ninja -C out \
-    && cp out/gn /usr/local/bin \
-    && cd .. \
-    && rm -rf gn \
     && : # last line
 
 # Install bloat comparison tools
@@ -172,38 +129,13 @@ RUN set -x \
     && mkdir -p bloaty/build \
     && cd bloaty/build \
     && cmake -DCMAKE_BUILD_TYPE=MinSizeRel ../ \
-    && make -j8 \
+    && make -j$(nproc) \
     && strip bloaty \
     && make install \
     && cd ../.. \
     && rm -rf bloaty \
     && : # last line
 
-# Need newer version of include-what-you-use
-RUN set -x \
-    && apt-get update \
-    # Install build and runtime requirements for IWYU
-    && DEBIAN_FRONTEND=noninteractive apt-get install -fy --fix-missing clang-12 libclang-12-dev \
-    # Build and install IWYU
-    && git clone --depth=1 --branch=clang_12 https://github.com/include-what-you-use/include-what-you-use.git \
-    && mkdir -p include-what-you-use/build \
-    && cd include-what-you-use/build \
-    && cmake -DCMAKE_BUILD_TYPE=MinSizeRel -DCMAKE_INSTALL_PREFIX=/usr -DCMAKE_PREFIX_PATH=/usr/lib/llvm-12 -DIWYU_LINK_CLANG_DYLIB=OFF .. \
-    && make -j8 \
-    && strip bin/include-what-you-use \
-    && make install \
-    # Save clang-12 files, so we can restore them after build dependencies cleanup
-    && tar -cf clang-12-files.tar $(dpkg -L libclang-common-12-dev |grep /include) /usr/lib/llvm-12/lib/libLLVM-12.so.1 \
-    # Cleanup build dependencies
-    && apt autopurge -fy clang-12 libclang-12-dev \
-    && rm -rf /var/lib/apt/lists/ \
-    # Restore clang-12 files
-    && tar -xf clang-12-files.tar -C / \
-    # Cleanup
-    && cd ../.. \
-    && rm -rf include-what-you-use \
-    && : # last line
-
 # Build glib-2.0 from source with enabled thread sanitizer. This is needed for
 # running CHIP tests with TSAN enabled. When running applications with TSAN
 # all shared libraries should be built with TSAN enabled, otherwise TSAN might
@@ -219,7 +151,7 @@ RUN case ${TARGETPLATFORM} in \
     && git clone --depth=1 --branch=$GLIB_VERSION https://github.com/GNOME/glib.git \
     && CFLAGS="-O2 -g -fsanitize=thread" meson glib/build glib -Dtests=false \
     && DESTDIR=../build-image ninja -C glib/build install \
-    && mv glib/build-image/usr/local/lib/x86_64-linux-gnu/lib* $LD_LIBRARY_PATH_TSAN \
+    && mv glib/build-image/usr/local/lib64/lib* $LD_LIBRARY_PATH_TSAN \
     && rm -rf glib \
     ;; \
     "linux/arm64") \
@@ -231,41 +163,6 @@ RUN case ${TARGETPLATFORM} in \
     ;; \
     esac
 
-# NodeJS: install a newer version than what apt-get would read
-# This installs the latest LTS version of nodejs
-#
-# NodeJS is required by github actions, we use Wandalen/[email protected]
-# and that seems to use the built-in node installation in the image
-#
-# This is not a CHIP dependency directly, but used by CI
-ENV CHIP_NODE_VERSION=v16.13.2
-ENV NODE_PLATFORM_VERSION=
-RUN case ${TARGETPLATFORM} in \
-    "linux/amd64") NODE_PLATFORM_VERSION=x64;; \
-    "linux/arm64") NODE_PLATFORM_VERSION=arm64;; \
-    *) \
-    test -n "$TARGETPLATFORM" \
-    echo "Unsupported platform ${TARGETPLATFORM}" \
-    && return 1 ;\
-    ;; \
-    esac \
-    && set -x \
-    && mkdir node_js \
-    && cd node_js \
-    && wget https://nodejs.org/dist/$CHIP_NODE_VERSION/node-$CHIP_NODE_VERSION-linux-$NODE_PLATFORM_VERSION.tar.xz \
-    && tar xfvJ node-$CHIP_NODE_VERSION-linux-$NODE_PLATFORM_VERSION.tar.xz \
-    && mv node-$CHIP_NODE_VERSION-linux-$NODE_PLATFORM_VERSION /opt/ \
-    && ln -s /opt/node-$CHIP_NODE_VERSION-linux-$NODE_PLATFORM_VERSION /opt/node \
-    && ln -s /opt/node/bin/* /usr/bin \
-    && cd .. \
-    && rm -rf node_js \
-    && : # last line
-
 # Some things that save space
 # Protoc goes from 108M to 4.6M
 RUN strip /usr/local/bin/protoc*
-
-# CMake documentation not needed, saves 34MB
-# /usr/local/man contains cmake documentation
-RUN rm -rf /usr/local/doc/cmake
-RUN rm -rf /usr/local/man

integrations/docker/images/base/chip-build/version

@@ -1 +1 @@
-54 : [Telink] Update Docker image (Zephyr update)
+55 : Update to Ubuntu 24.04 as the base build image

integrations/docker/images/chip-cert-bins/Dockerfile

@@ -1,5 +1,5 @@
 # Stage 1: Setup dependencies (based on chip-build).
-FROM ubuntu:22.04 as chip-build-cert
+FROM ubuntu:24.04 as chip-build-cert
 LABEL org.opencontainers.image.source https://github.com/project-chip/connectedhomeip
 ARG TARGETPLATFORM
 # COMMITHASH defines the target commit to build from. May be passed in using --build-arg.
@@ -292,12 +292,12 @@ COPY --from=chip-build-cert-bins /root/connectedhomeip/out/python_env python_env
 COPY --from=chip-build-cert-bins /root/connectedhomeip/src/python_testing python_testing
 
 COPY --from=chip-build-cert-bins /root/connectedhomeip/scripts/tests/requirements.txt /tmp/requirements.txt
-RUN pip install -r /tmp/requirements.txt && rm /tmp/requirements.txt
+RUN pip install --break-system-packages -r /tmp/requirements.txt && rm /tmp/requirements.txt
 
 COPY --from=chip-build-cert-bins /root/connectedhomeip/src/python_testing/requirements.txt /tmp/requirements.txt
-RUN pip install -r /tmp/requirements.txt && rm /tmp/requirements.txt
+RUN pip install --break-system-packages -r /tmp/requirements.txt && rm /tmp/requirements.txt
 
 # PIP requires MASON package compilation, which seems to require a JDK
 RUN set -x && DEBIAN_FRONTEND=noninteractive apt-get install -fy openjdk-8-jdk
 
-RUN pip install --no-cache-dir python_lib/controller/python/chip*.whl
+RUN pip install --break-system-packages --no-cache-dir python_lib/controller/python/chip*.whl

integrations/docker/images/stage-1/chip-build-crosscompile/Dockerfile

@@ -6,6 +6,7 @@ RUN set -x \
     && apt-get update \
     && DEBIAN_FRONTEND=noninteractive apt-get install -fy --no-install-recommends \
     git \
+    xz-utils \
     && apt-get clean \
     && rm -rf /var/lib/apt/lists/ \
     && : # last line

integrations/docker/images/stage-2/chip-build-ameba/Dockerfile

@@ -2,6 +2,13 @@ ARG VERSION=1
 FROM ghcr.io/project-chip/chip-build:${VERSION}
 LABEL org.opencontainers.image.source https://github.com/project-chip/connectedhomeip
 
+RUN set -x \
+    && apt-get update \
+    && DEBIAN_FRONTEND=noninteractive apt-get install -fy --no-install-recommends \
+    bzip2 \
+    && rm -rf /var/lib/apt/lists/ \
+    && : # last line
+
 # Setup Ameba
 ARG AMEBA_DIR=/opt/ameba
 ARG TAG_NAME=ameba_update_2024_03_22

integrations/docker/images/stage-2/chip-build-asr/Dockerfile

@@ -5,6 +5,8 @@ LABEL org.opencontainers.image.source https://github.com/project-chip/connectedh
 RUN set -x \
     && apt-get update \
     && DEBIAN_FRONTEND=noninteractive apt-get install -fy --no-install-recommends \
+    bzip2 \
+    tar \
     wget \
     && apt-get clean \
     && rm -rf /var/lib/apt/lists/ \