Enable buildx support in CICO scripts

Signed-off-by: Shahid Shaikh <[email protected]>

Correctional changes

Signed-off-by: Shahid Shaikh <[email protected]>

Testing changes

Signed-off-by: Shahid Shaikh <[email protected]>

build.include

@@ -47,6 +47,31 @@ parse() {
   done
 }
 
+# Sorts two versions and returns the lower version to check if its supported for using docker buildx.
+check_supported_version() {
+  local query=$1
+  local target=$2
+  echo "$target" "$query" | tr ' ' '\n' | sort -V | head -n1 2> /dev/null
+}
+
+# Checks whether docker and kernel versions are greater than or equal to the specified version, such that docker buildx requirements are met.
+# Returns 0 when buildx support is available, returns 1 if its not available.
+check_buildx_support() {
+  docker_version="$(docker --version | cut -d' ' -f3 | tr -cd '0-9.')"
+  if [[ $(check_supported_version "$docker_version" "19.03") != 19.03 ]]; then
+    echo "CICO: Docker $docker_version greater than or equal to 19.03 is required."
+    return 1
+  else
+    # Kernel
+    kernel_version="$(uname -r)"
+    if [[ $(check_supported_version "$kernel_version" "4.8") != "4.8" ]]; then
+      return 1
+    else
+      return 0
+    fi
+  fi
+}
+
 is_publish_images() {
   if [[ "${PUBLISH_IMAGES}" == "true" ]]; then
     return 0
@@ -73,6 +98,12 @@ getImages() {
 }
 
 buildImages() {
+  if [[ -n "${THEIA_DOCKER_IMAGE_VERSION}" ]]; then
+    export THEIA_DOCKER_IMAGE_VERSION=""
+  else
+    export THEIA_DOCKER_IMAGE_VERSION=${THEIA_DOCKER_IMAGE_VERSION}
+  fi
+
   IFS=" " read -r -a IMG_LIST <<< "$(getImages)"
   for image_dir in "${IMG_LIST[@]}"
   do

build.sh

@@ -15,6 +15,8 @@ set -o pipefail
 parse "$@"
 yarn ${YARN_OPTS}
 
+export BUILDX=0
+
 buildImages
 
 if is_publish_images; then

cico_build_master.sh

@@ -29,8 +29,14 @@ install_deps
 set +x
 load_jenkins_vars
 set -x
-buildImages
-publishImagesOnQuay
+export BUILDX=0
+if [[ $(check_buildx_support; echo $?) -eq 0 ]]; then
+  export BUILDX=1
+  buildImages
+else
+  buildImages
+  publishImagesOnQuay
+fi
 
 set +x
 # Release npm packages

cico_build_pr.sh

@@ -29,4 +29,5 @@ sed -i -e 's/IMAGE_TAG="..*"/IMAGE_TAG="'${GIT_COMMIT}'"/' build.include
 
 parse "$@"
 
+export BUILDX=0
 buildImages

cico_common.sh

@@ -47,6 +47,13 @@ function install_deps() {
   yum install -y docker-ce git nodejs yarn gcc-c++ make jq
 
   service docker start
+  #Set buildx environment variables
+  export DOCKER_BUILD_KIT=1
+  export DOCKER_CLI_EXPERIMENTAL=enabled
+  #Enable qemu and binfmt support
+  docker run --rm --privileged docker/binfmt:a7996909642ee92942dcd6cff44b9b95f08dad64
+  docker run --rm --privileged multiarch/qemu-user-static:4.2.0-7 --reset -p yes
+
   echo 'CICO: Dependencies installed'
 }
 

dockerfiles/build.include

@@ -117,6 +117,7 @@ init() {
   done
 
   IMAGE_NAME="$ORGANIZATION/$PREFIX-$NAME:$TAG"
+  IMAGE_NAME_NOTAG="$ORGANIZATION/$PREFIX-$NAME"
 }
 
 build() {
@@ -174,6 +175,13 @@ build_image() {
   content_docker=$(cat ${DIR}/${DOCKERFILE})
   update_macros "${content_docker}" > ${DIR}/.Dockerfile
 
+  if [ "${BUILDX}" == "1" ]; then
+    filename_to_check="${DIR}/docker/alpine/builder-from.dockerfile-multiarch"
+    if [ -f "${filename_to_check}" ]; then
+       sed -i 's|builder-from.dockerfile|builder-from.dockerfile-multiarch|g' ${DIR}/.Dockerfile
+    fi
+  fi
+
   # apply IF
   if_patterns=$(sed -n 's/.*\#{IF:\(.*\)\}/\1/p' ${DIR}/.Dockerfile)
    echo "$if_patterns" | while IFS= read -r conditional_arg ; do
@@ -217,25 +225,47 @@ build_image() {
     fi
   done
 
+  if [ "${BUILDX}" == "1" ]; then
+    REGISTRY="quay.io"
+    QUAY_USERNAME=${QUAY_ECLIPSE_CHE_USERNAME}
+    QUAY_PASSWORD=${QUAY_ECLIPSE_CHE_PASSWORD}
+    if [ -n "${QUAY_USERNAME}" ] && [ -n "${QUAY_PASSWORD}" ]; then
+      docker login -u "${QUAY_USERNAME}" -p "${QUAY_PASSWORD}" "${REGISTRY}"
+    else
+      echo "Could not login, missing credentials for pushing to the '${ORGANIZATION}' organization"
+      return
+    fi
+  fi
+
   if ! dry_run; then
-    cd "${DIR}" && docker build --cache-from ${IMAGE_NAME} -f ${DIR}/.Dockerfile -t ${IMAGE_NAME} ${BUILD_ARGS} ${DOCKER_BUILD_TARGET} .
+    if [ "${BUILDX}" == "1" ]; then
+        if [[ -n "${THEIA_DOCKER_IMAGE_VERSION}" ]]; then
+          cd "${DIR}" && docker buildx build --cache-from ${IMAGE_NAME} --platform linux/amd64,linux/s390x --file ${DIR}/.Dockerfile --output "type=image,push=true" --tag quay.io/${IMAGE_NAME_NOTAG}:${THEIA_DOCKER_IMAGE_VERSION} --tag quay.io/${IMAGE_NAME} --progress=plain .
+        else
+          cd "${DIR}" && docker buildx build --cache-from ${IMAGE_NAME} --platform linux/amd64,linux/s390x --file ${DIR}/.Dockerfile --output "type=image,push=true" --tag quay.io/${IMAGE_NAME} --progress=plain .
+        fi
+    else
+        cd "${DIR}" && docker build --cache-from ${IMAGE_NAME} -f ${DIR}/.Dockerfile -t ${IMAGE_NAME} ${BUILD_ARGS} ${DOCKER_BUILD_TARGET} .
+    fi
     rm ${DIR}/.Dockerfile
   fi
 
   if [ $? -eq 0 ]; then
     printf "Build of ${BLUE}${IMAGE_NAME} ${GREEN}[OK]${NC}\n"
-    if [ ! -z "${IMAGE_ALIASES}" ]; then
-      for TMP_IMAGE_NAME in ${IMAGE_ALIASES}
-      do
-        docker tag ${IMAGE_NAME} ${TMP_IMAGE_NAME}:${TAG}
-        if [ $? -eq 0 ]; then
-          printf "  /alias ${BLUE}${TMP_IMAGE_NAME}:${TAG}${NC} ${GREEN}[OK]${NC}\n"
-        else
-          printf "${RED}Failure when building docker image ${IMAGE_NAME}${NC}\n"
-          exit 1
-        fi
+    if [ "${BUILDX}" != "1" ]; then
+      if [ ! -z "${IMAGE_ALIASES}" ]; then
+        for TMP_IMAGE_NAME in ${IMAGE_ALIASES}
+        do
+          docker tag ${IMAGE_NAME} ${TMP_IMAGE_NAME}:${TAG}
+          if [ $? -eq 0 ]; then
+            printf "  /alias ${BLUE}${TMP_IMAGE_NAME}:${TAG}${NC} ${GREEN}[OK]${NC}\n"
+          else
+            printf "${RED}Failure when building docker image ${IMAGE_NAME}${NC}\n"
+            exit 1
+          fi
 
-      done
+        done
+      fi
     fi
     printf "${GREEN}Script run successfully: ${BLUE}${IMAGE_NAME}${NC}\n"
   else

dockerfiles/theia-endpoint-runtime-binary/docker/alpine/builder-from.dockerfile-multiarch

@@ -0,0 +1,2 @@
+FROM quay.io/eclipse/che-custom-nodejs-deasync:10.20.1 as custom-nodejs
+FROM quay.io/${BUILD_ORGANIZATION}/${BUILD_PREFIX}-theia:${BUILD_TAG} as builder

dockerfiles/theia-endpoint-runtime-binary/docker/ubi8/builder-from.dockerfile-multiarch

@@ -0,0 +1,2 @@
+FROM quay.io/eclipse/che-custom-nodejs-deasync:10.20.1 as custom-nodejs
+FROM quay.io/${BUILD_ORGANIZATION}/${BUILD_PREFIX}-theia:${BUILD_TAG} as builder

dockerfiles/theia/build.sh

@@ -39,8 +39,12 @@ fi
 
 build
 
-if ! skip_tests; then
-  bash "${base_dir}"/e2e/build.sh "$PREFIX-$NAME" "$@"
+if [ "${BUILDX}" != "1" ]; then
+  if ! skip_tests; then
+    bash "${base_dir}"/e2e/build.sh "$PREFIX-$NAME" "$@"
+  fi
+else
+  IMAGE_NAME="quay.io/${IMAGE_NAME}"
 fi
 
 if [[ -z "$DOCKER_BUILD_TARGET" ]]; then

dockerfiles/theia/docker/alpine/builder-from.dockerfile-multiarch

@@ -0,0 +1 @@
+FROM quay.io/${BUILD_ORGANIZATION}/${BUILD_PREFIX}-theia-dev:${BUILD_TAG} as builder

dockerfiles/theia/docker/ubi8/builder-from.dockerfile-multiarch

@@ -0,0 +1 @@
+FROM quay.io/${BUILD_ORGANIZATION}/${BUILD_PREFIX}-theia-dev:${BUILD_TAG} as builder