CRW-579 add three new scripts for adding digests, building map of dig…

…ests in registries, and introspecting containers (including scratch ones) to extract files (#202)

* add three new scripts for adding digests, building map of digests in registries, and introspecting containers (including scratch ones) to extract files

Change-Id: I12c280e867c7a105a52385253a298bf9c339a8de
Signed-off-by: nickboldt <[email protected]>

* shell scripts should be +x

Change-Id: I2a9c3ef028fd5e094cb247dfc85e2db3895b69a8
Signed-off-by: nickboldt <[email protected]>

* Check if podmand and yq are installed

* switch to docker by default olm/dockerContainerExtract.sh

* Use --wildcards

Co-authored-by: Anatolii Bazko <[email protected]>

olm/addDigests.sh

@@ -0,0 +1,67 @@
+#!/bin/bash
+#
+# Copyright (c) 2019 Red Hat, Inc.
+# This program and the accompanying materials are made
+# available under the terms of the Eclipse Public License 2.0
+# which is available at https://www.eclipse.org/legal/epl-2.0/
+#
+# SPDX-License-Identifier: EPL-2.0
+#
+# Contributors:
+#   Red Hat, Inc. - initial API and implementation
+
+SCRIPTS_DIR=$(cd "$(dirname "$0")"; pwd)
+BASE_DIR="$(pwd)"
+command -v podman >/dev/null 2>&1 || { echo "podman is not installed. Aborting."; exit 1; }
+command -v yq >/dev/null 2>&1 || { echo "yq is not installed. Aborting."; exit 1; }
+usage () {
+	echo "Usage:   $0 [-w WORKDIR] -s [SOURCE_PATH] -n [csv name] -v [VERSION] "
+	echo "Example: $0 -w $(pwd) -s eclipse-che-preview-openshift/deploy/olm-catalog/eclipse-che-preview-openshift -n eclipse-che-preview-openshift -v 7.9.0"
+	echo "Example: $0 -w $(pwd) -s controller-manifests -n codeready-workspaces -v 2.1.0"
+}
+
+if [[ $# -lt 1 ]]; then usage; exit; fi
+
+while [[ "$#" -gt 0 ]]; do
+  case $1 in
+    '-w') BASE_DIR="$2"; shift 1;;
+    '-s') SRC_DIR="$2"; shift 1;;
+    '-n') CSV_NAME="$2"; shift 1;;
+    '-v') VERSION="$2"; shift 1;;
+	'--help'|'-h') usage; exit;;
+  esac
+  shift 1
+done
+
+if [[ ! $SRC_DIR ]] || [[ ! $CSV_NAME ]] || [[ ! $VERSION ]]; then usage; exit 1; fi
+
+rm -Rf ${BASE_DIR}/generated/${CSV_NAME}/
+mkdir -p ${BASE_DIR}/generated/${CSV_NAME}/
+cp -R ${BASE_DIR}/${SRC_DIR}/* ${BASE_DIR}/generated/${CSV_NAME}/
+
+CSV_FILE="$(find ${BASE_DIR}/generated/${CSV_NAME}/*${VERSION}/ -name "${CSV_NAME}.*${VERSION}.clusterserviceversion.yaml" | tail -1)"; # echo "[INFO] CSV = ${CSV_FILE}"
+${SCRIPTS_DIR}/buildDigestMap.sh -w ${BASE_DIR} -c ${CSV_FILE}
+
+names=" "
+count=1
+RELATED_IMAGES='. * { spec : { relatedImages: [ '
+for mapping in $(cat ${BASE_DIR}/generated/digests-mapping.txt)
+do
+  source=$(echo "${mapping}" | sed -e 's/\(.*\)=.*/\1/')
+  dest=$(echo "${mapping}" | sed -e 's/.*=\(.*\)/\1/')
+  sed -i -e "s;${source};${dest};" ${CSV_FILE}
+  name=$(echo "${dest}" | sed -e 's;.*/\([^\/][^\/]*\)@.*;\1;')
+  nameWithSpaces=" ${name} "
+  if [[ "${names}" == *${nameWithSpaces}* ]]; then
+    name="${name}-${count}"
+    count=$(($count+1))
+  fi
+  if [ "${names}" != " " ]; then
+    RELATED_IMAGES="${RELATED_IMAGES},"
+  fi
+  RELATED_IMAGES="${RELATED_IMAGES} { name: \"${name}\", image: \"${dest}\"}"
+  names="${names} ${name} "
+done
+RELATED_IMAGES="${RELATED_IMAGES} ] } }"
+mv ${CSV_FILE} ${CSV_FILE}.old
+yq -Y "$RELATED_IMAGES" ${CSV_FILE}.old > ${CSV_FILE}

olm/buildDigestMap.sh

@@ -0,0 +1,80 @@
+#!/bin/bash
+#
+# Copyright (c) 2019 Red Hat, Inc.
+# This program and the accompanying materials are made
+# available under the terms of the Eclipse Public License 2.0
+# which is available at https://www.eclipse.org/legal/epl-2.0/
+#
+# SPDX-License-Identifier: EPL-2.0
+#
+# Contributors:
+#   Red Hat, Inc. - initial API and implementation
+
+SCRIPTS_DIR=$(cd "$(dirname "$0")"; pwd)
+BASE_DIR="$1"
+
+usage () {
+	echo "Usage:   $0 [-w WORKDIR] -c [/path/to/csv.yaml] "
+	echo "Example: $0 -w $(pwd) -c  $(pwd)/generated/eclipse-che-preview-openshift/7.9.0/eclipse-che-preview-openshift.v7.9.0.clusterserviceversion.yaml"
+}
+
+if [[ $# -lt 1 ]]; then usage; exit; fi
+
+while [[ "$#" -gt 0 ]]; do
+  case $1 in
+    '-w') BASE_DIR="$2"; shift 1;;
+    '-c') CSV="$2"; shift 1;;
+	'--help'|'-h') usage; exit;;
+  esac
+  shift 1
+done
+
+if [[ ! $CSV ]]; then usage; exit 1; fi
+
+mkdir -p ${BASE_DIR}/generated
+
+echo "[INFO] Get images from CSV ${CSV}"
+
+IMAGE_LIST=$(yq -r '.spec.install.spec.deployments[].spec.template.spec.containers[].env[] | select(.name | test("IMAGE_default_.*"; "g")) | .value' "${CSV}")
+OPERATOR_IMAGE=$(yq -r '.spec.install.spec.deployments[].spec.template.spec.containers[].image' "${CSV}")
+
+REGISTRY_LIST=$(yq -r '.spec.install.spec.deployments[].spec.template.spec.containers[].env[] | select(.name | test("IMAGE_default_.*_registry"; "g")) | .value' "${CSV}")
+REGISTRY_IMAGES=""
+for registry in ${REGISTRY_LIST}; do
+  extracted=$(${SCRIPTS_DIR}/dockerContainerExtract.sh ${registry} var/www/html/*/external_images.txt | tail -n 1)
+
+  # Container quay.io/eclipse/che-devfile-registry:7.9.0 unpacked to /tmp/quay.io-eclipse-che-devfile-registry-7.9.0-1584588272
+  extracted=${extracted##* } # the last token in the above line is the path we want
+
+  echo -n "[INFO] Extract images from registry ${registry} ... "
+  if [[ -d ${extracted} ]]; then
+    # cat ${extracted}/var/www/html/*/external_images.txt
+    REGISTRY_IMAGES="${REGISTRY_IMAGES} $(cat ${extracted}/var/www/html/*/external_images.txt)"
+  fi
+  echo "found $(cat ${extracted}/var/www/html/*/external_images.txt | wc -l)"
+  rm -fr ${extracted} 2>&1 >/dev/null
+done
+
+rm -Rf ${BASE_DIR}/generated/digests-mapping.txt
+touch ${BASE_DIR}/generated/digests-mapping.txt
+for image in ${OPERATOR_IMAGE} ${IMAGE_LIST} ${REGISTRY_IMAGES}; do
+  case ${image} in
+    *@sha256:*)
+      withDigest="${image}";;
+    *@)
+      continue;;
+    *)
+      echo "[INFO] Get digest from ${image}"
+      digest="$(skopeo inspect docker://${image} | jq -r '.Digest')"
+      withoutTag="$(echo "${image}" | sed -e 's/^\(.*\):[^:]*$/\1/')"
+      withDigest="${withoutTag}@${digest}";;
+  esac
+  dots="${withDigest//[^\.]}"
+  separators="${withDigest//[^\/]}"
+  if [ "${#separators}" == "1" ] && [ "${#dots}" == "0" ]; then
+    echo "[WARN] Add 'docker.io/' prefix to $image"
+    withDigest="docker.io/${withDigest}"
+  fi
+
+  echo "${image}=${withDigest}" >> ${BASE_DIR}/generated/digests-mapping.txt
+done

olm/dockerContainerExtract.sh

@@ -0,0 +1,37 @@
+#!/bin/bash
+set -e +x
+
+if [[ ! $1 ]]; then
+  echo "Usage: $0 CONTAINER [tar-extraction-flags]"
+  echo "Usage: $0 quay.io/crw/operator-metadata:latest"
+  echo "Usage: $0 quay.io/crw/devfileregistry-rhel8:latest var/www/html/*/external_images.txt"
+  echo "Usage: $0 quay.io/crw/pluginregistry-rhel8:latest var/www/html/*/external_images.txt"
+  exit
+fi
+
+PODMAN=docker # or user podman
+
+container="$1"; shift 1
+tmpcontainer="$(echo $container | tr "/:" "--")-$(date +%s)"
+unpackdir="/tmp/${tmpcontainer}"
+
+# get remote image
+echo "[INFO] Pulling $container ..."
+${PODMAN} pull $container 2>&1 >/dev/null
+
+# create local container
+${PODMAN} rm -f "${tmpcontainer}" 2>&1 >/dev/null || true
+# use sh for regular containers or ls for scratch containers
+${PODMAN} create --name="${tmpcontainer}" $container sh 2>&1 >/dev/null || ${PODMAN} create --name="${tmpcontainer}" $container ls 2>&1 >/dev/null 
+
+# export and unpack
+${PODMAN} export "${tmpcontainer}" > /tmp/${tmpcontainer}.tar
+rm -fr "$unpackdir"; mkdir -p "$unpackdir"
+echo "[INFO] Extract from container ..."
+tar xf /tmp/${tmpcontainer}.tar --wildcards -C "$unpackdir" $*
+
+# cleanup
+${PODMAN} rm -f "${tmpcontainer}" 2>&1 >/dev/null || true
+rm -fr  /tmp/${tmpcontainer}.tar
+
+echo "[INFO] Container $container unpacked to $unpackdir"