Auto download test-images and debug-images (#1692)

- add auto download test-images and debug-images - to simplify the story, auto download will only be triggered if users provide only one CUSTOMIZED_SDK_URL and URL filename contains OpenJ9-JDK. In this way, if users want to provide multiple separate URLs or multiple separate URLs from separate servers, they can still do so and auto-download will not be triggered. - Once auto-download is triggered, by default, the flag test_images_required and debug_images_required are set to true in get.sh. If the flag is set to true, it means the script will retry if the download fails and the program will fail if all retry attempts fail (the same behavior as downloading SDK). If the flag is set to false, it means the script will not try to auto download. - In JenkinsfileBase, pass in test_images_required and debug_images_required to get.sh if users want to overwrite the flag(s) - During unzipping process, if debug-images jar is found, overlay debug-images on top of sdk j2sdk-image/jre dir if it exists. Otherwise, extracts into j2sdk-image dir Issue: eclipse-openj9/openj9#8874 Signed-off-by: lanxia <[email protected]>
adoptium · Mar 30, 2020 · e9d03a7 · e9d03a7
1 parent f449885
commit e9d03a7
Show file tree

Hide file tree

Showing 2 changed files with 111 additions and 35 deletions.
diff --git a/buildenv/jenkins/JenkinsfileBase b/buildenv/jenkins/JenkinsfileBase
@@ -276,7 +276,12 @@ def setup() {
 			VENDOR_TEST_BRANCHES = (params.VENDOR_TEST_BRANCHES) ? "--vendor_branches \"${params.VENDOR_TEST_BRANCHES}\"" : ""
 			VENDOR_TEST_DIRS = (params.VENDOR_TEST_DIRS) ? "--vendor_dirs \"${params.VENDOR_TEST_DIRS}\"" : ""
 			VENDOR_TEST_SHAS = (params.VENDOR_TEST_SHAS) ? "--vendor_shas \"${params.VENDOR_TEST_SHAS}\"" : ""
-			GET_SH_CMD = "./openjdk-tests/get.sh -s `pwd` -t `pwd`/openjdk-tests -p $PLATFORM -r ${SDK_RESOURCE} ${JDK_VERSION_OPTION} ${JDK_IMPL_OPTION} ${CUSTOMIZED_SDK_URL_OPTION} ${CUSTOMIZED_SDK_SOURCE_URL_OPTION} ${CLONE_OPENJ9_OPTION} ${OPENJ9_REPO_OPTION} ${OPENJ9_BRANCH_OPTION} ${OPENJ9_SHA_OPTION} ${TKG_REPO_OPTION} ${TKG_BRANCH_OPTION} ${TKG_SHA_OPTION} ${VENDOR_TEST_REPOS} ${VENDOR_TEST_BRANCHES} ${VENDOR_TEST_DIRS} ${VENDOR_TEST_SHAS}"
+
+			// handle three cases (true/false/null) in params.TEST_IMAGES_REQUIRED and params.DEBUG_IMAGES_REQUIRED
+			// Only set image required to false if params is set to false. In get.sh, the default value is true
+			TEST_IMAGES_REQUIRED = (params.TEST_IMAGES_REQUIRED == false) ? "--test_images_required false" : ""
+			DEBUG_IMAGES_REQUIRED = (params.DEBUG_IMAGES_REQUIRED == false) ? "--debug_images_required false" : ""
+			GET_SH_CMD = "./openjdk-tests/get.sh -s `pwd` -t `pwd`/openjdk-tests -p $PLATFORM -r ${SDK_RESOURCE} ${JDK_VERSION_OPTION} ${JDK_IMPL_OPTION} ${CUSTOMIZED_SDK_URL_OPTION} ${CUSTOMIZED_SDK_SOURCE_URL_OPTION} ${CLONE_OPENJ9_OPTION} ${OPENJ9_REPO_OPTION} ${OPENJ9_BRANCH_OPTION} ${OPENJ9_SHA_OPTION} ${TKG_REPO_OPTION} ${TKG_BRANCH_OPTION} ${TKG_SHA_OPTION} ${VENDOR_TEST_REPOS} ${VENDOR_TEST_BRANCHES} ${VENDOR_TEST_DIRS} ${VENDOR_TEST_SHAS} ${TEST_IMAGES_REQUIRED} ${DEBUG_IMAGES_REQUIRED}"
 
 			dir( WORKSPACE) {
 				// use sshagent with Jenkins credentials ID for all platforms except zOS

diff --git a/get.sh b/get.sh
@@ -36,7 +36,8 @@ JDK_VERSION="8"
 JDK_IMPL="openj9"
 RELEASES="latest"
 TYPE="jdk"
-
+TEST_IMAGES_REQUIRED=true
+DEBUG_IMAGES_REQUIRED=true
 
 usage ()
 {
@@ -140,6 +141,12 @@ parseCommandLineArgs()
 			"--vendor_dirs" )
 				VENDOR_DIRS="$1"; shift;;
 
+			"--test_images_required" )
+				TEST_IMAGES_REQUIRED="$1"; shift;;
+
+			"--debug_images_required" )
+				DEBUG_IMAGES_REQUIRED="$1"; shift;;
+
 			"--help" | "-h" )
 				usage; exit 0;;
 
@@ -177,6 +184,24 @@ getBinaryOpenjdk()
 		if [ "$USERNAME" != "" ] && [ "$PASSWORD" != "" ]; then
 			curl_options="--user $USERNAME:$PASSWORD"
 		fi
+		images="test-images.tar.gz debug-image.tar.gz"
+		download_urls=($download_url)
+		# for now, auto-download is enabled only if users provide one URL and filename contains OpenJ9-JDK
+		if [[ "${#download_urls[@]}" == 1 ]]; then
+			download_filename=${download_url##*/}
+			if [[ "$download_filename" =~ "OpenJ9-JDK" ]]; then
+				link=${download_url%$download_filename}
+				for image in $images
+				do
+					required=true
+					checkURL "$image"
+					if [[ $required != false ]]; then
+						download_url+=" ${link}${image}"
+						echo "auto download: ${link}${image}"
+					fi
+				done
+			fi
+		fi
 	elif [ "$SDK_RESOURCE" == "nightly" ] || [ "$SDK_RESOURCE" == "releases" ]; then
 		os=${PLATFORM#*_}
 		os=${os%_xl}
@@ -219,7 +244,7 @@ getBinaryOpenjdk()
 					sleep $sleep_time
 
 					download_filename=${file##*/}
-					echo "check for $download_filename..."
+					echo "check for $download_filename. If found, the file will be removed."
 					if [ -f "$download_filename" ]; then
 						echo "remove $download_filename before retry..."
 						rm $download_filename
@@ -257,51 +282,97 @@ getBinaryOpenjdk()
 
 	jar_files=`ls`
 	jar_file_array=(${jar_files//\\n/ })
-	for jar_name in "${jar_file_array[@]}"
-		do
-			if [ -d "$SDKDIR/openjdkbinary/tmp" ]; then
-				rm -rf $SDKDIR/openjdkbinary/tmp/*
-			else
-				mkdir $SDKDIR/openjdkbinary/tmp
+
+	# if $jar_file_array contains debug-image, move debug-image element to the end of the array
+	# debug image jar needs to be extracted after jdk as debug image jar extraction location depends on jdk structure
+	# debug image jar extracts into j2sdk-image/jre dir if it exists. Otherwise, extracts into j2sdk-image dir
+	if [[ $DEBUG_IMAGES_REQUIRED = true ]]; then
+		last_index=$(( ${#jar_file_array[@]} -1 ))
+		for i in "${!jar_file_array[@]}"; do
+			if [[ "${jar_file_array[$i]}" =~ "debug-image" ]]; then
+				if [[ $i -ne $last_index ]]; then
+					debug_image_jar="${jar_file_array[$i]}"
+
+					#remove the element
+					unset jar_file_array[$i]
+
+					# add $debug_image_jar to the end of the array
+					jar_file_array=( "${jar_file_array[@]}" "${debug_image_jar}" )
+					break
+				fi
 			fi
+		done
+	fi
 
-			echo "unzip file: $jar_name ..."
-			if [[ $jar_name == *zip || $jar_name == *jar ]]; then
-				unzip -q $jar_name -d ./tmp
+	for jar_name in "${jar_file_array[@]}"
+		do
+			# if jar_name contains debug-image, extract into j2sdk-image/jre or j2sdk-image dir
+			# Otherwise, files will be extracted under ./tmp
+			if [[ "$jar_name"  =~ "debug-image" ]]; then
+				extract_dir="./j2sdk-image"
+				if [ -d "$SDKDIR/openjdkbinary/j2sdk-image/jre" ]; then
+					extract_dir="./j2sdk-image/jre"
+				fi
+				echo "unzip $jar_name in $extract_dir..."
+				if [[ $jar_name == *zip || $jar_name == *jar ]]; then
+					unzip -q $jar_name -d $extract_dir
+				else
+					# some debug-image tar has parent folder. --strip 1 is used to remove it
+					gzip -cd $jar_name | tar xof - -C $extract_dir --strip 1
+				fi
 			else
-				gzip -cd $jar_name | tar xof - -C ./tmp
-			fi
+				if [ -d "$SDKDIR/openjdkbinary/tmp" ]; then
+					rm -rf $SDKDIR/openjdkbinary/tmp/*
+				else
+					mkdir $SDKDIR/openjdkbinary/tmp
+				fi
+				echo "unzip file: $jar_name ..."
+				if [[ $jar_name == *zip || $jar_name == *jar ]]; then
+					unzip -q $jar_name -d ./tmp
+				else
+					gzip -cd $jar_name | tar xof - -C ./tmp
+				fi
 
-			cd ./tmp
-			jar_dirs=`ls -d */`
-			jar_dir_array=(${jar_dirs//\\n/ })
-			len=${#jar_dir_array[@]}
-			if [[ "$len" == 1 ]]; then
-				jar_dir_name=${jar_dir_array[0]}
-				if [[ "$jar_dir_name" =~ "test-image" && "$jar_dir_name" != "openjdk-test-image" ]]; then
-					mv $jar_dir_name ../openjdk-test-image
-				elif [[ "$jar_dir_name" =~ jre*  &&  "$jar_dir_name" != "j2re-image" ]]; then
-					mv $jar_dir_name ../j2re-image
-				elif [[ "$jar_dir_name" =~ jdk*  &&  "$jar_dir_name" != "j2sdk-image" ]]; then
-					mv $jar_dir_name ../j2sdk-image
-				# if native test libs folder is available, mv it under native-test-libs
-				elif [[ "$jar_dir_name"  =~ native-test-libs*  &&  "$jar_dir_name" != "native-test-libs" ]]; then
-					mv $jar_dir_name ../native-test-libs
-				#The following only needed if openj9 has a different image name convention
-				elif [[ "$jar_dir_name" != "j2sdk-image"  &&  "$jar_dir_name" != "native-test-libs" ]]; then
-					mv $jar_dir_name ../j2sdk-image
+				cd ./tmp
+				jar_dirs=`ls -d */`
+				jar_dir_array=(${jar_dirs//\\n/ })
+				len=${#jar_dir_array[@]}
+				if [[ "$len" == 1 ]]; then
+					jar_dir_name=${jar_dir_array[0]}
+					if [[ "$jar_dir_name" =~ "test-image" && "$jar_dir_name" != "openjdk-test-image" ]]; then
+						mv $jar_dir_name ../openjdk-test-image
+					elif [[ "$jar_dir_name" =~ jre*  &&  "$jar_dir_name" != "j2re-image" ]]; then
+						mv $jar_dir_name ../j2re-image
+					elif [[ "$jar_dir_name" =~ jdk*  &&  "$jar_dir_name" != "j2sdk-image" ]]; then
+						mv $jar_dir_name ../j2sdk-image
+					# if native test libs folder is available, mv it under native-test-libs
+					elif [[ "$jar_dir_name"  =~ native-test-libs*  &&  "$jar_dir_name" != "native-test-libs" ]]; then
+						mv $jar_dir_name ../native-test-libs
+					#The following only needed if openj9 has a different image name convention
+					elif [[ "$jar_dir_name" != "j2sdk-image"  &&  "$jar_dir_name" != "native-test-libs" ]]; then
+						mv $jar_dir_name ../j2sdk-image
+					fi
+				elif [[ "$len" > 1 ]]; then
+					mv ../tmp ../j2sdk-image
 				fi
-			elif [[ "$len" > 1 ]]; then
-				mv ../tmp ../j2sdk-image
+				cd $SDKDIR/openjdkbinary
 			fi
-			cd $SDKDIR/openjdkbinary
 		done
 
 	if [[ "$PLATFORM" == "s390x_zos" ]]; then
 		chmod -R 755 j2sdk-image
 	fi
 }
 
+checkURL() {
+	local filename="$1"
+	if [[ $filename =~ "test-image" && $TEST_IMAGES_REQUIRED = false ]]; then
+		required=false
+	elif [[ $filename =~ "debug-image" && $DEBUG_IMAGES_REQUIRED = false ]]; then
+		required=false
+	fi
+}
+
 getOpenJDKSources() {
 	echo "get jdk sources..."
 	cd $TESTDIR