[Ubuntu ] Sync latest repo changes with ubuntu-new-folder-structure b…

…ranch (#8679) * Update folder structure * Adjust packer templates and debug * Fix issues in ubuntu2204 template * Remove Debug steps * Rename template, sync pipelines and adjust resource gen script * fix post-gen path in ubuntu-minimal * fix shebang linter, workflow paths and adjust readme * update preimagedata * [MacOS] Add xcode 15.0.1 (#8604) * Updating readme file for macos-13 version 20231018.14 (#8601) Co-authored-by: Image generation service account <[email protected]> Co-authored-by: Actions service account <[email protected]> * Updating readme file for macos-13-arm64 version 20231017.1 (#8586) Co-authored-by: Image generation service account <[email protected]> * [MacOS] improve errexit option handling (#8432) * [MacOS] improve errexit option handling * Prevent prompting for override when unzipping * Explicitly allow failing when installing virtualbox * Remove libtcl symlinks * Add debug output * Updating readme file for ubuntu20 version 20231022.1.1 (#8637) Co-authored-by: Image generation service account <[email protected]> * Updating readme file for ubuntu22 version 20231022.1.1 (#8636) Co-authored-by: Image generation service account <[email protected]> * Updating readme file for win19 version 20231023.1.1 (#8644) Co-authored-by: Image generation service account <[email protected]> Co-authored-by: Actions service account <[email protected]> * Updating readme file for win22 version 20231023.1.1 (#8643) Co-authored-by: Image generation service account <[email protected]> Co-authored-by: Actions service account <[email protected]> * [macos] simplify "brew_smart_install" helper (#8639) * [macos] homebrew: use hardcoded condition for jq installation we cannot use "jq" if we are asked to install "jq" * [macos] always use "brew install" * [macos] add retries to "get_github_package_download_url" helper * [macos] add retries to chrome install script * [macos] add retries to OpenJDK install script * [macos] add retries to miniconda installer * Update images/macos/provision/core/openjdk.sh Co-authored-by: Vasilii Polikarpov <[email protected]> * fix copy-paste error * Update images/macos/provision/core/openjdk.sh Co-authored-by: Vasilii Polikarpov <[email protected]> * Update images/macos/provision/core/openjdk.sh Co-authored-by: Vasilii Polikarpov <[email protected]> --------- Co-authored-by: Vasilii Polikarpov <[email protected]> * Updating readme file for ubuntu22 version 20231025.1.1 (#8663) Co-authored-by: Image generation service account <[email protected]> Co-authored-by: Actions service account <[email protected]> * Updating readme file for ubuntu20 version 20231025.1.1 (#8664) Co-authored-by: Image generation service account <[email protected]> Co-authored-by: Actions service account <[email protected]> * Updating readme file for win22 version 20231024.1.1 (#8668) Co-authored-by: Image generation service account <[email protected]> Co-authored-by: Actions service account <[email protected]> * Windows Server 2019 (20231025) Image Update (#8667) * Updating readme file for win19 version 20231025.2.1 * Update Windows2019-Readme.md --------- Co-authored-by: Image generation service account <[email protected]> Co-authored-by: Mikhail Koliada <[email protected]> Co-authored-by: Actions service account <[email protected]> * [Mac OS] Fix logic for retrying AppleScript execution (#8662) * [Ubuntu] Add PGP validation for swift (#8652) * [Windows] Remove out-of-support vs components (#8678) * [MacOS] Unpin Virtualbox (#8653) * [MacOS] Unpin Virtualbox * Remove obsolete virtualbox kext pester test --------- Co-authored-by: sergei-pyshnoi <[email protected]> Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> Co-authored-by: Image generation service account <[email protected]> Co-authored-by: Actions service account <[email protected]> Co-authored-by: Vasilii Polikarpov <[email protected]> Co-authored-by: ilia-shipitsin <[email protected]> Co-authored-by: Mikhail Koliada <[email protected]>
actions · Oct 27, 2023 · 3eeee42 · 3eeee42
1 parent 54a830d
commit 3eeee42
Show file tree

Hide file tree

Showing 18 changed files with 487 additions and 518 deletions.
diff --git a/images/macos/macos-13-Readme.md b/images/macos/macos-13-Readme.md
diff --git a/images/macos/macos-13-arm64-Readme.md b/images/macos/macos-13-arm64-Readme.md
diff --git a/images/macos/provision/core/android-toolsets.sh b/images/macos/provision/core/android-toolsets.sh
@@ -126,7 +126,7 @@ sdkTools="android-sdk-tools.zip"
 sdkToolsVersion=$(get_toolset_value '.android."sdk-tools"')
 if [ "$sdkToolsVersion" != "null" ]; then
     download_with_retries "https://dl.google.com/android/repository/${sdkToolsVersion}" "." $sdkTools
-    unzip -qq $sdkTools -d ${ANDROID_SDK_ROOT}
+    unzip -o -qq $sdkTools -d ${ANDROID_SDK_ROOT}
     rm -f $sdkTools
 fi
 

diff --git a/images/macos/provision/core/chrome.sh b/images/macos/provision/core/chrome.sh
@@ -14,7 +14,8 @@ echo "Google Chrome version is $FULL_CHROME_VERSION"
 # Get Google Chrome versions information
 CHROME_PLATFORM="mac-$arch"
 CHROME_VERSIONS_URL="https://googlechromelabs.github.io/chrome-for-testing/latest-patch-versions-per-build-with-downloads.json"
-CHROME_VERSIONS_JSON=$(curl -fsSL "${CHROME_VERSIONS_URL}")
+download_with_retries "$CHROME_VERSIONS_URL" "/tmp" "latest-patch-versions-per-build-with-downloads.json"
+CHROME_VERSIONS_JSON=$(cat /tmp/latest-patch-versions-per-build-with-downloads.json)
 
 # Download and unpack the latest release of Chrome Driver
 CHROMEDRIVER_VERSION=$(echo "${CHROME_VERSIONS_JSON}" | jq -r '.builds["'"$CHROME_VERSION"'"].version')

diff --git a/images/macos/provision/core/commonutils.sh b/images/macos/provision/core/commonutils.sh
@@ -13,18 +13,7 @@ done
 cask_packages=$(get_toolset_value '.brew.cask_packages[]')
 for package in $cask_packages; do
     echo "Installing $package..."
-    if [[ $package == "virtualbox" ]]; then
-        if ! is_Ventura || ! is_VenturaArm64; then
-            # VirtualBox 7 crashes
-            # macOS host: Dropped all kernel extensions. VirtualBox relies fully on the hypervisor and vmnet frameworks provided by Apple now.
-            vbcask_url="https://raw.githubusercontent.com/Homebrew/homebrew-cask/aa3c55951fc9d687acce43e5c0338f42c1ddff7b/Casks/virtualbox.rb"
-            download_with_retries $vbcask_url
-            brew install ./virtualbox.rb
-            rm ./virtualbox.rb
-        fi
-    else
-        brew install --cask $package
-    fi
+    brew install --cask $package
 done
 
 # Load "Parallels International GmbH"
@@ -36,18 +25,24 @@ fi
 # System Preferences -> Security & Privacy -> General -> Unlock -> Allow -> Not now
 if is_Monterey; then
     if is_Veertu; then
-        retry=5
-        while [ $retry -gt 0 ]; do
+        for retry in {4..0}; do
+            echo "Executing AppleScript to change security preferences. Retries left: $retry"
             {
+                set -e
                 osascript -e 'tell application "System Events" to get application processes where visible is true'
-            }
-            osascript $HOME/utils/confirm-identified-developers.scpt $USER_PASSWORD
+                osascript $HOME/utils/confirm-identified-developers.scpt $USER_PASSWORD
+            } && break
+
+            if [ "$retry" -eq 0 ]; then
+                echo "Executing AppleScript failed. No retries left"
+                exit 1
+            fi
 
-            retry=$((retry-1))
-            echo "retries left "$retry
+            echo "Executing AppleScript failed. Sleeping for 10 seconds and retrying"
             sleep 10
         done
     else
+        echo "Executing AppleScript to change security preferences"
         osascript $HOME/utils/confirm-identified-developers.scpt $USER_PASSWORD
     fi
 fi
@@ -82,13 +77,5 @@ bazel
 # Install Azure DevOps extension for Azure Command Line Interface
 az extension add -n azure-devops
 
-# Workaround https://github.com/actions/runner-images/issues/4931
-# by making Tcl/Tk paths the same on macOS 10.15 and macOS 11
-if is_Monterey; then
-    version=$(brew info tcl-tk --json | jq -r '.[].installed[].version')
-    ln -s /usr/local/Cellar/tcl-tk/$version/lib/libtcl8.6.dylib /usr/local/lib/libtcl8.6.dylib
-    ln -s /usr/local/Cellar/tcl-tk/$version/lib/libtk8.6.dylib /usr/local/lib/libtk8.6.dylib
-fi
-
 # Invoke tests for all basic tools
 invoke_tests "BasicTools"
diff --git a/images/macos/provision/core/miniconda.sh b/images/macos/provision/core/miniconda.sh
@@ -1,9 +1,11 @@
 #!/bin/bash -e -o pipefail
 
-MINICONDA_INSTALLER="/tmp/miniconda.sh"
-curl -fsSL https://repo.anaconda.com/miniconda/Miniconda3-latest-MacOSX-x86_64.sh -o $MINICONDA_INSTALLER
-chmod +x $MINICONDA_INSTALLER
-sudo $MINICONDA_INSTALLER -b -p /usr/local/miniconda
+source ~/utils/utils.sh
+
+download_with_retries "https://repo.anaconda.com/miniconda/Miniconda3-latest-MacOSX-x86_64.sh" "/tmp" "miniconda.sh"
+
+chmod +x /tmp/miniconda.sh
+sudo /tmp/miniconda.sh -b -p /usr/local/miniconda
 
 # Chmod with full permissions recursively to avoid permissions restrictions
 sudo chmod -R 777 /usr/local/miniconda

diff --git a/images/macos/provision/core/openjdk.sh b/images/macos/provision/core/openjdk.sh
@@ -29,12 +29,12 @@ installOpenJDK() {
     local JAVA_VERSION=$1
 
     # Get link for Java binaries and Java version
-    assetUrl=$(curl -fsSL "https://api.adoptium.net/v3/assets/latest/${JAVA_VERSION}/hotspot")
+    download_with_retries "https://api.adoptium.net/v3/assets/latest/${JAVA_VERSION}/hotspot" "/tmp" "openjdk-hotspot.json"
 
     if [[ $arch == "arm64" ]]; then
-        asset=$(echo ${assetUrl} | jq -r '.[] | select(.binary.os=="mac" and .binary.image_type=="jdk" and .binary.architecture=="aarch64")')
+        asset=$(jq -r '.[] | select(.binary.os=="mac" and .binary.image_type=="jdk" and .binary.architecture=="aarch64")' /tmp/openjdk-hotspot.json)
     else
-        asset=$(echo ${assetUrl} | jq -r '.[] | select(.binary.os=="mac" and .binary.image_type=="jdk" and .binary.architecture=="x64")')
+        asset=$(jq -r '.[] | select(.binary.os=="mac" and .binary.image_type=="jdk" and .binary.architecture=="x64")' /tmp/openjdk-hotspot.json)
     fi
 
     archivePath=$(echo ${asset} | jq -r '.binary.package.link')

diff --git a/images/macos/provision/core/pypy.sh b/images/macos/provision/core/pypy.sh
@@ -11,7 +11,7 @@ function InstallPyPy
     PACKAGE_URL=$1
 
     PACKAGE_TAR_NAME=$(echo $PACKAGE_URL | awk -F/ '{print $NF}')
-    echo "Downloading tar archive '$PACKAGE_TAR_NAME' - '$PACKAGE_URL'"
+    echo "Downloading tar archive '$PACKAGE_TAR_NAME'"
     PACKAGE_TAR_TEMP_PATH="/tmp/$PACKAGE_TAR_NAME"
     download_with_retries $PACKAGE_URL "/tmp" "$PACKAGE_TAR_NAME"
 
@@ -89,4 +89,4 @@ for toolsetVersion in $toolsetVersions; do
         exit 1
     fi
     InstallPyPy $latestMajorPyPyVersion
-done
+done
diff --git a/images/macos/provision/utils/utils.sh b/images/macos/provision/utils/utils.sh
@@ -15,24 +15,31 @@ download_with_retries() {
         local COMMAND="curl $URL -4 -sL -o '$DEST/$NAME' -w '%{http_code}'"
     fi
 
+    # Save current errexit state and disable it to prevent unexpected exit on error
+    if echo $SHELLOPTS | grep '\(^\|:\)errexit\(:\|$\)' > /dev/null;
+    then
+        local ERR_EXIT_ENABLED=true
+    else
+        local ERR_EXIT_ENABLED=false
+    fi
+    set +e
+
     echo "Downloading '$URL' to '${DEST}/${NAME}'..."
     retries=20
     interval=30
     while [ $retries -gt 0 ]; do
         ((retries--))
-        # Temporary disable exit on error to retry on non-zero exit code
-        set +e
+        test "$ERR_EXIT_ENABLED" = true && set +e
         http_code=$(eval $COMMAND)
         exit_code=$?
+        test "$ERR_EXIT_ENABLED" = true && set -e
         if [ $http_code -eq 200 ] && [ $exit_code -eq 0 ]; then
             echo "Download completed"
             return 0
         else
             echo "Error — Either HTTP response code for '$URL' is wrong - '$http_code' or exit code is not 0 - '$exit_code'. Waiting $interval seconds before the next attempt, $retries attempts left"
             sleep 30
         fi
-        # Enable exit on error back
-        set -e
     done
 
     echo "Could not download $URL"
@@ -120,98 +127,50 @@ get_brew_os_keyword() {
     fi
 }
 
-should_build_from_source() {
-    local tool_name=$1
-    local os_name=$2
-    # If one of the parsers aborts with an error, 
-    # we will get an empty variable notification in the logs
-    set -u
-
-    # Geting tool info from brew to find available install methods except build from source
-    local tool_info=$(brew info --json=v1 $tool_name)
-
-    # No need to build from source if a bottle is disabled
-    local bottle_disabled=$(echo -E $tool_info | jq ".[0].bottle_disabled")
-    if [[ $bottle_disabled == "true" ]]; then
-        echo "false"
-        return
-    fi
-
-    # No need to build from source if a universal bottle is available    
-    local all_bottle=$(echo -E $tool_info | jq ".[0].bottle.stable.files.all")
-    if [[ "$all_bottle" != "null" ]]; then
-        echo "false"
-        return
-    fi
-
-    # No need to build from source if a bottle for current OS is available
-    local os_bottle=$(echo -E $tool_info | jq ".[0].bottle.stable.files.$os_name")
-    if [[ "$os_bottle" != "null" ]]; then
-        echo "false"
-        return
-    fi
-
-    # Available method wasn't found - should build from source
-    echo "true"
-}
-
 # brew provides package bottles for different macOS versions
 # The 'brew install' command will fail if a package bottle does not exist
 # Use the '--build-from-source' option to build from source in this case
 brew_smart_install() {
     local tool_name=$1
 
-    local os_name=$(get_brew_os_keyword)
-    if [[ "$os_name" == "null" ]]; then
-        echo "$OSTYPE is unknown operating system"
-        exit 1
-    fi
+    echo "Downloading $tool_name..."
 
-    local build_from_source=$(should_build_from_source "$tool_name" "$os_name")
-    if $build_from_source; then
-        echo "Bottle of the $tool_name for the $os_name was not found. Building $tool_name from source..."
-        brew install --build-from-source $tool_name
-    else
-        echo "Downloading $tool_name..."
+    # get deps & cache em
 
-        # get deps & cache em
+    failed=true
+    for i in {1..10}; do
+        brew deps $tool_name > /tmp/$tool_name && failed=false || sleep 60
+        [ "$failed" = false ] && break
+    done
 
-        failed=true
-        for i in {1..10}; do
-            brew deps $tool_name > /tmp/$tool_name && failed=false || sleep 60
-            [ "$failed" = false ] && break
-        done
+    if [ "$failed" = true ]; then
+       echo "Failed: brew deps $tool_name"
+       exit 1;
+    fi
 
-        if [ "$failed" = true ]; then
-           echo "Failed: brew deps $tool_name"
-           exit 1;
-        fi
+    for dep in $(cat /tmp/$tool_name) $tool_name; do
 
-        for dep in $(cat /tmp/$tool_name) $tool_name; do
-
-            failed=true
-            for i in {1..10}; do
-                brew --cache $dep >/dev/null && failed=false || sleep 60
-                [ "$failed" = false ] && break
-            done
-
-            if [ "$failed" = true ]; then
-               echo "Failed: brew --cache $dep"
-               exit 1;
-            fi
-        done
-
-        failed=true
-        for i in {1..10}; do
-            brew install $tool_name >/dev/null && failed=false || sleep 60
-            [ "$failed" = false ] && break
-        done
-
-        if [ "$failed" = true ]; then
-           echo "Failed: brew install $tool_name"
-           exit 1;
-        fi
+    failed=true
+    for i in {1..10}; do
+        brew --cache $dep >/dev/null && failed=false || sleep 60
+        [ "$failed" = false ] && break
+    done
+
+    if [ "$failed" = true ]; then
+       echo "Failed: brew --cache $dep"
+       exit 1;
+    fi
+    done
+
+    failed=true
+    for i in {1..10}; do
+        brew install $tool_name >/dev/null && failed=false || sleep 60
+        [ "$failed" = false ] && break
+    done
 
+    if [ "$failed" = true ]; then
+       echo "Failed: brew install $tool_name"
+       exit 1;
     fi
 }
 
@@ -240,7 +199,18 @@ get_github_package_download_url() {
 
     [ -n "$API_PAT" ] && authString=(-H "Authorization: token ${API_PAT}")
 
-    json=$(curl "${authString[@]}" -fsSL "https://api.github.com/repos/${REPO_ORG}/releases?per_page=${SEARCH_IN_COUNT}")
+    failed=true
+    for i in {1..10}; do
+        curl "${authString[@]}" -fsSL "https://api.github.com/repos/${REPO_ORG}/releases?per_page=${SEARCH_IN_COUNT}" >/tmp/get_github_package_download_url.json && failed=false || sleep 60
+        [ "$failed" = false ] && break
+    done
+
+    if [ "$failed" = true ]; then
+       echo "Failed: get_github_package_download_url"
+       exit 1;
+    fi
+
+    json=$(cat /tmp/get_github_package_download_url.json)
 
     if [[ "$VERSION" == "latest" ]]; then
         tagName=$(echo $json | jq -r '.[] | select((.prerelease==false) and (.assets | length > 0)).tag_name' | sort --unique --version-sort | egrep -v ".*-[a-z]" | tail -1)

diff --git a/images/macos/tests/BasicTools.Tests.ps1 b/images/macos/tests/BasicTools.Tests.ps1
@@ -67,6 +67,13 @@ Describe "Helm" -Skip:($os.IsMonterey -or $os.IsVentura -or $os.IsVenturaArm64)
     }
 }
 
+Describe "Tcl/Tk" {
+    It "libtcl" {
+        "file /usr/local/lib/libtcl8.6.dylib" | Should -ReturnZeroExitCode
+        "file /usr/local/lib/libtk8.6.dylib" | Should -ReturnZeroExitCode
+    }
+}
+
 Describe "bazelisk" {
     It "bazelisk" {
         "bazelisk version" | Should -ReturnZeroExitCode
@@ -164,9 +171,9 @@ Describe "Homebrew" {
 }
 
 Describe "Kotlin" {
-    $kotlinPackages =  @("kapt", "kotlin", "kotlinc", "kotlinc-jvm", "kotlin-dce-js")
+    $kotlinPackages = @("kapt", "kotlin", "kotlinc", "kotlinc-jvm", "kotlin-dce-js")
 
-    It "<toolName> is available" -TestCases ($kotlinPackages | ForEach-Object {  @{ toolName = $_ } })  { 
+    It "<toolName> is available" -TestCases ($kotlinPackages | ForEach-Object { @{ toolName = $_ } }) { 
         "$toolName -version" | Should -ReturnZeroExitCode
     }
 }
@@ -187,4 +194,4 @@ Describe "imagemagick" -Skip:($os.IsVentura -or $os.IsVenturaArm64) {
     It "imagemagick" {
         "magick -version" | Should -ReturnZeroExitCode
     }
-}
+}
diff --git a/images/macos/tests/Common.Tests.ps1 b/images/macos/tests/Common.Tests.ps1
@@ -123,12 +123,6 @@ Describe "Go" -Skip:($os.IsVentura -or $os.IsVenturaArm64) {
     }
 }
 
-Describe "VirtualBox" -Skip:($os.IsBigSur -or $os.IsVentura -or $os.IsVenturaArm64) {
-    It "Check kext kernel modules" {
-        kextstat | Out-String | Should -Match "org.virtualbox.kext"
-    }
-}
-
 Describe "CodeQL Bundle" {
     It "Is installed" {
         $CodeQLVersionWildcard = Join-Path $Env:AGENT_TOOLSDIRECTORY -ChildPath "CodeQL" | Join-Path -ChildPath "*"

diff --git a/images/macos/toolsets/toolset-13.json b/images/macos/toolsets/toolset-13.json
@@ -3,17 +3,15 @@
         "default": "14.3.1",
         "x64": {
             "versions": [
-                { "link": "15.0.1", "version": "15.0.1-Release.Candidate+15A507", "install_runtimes": "true"},
-                { "link": "15.0", "version": "15.0.0+15A240d", "install_runtimes": "true"},
-                { "link": "14.3.1", "version": "14.3.1+14E300c","symlinks": ["14.3"], "install_runtimes": "true" },
-                { "link": "14.2", "version": "14.2.0+14C18", "install_runtimes": "true" },
-                { "link": "14.1", "version": "14.1.0+14B47b", "install_runtimes": "true" }
+                { "link": "15.0.1", "version": "15.0.1+15A507", "symlinks": ["15.0"], "install_runtimes": "true"},
+                { "link": "14.3.1", "version": "14.3.1+14E300c","symlinks": ["14.3"], "install_runtimes": "true"},
+                { "link": "14.2", "version": "14.2.0+14C18", "install_runtimes": "true"},
+                { "link": "14.1", "version": "14.1.0+14B47b", "install_runtimes": "true"}
                 ]
         },
         "arm64":{
             "versions": [
-                { "link": "15.0.1", "version": "15.0.1-Release.Candidate+15A507", "install_runtimes": "true"},
-                { "link": "15.0", "version": "15.0.0+15A240d", "install_runtimes": "true"},
+                { "link": "15.0.1", "version": "15.0.1+15A507", "symlinks": ["15.0"], "install_runtimes": "true"},
                 { "link": "14.3.1", "version": "14.3.1+14E300c","symlinks": ["14.3"], "install_runtimes": "true" },
                 { "link": "14.2", "version": "14.2.0+14C18", "install_runtimes": "true" },
                 { "link": "14.1", "version": "14.1.0+14B47b", "install_runtimes": "true" }