Sync changes to ubuntu-new-folder-structure branch (#8764)

* Update folder structure * Adjust packer templates and debug * Fix issues in ubuntu2204 template * Remove Debug steps * Rename template, sync pipelines and adjust resource gen script * fix post-gen path in ubuntu-minimal * fix shebang linter, workflow paths and adjust readme * update preimagedata * [MacOS] Add xcode 15.0.1 (#8604) * Updating readme file for macos-13 version 20231018.14 (#8601) Co-authored-by: Image generation service account <[email protected]> Co-authored-by: Actions service account <[email protected]> * Updating readme file for macos-13-arm64 version 20231017.1 (#8586) Co-authored-by: Image generation service account <[email protected]> * [MacOS] improve errexit option handling (#8432) * [MacOS] improve errexit option handling * Prevent prompting for override when unzipping * Explicitly allow failing when installing virtualbox * Remove libtcl symlinks * Add debug output * Updating readme file for ubuntu20 version 20231022.1.1 (#8637) Co-authored-by: Image generation service account <[email protected]> * Updating readme file for ubuntu22 version 20231022.1.1 (#8636) Co-authored-by: Image generation service account <[email protected]> * Updating readme file for win19 version 20231023.1.1 (#8644) Co-authored-by: Image generation service account <[email protected]> Co-authored-by: Actions service account <[email protected]> * Updating readme file for win22 version 20231023.1.1 (#8643) Co-authored-by: Image generation service account <[email protected]> Co-authored-by: Actions service account <[email protected]> * [macos] simplify "brew_smart_install" helper (#8639) * [macos] homebrew: use hardcoded condition for jq installation we cannot use "jq" if we are asked to install "jq" * [macos] always use "brew install" * [macos] add retries to "get_github_package_download_url" helper * [macos] add retries to chrome install script * [macos] add retries to OpenJDK install script * [macos] add retries to miniconda installer * Update images/macos/provision/core/openjdk.sh Co-authored-by: Vasilii Polikarpov <[email protected]> * fix copy-paste error * Update images/macos/provision/core/openjdk.sh Co-authored-by: Vasilii Polikarpov <[email protected]> * Update images/macos/provision/core/openjdk.sh Co-authored-by: Vasilii Polikarpov <[email protected]> --------- Co-authored-by: Vasilii Polikarpov <[email protected]> * Updating readme file for ubuntu22 version 20231025.1.1 (#8663) Co-authored-by: Image generation service account <[email protected]> Co-authored-by: Actions service account <[email protected]> * Updating readme file for ubuntu20 version 20231025.1.1 (#8664) Co-authored-by: Image generation service account <[email protected]> Co-authored-by: Actions service account <[email protected]> * Updating readme file for win22 version 20231024.1.1 (#8668) Co-authored-by: Image generation service account <[email protected]> Co-authored-by: Actions service account <[email protected]> * Windows Server 2019 (20231025) Image Update (#8667) * Updating readme file for win19 version 20231025.2.1 * Update Windows2019-Readme.md --------- Co-authored-by: Image generation service account <[email protected]> Co-authored-by: Mikhail Koliada <[email protected]> Co-authored-by: Actions service account <[email protected]> * [Mac OS] Fix logic for retrying AppleScript execution (#8662) * [Ubuntu] Add PGP validation for swift (#8652) * [Windows] Remove out-of-support vs components (#8678) * [MacOS] Unpin Virtualbox (#8653) * [MacOS] Unpin Virtualbox * Remove obsolete virtualbox kext pester test * [Ubuntu] Update apt repos list in documentation (#8634) * fix: bump libssl version to ubuntu2.20 (#8682) Signed-off-by: Jakub Oskera <[email protected]> Co-authored-by: Jakub Oskera <[email protected]> * [macOS] increase timeout for the macOS ci (#8692) * Updating readme file for macos-13-arm64 version 20231024.2 (#8673) Co-authored-by: Image generation service account <[email protected]> Co-authored-by: Actions service account <[email protected]> * Escape quotes when running in PowerShell 7.0-7.2 (#8705) * fix: checksum search for Kotlin (#8704) * [Ubuntu] Fix checksum search for Kotlin * [Windows] Fixed checksum search for Kotlin * [Ubuntu] Pin bindgen cli version to avoid bug (#8714) * [Ubuntu] Add more checksum validations (#8660) * [Windows] Pin bindgen cli version to avoid bug (#8713) * Updating readme file for ubuntu22 version 20231030.2.1 (#8691) Co-authored-by: Image generation service account <[email protected]> Co-authored-by: Actions service account <[email protected]> * Updating readme file for ubuntu20 version 20231030.2.1 (#8690) Co-authored-by: Image generation service account <[email protected]> Co-authored-by: Actions service account <[email protected]> * Updating readme file for win19 version 20231029.1.1 (#8687) Co-authored-by: Image generation service account <[email protected]> Co-authored-by: Actions service account <[email protected]> * Revert "[Windows] Pin bindgen cli version to avoid bug (#8713)" (#8719) This reverts commit 1dac9b9. * Revert "[Ubuntu] Pin bindgen cli version to avoid bug (#8714)" (#8720) This reverts commit 3146c70. * [Mac OS] Temporary disable VNC for Mac OS 14 (#8717) * Updating readme file for win22 version 20231029.1.1 (#8686) Co-authored-by: Image generation service account <[email protected]> Co-authored-by: Actions service account <[email protected]> * Updating readme file for macOS-11 version 20231030.1 (#8694) Co-authored-by: Image generation service account <[email protected]> Co-authored-by: Actions service account <[email protected]> * [Mac OS] Add template for Mac OS 14 image (#8733) * Add gfortran-13 (#8702) * Revert "[MacOS] Unpin Virtualbox" (#8731) * Revert "[MacOS] Unpin Virtualbox (#8653)" This reverts commit f3d011e. * Add more info * [ubuntu] split git and git-lfs into separate install scripts (#8711) for better observability of integrity validation let us split install script into two * Updating readme file for macOS-12 version 20231029.1 (#8695) Co-authored-by: Image generation service account <[email protected]> Co-authored-by: Actions service account <[email protected]> * [Windows] Remove Android SDK Patch applier v4 (#8750) * remove ndk 23 from macos images (#8747) * [Mac OS] Remove 'LTS' suffix from OpenJDK version (#8751) * [windows] split docker install into 3 scripts (#8688) * [windows] split docker install into 3 scripts the idea is to end with 1 script per 1 component for better observability * Update images/win/scripts/Installers/Install-Docker-Compose.ps1 Co-authored-by: Erik Bershel <[email protected]> * Update images/win/scripts/Installers/Install-Docker-WinCred.ps1 Co-authored-by: Vasilii Polikarpov <[email protected]> * fix test --------- Co-authored-by: Erik Bershel <[email protected]> Co-authored-by: Vasilii Polikarpov <[email protected]> * remove ndk 23 from ubuntu images (#8745) * [macos] add android SDK to SW report on Ventura ARM64 (#8715) * [Mac OS] Update Anka Helpers to build Mac OS 14 ARM image (#8757) * Update Anka Helpers to build Mac OS 14 ARM image * Drop Powershell 5 support * [Mac OS] Add missing macOS Sonoma selector (#8759) * remove ndk23 from windows images (#8746) * Updating readme file for macos-13 version 20231025.2 (#8672) Co-authored-by: Image generation service account <[email protected]> --------- Signed-off-by: Jakub Oskera <[email protected]> Co-authored-by: sergei-pyshnoi <[email protected]> Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> Co-authored-by: Image generation service account <[email protected]> Co-authored-by: Actions service account <[email protected]> Co-authored-by: Vasilii Polikarpov <[email protected]> Co-authored-by: ilia-shipitsin <[email protected]> Co-authored-by: Mikhail Koliada <[email protected]> Co-authored-by: Jakub Oskera <[email protected]> Co-authored-by: Jakub Oskera <[email protected]> Co-authored-by: Luke Fritts <[email protected]> Co-authored-by: Erik Bershel <[email protected]> Co-authored-by: Harmen Stoppels <[email protected]>
actions · Nov 8, 2023 · 5739887 · 5739887
1 parent 48a3da7
commit 5739887
Show file tree

Hide file tree

Showing 43 changed files with 843 additions and 437 deletions.
diff --git a/images.CI/macos/anka/CreateCleanAnkaTemplate.ps1 b/images.CI/macos/anka/CreateCleanAnkaTemplate.ps1
@@ -182,6 +182,12 @@ Write-Host "`n[#2] Create a VM template:"
 Write-Host "`t[*] Deleting existed template with name '$TemplateName' before creating a new one"
 Remove-AnkaVM -VMName $TemplateName
 
+# Temporary disable VNC for macOS 14
+# It's probably Anka's bug fixed in 3.3.2
+if ($shortMacOSVersion -eq "14") {
+    $env:ANKA_CREATE_VNC = 0
+}
+
 Write-Host "`t[*] Creating Anka VM template with name '$TemplateName' and '$TemplateUsername' user"
 Write-Host "`t[*] CPU Count: $CPUCount, RamSize: ${RamSizeGb}G, DiskSizeGb: ${DiskSizeGb}G, InstallerPath: $macOSInstaller, TemplateName: $TemplateName"
 New-AnkaVMTemplate -InstallerPath "$macOSInstaller" `

diff --git a/images.CI/macos/anka/Service.Helpers.psm1 b/images.CI/macos/anka/Service.Helpers.psm1
@@ -63,24 +63,24 @@ function Get-AvailableIPSWVersions {
     )
 
     if ($IsBeta) {
-        $command = { mist list firmware "$MacOSCodeNameOrVersion" --include-betas --latest --export "/Applications/export.json"}
+        $command = { mist list firmware "$MacOSCodeNameOrVersion" --compatible --include-betas --latest --export "/Applications/export.json" }
     } elseif ($IsLatest) {
-        $command = { mist list firmware "$MacOSCodeNameOrVersion" --latest  --export "/Applications/export.json" }
+        $command = { mist list firmware "$MacOSCodeNameOrVersion" --compatible --latest --export "/Applications/export.json" }
     } else {
-        $command = { mist list firmware "$MacOSCodeNameOrVersion"  --export "/Applications/export.json" }
+        $command = { mist list firmware "$MacOSCodeNameOrVersion" --compatible --export "/Applications/export.json" }
     }
 
     $condition = { $LASTEXITCODE -eq 0 }
     Invoke-WithRetry -Command $command -BreakCondition $condition | Out-Null
     $softwareList = get-content -Path "/Applications/export.json"
-    $turgetVersion = ($softwareList | ConvertFrom-Json).version
-    if ($null -eq $turgetVersion) {
+    $availableBuilds = ($softwareList | ConvertFrom-Json).build
+    if ($null -eq $availableBuilds) {
         Write-Host "Requested macOS '$MacOSCodeNameOrVersion' version not found in the list of available installers."
         $command = { mist list firmware "$($MacOSCodeNameOrVersion.split('.')[0])" }
         Invoke-WithRetry -Command $command -BreakCondition $condition
         exit 1
     }
-    return $turgetVersion
+    return $availableBuilds
 }
 
 function Get-MacOSIPSWInstaller {
@@ -96,21 +96,26 @@ function Get-MacOSIPSWInstaller {
         $MacOSName = "macOS Monterey"
     } elseif ($MacOSVersion -eq [version] "13.0") {
         $MacOSName = "macOS Ventura"
+    } elseif ($MacOSVersion -eq [version] "14.0") {
+        $MacOSName = "macOS Sonoma"
     } else {
         $MacOSName = $MacOSVersion.ToString()
     }
 
-
     Write-Host "`t[*] Finding available full installers"
     if ($DownloadLatestVersion -eq $true) {
-        $targetVersion = Get-AvailableIPSWVersions -IsLatest $true -MacOSCodeNameOrVersion $MacOSName
-        Write-host "`t[*] The 'DownloadLatestVersion' flag is set to true. Latest macOS version is '$MacOSName' - '$targetVersion' now"
+        $targetBuild = Get-AvailableIPSWVersions -IsLatest $true -MacOSCodeNameOrVersion $MacOSName
+        Write-Host "`t[*] The 'DownloadLatestVersion' flag is set to true. Latest compatible macOS build of '$MacOSName' is '$targetBuild'"
     } elseif ($BetaSearch -eq $true) {
-        $targetVersion = Get-AvailableIPSWVersions -IsBeta $true -MacOSCodeNameOrVersion $MacOSName
-        Write-host "`t[*] The 'BetaSearch' flag is set to true. Latestbeta macOS version is '$MacOSName' - '$targetVersion' now"
+        $targetBuild = Get-AvailableIPSWVersions -IsBeta $true -MacOSCodeNameOrVersion $MacOSName
+        Write-Host "`t[*] The 'BetaSearch' flag is set to true. Latest compatible beta macOS build of '$MacOSName' is '$targetBuild'"
     } else {
-        $targetVersion = Get-AvailableIPSWVersions -MacOSCodeNameOrVersion $MacOSName -IsLatest $false
-        Write-host "`t[*] The exact version was specified - '$MacOSName' "
+        $targetBuild = Get-AvailableIPSWVersions -MacOSCodeNameOrVersion $MacOSName -IsLatest $false
+        Write-Host "`t[*] Available compatible macOS builds of '$MacOSName' are: $($targetBuild -join ', ')"
+        if ($targetBuild.Count -gt 1) {
+            Write-Error "`t[*] Please specify the exact build number of macOS you want to install"
+            exit 1
+        }
     }
 
     $installerPathPattern = "/Applications/Install ${macOSName}*.ipsw"
@@ -123,12 +128,12 @@ function Get-MacOSIPSWInstaller {
     # Download macOS installer
     $installerDir = "/Applications/"
     $installerName = "Install ${macOSName}.ipsw"
-    Write-Host "`t[*] Requested macOS '$targetVersion' version installer found, fetching it from mist database"
-    Invoke-WithRetry { mist download firmware "$targetVersion" --output-directory $installerDir --firmware-name "$installerName" } {$LASTEXITCODE -eq 0} | Out-Null
+    Write-Host "`t[*] Requested macOS '$targetBuild' version installer found, fetching it from mist database"
+    Invoke-WithRetry { mist download firmware "$targetBuild" --output-directory $installerDir --firmware-name "$installerName" } { $LASTEXITCODE -eq 0 } | Out-Null
     if (Test-Path "$installerDir$installerName") {
         $result = "$installerDir$installerName"
     } else {
-        Write-host "`t[*] Requested macOS '$targetVersion' version installer failed to download"
+        Write-Error "`t[*] Requested macOS '$targetBuild' version installer failed to download"
         exit 1
     }
     return $result
@@ -169,7 +174,7 @@ function Get-MacOSInstaller {
             exit 1
         }
         Show-StringWithFormat $filteredVersions
-        $osVersions = $filteredVersions.OSVersion | Sort-Object {[version]$_}
+        $osVersions = $filteredVersions.OSVersion | Sort-Object { [version]$_ }
         $MacOSVersion = $osVersions | Select-Object -Last 1
         Write-Host "`t[*] The 'DownloadLatestVersion' flag is set. Latest macOS version is '$MacOSVersion' now"
     }
@@ -248,15 +253,15 @@ function Install-SoftwareUpdate {
     $command = "sw_vers"
     $guestMacosVersion = Invoke-SSHPassCommand -HostName $HostName -Command $command
     if ($guestMacosVersion[1] -match "12") {
-        foreach ($update in $listOfUpdates){
+        foreach ($update in $listOfUpdates) {
             # Filtering updates that contain "Ventura" word
             if ($update -notmatch "Ventura") {
                 $command = "sudo /usr/sbin/softwareupdate --restart --verbose --install '$($update.trim())'"
                 Invoke-SSHPassCommand -HostName $HostName -Command $command
             }
         }
     } elseif ($guestMacosVersion[1] -match "13") {
-        foreach ($update in $listOfUpdates){
+        foreach ($update in $listOfUpdates) {
             # Filtering updates that contain "Sonoma" word
             if ($update -notmatch "Sonoma") {
                 $command = "sudo /usr/sbin/softwareupdate --restart --verbose --install '$($update.trim())'"
@@ -302,7 +307,11 @@ function Invoke-SSHPassCommand {
         "${env:SSHUSER}@${HostName}"
     )
     $sshPassOptions = $sshArg -join " "
-    $result = bash -c "$sshPassOptions \""$Command\"" 2>&1"
+    if ($PSVersionTable.PSVersion.Major -eq 7 -and $PSVersionTable.PSVersion.Minor -le 2) {
+        $result = bash -c "$sshPassOptions \""$Command\"" 2>&1"
+    } else {
+        $result = bash -c "$sshPassOptions `"$Command`" 2>&1"
+    }
     if ($LASTEXITCODE -ne 0) {
         Write-Error "There is an error during command execution:`n$result"
         exit 1