From db349895e236ffb122f3662032f28467e4c1d99f Mon Sep 17 00:00:00 2001 From: Ben Broderick Phillips Date: Fri, 15 Nov 2024 17:34:36 -0500 Subject: [PATCH 1/4] Support override of powershell module feeds --- .../scripts/Helpers/PSModule-Helpers.ps1 | 104 +++++++++--------- 1 file changed, 55 insertions(+), 49 deletions(-) diff --git a/eng/common/scripts/Helpers/PSModule-Helpers.ps1 b/eng/common/scripts/Helpers/PSModule-Helpers.ps1 index bfca8a4154d8..9ea3120a7bb0 100644 --- a/eng/common/scripts/Helpers/PSModule-Helpers.ps1 +++ b/eng/common/scripts/Helpers/PSModule-Helpers.ps1 @@ -55,68 +55,74 @@ function Install-ModuleIfNotInstalled() param( [string]$moduleName, [string]$version, - [string]$repositoryUrl = $DefaultPSRepositoryUrl + [string]$repositoryUrl ) - # Check installed modules - $modules = (Get-Module -ListAvailable $moduleName) - if ($version -as [Version]) { - $modules = $modules.Where({ [Version]$_.Version -ge [Version]$version }) + # List of modules+versions we want to replace with internal feed sources for reliability, security, etc. + $mirrorFeedOverrides = @{ + 'powershell-yaml;0.4.7' = 'https://pkgs.dev.azure.com/azure-sdk/public/_packaging/azure-sdk-tools/nuget/v2' } - if ($modules.Count -eq 0) - { - # Use double-checked locking to avoid locking when module is already installed + try { $mutex = New-Object System.Threading.Mutex($false, "Install-ModuleIfNotInstalled") $null = $mutex.WaitOne() - try { - # Check installed modules again after acquiring lock - $modules = (Get-Module -ListAvailable $moduleName) - if ($version -as [Version]) { - $modules = $modules.Where({ [Version]$_.Version -ge [Version]$version }) - } + # Check installed modules again after acquiring lock + $modules = (Get-Module -ListAvailable $moduleName) + if ($version -as [Version]) { + $modules = $modules.Where({ [Version]$_.Version -ge [Version]$version }) + } + + if ($modules.Count -gt 0) + { + Write-Host "Using module $($modules[0].Name) with version $($modules[0].Version)." + return $modules[0] + } - if ($modules.Count -eq 0) - { - $repositories = (Get-PSRepository).Where({ $_.SourceLocation -eq $repositoryUrl }) - if ($repositories.Count -eq 0) - { - Register-PSRepository -Name $repositoryUrl -SourceLocation $repositoryUrl -InstallationPolicy Trusted - $repositories = (Get-PSRepository).Where({ $_.SourceLocation -eq $repositoryUrl }) - if ($repositories.Count -eq 0) { - Write-Error "Failed to register package repository $repositoryUrl." - return - } - } - $repository = $repositories[0] - - if ($repository.InstallationPolicy -ne "Trusted") { - Set-PSRepository -Name $repository.Name -InstallationPolicy "Trusted" - } - - Write-Host "Installing module $moduleName with min version $version from $repositoryUrl" - # Install under CurrentUser scope so that the end up under $CurrentUserModulePath for caching - Install-Module $moduleName -MinimumVersion $version -Repository $repository.Name -Scope CurrentUser -Force - - # Ensure module installed - $modules = (Get-Module -ListAvailable $moduleName) - if ($version -as [Version]) { - $modules = $modules.Where({ [Version]$_.Version -ge [Version]$version }) - } - - if ($modules.Count -eq 0) { - Write-Error "Failed to install module $moduleName with version $version" - return - } + $repositoryUrl = if ($repositoryUrl) { + $repositoryUrl + } elseif ($mirrorFeedOverrides.Contains("${moduleName};${version}")) { + $mirrorFeedOverrides["${moduleName};${version}"] + } else { + $DefaultPSRepositoryUrl + } + + $repositories = (Get-PSRepository).Where({ $_.SourceLocation -eq $repositoryUrl }) + if ($repositories.Count -eq 0) + { + Register-PSRepository -Name $repositoryUrl -SourceLocation $repositoryUrl -InstallationPolicy Trusted + $repositories = (Get-PSRepository).Where({ $_.SourceLocation -eq $repositoryUrl }) + if ($repositories.Count -eq 0) { + Write-Error "Failed to register package repository $repositoryUrl." + return } } - finally { - $mutex.ReleaseMutex() + $repository = $repositories[0] + + if ($repository.InstallationPolicy -ne "Trusted") { + Set-PSRepository -Name $repository.Name -InstallationPolicy "Trusted" } + + Write-Host "Installing module $moduleName with min version $version from $repositoryUrl" + # Install under CurrentUser scope so that the end up under $CurrentUserModulePath for caching + Install-Module $moduleName -MinimumVersion $version -Repository $repository.Name -Scope CurrentUser -Force + + # Ensure module installed + $modules = (Get-Module -ListAvailable $moduleName) + if ($version -as [Version]) { + $modules = $modules.Where({ [Version]$_.Version -ge [Version]$version }) + } + + if ($modules.Count -eq 0) { + Write-Error "Failed to install module $moduleName with version $version" + return + } + + Write-Host "Using module $($modules[0].Name) with version $($modules[0].Version)." + } finally { + $mutex.ReleaseMutex() } - Write-Host "Using module $($modules[0].Name) with version $($modules[0].Version)." return $modules[0] } From 42c23221a6a2fe449034eae02e77207a9a44d132 Mon Sep 17 00:00:00 2001 From: Ben Broderick Phillips Date: Tue, 19 Nov 2024 16:06:42 -0500 Subject: [PATCH 2/4] snap --- eng/common/scripts/Helpers/PSModule-Helpers.ps1 | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/eng/common/scripts/Helpers/PSModule-Helpers.ps1 b/eng/common/scripts/Helpers/PSModule-Helpers.ps1 index 9ea3120a7bb0..94fe04bbd04c 100644 --- a/eng/common/scripts/Helpers/PSModule-Helpers.ps1 +++ b/eng/common/scripts/Helpers/PSModule-Helpers.ps1 @@ -59,8 +59,8 @@ function Install-ModuleIfNotInstalled() ) # List of modules+versions we want to replace with internal feed sources for reliability, security, etc. - $mirrorFeedOverrides = @{ - 'powershell-yaml;0.4.7' = 'https://pkgs.dev.azure.com/azure-sdk/public/_packaging/azure-sdk-tools/nuget/v2' + $packageFeedOverrides = @{ + 'powershell-yaml' = 'https://pkgs.dev.azure.com/azure-sdk/public/_packaging/azure-sdk-tools/nuget/v2' } try { @@ -81,8 +81,8 @@ function Install-ModuleIfNotInstalled() $repositoryUrl = if ($repositoryUrl) { $repositoryUrl - } elseif ($mirrorFeedOverrides.Contains("${moduleName};${version}")) { - $mirrorFeedOverrides["${moduleName};${version}"] + } elseif ($mirrorFeedOverrides.Contains("${moduleName}")) { + $mirrorFeedOverrides["${moduleName}"] } else { $DefaultPSRepositoryUrl } From 1ad6a720c28aeb09c9e7f97d3ccb7466d2a3d83e Mon Sep 17 00:00:00 2001 From: Ben Broderick Phillips Date: Wed, 20 Nov 2024 15:14:10 -0500 Subject: [PATCH 3/4] Add fallback repository support --- .../scripts/Helpers/PSModule-Helpers.ps1 | 134 +++++++++++------- 1 file changed, 81 insertions(+), 53 deletions(-) diff --git a/eng/common/scripts/Helpers/PSModule-Helpers.ps1 b/eng/common/scripts/Helpers/PSModule-Helpers.ps1 index 94fe04bbd04c..4e7224021eb0 100644 --- a/eng/common/scripts/Helpers/PSModule-Helpers.ps1 +++ b/eng/common/scripts/Helpers/PSModule-Helpers.ps1 @@ -1,4 +1,3 @@ -$DefaultPSRepositoryUrl = "https://www.powershellgallery.com/api/v2" $global:CurrentUserModulePath = "" function Update-PSModulePathForCI() @@ -47,83 +46,112 @@ function Update-PSModulePathForCI() } } -# Manual test at eng/common-tests/psmodule-helpers/Install-Module-Parallel.ps1 -# If we want to use another default repository other then PSGallery we can update the default parameters -function Install-ModuleIfNotInstalled() -{ - [CmdletBinding(SupportsShouldProcess = $true)] - param( - [string]$moduleName, - [string]$version, - [string]$repositoryUrl - ) - +function Get-ModuleRepositories([string]$moduleName) { + $DefaultPSRepositoryUrl = "https://www.powershellgallery.com/api/v2" # List of modules+versions we want to replace with internal feed sources for reliability, security, etc. $packageFeedOverrides = @{ 'powershell-yaml' = 'https://pkgs.dev.azure.com/azure-sdk/public/_packaging/azure-sdk-tools/nuget/v2' } - try { - $mutex = New-Object System.Threading.Mutex($false, "Install-ModuleIfNotInstalled") - $null = $mutex.WaitOne() + $repoUrls = if ($packageFeedOverrides.Contains("${moduleName}")) { + @($packageFeedOverrides["${moduleName}"], $DefaultPSRepositoryUrl) + } else { + @($DefaultPSRepositoryUrl) + } - # Check installed modules again after acquiring lock - $modules = (Get-Module -ListAvailable $moduleName) - if ($version -as [Version]) { - $modules = $modules.Where({ [Version]$_.Version -ge [Version]$version }) - } + return $repoUrls +} +function moduleIsInstalled([string]$moduleName, [string]$version) { + $modules = (Get-Module -ListAvailable $moduleName) + if ($version -as [Version]) { + $modules = $modules.Where({ [Version]$_.Version -ge [Version]$version }) if ($modules.Count -gt 0) { Write-Host "Using module $($modules[0].Name) with version $($modules[0].Version)." return $modules[0] } + } + return $null +} - $repositoryUrl = if ($repositoryUrl) { - $repositoryUrl - } elseif ($mirrorFeedOverrides.Contains("${moduleName}")) { - $mirrorFeedOverrides["${moduleName}"] - } else { - $DefaultPSRepositoryUrl +function installModule([string]$moduleName, [string]$version, $repoUrl) { + $repo = (Get-PSRepository).Where({ $_.SourceLocation -eq $repoUrl }) + if ($repo.Count -eq 0) + { + Register-PSRepository -Name $repoUrl -SourceLocation $repoUrl -InstallationPolicy Trusted + $repo = (Get-PSRepository).Where({ $_.SourceLocation -eq $repoUrl }) + if ($repo.Count -eq 0) { + throw "Failed to register package repository $repoUrl." } + } - $repositories = (Get-PSRepository).Where({ $_.SourceLocation -eq $repositoryUrl }) - if ($repositories.Count -eq 0) - { - Register-PSRepository -Name $repositoryUrl -SourceLocation $repositoryUrl -InstallationPolicy Trusted - $repositories = (Get-PSRepository).Where({ $_.SourceLocation -eq $repositoryUrl }) - if ($repositories.Count -eq 0) { - Write-Error "Failed to register package repository $repositoryUrl." - return - } - } - $repository = $repositories[0] + if ($repo.InstallationPolicy -ne "Trusted") { + Set-PSRepository -Name $repo.Name -InstallationPolicy "Trusted" + } - if ($repository.InstallationPolicy -ne "Trusted") { - Set-PSRepository -Name $repository.Name -InstallationPolicy "Trusted" - } + Write-Host "Installing module $moduleName with min version $version from $repoUrl" + # Install under CurrentUser scope so that the end up under $CurrentUserModulePath for caching + Install-Module $moduleName -MinimumVersion $version -Repository $repo.Name -Scope CurrentUser -Force + # Ensure module installed + $modules = (Get-Module -ListAvailable $moduleName) + if ($version -as [Version]) { + $modules = $modules.Where({ [Version]$_.Version -ge [Version]$version }) + } + if ($modules.Count -eq 0) { + throw "Failed to install module $moduleName with version $version" + } - Write-Host "Installing module $moduleName with min version $version from $repositoryUrl" - # Install under CurrentUser scope so that the end up under $CurrentUserModulePath for caching - Install-Module $moduleName -MinimumVersion $version -Repository $repository.Name -Scope CurrentUser -Force + return $modules[0] +} - # Ensure module installed - $modules = (Get-Module -ListAvailable $moduleName) - if ($version -as [Version]) { - $modules = $modules.Where({ [Version]$_.Version -ge [Version]$version }) - } +# Manual test at eng/common-tests/psmodule-helpers/Install-Module-Parallel.ps1 +# If we want to use another default repository other then PSGallery we can update the default parameters +function Install-ModuleIfNotInstalled() +{ + [CmdletBinding(SupportsShouldProcess = $true)] + param( + [string]$moduleName, + [string]$version, + [string]$repositoryUrl + ) + + # Check installed modules before after acquiring lock to avoid a big queue + $module = moduleIsInstalled -moduleName $moduleName -version $version + if ($module) { return $module } - if ($modules.Count -eq 0) { - Write-Error "Failed to install module $moduleName with version $version" - return + try { + $mutex = New-Object System.Threading.Mutex($false, "Install-ModuleIfNotInstalled") + $null = $mutex.WaitOne() + + # Check installed modules again after acquiring lock, in case it has been installed + $module = moduleIsInstalled -moduleName $moduleName -version $version + if ($module) { return $module } + + $repoUrls = Get-ModuleRepositories $moduleName + + foreach ($url in $repoUrls) { + try { + $module = installModule -moduleName $moduleName -version $version -repoUrl $url + } catch { + if ($url -ne $repoUrls[-1]) { + Write-Warning "Failed to install powershell module from '$url'. Retrying with fallback repository" + Write-Warning $_ + continue + } else { + Write-Warning "Failed to install powershell module from $url" + throw + } + } + break } - Write-Host "Using module $($modules[0].Name) with version $($modules[0].Version)." + Write-Host "Using module '$($module.Name)' with version '$($module.Version)'." } finally { $mutex.ReleaseMutex() } - return $modules[0] + return $module } if ($null -ne $env:SYSTEM_TEAMPROJECTID) { From 2e56d7498c6b6ef4d7be454189b849c786d1f8e4 Mon Sep 17 00:00:00 2001 From: Ben Broderick Phillips Date: Thu, 21 Nov 2024 16:05:46 -0500 Subject: [PATCH 4/4] Unregister repository after module install --- eng/common/scripts/Helpers/PSModule-Helpers.ps1 | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/eng/common/scripts/Helpers/PSModule-Helpers.ps1 b/eng/common/scripts/Helpers/PSModule-Helpers.ps1 index 4e7224021eb0..0c825cb70601 100644 --- a/eng/common/scripts/Helpers/PSModule-Helpers.ps1 +++ b/eng/common/scripts/Helpers/PSModule-Helpers.ps1 @@ -102,6 +102,10 @@ function installModule([string]$moduleName, [string]$version, $repoUrl) { throw "Failed to install module $moduleName with version $version" } + # Unregister repository as it can cause overlap issues with `dotnet tool install` + # commands using the same devops feed + Unregister-PSRepository -Name $repoUrl + return $modules[0] }