diff --git a/.github/actions/setup-node-npm-ci/action.yaml b/.github/actions/setup-node-npm-ci/action.yaml new file mode 100644 index 000000000000..f8e212657351 --- /dev/null +++ b/.github/actions/setup-node-npm-ci/action.yaml @@ -0,0 +1,22 @@ +name: Setup Node 20 and run `npm ci` +description: Uses specified Node version and runs npm commands to set up the environment for REST API CI + +inputs: + node-version: + description: 'Node version to use' + default: 20.x + +runs: + using: "composite" + + steps: + - uses: actions/setup-node@v4 + with: + node-version: ${{ inputs.node-version }} + + - run: npm ci + shell: bash + + - run: npm ls -a + shell: bash + continue-on-error: true diff --git a/.github/workflows/typespec-validation-all.yaml b/.github/workflows/typespec-validation-all.yaml new file mode 100644 index 000000000000..2ffb641af2e4 --- /dev/null +++ b/.github/workflows/typespec-validation-all.yaml @@ -0,0 +1,69 @@ +name: TypeSpec Validation All + +on: + push: + branches: + - main + - RPSaaSMaster + - typespec-next + + pull_request: + branches: + - main + - RPSaaSMaster + - typespec-next + paths: + - .gitattributes + - .prettierrc.json + - package-lock.json + - package.json + - tsconfig.json + - eng/** + - specification/suppressions.yaml + - specification/common-types/** + + # Workflow and workflow dependencies + - .github/workflows/typespec-validation-all.yaml + - .github/actions/setup-node-npm-ci/** + + schedule: + # Run 4x/day + - cron: '0 0,6,12,18 * * * ' + +jobs: + typespec-validation-all: + name: TypeSpec Validation All + strategy: + matrix: + os: [ubuntu-latest, windows-latest] + # shards must start at 0 and increment by 1 + shard: [0, 1, 2] + # total-shards must be an accurate count of the number of shards + total-shards: [3] + + runs-on: ${{ matrix.os }} + + steps: + - name: Enable git long paths + if: runner.os == 'Windows' + run: git config --global core.longpaths true + + - uses: actions/checkout@v4 + with: + fetch-depth: 2 + + - name: Setup Node 20 and run `npm ci` + uses: ./.github/actions/setup-node-npm-ci + + - name: Validate All Specs + run: | + ./eng/scripts/TypeSpec-Validation.ps1 ` + -Shard ${{ matrix.shard }} ` + -TotalShards ${{ matrix.total-shards }} ` + -CheckAll ` + -GitClean ` + -Verbose + + # Effectively the same as ignoreLASTEXITCODE: true in Azure DevOps + exit 0 + shell: pwsh diff --git a/.github/workflows/typespec-validation.yaml b/.github/workflows/typespec-validation.yaml new file mode 100644 index 000000000000..0e50527c1761 --- /dev/null +++ b/.github/workflows/typespec-validation.yaml @@ -0,0 +1,24 @@ +name: Typespec Validation + +on: pull_request + +jobs: + typespec-validation: + name: Typespec Validation + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 2 + + - name: Setup Node 20 and run `npm ci` + uses: ./.github/actions/setup-node-npm-ci + + - name: Validate Impacted Specs + run: | + ./eng/scripts/TypeSpec-Validation.ps1 -GitClean -Verbose + + # Effectively the same as ignoreLASTEXITCODE: true in Azure DevOps + exit 0 + shell: pwsh diff --git a/eng/pipelines/typespec-validation-all.yml b/eng/pipelines/typespec-validation-all.yml deleted file mode 100644 index e6be95c4bb43..000000000000 --- a/eng/pipelines/typespec-validation-all.yml +++ /dev/null @@ -1,47 +0,0 @@ -trigger: - branches: - include: - - main - - RPSaaSMaster - - typespec-next - -resources: - pipelines: - - pipeline: typespec - source: typespec - Publish - project: internal - trigger: - branches: - include: - - main - - pipeline: typespec-azure - source: typespec-azure - Publish - project: internal - trigger: - branches: - include: - - main - -jobs: -- job: - - strategy: - matrix: - Linux: - Pool: azsdk-pool-mms-ubuntu-2204-general - OsVmImage: ubuntu-22.04 - Windows: - Pool: azsdk-pool-mms-win-2022-general - OsVmImage: windows-2022 - - pool: - name: $(Pool) - vmImage: $(OSVmImage) - - steps: - - template: /eng/pipelines/templates/steps/npm-install.yml - - - pwsh: | - $(Build.SourcesDirectory)/eng/scripts/TypeSpec-Validation.ps1 -CheckAll -GitClean -Verbose - displayName: Validate All Specs - ignoreLASTEXITCODE: true diff --git a/eng/pipelines/typespec-validation.yml b/eng/pipelines/typespec-validation.yml deleted file mode 100644 index 59729ce5b1f8..000000000000 --- a/eng/pipelines/typespec-validation.yml +++ /dev/null @@ -1,16 +0,0 @@ -trigger: none - -jobs: -- job: - displayName: Validated Impacted Specs - pool: - name: azsdk-pool-mms-ubuntu-2204-general - vmImage: ubuntu-22.04 - - steps: - - template: /eng/pipelines/templates/steps/npm-install.yml - - - pwsh: | - $(Build.SourcesDirectory)/eng/scripts/TypeSpec-Validation.ps1 -GitClean -Verbose - displayName: Validate Impacted Specs - ignoreLASTEXITCODE: true diff --git a/eng/scripts/Array-Functions.ps1 b/eng/scripts/Array-Functions.ps1 new file mode 100644 index 000000000000..d5da35e182b5 --- /dev/null +++ b/eng/scripts/Array-Functions.ps1 @@ -0,0 +1,18 @@ +function ShardArray($array, $shard, $totalShards) { + if ($totalShards -lt 2) { + return $array + } + + if ($shard -ge $totalShards) { + throw "Shard index ($shard) must be less than total shards ($totalShards)" + } + + if ($totalShards -gt $array.Length) { + throw "Cannot shard array into more pieces than there are elements" + } + + $shardSize = [math]::Ceiling($array.Length / $totalShards) + $start = $shard * $shardSize + $end = [math]::Min($start + $shardSize, $array.Length) + return $array[$start..($end - 1)] + } diff --git a/eng/scripts/Tests/Array-Functions.Tests.ps1 b/eng/scripts/Tests/Array-Functions.Tests.ps1 new file mode 100644 index 000000000000..923cbe3914f9 --- /dev/null +++ b/eng/scripts/Tests/Array-Functions.Tests.ps1 @@ -0,0 +1,77 @@ +BeforeAll { + . "$PSScriptRoot\..\Array-Functions.ps1" +} + +# 0..78 makes an array of 79 (prime number) items + +Describe "ShardArray" { + Context "Input Validation" { + It "throws when sharding into more pieces than there are elements" { + $array = 0..78 + $shard = 0 + $totalShards = 150 + + { ShardArray $array $shard $totalShards } | Should -Throw + } + + It "returns the full array when given shards" -ForEach @( + @{ totalShards = 1 }, + @{ totalShards = 0 }, + @{ totalShards = -1 }, + @{ totalShards = -2 } + ) { + $array = 0..78 + $shard = 0 + + $result = ShardArray $array $shard $totalShards + + $result | Should -Be $array + } + + It "throws when the shard index is greater than the total shards" { + $array = 0..78 + $shard = 10 + $totalShards = 3 + + { ShardArray $array $shard $totalShards } | Should -Throw + } + + It "does not throw when totalShards equals the array length" { + $array = 0..78 + $shard = 0 + $totalShards = 79 + + { ShardArray $array $shard $totalShards } | Should -Not -Throw + } + + } + + Context "Shards arrays" -ForEach @( + @{ array = 0..78; totalShards = 1 }, + @{ array = 0..78; totalShards = 2 }, + @{ array = 0..11; totalShards = 3 }, + @{ array = 0..10; totalShards = 4 }, + @{ array = 0..78; totalShards = 79 } + ) { + It "returns all of the values in order when sharding (Total Shards: ) " { + $shards = New-Object object[] $totalShards + for ($i = 0; $i -lt $totalShards; $i++) { + # Assigning directly avoids flattening the array. + $shards[$i] = ShardArray $array $i $totalShards + } + + # Flatten the array for comparison + $actual = $shards | ForEach-Object { $_ } + + $actual | Should -Be $array + } + + It "returns arrays of expected valid size (Total Shards: )" { + $maxLength = [math]::Ceiling($array.Length / $totalShards) + for ($i = 0; $i -lt $totalShards; $i++) { + $shard = ShardArray $array $i $totalShards + $shard.Length | Should -BeLessOrEqual $maxLength + } + } + } +} diff --git a/eng/scripts/Tests/Copy-ApiVersion/Copy-ApiVersion.Tests.ps1 b/eng/scripts/Tests/Copy-ApiVersion.Tests.ps1 similarity index 75% rename from eng/scripts/Tests/Copy-ApiVersion/Copy-ApiVersion.Tests.ps1 rename to eng/scripts/Tests/Copy-ApiVersion.Tests.ps1 index 9bbaa3b230b8..a560cafc76f3 100644 --- a/eng/scripts/Tests/Copy-ApiVersion/Copy-ApiVersion.Tests.ps1 +++ b/eng/scripts/Tests/Copy-ApiVersion.Tests.ps1 @@ -1,7 +1,7 @@ Import-Module Pester BeforeAll { - . "$PSScriptRoot\..\..\Copy-ApiVersion-Functions.ps1" + . "$PSScriptRoot\..\Copy-ApiVersion-Functions.ps1" } Describe "Copy-ApiVersion regex tests" { @@ -11,25 +11,25 @@ Describe "Copy-ApiVersion regex tests" { version = "2024-01-01-preview" provider = "Microsoft.AgFoodPlatform" versionStatus = "preview" - specsDir = '..\..\..\..\..\specification\agrifood\resource-manager\Microsoft.AgFoodPlatform\preview\2023-06-01-preview' + specsDir = '..\..\..\..\specification\agrifood\resource-manager\Microsoft.AgFoodPlatform\preview\2023-06-01-preview' }, @{ version = "2024-01-01" provider = "Microsoft.Compute\ComputeRP" versionStatus = "stable" - specsDir = '..\..\..\..\..\specification\compute\resource-manager\Microsoft.Compute\ComputeRP\stable\2023-09-01' + specsDir = '..\..\..\..\specification\compute\resource-manager\Microsoft.Compute\ComputeRP\stable\2023-09-01' }, @{ version = "7.6-preview.1" provider = "Microsoft.KeyVault" versionStatus = "preview" - specsDir = '..\..\..\..\..\specification\keyvault\data-plane\Microsoft.KeyVault\preview\7.6-preview.1' + specsDir = '..\..\..\..\specification\keyvault\data-plane\Microsoft.KeyVault\preview\7.6-preview.1' }, @{ version = "7.5" provider = "Microsoft.Compute\ComputeRP" versionStatus = "stable" - specsDir = '..\..\..\..\..\specification\keyvault\data-plane\Microsoft.KeyVault\stable\7.5' + specsDir = '..\..\..\..\specification\keyvault\data-plane\Microsoft.KeyVault\stable\7.5' } ) { param($version, $provider, $versionStatus, $specsDir) @@ -48,12 +48,12 @@ Describe "Copy-ApiVersion regex tests" { # TODO: This is fragile. The tests stop working when a service team updates their readme.md. We should instead take fixed copies or something. It "Default version gets updated" -TestCases @( @{ - inputReadme = '..\..\..\..\..\specification\compute\resource-manager\readme.md' + inputReadme = '..\..\..\..\specification\compute\resource-manager\readme.md' apiVersion = "2024-01-01" versionStatus = "stable" }, @{ - inputReadme = '..\..\..\..\..\specification\keyvault\data-plane\readme.md' + inputReadme = '..\..\..\..\specification\keyvault\data-plane\readme.md' apiVersion = "7.6-preview.1" versionStatus = "preview" } diff --git a/eng/scripts/TypeSpec-Validation.ps1 b/eng/scripts/TypeSpec-Validation.ps1 index 031ae7d69000..0b60e6aa0631 100644 --- a/eng/scripts/TypeSpec-Validation.ps1 +++ b/eng/scripts/TypeSpec-Validation.ps1 @@ -1,16 +1,35 @@ [CmdletBinding()] param ( [switch]$CheckAll = $false, + [int]$Shard = 0, + [int]$TotalShards = 1, [switch]$GitClean = $false, [switch]$DryRun = $false, [string]$BaseCommitish = "HEAD^", [string]$TargetCommitish = "HEAD" ) +if ($TotalShards -gt 0 -and $Shard -ge $TotalShards) { + throw "Shard ($Shard) must be less than TotalShards ($TotalShards)" +} + . $PSScriptRoot/Logging-Functions.ps1 . $PSScriptRoot/Suppressions-Functions.ps1 +. $PSScriptRoot/Array-Functions.ps1 + +$typespecFolders, $checkedAll = &"$PSScriptRoot/Get-TypeSpec-Folders.ps1" ` + -BaseCommitish:$BaseCommitish ` + -TargetCommitish:$TargetCommitish ` + -CheckAll:$CheckAll -$typespecFolders, $checkedAll = &"$PSScriptRoot/Get-TypeSpec-Folders.ps1" -BaseCommitish:$BaseCommitish -TargetCommitish:$TargetCommitish -CheckAll:$CheckAll +if ($TotalShards -gt 1 -and $TotalShards -le $typespecFolders.Count) { + $typespecFolders = shardArray $typespecFolders $Shard $TotalShards +} + +Write-Host "Checking $($typespecFolders.Count) TypeSpec folders:" +foreach ($typespecFolder in $typespecFolders) { + Write-Host " $typespecFolder" +} $typespecFoldersWithFailures = @() if ($typespecFolders) {