From 063a8445e3f4af5138971f84da1f250223948f5f Mon Sep 17 00:00:00 2001 From: m-nash Date: Wed, 25 Jan 2023 15:13:18 -0800 Subject: [PATCH 1/6] move scripts to common --- eng/common/scripts/Cadl-Project-Generate.ps1 | 68 +++++++++ eng/common/scripts/Cadl-Project-Sync.ps1 | 138 +++++++++++++++++++ eng/common/scripts/common.ps1 | 2 + 3 files changed, 208 insertions(+) create mode 100644 eng/common/scripts/Cadl-Project-Generate.ps1 create mode 100644 eng/common/scripts/Cadl-Project-Sync.ps1 diff --git a/eng/common/scripts/Cadl-Project-Generate.ps1 b/eng/common/scripts/Cadl-Project-Generate.ps1 new file mode 100644 index 00000000000..0e622609988 --- /dev/null +++ b/eng/common/scripts/Cadl-Project-Generate.ps1 @@ -0,0 +1,68 @@ +[CmdletBinding()] +param ( + [Parameter(Position=0)] + [ValidateNotNullOrEmpty()] + [string] $ProjectDirectory +) + +$ErrorActionPreference = "Stop" +. $PSScriptRoot/Helpers/PSModule-Helpers.ps1 +Install-ModuleIfNotInstalled "powershell-yaml" "0.4.1" | Import-Module + +function NpmInstallForProject([string]$workingDirectory) { + Push-Location $workingDirectory + try { + $currentDur = Resolve-Path "." + Write-Host "Generating from $currentDur" + if (Test-Path "package.json") { + Remove-Item -Path "package.json" -Force + } + if (Test-Path ".npmrc") { + Remove-Item -Path ".npmrc" -Force + } + $replacementPackageJson = "$PSScriptRoot/../../emitter-package.json" + Write-Host("Copying package.json from $replacementPackageJson") + Copy-Item -Path $replacementPackageJson -Destination "package.json" -Force + npm install + if ($LASTEXITCODE) { exit $LASTEXITCODE } + } + finally { + Pop-Location + } +} + +Write-Host($GetEmitterNameFn) + +$emitterName = &$GetEmitterNameFn +$cadlConfigurationFile = Resolve-Path "$ProjectDirectory/cadl-location.yaml" + +Write-Host "Reading configuration from $cadlConfigurationFile" +$configuration = Get-Content -Path $cadlConfigurationFile -Raw | ConvertFrom-Yaml + +$specSubDirectory = $configuration["directory"] +$innerFolder = Split-Path $specSubDirectory -Leaf + +$tempFolder = "$ProjectDirectory/TempCadlFiles" +$npmWorkingDir = Resolve-Path $tempFolder/$innerFolder +$mainCadlFile = If (Test-Path "$npmWorkingDir/client.cadl") { Resolve-Path "$npmWorkingDir/client.cadl" } Else { Resolve-Path "$npmWorkingDir/main.cadl"} + +try { + Push-Location $npmWorkingDir + NpmInstallForProject $npmWorkingDir + + if ($LASTEXITCODE) { exit $LASTEXITCODE } + + $emitterAdditionalOptions = &$GetEmitterAdditionalOptionsFn + Write-Host("npx cadl compile $mainCadlFile --emit $emitterName $emitterAdditionalOptions") + npx cadl compile $mainCadlFile --emit $emitterName $emitterAdditionalOptions + + if ($LASTEXITCODE) { exit $LASTEXITCODE } +} +finally { + Pop-Location +} + +$shouldCleanUp = $configuration["cleanup"] ?? $true +if ($shouldCleanUp) { + Remove-Item $tempFolder -Recurse -Force +} \ No newline at end of file diff --git a/eng/common/scripts/Cadl-Project-Sync.ps1 b/eng/common/scripts/Cadl-Project-Sync.ps1 new file mode 100644 index 00000000000..136a5ed2293 --- /dev/null +++ b/eng/common/scripts/Cadl-Project-Sync.ps1 @@ -0,0 +1,138 @@ +[CmdletBinding()] +param ( + [Parameter(Position=0)] + [ValidateNotNullOrEmpty()] + [string] $ProjectDirectory +) + +$ErrorActionPreference = "Stop" +. $PSScriptRoot/Helpers/PSModule-Helpers.ps1 +Install-ModuleIfNotInstalled "powershell-yaml" "0.4.1" | Import-Module +$sparseCheckoutFile = ".git/info/sparse-checkout" + +function AddSparseCheckoutPath([string]$subDirectory) { + if (!(Test-Path $sparseCheckoutFile) -or !((Get-Content $sparseCheckoutFile).Contains($subDirectory))) { + Write-Output $subDirectory >> .git/info/sparse-checkout + } +} + +function CopySpecToProjectIfNeeded([string]$specCloneRoot, [string]$mainSpecDir, [string]$dest, [string[]]$specAdditionalSubDirectories) { + $source = "$specCloneRoot/$mainSpecDir" + Write-Host "Copying spec from $source" + # $mainSpecDir is the PR folder, we just need to copy its subfolders which include the cadl project folder + Get-ChildItem –Path "$source" -Exclude @("data-plane", "resource-manager")| + Foreach-Object { + Copy-Item -Path $_.FullName -Destination $dest -Recurse -Force + } + foreach($additionalDir in $specAdditionalSubDirectories) + { + $source = "$specCloneRoot/$additionalDir" + Write-Host "Copying spec from $source" + Copy-Item -Path $source -Destination $dest -Recurse -Force + } +} + +function UpdateSparseCheckoutFile([string]$mainSpecDir, [string[]]$specAdditionalSubDirectories) { + AddSparseCheckoutPath $mainSpecDir + foreach($subDir in $specAdditionalSubDirectories) + { + AddSparseCheckoutPath $subDir + } +} + +function GetGitRemoteValue([string]$repo) { + Push-Location $ProjectDirectory + $result = "" + try { + $gitRemotes = (git remote -v) + foreach ($remote in $gitRemotes) + { + if ($remote.StartsWith("origin")) { + if ($remote -match 'https://github.com/\S+[\.git]') { + $result = "https://github.com/$repo.git" + break + } elseif ($remote -match "git@github.com:\S+[\.git]"){ + $result = "git@github.com:$repo.git" + break + } else { + throw "Unknown git remote format found: $remote" + } + } + } + } + finally { + Pop-Location + } + + return $result +} + +function InitializeSparseGitClone([string]$repo) { + git clone --no-checkout --filter=tree:0 $repo . + if ($LASTEXITCODE) { exit $LASTEXITCODE } + git sparse-checkout init + if ($LASTEXITCODE) { exit $LASTEXITCODE } + Remove-Item $sparseCheckoutFile -Force +} + +function GetSpecCloneDir([string]$projectName) { + Push-Location $ProjectDirectory + try { + $root = git rev-parse --show-toplevel + } + finally { + Pop-Location + } + + $sparseSpecCloneDir = "$root/../sparse-spec/$projectName" + New-Item $sparseSpecCloneDir -Type Directory -Force | Out-Null + $createResult = Resolve-Path $sparseSpecCloneDir + return $createResult +} + +$cadlConfigurationFile = Resolve-Path "$ProjectDirectory/cadl-location.yaml" +Write-Host "Reading configuration from $cadlConfigurationFile" +$configuration = Get-Content -Path $cadlConfigurationFile -Raw | ConvertFrom-Yaml + +$pieces = $cadlConfigurationFile.Path.Replace("\","/").Split("/") +$projectName = $pieces[$pieces.Count - 3] + +# clone the whole RP directory which is the parent of $configuration["directory"] +if ($configuration["directory"] -match "^[^/\\]+[\\/]+[^/\\]+") +{ + $specSubDirectory = $Matches[0] +} +else +{ + throw "The directory in $cadlConfigurationFile is not expected" +} +if ( $configuration["repo"] -and $configuration["commit"]) { + $specCloneDir = GetSpecCloneDir $projectName + $gitRemoteValue = GetGitRemoteValue $configuration["repo"] + + Write-Host "Setting up sparse clone for $projectName at $specCloneDir" + + Push-Location $specCloneDir.Path + try { + if (!(Test-Path ".git")) { + InitializeSparseGitClone $gitRemoteValue + UpdateSparseCheckoutFile $specSubDirectory $configuration["additionalDirectories"] + } + git checkout $configuration["commit"] + if ($LASTEXITCODE) { exit $LASTEXITCODE } + } + finally { + Pop-Location + } +} elseif ( $configuration["spec-root-dir"] ) { + $specCloneDir = $configuration["spec-root-dir"] +} + + +$tempCadlDir = "$ProjectDirectory/TempCadlFiles" +New-Item $tempCadlDir -Type Directory -Force | Out-Null +CopySpecToProjectIfNeeded ` + -specCloneRoot $specCloneDir ` + -mainSpecDir $specSubDirectory ` + -dest $tempCadlDir ` + -specAdditionalSubDirectories $configuration["additionalDirectories"] diff --git a/eng/common/scripts/common.ps1 b/eng/common/scripts/common.ps1 index c78b1cfc2f1..8f9c9151432 100644 --- a/eng/common/scripts/common.ps1 +++ b/eng/common/scripts/common.ps1 @@ -57,3 +57,5 @@ $GetDocsMsTocChildrenForManagementPackagesFn = "Get-${Language}-DocsMsTocChildre $UpdateDocsMsTocFn = "Get-${Language}-UpdatedDocsMsToc" $GetPackageLevelReadmeFn = "Get-${Language}-PackageLevelReadme" $GetRepositoryLinkFn = "Get-${Language}-RepositoryLink" +$GetEmitterAdditionalOptionsFn = "Get-${Language}-EmitterAdditionalOptions" +$GetEmitterNameFn = "Get-${Language}-EmitterName" From de1581a598ee966b8a812a154fbae086b865cdd5 Mon Sep 17 00:00:00 2001 From: m-nash Date: Wed, 25 Jan 2023 15:34:59 -0800 Subject: [PATCH 2/6] add reference to common script --- eng/common/scripts/Cadl-Project-Generate.ps1 | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/eng/common/scripts/Cadl-Project-Generate.ps1 b/eng/common/scripts/Cadl-Project-Generate.ps1 index 0e622609988..ac00f481da9 100644 --- a/eng/common/scripts/Cadl-Project-Generate.ps1 +++ b/eng/common/scripts/Cadl-Project-Generate.ps1 @@ -7,6 +7,7 @@ param ( $ErrorActionPreference = "Stop" . $PSScriptRoot/Helpers/PSModule-Helpers.ps1 +. $PSScriptRoot/common.ps1 Install-ModuleIfNotInstalled "powershell-yaml" "0.4.1" | Import-Module function NpmInstallForProject([string]$workingDirectory) { @@ -31,8 +32,6 @@ function NpmInstallForProject([string]$workingDirectory) { } } -Write-Host($GetEmitterNameFn) - $emitterName = &$GetEmitterNameFn $cadlConfigurationFile = Resolve-Path "$ProjectDirectory/cadl-location.yaml" From 9273d024e4670d8fc1cd45d99e4a85517853eb49 Mon Sep 17 00:00:00 2001 From: m-nash Date: Thu, 26 Jan 2023 08:19:39 -0800 Subject: [PATCH 3/6] address feedback --- eng/common/scripts/Cadl-Project-Generate.ps1 | 23 ++++++++++++++++++-- eng/common/scripts/Cadl-Project-Sync.ps1 | 22 ++++++------------- 2 files changed, 28 insertions(+), 17 deletions(-) diff --git a/eng/common/scripts/Cadl-Project-Generate.ps1 b/eng/common/scripts/Cadl-Project-Generate.ps1 index ac00f481da9..86205703e47 100644 --- a/eng/common/scripts/Cadl-Project-Generate.ps1 +++ b/eng/common/scripts/Cadl-Project-Generate.ps1 @@ -15,16 +15,33 @@ function NpmInstallForProject([string]$workingDirectory) { try { $currentDur = Resolve-Path "." Write-Host "Generating from $currentDur" + if (Test-Path "package.json") { Remove-Item -Path "package.json" -Force } + if (Test-Path ".npmrc") { Remove-Item -Path ".npmrc" -Force } + + if (Test-Path "node_modules") { + Remove-Item -Path "node_modules" -Force -Recurse + } + + if (Test-Path "package-lock.json") { + Remove-Item -Path "package-lock.json" -Force + } + + #default to root/eng/emitter-package.json but you can override by writing + #Get-${Language}-EmitterPackageJsonPath in your Language-Settings.ps1 $replacementPackageJson = "$PSScriptRoot/../../emitter-package.json" + if (Test-Path "Function:$GetEmitterPackageJsonPathFn") { + $replacementPackageJson = &$GetEmitterPackageJsonPathFn + } + Write-Host("Copying package.json from $replacementPackageJson") Copy-Item -Path $replacementPackageJson -Destination "package.json" -Force - npm install + npm install --no-lock-file if ($LASTEXITCODE) { exit $LASTEXITCODE } } finally { @@ -51,7 +68,9 @@ try { if ($LASTEXITCODE) { exit $LASTEXITCODE } - $emitterAdditionalOptions = &$GetEmitterAdditionalOptionsFn + if (Test-Path "Function:$GetEmitterPackageJsonPathFn") { + $emitterAdditionalOptions = &$GetEmitterAdditionalOptionsFn + } Write-Host("npx cadl compile $mainCadlFile --emit $emitterName $emitterAdditionalOptions") npx cadl compile $mainCadlFile --emit $emitterName $emitterAdditionalOptions diff --git a/eng/common/scripts/Cadl-Project-Sync.ps1 b/eng/common/scripts/Cadl-Project-Sync.ps1 index 136a5ed2293..7944611f98d 100644 --- a/eng/common/scripts/Cadl-Project-Sync.ps1 +++ b/eng/common/scripts/Cadl-Project-Sync.ps1 @@ -19,13 +19,8 @@ function AddSparseCheckoutPath([string]$subDirectory) { function CopySpecToProjectIfNeeded([string]$specCloneRoot, [string]$mainSpecDir, [string]$dest, [string[]]$specAdditionalSubDirectories) { $source = "$specCloneRoot/$mainSpecDir" Write-Host "Copying spec from $source" - # $mainSpecDir is the PR folder, we just need to copy its subfolders which include the cadl project folder - Get-ChildItem –Path "$source" -Exclude @("data-plane", "resource-manager")| - Foreach-Object { - Copy-Item -Path $_.FullName -Destination $dest -Recurse -Force - } - foreach($additionalDir in $specAdditionalSubDirectories) - { + + foreach ($additionalDir in $specAdditionalSubDirectories) { $source = "$specCloneRoot/$additionalDir" Write-Host "Copying spec from $source" Copy-Item -Path $source -Destination $dest -Recurse -Force @@ -34,8 +29,7 @@ function CopySpecToProjectIfNeeded([string]$specCloneRoot, [string]$mainSpecDir, function UpdateSparseCheckoutFile([string]$mainSpecDir, [string[]]$specAdditionalSubDirectories) { AddSparseCheckoutPath $mainSpecDir - foreach($subDir in $specAdditionalSubDirectories) - { + foreach ($subDir in $specAdditionalSubDirectories) { AddSparseCheckoutPath $subDir } } @@ -45,8 +39,7 @@ function GetGitRemoteValue([string]$repo) { $result = "" try { $gitRemotes = (git remote -v) - foreach ($remote in $gitRemotes) - { + foreach ($remote in $gitRemotes) { if ($remote.StartsWith("origin")) { if ($remote -match 'https://github.com/\S+[\.git]') { $result = "https://github.com/$repo.git" @@ -98,14 +91,13 @@ $pieces = $cadlConfigurationFile.Path.Replace("\","/").Split("/") $projectName = $pieces[$pieces.Count - 3] # clone the whole RP directory which is the parent of $configuration["directory"] -if ($configuration["directory"] -match "^[^/\\]+[\\/]+[^/\\]+") -{ +if ($configuration["directory"] -match "^[^/\\]+[\\/]+[^/\\]+") { $specSubDirectory = $Matches[0] } -else -{ +else { throw "The directory in $cadlConfigurationFile is not expected" } + if ( $configuration["repo"] -and $configuration["commit"]) { $specCloneDir = GetSpecCloneDir $projectName $gitRemoteValue = GetGitRemoteValue $configuration["repo"] From ccc77383fa5af5fb26f2e87d2eebabbd3ed50d63 Mon Sep 17 00:00:00 2001 From: m-nash Date: Thu, 26 Jan 2023 09:57:42 -0800 Subject: [PATCH 4/6] couple tweaks to relative path updates --- eng/common/scripts/Cadl-Project-Sync.ps1 | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/eng/common/scripts/Cadl-Project-Sync.ps1 b/eng/common/scripts/Cadl-Project-Sync.ps1 index 7944611f98d..98e8f6c0e06 100644 --- a/eng/common/scripts/Cadl-Project-Sync.ps1 +++ b/eng/common/scripts/Cadl-Project-Sync.ps1 @@ -88,15 +88,9 @@ Write-Host "Reading configuration from $cadlConfigurationFile" $configuration = Get-Content -Path $cadlConfigurationFile -Raw | ConvertFrom-Yaml $pieces = $cadlConfigurationFile.Path.Replace("\","/").Split("/") -$projectName = $pieces[$pieces.Count - 3] +$projectName = $pieces[$pieces.Count - 2] -# clone the whole RP directory which is the parent of $configuration["directory"] -if ($configuration["directory"] -match "^[^/\\]+[\\/]+[^/\\]+") { - $specSubDirectory = $Matches[0] -} -else { - throw "The directory in $cadlConfigurationFile is not expected" -} +$specSubDirectory = $configuration["directory"] if ( $configuration["repo"] -and $configuration["commit"]) { $specCloneDir = GetSpecCloneDir $projectName From 8903ad95c286cb885b3389560cd28be87dc75f52 Mon Sep 17 00:00:00 2001 From: m-nash Date: Thu, 26 Jan 2023 10:01:37 -0800 Subject: [PATCH 5/6] update copy paste issue with emitter path --- eng/common/scripts/Cadl-Project-Generate.ps1 | 2 +- eng/common/scripts/common.ps1 | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/eng/common/scripts/Cadl-Project-Generate.ps1 b/eng/common/scripts/Cadl-Project-Generate.ps1 index 86205703e47..40f3fcc2f40 100644 --- a/eng/common/scripts/Cadl-Project-Generate.ps1 +++ b/eng/common/scripts/Cadl-Project-Generate.ps1 @@ -68,7 +68,7 @@ try { if ($LASTEXITCODE) { exit $LASTEXITCODE } - if (Test-Path "Function:$GetEmitterPackageJsonPathFn") { + if (Test-Path "Function:$GetEmitterAdditionalOptionsFn") { $emitterAdditionalOptions = &$GetEmitterAdditionalOptionsFn } Write-Host("npx cadl compile $mainCadlFile --emit $emitterName $emitterAdditionalOptions") diff --git a/eng/common/scripts/common.ps1 b/eng/common/scripts/common.ps1 index 8f9c9151432..8f9c707ee29 100644 --- a/eng/common/scripts/common.ps1 +++ b/eng/common/scripts/common.ps1 @@ -59,3 +59,4 @@ $GetPackageLevelReadmeFn = "Get-${Language}-PackageLevelReadme" $GetRepositoryLinkFn = "Get-${Language}-RepositoryLink" $GetEmitterAdditionalOptionsFn = "Get-${Language}-EmitterAdditionalOptions" $GetEmitterNameFn = "Get-${Language}-EmitterName" +$GetEmitterPackageJsonPathFn = "Get-${Language}-EmitterPackageJsonPath" From 981c5e88a6b5a62e84b419c4dad41b4e1448ecef Mon Sep 17 00:00:00 2001 From: m-nash Date: Mon, 30 Jan 2023 11:02:16 -0800 Subject: [PATCH 6/6] updates to make scripts more generic --- eng/common/scripts/Cadl-Project-Generate.ps1 | 11 ++++++++--- eng/common/scripts/Cadl-Project-Sync.ps1 | 5 +++-- 2 files changed, 11 insertions(+), 5 deletions(-) diff --git a/eng/common/scripts/Cadl-Project-Generate.ps1 b/eng/common/scripts/Cadl-Project-Generate.ps1 index 40f3fcc2f40..fa2b72c75be 100644 --- a/eng/common/scripts/Cadl-Project-Generate.ps1 +++ b/eng/common/scripts/Cadl-Project-Generate.ps1 @@ -49,6 +49,7 @@ function NpmInstallForProject([string]$workingDirectory) { } } +$resolvedProjectDirectory = Resolve-Path $ProjectDirectory $emitterName = &$GetEmitterNameFn $cadlConfigurationFile = Resolve-Path "$ProjectDirectory/cadl-location.yaml" @@ -69,10 +70,14 @@ try { if ($LASTEXITCODE) { exit $LASTEXITCODE } if (Test-Path "Function:$GetEmitterAdditionalOptionsFn") { - $emitterAdditionalOptions = &$GetEmitterAdditionalOptionsFn + $emitterAdditionalOptions = &$GetEmitterAdditionalOptionsFn $resolvedProjectDirectory + if ($emitterAdditionalOptions.Length -gt 0) { + $emitterAdditionalOptions = " $emitterAdditionalOptions" + } } - Write-Host("npx cadl compile $mainCadlFile --emit $emitterName $emitterAdditionalOptions") - npx cadl compile $mainCadlFile --emit $emitterName $emitterAdditionalOptions + $cadlCompileCommand = "npx cadl compile $mainCadlFile --emit $emitterName$emitterAdditionalOptions" + Write-Host($cadlCompileCommand) + Invoke-Expression $cadlCompileCommand if ($LASTEXITCODE) { exit $LASTEXITCODE } } diff --git a/eng/common/scripts/Cadl-Project-Sync.ps1 b/eng/common/scripts/Cadl-Project-Sync.ps1 index 98e8f6c0e06..21f63403c68 100644 --- a/eng/common/scripts/Cadl-Project-Sync.ps1 +++ b/eng/common/scripts/Cadl-Project-Sync.ps1 @@ -18,11 +18,12 @@ function AddSparseCheckoutPath([string]$subDirectory) { function CopySpecToProjectIfNeeded([string]$specCloneRoot, [string]$mainSpecDir, [string]$dest, [string[]]$specAdditionalSubDirectories) { $source = "$specCloneRoot/$mainSpecDir" - Write-Host "Copying spec from $source" + Copy-Item -Path $source -Destination $dest -Recurse -Force + Write-Host "Copying spec from $source to $dest" foreach ($additionalDir in $specAdditionalSubDirectories) { $source = "$specCloneRoot/$additionalDir" - Write-Host "Copying spec from $source" + Write-Host "Copying spec from $source to $dest" Copy-Item -Path $source -Destination $dest -Recurse -Force } }